diff --git a/scripts/bench-gateway-startup.ts b/scripts/bench-gateway-startup.ts index 07093b3841fa..8119d27bfe0e 100644 --- a/scripts/bench-gateway-startup.ts +++ b/scripts/bench-gateway-startup.ts @@ -18,6 +18,8 @@ import { import { selectSlowStartupTraceDurations } from "./lib/gateway-startup-trace-ranking.js"; type GatewayBenchCase = { + agentTopology?: "single" | "shared-eleven-plus-distinct-one"; + completionTracePhase?: string; config: Record; env?: Record; id: string; @@ -25,6 +27,8 @@ type GatewayBenchCase = { pluginActivationOnStartup?: boolean; pluginCount?: number; providerCatalogStallMs?: number; + providerStaticCatalogModelCount?: number; + providerStaticCatalogStallMs?: number; }; type ProbeResult = { @@ -42,6 +46,7 @@ type ProbeTransition = { }; type GatewaySample = { + completionMs: number | null; cpuCoreRatio: number | null; cpuMs: number | null; exitedBeforeTeardown?: boolean; @@ -72,6 +77,7 @@ type CaseResult = { name: string; samples: GatewaySample[]; summary: { + completionMs: SummaryStats | null; firstOutputMs: SummaryStats | null; cpuCoreRatio: SummaryStats | null; cpuMs: SummaryStats | null; @@ -178,6 +184,50 @@ const GATEWAY_CASES: readonly GatewayBenchCase[] = [ }, }, }, + { + id: "preparedRuntimeScaleOne", + name: "gateway, prepared runtime scale with one agent", + agentTopology: "single", + completionTracePhase: "sidecars.ready", + env: { OPENCLAW_SKIP_CHANNELS: "1" }, + providerStaticCatalogModelCount: 64, + providerStaticCatalogStallMs: 100, + config: { + ...BASE_CONFIG, + agents: { + defaults: { + model: { primary: `${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}` }, + models: { + [`${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}`]: { + agentRuntime: { id: "openclaw" }, + }, + }, + }, + }, + }, + }, + { + id: "preparedRuntimeScaleMany", + name: "gateway, prepared runtime scale with 11 shared-workspace agents and one distinct", + agentTopology: "shared-eleven-plus-distinct-one", + completionTracePhase: "sidecars.ready", + env: { OPENCLAW_SKIP_CHANNELS: "1" }, + providerStaticCatalogModelCount: 64, + providerStaticCatalogStallMs: 100, + config: { + ...BASE_CONFIG, + agents: { + defaults: { + model: { primary: `${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}` }, + models: { + [`${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}`]: { + agentRuntime: { id: "openclaw" }, + }, + }, + }, + }, + }, + }, { id: "oneInternalHook", name: "gateway, one configured internal hook", @@ -432,6 +482,11 @@ function summarizeCase(benchCase: GatewayBenchCase, samples: GatewaySample[]): C name: benchCase.name, samples, summary: { + completionMs: summarizeNumbers( + samples + .map((sample) => sample.completionMs) + .filter((value): value is number => typeof value === "number"), + ), firstOutputMs: summarizeNumbers( samples .map((sample) => sample.firstOutputMs) @@ -492,6 +547,9 @@ function collectResultFailures( if (sample.readyz.status !== 200 || sample.readyz.ms == null) { missing.push("/readyz"); } + if (sample.completionMs == null) { + missing.push("completion"); + } if (processMetricsRequired) { if (sample.cpuMs == null || sample.cpuCoreRatio == null) { missing.push("cpu"); @@ -561,7 +619,7 @@ function formatRatio(value: number | null): string { return value.toFixed(3); } -function formatStats(stats: SummaryStats | null): string { +function formatStats(stats: SummaryStats | null | undefined): string { if (!stats) { return "n/a"; } @@ -635,11 +693,32 @@ async function waitForProbe(params: { return { firstErrorKind, firstRecoveryMs, ms: null, status: lastStatus, transitions }; } +async function waitForStartupTracePhase(params: { + deadlineAt: number; + isDone: () => boolean; + phase: string; + startupTrace: Record; +}): Promise { + const totalKey = `${params.phase}.total`; + while (performance.now() < params.deadlineAt) { + if (Object.hasOwn(params.startupTrace, totalKey)) { + return params.startupTrace[totalKey] ?? null; + } + if (params.isDone()) { + return null; + } + await delay(25); + } + return null; +} + function writePluginFixtures( root: string, count: number, activationOnStartup?: boolean, providerCatalogStallMs?: number, + providerStaticCatalogStallMs?: number, + providerStaticCatalogModelCount?: number, ): PluginFixtureResult { const pluginIds: string[] = []; const pluginsDir = path.join(root, "plugins"); @@ -647,28 +726,44 @@ function writePluginFixtures( for (let index = 0; index < count; index += 1) { const id = `bench-plugin-${String(index + 1).padStart(2, "0")}`; const stallsProviderCatalog = providerCatalogStallMs !== undefined && index === 0; + const stallsProviderStaticCatalog = providerStaticCatalogStallMs !== undefined && index === 0; pluginIds.push(id); const pluginDir = path.join(pluginsDir, id); mkdirSync(pluginDir, { recursive: true }); const entry = path.join(pluginDir, "index.cjs"); - const model = { - id: STALLED_CATALOG_MODEL_ID, - name: "Benchmark Model", - reasoning: false, - input: ["text"], - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 128_000, - maxTokens: 8_192, - }; + const providerDiscoveryEntry = path.join(pluginDir, "provider-discovery.cjs"); + const models = Array.from( + { length: stallsProviderStaticCatalog ? (providerStaticCatalogModelCount ?? 1) : 1 }, + (_, modelIndex) => ({ + id: + modelIndex === 0 + ? STALLED_CATALOG_MODEL_ID + : `${STALLED_CATALOG_MODEL_ID}-${modelIndex + 1}`, + name: `Benchmark Model ${modelIndex + 1}`, + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 8_192, + }), + ); const provider = { baseUrl: "http://127.0.0.1:1/v1", api: "openai-completions", - models: [model], + models, }; const entrySource = stallsProviderCatalog ? `const provider = ${JSON.stringify(provider)};\nmodule.exports = { id: ${JSON.stringify(id)}, register(api) { api.registerProvider({ id: ${JSON.stringify(STALLED_CATALOG_PROVIDER_ID)}, label: "Benchmark Catalog Stall", auth: [], catalog: { order: "simple", run: async () => { const stopAt = Date.now() + ${providerCatalogStallMs}; while (Date.now() < stopAt) {} return { provider }; } }, staticCatalog: { order: "simple", run: async () => ({ provider }) } }); } };\n` - : `module.exports = { id: ${JSON.stringify(id)}, register() {} };\n`; + : stallsProviderStaticCatalog + ? `const provider = ${JSON.stringify(provider)};\nmodule.exports = { id: ${JSON.stringify(id)}, register(api) { api.registerProvider({ id: ${JSON.stringify(STALLED_CATALOG_PROVIDER_ID)}, label: "Benchmark Static Catalog Stall", auth: [], staticCatalog: { order: "simple", run: async () => ({ provider }) } }); } };\n` + : `module.exports = { id: ${JSON.stringify(id)}, register() {} };\n`; writeFileSync(entry, entrySource); + if (stallsProviderStaticCatalog) { + writeFileSync( + providerDiscoveryEntry, + `const provider = ${JSON.stringify(provider)};\nlet staticCatalogCallCount = 0;\nmodule.exports = { id: ${JSON.stringify(STALLED_CATALOG_PROVIDER_ID)}, label: "Benchmark Static Catalog Stall", auth: [], staticCatalog: { order: "simple", run: async () => { staticCatalogCallCount += 1; console.log("startup trace: benchmark preparedRuntimeStaticCatalogCallCount=" + staticCatalogCallCount); const stopAt = Date.now() + ${providerStaticCatalogStallMs}; while (Date.now() < stopAt) {} return { provider }; } } };\n`, + ); + } writeFileSync( path.join(pluginDir, "openclaw.plugin.json"), `${JSON.stringify( @@ -677,12 +772,19 @@ function writePluginFixtures( ...(activationOnStartup === undefined ? {} : { activation: { onStartup: activationOnStartup } }), - ...(stallsProviderCatalog + ...(stallsProviderCatalog || stallsProviderStaticCatalog ? { providers: [STALLED_CATALOG_PROVIDER_ID], - modelCatalog: { - providers: { [STALLED_CATALOG_PROVIDER_ID]: provider }, - }, + ...(stallsProviderStaticCatalog + ? { providerCatalogEntry: "./provider-discovery.cjs" } + : {}), + ...(stallsProviderCatalog + ? { + modelCatalog: { + providers: { [STALLED_CATALOG_PROVIDER_ID]: provider }, + }, + } + : {}), } : {}), configSchema: { type: "object", additionalProperties: false }, @@ -695,18 +797,53 @@ function writePluginFixtures( return { pluginIds, pluginsDir }; } +function buildBenchAgentList( + root: string, + topology: GatewayBenchCase["agentTopology"], +): Array<{ id: string; default?: boolean; workspace: string }> | undefined { + if (!topology) { + return undefined; + } + const sharedWorkspace = path.join(root, "shared-workspace"); + const distinctWorkspace = path.join(root, "distinct-workspace"); + mkdirSync(sharedWorkspace, { recursive: true }); + if (topology === "single") { + return [{ id: "main", default: true, workspace: sharedWorkspace }]; + } + mkdirSync(distinctWorkspace, { recursive: true }); + return Array.from({ length: 12 }, (_, index) => ({ + id: `agent-${String(index + 1).padStart(2, "0")}`, + ...(index === 0 ? { default: true } : {}), + workspace: index === 11 ? distinctWorkspace : sharedWorkspace, + })); +} + function writeConfig(root: string, benchCase: GatewayBenchCase): string { - const pluginCount = benchCase.providerCatalogStallMs === undefined ? benchCase.pluginCount : 1; + const hasCatalogFixture = + benchCase.providerCatalogStallMs !== undefined || + benchCase.providerStaticCatalogStallMs !== undefined; + const pluginCount = hasCatalogFixture ? 1 : benchCase.pluginCount; const pluginFixtures = pluginCount ? writePluginFixtures( root, pluginCount, - benchCase.providerCatalogStallMs === undefined ? benchCase.pluginActivationOnStartup : true, + hasCatalogFixture ? true : benchCase.pluginActivationOnStartup, benchCase.providerCatalogStallMs, + benchCase.providerStaticCatalogStallMs, + benchCase.providerStaticCatalogModelCount, ) : null; + const agentList = buildBenchAgentList(root, benchCase.agentTopology); const config = { ...benchCase.config, + ...(agentList + ? { + agents: { + ...(benchCase.config.agents as Record | undefined), + list: agentList, + }, + } + : {}), plugins: { ...(benchCase.config.plugins as Record | undefined), ...(pluginFixtures @@ -919,10 +1056,18 @@ async function runGatewaySample(options: { startAt, }), ]); - const readyAt = performance.now(); + const completionMs = options.benchCase.completionTracePhase + ? await waitForStartupTracePhase({ + deadlineAt, + isDone: () => childExited, + phase: options.benchCase.completionTracePhase, + startupTrace, + }) + : performance.now() - startAt; + const completedAt = performance.now(); const cpuEndMs = readProcessTreeCpuMs(child.pid); const cpuMs = cpuStartMs == null || cpuEndMs == null ? null : Math.max(0, cpuEndMs - cpuStartMs); - const cpuCoreRatio = cpuMs == null ? null : cpuMs / Math.max(1, readyAt - startAt); + const cpuCoreRatio = cpuMs == null ? null : cpuMs / Math.max(1, completedAt - startAt); const exit = await stopChild(child); clearInterval(rssTimer); sampleRss(); @@ -931,6 +1076,7 @@ async function runGatewaySample(options: { rmSync(root, { force: true, maxRetries: 3, recursive: true, retryDelay: 100 }); return { + completionMs, cpuCoreRatio, cpuMs, exitedBeforeTeardown: exit.exitedBeforeTeardown, @@ -973,8 +1119,18 @@ async function runCase(options: { samples.push(sample); const heapUsedMb = sample.startupTrace["memory.ready.heapUsedMb"] ?? null; console.log( - `[gateway-startup-bench] ${options.benchCase.id} run ${samples.length}/${options.runs}: healthz=${formatMs(sample.healthz.ms)} readyz=${formatMs(sample.readyz.ms)} httpListen=${formatMs(sample.httpListenLogMs)} gatewayReady=${formatMs(sample.gatewayReadyLogMs)} cpu=${formatMs(sample.cpuMs)} cpuCore=${formatRatio(sample.cpuCoreRatio)} rss=${formatMb(sample.maxRssMb)} heap=${formatMb(heapUsedMb)}`, + `[gateway-startup-bench] ${options.benchCase.id} run ${samples.length}/${options.runs}: completion=${formatMs(sample.completionMs)} healthz=${formatMs(sample.healthz.ms)} readyz=${formatMs(sample.readyz.ms)} httpListen=${formatMs(sample.httpListenLogMs)} gatewayReady=${formatMs(sample.gatewayReadyLogMs)} cpu=${formatMs(sample.cpuMs)} cpuCore=${formatRatio(sample.cpuCoreRatio)} rss=${formatMb(sample.maxRssMb)} heap=${formatMb(heapUsedMb)}`, ); + if ( + sample.outputTail && + (sample.completionMs == null || + sample.healthz.status !== 200 || + sample.readyz.status !== 200) + ) { + console.error( + `[gateway-startup-bench] ${options.benchCase.id} output tail:\n${sample.outputTail}`, + ); + } } else { const heapUsedMb = sample.startupTrace["memory.ready.heapUsedMb"] ?? null; console.log( @@ -987,6 +1143,7 @@ async function runCase(options: { function printResult(result: CaseResult): void { console.log(`\n${result.name} (${result.id})`); + console.log(` completion: ${formatStats(result.summary.completionMs)}`); console.log(` first output: ${formatStats(result.summary.firstOutputMs)}`); console.log(` CPU: ${formatStats(result.summary.cpuMs)}`); console.log(` CPU core: ${formatRatioStats(result.summary.cpuCoreRatio)}`); @@ -1001,6 +1158,9 @@ function printResult(result: CaseResult): void { console.log( ` post-ready memory: rss=${formatMemoryStats(result.summary.startupTrace["memory.post-ready.rssMb"])} heap=${formatMemoryStats(result.summary.startupTrace["memory.post-ready.heapUsedMb"])} external=${formatMemoryStats(result.summary.startupTrace["memory.post-ready.externalMb"])}`, ); + console.log( + ` prepared runtime: agents=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.agentCount"])} workspaces=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.workspaceGroupCount"])} configuredGroups=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.configuredFactsGroupCount"])} configuredModels=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.configuredRuntimeModelCount"])} generatedPlugins=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.generatedCatalogPluginCount"])} generatedReads=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.generatedCatalogReadCount"])} sources=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.catalogSourceCount"])} credentials=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.credentialGroupCount"])} catalogs=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.catalogGroupCount"])} registries=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.runtimeRegistryCount"])} workspaceFacts=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.workspaceFactsMs"])} runtimePlugins=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.runtimePluginMs"])} metadata=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.pluginMetadataMs"])} staticProviders=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.staticProviderCatalogMs"])} ambientAuth=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.ambientCredentialsMs"])} agentFacts=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.agentFactsMs"])} configuredProjection=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.configuredProjectionMs"])} sourceMs=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.catalogSourceMs"])} registryMs=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.registryMs"])} sourceLimit=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.sourceConcurrencyLimitCount"])} fullCatalogLimit=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.fullCatalogConcurrencyLimitCount"])} staticCatalog=${formatStats(result.summary.startupTrace["benchmark.preparedRuntimeStaticCatalogCallCount"])} pluginLoader=${formatStats(result.summary.startupTrace["sidecars.plugin-loader.callsCount"])} eventLoopMax=${formatStats(result.summary.startupTrace["sidecars.model-runtime.eventLoopMax"])}`, + ); const trace = selectSlowStartupTraceDurations(result.summary.startupTrace, 8); if (trace.length > 0) { console.log(" trace top:"); @@ -1077,6 +1237,7 @@ export const testing = { summarizeCase, validateCliArgs, waitForProbe, + waitForStartupTracePhase, writeConfig, }; diff --git a/src/agents/agent-auth-discovery.external-cli.test.ts b/src/agents/agent-auth-discovery.external-cli.test.ts index c2ed98a762f9..0e30e905f666 100644 --- a/src/agents/agent-auth-discovery.external-cli.test.ts +++ b/src/agents/agent-auth-discovery.external-cli.test.ts @@ -5,9 +5,6 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; const storeMocks = vi.hoisted(() => ({ ensureAuthProfileStore: vi.fn(() => ({ version: 1, profiles: {} })), ensureAuthProfileStoreWithoutExternalProfiles: vi.fn(() => ({ version: 1, profiles: {} })), - loadAuthProfileStoreWithoutExternalProfiles: vi.fn(() => ({ version: 1, profiles: {} })), - loadAuthProfileStoreForRuntime: vi.fn(() => ({ version: 1, profiles: {} })), - loadAuthProfileStoreForSecretsRuntime: vi.fn(() => ({ version: 1, profiles: {} })), })); const credentialMocks = vi.hoisted(() => ({ @@ -44,6 +41,7 @@ import { externalCliDiscoveryForProviders } from "./auth-profiles/external-cli-d describe("resolveAgentCredentialsForDiscovery external CLI scoping", () => { beforeEach(() => { vi.clearAllMocks(); + credentialMocks.resolveAgentCredentialMapFromStore.mockReturnValue({}); }); it("threads scoped external CLI discovery into writable auth store loading", () => { @@ -64,10 +62,9 @@ describe("resolveAgentCredentialsForDiscovery external CLI scoping", () => { config: cfg, externalCli, }); - expect(storeMocks.loadAuthProfileStoreForRuntime).not.toHaveBeenCalled(); }); - it("preserves scoped external CLI discovery for read-only auth store loading", () => { + it("reuses the active runtime generation for read-only auth discovery", () => { const cfg = {} as OpenClawConfig; const externalCli = externalCliDiscoveryForProviders({ cfg, @@ -81,7 +78,7 @@ describe("resolveAgentCredentialsForDiscovery external CLI scoping", () => { readOnly: true, }); - expect(storeMocks.loadAuthProfileStoreForRuntime).toHaveBeenCalledWith("/tmp/openclaw-agent", { + expect(storeMocks.ensureAuthProfileStore).toHaveBeenCalledWith("/tmp/openclaw-agent", { allowKeychainPrompt: false, config: cfg, externalCli, @@ -89,6 +86,29 @@ describe("resolveAgentCredentialsForDiscovery external CLI scoping", () => { }); }); + it("merges prepared ambient credentials without repeating ambient discovery", () => { + credentialMocks.resolveAgentCredentialMapFromStore.mockReturnValue({ + fireworks: { type: "api_key", key: "agent-key" }, + }); + + const credentials = resolveAgentCredentialsForDiscovery("/tmp/openclaw-agent", { + ambientCredentials: { + fireworks: { type: "api_key", key: "ambient-key" }, + "claude-cli": { type: "api_key", key: "synthetic-key" }, + }, + env: {}, + readOnly: true, + }); + + expect(credentials).toEqual({ + fireworks: { type: "api_key", key: "agent-key" }, + "claude-cli": { type: "api_key", key: "synthetic-key" }, + }); + expect(discoveryCoreMocks.addEnvBackedAgentCredentials).not.toHaveBeenCalled(); + expect(syntheticAuthMocks.resolveRuntimeSyntheticAuthProviderRefs).not.toHaveBeenCalled(); + expect(syntheticAuthMocks.resolveProviderSyntheticAuthWithPlugin).not.toHaveBeenCalled(); + }); + it("can skip runtime external auth overlays and scope synthetic auth discovery", () => { resolveAgentCredentialsForDiscovery("/tmp/openclaw-agent", { env: {}, @@ -106,6 +126,9 @@ describe("resolveAgentCredentialsForDiscovery external CLI scoping", () => { expect(syntheticAuthMocks.resolveRuntimeSyntheticAuthProviderRefs).not.toHaveBeenCalled(); expect(syntheticAuthMocks.resolveProviderSyntheticAuthWithPlugin).toHaveBeenCalledWith({ provider: "fireworks", + config: undefined, + workspaceDir: undefined, + env: {}, context: { config: undefined, provider: "fireworks", diff --git a/src/agents/agent-auth-discovery.ts b/src/agents/agent-auth-discovery.ts index fe8dc908ea98..e75463c7171e 100644 --- a/src/agents/agent-auth-discovery.ts +++ b/src/agents/agent-auth-discovery.ts @@ -14,13 +14,11 @@ import type { ExternalCliAuthDiscovery } from "./auth-profiles/external-cli-disc import { ensureAuthProfileStore, ensureAuthProfileStoreWithoutExternalProfiles, - loadAuthProfileStoreWithoutExternalProfiles, - loadAuthProfileStoreForRuntime, - loadAuthProfileStoreForSecretsRuntime, } from "./auth-profiles/store.js"; /** Options for discovering credentials without prompting for secret material. */ export type DiscoverAuthStorageOptions = { + ambientCredentials?: Readonly; externalCli?: ExternalCliAuthDiscovery; inheritedAuthDir?: string; readOnly?: boolean; @@ -29,6 +27,62 @@ export type DiscoverAuthStorageOptions = { syntheticAuthProviderRefs?: Iterable; } & AgentDiscoveryAuthLookupOptions; +type AmbientAgentCredentialOptions = AgentDiscoveryAuthLookupOptions & { + resolveSyntheticAuth?: (provider: string) => { apiKey?: string } | undefined; + syntheticAuthProviderRefs?: Iterable; +}; + +/** Resolves workspace/config/env-stable credentials independently of agent-local profiles. */ +export function resolveAmbientAgentCredentialsForDiscovery( + options: AmbientAgentCredentialOptions = {}, +): AgentCredentialMap { + const credentials = addEnvBackedAgentCredentials({}, options); + const syntheticAuthProviderRefs = + options.syntheticAuthProviderRefs ?? resolveRuntimeSyntheticAuthProviderRefs(); + const resolveSyntheticAuth = + options.resolveSyntheticAuth ?? + ((provider: string) => + resolveProviderSyntheticAuthWithPlugin({ + provider, + config: options.config, + workspaceDir: options.workspaceDir, + env: options.env, + context: { + config: options.config, + provider, + providerConfig: options.config?.models?.providers?.[provider], + }, + })); + for (const provider of syntheticAuthProviderRefs) { + if (credentials[provider]) { + continue; + } + if ( + !isAmbientCredentialAllowedByProviderAuthPin({ + config: options.config, + authAliasLookupParams: { + ...(options.env ? { env: options.env } : {}), + ...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}), + }, + provider, + type: "api_key", + }) + ) { + continue; + } + const resolved = resolveSyntheticAuth(provider); + const apiKey = resolved?.apiKey?.trim(); + if (!apiKey) { + continue; + } + credentials[provider] = { + type: "api_key", + key: apiKey, + }; + } + return credentials; +} + /** Resolves agent credentials from auth profiles, env, and synthetic auth hooks. */ export function resolveAgentCredentialsForDiscovery( agentDir: string, @@ -42,68 +96,33 @@ export function resolveAgentCredentialsForDiscovery( }; const store = options?.skipExternalAuthProfiles === true - ? options.readOnly === true - ? loadAuthProfileStoreWithoutExternalProfiles( - agentDir, - options.inheritedAuthDir ? { inheritedAuthDir: options.inheritedAuthDir } : undefined, - ) - : ensureAuthProfileStoreWithoutExternalProfiles(agentDir, { - allowKeychainPrompt: false, - ...(options?.inheritedAuthDir ? { inheritedAuthDir: options.inheritedAuthDir } : {}), - }) - : options?.readOnly === true - ? options.externalCli || options.config || options.inheritedAuthDir - ? loadAuthProfileStoreForRuntime(agentDir, { readOnly: true, ...storeOptions }) - : loadAuthProfileStoreForSecretsRuntime(agentDir) - : ensureAuthProfileStore(agentDir, storeOptions); - const credentials = addEnvBackedAgentCredentials( - resolveAgentCredentialMapFromStore(store, { - includeSecretRefPlaceholders: options?.readOnly === true, - config: options?.config, - }), - { + ? ensureAuthProfileStoreWithoutExternalProfiles(agentDir, { + allowKeychainPrompt: false, + ...(options?.inheritedAuthDir ? { inheritedAuthDir: options.inheritedAuthDir } : {}), + ...(options?.readOnly === true ? { readOnly: true } : {}), + }) + : ensureAuthProfileStore(agentDir, { + ...storeOptions, + ...(options?.readOnly === true ? { readOnly: true } : {}), + }); + const credentials = resolveAgentCredentialMapFromStore(store, { + includeSecretRefPlaceholders: options?.readOnly === true, + config: options?.config, + }); + const ambientCredentials = + options?.ambientCredentials ?? + resolveAmbientAgentCredentialsForDiscovery({ config: options?.config, workspaceDir: options?.workspaceDir, env: options?.env, - }, - ); - const syntheticAuthProviderRefs = - options?.syntheticAuthProviderRefs ?? resolveRuntimeSyntheticAuthProviderRefs(); - for (const provider of syntheticAuthProviderRefs) { + syntheticAuthProviderRefs: options?.syntheticAuthProviderRefs, + }); + for (const [provider, credential] of Object.entries(ambientCredentials)) { if (credentials[provider]) { continue; } - if ( - !isAmbientCredentialAllowedByProviderAuthPin({ - config: options?.config, - authAliasLookupParams: { - ...(options?.env ? { env: options.env } : {}), - ...(options?.workspaceDir ? { workspaceDir: options.workspaceDir } : {}), - }, - provider, - type: "api_key", - }) - ) { - continue; - } - // Synthetic auth is a plugin/runtime fallback. Only fill empty providers so - // persisted profiles and env-backed credentials remain authoritative. - const resolved = resolveProviderSyntheticAuthWithPlugin({ - provider, - context: { - config: undefined, - provider, - providerConfig: undefined, - }, - }); - const apiKey = resolved?.apiKey?.trim(); - if (!apiKey) { - continue; - } - credentials[provider] = { - type: "api_key", - key: apiKey, - }; + // Ambient auth is a lifecycle-owned fallback. Agent-local profiles remain authoritative. + credentials[provider] = credential; } return credentials; } diff --git a/src/agents/agent-model-discovery.synthetic-auth.test.ts b/src/agents/agent-model-discovery.synthetic-auth.test.ts index d07dcb4bdac2..0494cbffbc90 100644 --- a/src/agents/agent-model-discovery.synthetic-auth.test.ts +++ b/src/agents/agent-model-discovery.synthetic-auth.test.ts @@ -31,7 +31,7 @@ vi.mock("../plugins/provider-runtime.js", () => ({ vi.mock("./auth-profiles/store.js", () => ({ ensureAuthProfileStore: () => ({ version: 1, profiles: {} }), - loadAuthProfileStoreForSecretsRuntime: () => ({ version: 1, profiles: {} }), + ensureAuthProfileStoreWithoutExternalProfiles: () => ({ version: 1, profiles: {} }), })); vi.mock("./agent-auth-discovery-core.js", () => ({ @@ -74,6 +74,9 @@ describe("agent model discovery synthetic auth", () => { expect(resolveProviderSyntheticAuthWithPlugin).toHaveBeenCalledTimes(1); expect(resolveProviderSyntheticAuthWithPlugin).toHaveBeenCalledWith({ provider: "claude-cli", + config: undefined, + workspaceDir: undefined, + env: undefined, context: { config: undefined, provider: "claude-cli", diff --git a/src/agents/agent-model-discovery.test.ts b/src/agents/agent-model-discovery.test.ts index ef435a106020..7afdce937dae 100644 --- a/src/agents/agent-model-discovery.test.ts +++ b/src/agents/agent-model-discovery.test.ts @@ -5,7 +5,11 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { clearCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-state.js"; -import { discoverAuthStorage, discoverModels } from "./agent-model-discovery.js"; +import { + discoverAuthStorage, + discoverModels, + discoverModelsFromCapturedSources, +} from "./agent-model-discovery.js"; // Discovery must not cold-load bundled plugin runtime: with build artifacts // present, the openai plugin's normalizeResolvedModel currently overrides @@ -37,6 +41,33 @@ function writeModelsJson(agentDir: string, modelId: string): void { } describe("discoverModels", () => { + it("uses a directory-independent source label for lifecycle-captured catalogs", () => { + const firstAgentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-agent-models-first-")); + const secondAgentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-agent-models-second-")); + try { + const createRegistry = (agentDir: string) => + discoverModelsFromCapturedSources( + discoverAuthStorage(agentDir, { skipCredentials: true }), + { + includePluginCatalogs: true, + modelsJsonContents: "not valid json", + pluginCatalogs: [], + }, + ); + + const firstError = createRegistry(firstAgentDir).getError(); + const secondError = createRegistry(secondAgentDir).getError(); + + expect(firstError).toBe(secondError); + expect(firstError).toContain("captured:models.json"); + expect(firstError).not.toContain(firstAgentDir); + expect(secondError).not.toContain(secondAgentDir); + } finally { + fs.rmSync(firstAgentDir, { recursive: true, force: true }); + fs.rmSync(secondAgentDir, { recursive: true, force: true }); + } + }); + it("clears cached find results when the agent model registry refreshes", () => { const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-agent-models-")); writeModelsJson(agentDir, "old-model"); diff --git a/src/agents/agent-model-discovery.ts b/src/agents/agent-model-discovery.ts index 0df997d801a6..84ce639386c0 100644 --- a/src/agents/agent-model-discovery.ts +++ b/src/agents/agent-model-discovery.ts @@ -15,6 +15,7 @@ import { } from "./agent-auth-discovery.js"; import { resolveModelPluginMetadataSnapshot } from "./model-discovery-context.js"; import type { PluginModelCatalogMetadataSnapshot } from "./plugin-model-catalog.js"; +import type { PersistedPluginModelCatalog } from "./plugin-model-catalog.js"; import { AuthStorage, ModelRegistry, @@ -30,14 +31,27 @@ type DiscoveredProviderRuntimeModelLike = Omit api?: string | null; }; +const CAPTURED_MODELS_JSON_SOURCE_PATH = "captured:models.json"; + type DiscoverModelsOptions = { config?: OpenClawConfig; + includePluginCatalogs?: boolean; + modelsJsonContents?: string | null; + pluginCatalogs?: readonly PersistedPluginModelCatalog[]; providerFilter?: string; pluginMetadataSnapshot?: PluginModelCatalogMetadataSnapshot; workspaceDir?: string; normalizeModels?: boolean; }; +type DiscoverCapturedModelsOptions = Omit< + DiscoverModelsOptions, + "modelsJsonContents" | "normalizeModels" | "pluginCatalogs" +> & { + modelsJsonContents: string | null; + pluginCatalogs: readonly PersistedPluginModelCatalog[]; +}; + /** Applies plugin model normalization and transport hooks to discovered agent models. */ export function normalizeDiscoveredAgentModel( value: T, @@ -98,7 +112,7 @@ export function normalizeDiscoveredAgentModel( function createOpenClawModelRegistry( authStorage: AgentAuthStorage, modelsJsonPath: string, - agentDir: string, + agentDir: string | undefined, options?: DiscoverModelsOptions, ): AgentModelRegistry { const pluginMetadataSnapshot = resolveModelPluginMetadataSnapshot({ @@ -110,7 +124,16 @@ function createOpenClawModelRegistry( allowWorkspaceScopedCurrent: options?.workspaceDir === undefined, useRuntimeConfig: options?.config === undefined, }); - const registryOptions = pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}; + const registryOptions = { + ...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}), + ...(options?.includePluginCatalogs !== undefined + ? { includePluginCatalogs: options.includePluginCatalogs } + : {}), + ...(options?.modelsJsonContents !== undefined + ? { modelsJsonContents: options.modelsJsonContents } + : {}), + ...(options?.pluginCatalogs !== undefined ? { pluginCatalogs: options.pluginCatalogs } : {}), + }; const registry = ModelRegistry.create(authStorage, modelsJsonPath, registryOptions); const getAll = registry.getAll.bind(registry); const getAvailable = registry.getAvailable.bind(registry); @@ -121,8 +144,15 @@ function createOpenClawModelRegistry( !providerFilter || normalizeProviderId(entry.provider) === providerFilter; const shouldNormalize = options?.normalizeModels !== false; const findCache = new Map(); - const normalizeEntry = (entry: Model) => - shouldNormalize ? normalizeDiscoveredAgentModel(entry, agentDir, options) : entry; + const normalizeEntry = (entry: Model) => { + if (!shouldNormalize) { + return entry; + } + if (!agentDir) { + throw new Error("agent directory is required for model normalization"); + } + return normalizeDiscoveredAgentModel(entry, agentDir, options); + }; registry.getAll = () => { const entries = getAll().filter((entry: Model) => matchesProviderFilter(entry)); @@ -151,7 +181,6 @@ function createOpenClawModelRegistry( return registry; } -/** Creates auth storage for model discovery from stored and env-backed credentials. */ /** Builds auth storage for model discovery without prompting for secrets. */ export function discoverAuthStorage( agentDir: string, @@ -176,3 +205,17 @@ export function discoverModels( options, ); } + +/** + * Parses complete lifecycle-captured sources without retaining an agent-directory dependency. + * Callers may share the resulting immutable catalog snapshot across exact source generations. + */ +export function discoverModelsFromCapturedSources( + authStorage: AgentAuthStorage, + options: DiscoverCapturedModelsOptions, +): AgentModelRegistry { + return createOpenClawModelRegistry(authStorage, CAPTURED_MODELS_JSON_SOURCE_PATH, undefined, { + ...options, + normalizeModels: false, + }); +} diff --git a/src/agents/ai-transport-runtime-host.ts b/src/agents/ai-transport-runtime-host.ts index 82082b0d8e51..40df866b6347 100644 --- a/src/agents/ai-transport-runtime-host.ts +++ b/src/agents/ai-transport-runtime-host.ts @@ -26,6 +26,7 @@ import { import { attachModelProviderRequestTransport, getModelProviderRequestTransport, + inheritModelProviderMetadataOwners, resolveProviderRequestPolicyConfig, } from "./provider-request-config.js"; import { transformTransportMessages } from "./transport-message-transform.js"; @@ -89,9 +90,12 @@ export function configureAiTransportRuntimeHost(): void { return Boolean(request?.proxy || request?.tls || getModelProviderLocalService(model)); }, inheritManagedTransport: (source, target) => - attachModelProviderLocalService( - attachModelProviderRequestTransport(target, getModelProviderRequestTransport(source)), - getModelProviderLocalService(source), + inheritModelProviderMetadataOwners( + source, + attachModelProviderLocalService( + attachModelProviderRequestTransport(target, getModelProviderRequestTransport(source)), + getModelProviderLocalService(source), + ), ), transformTransportMessages, registerCustomApi: ensureCustomApiRegistered, diff --git a/src/agents/channel-tools.test.ts b/src/agents/channel-tools.test.ts index 26515f1a2373..f27933e42da2 100644 --- a/src/agents/channel-tools.test.ts +++ b/src/agents/channel-tools.test.ts @@ -2,12 +2,17 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { ChannelPlugin } from "../channels/plugins/types.public.js"; import type { OpenClawConfig } from "../config/config.js"; -import { EMPTY_PREPARED_MESSAGE_TOOL_CATALOG } from "../plugins/prepared-message-tool-catalog.js"; import { setActivePluginRegistry } from "../plugins/runtime.js"; import { defaultRuntime } from "../runtime.js"; import { createTestRegistry } from "../test-utils/channel-plugins.js"; import { listAllChannelSupportedActions, listChannelSupportedActions } from "./channel-tools.js"; +const EMPTY_PREPARED_MESSAGE_TOOL_CATALOG = { + version: 0, + channels: [], + getChannel: () => undefined, +} as const; + describe("channel tools", () => { const errorSpy = vi.spyOn(defaultRuntime, "error").mockImplementation(() => undefined); diff --git a/src/agents/embedded-agent-runner/model.configured-fallback.ts b/src/agents/embedded-agent-runner/model.configured-fallback.ts index fa856463f818..e13977481c9e 100644 --- a/src/agents/embedded-agent-runner/model.configured-fallback.ts +++ b/src/agents/embedded-agent-runner/model.configured-fallback.ts @@ -1,9 +1,11 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js"; import type { Model } from "../../llm/types.js"; +import type { PluginMetadataSnapshotOwnerMaps } from "../../plugins/plugin-metadata-snapshot.types.js"; import { DEFAULT_CONTEXT_TOKENS } from "../defaults.js"; import { resolveCatalogOwnedModelCompat } from "../model-compat-catalog.js"; import { attachModelProviderLocalService } from "../provider-local-service.js"; import { + attachModelProviderMetadataOwners, attachModelProviderRequestTransport, resolveProviderRequestConfig, sanitizeConfiguredModelProviderRequest, @@ -42,6 +44,7 @@ export function resolveConfiguredFallbackModel(params: { cfg?: OpenClawConfig; agentDir?: string; manifestAlias: ManifestModelCatalogProviderAliasMetadata; + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps; workspaceDir?: string; runtimeHooks?: ProviderRuntimeHooks; }): Model | undefined { @@ -138,6 +141,9 @@ export function resolveConfiguredFallbackModel(params: { provider, api: fallbackTransport.api ?? "openai-responses", baseUrl: fallbackTransport.baseUrl, + ...(params.providerMetadataOwners + ? { providerMetadataOwners: params.providerMetadataOwners } + : {}), discoveredHeaders: staticCatalogHeaders, providerHeaders, modelHeaders, @@ -171,52 +177,55 @@ export function resolveConfiguredFallbackModel(params: { cfg, agentDir, workspaceDir, - model: attachModelProviderLocalService( - attachModelProviderRequestTransport( - { - id: modelId, - name: metadataModel?.name ?? modelId, - api: requestConfig.api ?? "openai-responses", - provider, - baseUrl: requestConfig.baseUrl, - reasoning: fallbackReasoning, - input: resolveProviderModelInput({ + model: attachModelProviderMetadataOwners( + attachModelProviderLocalService( + attachModelProviderRequestTransport( + { + id: modelId, + name: metadataModel?.name ?? modelId, + api: requestConfig.api ?? "openai-responses", provider, - modelId, - modelName: metadataModel?.name ?? modelId, - input: metadataModel?.input, - }), - ...(configuredModel?.thinkingLevelMap !== undefined - ? { thinkingLevelMap: configuredModel.thinkingLevelMap } - : {}), - cost: metadataModel?.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: resolvedFallbackContextWindow, - contextTokens: - configuredModel?.contextTokens ?? - providerConfig?.contextTokens ?? - providerConfig?.models?.[0]?.contextTokens ?? - staticCatalogModel?.contextTokens, - // maxTokens is a wire-level output cap, not a context-budget fallback. - // Omit an unknown cap so strict providers can apply their own limit. - ...(normalizedResolvedFallbackMaxTokens !== undefined - ? { - maxTokens: normalizedResolvedFallbackMaxTokens, - maxTokensSource: - configuredFallbackMaxTokens !== undefined ? "configured" : "discovered", - } - : {}), - ...(resolvedParams ? { params: resolvedParams } : {}), - ...(requestTimeoutMs !== undefined ? { requestTimeoutMs } : {}), - headers: requestConfig.headers, - ...(providerConfig?.authHeader !== undefined - ? { authHeader: providerConfig.authHeader } - : {}), - compat: fallbackCompat, - mediaInput: fallbackMediaInput, - } as Model, - providerRequest, + baseUrl: requestConfig.baseUrl, + reasoning: fallbackReasoning, + input: resolveProviderModelInput({ + provider, + modelId, + modelName: metadataModel?.name ?? modelId, + input: metadataModel?.input, + }), + ...(configuredModel?.thinkingLevelMap !== undefined + ? { thinkingLevelMap: configuredModel.thinkingLevelMap } + : {}), + cost: metadataModel?.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: resolvedFallbackContextWindow, + contextTokens: + configuredModel?.contextTokens ?? + providerConfig?.contextTokens ?? + providerConfig?.models?.[0]?.contextTokens ?? + staticCatalogModel?.contextTokens, + // maxTokens is a wire-level output cap, not a context-budget fallback. + // Omit an unknown cap so strict providers can apply their own limit. + ...(normalizedResolvedFallbackMaxTokens !== undefined + ? { + maxTokens: normalizedResolvedFallbackMaxTokens, + maxTokensSource: + configuredFallbackMaxTokens !== undefined ? "configured" : "discovered", + } + : {}), + ...(resolvedParams ? { params: resolvedParams } : {}), + ...(requestTimeoutMs !== undefined ? { requestTimeoutMs } : {}), + headers: requestConfig.headers, + ...(providerConfig?.authHeader !== undefined + ? { authHeader: providerConfig.authHeader } + : {}), + compat: fallbackCompat, + mediaInput: fallbackMediaInput, + } as Model, + providerRequest, + ), + providerConfig?.localService, ), - providerConfig?.localService, + params.providerMetadataOwners, ), runtimeHooks, }); diff --git a/src/agents/embedded-agent-runner/model.configured-overrides.ts b/src/agents/embedded-agent-runner/model.configured-overrides.ts index ae80166bab8e..e7e644dc2fee 100644 --- a/src/agents/embedded-agent-runner/model.configured-overrides.ts +++ b/src/agents/embedded-agent-runner/model.configured-overrides.ts @@ -2,6 +2,7 @@ import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/st import type { ModelCompatConfig, ModelMediaInputConfig } from "../../config/types.models.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import type { Api, Model } from "../../llm/types.js"; +import type { PluginMetadataSnapshotOwnerMaps } from "../../plugins/plugin-metadata-snapshot.types.js"; import type { ProviderRuntimeModel } from "../../plugins/provider-runtime-model.types.js"; import { resolveCatalogOwnedModelCompat } from "../model-compat-catalog.js"; import { modelKey, normalizeStaticProviderModelId } from "../model-ref-shared.js"; @@ -9,6 +10,7 @@ import { findNormalizedProviderValue, normalizeProviderId } from "../model-selec import { shouldSuppressBuiltInModel, shouldUnconditionallySuppress } from "../model-suppression.js"; import { attachModelProviderLocalService } from "../provider-local-service.js"; import { + attachModelProviderMetadataOwners, attachModelProviderRequestTransport, resolveProviderRequestConfig, sanitizeConfiguredModelProviderRequest, @@ -328,6 +330,7 @@ export function applyConfiguredProviderOverrides(params: { modelId: string; cfg?: OpenClawConfig; manifestAlias: ManifestModelCatalogProviderAliasMetadata; + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps; runtimeHooks?: ProviderRuntimeHooks; preferDiscoveredModelMetadata?: boolean; preferDiscoveredTransport?: boolean; @@ -335,7 +338,10 @@ export function applyConfiguredProviderOverrides(params: { workspaceDir?: string; }): ProviderRuntimeModel { const { providerConfig, modelId } = params; - const discoveredModel = markDiscoveredMaxTokensSource(params.discoveredModel); + const discoveredModel = attachModelProviderMetadataOwners( + markDiscoveredMaxTokensSource(params.discoveredModel), + params.providerMetadataOwners, + ); const manifestAliasTransport = params.manifestAlias.transport; const requestTimeoutMs = resolveProviderRequestTimeoutMs(providerConfig?.timeoutSeconds); const defaultModelParams = findConfiguredAgentModelParams({ @@ -367,6 +373,9 @@ export function applyConfiguredProviderOverrides(params: { provider: params.provider, api: aliasTransport?.api ?? discoveredModel.api, baseUrl: aliasTransport?.baseUrl ?? discoveredModel.baseUrl, + ...(params.providerMetadataOwners + ? { providerMetadataOwners: params.providerMetadataOwners } + : {}), discoveredHeaders, capability: "llm", transport: "stream", @@ -419,6 +428,9 @@ export function applyConfiguredProviderOverrides(params: { provider: params.provider, api: discoveredModel.api, baseUrl: discoveredModel.baseUrl, + ...(params.providerMetadataOwners + ? { providerMetadataOwners: params.providerMetadataOwners } + : {}), discoveredHeaders, providerHeaders, modelHeaders: configuredHeaders, @@ -562,6 +574,9 @@ export function applyConfiguredProviderOverrides(params: { "openai-responses", baseUrl: resolvedTransport.baseUrl ?? configuredStaticCatalogModel?.baseUrl ?? discoveredModel.baseUrl, + ...(params.providerMetadataOwners + ? { providerMetadataOwners: params.providerMetadataOwners } + : {}), discoveredHeaders, providerHeaders, modelHeaders: configuredHeaders, @@ -570,47 +585,50 @@ export function applyConfiguredProviderOverrides(params: { capability: "llm", transport: "stream", }); - return attachModelProviderLocalService( - attachModelProviderRequestTransport( - { - ...discoveredModel, - provider: params.provider, - api: requestConfig.api ?? "openai-responses", - baseUrl: requestConfig.baseUrl ?? discoveredModel.baseUrl, - reasoning: resolvedReasoning, - input: normalizedInput, - cost: metadataOverrideModel?.cost ?? discoveredModel.cost, - contextWindow: resolvedContextWindow ?? discoveredModel.contextWindow, - contextTokens: - metadataOverrideModel?.contextTokens ?? - providerConfig.contextTokens ?? - discoveredModel.contextTokens, - ...(normalizedResolvedMaxTokens !== undefined - ? { - maxTokens: normalizedResolvedMaxTokens, - maxTokensSource: - configuredMaxTokens !== undefined - ? "configured" - : (discoveredModel.maxTokensSource ?? "discovered"), - } - : {}), - ...(resolvedParams ? { params: resolvedParams } : {}), - ...(requestTimeoutMs !== undefined ? { requestTimeoutMs } : {}), - headers: requestConfig.headers, - ...(providerConfig.authHeader !== undefined - ? { authHeader: providerConfig.authHeader } - : {}), - compat: resolvedCompat, - mediaInput: mergeModelMediaInput( - mergeModelMediaInput( - configuredStaticCatalogModel?.mediaInput, - discoveredModel.mediaInput, + return attachModelProviderMetadataOwners( + attachModelProviderLocalService( + attachModelProviderRequestTransport( + { + ...discoveredModel, + provider: params.provider, + api: requestConfig.api ?? "openai-responses", + baseUrl: requestConfig.baseUrl ?? discoveredModel.baseUrl, + reasoning: resolvedReasoning, + input: normalizedInput, + cost: metadataOverrideModel?.cost ?? discoveredModel.cost, + contextWindow: resolvedContextWindow ?? discoveredModel.contextWindow, + contextTokens: + metadataOverrideModel?.contextTokens ?? + providerConfig.contextTokens ?? + discoveredModel.contextTokens, + ...(normalizedResolvedMaxTokens !== undefined + ? { + maxTokens: normalizedResolvedMaxTokens, + maxTokensSource: + configuredMaxTokens !== undefined + ? "configured" + : (discoveredModel.maxTokensSource ?? "discovered"), + } + : {}), + ...(resolvedParams ? { params: resolvedParams } : {}), + ...(requestTimeoutMs !== undefined ? { requestTimeoutMs } : {}), + headers: requestConfig.headers, + ...(providerConfig.authHeader !== undefined + ? { authHeader: providerConfig.authHeader } + : {}), + compat: resolvedCompat, + mediaInput: mergeModelMediaInput( + mergeModelMediaInput( + configuredStaticCatalogModel?.mediaInput, + discoveredModel.mediaInput, + ), + metadataOverrideModel?.mediaInput, ), - metadataOverrideModel?.mediaInput, - ), - }, - providerRequest, + }, + providerRequest, + ), + providerConfig.localService, ), - providerConfig.localService, + params.providerMetadataOwners, ); } diff --git a/src/agents/embedded-agent-runner/model.inline-provider.ts b/src/agents/embedded-agent-runner/model.inline-provider.ts index e31aa1a67d94..47d038f98d5e 100644 --- a/src/agents/embedded-agent-runner/model.inline-provider.ts +++ b/src/agents/embedded-agent-runner/model.inline-provider.ts @@ -5,9 +5,11 @@ import { normalizeOptionalLowercaseString } from "@openclaw/normalization-core/s import type { ModelDefinitionConfig, ModelProviderConfig } from "../../config/types.js"; import { normalizeGoogleApiBaseUrl } from "../../infra/google-api-base-url.js"; import type { Api } from "../../llm/types.js"; +import type { PluginMetadataSnapshotOwnerMaps } from "../../plugins/plugin-metadata-snapshot.types.js"; import { isSecretRefHeaderValueMarker } from "../model-auth-markers.js"; import { attachModelProviderLocalService } from "../provider-local-service.js"; import { + attachModelProviderMetadataOwners, attachModelProviderRequestTransport, resolveProviderRequestConfig, sanitizeConfiguredModelProviderRequest, @@ -141,6 +143,7 @@ function resolveInlineProviderTransport(params: { api?: Api | null; baseUrl?: st /** Builds runtime model records from inline provider config, inheriting provider-level defaults. */ export function buildInlineProviderModels( providers: Record, + options: { providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps } = {}, ): InlineModelEntry[] { return Object.entries(providers).flatMap(([providerId, entry]) => { const trimmed = providerId.trim(); @@ -163,6 +166,9 @@ export function buildInlineProviderModels( provider: trimmed, api: transport.api ?? model.api, baseUrl: transport.baseUrl, + ...(options.providerMetadataOwners + ? { providerMetadataOwners: options.providerMetadataOwners } + : {}), providerHeaders, modelHeaders, authHeader: entry?.authHeader, @@ -170,27 +176,30 @@ export function buildInlineProviderModels( capability: "llm", transport: "stream", }); - return attachModelProviderLocalService( - attachModelProviderRequestTransport( - { - ...model, - contextWindow: model.contextWindow ?? entry?.contextWindow, - contextTokens: model.contextTokens ?? entry?.contextTokens, - maxTokens: model.maxTokens ?? entry?.maxTokens, - input: resolveProviderModelInput({ + return attachModelProviderMetadataOwners( + attachModelProviderLocalService( + attachModelProviderRequestTransport( + { + ...model, + contextWindow: model.contextWindow ?? entry?.contextWindow, + contextTokens: model.contextTokens ?? entry?.contextTokens, + maxTokens: model.maxTokens ?? entry?.maxTokens, + input: resolveProviderModelInput({ + provider: trimmed, + modelId: model.id, + modelName: model.name, + input: model.input, + }), provider: trimmed, - modelId: model.id, - modelName: model.name, - input: model.input, - }), - provider: trimmed, - baseUrl: requestConfig.baseUrl ?? transport.baseUrl, - api: requestConfig.api ?? model.api, - headers: requestConfig.headers, - }, - providerRequest, + baseUrl: requestConfig.baseUrl ?? transport.baseUrl, + api: requestConfig.api ?? model.api, + headers: requestConfig.headers, + }, + providerRequest, + ), + entry?.localService, ), - entry?.localService, + options.providerMetadataOwners, ); }); }); diff --git a/src/agents/embedded-agent-runner/model.provider-hooks.ts b/src/agents/embedded-agent-runner/model.provider-hooks.ts index 2a525f4876e1..7cc4cc5aac27 100644 --- a/src/agents/embedded-agent-runner/model.provider-hooks.ts +++ b/src/agents/embedded-agent-runner/model.provider-hooks.ts @@ -12,6 +12,7 @@ import { shouldPreferProviderRuntimeResolvedModel, } from "../../plugins/provider-runtime.js"; import { canonicalizeOpenAIModelId } from "../openai-routing.js"; +import { inheritModelProviderMetadataOwners } from "../provider-request-config.js"; import { normalizeResolvedTransportApi, resolveProviderModelInput, @@ -231,10 +232,13 @@ export function normalizeResolvedModel(params: { normalizedInputModel.requestTimeoutMs !== undefined ? { ...normalizedModel, requestTimeoutMs: normalizedInputModel.requestTimeoutMs } : normalizedModel; - return canonicalizeLegacyResolvedModel({ - provider: params.provider, - model: modelWithProviderTimeout, - }); + return inheritModelProviderMetadataOwners( + params.model, + canonicalizeLegacyResolvedModel({ + provider: params.provider, + model: modelWithProviderTimeout, + }), + ); } export function resolveProviderTransport(params: { diff --git a/src/agents/embedded-agent-runner/model.registry-resolution.ts b/src/agents/embedded-agent-runner/model.registry-resolution.ts index c93affc07160..81d1b3cc7531 100644 --- a/src/agents/embedded-agent-runner/model.registry-resolution.ts +++ b/src/agents/embedded-agent-runner/model.registry-resolution.ts @@ -1,6 +1,7 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js"; import type { ModelRegistry as CoreModelRegistry } from "../../llm/model-registry.js"; import type { Model } from "../../llm/types.js"; +import type { PluginMetadataSnapshotOwnerMaps } from "../../plugins/plugin-metadata-snapshot.types.js"; import { ensureAuthProfileStore, resolveAuthProfileOrder } from "../auth-profiles.js"; import type { AuthProfileCredential } from "../auth-profiles/types.js"; import { resolveAgentHarnessPolicy } from "../harness/policy.js"; @@ -35,6 +36,16 @@ type ExplicitModelResolution = | { kind: "resolved"; dropOnRuntimeMiss: boolean; model: Model; source: "registry" } | { kind: "suppressed" }; +function getRegistryProviderMetadataOwners( + modelRegistry: CoreModelRegistry, +): PluginMetadataSnapshotOwnerMaps | undefined { + return ( + modelRegistry as CoreModelRegistry & { + getProviderMetadataOwners?: () => PluginMetadataSnapshotOwnerMaps | undefined; + } + ).getProviderMetadataOwners?.(); +} + export function resolveExplicitModelWithRegistry(params: { provider: string; modelId: string; @@ -48,6 +59,7 @@ export function resolveExplicitModelWithRegistry(params: { preparedStaticCatalogModel?: StaticCatalogFallbackModel; }): ExplicitModelResolution | undefined { const { provider, modelId, modelRegistry, cfg, agentDir, workspaceDir, runtimeHooks } = params; + const providerMetadataOwners = getRegistryProviderMetadataOwners(modelRegistry); const providerConfig = resolveConfiguredProviderConfig(cfg, provider); const inlineMatch = findInlineModelMatch({ providers: cfg?.models?.providers ?? {}, @@ -100,6 +112,7 @@ export function resolveExplicitModelWithRegistry(params: { modelId, cfg, manifestAlias: params.manifestAlias, + providerMetadataOwners, runtimeHooks, workspaceDir, preferDiscoveredTransport: true, @@ -158,6 +171,7 @@ export function resolveExplicitModelWithRegistry(params: { modelId, cfg, manifestAlias: params.manifestAlias, + providerMetadataOwners, runtimeHooks, workspaceDir, }), @@ -309,6 +323,7 @@ function resolvePluginDynamicModelWithRegistry(params: { modelId, cfg, manifestAlias: params.manifestAlias, + providerMetadataOwners: getRegistryProviderMetadataOwners(modelRegistry), runtimeHooks, workspaceDir, preferDiscoveredModelMetadata, @@ -446,7 +461,12 @@ export function resolveModelWithPreparedRegistry( if (pluginDynamicModel) { return pluginDynamicModel; } - return params.skipConfiguredFallback ? undefined : resolveConfiguredFallbackModel(params); + return params.skipConfiguredFallback + ? undefined + : resolveConfiguredFallbackModel({ + ...params, + providerMetadataOwners: getRegistryProviderMetadataOwners(params.modelRegistry), + }); } export function resolveModelWithRegistry( diff --git a/src/agents/embedded-agent-runner/model.static-catalog.prepared.test.ts b/src/agents/embedded-agent-runner/model.static-catalog.prepared.test.ts new file mode 100644 index 000000000000..0ab939ad3bfb --- /dev/null +++ b/src/agents/embedded-agent-runner/model.static-catalog.prepared.test.ts @@ -0,0 +1,207 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { PluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.types.js"; +import { getModelProviderMetadataOwners } from "../provider-request-config.js"; + +const mocks = vi.hoisted(() => ({ + loadPluginManifestRegistry: vi.fn(), + normalizePluginDiscoveryResult: vi.fn(), + resolveBundledProviderCompatPluginIds: vi.fn(), + resolveRuntimePluginDiscoveryProviders: vi.fn(), + runProviderStaticCatalog: vi.fn(), +})); + +vi.mock("../../plugins/current-plugin-metadata-snapshot.js", () => ({ + getCurrentPluginMetadataSnapshot: () => undefined, +})); + +vi.mock("../../plugins/manifest-metadata-scan.js", () => ({ + listOpenClawPluginManifestMetadata: () => [], +})); + +vi.mock("../../plugins/manifest-owner-policy.js", () => ({ + passesManifestOwnerBasePolicy: () => true, +})); + +vi.mock("../../plugins/manifest-registry.js", () => ({ + loadPluginManifestRegistry: mocks.loadPluginManifestRegistry, +})); + +vi.mock("../../plugins/manifest.js", () => ({ + loadPluginManifest: vi.fn(), +})); + +vi.mock("../../plugins/providers.js", () => ({ + resolveActivatableProviderOwnerPluginIds: vi.fn(), + resolveBundledProviderCompatPluginIds: mocks.resolveBundledProviderCompatPluginIds, + resolveOwningPluginIdsForProviderRef: vi.fn(), +})); + +vi.mock("../../plugins/provider-discovery.js", () => ({ + normalizePluginDiscoveryResult: mocks.normalizePluginDiscoveryResult, + resolveRuntimePluginDiscoveryProviders: mocks.resolveRuntimePluginDiscoveryProviders, + runProviderStaticCatalog: mocks.runProviderStaticCatalog, +})); + +import { loadBundledProviderStaticCatalogContextModels } from "./model.static-catalog.js"; + +const cfg = { plugins: { entries: { google: { enabled: true } } } }; +const provider = { + id: "google", + pluginId: "google", + label: "Google", + auth: [], + staticCatalog: { run: vi.fn() }, +}; +const unconfiguredProvider = { + id: "anthropic", + pluginId: "anthropic", + label: "Anthropic", + auth: [], + staticCatalog: { run: vi.fn() }, +}; + +function createMetadataSnapshot(pluginIds: string[]): PluginMetadataSnapshot { + return { + manifestRegistry: { + diagnostics: [], + plugins: pluginIds.map((id) => ({ + id, + origin: "bundled", + providerDiscoverySource: `/fixtures/${id}/provider-discovery.ts`, + })), + }, + owners: { + providerEndpoints: [], + providerRequests: new Map(), + }, + } as unknown as PluginMetadataSnapshot; +} + +describe("prepared bundled provider static catalogs", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.resolveBundledProviderCompatPluginIds.mockReturnValue(["google"]); + mocks.loadPluginManifestRegistry.mockReturnValue({ + plugins: [ + { + id: "google", + origin: "bundled", + providerDiscoverySource: "/fixtures/google/provider-discovery.ts", + }, + ], + }); + }); + + it("projects prepared rows without rerunning hooks", async () => { + mocks.resolveRuntimePluginDiscoveryProviders.mockResolvedValue([provider]); + mocks.normalizePluginDiscoveryResult.mockReturnValue({ + google: { + models: [ + { + id: "gemini-3.1-pro-preview", + name: "Gemini Pro", + contextWindow: 1_048_576, + }, + ], + }, + }); + + const metadataSnapshot = createMetadataSnapshot(["google"]); + const models = await loadBundledProviderStaticCatalogContextModels({ + cfg, + metadataSnapshot, + preparedStaticProviderCatalog: { + entries: [{ provider, result: { marker: "prepared-static-result" } as never }], + }, + }); + + expect(models).toEqual([ + expect.objectContaining({ + id: "gemini-3.1-pro-preview", + provider: "google", + contextWindow: 1_048_576, + }), + ]); + expect(getModelProviderMetadataOwners(models[0]!)).toBe(metadataSnapshot.owners); + expect(mocks.resolveRuntimePluginDiscoveryProviders).toHaveBeenCalledOnce(); + expect(mocks.runProviderStaticCatalog).not.toHaveBeenCalled(); + }); + + it("runs hooks omitted from a configured-only prepared catalog on full load", async () => { + mocks.runProviderStaticCatalog.mockResolvedValue({ marker: "full-static-result" }); + mocks.normalizePluginDiscoveryResult.mockReturnValue({ + google: { + models: [{ id: "gemini-3.1-pro-preview", contextWindow: 1_048_576 }], + }, + }); + + await expect( + loadBundledProviderStaticCatalogContextModels({ + cfg, + metadataSnapshot: createMetadataSnapshot(["google"]), + preparedStaticProviderCatalog: { + providers: [provider], + entries: [], + }, + }), + ).resolves.toEqual([ + expect.objectContaining({ + id: "gemini-3.1-pro-preview", + provider: "google", + }), + ]); + expect(mocks.resolveRuntimePluginDiscoveryProviders).not.toHaveBeenCalled(); + expect(mocks.runProviderStaticCatalog).toHaveBeenCalledWith( + expect.objectContaining({ provider }), + ); + }); + + it("discovers unconfigured providers when the full catalog is requested", async () => { + mocks.resolveBundledProviderCompatPluginIds.mockReturnValue(["anthropic", "google"]); + mocks.loadPluginManifestRegistry.mockReturnValue({ + plugins: [ + { + id: "anthropic", + origin: "bundled", + providerDiscoverySource: "/fixtures/anthropic/provider-discovery.ts", + }, + { + id: "google", + origin: "bundled", + providerDiscoverySource: "/fixtures/google/provider-discovery.ts", + }, + ], + }); + mocks.resolveRuntimePluginDiscoveryProviders.mockResolvedValue([unconfiguredProvider]); + mocks.runProviderStaticCatalog.mockResolvedValue({ marker: "unconfigured-static-result" }); + mocks.normalizePluginDiscoveryResult.mockImplementation( + ({ provider: catalogProvider }: { provider: { id: string } }) => ({ + [catalogProvider.id]: { + models: [{ id: `${catalogProvider.id}-model`, contextWindow: 128_000 }], + }, + }), + ); + + await expect( + loadBundledProviderStaticCatalogContextModels({ + cfg, + metadataSnapshot: createMetadataSnapshot(["anthropic", "google"]), + preparedStaticProviderCatalog: { + providers: [provider], + entries: [{ provider, result: { marker: "prepared-static-result" } as never }], + }, + }), + ).resolves.toEqual([ + expect.objectContaining({ id: "anthropic-model", provider: "anthropic" }), + expect.objectContaining({ id: "google-model", provider: "google" }), + ]); + expect(mocks.resolveRuntimePluginDiscoveryProviders).toHaveBeenCalledOnce(); + expect(mocks.resolveRuntimePluginDiscoveryProviders).toHaveBeenCalledWith( + expect.objectContaining({ onlyPluginIds: ["anthropic"] }), + ); + expect(mocks.runProviderStaticCatalog).toHaveBeenCalledOnce(); + expect(mocks.runProviderStaticCatalog).toHaveBeenCalledWith( + expect.objectContaining({ provider: unconfiguredProvider }), + ); + }); +}); diff --git a/src/agents/embedded-agent-runner/model.static-catalog.test.ts b/src/agents/embedded-agent-runner/model.static-catalog.test.ts index cb5d3898c3a8..d5ba49289d75 100644 --- a/src/agents/embedded-agent-runner/model.static-catalog.test.ts +++ b/src/agents/embedded-agent-runner/model.static-catalog.test.ts @@ -841,12 +841,7 @@ describe("resolveBundledProviderStaticCatalogModel", () => { discoveryEntriesOnly: true, includeManifestModelCatalogProviders: false, }); - expect(providerMocks.runProviderStaticCatalog).toHaveBeenCalledWith({ - provider, - config: cfg, - workspaceDir: undefined, - env: process.env, - }); + expect(providerMocks.runProviderStaticCatalog).toHaveBeenCalledWith({ provider }); }); it("does not load bundled provider static catalogs when owner policy blocks the plugin", async () => { diff --git a/src/agents/embedded-agent-runner/model.static-catalog.ts b/src/agents/embedded-agent-runner/model.static-catalog.ts index f9424e573afc..93e2acc99eeb 100644 --- a/src/agents/embedded-agent-runner/model.static-catalog.ts +++ b/src/agents/embedded-agent-runner/model.static-catalog.ts @@ -17,6 +17,7 @@ import { normalizePluginDiscoveryResult, resolveRuntimePluginDiscoveryProviders, runProviderStaticCatalog, + type PreparedProviderStaticCatalog, } from "../../plugins/provider-discovery.js"; import type { ProviderRuntimeModel } from "../../plugins/provider-runtime-model.types.js"; import { @@ -92,10 +93,14 @@ function modelFromProviderStaticCatalog(params: { provider: string; providerConfig: ModelProviderConfig; model: ModelProviderConfig["models"][number]; + providerMetadataOwners?: PluginMetadataSnapshot["owners"]; }): ProviderRuntimeModel { - const [model] = buildInlineProviderModels({ - [params.provider]: { ...params.providerConfig, models: [params.model] }, - }); + const [model] = buildInlineProviderModels( + { + [params.provider]: { ...params.providerConfig, models: [params.model] }, + }, + { providerMetadataOwners: params.providerMetadataOwners }, + ); return { ...model, id: model?.id ?? params.model.id, @@ -275,6 +280,8 @@ type BundledProviderStaticCatalogResolverParams = { cfg?: OpenClawConfig; workspaceDir?: string; env?: NodeJS.ProcessEnv; + metadataSnapshot?: PluginMetadataSnapshot; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; providerIds?: readonly string[]; }; @@ -403,25 +410,50 @@ async function loadBundledProviderStaticCatalogModels(params: { cfg?: OpenClawConfig; workspaceDir?: string; env: NodeJS.ProcessEnv; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; + providerMetadataOwners?: PluginMetadataSnapshot["owners"]; }): Promise> { - const providers = await resolveRuntimePluginDiscoveryProviders({ - config: params.cfg, - workspaceDir: params.workspaceDir, - env: params.env, - onlyPluginIds: params.pluginIds, - includeUntrustedWorkspacePlugins: false, - requireCompleteDiscoveryEntryCoverage: true, - discoveryEntriesOnly: true, - includeManifestModelCatalogProviders: false, - }); + const pluginIds = new Set(params.pluginIds); + const preparedProviders = (params.preparedStaticProviderCatalog?.providers ?? []).filter( + (provider) => provider.pluginId !== undefined && pluginIds.has(provider.pluginId), + ); + const preparedPluginIds = new Set( + preparedProviders.flatMap((provider) => (provider.pluginId ? [provider.pluginId] : [])), + ); + const missingPluginIds = params.pluginIds.filter((pluginId) => !preparedPluginIds.has(pluginId)); + // Prepared provider lists are complete only for the plugin ids they name. Full catalog reads + // must still discover omitted plugins, while reusing cached hook results for covered providers. + const discoveredProviders = + missingPluginIds.length === 0 + ? [] + : await resolveRuntimePluginDiscoveryProviders({ + config: params.cfg, + workspaceDir: params.workspaceDir, + env: params.env, + onlyPluginIds: missingPluginIds, + includeUntrustedWorkspacePlugins: false, + requireCompleteDiscoveryEntryCoverage: true, + discoveryEntriesOnly: true, + includeManifestModelCatalogProviders: false, + }); + const providers = [...preparedProviders, ...discoveredProviders]; + const preparedEntries = params.preparedStaticProviderCatalog?.entries.filter( + ({ provider }) => provider.pluginId !== undefined && pluginIds.has(provider.pluginId), + ); + const preparedResults = preparedEntries + ? new Map( + preparedEntries.map(({ provider, result }) => [ + `${provider.pluginId ?? ""}\0${normalizeProviderId(provider.id)}`, + result, + ]), + ) + : undefined; const modelsByProvider = new Map(); for (const catalogProvider of providers) { - const result = await runProviderStaticCatalog({ - provider: catalogProvider, - config: params.cfg ?? {}, - workspaceDir: params.workspaceDir, - env: params.env, - }); + const preparedResultKey = `${catalogProvider.pluginId ?? ""}\0${normalizeProviderId(catalogProvider.id)}`; + const result = preparedResults?.has(preparedResultKey) + ? preparedResults.get(preparedResultKey) + : await runProviderStaticCatalog({ provider: catalogProvider }); const normalized = normalizePluginDiscoveryResult({ provider: catalogProvider, result, @@ -438,6 +470,9 @@ async function loadBundledProviderStaticCatalogModels(params: { provider, providerConfig, model, + ...(params.providerMetadataOwners + ? { providerMetadataOwners: params.providerMetadataOwners } + : {}), }), ), ); @@ -455,6 +490,7 @@ export async function loadBundledProviderStaticCatalogContextModels( const metadataSnapshot = resolveBundledStaticCatalogMetadataSnapshot({ cfg: params.cfg, env, + ...(params.metadataSnapshot ? { metadataSnapshot: params.metadataSnapshot } : {}), workspaceDir: params.workspaceDir, }); const discoveryEntryPluginIds = new Set( @@ -499,6 +535,10 @@ export async function loadBundledProviderStaticCatalogContextModels( cfg: params.cfg, workspaceDir: params.workspaceDir, env, + ...(params.preparedStaticProviderCatalog + ? { preparedStaticProviderCatalog: params.preparedStaticProviderCatalog } + : {}), + ...(metadataSnapshot ? { providerMetadataOwners: metadataSnapshot.owners } : {}), }), ), ); diff --git a/src/agents/models-config.plan.ts b/src/agents/models-config.plan.ts index 5f1334b2675f..8fe4bea79203 100644 --- a/src/agents/models-config.plan.ts +++ b/src/agents/models-config.plan.ts @@ -5,6 +5,7 @@ */ import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; +import type { PreparedProviderStaticCatalog } from "../plugins/provider-discovery.js"; import { isRecord } from "../utils.js"; import { mergeProviders, @@ -35,12 +36,16 @@ type ResolveImplicitProvidersForModelsJson = (params: { workspaceDir?: string; explicitProviders: Record; pluginMetadataSnapshot?: Pick; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; providerDiscoveryProviderIds?: readonly string[]; providerDiscoveryTimeoutMs?: number; providerDiscoveryEntriesOnly?: boolean; }) => Promise>; -/** Planned models.json write/noop/skip result plus plugin catalog sidecar writes. */ +/** + * Planned models.json result. When present, pluginCatalogWrites is the complete + * replacement set; omission means the plan is non-authoritative for plugin catalogs. + */ type ModelsJsonPlan = | { action: "skip"; @@ -99,6 +104,7 @@ async function resolveProvidersForModelsJsonWithDeps( env: NodeJS.ProcessEnv; workspaceDir?: string; pluginMetadataSnapshot?: Pick; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; providerDiscoveryProviderIds?: readonly string[]; providerDiscoveryTimeoutMs?: number; providerDiscoveryEntriesOnly?: boolean; @@ -128,6 +134,9 @@ async function resolveProvidersForModelsJsonWithDeps( ...(params.pluginMetadataSnapshot ? { pluginMetadataSnapshot: params.pluginMetadataSnapshot } : {}), + ...(params.preparedStaticProviderCatalog + ? { preparedStaticProviderCatalog: params.preparedStaticProviderCatalog } + : {}), ...(params.providerDiscoveryProviderIds ? { providerDiscoveryProviderIds: params.providerDiscoveryProviderIds } : {}), @@ -210,6 +219,7 @@ async function planOpenClawModelsJsonWithDeps( existingRaw: string; existingParsed: unknown; pluginMetadataSnapshot?: Pick; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; providerDiscoveryProviderIds?: readonly string[]; providerDiscoveryTimeoutMs?: number; providerDiscoveryEntriesOnly?: boolean; @@ -228,6 +238,9 @@ async function planOpenClawModelsJsonWithDeps( ...(params.pluginMetadataSnapshot ? { pluginMetadataSnapshot: params.pluginMetadataSnapshot } : {}), + ...(params.preparedStaticProviderCatalog + ? { preparedStaticProviderCatalog: params.preparedStaticProviderCatalog } + : {}), ...(params.providerDiscoveryProviderIds ? { providerDiscoveryProviderIds: params.providerDiscoveryProviderIds } : {}), diff --git a/src/agents/models-config.providers.implicit.discovery-scope.test.ts b/src/agents/models-config.providers.implicit.discovery-scope.test.ts index 83ced0606cfa..c91050954cc8 100644 --- a/src/agents/models-config.providers.implicit.discovery-scope.test.ts +++ b/src/agents/models-config.providers.implicit.discovery-scope.test.ts @@ -9,6 +9,7 @@ import type { ProviderPlugin } from "../plugins/types.js"; import { withEnvAsync } from "../test-utils/env.js"; const mocks = vi.hoisted(() => ({ + prepareProviderStaticCatalog: vi.fn(), resolveRuntimePluginDiscoveryProviders: vi.fn(), runProviderCatalog: vi.fn(), runProviderStaticCatalog: vi.fn(), @@ -32,9 +33,13 @@ vi.mock("../plugins/provider-discovery.js", () => ({ provider: ProviderPlugin; result?: { provider?: unknown; providers?: Record } | null; }) => result?.providers ?? (result?.provider ? { [provider.id]: result.provider } : {}), + prepareProviderStaticCatalog: mocks.prepareProviderStaticCatalog, })); -import { resolveImplicitProviders } from "./models-config.providers.implicit.js"; +import { + prepareImplicitProviderStaticCatalog, + resolveImplicitProviders, +} from "./models-config.providers.implicit.js"; function metadataOwners( overrides: Partial, @@ -131,6 +136,32 @@ describe("resolveImplicitProviders startup discovery scope", () => { }, }, }); + mocks.prepareProviderStaticCatalog.mockResolvedValue({ + providers: [], + entries: [], + }); + }); + + it("loads configured provider entrypoints but runs static hooks only for unresolved refs", async () => { + const openai = createStaticOnlyProvider("openai"); + const anthropic = createStaticOnlyProvider("anthropic"); + mocks.resolveRuntimePluginDiscoveryProviders.mockResolvedValue([openai, anthropic]); + mocks.prepareProviderStaticCatalog.mockResolvedValue({ + providers: [anthropic], + entries: [], + }); + + const prepared = await prepareImplicitProviderStaticCatalog({ + config: {}, + env: {} as NodeJS.ProcessEnv, + providerDiscoveryProviderIds: ["openai", "anthropic"], + staticCatalogProviderIds: ["anthropic"], + }); + + expect(mocks.prepareProviderStaticCatalog).toHaveBeenCalledWith({ + providers: [anthropic], + }); + expect(prepared.providers).toEqual([openai, anthropic]); }); it("passes startup provider scopes as plugin owner filters", async () => { @@ -207,6 +238,99 @@ describe("resolveImplicitProviders startup discovery scope", () => { expect(mocks.runProviderCatalog).not.toHaveBeenCalled(); }); + it("reuses prepared static results while preserving the requesting provider scope", async () => { + const openai = { ...createStaticOnlyProvider("openai"), pluginId: "openai" }; + const anthropic = { ...createStaticOnlyProvider("anthropic"), pluginId: "anthropic" }; + const providers = await resolveImplicitProviders({ + agentDir: "/tmp/openclaw-agent", + config: {}, + env: {} as NodeJS.ProcessEnv, + explicitProviders: {}, + pluginMetadataSnapshot: { + index: { plugins: [] } as never, + manifestRegistry: { plugins: [], diagnostics: [] }, + owners: metadataOwners({ + providers: new Map([ + ["openai", ["openai"]], + ["anthropic", ["anthropic"]], + ]), + }), + }, + preparedStaticProviderCatalog: { + providers: [openai, anthropic], + entries: [ + { + provider: openai, + result: { + providers: { + openai: { + baseUrl: "https://api.openai.com/v1", + api: "openai-responses", + models: [], + }, + }, + }, + }, + { + provider: anthropic, + result: { + providers: { + anthropic: { + baseUrl: "https://api.anthropic.com", + api: "anthropic-messages", + models: [], + }, + }, + }, + }, + ], + }, + providerDiscoveryEntriesOnly: true, + providerDiscoveryProviderIds: ["openai"], + }); + + expect(Object.keys(providers ?? {})).toEqual(["openai"]); + expect(mocks.resolveRuntimePluginDiscoveryProviders).not.toHaveBeenCalled(); + expect(mocks.runProviderStaticCatalog).not.toHaveBeenCalled(); + }); + + it("runs a prepared provider's static hook when its result was not prepared", async () => { + const anthropic = { ...createStaticOnlyProvider("anthropic"), pluginId: "anthropic" }; + mocks.runProviderStaticCatalog.mockResolvedValueOnce({ + providers: { + anthropic: { + baseUrl: "https://api.anthropic.com", + api: "anthropic-messages", + models: [], + }, + }, + }); + + const providers = await resolveImplicitProviders({ + agentDir: "/tmp/openclaw-agent", + config: {}, + env: {} as NodeJS.ProcessEnv, + explicitProviders: {}, + pluginMetadataSnapshot: { + index: { plugins: [] } as never, + manifestRegistry: { plugins: [], diagnostics: [] }, + owners: metadataOwners({ + providers: new Map([["anthropic", ["anthropic"]]]), + }), + }, + preparedStaticProviderCatalog: { + providers: [anthropic], + entries: [], + }, + providerDiscoveryEntriesOnly: true, + providerDiscoveryProviderIds: ["anthropic"], + }); + + expect(Object.keys(providers ?? {})).toEqual(["anthropic"]); + expect(mocks.resolveRuntimePluginDiscoveryProviders).not.toHaveBeenCalled(); + expect(mocks.runProviderStaticCatalog).toHaveBeenCalledWith({ provider: anthropic }); + }); + it("uses static-only provider catalogs for scoped startup discovery", async () => { mocks.resolveRuntimePluginDiscoveryProviders.mockResolvedValue([ createStaticOnlyProvider("openai"), diff --git a/src/agents/models-config.providers.implicit.ts b/src/agents/models-config.providers.implicit.ts index 67327144cf54..7cba64aea3a0 100644 --- a/src/agents/models-config.providers.implicit.ts +++ b/src/agents/models-config.providers.implicit.ts @@ -15,9 +15,11 @@ import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot import { groupPluginDiscoveryProvidersByOrder, normalizePluginDiscoveryResult, + prepareProviderStaticCatalog, resolveRuntimePluginDiscoveryProviders, runProviderCatalog, runProviderStaticCatalog, + type PreparedProviderStaticCatalog, } from "../plugins/provider-discovery.js"; import { resolveOwningPluginIdsForProviderRef } from "../plugins/providers.js"; import { ensureAuthProfileStore } from "./auth-profiles/store.js"; @@ -60,7 +62,9 @@ type ImplicitProviderParams = { workspaceDir?: string; explicitProviders?: Record | null; pluginMetadataSnapshot?: Pick; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; providerDiscoveryProviderIds?: readonly string[]; + staticCatalogProviderIds?: readonly string[]; providerDiscoveryTimeoutMs?: number; providerDiscoveryEntriesOnly?: boolean; }; @@ -361,6 +365,10 @@ async function resolvePluginImplicitProviders( ctx: ImplicitProviderContext, providers: import("../plugins/types.js").ProviderPlugin[], order: import("../plugins/types.js").ProviderCatalogOrder, + preparedStaticResults?: ReadonlyMap< + import("../plugins/types.js").ProviderPlugin, + PreparedProviderStaticCatalog["entries"][number]["result"] + >, ): Promise | undefined> { const byOrder = groupPluginDiscoveryProvidersByOrder(providers); const discovered: Record = {}; @@ -413,34 +421,28 @@ async function resolvePluginImplicitProviders( (ctx.providerDiscoveryEntriesOnly === true || !hasRuntimeProviderCatalog(provider)); // Static catalogs are preferred for entries-only discovery and as a fallback // when runtime discovery produces no usable provider config. - let result = useStaticCatalog - ? await runProviderStaticCatalog({ - provider, - config: catalogConfig, - agentDir: ctx.agentDir, - workspaceDir: ctx.workspaceDir, - env: ctx.env, - }) - : await runProviderCatalogWithTimeout({ - provider, - config: catalogConfig, - agentDir: ctx.agentDir, - workspaceDir: ctx.workspaceDir, - env: ctx.env, - resolveProviderApiKey: resolveCatalogProviderApiKey, - resolveProviderAuth: (providerId, options) => - ctx.resolveProviderAuth(providerId?.trim() || provider.id, options), - timeoutMs: ctx.providerDiscoveryTimeoutMs ?? resolveLiveProviderCatalogTimeoutMs(ctx.env), - }); - if (!result && !useStaticCatalog && provider.staticCatalog) { - result = await runProviderStaticCatalog({ + const hasPreparedStaticResult = preparedStaticResults?.has(provider) === true; + let result; + if (useStaticCatalog) { + result = hasPreparedStaticResult + ? preparedStaticResults.get(provider) + : await runProviderStaticCatalog({ provider }); + } else { + result = await runProviderCatalogWithTimeout({ provider, config: catalogConfig, agentDir: ctx.agentDir, workspaceDir: ctx.workspaceDir, env: ctx.env, + resolveProviderApiKey: resolveCatalogProviderApiKey, + resolveProviderAuth: (providerId, options) => + ctx.resolveProviderAuth(providerId?.trim() || provider.id, options), + timeoutMs: ctx.providerDiscoveryTimeoutMs ?? resolveLiveProviderCatalogTimeoutMs(ctx.env), }); } + if (!result && !useStaticCatalog && provider.staticCatalog) { + result = await runProviderStaticCatalog({ provider }); + } if (!result) { continue; } @@ -534,6 +536,58 @@ async function runProviderCatalogWithTimeout( } } +/** Prepares sterile provider catalog results for one workspace/config generation. */ +export async function prepareImplicitProviderStaticCatalog( + params: Pick< + ImplicitProviderParams, + | "config" + | "env" + | "pluginMetadataSnapshot" + | "providerDiscoveryProviderIds" + | "staticCatalogProviderIds" + | "workspaceDir" + >, +): Promise { + const env = params.env ?? process.env; + const providers = await resolveRuntimePluginDiscoveryProviders({ + config: params.config, + workspaceDir: params.workspaceDir, + env, + onlyPluginIds: resolveProviderDiscoveryFilter({ + config: params.config, + workspaceDir: params.workspaceDir, + env, + resolveOwners: params.pluginMetadataSnapshot + ? (provider) => resolvePluginMetadataProviderOwners(params.pluginMetadataSnapshot, provider) + : undefined, + providerIds: params.providerDiscoveryProviderIds, + }), + ...(params.pluginMetadataSnapshot + ? { pluginMetadataSnapshot: params.pluginMetadataSnapshot } + : {}), + discoveryEntriesOnly: true, + includeSyntheticAuthProviders: true, + }); + const staticCatalogProviderIds = params.staticCatalogProviderIds + ? new Set(params.staticCatalogProviderIds.map((provider) => normalizeProviderId(provider))) + : undefined; + const prepared = await prepareProviderStaticCatalog({ + providers: staticCatalogProviderIds + ? providers.filter((provider) => + [provider.id, ...(provider.aliases ?? []), ...(provider.hookAliases ?? [])].some((id) => + staticCatalogProviderIds.has(normalizeProviderId(id)), + ), + ) + : providers, + }); + // Synthetic auth consumes the complete configured provider entrypoint set. Static results may + // be narrower because startup only executes hooks for unresolved configured model refs. + return Object.freeze({ + providers: Object.freeze(providers), + entries: prepared.entries, + }); +} + /** Resolve all implicit provider configs contributed by runtime plugin discovery. */ export async function resolveImplicitProviders( params: ImplicitProviderParams, @@ -555,29 +609,82 @@ export async function resolveImplicitProviders( resolveProviderApiKey: createProviderApiKeyResolver(env, getAuthStore, params.config), resolveProviderAuth: createProviderAuthResolver(env, getAuthStore, params.config), }; - const discoveryProviders = await resolveRuntimePluginDiscoveryProviders({ + const discoveryPluginIds = resolveProviderDiscoveryFilter({ config: params.config, workspaceDir: params.workspaceDir, env, - onlyPluginIds: resolveProviderDiscoveryFilter({ - config: params.config, - workspaceDir: params.workspaceDir, - env, - resolveOwners: params.pluginMetadataSnapshot - ? (provider) => resolvePluginMetadataProviderOwners(params.pluginMetadataSnapshot, provider) - : undefined, - providerIds: params.providerDiscoveryProviderIds, - }), - ...(params.pluginMetadataSnapshot - ? { pluginMetadataSnapshot: params.pluginMetadataSnapshot } - : {}), - ...(params.providerDiscoveryEntriesOnly === true ? { discoveryEntriesOnly: true } : {}), + resolveOwners: params.pluginMetadataSnapshot + ? (provider) => resolvePluginMetadataProviderOwners(params.pluginMetadataSnapshot, provider) + : undefined, + providerIds: params.providerDiscoveryProviderIds, }); - + const preparedStaticEntries = params.preparedStaticProviderCatalog + ? params.preparedStaticProviderCatalog.entries.filter( + ({ provider }) => + discoveryPluginIds === undefined || + (provider.pluginId !== undefined && discoveryPluginIds.includes(provider.pluginId)), + ) + : undefined; + const preparedProviders = + params.providerDiscoveryEntriesOnly === true && params.preparedStaticProviderCatalog?.providers + ? params.preparedStaticProviderCatalog.providers.filter( + (provider) => + discoveryPluginIds === undefined || + (provider.pluginId !== undefined && discoveryPluginIds.includes(provider.pluginId)), + ) + : []; + const preparedPluginIds = new Set( + preparedProviders.flatMap((provider) => (provider.pluginId ? [provider.pluginId] : [])), + ); + const missingDiscoveryPluginIds = + discoveryPluginIds?.filter((pluginId) => !preparedPluginIds.has(pluginId)) ?? + (preparedProviders.length > 0 ? undefined : discoveryPluginIds); + const resolvedProviders = + missingDiscoveryPluginIds === undefined || missingDiscoveryPluginIds.length > 0 + ? await resolveRuntimePluginDiscoveryProviders({ + config: params.config, + workspaceDir: params.workspaceDir, + env, + onlyPluginIds: missingDiscoveryPluginIds, + ...(params.pluginMetadataSnapshot + ? { pluginMetadataSnapshot: params.pluginMetadataSnapshot } + : {}), + ...(params.providerDiscoveryEntriesOnly === true ? { discoveryEntriesOnly: true } : {}), + }) + : []; + const discoveryProviders = [ + ...new Map( + [...resolvedProviders, ...preparedProviders].map((provider) => [ + `${provider.pluginId ?? ""}\0${normalizeProviderId(provider.id)}`, + provider, + ]), + ).values(), + ]; + const preparedStaticResultsByProvider = new Map( + preparedStaticEntries?.map(({ provider, result }) => [ + `${provider.pluginId ?? ""}\0${normalizeProviderId(provider.id)}`, + result, + ]) ?? [], + ); + const preparedStaticResults = params.preparedStaticProviderCatalog + ? new Map( + discoveryProviders.flatMap((provider) => { + const key = `${provider.pluginId ?? ""}\0${normalizeProviderId(provider.id)}`; + return preparedStaticResultsByProvider.has(key) + ? [[provider, preparedStaticResultsByProvider.get(key)] as const] + : []; + }), + ) + : undefined; for (const order of PLUGIN_DISCOVERY_ORDERS) { mergeImplicitProviderSet( providers, - await resolvePluginImplicitProviders(context, discoveryProviders, order), + await resolvePluginImplicitProviders( + context, + discoveryProviders, + order, + preparedStaticResults, + ), ); } diff --git a/src/agents/models-config.ts b/src/agents/models-config.ts index c6cc8620a743..842b7b2b2f14 100644 --- a/src/agents/models-config.ts +++ b/src/agents/models-config.ts @@ -21,6 +21,7 @@ import { resolvePluginMetadataSnapshot, type PluginMetadataSnapshot, } from "../plugins/plugin-metadata-snapshot.js"; +import type { PreparedProviderStaticCatalog } from "../plugins/provider-discovery.js"; import { resolveAgentWorkspaceDir, resolveDefaultAgentDir, @@ -33,11 +34,15 @@ import { type ModelsJsonReadyState, } from "./models-config-state.js"; import { planOpenClawModelsJson } from "./models-config.plan.js"; +import { repairPluginModelCatalogTransportMetadata } from "./plugin-model-catalog-repair.js"; import { + decodePluginModelCatalogRelativePathPluginId, isGeneratedPluginModelCatalog, loadPersistedPluginModelCatalogs, + loadPersistedPluginModelCatalogsReadOnly, replacePersistedPluginModelCatalogs, resolvePluginModelCatalogOwnerPluginId, + type PersistedPluginModelCatalog, } from "./plugin-model-catalog.js"; import { stableStringify } from "./stable-stringify.js"; @@ -49,12 +54,19 @@ type PreparedOpenClawModelsJsonSource = ModelsJsonReadyResult & { type EnsureOpenClawModelsJsonOptions = { env?: NodeJS.ProcessEnv; pluginMetadataSnapshot?: Pick; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; workspaceDir?: string; providerDiscoveryProviderIds?: readonly string[]; providerDiscoveryTimeoutMs?: number; providerDiscoveryEntriesOnly?: boolean; }; +type PlannedOpenClawModelsJsonSource = Readonly<{ + agentDir: string; + modelsJsonContents: string | null; + pluginCatalogs: readonly PersistedPluginModelCatalog[]; +}>; + function listPreparedPluginModelCatalogs(agentDir: string) { const { catalogs, warnings } = loadPersistedPluginModelCatalogs(agentDir); if (warnings.length > 0) { @@ -167,14 +179,14 @@ if (process.env.VITEST || process.env.NODE_ENV === "test") { async function mergeGeneratedPluginCatalogProvidersIntoExistingParsed(params: { agentDir: string; existingParsed: unknown; + pluginCatalogs?: readonly PersistedPluginModelCatalog[]; pluginMetadataSnapshot?: Pick; }): Promise { const root = isRecord(params.existingParsed) ? params.existingParsed : {}; const providers = isRecord(root.providers) ? { ...root.providers } : {}; let changed = false; - for (const { pluginId: catalogPluginId, contents } of listPreparedPluginModelCatalogs( - params.agentDir, - )) { + const pluginCatalogs = params.pluginCatalogs ?? listPreparedPluginModelCatalogs(params.agentDir); + for (const { pluginId: catalogPluginId, contents } of pluginCatalogs) { let catalog: unknown; try { catalog = JSON.parse(contents) as unknown; @@ -206,6 +218,23 @@ async function mergeGeneratedPluginCatalogProvidersIntoExistingParsed(params: { return { ...root, providers }; } +function materializePlannedPluginCatalogs( + pluginCatalogWrites: Readonly>, +): PersistedPluginModelCatalog[] { + return Object.entries(pluginCatalogWrites) + .map(([relativePath, contents]) => { + const pluginId = decodePluginModelCatalogRelativePathPluginId(relativePath); + if (!pluginId) { + throw new Error(`Invalid generated plugin model catalog key: ${relativePath}`); + } + return { + pluginId, + contents: repairPluginModelCatalogTransportMetadata(contents).contents, + }; + }) + .toSorted((left, right) => left.pluginId.localeCompare(right.pluginId)); +} + function writePluginCatalogsForModelsJson(params: { agentDir: string; pluginCatalogWrites?: Record; @@ -360,6 +389,9 @@ async function prepareOpenClawModelsJsonSource( existingRaw: existingModelsFile.raw, existingParsed: existingParsedForMerge, ...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}), + ...(options.preparedStaticProviderCatalog + ? { preparedStaticProviderCatalog: options.preparedStaticProviderCatalog } + : {}), ...(options.providerDiscoveryProviderIds ? { providerDiscoveryProviderIds: options.providerDiscoveryProviderIds } : {}), @@ -442,6 +474,75 @@ async function prepareOpenClawModelsJsonSource( } } +/** + * Plans the complete root/plugin catalog generation without mutating agent-owned state. + * Control-plane inventory reads use this when their lifecycle generation may be superseded. + */ +export async function planOpenClawModelsJsonSource( + config?: OpenClawConfig, + agentDirOverride?: string, + options: EnsureOpenClawModelsJsonOptions = {}, +): Promise { + const resolved = resolveModelsConfigInput(config); + const cfg = resolved.config; + const workspaceDir = + options.workspaceDir ?? + (agentDirOverride?.trim() + ? undefined + : resolveAgentWorkspaceDir(cfg, resolveDefaultAgentId(cfg))); + const providerScopedDiscovery = Boolean(options.providerDiscoveryProviderIds?.length); + const pluginMetadataSnapshot = + options.pluginMetadataSnapshot ?? + resolvePluginMetadataSnapshot({ + config: cfg, + env: createConfigRuntimeEnv(cfg, options.env), + ...(workspaceDir ? { workspaceDir } : {}), + ...(providerScopedDiscovery ? { preferPersisted: false } : {}), + }); + const agentDir = agentDirOverride?.trim() ? agentDirOverride.trim() : resolveDefaultAgentDir(cfg); + const existingModelsFile = await readExistingModelsFile(path.join(agentDir, "models.json")); + const existingPluginCatalogs = loadPersistedPluginModelCatalogsReadOnly(agentDir); + const existingParsedForMerge = await mergeGeneratedPluginCatalogProvidersIntoExistingParsed({ + agentDir, + existingParsed: existingModelsFile.parsed, + pluginCatalogs: existingPluginCatalogs, + ...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}), + }); + const env = createConfigRuntimeEnv(cfg, options.env); + const plan = await planOpenClawModelsJson({ + cfg, + sourceConfigForSecrets: resolved.sourceConfigForSecrets, + agentDir, + env, + ...(workspaceDir ? { workspaceDir } : {}), + existingRaw: existingModelsFile.raw, + existingParsed: existingParsedForMerge, + ...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}), + ...(options.preparedStaticProviderCatalog + ? { preparedStaticProviderCatalog: options.preparedStaticProviderCatalog } + : {}), + ...(options.providerDiscoveryProviderIds + ? { providerDiscoveryProviderIds: options.providerDiscoveryProviderIds } + : {}), + ...(options.providerDiscoveryTimeoutMs !== undefined + ? { providerDiscoveryTimeoutMs: options.providerDiscoveryTimeoutMs } + : {}), + ...(options.providerDiscoveryEntriesOnly === true + ? { providerDiscoveryEntriesOnly: true } + : {}), + }); + return { + agentDir, + modelsJsonContents: plan.action === "write" ? plan.contents : existingModelsFile.raw || null, + // Planned writes share the writer's complete-replacement contract, including intentional + // stale-catalog deletion. Only a non-authoritative plan omits this field. + pluginCatalogs: + plan.pluginCatalogWrites === undefined + ? existingPluginCatalogs + : materializePlannedPluginCatalogs(plan.pluginCatalogWrites), + }; +} + /** Ensures models.json and the agent SQLite catalog cache are current. */ export async function ensureOpenClawModelsJson( config?: OpenClawConfig, diff --git a/src/agents/models-config.write-serialization.test.ts b/src/agents/models-config.write-serialization.test.ts index b0b168cddf5d..ea94503b1f10 100644 --- a/src/agents/models-config.write-serialization.test.ts +++ b/src/agents/models-config.write-serialization.test.ts @@ -57,6 +57,7 @@ let actualPrivateFileStore: installModelsConfigTestHooks(); let ensureOpenClawModelsJson: typeof import("./models-config.js").ensureOpenClawModelsJson; +let planOpenClawModelsJsonSource: typeof import("./models-config.js").planOpenClawModelsJsonSource; let clearCurrentPluginMetadataSnapshot: typeof import("../plugins/current-plugin-metadata-state.js").clearCurrentPluginMetadataSnapshot; let setCurrentPluginMetadataSnapshot: typeof import("../plugins/current-plugin-metadata-snapshot.js").setCurrentPluginMetadataSnapshot; @@ -159,7 +160,7 @@ beforeAll(async () => { }, }; }); - ({ ensureOpenClawModelsJson } = await import("./models-config.js")); + ({ ensureOpenClawModelsJson, planOpenClawModelsJsonSource } = await import("./models-config.js")); ({ clearCurrentPluginMetadataSnapshot } = await import("../plugins/current-plugin-metadata-state.js")); ({ setCurrentPluginMetadataSnapshot } = @@ -190,6 +191,54 @@ beforeEach(() => { }); describe("models-config write serialization", () => { + it("materializes an authoritative plugin catalog replacement without mutating state", async () => { + await withModelsTempHome(async (home) => { + const agentDir = path.join(home, "agent"); + const workspaceDir = path.join(home, "workspace"); + const rootContents = `${JSON.stringify({ providers: { existing: { models: [] } } })}\n`; + const existingPluginContents = `${JSON.stringify({ + generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY, + providers: {}, + })}\n`; + await fs.mkdir(agentDir, { recursive: true }); + await fs.writeFile(path.join(agentDir, "models.json"), rootContents); + replacePersistedPluginModelCatalogs({ + agentDir, + pluginCatalogWrites: { + [encodePluginModelCatalogRelativePath("existing-plugin")]: existingPluginContents, + }, + }); + const originalPluginRow = readRawCatalogCacheRow(agentDir, "existing-plugin"); + const plannedPluginContents = `${JSON.stringify({ + generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY, + providers: {}, + })}\n`; + planOpenClawModelsJsonMock.mockResolvedValue({ + action: "write", + contents: `${JSON.stringify({ providers: { discovered: { models: [] } } })}\n`, + pluginCatalogWrites: { + [encodePluginModelCatalogRelativePath("planned-plugin")]: plannedPluginContents, + }, + }); + + const planned = await planOpenClawModelsJsonSource({}, agentDir, { + workspaceDir, + pluginMetadataSnapshot: createPluginMetadataSnapshot(workspaceDir), + }); + + expect(planned.modelsJsonContents).toContain("discovered"); + expect(planned.pluginCatalogs).toEqual([ + { pluginId: "planned-plugin", contents: plannedPluginContents }, + ]); + expect(await fs.readFile(path.join(agentDir, "models.json"), "utf8")).toBe(rootContents); + expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([ + { pluginId: "existing-plugin", contents: existingPluginContents }, + ]); + expect(readRawCatalogCacheRow(agentDir, "existing-plugin")).toEqual(originalPluginRow); + expect(writePrivateStoreTextWriteMock).not.toHaveBeenCalled(); + }); + }); + it("does not reuse default workspace plugin metadata for explicit agent dirs without workspace", async () => { await withModelsTempHome(async (home) => { const snapshot = createPluginMetadataSnapshot(path.join(home, "default-workspace")); diff --git a/src/agents/plugin-model-catalog.test.ts b/src/agents/plugin-model-catalog.test.ts index 9b4f60fa3104..885d42fa69c5 100644 --- a/src/agents/plugin-model-catalog.test.ts +++ b/src/agents/plugin-model-catalog.test.ts @@ -19,6 +19,7 @@ import { decodePluginModelCatalogRelativePathPluginId, encodePluginModelCatalogRelativePath, loadPersistedPluginModelCatalogs, + loadPersistedPluginModelCatalogsReadOnly, migrateLegacyPluginModelCatalogs, PLUGIN_MODEL_CATALOG_GENERATED_BY, replacePersistedPluginModelCatalogs, @@ -83,6 +84,31 @@ afterEach(() => { }); describe("SQLite-backed plugin model catalogs", () => { + it("reads only named catalog rows without running migration or repair", () => { + const agentDir = createAgentDir(); + const zai = catalogContents("zai"); + const anthropic = catalogContents("anthropic"); + replacePersistedPluginModelCatalogs({ + agentDir, + pluginCatalogWrites: { + [encodePluginModelCatalogRelativePath("zai")]: zai, + [encodePluginModelCatalogRelativePath("anthropic")]: anthropic, + }, + }); + const legacyPath = join(agentDir, encodePluginModelCatalogRelativePath("legacy")); + mkdirSync(join(agentDir, "plugins", "legacy"), { recursive: true }); + writeFileSync(legacyPath, catalogContents("legacy"), "utf8"); + + expect(loadPersistedPluginModelCatalogsReadOnly(agentDir, ["zai"])).toEqual([ + { pluginId: "zai", contents: zai }, + ]); + expect(loadPersistedPluginModelCatalogsReadOnly(agentDir)).toEqual([ + { pluginId: "anthropic", contents: anthropic }, + { pluginId: "zai", contents: zai }, + ]); + expect(existsSync(legacyPath)).toBe(true); + }); + it("removes generated model rows whose API semantics cannot be derived", () => { const agentDir = createAgentDir(); const relativePath = encodePluginModelCatalogRelativePath("nvidia"); diff --git a/src/agents/plugin-model-catalog.ts b/src/agents/plugin-model-catalog.ts index a7dcbcabeda8..2483826921b9 100644 --- a/src/agents/plugin-model-catalog.ts +++ b/src/agents/plugin-model-catalog.ts @@ -43,7 +43,7 @@ export function isPluginModelCatalogMigrationFile(filename: string): boolean { type PluginModelCatalogDatabase = Pick; -type PersistedPluginModelCatalog = { +export type PersistedPluginModelCatalog = { pluginId: string; contents: string; }; @@ -84,6 +84,25 @@ function readPersistedPluginModelCatalogs(agentDir: string): PersistedPluginMode return readPersistedPluginModelCatalogEntries(agentDir, PLUGIN_MODEL_CATALOG_CACHE_SCOPE); } +/** + * Reads an exact plugin-catalog generation without migration or repair writes. + * Lifecycle preparation uses this for configured providers before atomic publication. + */ +export function loadPersistedPluginModelCatalogsReadOnly( + agentDir: string, + pluginIds?: readonly string[], +): PersistedPluginModelCatalog[] { + if (pluginIds?.length === 0) { + return []; + } + const catalogs = readPersistedPluginModelCatalogs(agentDir); + if (!pluginIds) { + return catalogs; + } + const allowed = new Set(pluginIds); + return catalogs.filter(({ pluginId }) => allowed.has(pluginId)); +} + function repairPersistedPluginModelCatalogs(params: { agentDir: string; catalogs: readonly PersistedPluginModelCatalog[]; diff --git a/src/agents/prepared-model-catalog.test.ts b/src/agents/prepared-model-catalog.test.ts index c93d5a51b951..c406a7e94727 100644 --- a/src/agents/prepared-model-catalog.test.ts +++ b/src/agents/prepared-model-catalog.test.ts @@ -105,6 +105,38 @@ describe("prepared model catalog access", () => { expect(mocks.releaseSnapshot).not.toHaveBeenCalled(); }); + it("keeps read-only catalog reads on configured facts and materializes full reads once", async () => { + const configuredCatalog = { + entries: [{ provider: "test", id: "configured", name: "Configured" }], + routeVariants: [], + }; + const discoveredCatalog = { + entries: [{ provider: "test", id: "discovered", name: "Discovered" }], + routeVariants: [], + }; + const loadFullModelCatalog = vi.fn(async () => discoveredCatalog); + const snapshot = { + ...fullSnapshot, + modelCatalog: configuredCatalog, + loadFullModelCatalog, + }; + mocks.prepareSnapshot.mockResolvedValue(snapshot); + + await expect(loadPreparedModelCatalogSnapshot({ readOnly: true })).resolves.toBe( + configuredCatalog, + ); + expect(loadFullModelCatalog).not.toHaveBeenCalled(); + + await expect(loadPreparedModelCatalogSnapshot({ readOnly: false })).resolves.toBe( + discoveredCatalog, + ); + expect(loadFullModelCatalog).toHaveBeenCalledOnce(); + + mocks.getSnapshot.mockReturnValue(snapshot); + expect(getPreparedModelCatalogSnapshot({ readOnly: true })).toBe(configuredCatalog); + expect(getPreparedModelCatalogSnapshot()).toBe(configuredCatalog); + }); + it("carries an explicit dynamic workspace into the read-only loader", async () => { mocks.prepareSnapshot.mockRejectedValue(new PreparedModelRuntimeOwnerNotPublishedError()); mocks.loadSnapshot.mockResolvedValue(readOnlySnapshot); diff --git a/src/agents/prepared-model-catalog.ts b/src/agents/prepared-model-catalog.ts index ebbb6502e726..8a6b9a1ac6b6 100644 --- a/src/agents/prepared-model-catalog.ts +++ b/src/agents/prepared-model-catalog.ts @@ -35,6 +35,19 @@ export type LoadPreparedModelCatalogParams = { type PreparedModelCatalogConfigPolicy = "exact" | "published"; +async function materializeRequestedModelCatalog( + snapshot: PreparedModelRuntimeSnapshot, + readOnly: boolean | undefined, +): Promise { + if (readOnly === true || !snapshot.loadFullModelCatalog) { + return snapshot; + } + const modelCatalog = await snapshot.loadFullModelCatalog(); + return modelCatalog === snapshot.modelCatalog + ? snapshot + : Object.freeze({ ...snapshot, modelCatalog }); +} + function acceptsPreparedSnapshotConfig( snapshot: PreparedModelRuntimeSnapshot, input: PreparedModelRuntimeInput, @@ -93,7 +106,7 @@ function resolveInputs(params: LoadPreparedModelCatalogParams = {}): { }; } -/** Returns the current published catalog without waiting or starting discovery. */ +/** Returns the configured catalog for the current generation without starting discovery. */ export function getPreparedModelCatalogSnapshot( params: LoadPreparedModelCatalogParams = {}, ): ModelCatalogSnapshot | undefined { @@ -124,7 +137,7 @@ export function getPreparedModelCatalogSnapshot( : undefined; } -async function loadPreparedModelCatalogOwnerSnapshotWithPolicy( +async function resolvePreparedModelCatalogOwnerSnapshotWithPolicy( params: LoadPreparedModelCatalogParams, configPolicy: PreparedModelCatalogConfigPolicy, ): Promise { @@ -208,6 +221,16 @@ async function loadPreparedModelCatalogOwnerSnapshotWithPolicy( } } +async function loadPreparedModelCatalogOwnerSnapshotWithPolicy( + params: LoadPreparedModelCatalogParams, + configPolicy: PreparedModelCatalogConfigPolicy, +): Promise { + return await materializeRequestedModelCatalog( + await resolvePreparedModelCatalogOwnerSnapshotWithPolicy(params, configPolicy), + params.readOnly, + ); +} + /** Resolves the lifecycle owner for an exact caller-supplied config. */ export async function loadPreparedModelCatalogOwnerSnapshot( params: LoadPreparedModelCatalogParams = {}, diff --git a/src/agents/prepared-model-runtime.build.ts b/src/agents/prepared-model-runtime.build.ts new file mode 100644 index 000000000000..62924bc82715 --- /dev/null +++ b/src/agents/prepared-model-runtime.build.ts @@ -0,0 +1,495 @@ +import { performance } from "node:perf_hooks"; +import pLimit from "p-limit"; +import { withTimeout } from "../node-host/with-timeout.js"; +import { runTasksWithConcurrency } from "../utils/run-with-concurrency.js"; +import type { ModelCatalogSnapshot } from "./model-catalog.types.js"; +import { + PreparedModelRuntimePublicationSupersededError, + toPreparedModelRuntimeError, +} from "./prepared-model-runtime.errors.js"; +import { + fingerprintPreparedRuntimeFacts, + prepareAgentCatalogSource, + prepareConfiguredRuntimeFactsBatch, + prepareFullCatalogFacts, + preparedModelRuntimeWorkspaceFactsKey, + prepareWorkspaceBuildGroup, + type PreparedModelRuntimeAgentFacts, + type PreparedModelRuntimeCatalogFacts, + type PreparedModelRuntimeCatalogSource, + type PreparedModelRuntimeWorkspaceFacts, +} from "./prepared-model-runtime.facts.js"; +import type { + PreparedModelRuntimeBuildStats, + PreparedModelRuntimeCatalogMode, + PreparedModelRuntimeInput, + PreparedModelRuntimeSnapshot, + PreparedModelRuntimeStores, +} from "./prepared-model-runtime.types.js"; +import { AuthStorage } from "./sessions/auth-storage.js"; + +const MAX_CONCURRENT_MODEL_RUNTIME_AGENT_SOURCE_BUILDS = 2; +const MAX_CONCURRENT_FULL_MODEL_CATALOG_BUILDS = 1; +const limitFullModelCatalogBuild = pLimit(MAX_CONCURRENT_FULL_MODEL_CATALOG_BUILDS); + +type PreparedModelRuntimeCatalogAccess = Readonly<{ + loadFullModelCatalog: () => Promise; +}>; +type PreparedModelRuntimeBuildGuards = + | ReadonlyMap boolean> + | (() => boolean); + +function runSerializedPreparedModelRuntimeTask(params: { + agentDir: string; + agentBuildCompletions: Map>; + isCurrent: () => boolean; + task: () => Promise; +}): Promise { + const previous = params.agentBuildCompletions.get(params.agentDir); + const pending = (async () => { + if (previous) { + await previous; + } + if (!params.isCurrent()) { + throw new PreparedModelRuntimePublicationSupersededError( + `prepared model runtime catalog generation was superseded for ${params.agentDir}`, + ); + } + return await params.task(); + })(); + const completion = pending.then( + () => undefined, + () => undefined, + ); + params.agentBuildCompletions.set(params.agentDir, completion); + void completion.then(() => { + if (params.agentBuildCompletions.get(params.agentDir) === completion) { + params.agentBuildCompletions.delete(params.agentDir); + } + }); + return pending; +} + +function assertPreparedModelRuntimeInputCurrent( + input: PreparedModelRuntimeInput, + guards: PreparedModelRuntimeBuildGuards, +): void { + const isCurrent = typeof guards === "function" ? guards : guards.get(input); + if (isCurrent && !isCurrent()) { + throw new PreparedModelRuntimePublicationSupersededError( + `prepared model runtime publication was superseded for ${input.agentDir}`, + ); + } +} + +function assertPreparedModelRuntimeInputsCurrent( + inputs: readonly PreparedModelRuntimeInput[], + guards: PreparedModelRuntimeBuildGuards, +): void { + for (const input of inputs) { + assertPreparedModelRuntimeInputCurrent(input, guards); + } +} + +function createFullModelCatalogAccess(params: { + agentFacts: PreparedModelRuntimeAgentFacts; + workspaceFacts: PreparedModelRuntimeWorkspaceFacts; + agentBuildCompletions: Map>; + isCurrent: () => boolean; + eagerCatalog?: ModelCatalogSnapshot; +}): PreparedModelRuntimeCatalogAccess { + let fullCatalog = params.eagerCatalog; + let pending: Promise | undefined; + const assertCurrent = () => { + if (!params.isCurrent()) { + throw new PreparedModelRuntimePublicationSupersededError( + `prepared model runtime catalog generation was superseded for ${params.agentFacts.input.agentDir}`, + ); + } + }; + return { + loadFullModelCatalog: () => { + if (fullCatalog) { + return Promise.resolve(fullCatalog); + } + if (!pending) { + pending = runSerializedPreparedModelRuntimeTask({ + agentDir: params.agentFacts.input.agentDir, + agentBuildCompletions: params.agentBuildCompletions, + isCurrent: params.isCurrent, + task: async () => + await limitFullModelCatalogBuild(async () => { + // Full inventory belongs to explicit control-plane reads. The generation queue + // prevents a stale plan from overlapping or following a replacement build. + assertCurrent(); + const fullCatalogMode: PreparedModelRuntimeCatalogMode = "live"; + const liveWorkspaceFacts = ( + await prepareWorkspaceBuildGroup([params.agentFacts.input], fullCatalogMode) + ).workspaceFacts; + assertCurrent(); + // Agent facts remain bound to the published turn generation. Auth mutations advance + // that owner generation, so these guards reject rather than mixing credential facts. + const catalogSource = await prepareAgentCatalogSource( + params.agentFacts, + liveWorkspaceFacts, + fullCatalogMode, + false, + ); + assertCurrent(); + const facts = await prepareFullCatalogFacts( + params.agentFacts, + liveWorkspaceFacts, + fullCatalogMode, + catalogSource, + ); + assertCurrent(); + fullCatalog = facts.modelCatalog; + return fullCatalog; + }), + }).finally(() => { + pending = undefined; + }); + } + return pending; + }, + }; +} + +function createSnapshot( + agentFacts: PreparedModelRuntimeAgentFacts, + workspaceFacts: PreparedModelRuntimeWorkspaceFacts, + catalogFacts: PreparedModelRuntimeCatalogFacts, + catalogAccess: PreparedModelRuntimeCatalogAccess, +): PreparedModelRuntimeSnapshot { + const { credentials, input } = agentFacts; + const { mediaCapabilityProviders, messageToolCatalog, pluginMetadataSnapshot } = workspaceFacts; + const { configuredRuntimeModels, inlineProviderModels, modelCatalog, templateModelRegistry } = + catalogFacts; + const createStores = (): PreparedModelRuntimeStores => { + // Runtime API keys and session extensions mutate these objects. Fork them per run while the + // credential map and parsed catalog remain owned by the lifecycle snapshot. + const authStorage = AuthStorage.inMemory(credentials); + return { authStorage, modelRegistry: templateModelRegistry.fork(authStorage) }; + }; + return Object.freeze({ + ...(input.agentId ? { agentId: input.agentId } : {}), + agentDir: input.agentDir, + activeProjectKeys: [], + ...(input.inheritedAuthDir ? { inheritedAuthDir: input.inheritedAuthDir } : {}), + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + config: input.config, + metadataSnapshot: pluginMetadataSnapshot, + ...(messageToolCatalog ? { messageToolCatalog } : {}), + ...(mediaCapabilityProviders ? { mediaCapabilityProviders } : {}), + modelCatalog, + loadFullModelCatalog: catalogAccess.loadFullModelCatalog, + configuredRuntimeModels, + inlineProviderModels, + createStores, + }); +} + +async function buildSnapshotBatch( + inputs: readonly PreparedModelRuntimeInput[], + catalogMode: PreparedModelRuntimeCatalogMode, + agentBuildCompletions: Map>, + generationGuards: ReadonlyMap boolean>, + buildGuards: PreparedModelRuntimeBuildGuards, + onBuildStats?: (stats: PreparedModelRuntimeBuildStats) => void, +): Promise { + const groups = new Map(); + for (const input of inputs) { + const key = preparedModelRuntimeWorkspaceFactsKey(input); + const group = groups.get(key); + if (group) { + group.push(input); + } else { + groups.set(key, [input]); + } + } + const preparedInputs = new Map(); + const workspaceFacts = new Map(); + const workspaceKeys = new Map(); + let runtimePluginMs = 0; + let pluginMetadataMs = 0; + let staticProviderCatalogMs = 0; + let ambientCredentialsMs = 0; + let agentFactsMs = 0; + let configuredProjectionMs = 0; + const workspaceFactsStartedAt = performance.now(); + // Workspace plugin loading and static hooks are intentionally sequential. Large parallel + // workspace fanout recreates the CPU/RSS spike this generation boundary is meant to contain. + for (const [key, group] of groups) { + if (typeof buildGuards === "function") { + assertPreparedModelRuntimeInputsCurrent(group, buildGuards); + } + const prepared = await prepareWorkspaceBuildGroup(group, catalogMode); + assertPreparedModelRuntimeInputsCurrent(group, buildGuards); + workspaceFacts.set(key, prepared.workspaceFacts); + runtimePluginMs += prepared.buildStats.runtimePluginMs; + pluginMetadataMs += prepared.buildStats.pluginMetadataMs; + staticProviderCatalogMs += prepared.buildStats.staticProviderCatalogMs; + ambientCredentialsMs += prepared.buildStats.ambientCredentialsMs; + agentFactsMs += prepared.buildStats.agentFactsMs; + configuredProjectionMs += prepared.buildStats.configuredProjectionMs; + for (const agentFacts of prepared.agentFacts) { + preparedInputs.set(agentFacts.input, agentFacts); + workspaceKeys.set(agentFacts.input, key); + } + } + const workspaceFactsMs = performance.now() - workspaceFactsStartedAt; + const catalogSourceStartedAt = performance.now(); + const catalogSources = new Map(); + if (catalogMode === "live") { + const sourceInputsByAgentDir = new Map(); + for (const input of inputs) { + const group = sourceInputsByAgentDir.get(input.agentDir); + if (group) { + group.push(input); + } else { + sourceInputsByAgentDir.set(input.agentDir, [input]); + } + } + const sourceErrors: unknown[] = []; + const sourceBuild = await runTasksWithConcurrency({ + limit: MAX_CONCURRENT_MODEL_RUNTIME_AGENT_SOURCE_BUILDS, + errorMode: "stop", + onTaskError: (error) => { + sourceErrors.push(error); + }, + tasks: [...sourceInputsByAgentDir.values()].map((sourceInputs) => async () => { + // Generated catalogs are agent-directory owned. Preserve write serialization within one + // directory while allowing bounded progress across distinct agents. + for (const input of sourceInputs) { + const prepared = preparedInputs.get(input); + const workspaceKey = workspaceKeys.get(input); + const facts = workspaceKey ? workspaceFacts.get(workspaceKey) : undefined; + if (!prepared) { + throw new Error(`prepared model runtime agent facts missing for ${input.agentDir}`); + } + if (!facts) { + throw new Error(`prepared model runtime workspace facts missing for ${input.agentDir}`); + } + // A replacement waits for this batch's completion. Stop the stale batch before another + // same-directory write so a superseded generation cannot overwrite catalog state. + assertPreparedModelRuntimeInputCurrent(input, buildGuards); + const catalogSource = await prepareAgentCatalogSource(prepared, facts, catalogMode); + assertPreparedModelRuntimeInputCurrent(input, buildGuards); + catalogSources.set(input, catalogSource); + } + }), + }); + if (sourceBuild.hasError) { + // A superseded owner is lifecycle control flow. Preserve any genuine in-flight sibling + // failure so auth refresh diagnostics do not disappear behind that expected cancellation. + throw toPreparedModelRuntimeError( + sourceErrors.find( + (error) => !(error instanceof PreparedModelRuntimePublicationSupersededError), + ) ?? sourceBuild.firstError, + ); + } + } + const catalogSourceMs = performance.now() - catalogSourceStartedAt; + const preparedCatalogs = new Map(); + let runtimeRegistryCount = 0; + const registryStartedAt = performance.now(); + if (catalogMode === "live") { + // Explicit live owners still request the complete inventory. Keep those builds sequential + // instead of multiplying heap and GC pressure when a command names several agents. + for (const input of inputs) { + const agentFacts = preparedInputs.get(input); + const workspaceKey = workspaceKeys.get(input); + const facts = workspaceKey ? workspaceFacts.get(workspaceKey) : undefined; + if (!agentFacts || !facts) { + throw new Error(`prepared model runtime facts missing for ${input.agentDir}`); + } + const catalogSource = catalogSources.get(input); + if (!catalogSource) { + throw new Error(`prepared model runtime catalog source missing for ${input.agentDir}`); + } + assertPreparedModelRuntimeInputCurrent(input, buildGuards); + preparedCatalogs.set( + input, + await prepareFullCatalogFacts(agentFacts, facts, catalogMode, catalogSource), + ); + assertPreparedModelRuntimeInputCurrent(input, buildGuards); + runtimeRegistryCount += 1; + } + } else { + for (const [workspaceKey, group] of groups) { + assertPreparedModelRuntimeInputsCurrent(group, buildGuards); + const facts = workspaceFacts.get(workspaceKey); + if (!facts) { + throw new Error(`prepared model runtime workspace facts missing for ${workspaceKey}`); + } + const batch = prepareConfiguredRuntimeFactsBatch({ + agentFacts: group.map((input) => { + const agentFacts = preparedInputs.get(input); + if (!agentFacts) { + throw new Error(`prepared model runtime facts missing for ${input.agentDir}`); + } + return agentFacts; + }), + workspaceFacts: facts, + }); + runtimeRegistryCount += batch.registryCount; + for (const [input, catalogFacts] of batch.catalogs) { + preparedCatalogs.set(input, catalogFacts); + } + assertPreparedModelRuntimeInputsCurrent(group, buildGuards); + } + } + const registryMs = performance.now() - registryStartedAt; + const preparedAgentFacts = [...preparedInputs.values()]; + const configuredRuntimeModelCount = preparedAgentFacts.reduce( + (count, facts) => count + facts.configuredRuntimeModels.length, + 0, + ); + const generatedCatalogPluginCount = new Set( + preparedAgentFacts.flatMap((facts) => facts.configuredGeneratedCatalogPluginIds), + ).size; + const generatedCatalogReadCount = preparedAgentFacts.reduce( + (count, facts) => count + facts.configuredGeneratedCatalogPluginIds.length, + 0, + ); + onBuildStats?.({ + agentCount: inputs.length, + workspaceGroupCount: groups.size, + configuredFactsGroupCount: groups.size, + catalogSourceCount: + catalogMode === "live" + ? [...preparedInputs.values()].filter(({ input }) => !input.readOnly).length + : 0, + credentialGroupCount: new Set( + [...preparedInputs.values()].map((agentFacts) => + fingerprintPreparedRuntimeFacts(agentFacts.credentials), + ), + ).size, + catalogGroupCount: catalogMode === "live" ? inputs.length : 0, + runtimeRegistryCount, + configuredRuntimeModelCount, + generatedCatalogPluginCount, + generatedCatalogReadCount, + workspaceFactsMs, + runtimePluginMs, + pluginMetadataMs, + staticProviderCatalogMs, + ambientCredentialsMs, + agentFactsMs, + configuredProjectionMs, + catalogSourceMs, + registryMs, + sourceConcurrencyLimit: MAX_CONCURRENT_MODEL_RUNTIME_AGENT_SOURCE_BUILDS, + fullCatalogConcurrencyLimit: MAX_CONCURRENT_FULL_MODEL_CATALOG_BUILDS, + }); + assertPreparedModelRuntimeInputsCurrent(inputs, buildGuards); + return inputs.map((input) => { + const agentFacts = preparedInputs.get(input); + const workspaceKey = workspaceKeys.get(input); + const facts = workspaceKey ? workspaceFacts.get(workspaceKey) : undefined; + const catalogFacts = preparedCatalogs.get(input); + if (!agentFacts || !facts || !catalogFacts) { + throw new Error(`prepared model runtime snapshot facts missing for ${input.agentDir}`); + } + return createSnapshot( + agentFacts, + facts, + catalogFacts, + createFullModelCatalogAccess({ + agentFacts, + workspaceFacts: facts, + agentBuildCompletions, + isCurrent: generationGuards.get(input) ?? (() => false), + ...(catalogMode === "live" ? { eagerCatalog: catalogFacts.modelCatalog } : {}), + }), + ); + }); +} + +export function startSerializedSnapshotBuildBatch( + inputs: readonly PreparedModelRuntimeInput[], + agentBuildCompletions: Map>, + buildTimeoutMs: number, + catalogMode: PreparedModelRuntimeCatalogMode = "live", + onBuildStats?: (stats: PreparedModelRuntimeBuildStats) => void, + generationGuards: ReadonlyMap boolean> = new Map(), + buildGuards: PreparedModelRuntimeBuildGuards = generationGuards, +): { + pending: Promise; + completion: Promise; +} { + const agentDirs = [...new Set(inputs.map((input) => input.agentDir))]; + const previousBuildCompletions = [ + ...new Set( + agentDirs + .map((agentDir) => agentBuildCompletions.get(agentDir)) + .filter((completion): completion is Promise => completion !== undefined), + ), + ]; + // Lifecycle events may overlap. The timeout covers queueing plus this build, while completion + // follows the real work so a timed-out generation can never overlap a replacement. + const startBuild = (async () => { + if (previousBuildCompletions.length > 0) { + await Promise.all(previousBuildCompletions); + } + return { + actualBuild: buildSnapshotBatch( + inputs, + catalogMode, + agentBuildCompletions, + generationGuards, + buildGuards, + onBuildStats, + ), + }; + })(); + const completion = startBuild + .then(async ({ actualBuild }) => await actualBuild) + .then( + () => undefined, + () => undefined, + ); + for (const agentDir of agentDirs) { + agentBuildCompletions.set(agentDir, completion); + void completion.then(() => { + if (agentBuildCompletions.get(agentDir) === completion) { + agentBuildCompletions.delete(agentDir); + } + }); + } + return { + pending: withTimeout( + async () => { + const { actualBuild } = await startBuild; + return await actualBuild; + }, + buildTimeoutMs, + "prepared model runtime publication", + ), + completion, + }; +} + +export function startSerializedSnapshotBuild( + input: PreparedModelRuntimeInput, + agentBuildCompletions: Map>, + buildTimeoutMs: number, + catalogMode: PreparedModelRuntimeCatalogMode = "live", + generationGuard: () => boolean = () => true, +): { + pending: Promise; + completion: Promise; +} { + const build = startSerializedSnapshotBuildBatch( + [input], + agentBuildCompletions, + buildTimeoutMs, + catalogMode, + undefined, + new Map([[input, generationGuard]]), + ); + return { + pending: build.pending.then((snapshots) => snapshots[0]!), + completion: build.completion, + }; +} diff --git a/src/agents/prepared-model-runtime.configured.ts b/src/agents/prepared-model-runtime.configured.ts new file mode 100644 index 000000000000..36c361a07095 --- /dev/null +++ b/src/agents/prepared-model-runtime.configured.ts @@ -0,0 +1,232 @@ +import { + collectConfiguredModelRefs, + type ConfiguredModelRef, +} from "@openclaw/model-catalog-core/configured-model-refs"; +import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; +import { MODEL_APIS } from "../config/types.models.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.types.js"; +import { + normalizePluginDiscoveryResult, + type PreparedProviderStaticCatalog, +} from "../plugins/provider-discovery.js"; +import type { ProviderRuntimeModel } from "../plugins/provider-runtime-model.types.js"; +import { resolveAgentEntry } from "./agent-scope-config.js"; +import { buildInlineProviderModels } from "./embedded-agent-runner/model.inline-provider.js"; +import { staticModelIdMatches } from "./embedded-agent-runner/model.static-id.js"; +import type { ModelCatalogEntry } from "./model-catalog.js"; +import type { AuthStorageData } from "./sessions/auth-storage.js"; + +export type PreparedConfiguredRuntimeModel = Readonly<{ + provider: string; + modelId: string; + model: ProviderRuntimeModel; +}>; + +/** Collects defaults, global refs, and only the selected agent's overrides. */ +export function collectPreparedModelRuntimeConfiguredRefs( + config: OpenClawConfig, + agentId: string | undefined, +): ConfiguredModelRef[] { + if (!agentId) { + return collectConfiguredModelRefs(config); + } + const entry = resolveAgentEntry(config, agentId); + return collectConfiguredModelRefs({ + ...config, + agents: { + ...(config.agents?.defaults ? { defaults: config.agents.defaults } : {}), + list: entry ? [entry] : [], + }, + }); +} + +function isCatalogModelApi( + value: string | undefined, +): value is NonNullable { + return value !== undefined && (MODEL_APIS as readonly string[]).includes(value); +} + +export function toStaticCatalogEntry(model: ProviderRuntimeModel): ModelCatalogEntry { + return { + id: model.id, + name: model.name ?? model.id, + provider: model.provider, + ...(isCatalogModelApi(model.api) ? { api: model.api } : {}), + ...(model.baseUrl ? { baseUrl: model.baseUrl } : {}), + ...(model.contextWindow ? { contextWindow: model.contextWindow } : {}), + ...(model.contextTokens ? { contextTokens: model.contextTokens } : {}), + ...(model.reasoning !== undefined ? { reasoning: model.reasoning } : {}), + ...(model.input ? { input: model.input } : {}), + ...(model.params ? { params: model.params } : {}), + ...(model.compat ? { compat: model.compat } : {}), + ...(model.mediaInput ? { mediaInput: model.mediaInput } : {}), + }; +} + +export function collectPreparedModelRuntimeProviderIds( + config: OpenClawConfig, + credentials: Readonly, + includeCredentialProviders: boolean, + configuredModelRefs: readonly ConfiguredModelRef[] = collectConfiguredModelRefs(config), +): string[] { + const providerIds = new Set(); + const addProviderId = (value: string) => { + const providerId = normalizeProviderId(value); + if (providerId) { + providerIds.add(providerId); + } + }; + if (includeCredentialProviders) { + for (const providerId of Object.keys(credentials)) { + addProviderId(providerId); + } + } + for (const providerId of Object.keys(config.models?.providers ?? {})) { + addProviderId(providerId); + } + for (const ref of configuredModelRefs) { + const separator = ref.value.indexOf("/"); + if (separator > 0) { + addProviderId(ref.value.slice(0, separator)); + } + } + return [...providerIds].toSorted((left, right) => left.localeCompare(right)); +} + +function hasConfiguredInlineProviderModel( + config: OpenClawConfig, + provider: string, + modelId: string, +): boolean { + return Object.entries(config.models?.providers ?? {}).some( + ([providerId, providerConfig]) => + normalizeProviderId(providerId) === provider && + (providerConfig.models ?? []).some((model) => + staticModelIdMatches({ + candidateId: model.id, + rowProvider: providerId, + provider, + modelId, + }), + ), + ); +} + +export function collectConfiguredProviderIdsNeedingStaticCatalog(params: { + config: OpenClawConfig; + configuredModelRefs?: readonly ConfiguredModelRef[]; + resolveStaticCatalogModel: (lookup: { + provider: string; + modelId: string; + }) => ProviderRuntimeModel | undefined; +}): string[] { + const providerIds = new Set(); + for (const { value } of params.configuredModelRefs ?? collectConfiguredModelRefs(params.config)) { + const separator = value.indexOf("/"); + if (separator <= 0 || separator >= value.length - 1) { + continue; + } + const provider = normalizeProviderId(value.slice(0, separator)); + const modelId = value.slice(separator + 1).trim(); + if ( + !provider || + !modelId || + hasConfiguredInlineProviderModel(params.config, provider, modelId) || + params.resolveStaticCatalogModel({ provider, modelId }) + ) { + continue; + } + providerIds.add(provider); + } + return [...providerIds].toSorted((left, right) => left.localeCompare(right)); +} + +export function prepareConfiguredRuntimeModels(params: { + config: OpenClawConfig; + configuredModelRefs?: readonly ConfiguredModelRef[]; + metadataSnapshot: PluginMetadataSnapshot; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; + providerStaticModels: readonly ProviderRuntimeModel[]; + resolveStaticCatalogModel: (lookup: { + provider: string; + modelId: string; + }) => ProviderRuntimeModel | undefined; +}): PreparedConfiguredRuntimeModel[] { + const prepared: PreparedConfiguredRuntimeModel[] = []; + const seen = new Set(); + for (const { value } of params.configuredModelRefs ?? collectConfiguredModelRefs(params.config)) { + const separator = value.indexOf("/"); + if (separator <= 0 || separator >= value.length - 1) { + continue; + } + const provider = normalizeProviderId(value.slice(0, separator)); + const modelId = value.slice(separator + 1).trim(); + if (!provider || !modelId) { + continue; + } + const key = `${provider}\0${modelId.toLowerCase()}`; + if (seen.has(key)) { + continue; + } + seen.add(key); + // Match request-time fallback precedence exactly: manifest/runtime-discovery rows win, + // and the provider-static catalog fills only models absent from that surface. + const model = + params.resolveStaticCatalogModel({ provider, modelId }) ?? + findPreparedProviderStaticCatalogModel({ + prepared: params.preparedStaticProviderCatalog, + metadataSnapshot: params.metadataSnapshot, + provider, + modelId, + }) ?? + params.providerStaticModels.find((candidate) => + staticModelIdMatches({ + candidateId: candidate.id, + rowProvider: candidate.provider, + provider, + modelId, + }), + ); + if (model) { + prepared.push({ provider, modelId, model }); + } + } + return prepared; +} + +function findPreparedProviderStaticCatalogModel(params: { + prepared: PreparedProviderStaticCatalog | undefined; + metadataSnapshot: PluginMetadataSnapshot; + provider: string; + modelId: string; +}): ProviderRuntimeModel | undefined { + if (!params.prepared) { + return undefined; + } + for (const { provider, result } of params.prepared.entries) { + for (const [providerId, providerConfig] of Object.entries( + normalizePluginDiscoveryResult({ provider, result }), + )) { + const model = (providerConfig.models ?? []).find((candidate) => + staticModelIdMatches({ + candidateId: candidate.id, + rowProvider: providerId, + provider: params.provider, + modelId: params.modelId, + }), + ); + if (!model) { + continue; + } + const [resolved] = buildInlineProviderModels( + { [providerId]: { ...providerConfig, models: [model] } }, + { providerMetadataOwners: params.metadataSnapshot.owners }, + ); + if (resolved) { + return resolved as ProviderRuntimeModel; + } + } + } + return undefined; +} diff --git a/src/agents/prepared-model-runtime.errors.ts b/src/agents/prepared-model-runtime.errors.ts new file mode 100644 index 000000000000..1c97d5717b75 --- /dev/null +++ b/src/agents/prepared-model-runtime.errors.ts @@ -0,0 +1,7 @@ +export class PreparedModelRuntimeOwnerNotPublishedError extends Error {} + +export class PreparedModelRuntimePublicationSupersededError extends PreparedModelRuntimeOwnerNotPublishedError {} + +export function toPreparedModelRuntimeError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} diff --git a/src/agents/prepared-model-runtime.facts.ts b/src/agents/prepared-model-runtime.facts.ts new file mode 100644 index 000000000000..112004a4b608 --- /dev/null +++ b/src/agents/prepared-model-runtime.facts.ts @@ -0,0 +1,688 @@ +import fs from "node:fs"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import type { ConfiguredModelRef } from "@openclaw/model-catalog-core/configured-model-refs"; +import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; +import type { PreparedMessageToolCatalog } from "../channels/plugins/message-action-discovery.js"; +import { hashRuntimeConfigValue } from "../config/runtime-snapshot.js"; +import { sha256Base64Url } from "../infra/crypto-digest.js"; +import { prepareMediaCapabilityProviders } from "../plugins/capability-provider-runtime.js"; +import { resolvePluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; +import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.types.js"; +import { + getPreparedMessageToolCatalog, + getPreparedMessageToolCatalogForRegistry, +} from "../plugins/prepared-message-tool-catalog.js"; +import type { PreparedProviderStaticCatalog } from "../plugins/provider-discovery.js"; +import type { ProviderRuntimeModel } from "../plugins/provider-runtime-model.types.js"; +import { resolveRuntimeSyntheticAuthProviderRefs } from "../plugins/synthetic-auth.runtime.js"; +import type { ProviderPlugin } from "../plugins/types.js"; +import type { AgentCredentialMap } from "./agent-auth-credentials.js"; +import { resolveAmbientAgentCredentialsForDiscovery } from "./agent-auth-discovery.js"; +import { + discoverAuthStorage, + discoverModels, + discoverModelsFromCapturedSources, +} from "./agent-model-discovery.js"; +import { + buildInlineProviderModels, + type InlineModelEntry, +} from "./embedded-agent-runner/model.inline-provider.js"; +import { + createBundledStaticCatalogModelResolver, + loadBundledProviderStaticCatalogContextModels, +} from "./embedded-agent-runner/model.static-catalog.js"; +import { buildPreparedModelCatalogSnapshot, type ModelCatalogEntry } from "./model-catalog.js"; +import type { ModelCatalogSnapshot } from "./model-catalog.types.js"; +import { buildConfiguredModelCatalog } from "./model-selection-shared.js"; +import { ensureOpenClawModelsJson, planOpenClawModelsJsonSource } from "./models-config.js"; +import { prepareImplicitProviderStaticCatalog } from "./models-config.providers.implicit.js"; +import { + loadPersistedPluginModelCatalogsReadOnly, + resolvePluginModelCatalogOwnerPluginId, + type PersistedPluginModelCatalog, +} from "./plugin-model-catalog.js"; +import { + collectPreparedModelRuntimeConfiguredRefs, + collectConfiguredProviderIdsNeedingStaticCatalog, + collectPreparedModelRuntimeProviderIds, + prepareConfiguredRuntimeModels, + toStaticCatalogEntry, + type PreparedConfiguredRuntimeModel, +} from "./prepared-model-runtime.configured.js"; +import type { + PreparedModelRuntimeBuildStats, + PreparedModelRuntimeCatalogMode, + PreparedModelRuntimeInput, +} from "./prepared-model-runtime.types.js"; +import { ensureRuntimePluginsLoaded } from "./runtime-plugins.js"; +import type { AuthStorage, AuthStorageData } from "./sessions/auth-storage.js"; +import type { ModelRegistry } from "./sessions/model-registry.js"; +import { stableStringify } from "./stable-stringify.js"; + +const MODEL_RUNTIME_PROVIDER_DISCOVERY_TIMEOUT_MS = 5_000; + +type PreparedModelRuntimeAgentBaseFacts = { + input: PreparedModelRuntimeInput; + env: NodeJS.ProcessEnv; + templateAuthStorage: AuthStorage; + credentials: Readonly; + providerIds: string[]; + configuredModelRefs: readonly ConfiguredModelRef[]; +}; + +export type PreparedModelRuntimeAgentFacts = PreparedModelRuntimeAgentBaseFacts & { + configuredRuntimeModels: readonly PreparedConfiguredRuntimeModel[]; + configuredGeneratedCatalogPluginIds: readonly string[]; +}; + +export type PreparedModelRuntimeWorkspaceFacts = { + pluginMetadataSnapshot: PluginMetadataSnapshot; + messageToolCatalog?: PreparedMessageToolCatalog; + mediaCapabilityProviders?: ReturnType; + preparedStaticProviderCatalog?: PreparedProviderStaticCatalog; + providerStaticModels?: readonly ProviderRuntimeModel[]; + providerStaticModelsComplete: boolean; + inlineProviderModels: readonly InlineModelEntry[]; + configuredCatalogEntries: readonly ModelCatalogEntry[]; +}; + +export type PreparedModelRuntimeCatalogFacts = { + templateModelRegistry: ModelRegistry; + modelCatalog: ModelCatalogSnapshot; + configuredRuntimeModels: readonly PreparedConfiguredRuntimeModel[]; + inlineProviderModels: readonly InlineModelEntry[]; +}; + +export type PreparedModelRuntimeCatalogSource = Readonly<{ + modelsJsonContents: string | null; + pluginCatalogs: readonly PersistedPluginModelCatalog[]; +}>; + +type PreparedConfiguredRegistryGroup = { + agentFacts: PreparedModelRuntimeAgentFacts[]; + modelsJsonContents: string | null; + oauthProviders: ReturnType; + pluginCatalogs: readonly PersistedPluginModelCatalog[]; +}; + +function prepareAgentFacts( + input: PreparedModelRuntimeInput, + catalogMode: PreparedModelRuntimeCatalogMode, + ambientCredentials: Readonly, +): PreparedModelRuntimeAgentBaseFacts { + const env = input.env ?? process.env; + const templateAuthStorage = discoverAuthStorage(input.agentDir, { + config: input.config, + // Snapshot construction never initializes, migrates, or externally syncs auth. ModelRegistry + // discovery only parses the credential generation captured here. + readOnly: true, + ambientCredentials, + ...(input.skipCredentials ? { skipCredentials: true } : {}), + ...(input.inheritedAuthDir ? { inheritedAuthDir: input.inheritedAuthDir } : {}), + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + ...(input.env ? { env } : {}), + }); + const credentials = templateAuthStorage.getAll(); + const configuredModelRefs = collectPreparedModelRuntimeConfiguredRefs( + input.config, + input.agentId, + ); + return { + input, + env, + templateAuthStorage, + credentials, + configuredModelRefs, + // Gateway startup prepares only providers named by config/model selection. An unrelated + // stored credential must not pull that provider's complete catalog into the admission path. + providerIds: collectPreparedModelRuntimeProviderIds( + input.config, + credentials, + catalogMode === "live", + configuredModelRefs, + ), + }; +} + +function listPreparedSyntheticAuthProviderRefs(providers: readonly ProviderPlugin[]): string[] { + return [ + ...new Set( + providers.flatMap((provider) => + typeof provider.resolveSyntheticAuth === "function" + ? [provider.id, ...(provider.aliases ?? []), ...(provider.hookAliases ?? [])] + : [], + ), + ), + ].toSorted((left, right) => left.localeCompare(right)); +} + +function resolvePreparedSyntheticAuth(params: { + config: PreparedModelRuntimeInput["config"]; + provider: string; + providers: readonly ProviderPlugin[]; +}): { apiKey?: string } | undefined { + const normalizedProvider = normalizeProviderId(params.provider); + const providerPlugin = params.providers.find((candidate) => + [candidate.id, ...(candidate.aliases ?? []), ...(candidate.hookAliases ?? [])].some( + (ref) => normalizeProviderId(ref) === normalizedProvider, + ), + ); + return ( + providerPlugin?.resolveSyntheticAuth?.({ + config: params.config, + provider: params.provider, + providerConfig: Object.entries(params.config.models?.providers ?? {}).find( + ([providerId]) => normalizeProviderId(providerId) === normalizedProvider, + )?.[1], + }) ?? undefined + ); +} + +export function preparedModelRuntimeWorkspaceFactsKey(input: PreparedModelRuntimeInput): string { + return JSON.stringify({ + // Config is the process generation. Agent-specific configured refs are projected after these + // workspace/plugin facts are shared. + config: hashRuntimeConfigValue(input.config), + env: hashRuntimeConfigValue(input.env ?? process.env), + readOnly: input.readOnly === true, + workspaceDir: input.workspaceDir, + }); +} + +export async function prepareWorkspaceBuildGroup( + inputs: readonly PreparedModelRuntimeInput[], + catalogMode: PreparedModelRuntimeCatalogMode, +): Promise<{ + agentFacts: PreparedModelRuntimeAgentFacts[]; + workspaceFacts: PreparedModelRuntimeWorkspaceFacts; + buildStats: Pick< + PreparedModelRuntimeBuildStats, + | "runtimePluginMs" + | "pluginMetadataMs" + | "staticProviderCatalogMs" + | "ambientCredentialsMs" + | "agentFactsMs" + | "configuredProjectionMs" + >; +}> { + const input = inputs[0]; + if (!input) { + throw new Error("prepared model runtime workspace group is empty"); + } + const env = input.env ?? process.env; + const runtimePluginStartedAt = performance.now(); + const runtimePluginRegistry = + catalogMode === "live" && !input.readOnly + ? ensureRuntimePluginsLoaded({ + config: input.config, + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }) + : undefined; + const runtimePluginMs = performance.now() - runtimePluginStartedAt; + const pluginMetadataStartedAt = performance.now(); + const pluginMetadataSnapshot = resolvePluginMetadataSnapshot({ + config: input.config, + env, + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }); + const pluginMetadataMs = performance.now() - pluginMetadataStartedAt; + const mediaCapabilityProviders = + input.readOnly || !runtimePluginRegistry + ? undefined + : prepareMediaCapabilityProviders({ + cfg: input.config, + pluginMetadataSnapshot, + registry: runtimePluginRegistry, + }); + const messageToolCatalog = runtimePluginRegistry + ? getPreparedMessageToolCatalogForRegistry(runtimePluginRegistry) + : catalogMode === "live" + ? getPreparedMessageToolCatalog() + : undefined; + const resolveManifestStaticCatalogModel = createBundledStaticCatalogModelResolver({ + cfg: input.config, + env, + includeRuntimeDiscovery: true, + metadataSnapshot: pluginMetadataSnapshot, + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }); + const configuredManifestModels = new Map(); + const resolveConfiguredManifestModel = (lookup: { provider: string; modelId: string }) => { + const key = `${normalizeProviderId(lookup.provider)}\0${lookup.modelId.trim().toLowerCase()}`; + if (configuredManifestModels.has(key)) { + return configuredManifestModels.get(key); + } + const model = resolveManifestStaticCatalogModel(lookup); + configuredManifestModels.set(key, model); + return model; + }; + const configuredProviderIds = collectPreparedModelRuntimeProviderIds(input.config, {}, false); + const staticCatalogProviderIds = collectConfiguredProviderIdsNeedingStaticCatalog({ + config: input.config, + resolveStaticCatalogModel: resolveConfiguredManifestModel, + }); + const staticProviderCatalogStartedAt = performance.now(); + const preparedStaticProviderCatalog = + catalogMode === "static" + ? await prepareImplicitProviderStaticCatalog({ + config: input.config, + env, + pluginMetadataSnapshot, + providerDiscoveryProviderIds: configuredProviderIds, + staticCatalogProviderIds, + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }) + : undefined; + const staticProviderCatalogMs = performance.now() - staticProviderCatalogStartedAt; + const preparedSyntheticAuthProviders = preparedStaticProviderCatalog?.providers ?? []; + // Static Gateway publication consumes provider discovery entrypoints without activating plugin + // runtimes. The run boundary already owns runtime activation for its exact workspace. + const ambientCredentialsStartedAt = performance.now(); + const ambientCredentials = resolveAmbientAgentCredentialsForDiscovery({ + config: input.config, + env, + syntheticAuthProviderRefs: + catalogMode === "static" + ? listPreparedSyntheticAuthProviderRefs(preparedSyntheticAuthProviders) + : resolveRuntimeSyntheticAuthProviderRefs({ + config: input.config, + env, + index: pluginMetadataSnapshot.index, + registryDiagnostics: pluginMetadataSnapshot.registryDiagnostics, + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }), + ...(catalogMode === "static" + ? { + resolveSyntheticAuth: (provider: string) => + resolvePreparedSyntheticAuth({ + config: input.config, + provider, + providers: preparedSyntheticAuthProviders, + }), + } + : {}), + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }); + const ambientCredentialsMs = performance.now() - ambientCredentialsStartedAt; + const agentFactsStartedAt = performance.now(); + const agentBaseFacts = inputs.map((candidate) => + prepareAgentFacts(candidate, catalogMode, ambientCredentials), + ); + const agentFactsMs = performance.now() - agentFactsStartedAt; + const configuredProjectionStartedAt = performance.now(); + const providerStaticModels = + catalogMode === "static" + ? [] + : await loadBundledProviderStaticCatalogContextModels({ + cfg: input.config, + env, + metadataSnapshot: pluginMetadataSnapshot, + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }); + // Provider definitions are process/config facts. Which refs are admitted remains agent-owned. + const inlineProviderModels = buildInlineProviderModels(input.config.models?.providers ?? {}, { + providerMetadataOwners: pluginMetadataSnapshot.owners, + }); + const configuredCatalogEntries = buildConfiguredModelCatalog({ + cfg: input.config, + manifestPlugins: pluginMetadataSnapshot.plugins, + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }); + const agentFacts: PreparedModelRuntimeAgentFacts[] = []; + for (const facts of agentBaseFacts) { + const configuredRuntimeModels = prepareConfiguredRuntimeModels({ + config: facts.input.config, + configuredModelRefs: facts.configuredModelRefs, + metadataSnapshot: pluginMetadataSnapshot, + ...(preparedStaticProviderCatalog ? { preparedStaticProviderCatalog } : {}), + providerStaticModels, + resolveStaticCatalogModel: resolveConfiguredManifestModel, + }); + const configuredEntryKeys = new Set(configuredCatalogEntries.map(modelCatalogEntryKey)); + for (const configured of configuredRuntimeModels) { + configuredEntryKeys.add( + modelCatalogEntryKey({ provider: configured.provider, id: configured.modelId }), + ); + } + const configuredGeneratedCatalogPluginIds = [ + ...new Set( + facts.configuredModelRefs.flatMap(({ value }) => { + const separator = value.indexOf("/"); + if (separator <= 0 || separator >= value.length - 1) { + return []; + } + const provider = normalizeProviderId(value.slice(0, separator)); + const modelId = value.slice(separator + 1).trim(); + if ( + !provider || + !modelId || + configuredEntryKeys.has(modelCatalogEntryKey({ provider, id: modelId })) + ) { + return []; + } + const pluginId = resolvePluginModelCatalogOwnerPluginId({ + providerId: provider, + pluginMetadataSnapshot, + }); + return pluginId ? [pluginId] : []; + }), + ), + ].toSorted((left, right) => left.localeCompare(right)); + agentFacts.push({ + ...facts, + configuredRuntimeModels, + configuredGeneratedCatalogPluginIds, + }); + } + const configuredProjectionMs = performance.now() - configuredProjectionStartedAt; + return { + agentFacts, + buildStats: { + runtimePluginMs, + pluginMetadataMs, + staticProviderCatalogMs, + ambientCredentialsMs, + agentFactsMs, + configuredProjectionMs, + }, + workspaceFacts: { + pluginMetadataSnapshot, + messageToolCatalog, + providerStaticModelsComplete: catalogMode === "live", + inlineProviderModels, + configuredCatalogEntries, + ...(mediaCapabilityProviders ? { mediaCapabilityProviders } : {}), + ...(preparedStaticProviderCatalog ? { preparedStaticProviderCatalog } : {}), + ...(providerStaticModels ? { providerStaticModels } : {}), + }, + }; +} + +export async function prepareFullCatalogFacts( + agentFacts: PreparedModelRuntimeAgentFacts, + workspaceFacts: PreparedModelRuntimeWorkspaceFacts, + catalogMode: PreparedModelRuntimeCatalogMode, + catalogSource?: PreparedModelRuntimeCatalogSource, +): Promise { + const { credentials, env, input, templateAuthStorage } = agentFacts; + const { pluginMetadataSnapshot, preparedStaticProviderCatalog } = workspaceFacts; + const templateModelRegistry = discoverModels(templateAuthStorage, input.agentDir, { + config: input.config, + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + pluginMetadataSnapshot, + ...(catalogMode === "static" ? { normalizeModels: false } : {}), + ...(catalogSource + ? { + includePluginCatalogs: true, + modelsJsonContents: catalogSource.modelsJsonContents, + pluginCatalogs: catalogSource.pluginCatalogs, + } + : {}), + }); + const modelCatalog = await buildPreparedModelCatalogSnapshot({ + agentDir: input.agentDir, + authCredentials: credentials, + config: input.config, + modelRegistry: templateModelRegistry, + metadataSnapshot: pluginMetadataSnapshot, + includeProviderPluginAugmentation: catalogMode === "live", + ...(input.env ? { env } : {}), + ...(input.readOnly ? { readOnly: true } : {}), + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }); + const providerStaticModels = + (workspaceFacts.providerStaticModelsComplete + ? workspaceFacts.providerStaticModels + : undefined) ?? + (await loadBundledProviderStaticCatalogContextModels({ + cfg: input.config, + env, + metadataSnapshot: pluginMetadataSnapshot, + ...(preparedStaticProviderCatalog ? { preparedStaticProviderCatalog } : {}), + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + })); + const configuredRuntimeModels = agentFacts.configuredRuntimeModels; + const staticModels = new Map(); + for (const model of [ + ...configuredRuntimeModels.map((configured) => configured.model), + ...providerStaticModels, + ]) { + const modelKey = `${normalizeProviderId(model.provider)}\0${model.id.trim().toLowerCase()}`; + if (!staticModels.has(modelKey)) { + staticModels.set(modelKey, model); + } + } + const staticEntries = [...staticModels.values()].map(toStaticCatalogEntry); + return { + templateModelRegistry, + modelCatalog: { ...modelCatalog, staticEntries }, + configuredRuntimeModels, + inlineProviderModels: workspaceFacts.inlineProviderModels, + }; +} + +function modelCatalogEntryKey(entry: Pick): string { + return `${normalizeProviderId(entry.provider)}\0${entry.id.trim().toLowerCase()}`; +} + +function createConfiguredModelCatalogSnapshot(params: { + agentFacts: PreparedModelRuntimeAgentFacts; + workspaceFacts: PreparedModelRuntimeWorkspaceFacts; + templateModelRegistry: ModelRegistry; + configuredRuntimeModels: readonly PreparedConfiguredRuntimeModel[]; +}): ModelCatalogSnapshot { + const entries = new Map(); + const addEntry = (entry: ModelCatalogEntry) => { + const key = modelCatalogEntryKey(entry); + if (!entries.has(key)) { + entries.set(key, entry); + } + }; + for (const entry of params.workspaceFacts.configuredCatalogEntries) { + addEntry(entry); + } + for (const configured of params.configuredRuntimeModels) { + addEntry(toStaticCatalogEntry(configured.model)); + } + for (const { value } of params.agentFacts.configuredModelRefs) { + const separator = value.indexOf("/"); + if (separator <= 0 || separator >= value.length - 1) { + continue; + } + const provider = normalizeProviderId(value.slice(0, separator)); + const modelId = value.slice(separator + 1).trim(); + if (!provider || !modelId) { + continue; + } + const model = params.templateModelRegistry.find(provider, modelId); + if (model) { + addEntry(toStaticCatalogEntry(model)); + } + } + const configuredEntries = [...entries.values()]; + const staticEntries = params.configuredRuntimeModels.map(({ model }) => + toStaticCatalogEntry(model), + ); + return { + entries: configuredEntries, + routeVariants: configuredEntries, + ...(staticEntries.length > 0 ? { staticEntries } : {}), + }; +} + +function prepareConfiguredRuntimeFacts( + agentFacts: PreparedModelRuntimeAgentFacts, + workspaceFacts: PreparedModelRuntimeWorkspaceFacts, + sharedTemplateModelRegistry: ModelRegistry, +): PreparedModelRuntimeCatalogFacts { + const { configuredRuntimeModels } = agentFacts; + const { inlineProviderModels } = workspaceFacts; + const templateModelRegistry = sharedTemplateModelRegistry; + return { + templateModelRegistry, + modelCatalog: createConfiguredModelCatalogSnapshot({ + agentFacts, + workspaceFacts, + templateModelRegistry, + configuredRuntimeModels, + }), + configuredRuntimeModels, + inlineProviderModels, + }; +} + +function captureModelsJsonContents(agentDir: string): string | null { + try { + return fs.readFileSync(path.join(agentDir, "models.json"), "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return null; + } + throw error; + } +} + +export function fingerprintPreparedRuntimeFacts(value: unknown): string { + return sha256Base64Url(stableStringify(value)); +} + +function hasSameOAuthProviderGeneration( + left: ReturnType, + right: ReturnType, +): boolean { + // OAuth descriptors carry executable hooks. Match those hooks by identity so equivalent + // AuthStorage instances share built-ins without merging distinct closure generations. + return ( + left.length === right.length && + left.every((provider, index) => { + const candidate = right[index]; + return ( + candidate !== undefined && + provider.id === candidate.id && + provider.name === candidate.name && + provider.usesCallbackServer === candidate.usesCallbackServer && + provider.login === candidate.login && + provider.refreshToken === candidate.refreshToken && + provider.getApiKey === candidate.getApiKey && + provider.modifyModels === candidate.modifyModels + ); + }) + ); +} + +function groupConfiguredRegistrySources( + agentFacts: readonly PreparedModelRuntimeAgentFacts[], +): PreparedConfiguredRegistryGroup[] { + const groups = new Map(); + for (const facts of agentFacts) { + const modelsJsonContents = captureModelsJsonContents(facts.input.agentDir); + const oauthProviders = facts.templateAuthStorage.getOAuthProviders(); + // Generated catalogs are agent-owned. Capture only plugins needed by unresolved configured + // refs, then group exact bytes and OAuth behavior so publication never mixes generations. + const pluginCatalogs = loadPersistedPluginModelCatalogsReadOnly( + facts.input.agentDir, + facts.configuredGeneratedCatalogPluginIds, + ); + const key = fingerprintPreparedRuntimeFacts({ + credentials: facts.credentials, + modelsJsonContents, + pluginCatalogs, + }); + const candidates = groups.get(key) ?? []; + const group = candidates.find((candidate) => + hasSameOAuthProviderGeneration(candidate.oauthProviders, oauthProviders), + ); + if (group) { + group.agentFacts.push(facts); + } else { + candidates.push({ + agentFacts: [facts], + modelsJsonContents, + oauthProviders, + pluginCatalogs, + }); + groups.set(key, candidates); + } + } + return [...groups.values()].flat(); +} + +export function prepareConfiguredRuntimeFactsBatch(params: { + agentFacts: readonly PreparedModelRuntimeAgentFacts[]; + workspaceFacts: PreparedModelRuntimeWorkspaceFacts; +}): { + catalogs: Map; + registryCount: number; +} { + const catalogs = new Map(); + let registryCount = 0; + for (const group of groupConfiguredRegistrySources(params.agentFacts)) { + const representative = group.agentFacts[0]; + if (!representative) { + continue; + } + // Catalog bytes, credentials, and OAuth provider behavior are identical inside this group. + // Parse once, then fork request auth without reopening filesystem or SQLite catalog sources. + const templateModelRegistry = discoverModelsFromCapturedSources( + representative.templateAuthStorage, + { + config: representative.input.config, + includePluginCatalogs: true, + modelsJsonContents: group.modelsJsonContents, + pluginCatalogs: group.pluginCatalogs, + pluginMetadataSnapshot: params.workspaceFacts.pluginMetadataSnapshot, + ...(representative.input.workspaceDir + ? { workspaceDir: representative.input.workspaceDir } + : {}), + }, + ); + registryCount += 1; + for (const facts of group.agentFacts) { + catalogs.set( + facts.input, + prepareConfiguredRuntimeFacts(facts, params.workspaceFacts, templateModelRegistry), + ); + } + } + return { catalogs, registryCount }; +} + +export async function prepareAgentCatalogSource( + agentFacts: PreparedModelRuntimeAgentFacts, + workspaceFacts: PreparedModelRuntimeWorkspaceFacts, + catalogMode: PreparedModelRuntimeCatalogMode, + persist = true, +): Promise { + const { env, input, providerIds } = agentFacts; + const options = { + pluginMetadataSnapshot: workspaceFacts.pluginMetadataSnapshot, + ...(workspaceFacts.preparedStaticProviderCatalog + ? { preparedStaticProviderCatalog: workspaceFacts.preparedStaticProviderCatalog } + : {}), + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + ...(input.env ? { env } : {}), + ...(catalogMode === "static" + ? { + providerDiscoveryEntriesOnly: true as const, + providerDiscoveryProviderIds: providerIds, + } + : { providerDiscoveryTimeoutMs: MODEL_RUNTIME_PROVIDER_DISCOVERY_TIMEOUT_MS }), + }; + if (!persist) { + const source = await planOpenClawModelsJsonSource(input.config, input.agentDir, options); + return { + modelsJsonContents: source.modelsJsonContents, + pluginCatalogs: source.pluginCatalogs, + }; + } + if (!input.readOnly) { + await ensureOpenClawModelsJson(input.config, input.agentDir, options); + } + // Capture immediately after the serialized write. Another owner may share this directory and + // publish a different workspace generation before full-catalog parsing begins. + return { + modelsJsonContents: captureModelsJsonContents(input.agentDir), + pluginCatalogs: loadPersistedPluginModelCatalogsReadOnly(input.agentDir), + }; +} diff --git a/src/agents/prepared-model-runtime.lifecycle.test.ts b/src/agents/prepared-model-runtime.lifecycle.test.ts index ea01f52fca07..296651d798d9 100644 --- a/src/agents/prepared-model-runtime.lifecycle.test.ts +++ b/src/agents/prepared-model-runtime.lifecycle.test.ts @@ -4,25 +4,38 @@ type LoadStaticCatalog = typeof import("./embedded-agent-runner/model.static-catalog.js").loadBundledProviderStaticCatalogContextModels; const mocks = vi.hoisted(() => ({ - authStorage: { getAll: vi.fn(() => ({ custom: { type: "api_key", key: "test-key" } })) }, + authStorage: { + getAll: vi.fn(() => ({ custom: { type: "api_key", key: "test-key" } })), + getOAuthProviders: vi.fn(() => []), + }, modelRegistry: { fork: vi.fn((authStorage: unknown) => ({ authStorage })), getAll: vi.fn(() => []), + find: vi.fn(() => null), }, - discoverAuthStorage: vi.fn(), + resolveAmbientCredentials: vi.fn((..._args: unknown[]) => ({})), + discoverAuthStorage: vi.fn((..._args: unknown[]) => undefined as unknown), discoverModels: vi.fn(), ensureOpenClawModelsJson: vi.fn(async (..._args: unknown[]) => ({ agentDir: "/tmp/agent", wrote: false, })), + planOpenClawModelsJsonSource: vi.fn(async (...args: unknown[]) => ({ + agentDir: String(args[1]), + modelsJsonContents: null, + pluginCatalogs: [], + })), buildPreparedModelCatalogSnapshot: vi.fn(async (..._args: unknown[]) => ({ entries: [], routeVariants: [], })), ensureRuntimePluginsLoaded: vi.fn(), loadStaticCatalog: vi.fn(async () => []), + prepareStaticCatalog: vi.fn(async (..._args: unknown[]) => ({ entries: [] })), resolveStaticCatalogModel: vi.fn(() => undefined), configuredAgentIds: [] as string[], + configuredAgentDirs: new Map(), + configuredWorkspaces: new Map(), warn: vi.fn(), mutationListener: undefined as | ((event: { agentDir?: string; affectsInheritedStores: boolean }) => void) @@ -34,23 +47,37 @@ vi.mock("./model-catalog.js", () => ({ mocks.buildPreparedModelCatalogSnapshot(...args), })); +vi.mock("./agent-auth-discovery.js", () => ({ + resolveAmbientAgentCredentialsForDiscovery: (...args: unknown[]) => + mocks.resolveAmbientCredentials(...args), +})); + vi.mock("./agent-model-discovery.js", () => ({ discoverAuthStorage: (...args: unknown[]) => { - mocks.discoverAuthStorage(...args); - return mocks.authStorage; + return mocks.discoverAuthStorage(...args) ?? mocks.authStorage; }, discoverModels: (...args: unknown[]) => { mocks.discoverModels(...args); return mocks.modelRegistry; }, + discoverModelsFromCapturedSources: (...args: unknown[]) => { + mocks.discoverModels(...args); + return mocks.modelRegistry; + }, +})); + +vi.mock("../plugins/synthetic-auth.runtime.js", () => ({ + resolveRuntimeSyntheticAuthProviderRefs: () => [], })); vi.mock("./agent-scope.js", () => ({ listAgentIds: () => mocks.configuredAgentIds, resolveAgentDir: (_config: unknown, agentId: string) => - agentId === "default" ? "/tmp/unused-agent" : `/tmp/configured-${agentId}`, + mocks.configuredAgentDirs.get(agentId) ?? + (agentId === "default" ? "/tmp/unused-agent" : `/tmp/configured-${agentId}`), resolveAgentWorkspaceDir: (_config: unknown, agentId: string) => - agentId === "default" ? "/tmp/unused-workspace" : `/tmp/workspace-${agentId}`, + mocks.configuredWorkspaces.get(agentId) ?? + (agentId === "default" ? "/tmp/unused-workspace" : `/tmp/workspace-${agentId}`), resolveDefaultAgentDir: () => "/tmp/unused-agent", resolveDefaultAgentId: () => "default", })); @@ -70,6 +97,11 @@ vi.mock("./model-discovery-context.js", () => ({ vi.mock("./models-config.js", () => ({ ensureOpenClawModelsJson: (...args: unknown[]) => mocks.ensureOpenClawModelsJson(...args), + planOpenClawModelsJsonSource: (...args: unknown[]) => mocks.planOpenClawModelsJsonSource(...args), +})); + +vi.mock("./models-config.providers.implicit.js", () => ({ + prepareImplicitProviderStaticCatalog: (...args: unknown[]) => mocks.prepareStaticCatalog(...args), })); vi.mock("./runtime-plugins.js", () => ({ @@ -108,15 +140,28 @@ describe("prepared model runtime snapshots", () => { beforeEach(() => { getTesting().resetPreparedModelRuntimeSnapshotsForTest(); mocks.discoverAuthStorage.mockClear(); + mocks.resolveAmbientCredentials.mockClear(); + mocks.discoverAuthStorage.mockImplementation(() => mocks.authStorage); mocks.discoverModels.mockClear(); mocks.ensureOpenClawModelsJson.mockReset(); mocks.ensureOpenClawModelsJson.mockResolvedValue({ agentDir: "/tmp/agent", wrote: false }); + mocks.planOpenClawModelsJsonSource.mockReset(); + mocks.planOpenClawModelsJsonSource.mockImplementation(async (_config, agentDir) => ({ + agentDir: String(agentDir), + modelsJsonContents: null, + pluginCatalogs: [], + })); mocks.buildPreparedModelCatalogSnapshot.mockClear(); mocks.ensureRuntimePluginsLoaded.mockClear(); mocks.loadStaticCatalog.mockClear(); + mocks.prepareStaticCatalog.mockClear(); + mocks.resolveStaticCatalogModel.mockClear(); mocks.modelRegistry.fork.mockClear(); + mocks.modelRegistry.find.mockClear(); mocks.warn.mockClear(); mocks.configuredAgentIds = []; + mocks.configuredAgentDirs.clear(); + mocks.configuredWorkspaces.clear(); }); it("does not discover missing owners from a gateway request", async () => { @@ -754,8 +799,9 @@ describe("prepared model runtime snapshots", () => { it("does not replay an auth mutation that occurs before the first owner is registered", async () => { getTesting().setModelRuntimeBuildTimeoutMsForTest(100); mocks.configuredAgentIds = ["default"]; - mocks.ensureRuntimePluginsLoaded.mockImplementationOnce(() => { + mocks.prepareStaticCatalog.mockImplementationOnce(async () => { mocks.mutationListener?.({ affectsInheritedStores: true }); + return { entries: [] }; }); mocks.ensureOpenClawModelsJson .mockResolvedValueOnce({ agentDir: "/tmp/unused-agent", wrote: false }) @@ -764,7 +810,10 @@ describe("prepared model runtime snapshots", () => { await expect( refreshPreparedModelRuntimeSnapshots({}, { gatewayLifecycle: true, catalogMode: "static" }), ).resolves.toBeUndefined(); - expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledOnce(); + expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); + expect(mocks.ensureRuntimePluginsLoaded).not.toHaveBeenCalled(); + expect(mocks.discoverAuthStorage).toHaveBeenCalledOnce(); + expect(mocks.discoverModels).toHaveBeenCalledOnce(); }); it("awaits auth invalidation queued during lifecycle publication", async () => { @@ -843,6 +892,31 @@ describe("prepared model runtime snapshots", () => { expect(mocks.warn).not.toHaveBeenCalled(); }); + it("rejects a deduplicated caller when an auth refresh is superseded", async () => { + const config = {}; + const agentDir = "/tmp/prepared-model-runtime-auth-pending-superseded"; + await publishPreparedModelRuntimeSnapshot({ config, agentDir }); + let finishFirstRefresh: (() => void) | undefined; + mocks.ensureOpenClawModelsJson.mockImplementationOnce( + async () => + await new Promise<{ agentDir: string; wrote: false }>((resolve) => { + finishFirstRefresh = () => resolve({ agentDir, wrote: false }); + }), + ); + + mocks.mutationListener?.({ agentDir, affectsInheritedStores: false }); + await vi.waitFor(() => expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(2)); + const deduplicated = publishPreparedModelRuntimeSnapshot({ config, agentDir }); + mocks.mutationListener?.({ agentDir, affectsInheritedStores: false }); + finishFirstRefresh?.(); + + await expect(deduplicated).rejects.toThrow("superseded"); + await vi.waitFor(() => expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(3)); + await expect(prepareModelRuntimeSnapshot({ config, agentDir })).resolves.toMatchObject({ + agentDir, + }); + }); + it("does not let a superseded owner hide a genuine sibling refresh failure", async () => { const config = {}; const supersededDir = "/tmp/prepared-model-runtime-auth-superseded-sibling"; @@ -1012,109 +1086,4 @@ describe("prepared model runtime snapshots", () => { expect.objectContaining({ workspaceDir: "/tmp/explicit-workspace" }), ); }); - - it("finds the configured gateway owner when request config omits its launch workspace", async () => { - mocks.configuredAgentIds = ["default"]; - const config = {}; - - await refreshPreparedModelRuntimeSnapshots(config, { - gatewayLifecycle: true, - defaultWorkspaceDir: "/tmp/gateway-launch-workspace", - }); - const snapshot = await prepareModelRuntimeSnapshot({ - config, - agentDir: "/tmp/unused-agent", - }); - - expect(snapshot.workspaceDir).toBe("/tmp/gateway-launch-workspace"); - expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledOnce(); - }); - - it("does not substitute a configured owner captured from another environment", async () => { - mocks.configuredAgentIds = ["default"]; - const config = {}; - await refreshPreparedModelRuntimeSnapshots(config, { - gatewayLifecycle: true, - defaultWorkspaceDir: "/tmp/gateway-launch-workspace", - }); - - await expect( - prepareModelRuntimeSnapshot({ - config, - agentDir: "/tmp/unused-agent", - env: { ...process.env, OPENCLAW_PREPARED_RUNTIME_TEST_SCOPE: "different" }, - }), - ).rejects.toThrow("prepared model runtime owner was not published"); - }); - - it("does not substitute a configured owner for an explicit workspace", async () => { - mocks.configuredAgentIds = ["default"]; - const config = {}; - - await refreshPreparedModelRuntimeSnapshots(config, { - gatewayLifecycle: true, - defaultWorkspaceDir: "/tmp/gateway-launch-workspace", - }); - - await expect( - prepareModelRuntimeSnapshot({ - config, - agentDir: "/tmp/unused-agent", - workspaceDir: "/tmp/other-explicit-workspace", - }), - ).rejects.toThrow("prepared model runtime owner was not published"); - }); - - it("does not choose between configured owners sharing one agent directory", async () => { - const config = {}; - const agentDir = "/tmp/shared-configured-agent"; - await publishPreparedModelRuntimeSnapshot( - { config, agentDir, workspaceDir: "/tmp/shared-workspace-a" }, - { provenance: "configured" }, - ); - await publishPreparedModelRuntimeSnapshot( - { config, agentDir, workspaceDir: "/tmp/shared-workspace-b" }, - { provenance: "configured" }, - ); - - await expect(prepareModelRuntimeSnapshot({ config, agentDir })).rejects.toThrow( - "prepared model runtime owner was not published", - ); - }); - - it("selects a configured owner by agent id when directories are shared", async () => { - const config = {}; - const agentDir = "/tmp/shared-agent-id-directory"; - await publishPreparedModelRuntimeSnapshot( - { agentId: "agent-a", config, agentDir, workspaceDir: "/tmp/shared-agent-id-workspace" }, - { provenance: "configured" }, - ); - const selected = await publishPreparedModelRuntimeSnapshot( - { agentId: "agent-b", config, agentDir, workspaceDir: "/tmp/shared-agent-id-workspace" }, - { provenance: "configured" }, - ); - - await expect( - prepareModelRuntimeSnapshot({ agentId: "agent-b", config, agentDir }), - ).resolves.toBe(selected); - }); - - it("retires configured owners removed by config reload", async () => { - mocks.configuredAgentIds = ["default", "removed"]; - const config = {}; - await refreshPreparedModelRuntimeSnapshots(config); - mocks.configuredAgentIds = ["default"]; - - await refreshPreparedModelRuntimeSnapshots(config); - - await expect( - prepareModelRuntimeSnapshot({ - config, - agentDir: "/tmp/configured-removed", - inheritedAuthDir: "/tmp/unused-agent", - workspaceDir: "/tmp/workspace-removed", - }), - ).rejects.toThrow("prepared model runtime owner was not published"); - expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(3); - }); }); diff --git a/src/agents/prepared-model-runtime.owner-selection.test.ts b/src/agents/prepared-model-runtime.owner-selection.test.ts new file mode 100644 index 000000000000..1b4f9399bb66 --- /dev/null +++ b/src/agents/prepared-model-runtime.owner-selection.test.ts @@ -0,0 +1,708 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +type CreateStaticCatalogResolver = + typeof import("./embedded-agent-runner/model.static-catalog.js").createBundledStaticCatalogModelResolver; +type StaticCatalogResolver = ReturnType; + +const mocks = vi.hoisted(() => ({ + authStorage: { + getAll: vi.fn(() => ({ custom: { type: "api_key", key: "test-key" } })), + getOAuthProviders: vi.fn(() => []), + }, + modelRegistry: { + fork: vi.fn((authStorage: unknown) => ({ authStorage })), + getAll: vi.fn(() => []), + find: vi.fn(() => null), + }, + configuredAgentIds: [] as string[], + configuredAgentDirs: new Map(), + configuredWorkspaces: new Map(), + buildPreparedModelCatalogSnapshot: vi.fn(async (..._args: unknown[]) => ({ + entries: [], + routeVariants: [], + })), + discoverAuthStorage: vi.fn((..._args: unknown[]) => undefined as unknown), + discoverModels: vi.fn(), + ensureOpenClawModelsJson: vi.fn(async (..._args: unknown[]) => ({ + agentDir: "/tmp/agent", + wrote: false, + })), + ensureRuntimePluginsLoaded: vi.fn(), + planOpenClawModelsJsonSource: vi.fn(async (...args: unknown[]) => ({ + agentDir: String(args[1]), + modelsJsonContents: null, + pluginCatalogs: [], + })), + prepareStaticCatalog: vi.fn(async (..._args: unknown[]) => ({ entries: [] })), + resolveAmbientCredentials: vi.fn((..._args: unknown[]) => ({})), + resolveStaticCatalogModel: vi.fn(() => undefined), + mutationListener: undefined as + | ((event: { agentDir?: string; affectsInheritedStores: boolean }) => void) + | undefined, +})); + +vi.mock("./model-catalog.js", () => ({ + buildPreparedModelCatalogSnapshot: (...args: unknown[]) => + mocks.buildPreparedModelCatalogSnapshot(...args), +})); + +vi.mock("./agent-auth-discovery.js", () => ({ + resolveAmbientAgentCredentialsForDiscovery: (...args: unknown[]) => + mocks.resolveAmbientCredentials(...args), +})); + +vi.mock("./agent-model-discovery.js", () => ({ + discoverAuthStorage: (...args: unknown[]) => + mocks.discoverAuthStorage(...args) ?? mocks.authStorage, + discoverModels: (...args: unknown[]) => { + mocks.discoverModels(...args); + return mocks.modelRegistry; + }, + discoverModelsFromCapturedSources: (...args: unknown[]) => { + mocks.discoverModels(...args); + return mocks.modelRegistry; + }, +})); + +vi.mock("../plugins/synthetic-auth.runtime.js", () => ({ + resolveRuntimeSyntheticAuthProviderRefs: () => [], +})); + +vi.mock("./agent-scope.js", () => ({ + listAgentIds: () => mocks.configuredAgentIds, + resolveAgentDir: (_config: unknown, agentId: string) => + mocks.configuredAgentDirs.get(agentId) ?? + (agentId === "default" ? "/tmp/unused-agent" : `/tmp/configured-${agentId}`), + resolveAgentWorkspaceDir: (_config: unknown, agentId: string) => + mocks.configuredWorkspaces.get(agentId) ?? + (agentId === "default" ? "/tmp/unused-workspace" : `/tmp/workspace-${agentId}`), + resolveDefaultAgentDir: () => "/tmp/unused-agent", + resolveDefaultAgentId: () => "default", +})); + +vi.mock("./auth-profiles/runtime-snapshots.js", () => ({ + registerRuntimeAuthProfileStoreMutationListener: ( + listener: (event: { agentDir?: string; affectsInheritedStores: boolean }) => void, + ) => { + mocks.mutationListener = listener; + return () => {}; + }, +})); + +vi.mock("./model-discovery-context.js", () => ({ + resolveModelPluginMetadataSnapshot: () => undefined, +})); + +vi.mock("./models-config.js", () => ({ + ensureOpenClawModelsJson: (...args: unknown[]) => mocks.ensureOpenClawModelsJson(...args), + planOpenClawModelsJsonSource: (...args: unknown[]) => mocks.planOpenClawModelsJsonSource(...args), +})); + +vi.mock("./models-config.providers.implicit.js", () => ({ + prepareImplicitProviderStaticCatalog: (...args: unknown[]) => mocks.prepareStaticCatalog(...args), +})); + +vi.mock("./runtime-plugins.js", () => ({ + ensureRuntimePluginsLoaded: (...args: unknown[]) => mocks.ensureRuntimePluginsLoaded(...args), +})); + +vi.mock("./embedded-agent-runner/model.static-catalog.js", () => ({ + loadBundledProviderStaticCatalogContextModels: async () => [], + createBundledStaticCatalogModelResolver: () => mocks.resolveStaticCatalogModel, +})); + +vi.mock("../logging/subsystem.js", () => ({ + createSubsystemLogger: () => ({ warn: vi.fn() }), +})); + +import { + getPreparedModelRuntimeSnapshot, + prepareModelRuntimeSnapshot, + publishPreparedModelRuntimeSnapshot, + refreshPreparedModelRuntimeSnapshots, +} from "./prepared-model-runtime.js"; + +describe("prepared model runtime owner selection", () => { + const getTesting = () => + (globalThis as Record)[ + Symbol.for("openclaw.preparedModelRuntimeTestApi") + ] as { + resetPreparedModelRuntimeSnapshotsForTest: () => void; + }; + + beforeEach(() => { + getTesting().resetPreparedModelRuntimeSnapshotsForTest(); + mocks.configuredAgentIds = []; + mocks.configuredAgentDirs.clear(); + mocks.configuredWorkspaces.clear(); + mocks.buildPreparedModelCatalogSnapshot.mockClear(); + mocks.discoverAuthStorage.mockReset(); + mocks.discoverAuthStorage.mockImplementation(() => mocks.authStorage); + mocks.discoverModels.mockClear(); + mocks.ensureOpenClawModelsJson.mockReset(); + mocks.ensureOpenClawModelsJson.mockResolvedValue({ agentDir: "/tmp/agent", wrote: false }); + mocks.ensureRuntimePluginsLoaded.mockClear(); + mocks.modelRegistry.fork.mockClear(); + mocks.planOpenClawModelsJsonSource.mockReset(); + mocks.planOpenClawModelsJsonSource.mockImplementation(async (_config, agentDir) => ({ + agentDir: String(agentDir), + modelsJsonContents: null, + pluginCatalogs: [], + })); + mocks.prepareStaticCatalog.mockClear(); + mocks.resolveAmbientCredentials.mockClear(); + mocks.resolveStaticCatalogModel.mockClear(); + }); + + it("serializes live catalog sources for owners sharing one agent directory", async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-shared-catalog-source-")); + try { + const agentDir = path.join(rootDir, "agent"); + fs.mkdirSync(agentDir); + mocks.configuredAgentIds = ["agent-a", "agent-b"]; + mocks.configuredAgentDirs.set("agent-a", agentDir); + mocks.configuredAgentDirs.set("agent-b", agentDir); + mocks.configuredWorkspaces.set("agent-a", "/tmp/source-workspace-a"); + mocks.configuredWorkspaces.set("agent-b", "/tmp/source-workspace-b"); + let activeWrites = 0; + let peakActiveWrites = 0; + mocks.ensureOpenClawModelsJson.mockImplementation(async (_config, targetDir, options) => { + activeWrites += 1; + peakActiveWrites = Math.max(peakActiveWrites, activeWrites); + await new Promise((resolve) => { + setImmediate(resolve); + }); + const workspaceDir = (options as { workspaceDir?: string }).workspaceDir ?? "unknown"; + fs.writeFileSync( + path.join(String(targetDir), "models.json"), + JSON.stringify({ + providers: { + custom: { + api: "openai-completions", + baseUrl: "https://models.example/v1", + models: [{ id: path.basename(workspaceDir) }], + }, + }, + }), + ); + activeWrites -= 1; + return { agentDir: String(targetDir), wrote: true }; + }); + + await refreshPreparedModelRuntimeSnapshots({}); + + expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(2); + expect(peakActiveWrites).toBe(1); + expect( + mocks.discoverModels.mock.calls.map((call) => { + const contents = (call[2] as { modelsJsonContents: string }).modelsJsonContents; + const parsed = JSON.parse(contents) as { + providers: { custom: { models: Array<{ id: string }> } }; + }; + return parsed.providers.custom.models[0]?.id; + }), + ).toEqual(["source-workspace-a", "source-workspace-b"]); + } finally { + fs.rmSync(rootDir, { recursive: true, force: true }); + } + }); + + it("finds the configured gateway owner when request config omits its launch workspace", async () => { + mocks.configuredAgentIds = ["default"]; + const config = {}; + + await refreshPreparedModelRuntimeSnapshots(config, { + gatewayLifecycle: true, + defaultWorkspaceDir: "/tmp/gateway-launch-workspace", + }); + const snapshot = await prepareModelRuntimeSnapshot({ + config, + agentDir: "/tmp/unused-agent", + }); + + expect(snapshot.workspaceDir).toBe("/tmp/gateway-launch-workspace"); + expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledOnce(); + }); + + it("does not substitute a configured owner captured from another environment", async () => { + mocks.configuredAgentIds = ["default"]; + const config = {}; + await refreshPreparedModelRuntimeSnapshots(config, { + gatewayLifecycle: true, + defaultWorkspaceDir: "/tmp/gateway-launch-workspace", + }); + + await expect( + prepareModelRuntimeSnapshot({ + config, + agentDir: "/tmp/unused-agent", + env: { ...process.env, OPENCLAW_PREPARED_RUNTIME_TEST_SCOPE: "different" }, + }), + ).rejects.toThrow("prepared model runtime owner was not published"); + }); + + it("does not substitute a configured owner for an explicit workspace", async () => { + mocks.configuredAgentIds = ["default"]; + const config = {}; + + await refreshPreparedModelRuntimeSnapshots(config, { + gatewayLifecycle: true, + defaultWorkspaceDir: "/tmp/gateway-launch-workspace", + }); + + await expect( + prepareModelRuntimeSnapshot({ + config, + agentDir: "/tmp/unused-agent", + workspaceDir: "/tmp/other-explicit-workspace", + }), + ).rejects.toThrow("prepared model runtime owner was not published"); + }); + + it("does not choose between configured owners sharing one agent directory", async () => { + const config = {}; + const agentDir = "/tmp/shared-configured-agent"; + await publishPreparedModelRuntimeSnapshot( + { config, agentDir, workspaceDir: "/tmp/shared-workspace-a" }, + { provenance: "configured" }, + ); + await publishPreparedModelRuntimeSnapshot( + { config, agentDir, workspaceDir: "/tmp/shared-workspace-b" }, + { provenance: "configured" }, + ); + + await expect(prepareModelRuntimeSnapshot({ config, agentDir })).rejects.toThrow( + "prepared model runtime owner was not published", + ); + }); + + it("selects a configured owner by agent id when directories are shared", async () => { + const config = {}; + const agentDir = "/tmp/shared-agent-id-directory"; + await publishPreparedModelRuntimeSnapshot( + { agentId: "agent-a", config, agentDir, workspaceDir: "/tmp/shared-agent-id-workspace" }, + { provenance: "configured" }, + ); + const selected = await publishPreparedModelRuntimeSnapshot( + { agentId: "agent-b", config, agentDir, workspaceDir: "/tmp/shared-agent-id-workspace" }, + { provenance: "configured" }, + ); + + await expect( + prepareModelRuntimeSnapshot({ agentId: "agent-b", config, agentDir }), + ).resolves.toBe(selected); + }); + + it("retires configured owners removed by config reload", async () => { + mocks.configuredAgentIds = ["default", "removed"]; + const config = {}; + await refreshPreparedModelRuntimeSnapshots(config); + mocks.configuredAgentIds = ["default"]; + + await refreshPreparedModelRuntimeSnapshots(config); + + await expect( + prepareModelRuntimeSnapshot({ + config, + agentDir: "/tmp/configured-removed", + inheritedAuthDir: "/tmp/unused-agent", + workspaceDir: "/tmp/workspace-removed", + }), + ).rejects.toThrow("prepared model runtime owner was not published"); + expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(3); + }); + + it("shares static workspace facts without eager per-agent catalog work", async () => { + mocks.configuredAgentIds = ["agent-a", "agent-b", "agent-c", "agent-d"]; + for (const agentId of ["agent-a", "agent-b", "agent-c"]) { + mocks.configuredWorkspaces.set(agentId, "/tmp/shared-prepared-runtime-workspace"); + } + mocks.configuredWorkspaces.set("agent-d", "/tmp/distinct-prepared-runtime-workspace"); + const config = { agents: { defaults: { model: "openai/gpt-5.5" } } }; + let stats: + | { + agentCount: number; + workspaceGroupCount: number; + configuredFactsGroupCount: number; + catalogSourceCount: number; + catalogGroupCount: number; + runtimeRegistryCount: number; + fullCatalogConcurrencyLimit: number; + } + | undefined; + + await refreshPreparedModelRuntimeSnapshots(config, { + gatewayLifecycle: true, + catalogMode: "static", + onBuildStats: (value) => { + stats = value; + }, + }); + + expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); + expect(mocks.ensureRuntimePluginsLoaded).not.toHaveBeenCalled(); + expect(mocks.resolveAmbientCredentials).toHaveBeenCalledTimes(2); + expect(mocks.prepareStaticCatalog).toHaveBeenCalledTimes(2); + expect(mocks.resolveStaticCatalogModel).toHaveBeenCalledTimes(2); + expect(mocks.buildPreparedModelCatalogSnapshot).not.toHaveBeenCalled(); + expect(mocks.discoverModels).toHaveBeenCalledTimes(2); + expect(stats).toMatchObject({ + agentCount: 4, + workspaceGroupCount: 2, + configuredFactsGroupCount: 2, + catalogSourceCount: 0, + catalogGroupCount: 0, + runtimeRegistryCount: 2, + fullCatalogConcurrencyLimit: 1, + }); + + const snapshot = getPreparedModelRuntimeSnapshot({ + agentId: "agent-a", + config, + agentDir: "/tmp/configured-agent-a", + inheritedAuthDir: "/tmp/unused-agent", + workspaceDir: "/tmp/shared-prepared-runtime-workspace", + }); + await snapshot?.loadFullModelCatalog?.(); + expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); + expect(mocks.planOpenClawModelsJsonSource).toHaveBeenCalledOnce(); + expect(mocks.buildPreparedModelCatalogSnapshot).toHaveBeenCalledOnce(); + }); + + it("shares workspace facts while isolating each agent's configured model projection", async () => { + mocks.configuredAgentIds = ["agent-a", "agent-b"]; + for (const agentId of mocks.configuredAgentIds) { + mocks.configuredWorkspaces.set(agentId, "/tmp/shared-agent-model-workspace"); + } + mocks.resolveStaticCatalogModel.mockImplementation( + ({ provider, modelId }: { provider: string; modelId: string }) => ({ + id: modelId, + name: modelId, + provider, + api: "openai-completions", + baseUrl: "https://models.example/v1", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 8_192, + }), + ); + const config = { + agents: { + defaults: { model: "custom/shared-model" }, + list: [ + { id: "agent-a", model: "custom/model-a" }, + { id: "agent-b", model: "custom/model-b" }, + ], + }, + }; + + await refreshPreparedModelRuntimeSnapshots(config, { + gatewayLifecycle: true, + catalogMode: "static", + }); + + for (const agentId of mocks.configuredAgentIds) { + const snapshot = getPreparedModelRuntimeSnapshot({ + agentId, + config, + agentDir: `/tmp/configured-${agentId}`, + inheritedAuthDir: "/tmp/unused-agent", + workspaceDir: "/tmp/shared-agent-model-workspace", + }); + expect(snapshot?.configuredRuntimeModels.map(({ modelId }) => modelId)).toEqual([ + "shared-model", + agentId === "agent-a" ? "model-a" : "model-b", + ]); + expect(snapshot?.modelCatalog.entries.map(({ id }) => id)).not.toContain( + agentId === "agent-a" ? "model-b" : "model-a", + ); + } + expect(mocks.prepareStaticCatalog).toHaveBeenCalledOnce(); + }); + + it("parses one static registry per exact agent catalog and credential generation", async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-prepared-registry-groups-")); + try { + mocks.configuredAgentIds = ["agent-a", "agent-b", "agent-c"]; + for (const agentId of mocks.configuredAgentIds) { + const agentDir = path.join(rootDir, agentId); + fs.mkdirSync(agentDir, { recursive: true }); + mocks.configuredAgentDirs.set(agentId, agentDir); + mocks.configuredWorkspaces.set(agentId, "/tmp/shared-prepared-runtime-workspace"); + } + const sharedCatalog = JSON.stringify({ + providers: { + custom: { + api: "openai-completions", + baseUrl: "https://models.example/v1", + models: [{ id: "shared-model" }], + }, + }, + }); + fs.writeFileSync(path.join(rootDir, "agent-a", "models.json"), sharedCatalog); + fs.writeFileSync(path.join(rootDir, "agent-b", "models.json"), sharedCatalog); + fs.writeFileSync( + path.join(rootDir, "agent-c", "models.json"), + JSON.stringify({ + providers: { + custom: { + api: "openai-completions", + baseUrl: "https://models.example/v1", + models: [{ id: "distinct-model" }], + }, + }, + }), + ); + let runtimeRegistryCount = 0; + + await refreshPreparedModelRuntimeSnapshots( + { agents: { defaults: { model: "openai/gpt-5.5" } } }, + { + gatewayLifecycle: true, + catalogMode: "static", + onBuildStats: (stats) => { + runtimeRegistryCount = stats.runtimeRegistryCount; + }, + }, + ); + + expect(mocks.discoverModels).toHaveBeenCalledTimes(2); + expect(runtimeRegistryCount).toBe(2); + expect( + mocks.discoverModels.mock.calls.map((call) => { + const options = call.length === 2 ? call[1] : call[2]; + return (options as { modelsJsonContents?: string }).modelsJsonContents; + }), + ).toEqual(expect.arrayContaining([sharedCatalog, expect.stringContaining("distinct-model")])); + } finally { + fs.rmSync(rootDir, { recursive: true, force: true }); + } + }); + + it("keeps registry parsing isolated across OAuth provider generations", async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-prepared-oauth-groups-")); + try { + mocks.configuredAgentIds = ["agent-a", "agent-b", "agent-c"]; + const sharedCatalog = JSON.stringify({ + providers: { + custom: { + api: "openai-completions", + baseUrl: "https://models.example/v1", + models: [{ id: "shared-model" }], + }, + }, + }); + const sharedProvider = { + id: "custom", + name: "OAuth A", + login: vi.fn(), + refreshToken: vi.fn(), + getApiKey: vi.fn(), + }; + const oauthProviders = { + "agent-a": sharedProvider, + "agent-b": { ...sharedProvider }, + "agent-c": { ...sharedProvider, name: "OAuth B", modifyModels: vi.fn() }, + }; + for (const agentId of mocks.configuredAgentIds) { + const agentDir = path.join(rootDir, agentId); + fs.mkdirSync(agentDir, { recursive: true }); + fs.writeFileSync(path.join(agentDir, "models.json"), sharedCatalog); + mocks.configuredAgentDirs.set(agentId, agentDir); + mocks.configuredWorkspaces.set(agentId, "/tmp/shared-prepared-runtime-workspace"); + } + mocks.discoverAuthStorage.mockImplementation((agentDir: unknown) => { + const agentId = path.basename(String(agentDir)) as keyof typeof oauthProviders; + return { + getAll: () => ({ custom: { type: "api_key" as const, key: "shared-key" } }), + getOAuthProviders: () => [oauthProviders[agentId]], + }; + }); + let runtimeRegistryCount = 0; + + await refreshPreparedModelRuntimeSnapshots( + { agents: { defaults: { model: "openai/gpt-5.5" } } }, + { + gatewayLifecycle: true, + catalogMode: "static", + onBuildStats: (stats) => { + runtimeRegistryCount = stats.runtimeRegistryCount; + }, + }, + ); + + expect(mocks.discoverModels).toHaveBeenCalledTimes(2); + expect(runtimeRegistryCount).toBe(2); + } finally { + fs.rmSync(rootDir, { recursive: true, force: true }); + } + }); + + it("serializes on-demand full catalogs while preserving agent credentials", async () => { + mocks.configuredAgentIds = ["agent-a", "agent-b"]; + mocks.configuredWorkspaces.set("agent-a", "/tmp/shared-prepared-runtime-workspace"); + mocks.configuredWorkspaces.set("agent-b", "/tmp/shared-prepared-runtime-workspace"); + mocks.discoverAuthStorage.mockImplementation((agentDir: unknown) => ({ + getAll: () => ({ + custom: { type: "api_key" as const, key: `test-key:${String(agentDir)}` }, + }), + getOAuthProviders: () => [], + })); + let activePlans = 0; + let peakActivePlans = 0; + mocks.planOpenClawModelsJsonSource.mockImplementation(async (_config, agentDir) => { + activePlans += 1; + peakActivePlans = Math.max(peakActivePlans, activePlans); + await Promise.resolve(); + activePlans -= 1; + return { agentDir: String(agentDir), modelsJsonContents: null, pluginCatalogs: [] }; + }); + const config = { agents: { defaults: { model: "openai/gpt-5.5" } } }; + + await refreshPreparedModelRuntimeSnapshots(config, { + gatewayLifecycle: true, + catalogMode: "static", + }); + + expect(mocks.ensureRuntimePluginsLoaded).not.toHaveBeenCalled(); + expect(mocks.prepareStaticCatalog).toHaveBeenCalledOnce(); + expect(mocks.discoverModels).toHaveBeenCalledTimes(2); + const loadAgentCatalog = (agentId: string) => + getPreparedModelRuntimeSnapshot({ + agentId, + config, + agentDir: `/tmp/configured-${agentId}`, + inheritedAuthDir: "/tmp/unused-agent", + workspaceDir: "/tmp/shared-prepared-runtime-workspace", + })?.loadFullModelCatalog?.(); + await Promise.all([loadAgentCatalog("agent-a"), loadAgentCatalog("agent-b")]); + + expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); + expect(mocks.planOpenClawModelsJsonSource).toHaveBeenCalledTimes(2); + expect(mocks.buildPreparedModelCatalogSnapshot).toHaveBeenCalledTimes(2); + expect(peakActivePlans).toBe(1); + expect( + mocks.buildPreparedModelCatalogSnapshot.mock.calls.map( + (call) => + (call[0] as { authCredentials: { custom: { key: string } } }).authCredentials.custom.key, + ), + ).toEqual(["test-key:/tmp/configured-agent-a", "test-key:/tmp/configured-agent-b"]); + }); + + it("serializes a lazy catalog plan before a superseding generation", async () => { + mocks.configuredAgentIds = ["agent-a"]; + mocks.configuredWorkspaces.set("agent-a", "/tmp/shared-prepared-runtime-workspace"); + const initialConfig = { agents: { defaults: { model: "openai/gpt-5.5" } } }; + await refreshPreparedModelRuntimeSnapshots(initialConfig, { + gatewayLifecycle: true, + catalogMode: "static", + }); + const snapshot = getPreparedModelRuntimeSnapshot({ + agentId: "agent-a", + config: initialConfig, + agentDir: "/tmp/configured-agent-a", + inheritedAuthDir: "/tmp/unused-agent", + workspaceDir: "/tmp/shared-prepared-runtime-workspace", + }); + let releaseLazyPlan: (() => void) | undefined; + mocks.planOpenClawModelsJsonSource.mockImplementation(async (_config, agentDir) => { + if (!releaseLazyPlan) { + await new Promise((resolve) => { + releaseLazyPlan = resolve; + }); + } + return { agentDir: String(agentDir), modelsJsonContents: null, pluginCatalogs: [] }; + }); + + const staleCatalogLoad = snapshot?.loadFullModelCatalog?.(); + await vi.waitFor(() => expect(releaseLazyPlan).toBeTypeOf("function")); + const replacement = refreshPreparedModelRuntimeSnapshots( + { agents: { defaults: { model: "openai/gpt-5.6" } } }, + { gatewayLifecycle: true, catalogMode: "live" }, + ); + await Promise.resolve(); + expect(mocks.planOpenClawModelsJsonSource).toHaveBeenCalledOnce(); + expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); + + releaseLazyPlan?.(); + await expect(staleCatalogLoad).rejects.toThrow("superseded"); + await replacement; + expect(mocks.planOpenClawModelsJsonSource).toHaveBeenCalledOnce(); + expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledOnce(); + }); + + it("stops a superseded same-directory batch before another catalog write", async () => { + mocks.configuredAgentIds = ["agent-a", "agent-b"]; + for (const agentId of mocks.configuredAgentIds) { + mocks.configuredAgentDirs.set(agentId, "/tmp/shared-catalog-agent-dir"); + mocks.configuredWorkspaces.set(agentId, `/tmp/catalog-workspace-${agentId}`); + } + const staleConfig = { agents: { defaults: { model: "openai/gpt-5.5" } } }; + const latestConfig = { agents: { defaults: { model: "openai/gpt-5.6" } } }; + let releaseStaleWrite: (() => void) | undefined; + mocks.ensureOpenClawModelsJson.mockImplementation(async (config) => { + if (config === staleConfig && !releaseStaleWrite) { + await new Promise((resolve) => { + releaseStaleWrite = resolve; + }); + } + return { agentDir: "/tmp/shared-catalog-agent-dir", wrote: false }; + }); + + const stale = refreshPreparedModelRuntimeSnapshots(staleConfig); + await vi.waitFor(() => expect(releaseStaleWrite).toBeTypeOf("function")); + const latest = refreshPreparedModelRuntimeSnapshots(latestConfig); + releaseStaleWrite?.(); + + await expect(stale).rejects.toThrow("superseded"); + await latest; + expect( + mocks.ensureOpenClawModelsJson.mock.calls.filter(([config]) => config === staleConfig), + ).toHaveLength(1); + expect( + mocks.ensureOpenClawModelsJson.mock.calls.filter(([config]) => config === latestConfig), + ).toHaveLength(2); + }); + + it("publishes a current sibling when another auth owner is superseded", async () => { + const config = {}; + const supersededDir = "/tmp/prepared-model-runtime-auth-retry-superseded"; + const siblingDir = "/tmp/prepared-model-runtime-auth-retry-sibling"; + await publishPreparedModelRuntimeSnapshot({ config, agentDir: supersededDir }); + const firstSibling = await publishPreparedModelRuntimeSnapshot({ + config, + agentDir: siblingDir, + }); + let releaseSupersededRefresh: (() => void) | undefined; + let blockedSupersededRefresh = true; + mocks.ensureOpenClawModelsJson.mockImplementation(async (_config, agentDir) => { + if (agentDir === supersededDir && blockedSupersededRefresh) { + blockedSupersededRefresh = false; + await new Promise((resolve) => { + releaseSupersededRefresh = resolve; + }); + } + return { agentDir: String(agentDir), wrote: false }; + }); + + mocks.mutationListener?.({ affectsInheritedStores: true }); + await vi.waitFor(() => expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(4)); + const siblingPending = publishPreparedModelRuntimeSnapshot({ + config, + agentDir: siblingDir, + }); + mocks.mutationListener?.({ agentDir: supersededDir, affectsInheritedStores: false }); + releaseSupersededRefresh?.(); + + await expect(siblingPending).resolves.not.toBe(firstSibling); + await vi.waitFor(() => expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(6)); + await expect( + prepareModelRuntimeSnapshot({ config, agentDir: supersededDir }), + ).resolves.toMatchObject({ agentDir: supersededDir }); + }); +}); diff --git a/src/agents/prepared-model-runtime.owner.ts b/src/agents/prepared-model-runtime.owner.ts index b2857569eee3..d8157dfa6391 100644 --- a/src/agents/prepared-model-runtime.owner.ts +++ b/src/agents/prepared-model-runtime.owner.ts @@ -1,22 +1,7 @@ import path from "node:path"; -import { collectConfiguredModelRefs } from "@openclaw/model-catalog-core/configured-model-refs"; -import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; -import type { PreparedMessageToolCatalog } from "../channels/plugins/message-action-discovery.js"; import { hashRuntimeConfigValue } from "../config/runtime-snapshot.js"; -import { MODEL_APIS } from "../config/types.models.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { withTimeout } from "../node-host/with-timeout.js"; -import { prepareMediaCapabilityProviders } from "../plugins/capability-provider-runtime.js"; -import { resolvePluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; -import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.types.js"; -import { - EMPTY_PREPARED_MESSAGE_TOOL_CATALOG, - getPreparedMessageToolCatalog, - getPreparedMessageToolCatalogForRegistry, -} from "../plugins/prepared-message-tool-catalog.js"; -import type { ProviderRuntimeModel } from "../plugins/provider-runtime-model.types.js"; import { isReservedSystemAgentId } from "../system-agent/agent-id.js"; -import { discoverAuthStorage, discoverModels } from "./agent-model-discovery.js"; import { listAgentIds, resolveAgentDir, @@ -25,101 +10,34 @@ import { resolveDefaultAgentId, } from "./agent-scope.js"; import { - buildInlineProviderModels, - type InlineModelEntry, -} from "./embedded-agent-runner/model.inline-provider.js"; + startSerializedSnapshotBuild, + startSerializedSnapshotBuildBatch, +} from "./prepared-model-runtime.build.js"; import { - createBundledStaticCatalogModelResolver, - loadBundledProviderStaticCatalogContextModels, -} from "./embedded-agent-runner/model.static-catalog.js"; -import { staticModelIdMatches } from "./embedded-agent-runner/model.static-id.js"; -import { buildPreparedModelCatalogSnapshot, type ModelCatalogEntry } from "./model-catalog.js"; -import type { ModelCatalogSnapshot } from "./model-catalog.types.js"; -import { ensureOpenClawModelsJson } from "./models-config.js"; -import { ensureRuntimePluginsLoaded } from "./runtime-plugins.js"; -import { AuthStorage, type AuthStorageData } from "./sessions/auth-storage.js"; -import type { ModelRegistry } from "./sessions/model-registry.js"; + PreparedModelRuntimeOwnerNotPublishedError, + PreparedModelRuntimePublicationSupersededError, + toPreparedModelRuntimeError, +} from "./prepared-model-runtime.errors.js"; +import type { + PreparedModelRuntimeCatalogMode, + PreparedModelRuntimeInput, + PreparedModelRuntimeOwner, + PreparedModelRuntimeReplacement, + PreparedModelRuntimeSnapshot, +} from "./prepared-model-runtime.types.js"; -const MODEL_RUNTIME_PROVIDER_DISCOVERY_TIMEOUT_MS = 5_000; - -type PreparedModelRuntimeCatalogMode = "live" | "static"; - -export type PreparedModelRuntimeSnapshot = Readonly<{ - agentId?: string; - agentDir: string; - inheritedAuthDir?: string; - workspaceDir?: string; - /** Run-prepared repository root; null means discovery completed without a match. */ - repoRoot?: string | null; - /** Stable identity derived from repoRoot; null means the run is outside a repository. */ - projectKey?: string | null; - /** Session active project set, ordered most-recent first; empty before run binding. */ - activeProjectKeys: readonly string[]; - config: OpenClawConfig; - metadataSnapshot: PluginMetadataSnapshot; - messageToolCatalog?: PreparedMessageToolCatalog; - mediaCapabilityProviders?: ReturnType; - modelCatalog: ModelCatalogSnapshot; - /** Full static models for configured refs, resolved once at the lifecycle boundary. */ - configuredRuntimeModels: readonly PreparedConfiguredRuntimeModel[]; - /** Inline provider projection prepared once for all resolutions owned by this snapshot. */ - inlineProviderModels: readonly InlineModelEntry[]; - createStores: () => PreparedModelRuntimeStores; -}>; - -type PreparedConfiguredRuntimeModel = Readonly<{ - provider: string; - modelId: string; - model: ProviderRuntimeModel; -}>; - -export type PreparedModelRuntimeStores = { - authStorage: AuthStorage; - modelRegistry: ModelRegistry; -}; - -export type PreparedModelRuntimeInput = { - agentId?: string; - agentDir: string; - inheritedAuthDir?: string; - workspaceDir?: string; - preserveWorkspaceDirOnRefresh?: boolean; - readOnly?: boolean; - skipCredentials?: boolean; - env?: NodeJS.ProcessEnv; - config: OpenClawConfig; -}; - -export type PreparedModelRuntimeLease = Readonly<{ - snapshot: PreparedModelRuntimeSnapshot; - release: () => void; -}>; - -export type PreparedModelRuntimePublicationOptions = { - force?: boolean; - provenance?: PreparedModelRuntimeOwner["provenance"]; - catalogMode?: PreparedModelRuntimeCatalogMode; -}; - -export type PreparedModelRuntimeRefreshOptions = { - gatewayLifecycle?: boolean; - defaultWorkspaceDir?: string; - catalogMode?: PreparedModelRuntimeCatalogMode; -}; - -export type PreparedModelRuntimeOwner = { - input: PreparedModelRuntimeInput; - environmentFingerprint: string; - catalogMode: PreparedModelRuntimeCatalogMode; - provenance: "configured" | "standalone" | "explicit" | "run" | "ephemeral"; - generation: number; - needsRefresh: boolean; - refreshError?: Error; - snapshot?: PreparedModelRuntimeSnapshot; - pending?: Promise; - buildCompletion?: Promise; - leaseCount?: number; -}; +export { startSerializedSnapshotBuildBatch }; +export type { + PreparedModelRuntimeInput, + PreparedModelRuntimeLease, + PreparedModelRuntimeOwner, + PreparedModelRuntimePublicationOptions, + PreparedModelRuntimeRefreshOptions, + PreparedModelRuntimeReplacement, + PreparedModelRuntimeReplacementGateId, + PreparedModelRuntimeSnapshot, + PreparedModelRuntimeStores, +} from "./prepared-model-runtime.types.js"; export function createPreparedModelRuntimeOwner( input: PreparedModelRuntimeInput, @@ -136,16 +54,10 @@ export function createPreparedModelRuntimeOwner( }; } -export type PreparedModelRuntimeReplacement = { - gateId: PreparedModelRuntimeReplacementGateId; - promise: Promise; - resolve: () => void; - reject: (error: Error) => void; +export { + PreparedModelRuntimeOwnerNotPublishedError, + PreparedModelRuntimePublicationSupersededError, }; -export type PreparedModelRuntimeReplacementGateId = symbol; -export class PreparedModelRuntimeOwnerNotPublishedError extends Error {} - -export class PreparedModelRuntimePublicationSupersededError extends PreparedModelRuntimeOwnerNotPublishedError {} function findConfiguredOwnerCandidates( owners: Map, @@ -264,107 +176,6 @@ export function effectiveEnvironmentFingerprint(input: PreparedModelRuntimeInput return hashRuntimeConfigValue(input.env ?? process.env); } -function isCatalogModelApi( - value: string | undefined, -): value is NonNullable { - return value !== undefined && (MODEL_APIS as readonly string[]).includes(value); -} - -function toStaticCatalogEntry( - model: Awaited>[number], -): ModelCatalogEntry { - return { - id: model.id, - name: model.name ?? model.id, - provider: model.provider, - ...(isCatalogModelApi(model.api) ? { api: model.api } : {}), - ...(model.baseUrl ? { baseUrl: model.baseUrl } : {}), - ...(model.contextWindow ? { contextWindow: model.contextWindow } : {}), - ...(model.contextTokens ? { contextTokens: model.contextTokens } : {}), - ...(model.reasoning !== undefined ? { reasoning: model.reasoning } : {}), - ...(model.input ? { input: model.input } : {}), - ...(model.params ? { params: model.params } : {}), - ...(model.compat ? { compat: model.compat } : {}), - ...(model.mediaInput ? { mediaInput: model.mediaInput } : {}), - }; -} - -function collectPreparedModelRuntimeProviderIds( - config: OpenClawConfig, - credentials: Readonly, -): string[] { - const providerIds = new Set(); - const addProviderId = (value: string) => { - const providerId = normalizeProviderId(value); - if (providerId) { - providerIds.add(providerId); - } - }; - for (const providerId of Object.keys(credentials)) { - addProviderId(providerId); - } - for (const providerId of Object.keys(config.models?.providers ?? {})) { - addProviderId(providerId); - } - for (const ref of collectConfiguredModelRefs(config)) { - const separator = ref.value.indexOf("/"); - if (separator > 0) { - addProviderId(ref.value.slice(0, separator)); - } - } - return [...providerIds].toSorted((left, right) => left.localeCompare(right)); -} - -function prepareConfiguredRuntimeModels(params: { - config: OpenClawConfig; - env: NodeJS.ProcessEnv; - metadataSnapshot: PluginMetadataSnapshot; - providerStaticModels: readonly ProviderRuntimeModel[]; - workspaceDir?: string; -}): PreparedConfiguredRuntimeModel[] { - const prepared: PreparedConfiguredRuntimeModel[] = []; - const seen = new Set(); - const resolveStaticCatalogModel = createBundledStaticCatalogModelResolver({ - cfg: params.config, - env: params.env, - includeRuntimeDiscovery: true, - metadataSnapshot: params.metadataSnapshot, - ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), - }); - for (const { value } of collectConfiguredModelRefs(params.config)) { - const separator = value.indexOf("/"); - if (separator <= 0 || separator >= value.length - 1) { - continue; - } - const provider = normalizeProviderId(value.slice(0, separator)); - const modelId = value.slice(separator + 1).trim(); - if (!provider || !modelId) { - continue; - } - const key = `${provider}\0${modelId.toLowerCase()}`; - if (seen.has(key)) { - continue; - } - seen.add(key); - // Match request-time fallback precedence exactly: manifest/runtime-discovery rows win, - // and the provider-static catalog fills only models absent from that surface. - const model = - resolveStaticCatalogModel({ provider, modelId }) ?? - params.providerStaticModels.find((candidate) => - staticModelIdMatches({ - candidateId: candidate.id, - rowProvider: candidate.provider, - provider, - modelId, - }), - ); - if (model) { - prepared.push({ provider, modelId, model }); - } - } - return prepared; -} - export function ownerKey(input: PreparedModelRuntimeInput): string { return JSON.stringify({ agentId: input.agentId, @@ -424,7 +235,7 @@ export function hasSameLifecycleInput( } export function toError(error: unknown): Error { - return error instanceof Error ? error : new Error(String(error)); + return toPreparedModelRuntimeError(error); } export function createPreparedModelRuntimeReplacement(): PreparedModelRuntimeReplacement { @@ -464,166 +275,136 @@ export function listConfiguredOwnerInputs( }); } -async function buildSnapshot( - input: PreparedModelRuntimeInput, - catalogMode: PreparedModelRuntimeCatalogMode, -): Promise { - const env = input.env ?? process.env; - const runtimePluginRegistry = !input.readOnly - ? ensureRuntimePluginsLoaded({ - config: input.config, - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), - }) - : undefined; - const pluginMetadataSnapshot = resolvePluginMetadataSnapshot({ - config: input.config, - env, - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), +export async function publishPreparedModelRuntimeOwnerBatch(params: { + entries: Array<{ + owner: PreparedModelRuntimeOwner; + input: PreparedModelRuntimeInput; + }>; + owners: Map; + agentBuildCompletions: Map>; + buildTimeoutMs: number; +}): Promise { + const candidates = params.entries.map(({ owner, input }) => { + owner.input = input; + owner.environmentFingerprint = effectiveEnvironmentFingerprint(input); + owner.generation += 1; + owner.needsRefresh = true; + owner.refreshError = undefined; + const generation = owner.generation; + const key = ownerKey(input); + return { + catalogMode: owner.catalogMode, + input, + isCurrent: () => owner.generation === generation && params.owners.get(key) === owner, + owner, + }; }); - const mediaCapabilityProviders = input.readOnly - ? undefined - : prepareMediaCapabilityProviders({ - cfg: input.config, - pluginMetadataSnapshot, - registry: runtimePluginRegistry, - }); - const messageToolCatalog = - (runtimePluginRegistry - ? getPreparedMessageToolCatalogForRegistry(runtimePluginRegistry) - : getPreparedMessageToolCatalog()) ?? EMPTY_PREPARED_MESSAGE_TOOL_CATALOG; - const templateAuthStorage = discoverAuthStorage(input.agentDir, { - config: input.config, - // Snapshot construction never initializes, migrates, or externally syncs auth. ModelRegistry - // discovery only parses the credential generation captured here. - readOnly: true, - ...(input.skipCredentials ? { skipCredentials: true } : {}), - ...(input.inheritedAuthDir ? { inheritedAuthDir: input.inheritedAuthDir } : {}), - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), - ...(input.env ? { env } : {}), - }); - const credentials = templateAuthStorage.getAll(); - const providerIds = collectPreparedModelRuntimeProviderIds(input.config, credentials); - if (!input.readOnly) { - await ensureOpenClawModelsJson(input.config, input.agentDir, { - pluginMetadataSnapshot, - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), - ...(input.env ? { env } : {}), - ...(catalogMode === "static" - ? { - providerDiscoveryEntriesOnly: true, - providerDiscoveryProviderIds: providerIds, + const groups = new Map(); + for (const candidate of candidates) { + const group = groups.get(candidate.catalogMode); + if (group) { + group.push(candidate); + } else { + groups.set(candidate.catalogMode, [candidate]); + } + } + const snapshots = new Map(); + const publication = (async () => { + try { + while (true) { + const attempt = candidates.filter( + (candidate) => candidate.isCurrent() && !snapshots.has(candidate.owner), + ); + if (attempt.length === 0) { + break; + } + try { + // Auth events can touch live and static owners together. Build mode groups in sequence + // so one mutation cannot reintroduce broad plugin/catalog fanout on constrained hosts. + for (const [catalogMode, group] of groups) { + const currentGroup = group.filter( + (candidate) => candidate.isCurrent() && !snapshots.has(candidate.owner), + ); + if (currentGroup.length === 0) { + continue; + } + const build = startSerializedSnapshotBuildBatch( + currentGroup.map(({ input }) => input), + params.agentBuildCompletions, + params.buildTimeoutMs, + catalogMode, + undefined, + new Map(currentGroup.map((candidate) => [candidate.input, candidate.isCurrent])), + ); + for (const candidate of currentGroup) { + candidate.owner.buildCompletion = build.completion; + void build.completion.then(() => { + if (candidate.owner.buildCompletion === build.completion) { + candidate.owner.buildCompletion = undefined; + } + }); + } + const built = await build.pending; + for (const [index, candidate] of currentGroup.entries()) { + snapshots.set(candidate.owner, built[index]!); + } } - : { providerDiscoveryTimeoutMs: MODEL_RUNTIME_PROVIDER_DISCOVERY_TIMEOUT_MS }), - }); - } - const templateModelRegistry = discoverModels(templateAuthStorage, input.agentDir, { - config: input.config, - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), - ...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}), - ...(catalogMode === "static" ? { normalizeModels: false } : {}), - }); - const modelCatalog = await buildPreparedModelCatalogSnapshot({ - agentDir: input.agentDir, - authCredentials: credentials, - config: input.config, - modelRegistry: templateModelRegistry, - metadataSnapshot: pluginMetadataSnapshot, - includeProviderPluginAugmentation: catalogMode === "live", - ...(input.env ? { env } : {}), - ...(input.readOnly ? { readOnly: true } : {}), - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), - }); - const providerStaticModels = await loadBundledProviderStaticCatalogContextModels({ - cfg: input.config, - env, - ...(catalogMode === "static" ? { providerIds } : {}), - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), - }); - const configuredRuntimeModels = prepareConfiguredRuntimeModels({ - config: input.config, - env, - metadataSnapshot: pluginMetadataSnapshot, - providerStaticModels, - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), - }); - const staticModels = new Map(); - for (const model of [ - ...configuredRuntimeModels.map((configured) => configured.model), - ...providerStaticModels, - ]) { - const modelKey = `${normalizeProviderId(model.provider)}\0${model.id.trim().toLowerCase()}`; - if (!staticModels.has(modelKey)) { - staticModels.set(modelKey, model); + break; + } catch (error) { + const refreshError = toError(error); + const lostCandidate = attempt.some((candidate) => !candidate.isCurrent()); + if ( + !(refreshError instanceof PreparedModelRuntimePublicationSupersededError) || + !lostCandidate + ) { + throw refreshError; + } + // Supersession belongs to one owner generation. Retry only still-current siblings so + // an agent-local mutation cannot discard an inherited-auth refresh built for others. + } + } + for (const candidate of candidates) { + if (!candidate.isCurrent()) { + continue; + } + const snapshot = snapshots.get(candidate.owner); + if (!snapshot) { + throw new Error( + `prepared model runtime snapshot missing after auth refresh for ${candidate.input.agentDir}`, + ); + } + candidate.owner.snapshot = snapshot; + candidate.owner.pending = undefined; + candidate.owner.needsRefresh = false; + } + } catch (error) { + const refreshError = toError(error); + for (const candidate of candidates) { + if (!candidate.isCurrent()) { + continue; + } + candidate.owner.pending = undefined; + candidate.owner.needsRefresh = true; + candidate.owner.refreshError = refreshError; + } + throw refreshError; } - } - const staticEntries = [...staticModels.values()].map(toStaticCatalogEntry); - // Config reload publishes a replacement snapshot. Keep the synchronous inline projection - // at that lifecycle boundary instead of rebuilding it on every model resolution in a turn. - const inlineProviderModels = buildInlineProviderModels(input.config.models?.providers ?? {}); - const createStores = (): PreparedModelRuntimeStores => { - // Runtime API keys and session extensions mutate these objects. Fork them per run while the - // credential map and parsed catalog remain owned by the lifecycle snapshot. - const authStorage = AuthStorage.inMemory(credentials); - return { authStorage, modelRegistry: templateModelRegistry.fork(authStorage) }; - }; - return Object.freeze({ - ...(input.agentId ? { agentId: input.agentId } : {}), - agentDir: input.agentDir, - activeProjectKeys: [], - ...(input.inheritedAuthDir ? { inheritedAuthDir: input.inheritedAuthDir } : {}), - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), - config: input.config, - metadataSnapshot: pluginMetadataSnapshot, - messageToolCatalog, - ...(mediaCapabilityProviders ? { mediaCapabilityProviders } : {}), - modelCatalog: { ...modelCatalog, staticEntries }, - configuredRuntimeModels, - inlineProviderModels, - createStores, - }); -} - -export function startSerializedSnapshotBuild( - input: PreparedModelRuntimeInput, - agentBuildCompletions: Map>, - buildTimeoutMs: number, - catalogMode: PreparedModelRuntimeCatalogMode = "live", -): { - pending: Promise; - completion: Promise; -} { - const previousBuildCompletion = agentBuildCompletions.get(input.agentDir); - // Lifecycle events may overlap. The timeout covers queueing plus this build, while completion - // follows the real work so a timed-out generation can never overlap a replacement. - const startBuild = (async () => { - if (previousBuildCompletion) { - await previousBuildCompletion; - } - return { actualBuild: buildSnapshot(input, catalogMode) }; })(); - const completion = startBuild - .then(async ({ actualBuild }) => await actualBuild) - .then( - () => undefined, - () => undefined, - ); - agentBuildCompletions.set(input.agentDir, completion); - void completion.then(() => { - if (agentBuildCompletions.get(input.agentDir) === completion) { - agentBuildCompletions.delete(input.agentDir); - } - }); - return { - pending: withTimeout( - async () => { - const { actualBuild } = await startBuild; - return await actualBuild; - }, - buildTimeoutMs, - "prepared model runtime publication", - ), - completion, - }; + for (const candidate of candidates) { + const pending = publication.then(() => { + // A newer auth publication may win while this batch finishes. Reject deduplicated callers + // at the owner boundary so the stale snapshot cannot escape despite being skipped at commit. + if (!candidate.isCurrent()) { + throw new PreparedModelRuntimePublicationSupersededError( + `prepared model runtime publication was superseded for ${candidate.input.agentDir}`, + ); + } + return snapshots.get(candidate.owner)!; + }); + candidate.owner.pending = pending; + void pending.catch(() => undefined); + } + await publication; } export async function publishModelRuntimeSnapshot( @@ -650,6 +431,7 @@ export async function publishModelRuntimeSnapshot( agentBuildCompletions, buildTimeoutMs, catalogMode, + () => owner.generation === generation && owners.get(key) === owner, ); owner.buildCompletion = build.completion; void build.completion.then(() => { diff --git a/src/agents/prepared-model-runtime.startup-static.test.ts b/src/agents/prepared-model-runtime.startup-static.test.ts index f374397e5666..2d9efa3fd4fd 100644 --- a/src/agents/prepared-model-runtime.startup-static.test.ts +++ b/src/agents/prepared-model-runtime.startup-static.test.ts @@ -1,10 +1,14 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +type CreateStaticCatalogResolver = + typeof import("./embedded-agent-runner/model.static-catalog.js").createBundledStaticCatalogModelResolver; +type StaticCatalogResolver = ReturnType; + const mocks = vi.hoisted(() => { const metadataSnapshot = { plugins: [], pluginIds: [], - index: { plugins: [] }, + index: { plugins: [{ pluginId: "openai", enabled: true }] }, manifestRegistry: { plugins: [], diagnostics: [] }, owners: { channels: new Map(), @@ -19,22 +23,73 @@ const mocks = vi.hoisted(() => { }; const authStorage = { getAll: vi.fn(() => ({ openai: { type: "api_key" as const, key: "test-openai-key" } })), + getOAuthProviders: vi.fn(() => []), }; const modelRegistry = { fork: vi.fn((nextAuthStorage: unknown) => ({ authStorage: nextAuthStorage })), getAll: vi.fn(() => []), + find: vi.fn(() => null), }; + const resolveSyntheticAuth = vi.fn(() => ({ + apiKey: "synthetic-openai-key", + source: "test", + mode: "api-key" as const, + })); return { authStorage, modelRegistry, metadataSnapshot, + resolveAmbientCredentials: vi.fn((..._args: unknown[]) => ({})), discoverAuthStorage: vi.fn(() => authStorage), discoverModels: vi.fn(() => modelRegistry), - ensureOpenClawModelsJson: vi.fn(async () => ({ agentDir: "/tmp/agent", wrote: false })), + ensureOpenClawModelsJson: vi.fn( + async (_config: unknown, _agentDir: unknown, _options?: unknown) => ({ + agentDir: "/tmp/agent", + wrote: false, + }), + ), + planOpenClawModelsJsonSource: vi.fn(async (_config: unknown, agentDir: unknown) => ({ + agentDir: String(agentDir), + modelsJsonContents: null, + pluginCatalogs: [], + })), buildPreparedModelCatalogSnapshot: vi.fn(async () => ({ entries: [], routeVariants: [] })), ensureRuntimePluginsLoaded: vi.fn(), loadStaticCatalog: vi.fn(async () => []), - resolveStaticCatalogModel: vi.fn(() => undefined), + prepareStaticCatalog: vi.fn(async (..._args: unknown[]) => ({ + providers: [ + { + id: "openai", + label: "OpenAI", + auth: [], + resolveSyntheticAuth, + }, + ], + entries: [ + { + provider: { id: "openai", label: "OpenAI", auth: [] }, + result: { + provider: { + baseUrl: "https://api.openai.com/v1", + api: "openai-responses", + models: [ + { + id: "gpt-5.5", + name: "GPT-5.5", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 8_192, + }, + ], + }, + }, + }, + ], + })), + resolveStaticCatalogModel: vi.fn(() => undefined), + resolveSyntheticAuth, mutationListener: undefined as | ((event: { agentDir?: string; affectsInheritedStores: boolean }) => void) | undefined, @@ -42,12 +97,22 @@ const mocks = vi.hoisted(() => { }); vi.mock("../plugins/plugin-metadata-snapshot.js", () => ({ + loadPluginMetadataSnapshot: () => mocks.metadataSnapshot, resolvePluginMetadataSnapshot: () => mocks.metadataSnapshot, })); +vi.mock("./agent-auth-discovery.js", () => ({ + resolveAmbientAgentCredentialsForDiscovery: mocks.resolveAmbientCredentials, +})); + vi.mock("./agent-model-discovery.js", () => ({ discoverAuthStorage: mocks.discoverAuthStorage, discoverModels: mocks.discoverModels, + discoverModelsFromCapturedSources: mocks.discoverModels, +})); + +vi.mock("../plugins/synthetic-auth.runtime.js", () => ({ + resolveRuntimeSyntheticAuthProviderRefs: () => [], })); vi.mock("./agent-scope.js", () => ({ @@ -73,6 +138,11 @@ vi.mock("./model-catalog.js", () => ({ vi.mock("./models-config.js", () => ({ ensureOpenClawModelsJson: mocks.ensureOpenClawModelsJson, + planOpenClawModelsJsonSource: mocks.planOpenClawModelsJsonSource, +})); + +vi.mock("./models-config.providers.implicit.js", () => ({ + prepareImplicitProviderStaticCatalog: mocks.prepareStaticCatalog, })); vi.mock("./runtime-plugins.js", () => ({ @@ -88,17 +158,65 @@ vi.mock("../logging/subsystem.js", () => ({ createSubsystemLogger: () => ({ warn: vi.fn() }), })); -const { refreshPreparedModelRuntimeSnapshots } = await import("./prepared-model-runtime.js"); +const { getPreparedModelRuntimeSnapshot, refreshPreparedModelRuntimeSnapshots } = + await import("./prepared-model-runtime.js"); const { resetPreparedModelRuntimeSnapshotsForTest } = await import("./prepared-model-runtime.test-support.js"); beforeEach(() => { resetPreparedModelRuntimeSnapshotsForTest(); vi.clearAllMocks(); + mocks.resolveStaticCatalogModel.mockReturnValue(undefined); }); describe("prepared model runtime Gateway catalog mode", () => { - it("keeps startup and auth refreshes on configured static provider facts", async () => { + it("does not publish a static catalog generation superseded while its hook is running", async () => { + const staleConfig = { agents: { defaults: { model: "openai/gpt-5.5" } } }; + const latestConfig = { agents: { defaults: { model: "openai/gpt-5.6" } } }; + const defaultPrepareStaticCatalog = mocks.prepareStaticCatalog.getMockImplementation(); + let releaseStaleHook: (() => void) | undefined; + let staleHookStarted!: () => void; + const staleHookPending = new Promise((resolve) => { + staleHookStarted = resolve; + }); + mocks.prepareStaticCatalog.mockImplementationOnce(async (...args: unknown[]) => { + staleHookStarted(); + await new Promise((resolve) => { + releaseStaleHook = resolve; + }); + if (!defaultPrepareStaticCatalog) { + throw new Error("expected default static catalog implementation"); + } + return await defaultPrepareStaticCatalog(...args); + }); + + const stale = refreshPreparedModelRuntimeSnapshots(staleConfig, { + gatewayLifecycle: true, + catalogMode: "static", + }); + await staleHookPending; + const latest = refreshPreparedModelRuntimeSnapshots(latestConfig, { + gatewayLifecycle: true, + catalogMode: "static", + }); + releaseStaleHook?.(); + + await expect(stale).rejects.toThrow("superseded"); + await latest; + expect( + getPreparedModelRuntimeSnapshot({ + agentId: "default", + config: latestConfig, + agentDir: "/tmp/prepared-static-agent", + inheritedAuthDir: "/tmp/prepared-static-agent", + workspaceDir: "/tmp/prepared-static-workspace", + })?.config, + ).toBe(latestConfig); + expect(mocks.prepareStaticCatalog).toHaveBeenCalledTimes(2); + expect(mocks.discoverModels).toHaveBeenCalledOnce(); + }); + + it("publishes configured turn facts without eagerly building a full catalog", async () => { const config = { agents: { defaults: { @@ -108,42 +226,138 @@ describe("prepared model runtime Gateway catalog mode", () => { }, }; + let configuredRuntimeModelCount = 0; + let generatedCatalogReadCount = -1; await refreshPreparedModelRuntimeSnapshots(config, { gatewayLifecycle: true, catalogMode: "static", + onBuildStats: (stats) => { + configuredRuntimeModelCount = stats.configuredRuntimeModelCount; + generatedCatalogReadCount = stats.generatedCatalogReadCount; + }, }); - const expectedStaticOptions = expect.objectContaining({ - pluginMetadataSnapshot: mocks.metadataSnapshot, - providerDiscoveryEntriesOnly: true, - providerDiscoveryProviderIds: ["openai"], - }); - expect(mocks.ensureOpenClawModelsJson).toHaveBeenLastCalledWith( - config, - "/tmp/prepared-static-agent", - expectedStaticOptions, + expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); + expect(mocks.ensureRuntimePluginsLoaded).not.toHaveBeenCalled(); + expect(mocks.prepareStaticCatalog).toHaveBeenCalledWith( + expect.objectContaining({ + providerDiscoveryProviderIds: ["openai"], + staticCatalogProviderIds: ["openai"], + }), ); + expect(mocks.resolveAmbientCredentials).toHaveBeenCalledWith( + expect.objectContaining({ + syntheticAuthProviderRefs: ["openai"], + resolveSyntheticAuth: expect.any(Function), + }), + ); + const ambientOptions = mocks.resolveAmbientCredentials.mock.calls[0]?.[0] as + | { resolveSyntheticAuth?: (provider: string) => { apiKey?: string } | undefined } + | undefined; + expect(ambientOptions?.resolveSyntheticAuth?.("openai")).toMatchObject({ + apiKey: "synthetic-openai-key", + }); + expect(mocks.resolveSyntheticAuth).toHaveBeenCalledWith({ + config, + provider: "openai", + providerConfig: undefined, + }); expect(mocks.discoverModels).toHaveBeenLastCalledWith( mocks.authStorage, + expect.objectContaining({ + includePluginCatalogs: true, + modelsJsonContents: null, + pluginCatalogs: [], + pluginMetadataSnapshot: mocks.metadataSnapshot, + workspaceDir: "/tmp/prepared-static-workspace", + }), + ); + expect(mocks.buildPreparedModelCatalogSnapshot).not.toHaveBeenCalled(); + expect(mocks.loadStaticCatalog).not.toHaveBeenCalled(); + expect(configuredRuntimeModelCount).toBe(1); + expect(generatedCatalogReadCount).toBe(0); + const snapshot = getPreparedModelRuntimeSnapshot({ + agentId: "default", + config, + agentDir: "/tmp/prepared-static-agent", + inheritedAuthDir: "/tmp/prepared-static-agent", + workspaceDir: "/tmp/prepared-static-workspace", + }); + expect(snapshot?.configuredRuntimeModels).toHaveLength(1); + expect(snapshot?.messageToolCatalog).toBeUndefined(); + expect(snapshot?.mediaCapabilityProviders).toBeUndefined(); + const fullCatalog = await snapshot?.loadFullModelCatalog?.(); + expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); + expect(mocks.planOpenClawModelsJsonSource).toHaveBeenCalledWith( + config, "/tmp/prepared-static-agent", - expect.objectContaining({ normalizeModels: false }), + expect.objectContaining({ + pluginMetadataSnapshot: mocks.metadataSnapshot, + providerDiscoveryTimeoutMs: 5_000, + }), ); - expect(mocks.buildPreparedModelCatalogSnapshot).toHaveBeenLastCalledWith( - expect.objectContaining({ includeProviderPluginAugmentation: false }), + expect(mocks.buildPreparedModelCatalogSnapshot).toHaveBeenCalledWith( + expect.objectContaining({ includeProviderPluginAugmentation: true }), ); - expect(mocks.loadStaticCatalog).toHaveBeenLastCalledWith( - expect.objectContaining({ providerIds: ["openai"] }), + expect(mocks.ensureRuntimePluginsLoaded).toHaveBeenCalledOnce(); + expect(mocks.ensureRuntimePluginsLoaded.mock.invocationCallOrder[0]).toBeLessThan( + mocks.buildPreparedModelCatalogSnapshot.mock.invocationCallOrder[0]!, + ); + expect(mocks.loadStaticCatalog).toHaveBeenCalledWith( + expect.objectContaining({ metadataSnapshot: mocks.metadataSnapshot }), ); mocks.mutationListener?.({ agentDir: "/tmp/prepared-static-agent", affectsInheritedStores: false, }); - await vi.waitFor(() => expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(2)); - expect(mocks.ensureOpenClawModelsJson).toHaveBeenLastCalledWith( - config, - "/tmp/prepared-static-agent", - expectedStaticOptions, + await expect(snapshot?.loadFullModelCatalog?.()).resolves.toBe(fullCatalog); + await vi.waitFor(() => expect(mocks.prepareStaticCatalog).toHaveBeenCalledTimes(2)); + expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); + expect(mocks.planOpenClawModelsJsonSource).toHaveBeenCalledOnce(); + expect(mocks.prepareStaticCatalog).toHaveBeenCalledTimes(2); + expect(mocks.discoverModels).toHaveBeenCalledTimes(3); + }); + + it("does not request a static provider hook when manifest facts resolve the configured model", async () => { + mocks.resolveStaticCatalogModel.mockReturnValue({ + id: "gpt-5.5", + name: "GPT-5.5", + provider: "openai", + api: "openai-responses", + baseUrl: "https://api.openai.com/v1", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 8_192, + }); + const config = { + agents: { + defaults: { + model: { primary: "openai/gpt-5.5" }, + }, + }, + }; + + await refreshPreparedModelRuntimeSnapshots(config, { + gatewayLifecycle: true, + catalogMode: "static", + }); + + expect(mocks.prepareStaticCatalog).toHaveBeenCalledWith( + expect.objectContaining({ + providerDiscoveryProviderIds: ["openai"], + staticCatalogProviderIds: [], + }), ); + const snapshot = getPreparedModelRuntimeSnapshot({ + agentId: "default", + config, + agentDir: "/tmp/prepared-static-agent", + inheritedAuthDir: "/tmp/prepared-static-agent", + workspaceDir: "/tmp/prepared-static-workspace", + }); + expect(snapshot?.configuredRuntimeModels).toHaveLength(1); }); }); diff --git a/src/agents/prepared-model-runtime.test.ts b/src/agents/prepared-model-runtime.test.ts index 94e77c384f13..f11c03cc6d2f 100644 --- a/src/agents/prepared-model-runtime.test.ts +++ b/src/agents/prepared-model-runtime.test.ts @@ -7,17 +7,27 @@ type CreateStaticCatalogResolver = type StaticCatalogResolver = ReturnType; const mocks = vi.hoisted(() => ({ - authStorage: { getAll: vi.fn(() => ({ custom: { type: "api_key", key: "test-key" } })) }, + authStorage: { + getAll: vi.fn(() => ({ custom: { type: "api_key", key: "test-key" } })), + getOAuthProviders: vi.fn(() => []), + }, modelRegistry: { fork: vi.fn((authStorage: unknown) => ({ authStorage })), getAll: vi.fn(() => []), + find: vi.fn(() => null), }, + resolveAmbientCredentials: vi.fn((..._args: unknown[]) => ({})), discoverAuthStorage: vi.fn(), discoverModels: vi.fn(), ensureOpenClawModelsJson: vi.fn(async (..._args: unknown[]) => ({ agentDir: "/tmp/agent", wrote: false, })), + planOpenClawModelsJsonSource: vi.fn(async (...args: unknown[]) => ({ + agentDir: String(args[1]), + modelsJsonContents: null, + pluginCatalogs: [], + })), buildPreparedModelCatalogSnapshot: vi.fn(async (..._args: unknown[]) => ({ entries: [], routeVariants: [], @@ -37,6 +47,11 @@ vi.mock("./model-catalog.js", () => ({ mocks.buildPreparedModelCatalogSnapshot(...args), })); +vi.mock("./agent-auth-discovery.js", () => ({ + resolveAmbientAgentCredentialsForDiscovery: (...args: unknown[]) => + mocks.resolveAmbientCredentials(...args), +})); + vi.mock("./agent-model-discovery.js", () => ({ discoverAuthStorage: (...args: unknown[]) => { mocks.discoverAuthStorage(...args); @@ -46,6 +61,14 @@ vi.mock("./agent-model-discovery.js", () => ({ mocks.discoverModels(...args); return mocks.modelRegistry; }, + discoverModelsFromCapturedSources: (...args: unknown[]) => { + mocks.discoverModels(...args); + return mocks.modelRegistry; + }, +})); + +vi.mock("../plugins/synthetic-auth.runtime.js", () => ({ + resolveRuntimeSyntheticAuthProviderRefs: () => [], })); vi.mock("./agent-scope.js", () => ({ @@ -73,6 +96,7 @@ vi.mock("./model-discovery-context.js", () => ({ vi.mock("./models-config.js", () => ({ ensureOpenClawModelsJson: (...args: unknown[]) => mocks.ensureOpenClawModelsJson(...args), + planOpenClawModelsJsonSource: (...args: unknown[]) => mocks.planOpenClawModelsJsonSource(...args), })); vi.mock("./runtime-plugins.js", () => ({ @@ -90,6 +114,7 @@ vi.mock("../logging/subsystem.js", () => ({ createSubsystemLogger: () => ({ warn: vi.fn() }), })); +import { startSerializedSnapshotBuild } from "./prepared-model-runtime.build.js"; import { acquireReadOnlyPreparedModelRuntime, activateStandalonePreparedModelRuntime, @@ -114,6 +139,7 @@ describe("prepared model runtime snapshots", () => { beforeEach(() => { getTesting().resetPreparedModelRuntimeSnapshotsForTest(); mocks.discoverAuthStorage.mockClear(); + mocks.resolveAmbientCredentials.mockClear(); mocks.discoverModels.mockClear(); mocks.ensureOpenClawModelsJson.mockClear(); mocks.buildPreparedModelCatalogSnapshot.mockClear(); @@ -126,6 +152,21 @@ describe("prepared model runtime snapshots", () => { mocks.configuredAgentIds = []; }); + it("allows a direct serialized build without a lifecycle generation guard", async () => { + const input = { + config: {}, + agentDir: "/tmp/direct-prepared-model-runtime-build", + readOnly: true, + }; + const build = startSerializedSnapshotBuild(input, new Map(), 1_000, "static"); + + await expect(build.pending).resolves.toMatchObject({ + agentDir: input.agentDir, + config: input.config, + }); + await expect(build.completion).resolves.toBeUndefined(); + }); + it("keeps an isolated setup probe exact after a gateway replacement", async () => { mocks.configuredAgentIds = ["default"]; const stagedConfig = { agents: { defaults: { model: "openai/gpt-5.6" } } }; @@ -258,11 +299,14 @@ describe("prepared model runtime snapshots", () => { workspaceDir: "/tmp/prepared-model-runtime-static-workspace", }); - expect(mocks.loadStaticCatalog).toHaveBeenCalledWith({ - cfg: {}, - env: process.env, - workspaceDir: "/tmp/prepared-model-runtime-static-workspace", - }); + expect(mocks.loadStaticCatalog).toHaveBeenCalledWith( + expect.objectContaining({ + cfg: {}, + env: process.env, + metadataSnapshot: snapshot.metadataSnapshot, + workspaceDir: "/tmp/prepared-model-runtime-static-workspace", + }), + ); expect(snapshot.modelCatalog.staticEntries).toEqual([ { provider: "nvidia", @@ -310,11 +354,14 @@ describe("prepared model runtime snapshots", () => { workspaceDir: "/tmp/prepared-model-runtime-manifest-workspace", }); - expect(mocks.loadStaticCatalog).toHaveBeenCalledWith({ - cfg: config, - env: process.env, - workspaceDir: "/tmp/prepared-model-runtime-manifest-workspace", - }); + expect(mocks.loadStaticCatalog).toHaveBeenCalledWith( + expect.objectContaining({ + cfg: config, + env: process.env, + metadataSnapshot: snapshot.metadataSnapshot, + workspaceDir: "/tmp/prepared-model-runtime-manifest-workspace", + }), + ); expect(mocks.createStaticCatalogResolver).toHaveBeenCalledOnce(); expect(mocks.createStaticCatalogResolver).toHaveBeenCalledWith( expect.objectContaining({ @@ -613,6 +660,7 @@ describe("prepared model runtime snapshots", () => { expect(Object.isFrozen(first)).toBe(true); expect(mocks.ensureOpenClawModelsJson).toHaveBeenCalledTimes(1); expect(mocks.discoverAuthStorage).toHaveBeenCalledTimes(1); + expect(mocks.resolveAmbientCredentials).toHaveBeenCalledTimes(1); expect(mocks.discoverModels).toHaveBeenCalledTimes(1); expect(mocks.buildPreparedModelCatalogSnapshot).toHaveBeenCalledWith( expect.objectContaining({ authCredentials: mocks.authStorage.getAll() }), diff --git a/src/agents/prepared-model-runtime.ts b/src/agents/prepared-model-runtime.ts index c657e8022a32..09cc213ebebc 100644 --- a/src/agents/prepared-model-runtime.ts +++ b/src/agents/prepared-model-runtime.ts @@ -16,10 +16,11 @@ import { normalizePreparedModelRuntimeInput, ownerKey, preparedModelRuntimeConfigsMatch, + publishPreparedModelRuntimeOwnerBatch, publishModelRuntimeSnapshot, rebindInputToCommittedConfiguredOwner, resolvePublishedOwner, - startSerializedSnapshotBuild, + startSerializedSnapshotBuildBatch, toError, type PreparedModelRuntimeOwner, type PreparedModelRuntimeInput, @@ -490,12 +491,11 @@ export function rejectPendingPreparedModelRuntimeReplacement( /** Rebuilds active owners after config/plugin runtime publication. */ async function refreshPreparedModelRuntimeSnapshotsNow( config: OpenClawConfig, - options: PreparedModelRuntimeRefreshOptions = {}, + options: PreparedModelRuntimeRefreshOptions, + publicationEpoch: number, ): Promise { const catalogMode = options.catalogMode ?? "live"; - if (options.gatewayLifecycle) { - gatewayLifecycleActive = true; - } + gatewayLifecycleActive ||= options.gatewayLifecycle === true; const staleError = new Error("prepared model runtime owner is stale after config publication"); for (const owner of owners.values()) { // Invalidate every prior generation before starting any replacement. A failed reload must @@ -552,26 +552,35 @@ async function refreshPreparedModelRuntimeSnapshotsNow( owner.needsRefresh = true; owner.refreshError = undefined; const generation = owner.generation; - const build = startSerializedSnapshotBuild( - input, - agentBuildCompletions, - modelRuntimeBuildTimeoutMs, - catalogMode, - ); - owner.buildCompletion = build.completion; - owners.set(ownerKey(input), owner); + const isCurrent = () => + publicationEpoch === refreshRequestEpoch && + owner.generation === generation && + owners.get(ownerKey(input)) === owner; + return { input, isCurrent, owner }; + }); + const build = startSerializedSnapshotBuildBatch( + candidates.map(({ input }) => input), + agentBuildCompletions, + modelRuntimeBuildTimeoutMs, + catalogMode, + options.onBuildStats, + new Map(candidates.map((candidate) => [candidate.input, candidate.isCurrent])), + () => publicationEpoch === refreshRequestEpoch, + ); + for (const candidate of candidates) { + owners.set(ownerKey(candidate.input), candidate.owner); + candidate.owner.buildCompletion = build.completion; void build.completion.then(() => { - if (owner.buildCompletion === build.completion) { - owner.buildCompletion = undefined; + if (candidate.owner.buildCompletion === build.completion) { + candidate.owner.buildCompletion = undefined; } }); - return { build, generation, owner }; - }); + } const publication = (async () => { try { - const snapshots = await Promise.all(candidates.map(({ build }) => build.pending)); + const snapshots = await build.pending; for (const [index, candidate] of candidates.entries()) { - if (candidate.owner.generation !== candidate.generation) { + if (!candidate.isCurrent()) { continue; } candidate.owner.snapshot = snapshots[index]!; @@ -581,9 +590,8 @@ async function refreshPreparedModelRuntimeSnapshotsNow( return snapshots; } catch (error) { const refreshError = toError(error); - await Promise.allSettled(candidates.map(({ build }) => build.pending)); for (const candidate of candidates) { - if (candidate.owner.generation !== candidate.generation) { + if (!candidate.isCurrent()) { continue; } candidate.owner.pending = undefined; @@ -594,7 +602,16 @@ async function refreshPreparedModelRuntimeSnapshotsNow( } })(); for (const [index, candidate] of candidates.entries()) { - const pending = publication.then((snapshots) => snapshots[index]!); + const pending = publication.then((snapshots) => { + // Config publication is atomic, including callers deduplicated against an individual owner. + // A superseded batch must not leak its unpublished snapshot through that pending promise. + if (!candidate.isCurrent()) { + throw new PreparedModelRuntimePublicationSupersededError( + `prepared model runtime publication was superseded for ${candidate.input.agentDir}`, + ); + } + return snapshots[index]!; + }); candidate.owner.pending = pending; void pending.catch(() => undefined); } @@ -614,7 +631,7 @@ export function refreshPreparedModelRuntimeSnapshots( if (requestEpoch !== refreshRequestEpoch) { return; } - await refreshPreparedModelRuntimeSnapshotsNow(config, options); + await refreshPreparedModelRuntimeSnapshotsNow(config, options, requestEpoch); if (requestEpoch !== refreshRequestEpoch) { return; } @@ -686,29 +703,12 @@ async function drainPendingAuthMutations(): Promise { entries.push({ owner, input: owner.input }); } } - const results = await Promise.allSettled( - entries.map( - async ({ owner, input }) => - await publishPreparedModelRuntimeSnapshot(input, { - force: true, - provenance: owner.provenance, - }), - ), - ); - // Supersession belongs to one owner generation. Wait for every sibling refresh before - // deciding the batch outcome so an expected race cannot hide a genuine owner failure. - const failures = results.flatMap((result) => - result.status === "rejected" && - !(result.reason instanceof PreparedModelRuntimePublicationSupersededError) - ? [result.reason] - : [], - ); - if (failures.length === 1) { - throw failures[0]; - } - if (failures.length > 1) { - throw new AggregateError(failures, `${failures.length} model runtime owner refreshes failed`); - } + await publishPreparedModelRuntimeOwnerBatch({ + entries, + owners, + agentBuildCompletions, + buildTimeoutMs: modelRuntimeBuildTimeoutMs, + }); } } diff --git a/src/agents/prepared-model-runtime.types.ts b/src/agents/prepared-model-runtime.types.ts new file mode 100644 index 000000000000..c465861f875d --- /dev/null +++ b/src/agents/prepared-model-runtime.types.ts @@ -0,0 +1,122 @@ +import type { PreparedMessageToolCatalog } from "../channels/plugins/message-action-discovery.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { prepareMediaCapabilityProviders } from "../plugins/capability-provider-runtime.js"; +import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.types.js"; +import type { InlineModelEntry } from "./embedded-agent-runner/model.inline-provider.js"; +import type { ModelCatalogSnapshot } from "./model-catalog.types.js"; +import type { PreparedConfiguredRuntimeModel } from "./prepared-model-runtime.configured.js"; +import type { AuthStorage } from "./sessions/auth-storage.js"; +import type { ModelRegistry } from "./sessions/model-registry.js"; + +export type PreparedModelRuntimeCatalogMode = "live" | "static"; + +export type PreparedModelRuntimeSnapshot = Readonly<{ + agentId?: string; + agentDir: string; + inheritedAuthDir?: string; + workspaceDir?: string; + /** Run-prepared repository root; null means discovery completed without a match. */ + repoRoot?: string | null; + /** Stable identity derived from repoRoot; null means the run is outside a repository. */ + projectKey?: string | null; + /** Session active project set, ordered most-recent first; empty before run binding. */ + activeProjectKeys: readonly string[]; + config: OpenClawConfig; + metadataSnapshot: PluginMetadataSnapshot; + messageToolCatalog?: PreparedMessageToolCatalog; + mediaCapabilityProviders?: ReturnType; + /** + * Configured model projection used by turn admission and synchronous callers. + * Full inventory discovery is deliberately outside the startup publication boundary. + */ + modelCatalog: ModelCatalogSnapshot; + /** Builds this generation's full control-plane catalog without replacing turn facts. */ + loadFullModelCatalog?: () => Promise; + /** Full static models for configured refs, resolved once at the lifecycle boundary. */ + configuredRuntimeModels: readonly PreparedConfiguredRuntimeModel[]; + /** Inline provider projection prepared once for all resolutions owned by this snapshot. */ + inlineProviderModels: readonly InlineModelEntry[]; + createStores: () => PreparedModelRuntimeStores; +}>; + +export type PreparedModelRuntimeStores = { + authStorage: AuthStorage; + modelRegistry: ModelRegistry; +}; + +export type PreparedModelRuntimeInput = { + agentId?: string; + agentDir: string; + inheritedAuthDir?: string; + workspaceDir?: string; + preserveWorkspaceDirOnRefresh?: boolean; + readOnly?: boolean; + skipCredentials?: boolean; + env?: NodeJS.ProcessEnv; + config: OpenClawConfig; +}; + +export type PreparedModelRuntimeLease = Readonly<{ + snapshot: PreparedModelRuntimeSnapshot; + release: () => void; +}>; + +export type PreparedModelRuntimePublicationOptions = { + force?: boolean; + provenance?: PreparedModelRuntimeOwner["provenance"]; + catalogMode?: PreparedModelRuntimeCatalogMode; +}; + +export type PreparedModelRuntimeRefreshOptions = { + gatewayLifecycle?: boolean; + defaultWorkspaceDir?: string; + catalogMode?: PreparedModelRuntimeCatalogMode; + onBuildStats?: (stats: PreparedModelRuntimeBuildStats) => void; +}; + +export type PreparedModelRuntimeBuildStats = Readonly<{ + agentCount: number; + workspaceGroupCount: number; + configuredFactsGroupCount: number; + catalogSourceCount: number; + credentialGroupCount: number; + catalogGroupCount: number; + runtimeRegistryCount: number; + configuredRuntimeModelCount: number; + generatedCatalogPluginCount: number; + generatedCatalogReadCount: number; + workspaceFactsMs: number; + runtimePluginMs: number; + pluginMetadataMs: number; + staticProviderCatalogMs: number; + ambientCredentialsMs: number; + agentFactsMs: number; + configuredProjectionMs: number; + catalogSourceMs: number; + registryMs: number; + sourceConcurrencyLimit: number; + fullCatalogConcurrencyLimit: number; +}>; + +export type PreparedModelRuntimeOwner = { + input: PreparedModelRuntimeInput; + environmentFingerprint: string; + catalogMode: PreparedModelRuntimeCatalogMode; + provenance: "configured" | "standalone" | "explicit" | "run" | "ephemeral"; + generation: number; + needsRefresh: boolean; + refreshError?: Error; + snapshot?: PreparedModelRuntimeSnapshot; + pending?: Promise; + buildCompletion?: Promise; + leaseCount?: number; +}; + +export type PreparedModelRuntimeReplacement = { + gateId: PreparedModelRuntimeReplacementGateId; + promise: Promise; + resolve: () => void; + reject: (error: Error) => void; +}; + +export type PreparedModelRuntimeReplacementGateId = symbol; diff --git a/src/agents/provider-attribution.test.ts b/src/agents/provider-attribution.test.ts index 3d577866bb5b..727f0c2f832c 100644 --- a/src/agents/provider-attribution.test.ts +++ b/src/agents/provider-attribution.test.ts @@ -287,6 +287,42 @@ describe("provider attribution", () => { expect(loadPluginMetadataSnapshot).not.toHaveBeenCalled(); }); + it("uses explicitly prepared provider facts without reading process metadata", () => { + providerMetadataState.pluginIdScoped = true; + providerMetadataState.snapshot = undefined; + const providerMetadataOwners = { + channels: new Map(), + channelConfigs: new Map(), + providers: new Map(), + modelCatalogProviders: new Map(), + cliBackends: new Map(), + setupProviders: new Map(), + commandAliases: new Map(), + contracts: new Map(), + providerEndpoints: [ + { + endpointClass: "anthropic-public" as const, + hosts: ["prepared.example"], + hostSuffixes: [], + baseUrls: [], + }, + ], + providerRequests: new Map([["prepared", { family: "prepared-family" }]]), + }; + + expect( + resolveProviderRequestPolicy({ + provider: "prepared", + baseUrl: "https://prepared.example", + providerMetadataOwners, + }), + ).toMatchObject({ + endpointClass: "anthropic-public", + knownProviderFamily: "prepared-family", + }); + expect(loadPluginMetadataSnapshot).not.toHaveBeenCalled(); + }); + it("resolves the canonical OpenClaw product and runtime version", () => { const identity = resolveProviderAttributionIdentity({ OPENCLAW_VERSION: "2026.3.99", diff --git a/src/agents/provider-attribution.ts b/src/agents/provider-attribution.ts index 1c1aeac8510b..8c602ac2a013 100644 --- a/src/agents/provider-attribution.ts +++ b/src/agents/provider-attribution.ts @@ -10,6 +10,7 @@ import type { } from "../plugins/manifest.js"; import { normalizePluginProviderBaseUrl } from "../plugins/plugin-metadata-provider-facts.js"; import { loadPluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; +import type { PluginMetadataSnapshotOwnerMaps } from "../plugins/plugin-metadata-snapshot.types.js"; import { asBoolean } from "../utils/boolean.js"; import type { RuntimeVersionEnv } from "../version.js"; import { resolveRuntimeServiceVersion } from "../version.js"; @@ -89,6 +90,7 @@ export type ProviderRequestPolicyInput = { baseUrl?: string | null; transport?: ProviderRequestTransport; capability?: ProviderRequestCapability; + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps; }; /** Provider policy facts consumed by transports before constructing a request. */ @@ -177,7 +179,15 @@ type ProviderMetadataOwners = { providerRequests: ReadonlyMap; }; -function resolveProviderMetadataOwners(): ProviderMetadataOwners { +function resolveProviderMetadataOwners( + prepared?: PluginMetadataSnapshotOwnerMaps, +): ProviderMetadataOwners { + if (prepared) { + return { + providerEndpoints: prepared.providerEndpoints ?? [], + providerRequests: prepared.providerRequests ?? new Map(), + }; + } const current = getCurrentPluginMetadataSnapshot({ allowWorkspaceScopedSnapshot: true, }); @@ -194,10 +204,15 @@ function resolveProviderMetadataOwners(): ProviderMetadataOwners { }; } -function resolveManifestProviderRequest( - provider: string | undefined, -): PluginManifestProviderRequestProvider | undefined { - return provider ? resolveProviderMetadataOwners().providerRequests.get(provider) : undefined; +function resolveManifestProviderRequest(params: { + provider: string | undefined; + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps; +}): PluginManifestProviderRequestProvider | undefined { + return params.provider + ? resolveProviderMetadataOwners(params.providerMetadataOwners).providerRequests.get( + params.provider, + ) + : undefined; } function hostMatchesSuffix(host: string, suffix: string): boolean { @@ -227,8 +242,10 @@ function buildManifestEndpointResolution( function resolveManifestProviderEndpoint(params: { host: string; normalizedBaseUrl?: string; + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps; }): ProviderEndpointResolution | undefined { - for (const endpoint of resolveProviderMetadataOwners().providerEndpoints) { + for (const endpoint of resolveProviderMetadataOwners(params.providerMetadataOwners) + .providerEndpoints) { if ((endpoint.hosts ?? []).includes(params.host)) { return buildManifestEndpointResolution(endpoint, params.host); } @@ -253,6 +270,7 @@ function isLocalEndpointHost(host: string): boolean { export function resolveProviderEndpoint( baseUrl: string | null | undefined, + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps, ): ProviderEndpointResolution { if (typeof baseUrl !== "string" || !baseUrl.trim()) { return { endpointClass: "default" }; @@ -263,7 +281,11 @@ export function resolveProviderEndpoint( return { endpointClass: "invalid" }; } const normalizedBaseUrl = normalizePluginProviderBaseUrl(baseUrl); - const manifestEndpoint = resolveManifestProviderEndpoint({ host, normalizedBaseUrl }); + const manifestEndpoint = resolveManifestProviderEndpoint({ + host, + normalizedBaseUrl, + ...(providerMetadataOwners ? { providerMetadataOwners } : {}), + }); if (manifestEndpoint) { return manifestEndpoint; } @@ -273,8 +295,14 @@ export function resolveProviderEndpoint( return { endpointClass: "custom", hostname: host }; } -function resolveKnownProviderFamily(provider: string | undefined): string { - const manifestFamily = resolveManifestProviderRequest(provider)?.family; +function resolveKnownProviderFamily( + provider: string | undefined, + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps, +): string { + const manifestFamily = resolveManifestProviderRequest({ + provider, + ...(providerMetadataOwners ? { providerMetadataOwners } : {}), + })?.family; if (manifestFamily) { return manifestFamily; } @@ -469,7 +497,7 @@ export function resolveProviderRequestPolicy( ): ProviderRequestPolicyResolution { const provider = normalizeProviderId(input.provider ?? ""); const policy = resolveProviderAttributionPolicy(provider, env); - const endpointResolution = resolveProviderEndpoint(input.baseUrl); + const endpointResolution = resolveProviderEndpoint(input.baseUrl, input.providerMetadataOwners); const endpointClass = endpointResolution.endpointClass; const usesConfiguredBaseUrl = endpointClass !== "default"; const usesKnownNativeOpenAIEndpoint = @@ -517,7 +545,10 @@ export function resolveProviderRequestPolicy( policy: attributionPolicy ?? policy, endpointClass, usesConfiguredBaseUrl, - knownProviderFamily: resolveKnownProviderFamily(provider || undefined), + knownProviderFamily: resolveKnownProviderFamily( + provider || undefined, + input.providerMetadataOwners, + ), attributionProvider, attributionHeaders, allowsHiddenAttribution: @@ -565,7 +596,12 @@ export function resolveProviderRequestCapabilities( endpointClass === "google-generative-ai" || endpointClass === "google-vertex"; - const manifestProviderRequest = resolveManifestProviderRequest(provider); + const manifestProviderRequest = resolveManifestProviderRequest({ + provider, + ...(input.providerMetadataOwners + ? { providerMetadataOwners: input.providerMetadataOwners } + : {}), + }); const compatibilityFamily = manifestProviderRequest?.compatibilityFamily; const isResponsesApi = isOpenAIResponsesApi(api); diff --git a/src/agents/provider-request-config.test.ts b/src/agents/provider-request-config.test.ts index 20bbd0222a06..565e15850b7d 100644 --- a/src/agents/provider-request-config.test.ts +++ b/src/agents/provider-request-config.test.ts @@ -4,7 +4,10 @@ import type { ConfiguredProviderRequest } from "../config/types.provider-request import type { SecretRef } from "../config/types.secrets.js"; import { applyPreparedRuntimeAuthToModel, + attachModelProviderMetadataOwners, buildProviderRequestDispatcherPolicy, + getModelProviderMetadataOwners, + inheritModelProviderMetadataOwners, mergeModelProviderRequestOverrides, resolveProviderRequestPolicyConfig, resolveProviderRequestConfig, @@ -14,6 +17,35 @@ import { } from "./provider-request-config.js"; describe("provider request config", () => { + it("carries lifecycle plugin metadata ownership through model projections", () => { + const owners = { + channels: new Map(), + channelConfigs: new Map(), + providers: new Map(), + modelCatalogProviders: new Map(), + cliBackends: new Map(), + setupProviders: new Map(), + commandAliases: new Map(), + contracts: new Map(), + providerEndpoints: [], + providerRequests: new Map([["prepared", { family: "prepared-family" }]]), + }; + const prepared = attachModelProviderMetadataOwners({ id: "prepared-model" }, owners); + const projected = inheritModelProviderMetadataOwners(prepared, { + ...prepared, + id: "projected-model", + }); + + expect(getModelProviderMetadataOwners(prepared)).toBe(owners); + expect(getModelProviderMetadataOwners(projected)).toBe(owners); + expect( + resolveProviderRequestPolicyConfig({ + provider: "prepared", + providerMetadataOwners: getModelProviderMetadataOwners(projected), + }).policy.knownProviderFamily, + ).toBe("prepared-family"); + }); + it("applies prepared runtime auth without retaining stale credential headers", () => { const model = { provider: "microsoft-foundry", diff --git a/src/agents/provider-request-config.ts b/src/agents/provider-request-config.ts index d09b4fe7b39c..563aed8b9c3e 100644 --- a/src/agents/provider-request-config.ts +++ b/src/agents/provider-request-config.ts @@ -12,6 +12,7 @@ import type { import { assertSecretInputResolved } from "../config/types.secrets.js"; import type { PinnedDispatcherPolicy } from "../infra/net/ssrf.js"; import type { Api } from "../llm/types.js"; +import type { PluginMetadataSnapshotOwnerMaps } from "../plugins/plugin-metadata-snapshot.types.js"; import type { ProviderRequestCapabilities, ProviderRequestCapability, @@ -168,6 +169,7 @@ type ResolveProviderRequestPolicyConfigParams = { provider?: string; api?: RequestApi; baseUrl?: string; + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps; defaultBaseUrl?: string; capability?: ProviderRequestCapability; transport?: ProviderRequestTransport; @@ -689,6 +691,9 @@ export function resolveProviderRequestPolicyConfig( provider: params.provider, api: params.api, baseUrl, + ...(params.providerMetadataOwners + ? { providerMetadataOwners: params.providerMetadataOwners } + : {}), capability, transport, } satisfies Parameters[0]; @@ -751,6 +756,7 @@ export function resolveProviderRequestConfig(params: { provider: string; api?: RequestApi; baseUrl?: string; + providerMetadataOwners?: PluginMetadataSnapshotOwnerMaps; capability?: ProviderRequestCapability; transport?: ProviderRequestTransport; discoveredHeaders?: Record; @@ -803,10 +809,14 @@ export function resolveProviderRequestHeaders(params: { const MODEL_PROVIDER_REQUEST_TRANSPORT_SYMBOL = Symbol.for( "openclaw.modelProviderRequestTransport", ); +const MODEL_PROVIDER_METADATA_OWNERS_SYMBOL = Symbol.for("openclaw.modelProviderMetadataOwners"); type ModelWithProviderRequestTransport = { [MODEL_PROVIDER_REQUEST_TRANSPORT_SYMBOL]?: ModelProviderRequestTransportOverrides; }; +type ModelWithProviderMetadataOwners = { + [MODEL_PROVIDER_METADATA_OWNERS_SYMBOL]?: PluginMetadataSnapshotOwnerMaps; +}; /** Attaches model-scoped provider request transport metadata without mutating the model. */ export function attachModelProviderRequestTransport( @@ -827,4 +837,32 @@ export function getModelProviderRequestTransport( ): ModelProviderRequestTransportOverrides | undefined { return (model as ModelWithProviderRequestTransport)[MODEL_PROVIDER_REQUEST_TRANSPORT_SYMBOL]; } + +/** Attaches the lifecycle-owned plugin metadata generation used for request policy. */ +export function attachModelProviderMetadataOwners( + model: TModel, + owners: PluginMetadataSnapshotOwnerMaps | undefined, +): TModel { + if (!owners) { + return model; + } + const next = { ...model } as TModel & ModelWithProviderMetadataOwners; + next[MODEL_PROVIDER_METADATA_OWNERS_SYMBOL] = owners; + return next; +} + +/** Reads the plugin metadata generation attached to a prepared model. */ +export function getModelProviderMetadataOwners( + model: object, +): PluginMetadataSnapshotOwnerMaps | undefined { + return (model as ModelWithProviderMetadataOwners)[MODEL_PROVIDER_METADATA_OWNERS_SYMBOL]; +} + +/** Carries request-policy ownership across provider/transport model projections. */ +export function inheritModelProviderMetadataOwners( + source: object, + target: TModel, +): TModel { + return attachModelProviderMetadataOwners(target, getModelProviderMetadataOwners(source)); +} /* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/agents/provider-transport-fetch.test.ts b/src/agents/provider-transport-fetch.test.ts index 2128798581ed..27b0bf592b3e 100644 --- a/src/agents/provider-transport-fetch.test.ts +++ b/src/agents/provider-transport-fetch.test.ts @@ -93,6 +93,7 @@ vi.mock("./provider-local-service.js", () => ({ vi.mock("./provider-request-config.js", () => ({ buildProviderRequestDispatcherPolicy: buildProviderRequestDispatcherPolicyMock, + getModelProviderMetadataOwners: vi.fn(() => undefined), getModelProviderRequestTransport: vi.fn(() => undefined), mergeModelProviderRequestOverrides: mergeModelProviderRequestOverridesMock, resolveProviderRequestPolicyConfig: resolveProviderRequestPolicyConfigMock, diff --git a/src/agents/provider-transport-fetch.ts b/src/agents/provider-transport-fetch.ts index 4aba7d309799..31e87c62180b 100644 --- a/src/agents/provider-transport-fetch.ts +++ b/src/agents/provider-transport-fetch.ts @@ -45,6 +45,7 @@ import { } from "./provider-local-service.js"; import { buildProviderRequestDispatcherPolicy, + getModelProviderMetadataOwners, getModelProviderRequestTransport, mergeModelProviderRequestOverrides, resolveProviderRequestPolicyConfig, @@ -590,10 +591,12 @@ function resolveModelRequestPolicy(model: Model) { } : undefined, }); + const providerMetadataOwners = getModelProviderMetadataOwners(model); return resolveProviderRequestPolicyConfig({ provider: model.provider, api: model.api, baseUrl: model.baseUrl, + ...(providerMetadataOwners ? { providerMetadataOwners } : {}), capability: "llm", transport: "stream", request, diff --git a/src/agents/sessions/model-registry.test.ts b/src/agents/sessions/model-registry.test.ts index ec0823ed6222..6e369dbed3a1 100644 --- a/src/agents/sessions/model-registry.test.ts +++ b/src/agents/sessions/model-registry.test.ts @@ -383,6 +383,97 @@ describe("ModelRegistry models.json auth", () => { expect(registry.find("zai", "glm-5.1")?.name).toBe("GLM 5.1"); }); + it("can parse authored models without opening generated plugin catalogs", () => { + const modelsPath = writeModelsJsonWithPluginCatalog({ + root: { + providers: { + custom: { + baseUrl: "https://models.example/v1", + api: "openai-completions", + models: [{ id: "authored-model", name: "Authored Model" }], + }, + }, + }, + pluginRelativePath: join("plugins", "zai", PLUGIN_MODEL_CATALOG_FILE), + pluginCatalog: { + generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY, + providers: { + zai: { + baseUrl: "https://api.z.ai/api/paas/v4", + api: "openai-completions", + models: [{ id: "glm-5.1", name: "GLM 5.1" }], + }, + }, + }, + }); + + const registry = ModelRegistry.create(AuthStorage.inMemory(), modelsPath, { + includePluginCatalogs: false, + pluginMetadataSnapshot: pluginOwnerSnapshot("zai", "zai"), + }); + + expect(registry.find("custom", "authored-model")?.name).toBe("Authored Model"); + expect(registry.find("zai", "glm-5.1")).toBeUndefined(); + }); + + it("can parse a lifecycle-captured models.json source without rereading the path", () => { + const modelsPath = writeModelsJson({ providers: {} }); + writeFileSync(modelsPath, "not valid json"); + const captured = JSON.stringify({ + providers: { + custom: { + baseUrl: "https://models.example/v1", + api: "openai-completions", + models: [{ id: "captured-model", name: "Captured Model" }], + }, + }, + }); + + const registry = ModelRegistry.create(AuthStorage.inMemory(), modelsPath, { + includePluginCatalogs: false, + modelsJsonContents: captured, + }); + + expect(registry.getError()).toBeUndefined(); + expect(registry.find("custom", "captured-model")?.name).toBe("Captured Model"); + }); + + it("loads only lifecycle-captured generated catalogs when the root catalog is absent", () => { + const modelsPath = writeModelsJson({ providers: {} }); + rmSync(modelsPath); + const capturedCatalog = { + pluginId: "zai", + contents: JSON.stringify({ + generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY, + providers: { + zai: { + baseUrl: "https://api.z.ai/api/paas/v4", + api: "openai-completions", + models: [{ id: "glm-5.1", name: "GLM 5.1" }], + }, + }, + }), + }; + + const pluginMetadataSnapshot = pluginOwnerSnapshotEntries([ + { providerId: "zai", pluginId: "zai" }, + { providerId: "other", pluginId: "other" }, + ]); + const registry = ModelRegistry.create(AuthStorage.inMemory(), modelsPath, { + includePluginCatalogs: true, + modelsJsonContents: null, + pluginCatalogs: [capturedCatalog], + pluginMetadataSnapshot, + }); + const fork = registry.fork(AuthStorage.inMemory()); + + expect(registry.getError()).toBeUndefined(); + expect(registry.find("zai", "glm-5.1")?.name).toBe("GLM 5.1"); + expect(registry.find("other", "unrelated-model")).toBeUndefined(); + expect(registry.getProviderMetadataOwners()).toBe(pluginMetadataSnapshot.owners); + expect(fork.getProviderMetadataOwners()).toBe(pluginMetadataSnapshot.owners); + }); + it("reports an unreadable legacy catalog while preserving healthy provider models", () => { if (process.getuid?.() === 0) { return; diff --git a/src/agents/sessions/model-registry.ts b/src/agents/sessions/model-registry.ts index 46f9128ca8b3..100a7707ea8a 100644 --- a/src/agents/sessions/model-registry.ts +++ b/src/agents/sessions/model-registry.ts @@ -25,6 +25,7 @@ import { filterGeneratedPluginModelCatalogProviders, isGeneratedPluginModelCatalog, loadPersistedPluginModelCatalogs, + type PersistedPluginModelCatalog, type PluginModelCatalogMetadataSnapshot, } from "../plugin-model-catalog.js"; import { getAuthStorageOAuthProviderRegistry } from "./auth-storage-oauth-registry.js"; @@ -264,6 +265,9 @@ function emptyCustomModelsResult(error?: string): CustomModelsResult { } type ModelRegistryOptions = { + includePluginCatalogs?: boolean; + modelsJsonContents?: string | null; + pluginCatalogs?: readonly PersistedPluginModelCatalog[]; pluginMetadataSnapshot?: PluginModelCatalogMetadataSnapshot; sourceSnapshot?: ModelRegistry; workspaceDir?: string; @@ -328,7 +332,10 @@ export class ModelRegistry { private loadError: string | undefined = undefined; readonly authStorage: AuthStorage; private modelsJsonPath: string | undefined; + private modelsJsonContents: string | null | undefined; + private pluginCatalogs: readonly PersistedPluginModelCatalog[] | undefined; private pluginMetadataSnapshot: PluginModelCatalogMetadataSnapshot | undefined; + private includePluginCatalogs = true; private baseCatalogSnapshot: ModelRegistryCatalogSnapshot | undefined; private sourceSnapshot: ModelRegistryCatalogSnapshot | undefined; @@ -338,6 +345,7 @@ export class ModelRegistry { options: ModelRegistryOptions = {}, ) { this.authStorage = authStorage; + this.includePluginCatalogs = options.includePluginCatalogs !== false; initializeModelRegistryRuntime(this); if (options.sourceSnapshot) { const source = options.sourceSnapshot; @@ -358,6 +366,8 @@ export class ModelRegistry { return; } this.modelsJsonPath = modelsJsonPath; + this.modelsJsonContents = options.modelsJsonContents; + this.pluginCatalogs = options.pluginCatalogs; this.pluginMetadataSnapshot = resolveModelPluginMetadataSnapshot({ ...(options.pluginMetadataSnapshot ? { pluginMetadataSnapshot: options.pluginMetadataSnapshot } @@ -447,20 +457,36 @@ export class ModelRegistry { return this.loadError; } + /** Returns the exact plugin metadata generation captured with this registry. */ + getProviderMetadataOwners() { + return this.pluginMetadataSnapshot?.owners; + } + private loadModels(): void { // Keep authored models.json separate from rebuildable provider catalogs // owned by the agent SQLite cache. - const { models: customModels, error } = this.modelsJsonPath - ? this.loadCustomModels(this.modelsJsonPath) - : emptyCustomModelsResult(); + const customResult = + this.modelsJsonPath && this.modelsJsonContents !== null + ? this.loadCustomModels(this.modelsJsonPath, { + ...(this.modelsJsonContents !== undefined ? { contents: this.modelsJsonContents } : {}), + includePluginCatalogs: this.includePluginCatalogs && this.pluginCatalogs === undefined, + }) + : emptyCustomModelsResult(); + const capturedPluginResult = + this.includePluginCatalogs && this.pluginCatalogs !== undefined + ? this.loadCapturedPluginCatalogs(this.pluginCatalogs) + : emptyCustomModelsResult(); + const errors = [customResult.error, capturedPluginResult.error].filter( + (error): error is string => Boolean(error), + ); - if (error) { - this.loadError = error; - log.warn(`model catalog load issue: ${error}`); + if (errors.length > 0) { + this.loadError = errors.join("\n\n"); + log.warn(`model catalog load issue: ${this.loadError}`); // Plugin catalog failures can return salvaged models; root failures return empty. } - let combined = customModels; + let combined = [...customResult.models, ...capturedPluginResult.models]; // Let OAuth providers modify their models (e.g., update baseUrl) for (const oauthProvider of this.authStorage.getOAuthProviders()) { @@ -473,6 +499,29 @@ export class ModelRegistry { this.models = combined; } + private loadCapturedPluginCatalogs( + pluginCatalogs: readonly PersistedPluginModelCatalog[], + ): CustomModelsResult { + const models: Model[] = []; + const errors: string[] = []; + for (const pluginCatalog of pluginCatalogs) { + const result = this.loadCustomModels( + `sqlite:plugin-model-catalog/${pluginCatalog.pluginId}`, + { + catalogPluginId: pluginCatalog.pluginId, + contents: pluginCatalog.contents, + includePluginCatalogs: false, + requireGeneratedCatalog: true, + }, + ); + models.push(...result.models); + if (result.error) { + errors.push(result.error); + } + } + return { models, error: errors.join("\n\n") || undefined }; + } + private loadCustomModels( modelsJsonPath: string, options: { @@ -538,11 +587,15 @@ export class ModelRegistry { ); const pluginCatalogErrors: string[] = []; if (options.includePluginCatalogs !== false) { - let pluginCatalogs: ReturnType["catalogs"] = []; + let pluginCatalogs: readonly PersistedPluginModelCatalog[] = []; try { - const loaded = loadPersistedPluginModelCatalogs(dirname(modelsJsonPath)); - pluginCatalogs = loaded.catalogs; - pluginCatalogErrors.push(...loaded.warnings); + if (this.pluginCatalogs) { + pluginCatalogs = this.pluginCatalogs; + } else { + const loaded = loadPersistedPluginModelCatalogs(dirname(modelsJsonPath)); + pluginCatalogs = loaded.catalogs; + pluginCatalogErrors.push(...loaded.warnings); + } } catch (error) { pluginCatalogErrors.push( `Failed to load generated plugin model catalogs: ${ diff --git a/src/agents/tools/message-tool.test.ts b/src/agents/tools/message-tool.test.ts index 7994314d7357..4b080306c837 100644 --- a/src/agents/tools/message-tool.test.ts +++ b/src/agents/tools/message-tool.test.ts @@ -16,7 +16,6 @@ import { MESSAGE_TOOL_DELIVERY_HINTS, MESSAGE_TOOL_ONLY_DELIVERY_HINT, } from "../../plugin-sdk/message-tool-delivery-hints.js"; -import { EMPTY_PREPARED_MESSAGE_TOOL_CATALOG } from "../../plugins/prepared-message-tool-catalog.js"; import { wrapToolWithBeforeToolCallHook } from "../agent-tools.before-tool-call.js"; type CreateMessageTool = typeof import("./message-tool.js").createMessageTool; type CreateOpenClawTools = typeof import("../openclaw-tools.js").createOpenClawTools; @@ -27,6 +26,11 @@ type CreateTestRegistry = typeof import("../../test-utils/channel-plugins.js").c const ROOM_EVENT_DELIVERY_HINT = MESSAGE_TOOL_DELIVERY_HINTS[3]; const CRITICAL_THRESHOLD = 20; +const EMPTY_PREPARED_MESSAGE_TOOL_CATALOG = { + version: 0, + channels: [], + getChannel: () => undefined, +} as const; let createMessageTool: CreateMessageTool; let createOpenClawTools: CreateOpenClawTools; diff --git a/src/channels/plugins/message-actions.test.ts b/src/channels/plugins/message-actions.test.ts index 022688b10cff..49f55259fccf 100644 --- a/src/channels/plugins/message-actions.test.ts +++ b/src/channels/plugins/message-actions.test.ts @@ -2,10 +2,7 @@ import { Type } from "typebox"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../../config/config.js"; -import { - EMPTY_PREPARED_MESSAGE_TOOL_CATALOG, - getPreparedMessageToolCatalog, -} from "../../plugins/prepared-message-tool-catalog.js"; +import { getPreparedMessageToolCatalog } from "../../plugins/prepared-message-tool-catalog.js"; import { setActivePluginRegistry } from "../../plugins/runtime.js"; import { defaultRuntime } from "../../runtime.js"; import { @@ -23,6 +20,11 @@ import type { ChannelMessageCapability } from "./message-capabilities.js"; import type { ChannelPlugin } from "./types.public.js"; const emptyRegistry = createTestRegistry([]); +const EMPTY_PREPARED_MESSAGE_TOOL_CATALOG = { + version: 0, + channels: [], + getChannel: () => undefined, +} as const; function createMessageActionsPlugin(params: { id: "demo-buttons" | "demo-cards"; diff --git a/src/flows/doctor-core-checks.runtime.ts b/src/flows/doctor-core-checks.runtime.ts index c502b0d26bbd..906c3632ec37 100644 --- a/src/flows/doctor-core-checks.runtime.ts +++ b/src/flows/doctor-core-checks.runtime.ts @@ -9,7 +9,6 @@ import { listAgentEntries, listAgentIds, resolveAgentDir, - resolveDefaultAgentDir, resolveAgentWorkspaceDir, resolveDefaultAgentId, } from "../agents/agent-scope.js"; @@ -606,7 +605,6 @@ export async function collectProviderCatalogProjectionFindings( const { runProviderStaticCatalog } = await import("../plugins/provider-discovery.js"); const { resolvePluginProviders } = await import("../plugins/providers.runtime.js"); const env = process.env; - const agentDir = resolveDefaultAgentDir(cfg); const workspaceDir = resolveAgentWorkspaceDir(cfg, resolveDefaultAgentId(cfg)); let providers: Awaited>; try { @@ -665,13 +663,7 @@ export async function collectProviderCatalogProjectionFindings( } let result: Awaited>; try { - result = await runProviderStaticCatalog({ - provider, - config: cfg, - agentDir, - workspaceDir, - env, - }); + result = await runProviderStaticCatalog({ provider }); } catch (error) { findings.push( providerCatalogProjectionFinding({ diff --git a/src/gateway/server-startup-model-runtime.event-loop.test.ts b/src/gateway/server-startup-model-runtime.event-loop.test.ts index 3f4964d61d45..4e4120f08ad2 100644 --- a/src/gateway/server-startup-model-runtime.event-loop.test.ts +++ b/src/gateway/server-startup-model-runtime.event-loop.test.ts @@ -102,7 +102,7 @@ afterEach(() => { }); describe("Gateway prepared model runtime startup", () => { - it("keeps health probes responsive without executing live provider catalogs", async () => { + it("keeps health probes responsive without executing unnecessary provider catalogs", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "openclaw-model-runtime-startup-")); const stateDir = path.join(root, "state"); const workspaceDir = path.join(root, "workspace"); @@ -132,14 +132,15 @@ describe("Gateway prepared model runtime startup", () => { }, agentDir, ); - providerMocks.staticCatalog.mockResolvedValue(providerConfig); - providerMocks.liveCatalog.mockImplementation(async () => { + const blockEventLoop = async () => { const stopAt = performance.now() + 1_500; while (performance.now() < stopAt) { // Deliberately model synchronous provider/plugin catalog work that starves timers. } return providerConfig; - }); + }; + providerMocks.staticCatalog.mockImplementation(blockEventLoop); + providerMocks.liveCatalog.mockImplementation(blockEventLoop); const healthServer = await listenHealthz(); try { @@ -164,10 +165,10 @@ describe("Gateway prepared model runtime startup", () => { const [{ elapsedMs, response }] = await Promise.all([probe, sidecars]); expect(response.status).toBe(200); - // Allow loaded CI hosts to finish static startup work while keeping the - // deliberately blocking live-catalog path well outside the guard. + // The configured model is already resolved from manifest facts. Either provider hook + // would deliberately block the event loop well beyond this responsiveness guard. expect(elapsedMs).toBeLessThan(1_000); - expect(providerMocks.staticCatalog).toHaveBeenCalled(); + expect(providerMocks.staticCatalog).not.toHaveBeenCalled(); expect(providerMocks.liveCatalog).not.toHaveBeenCalled(); }, ); diff --git a/src/gateway/server-startup-post-attach.test.ts b/src/gateway/server-startup-post-attach.test.ts index fbad938cacf7..05c0d0c39e04 100644 --- a/src/gateway/server-startup-post-attach.test.ts +++ b/src/gateway/server-startup-post-attach.test.ts @@ -62,7 +62,9 @@ const hoisted = vi.hoisted(() => { inCatalog: true, })); const prepareModelRuntimeSnapshot = vi.fn(async () => ({})); - const refreshPreparedModelRuntimeSnapshots = vi.fn(async (_cfg?: unknown) => {}); + const refreshPreparedModelRuntimeSnapshots = vi.fn( + async (_cfg?: unknown, _options?: unknown) => {}, + ); const ensureRuntimePluginsLoaded = vi.fn(); const ensureContextWindowCacheLoaded = vi.fn(async () => {}); const scheduleGatewayHandlerPrewarm = vi.fn(() => ({ stop: vi.fn() })); @@ -1753,11 +1755,112 @@ describe("startGatewayPostAttachRuntime", () => { }); expect(trace.measures).toContain("sidecars.channels"); expect(trace.measures).toContain("sidecars.channel-skip"); + expect(prewarmPrimaryModel).toHaveBeenCalledWith( + expect.objectContaining({ startupTrace: trace.startupTrace }), + ); expect(logChannels.info).toHaveBeenCalledWith( "skipping channel start (OPENCLAW_SKIP_CHANNELS=1 or OPENCLAW_SKIP_PROVIDERS=1)", ); }); + it("records prepared runtime build grouping in the startup trace", async () => { + const trace = createStartupTraceRecorder(); + + await startGatewaySidecars({ + cfg: { hooks: { internal: { enabled: false } } } as never, + pluginRegistry: createPostAttachParams().pluginRegistry, + defaultWorkspaceDir: "/tmp/openclaw-workspace", + deps: {} as never, + startChannels: vi.fn(async () => {}), + log: { warn: vi.fn() }, + logHooks: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, + logChannels: { info: vi.fn(), error: vi.fn() }, + startupTrace: trace.startupTrace, + }); + + const options = hoisted.refreshPreparedModelRuntimeSnapshots.mock.calls[0]?.[1] as + | { + onBuildStats?: (stats: { + agentCount: number; + workspaceGroupCount: number; + configuredFactsGroupCount: number; + catalogSourceCount: number; + credentialGroupCount: number; + catalogGroupCount: number; + runtimeRegistryCount: number; + configuredRuntimeModelCount: number; + generatedCatalogPluginCount: number; + generatedCatalogReadCount: number; + workspaceFactsMs: number; + runtimePluginMs: number; + pluginMetadataMs: number; + staticProviderCatalogMs: number; + ambientCredentialsMs: number; + agentFactsMs: number; + configuredProjectionMs: number; + catalogSourceMs: number; + registryMs: number; + sourceConcurrencyLimit: number; + fullCatalogConcurrencyLimit: number; + }) => void; + } + | undefined; + options?.onBuildStats?.({ + agentCount: 12, + workspaceGroupCount: 2, + configuredFactsGroupCount: 2, + catalogSourceCount: 0, + credentialGroupCount: 1, + catalogGroupCount: 0, + runtimeRegistryCount: 12, + configuredRuntimeModelCount: 2, + generatedCatalogPluginCount: 0, + generatedCatalogReadCount: 0, + workspaceFactsMs: 120, + runtimePluginMs: 0, + pluginMetadataMs: 40, + staticProviderCatalogMs: 50, + ambientCredentialsMs: 10, + agentFactsMs: 5, + configuredProjectionMs: 15, + catalogSourceMs: 0, + registryMs: 30, + sourceConcurrencyLimit: 2, + fullCatalogConcurrencyLimit: 1, + }); + + expect(trace.details).toContainEqual({ + name: "sidecars.model-runtime-build", + metrics: [ + ["agentCount", 12], + ["workspaceGroupCount", 2], + ["configuredFactsGroupCount", 2], + ["catalogSourceCount", 0], + ["credentialGroupCount", 1], + ["catalogGroupCount", 0], + ["runtimeRegistryCount", 12], + ["configuredRuntimeModelCount", 2], + ["generatedCatalogPluginCount", 0], + ["generatedCatalogReadCount", 0], + ["workspaceFactsMs", 120], + ["runtimePluginMs", 0], + ["pluginMetadataMs", 40], + ["staticProviderCatalogMs", 50], + ["ambientCredentialsMs", 10], + ["agentFactsMs", 5], + ["configuredProjectionMs", 15], + ["catalogSourceMs", 0], + ["registryMs", 30], + ["sourceConcurrencyLimitCount", 2], + ["fullCatalogConcurrencyLimitCount", 1], + ], + }); + }); + it("marks startup main-session orphans before channel startup", async () => { const events: string[] = []; let releaseMarking: (() => void) | undefined; diff --git a/src/gateway/server-startup-post-attach.ts b/src/gateway/server-startup-post-attach.ts index cf7f2136e266..cf27e691e339 100644 --- a/src/gateway/server-startup-post-attach.ts +++ b/src/gateway/server-startup-post-attach.ts @@ -535,6 +535,7 @@ async function prewarmConfiguredPrimaryModel(params: { cfg: OpenClawConfig; workspaceDir?: string; log: { warn: (msg: string) => void }; + startupTrace?: GatewayStartupTrace; }): Promise { await publishConfiguredModelRuntimeSnapshots(params); } @@ -543,6 +544,7 @@ async function publishConfiguredModelRuntimeSnapshots(params: { cfg: OpenClawConfig; workspaceDir?: string; log: { warn: (msg: string) => void }; + startupTrace?: GatewayStartupTrace; }): Promise { const { refreshPreparedModelRuntimeSnapshots } = await import("../agents/prepared-model-runtime.js"); @@ -550,6 +552,34 @@ async function publishConfiguredModelRuntimeSnapshots(params: { gatewayLifecycle: true, catalogMode: "static", ...(params.workspaceDir ? { defaultWorkspaceDir: params.workspaceDir } : {}), + ...(params.startupTrace + ? { + onBuildStats: (stats) => + params.startupTrace?.detail("sidecars.model-runtime-build", [ + ["agentCount", stats.agentCount], + ["workspaceGroupCount", stats.workspaceGroupCount], + ["configuredFactsGroupCount", stats.configuredFactsGroupCount], + ["catalogSourceCount", stats.catalogSourceCount], + ["credentialGroupCount", stats.credentialGroupCount], + ["catalogGroupCount", stats.catalogGroupCount], + ["runtimeRegistryCount", stats.runtimeRegistryCount], + ["configuredRuntimeModelCount", stats.configuredRuntimeModelCount], + ["generatedCatalogPluginCount", stats.generatedCatalogPluginCount], + ["generatedCatalogReadCount", stats.generatedCatalogReadCount], + ["workspaceFactsMs", stats.workspaceFactsMs], + ["runtimePluginMs", stats.runtimePluginMs], + ["pluginMetadataMs", stats.pluginMetadataMs], + ["staticProviderCatalogMs", stats.staticProviderCatalogMs], + ["ambientCredentialsMs", stats.ambientCredentialsMs], + ["agentFactsMs", stats.agentFactsMs], + ["configuredProjectionMs", stats.configuredProjectionMs], + ["catalogSourceMs", stats.catalogSourceMs], + ["registryMs", stats.registryMs], + ["sourceConcurrencyLimitCount", stats.sourceConcurrencyLimit], + ["fullCatalogConcurrencyLimitCount", stats.fullCatalogConcurrencyLimit], + ]), + } + : {}), }); } @@ -558,6 +588,7 @@ async function publishStartupModelRuntime( cfg: OpenClawConfig; workspaceDir?: string; log: { warn: (msg: string) => void }; + startupTrace?: GatewayStartupTrace; }, prewarm: typeof prewarmConfiguredPrimaryModel = prewarmConfiguredPrimaryModel, ): Promise { @@ -635,6 +666,7 @@ export async function startGatewaySidecars(params: { cfg: params.cfg, workspaceDir: params.defaultWorkspaceDir, log: params.log, + startupTrace: params.startupTrace, }, params.prewarmPrimaryModel, ), diff --git a/src/plugins/prepared-message-tool-catalog.ts b/src/plugins/prepared-message-tool-catalog.ts index aaf55d9fdbb9..1f6b775ca01e 100644 --- a/src/plugins/prepared-message-tool-catalog.ts +++ b/src/plugins/prepared-message-tool-catalog.ts @@ -22,12 +22,6 @@ export type PreparedMessageToolCatalog = Readonly<{ const catalogsByRegistry = new WeakMap>(); const latestCatalogByRegistry = new WeakMap(); -export const EMPTY_PREPARED_MESSAGE_TOOL_CATALOG: PreparedMessageToolCatalog = Object.freeze({ - version: 0, - channels: Object.freeze([]), - getChannel: () => undefined, -}); - function selectedRegistry( snapshot: ActivePluginChannelRegistrySnapshot, ): PluginRegistry | undefined { diff --git a/src/plugins/provider-discovery.runtime.test.ts b/src/plugins/provider-discovery.runtime.test.ts index db31e59464d8..08d79e0171f1 100644 --- a/src/plugins/provider-discovery.runtime.test.ts +++ b/src/plugins/provider-discovery.runtime.test.ts @@ -669,6 +669,30 @@ describe("resolvePluginDiscoveryProvidersRuntime", () => { expect(mocks.resolvePluginProviders).not.toHaveBeenCalled(); }); + it("returns synthetic-auth discovery entries only when explicitly requested", () => { + const syntheticProvider: ProviderPlugin = { + id: "claude-cli", + label: "Claude CLI", + auth: [], + resolveSyntheticAuth: () => ({ + apiKey: "synthetic-token", + source: "test", + mode: "oauth", + }), + }; + mocks.loadSource.mockReturnValue(syntheticProvider); + + expect(resolvePluginDiscoveryProvidersRuntime({ discoveryEntriesOnly: true })).toEqual([]); + const providers = resolvePluginDiscoveryProvidersRuntime({ + discoveryEntriesOnly: true, + includeSyntheticAuthProviders: true, + }); + + expect(providers).toHaveLength(1); + expect(providers[0]).toMatchObject({ id: "claude-cli", pluginId: "deepseek" }); + expect(mocks.resolvePluginProviders).not.toHaveBeenCalled(); + }); + it("returns manifest model catalogs as static discovery entries", async () => { mocks.resolveDiscoveredProviderPluginIds.mockReturnValue(["openai"]); mocks.loadPluginMetadataSnapshot.mockReturnValue({ diff --git a/src/plugins/provider-discovery.runtime.ts b/src/plugins/provider-discovery.runtime.ts index dfa1c1fed9a2..02bf45f70089 100644 --- a/src/plugins/provider-discovery.runtime.ts +++ b/src/plugins/provider-discovery.runtime.ts @@ -443,12 +443,18 @@ export function resolvePluginDiscoveryProvidersRuntime(params: { requireCompleteDiscoveryEntryCoverage?: boolean; discoveryEntriesOnly?: boolean; includeManifestModelCatalogProviders?: boolean; + includeSyntheticAuthProviders?: boolean; pluginMetadataSnapshot?: PluginMetadataRegistryView; }): ProviderPlugin[] { const env = params.env ?? process.env; const bundledProviderVitestCompat = params.bundledProviderVitestCompat ?? env.VITEST === "true"; const entryResult = resolveProviderDiscoveryEntryPlugins({ ...params, env }); - const entryProviders = entryResult.providers.filter(hasProviderCatalogHook); + const entryProviders = entryResult.providers.filter( + (provider) => + hasProviderCatalogHook(provider) || + (params.includeSyntheticAuthProviders === true && + typeof provider.resolveSyntheticAuth === "function"), + ); const runtimeEntryProviders = resolveRuntimeEntryProviders(entryResult); if (params.discoveryEntriesOnly === true) { return entryProviders; diff --git a/src/plugins/provider-discovery.test.ts b/src/plugins/provider-discovery.test.ts index 90b500976bba..534360fd663c 100644 --- a/src/plugins/provider-discovery.test.ts +++ b/src/plugins/provider-discovery.test.ts @@ -380,27 +380,7 @@ describe("runProviderStaticCatalog", () => { }, }; - await expect( - runProviderStaticCatalog({ - provider, - config: { - models: { - providers: { - demo: { - baseUrl: "https://configured.example/v1", - models: [], - apiKey: "secret-value", - }, - }, - }, - }, - agentDir: "/tmp/agent", - workspaceDir: "/tmp/workspace", - env: { - SECRET_TOKEN: "secret-value", - }, - }), - ).resolves.toEqual({ + await expect(runProviderStaticCatalog({ provider })).resolves.toEqual({ provider: { baseUrl: "https://static.example/v1", models: [], diff --git a/src/plugins/provider-discovery.ts b/src/plugins/provider-discovery.ts index 0ed4d50f0a5e..bd26985908a1 100644 --- a/src/plugins/provider-discovery.ts +++ b/src/plugins/provider-discovery.ts @@ -33,6 +33,17 @@ function isSafeProviderConfigKey(value: string): boolean { return value !== "" && !DANGEROUS_PROVIDER_KEYS.has(value); } +type PreparedProviderStaticCatalogEntry = Readonly<{ + provider: ProviderPlugin; + result: Awaited>; +}>; + +export type PreparedProviderStaticCatalog = Readonly<{ + /** Discovery-entry providers captured for this config/workspace generation. */ + providers?: readonly ProviderPlugin[]; + entries: readonly PreparedProviderStaticCatalogEntry[]; +}>; + /** Options for resolving plugin providers that can contribute model catalog entries. */ type ResolveRuntimePluginDiscoveryProvidersParams = { config?: OpenClawConfig; @@ -44,6 +55,7 @@ type ResolveRuntimePluginDiscoveryProvidersParams = { requireCompleteDiscoveryEntryCoverage?: boolean; discoveryEntriesOnly?: boolean; includeManifestModelCatalogProviders?: boolean; + includeSyntheticAuthProviders?: boolean; pluginMetadataSnapshot?: PluginMetadataRegistryView; }; @@ -53,7 +65,12 @@ export async function resolveRuntimePluginDiscoveryProviders( ): Promise { return (await loadProviderRuntime()) .resolvePluginDiscoveryProvidersRuntime(params) - .filter((provider) => resolveProviderCatalogOrderHook(provider)); + .filter( + (provider) => + resolveProviderCatalogOrderHook(provider) || + (params.includeSyntheticAuthProviders === true && + typeof provider.resolveSyntheticAuth === "function"), + ); } /** Groups plugin providers into stable discovery phases for catalog probing. */ @@ -155,13 +172,7 @@ export function runProviderCatalog(params: { }); } -export function runProviderStaticCatalog(params: { - provider: ProviderPlugin; - config: OpenClawConfig; - agentDir?: string; - workspaceDir?: string; - env: NodeJS.ProcessEnv; -}) { +export function runProviderStaticCatalog(params: { provider: ProviderPlugin }) { return params.provider.staticCatalog?.run({ config: {}, env: {}, @@ -175,3 +186,31 @@ export function runProviderStaticCatalog(params: { }), }); } + +/** + * Runs sterile provider catalogs once so lifecycle owners can reuse the immutable results. + * Providers remain attached to their plugin identity for later agent-specific scope filtering. + */ +export async function prepareProviderStaticCatalog(params: { + providers: readonly ProviderPlugin[]; +}): Promise { + const entries: PreparedProviderStaticCatalogEntry[] = []; + const byOrder = groupPluginDiscoveryProvidersByOrder([...params.providers]); + for (const order of DISCOVERY_ORDER) { + for (const provider of byOrder[order]) { + if (!provider.staticCatalog) { + continue; + } + entries.push( + Object.freeze({ + provider, + result: await runProviderStaticCatalog({ provider }), + }), + ); + } + } + return Object.freeze({ + providers: Object.freeze([...params.providers]), + entries: Object.freeze(entries), + }); +} diff --git a/test/scripts/bench-gateway-startup.test.ts b/test/scripts/bench-gateway-startup.test.ts index e53412937145..49fa97524f37 100644 --- a/test/scripts/bench-gateway-startup.test.ts +++ b/test/scripts/bench-gateway-startup.test.ts @@ -176,6 +176,7 @@ describe("gateway startup benchmark script", () => { it("summarizes split ready log timings without the ambiguous readyLogMs field", () => { const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [ { + completionMs: 50, cpuCoreRatio: null, cpuMs: null, exitCode: null, @@ -205,6 +206,7 @@ describe("gateway startup benchmark script", () => { }, ]); + expect(result.summary.completionMs?.p50).toBe(50); expect(result.summary.httpListenLogMs?.p50).toBe(10); expect(result.summary.gatewayReadyLogMs?.p50).toBe(40); expect("readyLogMs" in result.summary).toBe(false); @@ -213,6 +215,7 @@ describe("gateway startup benchmark script", () => { it("flags samples that never produced readiness or process metrics", () => { const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [ { + completionMs: null, cpuCoreRatio: null, cpuMs: null, exitCode: 1, @@ -245,7 +248,7 @@ describe("gateway startup benchmark script", () => { expect(testing.collectResultFailures([result], { processMetricsRequired: true })).toEqual([ { id: "demo", - reason: "missing /healthz, /readyz, cpu, rss", + reason: "missing /healthz, /readyz, completion, cpu, rss", sampleIndex: 1, }, ]); @@ -254,6 +257,7 @@ describe("gateway startup benchmark script", () => { it("flags samples that become ready and then exit nonzero", () => { const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [ { + completionMs: 20, cpuCoreRatio: 0.5, cpuMs: 100, exitedBeforeTeardown: true, @@ -296,6 +300,7 @@ describe("gateway startup benchmark script", () => { it("does not flag nonzero exits from intentional teardown", () => { const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [ { + completionMs: 20, cpuCoreRatio: 0.5, cpuMs: 100, exitedBeforeTeardown: false, @@ -332,6 +337,7 @@ describe("gateway startup benchmark script", () => { it("flags samples that become ready and then die from a signal", () => { const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [ { + completionMs: 20, cpuCoreRatio: 0.5, cpuMs: 100, exitedBeforeTeardown: true, @@ -388,6 +394,35 @@ describe("gateway startup benchmark script", () => { expect(startupTrace["sidecars.acp.runtime-ready.readyCount"]).toBe(1); }); + it("collects prepared runtime grouping counts", () => { + const startupTrace: Record = {}; + + testing.collectStartupTrace( + "[gateway] startup trace: sidecars.model-runtime-build agentCount=12 workspaceGroupCount=2 configuredFactsGroupCount=2 catalogSourceCount=0 credentialGroupCount=1 catalogGroupCount=0 runtimeRegistryCount=2 sourceConcurrencyLimitCount=2 fullCatalogConcurrencyLimitCount=1", + startupTrace, + ); + + expect(startupTrace["sidecars.model-runtime-build.agentCount"]).toBe(12); + expect(startupTrace["sidecars.model-runtime-build.configuredFactsGroupCount"]).toBe(2); + expect(startupTrace["sidecars.model-runtime-build.catalogGroupCount"]).toBe(0); + expect(startupTrace["sidecars.model-runtime-build.runtimeRegistryCount"]).toBe(2); + }); + + it("uses the recorded trace total for completion timing", async () => { + const startedAt = performance.now(); + const completionMs = await testing.waitForStartupTracePhase({ + deadlineAt: startedAt + 1_000, + isDone: () => false, + phase: "sidecars.ready", + startupTrace: { + "sidecars.ready": 20, + "sidecars.ready.total": 50, + }, + }); + + expect(completionMs).toBe(50); + }); + it("keeps counts and memory metrics out of the slow-duration ranking", () => { expect(isStartupTraceDuration("plugins.runtime-post-bind")).toBe(true); expect(isStartupTraceDuration("plugins.gateway-load.loadMs")).toBe(true); @@ -480,6 +515,43 @@ describe("gateway startup benchmark script", () => { } }); + it("builds prepared-runtime scale cases with shared and distinct workspaces", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-config-test-")); + try { + const benchCase = testing.parseOptions(["--case", "preparedRuntimeScaleMany"]).cases[0]; + if (!benchCase) { + throw new Error("expected prepared runtime scale case"); + } + const configPath = testing.writeConfig(root, benchCase); + const config = JSON.parse(fs.readFileSync(configPath, "utf8")) as { + agents?: { list?: Array<{ id: string; workspace: string }> }; + plugins?: { allow?: string[] }; + }; + const agents = config.agents?.list ?? []; + expect(agents).toHaveLength(12); + expect(new Set(agents.slice(0, 11).map((agent) => agent.workspace)).size).toBe(1); + expect(agents[11]?.workspace).not.toBe(agents[0]?.workspace); + const pluginId = config.plugins?.allow?.[0]; + const manifest = JSON.parse( + fs.readFileSync( + path.join(root, "plugins", pluginId ?? "missing", "openclaw.plugin.json"), + "utf8", + ), + ) as { modelCatalog?: unknown; providerCatalogEntry?: string; providers?: string[] }; + expect(manifest.providers).toEqual(["bench-catalog-stall"]); + expect(manifest.providerCatalogEntry).toBe("./provider-discovery.cjs"); + expect(manifest.modelCatalog).toBeUndefined(); + expect( + fs.readFileSync( + path.join(root, "plugins", "bench-plugin-01", "provider-discovery.cjs"), + "utf8", + ), + ).toContain("preparedRuntimeStaticCatalogCallCount"); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + it("keeps startup-lazy plugin fixtures opted out of startup activation", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-config-test-")); try {