From 01e888795978ede45f9cbeaf6d8bfc4bbbb9987d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 21 Aug 2026 11:40:52 -0700 Subject: [PATCH] refactor(providers): return prepared dynamic models directly (#126574) --- docs/plugins/architecture-internals.md | 2 +- docs/plugins/sdk-provider-plugins.md | 8 +- extensions/llama-cpp/index.test.ts | 24 ++ .../src/external-server/provider.test.ts | 221 +++++++++--------- .../llama-cpp/src/external-server/provider.ts | 77 ++---- extensions/llama-cpp/src/managed-provider.ts | 12 +- extensions/lmstudio/api.ts | 2 +- extensions/lmstudio/index.test.ts | 114 ++++++++- extensions/lmstudio/index.ts | 12 +- extensions/lmstudio/src/api.ts | 2 +- extensions/lmstudio/src/setup.ts | 27 ++- extensions/ollama/index.test.ts | 103 ++++---- extensions/ollama/index.ts | 130 ++--------- .../model.provider-hooks.ts | 2 +- .../model.registry-resolution.ts | 52 ++--- .../embedded-agent-runner/model.test.ts | 121 ++++++++++ src/agents/embedded-agent-runner/model.ts | 3 +- .../live-model-dynamic-candidates.test.ts | 21 ++ .../live-model-dynamic-candidates.ts | 22 +- src/plugins/provider-plugin.types.ts | 14 +- src/plugins/provider-runtime.test.ts | 14 +- src/plugins/provider-runtime.ts | 4 +- src/plugins/provider-runtime.types.ts | 7 +- 23 files changed, 562 insertions(+), 432 deletions(-) diff --git a/docs/plugins/architecture-internals.md b/docs/plugins/architecture-internals.md index 343c73dd91e5..7e86b927e3a0 100644 --- a/docs/plugins/architecture-internals.md +++ b/docs/plugins/architecture-internals.md @@ -272,7 +272,7 @@ listed here. | `resolveExternalAuthProfiles` | Overlay provider-owned external auth profiles; default `persistence` is `runtime-only` for CLI/app-owned creds | Provider reuses external auth credentials without persisting copied refresh tokens; declare `contracts.externalAuthProviders` in the manifest | | `shouldDeferSyntheticProfileAuth` | Lower stored synthetic profile placeholders behind env/config-backed auth | Provider stores synthetic placeholder profiles that should not win precedence | | `resolveDynamicModel` | Sync fallback for provider-owned model ids not in the local registry yet | Provider accepts arbitrary upstream model ids | -| `prepareDynamicModel` | Async warm-up, then `resolveDynamicModel` runs again | Provider needs network metadata before resolving unknown ids | +| `prepareDynamicModel` | Return an asynchronously prepared model, or warm reusable metadata before retrying `resolveDynamicModel` | Provider needs network metadata before resolving unknown ids | | `normalizeResolvedModel` | Final rewrite before the embedded runner uses the resolved model | Provider needs transport rewrites but still uses a core transport | | `normalizeToolSchemas` | Normalize tool schemas before the embedded runner sees them | Provider needs transport-family schema cleanup | | `inspectToolSchemas` | Surface provider-owned schema diagnostics after normalization | Provider wants keyword warnings without teaching core provider-specific rules | diff --git a/docs/plugins/sdk-provider-plugins.md b/docs/plugins/sdk-provider-plugins.md index 9b24294591cd..eda32122a4c0 100644 --- a/docs/plugins/sdk-provider-plugins.md +++ b/docs/plugins/sdk-provider-plugins.md @@ -485,8 +485,10 @@ catalog, API-key auth, and dynamic model resolution. }); ``` - If resolving requires a network call, use `prepareDynamicModel` for async - warm-up - `resolveDynamicModel` runs again after it completes. + If resolving requires a network call, return the requested model directly + from `prepareDynamicModel`. OpenClaw applies the same configured overrides + and normalization as synchronous dynamic resolution. Existing hooks that + return nothing still retry `resolveDynamicModel` after preparation. @@ -683,7 +685,7 @@ catalog, API-key auth, and dynamic model resolution. | `resolveExternalAuthProfiles` | Overlay provider-owned external auth profiles for CLI/app-managed credentials | | `shouldDeferSyntheticProfileAuth` | Lower synthetic stored-profile placeholders behind env/config auth | | `resolveDynamicModel` | Accept arbitrary upstream model IDs | - | `prepareDynamicModel` | Async metadata fetch before resolving | + | `prepareDynamicModel` | Return an asynchronously discovered model, or warm reusable metadata before sync resolution | | `normalizeResolvedModel` | Transport rewrites before the runner | | `normalizeToolSchemas` | Provider-owned tool-schema cleanup before registration | | `inspectToolSchemas` | Provider-owned tool-schema diagnostics | diff --git a/extensions/llama-cpp/index.test.ts b/extensions/llama-cpp/index.test.ts index 032401eb003b..3d9dac91f60c 100644 --- a/extensions/llama-cpp/index.test.ts +++ b/extensions/llama-cpp/index.test.ts @@ -18,6 +18,7 @@ import type { ProviderPlugin } from "openclaw/plugin-sdk/provider-model-shared"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ + discoverServer: vi.fn(), ensureModel: vi.fn(), prepareServer: vi.fn(), inspectRuntime: vi.fn(), @@ -36,6 +37,11 @@ vi.mock("./src/managed-server.js", async (importOriginal) => ({ inspectLlamaServerRuntime: mocks.inspectRuntime, })); +vi.mock("./src/external-server/discovery.js", async (importOriginal) => ({ + ...(await importOriginal()), + discoverLlamaServer: mocks.discoverServer, +})); + import llamaCppPlugin from "./index.js"; import { DEFAULT_LLAMA_CPP_EMBEDDING_CACHE_FILE, @@ -52,6 +58,7 @@ let previousPluginRegistry: ReturnType; beforeEach(() => { previousPluginRegistry = getActivePluginRegistry(); + mocks.discoverServer.mockReset(); mocks.ensureModel.mockResolvedValue("/models/model.gguf"); mocks.prepareServer.mockResolvedValue({}); mocks.inspectRuntime.mockResolvedValue({ @@ -171,6 +178,23 @@ describe("llama.cpp provider plugin", () => { expect(provider).not.toHaveProperty("createStreamFn"); }); + it("never discovers external models for a managed local service", async () => { + const provider = registerTextProvider(); + const prepareDynamicModel = expectDefined(provider.prepareDynamicModel, "dynamic model hook"); + const { config } = configuredOptions(); + + await expect( + prepareDynamicModel({ + config, + provider: LLAMA_CPP_PROVIDER_ID, + modelId: "gemma-4-e4b-it-q4_k_m", + modelRegistry: {} as never, + providerConfig: config.models.providers[LLAMA_CPP_PROVIDER_ID], + }), + ).resolves.toBeUndefined(); + expect(mocks.discoverServer).not.toHaveBeenCalled(); + }); + it("registers local embeddings through the generic provider contract", () => { const { config, registry } = createPluginRegistryFixture(); registerVirtualTestPlugin({ diff --git a/extensions/llama-cpp/src/external-server/provider.test.ts b/extensions/llama-cpp/src/external-server/provider.test.ts index e0cd543f0c64..43e0a79db26f 100644 --- a/extensions/llama-cpp/src/external-server/provider.test.ts +++ b/extensions/llama-cpp/src/external-server/provider.test.ts @@ -3,11 +3,7 @@ import type { ProviderPrepareDynamicModelContext, } from "openclaw/plugin-sdk/plugin-entry"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { - discoverLlamaServerProvider, - prepareLlamaServerDynamicModels, - resolveLlamaServerDynamicModel, -} from "./provider.js"; +import { discoverLlamaServerProvider, prepareLlamaServerDynamicModel } from "./provider.js"; const discoverMock = vi.hoisted(() => vi.fn()); const runtimeApiKeyMock = vi.hoisted(() => vi.fn()); @@ -64,6 +60,22 @@ function catalogContext(): ProviderCatalogContext { }; } +function dynamicContext( + overrides: Partial = {}, +): ProviderPrepareDynamicModelContext { + return { + config: {}, + provider: "llama-cpp", + modelId: "org/model:Q4", + modelRegistry: {} as never, + providerConfig: { + baseUrl: "http://localhost:8080/v1", + api: "openai-completions", + }, + ...overrides, + }; +} + describe("llama-server provider discovery", () => { beforeEach(() => { discoverMock.mockReset(); @@ -131,7 +143,29 @@ describe("llama-server provider discovery", () => { }); }); - it("scopes dynamic catalogs by agent runtime and auth profile", async () => { + it("returns only the requested discovered model directly to its preparation owner", async () => { + discoverMock.mockResolvedValue({ + ...success(), + models: [ + model(), + { + ...model(), + config: { ...model().config, id: "org/requested:Q8", name: "Requested model" }, + }, + ], + }); + await expect( + prepareLlamaServerDynamicModel(dynamicContext({ modelId: "org/requested:Q8" })), + ).resolves.toMatchObject({ + provider: "llama-cpp", + id: "org/requested:Q8", + name: "Requested model", + baseUrl: "http://localhost:8080/v1", + api: "openai-completions", + }); + }); + + it("keeps requested models and API keys isolated by agent runtime and auth profile", async () => { const first = success(); const second = { ...success(), @@ -143,29 +177,20 @@ describe("llama-server provider discovery", () => { ], }; discoverMock.mockResolvedValueOnce(first).mockResolvedValueOnce(second); - const base = { - config: {}, - provider: "llama-cpp", - modelId: "org/model:Q4", - modelRegistry: {}, - providerConfig: { - baseUrl: "http://localhost:8080/v1", - api: "openai-completions", - }, - }; - const firstCtx = { - ...base, + runtimeApiKeyMock + .mockResolvedValueOnce("first-profile-key") + .mockResolvedValueOnce("second-profile-key"); + const firstCtx = dynamicContext({ agentRuntimeId: "runtime-one", authProfileId: "profile-one", - } as unknown as ProviderPrepareDynamicModelContext; - const secondCtx = { - ...base, + }); + const secondCtx = dynamicContext({ agentRuntimeId: "runtime-two", authProfileId: "profile-two", - } as unknown as ProviderPrepareDynamicModelContext; + }); - await prepareLlamaServerDynamicModels(firstCtx); - await prepareLlamaServerDynamicModels(secondCtx); + const firstModel = await prepareLlamaServerDynamicModel(firstCtx); + const secondModel = await prepareLlamaServerDynamicModel(secondCtx); expect(runtimeApiKeyMock).toHaveBeenNthCalledWith( 1, @@ -175,111 +200,87 @@ describe("llama-server provider discovery", () => { 2, expect.objectContaining({ profileId: "profile-two" }), ); - expect(resolveLlamaServerDynamicModel(firstCtx)?.name).toBe("org/model:Q4"); - expect(resolveLlamaServerDynamicModel(secondCtx)?.name).toBe("second scope"); - }); - - it("bounds dynamic model snapshots by scope", async () => { - discoverMock.mockResolvedValue(success()); - const contexts = Array.from( - { length: 101 }, - (_, index) => - ({ - config: {}, - provider: "llama-cpp", - modelId: "org/model:Q4", - modelRegistry: {}, - agentRuntimeId: `runtime-${index}`, - providerConfig: { - baseUrl: "http://localhost:8080/v1", - api: "openai-completions", - }, - }) as unknown as ProviderPrepareDynamicModelContext, + expect(discoverMock).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ apiKey: "first-profile-key" }), ); - - for (const ctx of contexts) { - await prepareLlamaServerDynamicModels(ctx); - } - - expect(resolveLlamaServerDynamicModel(contexts[0]!)).toBeUndefined(); - expect(resolveLlamaServerDynamicModel(contexts.at(-1)!)).toMatchObject({ - id: "org/model:Q4", - }); + expect(discoverMock).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ apiKey: "second-profile-key" }), + ); + expect(firstModel?.name).toBe("org/model:Q4"); + expect(secondModel?.name).toBe("second scope"); }); - it("keeps dynamic snapshots separate when only the endpoint changes", async () => { + it("keeps requested models separate when only the endpoint changes", async () => { discoverMock.mockResolvedValueOnce(success()).mockResolvedValueOnce({ ...success(), models: [{ ...model(), config: { ...model().config, name: "second endpoint" } }], }); const base = { - config: {}, - provider: "llama-cpp", - modelId: "org/model:Q4", - modelRegistry: {}, agentRuntimeId: "endpoint-runtime", authProfileId: "endpoint-profile", }; - const first = { + const first = dynamicContext({ ...base, providerConfig: { baseUrl: "http://localhost:8080/v1", api: "openai-completions" }, - } as unknown as ProviderPrepareDynamicModelContext; - const second = { + }); + const second = dynamicContext({ ...base, providerConfig: { baseUrl: "http://localhost:8081/v1", api: "openai-completions" }, - } as unknown as ProviderPrepareDynamicModelContext; - - await prepareLlamaServerDynamicModels(first); - await prepareLlamaServerDynamicModels(second); - - expect(resolveLlamaServerDynamicModel(first)?.name).toBe("org/model:Q4"); - expect(resolveLlamaServerDynamicModel(second)?.name).toBe("second endpoint"); - }); - - it("clears a scope snapshot when its refresh cannot discover the server", async () => { - discoverMock.mockResolvedValueOnce(success()).mockResolvedValueOnce({ - kind: "unreachable", - endpoint: { origin: "http://localhost:8080", inferenceBaseUrl: "http://localhost:8080/v1" }, - error: new Error("offline"), }); - const ctx = { - config: {}, - provider: "llama-cpp", - modelId: "org/model:Q4", - modelRegistry: {}, - agentRuntimeId: "failed-refresh-runtime", - providerConfig: { - baseUrl: "http://localhost:8080/v1", - api: "openai-completions", - }, - } as unknown as ProviderPrepareDynamicModelContext; - await prepareLlamaServerDynamicModels(ctx); - expect(resolveLlamaServerDynamicModel(ctx)).toMatchObject({ id: "org/model:Q4" }); - await prepareLlamaServerDynamicModels(ctx); - expect(resolveLlamaServerDynamicModel(ctx)).toBeUndefined(); - }); + const firstModel = await prepareLlamaServerDynamicModel(first); + const secondModel = await prepareLlamaServerDynamicModel(second); - it("refreshes and resolves dynamic model ids containing slashes", async () => { - discoverMock.mockResolvedValue(success()); - const ctx = { - config: {}, - provider: "llama-cpp", - modelId: "org/model:Q4", - modelRegistry: {}, - providerConfig: { - baseUrl: "http://localhost:8080/v1", - api: "openai-completions", - }, - } as unknown as ProviderPrepareDynamicModelContext; - - await prepareLlamaServerDynamicModels(ctx); - - expect(resolveLlamaServerDynamicModel(ctx)).toMatchObject({ - provider: "llama-cpp", - id: "org/model:Q4", + expect(firstModel).toMatchObject({ + name: "org/model:Q4", baseUrl: "http://localhost:8080/v1", - api: "openai-completions", }); + expect(secondModel).toMatchObject({ + name: "second endpoint", + baseUrl: "http://localhost:8081/v1", + }); + }); + + it("prefers explicit Authorization over the profile API key during model preparation", async () => { + runtimeApiKeyMock.mockResolvedValue("profile-key"); + discoverMock.mockResolvedValue(success()); + const headers = { Authorization: "Bearer endpoint-key" }; + + await prepareLlamaServerDynamicModel( + dynamicContext({ + providerConfig: { baseUrl: "http://localhost:8080/v1", headers }, + }), + ); + + expect(discoverMock).toHaveBeenCalledWith( + expect.objectContaining({ apiKey: undefined, headers, cacheTtlMs: 0 }), + ); + }); + + it.each([ + { + label: "the server is unavailable", + discovery: { + kind: "unreachable" as const, + endpoint: { origin: "http://localhost:8080", inferenceBaseUrl: "http://localhost:8080/v1" }, + error: new Error("offline"), + }, + }, + { + label: "the requested model disappears", + discovery: { ...success(), models: [] }, + }, + ])("returns no stale model when $label", async ({ discovery }) => { + discoverMock.mockResolvedValueOnce(success()).mockResolvedValueOnce(discovery); + const ctx = dynamicContext({ + agentRuntimeId: "failed-refresh-runtime", + }); + + await expect(prepareLlamaServerDynamicModel(ctx)).resolves.toMatchObject({ + id: "org/model:Q4", + }); + await expect(prepareLlamaServerDynamicModel(ctx)).resolves.toBeUndefined(); }); }); diff --git a/extensions/llama-cpp/src/external-server/provider.ts b/extensions/llama-cpp/src/external-server/provider.ts index 2138214413a2..080d9eb2e73f 100644 --- a/extensions/llama-cpp/src/external-server/provider.ts +++ b/extensions/llama-cpp/src/external-server/provider.ts @@ -1,8 +1,6 @@ -import { pruneMapToMaxSize } from "openclaw/plugin-sdk/collection-runtime"; import type { ProviderCatalogContext, ProviderPrepareDynamicModelContext, - ProviderResolveDynamicModelContext, ProviderRuntimeModel, } from "openclaw/plugin-sdk/plugin-entry"; import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared"; @@ -14,47 +12,7 @@ import { } from "./auth.js"; import { discoverLlamaServer } from "./discovery.js"; import { resolveLlamaServerEndpoint } from "./endpoint.js"; -import { buildLlamaServerProviderConfig, type LlamaServerDiscoveredModel } from "./models.js"; - -const dynamicModels = new Map(); -const LLAMA_SERVER_DYNAMIC_MODEL_MAX_SCOPES = 100; - -function cacheDynamicModels(key: string, models: ProviderRuntimeModel[]): void { - dynamicModels.delete(key); - dynamicModels.set(key, models); - pruneMapToMaxSize(dynamicModels, LLAMA_SERVER_DYNAMIC_MODEL_MAX_SCOPES); -} - -function dynamicModelScopeKey( - ctx: Pick< - ProviderResolveDynamicModelContext, - "agentRuntimeId" | "agentDir" | "authProfileId" | "providerConfig" - >, -): string { - return [ - ctx.agentRuntimeId ?? ctx.agentDir ?? "", - ctx.authProfileId ?? "", - ctx.providerConfig?.baseUrl ?? "", - ].join("\u0000"); -} - -function toRuntimeModel( - model: LlamaServerDiscoveredModel, - providerConfig: { - baseUrl?: string; - api?: ProviderRuntimeModel["api"]; - }, -): ProviderRuntimeModel { - return { - ...model.config, - provider: LLAMA_CPP_PROVIDER_ID, - api: providerConfig.api ?? "openai-completions", - baseUrl: resolveLlamaServerEndpoint(providerConfig.baseUrl).inferenceBaseUrl, - input: model.config.input.filter( - (entry): entry is "text" | "image" => entry === "text" || entry === "image", - ), - }; -} +import { buildLlamaServerProviderConfig } from "./models.js"; /** Discovers external llama-server models for provider runtime resolution. */ export async function discoverLlamaServerProvider( @@ -96,9 +54,9 @@ export async function discoverLlamaServerProvider( }; } -export async function prepareLlamaServerDynamicModels( +export async function prepareLlamaServerDynamicModel( ctx: ProviderPrepareDynamicModelContext, -): Promise { +): Promise { const apiKey = await resolveLlamaServerRuntimeApiKey({ config: ctx.config, agentDir: ctx.agentDir, @@ -115,19 +73,20 @@ export async function prepareLlamaServerDynamicModels( headers, cacheTtlMs: 0, }); - const key = dynamicModelScopeKey(ctx); - cacheDynamicModels( - key, + const model = discovery.kind === "success" - ? discovery.models.map((model) => toRuntimeModel(model, ctx.providerConfig ?? {})) - : [], - ); -} - -export function resolveLlamaServerDynamicModel( - params: ProviderResolveDynamicModelContext, -): ProviderRuntimeModel | undefined { - return dynamicModels - .get(dynamicModelScopeKey(params)) - ?.find((model) => model.id === params.modelId); + ? discovery.models.find((entry) => entry.config.id === ctx.modelId) + : undefined; + if (!model) { + return undefined; + } + return { + ...model.config, + provider: LLAMA_CPP_PROVIDER_ID, + api: ctx.providerConfig?.api ?? "openai-completions", + baseUrl: resolveLlamaServerEndpoint(ctx.providerConfig?.baseUrl).inferenceBaseUrl, + input: model.config.input.filter( + (entry): entry is "text" | "image" => entry === "text" || entry === "image", + ), + }; } diff --git a/extensions/llama-cpp/src/managed-provider.ts b/extensions/llama-cpp/src/managed-provider.ts index 607348df4e2d..0b3f76b67614 100644 --- a/extensions/llama-cpp/src/managed-provider.ts +++ b/extensions/llama-cpp/src/managed-provider.ts @@ -21,8 +21,7 @@ import { import { normalizeLlamaServerProviderConfig } from "./external-server/endpoint.js"; import { discoverLlamaServerProvider, - prepareLlamaServerDynamicModels, - resolveLlamaServerDynamicModel, + prepareLlamaServerDynamicModel, } from "./external-server/provider.js"; import { configureLlamaServerNonInteractive, @@ -119,15 +118,10 @@ export function registerLlamaCppProvider(api: OpenClawPluginApi): void { providerConfig.localService ? providerConfig : normalizeLlamaServerProviderConfig(providerConfig), - prepareDynamicModel: async (ctx) => { - if (!ctx.config?.models?.providers?.[LLAMA_CPP_PROVIDER_ID]?.localService) { - await prepareLlamaServerDynamicModels(ctx); - } - }, - resolveDynamicModel: (ctx) => + prepareDynamicModel: async (ctx) => ctx.config?.models?.providers?.[LLAMA_CPP_PROVIDER_ID]?.localService ? undefined - : resolveLlamaServerDynamicModel(ctx), + : await prepareLlamaServerDynamicModel(ctx), wrapStreamFn: (ctx) => { const providerConfig = ctx.config?.models?.providers?.[LLAMA_CPP_PROVIDER_ID]; if (!providerConfig?.localService) { diff --git a/extensions/lmstudio/api.ts b/extensions/lmstudio/api.ts index 4b332008df89..ce2cd4d9b5fd 100644 --- a/extensions/lmstudio/api.ts +++ b/extensions/lmstudio/api.ts @@ -25,7 +25,7 @@ export { normalizeLmstudioConfiguredCatalogEntry, normalizeLmstudioProviderConfig, prepareAppGuidedLmstudioSetup, - prepareLmstudioDynamicModels, + prepareLmstudioDynamicModel, promptAndConfigureLmstudioInteractive, resolveLmstudioConfiguredApiKey, resolveLmstudioInferenceBase, diff --git a/extensions/lmstudio/index.test.ts b/extensions/lmstudio/index.test.ts index 2cb99c2d52ea..71fa72c9722a 100644 --- a/extensions/lmstudio/index.test.ts +++ b/extensions/lmstudio/index.test.ts @@ -1,17 +1,29 @@ // Lmstudio tests cover index plugin behavior. -import type { OpenClawConfig, ProviderAuthMethod } from "openclaw/plugin-sdk/plugin-entry"; +import { createDeferred } from "openclaw/plugin-sdk/extension-shared"; +import type { + OpenClawConfig, + ProviderAuthMethod, + ProviderPrepareDynamicModelContext, +} from "openclaw/plugin-sdk/plugin-entry"; import { capturePluginRegistration } from "openclaw/plugin-sdk/plugin-test-runtime"; import { CUSTOM_LOCAL_AUTH_MARKER } from "openclaw/plugin-sdk/provider-auth"; -import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared"; +import type { + ModelDefinitionConfig, + ModelProviderConfig, +} from "openclaw/plugin-sdk/provider-model-shared"; import { beforeEach, describe, expect, it, vi } from "vitest"; import plugin from "./index.js"; import { LMSTUDIO_LOCAL_API_KEY_PLACEHOLDER } from "./src/defaults.js"; const fetchLmstudioModelsMock = vi.hoisted(() => vi.fn()); +const discoverLmstudioModelsMock = vi.hoisted(() => + vi.fn(), +); vi.mock("./src/models.fetch.js", async (importOriginal) => ({ ...(await importOriginal()), fetchLmstudioModels: fetchLmstudioModelsMock, + discoverLmstudioModels: discoverLmstudioModelsMock, })); function registerProvider() { @@ -83,9 +95,42 @@ function createRemoteProviderConfig(overrides?: Partial): M }; } +function createDynamicModelContext( + profile: "first" | "second", +): ProviderPrepareDynamicModelContext { + return { + provider: "lmstudio", + modelId: "shared-model", + modelRegistry: { + getAll: () => [], + getAvailable: () => [], + find: () => undefined, + hasConfiguredAuth: () => false, + }, + authProfileId: `lmstudio:${profile}`, + providerConfig: { + baseUrl: "http://lmstudio.internal:1234/v1", + headers: { Authorization: `Bearer ${profile}-profile` }, + }, + }; +} + +function createDiscoveredModel(name: string, contextWindow: number): ModelDefinitionConfig { + return { + id: "shared-model", + name, + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow, + maxTokens: 8192, + }; +} + describe("lmstudio plugin", () => { beforeEach(() => { fetchLmstudioModelsMock.mockReset(); + discoverLmstudioModelsMock.mockReset(); }); it("registers llama.cpp GBNF tool-schema projection", () => { @@ -95,6 +140,71 @@ describe("lmstudio plugin", () => { }); }); + it("keeps concurrent model preparations isolated when shared-endpoint profiles finish in reverse order", async () => { + const provider = registerProvider(); + const prepareDynamicModel = provider.prepareDynamicModel; + if (!prepareDynamicModel) { + throw new Error("expected the LM Studio provider to prepare dynamic models"); + } + const firstDiscovery = createDeferred(); + const secondDiscovery = createDeferred(); + discoverLmstudioModelsMock.mockImplementation(({ headers }) => + headers?.Authorization === "Bearer first-profile" + ? firstDiscovery.promise + : secondDiscovery.promise, + ); + + const firstPreparation = prepareDynamicModel(createDynamicModelContext("first")); + const secondPreparation = prepareDynamicModel(createDynamicModelContext("second")); + await vi.waitFor(() => expect(discoverLmstudioModelsMock).toHaveBeenCalledTimes(2)); + + secondDiscovery.resolve([createDiscoveredModel("Second profile model", 65_536)]); + const secondPrepared = await secondPreparation; + firstDiscovery.resolve([createDiscoveredModel("First profile model", 32_768)]); + const firstPrepared = await firstPreparation; + + expect(secondPrepared).toMatchObject({ + id: "shared-model", + name: "Second profile model", + contextWindow: 65_536, + }); + expect(firstPrepared).toMatchObject({ + id: "shared-model", + name: "First profile model", + contextWindow: 32_768, + }); + }); + + it("returns only the requested discovered model without retaining stale endpoint results", async () => { + const provider = registerProvider(); + const prepareDynamicModel = provider.prepareDynamicModel; + if (!prepareDynamicModel) { + throw new Error("expected the LM Studio provider to prepare dynamic models"); + } + const otherModel = { ...createDiscoveredModel("Other model", 16_384), id: "other-model" }; + const requestedModel = createDiscoveredModel("Requested model", 32_768); + discoverLmstudioModelsMock + .mockResolvedValueOnce([otherModel, requestedModel]) + .mockResolvedValueOnce([otherModel]); + + const context = createDynamicModelContext("first"); + await expect(prepareDynamicModel(context)).resolves.toMatchObject({ + id: "shared-model", + name: "Requested model", + provider: "lmstudio", + api: "openai-completions", + baseUrl: "http://lmstudio.internal:1234/v1", + contextWindow: 32_768, + }); + await expect(prepareDynamicModel(context)).resolves.toBeUndefined(); + expect(discoverLmstudioModelsMock).toHaveBeenCalledWith({ + baseUrl: "http://lmstudio.internal:1234/v1", + apiKey: "", + headers: { Authorization: "Bearer first-profile" }, + quiet: true, + }); + }); + it("preflights the requested LM Studio model before destructive non-interactive reset", async () => { fetchLmstudioModelsMock.mockResolvedValue({ reachable: true, diff --git a/extensions/lmstudio/index.ts b/extensions/lmstudio/index.ts index 5590607b7414..dd21fac56fa6 100644 --- a/extensions/lmstudio/index.ts +++ b/extensions/lmstudio/index.ts @@ -8,7 +8,6 @@ import { type ProviderAuthMethod, type ProviderAuthMethodNonInteractiveContext, type ProviderAuthResult, - type ProviderRuntimeModel, } from "openclaw/plugin-sdk/plugin-entry"; import { CUSTOM_LOCAL_AUTH_MARKER, @@ -33,8 +32,6 @@ import { shouldUseLmstudioSyntheticAuth } from "./src/provider-auth.js"; import { wrapLmstudioInferencePreload } from "./src/stream.js"; const PROVIDER_ID = "lmstudio"; -// Intentional: dynamic models are cached per LM Studio endpoint (`baseUrl`) only. -const cachedDynamicModels = new Map(); type LmstudioNonInteractiveValidationContext = Parameters< NonNullable @@ -233,15 +230,8 @@ export default definePluginEntry({ normalizeConfig: ({ providerConfig }) => normalizeLmstudioProviderConfig(providerConfig), prepareDynamicModel: async (ctx) => { const providerSetup = await loadProviderSetup(); - cachedDynamicModels.set( - ctx.providerConfig?.baseUrl ?? "", - await providerSetup.prepareLmstudioDynamicModels(ctx), - ); + return await providerSetup.prepareLmstudioDynamicModel(ctx); }, - resolveDynamicModel: (ctx) => - cachedDynamicModels - .get(ctx.providerConfig?.baseUrl ?? "") - ?.find((model) => model.id === ctx.modelId), augmentModelCatalog: (ctx) => resolveLmstudioAugmentedCatalogEntries(ctx.config), wrapStreamFn: wrapLmstudioInferencePreload, ...buildProviderToolCompatFamilyHooks("llamacpp-gbnf"), diff --git a/extensions/lmstudio/src/api.ts b/extensions/lmstudio/src/api.ts index 730801c24bf4..e601b34351ba 100644 --- a/extensions/lmstudio/src/api.ts +++ b/extensions/lmstudio/src/api.ts @@ -40,6 +40,6 @@ export { detectAppGuidedLmstudioAvailability, discoverLmstudioProvider, prepareAppGuidedLmstudioSetup, - prepareLmstudioDynamicModels, + prepareLmstudioDynamicModel, promptAndConfigureLmstudioInteractive, } from "./setup.js"; diff --git a/extensions/lmstudio/src/setup.ts b/extensions/lmstudio/src/setup.ts index 97694a40a82c..4dd8dcbc53ca 100644 --- a/extensions/lmstudio/src/setup.ts +++ b/extensions/lmstudio/src/setup.ts @@ -1046,9 +1046,9 @@ export async function discoverLmstudioProvider(ctx: ProviderCatalogContext): Pro }; } -export async function prepareLmstudioDynamicModels( +export async function prepareLmstudioDynamicModel( ctx: ProviderPrepareDynamicModelContext, -): Promise { +): Promise { const baseUrl = resolveLmstudioInferenceBase(ctx.providerConfig?.baseUrl); const { apiKey, headers } = await resolveLmstudioRequestContext({ config: ctx.config, @@ -1062,15 +1062,18 @@ export async function prepareLmstudioDynamicModels( headers, quiet: true, }); - return discoveredModels.map((model) => - Object.assign({}, model, { - provider: PROVIDER_ID, - api: ctx.providerConfig?.api ?? `openai-completions`, - baseUrl, - input: model.input.filter( - (entry): entry is "text" | "image" => entry === "text" || entry === "image", - ), - }), - ); + const model = discoveredModels.find((candidate) => candidate.id === ctx.modelId); + if (!model) { + return undefined; + } + return { + ...model, + provider: PROVIDER_ID, + api: ctx.providerConfig?.api ?? "openai-completions", + baseUrl, + input: model.input.filter( + (entry): entry is "text" | "image" => entry === "text" || entry === "image", + ), + }; } /* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/extensions/ollama/index.test.ts b/extensions/ollama/index.test.ts index 0a20239cf55b..e9175aef0bab 100644 --- a/extensions/ollama/index.test.ts +++ b/extensions/ollama/index.test.ts @@ -1084,9 +1084,7 @@ describe("ollama plugin", () => { const context = createDynamicModelContext("llama3.2:latest"); try { - await provider.prepareDynamicModel?.(context as never); - - const resolved = provider.resolveDynamicModel?.(context as never); + const resolved = await provider.prepareDynamicModel?.(context as never); expect(resolved?.provider).toBe("ollama"); expect(resolved?.id).toBe("llama3.2:latest"); expect(resolved?.api).toBe("ollama"); @@ -1134,9 +1132,7 @@ describe("ollama plugin", () => { }; const context = createDynamicModelContext("qwen3-coder:cloud", config); - await provider.prepareDynamicModel?.(context as never); - - const resolved = provider.resolveDynamicModel?.(context as never); + const resolved = await provider.prepareDynamicModel?.(context as never); expect(resolved?.provider).toBe("ollama"); expect(resolved?.id).toBe("qwen3-coder:cloud"); expect(resolved?.api).toBe("openai-completions"); @@ -1172,7 +1168,7 @@ describe("ollama plugin", () => { mockDiscoveredOllamaProvider([], { baseUrl, once: true }); const context = createDynamicModelContext("private-dynamic-model", config); - await provider.prepareDynamicModel?.(context as never); + const resolved = await provider.prepareDynamicModel?.(context as never); expect(buildOllamaProviderMock).toHaveBeenCalledWith(baseUrl, { quiet: true, @@ -1181,10 +1177,10 @@ describe("ollama plugin", () => { expect(queryOllamaModelShowInfoMock).toHaveBeenCalledWith(baseUrl, "private-dynamic-model", { apiKey: "dynamic-discovery-access", }); - expect(provider.resolveDynamicModel?.(context as never)?.id).toBe("private-dynamic-model"); + expect(resolved?.id).toBe("private-dynamic-model"); }); - it("scopes dynamic Ollama model caches to the effective credential", async () => { + it("returns the exact prepared Ollama model for concurrent credential profiles", async () => { const provider = registerProvider(); const baseUrl = "https://shared-dynamic-ollama.example.com"; const modelId = "tenant-dynamic-model"; @@ -1197,25 +1193,34 @@ describe("ollama plugin", () => { }); const discoveredFor = (name: string) => ({ baseUrl, - api: "ollama", + api: "ollama" as const, models: [{ id: modelId, name, contextWindow: 8192, maxTokens: 2048 }], }); - buildOllamaProviderMock - .mockResolvedValueOnce(discoveredFor("First tenant model")) - .mockResolvedValueOnce(discoveredFor("Second tenant model")); + const completeDiscovery: Array<(result: ReturnType) => void> = []; + buildOllamaProviderMock.mockImplementation( + () => + new Promise>((resolve) => { + completeDiscovery.push(resolve); + }), + ); - for (const config of [configFor("first-tenant-access"), configFor("second-tenant-access")]) { - await provider.prepareDynamicModel?.(createDynamicModelContext(modelId, config) as never); - } + const prepareFor = (apiKey: string, authProfileId: string) => + provider.prepareDynamicModel?.({ + ...createDynamicModelContext(modelId, configFor(apiKey)), + authProfileId, + } as never); + const firstPrepared = prepareFor("first-tenant-access", "ollama:first"); + const secondPrepared = prepareFor("second-tenant-access", "ollama:second"); - const resolveFor = (apiKey: string) => - provider.resolveDynamicModel?.( - createDynamicModelContext(modelId, configFor(apiKey)) as never, - ); + await vi.waitFor(() => expect(buildOllamaProviderMock).toHaveBeenCalledTimes(2)); + completeDiscovery[1]?.(discoveredFor("Second tenant model")); + await expect(secondPrepared).resolves.toMatchObject({ + id: modelId, + name: "Second tenant model", + }); + completeDiscovery[0]?.(discoveredFor("First tenant model")); + await expect(firstPrepared).resolves.toMatchObject({ id: modelId, name: "First tenant model" }); - expect(resolveFor("first-tenant-access")?.name).toBe("First tenant model"); - expect(resolveFor("second-tenant-access")?.name).toBe("Second tenant model"); - expect(resolveFor("unprepared-tenant-access")).toBeUndefined(); expect(buildOllamaProviderMock).toHaveBeenNthCalledWith(1, baseUrl, { quiet: true, apiKey: "first-tenant-access", @@ -1250,8 +1255,9 @@ describe("ollama plugin", () => { const context = createDynamicModelContext("secretref-dynamic-model", config); try { - await provider.prepareDynamicModel?.(context as never); + const resolved = await provider.prepareDynamicModel?.(context as never); + expect(resolved?.id).toBe("secretref-dynamic-model"); expect(buildOllamaProviderMock).toHaveBeenCalledWith(baseUrl, { quiet: true, apiKey: secretValue, @@ -1290,7 +1296,7 @@ describe("ollama plugin", () => { }); try { - await provider.prepareDynamicModel?.(context as never); + await expect(provider.prepareDynamicModel?.(context as never)).resolves.toBeUndefined(); expect(buildOllamaProviderMock).not.toHaveBeenCalled(); expect(queryOllamaModelShowInfoMock).not.toHaveBeenCalled(); @@ -1301,7 +1307,7 @@ describe("ollama plugin", () => { } }); - it("invalidates managed dynamic model caches when their SecretRef stops resolving", async () => { + it("keeps rotated managed SecretRefs request-owned and fails closed when unavailable", async () => { const provider = registerProvider(); const baseUrl = "https://managed-dynamic-ollama.example.com"; const modelId = "managed-private-model"; @@ -1319,20 +1325,32 @@ describe("ollama plugin", () => { }; resolveConfiguredSecretInputStringMock .mockResolvedValueOnce({ value: "managed-dynamic-access" }) + .mockResolvedValueOnce({ value: "rotated-managed-access" }) .mockResolvedValueOnce({ unresolvedRefReason: "managed credential is unavailable" }); mockDiscoveredOllamaProvider( [{ id: modelId, name: "Managed private model", contextWindow: 8192 }], { baseUrl, once: true }, ); + mockDiscoveredOllamaProvider( + [{ id: modelId, name: "Rotated managed model", contextWindow: 8192 }], + { baseUrl, once: true }, + ); const context = createDynamicModelContext(modelId, config); - await provider.prepareDynamicModel?.(context as never); - expect(provider.resolveDynamicModel?.(context as never)?.id).toBe(modelId); - - await provider.prepareDynamicModel?.(context as never); - - expect(provider.resolveDynamicModel?.(context as never)).toBeUndefined(); - expect(buildOllamaProviderMock).toHaveBeenCalledOnce(); + await expect(provider.prepareDynamicModel?.(context as never)).resolves.toMatchObject({ + id: modelId, + name: "Managed private model", + }); + await expect(provider.prepareDynamicModel?.(context as never)).resolves.toMatchObject({ + id: modelId, + name: "Rotated managed model", + }); + await expect(provider.prepareDynamicModel?.(context as never)).resolves.toBeUndefined(); + expect(buildOllamaProviderMock).toHaveBeenNthCalledWith(2, baseUrl, { + quiet: true, + apiKey: "rotated-managed-access", + }); + expect(buildOllamaProviderMock).toHaveBeenCalledTimes(2); }); it("isolates identically named managed SecretRefs by their resolved configuration", async () => { @@ -1371,15 +1389,11 @@ describe("ollama plugin", () => { ); const contextFor = (config: typeof firstConfig) => createDynamicModelContext(modelId, config); - await provider.prepareDynamicModel?.(contextFor(firstConfig) as never); - await provider.prepareDynamicModel?.(contextFor(secondConfig) as never); + const firstModel = await provider.prepareDynamicModel?.(contextFor(firstConfig) as never); + const secondModel = await provider.prepareDynamicModel?.(contextFor(secondConfig) as never); - expect(provider.resolveDynamicModel?.(contextFor(firstConfig) as never)?.name).toBe( - "First managed tenant model", - ); - expect(provider.resolveDynamicModel?.(contextFor(secondConfig) as never)?.name).toBe( - "Second managed tenant model", - ); + expect(firstModel?.name).toBe("First managed tenant model"); + expect(secondModel?.name).toBe("Second managed tenant model"); expect(buildOllamaProviderMock).toHaveBeenNthCalledWith(1, baseUrl, { quiet: true, apiKey: "first-managed-tenant-access", @@ -1412,13 +1426,12 @@ describe("ollama plugin", () => { const context = createDynamicModelContext("deepseek-v4-pro:cloud"); try { - await provider.prepareDynamicModel?.(context as never); + const resolved = await provider.prepareDynamicModel?.(context as never); expect(queryOllamaModelShowInfoMock).toHaveBeenCalledWith( "http://127.0.0.1:11434", "deepseek-v4-pro:cloud", ); - const resolved = provider.resolveDynamicModel?.(context as never); expect(resolved?.provider).toBe("ollama"); expect(resolved?.id).toBe("deepseek-v4-pro:cloud"); expect(resolved?.api).toBe("ollama"); @@ -1971,9 +1984,7 @@ describe("ollama plugin", () => { const context = createDynamicModelContext("depseek-v4-pro:cloud"); try { - await provider.prepareDynamicModel?.(context as never); - - expect(provider.resolveDynamicModel?.(context as never)).toBeUndefined(); + await expect(provider.prepareDynamicModel?.(context as never)).resolves.toBeUndefined(); } finally { if (previous === undefined) { delete process.env.OLLAMA_API_KEY; diff --git a/extensions/ollama/index.ts b/extensions/ollama/index.ts index 9fb7b6c74ab4..40dd4356de03 100644 --- a/extensions/ollama/index.ts +++ b/extensions/ollama/index.ts @@ -1,5 +1,4 @@ // Ollama plugin entrypoint registers its OpenClaw integration. -import { createHash } from "node:crypto"; import { collectConfiguredModelRefValues } from "@openclaw/model-catalog-core/configured-model-refs"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime"; @@ -161,8 +160,6 @@ function classifyOllamaFailoverReason(errorMessage: string): "server_error" | un return errorMessage.trim() === OLLAMA_INCOMPLETE_STREAM_ERROR ? "server_error" : undefined; } -const dynamicModelCache = new Map(); -const dynamicManagedCredentialFingerprints = new WeakMap>(); const OLLAMA_CLOUD_DEFAULT_MODEL_REF = `${OLLAMA_CLOUD_PROVIDER_ID}/${OLLAMA_CLOUD_DEFAULT_MODELS[0].id}`; const OLLAMA_CONFIGURED_SHOW_CONCURRENCY = 4; const OLLAMA_CONFIGURED_SHOW_MAX_MODELS = 8; @@ -426,44 +423,6 @@ async function discoverAppGuidedOllamaModel( }; } -function buildDynamicManagedSecretScope( - provider: string, - baseUrl: string | undefined, - configuredApiKey: unknown, -): string | undefined { - const secretRef = coerceSecretRef(configuredApiKey); - if (!secretRef || secretRef.source === "env") { - return undefined; - } - return `${provider}\0${resolveOllamaApiBase(baseUrl)}\0${secretRef.source}\0${secretRef.provider}\0${secretRef.id}`; -} - -function buildDynamicCacheKey( - provider: string, - baseUrl: string | undefined, - configuredApiKey: unknown, - config?: OpenClawConfig, -): string { - const secretRef = coerceSecretRef(configuredApiKey); - const managedSecretScope = buildDynamicManagedSecretScope(provider, baseUrl, configuredApiKey); - const apiKey = readUsableOllamaShowApiKey({ - env: process.env, - allowAmbientEnvFallback: !isLocalOllamaBaseUrl(baseUrl), - explicitApiKey: configuredApiKey, - }); - // Managed secrets resolve asynchronously; retain their resolved fingerprint - // per config so synchronous lookups cannot cross secret-provider ownership. - const managedCredentialFingerprint = - managedSecretScope && config - ? dynamicManagedCredentialFingerprints.get(config)?.get(managedSecretScope) - : undefined; - const credentialScope = - apiKey ?? (secretRef ? `${secretRef.source}\0${secretRef.provider}\0${secretRef.id}` : ""); - const credentialFingerprint = - managedCredentialFingerprint ?? createHash("sha256").update(credentialScope).digest("hex"); - return `${provider}\0${resolveOllamaApiBase(baseUrl)}\0${credentialFingerprint}`; -} - function hasOllamaDiscoverySignal(providerConfig: ModelProviderConfig | undefined): boolean { return ( Boolean(process.env.OLLAMA_API_KEY?.trim()) || @@ -1173,20 +1132,9 @@ export default definePluginEntry({ providerId: ctx.provider, }); if (!hasOllamaDiscoverySignal(providerConfig)) { - return; + return undefined; } const baseUrl = readProviderBaseUrl(providerConfig); - const managedSecretScope = buildDynamicManagedSecretScope( - ctx.provider, - baseUrl, - providerConfig?.apiKey, - ); - let dynamicCacheKey = buildDynamicCacheKey( - ctx.provider, - baseUrl, - providerConfig?.apiKey, - ctx.config, - ); let discoveryApiKey: string | undefined; if (providerConfig?.apiKey !== undefined && providerConfig.apiKey !== null) { const resolved = await resolveConfiguredSecretInputString({ @@ -1197,11 +1145,7 @@ export default definePluginEntry({ unresolvedReasonStyle: "detailed", }); if (resolved.unresolvedRefReason) { - dynamicModelCache.delete(dynamicCacheKey); - if (managedSecretScope && ctx.config) { - dynamicManagedCredentialFingerprints.get(ctx.config)?.delete(managedSecretScope); - } - return; + return undefined; } const resolvedApiKey = readConfiguredOllamaApiKey(resolved.value); const configuredSecretRef = coerceSecretRef(providerConfig.apiKey); @@ -1211,38 +1155,11 @@ export default definePluginEntry({ ? readConcreteOllamaApiKey(process.env.OLLAMA_API_KEY) : readConcreteOllamaApiKey(resolvedApiKey); if (configuredSecretRef && !discoveryApiKey) { - dynamicModelCache.delete(dynamicCacheKey); - if (managedSecretScope && ctx.config) { - dynamicManagedCredentialFingerprints.get(ctx.config)?.delete(managedSecretScope); - } - return; + return undefined; } } else if (!isLocalOllamaBaseUrl(baseUrl)) { discoveryApiKey = readConcreteOllamaApiKey(process.env.OLLAMA_API_KEY); } - if (managedSecretScope && ctx.config && discoveryApiKey) { - let fingerprints = dynamicManagedCredentialFingerprints.get(ctx.config); - if (!fingerprints) { - fingerprints = new Map(); - dynamicManagedCredentialFingerprints.set(ctx.config, fingerprints); - } - const resolvedCredentialFingerprint = createHash("sha256") - .update(discoveryApiKey) - .digest("hex"); - if ( - fingerprints.has(managedSecretScope) && - fingerprints.get(managedSecretScope) !== resolvedCredentialFingerprint - ) { - dynamicModelCache.delete(dynamicCacheKey); - } - fingerprints.set(managedSecretScope, resolvedCredentialFingerprint); - dynamicCacheKey = buildDynamicCacheKey( - ctx.provider, - baseUrl, - providerConfig?.apiKey, - ctx.config, - ); - } const provider = await buildLocalOllamaProvider(baseUrl, { quiet: true, ...(discoveryApiKey ? { apiKey: discoveryApiKey } : {}), @@ -1257,42 +1174,21 @@ export default definePluginEntry({ }), api: dynamicApi, }; - const dynamicModels = (dynamicProvider.models ?? []).map((model) => - toDynamicOllamaModel({ + const discoveredModel = dynamicProvider.models?.find((model) => model.id === ctx.modelId); + if (discoveredModel) { + return toDynamicOllamaModel({ provider: ctx.provider, providerConfig: dynamicProvider, - model, - }), - ); - if (!dynamicModels.some((model) => model.id === ctx.modelId)) { - const requestedModel = await resolveRequestedDynamicOllamaModel({ - provider: ctx.provider, - providerConfig: dynamicProvider, - modelId: ctx.modelId, - showApiKey: discoveryApiKey, - capContextTokens: true, + model: discoveredModel, }); - if (requestedModel) { - dynamicModels.push(requestedModel); - } } - dynamicModelCache.set(dynamicCacheKey, dynamicModels); - }, - resolveDynamicModel: (ctx) => { - const providerConfig = resolveConfiguredOllamaProviderConfig({ - config: ctx.config, - providerId: ctx.provider, + return await resolveRequestedDynamicOllamaModel({ + provider: ctx.provider, + providerConfig: dynamicProvider, + modelId: ctx.modelId, + showApiKey: discoveryApiKey, + capContextTokens: true, }); - return dynamicModelCache - .get( - buildDynamicCacheKey( - ctx.provider, - readProviderBaseUrl(providerConfig), - providerConfig?.apiKey, - ctx.config, - ), - ) - ?.find((model) => model.id === ctx.modelId); }, buildUnknownModelHint: () => "Ollama requires authentication to be registered as a provider. " + diff --git a/src/agents/embedded-agent-runner/model.provider-hooks.ts b/src/agents/embedded-agent-runner/model.provider-hooks.ts index d14434bd96a4..9db8ae68b063 100644 --- a/src/agents/embedded-agent-runner/model.provider-hooks.ts +++ b/src/agents/embedded-agent-runner/model.provider-hooks.ts @@ -29,7 +29,7 @@ export type ProviderRuntimeHooks = { ) => string | undefined; prepareProviderDynamicModel: ( params: Parameters[0], - ) => Promise; + ) => ReturnType; runProviderDynamicModel: (params: Parameters[0]) => unknown; shouldPreferProviderRuntimeResolvedModel?: ( params: Parameters[0], diff --git a/src/agents/embedded-agent-runner/model.registry-resolution.ts b/src/agents/embedded-agent-runner/model.registry-resolution.ts index ba3127181b09..b3829a4a3bb5 100644 --- a/src/agents/embedded-agent-runner/model.registry-resolution.ts +++ b/src/agents/embedded-agent-runner/model.registry-resolution.ts @@ -2,6 +2,7 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js"; import type { ModelRegistry as CoreModelRegistry } from "../../llm/model-registry.js"; import type { Model } from "../../llm/types.js"; import type { PluginMetadataSnapshotOwnerMaps } from "../../plugins/plugin-metadata-snapshot.types.js"; +import type { ProviderRuntimeModel } from "../../plugins/provider-runtime-model.types.js"; import { ensureAuthProfileStore, resolveAuthProfileOrder } from "../auth-profiles.js"; import type { AuthProfileCredential } from "../auth-profiles/types.js"; import { resolveAgentHarnessPolicy } from "../harness/policy.js"; @@ -252,21 +253,9 @@ export function resolveDynamicModelAuthProfile(params: { }; } -function resolvePluginDynamicModelWithRegistry(params: { - provider: string; - modelId: string; - modelRegistry: CoreModelRegistry; - cfg?: OpenClawConfig; - agentDir?: string; - agentRuntimeId?: string; - manifestAlias: ManifestModelCatalogProviderAliasMetadata; - workspaceDir?: string; - authProfileId?: string; - authProfileMode?: AuthProfileCredential["type"] | "aws-sdk"; - preferredProfile?: string; - runtimeHooks?: ProviderRuntimeHooks; - getStaticCatalogModel?: () => StaticCatalogFallbackModel | undefined; -}): Model | undefined { +function resolvePluginDynamicModelWithRegistry( + params: ResolveModelWithPreparedRegistryParams, +): Model | undefined { const { provider, modelId, modelRegistry, cfg, agentDir, workspaceDir } = params; const runtimeHooks = params.runtimeHooks ?? DEFAULT_PROVIDER_RUNTIME_HOOKS; const providerConfig = resolveConfiguredProviderConfig(cfg, provider); @@ -294,22 +283,24 @@ function resolvePluginDynamicModelWithRegistry(params: { workspaceDir, runtimeHooks, }); - const pluginDynamicModel = runtimeHooks.runProviderDynamicModel({ - provider, - config: cfg, - workspaceDir, - context: { - config: cfg, - agentDir, - workspaceDir, - ...(agentRuntimeId ? { agentRuntimeId } : {}), + const pluginDynamicModel = + params.preparedDynamicModel ?? + (runtimeHooks.runProviderDynamicModel({ provider, - modelId, - modelRegistry, - providerConfig, - ...authProfile, - }, - }) as Model | undefined; + config: cfg, + workspaceDir, + context: { + config: cfg, + agentDir, + workspaceDir, + ...(agentRuntimeId ? { agentRuntimeId } : {}), + provider, + modelId, + modelRegistry, + providerConfig, + ...authProfile, + }, + }) as ProviderRuntimeModel | undefined); if (!pluginDynamicModel) { return undefined; } @@ -425,6 +416,7 @@ type ResolveModelWithRegistryParams = { type ResolveModelWithPreparedRegistryParams = ResolveModelWithRegistryParams & { manifestAlias: ManifestModelCatalogProviderAliasMetadata; + preparedDynamicModel?: ProviderRuntimeModel; getStaticCatalogModel?: () => StaticCatalogFallbackModel | undefined; }; diff --git a/src/agents/embedded-agent-runner/model.test.ts b/src/agents/embedded-agent-runner/model.test.ts index 20a9ddd3d141..660170d93523 100644 --- a/src/agents/embedded-agent-runner/model.test.ts +++ b/src/agents/embedded-agent-runner/model.test.ts @@ -582,6 +582,127 @@ function makeVllmQwenConfig( } describe("resolveModel", () => { + it("consumes a directly prepared model through configured overrides and normalization", async () => { + const preparedModel = { + ...makeModel("prepared-model"), + provider: "acme", + name: "Prepared Model", + api: "openai-completions" as const, + baseUrl: "https://discovered.example/v1", + input: ["text" as const], + contextWindow: 65_536, + maxTokens: 8_192, + }; + const prepareProviderDynamicModel = vi.fn(async () => preparedModel); + const runProviderDynamicModel = vi.fn(() => undefined); + const normalizeProviderResolvedModelWithPlugin = vi.fn( + ({ context }: { context: { model: Model } }) => ({ + ...context.model, + name: "Normalized Prepared Model", + }), + ); + const cfg = makeProviderConfig("acme", { + api: "openai-responses", + baseUrl: "https://configured.example/v1", + headers: { "X-Tenant": "tenant-a" }, + }); + + const result = await resolveModelAsync("acme", "prepared-model", "/tmp/agent", cfg, { + runtimeHooks: { + ...createRuntimeHooks(), + prepareProviderDynamicModel, + runProviderDynamicModel, + normalizeProviderResolvedModelWithPlugin, + }, + skipAgentDiscovery: true, + }); + + expectRecordFields(expectResolvedModel(result), { + provider: "acme", + id: "prepared-model", + name: "Normalized Prepared Model", + api: "openai-responses", + baseUrl: "https://configured.example/v1", + contextWindow: 65_536, + maxTokens: 8_192, + }); + expect(expectResolvedModel(result).headers).toEqual( + expect.objectContaining({ "X-Tenant": "tenant-a" }), + ); + expect(prepareProviderDynamicModel).toHaveBeenCalledOnce(); + expect(normalizeProviderResolvedModelWithPlugin).toHaveBeenCalledOnce(); + expect(runProviderDynamicModel).not.toHaveBeenCalled(); + }); + + it.each([ + { + description: "keeps explicit configured models ahead of prepared models", + preferRuntime: false, + expectedName: "Configured Model", + expectedPreparationCount: 0, + }, + { + description: "preserves manually configured limits during runtime comparison", + preferRuntime: true, + expectedName: "Prepared Model", + expectedPreparationCount: 1, + }, + { + description: "replaces models-add metadata with a preferred prepared model", + preferRuntime: true, + metadataSource: "models-add" as const, + expectedName: "Prepared Model", + expectedPreparationCount: 1, + }, + ])( + "$description", + async ({ preferRuntime, metadataSource, expectedName, expectedPreparationCount }) => { + const prepareProviderDynamicModel = vi.fn(async () => ({ + ...makeModel("prepared-model"), + provider: "acme", + name: "Prepared Model", + api: "openai-completions" as const, + baseUrl: "https://discovered.example/v1", + input: ["text" as const], + contextWindow: 65_536, + maxTokens: 8_192, + })); + const runProviderDynamicModel = vi.fn(() => undefined); + const cfg = makeProviderConfig("acme", { + api: "openai-responses", + baseUrl: "https://configured.example/v1", + models: [ + { + ...makeModel("prepared-model"), + name: "Configured Model", + contextWindow: 32_768, + maxTokens: 4_096, + ...(metadataSource ? { metadataSource } : {}), + }, + ], + }); + + const result = await resolveModelAsync("acme", "prepared-model", "/tmp/agent", cfg, { + runtimeHooks: { + ...createRuntimeHooks(), + prepareProviderDynamicModel, + runProviderDynamicModel, + shouldPreferProviderRuntimeResolvedModel: () => preferRuntime, + }, + skipAgentDiscovery: true, + }); + + expectRecordFields(expectResolvedModel(result), { + name: expectedName, + api: "openai-responses", + baseUrl: "https://configured.example/v1", + contextWindow: metadataSource ? 65_536 : 32_768, + }); + expect(prepareProviderDynamicModel).toHaveBeenCalledTimes(expectedPreparationCount); + expect(runProviderDynamicModel).not.toHaveBeenCalled(); + }, + ); + it("reuses agent discovery stores while the agent model files are unchanged", async () => { mockModelDiscovery(); diff --git a/src/agents/embedded-agent-runner/model.ts b/src/agents/embedded-agent-runner/model.ts index 9b8c32722088..b65e4ddd80d8 100644 --- a/src/agents/embedded-agent-runner/model.ts +++ b/src/agents/embedded-agent-runner/model.ts @@ -419,7 +419,7 @@ export async function resolveModelAsync( }); }; const resolveDynamicAttempt = async () => { - await runtimeHooks.prepareProviderDynamicModel({ + const preparedDynamicModel = await runtimeHooks.prepareProviderDynamicModel({ provider: normalizedRef.provider, config: cfg, workspaceDir, @@ -448,6 +448,7 @@ export async function resolveModelAsync( authProfileMode: options?.authProfileMode, preferredProfile: options?.preferredProfile, runtimeHooks, + ...(preparedDynamicModel ? { preparedDynamicModel } : {}), getStaticCatalogModel: getManifestStaticCatalogModel, ...(options?.allowBundledStaticCatalogFallback ? { skipConfiguredFallback: true } : {}), }); diff --git a/src/agents/live-model-dynamic-candidates.test.ts b/src/agents/live-model-dynamic-candidates.test.ts index 8bbe6b37bd0d..952b2928fc7a 100644 --- a/src/agents/live-model-dynamic-candidates.test.ts +++ b/src/agents/live-model-dynamic-candidates.test.ts @@ -155,6 +155,27 @@ describe("appendPrioritizedDynamicLiveModels", () => { expect(resolveDynamicModel).not.toHaveBeenCalled(); }); + it("materializes a directly prepared model without retrying synchronous resolution", async () => { + const preparedModel = model(DYNAMIC_PROVIDER, "glm-5"); + providerRuntimeMocks.prepareProviderDynamicModel.mockResolvedValue(preparedModel); + + const result = await appendPrioritizedDynamicLiveModels({ + models: [], + agentDir: "/tmp/openclaw-agent", + modelRegistry: REGISTRY, + refs: [{ provider: DYNAMIC_PROVIDER, id: "glm-5" }], + }); + + expect(result.added).toEqual([preparedModel]); + expect(providerRuntimeMocks.prepareProviderDynamicModel).toHaveBeenCalledOnce(); + expect(providerRuntimeMocks.runProviderDynamicModel).not.toHaveBeenCalled(); + expect(normalizeDiscoveredAgentModelMock).toHaveBeenCalledWith( + preparedModel, + "/tmp/openclaw-agent", + { config: undefined, workspaceDir: undefined }, + ); + }); + it("uses default provider runtime hooks when resolvers are not injected", async () => { providerRuntimeMocks.runProviderDynamicModel.mockImplementation((params) => params.context.provider === DYNAMIC_PROVIDER && params.context.modelId === "glm-5" diff --git a/src/agents/test-helpers/live-model-dynamic-candidates.ts b/src/agents/test-helpers/live-model-dynamic-candidates.ts index a1a827899266..f41c2438bc1c 100644 --- a/src/agents/test-helpers/live-model-dynamic-candidates.ts +++ b/src/agents/test-helpers/live-model-dynamic-candidates.ts @@ -29,9 +29,9 @@ const providerRuntimeLoader = createLazyImportLoader( async function prepareProviderDynamicModelDefault( params: Parameters[0], -): Promise { +): ReturnType { const { prepareProviderDynamicModel } = await providerRuntimeLoader.load(); - await prepareProviderDynamicModel(params); + return await prepareProviderDynamicModel(params); } async function runProviderDynamicModelDefault( @@ -108,20 +108,22 @@ export async function appendPrioritizedDynamicLiveModels(params: { modelRegistry: params.modelRegistry, providerConfig, }; - await prepareDynamicModel({ - provider: ref.provider, - config: params.config, - workspaceDir: params.workspaceDir, - env: params.env, - context, - }); - const resolved = await resolveDynamicModel({ + const prepared = await prepareDynamicModel({ provider: ref.provider, config: params.config, workspaceDir: params.workspaceDir, env: params.env, context, }); + const resolved = + prepared ?? + (await resolveDynamicModel({ + provider: ref.provider, + config: params.config, + workspaceDir: params.workspaceDir, + env: params.env, + context, + })); if (!resolved) { continue; } diff --git a/src/plugins/provider-plugin.types.ts b/src/plugins/provider-plugin.types.ts index f8a512b37d50..d7917bdf6b3c 100644 --- a/src/plugins/provider-plugin.types.ts +++ b/src/plugins/provider-plugin.types.ts @@ -154,19 +154,21 @@ export type ProviderPlugin = { * 3. core fallback heuristics * 4. generic provider-config fallback * - * Keep this hook cheap and deterministic. If you need network I/O first, use - * `prepareDynamicModel` to prime state for the async retry path. + * Keep this hook cheap and deterministic. Async model discovery belongs in + * `prepareDynamicModel`, which can return the prepared model directly. */ resolveDynamicModel?: ( ctx: ProviderResolveDynamicModelContext, ) => ProviderRuntimeModel | null | undefined; /** - * Optional async prefetch for dynamic model resolution. + * Optional async preparation for dynamic model resolution. * - * OpenClaw calls this only from async model resolution paths. After it - * completes, `resolveDynamicModel` is called again. + * OpenClaw calls this only from async model resolution paths. Return the + * requested model directly, or return nothing to retry `resolveDynamicModel`. */ - prepareDynamicModel?: (ctx: ProviderPrepareDynamicModelContext) => Promise; + prepareDynamicModel?: ( + ctx: ProviderPrepareDynamicModelContext, + ) => Promise; /** * Lets a provider plugin opt exact configured models into a runtime * metadata comparison pass before the embedded runner returns the explicit diff --git a/src/plugins/provider-runtime.test.ts b/src/plugins/provider-runtime.test.ts index 0bd99bdcfb81..7f6c96232ed9 100644 --- a/src/plugins/provider-runtime.test.ts +++ b/src/plugins/provider-runtime.test.ts @@ -2041,7 +2041,7 @@ describe("provider-runtime", () => { { pluginId: "demo", providerIds: ["demo"] }, ]); resolveExternalAuthProfileProviderPluginIdsMock.mockReturnValue(["demo"]); - const prepareDynamicModel = vi.fn(async () => undefined); + const prepareDynamicModel = vi.fn(async () => MODEL); const createStreamFn = vi.fn(() => vi.fn()); const sanitizeReplayHistory = vi.fn( async ({ @@ -2245,12 +2245,14 @@ describe("provider-runtime", () => { }), ).toBe("DEMO_PROFILE"); - await prepareProviderDynamicModel({ - provider: DEMO_PROVIDER_ID, - context: createDemoRuntimeContext({ - modelRegistry: EMPTY_MODEL_REGISTRY, + expect( + await prepareProviderDynamicModel({ + provider: DEMO_PROVIDER_ID, + context: createDemoRuntimeContext({ + modelRegistry: EMPTY_MODEL_REGISTRY, + }), }), - }); + ).toEqual(MODEL); expect( resolveProviderReasoningOutputModeWithPlugin({ diff --git a/src/plugins/provider-runtime.ts b/src/plugins/provider-runtime.ts index 317513de30c9..2fd908939b83 100644 --- a/src/plugins/provider-runtime.ts +++ b/src/plugins/provider-runtime.ts @@ -282,8 +282,8 @@ export async function prepareProviderDynamicModel(params: { workspaceDir?: string; env?: NodeJS.ProcessEnv; context: ProviderPrepareDynamicModelContext; -}): Promise { - await resolveProviderRuntimePlugin(params)?.prepareDynamicModel?.(params.context); +}): Promise { + return resolveProviderRuntimePlugin(params)?.prepareDynamicModel?.(params.context); } export function shouldPreferProviderRuntimeResolvedModel(params: { diff --git a/src/plugins/provider-runtime.types.ts b/src/plugins/provider-runtime.types.ts index 53af7112504f..77a76dc20363 100644 --- a/src/plugins/provider-runtime.types.ts +++ b/src/plugins/provider-runtime.types.ts @@ -41,11 +41,10 @@ export type ProviderResolveDynamicModelContext = { }; /** - * Optional async warm-up for dynamic model resolution. + * Optional async preparation for dynamic model resolution. * - * Called only from async model resolution paths, before retrying - * `resolveDynamicModel`. This is the place to refresh caches or fetch provider - * metadata over the network. + * Called only from async model resolution paths. Providers can return the + * requested model directly or refresh reusable metadata before the sync retry. */ export type ProviderPrepareDynamicModelContext = ProviderResolveDynamicModelContext;