diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index 0db0c7b33af2..63c85bd1d2f4 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -503,7 +503,6 @@ src/agents/openclaw-tools.media-factory-plan.test.ts src/agents/openclaw-tools.session-status.test.ts src/agents/openclaw-tools.sessions.test.ts src/agents/provider-attribution.test.ts -src/agents/provider-attribution.ts src/agents/provider-local-service.ts src/agents/provider-request-config.ts src/agents/provider-transport-fetch.test.ts diff --git a/src/agents/openclaw-tools.media-factory-plan.test.ts b/src/agents/openclaw-tools.media-factory-plan.test.ts index d75352afbbc1..1b37e293e078 100644 --- a/src/agents/openclaw-tools.media-factory-plan.test.ts +++ b/src/agents/openclaw-tools.media-factory-plan.test.ts @@ -14,7 +14,10 @@ import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot import { resetPluginRuntimeStateForTest } from "../plugins/runtime.js"; import { clearSecretsRuntimeSnapshot } from "../secrets/runtime.js"; import type { AuthProfileStore } from "./auth-profiles/types.js"; -import { resolveOptionalMediaToolFactoryPlan } from "./openclaw-tools.media-factory-plan.js"; +import { + resolveImageToolFactoryAvailable, + resolveOptionalMediaToolFactoryPlan, +} from "./openclaw-tools.media-factory-plan.js"; import { DEFAULT_PLUGIN_TOOLS_ALLOWLIST_ENTRY } from "./tool-policy.js"; import { loadCapabilityMetadataSnapshot } from "./tools/manifest-capability-availability.js"; import * as pdfModelConfigModule from "./tools/pdf-tool.model-config.js"; @@ -156,6 +159,7 @@ function installSnapshot( }, } satisfies PluginMetadataSnapshot; setCurrentPluginMetadataSnapshot(snapshot, { config }); + return snapshot; } describe("optional media tool factory planning", () => { @@ -193,6 +197,139 @@ describe("optional media tool factory planning", () => { vi.unstubAllEnvs(); }); + it("uses the prepared media family for image-tool availability", () => { + const config: OpenClawConfig = {}; + const snapshot = installSnapshot(config, [ + createPlugin({ + id: "media-owner", + contracts: { mediaUnderstandingProviders: ["media-owner"] }, + setupProviders: [{ id: "media-owner" }], + }), + ]); + const base = { + config, + agentDir: "/agent", + authStore: createAuthStore(["media-owner"]), + }; + + expect( + resolveImageToolFactoryAvailable({ + ...base, + preparedModelRuntime: { + metadataSnapshot: snapshot, + mediaCapabilityProviders: { mediaUnderstandingProviders: [] }, + } as never, + }), + ).toBe(false); + expect( + resolveImageToolFactoryAvailable({ + ...base, + preparedModelRuntime: { + metadataSnapshot: snapshot, + mediaCapabilityProviders: { + mediaUnderstandingProviders: [{ id: "media-owner", capabilities: ["image"] }], + }, + } as never, + }), + ).toBe(true); + }); + + it("requires image capability and auth on the same prepared provider", () => { + const config: OpenClawConfig = {}; + const snapshot = installSnapshot(config, [ + createPlugin({ + id: "media-owner", + contracts: { + mediaUnderstandingProviders: ["audio-auth", "image-no-auth"], + }, + setupProviders: [{ id: "audio-auth" }, { id: "image-no-auth" }], + }), + ]); + + expect( + resolveImageToolFactoryAvailable({ + config, + agentDir: "/agent", + authStore: createAuthStore(["audio-auth"]), + preparedModelRuntime: { + metadataSnapshot: snapshot, + mediaCapabilityProviders: { + mediaUnderstandingProviders: [ + { id: "audio-auth", capabilities: ["audio"] }, + { id: "image-no-auth", capabilities: ["image"] }, + ], + }, + } as never, + }), + ).toBe(false); + }); + + it("keeps config vision routes while gating OpenAI subscription auth on prepared Codex", () => { + vi.stubEnv("OPENAI_API_KEY", ""); + const config = { + models: { + providers: { + custom: { + baseUrl: "https://vision.example/v1", + models: [{ id: "vision", input: ["text", "image"] }], + }, + openai: { + baseUrl: "https://api.openai.com/v1", + models: [{ id: "gpt-image", input: ["text", "image"] }], + }, + }, + }, + } as unknown as OpenClawConfig; + const snapshot = installSnapshot(config, []); + const preparedModelRuntime = { + metadataSnapshot: snapshot, + mediaCapabilityProviders: { mediaUnderstandingProviders: [] }, + } as never; + const oauthStore = createAuthStore(); + oauthStore.profiles["openai:default"] = { + provider: "openai", + type: "oauth", + access: "test", + refresh: "test", + expires: Date.now() + 60_000, + }; + + expect( + resolveImageToolFactoryAvailable({ + config, + agentDir: "/agent", + authStore: createAuthStore(["custom"]), + preparedModelRuntime, + }), + ).toBe(true); + expect( + resolveImageToolFactoryAvailable({ + config, + agentDir: "/agent", + authStore: oauthStore, + preparedModelRuntime, + }), + ).toBe(false); + for (const [capabilities, expected] of [ + [["audio"], false], + [["image"], true], + ] as const) { + expect( + resolveImageToolFactoryAvailable({ + config, + agentDir: "/agent", + authStore: oauthStore, + preparedModelRuntime: { + metadataSnapshot: snapshot, + mediaCapabilityProviders: { + mediaUnderstandingProviders: [{ id: "codex", capabilities }], + }, + } as never, + }), + ).toBe(expected); + } + }); + it("skips unavailable generation and PDF factories from snapshot and run auth facts", () => { const config: OpenClawConfig = {}; installSnapshot(config, [ diff --git a/src/agents/openclaw-tools.media-factory-plan.ts b/src/agents/openclaw-tools.media-factory-plan.ts index 12a05fb94826..a81b01eb45eb 100644 --- a/src/agents/openclaw-tools.media-factory-plan.ts +++ b/src/agents/openclaw-tools.media-factory-plan.ts @@ -1,22 +1,21 @@ -/** - * Optional media tool factory planner. - * - * Combines config, tool policy, plugin capability metadata, and auth-profile availability before tool construction. - */ import { uniqueStrings } from "@openclaw/normalization-core/string-normalization"; +import { findCapabilityProviderById } from "../../packages/media-generation-core/src/capability-model-ref.js"; import { resolveAgentModelFallbackValues, resolveAgentModelPrimaryValue, } from "../config/model-input.js"; import type { AgentModelConfig } from "../config/types.agents-shared.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { normalizeMediaProviderId } from "../media-understanding/provider-id.js"; import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.types.js"; import { listProfilesForProvider } from "./auth-profiles/profile-list.js"; import type { AuthProfileStore } from "./auth-profiles/types.js"; +import type { PreparedModelRuntimeSnapshot } from "./prepared-model-runtime.js"; import { isToolAllowedByPolicyName } from "./tool-policy-match.js"; import { DEFAULT_PLUGIN_TOOLS_ALLOWLIST_ENTRY } from "./tool-policy.js"; import { hasSnapshotCapabilityAvailability, + hasSnapshotCapabilityProviderAvailability, hasSnapshotProviderEnvAvailability, loadCapabilityMetadataSnapshot, } from "./tools/manifest-capability-availability.js"; @@ -116,6 +115,7 @@ export function resolveImageToolFactoryAvailable(params: { workspaceDir?: string; modelHasVision?: boolean; authStore?: AuthProfileStore; + preparedModelRuntime?: PreparedModelRuntimeSnapshot; }): boolean { if (!params.agentDir?.trim()) { return false; @@ -123,21 +123,39 @@ export function resolveImageToolFactoryAvailable(params: { if (params.modelHasVision || hasExplicitImageModelConfig(params.config)) { return true; } - const snapshot = loadCapabilityMetadataSnapshot({ - config: params.config, - ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), - }); - return ( - hasSnapshotCapabilityAvailability({ - snapshot, - authStore: params.authStore, - key: "mediaUnderstandingProviders", + const snapshot = + params.preparedModelRuntime?.metadataSnapshot ?? + loadCapabilityMetadataSnapshot({ config: params.config, - }) || + ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), + }); + const preparedProviders = + params.preparedModelRuntime?.mediaCapabilityProviders?.mediaUnderstandingProviders; + const hasPreparedImageProvider = preparedProviders?.some( + (provider) => + provider.capabilities?.includes("image") && + hasSnapshotCapabilityProviderAvailability({ + snapshot, + authStore: params.authStore, + key: "mediaUnderstandingProviders", + providerId: provider.id, + config: params.config, + }), + ); + return ( + (preparedProviders === undefined + ? hasSnapshotCapabilityAvailability({ + snapshot, + authStore: params.authStore, + key: "mediaUnderstandingProviders", + config: params.config, + }) + : hasPreparedImageProvider === true) || hasConfiguredVisionModelAuthSignal({ config: params.config, snapshot, authStore: params.authStore, + preparedProviders, }) ); } @@ -146,6 +164,9 @@ function hasConfiguredVisionModelAuthSignal(params: { config?: OpenClawConfig; snapshot: Pick; authStore?: AuthProfileStore; + preparedProviders?: NonNullable< + PreparedModelRuntimeSnapshot["mediaCapabilityProviders"] + >["mediaUnderstandingProviders"]; }): boolean { const providers = params.config?.models?.providers; if (!providers || typeof providers !== "object") { @@ -159,16 +180,34 @@ function hasConfiguredVisionModelAuthSignal(params: { ) { continue; } - if (params.authStore && listProfilesForProvider(params.authStore, providerId).length > 0) { - return true; - } + const profileIds = params.authStore + ? listProfilesForProvider(params.authStore, providerId) + : []; + const hasDirectProfile = profileIds.some( + (profileId) => params.authStore?.profiles[profileId]?.type === "api_key", + ); + const hasEnv = hasSnapshotProviderEnvAvailability({ + snapshot: params.snapshot, + providerId, + config: params.config, + }); + const needsPreparedCodex = + normalizeMediaProviderId(providerId) === "openai" && + profileIds.length > 0 && + !hasDirectProfile && + !hasEnv; if ( - hasSnapshotProviderEnvAvailability({ - snapshot: params.snapshot, - providerId, - config: params.config, - }) + needsPreparedCodex && + params.preparedProviders !== undefined && + !findCapabilityProviderById({ + providers: params.preparedProviders, + providerId: "codex", + normalizeProviderId: normalizeMediaProviderId, + })?.capabilities?.includes("image") ) { + continue; + } + if (profileIds.length > 0 || hasEnv) { return true; } } @@ -182,6 +221,7 @@ export function resolveOptionalMediaToolFactoryPlan(params: { authStore?: AuthProfileStore; toolAllowlist?: string[]; toolDenylist?: string[]; + preparedModelRuntime?: PreparedModelRuntimeSnapshot; }): OptionalMediaToolFactoryPlan { const defaults = params.config?.agents?.defaults; const toolAllowlist = mergeBuiltInFactoryAllowlist( @@ -223,13 +263,19 @@ export function resolveOptionalMediaToolFactoryPlan(params: { pdf: false, }; } - const snapshot = loadCapabilityMetadataSnapshot({ - config: params.config, - ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), - }); + const snapshot = + params.preparedModelRuntime?.metadataSnapshot ?? + loadCapabilityMetadataSnapshot({ + config: params.config, + ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), + }); + const preparedProviders = params.preparedModelRuntime?.mediaCapabilityProviders; + const preparedFamilyAvailable = (providers: readonly unknown[] | undefined) => + providers === undefined || providers.length > 0; return { imageGenerate: allowImageGenerate && + preparedFamilyAvailable(preparedProviders?.imageGenerationProviders) && (explicitImageGeneration || hasSnapshotCapabilityAvailability({ snapshot, @@ -239,6 +285,7 @@ export function resolveOptionalMediaToolFactoryPlan(params: { })), videoGenerate: allowVideoGenerate && + preparedFamilyAvailable(preparedProviders?.videoGenerationProviders) && (explicitVideoGeneration || hasSnapshotCapabilityAvailability({ snapshot, @@ -248,6 +295,7 @@ export function resolveOptionalMediaToolFactoryPlan(params: { })), musicGenerate: allowMusicGenerate && + preparedFamilyAvailable(preparedProviders?.musicGenerationProviders) && (explicitMusicGeneration || hasSnapshotCapabilityAvailability({ snapshot, diff --git a/src/agents/openclaw-tools.ts b/src/agents/openclaw-tools.ts index c46d8b9ac7f1..1374911cbab1 100644 --- a/src/agents/openclaw-tools.ts +++ b/src/agents/openclaw-tools.ts @@ -1,4 +1,3 @@ -/** Builds the per-run built-in and plugin tool inventory. */ import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; import type { SourceReplyDeliveryMode, @@ -239,18 +238,15 @@ export function createOpenClawTools( config: resolvedConfig, agentId: options?.requesterAgentIdOverride, }); - const effectiveRequesterAgentId = sessionAgentId; const swarmToolGroups = createOpenClawSwarmToolGroups({ config: resolvedConfig, - effectiveRequesterAgentId, + effectiveRequesterAgentId: sessionAgentId, agentSessionKey: options?.agentSessionKey, runSessionKey: options?.runSessionKey, runId: options?.runId, swarmCollector: options?.swarmCollector, swarmOutputSchema: options?.swarmOutputSchema, }); - // Fall back to the session agent workspace so plugin loading stays workspace-stable - // even when a caller forgets to thread workspaceDir explicitly. const inferredWorkspaceDir = options?.workspaceDir || !resolvedConfig ? undefined @@ -280,46 +276,45 @@ export function createOpenClawTools( authStore: options?.authProfileStore, toolAllowlist: options?.pluginToolAllowlist, toolDenylist: options?.pluginToolDenylist, + preparedModelRuntime: options?.preparedModelRuntime, }); const trimmedRunSessionKey = options?.runSessionKey?.trim(); const mediaGenerationAgentSessionKey = trimmedRunSessionKey && isCronRunSessionKey(trimmedRunSessionKey) ? trimmedRunSessionKey : options?.agentSessionKey; - const mediaGenerationAsyncStartCallback = mediaGenerationAgentSessionKey - ? isCronRunSessionKey(mediaGenerationAgentSessionKey) + const mediaGenerationAsyncStartCallback = + mediaGenerationAgentSessionKey && isCronRunSessionKey(mediaGenerationAgentSessionKey) ? undefined - : options?.onYield - : options?.onYield; - const taskSuggestionSessionKey = normalizeOptionalString( - options?.runSessionKey ?? options?.agentSessionKey, - ); - const requesterSessionKey = options?.agentSessionKey; - const requesterTurnRunId = options?.runId; - const imageToolAgentDir = options?.agentDir; - const imageTool = resolveImageToolFactoryAvailable({ - config: availabilityConfig ?? resolvedConfig, - agentDir: imageToolAgentDir, - workspaceDir, - modelHasVision: options?.modelHasVision, - authStore: options?.authProfileStore, - }) - ? createImageTool({ - config: availabilityConfig ?? options?.config, - agentId: sessionAgentId, - agentDir: imageToolAgentDir!, - preparedModelRuntime: options?.preparedModelRuntime, - authProfileStore: options?.authProfileStore, - workspaceDir, - sandbox, - fsPolicy: options?.fsPolicy, - agentChannel: options?.agentChannel, - agentAccountId: options?.agentAccountId, - currentChannelId: options?.currentChannelId, - modelHasVision: options?.modelHasVision, - deferAutoModelResolution: true, - }) - : null; + : options?.onYield; + const taskKey = normalizeOptionalString(options?.runSessionKey ?? options?.agentSessionKey); + const { agentSessionKey: requesterSessionKey, runId: requesterTurnRunId } = options ?? {}; + const imageTool = + options?.agentDir && + resolveImageToolFactoryAvailable({ + config: availabilityConfig ?? resolvedConfig, + agentDir: options.agentDir, + workspaceDir, + modelHasVision: options?.modelHasVision, + authStore: options?.authProfileStore, + preparedModelRuntime: options?.preparedModelRuntime, + }) + ? createImageTool({ + config: availabilityConfig ?? options?.config, + agentId: sessionAgentId, + agentDir: options.agentDir, + preparedModelRuntime: options?.preparedModelRuntime, + authProfileStore: options?.authProfileStore, + workspaceDir, + sandbox, + fsPolicy: options?.fsPolicy, + agentChannel: options?.agentChannel, + agentAccountId: options?.agentAccountId, + currentChannelId: options?.currentChannelId, + modelHasVision: options?.modelHasVision, + deferAutoModelResolution: true, + }) + : null; options?.recordToolPrepStage?.("openclaw-tools:image-tool"); const imageGenerateTool = optionalMediaTools.imageGenerate ? createImageGenerateTool({ @@ -329,6 +324,7 @@ export function createOpenClawTools( agentSessionKey: mediaGenerationAgentSessionKey, requesterOrigin: deliveryContext ?? undefined, workspaceDir, + preparedModelRuntime: options?.preparedModelRuntime, sandbox, fsPolicy: options?.fsPolicy, onAsyncTaskStarted: mediaGenerationAsyncStartCallback, @@ -343,6 +339,7 @@ export function createOpenClawTools( agentSessionKey: mediaGenerationAgentSessionKey, requesterOrigin: deliveryContext ?? undefined, workspaceDir, + preparedModelRuntime: options?.preparedModelRuntime, sandbox, fsPolicy: options?.fsPolicy, onAsyncTaskStarted: mediaGenerationAsyncStartCallback, @@ -357,6 +354,7 @@ export function createOpenClawTools( agentSessionKey: mediaGenerationAgentSessionKey, requesterOrigin: deliveryContext ?? undefined, workspaceDir, + preparedModelRuntime: options?.preparedModelRuntime, sandbox, fsPolicy: options?.fsPolicy, onAsyncTaskStarted: mediaGenerationAsyncStartCallback, @@ -452,15 +450,14 @@ export function createOpenClawTools( resolvedConfig?.tools?.deny, options?.pluginToolDenylist, ); - const messageExplicitlyAllowed = isToolExplicitlyAllowedByFactoryPolicy({ - toolName: "message", - allowlist: explicitFactoryAllowlist, - denylist: explicitFactoryDenylist, - }); const includeMessageTool = !embedded || options?.sourceReplyDeliveryMode === "message_tool_only" || - messageExplicitlyAllowed; + isToolExplicitlyAllowedByFactoryPolicy({ + toolName: "message", + allowlist: explicitFactoryAllowlist, + denylist: explicitFactoryDenylist, + }); const includeSubagentSpawnTool = !embedded || options?.allowGatewaySubagentBinding === true; const effectiveCallGateway = embedded ? createEmbeddedCallGateway() : callGateway; const includeUpdatePlanTool = shouldIncludeUpdatePlanToolForOpenClawTools({ @@ -472,8 +469,6 @@ export function createOpenClawTools( pluginToolAllowlist: options?.pluginToolAllowlist, pluginToolDenylist: options?.pluginToolDenylist, }); - // isEmbeddedMode() marks the TUI-embedded host, not the embedded agent runner; - // gating on it would hide ask_user from every normal gateway run. const includeAskUserTool = shouldIncludeAskUserToolForOpenClawTools({ config: resolvedConfig, agentSessionKey: options?.runSessionKey ?? options?.agentSessionKey, @@ -495,8 +490,7 @@ export function createOpenClawTools( createComputerTool({ config: options?.config, modelHasVision: options?.modelHasVision, - // Run ids survive attempt/session reconstruction but do not - // span later assistant runs that may reuse a provider call id. + // Run ids expire before later assistant runs can reuse a provider call id. idempotencyScope: options?.runId, contextEpoch: options?.computerContextEpoch, }), @@ -532,16 +526,15 @@ export function createOpenClawTools( }), ]), ]), - ...(!embedded && taskSuggestionSessionKey && options?.taskSuggestionDeliveryMode === "gateway" + ...(!embedded && taskKey && options?.taskSuggestionDeliveryMode === "gateway" ? createTaskSuggestionTools({ - sessionKey: taskSuggestionSessionKey, + sessionKey: taskKey, agentId: sessionAgentId, cwd: runtimeCwd, }) : []), ...(messageTool && includeMessageTool ? [messageTool] : []), - // Discord sessions get the Discord plugin's own show_widget (Activities - // delivery); registering the core tool there would collide on the name. + // Discord owns show_widget; registering the core tool would collide. ...(options?.agentChannel === "discord" || !isCoreCanvasHostEnabled(resolvedConfig) ? [] : [ @@ -683,7 +676,7 @@ export function createOpenClawTools( agentMemberRoleIds: options?.agentMemberRoleIds, sandboxed: options?.sandboxed, config: resolvedConfig, - requesterAgentIdOverride: effectiveRequesterAgentId, + requesterAgentIdOverride: sessionAgentId, requesterRunId: options?.runId, swarmCollector: options?.swarmCollector, workspaceDir: spawnWorkspaceDir, diff --git a/src/agents/prepared-model-runtime.owner.ts b/src/agents/prepared-model-runtime.owner.ts index dc99179bacd1..cd34ba1f5bc9 100644 --- a/src/agents/prepared-model-runtime.owner.ts +++ b/src/agents/prepared-model-runtime.owner.ts @@ -1,4 +1,3 @@ -/** Construction and owner identity for prepared model runtime generations. */ import path from "node:path"; import { collectConfiguredModelRefs } from "@openclaw/model-catalog-core/configured-model-refs"; import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; @@ -6,6 +5,7 @@ import { hashRuntimeConfigValue } from "../config/runtime-snapshot.js"; import { MODEL_APIS } from "../config/types.models.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { withTimeout } from "../node-host/with-timeout.js"; +import { prepareMediaCapabilityProviders } from "../plugins/capability-provider-runtime.js"; import { resolvePluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.types.js"; import { isReservedSystemAgentId } from "../system-agent/agent-id.js"; @@ -36,6 +36,7 @@ export type PreparedModelRuntimeSnapshot = Readonly<{ workspaceDir?: string; config: OpenClawConfig; metadataSnapshot: PluginMetadataSnapshot; + mediaCapabilityProviders?: ReturnType; modelCatalog: ModelCatalogSnapshot; createStores: () => PreparedModelRuntimeStores; }>; @@ -386,19 +387,24 @@ async function buildSnapshot( catalogMode: PreparedModelRuntimeCatalogMode, ): Promise { const env = input.env ?? process.env; - if (!input.readOnly) { - // Writable lifecycle publication owns process-global runtime plugin activation. Read-only - // drafts consume manifest metadata only and must not mutate live hooks outside that gate. - ensureRuntimePluginsLoaded({ - config: input.config, - ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), - }); - } + const runtimePluginRegistry = !input.readOnly + ? ensureRuntimePluginsLoaded({ + config: input.config, + ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), + }) + : undefined; const pluginMetadataSnapshot = resolvePluginMetadataSnapshot({ config: input.config, env, ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), }); + const mediaCapabilityProviders = input.readOnly + ? undefined + : prepareMediaCapabilityProviders({ + cfg: input.config, + pluginMetadataSnapshot, + registry: runtimePluginRegistry, + }); const templateAuthStorage = discoverAuthStorage(input.agentDir, { config: input.config, // Snapshot construction never initializes, migrates, or externally syncs auth. ModelRegistry @@ -462,6 +468,7 @@ async function buildSnapshot( ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), config: input.config, metadataSnapshot: pluginMetadataSnapshot, + ...(mediaCapabilityProviders ? { mediaCapabilityProviders } : {}), modelCatalog: { ...modelCatalog, staticEntries }, createStores, }); diff --git a/src/agents/provider-attribution.catalog-endpoints.test.ts b/src/agents/provider-attribution.catalog-endpoints.test.ts index c62cdec1a2d2..ceb46045bb6e 100644 --- a/src/agents/provider-attribution.catalog-endpoints.test.ts +++ b/src/agents/provider-attribution.catalog-endpoints.test.ts @@ -1,24 +1,24 @@ // Verifies catalog-backed endpoint classification for externalized official providers. import { describe, expect, it, vi } from "vitest"; -// Simulates a built dist tree: externalized provider plugins (qwen, moonshot, -// zai, ...) are excluded from dist packaging, so no plugin manifest supplies -// their endpoint metadata. Classification must come from the bundled catalog. -// The single conflicting manifest entry proves installed manifests stay -// authoritative over catalog metadata (first match wins). -vi.mock("../plugins/manifest-metadata-scan.js", () => ({ - listOpenClawPluginManifestMetadata: () => [ - { - pluginDir: "installed-conflict-fixture", - manifest: { - providerEndpoints: [ - { endpointClass: "openai-public", hosts: ["coding.dashscope.aliyuncs.com"] }, - ], - }, - origin: "installed", - }, - ], -})); +// Simulates a built dist tree: externalized provider metadata comes from the +// catalog, while one installed manifest proves first-match precedence. +vi.mock("../plugins/current-plugin-metadata-snapshot.js", async () => { + const { buildPluginMetadataProviderFacts } = + await import("../plugins/plugin-metadata-provider-facts.js"); + return { + getCurrentPluginMetadataSnapshot: () => ({ + owners: buildPluginMetadataProviderFacts([ + { + id: "installed-conflict-fixture", + providerEndpoints: [ + { endpointClass: "openai-public", hosts: ["coding.dashscope.aliyuncs.com"] }, + ], + } as never, + ]), + }), + }; +}); import { resolveProviderEndpoint, diff --git a/src/agents/provider-attribution.test.ts b/src/agents/provider-attribution.test.ts index b7eab883d752..3f5b4dffca3e 100644 --- a/src/agents/provider-attribution.test.ts +++ b/src/agents/provider-attribution.test.ts @@ -1,5 +1,5 @@ // Verifies provider attribution headers and endpoint classification policies. -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; function expectRecordFields(record: unknown, expected: Record) { // Policy helpers return broad records; assertions pin only the relevant fields. @@ -116,20 +116,39 @@ const providerEndpointPlugins = vi.hoisted(() => [ }, ]); -vi.mock("../plugins/plugin-registry.js", () => ({ - loadPluginManifestRegistryForPluginRegistry: () => ({ - plugins: providerEndpointPlugins, - diagnostics: [], - }), +const providerMetadataState = vi.hoisted(() => ({ + compatible: true, + snapshot: undefined as unknown, })); -vi.mock("../plugins/manifest-metadata-scan.js", () => ({ - listOpenClawPluginManifestMetadata: () => - providerEndpointPlugins.map((manifest, index) => ({ - pluginDir: `provider-endpoint-fixture-${index}`, - manifest, - origin: "bundled", - })), +vi.mock("../plugins/current-plugin-metadata-snapshot.js", () => ({ + getCurrentPluginMetadataSnapshot: (params?: { + config?: unknown; + requireDefaultDiscoveryContext?: boolean; + }) => + params?.config !== undefined || + (params?.requireDefaultDiscoveryContext && !providerMetadataState.compatible) + ? undefined + : (providerMetadataState.snapshot ?? { + owners: { + providerEndpoints: providerEndpointPlugins.flatMap((manifest) => + (manifest.providerEndpoints ?? []).map((endpoint) => + Object.assign({}, endpoint, { + hosts: endpoint.hosts ?? [], + hostSuffixes: endpoint.hostSuffixes ?? [], + baseUrls: (endpoint.baseUrls ?? []).map((baseUrl) => + baseUrl.toLowerCase().replace(/\/+$/, ""), + ), + }), + ), + ), + providerRequests: new Map( + providerEndpointPlugins.flatMap((manifest) => + Object.entries(manifest.providerRequest?.providers ?? {}), + ), + ), + }, + }), })); import { @@ -168,6 +187,73 @@ function listProviderAttributionPolicies(env: ProviderAttributionTestEnv) { } describe("provider attribution", () => { + afterEach(() => { + providerMetadataState.compatible = true; + providerMetadataState.snapshot = undefined; + }); + + it("uses provider facts from the replacement plugin snapshot after reload", () => { + providerMetadataState.snapshot = { + owners: { + providerEndpoints: [ + { + endpointClass: "openai-public", + hosts: ["reload.example.com"], + hostSuffixes: [], + baseUrls: [], + }, + ], + providerRequests: new Map([["reload", { family: "before-reload" }]]), + }, + }; + expect(resolveProviderEndpoint("https://reload.example.com").endpointClass).toBe( + "openai-public", + ); + expect(resolveProviderRequestPolicy({ provider: "reload" }).knownProviderFamily).toBe( + "before-reload", + ); + + providerMetadataState.snapshot = { + owners: { + providerEndpoints: [ + { + endpointClass: "anthropic-public", + hosts: ["reload.example.com"], + hostSuffixes: [], + baseUrls: [], + }, + ], + providerRequests: new Map([["reload", { family: "after-reload" }]]), + }, + }; + + expect(resolveProviderEndpoint("https://reload.example.com").endpointClass).toBe( + "anthropic-public", + ); + expect(resolveProviderRequestPolicy({ provider: "reload" }).knownProviderFamily).toBe( + "after-reload", + ); + }); + + it("rejects provider facts from a scoped current snapshot", () => { + providerMetadataState.compatible = false; + providerMetadataState.snapshot = { + owners: { + providerEndpoints: [ + { + endpointClass: "openai-public", + hosts: ["scoped-only.example"], + hostSuffixes: [], + baseUrls: [], + }, + ], + providerRequests: new Map(), + }, + }; + + expect(resolveProviderEndpoint("https://scoped-only.example").endpointClass).toBe("custom"); + }); + it("resolves the canonical OpenClaw product and runtime version", () => { const identity = resolveProviderAttributionIdentity({ OPENCLAW_VERSION: "2026.3.99", @@ -861,6 +947,13 @@ describe("provider attribution", () => { }); }); + it("classifies WebSocket provider URLs by hostname", () => { + expectRecordFields(resolveProviderEndpoint("wss://api.openai.com/v1/realtime"), { + endpointClass: "openai-public", + hostname: "api.openai.com", + }); + }); + it("classifies Google Gemini and Vertex endpoints separately from custom hosts", () => { expectRecordFields(resolveProviderEndpoint("https://generativelanguage.googleapis.com"), { endpointClass: "google-generative-ai", diff --git a/src/agents/provider-attribution.ts b/src/agents/provider-attribution.ts index 7fe6c95699c3..62a5cd6d6499 100644 --- a/src/agents/provider-attribution.ts +++ b/src/agents/provider-attribution.ts @@ -1,18 +1,15 @@ -/** - * Provider endpoint attribution and request capability resolver. - * - * Classifies provider routes so transports know which attribution headers, payload features, and endpoint policies apply. - */ import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; -import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { - normalizeLowercaseStringOrEmpty, normalizeOptionalLowercaseString, normalizeOptionalString, } from "@openclaw/normalization-core/string-coerce"; -import { normalizeTrimmedStringList } from "@openclaw/normalization-core/string-normalization"; -import { listOpenClawPluginManifestMetadata } from "../plugins/manifest-metadata-scan.js"; -import { listOfficialExternalProviderEndpointManifests } from "../plugins/official-external-provider-endpoints.js"; +import { getCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-snapshot.js"; +import type { + PluginManifestProviderEndpoint, + PluginManifestProviderRequestProvider, +} from "../plugins/manifest.js"; +import { normalizePluginProviderBaseUrl } from "../plugins/plugin-metadata-provider-facts.js"; +import { loadPluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; import { asBoolean } from "../utils/boolean.js"; import type { RuntimeVersionEnv } from "../version.js"; import { resolveRuntimeServiceVersion } from "../version.js"; @@ -155,226 +152,53 @@ const OPENAI_RESPONSES_APIS = new Set([ "openai-chatgpt-responses", ]); const OPENAI_RESPONSES_PROVIDERS = new Set(["openai", "azure-openai", "azure-openai-responses"]); -const MANIFEST_PROVIDER_ENDPOINT_CLASSES = new Set([ - "anthropic-public", - "cerebras-native", - "chutes-native", - "deepseek-native", - "github-copilot-native", - "groq-native", - "meta-native", - "mistral-public", - "minimax-native", - "moonshot-native", - "modelstudio-native", - "nvidia-native", - "openai-public", - "openai", - "opencode-native", - "azure-openai", - "openrouter", - "xai-native", - "xiaomi-native", - "zai-native", - "google-generative-ai", - "google-vertex", -]); -type ManifestProviderEndpointCacheEntry = { - endpointClass: ProviderEndpointClass; - hosts: readonly string[]; - hostSuffixes: readonly string[]; - normalizedBaseUrls: readonly string[]; - googleVertexRegion?: string; - googleVertexRegionHostSuffix?: string; -}; -type ManifestProviderRequestCacheEntry = { - family?: string; - compatibilityFamily?: ProviderRequestCompatibilityFamily; - supportsOpenAICompletionsStreamingUsageCompat?: boolean; -}; -let manifestProviderEndpointCache: ManifestProviderEndpointCacheEntry[] | null = null; -let manifestProviderRequestCache: Map | null = null; function formatOpenClawUserAgent(version: string): string { return `${OPENCLAW_ATTRIBUTION_ORIGINATOR}/${version}`; } -function tryParseHostname(value: string): string | undefined { - try { - return normalizeOptionalLowercaseString(new URL(value).hostname); - } catch { - return undefined; - } -} - -function isSchemelessHostnameCandidate(value: string): boolean { - return /^[a-z0-9.[\]-]+(?::\d+)?(?:[/?#].*)?$/i.test(value); -} - function resolveUrlHostname(value: unknown): string | undefined { const trimmed = normalizeOptionalString(value); if (!trimmed) { return undefined; } - const parsedHostname = tryParseHostname(trimmed); - if (parsedHostname) { - return parsedHostname; - } - if (!isSchemelessHostnameCandidate(trimmed)) { - return undefined; - } - return tryParseHostname(`https://${trimmed}`); -} - -function normalizeComparableBaseUrl(value: string): string | undefined { - const trimmed = normalizeOptionalString(value); - if (!trimmed) { - return undefined; - } - - const parsedValue = - tryParseHostname(trimmed) || !isSchemelessHostnameCandidate(trimmed) - ? trimmed - : `https://${trimmed}`; + const candidate = /^[a-z0-9.[\]-]+(?::\d+)?(?:[/?#].*)?$/i.test(trimmed) + ? `https://${trimmed}` + : trimmed; try { - const url = new URL(parsedValue); - if (url.protocol !== "http:" && url.protocol !== "https:") { - return undefined; - } - url.hash = ""; - url.search = ""; - return normalizeOptionalLowercaseString(url.toString().replace(/\/+$/, "")); + return normalizeOptionalLowercaseString(new URL(candidate).hostname); } catch { return undefined; } } -function isManifestProviderEndpointClass(value: string): value is ProviderEndpointClass { - return MANIFEST_PROVIDER_ENDPOINT_CLASSES.has(value as ProviderEndpointClass); -} +type ProviderMetadataOwners = { + providerEndpoints: readonly PluginManifestProviderEndpoint[]; + providerRequests: ReadonlyMap; +}; -function readManifestProviderEndpoints( - manifest: Record, -): ManifestProviderEndpointCacheEntry[] { - if (!Array.isArray(manifest.providerEndpoints)) { - return []; +function resolveProviderMetadataOwners(): ProviderMetadataOwners { + const current = getCurrentPluginMetadataSnapshot({ + allowWorkspaceScopedSnapshot: true, + requireDefaultDiscoveryContext: true, + }); + if (current) { + return { + providerEndpoints: current.owners?.providerEndpoints ?? [], + providerRequests: current.owners?.providerRequests ?? new Map(), + }; } - const entries: ManifestProviderEndpointCacheEntry[] = []; - for (const rawEndpoint of manifest.providerEndpoints) { - if (!isRecord(rawEndpoint)) { - continue; - } - const endpointClassRaw = normalizeOptionalString(rawEndpoint.endpointClass); - if (!endpointClassRaw || !isManifestProviderEndpointClass(endpointClassRaw)) { - continue; - } - entries.push({ - endpointClass: endpointClassRaw, - hosts: normalizeTrimmedStringList(rawEndpoint.hosts).map((host) => host.toLowerCase()), - hostSuffixes: normalizeTrimmedStringList(rawEndpoint.hostSuffixes).map((host) => - host.toLowerCase(), - ), - normalizedBaseUrls: normalizeTrimmedStringList(rawEndpoint.baseUrls) - .map((baseUrl) => normalizeComparableBaseUrl(baseUrl)) - .filter((baseUrl): baseUrl is string => baseUrl !== undefined), - ...(normalizeOptionalString(rawEndpoint.googleVertexRegion) - ? { googleVertexRegion: normalizeOptionalString(rawEndpoint.googleVertexRegion) } - : {}), - ...(normalizeOptionalString(rawEndpoint.googleVertexRegionHostSuffix) - ? { - googleVertexRegionHostSuffix: normalizeOptionalString( - rawEndpoint.googleVertexRegionHostSuffix, - ), - } - : {}), - }); - } - return entries; -} - -function readManifestProviderRequests( - manifest: Record, -): Array<[string, ManifestProviderRequestCacheEntry]> { - const providerRequest = manifest.providerRequest; - if (!isRecord(providerRequest) || !isRecord(providerRequest.providers)) { - return []; - } - const entries: Array<[string, ManifestProviderRequestCacheEntry]> = []; - for (const [providerRaw, requestRaw] of Object.entries(providerRequest.providers)) { - if (!isRecord(requestRaw)) { - continue; - } - const provider = normalizeLowercaseStringOrEmpty(providerRaw); - if (!provider) { - continue; - } - const compatibilityFamily = - normalizeOptionalString(requestRaw.compatibilityFamily) === "moonshot" - ? "moonshot" - : undefined; - const supportsStreamingUsage = isRecord(requestRaw.openAICompletions) - ? requestRaw.openAICompletions.supportsStreamingUsage - : undefined; - entries.push([ - provider, - { - ...(normalizeOptionalString(requestRaw.family) - ? { family: normalizeOptionalString(requestRaw.family) } - : {}), - ...(compatibilityFamily ? { compatibilityFamily } : {}), - ...(typeof supportsStreamingUsage === "boolean" - ? { supportsOpenAICompletionsStreamingUsageCompat: supportsStreamingUsage } - : {}), - }, - ]); - } - return entries; -} - -function collectManifestProviderEndpoints(): ManifestProviderEndpointCacheEntry[] { - const entries: ManifestProviderEndpointCacheEntry[] = []; - for (const { manifest } of listOpenClawPluginManifestMetadata()) { - entries.push(...readManifestProviderEndpoints(manifest)); - } - // Externalized official provider plugins are excluded from dist builds, so - // their manifests are invisible unless installed. The bundled catalog keeps - // their endpoint classes resolvable: users can point a generic provider key - // at DashScope/Moonshot/... and still need native request policy. Matching - // is first-wins, so installed/bundled manifests stay authoritative. - for (const manifest of listOfficialExternalProviderEndpointManifests()) { - entries.push(...readManifestProviderEndpoints(manifest)); - } - return entries; -} - -function collectManifestProviderRequests(): Map { - const entries = new Map(); - for (const { manifest } of listOpenClawPluginManifestMetadata()) { - for (const [provider, request] of readManifestProviderRequests(manifest)) { - entries.set(provider, request); - } - } - return entries; -} - -function loadManifestProviderEndpointCache(): ManifestProviderEndpointCacheEntry[] { - if (!manifestProviderEndpointCache) { - manifestProviderEndpointCache = collectManifestProviderEndpoints(); - } - return manifestProviderEndpointCache; -} - -function loadManifestProviderRequestCache(): Map { - if (!manifestProviderRequestCache) { - manifestProviderRequestCache = collectManifestProviderRequests(); - } - return manifestProviderRequestCache; + const fallback = loadPluginMetadataSnapshot({ config: {} }).owners; + return { + providerEndpoints: fallback.providerEndpoints ?? [], + providerRequests: fallback.providerRequests ?? new Map(), + }; } function resolveManifestProviderRequest( provider: string | undefined, -): ManifestProviderRequestCacheEntry | undefined { - return provider ? loadManifestProviderRequestCache().get(provider) : undefined; +): PluginManifestProviderRequestProvider | undefined { + return provider ? resolveProviderMetadataOwners().providerRequests.get(provider) : undefined; } function hostMatchesSuffix(host: string, suffix: string): boolean { @@ -387,7 +211,7 @@ function hostMatchesSuffix(host: string, suffix: string): boolean { } function buildManifestEndpointResolution( - endpoint: ManifestProviderEndpointCacheEntry, + endpoint: PluginManifestProviderEndpoint, host: string, ): ProviderEndpointResolution { const regionSuffix = endpoint.googleVertexRegionHostSuffix; @@ -395,7 +219,7 @@ function buildManifestEndpointResolution( endpoint.googleVertexRegion ?? (regionSuffix && host.endsWith(regionSuffix) ? host.slice(0, -regionSuffix.length) : undefined); return { - endpointClass: endpoint.endpointClass, + endpointClass: endpoint.endpointClass as ProviderEndpointClass, hostname: host, ...(googleVertexRegion ? { googleVertexRegion } : {}), }; @@ -405,17 +229,14 @@ function resolveManifestProviderEndpoint(params: { host: string; normalizedBaseUrl?: string; }): ProviderEndpointResolution | undefined { - for (const endpoint of loadManifestProviderEndpointCache()) { - if (endpoint.hosts.includes(params.host)) { + for (const endpoint of resolveProviderMetadataOwners().providerEndpoints) { + if ((endpoint.hosts ?? []).includes(params.host)) { return buildManifestEndpointResolution(endpoint, params.host); } - if (endpoint.hostSuffixes.some((suffix) => hostMatchesSuffix(params.host, suffix))) { + if ((endpoint.hostSuffixes ?? []).some((suffix) => hostMatchesSuffix(params.host, suffix))) { return buildManifestEndpointResolution(endpoint, params.host); } - if ( - params.normalizedBaseUrl && - endpoint.normalizedBaseUrls.includes(params.normalizedBaseUrl) - ) { + if (params.normalizedBaseUrl && (endpoint.baseUrls ?? []).includes(params.normalizedBaseUrl)) { return buildManifestEndpointResolution(endpoint, params.host); } } @@ -442,7 +263,7 @@ export function resolveProviderEndpoint( if (!host) { return { endpointClass: "invalid" }; } - const normalizedBaseUrl = normalizeComparableBaseUrl(baseUrl); + const normalizedBaseUrl = normalizePluginProviderBaseUrl(baseUrl); const manifestEndpoint = resolveManifestProviderEndpoint({ host, normalizedBaseUrl }); if (manifestEndpoint) { return manifestEndpoint; @@ -806,7 +627,7 @@ export function resolveProviderRequestCapabilities( supportsNativeStreamingUsageCompat: endpointClass === "moonshot-native" || endpointClass === "modelstudio-native", supportsOpenAICompletionsStreamingUsageCompat: - manifestProviderRequest?.supportsOpenAICompletionsStreamingUsageCompat === true, + manifestProviderRequest?.openAICompletions?.supportsStreamingUsage === true, compatibilityFamily, }; } @@ -865,4 +686,3 @@ export function describeProviderRequestRoutingSummary( `policy=${routingPolicy}`, ].join(" "); } -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/agents/runtime-plugins.ts b/src/agents/runtime-plugins.ts index 45049a7173da..b70611864a30 100644 --- a/src/agents/runtime-plugins.ts +++ b/src/agents/runtime-plugins.ts @@ -1,10 +1,7 @@ -/** - * Ensures runtime plugin registries are loaded for agent execution. Startup - * plugin IDs from metadata scope the load when available. - */ import type { OpenClawConfig } from "../config/types.openclaw.js"; import { normalizePluginsConfig } from "../plugins/config-state.js"; import { getCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-snapshot.js"; +import type { PluginRegistry } from "../plugins/registry-types.js"; import { getActivePluginRuntimeSubagentMode } from "../plugins/runtime.js"; import { ensureStandaloneRuntimePluginRegistryLoaded } from "../plugins/runtime/standalone-runtime-registry-loader.js"; import { resolveUserPath } from "../utils.js"; @@ -37,9 +34,9 @@ export function ensureRuntimePluginsLoaded(params: { config?: OpenClawConfig; workspaceDir?: string | null; allowGatewaySubagentBinding?: boolean; -}): void { +}): PluginRegistry | undefined { if (params.config && !normalizePluginsConfig(params.config.plugins).enabled) { - return; + return undefined; } const workspaceDir = typeof params.workspaceDir === "string" && params.workspaceDir.trim() @@ -52,7 +49,7 @@ export function ensureRuntimePluginsLoaded(params: { const allowGatewaySubagentBinding = params.allowGatewaySubagentBinding === true || getActivePluginRuntimeSubagentMode() === "gateway-bindable"; - ensureStandaloneRuntimePluginRegistryLoaded({ + return ensureStandaloneRuntimePluginRegistryLoaded({ requiredPluginIds: startupPluginIds, loadOptions: { config: params.config, diff --git a/src/agents/tools/image-generate-tool.ts b/src/agents/tools/image-generate-tool.ts index 40b224424cda..4118d2989205 100644 --- a/src/agents/tools/image-generate-tool.ts +++ b/src/agents/tools/image-generate-tool.ts @@ -1,8 +1,4 @@ -/** - * image_generate built-in tool. - * - * Loads references, resolves providers/options, saves generated images, and supports detached background runs. - */ +/** Runs image generation, persistence, and detached completion. */ import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; import { Type } from "typebox"; import { findCapabilityProviderById } from "../../../packages/media-generation-core/src/capability-model-ref.js"; @@ -53,6 +49,7 @@ import { buildMediaGenerationRequestKey, recordRecentMediaGenerationTaskStartForSession, } from "../media-generation-task-status-shared.js"; +import type { PreparedModelRuntimeSnapshot } from "../prepared-model-runtime.js"; import { optionalStringEnum } from "../schema/string-enum.js"; import { ToolInputError, @@ -87,6 +84,7 @@ import { applyImageGenerationModelConfigDefaults, buildMediaReferenceDetails, buildTaskRunDetails, + createCapabilityProviderRuntimeDeps, hasGenerationToolAvailability, normalizeMediaReferenceInputs, readGenerationTimeoutMs, @@ -756,6 +754,7 @@ async function executeImageGenerationJob(params: { loadedReferenceImages: LoadedReferenceImage[]; taskHandle?: ImageGenerationTaskHandle | null; autoProviderFallback?: boolean; + providers?: ImageGenerationProvider[]; }) { if (params.taskHandle) { recordImageGenerationTaskProgress({ @@ -763,25 +762,28 @@ async function executeImageGenerationJob(params: { progressSummary: "Generating image", }); } - const result = await generateImage({ - cfg: params.effectiveCfg, - prompt: params.prompt, - agentDir: params.agentDir, - modelOverride: params.model, - autoProviderFallback: params.autoProviderFallback, - size: params.size, - aspectRatio: params.aspectRatio, - resolution: params.resolution, - inferredResolution: params.inferredResolution, - quality: params.quality, - outputFormat: params.outputFormat, - background: params.background, - count: params.count, - inputImages: params.inputImages, - timeoutMs: params.timeoutMs, - providerOptions: params.providerOptions, - ssrfPolicy: params.ssrfPolicy, - }); + const result = await generateImage( + { + cfg: params.effectiveCfg, + prompt: params.prompt, + agentDir: params.agentDir, + modelOverride: params.model, + autoProviderFallback: params.autoProviderFallback, + size: params.size, + aspectRatio: params.aspectRatio, + resolution: params.resolution, + inferredResolution: params.inferredResolution, + quality: params.quality, + outputFormat: params.outputFormat, + background: params.background, + count: params.count, + inputImages: params.inputImages, + timeoutMs: params.timeoutMs, + providerOptions: params.providerOptions, + ssrfPolicy: params.ssrfPolicy, + }, + createCapabilityProviderRuntimeDeps(params.providers), + ); if (params.taskHandle) { recordImageGenerationTaskProgress({ handle: params.taskHandle, @@ -902,12 +904,17 @@ export function createImageGenerateTool(options?: { agentSessionKey?: string; requesterOrigin?: DeliveryContext; workspaceDir?: string; + preparedModelRuntime?: PreparedModelRuntimeSnapshot; sandbox?: ImageGenerateSandboxConfig; fsPolicy?: ToolFsPolicy; scheduleBackgroundWork?: MediaGenerateBackgroundScheduler; onAsyncTaskStarted?: MediaGenerateAsyncStartCallback; }): AnyAgentTool | null { const cfg = options?.config ?? getRuntimeConfig(); + const preparedProviders = options?.preparedModelRuntime?.mediaCapabilityProviders + ?.imageGenerationProviders + ? [...options.preparedModelRuntime.mediaCapabilityProviders.imageGenerationProviders] + : undefined; if ( !hasGenerationToolAvailability({ cfg, @@ -916,6 +923,7 @@ export function createImageGenerateTool(options?: { authStore: options?.authProfileStore, modelConfig: cfg.agents?.defaults?.mediaModels?.image, providerKey: "imageGenerationProviders", + providers: preparedProviders, }) ) { return null; @@ -990,9 +998,8 @@ export function createImageGenerateTool(options?: { const outputFormat = normalizeOutputFormat(readStringParam(params, "outputFormat")); const background = normalizeBackground(readStringParam(params, "background")); const providerOptions = normalizeProviderOptions(params); - const imageGenerationProviders = listRuntimeImageGenerationProviders({ - config: effectiveCfg, - }); + const imageGenerationProviders = + preparedProviders ?? listRuntimeImageGenerationProviders({ config: effectiveCfg }); const selectedProvider = resolveSelectedImageGenerationProvider({ providers: imageGenerationProviders, imageGenerationModelConfig, @@ -1146,6 +1153,7 @@ export function createImageGenerateTool(options?: { loadedReferenceImages, taskHandle, autoProviderFallback: explicitModelConfig ? false : undefined, + providers: imageGenerationProviders, }), }); @@ -1204,6 +1212,7 @@ export function createImageGenerateTool(options?: { loadedReferenceImages, taskHandle, autoProviderFallback: explicitModelConfig ? false : undefined, + providers: imageGenerationProviders, }); completeImageGenerationTaskRun({ handle: taskHandle, diff --git a/src/agents/tools/image-tool.test-support.ts b/src/agents/tools/image-tool.test-support.ts index a928f2c3c500..0915fac1d5a0 100644 --- a/src/agents/tools/image-tool.test-support.ts +++ b/src/agents/tools/image-tool.test-support.ts @@ -15,6 +15,7 @@ import type { } from "../../plugin-sdk/media-understanding.js"; import type { AuthProfileStore } from "../auth-profiles/types.js"; import type { resolveBundledStaticCatalogModel } from "../embedded-agent-runner/model.static-catalog.js"; +import type { PreparedModelRuntimeSnapshot } from "../prepared-model-runtime.js"; import type { coerceImageAssistantText, decodeDataUrl, @@ -84,6 +85,7 @@ type ImageToolTestApi = { agentDir: string; workspaceDir?: string; authStore?: AuthProfileStore; + preparedModelRuntime?: PreparedModelRuntimeSnapshot; }): ImageModelConfig | null; }; diff --git a/src/agents/tools/image-tool.test.ts b/src/agents/tools/image-tool.test.ts index 12043db63f44..ed1830ce5a71 100644 --- a/src/agents/tools/image-tool.test.ts +++ b/src/agents/tools/image-tool.test.ts @@ -980,6 +980,43 @@ describe("image tool implicit imageModel config", () => { }); }); + it("does not mix a prepared media family with a live Codex provider", async () => { + await withTempAgentDir(async (agentDir) => { + await writeProfiles(agentDir, { "openai:chatgpt": openAiOAuthProfile() }); + installImageUnderstandingProviderStubs(minimaxProvider, moonshotProvider, codexMediaProvider); + + expect( + resolveImageModelConfigForTool({ + cfg: openAiPrimaryCfg, + agentDir, + preparedModelRuntime: { + mediaCapabilityProviders: { mediaUnderstandingProviders: [] }, + } as never, + }), + ).toBeNull(); + }); + }); + + it("resolves a Codex alias from the prepared media family", async () => { + await withTempAgentDir(async (agentDir) => { + await writeProfiles(agentDir, { "openai:chatgpt": openAiOAuthProfile() }); + + expect( + resolveImageModelConfigForTool({ + cfg: openAiPrimaryCfg, + agentDir, + preparedModelRuntime: { + mediaCapabilityProviders: { + mediaUnderstandingProviders: [ + { ...codexMediaProvider, id: "codex-owner", aliases: ["codex"] }, + ], + }, + } as never, + }), + ).toEqual(codexImageModel); + }); + }); + it.each(implicitImageRoutingCases)( "$name", async ({ cfg, profiles, codexProvider, openAiApiKey, expected }) => { diff --git a/src/agents/tools/image-tool.ts b/src/agents/tools/image-tool.ts index 79ec53c5242f..854752b3b2b6 100644 --- a/src/agents/tools/image-tool.ts +++ b/src/agents/tools/image-tool.ts @@ -1,10 +1,6 @@ -/** - * image built-in tool. - * - * Describes local, staged, web, and generated media through configured media-understanding providers. - */ import { resolve, isAbsolute } from "node:path"; import { Type } from "typebox"; +import { findCapabilityProviderById } from "../../../packages/media-generation-core/src/capability-model-ref.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import type { MediaUnderstandingModelConfig } from "../../config/types.tools.js"; import { @@ -263,6 +259,7 @@ function resolveImageModelConfigForTool(params: { agentDir: string; workspaceDir?: string; authStore?: AuthProfileStore; + preparedModelRuntime?: PreparedModelRuntimeSnapshot; }): ImageModelConfig | null { // Native-vision runs route post-prompt image bytes to the active model, not fallback config. const explicit = coerceImageModelConfig(params.cfg); @@ -276,10 +273,18 @@ function resolveImageModelConfigForTool(params: { const primary = resolveDefaultModelRef(params.cfg); let verifiedSubstituteProvider: string | undefined; const resolveCodexMediaRoute = () => { - const provider = imageToolProviderDeps.resolveRegisteredMediaUnderstandingProvider({ - providerId: "codex", - cfg: params.cfg, - }); + const preparedProviders = + params.preparedModelRuntime?.mediaCapabilityProviders?.mediaUnderstandingProviders; + const provider = preparedProviders + ? findCapabilityProviderById({ + providers: preparedProviders, + providerId: "codex", + normalizeProviderId: normalizeMediaProviderId, + }) + : imageToolProviderDeps.resolveRegisteredMediaUnderstandingProvider({ + providerId: "codex", + cfg: params.cfg, + }); if (!provider?.capabilities?.includes("image")) { return undefined; } @@ -686,7 +691,13 @@ async function runImagePrompt(params: { }> { const effectiveCfg = applyImageModelConfigDefaults(params.cfg, params.imageModelConfig); const providerCfg: OpenClawConfig = effectiveCfg ?? {}; - const providerRegistry = imageToolProviderDeps.buildProviderRegistry(undefined, providerCfg); + const preparedProviders = + params.preparedModelRuntime?.mediaCapabilityProviders?.mediaUnderstandingProviders; + const providerRegistry = imageToolProviderDeps.buildProviderRegistry( + undefined, + providerCfg, + preparedProviders, + ); const result = await runWithImageModelFallback({ cfg: effectiveCfg, @@ -844,6 +855,7 @@ export function createImageTool(options?: { agentDir, workspaceDir: options?.workspaceDir, authStore: options?.authProfileStore, + preparedModelRuntime: options?.preparedModelRuntime, }) : explicitImageModelConfig; if (!modelHasVision && !resolvedImageModelConfig && !options?.deferAutoModelResolution) { @@ -948,6 +960,7 @@ export function createImageTool(options?: { agentDir, workspaceDir: options?.workspaceDir, authStore: options?.authProfileStore, + preparedModelRuntime: options?.preparedModelRuntime, }); if (!imageModelConfig) { throw new Error( diff --git a/src/agents/tools/media-tool-shared.ts b/src/agents/tools/media-tool-shared.ts index 76fd5e696be1..ecb03e9bd968 100644 --- a/src/agents/tools/media-tool-shared.ts +++ b/src/agents/tools/media-tool-shared.ts @@ -1,8 +1,4 @@ -/** - * Shared media tool helpers. - * - * Resolves provider/model config, local roots, auth availability, SSRF policy, and media reference inputs. - */ +/** Shared media tool routing, auth, path, and reference helpers. */ import { normalizeInboundPathRoots } from "@openclaw/media-core/inbound-path-policy"; import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; import { parseBoolean } from "@openclaw/normalization-core/boolean-coercion"; @@ -239,6 +235,19 @@ export function isCapabilityProviderConfigured(par }); } +export function createCapabilityProviderRuntimeDeps( + providers: readonly T[] | undefined, +) { + const prepared = providers ? [...providers] : undefined; + return prepared + ? { + getProvider: (providerId?: string) => + findCapabilityProviderById({ providers: prepared, providerId, normalizeProviderId }), + listProviders: () => prepared, + } + : undefined; +} + /** * Resolves the provider implied by a model override or configured primary model. */ diff --git a/src/agents/tools/music-generate-tool.ts b/src/agents/tools/music-generate-tool.ts index ee1e98e48366..fc48c1ae35af 100644 --- a/src/agents/tools/music-generate-tool.ts +++ b/src/agents/tools/music-generate-tool.ts @@ -1,8 +1,4 @@ -/** - * music_generate built-in tool. - * - * Resolves music providers/options, saves generated tracks, and supports detached background runs. - */ +/** Runs music generation, persistence, and detached completion. */ import { normalizeOptionalLowercaseString } from "@openclaw/normalization-core/string-coerce"; import { Type } from "typebox"; import { getRuntimeConfig } from "../../config/config.js"; @@ -40,6 +36,7 @@ import { buildMediaGenerationRequestKey, recordRecentMediaGenerationTaskStartForSession, } from "../media-generation-task-status-shared.js"; +import type { PreparedModelRuntimeSnapshot } from "../prepared-model-runtime.js"; import { ToolInputError, readNumberParam, readStringParam } from "./common.js"; import { decodeDataUrl } from "./image-tool.helpers.js"; import { @@ -55,6 +52,7 @@ import { applyMusicGenerationModelConfigDefaults, buildMediaReferenceDetails, buildTaskRunDetails, + createCapabilityProviderRuntimeDeps, hasGenerationToolAvailability, normalizeMediaReferenceInputs, readBooleanToolParam, @@ -170,11 +168,12 @@ function hasExplicitMusicGenerationModelConfig(cfg?: OpenClawConfig): boolean { function resolveSelectedMusicGenerationProvider(params: { config?: OpenClawConfig; + providers?: MusicGenerationProvider[]; musicGenerationModelConfig: ToolModelConfig; modelOverride?: string; }): MusicGenerationProvider | undefined { return resolveSelectedCapabilityProvider({ - providers: listRuntimeMusicGenerationProviders({ config: params.config }), + providers: params.providers ?? listRuntimeMusicGenerationProviders({ config: params.config }), modelConfig: params.musicGenerationModelConfig, modelOverride: params.modelOverride, parseModelRef: parseMusicGenerationModelRef, @@ -429,6 +428,7 @@ async function executeMusicGenerationJob(params: { autoProviderFallback?: boolean; timeoutMs?: number; timeoutNormalization?: MusicGenerationTimeoutNormalization; + providers?: MusicGenerationProvider[]; }): Promise { if (params.taskHandle) { recordMusicGenerationTaskProgress({ @@ -436,19 +436,22 @@ async function executeMusicGenerationJob(params: { progressSummary: "Generating music", }); } - const result = await generateMusic({ - cfg: params.effectiveCfg, - prompt: params.prompt, - agentDir: params.agentDir, - modelOverride: params.model, - lyrics: params.lyrics, - instrumental: params.instrumental, - durationSeconds: params.durationSeconds, - format: params.format, - inputImages: params.loadedReferenceImages.map((entry) => entry.sourceImage), - autoProviderFallback: params.autoProviderFallback, - timeoutMs: params.timeoutMs, - }); + const result = await generateMusic( + { + cfg: params.effectiveCfg, + prompt: params.prompt, + agentDir: params.agentDir, + modelOverride: params.model, + lyrics: params.lyrics, + instrumental: params.instrumental, + durationSeconds: params.durationSeconds, + format: params.format, + inputImages: params.loadedReferenceImages.map((entry) => entry.sourceImage), + autoProviderFallback: params.autoProviderFallback, + timeoutMs: params.timeoutMs, + }, + createCapabilityProviderRuntimeDeps(params.providers), + ); if (params.taskHandle) { recordMusicGenerationTaskProgress({ handle: params.taskHandle, @@ -578,12 +581,17 @@ export function createMusicGenerateTool(options?: { agentSessionKey?: string; requesterOrigin?: DeliveryContext; workspaceDir?: string; + preparedModelRuntime?: PreparedModelRuntimeSnapshot; sandbox?: MusicGenerateSandboxConfig; fsPolicy?: ToolFsPolicy; scheduleBackgroundWork?: MediaGenerateBackgroundScheduler; onAsyncTaskStarted?: MediaGenerateAsyncStartCallback; }): AnyAgentTool | null { const cfg: OpenClawConfig = options?.config ?? getRuntimeConfig(); + const preparedProviders = options?.preparedModelRuntime?.mediaCapabilityProviders + ?.musicGenerationProviders + ? [...options.preparedModelRuntime.mediaCapabilityProviders.musicGenerationProviders] + : undefined; if ( !hasGenerationToolAvailability({ cfg, @@ -592,6 +600,7 @@ export function createMusicGenerateTool(options?: { authStore: options?.authProfileStore, modelConfig: cfg.agents?.defaults?.mediaModels?.music, providerKey: "musicGenerationProviders", + providers: preparedProviders, }) ) { return null; @@ -680,6 +689,7 @@ export function createMusicGenerateTool(options?: { const selectedProvider = shouldResolveSelectedProvider ? resolveSelectedMusicGenerationProvider({ config: effectiveCfg, + providers: preparedProviders, musicGenerationModelConfig, modelOverride: model, }) @@ -771,6 +781,7 @@ export function createMusicGenerateTool(options?: { autoProviderFallback: explicitModelConfig ? false : undefined, timeoutMs, timeoutNormalization: timeout.normalization, + providers: preparedProviders, }), }); @@ -829,6 +840,7 @@ export function createMusicGenerateTool(options?: { autoProviderFallback: explicitModelConfig ? false : undefined, timeoutMs, timeoutNormalization: timeout.normalization, + providers: preparedProviders, }); completeMusicGenerationTaskRun({ handle: taskHandle, diff --git a/src/agents/tools/video-generate-tool.ts b/src/agents/tools/video-generate-tool.ts index 883d46a113d2..fd2a1eeefad4 100644 --- a/src/agents/tools/video-generate-tool.ts +++ b/src/agents/tools/video-generate-tool.ts @@ -1,8 +1,4 @@ -/** - * video_generate built-in tool. - * - * Validates media references, resolves provider/model capabilities, and schedules video generation. - */ +/** Runs capability-aware video generation and persistence. */ import { Type, type TSchema } from "typebox"; import { getRuntimeConfig } from "../../config/config.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; @@ -45,6 +41,7 @@ import { recordRecentMediaGenerationTaskStartForSession, } from "../media-generation-task-status-shared.js"; import { getCustomProviderApiKey } from "../model-auth.js"; +import type { PreparedModelRuntimeSnapshot } from "../prepared-model-runtime.js"; import { resolveProviderIdForAuth } from "../provider-auth-aliases.js"; import { ToolInputError, readNumberParam, readStringParam } from "./common.js"; import { decodeDataUrl } from "./image-tool.helpers.js"; @@ -65,6 +62,7 @@ import { applyVideoGenerationModelConfigDefaults, buildMediaReferenceDetails, buildTaskRunDetails, + createCapabilityProviderRuntimeDeps, hasGenerationToolAvailability, normalizeMediaReferenceInputs, readBooleanToolParam, @@ -443,11 +441,12 @@ function normalizeReferenceInputs(params: { function resolveSelectedVideoGenerationProvider(params: { config?: OpenClawConfig; + providers?: VideoGenerationProvider[]; videoGenerationModelConfig: ToolModelConfig; modelOverride?: string; }): VideoGenerationProvider | undefined { return resolveSelectedCapabilityProvider({ - providers: listRuntimeVideoGenerationProviders({ config: params.config }), + providers: params.providers ?? listRuntimeVideoGenerationProviders({ config: params.config }), modelConfig: params.videoGenerationModelConfig, modelOverride: params.modelOverride, parseModelRef: parseVideoGenerationModelRef, @@ -706,6 +705,7 @@ async function executeVideoGenerationJob(params: { providerOptions?: Record; autoProviderFallback?: boolean; timeoutMs?: number; + providers?: VideoGenerationProvider[]; }): Promise { if (params.taskHandle) { recordVideoGenerationTaskProgress({ @@ -713,24 +713,27 @@ async function executeVideoGenerationJob(params: { progressSummary: "Generating video", }); } - const result = await generateVideo({ - cfg: params.effectiveCfg, - prompt: params.prompt, - agentDir: params.agentDir, - modelOverride: params.model, - size: params.size, - aspectRatio: params.aspectRatio, - resolution: params.resolution, - durationSeconds: params.durationSeconds, - audio: params.audio, - watermark: params.watermark, - inputImages: params.loadedReferenceImages.map((entry) => entry.sourceAsset), - inputVideos: params.loadedReferenceVideos.map((entry) => entry.sourceAsset), - inputAudios: params.loadedReferenceAudios.map((entry) => entry.sourceAsset), - autoProviderFallback: params.autoProviderFallback, - providerOptions: params.providerOptions, - timeoutMs: params.timeoutMs, - }); + const result = await generateVideo( + { + cfg: params.effectiveCfg, + prompt: params.prompt, + agentDir: params.agentDir, + modelOverride: params.model, + size: params.size, + aspectRatio: params.aspectRatio, + resolution: params.resolution, + durationSeconds: params.durationSeconds, + audio: params.audio, + watermark: params.watermark, + inputImages: params.loadedReferenceImages.map((entry) => entry.sourceAsset), + inputVideos: params.loadedReferenceVideos.map((entry) => entry.sourceAsset), + inputAudios: params.loadedReferenceAudios.map((entry) => entry.sourceAsset), + autoProviderFallback: params.autoProviderFallback, + providerOptions: params.providerOptions, + timeoutMs: params.timeoutMs, + }, + createCapabilityProviderRuntimeDeps(params.providers), + ); if (params.taskHandle) { recordVideoGenerationTaskProgress({ handle: params.taskHandle, @@ -940,12 +943,17 @@ export function createVideoGenerateTool(options?: { agentSessionKey?: string; requesterOrigin?: DeliveryContext; workspaceDir?: string; + preparedModelRuntime?: PreparedModelRuntimeSnapshot; sandbox?: VideoGenerateSandboxConfig; fsPolicy?: ToolFsPolicy; scheduleBackgroundWork?: MediaGenerateBackgroundScheduler; onAsyncTaskStarted?: MediaGenerateAsyncStartCallback; }): AnyAgentTool | null { const cfg: OpenClawConfig = options?.config ?? getRuntimeConfig(); + const preparedProviders = options?.preparedModelRuntime?.mediaCapabilityProviders + ?.videoGenerationProviders + ? [...options.preparedModelRuntime.mediaCapabilityProviders.videoGenerationProviders] + : undefined; if ( !hasGenerationToolAvailability({ cfg, @@ -954,6 +962,7 @@ export function createVideoGenerateTool(options?: { authStore: options?.authProfileStore, modelConfig: cfg.agents?.defaults?.mediaModels?.video, providerKey: "videoGenerationProviders", + providers: preparedProviders, }) ) { return null; @@ -1092,6 +1101,7 @@ export function createVideoGenerateTool(options?: { const selectedProvider = resolveSelectedVideoGenerationProvider({ config: effectiveCfg, + providers: preparedProviders, videoGenerationModelConfig, modelOverride: model, }); @@ -1236,6 +1246,7 @@ export function createVideoGenerateTool(options?: { providerOptions, autoProviderFallback: explicitModelConfig ? false : undefined, timeoutMs, + providers: preparedProviders, }), }); @@ -1299,6 +1310,7 @@ export function createVideoGenerateTool(options?: { providerOptions, autoProviderFallback: explicitModelConfig ? false : undefined, timeoutMs, + providers: preparedProviders, }); completeVideoGenerationTaskRun({ handle: taskHandle, diff --git a/src/media-understanding/provider-registry.ts b/src/media-understanding/provider-registry.ts index f3ce7f342177..ee39f0c4ef2d 100644 --- a/src/media-understanding/provider-registry.ts +++ b/src/media-understanding/provider-registry.ts @@ -1,5 +1,3 @@ -// Media-understanding provider registry combines plugin capability providers, -// config-derived image providers, and test/runtime overrides. import type { OpenClawConfig } from "../config/types.js"; import { resolvePluginCapabilityProviders } from "../plugins/capability-provider-runtime.js"; import { resolveImageCapableConfigProviderIds } from "./config-provider-models.js"; @@ -52,12 +50,16 @@ export { normalizeMediaExecutionProviderId, normalizeMediaProviderId } from "./p export function buildMediaUnderstandingRegistry( overrides?: Record, cfg?: OpenClawConfig, + preparedProviders?: readonly MediaUnderstandingProvider[], ): Map { const registry = new Map(); - for (const provider of resolvePluginCapabilityProviders({ - key: "mediaUnderstandingProviders", - cfg, - })) { + const providers = + preparedProviders ?? + resolvePluginCapabilityProviders({ + key: "mediaUnderstandingProviders", + cfg, + }); + for (const provider of providers) { mergeProviderIntoRegistry(registry, provider); } // Auto-register media-understanding for config providers with image-capable models (#51392) diff --git a/src/plugins/capability-provider-runtime.test.ts b/src/plugins/capability-provider-runtime.test.ts index 8d2828bbdfbd..818a850db86c 100644 --- a/src/plugins/capability-provider-runtime.test.ts +++ b/src/plugins/capability-provider-runtime.test.ts @@ -50,7 +50,8 @@ vi.mock("./loader.js", () => ({ resolvePluginRegistryLoadCacheKey: mocks.resolvePluginRegistryLoadCacheKey, })); -vi.mock("./active-runtime-registry.js", () => ({ +vi.mock("./active-runtime-registry.js", async (importOriginal) => ({ + ...(await importOriginal()), getLoadedRuntimePluginRegistry: (params?: { requiredPluginIds?: string[] }) => { if (params === undefined) { return mocks.resolveRuntimePluginRegistry(); @@ -124,6 +125,7 @@ vi.mock("./bundled-compat.js", () => ({ let resolvePluginCapabilityProviders: typeof import("./capability-provider-runtime.js").resolvePluginCapabilityProviders; let resolvePluginCapabilityProvider: typeof import("./capability-provider-runtime.js").resolvePluginCapabilityProvider; +let prepareMediaCapabilityProviders: typeof import("./capability-provider-runtime.js").prepareMediaCapabilityProviders; let clearCurrentPluginMetadataSnapshot: typeof import("./current-plugin-metadata-snapshot.js").clearCurrentPluginMetadataSnapshot; let setCurrentPluginMetadataSnapshot: typeof import("./current-plugin-metadata-snapshot.js").setCurrentPluginMetadataSnapshot; let clearPluginMetadataLifecycleCaches: typeof import("./plugin-metadata-lifecycle.js").clearPluginMetadataLifecycleCaches; @@ -282,8 +284,11 @@ function expectCompatChainApplied(params: { describe("resolvePluginCapabilityProviders", () => { beforeAll(async () => { vi.resetModules(); - ({ resolvePluginCapabilityProvider, resolvePluginCapabilityProviders } = - await import("./capability-provider-runtime.js")); + ({ + prepareMediaCapabilityProviders, + resolvePluginCapabilityProvider, + resolvePluginCapabilityProviders, + } = await import("./capability-provider-runtime.js")); ({ clearCurrentPluginMetadataSnapshot, setCurrentPluginMetadataSnapshot } = await import("./current-plugin-metadata-snapshot.js")); ({ clearPluginMetadataLifecycleCaches } = await import("./plugin-metadata-lifecycle.js")); @@ -377,6 +382,127 @@ describe("resolvePluginCapabilityProviders", () => { expect(mocks.loadPluginManifestRegistry).not.toHaveBeenCalled(); }); + it("prepares media capability families from one supplied metadata snapshot", () => { + const loaded = createEmptyPluginRegistry(); + for (const key of [ + "mediaUnderstandingProviders", + "imageGenerationProviders", + "videoGenerationProviders", + "musicGenerationProviders", + ] as const) { + loaded[key].push({ + pluginId: "media", + pluginName: "media", + source: "test", + provider: { id: key }, + } as never); + } + mocks.resolveRuntimePluginRegistry.mockReturnValue(loaded); + const pluginMetadataSnapshot = { + index: { plugins: [] }, + plugins: [ + { + id: "media", + origin: "bundled", + contracts: { + mediaUnderstandingProviders: ["mediaUnderstandingProviders"], + imageGenerationProviders: ["imageGenerationProviders"], + videoGenerationProviders: ["videoGenerationProviders"], + musicGenerationProviders: ["musicGenerationProviders"], + }, + }, + ], + } as never; + const prepared = prepareMediaCapabilityProviders({ pluginMetadataSnapshot, registry: loaded }); + + expect(prepared.mediaUnderstandingProviders?.map((provider) => provider.id)).toEqual([ + "mediaUnderstandingProviders", + ]); + expect(prepared.imageGenerationProviders?.map((provider) => provider.id)).toEqual([ + "imageGenerationProviders", + ]); + expect(prepared.videoGenerationProviders?.map((provider) => provider.id)).toEqual([ + "videoGenerationProviders", + ]); + expect(prepared.musicGenerationProviders?.map((provider) => provider.id)).toEqual([ + "musicGenerationProviders", + ]); + expect(mocks.loadPluginManifestRegistry).not.toHaveBeenCalled(); + }); + + it("leaves a media family unresolved for loaded providers without contracts", () => { + const loaded = createEmptyPluginRegistry(); + loaded.imageGenerationProviders.push({ + pluginId: "legacy-image", + pluginName: "legacy-image", + source: "test", + provider: { id: "legacy-image" }, + } as never); + + const prepared = prepareMediaCapabilityProviders({ + registry: loaded, + pluginMetadataSnapshot: { index: { plugins: [] }, plugins: [] } as never, + }); + + expect(prepared.imageGenerationProviders).toBeUndefined(); + }); + + it("leaves a media family unresolved when an eligible owner is not loaded", () => { + const loaded = createEmptyPluginRegistry(); + loaded.imageGenerationProviders.push({ + pluginId: "loaded-image", + pluginName: "loaded-image", + source: "test", + provider: { id: "loaded-image" }, + } as never); + mocks.resolveRuntimePluginRegistry.mockImplementation((params?: unknown) => + params && (params as { onlyPluginIds?: string[] }).onlyPluginIds?.includes("lazy-image") + ? undefined + : loaded, + ); + + const prepared = prepareMediaCapabilityProviders({ + cfg: { plugins: { allow: ["loaded-image"] } }, + registry: loaded, + pluginMetadataSnapshot: { + index: { plugins: [] }, + plugins: [ + { + id: "loaded-image", + origin: "bundled", + contracts: { imageGenerationProviders: ["loaded-image"] }, + }, + { + id: "lazy-image", + origin: "bundled", + contracts: { imageGenerationProviders: ["lazy-image"] }, + }, + ], + } as never, + }); + + expect(prepared.imageGenerationProviders).toBeUndefined(); + }); + + it("prepares disabled media families as authoritative empty arrays", () => { + const loaded = createEmptyPluginRegistry(); + loaded.imageGenerationProviders.push({ + pluginId: "loaded-image", + pluginName: "loaded-image", + source: "test", + provider: { id: "loaded-image" }, + } as never); + mocks.resolveRuntimePluginRegistry.mockReturnValue(loaded); + + const prepared = prepareMediaCapabilityProviders({ + cfg: { plugins: { enabled: false } }, + registry: loaded, + pluginMetadataSnapshot: { index: { plugins: [] }, plugins: [] } as never, + }); + + expect(prepared.imageGenerationProviders).toEqual([]); + }); + it("resolves enabled external capability plugins from the current metadata snapshot", () => { const loaded = createEmptyPluginRegistry(); loaded.imageGenerationProviders.push({ @@ -1495,40 +1621,6 @@ describe("resolvePluginCapabilityProviders", () => { expect(mocks.loadPluginManifestRegistry).toHaveBeenCalledTimes(1); }); - it("reuses capability snapshot loads for the same config object", () => { - const { cfg, enablementCompat } = createCompatChainConfig(); - const loaded = createEmptyPluginRegistry(); - loaded.mediaUnderstandingProviders.push({ - pluginId: "openai", - pluginName: "openai", - source: "test", - provider: { - id: "openai", - capabilities: ["image"], - }, - } as never); - setBundledCapabilityFixture("mediaUnderstandingProviders"); - mocks.withBundledPluginEnablementCompat.mockReturnValue(enablementCompat); - mocks.withBundledPluginVitestCompat.mockReturnValue(enablementCompat); - mocks.resolveRuntimePluginRegistry.mockImplementation((params?: unknown) => - params === undefined ? undefined : loaded, - ); - - expectResolvedCapabilityProviderIds( - resolvePluginCapabilityProviders({ key: "mediaUnderstandingProviders", cfg }), - ["openai"], - ); - expectResolvedCapabilityProviderIds( - resolvePluginCapabilityProviders({ key: "mediaUnderstandingProviders", cfg }), - ["openai"], - ); - - const snapshotLoads = mocks.resolveRuntimePluginRegistry.mock.calls.filter( - ([options]) => options !== undefined, - ); - expect(snapshotLoads).toHaveLength(1); - }); - it("reuses equivalent manifest metadata while applying bundled compat", () => { const first = createCompatChainConfig(); const second = createCompatChainConfig(); diff --git a/src/plugins/capability-provider-runtime.ts b/src/plugins/capability-provider-runtime.ts index dca648f99b49..2d7085ce82f2 100644 --- a/src/plugins/capability-provider-runtime.ts +++ b/src/plugins/capability-provider-runtime.ts @@ -1,27 +1,21 @@ -/** Resolves plugin capability providers through manifest contracts, bundled compat, and runtime registries. */ import { sortUniqueStrings } from "@openclaw/normalization-core/string-normalization"; import { resolveVoiceModelRefs } from "../../packages/speech-core/voice-models.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { getLoadedRuntimePluginRegistry } from "./active-runtime-registry.js"; +import { + getLoadedRuntimePluginRegistry, + registryContainsRuntimePluginIds, +} from "./active-runtime-registry.js"; import { loadBundledCapabilityRuntimeRegistry } from "./bundled-capability-runtime.js"; import { withBundledPluginEnablementCompat, withBundledPluginVitestCompat, } from "./bundled-compat.js"; -import { - resolvePluginRegistryLoadCacheKey, - resolveRuntimePluginRegistry, - type PluginLoadOptions, -} from "./loader.js"; +import { resolveRuntimePluginRegistry, type PluginLoadOptions } from "./loader.js"; import { hasManifestContractValue, isManifestPluginAvailableForControlPlane, loadManifestContractSnapshot, } from "./manifest-contract-eligibility.js"; -import { - resolveConfigScopedRuntimeCacheValue, - type ConfigScopedRuntimeCache, -} from "./plugin-cache-primitives.js"; import type { PluginMetadataSnapshot } from "./plugin-metadata-snapshot.types.js"; import { normalizeCapabilityProviderId } from "./provider-registry-shared.js"; import type { PluginRegistry } from "./registry-types.js"; @@ -50,17 +44,16 @@ type CapabilityContractKey = | "videoGenerationProviders" | "musicGenerationProviders"; -type CapabilityProviderForKey = - PluginRegistry[K][number] extends { provider: infer T } ? T : never; -type CapabilityProviderEntries = PluginRegistry[CapabilityProviderRegistryKey]; +type ProviderFor = PluginRegistry[K][number] extends { + provider: infer T; +} + ? T + : never; type CapabilityPluginResolution = { runtimePluginIds: string[]; bundledCompatPluginIds: string[]; }; -const capabilityProviderSnapshotCache: ConfigScopedRuntimeCache = - new WeakMap(); - const CAPABILITY_CONTRACT_KEY: Record = { embeddingProviders: "embeddingProviders", memoryEmbeddingProviders: "memoryEmbeddingProviders", @@ -74,10 +67,6 @@ const CAPABILITY_CONTRACT_KEY: Record; }): Pick { + if (params.pluginMetadataSnapshot) { + return params.pluginMetadataSnapshot; + } return loadManifestContractSnapshot({ config: params.cfg, ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), @@ -112,6 +102,7 @@ function resolveCapabilityPluginIds(params: { cfg?: OpenClawConfig; workspaceDir?: string; providerId?: string; + pluginMetadataSnapshot?: Pick; }): CapabilityPluginResolution { const contractKey = CAPABILITY_CONTRACT_KEY[params.key]; const snapshot = loadCapabilityManifestSnapshot(params); @@ -140,64 +131,37 @@ function resolveCapabilityPluginIds(params: { }; } -function resolveBundledCapabilityCompatPluginIds(params: { - key: CapabilityProviderRegistryKey; +function createCapabilityProviderLoadOptions(params: { cfg?: OpenClawConfig; - workspaceDir?: string; - providerId?: string; -}): string[] { - return resolveCapabilityPluginIds(params).bundledCompatPluginIds; -} - -function resolveCapabilityProviderConfig(params: { - key: CapabilityProviderRegistryKey; - cfg?: OpenClawConfig; - workspaceDir?: string; - pluginIds?: string[]; -}) { - const pluginIds = params.pluginIds ?? resolveBundledCapabilityCompatPluginIds(params); + resolution: CapabilityPluginResolution; +}): PluginLoadOptions { + const pluginIds = params.resolution.bundledCompatPluginIds; const enablementCompat = withBundledPluginEnablementCompat({ config: params.cfg, pluginIds, }); - return withBundledPluginVitestCompat({ + const config = withBundledPluginVitestCompat({ config: enablementCompat, pluginIds, env: process.env, }); -} - -function createCapabilityProviderFallbackLoadOptions(params: { - compatConfig?: OpenClawConfig; - pluginIds: string[]; -}): PluginLoadOptions { return { - ...(params.compatConfig === undefined ? {} : { config: params.compatConfig }), - onlyPluginIds: params.pluginIds, + ...(config === undefined ? {} : { config }), + onlyPluginIds: params.resolution.runtimePluginIds, activate: false, }; } -function resolveCapabilityProviderSnapshotCacheKey(params: { - key: CapabilityProviderRegistryKey; - loadOptions: PluginLoadOptions; -}): string { - return JSON.stringify({ - key: params.key, - load: resolvePluginRegistryLoadCacheKey(params.loadOptions), - }); -} - function findProviderById( entries: PluginRegistry[K], providerId: string, -): CapabilityProviderForKey | undefined { +): ProviderFor | undefined { const normalizedProviderId = normalizeCapabilityProviderId(providerId); if (!normalizedProviderId) { return undefined; } const providerEntries = entries as unknown as Array<{ - provider: CapabilityProviderForKey & { id?: unknown; aliases?: unknown }; + provider: ProviderFor & { id?: unknown; aliases?: unknown }; }>; for (const entry of providerEntries) { if ( @@ -225,12 +189,12 @@ function findProviderById( function mergeCapabilityProviders( left: PluginRegistry[K], right: PluginRegistry[K], -): CapabilityProviderForKey[] { - const merged = new Map>(); - const unnamed: CapabilityProviderForKey[] = []; +): ProviderFor[] { + const merged = new Map>(); + const unnamed: ProviderFor[] = []; const addEntries = (entries: PluginRegistry[K]) => { for (const entry of entries) { - const provider = entry.provider as CapabilityProviderForKey & { id?: string }; + const provider = entry.provider as ProviderFor & { id?: string }; if (!provider.id) { unnamed.push(provider); continue; @@ -501,7 +465,7 @@ export function resolvePluginCapabilityProvider | undefined { +}): ProviderFor | undefined { if (shouldSkipCapabilityResolution(params)) { return undefined; } @@ -527,58 +491,23 @@ export function resolvePluginCapabilityProvider - loadCapabilityProviderEntries({ - key: params.key, - bundledCompatPluginIds: pluginIds.bundledCompatPluginIds, - loadOptions, - requested: new Set([params.providerId.toLowerCase()]), - }) as CapabilityProviderEntries, - }) as PluginRegistry[K]; return findProviderById(loadedProviders, params.providerId); } -function resolveCachedCapabilityProviderEntries(params: { - key: K; - cfg?: OpenClawConfig; - bundledCompatPluginIds: string[]; - loadOptions: PluginLoadOptions; - requested?: Set; -}): PluginRegistry[K] { - return resolveConfigScopedRuntimeCacheValue({ - cache: capabilityProviderSnapshotCache, - config: params.cfg, - key: resolveCapabilityProviderSnapshotCacheKey({ - key: params.key, - loadOptions: params.loadOptions, - }), - load: () => - loadCapabilityProviderEntries({ - key: params.key, - bundledCompatPluginIds: params.bundledCompatPluginIds, - loadOptions: params.loadOptions, - requested: params.requested, - }) as CapabilityProviderEntries, - }) as PluginRegistry[K]; -} - export function resolvePluginCapabilityProviders(params: { key: K; cfg?: OpenClawConfig; -}): CapabilityProviderForKey[] { +}): ProviderFor[] { if (shouldSkipCapabilityResolution(params)) { return []; } @@ -597,12 +526,12 @@ export function resolvePluginCapabilityProviders 0 && params.key !== "memoryEmbeddingProviders") { if (!missingRequestedProviders && !shouldMergeManifestProvidersWhenActive(params.key)) { - return activeProviders.map((entry) => entry.provider) as CapabilityProviderForKey[]; + return activeProviders.map((entry) => entry.provider) as ProviderFor[]; } if (missingRequestedProviders) { removeActiveProviderIds(missingRequestedProviders, activeProviders); if (missingRequestedProviders.size === 0) { - return activeProviders.map((entry) => entry.provider) as CapabilityProviderForKey[]; + return activeProviders.map((entry) => entry.provider) as ProviderFor[]; } } } @@ -633,18 +562,12 @@ export function resolvePluginCapabilityProviders; + registry?: PluginRegistry; +}) { + const providers = ( + key: K, + ): readonly ProviderFor[] | undefined => { + if (shouldSkipCapabilityResolution({ key, cfg: params.cfg })) { + return []; + } + const resolution = resolveCapabilityPluginIds({ + key, + cfg: params.cfg, + pluginMetadataSnapshot: params.pluginMetadataSnapshot, + }); + const requiredPluginIds = sortUniqueStrings([ + ...resolution.runtimePluginIds, + ...resolution.bundledCompatPluginIds, + ]); + if (!params.registry || !registryContainsRuntimePluginIds(params.registry, requiredPluginIds)) { + return undefined; + } + const eligiblePluginIds = new Set(requiredPluginIds); + if (params.registry[key].some((entry) => !eligiblePluginIds.has(entry.pluginId))) { + return undefined; + } + return Object.freeze( + params.registry[key].map((entry) => entry.provider), + ) as readonly ProviderFor[]; + }; + return Object.freeze({ + mediaUnderstandingProviders: providers("mediaUnderstandingProviders"), + imageGenerationProviders: providers("imageGenerationProviders"), + videoGenerationProviders: providers("videoGenerationProviders"), + musicGenerationProviders: providers("musicGenerationProviders"), + }); +} diff --git a/src/plugins/manifest.ts b/src/plugins/manifest.ts index f8c2f87c3694..39aa09c90929 100644 --- a/src/plugins/manifest.ts +++ b/src/plugins/manifest.ts @@ -132,7 +132,7 @@ export type PluginManifestProviderEndpoint = { googleVertexRegionHostSuffix?: string; }; -type PluginManifestProviderRequestProvider = { +export type PluginManifestProviderRequestProvider = { family?: string; compatibilityFamily?: "moonshot"; openAICompletions?: { diff --git a/src/plugins/plugin-metadata-provider-facts.ts b/src/plugins/plugin-metadata-provider-facts.ts new file mode 100644 index 000000000000..e6671bed0253 --- /dev/null +++ b/src/plugins/plugin-metadata-provider-facts.ts @@ -0,0 +1,109 @@ +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { + normalizeLowercaseStringOrEmpty, + normalizeOptionalLowercaseString, + normalizeOptionalString, +} from "@openclaw/normalization-core/string-coerce"; +import type { PluginManifestRecord } from "./manifest-registry.js"; +import type { + PluginManifestProviderEndpoint, + PluginManifestProviderRequestProvider, +} from "./manifest.js"; +import { listOfficialExternalProviderEndpointManifests } from "./official-external-provider-endpoints.js"; + +const PROVIDER_ENDPOINT_CLASSES = new Set( + "anthropic-public cerebras-native chutes-native deepseek-native github-copilot-native groq-native meta-native mistral-public minimax-native moonshot-native modelstudio-native nvidia-native openai-public openai opencode-native azure-openai openrouter xai-native xiaomi-native zai-native google-generative-ai google-vertex".split( + " ", + ), +); + +function normalizeProviderHosts(value: unknown): string[] { + return Array.isArray(value) + ? value + .filter((entry): entry is string => typeof entry === "string") + .map((entry) => entry.trim().toLowerCase()) + .filter(Boolean) + : []; +} + +export function normalizePluginProviderBaseUrl(value: string): string | undefined { + const trimmed = normalizeOptionalString(value); + const schemeless = trimmed && /^[a-z0-9.[\]-]+(?::\d+)?(?:[/?#].*)?$/i.test(trimmed); + const url = trimmed ? URL.parse(schemeless ? `https://${trimmed}` : trimmed) : null; + if (!url || (url.protocol !== "http:" && url.protocol !== "https:")) { + return undefined; + } + url.hash = ""; + url.search = ""; + return normalizeOptionalLowercaseString(url.toString().replace(/\/+$/, "")); +} + +function prepareProviderEndpoints(value: unknown): PluginManifestProviderEndpoint[] { + if (!Array.isArray(value)) { + return []; + } + return value + .filter(isRecord) + .filter((endpoint) => { + const endpointClass = normalizeOptionalString(endpoint.endpointClass); + return endpointClass ? PROVIDER_ENDPOINT_CLASSES.has(endpointClass) : false; + }) + .map((endpoint) => { + const endpointClass = normalizeOptionalString(endpoint.endpointClass)!; + const googleVertexRegion = normalizeOptionalString(endpoint.googleVertexRegion); + const googleVertexRegionHostSuffix = normalizeOptionalString( + endpoint.googleVertexRegionHostSuffix, + )?.toLowerCase(); + return Object.assign( + { + endpointClass, + hosts: normalizeProviderHosts(endpoint.hosts), + hostSuffixes: normalizeProviderHosts(endpoint.hostSuffixes), + baseUrls: normalizeProviderHosts(endpoint.baseUrls) + .map(normalizePluginProviderBaseUrl) + .filter((baseUrl): baseUrl is string => baseUrl !== undefined), + }, + googleVertexRegion ? { googleVertexRegion } : {}, + googleVertexRegionHostSuffix ? { googleVertexRegionHostSuffix } : {}, + ); + }); +} + +export function buildPluginMetadataProviderFacts(plugins: readonly PluginManifestRecord[]) { + const providerEndpoints = plugins.flatMap((plugin) => + prepareProviderEndpoints(plugin.providerEndpoints), + ); + const providerRequests = new Map(); + for (const plugin of plugins) { + const requests = isRecord(plugin.providerRequest?.providers) + ? plugin.providerRequest.providers + : {}; + for (const [rawProvider, request] of Object.entries(requests)) { + if (!isRecord(request)) { + continue; + } + const provider = normalizeLowercaseStringOrEmpty(rawProvider); + if (!provider) { + continue; + } + const supportsStreamingUsage = isRecord(request.openAICompletions) + ? request.openAICompletions.supportsStreamingUsage + : undefined; + providerRequests.set(provider, { + ...(normalizeOptionalString(request.family) + ? { family: normalizeOptionalString(request.family) } + : {}), + ...(normalizeOptionalString(request.compatibilityFamily) === "moonshot" + ? { compatibilityFamily: "moonshot" as const } + : {}), + ...(typeof supportsStreamingUsage === "boolean" + ? { openAICompletions: { supportsStreamingUsage } } + : {}), + }); + } + } + for (const manifest of listOfficialExternalProviderEndpointManifests()) { + providerEndpoints.push(...prepareProviderEndpoints(manifest.providerEndpoints)); + } + return { providerEndpoints, providerRequests }; +} diff --git a/src/plugins/plugin-metadata-snapshot.memo.test.ts b/src/plugins/plugin-metadata-snapshot.memo.test.ts index f40acfd54afc..3e30391638c6 100644 --- a/src/plugins/plugin-metadata-snapshot.memo.test.ts +++ b/src/plugins/plugin-metadata-snapshot.memo.test.ts @@ -346,6 +346,82 @@ describe("loadPluginMetadataSnapshot process memo", () => { expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(2); }); + it("prepares provider endpoint and request facts with the metadata snapshot", () => { + const index = makeIndex("demo"); + const registry = makeManifestRegistry("demo"); + const plugin = registry.plugins[0]; + if (!plugin) { + throw new Error("expected manifest plugin fixture"); + } + plugin.providerEndpoints = [ + { + endpointClass: "openai-public", + hosts: [" API.EXAMPLE.COM "], + baseUrls: ["https://api.example.com/v1/"], + googleVertexRegion: " global ", + googleVertexRegionHostSuffix: " -AIPLATFORM.GOOGLEAPIS.COM ", + }, + ]; + plugin.providerRequest = { + providers: { + demo: { + family: " demo-family ", + compatibilityFamily: " moonshot " as never, + openAICompletions: { supportsStreamingUsage: true }, + }, + }, + }; + loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "provided", + snapshot: index, + diagnostics: [], + }); + loadPluginManifestRegistryForInstalledIndex.mockReturnValue(registry); + + const snapshot = loadPluginMetadataSnapshot({ config: {}, env: {}, index }); + + expect(snapshot.owners.providerEndpoints).toContainEqual({ + endpointClass: "openai-public", + hosts: ["api.example.com"], + hostSuffixes: [], + baseUrls: ["https://api.example.com/v1"], + googleVertexRegion: "global", + googleVertexRegionHostSuffix: "-aiplatform.googleapis.com", + }); + expect(snapshot.owners.providerRequests?.get("demo")).toEqual({ + family: "demo-family", + compatibilityFamily: "moonshot", + openAICompletions: { supportsStreamingUsage: true }, + }); + }); + + it("ignores malformed optional provider facts", () => { + const index = makeIndex("demo"); + const registry = makeManifestRegistry("demo"); + const plugin = registry.plugins[0]; + if (!plugin) { + throw new Error("expected manifest plugin fixture"); + } + plugin.providerEndpoints = [ + { endpointClass: "openai-public", hosts: { invalid: true } } as never, + null as never, + ]; + plugin.providerRequest = { providers: { demo: null } } as never; + loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "provided", + snapshot: index, + diagnostics: [], + }); + loadPluginManifestRegistryForInstalledIndex.mockReturnValue(registry); + + const snapshot = loadPluginMetadataSnapshot({ config: {}, env: {}, index }); + + expect(snapshot.owners.providerRequests?.has("demo")).toBe(false); + expect( + snapshot.owners.providerEndpoints?.some((endpoint) => (endpoint.hosts ?? []).length === 0), + ).toBe(true); + }); + it("keeps scoped and unscoped metadata snapshots in separate memo slots", () => { const index = makeIndex(); loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ diff --git a/src/plugins/plugin-metadata-snapshot.ts b/src/plugins/plugin-metadata-snapshot.ts index 2de384192804..73d0cf7fd803 100644 --- a/src/plugins/plugin-metadata-snapshot.ts +++ b/src/plugins/plugin-metadata-snapshot.ts @@ -1,4 +1,3 @@ -// Builds plugin metadata snapshots for gateway and diagnostics. import fs from "node:fs"; import path from "node:path"; import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; @@ -24,6 +23,7 @@ import { import { loadPluginManifestRegistry, type PluginManifestRecord } from "./manifest-registry.js"; import { resolvePluginControlPlaneFingerprint } from "./plugin-control-plane-context.js"; import { registerPluginMetadataProcessMemoLifecycleClear } from "./plugin-metadata-lifecycle.js"; +import { buildPluginMetadataProviderFacts } from "./plugin-metadata-provider-facts.js"; import type { LoadPluginMetadataSnapshotParams, PluginMetadataSnapshot, @@ -540,6 +540,7 @@ function buildPluginMetadataOwnerMaps( setupProviders: freezeOwnerMap(setupProviders), commandAliases: freezeOwnerMap(commandAliases), contracts: freezeOwnerMap(contracts), + ...buildPluginMetadataProviderFacts(plugins), }; } diff --git a/src/plugins/plugin-metadata-snapshot.types.ts b/src/plugins/plugin-metadata-snapshot.types.ts index 2b983dd89cb2..800f7a969033 100644 --- a/src/plugins/plugin-metadata-snapshot.types.ts +++ b/src/plugins/plugin-metadata-snapshot.types.ts @@ -1,9 +1,12 @@ -// Defines plugin metadata snapshot types used by gateway and diagnostics. import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { PluginDiscoveryResult } from "./discovery.js"; import type { InstalledPluginIndex } from "./installed-plugin-index-types.js"; import type { PluginManifestRecord, PluginManifestRegistry } from "./manifest-registry.js"; import type { PluginDiagnostic } from "./manifest-types.js"; +import type { + PluginManifestProviderEndpoint, + PluginManifestProviderRequestProvider, +} from "./manifest.js"; import type { PluginRegistrySnapshotSource } from "./plugin-registry-snapshot.types.js"; export type PluginMetadataSnapshotPluginIdScope = { @@ -20,6 +23,8 @@ export type PluginMetadataSnapshotOwnerMaps = { setupProviders: ReadonlyMap; commandAliases: ReadonlyMap; contracts: ReadonlyMap; + providerEndpoints?: readonly PluginManifestProviderEndpoint[]; + providerRequests?: ReadonlyMap; }; export type PluginMetadataSnapshotMetrics = {