diff --git a/CHANGELOG.md b/CHANGELOG.md index c78117bd8de8..3795ccbedd7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Docs: https://docs.openclaw.ai ### Changes +- Gateway/perf: reuse immutable plugin metadata snapshots across startup, config, model, channel, setup, and secret metadata readers so hot paths avoid repeated plugin file stats and manifest registry reloads. - Gateway/perf: lazy-load startup-idle plugin work, core gateway method handlers, and the embedded ACPX runtime so Gateway health and ready signals no longer wait on unused handler trees or ACPX probes. - Gateway/perf: cache plugin SDK public-surface alias maps and skip irrelevant macOS Linuxbrew PATH probes so Gateway startup avoids repeated filesystem walks and slow missing-directory stats. - Image tool: add adaptive model-aware image compression with an `agents.defaults.imageQuality` preference for choosing token-efficient, balanced, or high-detail media handling. diff --git a/src/agents/model-catalog.test.ts b/src/agents/model-catalog.test.ts index 72cee708a512..fa7248610a5c 100644 --- a/src/agents/model-catalog.test.ts +++ b/src/agents/model-catalog.test.ts @@ -13,8 +13,8 @@ let modelSupportsInput: typeof import("./model-catalog.js").modelSupportsInput; let resetModelCatalogCacheForTest: typeof import("./model-catalog.js").resetModelCatalogCacheForTest; let augmentCatalogMock: ReturnType; let ensureOpenClawModelsJsonMock: ReturnType; -let currentPluginMetadataSnapshotMock: ReturnType; -let loadPluginMetadataSnapshotMock: ReturnType; +let currentPluginMetadataSnapshotMock: ReturnType unknown>>; +let loadPluginMetadataSnapshotMock: ReturnType unknown>>; let readFileMock: ReturnType; vi.mock("./model-suppression.runtime.js", () => ({ @@ -184,13 +184,15 @@ describe("loadModelCatalog", () => { vi.doMock("../plugins/provider-runtime.runtime.js", () => ({ augmentModelCatalogWithProviderPlugins: vi.fn().mockResolvedValue([]), })); - currentPluginMetadataSnapshotMock = vi.fn(); - loadPluginMetadataSnapshotMock = vi.fn(); + currentPluginMetadataSnapshotMock = vi.fn<(...args: unknown[]) => unknown>(); + loadPluginMetadataSnapshotMock = vi.fn<(...args: unknown[]) => unknown>(); vi.doMock("../plugins/current-plugin-metadata-snapshot.js", () => ({ getCurrentPluginMetadataSnapshot: currentPluginMetadataSnapshotMock, })); vi.doMock("../plugins/plugin-metadata-snapshot.js", () => ({ loadPluginMetadataSnapshot: loadPluginMetadataSnapshotMock, + resolvePluginMetadataSnapshot: (...args: unknown[]) => + currentPluginMetadataSnapshotMock(...args) ?? loadPluginMetadataSnapshotMock(...args), })); ({ @@ -215,7 +217,7 @@ describe("loadModelCatalog", () => { ensureOpenClawModelsJsonMock.mockClear(); augmentCatalogMock.mockClear(); currentPluginMetadataSnapshotMock.mockReset(); - currentPluginMetadataSnapshotMock.mockReturnValue(emptyPluginMetadataSnapshot()); + currentPluginMetadataSnapshotMock.mockReturnValue(undefined); loadPluginMetadataSnapshotMock.mockReset(); loadPluginMetadataSnapshotMock.mockReturnValue(emptyPluginMetadataSnapshot()); }); @@ -511,7 +513,7 @@ describe("loadModelCatalog", () => { }); it("normalizes persisted read-only catalog rows with manifest model id policies", async () => { - currentPluginMetadataSnapshotMock.mockReturnValueOnce(modelIdNormalizationSnapshot()); + currentPluginMetadataSnapshotMock.mockReturnValue(modelIdNormalizationSnapshot()); readFileMock.mockResolvedValueOnce( JSON.stringify({ providers: { diff --git a/src/agents/model-catalog.ts b/src/agents/model-catalog.ts index ed1e2b2b8021..b5c6de3b00be 100644 --- a/src/agents/model-catalog.ts +++ b/src/agents/model-catalog.ts @@ -9,7 +9,7 @@ import { isManifestPluginAvailableForControlPlane, loadManifestMetadataSnapshot, } from "../plugins/manifest-contract-eligibility.js"; -import { loadPluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; +import { resolvePluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.types.js"; import { augmentModelCatalogWithProviderPlugins } from "../plugins/provider-runtime.runtime.js"; import { createLazyImportLoader } from "../shared/lazy-promise.js"; @@ -137,22 +137,20 @@ export function loadManifestModelCatalog(params: { fallbackToMetadataScan?: boolean; metadataSnapshot?: PluginMetadataSnapshot; }): ModelCatalogEntry[] { - const snapshot = - params.metadataSnapshot ?? - getCurrentPluginMetadataSnapshot({ - config: params.config, - env: params.env, - ...(params.workspaceDir !== undefined ? { workspaceDir: params.workspaceDir } : {}), - ...(params.workspaceDir === undefined ? { allowWorkspaceScopedSnapshot: true } : {}), - }); const resolvedSnapshot = - snapshot ?? + params.metadataSnapshot ?? (params.fallbackToMetadataScan === false - ? undefined - : loadPluginMetadataSnapshot({ + ? getCurrentPluginMetadataSnapshot({ + config: params.config, + env: params.env, + ...(params.workspaceDir !== undefined ? { workspaceDir: params.workspaceDir } : {}), + ...(params.workspaceDir === undefined ? { allowWorkspaceScopedSnapshot: true } : {}), + }) + : resolvePluginMetadataSnapshot({ config: params.config, ...(params.workspaceDir !== undefined ? { workspaceDir: params.workspaceDir } : {}), env: params.env ?? process.env, + allowWorkspaceScopedCurrent: params.workspaceDir === undefined, })); if (!resolvedSnapshot) { return []; @@ -362,9 +360,10 @@ function loadReadOnlyStaticModelCatalog(params?: { const configuredManifestPlugins = hasConfiguredProviderRowsNeedingManifestLookup(cfg) ? (params?.metadataSnapshot?.plugins ?? - loadPluginMetadataSnapshot({ + resolvePluginMetadataSnapshot({ config: cfg, env: process.env, + allowWorkspaceScopedCurrent: true, }).plugins) : []; const configuredModels = buildConfiguredModelCatalog({ diff --git a/src/agents/models-config.ts b/src/agents/models-config.ts index 9644eee53a1a..6704d5f8d311 100644 --- a/src/agents/models-config.ts +++ b/src/agents/models-config.ts @@ -8,10 +8,9 @@ import { } from "../config/config.js"; import { createConfigRuntimeEnv } from "../config/env-vars.js"; import { privateFileStore } from "../infra/private-file-store.js"; -import { getCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-snapshot.js"; import { resolveInstalledManifestRegistryIndexFingerprint } from "../plugins/manifest-registry-installed.js"; import { - loadPluginMetadataSnapshot, + resolvePluginMetadataSnapshot, type PluginMetadataSnapshot, } from "../plugins/plugin-metadata-snapshot.js"; import { @@ -175,14 +174,11 @@ export async function ensureOpenClawModelsJson( : resolveAgentWorkspaceDir(cfg, resolveDefaultAgentId(cfg))); const pluginMetadataSnapshot = options.pluginMetadataSnapshot ?? - getCurrentPluginMetadataSnapshot({ - config: cfg, - ...(workspaceDir ? { workspaceDir } : {}), - }) ?? - loadPluginMetadataSnapshot({ + resolvePluginMetadataSnapshot({ config: cfg, env: createConfigRuntimeEnv(cfg), ...(workspaceDir ? { workspaceDir } : {}), + allowWorkspaceScopedCurrent: workspaceDir === undefined, }); const agentDir = agentDirOverride?.trim() ? agentDirOverride.trim() : resolveDefaultAgentDir(cfg); const targetPath = path.join(agentDir, "models.json"); diff --git a/src/agents/skills/plugin-skills.test.ts b/src/agents/skills/plugin-skills.test.ts index 8dd71dd64c79..f7533faffae6 100644 --- a/src/agents/skills/plugin-skills.test.ts +++ b/src/agents/skills/plugin-skills.test.ts @@ -43,6 +43,7 @@ vi.mock("../../plugins/plugin-registry.js", () => ({ vi.mock("../../plugins/plugin-metadata-snapshot.js", () => ({ loadPluginMetadataSnapshot: hoisted.loadPluginMetadataSnapshot, + resolvePluginMetadataSnapshot: hoisted.loadPluginMetadataSnapshot, })); let resolvePluginSkillDirs: typeof import("./plugin-skills.js").resolvePluginSkillDirs; diff --git a/src/agents/skills/plugin-skills.ts b/src/agents/skills/plugin-skills.ts index 6c294aeac702..dd9e74dd8e7e 100644 --- a/src/agents/skills/plugin-skills.ts +++ b/src/agents/skills/plugin-skills.ts @@ -9,8 +9,7 @@ import { resolveEffectivePluginActivationState, resolveMemorySlotDecision, } from "../../plugins/config-policy.js"; -import { getCurrentPluginMetadataSnapshot } from "../../plugins/current-plugin-metadata-snapshot.js"; -import { loadPluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.js"; +import { resolvePluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.js"; import { hasKind } from "../../plugins/slots.js"; import { isPathInsideWithRealpath } from "../../security/scan-paths.js"; import { CONFIG_DIR } from "../../utils.js"; @@ -33,17 +32,12 @@ export function resolvePluginSkillDirs(params: { return []; } const config = params.config ?? {}; - const metadataSnapshot = - getCurrentPluginMetadataSnapshot({ - config, - env: process.env, - workspaceDir, - }) ?? - loadPluginMetadataSnapshot({ - workspaceDir, - config, - env: process.env, - }); + const metadataSnapshot = resolvePluginMetadataSnapshot({ + workspaceDir, + config, + env: process.env, + allowWorkspaceScopedCurrent: true, + }); const registry = metadataSnapshot.manifestRegistry; if (registry.plugins.length === 0) { publishPluginSkills([], { diff --git a/src/agents/tools/manifest-capability-availability.ts b/src/agents/tools/manifest-capability-availability.ts index 6843682a3e14..b166561237b2 100644 --- a/src/agents/tools/manifest-capability-availability.ts +++ b/src/agents/tools/manifest-capability-availability.ts @@ -1,9 +1,6 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { getCurrentPluginMetadataSnapshot } from "../../plugins/current-plugin-metadata-snapshot.js"; -import { - isManifestPluginAvailableForControlPlane, - loadManifestContractSnapshot, -} from "../../plugins/manifest-contract-eligibility.js"; +import { isManifestPluginAvailableForControlPlane } from "../../plugins/manifest-contract-eligibility.js"; import type { PluginManifestRecord } from "../../plugins/manifest-registry.js"; import { hasNonEmptyManifestEnvCandidate, @@ -11,7 +8,7 @@ import { manifestPluginSetupProviderEnvVars, manifestProviderBaseUrlGuardPasses, } from "../../plugins/manifest-tool-availability.js"; -import { loadPluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.js"; +import { resolvePluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.js"; import type { PluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.types.js"; import { getActivePluginRegistryWorkspaceDirFromState } from "../../plugins/runtime-state.js"; import { listProfilesForProvider } from "../auth-profiles/profile-list.js"; @@ -81,23 +78,12 @@ export function loadCapabilityMetadataSnapshot(params: { env?: NodeJS.ProcessEnv; }): Pick { const workspaceDir = params.workspaceDir ?? getActivePluginRegistryWorkspaceDirFromState(); - const current = getCurrentPluginMetadataSnapshot({ - config: params.config, + return resolvePluginMetadataSnapshot({ + config: params.config ?? {}, + env: params.env ?? process.env, ...(workspaceDir ? { workspaceDir } : {}), + allowWorkspaceScopedCurrent: workspaceDir === undefined, }); - if (current) { - return current; - } - return workspaceDir - ? loadManifestContractSnapshot({ - config: params.config, - env: params.env, - workspaceDir, - }) - : loadPluginMetadataSnapshot({ - config: params.config ?? {}, - env: params.env ?? process.env, - }); } export function hasSnapshotCapabilityAvailability(params: { diff --git a/src/channels/plugins/read-only-command-defaults.test.ts b/src/channels/plugins/read-only-command-defaults.test.ts index ec38f03817fd..e83301b6a4a2 100644 --- a/src/channels/plugins/read-only-command-defaults.test.ts +++ b/src/channels/plugins/read-only-command-defaults.test.ts @@ -4,6 +4,7 @@ const loadPluginMetadataSnapshot = vi.hoisted(() => vi.fn()); vi.mock("../../plugins/plugin-metadata-snapshot.js", () => ({ loadPluginMetadataSnapshot, + resolvePluginMetadataSnapshot: loadPluginMetadataSnapshot, })); import { resolveReadOnlyChannelCommandDefaults } from "./read-only-command-defaults.js"; @@ -59,6 +60,7 @@ describe("resolveReadOnlyChannelCommandDefaults", () => { nativeSkillsAutoEnabled: false, }); expect(loadPluginMetadataSnapshot).toHaveBeenCalledWith({ + allowWorkspaceScopedCurrent: true, config: {}, env, stateDir: "/state", diff --git a/src/channels/plugins/read-only-command-defaults.ts b/src/channels/plugins/read-only-command-defaults.ts index 15337d33dfda..6b0094ebb267 100644 --- a/src/channels/plugins/read-only-command-defaults.ts +++ b/src/channels/plugins/read-only-command-defaults.ts @@ -1,9 +1,8 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { isBlockedObjectKey } from "../../infra/prototype-keys.js"; -import { getCurrentPluginMetadataSnapshot } from "../../plugins/current-plugin-metadata-snapshot.js"; import { isInstalledPluginEnabled } from "../../plugins/installed-plugin-index.js"; import type { PluginManifestRecord } from "../../plugins/manifest-registry.js"; -import { loadPluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.js"; +import { resolvePluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.js"; import { normalizeOptionalString } from "../../shared/string-coerce.js"; import type { ChannelPlugin } from "./types.plugin.js"; @@ -66,22 +65,13 @@ export function resolveReadOnlyChannelCommandDefaults( return undefined; } const env = options.env ?? process.env; - const snapshot = - options.stateDir === undefined - ? getCurrentPluginMetadataSnapshot({ - config: options.config, - env, - workspaceDir: options.workspaceDir, - }) - : undefined; - const resolvedSnapshot = - snapshot ?? - loadPluginMetadataSnapshot({ - config: options.config, - stateDir: options.stateDir, - workspaceDir: options.workspaceDir, - env, - }); + const resolvedSnapshot = resolvePluginMetadataSnapshot({ + config: options.config, + stateDir: options.stateDir, + workspaceDir: options.workspaceDir, + env, + allowWorkspaceScopedCurrent: true, + }); for (const record of resolvedSnapshot.plugins) { if (!record.channels.includes(normalizedChannelId)) { continue; diff --git a/src/channels/plugins/read-only.ts b/src/channels/plugins/read-only.ts index 426012404b57..d277c90715f9 100644 --- a/src/channels/plugins/read-only.ts +++ b/src/channels/plugins/read-only.ts @@ -10,14 +10,13 @@ import { listConfiguredChannelIdsForReadOnlyScope, resolveDiscoverableScopedChannelPluginIds, } from "../../plugins/channel-plugin-ids.js"; -import { getCurrentPluginMetadataSnapshot } from "../../plugins/current-plugin-metadata-snapshot.js"; import { channelPluginIdBelongsToManifest, resolveSetupChannelRegistration, } from "../../plugins/loader-channel-setup.js"; import type { PluginManifestRecord } from "../../plugins/manifest-registry.js"; import type { PluginDiagnostic } from "../../plugins/manifest-types.js"; -import { loadPluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.js"; +import { resolvePluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.js"; import { getCachedPluginModuleLoader, type PluginModuleLoaderCache, @@ -759,22 +758,13 @@ export function resolveReadOnlyChannelPluginsForConfig( ): ReadOnlyChannelPluginResolution { const env = options.env ?? process.env; const workspaceDir = resolveReadOnlyWorkspaceDir(cfg, options); - const metadataSnapshot = - options.stateDir === undefined - ? getCurrentPluginMetadataSnapshot({ - config: cfg, - env, - workspaceDir, - }) - : undefined; - const manifestRecords = - metadataSnapshot?.plugins ?? - loadPluginMetadataSnapshot({ - config: cfg, - stateDir: options.stateDir, - workspaceDir, - env, - }).plugins; + const manifestRecords = resolvePluginMetadataSnapshot({ + config: cfg, + stateDir: options.stateDir, + workspaceDir, + env, + allowWorkspaceScopedCurrent: true, + }).plugins; const bundledManifestRecords = listBundledChannelManifestRecords(manifestRecords); const externalManifestRecords = listExternalChannelManifestRecords(manifestRecords); const configuredChannelIds = [ diff --git a/src/cli/config-cli.test.ts b/src/cli/config-cli.test.ts index 41615ed76b8e..67cd0c574a16 100644 --- a/src/cli/config-cli.test.ts +++ b/src/cli/config-cli.test.ts @@ -56,6 +56,8 @@ vi.mock("../plugins/plugin-metadata-snapshot.js", async (importOriginal) => { return { ...actual, loadPluginMetadataSnapshot: (config: unknown) => mockLoadPluginMetadataSnapshot(config), + resolvePluginMetadataSnapshot: (params: { config?: unknown }) => + mockLoadPluginMetadataSnapshot(params.config), }; }); diff --git a/src/commands/doctor/shared/missing-configured-plugin-install.test.ts b/src/commands/doctor/shared/missing-configured-plugin-install.test.ts index 7f7ebff28e61..7b42d51de9b3 100644 --- a/src/commands/doctor/shared/missing-configured-plugin-install.test.ts +++ b/src/commands/doctor/shared/missing-configured-plugin-install.test.ts @@ -134,6 +134,7 @@ vi.mock("../../../plugins/clawhub.js", () => ({ vi.mock("../../../plugins/plugin-metadata-snapshot.js", () => ({ loadPluginMetadataSnapshot: mocks.loadPluginMetadataSnapshot, + resolvePluginMetadataSnapshot: mocks.loadPluginMetadataSnapshot, })); vi.mock("../../../plugins/official-external-plugin-catalog.js", () => ({ diff --git a/src/commands/models/list.manifest-catalog.test.ts b/src/commands/models/list.manifest-catalog.test.ts index 897b347be133..0a7263bb94fa 100644 --- a/src/commands/models/list.manifest-catalog.test.ts +++ b/src/commands/models/list.manifest-catalog.test.ts @@ -15,6 +15,7 @@ vi.mock("../../plugins/plugin-registry.js", () => ({ vi.mock("../../plugins/plugin-metadata-snapshot.js", () => ({ loadPluginMetadataSnapshot: mocks.loadPluginMetadataSnapshot, + resolvePluginMetadataSnapshot: mocks.loadPluginMetadataSnapshot, })); const moonshotPlugin = { @@ -71,6 +72,7 @@ describe("loadStaticManifestCatalogRowsForList", () => { }).map((row) => row.ref), ).toEqual(["moonshot/kimi-k2.6"]); expect(mocks.loadPluginMetadataSnapshot).toHaveBeenCalledWith({ + allowWorkspaceScopedCurrent: true, config: {}, env: process.env, }); diff --git a/src/config/io.ts b/src/config/io.ts index fc8b357f2fda..654c7432255c 100644 --- a/src/config/io.ts +++ b/src/config/io.ts @@ -22,7 +22,7 @@ import { writePersistedInstalledPluginIndexInstallRecordsSync, } from "../plugins/installed-plugin-index-records.js"; import { - loadPluginMetadataSnapshot, + resolvePluginMetadataSnapshot, type PluginMetadataSnapshot, } from "../plugins/plugin-metadata-snapshot.js"; import { sanitizeTerminalText } from "../terminal/safe-text.js"; @@ -1598,10 +1598,11 @@ export function createConfigIO( effectiveConfigRaw, ); const defaultAgentId = resolveDefaultAgentId(metadataConfig); - pluginMetadataSnapshot = loadPluginMetadataSnapshot({ + pluginMetadataSnapshot = resolvePluginMetadataSnapshot({ config: metadataConfig, workspaceDir: resolveAgentWorkspaceDir(metadataConfig, defaultAgentId), env: deps.env, + allowWorkspaceScopedCurrent: true, }); return pluginMetadataSnapshot; }; @@ -1817,10 +1818,11 @@ export function createConfigIO( effectiveConfigRaw, ); const defaultAgentId = resolveDefaultAgentId(metadataConfig); - pluginMetadataSnapshot = loadPluginMetadataSnapshot({ + pluginMetadataSnapshot = resolvePluginMetadataSnapshot({ config: metadataConfig, workspaceDir: resolveAgentWorkspaceDir(metadataConfig, defaultAgentId), env: deps.env, + allowWorkspaceScopedCurrent: true, }); return pluginMetadataSnapshot; }; diff --git a/src/config/runtime-schema.test.ts b/src/config/runtime-schema.test.ts index 87d2f100aac8..d2033337113b 100644 --- a/src/config/runtime-schema.test.ts +++ b/src/config/runtime-schema.test.ts @@ -38,6 +38,10 @@ vi.mock("../plugins/plugin-metadata-snapshot.js", () => ({ loadPluginMetadataSnapshot: (...args: unknown[]) => ({ manifestRegistry: mockLoadPluginManifestRegistry(...args), }), + resolvePluginMetadataSnapshot: (...args: unknown[]) => + mockGetCurrentPluginMetadataSnapshot(...args) ?? { + manifestRegistry: mockLoadPluginManifestRegistry(...args), + }, })); vi.mock("../plugins/current-plugin-metadata-snapshot.js", () => ({ diff --git a/src/config/runtime-schema.ts b/src/config/runtime-schema.ts index 11b38f620478..b952d518c6f4 100644 --- a/src/config/runtime-schema.ts +++ b/src/config/runtime-schema.ts @@ -1,6 +1,5 @@ import { resolveAgentWorkspaceDir, resolveDefaultAgentId } from "../agents/agent-scope.js"; -import { getCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-snapshot.js"; -import { loadPluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; +import { resolvePluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; import { collectChannelSchemaMetadata, collectPluginSchemaMetadata, @@ -11,14 +10,11 @@ import { buildConfigSchema, type ConfigSchemaResponse } from "./schema.js"; function loadManifestRegistry(config: OpenClawConfig, env?: NodeJS.ProcessEnv) { const workspaceDir = resolveAgentWorkspaceDir(config, resolveDefaultAgentId(config)); - const currentSnapshot = getCurrentPluginMetadataSnapshot({ config, env, workspaceDir }); - if (currentSnapshot) { - return currentSnapshot.manifestRegistry; - } - return loadPluginMetadataSnapshot({ + return resolvePluginMetadataSnapshot({ config, env: env ?? process.env, workspaceDir, + allowWorkspaceScopedCurrent: true, }).manifestRegistry; } diff --git a/src/config/validation.channel-metadata.test.ts b/src/config/validation.channel-metadata.test.ts index d5c2de63ccf1..11ce9a7a20d7 100644 --- a/src/config/validation.channel-metadata.test.ts +++ b/src/config/validation.channel-metadata.test.ts @@ -153,6 +153,9 @@ vi.mock("../plugins/plugin-metadata-snapshot.js", () => ({ loadPluginMetadataSnapshot: () => ({ manifestRegistry: mockLoadPluginManifestRegistry(), }), + resolvePluginMetadataSnapshot: () => ({ + manifestRegistry: mockLoadPluginManifestRegistry(), + }), })); vi.mock("../plugins/doctor-contract-registry.js", () => ({ diff --git a/src/config/validation.ts b/src/config/validation.ts index e3a1c736eed9..dd581517e20f 100644 --- a/src/config/validation.ts +++ b/src/config/validation.ts @@ -18,7 +18,7 @@ import { resolveOfficialExternalPluginInstall, } from "../plugins/official-external-plugin-catalog.js"; import { - loadPluginMetadataSnapshot, + resolvePluginMetadataSnapshot, type PluginMetadataSnapshot, } from "../plugins/plugin-metadata-snapshot.js"; import { validateJsonSchemaValue } from "../plugins/schema-validator.js"; @@ -1036,10 +1036,11 @@ function validateConfigObjectWithPluginsBase( return registryInfo; } const workspaceDir = resolveAgentWorkspaceDir(config, resolveDefaultAgentId(config)); - const registry = loadPluginMetadataSnapshot({ + const registry = resolvePluginMetadataSnapshot({ config, workspaceDir: workspaceDir ?? undefined, env: opts.env ?? process.env, + allowWorkspaceScopedCurrent: true, }).manifestRegistry; registryInfo = { registry }; return registryInfo; diff --git a/src/gateway/model-pricing-cache.ts b/src/gateway/model-pricing-cache.ts index bde6810ad212..b388736659c1 100644 --- a/src/gateway/model-pricing-cache.ts +++ b/src/gateway/model-pricing-cache.ts @@ -19,7 +19,7 @@ import type { PluginManifestModelPricingProvider, PluginManifestModelPricingSource, } from "../plugins/manifest.js"; -import { loadPluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; +import { resolvePluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; import type { PluginMetadataRegistryView } from "../plugins/plugin-metadata-snapshot.types.js"; import type { PluginRegistrySnapshot } from "../plugins/plugin-registry.js"; import { normalizeOptionalString, resolvePrimaryStringValue } from "../shared/string-coerce.js"; @@ -459,10 +459,12 @@ function resolveModelPricingManifestMetadata(params: { activeRegistry: emptyRegistry, }; } - const snapshot = loadPluginMetadataSnapshot({ + const env = params.env ?? process.env; + const snapshot = resolvePluginMetadataSnapshot({ config: params.config, - env: params.env ?? process.env, + env, ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), + allowWorkspaceScopedCurrent: params.workspaceDir === undefined, }); return { allRegistry: snapshot.manifestRegistry, diff --git a/src/plugins/gateway-startup-plugin-ids.ts b/src/plugins/gateway-startup-plugin-ids.ts index 7c7a6408bf87..58863db3786d 100644 --- a/src/plugins/gateway-startup-plugin-ids.ts +++ b/src/plugins/gateway-startup-plugin-ids.ts @@ -23,7 +23,7 @@ import type { InstalledPluginIndexRecord } from "./installed-plugin-index.js"; import type { PluginManifestRecord, PluginManifestRegistry } from "./manifest-registry.js"; import { isPluginMetadataSnapshotCompatible, - loadPluginMetadataSnapshot, + resolvePluginMetadataSnapshot, type PluginMetadataSnapshot, } from "./plugin-metadata-snapshot.js"; import { @@ -938,10 +938,11 @@ export function loadGatewayStartupPluginPlan(params: { index: params.index, }) ? params.metadataSnapshot - : loadPluginMetadataSnapshot({ + : resolvePluginMetadataSnapshot({ config: snapshotConfig, workspaceDir: params.workspaceDir, env: params.env, + allowWorkspaceScopedCurrent: params.workspaceDir === undefined, ...(params.index ? { index: params.index } : {}), }); return resolveGatewayStartupPluginPlanFromRegistry({ diff --git a/src/plugins/manifest-contract-eligibility.ts b/src/plugins/manifest-contract-eligibility.ts index 4c8e667cfdee..fbff47a77a4d 100644 --- a/src/plugins/manifest-contract-eligibility.ts +++ b/src/plugins/manifest-contract-eligibility.ts @@ -1,8 +1,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { getCurrentPluginMetadataSnapshot } from "./current-plugin-metadata-snapshot.js"; import { isInstalledPluginEnabled } from "./installed-plugin-index.js"; import type { PluginManifestContractListKey, PluginManifestRecord } from "./manifest-registry.js"; -import { loadPluginMetadataSnapshot } from "./plugin-metadata-snapshot.js"; +import { resolvePluginMetadataSnapshot } from "./plugin-metadata-snapshot.js"; import type { PluginMetadataManifestView, PluginMetadataRegistryView, @@ -98,18 +97,10 @@ export function loadManifestMetadataSnapshot(params: { }): PluginMetadataSnapshot { const config = params.config ?? {}; const env = params.env ?? process.env; - const current = getCurrentPluginMetadataSnapshot({ - config, - env, - ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), - ...(params.workspaceDir === undefined ? { allowWorkspaceScopedSnapshot: true } : {}), - }); - if (current) { - return current; - } - return loadPluginMetadataSnapshot({ + return resolvePluginMetadataSnapshot({ config, env, ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), + allowWorkspaceScopedCurrent: params.workspaceDir === undefined, }); } diff --git a/src/plugins/manifest-model-id-normalization.ts b/src/plugins/manifest-model-id-normalization.ts index 33ef1de2302a..6049bdc76075 100644 --- a/src/plugins/manifest-model-id-normalization.ts +++ b/src/plugins/manifest-model-id-normalization.ts @@ -1,10 +1,9 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js"; -import { getCurrentPluginMetadataSnapshot } from "./current-plugin-metadata-snapshot.js"; import type { PluginManifestRecord } from "./manifest-registry.js"; import type { PluginManifestModelIdNormalizationProvider } from "./manifest.js"; import { - loadPluginMetadataSnapshot, + resolvePluginMetadataSnapshot, type PluginMetadataSnapshot, } from "./plugin-metadata-snapshot.js"; import { getActivePluginRegistryWorkspaceDirFromState } from "./runtime-state.js"; @@ -43,21 +42,14 @@ function resolveMetadataSnapshotForPolicies( } { const env = params.env ?? process.env; const workspaceDir = params.workspaceDir ?? getActivePluginRegistryWorkspaceDirFromState(); - const current = getCurrentPluginMetadataSnapshot({ - config: params.config, - env, - workspaceDir, - }); - if (current) { - return { snapshot: current, cacheable: true }; - } return { - snapshot: loadPluginMetadataSnapshot({ + snapshot: resolvePluginMetadataSnapshot({ config: params.config ?? {}, env, workspaceDir, + allowWorkspaceScopedCurrent: true, }), - cacheable: false, + cacheable: true, }; } diff --git a/src/plugins/plugin-lookup-table.ts b/src/plugins/plugin-lookup-table.ts index 3b4089b0a275..3510ed58f1ae 100644 --- a/src/plugins/plugin-lookup-table.ts +++ b/src/plugins/plugin-lookup-table.ts @@ -6,7 +6,7 @@ import { import { hashJson } from "./installed-plugin-index-hash.js"; import { isPluginMetadataSnapshotCompatible, - loadPluginMetadataSnapshot, + resolvePluginMetadataSnapshot, type PluginMetadataSnapshot, type PluginMetadataSnapshotOwnerMaps, } from "./plugin-metadata-snapshot.js"; @@ -59,10 +59,11 @@ export function loadPluginLookUpTable(params: LoadPluginLookUpTableParams): Plug index: params.index, }) ? params.metadataSnapshot - : loadPluginMetadataSnapshot({ + : resolvePluginMetadataSnapshot({ config: requestedSnapshotConfig, workspaceDir: params.workspaceDir, env: params.env, + allowWorkspaceScopedCurrent: params.workspaceDir === undefined, ...(params.index ? { index: params.index } : {}), }); const { index, manifestRegistry } = metadataSnapshot; diff --git a/src/plugins/plugin-metadata-snapshot.memo.test.ts b/src/plugins/plugin-metadata-snapshot.memo.test.ts index d0123b951ce8..71fa593cac7e 100644 --- a/src/plugins/plugin-metadata-snapshot.memo.test.ts +++ b/src/plugins/plugin-metadata-snapshot.memo.test.ts @@ -2,11 +2,17 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + clearCurrentPluginMetadataSnapshot, + setCurrentPluginMetadataSnapshot, +} from "./current-plugin-metadata-snapshot.js"; +import { resolveInstalledPluginIndexPolicyHash } from "./installed-plugin-index-policy.js"; import type { InstalledPluginIndex } from "./installed-plugin-index.js"; import type { PluginManifestRecord, PluginManifestRegistry } from "./manifest-registry.js"; import { clearLoadPluginMetadataSnapshotMemo, loadPluginMetadataSnapshot, + resolvePluginMetadataSnapshot, } from "./plugin-metadata-snapshot.js"; const loadPluginRegistrySnapshotWithMetadata = vi.hoisted(() => vi.fn()); @@ -222,6 +228,7 @@ describe("loadPluginMetadataSnapshot process memo", () => { afterEach(() => { clearLoadPluginMetadataSnapshotMemo(); + clearCurrentPluginMetadataSnapshot(); for (const dir of tempDirs.splice(0)) { fs.rmSync(dir, { recursive: true, force: true }); } @@ -254,6 +261,130 @@ describe("loadPluginMetadataSnapshot process memo", () => { expect(second.byPluginId.get("demo")).toBe(second.plugins[0]); }); + it("keeps hot persisted snapshots for alternating config callers", () => { + const stateDir = tempStateDir(); + touchPersistedIndex(stateDir); + loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "persisted", + snapshot: makeIndex(), + diagnostics: [], + }); + + loadPluginMetadataSnapshot({ + config: { plugins: { allow: ["demo"] } }, + env: {}, + stateDir, + }); + loadPluginMetadataSnapshot({ + config: { plugins: { allow: ["other"] } }, + env: {}, + stateDir, + }); + loadPluginMetadataSnapshot({ + config: { plugins: { allow: ["demo"] } }, + env: {}, + stateDir, + }); + + expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledTimes(2); + expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(2); + }); + + it("reuses workspace-scoped current snapshots when the caller opts in", () => { + const index = makeIndex(); + index.policyHash = resolveInstalledPluginIndexPolicyHash({}); + loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "runtime", + snapshot: index, + diagnostics: [], + }); + const snapshot = loadPluginMetadataSnapshot({ + config: {}, + env: {}, + index, + workspaceDir: "/workspace/a", + }); + setCurrentPluginMetadataSnapshot(snapshot, { + config: {}, + env: {}, + workspaceDir: "/workspace/a", + }); + loadPluginRegistrySnapshotWithMetadata.mockClear(); + loadPluginManifestRegistryForInstalledIndex.mockClear(); + + expect( + resolvePluginMetadataSnapshot({ + config: {}, + env: {}, + allowWorkspaceScopedCurrent: true, + }), + ).toBe(snapshot); + expect(loadPluginRegistrySnapshotWithMetadata).not.toHaveBeenCalled(); + expect(loadPluginManifestRegistryForInstalledIndex).not.toHaveBeenCalled(); + }); + + it("reuses compatible current snapshots without reloading metadata", () => { + const sourceConfig = { plugins: { allow: ["demo"] } }; + const compatibleConfig = { plugins: { entries: { demo: { enabled: true } } } }; + const index = makeIndex(); + index.policyHash = resolveInstalledPluginIndexPolicyHash(sourceConfig); + loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "runtime", + snapshot: index, + diagnostics: [], + }); + const snapshot = loadPluginMetadataSnapshot({ + config: sourceConfig, + env: {}, + index, + workspaceDir: "/workspace/a", + }); + setCurrentPluginMetadataSnapshot(snapshot, { + config: sourceConfig, + compatibleConfigs: [compatibleConfig], + env: {}, + workspaceDir: "/workspace/a", + }); + loadPluginRegistrySnapshotWithMetadata.mockClear(); + loadPluginManifestRegistryForInstalledIndex.mockClear(); + + expect( + resolvePluginMetadataSnapshot({ + config: compatibleConfig, + env: {}, + workspaceDir: "/workspace/a", + }), + ).toBe(snapshot); + expect(loadPluginRegistrySnapshotWithMetadata).not.toHaveBeenCalled(); + expect(loadPluginManifestRegistryForInstalledIndex).not.toHaveBeenCalled(); + }); + + it("does not scan persisted registry files when the caller provides an index", () => { + const stateDir = tempStateDir(); + writePersistedIndex({ pluginId: "demo", stateDir }); + const index = makeIndex(); + loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "provided", + snapshot: index, + diagnostics: [], + }); + loadPluginMetadataSnapshot({ config: {}, env: {}, index, stateDir }); + const statSpy = vi.spyOn(fs, "statSync"); + const readSpy = vi.spyOn(fs, "readFileSync"); + + try { + loadPluginMetadataSnapshot({ config: {}, env: {}, index, stateDir }); + } finally { + statSpy.mockRestore(); + readSpy.mockRestore(); + } + + expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledOnce(); + expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledOnce(); + expect(statSpy).not.toHaveBeenCalled(); + expect(readSpy).not.toHaveBeenCalled(); + }); + it("does not memoize policy-stale derived snapshots", () => { const stateDir = tempStateDir(); touchPersistedIndex(stateDir); @@ -366,7 +497,7 @@ describe("loadPluginMetadataSnapshot process memo", () => { ["source", "index.js", "source"], ["setup source", "setup.js", "setupSource"], ["package manifest", "package.json", "packageJsonPath"], - ])("refreshes when persisted plugin %s changes in the same process", (_, fileName, field) => { + ])("requires reload before persisted plugin %s edits are visible", (_, fileName, field) => { const stateDir = tempStateDir(); const filePath = path.join(stateDir, "extensions", "demo", fileName); writePersistedIndex({ [field]: filePath, pluginId: "demo", stateDir }); @@ -380,8 +511,8 @@ describe("loadPluginMetadataSnapshot process memo", () => { writeJson(filePath, { id: "demo", version: "0.2.0" }); loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); - expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledTimes(2); - expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(2); + expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledOnce(); + expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledOnce(); }); it.each([ @@ -398,7 +529,7 @@ describe("loadPluginMetadataSnapshot process memo", () => { (homeDir: string) => path.join(homeDir, "tracked-plugin", "package.json"), ], ])( - "refreshes when home-relative install record %s changes", + "requires reload before home-relative install record %s changes are visible", (_, recordPath, record, targetPath) => { const stateDir = tempStateDir(); const homeDir = path.join(stateDir, "home"); @@ -415,12 +546,12 @@ describe("loadPluginMetadataSnapshot process memo", () => { writeJson(filePath, { version: "1.0.1000" }); loadPluginMetadataSnapshot({ config: {}, env: { HOME: homeDir }, stateDir }); - expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledTimes(2); - expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(2); + expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledOnce(); + expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledOnce(); }, ); - it("does not reuse home-relative install record watches across env changes", () => { + it("does not reuse home-relative install record memo state across env changes", () => { const stateDir = tempStateDir(); const firstHomeDir = path.join(stateDir, "first-home"); const secondHomeDir = path.join(stateDir, "second-home"); @@ -442,11 +573,11 @@ describe("loadPluginMetadataSnapshot process memo", () => { writeJson(secondPackageJsonPath, { version: "1.0.1000" }); loadPluginMetadataSnapshot({ config: {}, env: { HOME: secondHomeDir }, stateDir }); - expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledTimes(3); - expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(3); + expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledTimes(2); + expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(2); }); - it("refreshes when recovered managed npm package metadata changes", () => { + it("requires reload before recovered managed npm package metadata changes are visible", () => { const stateDir = tempStateDir(); writeRecoverableNpmPlugin({ packageName: "recovered-plugin", @@ -470,11 +601,11 @@ describe("loadPluginMetadataSnapshot process memo", () => { }); loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); - expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledTimes(2); - expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(2); + expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledOnce(); + expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledOnce(); }); - it("refreshes when a declared recovered managed npm package appears", () => { + it("requires reload before a declared recovered managed npm package appears", () => { const stateDir = tempStateDir(); writeJson(path.join(stateDir, "npm", "package.json"), { dependencies: { @@ -497,11 +628,11 @@ describe("loadPluginMetadataSnapshot process memo", () => { }); loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); - expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledTimes(2); - expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(2); + expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledOnce(); + expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledOnce(); }); - it("refreshes when an in-root package manifest symlink target changes", () => { + it("requires reload before an in-root package manifest symlink target change is visible", () => { const stateDir = tempStateDir(); const pluginDir = path.join(stateDir, "extensions", "demo"); const packageJsonPath = path.join(pluginDir, "package.json"); @@ -520,8 +651,8 @@ describe("loadPluginMetadataSnapshot process memo", () => { writeJson(outsidePackageJsonPath, { name: "outside", version: "1.0.1" }); loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); - expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledTimes(2); - expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(2); + expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledOnce(); + expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledOnce(); }); it("does not fingerprint persisted plugin paths outside the plugin root", () => { diff --git a/src/plugins/plugin-metadata-snapshot.ts b/src/plugins/plugin-metadata-snapshot.ts index 4f2ad9ed3d47..06c093416bf7 100644 --- a/src/plugins/plugin-metadata-snapshot.ts +++ b/src/plugins/plugin-metadata-snapshot.ts @@ -8,10 +8,10 @@ import { } from "../infra/diagnostics-timeline.js"; import { resolveUserPath } from "../utils.js"; import { resolveCompatibilityHostVersion } from "../version.js"; +import { getCurrentPluginMetadataSnapshot } from "./current-plugin-metadata-snapshot.js"; import { resolveDefaultPluginNpmDir } from "./install-paths.js"; import { hashJson } from "./installed-plugin-index-hash.js"; import { resolveInstalledPluginIndexPolicyHash } from "./installed-plugin-index-policy.js"; -import { loadInstalledPluginIndexInstallRecordsSync } from "./installed-plugin-index-record-reader.js"; import { resolveInstalledPluginIndexStorePath } from "./installed-plugin-index-store-path.js"; import type { InstalledPluginIndex } from "./installed-plugin-index.js"; import { @@ -24,6 +24,7 @@ import type { LoadPluginMetadataSnapshotParams, PluginMetadataSnapshot, PluginMetadataSnapshotOwnerMaps, + ResolvePluginMetadataSnapshotParams, } from "./plugin-metadata-snapshot.types.js"; import { createPluginRegistryIdNormalizer } from "./plugin-registry-id-normalizer.js"; import { @@ -41,14 +42,14 @@ type PersistedRegistryMemoState = { contextHash: string; fastHash: string; fingerprint: unknown; - watchedFilesHash: string; - watchedFiles: readonly string[]; }; -let pluginMetadataSnapshotMemo: PluginMetadataSnapshotMemo | undefined; +const MAX_PLUGIN_METADATA_SNAPSHOT_MEMOS = 8; + +let pluginMetadataSnapshotMemos: PluginMetadataSnapshotMemo[] = []; export function clearLoadPluginMetadataSnapshotMemo(): void { - pluginMetadataSnapshotMemo = undefined; + pluginMetadataSnapshotMemos = []; } const MEMO_RELEVANT_ENV_KEYS = [ @@ -74,6 +75,7 @@ export type { PluginMetadataSnapshotMetrics, PluginMetadataSnapshotOwnerMaps, PluginMetadataSnapshotRegistryDiagnostic, + ResolvePluginMetadataSnapshotParams, } from "./plugin-metadata-snapshot.types.js"; function fileFingerprint(filePath: string): unknown { @@ -99,10 +101,6 @@ function readJsonObject(filePath: string): Record | undefined { } } -function normalizeString(value: unknown): string | undefined { - return typeof value === "string" && value.trim() ? value.trim() : undefined; -} - function stableMemoValue(value: unknown): unknown { if (Array.isArray(value)) { return value.map(stableMemoValue); @@ -117,162 +115,6 @@ function stableMemoValue(value: unknown): unknown { ); } -function isPathInsideOrEqual(childPath: string, parentPath: string): boolean { - const relative = path.relative(parentPath, childPath); - return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); -} - -function tryRealpath(filePath: string): string | null { - try { - return fs.realpathSync(filePath); - } catch { - return null; - } -} - -function resolvePluginFilePath( - pluginDir: string, - filePath: string | undefined, - options: { allowSymlinkOutsideRoot?: boolean } = {}, -): - | { status: "ok"; path: string } - | { status: "outside-root"; path: string } - | { status: "missing-root"; path: string } { - if (!filePath) { - return { status: "missing-root", path: "" }; - } - const rootDir = path.resolve(pluginDir); - const resolved = path.isAbsolute(filePath) - ? path.resolve(filePath) - : path.resolve(rootDir, filePath); - if (!isPathInsideOrEqual(resolved, rootDir)) { - return { status: "outside-root", path: resolved }; - } - const rootRealPath = tryRealpath(rootDir); - const targetRealPath = tryRealpath(resolved); - if ( - rootRealPath && - targetRealPath && - !isPathInsideOrEqual(targetRealPath, rootRealPath) && - !options.allowSymlinkOutsideRoot - ) { - return { status: "outside-root", path: resolved }; - } - return { status: "ok", path: resolved }; -} - -function persistedPluginFileFingerprint( - rootDir: string | undefined, - filePath: string | undefined, - options: { allowSymlinkOutsideRoot?: boolean; watchedFiles?: Set } = {}, -): unknown { - if (!filePath) { - return null; - } - if (!rootDir) { - return [filePath, "missing-root"]; - } - const resolved = resolvePluginFilePath(rootDir, filePath, { - allowSymlinkOutsideRoot: options.allowSymlinkOutsideRoot, - }); - if (resolved.status !== "ok") { - return [filePath, resolved.status]; - } - options.watchedFiles?.add(resolved.path); - return fileFingerprint(resolved.path); -} - -function watchedFileFingerprint(filePath: string | undefined, watchedFiles: Set): unknown { - if (!filePath) { - return null; - } - watchedFiles.add(filePath); - return fileFingerprint(filePath); -} - -function resolveInstallRecordPath(value: unknown, env: NodeJS.ProcessEnv): string | undefined { - const normalized = normalizeString(value); - return normalized ? resolveUserPath(normalized, env) : undefined; -} - -function installRecordPathFingerprints( - env: NodeJS.ProcessEnv, - records: unknown, - watchedFiles: Set, -): readonly unknown[] { - if (!isRecord(records)) { - return []; - } - return Object.entries(records) - .toSorted(([left], [right]) => left.localeCompare(right)) - .map(([pluginId, rawRecord]) => { - if (!isRecord(rawRecord)) { - return [pluginId, rawRecord]; - } - const installPath = normalizeString(rawRecord.installPath); - const sourcePath = normalizeString(rawRecord.sourcePath); - const resolvedInstallPath = resolveInstallRecordPath(rawRecord.installPath, env); - const resolvedSourcePath = resolveInstallRecordPath(rawRecord.sourcePath, env); - return [ - pluginId, - installPath, - sourcePath, - watchedFileFingerprint( - resolvedInstallPath ? path.join(resolvedInstallPath, "package.json") : undefined, - watchedFiles, - ), - watchedFileFingerprint( - resolvedInstallPath ? path.join(resolvedInstallPath, "openclaw.plugin.json") : undefined, - watchedFiles, - ), - watchedFileFingerprint(resolvedSourcePath, watchedFiles), - watchedFileFingerprint( - resolvedSourcePath ? path.join(resolvedSourcePath, "package.json") : undefined, - watchedFiles, - ), - watchedFileFingerprint( - resolvedSourcePath ? path.join(resolvedSourcePath, "openclaw.plugin.json") : undefined, - watchedFiles, - ), - ]; - }); -} - -function managedNpmDependencyMetadataFingerprints( - npmRoot: string, - watchedFiles: Set, -): readonly unknown[] { - const rootManifest = readJsonObject(path.join(npmRoot, "package.json")); - const dependencies = isRecord(rootManifest?.dependencies) ? rootManifest.dependencies : {}; - const nodeModulesRoot = path.join(npmRoot, "node_modules"); - return Object.entries(dependencies) - .toSorted(([left], [right]) => left.localeCompare(right)) - .map(([packageName, rawSpec]) => { - const dependencySpec = normalizeString(rawSpec); - if (!dependencySpec) { - return [packageName, rawSpec]; - } - const packageDir = path.resolve(nodeModulesRoot, packageName); - if (!isPathInsideOrEqual(packageDir, path.resolve(nodeModulesRoot))) { - return [packageName, dependencySpec, "outside-node-modules"]; - } - return [ - packageName, - dependencySpec, - watchedFileFingerprint(path.join(packageDir, "package.json"), watchedFiles), - watchedFileFingerprint(path.join(packageDir, "openclaw.plugin.json"), watchedFiles), - ]; - }); -} - -function resolveRecordPackageJsonPath(record: Record): string | undefined { - const packageJson = record.packageJson; - if (!isRecord(packageJson)) { - return undefined; - } - return normalizeString(packageJson.path); -} - function pickMemoRelevantEnv(env: NodeJS.ProcessEnv): Record { return Object.fromEntries( MEMO_RELEVANT_ENV_KEYS.flatMap((key) => { @@ -376,10 +218,6 @@ function resolvePersistedRegistryMemoContextHash(params: { }); } -function hashWatchedFiles(watchedFiles: readonly string[]): string { - return hashJson(watchedFiles.map((filePath) => fileFingerprint(filePath))); -} - function resolvePersistedRegistryMemoState(params: { env: NodeJS.ProcessEnv; index?: InstalledPluginIndex; @@ -397,100 +235,20 @@ function resolvePersistedRegistryMemoState(params: { contextHash, fastHash, fingerprint: fastFingerprint, - watchedFiles: [], - watchedFilesHash: hashJson([]), }; } const indexPath = resolveInstalledPluginIndexStorePath({ env: params.env, ...(params.stateDir ? { stateDir: params.stateDir } : {}), }); - const npmRoot = params.stateDir - ? path.join(params.stateDir, "npm") - : resolveDefaultPluginNpmDir(params.env); const index = params.index ?? readJsonObject(indexPath); - const plugins = Array.isArray(index?.plugins) ? index.plugins : []; - const diagnostics = Array.isArray(index?.diagnostics) ? index.diagnostics : []; - const pluginRootById = new Map(); - const watchedFiles = new Set(); - for (const rawPlugin of plugins) { - if (!isRecord(rawPlugin)) { - continue; - } - const pluginId = normalizeString(rawPlugin.pluginId); - const rootDir = normalizeString(rawPlugin.rootDir); - if (pluginId && rootDir) { - pluginRootById.set(pluginId, rootDir); - } - } - const installRecords = - params.index?.installRecords ?? - loadInstalledPluginIndexInstallRecordsSync({ - env: params.env, - ...(params.stateDir ? { stateDir: params.stateDir } : {}), - }); - const watchedPlugins = plugins.map((rawPlugin) => { - if (!isRecord(rawPlugin)) { - return rawPlugin; - } - const rootDir = normalizeString(rawPlugin.rootDir); - const manifestPath = normalizeString(rawPlugin.manifestPath); - const packageJsonPath = resolveRecordPackageJsonPath(rawPlugin); - const source = normalizeString(rawPlugin.source); - const setupSource = normalizeString(rawPlugin.setupSource); - return [ - normalizeString(rawPlugin.pluginId), - rootDir, - rootDir ? fileFingerprint(rootDir) : null, - manifestPath, - persistedPluginFileFingerprint(rootDir, manifestPath, { watchedFiles }), - source, - persistedPluginFileFingerprint(rootDir, source, { watchedFiles }), - setupSource, - persistedPluginFileFingerprint(rootDir, setupSource, { watchedFiles }), - packageJsonPath, - persistedPluginFileFingerprint(rootDir, packageJsonPath, { - allowSymlinkOutsideRoot: true, - watchedFiles, - }), - ]; - }); - const watchedDiagnostics = diagnostics.map((rawDiagnostic) => { - if (!isRecord(rawDiagnostic)) { - return rawDiagnostic; - } - const pluginId = normalizeString(rawDiagnostic.pluginId); - const source = normalizeString(rawDiagnostic.source); - return [ - pluginId, - source, - persistedPluginFileFingerprint(pluginId ? pluginRootById.get(pluginId) : undefined, source, { - watchedFiles, - }), - ]; - }); - const installRecordFiles = installRecordPathFingerprints( - params.env, - installRecords, - watchedFiles, - ); - const managedNpmDependencyFiles = managedNpmDependencyMetadataFingerprints(npmRoot, watchedFiles); - const watchedFilesList = [...watchedFiles].toSorted(); return { contextHash, fastHash, fingerprint: { ...fastFingerprint, indexHash: hashJson(stableMemoValue(index) ?? null), - installRecords: hashJson(stableMemoValue(installRecords)), - installRecordFiles, - managedNpmDependencyFiles, - npmPackageJson: fileFingerprint(path.join(npmRoot, "package.json")), - plugins: watchedPlugins, - diagnostics: watchedDiagnostics, }, - watchedFiles: watchedFilesList, - watchedFilesHash: hashWatchedFiles(watchedFilesList), }; } @@ -500,7 +258,7 @@ function resolvePersistedRegistryMemoStateForLookup( preferPersisted?: boolean; stateDir?: string; }, - memo: PluginMetadataSnapshotMemo | undefined, + memos: readonly PluginMetadataSnapshotMemo[], ): PersistedRegistryMemoState { const fastFingerprint = resolvePersistedRegistryFastMemoFingerprint(params); const fastHash = hashJson(fastFingerprint); @@ -508,18 +266,53 @@ function resolvePersistedRegistryMemoStateForLookup( ...params, fastFingerprint, }); - const registryState = memo?.registryState; - if ( - registryState && - registryState.contextHash === contextHash && - registryState.fastHash === fastHash && - hashWatchedFiles(registryState.watchedFiles) === registryState.watchedFilesHash - ) { - return registryState; + for (const memo of memos) { + const registryState = memo.registryState; + if ( + registryState && + registryState.contextHash === contextHash && + registryState.fastHash === fastHash + ) { + // Plugin files are immutable for a running gateway; plugin edits require + // an explicit reload/restart, so hot lookups only validate the registry envelope. + return registryState; + } } return resolvePersistedRegistryMemoState(params); } +function resolveProvidedIndexMemoState(index: InstalledPluginIndex): PersistedRegistryMemoState { + const fingerprint = { + providedIndex: resolveInstalledManifestRegistryIndexFingerprint(index), + }; + const fingerprintHash = hashJson(fingerprint); + return { + contextHash: fingerprintHash, + fastHash: fingerprintHash, + fingerprint, + }; +} + +function findPluginMetadataSnapshotMemo(key: string): PluginMetadataSnapshotMemo | undefined { + const index = pluginMetadataSnapshotMemos.findIndex((memo) => memo.key === key); + if (index === -1) { + return undefined; + } + const [memo] = pluginMetadataSnapshotMemos.splice(index, 1); + if (!memo) { + return undefined; + } + pluginMetadataSnapshotMemos.unshift(memo); + return memo; +} + +function rememberPluginMetadataSnapshotMemo(memo: PluginMetadataSnapshotMemo): void { + pluginMetadataSnapshotMemos = [ + memo, + ...pluginMetadataSnapshotMemos.filter((existing) => existing.key !== memo.key), + ].slice(0, MAX_PLUGIN_METADATA_SNAPSHOT_MEMOS); +} + function computePluginMetadataSnapshotMemoKey(params: { params: LoadPluginMetadataSnapshotParams; registryState: PersistedRegistryMemoState; @@ -702,17 +495,21 @@ export function loadPluginMetadataSnapshot( params: LoadPluginMetadataSnapshotParams, ): PluginMetadataSnapshot { const activeTimelineSpan = getActiveDiagnosticsTimelineSpan(); - const memo = pluginMetadataSnapshotMemo; const env = params.env ?? process.env; - const registryState = resolvePersistedRegistryMemoStateForLookup( - { - env, - ...(params.stateDir ? { stateDir: resolveUserPath(params.stateDir, env) } : {}), - ...(params.preferPersisted !== undefined ? { preferPersisted: params.preferPersisted } : {}), - }, - memo, - ); + const registryState = params.index + ? resolveProvidedIndexMemoState(params.index) + : resolvePersistedRegistryMemoStateForLookup( + { + env, + ...(params.stateDir ? { stateDir: resolveUserPath(params.stateDir, env) } : {}), + ...(params.preferPersisted !== undefined + ? { preferPersisted: params.preferPersisted } + : {}), + }, + pluginMetadataSnapshotMemos, + ); const memoKey = computePluginMetadataSnapshotMemoKey({ params, registryState }); + const memo = findPluginMetadataSnapshotMemo(memoKey); if (memo?.key === memoKey) { return measureDiagnosticsTimelineSpanSync( "plugins.metadata.scan", @@ -755,11 +552,11 @@ export function loadPluginMetadataSnapshot( : {}), }) : registryState; - pluginMetadataSnapshotMemo = { + rememberPluginMetadataSnapshotMemo({ key: computePluginMetadataSnapshotMemoKey({ params, registryState: cachedRegistryState }), registryState: cachedRegistryState, snapshot: clonePluginMetadataSnapshot(result.snapshot), - }; + }); } return result.snapshot; } @@ -771,6 +568,45 @@ function canMemoizePluginMetadataSnapshotResult(result: { return result.registrySource !== "derived" && result.snapshot.index.plugins.length > 0; } +export function resolvePluginMetadataSnapshot( + params: ResolvePluginMetadataSnapshotParams, +): PluginMetadataSnapshot { + const canUseCurrentSnapshot = + params.allowCurrent !== false && + params.stateDir === undefined && + params.preferPersisted !== false; + if (canUseCurrentSnapshot) { + const current = getCurrentPluginMetadataSnapshot({ + config: params.config, + env: params.env, + ...(params.workspaceDir !== undefined ? { workspaceDir: params.workspaceDir } : {}), + ...(params.allowWorkspaceScopedCurrent === true + ? { allowWorkspaceScopedSnapshot: true } + : {}), + }); + if (!current) { + return loadPluginMetadataSnapshot(params); + } + if (!params.index) { + return current; + } + if ( + isPluginMetadataSnapshotCompatible({ + snapshot: current, + config: params.config, + env: params.env, + workspaceDir: + params.workspaceDir ?? + (params.allowWorkspaceScopedCurrent === true ? current.workspaceDir : undefined), + index: params.index, + }) + ) { + return current; + } + } + return loadPluginMetadataSnapshot(params); +} + function loadPluginMetadataSnapshotImpl(params: LoadPluginMetadataSnapshotParams): { snapshot: PluginMetadataSnapshot; registrySource: PluginRegistrySnapshotSource; diff --git a/src/plugins/plugin-metadata-snapshot.types.ts b/src/plugins/plugin-metadata-snapshot.types.ts index 3943f4975bed..c6b064c209ff 100644 --- a/src/plugins/plugin-metadata-snapshot.types.ts +++ b/src/plugins/plugin-metadata-snapshot.types.ts @@ -62,3 +62,8 @@ export type LoadPluginMetadataSnapshotParams = { index?: InstalledPluginIndex; preferPersisted?: boolean; }; + +export type ResolvePluginMetadataSnapshotParams = LoadPluginMetadataSnapshotParams & { + allowCurrent?: boolean; + allowWorkspaceScopedCurrent?: boolean; +}; diff --git a/src/plugins/runtime/load-context.test.ts b/src/plugins/runtime/load-context.test.ts index 003f152021e5..9481de1fee6a 100644 --- a/src/plugins/runtime/load-context.test.ts +++ b/src/plugins/runtime/load-context.test.ts @@ -45,6 +45,7 @@ vi.mock("../../agents/agent-scope.js", () => ({ vi.mock("../plugin-metadata-snapshot.js", () => ({ loadPluginMetadataSnapshot: loadPluginMetadataSnapshotMock, + resolvePluginMetadataSnapshot: loadPluginMetadataSnapshotMock, })); vi.mock("../current-plugin-metadata-snapshot.js", () => ({ @@ -121,6 +122,7 @@ describe("resolvePluginRuntimeLoadContext", () => { installRecords: {}, }); expect(loadPluginMetadataSnapshotMock).toHaveBeenCalledWith({ + allowWorkspaceScopedCurrent: true, config: rawConfig, env, workspaceDir: "/resolved-workspace", diff --git a/src/plugins/runtime/load-context.ts b/src/plugins/runtime/load-context.ts index e4ebbaf7da04..14bafd876bbe 100644 --- a/src/plugins/runtime/load-context.ts +++ b/src/plugins/runtime/load-context.ts @@ -7,14 +7,13 @@ import { createSubsystemLogger } from "../../logging.js"; import { resolvePluginActivationSourceConfig } from "../activation-source-config.js"; import { clearCurrentPluginMetadataSnapshot, - getCurrentPluginMetadataSnapshot, isReusableCurrentPluginMetadataSnapshot, setCurrentPluginMetadataSnapshot, } from "../current-plugin-metadata-snapshot.js"; import { extractPluginInstallRecordsFromInstalledPluginIndex } from "../installed-plugin-index-install-records.js"; import type { PluginLoadOptions } from "../loader.js"; import type { PluginManifestRegistry } from "../manifest-registry.js"; -import { loadPluginMetadataSnapshot } from "../plugin-metadata-snapshot.js"; +import { resolvePluginMetadataSnapshot } from "../plugin-metadata-snapshot.js"; import type { PluginLogger } from "../types.js"; const log = createSubsystemLogger("plugins"); @@ -70,16 +69,12 @@ export function resolvePluginRuntimeLoadContext( options?.workspaceDir ?? resolveAgentWorkspaceDir(rawConfig, resolveDefaultAgentId(rawConfig)); const metadataSnapshot = options?.manifestRegistry ? undefined - : (getCurrentPluginMetadataSnapshot({ + : resolvePluginMetadataSnapshot({ config: rawConfig, env, workspaceDir: rawWorkspaceDir, - }) ?? - loadPluginMetadataSnapshot({ - config: rawConfig, - env, - workspaceDir: rawWorkspaceDir, - })); + allowWorkspaceScopedCurrent: true, + }); const manifestRegistry = options?.manifestRegistry ?? metadataSnapshot?.manifestRegistry; const installRecords = metadataSnapshot ? extractPluginInstallRecordsFromInstalledPluginIndex(metadataSnapshot.index) diff --git a/src/plugins/setup-registry.runtime.test.ts b/src/plugins/setup-registry.runtime.test.ts index b5d34203b9a3..945cfdd2352b 100644 --- a/src/plugins/setup-registry.runtime.test.ts +++ b/src/plugins/setup-registry.runtime.test.ts @@ -22,9 +22,23 @@ vi.mock("./manifest-registry-installed.js", async (importOriginal) => ({ ...(await importOriginal()), loadPluginManifestRegistryForInstalledIndex: loadPluginManifestRegistryForInstalledIndexMock, })); -vi.mock("./plugin-metadata-snapshot.js", () => ({ - loadPluginMetadataSnapshot: loadPluginMetadataSnapshotMock, -})); +vi.mock("./plugin-metadata-snapshot.js", async () => { + const current = await import("./current-plugin-metadata-snapshot.js"); + return { + loadPluginMetadataSnapshot: loadPluginMetadataSnapshotMock, + resolvePluginMetadataSnapshot: ( + params: Parameters[0] & { + allowWorkspaceScopedCurrent?: boolean; + }, + ) => + current.getCurrentPluginMetadataSnapshot({ + config: params.config, + env: params.env, + workspaceDir: params.workspaceDir, + allowWorkspaceScopedSnapshot: params.allowWorkspaceScopedCurrent, + }) ?? loadPluginMetadataSnapshotMock(params), + }; +}); afterEach(() => { clearCurrentPluginMetadataSnapshot(); diff --git a/src/plugins/setup-registry.runtime.ts b/src/plugins/setup-registry.runtime.ts index 95d835e187b4..e42758cfb383 100644 --- a/src/plugins/setup-registry.runtime.ts +++ b/src/plugins/setup-registry.runtime.ts @@ -1,10 +1,9 @@ import { createRequire } from "node:module"; import { normalizeProviderId } from "../agents/provider-id.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { getCurrentPluginMetadataSnapshot } from "./current-plugin-metadata-snapshot.js"; import { isInstalledPluginEnabled } from "./installed-plugin-index.js"; import { - loadPluginMetadataSnapshot, + resolvePluginMetadataSnapshot, type PluginMetadataSnapshot, } from "./plugin-metadata-snapshot.js"; import { getActivePluginRegistryWorkspaceDirFromState } from "./runtime-state.js"; @@ -57,21 +56,19 @@ function resolveMetadataSnapshotForSetupCliBackends( } { const env = params.env ?? process.env; const workspaceDir = params.workspaceDir ?? getActivePluginRegistryWorkspaceDirFromState(); - const current = getCurrentPluginMetadataSnapshot({ - config: params.config, + const snapshot = resolvePluginMetadataSnapshot({ + config: params.config ?? {}, env, - workspaceDir, + ...(workspaceDir !== undefined + ? { + workspaceDir, + allowWorkspaceScopedCurrent: true, + } + : {}), }); - if (current) { - return { snapshot: current, cacheable: true }; - } return { - snapshot: loadPluginMetadataSnapshot({ - config: params.config ?? {}, - env, - workspaceDir, - }), - cacheable: false, + snapshot, + cacheable: true, }; } diff --git a/src/secrets/target-registry-data.current-snapshot.test.ts b/src/secrets/target-registry-data.current-snapshot.test.ts index d3bb794e4644..7bbc48e3bd49 100644 --- a/src/secrets/target-registry-data.current-snapshot.test.ts +++ b/src/secrets/target-registry-data.current-snapshot.test.ts @@ -1,45 +1,34 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const metadataMocks = vi.hoisted(() => ({ - getCurrentPluginMetadataSnapshot: vi.fn(() => undefined), - loadPluginMetadataSnapshot: vi.fn(() => ({ plugins: [] })), -})); - -vi.mock("../plugins/current-plugin-metadata-snapshot.js", () => ({ - getCurrentPluginMetadataSnapshot: metadataMocks.getCurrentPluginMetadataSnapshot, + resolvePluginMetadataSnapshot: vi.fn(() => ({ plugins: [] })), })); vi.mock("../plugins/plugin-metadata-snapshot.js", () => ({ - loadPluginMetadataSnapshot: metadataMocks.loadPluginMetadataSnapshot, + resolvePluginMetadataSnapshot: metadataMocks.resolvePluginMetadataSnapshot, })); describe("getSecretTargetRegistry metadata reuse", () => { beforeEach(() => { vi.resetModules(); - metadataMocks.getCurrentPluginMetadataSnapshot.mockClear(); - metadataMocks.getCurrentPluginMetadataSnapshot.mockReturnValue(undefined); - metadataMocks.loadPluginMetadataSnapshot.mockClear(); - metadataMocks.loadPluginMetadataSnapshot.mockReturnValue({ plugins: [] }); + metadataMocks.resolvePluginMetadataSnapshot.mockClear(); + metadataMocks.resolvePluginMetadataSnapshot.mockReturnValue({ plugins: [] }); }); - it("does not request workspace-scoped current metadata for the configless global cache", async () => { + it("uses configless global metadata without a workspace-scoped current request", async () => { const { getSecretTargetRegistry } = await import("./target-registry-data.js"); getSecretTargetRegistry(); - expect(metadataMocks.getCurrentPluginMetadataSnapshot).toHaveBeenCalledWith({ + expect(metadataMocks.resolvePluginMetadataSnapshot).toHaveBeenCalledWith({ config: {}, env: process.env, }); - const calls = metadataMocks.getCurrentPluginMetadataSnapshot.mock.calls as unknown as Array< - [{ allowWorkspaceScopedSnapshot?: boolean }] + const calls = metadataMocks.resolvePluginMetadataSnapshot.mock.calls as unknown as Array< + [{ allowWorkspaceScopedCurrent?: boolean }] >; for (const [call] of calls) { - expect(call.allowWorkspaceScopedSnapshot).not.toBe(true); + expect(call.allowWorkspaceScopedCurrent).not.toBe(true); } - expect(metadataMocks.loadPluginMetadataSnapshot).toHaveBeenCalledWith({ - config: {}, - env: process.env, - }); }); }); diff --git a/src/secrets/target-registry-data.ts b/src/secrets/target-registry-data.ts index 40394af07262..9dcfb589fce3 100644 --- a/src/secrets/target-registry-data.ts +++ b/src/secrets/target-registry-data.ts @@ -1,6 +1,5 @@ -import { getCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-snapshot.js"; import type { PluginManifestRecord } from "../plugins/manifest-registry.js"; -import { loadPluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; +import { resolvePluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; import { loadChannelSecretContractApiForRecord } from "./channel-contract-api.js"; import type { SecretTargetRegistryEntry } from "./target-registry-types.js"; @@ -469,19 +468,11 @@ function loadSecretTargetRegistryFromPluginMetadata(params: { env: NodeJS.ProcessEnv; preferPersisted?: boolean; }): SecretTargetRegistryEntry[] { - const plugins = - (params.preferPersisted === false - ? undefined - : getCurrentPluginMetadataSnapshot({ - config: {}, - env: params.env, - }) - )?.plugins ?? - loadPluginMetadataSnapshot({ - config: {}, - env: params.env, - ...(params.preferPersisted !== undefined ? { preferPersisted: params.preferPersisted } : {}), - }).plugins; + const plugins = resolvePluginMetadataSnapshot({ + config: {}, + env: params.env, + ...(params.preferPersisted !== undefined ? { preferPersisted: params.preferPersisted } : {}), + }).plugins; const bundledPlugins = plugins.filter((record) => record.origin === "bundled"); const channelPlugins = plugins.filter((record) => record.channels.length > 0); return [