mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
perf(gateway): reuse lifecycle plugin metadata instead of per-turn rescans (#120344)
* perf(gateway): reuse lifecycle plugin metadata * test(commands): expect workspace-scoped snapshot reuse in sessions metadata prep
This commit is contained in:
committed by
GitHub
parent
45b1dbf962
commit
0aa85c7f83
@@ -69,6 +69,9 @@ export async function prepareEmbeddedAttemptTrajectory(input: {
|
||||
buildTrajectoryRunMetadata({
|
||||
env: process.env,
|
||||
config: attempt.config,
|
||||
...(attempt.preparedModelRuntime?.metadataSnapshot
|
||||
? { pluginMetadataSnapshot: attempt.preparedModelRuntime.metadataSnapshot }
|
||||
: {}),
|
||||
workspaceDir: input.effectiveWorkspace,
|
||||
sessionFile: attempt.sessionFile,
|
||||
sessionKey: attempt.sessionKey,
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
type PluginMetadataSnapshotScopeRunner,
|
||||
} from "../../../plugins/current-plugin-metadata-snapshot.js";
|
||||
import {
|
||||
completePluginMetadataSnapshot,
|
||||
isPluginMetadataSnapshotCompatible,
|
||||
loadPluginMetadataSnapshot,
|
||||
type PluginMetadataSnapshot,
|
||||
@@ -24,15 +25,7 @@ export function completeDoctorPluginMetadataSnapshot(params: {
|
||||
config: OpenClawConfig;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): PluginMetadataSnapshot | undefined {
|
||||
if (!params.snapshot || params.snapshot.pluginIds === undefined) {
|
||||
return params.snapshot;
|
||||
}
|
||||
return loadPluginMetadataSnapshot({
|
||||
config: params.config,
|
||||
env: params.env ?? process.env,
|
||||
index: params.snapshot.index,
|
||||
...(params.snapshot.workspaceDir ? { workspaceDir: params.snapshot.workspaceDir } : {}),
|
||||
});
|
||||
return completePluginMetadataSnapshot(params);
|
||||
}
|
||||
|
||||
/** Reuses one exact immutable plugin metadata generation per Doctor workspace. */
|
||||
|
||||
@@ -86,6 +86,7 @@ describe("sessions plugin metadata preparation", () => {
|
||||
expect(resolvePluginMetadataSnapshotMock).toHaveBeenCalledWith({
|
||||
config,
|
||||
env: process.env,
|
||||
allowWorkspaceScopedCurrent: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -8,6 +8,7 @@ import { getRuntimeConfig } from "../config/io.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { setCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-snapshot.js";
|
||||
import { completePluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js";
|
||||
import type { ExecApprovalManager } from "./exec-approval-manager.js";
|
||||
import { revokeAttachGrantsForSession } from "./mcp-grant-store.js";
|
||||
import { ADMIN_SCOPE } from "./method-scopes.js";
|
||||
@@ -495,7 +496,13 @@ export async function startGatewayCoreRuntime(input: {
|
||||
pluginLookUpTable: nextPluginLookUpTable,
|
||||
ambientEnvTriggers,
|
||||
});
|
||||
setCurrentPluginMetadataSnapshot(nextPluginLookUpTable, {
|
||||
const nextPluginMetadataSnapshot = completePluginMetadataSnapshot({
|
||||
snapshot: nextPluginLookUpTable,
|
||||
config: params.nextConfig,
|
||||
env: params.env,
|
||||
workspaceDir: defaultWorkspaceDir,
|
||||
});
|
||||
setCurrentPluginMetadataSnapshot(nextPluginMetadataSnapshot, {
|
||||
config: params.nextConfig,
|
||||
env: params.env,
|
||||
workspaceDir: defaultWorkspaceDir,
|
||||
|
||||
@@ -35,6 +35,7 @@ import { setGatewaySigusr1RestartPolicy, setPreRestartDeferralCheck } from "../i
|
||||
import { enqueueSystemEvent } from "../infra/system-events.js";
|
||||
import type { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { setCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-snapshot.js";
|
||||
import { completePluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js";
|
||||
import { getTotalQueueSize } from "../process/command-queue.js";
|
||||
import { getActiveGatewayRootWorkCount } from "../process/gateway-work-admission.js";
|
||||
import { createLazyPromise } from "../shared/lazy-runtime.js";
|
||||
@@ -493,6 +494,7 @@ export async function prepareGatewayServerBootstrap(input: {
|
||||
defaultWorkspaceDir,
|
||||
startupPluginIds,
|
||||
pluginManifestRecords,
|
||||
pluginMetadataSnapshot,
|
||||
pluginLookUpTable,
|
||||
baseMethods,
|
||||
ambientAutostartSuppressedChannelIds,
|
||||
@@ -505,7 +507,13 @@ export async function prepareGatewayServerBootstrap(input: {
|
||||
sourceConfig: startupLastGoodSnapshot.sourceConfig,
|
||||
});
|
||||
const coreGatewayMethodNames = listCoreGatewayMethodNames();
|
||||
setCurrentPluginMetadataSnapshot(pluginLookUpTable, {
|
||||
const currentPluginMetadataSnapshot = completePluginMetadataSnapshot({
|
||||
snapshot: pluginMetadataSnapshot,
|
||||
config: startupActivationSourceConfig,
|
||||
env: process.env,
|
||||
workspaceDir: defaultWorkspaceDir,
|
||||
});
|
||||
setCurrentPluginMetadataSnapshot(currentPluginMetadataSnapshot, {
|
||||
config: startupActivationSourceConfig,
|
||||
compatibleConfigs: [startupRuntimeConfig, cfgAtStart, gatewayPluginConfigAtStart],
|
||||
env: process.env,
|
||||
@@ -558,6 +566,7 @@ export async function prepareGatewayServerBootstrap(input: {
|
||||
defaultWorkspaceDir,
|
||||
startupPluginIds,
|
||||
pluginManifestRecords,
|
||||
pluginMetadataSnapshot,
|
||||
pluginLookUpTable,
|
||||
baseMethods,
|
||||
ambientAutostartSuppressedChannelIds,
|
||||
|
||||
@@ -203,6 +203,7 @@ function slackConfig(): OpenClawConfig {
|
||||
async function prepareBootstrapWithRuntimeConfig(
|
||||
cfg: OpenClawConfig,
|
||||
options: {
|
||||
pluginMetadataSnapshot?: PluginMetadataSnapshot;
|
||||
workerProviderIds?: readonly string[];
|
||||
} = {},
|
||||
) {
|
||||
@@ -481,9 +482,11 @@ describe("prepareGatewayPluginBootstrap startup plugins", () => {
|
||||
} as OpenClawConfig;
|
||||
|
||||
const result = await prepareBootstrapWithRuntimeConfig(cfg, {
|
||||
pluginMetadataSnapshot,
|
||||
workerProviderIds: ["static-ssh"],
|
||||
});
|
||||
expect(result.startupPluginIds).toEqual([]);
|
||||
expect(result.pluginMetadataSnapshot).toBe(pluginMetadataSnapshot);
|
||||
expect(result.pluginLookUpTable).toBeUndefined();
|
||||
expect(result.baseGatewayMethods).toEqual(["ping"]);
|
||||
|
||||
|
||||
@@ -145,8 +145,8 @@ export async function prepareGatewayPluginBootstrap(params: {
|
||||
workerProviderIds: params.workerProviderIds ?? [],
|
||||
ambientEnvTriggers: params.ambientEnvTriggers,
|
||||
});
|
||||
// Startup logging consumes the same process-stable manifest snapshot used for
|
||||
// activation planning. Minimal gateways deliberately have no plugin metadata.
|
||||
// Startup logging and lifecycle publication consume the process-stable metadata snapshot.
|
||||
// Minimal gateways skip runtime lookup-table construction, not metadata ownership.
|
||||
const pluginManifestRecords =
|
||||
pluginLookUpTable?.manifestRegistry.plugins ??
|
||||
params.pluginMetadataSnapshot?.manifestRegistry.plugins ??
|
||||
@@ -179,6 +179,7 @@ export async function prepareGatewayPluginBootstrap(params: {
|
||||
defaultWorkspaceDir,
|
||||
startupPluginIds,
|
||||
pluginManifestRecords,
|
||||
pluginMetadataSnapshot: pluginLookUpTable ?? params.pluginMetadataSnapshot,
|
||||
pluginLookUpTable,
|
||||
baseMethods,
|
||||
pluginRegistry,
|
||||
|
||||
@@ -7,6 +7,7 @@ import { resolveInstalledPluginIndexPolicyHash } from "./installed-plugin-index-
|
||||
import type { InstalledPluginIndex } from "./installed-plugin-index.js";
|
||||
import type { PluginManifestRecord, PluginManifestRegistry } from "./manifest-registry.js";
|
||||
import {
|
||||
completePluginMetadataSnapshot,
|
||||
loadPluginMetadataSnapshot,
|
||||
resolvePluginMetadataSnapshot,
|
||||
} from "./plugin-metadata-snapshot.js";
|
||||
@@ -114,6 +115,48 @@ describe("plugin metadata snapshot", () => {
|
||||
expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("promotes one scoped lifecycle graph and reuses it across runtime resolutions", () => {
|
||||
const config = {};
|
||||
const workspaceDir = "/workspace";
|
||||
const index = makeIndex();
|
||||
index.policyHash = resolveInstalledPluginIndexPolicyHash(config);
|
||||
loadPluginRegistrySnapshotWithMetadata.mockReturnValue({
|
||||
source: "provided",
|
||||
snapshot: index,
|
||||
diagnostics: [],
|
||||
});
|
||||
const scoped = loadPluginMetadataSnapshot({
|
||||
config,
|
||||
env: {},
|
||||
index,
|
||||
pluginIds: ["demo"],
|
||||
workspaceDir,
|
||||
});
|
||||
|
||||
const complete = completePluginMetadataSnapshot({
|
||||
snapshot: scoped,
|
||||
config,
|
||||
env: {},
|
||||
workspaceDir,
|
||||
});
|
||||
expect(complete?.pluginIds).toBeUndefined();
|
||||
setCurrentPluginMetadataSnapshot(complete, { config, env: {}, workspaceDir });
|
||||
loadPluginRegistrySnapshotWithMetadata.mockClear();
|
||||
loadPluginManifestRegistryForInstalledIndex.mockClear();
|
||||
|
||||
expect(
|
||||
completePluginMetadataSnapshot({ snapshot: complete, config, env: {}, workspaceDir }),
|
||||
).toBe(complete);
|
||||
expect(resolvePluginMetadataSnapshot({ env: {}, allowWorkspaceScopedCurrent: true })).toBe(
|
||||
complete,
|
||||
);
|
||||
for (let iteration = 0; iteration < 20; iteration += 1) {
|
||||
expect(resolvePluginMetadataSnapshot({ config, env: {}, workspaceDir })).toBe(complete);
|
||||
}
|
||||
expect(loadPluginRegistrySnapshotWithMetadata).not.toHaveBeenCalled();
|
||||
expect(loadPluginManifestRegistryForInstalledIndex).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rewalks collection-bearing manifest graphs after prototype mutation", () => {
|
||||
const index = makeIndex();
|
||||
const registry = makeManifestRegistry();
|
||||
|
||||
@@ -292,6 +292,25 @@ export function loadPluginMetadataSnapshot(
|
||||
);
|
||||
}
|
||||
|
||||
/** Promotes a planning-scoped graph to the complete process-lifecycle metadata snapshot. */
|
||||
export function completePluginMetadataSnapshot(params: {
|
||||
snapshot?: PluginMetadataSnapshot;
|
||||
config: OpenClawConfig;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
workspaceDir?: string;
|
||||
}): PluginMetadataSnapshot | undefined {
|
||||
if (!params.snapshot || params.snapshot.pluginIds === undefined) {
|
||||
return params.snapshot;
|
||||
}
|
||||
const workspaceDir = params.workspaceDir ?? params.snapshot.workspaceDir;
|
||||
return loadPluginMetadataSnapshot({
|
||||
config: params.config,
|
||||
env: params.env ?? process.env,
|
||||
index: params.snapshot.index,
|
||||
...(workspaceDir ? { workspaceDir } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
export function resolvePluginMetadataSnapshot(
|
||||
params: ResolvePluginMetadataSnapshotParams,
|
||||
): PluginMetadataSnapshot {
|
||||
@@ -303,6 +322,7 @@ export function resolvePluginMetadataSnapshot(
|
||||
const current = getCurrentPluginMetadataSnapshot({
|
||||
config: params.config,
|
||||
env: params.env,
|
||||
...(params.config === undefined ? { requireDefaultDiscoveryContext: true } : {}),
|
||||
...(params.pluginIds !== undefined ? { pluginIds: params.pluginIds } : {}),
|
||||
...(params.pluginIdScope !== undefined ? { pluginIdScope: params.pluginIdScope } : {}),
|
||||
...(params.workspaceDir !== undefined ? { workspaceDir: params.workspaceDir } : {}),
|
||||
|
||||
@@ -1061,9 +1061,9 @@ export function resolveExternalAuthProfilesWithPlugins(params: {
|
||||
const env = params.env ?? process.env;
|
||||
const config = params.config ?? {};
|
||||
const currentMetadataSnapshot = getCurrentPluginMetadataSnapshot({
|
||||
config,
|
||||
env,
|
||||
...(workspaceDir === undefined ? { allowWorkspaceScopedSnapshot: true } : { workspaceDir }),
|
||||
...(params.config ? { config } : { requireDefaultDiscoveryContext: true }),
|
||||
...(workspaceDir ? { workspaceDir } : { allowWorkspaceScopedSnapshot: true }),
|
||||
});
|
||||
const { manifestRegistry } =
|
||||
currentMetadataSnapshot ?? resolvePluginMetadataSnapshot({ config, workspaceDir, env });
|
||||
|
||||
@@ -149,7 +149,7 @@ describe("setup-registry descriptor lookup", () => {
|
||||
expect(resolvePluginSetupCliBackendDescriptor({ backend: "disabled-cli" })).toBeUndefined();
|
||||
expect(loadPluginMetadataSnapshotMock).toHaveBeenCalledTimes(3);
|
||||
expect(loadPluginMetadataSnapshotMock).toHaveBeenCalledWith({
|
||||
config: {},
|
||||
allowWorkspaceScopedCurrent: true,
|
||||
env: process.env,
|
||||
});
|
||||
});
|
||||
@@ -232,7 +232,7 @@ describe("setup-registry descriptor lookup", () => {
|
||||
expect(loadPluginMetadataSnapshotMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not reuse workspace-scoped current metadata without a workspace context", async () => {
|
||||
it("reuses the lifecycle-owned workspace when no runtime workspace is active", async () => {
|
||||
loadPluginMetadataSnapshotMock.mockReturnValue({
|
||||
index: {
|
||||
diagnostics: [],
|
||||
@@ -252,12 +252,10 @@ describe("setup-registry descriptor lookup", () => {
|
||||
{ config: {}, env: process.env },
|
||||
);
|
||||
|
||||
expect(
|
||||
resolvePluginSetupCliBackendDescriptor({ backend: "codex-cli", config: {} }),
|
||||
).toBeUndefined();
|
||||
expect(loadPluginMetadataSnapshotMock).toHaveBeenCalledWith({
|
||||
config: {},
|
||||
env: process.env,
|
||||
expect(resolvePluginSetupCliBackendDescriptor({ backend: "codex-cli", config: {} })).toEqual({
|
||||
pluginId: "openai",
|
||||
backend: { id: "Codex-CLI" },
|
||||
});
|
||||
expect(loadPluginMetadataSnapshotMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -37,14 +37,10 @@ function resolveMetadataSnapshotForSetupCliBackends(
|
||||
const env = params.env ?? process.env;
|
||||
const workspaceDir = params.workspaceDir ?? getActivePluginRegistryWorkspaceDirFromState();
|
||||
const snapshot = resolvePluginMetadataSnapshot({
|
||||
config: params.config ?? {},
|
||||
...(params.config ? { config: params.config } : {}),
|
||||
env,
|
||||
...(workspaceDir !== undefined
|
||||
? {
|
||||
workspaceDir,
|
||||
allowWorkspaceScopedCurrent: true,
|
||||
}
|
||||
: {}),
|
||||
...(workspaceDir ? { workspaceDir } : {}),
|
||||
allowWorkspaceScopedCurrent: true,
|
||||
});
|
||||
return {
|
||||
snapshot,
|
||||
|
||||
@@ -16,20 +16,20 @@ describe("getSecretTargetRegistry metadata reuse", () => {
|
||||
metadataMocks.resolvePluginMetadataSnapshot.mockReturnValue({ plugins: [] });
|
||||
});
|
||||
|
||||
it("uses configless global metadata without a workspace-scoped current request", async () => {
|
||||
it("allows configless runtime targets to reuse the lifecycle workspace", async () => {
|
||||
const { getSecretTargetRegistry } = await import("./target-registry-data.js");
|
||||
|
||||
getSecretTargetRegistry();
|
||||
|
||||
expect(metadataMocks.resolvePluginMetadataSnapshot).toHaveBeenCalledWith({
|
||||
config: {},
|
||||
allowWorkspaceScopedCurrent: true,
|
||||
env: process.env,
|
||||
});
|
||||
const calls = metadataMocks.resolvePluginMetadataSnapshot.mock.calls as unknown as Array<
|
||||
[{ allowWorkspaceScopedCurrent?: boolean }]
|
||||
>;
|
||||
for (const [call] of calls) {
|
||||
expect(call.allowWorkspaceScopedCurrent).not.toBe(true);
|
||||
expect(call.allowWorkspaceScopedCurrent).toBe(true);
|
||||
}
|
||||
});
|
||||
it("registers secret targets for installed-origin plugins (#104320)", async () => {
|
||||
|
||||
@@ -467,8 +467,8 @@ function loadSecretTargetRegistryFromPluginMetadata(params: {
|
||||
preferPersisted?: boolean;
|
||||
}): SecretTargetRegistryEntry[] {
|
||||
const plugins = resolvePluginMetadataSnapshot({
|
||||
config: {},
|
||||
env: params.env,
|
||||
allowWorkspaceScopedCurrent: true,
|
||||
...(params.preferPersisted !== undefined ? { preferPersisted: params.preferPersisted } : {}),
|
||||
}).plugins;
|
||||
const channelPlugins = plugins.filter((record) => record.channels.length > 0);
|
||||
|
||||
@@ -44,9 +44,37 @@ import { buildTrajectoryArtifacts, buildTrajectoryRunMetadata } from "./metadata
|
||||
|
||||
afterEach(() => {
|
||||
resetPluginRuntimeStateForTest();
|
||||
loadPluginManifestRegistry.mockClear();
|
||||
});
|
||||
|
||||
describe("trajectory metadata", () => {
|
||||
it("uses prepared plugin metadata without rescanning manifests", () => {
|
||||
const metadata = buildTrajectoryRunMetadata({
|
||||
pluginMetadataSnapshot: {
|
||||
plugins: [
|
||||
{
|
||||
id: "prepared-plugin",
|
||||
name: "Prepared Plugin",
|
||||
origin: "bundled",
|
||||
channels: [],
|
||||
providers: [],
|
||||
cliBackends: [],
|
||||
hooks: [],
|
||||
skills: [],
|
||||
},
|
||||
],
|
||||
} as never,
|
||||
workspaceDir: "/tmp/workspace",
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
|
||||
expect(metadata.plugins).toMatchObject({
|
||||
source: "manifest-registry",
|
||||
entries: [{ id: "prepared-plugin" }],
|
||||
});
|
||||
expect(loadPluginManifestRegistry).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("redacts harness argv and local paths with the support redaction rules", () => {
|
||||
const originalArgv = process.argv;
|
||||
process.argv = [
|
||||
|
||||
@@ -10,7 +10,10 @@ import {
|
||||
sanitizeSupportSnapshotValue,
|
||||
type SupportRedactionContext,
|
||||
} from "../logging/diagnostic-support-redaction.js";
|
||||
import { loadPluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js";
|
||||
import {
|
||||
loadPluginMetadataSnapshot,
|
||||
type PluginMetadataSnapshot,
|
||||
} from "../plugins/plugin-metadata-snapshot.js";
|
||||
import { getActivePluginRegistry, listImportedRuntimePluginIds } from "../plugins/runtime.js";
|
||||
import type { SkillSnapshot } from "../skills/types.js";
|
||||
import { VERSION } from "../version.js";
|
||||
@@ -20,6 +23,7 @@ import { VERSION } from "../version.js";
|
||||
type BuildTrajectoryRunMetadataParams = {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
config?: OpenClawConfig;
|
||||
pluginMetadataSnapshot?: PluginMetadataSnapshot;
|
||||
workspaceDir: string;
|
||||
sessionFile?: string;
|
||||
sessionKey?: string;
|
||||
@@ -139,16 +143,19 @@ function buildPluginsFromActiveRegistry() {
|
||||
|
||||
function buildPluginsFromManifest(params: {
|
||||
config?: OpenClawConfig;
|
||||
pluginMetadataSnapshot?: PluginMetadataSnapshot;
|
||||
workspaceDir?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}) {
|
||||
// Startup captures can happen before runtime activation. Fall back to the
|
||||
// manifest snapshot so exported runs still show configured plugin surfaces.
|
||||
const snapshot = loadPluginMetadataSnapshot({
|
||||
config: params.config ?? {},
|
||||
workspaceDir: params.workspaceDir,
|
||||
env: params.env ?? process.env,
|
||||
});
|
||||
const snapshot =
|
||||
params.pluginMetadataSnapshot ??
|
||||
loadPluginMetadataSnapshot({
|
||||
config: params.config ?? {},
|
||||
workspaceDir: params.workspaceDir,
|
||||
env: params.env ?? process.env,
|
||||
});
|
||||
return {
|
||||
source: "manifest-registry",
|
||||
entries: snapshot.plugins
|
||||
@@ -235,6 +242,9 @@ export function buildTrajectoryRunMetadata(
|
||||
buildPluginsFromActiveRegistry() ??
|
||||
buildPluginsFromManifest({
|
||||
config: params.config,
|
||||
...(params.pluginMetadataSnapshot
|
||||
? { pluginMetadataSnapshot: params.pluginMetadataSnapshot }
|
||||
: {}),
|
||||
workspaceDir: params.workspaceDir,
|
||||
env,
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user