From 824d1706801f48c2edd76ef2dbbaca1ebe114eca Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Thu, 6 Aug 2026 02:07:57 +0800 Subject: [PATCH] fix(memory): keep gateway RPCs responsive during startup (#119676) * fix(memory): defer startup manager warmup * test(plugins): register memory stop hook contract --- extensions/memory-core/index.test.ts | 126 +++++++++++++++++- extensions/memory-core/index.ts | 77 ++++++++--- .../contracts/boundary-invariants.test.ts | 7 +- 3 files changed, 186 insertions(+), 24 deletions(-) diff --git a/extensions/memory-core/index.test.ts b/extensions/memory-core/index.test.ts index 1bfebd0f8a90..e5b304aa36cc 100644 --- a/extensions/memory-core/index.test.ts +++ b/extensions/memory-core/index.test.ts @@ -3,7 +3,7 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import type { OpenClawPluginApi, OpenClawPluginCommandDefinition } from "openclaw/plugin-sdk/core"; import type { MemoryPluginRuntime } from "openclaw/plugin-sdk/memory-core-host-runtime-core"; import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { buildMemoryFlushPlan } from "./src/flush-plan.js"; import type { MemoryCoreRuntimeHost } from "./src/memory/runtime-host.js"; import { buildPromptSection } from "./src/prompt-section.js"; @@ -112,6 +112,10 @@ describe("memory-core plugin runtime registration", () => { vi.clearAllMocks(); }); + afterEach(() => { + vi.useRealTimers(); + }); + it("registers the dreaming runtime slash command", () => { let command: OpenClawPluginCommandDefinition | undefined; plugin.register( @@ -196,6 +200,7 @@ describe("memory-core plugin runtime registration", () => { }); it("warms each configured memory manager at gateway start and logs failures at debug", async () => { + vi.useFakeTimers(); const gatewayStartHandlers: Array<(event: unknown, ctx: { config: OpenClawConfig }) => void> = []; const syncMain = vi.fn(async () => {}); @@ -229,6 +234,8 @@ describe("memory-core plugin runtime registration", () => { } warmup({}, { config }); + expect(getMemorySearchManagerMock).not.toHaveBeenCalled(); + await vi.runOnlyPendingTimersAsync(); await vi.waitFor(() => { expect(getMemorySearchManagerMock).toHaveBeenCalledTimes(2); expect(syncMain).toHaveBeenCalledWith({ reason: "startup-warmup" }); @@ -239,7 +246,123 @@ describe("memory-core plugin runtime registration", () => { }); }); + it("defers memory manager initialization until after the gateway start turn", async () => { + vi.useFakeTimers(); + const gatewayStartHandlers: Array<(event: unknown, ctx: { config: OpenClawConfig }) => void> = + []; + getMemorySearchManagerMock.mockResolvedValue({ manager: null } as never); + const config = {} as OpenClawConfig; + plugin.register( + createTestPluginApi({ + config, + runtime: hostRuntime, + on(hookName, handler) { + if (hookName === "gateway_start") { + gatewayStartHandlers.push( + handler as unknown as (event: unknown, ctx: { config: OpenClawConfig }) => void, + ); + } + }, + }), + ); + const warmup = gatewayStartHandlers.at(-1); + if (!warmup) { + throw new Error("expected memory warmup gateway_start hook"); + } + + warmup({}, { config }); + + expect(getMemorySearchManagerMock).not.toHaveBeenCalled(); + await vi.runOnlyPendingTimersAsync(); + await vi.waitFor(() => expect(getMemorySearchManagerMock).toHaveBeenCalledTimes(1)); + }); + + it("cancels deferred memory warmup when the gateway stops", async () => { + vi.useFakeTimers(); + const gatewayStartHandlers: Array<(event: unknown, ctx: { config: OpenClawConfig }) => void> = + []; + const gatewayStopHandlers: Array<() => void> = []; + const config = {} as OpenClawConfig; + plugin.register( + createTestPluginApi({ + config, + runtime: hostRuntime, + on(hookName, handler) { + if (hookName === "gateway_start") { + gatewayStartHandlers.push( + handler as unknown as (event: unknown, ctx: { config: OpenClawConfig }) => void, + ); + } + if (hookName === "gateway_stop") { + gatewayStopHandlers.push(handler as unknown as () => void); + } + }, + }), + ); + const warmup = gatewayStartHandlers.at(-1); + const stop = gatewayStopHandlers.at(-1); + if (!warmup || !stop) { + throw new Error("expected memory warmup lifecycle hooks"); + } + + warmup({}, { config }); + stop(); + await vi.runOnlyPendingTimersAsync(); + + expect(getMemorySearchManagerMock).not.toHaveBeenCalled(); + }); + + it("prevents a restarted gateway generation from syncing a stale manager", async () => { + vi.useFakeTimers(); + const gatewayStartHandlers: Array<(event: unknown, ctx: { config: OpenClawConfig }) => void> = + []; + let resolveStaleManager: + | ((value: { manager: { sync: () => Promise } }) => void) + | undefined; + const staleManagerResult = new Promise<{ manager: { sync: () => Promise } }>( + (resolve) => { + resolveStaleManager = resolve; + }, + ); + const staleSync = vi.fn(async () => {}); + const currentSync = vi.fn(async () => {}); + getMemorySearchManagerMock + .mockReturnValueOnce(staleManagerResult as never) + .mockResolvedValueOnce({ manager: { sync: currentSync } } as never); + const config = {} as OpenClawConfig; + plugin.register( + createTestPluginApi({ + config, + runtime: hostRuntime, + on(hookName, handler) { + if (hookName === "gateway_start") { + gatewayStartHandlers.push( + handler as unknown as (event: unknown, ctx: { config: OpenClawConfig }) => void, + ); + } + }, + }), + ); + const warmup = gatewayStartHandlers.at(-1); + if (!warmup) { + throw new Error("expected memory warmup gateway_start hook"); + } + + warmup({}, { config }); + await vi.runOnlyPendingTimersAsync(); + await vi.waitFor(() => expect(getMemorySearchManagerMock).toHaveBeenCalledTimes(1)); + + warmup({}, { config }); + await vi.runOnlyPendingTimersAsync(); + await vi.waitFor(() => expect(currentSync).toHaveBeenCalledWith({ reason: "startup-warmup" })); + + resolveStaleManager?.({ manager: { sync: staleSync } }); + await vi.waitFor(() => expect(getMemorySearchManagerMock).toHaveBeenCalledTimes(2)); + expect(staleSync).not.toHaveBeenCalled(); + }); + it("leaves QMD startup synchronization to the backend boot policy", async () => { + vi.useFakeTimers(); const gatewayStartHandlers: Array<(event: unknown, ctx: { config: OpenClawConfig }) => void> = []; const sync = vi.fn(async () => {}); @@ -267,6 +390,7 @@ describe("memory-core plugin runtime registration", () => { warmup({}, { config }); + await vi.runOnlyPendingTimersAsync(); await vi.waitFor(() => expect(getMemorySearchManagerMock).toHaveBeenCalledTimes(1)); expect(sync).not.toHaveBeenCalled(); }); diff --git a/extensions/memory-core/index.ts b/extensions/memory-core/index.ts index 1b009e295a2c..82ba1dadc909 100644 --- a/extensions/memory-core/index.ts +++ b/extensions/memory-core/index.ts @@ -57,6 +57,8 @@ const loadRuntimeProviderModule = createLazyRuntimeModule( () => import("./src/runtime-provider.js"), ); +const MEMORY_MANAGER_WARMUP_DELAY_MS = 5_000; + function getToolConfig(options: MemoryToolOptions): OpenClawConfig | undefined { return options.getConfig?.() ?? options.config; } @@ -279,35 +281,66 @@ function registerMemoryManagerWarmup( api: OpenClawPluginApi, memoryRuntime: MemoryPluginRuntime, ): void { + let lifecycleGeneration = 0; + let warmupTimer: ReturnType | null = null; + + const cancelWarmup = () => { + lifecycleGeneration += 1; + if (warmupTimer) { + clearTimeout(warmupTimer); + warmupTimer = null; + } + }; + api.on("gateway_start", (_event, ctx) => { + cancelWarmup(); + const generation = lifecycleGeneration; const config = (api.runtime.config?.current?.() ?? ctx.config ?? api.config) as OpenClawConfig; if (normalizePluginsConfig(config.plugins).slots.memory !== "memory-core") { return; } - for (const agentId of listAgentIds(config)) { - const backend = memoryRuntime.resolveMemoryBackendConfig({ cfg: config, agentId }); - void memoryRuntime - .getMemorySearchManager({ cfg: config, agentId }) - .then(async ({ manager, error }) => { - if (!manager) { - if (error) { - api.logger.debug?.(`memory-core: startup index warmup unavailable: ${error}`); + + // Leave a bounded idle window for initial post-ready RPCs before loading the manager. + const timer = setTimeout(() => { + if (warmupTimer !== timer || generation !== lifecycleGeneration) { + return; + } + warmupTimer = null; + for (const agentId of listAgentIds(config)) { + const backend = memoryRuntime.resolveMemoryBackendConfig({ cfg: config, agentId }); + void memoryRuntime + .getMemorySearchManager({ cfg: config, agentId }) + .then(async ({ manager, error }) => { + if (generation !== lifecycleGeneration) { + return; } - return; - } - if (backend.backend === "builtin") { - await manager.sync?.({ reason: "startup-warmup" }); - } - }) - .catch((error: unknown) => { - api.logger.debug?.( - `memory-core: startup index warmup failed for ${agentId}: ${ - error instanceof Error ? error.message : String(error) - }`, - ); - }); - } + if (!manager) { + if (error) { + api.logger.debug?.(`memory-core: startup index warmup unavailable: ${error}`); + } + return; + } + if (backend.backend === "builtin") { + await manager.sync?.({ reason: "startup-warmup" }); + } + }) + .catch((error: unknown) => { + if (generation !== lifecycleGeneration) { + return; + } + api.logger.debug?.( + `memory-core: startup index warmup failed for ${agentId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + }); + } + }, MEMORY_MANAGER_WARMUP_DELAY_MS); + warmupTimer = timer; + warmupTimer.unref?.(); }); + + api.on("gateway_stop", cancelWarmup); } export default definePluginEntry({ diff --git a/src/plugins/contracts/boundary-invariants.test.ts b/src/plugins/contracts/boundary-invariants.test.ts index 75fc5153a160..a8b9b2cd1ffc 100644 --- a/src/plugins/contracts/boundary-invariants.test.ts +++ b/src/plugins/contracts/boundary-invariants.test.ts @@ -42,7 +42,12 @@ const BUNDLED_TYPED_HOOK_REGISTRATION_GUARDS = { "extensions/feishu/subagent-hooks-api.ts": ["subagent_delivery_target", "subagent_ended"], "extensions/matrix/subagent-hooks-api.ts": ["subagent_delivery_target", "subagent_ended"], "extensions/memory-core/src/dreaming.ts": ["before_agent_reply", "gateway_start", "gateway_stop"], - "extensions/memory-core/index.ts": ["before_agent_reply", "before_prompt_build", "gateway_start"], + "extensions/memory-core/index.ts": [ + "before_agent_reply", + "before_prompt_build", + "gateway_start", + "gateway_stop", + ], "extensions/memory-lancedb/index.ts": ["agent_end", "before_prompt_build", "session_end"], "extensions/onepassword/index.ts": ["before_tool_call", "tool_result_persist"], "extensions/thread-ownership/index.ts": ["message_received", "message_sending"],