diff --git a/docs/.generated/config-baseline.sha256 b/docs/.generated/config-baseline.sha256 index ef1294e5ceaa..015b6c066d17 100644 --- a/docs/.generated/config-baseline.sha256 +++ b/docs/.generated/config-baseline.sha256 @@ -1,4 +1,4 @@ -b5887d8c887a53a997dc4c5220f6b9d5adffbddb58e4b25ae0d20ca06850d0ca config-baseline.json +0485ba902d2afd89d2c41cde7180d0cec2900b2db6804b9f97d42b7d85cd3af5 config-baseline.json 72bb80be618406f3337eaa2560d2559a35e49bd29576de8dd4a3aec1a6a94d92 config-baseline.core.json 1218f5555541b61bd5ddcac6441f15061b44789e2471d4ffecbe3059777c55c1 config-baseline.channel.json -b0dec5acfe60557e728e5ad03cc36d19d2432d51f755656c97846afa7fbe374a config-baseline.plugin.json +a14ac4261e98403d1a7e047070e6f151938444e27382b860315bd0c74fda4861 config-baseline.plugin.json diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index e1878251ce76..52299fef8858 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -ab2a32b037be61953ad32d2498468bc812b0794ba6135530cefd1c8326d69de8 plugin-sdk-api-baseline.json -2c7bca3b46e0edd08ed445a241bb0a80b77635bf82825d5201ce41a8759c0e56 plugin-sdk-api-baseline.jsonl +85c3572e6ed2bfe3df92c7d53cef465b30d2e861ad9529009faa287cdc5aec71 plugin-sdk-api-baseline.json +0d7c7e42d04b97d40519c5a23ba96599b05868c71a997eb913b9fccbc5fb2515 plugin-sdk-api-baseline.jsonl diff --git a/docs/concepts/active-memory.md b/docs/concepts/active-memory.md index b23108a06220..3494e23a8bf2 100644 --- a/docs/concepts/active-memory.md +++ b/docs/concepts/active-memory.md @@ -479,6 +479,9 @@ names that plugin registers. Active Memory lists those tools in the recall prompt and passes the same list to the embedded sub-agent. If none of the configured tools are available, or the memory sub-agent fails, Active Memory skips recall for that turn and the main reply continues without memory context. +For custom recall tools, non-empty model-visible tool output counts as recall +evidence unless structured result fields explicitly report an empty result or +failure. `toolsAllow` only accepts concrete memory tool names. Wildcards, `group:*` entries, and core agent tools such as `read`, `exec`, `message`, and `web_search` are ignored before the hidden memory sub-agent starts. @@ -743,7 +746,11 @@ Before v2026.5.2 the plugin silently extended your configured `timeoutMs` by an extra 30000 ms during cold-start so model warm-up, embedding-index load, and the first recall could share one larger budget. v2026.5.2 moved that grace behind an explicit `setupGraceTimeoutMs` config — your configured `timeoutMs` -is now the budget by default, unless you opt in. +is now the recall-work budget by default, unless you opt in. The blocking hook +uses two bounded phases around that budget: up to 1500 ms for session/config +preflight before recall starts, then a separate fixed 1500 ms for abort +settlement and transcript recovery after recall work stops. Neither allowance +extends model or tool execution. If you upgraded from v2026.4.x and you set `timeoutMs` to a value tuned for the old implicit-grace world (the recommended starter `timeoutMs: 15000` is one @@ -765,14 +772,16 @@ outer watchdog budgets back to the pre-v5.2 effective values: } ``` -Per the v2026.5.2 changelog: _"use the configured recall timeout as the -blocking prompt-build hook budget by default and move cold-start setup grace -behind explicit `setupGraceTimeoutMs` config, so the plugin no longer silently -extends 15000 ms configs to 45000 ms on the main lane."_ +The v2026.5.2 change removed the old implicit 30000 ms cold-start extension. +Beyond the configured recall-work budget, the hook can use up to 1500 ms for +preflight and another 1500 ms for post-recall completion. Its worst-case +blocking time is therefore `timeoutMs + setupGraceTimeoutMs + 3000` ms. The embedded recall runner uses the same effective timeout budget, so `setupGraceTimeoutMs` covers both the outer prompt-build watchdog and the inner -blocking recall run. +blocking recall run. The preflight cap covers session/config checks before that +budget begins. The post-recall allowance lets the outer hook settle abort +cleanup and read any final transcript state. For resource-tight gateways where cold-start latency is a known trade-off, lower values (5000–15000 ms) work too — the trade-off is a higher chance of diff --git a/extensions/active-memory/index.test.ts b/extensions/active-memory/index.test.ts index 4e567970954a..d7f6e7e30d53 100644 --- a/extensions/active-memory/index.test.ts +++ b/extensions/active-memory/index.test.ts @@ -217,6 +217,18 @@ describe("active-memory plugin", () => { "utf8", ); }; + const writeUsableMemoryTranscript = async (sessionFile: string, text: string) => { + await writeTranscriptJsonl(sessionFile, [ + { + message: { + role: "toolResult", + toolName: "memory_search", + details: { results: [{ text }] }, + content: [{ type: "text", text: JSON.stringify({ results: [{ text }] }) }], + }, + }, + ]); + }; const waitForAbort = async (abortSignal?: AbortSignal): Promise => { if (abortSignal?.aborted) { throw toLintErrorObject( @@ -372,8 +384,11 @@ describe("active-memory plugin", () => { for (const key of Object.keys(registeredCommands)) { delete registeredCommands[key]; } - runEmbeddedAgent.mockResolvedValue({ - payloads: [{ text: "- lemon pepper wings\n- blue cheese" }], + runEmbeddedAgent.mockImplementation(async (params: { sessionFile: string }) => { + await writeUsableMemoryTranscript(params.sessionFile, "lemon pepper wings with blue cheese"); + return { + payloads: [{ text: "- lemon pepper wings\n- blue cheese" }], + }; }); testing.resetActiveRecallCacheForTests(); testing.setTimeoutPartialDataGraceMsForTests(5); @@ -394,21 +409,21 @@ describe("active-memory plugin", () => { const [hookName, handler, options] = firstHookRegistration(); expect(hookName).toBe("before_prompt_build"); expect(typeof handler).toBe("function"); - expect(options).toEqual({ timeoutMs: 15_000 }); - expect(hookOptions.before_prompt_build?.timeoutMs).toBe(15_000); + expect(options).toEqual({ timeoutMs: 153_000 }); + expect(hookOptions.before_prompt_build?.timeoutMs).toBe(153_000); }); - it("registers before_prompt_build with the configured recall timeout", () => { + it("keeps the outer hook timeout at the live-config ceiling", () => { api.pluginConfig = { agents: ["main"], timeoutMs: 90_000, }; plugin.register(api as unknown as OpenClawPluginApi); - expect(hookOptions.before_prompt_build?.timeoutMs).toBe(90_000); + expect(hookOptions.before_prompt_build?.timeoutMs).toBe(153_000); }); - it("registers before_prompt_build with explicit setup grace when configured", () => { + it("covers the maximum recall and setup-grace budgets", () => { api.pluginConfig = { agents: ["main"], timeoutMs: 90_000, @@ -416,7 +431,7 @@ describe("active-memory plugin", () => { }; plugin.register(api as unknown as OpenClawPluginApi); - expect(hookOptions.before_prompt_build?.timeoutMs).toBe(120_000); + expect(hookOptions.before_prompt_build?.timeoutMs).toBe(153_000); }); it("runs recall without recording shared auth-profile failures", async () => { @@ -1822,8 +1837,11 @@ describe("active-memory plugin", () => { }); it("preserves leading digits in a plain-text summary", async () => { - runEmbeddedAgent.mockResolvedValueOnce({ - payloads: [{ text: "2024 trip to tokyo and 2% milk both matter here." }], + runEmbeddedAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + await writeUsableMemoryTranscript(params.sessionFile, "2024 trip to tokyo and 2% milk"); + return { + payloads: [{ text: "2024 trip to tokyo and 2% milk both matter here." }], + }; }); const result = await hooks.before_prompt_build( @@ -2021,7 +2039,8 @@ describe("active-memory plugin", () => { sessionId: "s-main", updatedAt: 0, }; - runEmbeddedAgent.mockImplementationOnce(async () => { + runEmbeddedAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + await writeUsableMemoryTranscript(params.sessionFile, "lemon pepper wings"); return { meta: { activeMemorySearchDebug: { @@ -2114,6 +2133,370 @@ describe("active-memory plugin", () => { expect(line).toContain("hits=3"); }); + it("recognizes usable persisted memory results after details are capped", () => { + const cappedDetails = { + persistedDetailsTruncated: true, + originalDetailKeys: ["results"], + }; + expect( + testing.hasUsableMemoryResultInSessionRecord({ + message: { + role: "toolResult", + toolName: "memory_search", + details: cappedDetails, + content: [{ type: "text", text: '{\n "results": [\n {"text": "ramen"}\n ]\n}' }], + }, + }), + ).toBe(true); + expect( + testing.hasUsableMemoryResultInSessionRecord({ + message: { + role: "toolResult", + toolName: "memory_search", + details: cappedDetails, + content: [{ type: "text", text: '{\n "results": []\n}' }], + }, + }), + ).toBe(false); + expect( + testing.hasUsableMemoryResultInSessionRecord({ + message: { + role: "toolResult", + toolName: "memory_recall", + details: cappedDetails, + content: [{ type: "text", text: "Found 2 memories:\n\n1. ramen\n2. chili crisp" }], + }, + }), + ).toBe(true); + expect( + testing.hasUsableMemoryResultInSessionRecord({ + message: { + role: "toolResult", + toolName: "memory_recall", + details: cappedDetails, + content: [{ type: "text", text: "No relevant memories found." }], + }, + }), + ).toBe(false); + expect( + testing.hasUsableMemoryResultInSessionRecord({ + message: { + role: "toolResult", + toolName: "memory_get", + details: { path: "memory/food.md", text: "User usually orders ramen." }, + content: [{ type: "text", text: '{"text":"User usually orders ramen."}' }], + }, + }), + ).toBe(true); + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "memory_lookup_custom", + details: {}, + content: [{ type: "text", text: "User usually orders ramen." }], + }, + }, + ["memory_lookup_custom"], + ), + ).toBe(true); + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "memory_lookup_custom", + details: { results: [] }, + content: [{ type: "text", text: "No memories found." }], + }, + }, + ["memory_lookup_custom"], + ), + ).toBe(false); + for (const status of [ + "failed", + "error", + "failure", + "timeout", + "TIMED OUT", + "timed_out", + "timed-out", + "unavailable", + "disabled", + "denied", + "cancelled", + "canceled", + "aborted", + "killed", + "invalid", + "forbidden", + "blocked", + ]) { + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "memory_lookup_custom", + details: { status }, + content: [{ type: "text", text: "The memory backend is unavailable." }], + }, + }, + ["memory_lookup_custom"], + ), + ).toBe(false); + } + for (const status of ["ok", "error_free", "not_failed", "not_cancelled"]) { + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "memory_lookup_custom", + details: { status }, + content: [{ type: "text", text: "User usually orders ramen." }], + }, + }, + ["memory_lookup_custom"], + ), + ).toBe(true); + } + for (const status of ["not_found", "empty", "no_results", "no_matches"]) { + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "memory_lookup_custom", + details: { status }, + content: [{ type: "text", text: "No memories found." }], + }, + }, + ["memory_lookup_custom"], + ), + ).toBe(false); + } + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "memory_lookup_custom", + details: {}, + content: [ + { type: "text", text: '{"status":"aborted"}' }, + { type: "text", text: "The memory lookup was cancelled." }, + ], + }, + }, + ["memory_lookup_custom"], + ), + ).toBe(false); + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "memory_lookup_custom", + details: {}, + content: [ + { type: "text", text: '{"status":"not_found"}' }, + { type: "text", text: "No memories found." }, + ], + }, + }, + ["memory_lookup_custom"], + ), + ).toBe(false); + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "memory_lookup_custom", + details: {}, + content: [{ type: "text", text: "[]" }], + }, + }, + ["memory_lookup_custom"], + ), + ).toBe(false); + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "memory_lookup_custom", + details: {}, + content: [ + { type: "text", text: '{"results":[]}' }, + { type: "text", text: "No matching memories were found." }, + ], + }, + }, + ["memory_lookup_custom"], + ), + ).toBe(false); + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "memory_lookup_custom", + details: { results: [{ id: "ramen" }] }, + content: [ + { + type: "text", + text: '{"results":[{"content":"User usually orders ramen."}]}', + }, + ], + }, + }, + ["memory_lookup_custom"], + ), + ).toBe(true); + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "memory_lookup_custom", + details: { + persistedDetailsTruncated: true, + originalDetailKeys: ["results"], + }, + content: [ + { + type: "text", + text: '{"results":[{"content":"User usually orders ramen."}]}', + }, + ], + }, + }, + ["memory_lookup_custom"], + ), + ).toBe(true); + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "memory_lookup_custom", + details: { + persistedDetailsTruncated: true, + originalDetailKeys: ["results"], + }, + content: [{ type: "text", text: '{"results":[]}' }], + }, + }, + ["memory_lookup_custom"], + ), + ).toBe(false); + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "lcm_grep", + details: { totalMatches: 1, messageCount: 1, summaryCount: 0 }, + content: [{ type: "text", text: "User usually orders ramen." }], + }, + }, + ["lcm_grep"], + ), + ).toBe(true); + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "lcm_grep", + details: { totalMatches: 0, messageCount: 0, summaryCount: 0 }, + content: [{ type: "text", text: "No matches found." }], + }, + }, + ["lcm_grep"], + ), + ).toBe(false); + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "lcm_describe", + details: { id: "sum_123", type: "summary", summary: { tokenCount: 12 } }, + content: [{ type: "text", text: "User usually orders ramen." }], + }, + }, + ["lcm_describe"], + ), + ).toBe(true); + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "lcm_expand_query", + details: { + answer: "User usually orders ramen.", + expandedSummaryCount: 1, + citedIds: ["sum_123"], + }, + content: [{ type: "text", text: '{"answer":"User usually orders ramen."}' }], + }, + }, + ["lcm_expand_query"], + ), + ).toBe(true); + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "lcm_grep", + details: { + persistedDetailsTruncated: true, + originalDetailKeys: ["totalMatches", "messages", "summaries"], + }, + content: [ + { + type: "text", + text: "## LCM Grep Results\n**Pattern:** `ramen`\n**Total matches:** 2\n\n### Messages", + }, + ], + }, + }, + ["lcm_grep"], + ), + ).toBe(true); + expect( + testing.hasUsableMemoryResultInSessionRecord( + { + message: { + role: "toolResult", + toolName: "lcm_expand_query", + details: { + persistedDetailsTruncated: true, + originalDetailKeys: ["answer", "expandedSummaryCount"], + }, + content: [ + { + type: "text", + text: JSON.stringify({ + answer: "User usually orders ramen.", + expandedSummaryCount: 1, + citedIds: ["sum_123"], + }), + }, + ], + }, + }, + ["lcm_expand_query"], + ), + ).toBe(true); + }); + it("replaces stale structured active-memory lines on a later empty run", async () => { const sessionKey = "agent:main:stale-active-memory-lines"; hoisted.sessionStore[sessionKey] = { @@ -2354,9 +2737,10 @@ describe("active-memory plugin", () => { it("returns partial transcript text on timeout when the subagent has already written assistant output", async () => { testing.setMinimumTimeoutMsForTests(1); testing.setSetupGraceTimeoutMsForTests(0); + testing.setTimeoutPartialDataGraceMsForTests(200); api.pluginConfig = { agents: ["main"], - timeoutMs: 25, + timeoutMs: 1_000, maxSummaryChars: 40, persistTranscripts: true, logging: true, @@ -2528,7 +2912,7 @@ describe("active-memory plugin", () => { testing.setSetupGraceTimeoutMsForTests(0); api.pluginConfig = { agents: ["main"], - timeoutMs: 1, + timeoutMs: 100, logging: true, }; plugin.register(api as unknown as OpenClawPluginApi); @@ -2912,7 +3296,7 @@ describe("active-memory plugin", () => { expect(result).toBeUndefined(); await vi.waitFor(() => { - expect(hoisted.closeActiveMemorySearchManager).toHaveBeenCalledTimes(1); + expect(hoisted.closeActiveMemorySearchManager).toHaveBeenCalled(); }); expect(hoisted.closeActiveMemorySearchManager).toHaveBeenCalledWith({ cfg: configFile, @@ -2920,6 +3304,76 @@ describe("active-memory plugin", () => { }); }); + it("schedules timeout cleanup before slow status persistence", async () => { + vi.useFakeTimers(); + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 1, + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + runEmbeddedAgent.mockImplementationOnce(() => new Promise(() => {})); + hoisted.updateSessionStore.mockImplementationOnce( + async () => + await new Promise((resolve) => { + setTimeout(resolve, 5_000); + }), + ); + + const resultPromise = hooks.before_prompt_build( + { prompt: "what wings should i order? slow timeout persistence", messages: [] }, + { + agentId: "main", + trigger: "user", + sessionKey: "agent:main:slow-timeout-persistence", + messageProvider: "webchat", + }, + ); + await vi.advanceTimersByTimeAsync(1_501); + + await expect(resultPromise).resolves.toBeUndefined(); + expect(hoisted.closeActiveMemorySearchManager).toHaveBeenCalledTimes(1); + }); + + it("does not clean up memory managers when only successful status persistence stalls", async () => { + vi.useFakeTimers(); + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 25, + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + let markPersistenceStarted: (() => void) | undefined; + const persistenceStarted = new Promise((resolve) => { + markPersistenceStarted = resolve; + }); + hoisted.updateSessionStore.mockImplementationOnce(async () => { + markPersistenceStarted?.(); + await new Promise((resolve) => { + setTimeout(resolve, 5_000); + }); + }); + + const resultPromise = hooks.before_prompt_build( + { prompt: "what wings should i order? slow successful persistence", messages: [] }, + { + agentId: "main", + trigger: "user", + sessionKey: "agent:main:slow-success-persistence", + messageProvider: "webchat", + }, + ); + await persistenceStarted; + await vi.advanceTimersByTimeAsync(1_525); + + await expect(resultPromise).resolves.toBeUndefined(); + expect(hoisted.closeActiveMemorySearchManager).not.toHaveBeenCalled(); + }); + it("does not share cached recall results across session-id-only contexts", async () => { api.pluginConfig = { agents: ["main"], @@ -3011,10 +3465,11 @@ describe("active-memory plugin", () => { logging: true, }; plugin.register(api as unknown as OpenClawPluginApi); - runEmbeddedAgent.mockImplementationOnce(async () => { + runEmbeddedAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { await new Promise((resolve) => { setTimeout(resolve, CONFIGURED_TIMEOUT_MS + 5); }); + await writeUsableMemoryTranscript(params.sessionFile, "remember the ramen place"); return { payloads: [{ text: "remember the ramen place" }] }; }); @@ -3162,6 +3617,877 @@ describe("active-memory plugin", () => { expectLinesToContain(lines, "🔎 Active Memory Debug: backend=qmd searchMs=8 hits=0"); }); + it("uses a late verbose summary after a successful result and later unavailable trace", async () => { + const CONFIGURED_TIMEOUT_MS = 1_000; + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + testing.setTimeoutPartialDataGraceMsForTests(5); + api.pluginConfig = { + agents: ["main"], + timeoutMs: CONFIGURED_TIMEOUT_MS, + maxSummaryChars: 120, + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:terminal-unavailable-then-summary"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-terminal-unavailable-then-summary", + updatedAt: 0, + }; + const verboseSummary = + "This memory says the user usually orders tonkotsu ramen, keeps chili crisp nearby, and prefers short dinner suggestions without menu preamble."; + runEmbeddedAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + await writeTranscriptJsonl(params.sessionFile, [ + { + message: { + role: "toolResult", + toolName: "memory_search", + details: { + persistedDetailsTruncated: true, + originalDetailKeys: ["results", "debug"], + }, + content: [ + { + type: "text", + text: JSON.stringify( + { + results: [ + { path: "memory/food.md", text: "User usually orders tonkotsu ramen." }, + ], + debug: { backend: "qmd", hits: 1, searchMs: 8 }, + }, + null, + 2, + ), + }, + ], + }, + }, + { + message: { + role: "toolResult", + toolName: "memory_search", + details: { + disabled: true, + warning: "Memory search is unavailable due to an embedding/provider error.", + action: "Check the embedding provider configuration, then retry memory_search.", + error: "embedding request failed", + }, + }, + }, + ]); + await new Promise((resolve) => { + setTimeout(resolve, 550); + }); + const activeSessionFile = path.join(path.dirname(params.sessionFile), "rotated.jsonl"); + await writeTranscriptJsonl(activeSessionFile, [ + { + message: { + role: "toolResult", + toolName: "memory_search", + details: { + disabled: true, + error: "embedding request failed", + }, + }, + }, + ]); + return { + payloads: [{ text: verboseSummary }], + meta: { agentMeta: { sessionFile: activeSessionFile } }, + }; + }); + + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? unavailable then summary", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + + expect(requirePrependContext(result)).toContain( + "This memory says the user usually orders tonkotsu ramen", + ); + const infoLines = vi + .mocked(api.logger.info) + .mock.calls.map((call: unknown[]) => String(call[0])); + expectLinesToContain(infoLines, "done status=ok"); + expectLinesNotToContain(infoLines, "done status=unavailable"); + const lines = getActiveMemoryLines(sessionKey); + expect(lines).toHaveLength(2); + expectLinesToContain(lines, "Active Memory: status=ok"); + }); + + it("does not recover transcript partials after a later unavailable search times out", async () => { + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + testing.setTimeoutPartialDataGraceMsForTests(100); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 250, + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:grounded-terminal-timeout-partial"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-grounded-terminal-timeout-partial", + updatedAt: 0, + }; + runEmbeddedAgent.mockImplementationOnce( + async (params: { sessionFile: string; abortSignal?: AbortSignal }) => { + await writeTranscriptJsonl(params.sessionFile, [ + { + message: { + role: "assistant", + content: "I will inspect memory before answering.", + }, + }, + { + message: { + role: "toolResult", + toolName: "memory_search", + details: { + results: [{ path: "memory/food.md", text: "User usually orders ramen." }], + }, + }, + }, + { + message: { + role: "toolResult", + toolName: "memory_search", + details: { + disabled: true, + warning: "Memory search is disabled for this session.", + }, + }, + }, + ]); + return await waitForAbort(params.abortSignal); + }, + ); + + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? grounded timeout", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + + expect(result).toBeUndefined(); + const lines = getActiveMemoryLines(sessionKey); + expectLinesToContain(lines, "Active Memory: status=timeout"); + expectLinesNotToContain(lines, "timeout_partial"); + }); + + it("does not recover a timeout partial when unavailable debug arrives after the last poll", async () => { + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + testing.setTimeoutPartialDataGraceMsForTests(100); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 250, + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:late-unavailable-timeout"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-late-unavailable-timeout", + updatedAt: 0, + }; + runEmbeddedAgent.mockImplementationOnce( + async (params: { sessionFile: string; abortSignal?: AbortSignal }) => { + await new Promise((resolve) => { + if (params.abortSignal?.aborted) { + resolve(); + return; + } + params.abortSignal?.addEventListener("abort", () => resolve(), { once: true }); + }); + await writeTranscriptJsonl(params.sessionFile, [ + { + message: { + role: "toolResult", + toolName: "memory_search", + details: { + disabled: true, + warning: "Memory search is disabled for this session.", + }, + }, + }, + { + message: { + role: "assistant", + content: "This text must not become recalled context.", + }, + }, + ]); + return { payloads: [] }; + }, + ); + + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? late unavailable", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + + expect(result).toBeUndefined(); + const lines = getActiveMemoryLines(sessionKey); + expectLinesToContain(lines, "Active Memory: status=timeout"); + expectLinesNotToContain(lines, "timeout_partial"); + }); + + it("does not recover a timeout partial while abort cleanup is still settling", async () => { + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + testing.setTimeoutPartialDataGraceMsForTests(50); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 250, + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:unsettled-timeout"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-unsettled-timeout", + updatedAt: 0, + }; + let resolveLateWrite: () => void = () => {}; + const lateWriteDone = new Promise((resolve) => { + resolveLateWrite = resolve; + }); + runEmbeddedAgent.mockImplementationOnce( + async (params: { sessionFile: string; abortSignal?: AbortSignal }) => { + await new Promise((resolve) => { + if (params.abortSignal?.aborted) { + resolve(); + return; + } + params.abortSignal?.addEventListener("abort", () => resolve(), { once: true }); + }); + await writeTranscriptJsonl(params.sessionFile, [ + { + message: { + role: "assistant", + content: "This unsettled text must not become recalled context.", + }, + }, + ]); + await new Promise((resolve) => { + setTimeout(resolve, 100); + }); + await writeTranscriptJsonl(params.sessionFile, [ + { + message: { + role: "assistant", + content: "This unsettled text must not become recalled context.", + }, + }, + { + message: { + role: "toolResult", + toolName: "memory_search", + details: { disabled: true }, + }, + }, + ]); + resolveLateWrite(); + return { payloads: [] }; + }, + ); + + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? unsettled timeout", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + + expect(result).toBeUndefined(); + const lines = getActiveMemoryLines(sessionKey); + expectLinesToContain(lines, "Active Memory: status=timeout"); + expectLinesNotToContain(lines, "timeout_partial"); + await lateWriteDone; + }); + + it("does not recover a timeout partial after an unmirrored custom memory tool fails", async () => { + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + testing.setTimeoutPartialDataGraceMsForTests(100); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 250, + toolsAllow: ["memory_lookup_custom"], + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:custom-tool-timeout-failure"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-custom-tool-timeout-failure", + updatedAt: 0, + }; + runEmbeddedAgent.mockImplementationOnce( + async (params: { + sessionFile: string; + abortSignal?: AbortSignal; + onAgentToolResult?: (event: { + toolName: string; + result: unknown; + isError: boolean; + }) => void; + }) => { + params.onAgentToolResult?.({ + toolName: "memory_lookup_custom", + isError: true, + result: { + content: [{ type: "text", text: "upstream unavailable" }], + details: { status: "failed", error: "upstream unavailable" }, + }, + }); + await new Promise((resolve) => { + if (params.abortSignal?.aborted) { + resolve(); + return; + } + params.abortSignal?.addEventListener("abort", () => resolve(), { once: true }); + }); + await writeTranscriptJsonl(params.sessionFile, [ + { + message: { + role: "assistant", + content: "This custom-tool failure must not become recalled context.", + }, + }, + ]); + return { payloads: [] }; + }, + ); + + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? custom timeout failure", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + + expect(result).toBeUndefined(); + const lines = getActiveMemoryLines(sessionKey); + expectLinesToContain(lines, "Active Memory: status=timeout"); + expectLinesNotToContain(lines, "timeout_partial"); + }); + + it("waits for configured custom-tool evidence after memory_search fails", async () => { + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 1_000, + toolsAllow: ["memory_lookup_custom", "memory_search"], + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:custom-tool-evidence"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-custom-tool-evidence", + updatedAt: 0, + }; + runEmbeddedAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + await writeTranscriptJsonl(params.sessionFile, [ + { + message: { + role: "toolResult", + toolName: "memory_search", + details: { disabled: true, error: "embedding request failed" }, + }, + }, + ]); + await new Promise((resolve) => { + setTimeout(resolve, 75); + }); + await writeTranscriptJsonl(params.sessionFile, [ + { + message: { + role: "toolResult", + toolName: "memory_search", + details: { disabled: true, error: "embedding request failed" }, + }, + }, + { + message: { + role: "toolResult", + toolName: "memory_lookup_custom", + details: { + persistedDetailsTruncated: true, + success: true, + originalDetailKeys: ["success", "results"], + }, + content: [ + { + type: "text", + text: "User usually orders ramen.", + }, + ], + }, + }, + ]); + return { payloads: [{ text: "User usually orders ramen." }] }; + }); + + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? custom evidence", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + + expectPrependContextContains(result, "User usually orders ramen."); + expectLinesToContain(getActiveMemoryLines(sessionKey), "Active Memory: status=ok"); + }); + + it("matches configured memory tool names case-insensitively", async () => { + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 1_000, + toolsAllow: [" MEMORY_SEARCH "], + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:case-insensitive-tool-evidence"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-case-insensitive-tool-evidence", + updatedAt: 0, + }; + runEmbeddedAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + await writeUsableMemoryTranscript(params.sessionFile, "User usually orders ramen."); + return { payloads: [{ text: "User usually orders ramen." }] }; + }); + + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? case insensitive", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + + expect(lastEmbeddedRunParams().toolsAllow).toEqual(["memory_search"]); + expectPrependContextContains(result, "User usually orders ramen."); + }); + + it("allows a configured custom tool to succeed after a failed attempt", async () => { + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 1_000, + toolsAllow: ["memory_lookup_custom"], + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:custom-tool-retry"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-custom-tool-retry", + updatedAt: 0, + }; + const failedResult = { + message: { + role: "toolResult", + toolName: "memory_lookup_custom", + details: { status: "failed", error: "query was too broad" }, + }, + }; + runEmbeddedAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + await writeTranscriptJsonl(params.sessionFile, [failedResult]); + await new Promise((resolve) => { + setTimeout(resolve, 75); + }); + await writeTranscriptJsonl(params.sessionFile, [ + failedResult, + { + message: { + role: "toolResult", + toolName: "memory_lookup_custom", + details: { status: "success", results: [{ text: "User usually orders ramen." }] }, + content: [{ type: "text", text: "User usually orders ramen." }], + }, + }, + ]); + return { payloads: [{ text: "User usually orders ramen." }] }; + }); + + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? custom retry", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + + expectPrependContextContains(result, "User usually orders ramen."); + expectLinesToContain(getActiveMemoryLines(sessionKey), "Active Memory: status=ok"); + }); + + it("uses harness-native tool results when the runtime does not mirror them to the transcript", async () => { + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 1_000, + toolsAllow: ["memory_search"], + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:harness-tool-evidence"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-harness-tool-evidence", + updatedAt: 0, + }; + runEmbeddedAgent.mockImplementationOnce( + async (params: { + onAgentToolResult?: (event: { + toolName: string; + result: unknown; + isError: boolean; + }) => void; + }) => { + params.onAgentToolResult?.({ + toolName: "memory_search", + isError: false, + result: { + content: [ + { + type: "text", + text: '{"results":[{"text":"User usually orders ramen."}]}', + }, + ], + details: { results: [{ text: "User usually orders ramen." }] }, + }, + }); + return { payloads: [{ text: "User usually orders ramen." }] }; + }, + ); + + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? harness evidence", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + + expectPrependContextContains(result, "User usually orders ramen."); + expectLinesToContain(getActiveMemoryLines(sessionKey), "Active Memory: status=ok"); + }); + + it("rejects completed output after a configured custom tool reports a content-only timeout", async () => { + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 1_000, + toolsAllow: ["memory_lookup_custom"], + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:custom-tool-content-failure"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-custom-tool-content-failure", + updatedAt: 0, + }; + runEmbeddedAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + await writeTranscriptJsonl(params.sessionFile, [ + { + message: { + role: "toolResult", + toolName: "memory_lookup_custom", + details: { success: true }, + content: [ + { + type: "text", + text: '{"status":"timed_out"}', + }, + { + type: "text", + text: "The custom backend returned a diagnostic.", + }, + ], + }, + }, + ]); + return { payloads: [{ text: "This ungrounded summary must not become recalled context." }] }; + }); + + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? custom content failure", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + + expect(result).toBeUndefined(); + expectLinesToContain(getActiveMemoryLines(sessionKey), "Active Memory: status=unavailable"); + }); + + it("fails open at the live deadline when pre-recall session state stalls", async () => { + vi.useFakeTimers(); + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 25, + logging: true, + }; + let resolveLookup: ((value: undefined) => void) | undefined; + vi.spyOn(api.runtime.state, "openKeyedStore").mockReturnValue({ + lookup: () => + new Promise((resolve) => { + resolveLookup = resolve; + }), + }); + plugin.register(api as unknown as OpenClawPluginApi); + + const resultPromise = hooks.before_prompt_build( + { prompt: "what food do i usually order? stalled toggle lookup", messages: [] }, + { + agentId: "main", + trigger: "user", + sessionKey: "agent:main:stalled-toggle", + messageProvider: "webchat", + }, + ); + await vi.advanceTimersByTimeAsync(1_525); + + await expect(resultPromise).resolves.toBeUndefined(); + expect(runEmbeddedAgent).not.toHaveBeenCalled(); + const warnLines = vi + .mocked(api.logger.warn) + .mock.calls.map((call: unknown[]) => String(call[0])); + expectLinesToContain(warnLines, "before_prompt_build preflight timed out after 1500ms"); + resolveLookup?.(undefined); + await vi.advanceTimersByTimeAsync(0); + expect(runEmbeddedAgent).not.toHaveBeenCalled(); + }); + + it("preserves recall settlement time after near-limit preflight latency", async () => { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + testing.setTimeoutPartialDataGraceMsForTests(100); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 25, + logging: true, + }; + vi.spyOn(api.runtime.state, "openKeyedStore").mockReturnValue({ + lookup: async () => + await new Promise((resolve) => { + setTimeout(() => resolve(undefined), 1_490); + }), + }); + plugin.register(api as unknown as OpenClawPluginApi); + runEmbeddedAgent.mockImplementationOnce(() => new Promise(() => {})); + + const resultPromise = hooks.before_prompt_build( + { prompt: "what food do i usually order? delayed recall start", messages: [] }, + { + agentId: "main", + trigger: "user", + sessionKey: "agent:main:delayed-recall-start", + messageProvider: "webchat", + }, + ); + await vi.advanceTimersByTimeAsync(1_490); + const hasStartedRecall = () => + vi + .mocked(api.logger.info) + .mock.calls.some((call: unknown[]) => + String(call[0]).includes("session=agent:main:delayed-recall-start"), + ); + for (let attempt = 0; attempt < 20 && !hasStartedRecall(); attempt += 1) { + await new Promise((resolve) => { + setImmediate(resolve); + }); + } + expect(hasStartedRecall()).toBe(true); + await vi.advanceTimersByTimeAsync(25); + await vi.advanceTimersByTimeAsync(1_499); + + await expect(resultPromise).resolves.toBeUndefined(); + expect(hoisted.closeActiveMemorySearchManager).toHaveBeenCalled(); + const circuitBreakerKey = testing.buildCircuitBreakerKey( + "main", + "github-copilot", + "gpt-5.4-mini", + ); + expect(testing.isCircuitBreakerOpen(circuitBreakerKey, 1, 60_000)).toBe(true); + }); + + it("rejects completed output after a memory search returns no recall evidence", async () => { + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 1_000, + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:empty-search-completed-output"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-empty-search-completed-output", + updatedAt: 0, + }; + runEmbeddedAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + await writeTranscriptJsonl(params.sessionFile, [ + { + message: { + role: "toolResult", + toolName: "memory_search", + details: { results: [] }, + content: [{ type: "text", text: '{"results":[]}' }], + }, + }, + ]); + return { payloads: [{ text: "This ungrounded summary must not become recalled context." }] }; + }); + + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? empty search", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + + expect(result).toBeUndefined(); + expectLinesToContain(getActiveMemoryLines(sessionKey), "status=no_relevant_memory"); + }); + + it("does not recover arbitrary assistant text without successful memory evidence", async () => { + const CONFIGURED_TIMEOUT_MS = 1_000; + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + testing.setTimeoutPartialDataGraceMsForTests(200); + api.pluginConfig = { + agents: ["main"], + timeoutMs: CONFIGURED_TIMEOUT_MS, + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:terminal-unavailable-then-diagnostic"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-terminal-unavailable-then-diagnostic", + updatedAt: 0, + }; + const warning = "Memory search is unavailable due to an embedding/provider error."; + const action = "Check the embedding provider configuration, then retry memory_search."; + runEmbeddedAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + await writeTranscriptJsonl(params.sessionFile, [ + { + message: { + role: "toolResult", + toolName: "memory_search", + details: { + disabled: true, + warning, + action, + error: "embedding request failed", + }, + }, + }, + ]); + return { payloads: [{ text: "User usually orders tonkotsu ramen." }] }; + }); + + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? unavailable diagnostic", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + + expect(result).toBeUndefined(); + const infoLines = vi + .mocked(api.logger.info) + .mock.calls.map((call: unknown[]) => String(call[0])); + expectLinesToContain(infoLines, "done status=unavailable"); + expectLinesNotToContain(infoLines, "done status=ok"); + const lines = getActiveMemoryLines(sessionKey); + expect(lines).toHaveLength(2); + expectLinesToContain(lines, "Active Memory: status=unavailable"); + expectLinesToContain(lines, `Active Memory Debug: ${warning} ${action}`); + }); + + it("uses configured memory evidence from a rotated embedded transcript", async () => { + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 1_000, + toolsAllow: ["memory_get", "memory_search"], + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:rotated-memory-evidence"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-rotated-memory-evidence", + updatedAt: 0, + }; + runEmbeddedAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + await writeTranscriptJsonl(params.sessionFile, [ + { + message: { + role: "toolResult", + toolName: "memory_get", + details: { path: "memory/food.md", text: "User usually orders ramen." }, + }, + }, + ]); + const activeSessionFile = path.join(path.dirname(params.sessionFile), "rotated.jsonl"); + await writeTranscriptJsonl(activeSessionFile, [ + { + message: { + role: "toolResult", + toolName: "memory_search", + details: { + disabled: true, + error: "embedding request failed", + }, + }, + }, + ]); + return { + payloads: [{ text: "User usually orders ramen." }], + meta: { agentMeta: { sessionFile: activeSessionFile } }, + }; + }); + + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? rotated transcript", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + + expectPrependContextContains(result, "User usually orders ramen."); + expectLinesToContain(getActiveMemoryLines(sessionKey), "status=ok"); + }); + + it("rejects completed output when only a rotated transcript reports unavailable memory", async () => { + testing.setMinimumTimeoutMsForTests(1); + testing.setSetupGraceTimeoutMsForTests(0); + api.pluginConfig = { + agents: ["main"], + timeoutMs: 1_000, + logging: true, + }; + plugin.register(api as unknown as OpenClawPluginApi); + const sessionKey = "agent:main:rotated-memory-unavailable"; + hoisted.sessionStore[sessionKey] = { + sessionId: "s-rotated-memory-unavailable", + updatedAt: 0, + }; + runEmbeddedAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + const activeSessionFile = path.join(path.dirname(params.sessionFile), "rotated.jsonl"); + await writeTranscriptJsonl(activeSessionFile, [ + { + message: { + role: "toolResult", + toolName: "memory_search", + details: { + disabled: true, + warning: "Memory search is disabled for this session.", + }, + }, + }, + ]); + return { + payloads: [{ text: "This arbitrary output must not become recalled context." }], + meta: { agentMeta: { sessionFile: activeSessionFile } }, + }; + }); + + const result = await hooks.before_prompt_build( + { prompt: "what food do i usually order? rotated unavailable", messages: [] }, + { agentId: "main", trigger: "user", sessionKey, messageProvider: "webchat" }, + ); + + expect(result).toBeUndefined(); + expectLinesToContain(getActiveMemoryLines(sessionKey), "status=unavailable"); + }); + it("fast-fails configured-provider-missing memory_search results without injecting provider errors", async () => { const CONFIGURED_TIMEOUT_MS = 1_000; testing.setMinimumTimeoutMsForTests(1); @@ -3215,7 +4541,7 @@ describe("active-memory plugin", () => { ); }); - it("does not treat memory_get misses as terminal recall results", async () => { + it("does not fast-fail memory_get misses but rejects ungrounded completed output", async () => { testing.setMinimumTimeoutMsForTests(1); testing.setSetupGraceTimeoutMsForTests(0); api.pluginConfig = { @@ -3253,7 +4579,8 @@ describe("active-memory plugin", () => { }, ); - expect(result?.prependContext).toContain("User usually orders ramen after late flights."); + expect(result).toBeUndefined(); + expectLinesToContain(getActiveMemoryLines("agent:main:memory-get-miss"), "status=unavailable"); }); it("returns undefined instead of throwing when an unexpected error escapes prompt building", async () => { @@ -3878,8 +5205,11 @@ describe("active-memory plugin", () => { }); it("trusts the subagent's relevance decision for explicit preference recall prompts", async () => { - runEmbeddedAgent.mockResolvedValueOnce({ - payloads: [{ text: "User prefers aisle seats and extra buffer on connections." }], + runEmbeddedAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + await writeUsableMemoryTranscript(params.sessionFile, "aisle seats and connection buffer"); + return { + payloads: [{ text: "User prefers aisle seats and extra buffer on connections." }], + }; }); const result = await hooks.before_prompt_build( @@ -3903,12 +5233,15 @@ describe("active-memory plugin", () => { maxSummaryChars: 40, }; plugin.register(api as unknown as OpenClawPluginApi); - runEmbeddedAgent.mockResolvedValueOnce({ - payloads: [ - { - text: "alpha beta gamma delta epsilon zetalongword", - }, - ], + runEmbeddedAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { + await writeUsableMemoryTranscript(params.sessionFile, "alpha beta gamma"); + return { + payloads: [ + { + text: "alpha beta gamma delta epsilon zetalongword", + }, + ], + }; }); const result = await hooks.before_prompt_build( @@ -3981,7 +5314,7 @@ describe("active-memory plugin", () => { logging: true, }; plugin.register(api as unknown as OpenClawPluginApi); - const mkdirSpy = vi.spyOn(fs, "mkdir").mockResolvedValue(undefined); + const mkdirSpy = vi.spyOn(fs, "mkdir"); const mkdtempSpy = vi.spyOn(fs, "mkdtemp"); const rmSpy = vi.spyOn(fs, "rm").mockResolvedValue(undefined); diff --git a/extensions/active-memory/index.ts b/extensions/active-memory/index.ts index efe93db33f47..77d68b04974d 100644 --- a/extensions/active-memory/index.ts +++ b/extensions/active-memory/index.ts @@ -31,6 +31,7 @@ import { parseAgentSessionKey, parseThreadSessionSuffix } from "openclaw/plugin- import { isPathInside } from "openclaw/plugin-sdk/security-runtime"; import { asOptionalRecord as asRecord, + normalizeLowercaseStringOrEmpty, normalizeOptionalString, normalizeStringEntries, uniqueStrings, @@ -49,6 +50,8 @@ const DEFAULT_MAX_CACHE_ENTRIES = 1000; const CACHE_SWEEP_INTERVAL_MS = 1000; const DEFAULT_MIN_TIMEOUT_MS = 250; const DEFAULT_SETUP_GRACE_TIMEOUT_MS = 0; +const MAX_TIMEOUT_MS = 120_000; +const MAX_SETUP_GRACE_TIMEOUT_MS = 30_000; const DEFAULT_QUERY_MODE = "recent" as const; const DEFAULT_QMD_SEARCH_MODE = "search" as const; const DEFAULT_TRANSCRIPT_DIR = "active-memory"; @@ -58,6 +61,29 @@ const DEFAULT_CIRCUIT_BREAKER_COOLDOWN_MS = 60_000; const DEFAULT_ACTIVE_MEMORY_TOOLS_ALLOW = ["memory_search", "memory_get"] as const; const LANCEDB_ACTIVE_MEMORY_TOOLS_ALLOW = ["memory_recall"] as const; const MAX_ACTIVE_MEMORY_TOOLS_ALLOW = 32; +const STRUCTURED_MEMORY_FAILURE_STATUSES = new Set([ + "error", + "failed", + "failure", + "timeout", + "timed_out", + "denied", + "cancelled", + "canceled", + "aborted", + "killed", + "invalid", + "forbidden", + "unavailable", + "disabled", + "blocked", +]); +const STRUCTURED_MEMORY_EMPTY_STATUSES = new Set([ + "not_found", + "empty", + "no_results", + "no_matches", +]); const ACTIVE_MEMORY_RESERVED_TOOLS_ALLOW = new Set([ "*", "agents_list", @@ -96,6 +122,7 @@ const DEFAULT_PARTIAL_TRANSCRIPT_MAX_CHARS = 32_000; const DEFAULT_TRANSCRIPT_READ_MAX_LINES = 2_000; const DEFAULT_TRANSCRIPT_READ_MAX_BYTES = 50 * 1024 * 1024; const TIMEOUT_PARTIAL_DATA_GRACE_MS = 500; +const HOOK_TIMEOUT_RECOVERY_GRACE_MS = TIMEOUT_PARTIAL_DATA_GRACE_MS + 1_000; const MAX_ACTIVE_MEMORY_SEARCH_QUERY_CHARS = 480; const TERMINAL_MEMORY_SEARCH_POLL_INTERVAL_MS = 25; @@ -258,6 +285,7 @@ type ActiveRecallResult = type ActiveMemoryPartialTimeoutError = Error & { activeMemoryPartialReply?: string; activeMemorySearchDebug?: ActiveMemorySearchDebug; + activeMemoryUnavailableMemorySearch?: boolean; }; type TranscriptReadLimits = { @@ -271,10 +299,13 @@ type RecallSubagentResult = { resultStatus?: "failed" | "unavailable"; transcriptPath?: string; searchDebug?: ActiveMemorySearchDebug; + hasUsableMemoryResult?: boolean; + hasUnavailableMemorySearchResult?: boolean; }; type TerminalMemorySearchResult = { status: "unavailable"; + hasUsableMemoryResult: boolean; searchDebug?: ActiveMemorySearchDebug; }; @@ -427,12 +458,12 @@ function normalizeConfiguredToolsAllow(value: unknown): string[] | undefined { if (typeof entry !== "string") { continue; } - const trimmed = entry.trim(); - if (!trimmed || isReservedActiveMemoryToolsAllowEntry(trimmed) || seen.has(trimmed)) { + const normalized = normalizeLowercaseStringOrEmpty(entry); + if (!normalized || isReservedActiveMemoryToolsAllowEntry(normalized) || seen.has(normalized)) { continue; } - seen.add(trimmed); - out.push(trimmed); + seen.add(normalized); + out.push(normalized); if (out.length >= MAX_ACTIVE_MEMORY_TOOLS_ALLOW) { break; } @@ -839,9 +870,14 @@ function normalizePluginConfig( parseOptionalPositiveInt(raw.timeoutMs, DEFAULT_TIMEOUT_MS), DEFAULT_TIMEOUT_MS, minimumTimeoutMs, - 120_000, + MAX_TIMEOUT_MS, + ), + setupGraceTimeoutMs: clampInt( + raw.setupGraceTimeoutMs, + setupGraceTimeoutMs, + 0, + MAX_SETUP_GRACE_TIMEOUT_MS, ), - setupGraceTimeoutMs: clampInt(raw.setupGraceTimeoutMs, setupGraceTimeoutMs, 0, 30_000), queryMode: raw.queryMode === "message" || raw.queryMode === "recent" || raw.queryMode === "full" ? raw.queryMode @@ -1662,10 +1698,11 @@ function extractActiveMemorySearchDebugFromSessionRecord( ): ActiveMemorySearchDebug | undefined { const record = asRecord(value); const nestedMessage = asRecord(record?.message); + const recordToolName = normalizeLowercaseStringOrEmpty(record?.toolName); const topLevelMessage = record?.role === "toolResult" || - record?.toolName === "memory_search" || - record?.toolName === "memory_recall" + recordToolName === "memory_search" || + recordToolName === "memory_recall" ? record : undefined; const message = nestedMessage ?? topLevelMessage; @@ -1673,7 +1710,7 @@ function extractActiveMemorySearchDebugFromSessionRecord( return undefined; } const role = normalizeOptionalString(message.role); - const toolName = normalizeOptionalString(message.toolName); + const toolName = normalizeLowercaseStringOrEmpty(message.toolName); if (role !== "toolResult" || (toolName !== "memory_search" && toolName !== "memory_recall")) { return undefined; } @@ -1701,77 +1738,304 @@ function extractActiveMemorySearchDebugFromSessionRecord( }; } -function extractTerminalMemorySearchResultFromSessionRecord( - value: unknown, -): TerminalMemorySearchResult | undefined { +function extractToolResultNameFromSessionRecord(value: unknown): string | undefined { const record = asRecord(value); const nestedMessage = asRecord(record?.message); - const topLevelMessage = - record?.role === "toolResult" || - record?.toolName === "memory_search" || - record?.toolName === "memory_recall" - ? record - : undefined; + const topLevelMessage = record?.role === "toolResult" ? record : undefined; const message = nestedMessage ?? topLevelMessage; if (!message) { return undefined; } const role = normalizeOptionalString(message.role); - const toolName = normalizeOptionalString(message.toolName); - if (role !== "toolResult" || (toolName !== "memory_search" && toolName !== "memory_recall")) { - return undefined; - } - const details = asRecord(message.details); - const debug = extractActiveMemorySearchDebugFromSessionRecord(value); - const disabled = details?.disabled === true; - const unavailable = disabled || Boolean(debug?.error) || Boolean(details?.error); - if (unavailable) { - return { status: "unavailable", searchDebug: debug }; - } - return undefined; + const toolName = normalizeLowercaseStringOrEmpty(message.toolName); + return role === "toolResult" && toolName ? toolName : undefined; } -async function readActiveMemorySearchDebug( +function hasUnavailableMemoryResultInSessionRecord( + value: unknown, + toolsAllow: readonly string[] = [ + ...DEFAULT_ACTIVE_MEMORY_TOOLS_ALLOW, + ...LANCEDB_ACTIVE_MEMORY_TOOLS_ALLOW, + ], +): boolean { + const record = asRecord(value); + const nestedMessage = asRecord(record?.message); + const topLevelMessage = record?.role === "toolResult" ? record : undefined; + const message = nestedMessage ?? topLevelMessage; + if (!message || normalizeOptionalString(message.role) !== "toolResult") { + return false; + } + const toolName = normalizeLowercaseStringOrEmpty(message.toolName); + if (!toolName || !toolsAllow.includes(toolName)) { + return false; + } + const details = asRecord(message.details); + const unavailable = message.isError === true || readStructuredMemoryFailure(details) === true; + if (unavailable) { + return true; + } + return readStructuredMemoryFailureFromContent(message.content) === true; +} + +function hasTerminalUnavailableMemoryResultInSessionRecord( + value: unknown, + toolsAllow: readonly string[], +): boolean { + const record = asRecord(value); + const nestedMessage = asRecord(record?.message); + const topLevelMessage = record?.role === "toolResult" ? record : undefined; + const message = nestedMessage ?? topLevelMessage; + if (!message || normalizeOptionalString(message.role) !== "toolResult") { + return false; + } + const toolName = normalizeLowercaseStringOrEmpty(message.toolName); + if (!toolName || !toolsAllow.includes(toolName)) { + return false; + } + const details = asRecord(message.details); + if (details?.disabled === true || details?.unavailable === true) { + return true; + } + const status = normalizeOptionalString(details?.status) + ?.toLowerCase() + .replace(/[\s-]+/g, "_"); + if (status === "disabled" || status === "unavailable") { + return true; + } + if (toolName !== "memory_search" && toolName !== "memory_recall") { + return false; + } + const debug = extractActiveMemorySearchDebugFromSessionRecord(value); + return Boolean(debug?.error) || Boolean(details?.error); +} + +function createActiveMemoryHookDeadline() { + const timeoutSentinel = Symbol("active-memory-hook-timeout"); + let timeoutId: ReturnType | undefined; + let resolveTimeout: (value: typeof timeoutSentinel) => void = () => {}; + const promise = new Promise((resolve) => { + resolveTimeout = resolve; + }); + const stop = () => { + if (timeoutId) { + clearTimeout(timeoutId); + timeoutId = undefined; + } + }; + const arm = (timeoutMs: number, onTimeout: () => void) => { + stop(); + timeoutId = setTimeout(() => { + onTimeout(); + resolveTimeout(timeoutSentinel); + }, timeoutMs); + timeoutId.unref?.(); + }; + return { arm, promise, stop }; +} + +function hasUsableMemoryResultInSessionRecord( + value: unknown, + toolsAllow: readonly string[] = [ + ...DEFAULT_ACTIVE_MEMORY_TOOLS_ALLOW, + ...LANCEDB_ACTIVE_MEMORY_TOOLS_ALLOW, + ], +): boolean { + const record = asRecord(value); + const nestedMessage = asRecord(record?.message); + const recordToolName = normalizeLowercaseStringOrEmpty(record?.toolName); + const topLevelMessage = + record?.role === "toolResult" || + recordToolName === "memory_search" || + recordToolName === "memory_recall" + ? record + : undefined; + const message = nestedMessage ?? topLevelMessage; + if (!message || normalizeOptionalString(message.role) !== "toolResult") { + return false; + } + const toolName = normalizeLowercaseStringOrEmpty(message.toolName); + if (!toolName || !toolsAllow.includes(toolName)) { + return false; + } + if (hasUnavailableMemoryResultInSessionRecord(value, toolsAllow)) { + return false; + } + const details = asRecord(message.details); + const content = extractTextContent(message.content); + if (toolName === "memory_search") { + if (Array.isArray(details?.results)) { + return details.results.length > 0; + } + // Oversized details are capped before transcript persistence, while the + // leading model-visible JSON still preserves whether results were present. + return /"results"\s*:\s*\[\s*([^\s\]])/.test(content); + } + if (toolName === "memory_recall") { + if (Array.isArray(details?.memories)) { + return details.memories.length > 0; + } + return /^Found [1-9]\d* memories:/.test(content); + } + if (toolName === "memory_get") { + const text = normalizeOptionalString(details?.text); + return text !== undefined ? text.length > 0 : /"text"\s*:\s*"(?!")/.test(content); + } + if (toolName === "lcm_grep") { + if ( + typeof details?.totalMatches === "number" && + Number.isFinite(details.totalMatches) && + details.totalMatches > 0 + ) { + return true; + } + return /^## LCM Grep Results[\s\S]*^\*\*Total matches:\*\*\s+[1-9]\d*$/m.test(content); + } + if (toolName === "lcm_describe") { + const type = normalizeOptionalString(details?.type); + if (normalizeOptionalString(details?.id) && (type === "summary" || type === "file")) { + return true; + } + return /^LCM_SUMMARY \S+/m.test(content) || /^## LCM File: \S+/m.test(content); + } + if (toolName === "lcm_expand_query") { + if ( + typeof details?.expandedSummaryCount === "number" && + Number.isFinite(details.expandedSummaryCount) && + details.expandedSummaryCount > 0 && + Boolean(normalizeOptionalString(details?.answer)) + ) { + return true; + } + try { + const parsed = asRecord(JSON.parse(content)); + return ( + typeof parsed?.expandedSummaryCount === "number" && + Number.isFinite(parsed.expandedSummaryCount) && + parsed.expandedSummaryCount > 0 && + Boolean(normalizeOptionalString(parsed?.answer)) + ); + } catch { + return false; + } + } + const normalizedContent = normalizeOptionalString(content); + const explicitEvidence = details ? readExplicitMemoryEvidence(details) : undefined; + const structuredEvidence = normalizedContent + ? readStructuredMemoryEvidenceFromContent(message.content) + : undefined; + // Custom recall tools have a shipped native-output contract. Preserve + // non-empty model-visible results unless structured fields explicitly say + // the lookup was empty; explicit failures are rejected above. + return Boolean(normalizedContent) && explicitEvidence !== false && structuredEvidence !== false; +} + +async function readActiveMemoryTranscriptState( sessionFile: string, limits?: TranscriptReadLimits, -): Promise { - let found: ActiveMemorySearchDebug | undefined; + toolsAllow?: readonly string[], +): Promise<{ + searchDebug?: ActiveMemorySearchDebug; + hasUsableMemoryResult: boolean; + hasUnavailableMemorySearchResult: boolean; +}> { + let searchDebug: ActiveMemorySearchDebug | undefined; + let hasUsableMemoryResult = false; + let hasUnavailableMemorySearchResult = false; await streamBoundedTranscriptJsonl({ sessionFile, limits, onRecord: (record) => { const debug = extractActiveMemorySearchDebugFromSessionRecord(record); if (debug) { - found = debug; + searchDebug = debug; } + hasUnavailableMemorySearchResult ||= hasUnavailableMemoryResultInSessionRecord( + record, + toolsAllow, + ); + hasUsableMemoryResult ||= hasUsableMemoryResultInSessionRecord(record, toolsAllow); }, }); - return found; + return { searchDebug, hasUsableMemoryResult, hasUnavailableMemorySearchResult }; +} + +async function readActiveMemorySearchDebug( + sessionFile: string, + limits?: TranscriptReadLimits, +): Promise { + return (await readActiveMemoryTranscriptState(sessionFile, limits)).searchDebug; +} + +async function readMergedActiveMemoryTranscriptState(params: { + sessionFiles: readonly string[]; + toolsAllow: readonly string[]; +}): Promise<{ + searchDebug?: ActiveMemorySearchDebug; + hasUsableMemoryResult: boolean; + hasUnavailableMemorySearchResult: boolean; +}> { + let searchDebug: ActiveMemorySearchDebug | undefined; + let hasUsableMemoryResult = false; + let hasUnavailableMemorySearchResult = false; + for (const sessionFile of new Set(params.sessionFiles)) { + const state = await readActiveMemoryTranscriptState(sessionFile, undefined, params.toolsAllow); + searchDebug = state.searchDebug ?? searchDebug; + hasUsableMemoryResult ||= state.hasUsableMemoryResult; + hasUnavailableMemorySearchResult ||= state.hasUnavailableMemorySearchResult; + } + return { searchDebug, hasUsableMemoryResult, hasUnavailableMemorySearchResult }; } async function readTerminalMemorySearchResult( sessionFile: string, limits?: TranscriptReadLimits, + toolsAllow?: readonly string[], ): Promise { - let found: TerminalMemorySearchResult | undefined; + // memory_get consumes a path discovered by another tool; it is not an + // independent fallback that should delay terminal unavailability. + const recallPathNames = new Set( + toolsAllow + ?.map((toolName) => normalizeLowercaseStringOrEmpty(toolName)) + .filter((toolName) => toolName && toolName !== "memory_get"), + ); + if (recallPathNames.size === 0) { + return undefined; + } + const unavailablePathNames = new Set(); + let hasUsableMemoryResult = false; + let searchDebug: ActiveMemorySearchDebug | undefined; await streamBoundedTranscriptJsonl({ sessionFile, limits, onRecord: (record) => { - const result = extractTerminalMemorySearchResultFromSessionRecord(record); - if (result) { - found = result; - return true; + hasUsableMemoryResult ||= hasUsableMemoryResultInSessionRecord(record, toolsAllow); + searchDebug = extractActiveMemorySearchDebugFromSessionRecord(record) ?? searchDebug; + const toolName = extractToolResultNameFromSessionRecord(record); + if (!toolName || !recallPathNames.has(toolName)) { + return false; + } + if (hasTerminalUnavailableMemoryResultInSessionRecord(record, toolsAllow ?? [])) { + unavailablePathNames.add(toolName); + } else { + unavailablePathNames.delete(toolName); } return false; }, }); - return found; + if (unavailablePathNames.size !== recallPathNames.size) { + return undefined; + } + return { + status: "unavailable", + hasUsableMemoryResult, + searchDebug, + }; } function watchTerminalMemorySearchResult(params: { getSessionFile: () => string | undefined; abortSignal: AbortSignal; + toolsAllow: readonly string[]; }): TerminalMemorySearchWatch { let stopped = false; let timeoutId: ReturnType | undefined; @@ -1812,7 +2076,9 @@ function watchTerminalMemorySearchResult(params: { inFlight = true; try { const sessionFile = params.getSessionFile(); - const result = sessionFile ? await readTerminalMemorySearchResult(sessionFile) : undefined; + const result = sessionFile + ? await readTerminalMemorySearchResult(sessionFile, undefined, params.toolsAllow) + : undefined; if (result) { finish(result); return; @@ -1883,6 +2149,51 @@ function readActiveMemorySearchDebugFromRunResult( ); } +function readActiveMemorySessionFileFromRunResult(result: unknown): string | undefined { + const record = asRecord(result); + const meta = asRecord(record?.meta); + const agentMeta = asRecord(meta?.agentMeta); + return ( + normalizeOptionalString(agentMeta?.sessionFile) ?? normalizeOptionalString(meta?.sessionFile) + ); +} + +function readMemoryToolResultEvidence(params: { + toolName: string; + result: unknown; + isError: boolean; + toolsAllow: readonly string[]; +}): { + hasUsableMemoryResult: boolean; + hasUnavailableMemorySearchResult: boolean; +} { + const result = asRecord(params.result); + const rawContent = result?.content; + const textContent = + normalizeOptionalString(result?.detailedContent) ?? + (typeof rawContent === "string" ? normalizeOptionalString(rawContent) : undefined); + const record = { + message: { + role: "toolResult", + toolName: params.toolName, + isError: params.isError, + content: Array.isArray(rawContent) + ? rawContent + : textContent + ? [{ type: "text", text: textContent }] + : [], + details: result?.details, + }, + }; + return { + hasUsableMemoryResult: hasUsableMemoryResultInSessionRecord(record, params.toolsAllow), + hasUnavailableMemorySearchResult: hasUnavailableMemoryResultInSessionRecord( + record, + params.toolsAllow, + ), + }; +} + function extractAssistantTextFromSessionRecord(value: unknown): string { const record = asRecord(value); if (!record) { @@ -1939,6 +2250,7 @@ function attachPartialTimeoutData( error: unknown, partialReply: string | null, searchDebug: ActiveMemorySearchDebug | undefined, + hasUnavailableMemorySearchResult: boolean, ): void { if (!error || typeof error !== "object") { return; @@ -1950,11 +2262,15 @@ function attachPartialTimeoutData( if (searchDebug) { target.activeMemorySearchDebug = searchDebug; } + if (hasUnavailableMemorySearchResult) { + target.activeMemoryUnavailableMemorySearch = true; + } } function readPartialTimeoutData(error: unknown): { rawReply?: string; searchDebug?: ActiveMemorySearchDebug; + hasUnavailableMemorySearchResult?: boolean; } { if (!error || typeof error !== "object") { return {}; @@ -1963,6 +2279,7 @@ function readPartialTimeoutData(error: unknown): { return { rawReply: normalizeOptionalString(source.activeMemoryPartialReply), searchDebug: source.activeMemorySearchDebug, + hasUnavailableMemorySearchResult: source.activeMemoryUnavailableMemorySearch, }; } @@ -1971,25 +2288,25 @@ async function waitForSubagentPartialTimeoutData( ): Promise<{ rawReply?: string; searchDebug?: ActiveMemorySearchDebug; + hasUnavailableMemorySearchResult?: boolean; + settled: boolean; }> { if (!subagentPromise) { - return {}; + return { settled: true }; } let timeoutId: ReturnType | undefined; - const timeoutPromise = new Promise((resolve) => { - timeoutId = setTimeout(() => resolve(undefined), timeoutPartialDataGraceMs); + const timeoutPromise = new Promise<{ settled: false }>((resolve) => { + timeoutId = setTimeout(() => resolve({ settled: false }), timeoutPartialDataGraceMs); timeoutId.unref?.(); }); try { - return ( - (await Promise.race([ - subagentPromise.then( - () => undefined, - (error: unknown) => readPartialTimeoutData(error), - ), - timeoutPromise, - ])) ?? {} - ); + return await Promise.race([ + subagentPromise.then( + () => ({ settled: true as const }), + (error: unknown) => ({ ...readPartialTimeoutData(error), settled: true as const }), + ), + timeoutPromise, + ]); } finally { if (timeoutId) { clearTimeout(timeoutId); @@ -2003,12 +2320,13 @@ async function buildTimeoutRecallResult(params: { sessionFile?: string; rawReply?: string; searchDebug?: ActiveMemorySearchDebug; + hasUnavailableMemorySearchResult?: boolean; subagentPromise?: Promise; + toolsAllow: readonly string[]; }): Promise { - const subagentPartialData = - params.rawReply || params.searchDebug - ? {} - : await waitForSubagentPartialTimeoutData(params.subagentPromise); + const subagentPartialData = params.rawReply + ? { settled: true as const } + : await waitForSubagentPartialTimeoutData(params.subagentPromise); const rawReply = params.rawReply ?? subagentPartialData.rawReply ?? @@ -2017,11 +2335,19 @@ async function buildTimeoutRecallResult(params: { normalizeActiveSummary(rawReply ?? "") ?? "", params.maxSummaryChars, ); + const transcriptState = params.sessionFile + ? await readActiveMemoryTranscriptState(params.sessionFile, undefined, params.toolsAllow) + : undefined; const searchDebug = - params.searchDebug ?? - subagentPartialData.searchDebug ?? - (params.sessionFile ? await readActiveMemorySearchDebug(params.sessionFile) : undefined); - if (summary.length === 0) { + params.searchDebug ?? subagentPartialData.searchDebug ?? transcriptState?.searchDebug; + if ( + summary.length === 0 || + isUnavailableMemorySearchDebug(searchDebug) || + !subagentPartialData.settled || + params.hasUnavailableMemorySearchResult || + subagentPartialData.hasUnavailableMemorySearchResult || + transcriptState?.hasUnavailableMemorySearchResult + ) { return { status: "timeout", elapsedMs: params.elapsedMs, @@ -2037,6 +2363,54 @@ async function buildTimeoutRecallResult(params: { }; } +function buildSubagentRecallResult(params: { + subagentResult: RecallSubagentResult; + fallbackSearchDebug?: ActiveMemorySearchDebug; + fallbackHasUsableMemoryResult?: boolean; + elapsedMs: number; + maxSummaryChars: number; +}): ActiveRecallResult { + const { rawReply, resultStatus } = params.subagentResult; + const searchDebug = params.subagentResult.searchDebug ?? params.fallbackSearchDebug; + const summary = truncateSummary(normalizeActiveSummary(rawReply) ?? "", params.maxSummaryChars); + const hasUsableMemoryResult = + params.subagentResult.hasUsableMemoryResult === true || + params.fallbackHasUsableMemoryResult === true; + const hasUnavailableMemorySearchResult = + params.subagentResult.hasUnavailableMemorySearchResult === true; + const canUseSummary = hasUsableMemoryResult; + return summary.length > 0 && canUseSummary + ? { + status: "ok", + elapsedMs: params.elapsedMs, + rawReply, + summary, + searchDebug, + } + : resultStatus === "failed" + ? { + status: "failed", + elapsedMs: params.elapsedMs, + summary: null, + searchDebug, + } + : resultStatus === "unavailable" || + isUnavailableMemorySearchDebug(searchDebug) || + hasUnavailableMemorySearchResult + ? { + status: "unavailable", + elapsedMs: params.elapsedMs, + summary: null, + searchDebug, + } + : { + status: "no_relevant_memory", + elapsedMs: params.elapsedMs, + summary: null, + searchDebug, + }; +} + function escapeXml(str: string): string { return str .replace(/&/g, "&") @@ -2050,6 +2424,100 @@ function normalizeNoRecallValue(value: string): boolean { return NO_RECALL_VALUES.has(value.trim().toLowerCase()); } +function readExplicitMemoryEvidence(source: Record): boolean | undefined { + const status = normalizeOptionalString(source.status) + ?.toLowerCase() + .replace(/[\s-]+/g, "_"); + if (status !== undefined && STRUCTURED_MEMORY_EMPTY_STATUSES.has(status)) { + return false; + } + const resultCollections = [source.results, source.memories, source.items]; + if (resultCollections.some((entry) => Array.isArray(entry))) { + return resultCollections.some((entry) => Array.isArray(entry) && entry.length > 0); + } + const resultCounts = [ + source.count, + source.matches, + source.memoryCount, + source.resultCount, + source.totalMatches, + ]; + if (resultCounts.some((entry) => typeof entry === "number" && Number.isFinite(entry))) { + return resultCounts.some( + (entry) => typeof entry === "number" && Number.isFinite(entry) && entry > 0, + ); + } + if (typeof source.found === "boolean" || typeof source.hasResults === "boolean") { + return source.found === true || source.hasResults === true; + } + return undefined; +} + +function readStructuredMemoryFailure(source: unknown): boolean | undefined { + const record = asRecord(source); + if (!record) { + return undefined; + } + const status = normalizeOptionalString(record.status) + ?.toLowerCase() + .replace(/[\s-]+/g, "_"); + const hasFailureStatus = status !== undefined && STRUCTURED_MEMORY_FAILURE_STATUSES.has(status); + const hasFailureFields = + hasFailureStatus || + ["disabled", "unavailable", "success", "error"].some((key) => key in record); + if (!hasFailureFields) { + return undefined; + } + return ( + hasFailureStatus || + record.disabled === true || + record.unavailable === true || + record.success === false || + Boolean(record.error) + ); +} + +function readStructuredMemoryEvidence(source: unknown): boolean | undefined { + if (Array.isArray(source)) { + return source.length > 0; + } + const record = asRecord(source); + return record ? readExplicitMemoryEvidence(record) : undefined; +} + +function readStructuredContentState( + content: unknown, + readState: (source: unknown) => boolean | undefined, + decisiveState: boolean, +): boolean | undefined { + const parts = extractTextContentParts(content); + let sawOtherState = false; + for (const part of parts) { + try { + const state = readState(JSON.parse(part)); + if (state === decisiveState) { + return decisiveState; + } + sawOtherState ||= state === !decisiveState; + } catch {} + } + try { + const state = readState(JSON.parse(parts.join(" ").trim())); + if (state !== undefined) { + return state; + } + } catch {} + return sawOtherState ? !decisiveState : undefined; +} + +function readStructuredMemoryFailureFromContent(content: unknown): boolean | undefined { + return readStructuredContentState(content, readStructuredMemoryFailure, true); +} + +function readStructuredMemoryEvidenceFromContent(content: unknown): boolean | undefined { + return readStructuredContentState(content, readStructuredMemoryEvidence, false); +} + function isTimeoutBoilerplateSummary(value: string): boolean { return TIMEOUT_BOILERPLATE_PATTERNS.some((pattern) => pattern.test(value)); } @@ -2247,12 +2715,12 @@ function buildSearchQuery(params: { return clampSearchQuery(context ? `${context} ${latest}` : latest); } -function extractTextContent(content: unknown): string { +function extractTextContentParts(content: unknown): string[] { if (typeof content === "string") { - return content; + return content.trim() ? [content] : []; } if (!Array.isArray(content)) { - return ""; + return []; } const parts: string[] = []; for (const item of content) { @@ -2272,7 +2740,11 @@ function extractTextContent(content: unknown): string { parts.push(typed.content); } } - return parts.join(" ").trim(); + return parts.map((part) => part.trim()).filter(Boolean); +} + +function extractTextContent(content: unknown): string { + return extractTextContentParts(content).join(" ").trim(); } function stripRecalledContextNoise(text: string): string { @@ -2486,6 +2958,9 @@ async function runRecallSubagent(params: { channelId: params.channelId, }); + let activeSessionFile = sessionFile; + let harnessHasUsableMemoryResult = false; + let harnessHasUnavailableMemorySearchResult = false; try { const embeddedConfig = applyActiveMemoryRuntimeConfigSnapshot(params.api.config, params.config); const embeddedTimeoutMs = params.config.timeoutMs + params.config.setupGraceTimeoutMs; @@ -2517,7 +2992,16 @@ async function runRecallSubagent(params: { authProfileFailurePolicy: "local", cleanupBundleMcpOnRunEnd: true, abortSignal: params.abortSignal, + onAgentToolResult: (event) => { + const evidence = readMemoryToolResultEvidence({ + ...event, + toolsAllow: params.config.toolsAllow, + }); + harnessHasUsableMemoryResult ||= evidence.hasUsableMemoryResult; + harnessHasUnavailableMemorySearchResult ||= evidence.hasUnavailableMemorySearchResult; + }, }); + activeSessionFile = readActiveMemorySessionFileFromRunResult(result) ?? sessionFile; if (params.abortSignal?.aborted) { const reason = params.abortSignal.reason; if (reason instanceof Error) { @@ -2535,19 +3019,34 @@ async function runRecallSubagent(params: { .filter(Boolean) .join("\n") .trim(); + const transcriptState = await readMergedActiveMemoryTranscriptState({ + sessionFiles: [sessionFile, activeSessionFile], + toolsAllow: params.config.toolsAllow, + }); const searchDebug = - (await readActiveMemorySearchDebug(sessionFile)) ?? - readActiveMemorySearchDebugFromRunResult(result); + transcriptState.searchDebug ?? readActiveMemorySearchDebugFromRunResult(result); return { rawReply: rawReply || "NONE", - transcriptPath: params.config.persistTranscripts ? sessionFile : undefined, + transcriptPath: params.config.persistTranscripts ? activeSessionFile : undefined, searchDebug, + hasUsableMemoryResult: transcriptState.hasUsableMemoryResult || harnessHasUsableMemoryResult, + hasUnavailableMemorySearchResult: + transcriptState.hasUnavailableMemorySearchResult || harnessHasUnavailableMemorySearchResult, }; } catch (error) { if (params.abortSignal?.aborted) { - const partialReply = await readPartialAssistantText(sessionFile); - const searchDebug = await readActiveMemorySearchDebug(sessionFile); - attachPartialTimeoutData(error, partialReply, searchDebug); + const partialReply = await readPartialAssistantText(activeSessionFile); + const transcriptState = await readActiveMemoryTranscriptState( + activeSessionFile, + undefined, + params.config.toolsAllow, + ); + attachPartialTimeoutData( + error, + partialReply, + transcriptState.searchDebug, + transcriptState.hasUnavailableMemorySearchResult || harnessHasUnavailableMemorySearchResult, + ); } if ( !params.abortSignal?.aborted && @@ -2583,7 +3082,9 @@ async function maybeResolveActiveRecall(params: { searchQuery: string; currentModelProviderId?: string; currentModelId?: string; + abortSignal?: AbortSignal; }): Promise { + params.abortSignal?.throwIfAborted(); const startedAt = Date.now(); const cacheKey = buildCacheKey({ agentId: params.agentId, @@ -2607,6 +3108,7 @@ async function maybeResolveActiveRecall(params: { : []), ].join(" "); if (cached) { + params.abortSignal?.throwIfAborted(); await persistPluginStatusLines({ api: params.api, agentId: params.agentId, @@ -2615,6 +3117,7 @@ async function maybeResolveActiveRecall(params: { debugSummary: buildPersistedDebugSummary(cached), searchDebug: cached.searchDebug, }); + params.abortSignal?.throwIfAborted(); if (params.config.logging) { params.api.logger.info?.( `${logPrefix} cached status=${cached.status} summaryChars=${String(cached.summary?.length ?? 0)} queryChars=${String(params.query.length)}`, @@ -2630,6 +3133,22 @@ async function maybeResolveActiveRecall(params: { resolvedModelRef?.provider, resolvedModelRef?.model, ); + let timeoutCleanupScheduled = false; + const scheduleTimeoutCleanup = () => { + if (timeoutCleanupScheduled) { + return; + } + timeoutCleanupScheduled = true; + scheduleMemorySearchCleanupAfterTimeout(params.api, logPrefix, params.agentId); + }; + let circuitBreakerTimeoutRecorded = false; + const recordRecallTimeout = () => { + if (!circuitBreakerTimeoutRecorded) { + circuitBreakerTimeoutRecorded = true; + recordCircuitBreakerTimeout(cbKey); + } + scheduleTimeoutCleanup(); + }; if ( isCircuitBreakerOpen( cbKey, @@ -2647,6 +3166,7 @@ async function maybeResolveActiveRecall(params: { `${logPrefix} skipped (circuit breaker open after consecutive timeouts)`, ); } + params.abortSignal?.throwIfAborted(); await persistPluginStatusLines({ api: params.api, agentId: params.agentId, @@ -2665,10 +3185,20 @@ async function maybeResolveActiveRecall(params: { } const controller = new AbortController(); + const abortFromParent = () => controller.abort(params.abortSignal?.reason); + params.abortSignal?.addEventListener("abort", abortFromParent, { once: true }); + if (params.abortSignal?.aborted) { + abortFromParent(); + } const TIMEOUT_SENTINEL = Symbol("timeout"); let sessionFile: string | undefined; + let recallTimedOut = false; const watchdogTimeoutMs = params.config.timeoutMs + params.config.setupGraceTimeoutMs; const timeoutId = setTimeout(() => { + if (params.abortSignal?.aborted) { + return; + } + recallTimedOut = true; controller.abort(new Error(`active-memory timeout after ${watchdogTimeoutMs}ms`)); }, watchdogTimeoutMs); timeoutId.unref?.(); @@ -2684,7 +3214,9 @@ async function maybeResolveActiveRecall(params: { }); let terminalMemorySearchWatch: TerminalMemorySearchWatch | undefined; + let recallInFlight = false; try { + recallInFlight = true; const subagentPromise = runRecallSubagent({ ...params, modelRef: resolvedModelRef, @@ -2696,30 +3228,62 @@ async function maybeResolveActiveRecall(params: { terminalMemorySearchWatch = watchTerminalMemorySearchResult({ getSessionFile: () => sessionFile, abortSignal: controller.signal, + toolsAllow: params.config.toolsAllow, }); // Silently catch late rejections after timeout so they don't become // unhandled promise rejections. subagentPromise.catch(() => undefined); - const raceResult = await Promise.race([ + let raceResult = await Promise.race([ subagentPromise, timeoutPromise, terminalMemorySearchWatch.promise, ]); terminalMemorySearchWatch.stop(); + let fallbackSearchDebug: ActiveMemorySearchDebug | undefined; + let fallbackHasUsableMemoryResult = false; + if ( + raceResult !== TIMEOUT_SENTINEL && + "status" in raceResult && + raceResult.hasUsableMemoryResult + ) { + // A later unavailable call must not discard a summary grounded in an + // earlier successful recall. The existing watchdog remains the deadline. + fallbackSearchDebug = raceResult.searchDebug; + fallbackHasUsableMemoryResult = true; + raceResult = await Promise.race([subagentPromise, timeoutPromise]); + } + if (raceResult !== TIMEOUT_SENTINEL) { + recallInFlight = false; + } if (raceResult === TIMEOUT_SENTINEL) { - const result = await buildTimeoutRecallResult({ - elapsedMs: Date.now() - startedAt, - maxSummaryChars: params.config.maxSummaryChars, - sessionFile, - subagentPromise, - }); + if (recallTimedOut) { + recordRecallTimeout(); + } else if (params.abortSignal?.aborted && recallInFlight) { + scheduleTimeoutCleanup(); + } + const elapsedMs = Date.now() - startedAt; + const result: ActiveRecallResult = fallbackHasUsableMemoryResult + ? { + status: "timeout", + elapsedMs, + summary: null, + searchDebug: fallbackSearchDebug, + } + : await buildTimeoutRecallResult({ + elapsedMs, + maxSummaryChars: params.config.maxSummaryChars, + sessionFile, + subagentPromise, + toolsAllow: params.config.toolsAllow, + }); if (params.config.logging) { params.api.logger.info?.( `${logPrefix} done status=${result.status} elapsedMs=${String(result.elapsedMs)} summaryChars=${String(result.summary?.length ?? 0)}`, ); } + params.abortSignal?.throwIfAborted(); await persistPluginStatusLines({ api: params.api, agentId: params.agentId, @@ -2728,8 +3292,7 @@ async function maybeResolveActiveRecall(params: { debugSummary: buildPersistedDebugSummary(result), searchDebug: result.searchDebug, }); - recordCircuitBreakerTimeout(cbKey); - scheduleMemorySearchCleanupAfterTimeout(params.api, logPrefix, params.agentId); + params.abortSignal?.throwIfAborted(); return result; } @@ -2743,9 +3306,11 @@ async function maybeResolveActiveRecall(params: { }; if (params.config.logging) { params.api.logger.info?.( - `${logPrefix} done status=${result.status} elapsedMs=${String(result.elapsedMs)} summaryChars=0`, + `${logPrefix} done status=${result.status} elapsedMs=${String(result.elapsedMs)} summaryChars=${String(result.summary?.length ?? 0)}`, ); } + resetCircuitBreaker(cbKey); + params.abortSignal?.throwIfAborted(); await persistPluginStatusLines({ api: params.api, agentId: params.agentId, @@ -2753,55 +3318,31 @@ async function maybeResolveActiveRecall(params: { statusLine: buildPluginStatusLine({ result, config: params.config }), searchDebug: result.searchDebug, }); + params.abortSignal?.throwIfAborted(); if (shouldCacheResult(result)) { setCachedResult(cacheKey, result, params.config.cacheTtlMs); } - resetCircuitBreaker(cbKey); return result; } - const { rawReply, resultStatus, transcriptPath, searchDebug } = raceResult; - const summary = truncateSummary( - normalizeActiveSummary(rawReply) ?? "", - params.config.maxSummaryChars, - ); + const { transcriptPath } = raceResult; if (params.config.logging && transcriptPath) { params.api.logger.info?.(`${logPrefix} transcript=${transcriptPath}`); } - const result: ActiveRecallResult = - summary.length > 0 - ? { - status: "ok", - elapsedMs: Date.now() - startedAt, - rawReply, - summary, - searchDebug, - } - : resultStatus === "failed" - ? { - status: "failed", - elapsedMs: Date.now() - startedAt, - summary: null, - searchDebug, - } - : resultStatus === "unavailable" || isUnavailableMemorySearchDebug(searchDebug) - ? { - status: "unavailable", - elapsedMs: Date.now() - startedAt, - summary: null, - searchDebug, - } - : { - status: "no_relevant_memory", - elapsedMs: Date.now() - startedAt, - summary: null, - searchDebug, - }; + const result = buildSubagentRecallResult({ + subagentResult: raceResult, + fallbackSearchDebug, + fallbackHasUsableMemoryResult, + elapsedMs: Date.now() - startedAt, + maxSummaryChars: params.config.maxSummaryChars, + }); if (params.config.logging) { params.api.logger.info?.( `${logPrefix} done status=${result.status} elapsedMs=${String(result.elapsedMs)} summaryChars=${String(result.summary?.length ?? 0)}`, ); } + resetCircuitBreaker(cbKey); + params.abortSignal?.throwIfAborted(); await persistPluginStatusLines({ api: params.api, agentId: params.agentId, @@ -2810,13 +3351,24 @@ async function maybeResolveActiveRecall(params: { debugSummary: buildPersistedDebugSummary(result), searchDebug: result.searchDebug, }); + params.abortSignal?.throwIfAborted(); if (shouldCacheResult(result)) { setCachedResult(cacheKey, result, params.config.cacheTtlMs); } - resetCircuitBreaker(cbKey); return result; } catch (error) { + if (params.abortSignal?.aborted) { + if (recallTimedOut) { + recordRecallTimeout(); + } else if (recallInFlight) { + scheduleTimeoutCleanup(); + } + params.abortSignal.throwIfAborted(); + } if (controller.signal.aborted) { + if (recallTimedOut) { + recordRecallTimeout(); + } const partialTimeoutData = readPartialTimeoutData(error); const result = await buildTimeoutRecallResult({ elapsedMs: Date.now() - startedAt, @@ -2824,12 +3376,15 @@ async function maybeResolveActiveRecall(params: { sessionFile, rawReply: partialTimeoutData.rawReply, searchDebug: partialTimeoutData.searchDebug, + hasUnavailableMemorySearchResult: partialTimeoutData.hasUnavailableMemorySearchResult, + toolsAllow: params.config.toolsAllow, }); if (params.config.logging) { params.api.logger.info?.( `${logPrefix} done status=${result.status} elapsedMs=${String(result.elapsedMs)} summaryChars=${String(result.summary?.length ?? 0)}`, ); } + params.abortSignal?.throwIfAborted(); await persistPluginStatusLines({ api: params.api, agentId: params.agentId, @@ -2838,8 +3393,7 @@ async function maybeResolveActiveRecall(params: { debugSummary: buildPersistedDebugSummary(result), searchDebug: result.searchDebug, }); - recordCircuitBreakerTimeout(cbKey); - scheduleMemorySearchCleanupAfterTimeout(params.api, logPrefix, params.agentId); + params.abortSignal?.throwIfAborted(); return result; } const message = toSingleLineLogValue(error instanceof Error ? error.message : String(error)); @@ -2851,6 +3405,7 @@ async function maybeResolveActiveRecall(params: { elapsedMs: Date.now() - startedAt, summary: null, }; + params.abortSignal?.throwIfAborted(); await persistPluginStatusLines({ api: params.api, agentId: params.agentId, @@ -2860,6 +3415,7 @@ async function maybeResolveActiveRecall(params: { }); return result; } finally { + params.abortSignal?.removeEventListener("abort", abortFromParent); terminalMemorySearchWatch?.stop(); clearTimeout(timeoutId); } @@ -3000,116 +3556,158 @@ export default definePluginEntry({ }, }); - const beforePromptBuildTimeoutMs = config.timeoutMs + config.setupGraceTimeoutMs; + // Preflight and recall own separate deadlines. Reserve enough hook time for + // both maxima so preflight latency cannot consume recall settlement time. + const beforePromptBuildTimeoutMs = + MAX_TIMEOUT_MS + MAX_SETUP_GRACE_TIMEOUT_MS + HOOK_TIMEOUT_RECOVERY_GRACE_MS * 2; api.on( "before_prompt_build", async (event, ctx) => { - try { - refreshLiveConfigFromRuntime(); - const resolvedAgentId = resolveStatusUpdateAgentId(ctx); - const resolvedSessionKey = - ctx.sessionKey?.trim() || - (resolvedAgentId - ? resolveCanonicalSessionKeyFromSessionId({ - api, - agentId: resolvedAgentId, - sessionId: ctx.sessionId, - }) - : undefined); - const effectiveAgentId = - resolvedAgentId || resolveStatusUpdateAgentId({ sessionKey: resolvedSessionKey }); - if (await isSessionActiveMemoryDisabled({ api, sessionKey: resolvedSessionKey })) { - await persistPluginStatusLines({ + refreshLiveConfigFromRuntime(); + const invocationConfig = config; + const liveRecallTimeoutMs = + invocationConfig.timeoutMs + + invocationConfig.setupGraceTimeoutMs + + HOOK_TIMEOUT_RECOVERY_GRACE_MS; + const deadlineController = new AbortController(); + const hookDeadline = createActiveMemoryHookDeadline(); + const armHookDeadline = (timeoutMs: number, phase: "preflight" | "recall") => { + hookDeadline.arm(timeoutMs, () => { + deadlineController.abort( + new Error(`active-memory ${phase} timeout after ${timeoutMs}ms`), + ); + api.logger.warn?.( + `active-memory: before_prompt_build ${phase} timed out after ${String(timeoutMs)}ms; skipping memory lookup`, + ); + }); + }; + armHookDeadline(HOOK_TIMEOUT_RECOVERY_GRACE_MS, "preflight"); + const handlerPromise = (async () => { + try { + const resolvedAgentId = resolveStatusUpdateAgentId(ctx); + const resolvedSessionKey = + ctx.sessionKey?.trim() || + (resolvedAgentId + ? resolveCanonicalSessionKeyFromSessionId({ + api, + agentId: resolvedAgentId, + sessionId: ctx.sessionId, + }) + : undefined); + const effectiveAgentId = + resolvedAgentId || resolveStatusUpdateAgentId({ sessionKey: resolvedSessionKey }); + const sessionDisabled = await isSessionActiveMemoryDisabled({ api, - agentId: effectiveAgentId, sessionKey: resolvedSessionKey, }); - return undefined; - } - if (!isEnabledForAgent(config, effectiveAgentId)) { - await persistPluginStatusLines({ + deadlineController.signal.throwIfAborted(); + if (sessionDisabled) { + await persistPluginStatusLines({ + api, + agentId: effectiveAgentId, + sessionKey: resolvedSessionKey, + }); + return undefined; + } + if (!isEnabledForAgent(invocationConfig, effectiveAgentId)) { + await persistPluginStatusLines({ + api, + agentId: effectiveAgentId, + sessionKey: resolvedSessionKey, + }); + return undefined; + } + if (!isEligibleInteractiveSession(ctx)) { + await persistPluginStatusLines({ + api, + agentId: effectiveAgentId, + sessionKey: resolvedSessionKey, + }); + return undefined; + } + if ( + !isAllowedChatType(invocationConfig, { + ...ctx, + sessionKey: resolvedSessionKey ?? ctx.sessionKey, + mainKey: api.config.session?.mainKey, + }) + ) { + await persistPluginStatusLines({ + api, + agentId: effectiveAgentId, + sessionKey: resolvedSessionKey, + }); + return undefined; + } + if ( + !isAllowedChatId(invocationConfig, { + sessionKey: resolvedSessionKey ?? ctx.sessionKey, + messageProvider: ctx.messageProvider, + }) + ) { + await persistPluginStatusLines({ + api, + agentId: effectiveAgentId, + sessionKey: resolvedSessionKey, + }); + return undefined; + } + const recentTurns = extractRecentTurns(event.messages); + const query = buildQuery({ + latestUserMessage: event.prompt, + recentTurns, + config: invocationConfig, + }); + const searchQuery = buildSearchQuery({ + latestUserMessage: event.prompt, + recentTurns, + }); + // Start recall with its full configured budget. The preceding + // session/config checks must not consume abort-settlement time. + armHookDeadline(liveRecallTimeoutMs, "recall"); + const result = await maybeResolveActiveRecall({ api, + config: invocationConfig, agentId: effectiveAgentId, sessionKey: resolvedSessionKey, - }); - return undefined; - } - if (!isEligibleInteractiveSession(ctx)) { - await persistPluginStatusLines({ - api, - agentId: effectiveAgentId, - sessionKey: resolvedSessionKey, - }); - return undefined; - } - if ( - !isAllowedChatType(config, { - ...ctx, - sessionKey: resolvedSessionKey ?? ctx.sessionKey, - mainKey: api.config.session?.mainKey, - }) - ) { - await persistPluginStatusLines({ - api, - agentId: effectiveAgentId, - sessionKey: resolvedSessionKey, - }); - return undefined; - } - if ( - !isAllowedChatId(config, { - sessionKey: resolvedSessionKey ?? ctx.sessionKey, + sessionId: ctx.sessionId, messageProvider: ctx.messageProvider, - }) - ) { - await persistPluginStatusLines({ - api, - agentId: effectiveAgentId, - sessionKey: resolvedSessionKey, + channelId: ctx.channelId, + query, + searchQuery, + currentModelProviderId: ctx.modelProviderId, + currentModelId: ctx.modelId, + abortSignal: deadlineController.signal, }); + deadlineController.signal.throwIfAborted(); + if (!result.summary) { + return undefined; + } + const promptPrefix = buildPromptPrefix(result.summary); + if (!promptPrefix) { + return undefined; + } + return { + prependContext: promptPrefix, + }; + } catch (error) { + if (deadlineController.signal.aborted) { + return undefined; + } + const message = toSingleLineLogValue( + error instanceof Error ? error.message : String(error), + ); + api.logger.warn?.( + `active-memory: before_prompt_build failed, skipping memory lookup: ${message}`, + ); return undefined; } - const recentTurns = extractRecentTurns(event.messages); - const query = buildQuery({ - latestUserMessage: event.prompt, - recentTurns, - config, - }); - const searchQuery = buildSearchQuery({ - latestUserMessage: event.prompt, - recentTurns, - }); - const result = await maybeResolveActiveRecall({ - api, - config, - agentId: effectiveAgentId, - sessionKey: resolvedSessionKey, - sessionId: ctx.sessionId, - messageProvider: ctx.messageProvider, - channelId: ctx.channelId, - query, - searchQuery, - currentModelProviderId: ctx.modelProviderId, - currentModelId: ctx.modelId, - }); - if (!result.summary) { - return undefined; - } - const promptPrefix = buildPromptPrefix(result.summary); - if (!promptPrefix) { - return undefined; - } - return { - prependContext: promptPrefix, - }; - } catch (error) { - const message = toSingleLineLogValue( - error instanceof Error ? error.message : String(error), - ); - api.logger.warn?.( - `active-memory: before_prompt_build failed, skipping memory lookup: ${message}`, - ); - return undefined; + })(); + try { + const result = await Promise.race([handlerPromise, hookDeadline.promise]); + return typeof result === "symbol" ? undefined : result; + } finally { + hookDeadline.stop(); } }, { timeoutMs: beforePromptBuildTimeoutMs }, @@ -3124,6 +3722,7 @@ const testing = { buildPluginStatusLine, buildPromptPrefix, getCachedResult, + hasUsableMemoryResultInSessionRecord, isCircuitBreakerOpen, isMissingRegisteredMemoryToolsError, normalizePluginConfig, diff --git a/extensions/active-memory/openclaw.plugin.json b/extensions/active-memory/openclaw.plugin.json index cfcc47b1de35..c335dc0627de 100644 --- a/extensions/active-memory/openclaw.plugin.json +++ b/extensions/active-memory/openclaw.plugin.json @@ -123,11 +123,12 @@ "help": "Optional explicit denylist of chat/user IDs. Sessions whose resolved conversation id matches the list are skipped even when the chat type is allowed. Applied after allowedChatIds." }, "timeoutMs": { - "label": "Timeout (ms)" + "label": "Timeout (ms)", + "help": "Recall work budget on the main lane. Before recall, the hook allows up to 1500 ms for session/config preflight. After recall starts, it reserves another fixed 1500 ms only for abort settlement and transcript recovery." }, "setupGraceTimeoutMs": { "label": "Setup Grace Timeout (ms)", - "help": "Advanced: extra blocking budget for cold embedded-run setup before the recall timeout is considered exhausted. Defaults to 0 so timeoutMs remains the main-lane hook budget unless you opt in." + "help": "Advanced: extra recall-work budget for cold embedded-run setup. Defaults to 0. The separate 1500 ms preflight cap and 1500 ms post-recall completion allowance still apply." }, "queryMode": { "label": "Query Mode", diff --git a/extensions/codex/src/app-server/dynamic-tool-execution.test.ts b/extensions/codex/src/app-server/dynamic-tool-execution.test.ts index d4efd5780597..569f21386e04 100644 --- a/extensions/codex/src/app-server/dynamic-tool-execution.test.ts +++ b/extensions/codex/src/app-server/dynamic-tool-execution.test.ts @@ -183,6 +183,7 @@ describe("dynamic tool execution helpers", () => { vi.useFakeTimers(); let capturedSignal: AbortSignal | undefined; const onTimeout = vi.fn(); + const onAgentToolResult = vi.fn(); const response = handleDynamicToolCallWithTimeout({ call: { threadId: "thread-1", @@ -200,6 +201,7 @@ describe("dynamic tool execution helpers", () => { }, signal: new AbortController().signal, timeoutMs: 1, + onAgentToolResult, onTimeout, }); @@ -216,6 +218,64 @@ describe("dynamic tool execution helpers", () => { }); expect(capturedSignal?.aborted).toBe(true); expect(onTimeout).toHaveBeenCalledTimes(1); + expect(onAgentToolResult).toHaveBeenCalledWith({ + toolName: "message", + result: { + content: [ + { + type: "text", + text: "OpenClaw dynamic tool call timed out after 1ms while running tool message.", + }, + ], + details: { + status: "failed", + error: "OpenClaw dynamic tool call timed out after 1ms while running tool message.", + }, + }, + isError: true, + }); + }); + + it("reports pre-execution aborts to the private result observer", async () => { + const controller = new AbortController(); + controller.abort(new Error("run cancelled")); + const onAgentToolResult = vi.fn(); + const handleToolCall = vi.fn(); + + const result = await handleDynamicToolCallWithTimeout({ + call: { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-aborted", + namespace: null, + tool: "memory_search", + arguments: {}, + }, + toolBridge: { handleToolCall }, + signal: controller.signal, + timeoutMs: 1_000, + onAgentToolResult, + }); + + expect(result).toEqual({ + success: false, + contentItems: [ + { type: "inputText", text: "OpenClaw dynamic tool call aborted before execution." }, + ], + }); + expect(handleToolCall).not.toHaveBeenCalled(); + expect(onAgentToolResult).toHaveBeenCalledOnce(); + expect(onAgentToolResult).toHaveBeenCalledWith({ + toolName: "memory_search", + result: { + content: [{ type: "text", text: "OpenClaw dynamic tool call aborted before execution." }], + details: { + status: "failed", + error: "OpenClaw dynamic tool call aborted before execution.", + }, + }, + isError: true, + }); }); it("logs process poll timeout context separately from session idle", async () => { diff --git a/extensions/codex/src/app-server/dynamic-tool-execution.ts b/extensions/codex/src/app-server/dynamic-tool-execution.ts index c4aac342aa75..a126927e4259 100644 --- a/extensions/codex/src/app-server/dynamic-tool-execution.ts +++ b/extensions/codex/src/app-server/dynamic-tool-execution.ts @@ -126,10 +126,41 @@ export async function handleDynamicToolCallWithTimeout(params: { toolBridge: Pick; signal: AbortSignal; timeoutMs: number; + onAgentToolResult?: EmbeddedRunAttemptParams["onAgentToolResult"]; onTimeout?: () => void; }): Promise { + // Timeout or run abort can win while a tool ignores cancellation. Keep the + // private observer terminal result exactly once across those competing paths. + let didNotifyAgentToolResult = false; + const notifyAgentToolResult = ( + event: Parameters>[0], + ) => { + if (didNotifyAgentToolResult) { + return; + } + didNotifyAgentToolResult = true; + try { + params.onAgentToolResult?.(event); + } catch (error) { + embeddedAgentLog.warn( + `onAgentToolResult handler failed: tool=${params.call.tool} error=${String(error)}`, + ); + } + }; + const notifyFailedToolResult = (message: string) => { + notifyAgentToolResult({ + toolName: params.call.tool, + result: { + content: [{ type: "text", text: message }], + details: { status: "failed", error: message }, + }, + isError: true, + }); + }; if (params.signal.aborted) { - return failedDynamicToolResponse("OpenClaw dynamic tool call aborted before execution."); + const message = "OpenClaw dynamic tool call aborted before execution."; + notifyFailedToolResult(message); + return failedDynamicToolResponse(message); } const controller = new AbortController(); @@ -139,6 +170,7 @@ export async function handleDynamicToolCallWithTimeout(params: { const abortFromRun = () => { const message = "OpenClaw dynamic tool call aborted."; controller.abort(params.signal.reason ?? new Error(message)); + notifyFailedToolResult(message); resolveAbort?.(failedDynamicToolResponse(message, { sideEffectEvidence: true })); }; const abortPromise = new Promise((resolve) => { @@ -155,6 +187,7 @@ export async function handleDynamicToolCallWithTimeout(params: { ...timeoutDetails.meta, consoleMessage: timeoutDetails.consoleMessage, }); + notifyFailedToolResult(timeoutDetails.responseMessage); resolve( failedDynamicToolResponse(timeoutDetails.responseMessage, { sideEffectEvidence: true }), ); @@ -167,13 +200,22 @@ export async function handleDynamicToolCallWithTimeout(params: { if (params.signal.aborted) { abortFromRun(); } - return await Promise.race([ - params.toolBridge.handleToolCall(params.call, { signal: controller.signal }), + const response = await Promise.race([ + params.toolBridge.handleToolCall(params.call, { + signal: controller.signal, + onAgentToolResult: notifyAgentToolResult, + }), abortPromise, timeoutPromise, ]); + if (!response.success && !didNotifyAgentToolResult) { + notifyFailedToolResult(readDynamicToolResponseText(response)); + } + return response; } catch (error) { - return failedDynamicToolResponse(error instanceof Error ? error.message : String(error), { + const message = error instanceof Error ? error.message : String(error); + notifyFailedToolResult(message); + return failedDynamicToolResponse(message, { sideEffectEvidence: true, }); } finally { @@ -188,6 +230,16 @@ export async function handleDynamicToolCallWithTimeout(params: { } } +function readDynamicToolResponseText(response: CodexDynamicToolCallResponse): string { + const text = response.contentItems + .flatMap((item) => + item.type === "inputText" && typeof item.text === "string" ? [item.text] : [], + ) + .join("\n") + .trim(); + return text || "OpenClaw dynamic tool call failed."; +} + function failedDynamicToolResponse( message: string, options?: { sideEffectEvidence?: boolean }, diff --git a/extensions/codex/src/app-server/dynamic-tools.test.ts b/extensions/codex/src/app-server/dynamic-tools.test.ts index 2327c418a670..67c8de15dc96 100644 --- a/extensions/codex/src/app-server/dynamic-tools.test.ts +++ b/extensions/codex/src/app-server/dynamic-tools.test.ts @@ -222,6 +222,7 @@ describe("createCodexDynamicToolBridge", () => { it("can register a durable tool schema while denying execution for the current turn", async () => { const heartbeatExecute = vi.fn(async () => textToolResult("heartbeat recorded")); + const onAgentToolResult = vi.fn(); const bridge = createCodexDynamicToolBridge({ tools: [createTool({ name: "message" })], registeredTools: [ @@ -237,14 +238,17 @@ describe("createCodexDynamicToolBridge", () => { HEARTBEAT_RESPONSE_TOOL_NAME, ]); - const result = await bridge.handleToolCall({ - threadId: "thread-1", - turnId: "turn-1", - callId: "call-1", - namespace: null, - tool: HEARTBEAT_RESPONSE_TOOL_NAME, - arguments: {}, - }); + const result = await bridge.handleToolCall( + { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-1", + namespace: null, + tool: HEARTBEAT_RESPONSE_TOOL_NAME, + arguments: {}, + }, + { onAgentToolResult }, + ); expect(result).toEqual({ success: false, @@ -256,6 +260,22 @@ describe("createCodexDynamicToolBridge", () => { ], }); expect(heartbeatExecute).not.toHaveBeenCalled(); + expect(onAgentToolResult).toHaveBeenCalledWith({ + toolName: HEARTBEAT_RESPONSE_TOOL_NAME, + result: { + content: [ + { + type: "text", + text: `OpenClaw tool is not available for this turn: ${HEARTBEAT_RESPONSE_TOOL_NAME}`, + }, + ], + details: { + status: "failed", + error: `OpenClaw tool is not available for this turn: ${HEARTBEAT_RESPONSE_TOOL_NAME}`, + }, + }, + isError: true, + }); }); it("keeps available and registered schemas paired with their tools", () => { @@ -1027,6 +1047,152 @@ describe("createCodexDynamicToolBridge", () => { expectContextFields(callArg(handler, 0, 1, "middleware context"), { runtime: "codex" }); }); + it("keeps unrecognized non-success statuses fail-closed", async () => { + const onAgentToolResult = vi.fn(); + const bridge = createCodexDynamicToolBridge({ + tools: [ + createTool({ + name: "exec", + execute: vi.fn(async () => + textToolResult("Approval is unavailable.", { status: "approval-unavailable" }), + ), + }), + ], + signal: new AbortController().signal, + }); + + const result = await bridge.handleToolCall( + { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-1", + namespace: null, + tool: "exec", + arguments: { command: "pwd" }, + }, + { onAgentToolResult }, + ); + + expect(result).toMatchObject({ success: false }); + expect(onAgentToolResult).toHaveBeenCalledWith({ + toolName: "exec", + result: textToolResult("Approval is unavailable.", { status: "approval-unavailable" }), + isError: true, + }); + }); + + it("preserves explicitly successful cancellation outcomes", async () => { + const onAgentToolResult = vi.fn(); + const cancelledResult = textToolResult("Approval rejected.", { + ok: true, + status: "cancelled", + }); + const bridge = createCodexDynamicToolBridge({ + tools: [ + createTool({ + name: "lobster", + execute: vi.fn(async () => cancelledResult), + }), + ], + signal: new AbortController().signal, + }); + + const result = await bridge.handleToolCall( + { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-1", + namespace: null, + tool: "lobster", + arguments: {}, + }, + { onAgentToolResult }, + ); + + expect(result).toMatchObject({ success: true }); + expect(onAgentToolResult).toHaveBeenCalledWith({ + toolName: "lobster", + result: cancelledResult, + isError: false, + }); + }); + + it("reports sanitized dynamic tool results to the private result observer", async () => { + const onAgentToolResult = vi.fn(); + const bridge = createCodexDynamicToolBridge({ + tools: [ + createTool({ + name: "memory_lookup_custom", + execute: vi.fn(async () => + textToolResult("OPENROUTER_API_KEY=sk-or-v1-abcdef0123456789", { + status: "failed", + error: "backend unavailable", + }), + ), + }), + ], + signal: new AbortController().signal, + }); + + await bridge.handleToolCall( + { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-1", + namespace: null, + tool: "memory_lookup_custom", + arguments: {}, + }, + { onAgentToolResult }, + ); + + expect(onAgentToolResult).toHaveBeenCalledOnce(); + expect(onAgentToolResult).toHaveBeenCalledWith({ + toolName: "memory_lookup_custom", + result: { + content: [{ type: "text", text: "OPENROUTER_API_KEY=sk-or-…6789" }], + details: { status: "failed", error: "backend unavailable" }, + }, + isError: true, + }); + }); + + it("reports thrown dynamic tool failures to the private result observer", async () => { + const onAgentToolResult = vi.fn(); + const bridge = createCodexDynamicToolBridge({ + tools: [ + createTool({ + name: "memory_lookup_custom", + execute: vi.fn(async () => { + throw new Error("backend unavailable"); + }), + }), + ], + signal: new AbortController().signal, + }); + + await bridge.handleToolCall( + { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-1", + namespace: null, + tool: "memory_lookup_custom", + arguments: {}, + }, + { onAgentToolResult }, + ); + + expect(onAgentToolResult).toHaveBeenCalledWith({ + toolName: "memory_lookup_custom", + result: { + content: [{ type: "text", text: "backend unavailable" }], + details: { status: "failed", error: "backend unavailable" }, + }, + isError: true, + }); + }); + it("preserves terminal async tool results without marking them as errors", async () => { const bridge = createBridgeWithToolResult("image_generate", { content: [{ type: "text", text: "Background task started." }], diff --git a/extensions/codex/src/app-server/dynamic-tools.ts b/extensions/codex/src/app-server/dynamic-tools.ts index afd2dd5c3837..a15796c6cce9 100644 --- a/extensions/codex/src/app-server/dynamic-tools.ts +++ b/extensions/codex/src/app-server/dynamic-tools.ts @@ -12,11 +12,13 @@ import { embeddedAgentLog, type EmbeddedRunAttemptParams, isToolWrappedWithBeforeToolCallHook, + isToolResultError, isMessagingTool, isMessagingToolSendAction, normalizeHeartbeatToolResponse, projectRuntimeToolInputSchema, runAgentHarnessAfterToolCallHook, + sanitizeToolResult, setBeforeToolCallDiagnosticsEnabled, type AnyAgentTool, type HeartbeatToolResponse, @@ -71,7 +73,10 @@ export type CodexDynamicToolBridge = { specs: CodexDynamicToolSpec[]; handleToolCall: ( params: CodexDynamicToolCallParams, - options?: { signal?: AbortSignal }, + options?: { + signal?: AbortSignal; + onAgentToolResult?: EmbeddedRunAttemptParams["onAgentToolResult"]; + }, ) => Promise; telemetry: { didSendViaMessagingTool: boolean; @@ -155,7 +160,6 @@ export function createCodexDynamicToolBridge(params: { ...ALWAYS_DIRECT_DYNAMIC_TOOL_NAMES, ...(params.directToolNames ?? []), ]); - return { availableSpecs: availableTools.map((entry) => createCodexDynamicToolSpec({ @@ -175,19 +179,28 @@ export function createCodexDynamicToolBridge(params: { handleToolCall: async (call, options) => { const toolEntry = toolMap.get(call.tool); if (!toolEntry) { + const message = registeredToolNames.has(call.tool) + ? `OpenClaw tool is not available for this turn: ${call.tool}` + : `Unknown OpenClaw tool: ${call.tool}`; + notifyAgentToolResult( + options?.onAgentToolResult, + call.tool, + failedToolResult(message), + true, + ); if (registeredToolNames.has(call.tool)) { return { contentItems: [ { type: "inputText", - text: `OpenClaw tool is not available for this turn: ${call.tool}`, + text: message, }, ], success: false, }; } return { - contentItems: [{ type: "inputText", text: `Unknown OpenClaw tool: ${call.tool}` }], + contentItems: [{ type: "inputText", text: message }], success: false, }; } @@ -202,7 +215,7 @@ export function createCodexDynamicToolBridge(params: { const preparedArgs = tool.prepareArguments ? tool.prepareArguments(args) : args; didStartExecution = true; const rawResult = await tool.execute(call.callId, preparedArgs, signal); - const rawIsError = isToolResultError(rawResult); + const rawIsError = isCodexToolResultError(rawResult); const middlewareResult = await middlewareRunner.applyToolResultMiddleware({ threadId: call.threadId, turnId: call.turnId, @@ -220,7 +233,8 @@ export function createCodexDynamicToolBridge(params: { args, result: middlewareResult, }); - const resultIsError = rawIsError || isToolResultError(result); + const resultIsError = rawIsError || isCodexToolResultError(result); + notifyAgentToolResult(options?.onAgentToolResult, toolName, result, resultIsError); collectToolTelemetry({ toolName, args, @@ -262,6 +276,13 @@ export function createCodexDynamicToolBridge(params: { ); return withSideEffectEvidence(response, terminalType !== "blocked"); } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + notifyAgentToolResult( + options?.onAgentToolResult, + toolName, + failedToolResult(errorMessage), + true, + ); collectToolTelemetry({ toolName, args, @@ -278,7 +299,7 @@ export function createCodexDynamicToolBridge(params: { sessionKey: toolResultHookContext.sessionKey, channelId: toolResultHookContext.channelId, startArgs: args, - error: error instanceof Error ? error.message : String(error), + error: errorMessage, startedAt, }); return withSideEffectEvidence( @@ -287,7 +308,7 @@ export function createCodexDynamicToolBridge(params: { contentItems: [ { type: "inputText", - text: error instanceof Error ? error.message : String(error), + text: errorMessage, }, ], success: false, @@ -301,6 +322,32 @@ export function createCodexDynamicToolBridge(params: { }; } +function notifyAgentToolResult( + observer: EmbeddedRunAttemptParams["onAgentToolResult"] | undefined, + toolName: string, + result: unknown, + isError: boolean, +) { + try { + observer?.({ + toolName, + result: sanitizeToolResult(result), + isError, + }); + } catch (error) { + embeddedAgentLog.warn( + `onAgentToolResult handler failed: tool=${toolName} error=${String(error)}`, + ); + } +} + +function failedToolResult(message: string): AgentToolResult { + return { + content: [{ type: "text", text: message }], + details: { status: "failed", error: message }, + }; +} + function wrapProjectedCodexDynamicTools( tools: readonly ProjectedCodexDynamicTool[], hookContext: CodexDynamicToolHookContext | undefined, @@ -688,11 +735,17 @@ function readPositiveInteger(value: unknown): number | undefined { return Math.floor(value); } -function isToolResultError(result: AgentToolResult): boolean { +function isCodexToolResultError(result: AgentToolResult): boolean { + if (isToolResultError(result)) { + return true; + } const details = result.details; if (!isRecord(details)) { return false; } + if (details.ok === true || details.success === true) { + return false; + } if (details.timedOut === true) { return true; } diff --git a/extensions/codex/src/app-server/run-attempt.ts b/extensions/codex/src/app-server/run-attempt.ts index 2472acace56b..13eeedf6fe28 100644 --- a/extensions/codex/src/app-server/run-attempt.ts +++ b/extensions/codex/src/app-server/run-attempt.ts @@ -1846,6 +1846,7 @@ export async function runCodexAppServerAttempt( toolBridge, signal: runAbortController.signal, timeoutMs: dynamicToolTimeoutMs, + onAgentToolResult: params.onAgentToolResult, onTimeout: () => { trajectoryRecorder?.recordEvent("tool.timeout", { threadId: call.threadId, diff --git a/extensions/copilot/src/tool-bridge.test.ts b/extensions/copilot/src/tool-bridge.test.ts index 9c2fc6a14787..d9715574d1fc 100644 --- a/extensions/copilot/src/tool-bridge.test.ts +++ b/extensions/copilot/src/tool-bridge.test.ts @@ -1213,14 +1213,70 @@ describe("convertOpenClawToolToSdkTool", () => { }); it("converts single text content to an exact textResultForLlm", async () => { - const sdkTool = convertOpenClawToolToSdkTool( - makeTool({}, { content: [{ text: "hello", type: "text" }], details: null }), - {}, - ); + const onAgentToolResult = vi.fn(); + const sourceResult = { + content: [{ text: "hello", type: "text" }], + details: { results: [{ text: "hello" }] }, + }; + const sdkTool = convertOpenClawToolToSdkTool(makeTool({}, sourceResult), { onAgentToolResult }); const result = await runSdkTool(sdkTool, {}); expect(result).toEqual({ resultType: "success", textResultForLlm: "hello" }); + expect(onAgentToolResult).toHaveBeenCalledWith({ + toolName: "tool-a", + result: sourceResult, + isError: false, + }); + }); + + it("reports thrown tool failures to the private result observer", async () => { + const error = new Error("backend unavailable"); + const onAgentToolResult = vi.fn(); + const sdkTool = convertOpenClawToolToSdkTool( + makeTool({ + execute: vi.fn(async () => { + throw error; + }), + }), + { onAgentToolResult }, + ); + + await runSdkTool(sdkTool, {}); + + expect(onAgentToolResult).toHaveBeenCalledWith({ + toolName: "tool-a", + result: { + content: [ + { + type: "text", + text: "[copilot-tool-bridge] tool 'tool-a' failed: backend unavailable", + }, + ], + details: { status: "failed", error: "backend unavailable" }, + }, + isError: true, + }); + }); + + it("reports returned OpenClaw error results as observer failures", async () => { + const onAgentToolResult = vi.fn(); + const sourceResult = { + content: [{ text: '{"status":"error","error":"backend unavailable"}', type: "text" }], + details: { status: "error", error: "backend unavailable" }, + }; + const sdkTool = convertOpenClawToolToSdkTool(makeTool({}, sourceResult), { + onAgentToolResult, + }); + + const result = await runSdkTool(sdkTool, {}); + + expect(result).toMatchObject({ resultType: "success" }); + expect(onAgentToolResult).toHaveBeenCalledWith({ + toolName: "tool-a", + result: sourceResult, + isError: true, + }); }); it("joins multiple text blocks with newlines", async () => { @@ -1276,16 +1332,12 @@ describe("convertOpenClawToolToSdkTool", () => { }); it("returns a failure result for unsupported content shapes", async () => { - const sdkTool = convertOpenClawToolToSdkTool( - makeTool( - {}, - { - content: [{ type: "resource" }], - details: null, - }, - ), - {}, - ); + const onAgentToolResult = vi.fn(); + const sourceResult = { + content: [{ type: "resource" }], + details: null, + }; + const sdkTool = convertOpenClawToolToSdkTool(makeTool({}, sourceResult), { onAgentToolResult }); const result = await runSdkTool(sdkTool, {}); @@ -1296,6 +1348,11 @@ describe("convertOpenClawToolToSdkTool", () => { expect(getError(result as ToolResultObject)).toBe( "[copilot-tool-bridge] unsupported AgentToolResult content shape: resource", ); + expect(onAgentToolResult).toHaveBeenCalledWith({ + toolName: "tool-a", + result: sourceResult, + isError: true, + }); }); it("returns a failure result when execute throws and preserves the error", async () => { diff --git a/extensions/copilot/src/tool-bridge.ts b/extensions/copilot/src/tool-bridge.ts index b77e6db43f39..ec02aab4d6f5 100644 --- a/extensions/copilot/src/tool-bridge.ts +++ b/extensions/copilot/src/tool-bridge.ts @@ -10,9 +10,11 @@ import { buildEmbeddedAttemptToolRunContext, getPluginToolMeta, isSubagentSessionKey, + isToolResultError, resolveAttemptSpawnWorkspaceDir, resolveEmbeddedAttemptToolConstructionPlan, resolveModelAuthMode, + sanitizeToolResult, } from "openclaw/plugin-sdk/agent-harness-runtime"; type CreateOpenClawCodingTools = @@ -205,6 +207,7 @@ export async function createCopilotToolBridge( convertOpenClawToolToSdkTool(sourceTool, { abortSignal: input.abortSignal, beforeExecute: input.beforeExecute, + onAgentToolResult: input.attemptParams?.onAgentToolResult, }), ), sourceTools: filteredTools, @@ -384,6 +387,7 @@ export function convertOpenClawToolToSdkTool( ctx: { abortSignal?: AbortSignal; beforeExecute?: CopilotToolBridgeInput["beforeExecute"]; + onAgentToolResult?: CopilotToolAttemptParams["onAgentToolResult"]; }, ): SdkTool { if (typeof sourceTool.name !== "string" || sourceTool.name.trim().length === 0) { @@ -397,13 +401,30 @@ export function convertOpenClawToolToSdkTool( } let sequentialLock = Promise.resolve(); + const notifyToolResult = (result: unknown, isError: boolean) => { + try { + ctx.onAgentToolResult?.({ toolName: sourceTool.name, result, isError }); + } catch (error) { + console.warn("[copilot-tool-bridge] onAgentToolResult handler threw; continuing", error); + } + }; + const failureResult = (message: string, error: unknown): ToolResultObject => { + notifyToolResult( + sanitizeToolResult({ + content: [{ type: "text", text: message }], + details: { status: "failed", error: toError(error).message }, + }), + true, + ); + return createFailureResult(message, error); + }; const executeOnce = async ( args: unknown, invocation: ToolInvocation, ): Promise => { if (ctx.abortSignal?.aborted) { const error = new Error("[copilot-tool-bridge] aborted before execution"); - return createFailureResult(error.message, error); + return failureResult(error.message, error); } try { @@ -415,7 +436,7 @@ export function convertOpenClawToolToSdkTool( toolName: sourceTool.name, }); } catch (error: unknown) { - return createFailureResult( + return failureResult( `[copilot-tool-bridge] beforeExecute failed for tool '${sourceTool.name}': ${toError(error).message}`, error, ); @@ -425,7 +446,7 @@ export function convertOpenClawToolToSdkTool( try { preparedArgs = sourceTool.prepareArguments ? sourceTool.prepareArguments(args) : args; } catch (error: unknown) { - return createFailureResult( + return failureResult( `[copilot-tool-bridge] prepareArguments failed for tool '${sourceTool.name}': ${toError(error).message}`, error, ); @@ -440,13 +461,19 @@ export function convertOpenClawToolToSdkTool( undefined, ); } catch (error: unknown) { - return createFailureResult( + return failureResult( `[copilot-tool-bridge] tool '${sourceTool.name}' failed: ${toError(error).message}`, error, ); } - return agentToolResultToSdk(result); + const sdkResult = agentToolResultToSdk(result); + const sanitizedResult = sanitizeToolResult(result); + notifyToolResult( + sanitizedResult, + sdkResult.resultType === "failure" || isToolResultError(sanitizedResult), + ); + return sdkResult; }; const handler = diff --git a/src/agents/embedded-agent-runner/run.overflow-compaction.test.ts b/src/agents/embedded-agent-runner/run.overflow-compaction.test.ts index 114df2c58c65..38971fe65637 100644 --- a/src/agents/embedded-agent-runner/run.overflow-compaction.test.ts +++ b/src/agents/embedded-agent-runner/run.overflow-compaction.test.ts @@ -61,6 +61,7 @@ type RuntimePlanOverrides = Partial didDeliverSourceReplyViaMessageTool, + onAgentToolResult: params.onAgentToolResult, onToolResult: params.onToolResult, onReasoningStream: params.onReasoningStream, onReasoningEnd: params.onReasoningEnd, diff --git a/src/agents/embedded-agent-runner/run/params.ts b/src/agents/embedded-agent-runner/run/params.ts index cbeeb316fe21..0602b415b323 100644 --- a/src/agents/embedded-agent-runner/run/params.ts +++ b/src/agents/embedded-agent-runner/run/params.ts @@ -210,6 +210,8 @@ export type RunEmbeddedAgentParams = { }) => void | Promise; onReasoningEnd?: () => void | Promise; onToolResult?: (payload: ReplyPayload) => void | Promise; + /** Synchronous private observer for the sanitized per-tool result. */ + onAgentToolResult?: (event: { toolName: string; result: unknown; isError: boolean }) => void; onAgentEvent?: (evt: { stream: string; data: Record; diff --git a/src/agents/embedded-agent-subscribe.handlers.tools.test.ts b/src/agents/embedded-agent-subscribe.handlers.tools.test.ts index 6f23ceea21e3..aa9d94b65278 100644 --- a/src/agents/embedded-agent-subscribe.handlers.tools.test.ts +++ b/src/agents/embedded-agent-subscribe.handlers.tools.test.ts @@ -384,6 +384,35 @@ describe("handleToolExecutionEnd cron.add commitment tracking", () => { }); }); +describe("handleToolExecutionEnd private result observer", () => { + it("reports the sanitized original tool result", async () => { + const { ctx } = createTestContext(); + const onAgentToolResult = vi.fn(); + ctx.params.onAgentToolResult = onAgentToolResult; + const result = { + content: [{ type: "text", text: '{"results":[{"text":"ramen"}]}' }], + details: { results: [{ text: "ramen" }] }, + }; + + await handleToolExecutionEnd( + ctx as never, + { + type: "tool_execution_end", + toolName: "memory_search", + toolCallId: "tool-memory-search", + isError: false, + result, + } as never, + ); + + expect(onAgentToolResult).toHaveBeenCalledWith({ + toolName: "memory_search", + result, + isError: false, + }); + }); +}); + describe("handleToolExecutionEnd sessions_spawn terminal success tracking", () => { it("records accepted sessions_spawn identifiers", async () => { const { ctx } = createTestContext(); @@ -931,7 +960,9 @@ describe("handleToolExecutionEnd exec approval prompts", () => { it("emits a deterministic unavailable payload when the initiating surface cannot approve", async () => { const { ctx } = createTestContext(); const onToolResult = vi.fn(); + const onAgentToolResult = vi.fn(); ctx.params.onToolResult = onToolResult; + ctx.params.onAgentToolResult = onAgentToolResult; await handleToolExecutionEnd( ctx as never, @@ -961,6 +992,13 @@ describe("handleToolExecutionEnd exec approval prompts", () => { expect(text).not.toContain("Pending command:"); expect(text).not.toContain("Host:"); expect(text).not.toContain("CWD:"); + expect(onAgentToolResult).toHaveBeenCalledWith({ + toolName: "exec", + result: expect.objectContaining({ + details: expect.objectContaining({ status: "approval-unavailable" }), + }), + isError: true, + }); expect(ctx.state.deterministicApprovalPromptSent).toBe(true); }); diff --git a/src/agents/embedded-agent-subscribe.handlers.tools.ts b/src/agents/embedded-agent-subscribe.handlers.tools.ts index ca798fd0030e..db73ae8d8e74 100644 --- a/src/agents/embedded-agent-subscribe.handlers.tools.ts +++ b/src/agents/embedded-agent-subscribe.handlers.tools.ts @@ -1166,8 +1166,21 @@ export async function handleToolExecutionEnd( const runId = ctx.params.runId; const isError = evt.isError; const result = evt.result; - const isToolError = isError || isToolResultError(result); + const observerIsError = isError || isToolResultError(result); const sanitizedResult = sanitizeToolResult(result); + const approvalUnavailable = + isExecToolName(toolName) && + readExecToolDetails(sanitizedResult)?.status === "approval-unavailable"; + const isToolError = observerIsError && !approvalUnavailable; + try { + ctx.params.onAgentToolResult?.({ + toolName, + result: sanitizedResult, + isError: observerIsError, + }); + } catch (error) { + ctx.log.warn(`onAgentToolResult handler failed: tool=${toolName} error=${String(error)}`); + } const eventResult = isExecToolName(toolName) ? capLiveExecResult(sanitizedResult) : sanitizedResult; diff --git a/src/agents/embedded-agent-subscribe.handlers.types.ts b/src/agents/embedded-agent-subscribe.handlers.types.ts index 37439da94f92..c2ba842652b2 100644 --- a/src/agents/embedded-agent-subscribe.handlers.types.ts +++ b/src/agents/embedded-agent-subscribe.handlers.types.ts @@ -271,6 +271,7 @@ type ToolHandlerParams = Pick< | "onAgentEvent" | "onExecutionPhase" | "onHeartbeatToolResponse" + | "onAgentToolResult" | "onToolResult" | "sessionKey" | "sessionId" diff --git a/src/agents/embedded-agent-subscribe.tools.test.ts b/src/agents/embedded-agent-subscribe.tools.test.ts index 4f654b38c69b..6a59ed3f8455 100644 --- a/src/agents/embedded-agent-subscribe.tools.test.ts +++ b/src/agents/embedded-agent-subscribe.tools.test.ts @@ -6,6 +6,7 @@ import { buildToolLifecycleErrorResult, extractToolErrorCode, extractToolErrorMessage, + isToolResultError, sanitizeToolArgs, sanitizeToolResult, } from "./embedded-agent-subscribe.tools.js"; @@ -125,6 +126,23 @@ describe("extractToolErrorMessage", () => { }); }); +describe("isToolResultError", () => { + it("recognizes returned failures and nonzero exits", () => { + expect(isToolResultError({ details: { status: "failed" } })).toBe(true); + expect(isToolResultError({ details: { status: "blocked" } })).toBe(true); + expect(isToolResultError({ details: { status: "approval-unavailable" } })).toBe(true); + expect(isToolResultError({ details: { status: "completed", timedOut: true } })).toBe(true); + expect(isToolResultError({ details: { status: "completed", exitCode: 1 } })).toBe(true); + expect(isToolResultError({ details: { status: "completed", exitCode: 0 } })).toBe(false); + expect(isToolResultError({ details: { ok: true, status: "cancelled" } })).toBe(false); + expect(isToolResultError({ details: { success: true, status: "canceled" } })).toBe(false); + expect(isToolResultError({ details: { ok: false, status: "completed" } })).toBe(true); + expect(isToolResultError({ details: { ok: true, status: "cancelled", timedOut: true } })).toBe( + true, + ); + }); +}); + function getTextContent(result: unknown, index = 0): string { // Sanitizer tests assert text redaction while keeping the result shape opaque. const record = result as { content: Array<{ text: string }> }; diff --git a/src/agents/embedded-agent-subscribe.tools.ts b/src/agents/embedded-agent-subscribe.tools.ts index b37f6c414c91..ed48b8ff3994 100644 --- a/src/agents/embedded-agent-subscribe.tools.ts +++ b/src/agents/embedded-agent-subscribe.tools.ts @@ -541,11 +541,37 @@ export function extractToolResultMediaPaths(result: unknown): string[] { } export function isToolResultError(result: unknown): boolean { + const details = readToolResultDetails(result); const normalized = readToolResultStatus(result); - if (!normalized) { - return false; + const explicitlySuccessful = details?.ok === true || details?.success === true; + if (details?.ok === false || details?.success === false) { + return true; } - return normalized === "error" || normalized === "timeout"; + const hasFailureStatus = + normalized === "error" || + normalized === "failed" || + normalized === "failure" || + normalized === "timeout" || + normalized === "timed_out" || + normalized === "blocked" || + normalized === "denied" || + normalized === "forbidden" || + normalized === "unavailable" || + normalized === "approval-unavailable" || + normalized === "disabled" || + normalized === "aborted" || + normalized === "cancelled" || + normalized === "canceled" || + normalized === "killed" || + normalized === "invalid"; + if (hasFailureStatus && !explicitlySuccessful) { + return true; + } + if (details?.timedOut === true || Boolean(details?.error)) { + return true; + } + const exitCode = details?.exitCode; + return typeof exitCode === "number" && Number.isFinite(exitCode) && exitCode !== 0; } export function extractToolErrorCode(result: unknown): string | undefined { diff --git a/src/agents/embedded-agent-subscribe.types.ts b/src/agents/embedded-agent-subscribe.types.ts index 6189b1bb3818..15f0d75c1e19 100644 --- a/src/agents/embedded-agent-subscribe.types.ts +++ b/src/agents/embedded-agent-subscribe.types.ts @@ -46,6 +46,7 @@ export type SubscribeEmbeddedAgentSessionParams = { /** Attempt-owned delivery proof for message-tool-only source replies. */ hasDeliveredMessageToolOnlySourceReply?: () => boolean; onToolResult?: (payload: ReplyPayload) => void | Promise; + onAgentToolResult?: (event: { toolName: string; result: unknown; isError: boolean }) => void; onReasoningStream?: (payload: { text?: string; mediaUrls?: string[]; diff --git a/src/agents/session-tool-result-guard.tool-result-persist-hook.test.ts b/src/agents/session-tool-result-guard.tool-result-persist-hook.test.ts index 51bb57424161..4db630f8533d 100644 --- a/src/agents/session-tool-result-guard.tool-result-persist-hook.test.ts +++ b/src/agents/session-tool-result-guard.tool-result-persist-hook.test.ts @@ -142,6 +142,38 @@ describe("tool_result_persist hook", () => { expect(toolResult.details.originalDetailsBytesAtLeast).toBeGreaterThan(8_192); }); + it("preserves result state values when capping oversized details", () => { + const sm = guardSessionManager(SessionManager.inMemory(), { + agentId: "main", + sessionKey: "main", + }); + const appendMessage = sm.appendMessage.bind(sm) as unknown as (message: AgentMessage) => void; + appendMessage({ + role: "assistant", + content: [{ type: "toolCall", id: "call_1", name: "lookup", arguments: {} }], + } as AgentMessage); + appendMessage({ + role: "toolResult", + toolCallId: "call_1", + isError: false, + content: [{ type: "text", text: "visible output stays small" }], + details: { + success: true, + disabled: false, + unavailable: false, + error: null, + payload: "x".repeat(10_000), + }, + } as any); + + const details = requirePersistedToolResult(sm).details; + expect(details.persistedDetailsTruncated).toBe(true); + expect(details.success).toBe(true); + expect(details.disabled).toBe(false); + expect(details.unavailable).toBe(false); + expect(details.error).toBeUndefined(); + }); + it("redacts small toolResult details before persistence", () => { const tokenValue = "abcdefghijklmnopqrstuvwx1234567890"; const bearerValue = "bearerdiagnosticvalue1234567890"; @@ -607,6 +639,8 @@ describe("tool_result_persist hook", () => { details: { status: "completed".repeat(250), sessionId: "exec-oversized", + success: false, + error: "upstream unavailable", cwd: "/tmp/very-long-working-directory".repeat(250), name: "noisy process".repeat(250), fullOutputPath: "/tmp/output.log".repeat(250), @@ -631,6 +665,8 @@ describe("tool_result_persist hook", () => { expect(details.finalDetailsTruncated).toBe(true); expect(details.aggregated).toBeUndefined(); expect(details.tail).toBeUndefined(); + expect(details.success).toBe(false); + expect(details.error).toBe("upstream unavailable"); expect(Buffer.byteLength(JSON.stringify(details), "utf-8")).toBeLessThan(8_192); }); diff --git a/src/agents/session-tool-result-guard.ts b/src/agents/session-tool-result-guard.ts index b10fed4f2888..ad69300ced1b 100644 --- a/src/agents/session-tool-result-guard.ts +++ b/src/agents/session-tool-result-guard.ts @@ -306,6 +306,24 @@ function sanitizePersistedSessionDetail( return out; } +function copyPersistedResultStateFields( + out: Record, + src: Record, + maxStringChars: number, + redactionConfig?: ToolResultDetailRedactionConfig, +): void { + for (const key of ["disabled", "unavailable", "success"] as const) { + if (typeof src[key] === "boolean") { + out[key] = src[key]; + } + } + if (typeof src.error === "string" && src.error) { + out.error = redactPersistedDetailString(src.error, maxStringChars, redactionConfig); + } else if (src.error) { + out.error = true; + } +} + function buildPersistedDetailsFallback( src: Record | undefined, originalSize: BoundedJsonUtf8Bytes, @@ -336,6 +354,12 @@ function buildPersistedDetailsFallback( ); } } + copyPersistedResultStateFields( + fallback, + src, + MAX_PERSISTED_DETAIL_FALLBACK_STRING_CHARS, + redactionConfig, + ); } return fallback; } @@ -457,6 +481,7 @@ function sanitizeToolResultDetailsForPersistence( ); } } + copyPersistedResultStateFields(out, src, MAX_PERSISTED_DETAIL_STRING_CHARS, redactionConfig); if (typeof src.tail === "string") { out.tail = redactPersistedDetailString( src.tail, diff --git a/src/plugin-sdk/agent-harness-runtime.ts b/src/plugin-sdk/agent-harness-runtime.ts index eb18dfe01dce..8b1ba68edda5 100644 --- a/src/plugin-sdk/agent-harness-runtime.ts +++ b/src/plugin-sdk/agent-harness-runtime.ts @@ -131,6 +131,8 @@ export { isMessagingTool, isMessagingToolSendAction } from "../agents/embedded-a export { extractToolResultMediaArtifact, filterToolResultMediaUrls, + isToolResultError, + sanitizeToolResult, } from "../agents/embedded-agent-subscribe.tools.js"; export { normalizeUsage } from "../agents/usage.js"; export { resolveOpenClawAgentDir } from "./agent-dir-compat.js";