diff --git a/extensions/anthropic/session-catalog.test.ts b/extensions/anthropic/session-catalog.test.ts index fd27ec7fb1e5..990606636687 100644 --- a/extensions/anthropic/session-catalog.test.ts +++ b/extensions/anthropic/session-catalog.test.ts @@ -1407,6 +1407,45 @@ describe("Claude session catalog", () => { ]); }); + it("bounds how long a hung paired-node catalog can delay the caller", async () => { + vi.useFakeTimers(); + try { + const invoke = vi.fn( + async () => await new Promise(() => {}), + ); + const provider = captureCatalogProvider({ + nodes: { + list: vi.fn().mockResolvedValue({ + nodes: [ + { + nodeId: "slow-node", + displayName: "Slow node", + connected: true, + commands: [CLAUDE_SESSIONS_LIST_COMMAND], + }, + ], + }), + invoke, + }, + } as unknown as PluginRuntime); + const pending = provider.list({ hostIds: ["node:slow-node"] }); + + await vi.advanceTimersByTimeAsync(8_000); + + await expect(pending).resolves.toEqual([ + expect.objectContaining({ + hostId: "node:slow-node", + connected: true, + sessions: [], + error: expect.objectContaining({ code: "NODE_INVOKE_FAILED" }), + }), + ]); + expect(invoke).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 30_000 })); + } finally { + vi.useRealTimers(); + } + }); + it("starts paired-node discovery while the local catalog is still reading", async () => { const home = await createHome(); process.env.HOME = home; diff --git a/extensions/anthropic/session-catalog.ts b/extensions/anthropic/session-catalog.ts index f828b7f77b10..f7a05b35ddaa 100644 --- a/extensions/anthropic/session-catalog.ts +++ b/extensions/anthropic/session-catalog.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { resolveDefaultAgentId } from "openclaw/plugin-sdk/agent-runtime"; import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime"; +import { withTimeout } from "openclaw/plugin-sdk/security-runtime"; import type { SessionCatalogHost, SessionCatalogProvider, @@ -66,6 +67,9 @@ const MAX_TRANSCRIPT_SCAN_BYTES = 64 * 1024 * 1024; const MAX_TRANSCRIPT_PAGE_BYTES = 20 * 1024 * 1024; const NODE_INVOKE_TIMEOUT_MS = 30_000; +// Catalog refresh is fail-soft: one unhealthy machine must not hold the whole sidebar. +// The node invoke keeps running so cold native discovery can warm the next poll. +const NODE_CATALOG_LIST_RESPONSE_TIMEOUT_MS = 8_000; const CLAUDE_HISTORY_IMPORT_MAX_ITEMS = 200; const CLAUDE_HISTORY_IMPORT_MAX_BYTES = 512 * 1024; @@ -1061,17 +1065,21 @@ async function listClaudeSessionCatalog(params: { }); } try { - const raw = await params.runtime.nodes.invoke({ - nodeId: node.nodeId, - command: CLAUDE_SESSIONS_LIST_COMMAND, - params: { - limit: query.limitPerHost, - ...(query.search ? { searchTerm: query.search } : {}), - ...(query.cursors?.[hostId] ? { cursor: query.cursors[hostId] } : {}), - }, - timeoutMs: NODE_INVOKE_TIMEOUT_MS, - scopes: ["operator.write"], - }); + const raw = await withTimeout( + params.runtime.nodes.invoke({ + nodeId: node.nodeId, + command: CLAUDE_SESSIONS_LIST_COMMAND, + params: { + limit: query.limitPerHost, + ...(query.search ? { searchTerm: query.search } : {}), + ...(query.cursors?.[hostId] ? { cursor: query.cursors[hostId] } : {}), + }, + timeoutMs: NODE_INVOKE_TIMEOUT_MS, + scopes: ["operator.write"], + }), + NODE_CATALOG_LIST_RESPONSE_TIMEOUT_MS, + { message: "paired node Claude session catalog timed out" }, + ); return Object.assign(common, parseCatalogPage(unwrapNodePayload(raw))); } catch { return Object.assign(common, { diff --git a/extensions/codex/src/session-catalog-node-continue.ts b/extensions/codex/src/session-catalog-node-continue.ts index 0bd5f9fecdf7..ec8cd0383e3e 100644 --- a/extensions/codex/src/session-catalog-node-continue.ts +++ b/extensions/codex/src/session-catalog-node-continue.ts @@ -2,6 +2,7 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime"; import type { CodexThread } from "./app-server/protocol.js"; +import { withTimeout } from "./app-server/timeout.js"; import { createCodexCliNodeConversationBindingData } from "./conversation-binding-data.js"; import { CODEX_CLI_SESSION_RESUME_COMMAND } from "./node-cli-sessions.js"; import { @@ -44,6 +45,10 @@ const CODEX_NODE_CONTINUE_COMMANDS = [ CODEX_CLI_SESSION_RESUME_COMMAND, ] as const; +// Catalog refresh is fail-soft: one unhealthy machine must not hold the whole sidebar. +// The node invoke keeps running so cold native discovery can warm the next poll. +const NODE_CATALOG_LIST_RESPONSE_TIMEOUT_MS = 8_000; + export type CatalogNode = Awaited>["nodes"][number]; export function nodeLabel(node: CatalogNode): string { @@ -96,17 +101,21 @@ export async function listPairedNode(params: { }; } try { - const raw = await params.runtime.nodes.invoke({ - nodeId: params.node.nodeId, - command: CODEX_APP_SERVER_THREADS_LIST_COMMAND, - params: { - cursor: params.query.cursors?.[hostId], - limit: params.query.limitPerHost, - searchTerm: params.query.search, - }, - timeoutMs: NODE_INVOKE_TIMEOUT_MS, - scopes: ["operator.write"], - }); + const raw = await withTimeout( + params.runtime.nodes.invoke({ + nodeId: params.node.nodeId, + command: CODEX_APP_SERVER_THREADS_LIST_COMMAND, + params: { + cursor: params.query.cursors?.[hostId], + limit: params.query.limitPerHost, + searchTerm: params.query.search, + }, + timeoutMs: NODE_INVOKE_TIMEOUT_MS, + scopes: ["operator.write"], + }), + NODE_CATALOG_LIST_RESPONSE_TIMEOUT_MS, + "paired node Codex session catalog timed out", + ); const page = filterCatalogPageByTitle( parseCatalogPage(unwrapNodeInvokePayload(raw)), params.query.search, diff --git a/extensions/codex/src/session-catalog.test.ts b/extensions/codex/src/session-catalog.test.ts index 5182d5b79400..f0ba04e32c91 100644 --- a/extensions/codex/src/session-catalog.test.ts +++ b/extensions/codex/src/session-catalog.test.ts @@ -17,6 +17,7 @@ import { type CodexAppServerBindingStore, type CodexAppServerThreadBinding, } from "./app-server/session-binding.test-helpers.js"; +import { listPairedNode } from "./session-catalog-node-continue.js"; import { catalogError } from "./session-catalog-parsing.js"; import { CODEX_TERMINAL_RESUME_COMMAND } from "./session-catalog-terminal.js"; import { @@ -995,6 +996,38 @@ describe("Codex supervision catalog", () => { expect(JSON.stringify(result)).not.toContain("private transcript"); }); + it("bounds how long a hung paired-node catalog can delay the caller", async () => { + vi.useFakeTimers(); + try { + const invoke = vi.fn( + async () => await new Promise(() => {}), + ); + const pending = listPairedNode({ + runtime: { nodes: { invoke } } as unknown as PluginRuntime, + node: { + nodeId: "slow-node", + displayName: "Slow node", + connected: true, + commands: [CODEX_APP_SERVER_THREADS_LIST_COMMAND], + }, + query: { limitPerHost: 40 }, + adoptedSessions: new Map(), + }); + + await vi.advanceTimersByTimeAsync(8_000); + + await expect(pending).resolves.toMatchObject({ + hostId: "node:slow-node", + connected: true, + sessions: [], + error: { code: "NODE_INVOKE_FAILED" }, + }); + expect(invoke).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 65_000 })); + } finally { + vi.useRealTimers(); + } + }); + it("serves one bounded transcript page from the node host command", async () => { const listTurnPage = vi.fn(async () => ({ data: [