From 41909b4f3fc730171066b9bde9b988beb4431d28 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Mon, 3 Aug 2026 02:13:27 -0700 Subject: [PATCH 01/57] fix(ci): resolve external hydrate dependencies --- scripts/lib/package-dist-imports.mjs | 4 +- .../postinstall-bundled-plugins.test.ts | 44 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/scripts/lib/package-dist-imports.mjs b/scripts/lib/package-dist-imports.mjs index 80ffe22feb19..7183b12bda31 100644 --- a/scripts/lib/package-dist-imports.mjs +++ b/scripts/lib/package-dist-imports.mjs @@ -1,8 +1,10 @@ // Scans packaged dist JavaScript for relative imports and missing closure entries. +import { createRequire } from "node:module"; import path from "node:path"; -import ts from "typescript"; import { visitModuleSpecifiers } from "./guard-inventory-utils.mjs"; +const require = createRequire(import.meta.url); +const ts = require("typescript"); const JS_DIST_FILE_RE = /^dist\/.*\.(?:cjs|js|mjs)$/u; function normalizePackagePath(value) { diff --git a/test/scripts/postinstall-bundled-plugins.test.ts b/test/scripts/postinstall-bundled-plugins.test.ts index d3881c54fb82..fe8274e80888 100644 --- a/test/scripts/postinstall-bundled-plugins.test.ts +++ b/test/scripts/postinstall-bundled-plugins.test.ts @@ -68,6 +68,50 @@ async function writeBaileysMediaFile(packageRoot: string, text: string) { } describe("bundled plugin postinstall", () => { + it("resolves TypeScript from NODE_PATH during external modules-dir installs", async () => { + const packageRoot = await createTempDirAsync("openclaw-postinstall-node-path-"); + const scriptRoot = path.join(packageRoot, "scripts"); + const externalModulesDir = path.join(packageRoot, "external-node-modules"); + await fs.mkdir(path.join(scriptRoot, "lib"), { recursive: true }); + await fs.mkdir(externalModulesDir, { recursive: true }); + await fs.writeFile( + path.join(packageRoot, "package.json"), + '{"name":"openclaw","type":"module","version":"2026.7.2"}\n', + ); + for (const relativePath of [ + "scripts/postinstall-bundled-plugins.mjs", + "scripts/lib/package-dist-imports.mjs", + "scripts/lib/guard-inventory-utils.mjs", + ]) { + await fs.copyFile( + fileURLToPath(new URL(`../../${relativePath}`, import.meta.url)), + path.join(packageRoot, relativePath), + ); + } + await fs.symlink( + fileURLToPath(new URL("../../node_modules/typescript", import.meta.url)), + path.join(externalModulesDir, "typescript"), + process.platform === "win32" ? "junction" : "dir", + ); + + const result = spawnSync( + process.execPath, + [path.join(scriptRoot, "postinstall-bundled-plugins.mjs")], + { + cwd: packageRoot, + encoding: "utf8", + env: { + ...process.env, + NODE_PATH: [externalModulesDir, process.env.NODE_PATH] + .filter(Boolean) + .join(path.delimiter), + }, + }, + ); + + expect(result.status, result.stderr).toBe(0); + }); + it("recognizes direct invocation through symlinked temp prefixes", () => { const realpathSync = vi.fn((value: string) => value.replace(/^\/var\/folders\//u, "/private/var/folders/"), From e97f484ef92e5a73f50842275a42d595b02497c7 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 02:13:42 -0700 Subject: [PATCH 02/57] fix(discord): report pending message search indexes (#118573) Co-authored-by: Peter Steinberger --- extensions/discord/src/send.messages.test.ts | 32 ++++++++++++++++++++ extensions/discord/src/send.messages.ts | 17 ++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/extensions/discord/src/send.messages.test.ts b/extensions/discord/src/send.messages.test.ts index 2dfaa119b6b2..d5fb287ec012 100644 --- a/extensions/discord/src/send.messages.test.ts +++ b/extensions/discord/src/send.messages.test.ts @@ -66,6 +66,38 @@ describe("searchMessagesDiscord", () => { expect(result).toEqual(results); }); + it("preserves valid empty Discord search results", async () => { + const results = { messages: [], total_results: 0 }; + restMock.get.mockResolvedValueOnce(results); + + await expect( + searchMessagesDiscord({ guildId: "G1", content: "test" }, { cfg: {} as never }), + ).resolves.toEqual(results); + }); + + it("surfaces a pending Discord search index and its retry delay", async () => { + restMock.get.mockResolvedValueOnce({ + message: "Index not yet available. Try again later", + code: 110000, + documents_indexed: 0, + retry_after: 2, + }); + + await expect( + searchMessagesDiscord({ guildId: "G1", content: "test" }, { cfg: {} as never }), + ).rejects.toThrow( + "Discord message search unavailable: Index not yet available. Try again later (retry after 2s)", + ); + }); + + it("rejects object search responses without a messages array", async () => { + restMock.get.mockResolvedValueOnce({ total_results: 1 }); + + await expect( + searchMessagesDiscord({ guildId: "G1", content: "test" }, { cfg: {} as never }), + ).rejects.toThrow("Unexpected Discord response for message search: expected messages array."); + }); + it("throws a clear error when Discord returns a non-object search response", async () => { restMock.get.mockResolvedValueOnce("\u001f\ufffd\u0008raw gzip bytes"); diff --git a/extensions/discord/src/send.messages.ts b/extensions/discord/src/send.messages.ts index 9148c7087ccb..63699b2708f7 100644 --- a/extensions/discord/src/send.messages.ts +++ b/extensions/discord/src/send.messages.ts @@ -17,6 +17,7 @@ import { searchGuildMessages, unpinChannelMessage, } from "./internal/discord.js"; +import { parseDiscordRetryAfterBodySeconds } from "./retry-after.js"; import { resolveDiscordRest } from "./send.shared.js"; import type { DiscordMessageEdit, @@ -248,8 +249,22 @@ export async function searchMessagesDiscord(query: DiscordSearchQuery, opts: Dis const limit = Math.min(Math.max(Math.floor(query.limit), 1), 25); params.set("limit", String(limit)); } - return assertDiscordResponseObject( + const result = assertDiscordResponseObject( await searchGuildMessages(rest, query.guildId, params), "message search", ); + // Discord returns HTTP 202 with code 110000 while the guild search index is warming. + if (result.code === 110000) { + const message = + typeof result.message === "string" && result.message.trim() + ? result.message.trim() + : "Discord search index is not yet available"; + const retryAfter = parseDiscordRetryAfterBodySeconds(result.retry_after); + const retryHint = retryAfter === undefined ? "" : ` (retry after ${retryAfter}s)`; + throw new Error(`Discord message search unavailable: ${message}${retryHint}`); + } + if (!Array.isArray(result.messages)) { + throw new Error("Unexpected Discord response for message search: expected messages array."); + } + return result; } From d2046614f9fe3a5ff25759f1d478937a68ff1047 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Mon, 3 Aug 2026 17:22:29 +0800 Subject: [PATCH 03/57] fix(qa): restore Telegram release validation (#118542) --- .../openclaw-release-telegram-qa.yml | 9 +++- .../telegram/adapter.runtime.test.ts | 51 ++++++++++++++++++- .../telegram/adapter.runtime.ts | 20 ++++---- .../live-transports/telegram/profiles.test.ts | 4 +- .../telegram/telegram-api.runtime.test.ts | 23 +++++++++ .../telegram/telegram-api.runtime.ts | 3 +- .../qa-lab/src/profile-selection.test.ts | 2 +- .../src/scenario-catalog-channels.test.ts | 30 +++++------ .../channels/channel-message-flows.yaml | 3 -- .../native-command-session-target.yaml | 12 ++++- ...am-assistant-transcript-role-boundary.yaml | 7 +-- ...nclaw-release-telegram-qa-workflow.test.ts | 15 ++++++ 12 files changed, 137 insertions(+), 42 deletions(-) diff --git a/.github/workflows/openclaw-release-telegram-qa.yml b/.github/workflows/openclaw-release-telegram-qa.yml index 9863d6834de8..98372e372156 100644 --- a/.github/workflows/openclaw-release-telegram-qa.yml +++ b/.github/workflows/openclaw-release-telegram-qa.yml @@ -1819,8 +1819,15 @@ jobs: sut_tmp="${temp_root}/sut-tmp" install -d -o "$SUT_UID" -g "$SUT_GID" -m 0700 "$sut_tmp" export TMPDIR="$sut_tmp" + workspace="${temp_root}/workspace" + [[ -d "$workspace" && ! -L "$workspace" ]] + [[ "$(realpath -e "$workspace")" == "$workspace" ]] + # The trusted scenario host creates fixtures while the isolated SUT reads + # and mutates the workspace. Keep every other runtime directory SUT-private. + chown -R "$RUNNER_UID:$SUT_GID" "$workspace" + chmod -R u=rwX,g=rwX,o= "$workspace" + find "$workspace" -type d -exec chmod g+s {} + for path in \ - "$temp_root/workspace" \ "${OPENCLAW_HOME:?}" \ "${OPENCLAW_STATE_DIR:?}" \ "${XDG_CACHE_HOME:?}" \ diff --git a/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.test.ts b/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.test.ts index 90cb0f27a580..a1f224f938ef 100644 --- a/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.test.ts @@ -96,10 +96,29 @@ describe("Telegram QA transport adapter", () => { const addOutboundMessage = vi.fn().mockResolvedValue({ id: "out-1" }); const editMessage = vi.fn().mockResolvedValue({ id: "out-1" }); const adapter = await createTelegramQaTransportAdapter({ - adapterOptions: { sutAccountId: "sut" }, + adapterOptions: { + sutAccountId: "sut", + transportPolicy: { requireGroupMention: true }, + }, messages: { addInboundMessage, addOutboundMessage, editMessage }, } as never); + expect(adapter.createGatewayConfig?.({ baseUrl: "http://127.0.0.1:1234" })).toMatchObject({ + channels: { + telegram: { + accounts: { + sut: { + groups: { + "-100123": { + requireMention: true, + }, + }, + }, + }, + }, + }, + }); + await vi.waitFor(() => expect(pollResolvers).toHaveLength(1)); await adapter.sendInbound?.({ conversation: { id: "logical-room", kind: "group" }, @@ -185,10 +204,38 @@ describe("Telegram QA transport adapter", () => { expect.objectContaining({ messageId: "out-1", text: "final", timestamp: 101_000 }), ); + await adapter.resetTransport?.(); await vi.waitFor(() => expect(pollResolvers).toHaveLength(3)); + await adapter.sendInbound?.({ + conversation: { id: "next-room", kind: "group" }, + senderId: "driver", + text: "next", + }); + pollResolvers[2]?.([ + { + update_id: 3, + edited_message: { + message_id: 13, + date: 102, + chat: { id: -100123 }, + from: { id: 2, is_bot: true, username: "openclaw_qa_bot" }, + text: "orphan final", + }, + }, + ]); + await vi.waitFor(() => expect(addOutboundMessage).toHaveBeenCalledTimes(2)); + expect(addOutboundMessage).toHaveBeenLastCalledWith( + expect.objectContaining({ + to: "group:next-room", + text: "orphan final", + timestamp: 102_000, + }), + ); + + await vi.waitFor(() => expect(pollResolvers).toHaveLength(4)); mocks.heartbeatStop.mockRejectedValueOnce(new Error("heartbeat stop failed")); const cleanup = adapter.cleanup?.(); - pollResolvers[2]?.([]); + pollResolvers[3]?.([]); await cleanup; expect(mocks.shouldRetainQaGatewayCredentialLease).not.toHaveBeenCalled(); await expect(adapter.cleanupAfterGatewayStop?.()).rejects.toThrow("heartbeat stop failed"); diff --git a/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts index 110f92f73782..60bd468c4da5 100644 --- a/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts +++ b/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts @@ -134,17 +134,17 @@ export async function createTelegramQaTransportAdapter( continue; } const existingMessageId = busMessageIds.get(message.messageId); - if (update.edited_message) { - if (existingMessageId) { - await context.messages.editMessage({ - accountId, - messageId: existingMessageId, - text: message.text, - timestamp: message.timestamp, - }); - } + if (update.edited_message && existingMessageId) { + await context.messages.editMessage({ + accountId, + messageId: existingMessageId, + text: message.text, + timestamp: message.timestamp, + }); continue; } + // Telegram may expose only the final edit after the adapter resets between + // scenarios. Adopt that edit so the live observation cannot disappear. const outbound = await context.messages.addOutboundMessage({ accountId, to: `${logicalConversationKind}:${logicalConversationId}`, @@ -223,6 +223,8 @@ export async function createTelegramQaTransportAdapter( sutToken: runtimeEnv.sutToken, driverBotId: driverIdentity.id, sutAccountId: accountId, + // Mention-gating scenarios opt in through the shared transport policy. + requireMention: options.transportPolicy?.requireGroupMention === true, }), waitReady: async ({ gateway, timeoutMs, pollIntervalMs }) => await waitForTelegramChannelRunning(gateway, accountId, { diff --git a/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts b/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts index 0f389a5076bd..22c7cc6b6993 100644 --- a/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts +++ b/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts @@ -25,7 +25,7 @@ describe("Telegram QA profiles", () => { expect(live).not.toContain("telegram-long-final-reuses-preview"); expect(mock).toContain("telegram-long-final-reuses-preview"); - expect(mock).toContain("telegram-assistant-transcript-role-boundary"); + expect(mock).not.toContain("telegram-assistant-transcript-role-boundary"); expect(mock).not.toContain("telegram-startup-getme-live"); }); @@ -36,7 +36,7 @@ describe("Telegram QA profiles", () => { }); expect(scenarioIds).toContain("channel-message-flows"); - expect(scenarioIds).toContain("native-command-session-target"); + expect(scenarioIds).not.toContain("native-command-session-target"); }); it("lets explicit scenarios override profile selection", () => { diff --git a/extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.test.ts b/extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.test.ts index d691b9d725d0..703b493f6050 100644 --- a/extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.test.ts @@ -76,6 +76,7 @@ describe("Telegram QA API boundary", () => { sutToken: "placeholder", driverBotId: 1, sutAccountId: "sut", + requireMention: true, }, ); @@ -100,6 +101,28 @@ describe("Telegram QA API boundary", () => { }); }); + it("disables mention gating only inside the exact leased QA group", () => { + const config = buildTelegramQaConfig( + {}, + { + groupId: "-100123", + sutToken: "placeholder", + driverBotId: 1, + sutAccountId: "sut", + requireMention: false, + }, + ); + + expect(config.channels?.telegram?.groups).toBeUndefined(); + expect(config.channels?.telegram?.accounts?.sut?.groups).toEqual({ + "-100123": { + groupPolicy: "allowlist", + allowFrom: ["1"], + requireMention: false, + }, + }); + }); + it("waits for the selected Telegram account to become connected", async () => { const call = vi .fn() diff --git a/extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.ts index c152523f7b94..642569465df5 100644 --- a/extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.ts +++ b/extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.ts @@ -257,6 +257,7 @@ export function buildTelegramQaConfig( sutToken: string; driverBotId: number; sutAccountId: string; + requireMention: boolean; }, ): OpenClawConfig { return { @@ -305,7 +306,7 @@ export function buildTelegramQaConfig( [params.groupId]: { groupPolicy: "allowlist", allowFrom: [String(params.driverBotId)], - requireMention: true, + requireMention: params.requireMention, }, }, }, diff --git a/extensions/qa-lab/src/profile-selection.test.ts b/extensions/qa-lab/src/profile-selection.test.ts index 4d997ea5566b..7d45e70842fc 100644 --- a/extensions/qa-lab/src/profile-selection.test.ts +++ b/extensions/qa-lab/src/profile-selection.test.ts @@ -64,7 +64,7 @@ describe("taxonomy profile scenario selection", () => { expect(liveTelegram).toContain("telegram-help-command"); expect(liveTelegram).not.toContain("telegram-assistant-transcript-role-boundary"); - expect(mockTelegram).toContain("telegram-assistant-transcript-role-boundary"); + expect(mockTelegram).not.toContain("telegram-assistant-transcript-role-boundary"); expect(mockTelegram).not.toContain("discord-canary"); }); diff --git a/extensions/qa-lab/src/scenario-catalog-channels.test.ts b/extensions/qa-lab/src/scenario-catalog-channels.test.ts index 880110234252..e457162cf2dd 100644 --- a/extensions/qa-lab/src/scenario-catalog-channels.test.ts +++ b/extensions/qa-lab/src/scenario-catalog-channels.test.ts @@ -25,17 +25,17 @@ describe("qa scenario catalog channel contracts", () => { const scenario = readQaScenarioById("native-command-session-target"); const config = readQaScenarioExecutionConfig("native-command-session-target") as | { + requiredChannelDriver?: string; requiredProviderMode?: string; - sessionKey?: string; } | undefined; expect(scenario.execution.channel).toBe("telegram"); expect(config?.requiredProviderMode).toBe("mock-openai"); - expect(config?.sessionKey).toBe("agent:main:telegram:direct:qa-native-operator"); - expect(JSON.stringify(requireFlowScenario(scenario).execution.flow)).toContain( - "session.key === config.sessionKey && session.hasActiveRun === true", - ); + expect(config?.requiredChannelDriver).toBe("crabline"); + const flow = JSON.stringify(requireFlowScenario(scenario).execution.flow); + expect(flow).toContain("transport.buildAgentDelivery"); + expect(flow).toContain("peer: { kind: 'group', id: delivery.replyTo }"); }); it("keeps channel-owned scenarios independent from the driver implementation", () => { @@ -151,21 +151,21 @@ describe("qa scenario catalog channel contracts", () => { expect(scenario.coverage?.primary).toEqual(["channels.streaming-final-reply"]); expect(scenario.coverage?.secondary).toEqual([`${agentRuntime}.streaming-replies-delivery`]); expect(scenario.gatewayConfigPatch).toMatchObject({ - channels: { - telegram: { - groups: { "*": { requireMention: false } }, - streaming: { mode: "partial" }, - }, - }, + channels: { telegram: { streaming: { mode: "partial" } } }, }); + expect(scenario.gatewayConfigPatch).not.toHaveProperty("channels.telegram.groups"); }); - it("disables Telegram mention gating for deterministic group delivery proofs", () => { + it("keeps transcript-role delivery on the Crabline driver", () => { const scenario = readQaScenarioById("telegram-assistant-transcript-role-boundary"); + const config = readQaScenarioExecutionConfig("telegram-assistant-transcript-role-boundary") as + | { + requiredChannelDriver?: string; + } + | undefined; - expect(scenario.gatewayConfigPatch).toMatchObject({ - channels: { telegram: { groups: { "*": { requireMention: false } } } }, - }); + expect(scenario.gatewayConfigPatch).toBeUndefined(); + expect(config?.requiredChannelDriver).toBe("crabline"); }); it("rejects malformed string matcher lists before running a flow", () => { diff --git a/qa/scenarios/channels/channel-message-flows.yaml b/qa/scenarios/channels/channel-message-flows.yaml index cafb29c6d864..c7fd1dfa0e07 100644 --- a/qa/scenarios/channels/channel-message-flows.yaml +++ b/qa/scenarios/channels/channel-message-flows.yaml @@ -12,9 +12,6 @@ scenario: gatewayConfigPatch: channels: telegram: - groups: - "*": - requireMention: false streaming: mode: partial successCriteria: diff --git a/qa/scenarios/channels/native-command-session-target.yaml b/qa/scenarios/channels/native-command-session-target.yaml index a70eecb76453..455323792b68 100644 --- a/qa/scenarios/channels/native-command-session-target.yaml +++ b/qa/scenarios/channels/native-command-session-target.yaml @@ -32,9 +32,9 @@ scenario: summary: Start a real delayed channel turn, abort it through native `/stop`, then prove the conversation is unblocked. config: requiredProviderMode: mock-openai + requiredChannelDriver: crabline conversationId: native-stop-target senderId: qa-native-operator - sessionKey: agent:main:telegram:direct:qa-native-operator delayedPrompt: "Subagent recovery worker native command target proof. Wait until stopped." abortReplyNeedle: Agent was aborted recoveryMarker: QA-NATIVE-STOP-RECOVERY-OK @@ -55,6 +55,14 @@ flow: - ref: env - 60000 - resetTransport: true + # Telegram maps the logical QA conversation onto the leased physical group. + # The adapter delivery target is therefore the canonical routing peer. + - set: delivery + value: + expr: "transport.buildAgentDelivery({ target: `dm:${config.conversationId}` })" + - set: sessionKey + value: + expr: "buildAgentSessionKey({ agentId: env.cfg.agents?.list?.find((agent) => agent.default)?.id ?? env.cfg.agents?.list?.[0]?.id ?? 'qa', channel: delivery.channel, accountId: transport.accountId, peer: { kind: 'group', id: delivery.replyTo } })" - sendInbound: conversation: id: @@ -70,7 +78,7 @@ flow: args: - lambda: async: true - expr: "env.gateway.call('sessions.list', {}).then((result) => result.sessions?.find((session) => session.key === config.sessionKey && session.hasActiveRun === true))" + expr: "env.gateway.call('sessions.list', {}).then((result) => result.sessions?.find((session) => session.key === sessionKey && session.hasActiveRun === true))" - expr: liveTurnTimeoutMs(env, 30000) - 100 - set: startIndex diff --git a/qa/scenarios/channels/telegram-assistant-transcript-role-boundary.yaml b/qa/scenarios/channels/telegram-assistant-transcript-role-boundary.yaml index 742ea1e9564c..bcdca400b0fe 100644 --- a/qa/scenarios/channels/telegram-assistant-transcript-role-boundary.yaml +++ b/qa/scenarios/channels/telegram-assistant-transcript-role-boundary.yaml @@ -7,12 +7,6 @@ scenario: primary: - channels.automatic-final-reply objective: Verify Telegram renders transcript-role-looking assistant text as inert authorship-marked content. - gatewayConfigPatch: - channels: - telegram: - groups: - "*": - requireMention: false successCriteria: - The controlled model reply reaches the real Telegram plugin through Crabline. - Telegram HTML wraps only the transcript-role header in a code element. @@ -31,6 +25,7 @@ scenario: summary: Deliver a controlled role-looking reply through Telegram and inspect its API payload. config: requiredProviderMode: mock-openai + requiredChannelDriver: crabline conversationId: "-1001234567890" senderId: "100001" header: user[Thu 2026-07-02 18:14 EDT] diff --git a/test/scripts/openclaw-release-telegram-qa-workflow.test.ts b/test/scripts/openclaw-release-telegram-qa-workflow.test.ts index 3922b912d6fc..ccc5bb98db9e 100644 --- a/test/scripts/openclaw-release-telegram-qa-workflow.test.ts +++ b/test/scripts/openclaw-release-telegram-qa-workflow.test.ts @@ -454,4 +454,19 @@ describe("release Telegram QA workflow", () => { .status, ).not.toBe(0); }); + + it("shares only the isolated workspace with the trusted scenario host", () => { + const createSut = requireRun( + "run_telegram", + "Create isolated Telegram SUT identity and launcher", + ); + + expect(createSut).toContain('workspace="${temp_root}/workspace"'); + expect(createSut).toContain('chown -R "$RUNNER_UID:$SUT_GID" "$workspace"'); + expect(createSut).toContain('chmod -R u=rwX,g=rwX,o= "$workspace"'); + expect(createSut).toContain('find "$workspace" -type d -exec chmod g+s {} +'); + expect(createSut).not.toContain( + 'for path in \\\n "$temp_root/workspace" \\\n "${OPENCLAW_HOME:?}"', + ); + }); }); From ecc49b5a870b8c35108c656ef619dc528abb149e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 02:25:48 -0700 Subject: [PATCH 04/57] refactor(tts): absorb speech core package (#118513) * refactor(tts): absorb speech core package * fix(tts): preserve runtime SDK exports * refactor(tts): remove private package exports * test(tts): align canonical runtime mocks * test(tts): complete canonical settings mocks * chore(plugin-sdk): regenerate API baseline for #118513 --- config/knip.config.ts | 5 - config/max-lines-baseline.txt | 1 - .../.generated/plugin-sdk-api-baseline.sha256 | 6 +- .../messaging/trusted-media-path.test.ts | 2 +- .../speech-core-runtime-api.d.ts | 34 - extensions/xai/tsconfig.json | 3 - packages/speech-core/package.json | 34 - packages/speech-core/runtime-api.ts | 49 - packages/speech-core/src/tts-settings.ts | 401 ---- packages/speech-core/src/tts.test.ts | 1877 ----------------- packages/speech-core/tsconfig.json | 16 - pnpm-lock.yaml | 6 - qa/scenarios/media/webchat-auto-tts.yaml | 2 +- scripts/lib/extension-package-boundary.ts | 1 - scripts/lib/tsdown-output-roots.mjs | 1 - src/agents/cli-runner.reliability.test.ts | 2 + .../cli-runner/helpers.system-prompt.test.ts | 2 + src/agents/cli-runner/prepare.test.ts | 2 + .../attempt.spawn-workspace.test-support.ts | 2 + .../system-prompt.test.ts | 2 + src/agents/system-prompt-config.test.ts | 2 + .../reply/commands-system-prompt.test.ts | 2 + src/channels/plugins/types.core.ts | 2 +- src/gateway/server-methods/talk-shared.ts | 12 +- src/gateway/server-methods/talk.ts | 15 +- src/gateway/server.talk-runtime.test.ts | 2 +- src/infra/outbound/reply-payload-parts.ts | 71 + src/plugin-sdk/facade-runtime.test.ts | 2 +- src/plugin-sdk/reply-payload.ts | 82 +- src/plugin-sdk/tts-runtime.ts | 21 +- src/plugins/capability-provider-runtime.ts | 2 +- src/plugins/model-catalog-registration.ts | 2 +- src/plugins/runtime/runtime-tts-request.ts | 4 +- .../src/tts.ts => src/tts/runtime-api.ts | 54 +- .../src => src/tts}/runtime-availability.ts | 2 +- {packages/speech-core => src/tts}/speaker.ts | 2 +- .../src => src/tts}/speech-text.test.ts | 2 +- .../src => src/tts}/speech-text.ts | 2 +- src/tts/tts-audio-store.ts | 4 +- .../src => src/tts}/tts-payload.ts | 26 +- .../tts}/tts-provider-resolution.ts | 40 +- .../src => src/tts}/tts-request.ts | 21 +- src/tts/tts-runtime-fallbacks.test.ts | 462 ++++ src/tts/tts-runtime-models.test.ts | 341 +++ src/tts/tts-runtime-personas.test.ts | 496 +++++ src/tts/tts-runtime-routing.test.ts | 447 ++++ .../tts/tts-runtime-types.ts | 0 src/tts/tts-runtime.test-support.ts | 297 +++ .../src => src/tts}/tts-settings-writes.ts | 6 +- src/tts/tts-settings.ts | 402 +++- .../src => src/tts}/tts-streaming.ts | 6 +- .../src => src/tts}/tts-synthesis-support.ts | 24 +- .../src => src/tts}/tts-synthesis.ts | 12 +- .../src => src/tts}/tts-telephony.ts | 6 +- src/tts/tts.test.ts | 31 +- src/tts/tts.ts | 21 +- .../speech-core => src/tts}/voice-models.ts | 4 +- test/e2e/qa-lab/runtime/media-talk-gateway.ts | 2 +- test/vitest-scoped-config.test.ts | 4 +- tsconfig.json | 5 - tsdown.config.ts | 18 - 61 files changed, 2693 insertions(+), 2711 deletions(-) delete mode 100644 extensions/xai/.boundary-stubs/speech-core-runtime-api.d.ts delete mode 100644 packages/speech-core/package.json delete mode 100644 packages/speech-core/runtime-api.ts delete mode 100644 packages/speech-core/src/tts-settings.ts delete mode 100644 packages/speech-core/src/tts.test.ts delete mode 100644 packages/speech-core/tsconfig.json create mode 100644 src/infra/outbound/reply-payload-parts.ts rename packages/speech-core/src/tts.ts => src/tts/runtime-api.ts (57%) rename {packages/speech-core/src => src/tts}/runtime-availability.ts (89%) rename {packages/speech-core => src/tts}/speaker.ts (96%) rename {packages/speech-core/src => src/tts}/speech-text.test.ts (98%) rename {packages/speech-core/src => src/tts}/speech-text.ts (98%) rename {packages/speech-core/src => src/tts}/tts-payload.ts (92%) rename {packages/speech-core/src => src/tts}/tts-provider-resolution.ts (96%) rename {packages/speech-core/src => src/tts}/tts-request.ts (87%) create mode 100644 src/tts/tts-runtime-fallbacks.test.ts create mode 100644 src/tts/tts-runtime-models.test.ts create mode 100644 src/tts/tts-runtime-personas.test.ts create mode 100644 src/tts/tts-runtime-routing.test.ts rename packages/speech-core/src/tts-types.ts => src/tts/tts-runtime-types.ts (100%) create mode 100644 src/tts/tts-runtime.test-support.ts rename {packages/speech-core/src => src/tts}/tts-settings-writes.ts (87%) rename {packages/speech-core/src => src/tts}/tts-streaming.ts (94%) rename {packages/speech-core/src => src/tts}/tts-synthesis-support.ts (95%) rename {packages/speech-core/src => src/tts}/tts-synthesis.ts (94%) rename {packages/speech-core/src => src/tts}/tts-telephony.ts (89%) rename {packages/speech-core => src/tts}/voice-models.ts (98%) diff --git a/config/knip.config.ts b/config/knip.config.ts index 17fdce530266..865c7a969a74 100644 --- a/config/knip.config.ts +++ b/config/knip.config.ts @@ -620,11 +620,6 @@ const config = { entry: ["src/*.ts!", "src/host/embeddings-worker-child.ts!"], project: ["src/**/*.ts!"], }, - "packages/speech-core": { - entry: ["runtime-api.ts!", "speaker.ts!", "voice-models.ts!"], - project: ["**/*.ts!"], - ignoreDependencies: ["openclaw"], - }, "packages/*": { entry: ["index.js!", "scripts/postinstall.js!"], project: ["index.js!", "scripts/**/*.js!"], diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index 1d197eaaf706..e40c65b6a56b 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -320,7 +320,6 @@ packages/markdown-core/src/ir.ts packages/memory-host-sdk/src/host/session-files.ts packages/sdk/src/client.ts packages/sdk/src/index.test.ts -packages/speech-core/src/tts.test.ts packages/tool-call-repair/src/stream-normalizer.test.ts packages/tool-call-repair/src/stream-normalizer.ts src/acp/control-plane/manager.test.ts diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index bca39b21bf78..fc1b08e20761 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -6,7 +6,7 @@ e5e67ddf3cab38fcbf9220bc3160715897e2709d9a9ff6ff36f1ecc9453c2367 module/agent-c 74daa746deb548379d3f0d6eac3c4d082df1034c4360cc03bf51fee0f10a2e4d module/agent-harness 95a907e1c33305b9473be64cc8d723e1a12b909eda86b15b94cee91879fb6a89 module/agent-harness-runtime 5168648cd946abad8a92822889f13ceacc87ed502314a66190d0b1eb8ebe76ea module/agent-media-payload -fd54eb654443d646d6430d2be99d1f25c32701f1c45aaa870ffe74aefc7d7f00 module/agent-runtime +6ee8bb70cd7b8a5a976ee84cd0c6e632dbfa5e4a047f616cdc00fa5b27879b27 module/agent-runtime 56b6d5fb6af3d95af1200065aca2e7d4f59e5fa59740505fe6ff433077ef6646 module/allow-from 55cea5390d68839ca7768b4a0cc570b17b65fa0fa3bc4d76130ef0f16cb79ede module/allowlist-config-edit 7ddd81bd5f55de9adf64bf4d92d012f24b37b6da0a72805a3a220d8feff24ca3 module/approval-auth-runtime @@ -29,7 +29,7 @@ c0f910ebfa3dbf283145fb1e3b9c016d03e853ef13e70402b09f3b9d9c2f4ab0 module/channel 9a5aaf650f9242523bb57bdc2556c323ab64e55aa11e25e2685b73b23ee12534 module/channel-dm-policy ba41c40956d6b4565605fa38c2d12f4b8471a0f8afe842798716b9032ee4d74a module/channel-entry-contract 982f29a18e07228e3da82cae67d06ff38249592a29c2fd28f01f0d2016ff80d9 module/channel-feedback -d645d24bcb7a5f68cc46c692ad0d1fbd19be0a99996f31e9479ce9cffce301c1 module/channel-inbound +1d13edd07a8ae8e21ece6c542cb0cde22e5b6e11d3bb9b2bfaa5ebd2a5239bf8 module/channel-inbound 76bb7f531f3702c801e8fe7479e9e499f601fb361a4303afdcb45fc0da440e4b module/channel-inbound-debounce df567ce2f4a4ba8a0937f825c46e83763412a724e36b72ce727dc4203c7dd134 module/channel-ingress-runtime 0e6efb79730fae59bb549ad00d9af2848c139b4bb1762e83b66886284e1c1421 module/channel-lifecycle @@ -107,7 +107,7 @@ aa2a56b4448c8ebdec9d06aac95d809995f533093d42fa32cd75e1d852967245 module/questio 2e09c3181e79e157ed5366b144d116ef8cc06023256ace3fa59b35c43cab513a module/reply-chunking 7994045066b29af1fc6b36ae32068f2a6f277195971af84701cb739cc23d0579 module/reply-dispatch-runtime ac2b199e95c5c8b1e2a65e62bd41d1b6322e531bca294ef4979a297a12640bce module/reply-history -f394fe4d5a7ed9e4d574063ae44e8d6af85c9a0e7d8b329f750ca16b0664325f module/reply-payload +ad69a4a6970cfac86f9379efb927beac898217946d025c3aea55e6f399da08bb module/reply-payload 1f899eb54013f268d6698ce8e6943289ea0e872747e8e868259106829518db86 module/reply-runtime b4043b356372f6af64ee3c26e4d6a6d623b817e4d95346358dc0e64a3b61d1e0 module/root-walk 97fc4ed1ac6e62b7af95b4352cee3893252d8b691fef71a000c2602b3128541a module/routing diff --git a/extensions/qqbot/src/engine/messaging/trusted-media-path.test.ts b/extensions/qqbot/src/engine/messaging/trusted-media-path.test.ts index 26a491c183c3..d3e9e5b95123 100644 --- a/extensions/qqbot/src/engine/messaging/trusted-media-path.test.ts +++ b/extensions/qqbot/src/engine/messaging/trusted-media-path.test.ts @@ -20,7 +20,7 @@ afterEach(() => { }); function makeTtsStyleVoiceFile(): string { - // Mirrors cron auto-TTS: speech-core writes the voice file under the preferred + // Mirrors cron auto-TTS: the TTS runtime writes the voice file under the preferred // OpenClaw temp root, which is outside the QQ Bot media storage tree. const tmpRoot = resolvePreferredOpenClawTmpDir(); const ttsDir = makeTrackedDir(tmpRoot, "tts-"); diff --git a/extensions/xai/.boundary-stubs/speech-core-runtime-api.d.ts b/extensions/xai/.boundary-stubs/speech-core-runtime-api.d.ts deleted file mode 100644 index 1845352dfd8a..000000000000 --- a/extensions/xai/.boundary-stubs/speech-core-runtime-api.d.ts +++ /dev/null @@ -1,34 +0,0 @@ -// Xai type declarations define plugin contracts. -export type ResolvedTtsConfig = unknown; -export type ResolvedTtsModelOverrides = unknown; -export type TtsDirectiveOverrides = unknown; -export type TtsDirectiveParseResult = unknown; -export type TtsResult = unknown; -export type TtsSynthesisResult = unknown; -export type TtsTelephonyResult = unknown; - -export const testApi: unknown; -export { testApi as _test }; -export const buildTtsSystemPromptHint: (...args: unknown[]) => unknown; -export const getLastTtsAttempt: (...args: unknown[]) => unknown; -export const getResolvedSpeechProviderConfig: (...args: unknown[]) => unknown; -export const getTtsMaxLength: (...args: unknown[]) => unknown; -export const getTtsProvider: (...args: unknown[]) => unknown; -export const isSummarizationEnabled: (...args: unknown[]) => unknown; -export const isTtsEnabled: (...args: unknown[]) => unknown; -export const isTtsProviderConfigured: (...args: unknown[]) => unknown; -export const listSpeechVoices: (...args: unknown[]) => unknown; -export const maybeApplyTtsToPayload: (...args: unknown[]) => unknown; -export const resolveTtsAutoMode: (...args: unknown[]) => unknown; -export const resolveTtsConfig: (...args: unknown[]) => unknown; -export const resolveTtsPrefsPath: (...args: unknown[]) => unknown; -export const resolveTtsProviderOrder: (...args: unknown[]) => unknown; -export const setLastTtsAttempt: (...args: unknown[]) => unknown; -export const setSummarizationEnabled: (...args: unknown[]) => unknown; -export const setTtsAutoMode: (...args: unknown[]) => unknown; -export const setTtsEnabled: (...args: unknown[]) => unknown; -export const setTtsMaxLength: (...args: unknown[]) => unknown; -export const setTtsProvider: (...args: unknown[]) => unknown; -export const synthesizeSpeech: (...args: unknown[]) => unknown; -export const textToSpeech: (...args: unknown[]) => unknown; -export const textToSpeechTelephony: (...args: unknown[]) => unknown; diff --git a/extensions/xai/tsconfig.json b/extensions/xai/tsconfig.json index 994c2c99ade7..43db97d95e7f 100644 --- a/extensions/xai/tsconfig.json +++ b/extensions/xai/tsconfig.json @@ -848,9 +848,6 @@ ], "@openclaw/ollama/runtime-api.js": [ "./.boundary-stubs/ollama-runtime-api.d.ts" - ], - "@openclaw/speech-core/runtime-api.js": [ - "./.boundary-stubs/speech-core-runtime-api.d.ts" ] } } diff --git a/packages/speech-core/package.json b/packages/speech-core/package.json deleted file mode 100644 index 1fd094355af6..000000000000 --- a/packages/speech-core/package.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "name": "@openclaw/speech-core", - "version": "2026.5.31", - "private": true, - "description": "OpenClaw speech runtime package", - "type": "module", - "main": "./dist/runtime-api.mjs", - "types": "./dist/runtime-api.d.mts", - "exports": { - ".": { - "types": "./dist/runtime-api.d.mts", - "import": "./dist/runtime-api.mjs", - "default": "./dist/runtime-api.mjs" - }, - "./runtime-api": { - "types": "./dist/runtime-api.d.mts", - "import": "./dist/runtime-api.mjs", - "default": "./dist/runtime-api.mjs" - }, - "./speaker": { - "types": "./dist/speaker.d.mts", - "import": "./dist/speaker.mjs", - "default": "./dist/speaker.mjs" - }, - "./voice-models": { - "types": "./dist/voice-models.d.mts", - "import": "./dist/voice-models.mjs", - "default": "./dist/voice-models.mjs" - } - }, - "dependencies": { - "openclaw": "workspace:*" - } -} diff --git a/packages/speech-core/runtime-api.ts b/packages/speech-core/runtime-api.ts deleted file mode 100644 index b63c16349142..000000000000 --- a/packages/speech-core/runtime-api.ts +++ /dev/null @@ -1,49 +0,0 @@ -// Runtime speech API barrel for TTS preferences, synthesis, streaming, and test -// helpers used by speech-capable plugins. -export { setSpeechRuntimeAvailabilityGuard } from "./src/runtime-availability.js"; -export { - buildTtsSystemPromptHint, - getTtsMaxLength, - getTtsPersona, - isSummarizationEnabled, - isTtsEnabled, - listTtsPersonas, - resolveTtsAutoMode, - resolveTtsConfig, - resolveTtsPrefsPath, - setTtsMachinePrefsPathResolver, - type ResolvedTtsConfig, - type ResolvedTtsModelOverrides, -} from "./src/tts-settings.js"; -export { - setSummarizationEnabled, - setTtsAutoMode, - setTtsEnabled, - setTtsMaxLength, - setTtsPersona, - setTtsProvider, -} from "./src/tts-settings-writes.js"; -export { - getLastTtsAttempt, - getResolvedSpeechProviderConfig, - getTtsProvider, - isTtsProviderConfigured, - listSpeechVoices, - prepareTtsRequest, - resolveExplicitTtsOverrides, - resolveTtsProviderOrder, - setLastTtsAttempt, - synthesizeSpeech, - streamSpeech, - textToSpeechStream, - textToSpeechTelephony, - testApi as _test, - testApi, - type TtsDirectiveOverrides, - type TtsDirectiveParseResult, - type PreparedTtsRequest, - type TtsSynthesisResult, - type TtsSynthesisStreamResult, - type TtsStreamResult, - type TtsTelephonyResult, -} from "./src/tts.js"; diff --git a/packages/speech-core/src/tts-settings.ts b/packages/speech-core/src/tts-settings.ts deleted file mode 100644 index cfd56072a276..000000000000 --- a/packages/speech-core/src/tts-settings.ts +++ /dev/null @@ -1,401 +0,0 @@ -// Lightweight TTS settings resolution shared by agent prompts, status, and speech runtime. -import { existsSync, readFileSync } from "node:fs"; -import path from "node:path"; -import type { - OpenClawConfig, - ResolvedTtsPersona, - TtsAutoMode, - TtsConfig, - TtsModelOverrideConfig, - TtsProvider, -} from "openclaw/plugin-sdk/config-contracts"; -import { - getRuntimeConfigSnapshot, - getRuntimeConfigSourceSnapshot, - selectApplicableRuntimeConfig, -} from "openclaw/plugin-sdk/runtime-config-snapshot"; -import type { SpeechProviderConfig } from "openclaw/plugin-sdk/speech-core"; -import { - normalizeSpeechProviderId, - normalizeTtsAutoMode, - resolveEffectiveTtsConfig, - type ResolvedTtsConfig, - type ResolvedTtsModelOverrides, - type TtsConfigResolutionContext, -} from "openclaw/plugin-sdk/speech-settings"; -import { - normalizeOptionalLowercaseString, - normalizeOptionalString, -} from "openclaw/plugin-sdk/string-coerce-runtime"; -import { resolveConfigDir, resolveUserPath } from "openclaw/plugin-sdk/text-utility-runtime"; -import { withSpeakerSelectionCompat } from "../speaker.js"; - -export type { ResolvedTtsConfig, ResolvedTtsModelOverrides }; - -export const DEFAULT_TTS_TIMEOUT_MS = 30_000; -const DEFAULT_TTS_MAX_LENGTH = 1500; -const DEFAULT_TTS_SUMMARIZE = true; -const DEFAULT_MAX_TEXT_LENGTH = 4096; -let machinePrefsPathResolver: () => string | undefined = () => undefined; - -export function setTtsMachinePrefsPathResolver(resolver?: () => string | undefined): void { - machinePrefsPathResolver = resolver ?? (() => undefined); -} - -export type TtsUserPrefs = { - tts?: { - auto?: TtsAutoMode; - enabled?: boolean; - provider?: TtsProvider; - persona?: string | null; - maxLength?: number; - summarize?: boolean; - }; -}; - -function resolveConfiguredTtsAutoMode(raw: TtsConfig): TtsAutoMode { - return normalizeTtsAutoMode(raw.auto) ?? (raw.enabled ? "always" : "off"); -} - -export function normalizeConfiguredSpeechProviderId( - providerId: string | undefined, -): TtsProvider | undefined { - const normalized = normalizeSpeechProviderId(providerId); - if (!normalized) { - return undefined; - } - return normalized === "edge" ? "microsoft" : normalized; -} - -export function normalizeTtsPersonaId(personaId: string | null | undefined): string | undefined { - return normalizeOptionalLowercaseString(personaId ?? undefined); -} - -function resolveTtsPrefsPathValue(prefsPath: string | undefined): string { - // Scoped agent paths must win over the migrated machine-wide default. - if (prefsPath?.trim()) { - return resolveUserPath(prefsPath.trim()); - } - const envPath = process.env.OPENCLAW_TTS_PREFS?.trim(); - if (envPath) { - return resolveUserPath(envPath); - } - const machinePath = machinePrefsPathResolver()?.trim(); - if (machinePath) { - return resolveUserPath(machinePath); - } - return path.join(resolveConfigDir(process.env), "settings", "tts.json"); -} - -export function resolveModelOverridePolicy( - overrides: TtsModelOverrideConfig | undefined, -): ResolvedTtsModelOverrides { - const enabled = overrides?.enabled ?? true; - if (!enabled) { - return { - enabled: false, - allowText: false, - allowProvider: false, - allowVoice: false, - allowModelId: false, - allowVoiceSettings: false, - allowNormalization: false, - allowSeed: false, - }; - } - const allow = (value: boolean | undefined, defaultValue = true) => value ?? defaultValue; - return { - enabled: true, - allowText: allow(overrides?.allowText), - allowProvider: allow(overrides?.allowProvider, false), - allowVoice: allow(overrides?.allowVoice), - allowModelId: allow(overrides?.allowModelId), - allowVoiceSettings: allow(overrides?.allowVoiceSettings), - allowNormalization: allow(overrides?.allowNormalization), - allowSeed: allow(overrides?.allowSeed), - }; -} - -export function resolveTtsRuntimeConfig(cfg: OpenClawConfig): OpenClawConfig { - return ( - selectApplicableRuntimeConfig({ - inputConfig: cfg, - runtimeConfig: getRuntimeConfigSnapshot(), - runtimeSourceConfig: getRuntimeConfigSourceSnapshot(), - }) ?? cfg - ); -} - -export function asProviderConfig(value: unknown): SpeechProviderConfig { - return typeof value === "object" && value !== null && !Array.isArray(value) - ? withSpeakerSelectionCompat(value as SpeechProviderConfig) - : {}; -} - -export function asProviderConfigMap(value: unknown): Record { - return typeof value === "object" && value !== null && !Array.isArray(value) - ? (value as Record) - : {}; -} - -export function hasOwnProperty(value: object, key: string): boolean { - return Object.hasOwn(value, key); -} - -function normalizeProviderConfigMap( - value: unknown, -): Record | undefined { - const rawMap = asProviderConfigMap(value); - if (Object.keys(rawMap).length === 0) { - return undefined; - } - const next: Record = {}; - for (const [providerId, providerConfig] of Object.entries(rawMap)) { - const normalized = normalizeConfiguredSpeechProviderId(providerId) ?? providerId; - next[normalized] = asProviderConfig(providerConfig); - } - return next; -} - -function collectTtsPersonas(raw: TtsConfig): Record { - const rawPersonas = asProviderConfigMap(raw.personas); - const personas: Record = {}; - for (const [id, value] of Object.entries(rawPersonas)) { - const normalizedId = normalizeTtsPersonaId(id); - if (!normalizedId || typeof value !== "object" || value === null || Array.isArray(value)) { - continue; - } - const persona = value as Omit; - personas[normalizedId] = { - ...persona, - id: normalizedId, - provider: normalizeConfiguredSpeechProviderId(persona.provider) ?? persona.provider, - providers: normalizeProviderConfigMap(persona.providers), - }; - } - return personas; -} - -function collectDirectProviderConfigEntries(raw: TtsConfig): Record { - const entries: Record = {}; - const rawProviders = asProviderConfigMap(raw.providers); - for (const [providerId, value] of Object.entries(rawProviders)) { - const normalized = normalizeConfiguredSpeechProviderId(providerId) ?? providerId; - entries[normalized] = asProviderConfig(value); - } - const reservedKeys = new Set([ - "auto", - "enabled", - "maxTextLength", - "mode", - "modelOverrides", - "persona", - "personas", - "prefsPath", - "provider", - "providers", - "summaryModel", - "timeoutMs", - ]); - for (const [key, value] of Object.entries(raw as Record)) { - if (reservedKeys.has(key)) { - continue; - } - if (typeof value !== "object" || value === null || Array.isArray(value)) { - continue; - } - const normalized = normalizeConfiguredSpeechProviderId(key) ?? key; - entries[normalized] ??= asProviderConfig(value); - } - return entries; -} - -export function resolveTtsConfig( - cfgInput: OpenClawConfig, - contextOrAgentId?: string | TtsConfigResolutionContext, -): ResolvedTtsConfig { - const cfg = resolveTtsRuntimeConfig(cfgInput); - const raw: TtsConfig = resolveEffectiveTtsConfig(cfg, contextOrAgentId); - const providerSource = raw.provider ? "config" : "default"; - const timeoutMs = raw.timeoutMs ?? DEFAULT_TTS_TIMEOUT_MS; - const timeoutMsSource = raw.timeoutMs === undefined ? "default" : "config"; - return { - auto: resolveConfiguredTtsAutoMode(raw), - mode: raw.mode ?? "final", - provider: - normalizeConfiguredSpeechProviderId(raw.provider) ?? - (providerSource === "config" ? (normalizeOptionalLowercaseString(raw.provider) ?? "") : ""), - providerSource, - persona: normalizeTtsPersonaId(raw.persona), - personas: collectTtsPersonas(raw), - summaryModel: normalizeOptionalString(raw.summaryModel), - modelOverrides: resolveModelOverridePolicy(raw.modelOverrides), - providerConfigs: collectDirectProviderConfigEntries(raw), - prefsPath: (raw as TtsConfig & { prefsPath?: string }).prefsPath, - maxTextLength: raw.maxTextLength ?? DEFAULT_MAX_TEXT_LENGTH, - timeoutMs, - timeoutMsSource, - rawConfig: raw, - sourceConfig: cfg, - }; -} - -export function resolveTtsPrefsPath(config: ResolvedTtsConfig): string { - return resolveTtsPrefsPathValue(config.prefsPath); -} - -export function readTtsPrefs(prefsPath: string): TtsUserPrefs { - try { - if (!existsSync(prefsPath)) { - return {}; - } - const parsed: unknown = JSON.parse(readFileSync(prefsPath, "utf8")); - return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) - ? (parsed as TtsUserPrefs) - : {}; - } catch { - return {}; - } -} - -function resolveTtsAutoModeFromPrefs(prefs: TtsUserPrefs): TtsAutoMode | undefined { - const auto = normalizeTtsAutoMode(prefs.tts?.auto); - if (auto) { - return auto; - } - if (typeof prefs.tts?.enabled === "boolean") { - return prefs.tts.enabled ? "always" : "off"; - } - return undefined; -} - -export function resolveTtsAutoMode(params: { - config: ResolvedTtsConfig; - prefsPath: string; - sessionAuto?: string; -}): TtsAutoMode { - const sessionAuto = normalizeTtsAutoMode(params.sessionAuto); - if (sessionAuto) { - return sessionAuto; - } - return resolveTtsAutoModeFromPrefs(readTtsPrefs(params.prefsPath)) ?? params.config.auto; -} - -function resolveTtsPersonaIdFromPrefs( - config: ResolvedTtsConfig, - prefs: TtsUserPrefs, -): string | undefined { - if (prefs.tts && hasOwnProperty(prefs.tts, "persona")) { - return normalizeTtsPersonaId(prefs.tts.persona); - } - return normalizeTtsPersonaId(config.persona); -} - -export function resolveTtsPersonaFromPrefs( - config: ResolvedTtsConfig, - prefs: TtsUserPrefs, -): ResolvedTtsPersona | undefined { - const personaId = resolveTtsPersonaIdFromPrefs(config, prefs); - return personaId ? config.personas[personaId] : undefined; -} - -type ResolvedTtsSettingsSnapshot = { - autoMode: TtsAutoMode; - config: ResolvedTtsConfig; - maxLength: number; - persona?: ResolvedTtsPersona; - personaId?: string; - preferredProvider?: TtsProvider; - prefsPath: string; - summarize: boolean; -}; - -export function resolveTtsSettingsSnapshot(params: { - cfg: OpenClawConfig; - sessionAuto?: string; - agentId?: string; - channelId?: string; - accountId?: string; -}): ResolvedTtsSettingsSnapshot { - const config = resolveTtsConfig(params.cfg, { - agentId: params.agentId, - channelId: params.channelId, - accountId: params.accountId, - }); - const prefsPath = resolveTtsPrefsPath(config); - const prefs = readTtsPrefs(prefsPath); - const personaId = resolveTtsPersonaIdFromPrefs(config, prefs); - const persona = personaId ? config.personas[personaId] : undefined; - const preferredProvider = - normalizeConfiguredSpeechProviderId(prefs.tts?.provider) ?? - normalizeConfiguredSpeechProviderId(persona?.provider) ?? - (config.providerSource === "config" - ? (normalizeConfiguredSpeechProviderId(config.provider) ?? config.provider) - : undefined); - return { - autoMode: - normalizeTtsAutoMode(params.sessionAuto) ?? resolveTtsAutoModeFromPrefs(prefs) ?? config.auto, - config, - maxLength: prefs.tts?.maxLength ?? DEFAULT_TTS_MAX_LENGTH, - ...(persona ? { persona } : {}), - ...(personaId ? { personaId } : {}), - ...(preferredProvider ? { preferredProvider } : {}), - prefsPath, - summarize: prefs.tts?.summarize ?? DEFAULT_TTS_SUMMARIZE, - }; -} - -export function buildTtsSystemPromptHint( - cfg: OpenClawConfig, - agentId?: string, -): string | undefined { - const settings = resolveTtsSettingsSnapshot({ cfg, agentId }); - if (settings.autoMode === "off") { - return undefined; - } - const autoHint = - settings.autoMode === "inbound" - ? "Only use TTS when the user's last message includes audio/voice." - : settings.autoMode === "tagged" - ? "Only use TTS when you include [[tts:key=value]] directives or a [[tts:text]]...[[/tts:text]] block." - : undefined; - return [ - "Voice (TTS) is enabled.", - autoHint, - settings.persona - ? `Active TTS persona: ${settings.persona.label ?? settings.persona.id}${settings.persona.description ? ` - ${settings.persona.description}` : ""}.` - : undefined, - `Keep spoken text ≤${settings.maxLength} chars to avoid auto-summary (summary ${settings.summarize ? "on" : "off"}).`, - "If workspace context (especially MEMORY.md) tells you not to use [[tts:...]] or to use a local/non-tagged voice workflow, follow that workspace instruction instead.", - "Use [[tts:...]] and optional [[tts:text]]...[[/tts:text]] to control voice/expressiveness.", - ] - .filter(Boolean) - .join("\n"); -} - -export function isTtsEnabled( - config: ResolvedTtsConfig, - prefsPath: string, - sessionAuto?: string, -): boolean { - return resolveTtsAutoMode({ config, prefsPath, sessionAuto }) !== "off"; -} - -export function getTtsPersona( - config: ResolvedTtsConfig, - prefsPath: string, -): ResolvedTtsPersona | undefined { - return resolveTtsPersonaFromPrefs(config, readTtsPrefs(prefsPath)); -} - -export function listTtsPersonas(config: ResolvedTtsConfig): ResolvedTtsPersona[] { - return Object.values(config.personas).toSorted((left, right) => left.id.localeCompare(right.id)); -} - -export function getTtsMaxLength(prefsPath: string): number { - return readTtsPrefs(prefsPath).tts?.maxLength ?? DEFAULT_TTS_MAX_LENGTH; -} - -export function isSummarizationEnabled(prefsPath: string): boolean { - return readTtsPrefs(prefsPath).tts?.summarize ?? DEFAULT_TTS_SUMMARIZE; -} diff --git a/packages/speech-core/src/tts.test.ts b/packages/speech-core/src/tts.test.ts deleted file mode 100644 index fc9e6ded0438..000000000000 --- a/packages/speech-core/src/tts.test.ts +++ /dev/null @@ -1,1877 +0,0 @@ -// Speech Core tests cover tts behavior. -import crypto from "node:crypto"; -import { mkdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import type { OpenClawConfig, TtsConfig } from "openclaw/plugin-sdk/config-contracts"; -import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime"; -import type { ReplyPayload } from "openclaw/plugin-sdk/reply-payload"; -import { - clearRuntimeConfigSnapshot, - setRuntimeConfigSnapshot, -} from "openclaw/plugin-sdk/runtime-config-snapshot"; -import type { - SpeechListVoicesRequest, - SpeechProviderPlugin, - SpeechProviderPrepareSynthesisContext, - SpeechSynthesisRequest, - SpeechTelephonySynthesisRequest, -} from "openclaw/plugin-sdk/speech-core"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { CODE_HEAVY_SPOKEN_FALLBACK } from "./speech-text.js"; -import type { TtsAudioPersistence } from "./tts-synthesis.js"; - -type MockSpeechSynthesisResult = Awaited>; - -const synthesizeMock = vi.hoisted(() => - vi.fn( - async (request: SpeechSynthesisRequest): Promise => ({ - audioBuffer: Buffer.from("voice"), - fileExtension: ".ogg", - outputFormat: "ogg", - voiceCompatible: request.target === "voice-note", - }), - ), -); -const prepareSynthesisMock = vi.hoisted(() => - vi.fn(async (_ctx: SpeechProviderPrepareSynthesisContext) => undefined), -); - -const listSpeechProvidersMock = vi.hoisted(() => vi.fn()); -const getSpeechProviderMock = vi.hoisted(() => vi.fn()); -const transcodeAudioBufferMock = vi.hoisted(() => - // Default off: most tests rely on the synthesized buffer reaching the - // channel unchanged. Tests that exercise the pre-transcode branch override - // per-call via `transcodeAudioBufferMock.mockResolvedValueOnce(...)`. - // Typed as the helper's full return shape so per-call overrides aren't - // narrowed to the default's literal. - vi.fn< - () => Promise< - | { ok: true; buffer: Buffer } - | { - ok: false; - reason: - | "platform-unsupported" - | "invalid-extension" - | "noop-same-container" - | "no-recipe" - | "transcoder-failed"; - detail?: string; - } - > - >(async () => ({ ok: false, reason: "platform-unsupported" })), -); - -vi.mock("openclaw/plugin-sdk/media-runtime", () => ({ - transcodeAudioBuffer: transcodeAudioBufferMock, -})); - -vi.mock("openclaw/plugin-sdk/channel-targets", () => ({ - normalizeChannelId: (channel: string | undefined) => channel?.trim().toLowerCase() ?? null, - resolveChannelTtsVoiceDelivery: (channel: string | undefined) => { - const normalized = channel?.trim().toLowerCase(); - if (normalized === "voice-memo-chat") { - return { - synthesisTarget: "audio-file", - audioFileFormats: ["mp3", "caf", "audio/mpeg", "audio/x-caf"], - preferAudioFileFormat: "caf", - }; - } - if (normalized === "feishu" || normalized === "whatsapp") { - return { synthesisTarget: "voice-note", transcodesAudio: true }; - } - if (normalized === "discord" || normalized === "matrix" || normalized === "telegram") { - return { synthesisTarget: "voice-note" }; - } - return undefined; - }, -})); - -vi.mock("openclaw/plugin-sdk/speech-core", async () => { - const actual = await vi.importActual("openclaw/plugin-sdk/speech-core"); - const mockProvider: SpeechProviderPlugin = { - id: "mock", - label: "Mock", - autoSelectOrder: 1, - isConfigured: () => true, - prepareSynthesis: prepareSynthesisMock, - synthesize: synthesizeMock, - }; - listSpeechProvidersMock.mockImplementation(() => [mockProvider]); - getSpeechProviderMock.mockImplementation((providerId: string) => - providerId === "mock" ? mockProvider : null, - ); - return { - ...actual, - canonicalizeSpeechProviderId: (providerId: string | undefined) => - providerId?.trim().toLowerCase() || undefined, - normalizeSpeechProviderId: (providerId: string | undefined) => - providerId?.trim().toLowerCase() || undefined, - getSpeechProvider: getSpeechProviderMock, - listSpeechProviders: listSpeechProvidersMock, - scheduleCleanup: vi.fn(), - }; -}); - -const { - testApi, - buildTtsSystemPromptHint, - getTtsPersona, - getTtsProvider, - isTtsProviderConfigured, - listSpeechVoices, - prepareTtsRequest, - resolveTtsConfig, - resolveTtsPrefsPath, - setTtsMachinePrefsPathResolver, - setSummarizationEnabled, - setTtsMaxLength, - synthesizeSpeech, - textToSpeechStream, - textToSpeechTelephony, -} = await import("../runtime-api.js"); -const { maybeApplyTtsToPayload: maybeApplyTtsToPayloadCore } = await import("./tts-payload.js"); -const { textToSpeech: textToSpeechCore } = await import("./tts-synthesis.js"); - -const nativeVoiceNoteChannels = ["discord", "feishu", "matrix", "telegram", "whatsapp"] as const; - -function createMockSpeechProvider( - id = "mock", - options: Partial = {}, -): SpeechProviderPlugin { - return { - id, - label: id, - autoSelectOrder: id === "mock" ? 1 : 2, - isConfigured: () => true, - prepareSynthesis: prepareSynthesisMock, - synthesize: synthesizeMock, - ...options, - }; -} - -function installSpeechProviders(providers: SpeechProviderPlugin[]): void { - listSpeechProvidersMock.mockImplementation(() => providers); - getSpeechProviderMock.mockImplementation( - (providerId: string) => providers.find((provider) => provider.id === providerId) ?? null, - ); -} - -// macOS os.tmpdir() is a /var -> /private/var symlink and fs-safe rejects -// symlinked store roots; resolve the canonical dir before writing prefs. -const PREFS_TMP_DIR = realpathSync(os.tmpdir()); - -async function persistTestTtsAudio({ - audioBuffer, - fileExtension, -}: Parameters[0]): Promise { - const dir = path.join(PREFS_TMP_DIR, `openclaw-speech-core-media-${crypto.randomUUID()}`); - mkdirSync(dir, { recursive: true }); - const audioPath = path.join(dir, `voice---${crypto.randomUUID()}${fileExtension}`); - writeFileSync(audioPath, audioBuffer); - return audioPath; -} - -function textToSpeech(params: Parameters[0]) { - return textToSpeechCore(params, persistTestTtsAudio); -} - -function maybeApplyTtsToPayload(params: Parameters[0]) { - return maybeApplyTtsToPayloadCore(params, persistTestTtsAudio); -} - -function prefsPathFor(prefsName: string): string { - return path.join(PREFS_TMP_DIR, `${prefsName}.json`); -} - -function createTtsConfig(prefsName: string): OpenClawConfig { - setTtsMachinePrefsPathResolver(() => prefsPathFor(prefsName)); - return { - tts: { - enabled: true, - provider: "mock", - }, - }; -} - -function requireRecord(value: unknown, label: string): Record { - if (value === null || typeof value !== "object" || Array.isArray(value)) { - throw new Error(`expected ${label} to be a record`); - } - return value as Record; -} - -function requireFirstCallParam(calls: ReadonlyArray, label: string) { - const call = calls[0]; - if (!call) { - throw new Error(`expected ${label} call`); - } - return call[0]; -} - -function requireFirstSynthesisRequest(label: string): Record { - return requireRecord(requireFirstCallParam(synthesizeMock.mock.calls, label), label); -} - -function requireAttempt(attempts: unknown[] | undefined, index: number) { - if (!attempts) { - throw new Error("expected synthesis attempts"); - } - return requireRecord(attempts[index], `synthesis attempt ${index}`); -} - -async function expectTtsPayloadResult(params: { - channel: string; - prefsName: string; - text: string; - target: "voice-note" | "audio-file"; - audioAsVoice: true | undefined; - providerResult?: MockSpeechSynthesisResult; - mediaExtension?: string; - kind?: "tool" | "block" | "final"; -}) { - if (params.providerResult) { - synthesizeMock.mockResolvedValueOnce(params.providerResult); - } - const cfg = createTtsConfig(params.prefsName); - let mediaDir: string | undefined; - try { - const result = await maybeApplyTtsToPayload({ - payload: { text: params.text }, - cfg, - channel: params.channel, - kind: params.kind ?? "final", - }); - - expect(synthesizeMock).toHaveBeenCalled(); - const request = requireRecord( - synthesizeMock.mock.calls.at(-1)?.[0], - "latest synthesis request", - ); - expect(request.target).toBe(params.target); - expect(result.audioAsVoice).toBe(params.audioAsVoice); - expect(result.mediaUrl).toMatch( - new RegExp(`voice---[a-f0-9-]+\\.${params.mediaExtension ?? "ogg"}$`), - ); - expect(result.spokenText).toBe(params.text); - expect(result.ttsSupplement).toEqual({ spokenText: params.text }); - expect((result as { trustedLocalMedia?: boolean }).trustedLocalMedia).toBe(true); - - mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; - } finally { - if (mediaDir) { - rmSync(mediaDir, { recursive: true, force: true }); - } - } -} - -describe("speech-core native voice-note routing", () => { - afterEach(() => { - setTtsMachinePrefsPathResolver(); - clearRuntimeConfigSnapshot(); - delete (Object.prototype as Record).polluted; - synthesizeMock.mockClear(); - prepareSynthesisMock.mockClear(); - transcodeAudioBufferMock.mockClear(); - installSpeechProviders([createMockSpeechProvider()]); - }); - - it("prefers the environment preference path over migrated machine state", () => { - const previousEnvPath = process.env.OPENCLAW_TTS_PREFS; - const envPath = prefsPathFor("env-override"); - setTtsMachinePrefsPathResolver(() => prefsPathFor("machine-state")); - process.env.OPENCLAW_TTS_PREFS = envPath; - try { - expect(resolveTtsPrefsPath(resolveTtsConfig({}))).toBe(envPath); - } finally { - if (previousEnvPath === undefined) { - delete process.env.OPENCLAW_TTS_PREFS; - } else { - process.env.OPENCLAW_TTS_PREFS = previousEnvPath; - } - } - }); - - it("resolves voice delivery support from channel capabilities", () => { - for (const channel of nativeVoiceNoteChannels) { - expect(testApi.supportsNativeVoiceNoteTts(channel)).toBe(true); - expect(testApi.supportsNativeVoiceNoteTts(channel.toUpperCase())).toBe(true); - } - expect(testApi.supportsNativeVoiceNoteTts("slack")).toBe(false); - expect(testApi.supportsNativeVoiceNoteTts(undefined)).toBe(false); - }); - - it("tells generic TTS guidance to defer to MEMORY voice-delivery instructions", () => { - const hint = buildTtsSystemPromptHint(createTtsConfig("openclaw-speech-core-tts-hint-test")); - - expect(hint).toContain("Voice (TTS) is enabled."); - expect(hint).toContain( - "If workspace context (especially MEMORY.md) tells you not to use [[tts:...]] or to use a local/non-tagged voice workflow, follow that workspace instruction instead.", - ); - expect(hint).toContain( - "Use [[tts:...]] and optional [[tts:text]]...[[/tts:text]] to control voice/expressiveness.", - ); - }); - - it("prepares deep-merged surface config and directive inputs", () => { - const cfg: OpenClawConfig = { - tts: { - provider: "mock", - modelOverrides: { allowProvider: false }, - providers: { - mock: { - model: "base-model", - voiceSettings: { stability: 0.4 }, - }, - }, - }, - }; - - const prepared = prepareTtsRequest({ - cfg, - override: { - modelOverrides: { allowProvider: true }, - providers: { - mock: { - voice: "surface-voice", - voiceSettings: { speed: 1.1 }, - }, - }, - }, - text: "Hello [[tts:text]]Speak this instead[[/tts:text]] caller", - }); - - expect(prepared.cfg).not.toBe(cfg); - expect(prepared.cfg.tts?.providers?.mock).toEqual({ - model: "base-model", - voice: "surface-voice", - voiceSettings: { stability: 0.4, speed: 1.1 }, - }); - expect(prepared.cfg.tts?.modelOverrides?.allowProvider).toBe(true); - expect(prepared.directives).toEqual({ - cleanedText: "Hello caller", - hasDirective: true, - overrides: { - ttsText: "Speak this instead", - }, - ttsText: "Speak this instead", - warnings: [], - }); - expect(cfg.tts?.providers?.mock).toEqual({ - model: "base-model", - voiceSettings: { stability: 0.4 }, - }); - }); - - it("sanitizes blocked override keys while preparing TTS config", () => { - const prepared = prepareTtsRequest({ - cfg: { - tts: { - provider: "mock", - providers: { mock: { model: "base-model" } }, - }, - }, - override: JSON.parse( - '{"__proto__":{"polluted":"top"},"providers":{"mock":{"voice":"safe","__proto__":{"polluted":"nested"}}}}', - ) as TtsConfig, - text: "[[tts:text]]Speak this instead[[/tts:text]]", - }); - - expect((Object.prototype as Record).polluted).toBeUndefined(); - expect(prepared.cfg.tts).not.toHaveProperty("polluted"); - expect(prepared.cfg.tts?.providers?.mock).toEqual({ - model: "base-model", - voice: "safe", - }); - expect(prepared.directives.cleanedText).toBe(""); - expect(prepared.directives.ttsText).toBe("Speak this instead"); - }); - - it("marks Discord auto TTS replies as native voice messages", async () => { - await expectTtsPayloadResult({ - channel: "discord", - prefsName: "openclaw-speech-core-tts-test", - text: "This Discord reply should be delivered as a native voice note.", - target: "voice-note", - audioAsVoice: true, - }); - }); - - it("keeps compatible audio-file synthesis deliverable as a voice memo", async () => { - await expectTtsPayloadResult({ - channel: "voice-memo-chat", - prefsName: "openclaw-speech-core-tts-voice-memo-mp3-test", - text: "This reply should be delivered as a native voice memo.", - target: "audio-file", - audioAsVoice: true, - mediaExtension: "mp3", - providerResult: { - audioBuffer: Buffer.from("mp3"), - outputFormat: "mp3", - fileExtension: ".mp3", - voiceCompatible: false, - }, - }); - }); - - it("does not mark unsupported audio-file output as a voice memo", async () => { - await expectTtsPayloadResult({ - channel: "voice-memo-chat", - prefsName: "openclaw-speech-core-tts-voice-memo-ogg-test", - text: "This reply should stay a regular audio attachment.", - target: "audio-file", - audioAsVoice: undefined, - }); - }); - - it("pre-transcodes synthesized mp3 to opus-in-CAF when the host can satisfy preferAudioFileFormat", async () => { - transcodeAudioBufferMock.mockResolvedValueOnce({ - ok: true, - buffer: Buffer.from("transcoded-caf"), - }); - await expectTtsPayloadResult({ - channel: "voice-memo-chat", - prefsName: "openclaw-speech-core-tts-voice-memo-caf-transcode-test", - text: "This reply should be pre-transcoded to a native voice-memo CAF.", - target: "audio-file", - audioAsVoice: true, - mediaExtension: "caf", - providerResult: { - audioBuffer: Buffer.from("mp3"), - outputFormat: "mp3", - fileExtension: ".mp3", - voiceCompatible: false, - }, - }); - expect(transcodeAudioBufferMock).toHaveBeenCalledOnce(); - const transcodeRequest = requireRecord( - requireFirstCallParam(transcodeAudioBufferMock.mock.calls as unknown[][], "transcode"), - "transcode request", - ); - expect(transcodeRequest.sourceExtension).toBe("mp3"); - expect(transcodeRequest.targetExtension).toBe("caf"); - }); - - it("falls back to the original mp3 buffer when the host transcoder fails", async () => { - transcodeAudioBufferMock.mockResolvedValueOnce({ - ok: false, - reason: "transcoder-failed", - detail: "exit-1", - }); - // Even though the transcode failed, the original mp3 still satisfies the - // channel audioFileFormats list, so the channel still flips audioAsVoice. - // The user gets a voice memo bubble, possibly with bad duration, instead - // of a regression. The failure is logged via the call site in tts.ts. - await expectTtsPayloadResult({ - channel: "voice-memo-chat", - prefsName: "openclaw-speech-core-tts-voice-memo-caf-fallback-test", - text: "This reply should fall back to the original mp3.", - target: "audio-file", - audioAsVoice: true, - mediaExtension: "mp3", - providerResult: { - audioBuffer: Buffer.from("mp3"), - outputFormat: "mp3", - fileExtension: ".mp3", - voiceCompatible: false, - }, - }); - }); - - it("uses the active runtime snapshot when source config still contains TTS SecretRefs", async () => { - const sourceConfig = { - tts: { - enabled: true, - provider: "mock", - providers: { - mock: { - apiKey: { source: "exec", provider: "mockexec", id: "minimax/tts/apiKey" }, - }, - }, - }, - } as unknown as OpenClawConfig; - const runtimeConfig = { - tts: { - enabled: true, - provider: "mock", - providers: { - mock: { - apiKey: "resolved-minimax-key", - }, - }, - }, - } as unknown as OpenClawConfig; - installSpeechProviders([ - createMockSpeechProvider("mock", { - isConfigured: ({ providerConfig }) => providerConfig.apiKey === "resolved-minimax-key", - resolveConfig: ({ rawConfig }) => { - const providers = rawConfig.providers as Record | undefined; - return { - apiKey: providers?.mock?.apiKey, - }; - }, - }), - ]); - setRuntimeConfigSnapshot(runtimeConfig, sourceConfig); - - const result = await synthesizeSpeech({ - text: "Runtime snapshot TTS SecretRef", - cfg: sourceConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - expect(synthesizeMock).toHaveBeenCalled(); - const request = requireFirstSynthesisRequest("runtime snapshot synthesis request"); - expect(request.cfg).toBe(runtimeConfig); - const providerConfig = requireRecord(request.providerConfig, "provider config"); - expect(providerConfig.apiKey).toBe("resolved-minimax-key"); - }); - - it("uses provider default TTS timeout when the call and config omit timeoutMs", async () => { - installSpeechProviders([createMockSpeechProvider("mock", { defaultTimeoutMs: 600_000 })]); - - const result = await synthesizeSpeech({ - text: "Use provider timeout.", - cfg: { - tts: { - enabled: true, - provider: "mock", - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - const request = requireFirstSynthesisRequest("provider default timeout synthesis request"); - expect(request.timeoutMs).toBe(600_000); - }); - - it("normalizes non-streaming synthesis text before calling the provider", async () => { - const result = await synthesizeSpeech({ - text: "## Update\n\nRead the [guide](https://example.com/guide)!!!!!", - cfg: createTtsConfig("openclaw-speech-core-talk-markdown-test"), - disableFallback: true, - }); - - expect(result.success).toBe(true); - const request = requireFirstSynthesisRequest("normalized talk synthesis request"); - expect(request.text).toBe("Update\n\nRead the guide!"); - }); - - it("speaks stripped code through the explicit textToSpeech conversion path", async () => { - let mediaDir: string | undefined; - try { - const result = await textToSpeech({ - text: "```ts\nconst answer = 42;\n```", - cfg: createTtsConfig("openclaw-speech-core-code-convert-test"), - }); - - expect(result.success).toBe(true); - const request = requireFirstSynthesisRequest("explicit code conversion request"); - expect(request.text).toBe("const answer = 42;"); - expect(request.text).not.toBe(CODE_HEAVY_SPOKEN_FALLBACK); - mediaDir = result.audioPath ? path.dirname(result.audioPath) : undefined; - } finally { - if (mediaDir) { - rmSync(mediaDir, { recursive: true, force: true }); - } - } - }); - - it("returns a normal TTS failure when audio persistence rejects", async () => { - const result = await textToSpeechCore( - { - text: "Store this synthesized reply.", - cfg: createTtsConfig("openclaw-speech-core-persistence-failure-test"), - }, - async () => { - throw new Error("Media exceeds configured limit"); - }, - ); - - expect(result).toMatchObject({ - success: false, - error: "TTS audio persistence failed", - provider: "mock", - }); - }); - - it("resolves the configured timeout for voice listing", async () => { - const listVoicesMock = vi.fn(async (_request: SpeechListVoicesRequest) => []); - installSpeechProviders([ - createMockSpeechProvider("mock", { - defaultTimeoutMs: 60_000, - listVoices: listVoicesMock, - }), - ]); - - await listSpeechVoices({ - provider: "mock", - cfg: { - tts: { - enabled: true, - provider: "mock", - timeoutMs: 45_000, - }, - } as OpenClawConfig, - }); - - expect(listVoicesMock).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 45_000 })); - }); - - it("caps oversized provider default TTS timeouts before synthesis", async () => { - installSpeechProviders([ - createMockSpeechProvider("mock", { defaultTimeoutMs: Number.MAX_SAFE_INTEGER }), - ]); - - const result = await synthesizeSpeech({ - text: "Use capped provider timeout.", - cfg: { - tts: { - enabled: true, - provider: "mock", - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - const request = requireFirstSynthesisRequest("provider default capped timeout request"); - expect(request.timeoutMs).toBe(MAX_TIMER_TIMEOUT_MS); - }); - - it("ignores nonpositive provider default TTS timeouts", async () => { - installSpeechProviders([createMockSpeechProvider("mock", { defaultTimeoutMs: 0 })]); - - const result = await synthesizeSpeech({ - text: "Use fallback timeout.", - cfg: { - tts: { - enabled: true, - provider: "mock", - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - const request = requireFirstSynthesisRequest("provider default fallback timeout request"); - expect(request.timeoutMs).toBe(30_000); - }); - - it("keeps explicit TTS config timeout ahead of provider default timeout", async () => { - installSpeechProviders([createMockSpeechProvider("mock", { defaultTimeoutMs: 600_000 })]); - - await synthesizeSpeech({ - text: "Use configured timeout.", - cfg: { - tts: { - enabled: true, - provider: "mock", - timeoutMs: 45_000, - }, - } as OpenClawConfig, - disableFallback: true, - }); - - const request = requireFirstSynthesisRequest("configured timeout synthesis request"); - expect(request.timeoutMs).toBe(45_000); - }); - - it("caps oversized voice model TTS timeouts before synthesis", async () => { - installSpeechProviders([ - createMockSpeechProvider("mock", { autoSelectOrder: 1, models: ["mock-tts"] }), - ]); - - const result = await synthesizeSpeech({ - text: "Use capped explicit timeout.", - cfg: { - agents: { - defaults: { - voiceModel: { primary: "mock/mock-tts", timeoutMs: Number.MAX_SAFE_INTEGER }, - }, - }, - tts: { - enabled: true, - provider: "mock", - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - const request = requireFirstSynthesisRequest("voice model capped timeout request"); - expect(request.timeoutMs).toBe(MAX_TIMER_TIMEOUT_MS); - }); - - it("uses agents.defaults.voiceModel as the default speech provider and model", async () => { - installSpeechProviders([ - createMockSpeechProvider("mock", { autoSelectOrder: 1 }), - createMockSpeechProvider("openai", { - autoSelectOrder: 10, - models: ["gpt-4o-mini-tts"], - resolveConfig: ({ rawConfig }) => { - const providers = requireRecord(rawConfig.providers, "raw provider configs"); - return { - model: "provider-default-model", - modelId: "provider-default-model", - ...requireRecord(providers.openai, "raw openai provider config"), - }; - }, - }), - ]); - - const result = await synthesizeSpeech({ - text: "Use configured voice model.", - cfg: { - agents: { - defaults: { - voiceModel: { primary: "openai/gpt-4o-mini-tts", timeoutMs: 12_345 }, - }, - }, - tts: { - enabled: true, - prefsPath: "/tmp/openclaw-speech-core-voice-model-default-test.json", - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - expect(result.provider).toBe("openai"); - expect(result.providerModel).toBe("gpt-4o-mini-tts"); - const request = requireFirstSynthesisRequest("voice model synthesis request"); - expect(request.providerConfig).toMatchObject({ - model: "gpt-4o-mini-tts", - modelId: "gpt-4o-mini-tts", - }); - expect(request.timeoutMs).toBe(12_345); - }); - - it("keeps explicit provider model aliases ahead of voiceModel defaults", async () => { - installSpeechProviders([ - createMockSpeechProvider("openrouter", { - models: ["explicit-model", "default-model"], - resolveConfig: ({ rawConfig }) => { - const providers = requireRecord(rawConfig.providers, "raw provider configs"); - return requireRecord(providers.openrouter, "raw openrouter provider config"); - }, - }), - ]); - - const result = await synthesizeSpeech({ - text: "Prefer explicit model alias.", - cfg: { - agents: { - defaults: { - voiceModel: { primary: "openrouter/default-model" }, - }, - }, - tts: { - enabled: true, - provider: "openrouter", - prefsPath: "/tmp/openclaw-speech-core-explicit-model-alias-test.json", - providers: { - openrouter: { - modelId: "explicit-model", - }, - }, - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - const request = requireFirstSynthesisRequest("explicit model alias synthesis request"); - const providerConfig = requireRecord(request.providerConfig, "provider config"); - expect(providerConfig).toMatchObject({ - modelId: "explicit-model", - }); - expect(providerConfig.model).toBeUndefined(); - }); - - it("tries voiceModel fallbacks before auto-selected speech providers", async () => { - installSpeechProviders([ - createMockSpeechProvider("mock", { autoSelectOrder: 1 }), - createMockSpeechProvider("openai", { - autoSelectOrder: 10, - models: ["gpt-4o-mini-tts"], - isConfigured: () => false, - }), - createMockSpeechProvider("elevenlabs", { - autoSelectOrder: 99, - models: ["eleven_multilingual_v2"], - }), - ]); - - const result = await synthesizeSpeech({ - text: "Use configured voice model fallback.", - cfg: { - agents: { - defaults: { - voiceModel: { - primary: "openai/gpt-4o-mini-tts", - fallbacks: ["elevenlabs/eleven_multilingual_v2"], - }, - }, - }, - tts: { - enabled: true, - prefsPath: "/tmp/openclaw-speech-core-voice-model-fallback-test.json", - }, - } as OpenClawConfig, - }); - - expect(result.success).toBe(true); - expect(result.provider).toBe("elevenlabs"); - expect(result.fallbackFrom).toBe("openai"); - expect(result.providerModel).toBe("eleven_multilingual_v2"); - }); - - it("tries same-provider voiceModel fallbacks as separate model attempts", async () => { - const synthesize = vi.fn(async (request: SpeechSynthesisRequest) => { - if (request.providerConfig.model === "bad-tts") { - throw new Error("unavailable model"); - } - return { - audioBuffer: Buffer.from("voice"), - fileExtension: ".ogg", - outputFormat: "ogg", - voiceCompatible: request.target === "voice-note", - }; - }); - installSpeechProviders([ - createMockSpeechProvider("openai", { - autoSelectOrder: 10, - models: ["bad-tts", "good-tts"], - synthesize, - }), - ]); - - const result = await synthesizeSpeech({ - text: "Use same-provider fallback model.", - cfg: { - agents: { - defaults: { - voiceModel: { - primary: "openai/bad-tts", - fallbacks: ["openai/good-tts"], - }, - }, - }, - tts: { - enabled: true, - prefsPath: "/tmp/openclaw-speech-core-same-provider-voice-model-fallback-test.json", - }, - } as OpenClawConfig, - }); - - expect(result.success).toBe(true); - expect(result.provider).toBe("openai"); - expect(result.providerModel).toBe("good-tts"); - expect(result.attemptedProviders).toEqual(["openai", "openai"]); - expect(synthesize.mock.calls.map(([request]) => request.providerConfig.model)).toEqual([ - "bad-tts", - "good-tts", - ]); - }); - - it("skips non-streaming providers before using a streaming fallback", async () => { - const release = vi.fn(async () => {}); - const streamSynthesize = vi.fn(async () => ({ - audioStream: new ReadableStream({ - start(controller) { - controller.close(); - }, - }), - fileExtension: ".pcm", - outputFormat: "pcm", - voiceCompatible: false, - release, - })); - installSpeechProviders([ - createMockSpeechProvider("buffered", { autoSelectOrder: 1 }), - createMockSpeechProvider("streaming", { - autoSelectOrder: 2, - streamSynthesize, - }), - ]); - - const result = await textToSpeechStream({ - text: "Use streaming fallback.", - cfg: { - tts: { - enabled: true, - provider: "buffered", - prefsPath: "/tmp/openclaw-speech-core-streaming-fallback-test.json", - }, - } as OpenClawConfig, - }); - - expect(result.success).toBe(true); - expect(result.provider).toBe("streaming"); - expect(result.fallbackFrom).toBe("buffered"); - expect(result.attemptedProviders).toEqual(["buffered", "streaming"]); - expect(result.outputFormat).toBe("pcm"); - expect(result.fileExtension).toBe(".pcm"); - expect(result.target).toBe("audio-file"); - expect(result.release).toBe(release); - const skippedAttempt = requireAttempt(result.attempts, 0); - expect(skippedAttempt).toMatchObject({ - provider: "buffered", - outcome: "skipped", - reasonCode: "unsupported_for_streaming", - personaBinding: "none", - error: "buffered does not support streaming TTS", - }); - expect(skippedAttempt).not.toHaveProperty("latencyMs"); - expect(requireAttempt(result.attempts, 1)).toMatchObject({ - provider: "streaming", - outcome: "success", - reasonCode: "success", - }); - expect(streamSynthesize).toHaveBeenCalledOnce(); - }); - - it("classifies streaming timeouts before falling back with raw text", async () => { - const timeoutStreamSynthesize = vi.fn(async () => { - const error = new Error("stalled"); - error.name = "AbortError"; - throw error; - }); - const fallbackStreamSynthesize = vi.fn(async () => ({ - audioStream: new ReadableStream({ - start(controller) { - controller.close(); - }, - }), - fileExtension: ".pcm", - outputFormat: "pcm", - voiceCompatible: false, - })); - installSpeechProviders([ - createMockSpeechProvider("primary", { - autoSelectOrder: 1, - streamSynthesize: timeoutStreamSynthesize, - }), - createMockSpeechProvider("fallback", { - autoSelectOrder: 2, - streamSynthesize: fallbackStreamSynthesize, - }), - ]); - const text = "## Keep [streaming Markdown](https://example.com) raw!!!!!"; - - const result = await textToSpeechStream({ - text, - cfg: { - tts: { - enabled: true, - provider: "primary", - prefsPath: "/tmp/openclaw-speech-core-streaming-timeout-test.json", - }, - } as OpenClawConfig, - }); - - expect(result.success).toBe(true); - expect(result.provider).toBe("fallback"); - expect(result.fallbackFrom).toBe("primary"); - expect(requireAttempt(result.attempts, 0)).toMatchObject({ - provider: "primary", - outcome: "failed", - reasonCode: "timeout", - error: "primary: request timed out", - }); - expect(requireAttempt(result.attempts, 1)).toMatchObject({ - provider: "fallback", - outcome: "success", - reasonCode: "success", - }); - expect(fallbackStreamSynthesize).toHaveBeenCalledWith(expect.objectContaining({ text })); - }); - - it("ignores voiceModel refs that are not speech models", async () => { - installSpeechProviders([ - createMockSpeechProvider("openai", { - autoSelectOrder: 10, - defaultModel: "gpt-4o-mini-tts", - models: ["gpt-4o-mini-tts"], - resolveConfig: ({ rawConfig }) => { - const providers = requireRecord(rawConfig.providers, "raw provider configs"); - return { - model: "gpt-4o-mini-tts", - modelId: "gpt-4o-mini-tts", - ...requireRecord(providers.openai, "raw openai provider config"), - }; - }, - }), - ]); - - const result = await synthesizeSpeech({ - text: "Use speech provider default for unsupported realtime model.", - cfg: { - agents: { - defaults: { - voiceModel: { primary: "openai/gpt-realtime-2" }, - }, - }, - tts: { - enabled: true, - provider: "openai", - prefsPath: "/tmp/openclaw-speech-core-realtime-voice-model-ignored-test.json", - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - expect(result.provider).toBe("openai"); - expect(result.providerModel).toBe("gpt-4o-mini-tts"); - const request = requireFirstSynthesisRequest("speech model fallback request"); - expect(request.providerConfig).toMatchObject({ - model: "gpt-4o-mini-tts", - modelId: "gpt-4o-mini-tts", - }); - }); - - it("uses the first speech-supported voiceModel fallback as the default provider", async () => { - installSpeechProviders([ - createMockSpeechProvider("openai", { - autoSelectOrder: 1, - models: ["gpt-4o-mini-tts"], - }), - createMockSpeechProvider("elevenlabs", { - autoSelectOrder: 99, - models: ["eleven_multilingual_v2"], - }), - ]); - - const result = await synthesizeSpeech({ - text: "Use first speech-supported voice model.", - cfg: { - agents: { - defaults: { - voiceModel: { - primary: "openai/gpt-realtime-2", - fallbacks: ["elevenlabs/eleven_multilingual_v2"], - }, - }, - }, - tts: { - enabled: true, - prefsPath: "/tmp/openclaw-speech-core-supported-voice-model-provider-test.json", - }, - } as OpenClawConfig, - }); - - expect(result.success).toBe(true); - expect(result.provider).toBe("elevenlabs"); - expect(result.providerModel).toBe("eleven_multilingual_v2"); - expect(result.attemptedProviders).toEqual(["elevenlabs"]); - }); - - it("maps speakerVoice provider config to provider-compatible voice fields", async () => { - const result = await synthesizeSpeech({ - text: "Use the configured speaker.", - cfg: { - tts: { - enabled: true, - provider: "mock", - providers: { - mock: { - speakerVoice: "cedar", - speakerVoiceId: "voice-123", - voice: "legacy-voice", - voiceName: "legacy-name", - voiceId: "legacy-id", - }, - }, - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - expect(result.providerVoice).toBe("voice-123"); - const request = requireFirstSynthesisRequest("speaker voice synthesis request"); - expect(request.providerConfig).toMatchObject({ - speakerVoice: "cedar", - voice: "cedar", - voiceName: "cedar", - speakerVoiceId: "voice-123", - voiceId: "voice-123", - }); - }); - - it("preserves alias-keyed provider config when resolving canonical providers", async () => { - installSpeechProviders([ - createMockSpeechProvider("xiaomi", { - aliases: ["mimo"], - resolveConfig: ({ rawConfig }) => { - const providers = requireRecord(rawConfig.providers, "raw provider configs"); - return requireRecord(providers.xiaomi ?? providers.mimo, "raw xiaomi provider config"); - }, - }), - ]); - - const result = await synthesizeSpeech({ - text: "Use alias provider config.", - cfg: { - tts: { - enabled: true, - provider: "xiaomi", - providers: { - mimo: { apiKey: "mimo-key" }, - }, - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - const request = requireFirstSynthesisRequest("alias provider synthesis request"); - expect(request.providerConfig).toMatchObject({ apiKey: "mimo-key" }); - }); - - it("maps speakerVoice persona provider config to provider-compatible voice fields", async () => { - const result = await synthesizeSpeech({ - text: "Use the persona speaker.", - cfg: { - tts: { - enabled: true, - provider: "mock", - persona: "narrator", - personas: { - narrator: { - providers: { - mock: { - speakerVoice: "marin", - }, - }, - }, - }, - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - expect(result.providerVoice).toBe("marin"); - const request = requireFirstSynthesisRequest("persona speaker voice synthesis request"); - expect(request.providerConfig).toMatchObject({ - speakerVoice: "marin", - voice: "marin", - voiceName: "marin", - }); - }); - - it.each(["feishu", "whatsapp"] as const)( - "marks %s voice-note TTS for channel-side transcoding when provider returns mp3", - async (channel) => { - expect(testApi.supportsTranscodedVoiceNoteTts(channel)).toBe(true); - await expectTtsPayloadResult({ - channel, - prefsName: `openclaw-speech-core-tts-${channel}-mp3-test`, - text: `This ${channel} reply should be transcoded by the channel.`, - target: "voice-note", - audioAsVoice: true, - mediaExtension: "mp3", - providerResult: { - audioBuffer: Buffer.from("mp3"), - outputFormat: "mp3", - fileExtension: ".mp3", - voiceCompatible: false, - }, - }); - }, - ); - - it("keeps non-native voice-note channels as regular audio files", async () => { - await expectTtsPayloadResult({ - channel: "slack", - prefsName: "openclaw-speech-core-tts-slack-test", - text: "Slack replies should be delivered as regular audio attachments.", - target: "audio-file", - audioAsVoice: undefined, - }); - }); - - it("preserves the text reply when auto-TTS audio persistence fails", async () => { - const payload = { text: "This text must still be delivered when media storage rejects audio." }; - const result = await maybeApplyTtsToPayloadCore( - { - payload, - cfg: createTtsConfig("openclaw-speech-core-auto-persistence-failure-test"), - channel: "slack", - kind: "final", - }, - async () => { - throw new Error("Media exceeds configured limit"); - }, - ); - - expect(result).toBe(payload); - }); - - it("normalizes voice-note Markdown once before synthesis", async () => { - const text = - 'This short explanation keeps the fenced literal below from becoming code-heavy.\n\n```md\nconst literal = "[x](y)";\n```'; - let mediaDir: string | undefined; - try { - const result = await maybeApplyTtsToPayload({ - payload: { text }, - cfg: createTtsConfig("openclaw-speech-core-once-normalized-markdown-test"), - channel: "telegram", - kind: "final", - }); - - const request = requireFirstSynthesisRequest("once-normalized voice-note synthesis request"); - expect(request.text).toBe( - 'This short explanation keeps the fenced literal below from becoming code-heavy.\n\nconst literal = "[x](y)";', - ); - expect(result.text).toBe(text); - mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; - } finally { - if (mediaDir) { - rmSync(mediaDir, { recursive: true, force: true }); - } - } - }); - - it("skips channel auto-TTS audio for code-heavy replies", async () => { - const text = "```ts\nexport function answer() {\n return 42;\n}\n```"; - const result = await maybeApplyTtsToPayload({ - payload: { text }, - cfg: createTtsConfig("openclaw-speech-core-code-heavy-voice-note-test"), - channel: "telegram", - kind: "final", - }); - - expect(synthesizeMock).not.toHaveBeenCalled(); - expect(result).toEqual({ text }); - }); - - it("synthesizes code-heavy explicitly tagged hidden TTS text", async () => { - const cfg = createTtsConfig("openclaw-speech-core-code-heavy-hidden-tts-test"); - let mediaDir: string | undefined; - try { - const result = await maybeApplyTtsToPayload({ - payload: { - text: '[[tts:text]]```ts\nconst detailedAnswer = "this code should still be spoken";\n```[[/tts:text]]', - audioAsVoice: true, - }, - cfg, - channel: "telegram", - kind: "final", - }); - - expect(synthesizeMock).toHaveBeenCalled(); - const request = requireFirstSynthesisRequest("code-heavy hidden TTS request"); - expect(request.text).toBe('const detailedAnswer = "this code should still be spoken";'); - expect(result.text).toBeUndefined(); - mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; - } finally { - if (mediaDir) { - rmSync(mediaDir, { recursive: true, force: true }); - } - } - }); - - it("synthesizes explicitly tagged short hidden TTS text", async () => { - const cfg = createTtsConfig("openclaw-speech-core-short-hidden-tts-test"); - let mediaDir: string | undefined; - try { - const result = await maybeApplyTtsToPayload({ - payload: { - text: "[[tts:text]]hello[[/tts:text]]", - audioAsVoice: true, - }, - cfg, - channel: "telegram", - kind: "final", - }); - - expect(synthesizeMock).toHaveBeenCalled(); - const request = requireFirstSynthesisRequest("hidden TTS request"); - expect(request.text).toBe("hello"); - expect(result.mediaUrl).toMatch(/voice---[a-f0-9-]+\.ogg$/); - expect(result.audioAsVoice).toBe(true); - expect(result.text).toBeUndefined(); - expect(result.ttsSupplement).toBeUndefined(); - mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; - } finally { - if (mediaDir) { - rmSync(mediaDir, { recursive: true, force: true }); - } - } - }); - - it("truncates long TTS text on a UTF-16 boundary", async () => { - const prefsName = "openclaw-speech-core-utf16-truncate-test"; - const prefsPath = prefsPathFor(prefsName); - const cfg = createTtsConfig(prefsName); - setTtsMaxLength(prefsPath, 11); - setSummarizationEnabled(prefsPath, false); - let mediaDir: string | undefined; - try { - const result = await maybeApplyTtsToPayload({ - payload: { text: `${"a".repeat(7)}😀tail long enough for TTS` }, - cfg, - channel: "telegram", - kind: "final", - }); - - expect(synthesizeMock).toHaveBeenCalled(); - const request = requireFirstSynthesisRequest("utf16 truncated TTS request"); - const spokenText = String(request.text); - expect(spokenText).toBe(`${"a".repeat(7)}...`); - expect(result.spokenText).toBe(spokenText); - mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; - } finally { - rmSync(prefsPath, { force: true }); - if (mediaDir) { - rmSync(mediaDir, { recursive: true, force: true }); - } - } - }); - - it("skips block delivery kind in final mode (accumulated final tail synthesizes instead)", async () => { - synthesizeMock.mockClear(); - const cfg = createTtsConfig("openclaw-speech-core-block-kind-tts-test"); - const result = await maybeApplyTtsToPayload({ - payload: { text: "WebChat block stream chunks defer TTS to the final tail." }, - cfg, - channel: "webchat", - kind: "block", - }); - - expect(synthesizeMock).not.toHaveBeenCalled(); - expect((result as { trustedLocalMedia?: boolean }).trustedLocalMedia).toBeUndefined(); - expect(result.text).toBe("WebChat block stream chunks defer TTS to the final tail."); - }); - - it("skips tool delivery kind in final mode", async () => { - synthesizeMock.mockClear(); - const cfg = createTtsConfig("openclaw-speech-core-tool-kind-tts-test"); - const result = await maybeApplyTtsToPayload({ - payload: { text: "Intermediate tool output should not be spoken." }, - cfg, - channel: "webchat", - kind: "tool", - }); - - expect(synthesizeMock).not.toHaveBeenCalled(); - expect((result as { trustedLocalMedia?: boolean }).trustedLocalMedia).toBeUndefined(); - expect(result.text).toBe("Intermediate tool output should not be spoken."); - }); - - it("keeps skipping untagged short TTS text", async () => { - const cfg = createTtsConfig("openclaw-speech-core-short-plain-tts-test"); - const result = await maybeApplyTtsToPayload({ - payload: { - text: "hello", - audioAsVoice: true, - }, - cfg, - channel: "telegram", - kind: "final", - }); - - expect(synthesizeMock).not.toHaveBeenCalled(); - expect(result).toEqual({ - text: "hello", - audioAsVoice: true, - }); - }); - - it("skips auto TTS for legacy final media directives", async () => { - synthesizeMock.mockClear(); - const cfg = createTtsConfig("openclaw-speech-core-media-directive-tts-test"); - const result = await maybeApplyTtsToPayload({ - payload: { text: "Here is the render.\nMEDIA:/tmp/render.png" }, - cfg, - channel: "telegram", - kind: "final", - }); - - expect(synthesizeMock).not.toHaveBeenCalled(); - expect(result).toEqual({ text: "Here is the render.\nMEDIA:/tmp/render.png" }); - }); - - it("keeps skipping explicit tagged TTS text that strips to empty markdown", async () => { - const cfg = createTtsConfig("openclaw-speech-core-empty-hidden-tts-test"); - const result = await maybeApplyTtsToPayload({ - payload: { - text: "[[tts:text]]***[[/tts:text]]", - audioAsVoice: true, - }, - cfg, - channel: "telegram", - kind: "final", - }); - - expect(synthesizeMock).not.toHaveBeenCalled(); - expect(result).toEqual({ - audioAsVoice: true, - }); - }); - - it("selects persona preferred provider before config fallback", () => { - const cfg: OpenClawConfig = { - tts: { - enabled: true, - provider: "other", - persona: "alfred", - personas: { - alfred: { - label: "Alfred", - provider: "mock", - providers: { - mock: { - voice: "Algieba", - }, - }, - }, - }, - }, - }; - const config = resolveTtsConfig(cfg); - const prefsPath = "/tmp/openclaw-speech-core-persona-provider.json"; - - expect(getTtsPersona(config, prefsPath)?.id).toBe("alfred"); - expect(getTtsProvider(config, prefsPath)).toBe("mock"); - }); - - it("treats provider configuration errors as unconfigured", () => { - installSpeechProviders([ - createMockSpeechProvider("broken", { - resolveConfig: () => { - throw new Error("invalid provider URL"); - }, - }), - ]); - const prefsPath = "/tmp/openclaw-speech-core-invalid-provider.json"; - setTtsMachinePrefsPathResolver(() => prefsPath); - const cfg = { - tts: { - providers: { broken: {} }, - }, - } as OpenClawConfig; - const config = resolveTtsConfig(cfg); - - expect(isTtsProviderConfigured(config, "broken", cfg)).toBe(false); - expect(getTtsProvider(config, prefsPath)).toBe(""); - }); - - it("merges active persona provider binding into synthesis config", async () => { - setTtsMachinePrefsPathResolver(() => "/tmp/openclaw-speech-core-persona-merge.json"); - const cfg: OpenClawConfig = { - tts: { - enabled: true, - provider: "mock", - providers: { - mock: { - model: "base-model", - voice: "base-voice", - }, - }, - persona: "alfred", - personas: { - alfred: { - provider: "mock", - providers: { - mock: { - voice: "persona-voice", - style: "dry", - }, - }, - }, - }, - }, - }; - - const payload: ReplyPayload = { - text: "This reply should use persona-specific provider configuration.", - }; - - let mediaDir: string | undefined; - try { - const result = await maybeApplyTtsToPayload({ - payload, - cfg, - channel: "slack", - kind: "final", - }); - - expect(synthesizeMock).toHaveBeenCalled(); - const request = requireFirstSynthesisRequest("persona synthesis request"); - const providerConfig = requireRecord(request.providerConfig, "persona provider config"); - expect(providerConfig.model).toBe("base-model"); - expect(providerConfig.voice).toBe("persona-voice"); - expect(providerConfig.style).toBe("dry"); - expect(result.mediaUrl).toMatch(/voice---[a-f0-9-]+\.ogg$/); - - mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; - } finally { - if (mediaDir) { - rmSync(mediaDir, { recursive: true, force: true }); - } - } - }); - - it("does not mark skipped unregistered providers as missing persona bindings", async () => { - const result = await synthesizeSpeech({ - text: "Use fallback provider.", - cfg: { - tts: { - enabled: true, - provider: "missing", - persona: "alfred", - personas: { - alfred: { - providers: { - missing: { - voice: "configured-but-unregistered", - }, - }, - }, - }, - }, - }, - }); - - expect(result.success).toBe(true); - const attempt = requireAttempt(result.attempts, 0); - expect(attempt.provider).toBe("missing"); - expect(attempt.outcome).toBe("skipped"); - expect(attempt.reasonCode).toBe("no_provider_registered"); - expect(attempt.persona).toBe("alfred"); - expect(attempt).not.toHaveProperty("personaBinding"); - }); - - it("does not mark skipped telephony providers as missing persona bindings", async () => { - const result = await textToSpeechTelephony({ - text: "Use telephony provider.", - cfg: { - tts: { - enabled: true, - provider: "mock", - persona: "alfred", - personas: { - alfred: { - providers: { - mock: { - voice: "persona-voice", - }, - }, - }, - }, - }, - }, - }); - - expect(result.success).toBe(false); - const attempt = requireAttempt(result.attempts, 0); - expect(attempt.provider).toBe("mock"); - expect(attempt.outcome).toBe("skipped"); - expect(attempt.reasonCode).toBe("unsupported_for_telephony"); - expect(attempt.persona).toBe("alfred"); - expect(attempt).not.toHaveProperty("personaBinding"); - }); - - it("passes directive overrides to telephony synthesis providers", async () => { - const synthesizeTelephonyMock = vi.fn(async (_request: SpeechTelephonySynthesisRequest) => ({ - audioBuffer: Buffer.from("voice"), - outputFormat: "pcm", - sampleRate: 24_000, - })); - installSpeechProviders([ - createMockSpeechProvider("mock", { - synthesizeTelephony: synthesizeTelephonyMock, - }), - ]); - - const text = "## Keep [telephony Markdown](https://example.com) raw!!!!!"; - const result = await textToSpeechTelephony({ - text, - cfg: { - tts: { - enabled: true, - provider: "mock", - providers: { - mock: { - modelId: "telephony-model", - voiceId: "default-voice", - }, - }, - }, - }, - overrides: { - providerOverrides: { - mock: { - speakerVoice: "directed-voice", - speed: 1.5, - }, - }, - }, - }); - - expect(result.success).toBe(true); - expect(result.providerModel).toBe("telephony-model"); - expect(result.providerVoice).toBe("directed-voice"); - expect(synthesizeTelephonyMock).toHaveBeenCalledOnce(); - const telephonyRequest = requireRecord( - requireFirstCallParam(synthesizeTelephonyMock.mock.calls, "telephony synthesis"), - "telephony synthesis request", - ); - expect(telephonyRequest.providerOverrides).toEqual({ - speakerVoice: "directed-voice", - speed: 1.5, - }); - expect(telephonyRequest.text).toBe(text); - expect(telephonyRequest).not.toHaveProperty("target"); - }); - - it("uses provider defaults when fallback policy allows missing persona bindings", async () => { - await synthesizeSpeech({ - text: "Use neutral provider defaults.", - cfg: { - tts: { - enabled: true, - provider: "mock", - persona: "alfred", - personas: { - alfred: { - fallbackPolicy: "provider-defaults", - }, - }, - }, - }, - }); - - expect(prepareSynthesisMock).toHaveBeenCalledOnce(); - const prepareContext = requireRecord( - requireFirstCallParam(prepareSynthesisMock.mock.calls, "prepare synthesis"), - "prepare synthesis context", - ); - expect(prepareContext.persona).toBeUndefined(); - expect(prepareContext.personaProviderConfig).toBeUndefined(); - }); - - it("preserves persona metadata by default when provider bindings are missing", async () => { - await synthesizeSpeech({ - text: "Use persona prompt.", - cfg: { - tts: { - enabled: true, - provider: "mock", - persona: "alfred", - personas: { - alfred: { - label: "Alfred", - }, - }, - }, - }, - }); - - expect(prepareSynthesisMock).toHaveBeenCalledOnce(); - const prepareContext = requireRecord( - requireFirstCallParam(prepareSynthesisMock.mock.calls, "prepare synthesis"), - "prepare synthesis context", - ); - const persona = requireRecord(prepareContext.persona, "prepare synthesis persona"); - expect(persona.id).toBe("alfred"); - expect(prepareContext.personaProviderConfig).toBeUndefined(); - }); - - it("skips unbound providers under fail policy while allowing bound fallbacks", async () => { - installSpeechProviders([ - createMockSpeechProvider("mock", { autoSelectOrder: 1 }), - createMockSpeechProvider("fallback", { autoSelectOrder: 2 }), - ]); - - const result = await synthesizeSpeech({ - text: "Use the first persona-bound provider.", - cfg: { - tts: { - enabled: true, - provider: "mock", - persona: "alfred", - personas: { - alfred: { - fallbackPolicy: "fail", - providers: { - fallback: { - voice: "fallback-voice", - }, - }, - }, - }, - }, - }, - }); - - expect(result.success).toBe(true); - expect(result.provider).toBe("fallback"); - expect(result.fallbackFrom).toBe("mock"); - const skippedAttempt = requireAttempt(result.attempts, 0); - expect(skippedAttempt.provider).toBe("mock"); - expect(skippedAttempt.outcome).toBe("skipped"); - expect(skippedAttempt.reasonCode).toBe("not_configured"); - expect(skippedAttempt.persona).toBe("alfred"); - expect(skippedAttempt.personaBinding).toBe("missing"); - expect(skippedAttempt.error).toBe("mock: persona alfred has no provider binding"); - const successAttempt = requireAttempt(result.attempts, 1); - expect(successAttempt.provider).toBe("fallback"); - expect(successAttempt.outcome).toBe("success"); - expect(successAttempt.persona).toBe("alfred"); - expect(successAttempt.personaBinding).toBe("applied"); - }); -}); - -describe("speech-core per-agent TTS config", () => { - it("deep-merges the active agent TTS override over tts", () => { - const cfg = { - tts: { - enabled: true, - provider: "openai", - providers: { - openai: { - apiKey: "${OPENAI_API_KEY}", - voice: "coral", - speed: 1, - }, - }, - }, - agents: { - list: [ - { - id: "reader", - tts: { - provider: "openai", - providers: { - openai: { - voice: "nova", - }, - }, - }, - }, - ], - }, - } satisfies OpenClawConfig; - - const resolved = resolveTtsConfig(cfg, "reader"); - - const rawConfig = requireRecord(resolved.rawConfig, "resolved raw TTS config"); - expect(rawConfig.enabled).toBe(true); - expect(rawConfig.provider).toBe("openai"); - const providers = requireRecord(rawConfig.providers, "resolved raw TTS providers"); - const openai = requireRecord(providers.openai, "resolved OpenAI TTS provider config"); - expect(openai.apiKey).toBe("${OPENAI_API_KEY}"); - expect(openai.voice).toBe("nova"); - expect(openai.speed).toBe(1); - }); - - it("composes per-agent TTS overrides with active persona bindings", async () => { - const cfg = { - tts: { - enabled: true, - provider: "mock", - providers: { - mock: { - model: "base-model", - voice: "base-voice", - }, - }, - persona: "alfred", - personas: { - alfred: { - provider: "mock", - providers: { - mock: { - voice: "alfred-voice", - }, - }, - }, - jarvis: { - provider: "mock", - providers: { - mock: { - style: "jarvis-style", - }, - }, - }, - }, - }, - agents: { - list: [ - { - id: "reader", - tts: { - persona: "jarvis", - providers: { - mock: { - voice: "agent-voice", - }, - }, - }, - }, - ], - }, - } satisfies OpenClawConfig; - - let mediaDir: string | undefined; - try { - const result = await maybeApplyTtsToPayload({ - payload: { text: "This agent reply should use the composed persona config." }, - cfg, - channel: "slack", - kind: "final", - agentId: "reader", - }); - - expect(synthesizeMock).toHaveBeenCalled(); - const request = requireFirstSynthesisRequest("agent persona synthesis request"); - const providerConfig = requireRecord(request.providerConfig, "agent persona provider config"); - expect(providerConfig.model).toBe("base-model"); - expect(providerConfig.voice).toBe("agent-voice"); - expect(providerConfig.style).toBe("jarvis-style"); - expect(result.mediaUrl).toMatch(/voice---[a-f0-9-]+\.ogg$/); - mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; - } finally { - if (mediaDir) { - rmSync(mediaDir, { recursive: true, force: true }); - } - } - }); - - it("ignores prototype-pollution keys in agent TTS overrides", () => { - const cfg = { - tts: { - provider: "openai", - providers: { - openai: { - voice: "coral", - }, - }, - }, - agents: { - list: [ - { - id: "reader", - tts: JSON.parse( - '{"providers":{"openai":{"voice":"nova","__proto__":{"polluted":true}}}}', - ), - }, - ], - }, - } as OpenClawConfig; - - const resolved = resolveTtsConfig(cfg, "reader"); - - expect(resolved.rawConfig?.providers?.openai).toEqual({ voice: "nova" }); - expect(({} as Record).polluted).toBeUndefined(); - }); -}); -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/packages/speech-core/tsconfig.json b/packages/speech-core/tsconfig.json deleted file mode 100644 index 329cf33d90d7..000000000000 --- a/packages/speech-core/tsconfig.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "rootDir": "../.." - }, - "include": ["./*.ts", "./src/**/*.ts"], - "exclude": [ - "./**/*.test.ts", - "./dist/**", - "./node_modules/**", - "./src/test-support/**", - "./src/**/*test-helpers.ts", - "./src/**/*test-harness.ts", - "./src/**/*test-support.ts" - ] -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 43b0ed6ff41c..5e273594d4d1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2323,12 +2323,6 @@ importers: packages/session-url-contract: {} - packages/speech-core: - dependencies: - openclaw: - specifier: workspace:* - version: link:../.. - packages/terminal-core: dependencies: '@clack/prompts': diff --git a/qa/scenarios/media/webchat-auto-tts.yaml b/qa/scenarios/media/webchat-auto-tts.yaml index c6326cc3bfad..e9cbb21c4f33 100644 --- a/qa/scenarios/media/webchat-auto-tts.yaml +++ b/qa/scenarios/media/webchat-auto-tts.yaml @@ -19,7 +19,7 @@ scenario: - docs/tools/media-overview.md - docs/concepts/qa-e2e-automation.md codeRefs: - - packages/speech-core/src/tts.ts + - src/tts/runtime-api.ts - src/gateway/server-methods/chat-webchat-media.ts - src/gateway/managed-image-attachments.ts - src/gateway/server-methods/artifacts.ts diff --git a/scripts/lib/extension-package-boundary.ts b/scripts/lib/extension-package-boundary.ts index 5c677d3b6608..0ef871cb64e0 100644 --- a/scripts/lib/extension-package-boundary.ts +++ b/scripts/lib/extension-package-boundary.ts @@ -287,7 +287,6 @@ export const EXTENSION_PACKAGE_BOUNDARY_XAI_PATHS = { "@openclaw/anthropic-vertex/api.js": ["./.boundary-stubs/anthropic-vertex-api.d.ts"], "@openclaw/ollama/api.js": ["./.boundary-stubs/ollama-api.d.ts"], "@openclaw/ollama/runtime-api.js": ["./.boundary-stubs/ollama-runtime-api.d.ts"], - "@openclaw/speech-core/runtime-api.js": ["./.boundary-stubs/speech-core-runtime-api.d.ts"], } as const; type ExtensionPackageBoundaryTsConfigJson = { diff --git a/scripts/lib/tsdown-output-roots.mjs b/scripts/lib/tsdown-output-roots.mjs index b1ed9a7981d4..d7dcaa14f420 100644 --- a/scripts/lib/tsdown-output-roots.mjs +++ b/scripts/lib/tsdown-output-roots.mjs @@ -13,7 +13,6 @@ const TSDOWN_PACKAGE_NAMES = [ "net-policy", "normalization-core", "retry", - "speech-core", "terminal-core", "acp-core", ]; diff --git a/src/agents/cli-runner.reliability.test.ts b/src/agents/cli-runner.reliability.test.ts index ec78b66eed8d..7d16594be296 100644 --- a/src/agents/cli-runner.reliability.test.ts +++ b/src/agents/cli-runner.reliability.test.ts @@ -97,6 +97,8 @@ vi.mock("../skills/research/autocapture.js", () => ({ vi.mock("../tts/tts-settings.js", () => ({ buildTtsSystemPromptHint: vi.fn(() => undefined), + resolveModelOverridePolicy: vi.fn(), + setTtsMachinePrefsPathResolver: vi.fn(), })); const mockGetGlobalHookRunner = vi.mocked(getGlobalHookRunner); diff --git a/src/agents/cli-runner/helpers.system-prompt.test.ts b/src/agents/cli-runner/helpers.system-prompt.test.ts index d07c3fd216be..d4a78be19358 100644 --- a/src/agents/cli-runner/helpers.system-prompt.test.ts +++ b/src/agents/cli-runner/helpers.system-prompt.test.ts @@ -5,6 +5,8 @@ import { buildCliAgentSystemPrompt } from "./helpers.js"; vi.mock("../../tts/tts-settings.js", () => ({ buildTtsSystemPromptHint: vi.fn(() => undefined), + resolveModelOverridePolicy: vi.fn(), + setTtsMachinePrefsPathResolver: vi.fn(), })); describe("buildCliAgentSystemPrompt", () => { diff --git a/src/agents/cli-runner/prepare.test.ts b/src/agents/cli-runner/prepare.test.ts index f41442825574..a26cf2cc41d3 100644 --- a/src/agents/cli-runner/prepare.test.ts +++ b/src/agents/cli-runner/prepare.test.ts @@ -100,6 +100,8 @@ vi.mock("../../plugins/hook-runner-global.js", () => ({ vi.mock("../../tts/tts-settings.js", () => ({ buildTtsSystemPromptHint: vi.fn(() => undefined), + resolveModelOverridePolicy: vi.fn(), + setTtsMachinePrefsPathResolver: vi.fn(), })); vi.mock("../video-generation-task-status.js", () => ({ diff --git a/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts b/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts index 06f4e1a7c4cc..de4777a45bff 100644 --- a/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts +++ b/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts @@ -466,6 +466,8 @@ vi.mock("../../../infra/net/undici-global-dispatcher.js", () => ({ vi.mock("../../../tts/tts-settings.js", () => ({ buildTtsSystemPromptHint: () => undefined, + resolveModelOverridePolicy: () => undefined, + setTtsMachinePrefsPathResolver: () => undefined, })); vi.mock("../../bootstrap-files.js", async () => { diff --git a/src/agents/embedded-agent-runner/system-prompt.test.ts b/src/agents/embedded-agent-runner/system-prompt.test.ts index 4ae50bbe9c4b..c0a7e2f81a0a 100644 --- a/src/agents/embedded-agent-runner/system-prompt.test.ts +++ b/src/agents/embedded-agent-runner/system-prompt.test.ts @@ -10,6 +10,8 @@ import { applySystemPromptToSession, buildEmbeddedSystemPrompt } from "./system- vi.mock("../../tts/tts-settings.js", () => ({ buildTtsSystemPromptHint: vi.fn(() => undefined), + resolveModelOverridePolicy: vi.fn(), + setTtsMachinePrefsPathResolver: vi.fn(), })); describe("applySystemPromptToSession", () => { diff --git a/src/agents/system-prompt-config.test.ts b/src/agents/system-prompt-config.test.ts index f23f1f9389e2..d31bf1388fa3 100644 --- a/src/agents/system-prompt-config.test.ts +++ b/src/agents/system-prompt-config.test.ts @@ -6,6 +6,8 @@ import { buildConfiguredAgentSystemPrompt } from "./system-prompt-config.js"; vi.mock("../tts/tts-settings.js", () => ({ buildTtsSystemPromptHint: vi.fn(() => undefined), + resolveModelOverridePolicy: vi.fn(), + setTtsMachinePrefsPathResolver: vi.fn(), })); function buildPrompt(config: OpenClawConfig, agentId = "main"): string { diff --git a/src/auto-reply/reply/commands-system-prompt.test.ts b/src/auto-reply/reply/commands-system-prompt.test.ts index 817bdc7b961d..8ad29a3e7fa0 100644 --- a/src/auto-reply/reply/commands-system-prompt.test.ts +++ b/src/auto-reply/reply/commands-system-prompt.test.ts @@ -71,6 +71,8 @@ vi.mock("../../agents/agent-tools.js", () => ({ vi.mock("../../tts/tts-settings.js", () => ({ buildTtsSystemPromptHint: vi.fn(() => undefined), + resolveModelOverridePolicy: vi.fn(), + setTtsMachinePrefsPathResolver: vi.fn(), })); function makeParams(): HandleCommandsParams { diff --git a/src/channels/plugins/types.core.ts b/src/channels/plugins/types.core.ts index 281e2f99e974..cf0b4a096640 100644 --- a/src/channels/plugins/types.core.ts +++ b/src/channels/plugins/types.core.ts @@ -254,7 +254,7 @@ export type ChannelGroupContext = { /** TTS voice delivery behavior advertised by a channel plugin. */ /** * Container tokens (file-extension shape, no leading dot) that the host - * speech-core pipeline knows how to pre-transcode synthesized audio into. + * TTS pipeline knows how to pre-transcode synthesized audio into. * Channels that benefit from a specific container — currently only * iMessage, which needs Apple's native voice-memo CAF descriptor — name * one here. Adding a new entry requires extending the host transcoder diff --git a/src/gateway/server-methods/talk-shared.ts b/src/gateway/server-methods/talk-shared.ts index 455a8b736f8b..8f18b000779f 100644 --- a/src/gateway/server-methods/talk-shared.ts +++ b/src/gateway/server-methods/talk-shared.ts @@ -7,12 +7,6 @@ import { normalizeOptionalString, } from "@openclaw/normalization-core/string-coerce"; import { ErrorCodes } from "../../../packages/gateway-protocol/src/index.js"; -import { - getVoiceProviderConfig, - providerMatchesId, - resolveSupportedVoiceModelRefs, - type VoiceModelProvider, -} from "../../../packages/speech-core/voice-models.js"; import { resolveRealtimeBootstrapContextInstructions } from "../../agents/realtime-bootstrap-context.js"; import type { TalkRealtimeConfig } from "../../config/types.gateway.js"; import type { OpenClawConfig } from "../../config/types.js"; @@ -32,6 +26,12 @@ import type { RealtimeVoiceProviderConfig, } from "../../talk/provider-types.js"; import type { TalkBrain, TalkEvent, TalkMode, TalkTransport } from "../../talk/talk-events.js"; +import { + getVoiceProviderConfig, + providerMatchesId, + resolveSupportedVoiceModelRefs, + type VoiceModelProvider, +} from "../../tts/voice-models.js"; import { ADMIN_SCOPE } from "../operator-scopes.js"; import type { TalkHandoffTurnResult } from "../talk-handoff.js"; diff --git a/src/gateway/server-methods/talk.ts b/src/gateway/server-methods/talk.ts index a8fab5fee74f..7bd8ac06fb83 100644 --- a/src/gateway/server-methods/talk.ts +++ b/src/gateway/server-methods/talk.ts @@ -15,15 +15,6 @@ import { validateTalkModeParams, validateTalkSpeakParams, } from "../../../packages/gateway-protocol/src/index.js"; -import { - withSpeakerSelectionCompat, - withSpeakerSelectionFallbackCompat, -} from "../../../packages/speech-core/speaker.js"; -import { - CODE_HEAVY_SPOKEN_FALLBACK, - isCodeHeavySpeechText, -} from "../../../packages/speech-core/src/speech-text.js"; -import { getVoiceProviderConfig } from "../../../packages/speech-core/voice-models.js"; import { readConfigFileSnapshot } from "../../config/config.js"; import { redactConfigObject } from "../../config/redact-snapshot.js"; import { @@ -54,12 +45,18 @@ import { getSpeechProvider, listSpeechProviders, } from "../../tts/provider-registry.js"; +import { + withSpeakerSelectionCompat, + withSpeakerSelectionFallbackCompat, +} from "../../tts/speaker.js"; +import { CODE_HEAVY_SPOKEN_FALLBACK, isCodeHeavySpeechText } from "../../tts/speech-text.js"; import { getResolvedSpeechProviderConfig, resolveTtsConfig, synthesizeSpeech, type TtsDirectiveOverrides, } from "../../tts/tts.js"; +import { getVoiceProviderConfig } from "../../tts/voice-models.js"; import { ADMIN_SCOPE, READ_SCOPE, TALK_SECRETS_SCOPE } from "../operator-scopes.js"; import { resolveConfiguredSecretInputString } from "../resolve-configured-secret-input-string.js"; import { formatForLog } from "../ws-log.js"; diff --git a/src/gateway/server.talk-runtime.test.ts b/src/gateway/server.talk-runtime.test.ts index 671a9a1d952c..4336ecf37eb6 100644 --- a/src/gateway/server.talk-runtime.test.ts +++ b/src/gateway/server.talk-runtime.test.ts @@ -2,7 +2,7 @@ * Tests gateway talk runtime wiring for speech provider execution. */ import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { CODE_HEAVY_SPOKEN_FALLBACK } from "../../packages/speech-core/src/speech-text.js"; +import { CODE_HEAVY_SPOKEN_FALLBACK } from "../tts/speech-text.js"; import { invokeTalkSpeakDirect, type TalkSpeakTestPayload, diff --git a/src/infra/outbound/reply-payload-parts.ts b/src/infra/outbound/reply-payload-parts.ts new file mode 100644 index 000000000000..239f11dbefb0 --- /dev/null +++ b/src/infra/outbound/reply-payload-parts.ts @@ -0,0 +1,71 @@ +import { normalizeStringEntries } from "../../../packages/normalization-core/src/string-normalization.js"; + +/** Derived sendability facts for text/media outbound payload delivery. */ +export type SendableOutboundReplyParts = { + /** Raw text selected for delivery before trimming. */ + text: string; + /** Text after trimming whitespace for sendability checks. */ + trimmedText: string; + /** Normalized non-empty media URLs. */ + mediaUrls: string[]; + /** Number of normalized media URLs. */ + mediaCount: number; + /** Whether trimmed text is sendable. */ + hasText: boolean; + /** Whether at least one media URL is sendable. */ + hasMedia: boolean; + /** Whether the payload has any sendable text or media. */ + hasContent: boolean; +}; + +/** Prefer multi-attachment payloads, then fall back to the legacy single-media field. */ +export function resolveOutboundMediaUrls(payload: { + mediaUrls?: string[]; + mediaUrl?: string; +}): string[] { + if (payload.mediaUrls?.length) { + return payload.mediaUrls; + } + if (payload.mediaUrl) { + return [payload.mediaUrl]; + } + return []; +} + +/** Count outbound media items after legacy single-media fallback normalization. */ +export function countOutboundMedia(payload: { mediaUrls?: string[]; mediaUrl?: string }): number { + return resolveOutboundMediaUrls(payload).length; +} + +/** Check whether an outbound payload includes any media after normalization. */ +export function hasOutboundMedia(payload: { mediaUrls?: string[]; mediaUrl?: string }): boolean { + return countOutboundMedia(payload) > 0; +} + +/** Check whether an outbound payload includes text, optionally trimming whitespace first. */ +export function hasOutboundText(payload: { text?: string }, options?: { trim?: boolean }): boolean { + const text = options?.trim ? payload.text?.trim() : payload.text; + return Boolean(text); +} + +/** Normalize reply payload text/media into a trimmed, sendable shape for delivery paths. */ +export function resolveSendableOutboundReplyParts( + payload: { text?: string; mediaUrls?: string[]; mediaUrl?: string }, + options?: { text?: string }, +): SendableOutboundReplyParts { + const text = options?.text ?? payload.text ?? ""; + const trimmedText = text.trim(); + const mediaUrls = normalizeStringEntries(resolveOutboundMediaUrls(payload)); + const mediaCount = mediaUrls.length; + const hasText = Boolean(trimmedText); + const hasMedia = mediaCount > 0; + return { + text, + trimmedText, + mediaUrls, + mediaCount, + hasText, + hasMedia, + hasContent: hasText || hasMedia, + }; +} diff --git a/src/plugin-sdk/facade-runtime.test.ts b/src/plugin-sdk/facade-runtime.test.ts index 75c253aab303..452676556f45 100644 --- a/src/plugin-sdk/facade-runtime.test.ts +++ b/src/plugin-sdk/facade-runtime.test.ts @@ -700,7 +700,7 @@ describe("plugin-sdk facade runtime", () => { } }); - it("does not treat package-backed speech-core as a bundled extension facade", () => { + it("does not treat the core-owned speech runtime as a bundled extension facade", () => { setRuntimeConfigSnapshot({}); expect( diff --git a/src/plugin-sdk/reply-payload.ts b/src/plugin-sdk/reply-payload.ts index 50059bcfdb58..c6e5df1673bf 100644 --- a/src/plugin-sdk/reply-payload.ts +++ b/src/plugin-sdk/reply-payload.ts @@ -1,9 +1,15 @@ // Reply payload helpers normalize plugin reply targets, text, media, and approval metadata. import { normalizeLowercaseStringOrEmpty } from "../../packages/normalization-core/src/string-coerce.js"; -import { normalizeStringEntries } from "../../packages/normalization-core/src/string-normalization.js"; import type { ReplyPayload as InternalReplyPayload } from "../auto-reply/reply-payload.js"; import type { ChannelOutboundAdapter } from "../channels/plugins/outbound.types.js"; import { normalizeOutboundReplyPayload as normalizeCoreOutboundReplyPayload } from "../infra/outbound/reply-payload-normalize.js"; +import { + countOutboundMedia, + hasOutboundMedia, + hasOutboundText, + resolveOutboundMediaUrls, + resolveSendableOutboundReplyParts, +} from "../infra/outbound/reply-payload-parts.js"; import { createReplyToFanout } from "../infra/outbound/reply-policy.js"; import { hasReplyPayloadContent } from "../interactive/payload.js"; @@ -68,21 +74,13 @@ export type ReasoningReplyPayload = { }; /** Derived sendability facts for text/media outbound payload delivery. */ -export type SendableOutboundReplyParts = { - /** Raw text selected for delivery before trimming. */ - text: string; - /** Text after trimming whitespace for sendability checks. */ - trimmedText: string; - /** Normalized non-empty media URLs. */ - mediaUrls: string[]; - /** Number of normalized media URLs. */ - mediaCount: number; - /** Whether trimmed text is sendable. */ - hasText: boolean; - /** Whether at least one media URL is sendable. */ - hasMedia: boolean; - /** Whether the payload has any sendable text or media. */ - hasContent: boolean; +export type { SendableOutboundReplyParts } from "../infra/outbound/reply-payload-parts.js"; +export { + countOutboundMedia, + hasOutboundMedia, + hasOutboundText, + resolveOutboundMediaUrls, + resolveSendableOutboundReplyParts, }; type SendPayloadContext = Parameters>[0]; @@ -139,41 +137,11 @@ export function createNormalizedOutboundDeliverer( }; } -/** Prefer multi-attachment payloads, then fall back to the legacy single-media field. */ -export function resolveOutboundMediaUrls(payload: { - mediaUrls?: string[]; - mediaUrl?: string; -}): string[] { - if (payload.mediaUrls?.length) { - return payload.mediaUrls; - } - if (payload.mediaUrl) { - return [payload.mediaUrl]; - } - return []; -} - /** Resolve media URLs from a channel sendPayload context after legacy fallback normalization. */ export function resolvePayloadMediaUrls(payload: SendPayloadContext["payload"]): string[] { return resolveOutboundMediaUrls(payload); } -/** Count outbound media items after legacy single-media fallback normalization. */ -export function countOutboundMedia(payload: { mediaUrls?: string[]; mediaUrl?: string }): number { - return resolveOutboundMediaUrls(payload).length; -} - -/** Check whether an outbound payload includes any media after normalization. */ -export function hasOutboundMedia(payload: { mediaUrls?: string[]; mediaUrl?: string }): boolean { - return countOutboundMedia(payload) > 0; -} - -/** Check whether an outbound payload includes text, optionally trimming whitespace first. */ -export function hasOutboundText(payload: { text?: string }, options?: { trim?: boolean }): boolean { - const text = options?.trim ? payload.text?.trim() : payload.text; - return Boolean(text); -} - /** Check whether an outbound payload includes any sendable text, media, or rich reply content. */ export function hasOutboundReplyContent( payload: { @@ -189,28 +157,6 @@ export function hasOutboundReplyContent( return hasReplyPayloadContent(payload, { trimText: options?.trimText }); } -/** Normalize reply payload text/media into a trimmed, sendable shape for delivery paths. */ -export function resolveSendableOutboundReplyParts( - payload: { text?: string; mediaUrls?: string[]; mediaUrl?: string }, - options?: { text?: string }, -): SendableOutboundReplyParts { - const text = options?.text ?? payload.text ?? ""; - const trimmedText = text.trim(); - const mediaUrls = normalizeStringEntries(resolveOutboundMediaUrls(payload)); - const mediaCount = mediaUrls.length; - const hasText = Boolean(trimmedText); - const hasMedia = mediaCount > 0; - return { - text, - trimmedText, - mediaUrls, - mediaCount, - hasText, - hasMedia, - hasContent: hasText || hasMedia, - }; -} - /** Preserve caller-provided chunking, but fall back to the full text when chunkers return nothing. */ export function resolveTextChunksWithFallback(text: string, chunks: readonly string[]): string[] { if (chunks.length > 0) { diff --git a/src/plugin-sdk/tts-runtime.ts b/src/plugin-sdk/tts-runtime.ts index 97f0cb1aed36..896bc89f8d85 100644 --- a/src/plugin-sdk/tts-runtime.ts +++ b/src/plugin-sdk/tts-runtime.ts @@ -1,17 +1,6 @@ -// TTS runtime exports expose text-to-speech runtime helpers through the plugin SDK. -import { maybeApplyTtsToPayload as maybeApplyTtsToPayloadCore } from "../../packages/speech-core/src/tts-payload.js"; -import { textToSpeech as textToSpeechCore } from "../../packages/speech-core/src/tts-synthesis.js"; -import { persistTtsAudioToMediaStore } from "../tts/tts-audio-store.js"; - -export type { TtsResult } from "../../packages/speech-core/src/tts-types.js"; - -export function textToSpeech(params: Parameters[0]) { - return textToSpeechCore(params, persistTtsAudioToMediaStore); -} - -export function maybeApplyTtsToPayload(params: Parameters[0]) { - return maybeApplyTtsToPayloadCore(params, persistTtsAudioToMediaStore); -} +// TTS runtime exports expose host-owned text-to-speech helpers through the plugin SDK. +export { maybeApplyTtsToPayload, textToSpeech } from "../tts/tts.js"; +export type { TtsResult } from "../tts/tts-runtime-types.js"; export { TtsAutoSchema, @@ -23,8 +12,6 @@ export { /** Compatibility no-op retained for callers that prewarm facade runtimes generically. */ export function prewarmTtsRuntimeFacade(): void {} -// Pure synthesis stays in speech-core. File-backed helpers above inject the -// core media-store owner so package code never imports from src. export { buildTtsSystemPromptHint, getLastTtsAttempt, @@ -63,4 +50,4 @@ export { type TtsSynthesisStreamResult, type TtsStreamResult, type TtsTelephonyResult, -} from "../../packages/speech-core/runtime-api.js"; +} from "../tts/runtime-api.js"; diff --git a/src/plugins/capability-provider-runtime.ts b/src/plugins/capability-provider-runtime.ts index 2d7085ce82f2..75a28dc732b0 100644 --- a/src/plugins/capability-provider-runtime.ts +++ b/src/plugins/capability-provider-runtime.ts @@ -1,6 +1,6 @@ import { sortUniqueStrings } from "@openclaw/normalization-core/string-normalization"; -import { resolveVoiceModelRefs } from "../../packages/speech-core/voice-models.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { resolveVoiceModelRefs } from "../tts/voice-models.js"; import { getLoadedRuntimePluginRegistry, registryContainsRuntimePluginIds, diff --git a/src/plugins/model-catalog-registration.ts b/src/plugins/model-catalog-registration.ts index 2c2404e2fd86..1db05fec4d65 100644 --- a/src/plugins/model-catalog-registration.ts +++ b/src/plugins/model-catalog-registration.ts @@ -14,7 +14,7 @@ import { synthesizeVoiceModelCatalogEntries, type VoiceModelCapabilities, type VoiceModelProvider, -} from "../../packages/speech-core/voice-models.js"; +} from "../tts/voice-models.js"; import type { PluginDiagnostic } from "./manifest-types.js"; import { projectProviderCatalogResultToUnifiedTextRows } from "./provider-catalog-unified-text.js"; import type { PluginRecord, PluginRegistry } from "./registry-types.js"; diff --git a/src/plugins/runtime/runtime-tts-request.ts b/src/plugins/runtime/runtime-tts-request.ts index e735cd4a6a25..1376af3261e8 100644 --- a/src/plugins/runtime/runtime-tts-request.ts +++ b/src/plugins/runtime/runtime-tts-request.ts @@ -1,2 +1,2 @@ -// Lazy runtime bridge for speech-core request pre-resolution. -export { prepareTtsRequest } from "../../../packages/speech-core/runtime-api.js"; +// Lazy runtime bridge for TTS request pre-resolution. +export { prepareTtsRequest } from "../../tts/runtime-api.js"; diff --git a/packages/speech-core/src/tts.ts b/src/tts/runtime-api.ts similarity index 57% rename from packages/speech-core/src/tts.ts rename to src/tts/runtime-api.ts index 0c736b24b236..eaf9b7d8bccf 100644 --- a/packages/speech-core/src/tts.ts +++ b/src/tts/runtime-api.ts @@ -1,44 +1,62 @@ -import type { TtsProvider } from "openclaw/plugin-sdk/config-contracts"; -import { parseTtsDirectives, summarizeText } from "openclaw/plugin-sdk/speech-core"; +// Runtime speech API barrel for TTS preferences, synthesis, streaming, and test +// helpers used by speech-capable plugins. +import type { TtsProvider } from "../config/types.js"; +import { parseTtsDirectives } from "./directives.js"; +import { summarizeText } from "./tts-core.js"; import { getResolvedSpeechProviderConfig, resolveTtsProvider } from "./tts-provider-resolution.js"; import { resolveModelOverridePolicy, type ResolvedTtsConfig } from "./tts-settings.js"; import { formatTtsProviderError, sanitizeTtsErrorForLog } from "./tts-synthesis-support.js"; import { + resolveTtsSynthesisTarget, shouldDeliverTtsAsVoice, supportsNativeVoiceNoteTts, supportsTranscodedVoiceNoteTts, - resolveTtsSynthesisTarget, } from "./tts-synthesis.js"; -export type { - TtsDirectiveOverrides, - TtsDirectiveParseResult, -} from "openclaw/plugin-sdk/speech-core"; - -export function getTtsProvider(config: ResolvedTtsConfig, prefsPath: string): TtsProvider { - return resolveTtsProvider(config, prefsPath); -} - +export { setSpeechRuntimeAvailabilityGuard } from "./runtime-availability.js"; +export { + buildTtsSystemPromptHint, + getTtsMaxLength, + getTtsPersona, + isSummarizationEnabled, + isTtsEnabled, + listTtsPersonas, + resolveTtsAutoMode, + resolveTtsConfig, + resolveTtsPrefsPath, + setTtsMachinePrefsPathResolver, + type ResolvedTtsConfig, + type ResolvedTtsModelOverrides, +} from "./tts-settings.js"; +export { + setSummarizationEnabled, + setTtsAutoMode, + setTtsEnabled, + setTtsMaxLength, + setTtsPersona, + setTtsProvider, +} from "./tts-settings-writes.js"; export { getLastTtsAttempt, listSpeechVoices, setLastTtsAttempt } from "./tts-payload.js"; export { getResolvedSpeechProviderConfig, isTtsProviderConfigured, resolveTtsProviderOrder, } from "./tts-provider-resolution.js"; -export { - prepareTtsRequest, - resolveExplicitTtsOverrides, - type PreparedTtsRequest, -} from "./tts-request.js"; +export { prepareTtsRequest, resolveExplicitTtsOverrides } from "./tts-request.js"; export { streamSpeech, textToSpeechStream } from "./tts-streaming.js"; export { synthesizeSpeech } from "./tts-synthesis.js"; export { textToSpeechTelephony } from "./tts-telephony.js"; +export type { TtsDirectiveOverrides, TtsDirectiveParseResult } from "./provider-types.js"; export type { TtsStreamResult, TtsSynthesisResult, TtsSynthesisStreamResult, TtsTelephonyResult, -} from "./tts-types.js"; +} from "./tts-runtime-types.js"; + +export function getTtsProvider(config: ResolvedTtsConfig, prefsPath: string): TtsProvider { + return resolveTtsProvider(config, prefsPath); +} export const testApi = { parseTtsDirectives, diff --git a/packages/speech-core/src/runtime-availability.ts b/src/tts/runtime-availability.ts similarity index 89% rename from packages/speech-core/src/runtime-availability.ts rename to src/tts/runtime-availability.ts index 56aeaf1a7e67..ff565ce92a15 100644 --- a/packages/speech-core/src/runtime-availability.ts +++ b/src/tts/runtime-availability.ts @@ -1,4 +1,4 @@ -/** Host-owned availability guard shared by every speech-core entrypoint. */ +/** Host-owned availability guard shared by every speech runtime entrypoint. */ let assertRuntimeAvailable: (() => void) | undefined; diff --git a/packages/speech-core/speaker.ts b/src/tts/speaker.ts similarity index 96% rename from packages/speech-core/speaker.ts rename to src/tts/speaker.ts index cb730fbd4a15..f1705d2f0953 100644 --- a/packages/speech-core/speaker.ts +++ b/src/tts/speaker.ts @@ -1,6 +1,6 @@ // Speaker-selection compatibility helpers for plugins that renamed voice fields // over time but still need one normalized config object. -export type SpeakerSelectionConfig = Record; +type SpeakerSelectionConfig = Record; function readString(value: unknown): string | undefined { return typeof value === "string" && value.trim() ? value.trim() : undefined; diff --git a/packages/speech-core/src/speech-text.test.ts b/src/tts/speech-text.test.ts similarity index 98% rename from packages/speech-core/src/speech-text.test.ts rename to src/tts/speech-text.test.ts index da1dbbb820c6..8e871cdab91d 100644 --- a/packages/speech-core/src/speech-text.test.ts +++ b/src/tts/speech-text.test.ts @@ -1,5 +1,5 @@ -import { stripMarkdown } from "openclaw/plugin-sdk/text-chunking"; import { describe, expect, it } from "vitest"; +import { stripMarkdown } from "../shared/text/strip-markdown.js"; import { CODE_HEAVY_SPOKEN_FALLBACK, isCodeHeavySpeechText, diff --git a/packages/speech-core/src/speech-text.ts b/src/tts/speech-text.ts similarity index 98% rename from packages/speech-core/src/speech-text.ts rename to src/tts/speech-text.ts index e53359241402..71ca409b63f3 100644 --- a/packages/speech-core/src/speech-text.ts +++ b/src/tts/speech-text.ts @@ -1,4 +1,4 @@ -import { stripMarkdown } from "openclaw/plugin-sdk/text-chunking"; +import { stripMarkdown } from "../shared/text/strip-markdown.js"; export const CODE_HEAVY_SPOKEN_FALLBACK = "I've put the detailed response on screen."; diff --git a/src/tts/tts-audio-store.ts b/src/tts/tts-audio-store.ts index 48ea8afbf954..9a733cec9475 100644 --- a/src/tts/tts-audio-store.ts +++ b/src/tts/tts-audio-store.ts @@ -1,8 +1,8 @@ -// File-backed TTS output is owned by the canonical media store, not speech-core. +// File-backed TTS output is owned by the canonical media store. import { mimeTypeFromFilePath } from "@openclaw/media-core/mime"; -import type { TtsAudioPersistence } from "../../packages/speech-core/src/tts-synthesis.js"; import { resolveGeneratedMediaMaxBytes } from "../media/configured-max-bytes.js"; import { saveMediaBuffer } from "../media/store.js"; +import type { TtsAudioPersistence } from "./tts-synthesis.js"; const TTS_MEDIA_SUBDIR = "tool-speech-synthesis"; diff --git a/packages/speech-core/src/tts-payload.ts b/src/tts/tts-payload.ts similarity index 92% rename from packages/speech-core/src/tts-payload.ts rename to src/tts/tts-payload.ts index 63baa369d04b..33524db196d9 100644 --- a/packages/speech-core/src/tts-payload.ts +++ b/src/tts/tts-payload.ts @@ -1,25 +1,20 @@ -import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import { - markReplyPayloadAsTtsSupplement, - resolveSendableOutboundReplyParts, - type ReplyPayload, -} from "openclaw/plugin-sdk/reply-payload"; -import { isVerbose, logVerbose } from "openclaw/plugin-sdk/runtime-env"; -import { - canonicalizeSpeechProviderId, - getSpeechProvider, - parseTtsDirectives, - summarizeText, - type SpeechVoiceOption, -} from "openclaw/plugin-sdk/speech-core"; -import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; +import { markReplyPayloadAsTtsSupplement, type ReplyPayload } from "../auto-reply/reply-payload.js"; +import type { OpenClawConfig } from "../config/types.js"; +import { isVerbose, logVerbose } from "../globals.js"; +import { resolveSendableOutboundReplyParts } from "../infra/outbound/reply-payload-parts.js"; +import { truncateUtf16Safe } from "../utils.js"; +import { parseTtsDirectives } from "./directives.js"; +import { canonicalizeSpeechProviderId, getSpeechProvider } from "./provider-registry.js"; +import type { SpeechVoiceOption } from "./provider-types.js"; import { assertSpeechRuntimeAvailable, isSpeechRuntimeAvailable } from "./runtime-availability.js"; import { isCodeHeavySpeechText, normalizeSpeechText } from "./speech-text.js"; +import { summarizeText } from "./tts-core.js"; import { getResolvedSpeechProviderConfig, resolveSpeechProviderTimeoutMs, resolveTtsProvider, } from "./tts-provider-resolution.js"; +import type { TtsStatusEntry } from "./tts-runtime-types.js"; import { getTtsMaxLength, isSummarizationEnabled, @@ -29,7 +24,6 @@ import { type ResolvedTtsConfig, } from "./tts-settings.js"; import { textToSpeech, type TtsAudioPersistence } from "./tts-synthesis.js"; -import type { TtsStatusEntry } from "./tts-types.js"; let lastTtsAttempt: TtsStatusEntry | undefined; diff --git a/packages/speech-core/src/tts-provider-resolution.ts b/src/tts/tts-provider-resolution.ts similarity index 96% rename from packages/speech-core/src/tts-provider-resolution.ts rename to src/tts/tts-provider-resolution.ts index d5c811e78bb4..06317b038f63 100644 --- a/packages/speech-core/src/tts-provider-resolution.ts +++ b/src/tts/tts-provider-resolution.ts @@ -1,33 +1,23 @@ +import { clampTimerTimeoutMs } from "../../packages/normalization-core/src/number-coercion.js"; +import { + normalizeLowercaseStringOrEmpty, + normalizeOptionalString, +} from "../../packages/normalization-core/src/string-coerce.js"; import type { OpenClawConfig, ResolvedTtsPersona, TtsConfig, TtsProvider, -} from "openclaw/plugin-sdk/config-contracts"; -import { clampTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime"; +} from "../config/types.js"; +import type { SpeechProviderPlugin } from "../plugins/types.js"; import { canonicalizeSpeechProviderId, getSpeechProvider, listSpeechProviders, normalizeSpeechProviderId, - type SpeechProviderConfig, - type SpeechProviderPlugin, -} from "openclaw/plugin-sdk/speech-core"; -import { - normalizeLowercaseStringOrEmpty, - normalizeOptionalString, -} from "openclaw/plugin-sdk/string-coerce-runtime"; -import { withSpeakerSelectionCompat } from "../speaker.js"; -import { - resolvePrimaryVoiceProviderCandidate, - resolveSupportedVoiceModelRefs, - resolveVoiceModelRefs, - resolveVoiceProviderCandidates, - voiceProviderSupportsModel, - type VoiceModelProvider, - type VoiceModelRef, - type VoiceProviderCandidate, -} from "../voice-models.js"; +} from "./provider-registry.js"; +import type { SpeechProviderConfig } from "./provider-types.js"; +import { withSpeakerSelectionCompat } from "./speaker.js"; import { DEFAULT_TTS_TIMEOUT_MS, asProviderConfig, @@ -39,6 +29,16 @@ import { resolveTtsRuntimeConfig, type ResolvedTtsConfig, } from "./tts-settings.js"; +import { + resolvePrimaryVoiceProviderCandidate, + resolveSupportedVoiceModelRefs, + resolveVoiceModelRefs, + resolveVoiceProviderCandidates, + voiceProviderSupportsModel, + type VoiceModelProvider, + type VoiceModelRef, + type VoiceProviderCandidate, +} from "./voice-models.js"; function resolvePositiveTimeoutMs(timeoutMs: number | undefined): number | undefined { return typeof timeoutMs === "number" && Number.isFinite(timeoutMs) && timeoutMs > 0 diff --git a/packages/speech-core/src/tts-request.ts b/src/tts/tts-request.ts similarity index 87% rename from packages/speech-core/src/tts-request.ts rename to src/tts/tts-request.ts index 8466a0ad7506..762e51fbd81c 100644 --- a/packages/speech-core/src/tts-request.ts +++ b/src/tts/tts-request.ts @@ -1,17 +1,16 @@ -import type { OpenClawConfig, TtsConfig } from "openclaw/plugin-sdk/config-contracts"; -import { mergeDeep } from "openclaw/plugin-sdk/plugin-config-runtime"; -import { - canonicalizeSpeechProviderId, - getSpeechProvider, - parseTtsDirectives, - type SpeechProviderOverrides, - type TtsDirectiveOverrides, - type TtsDirectiveParseResult, -} from "openclaw/plugin-sdk/speech-core"; +import type { OpenClawConfig, TtsConfig } from "../config/types.js"; +import { mergeDeep } from "../infra/deep-merge.js"; +import { parseTtsDirectives } from "./directives.js"; +import { canonicalizeSpeechProviderId, getSpeechProvider } from "./provider-registry.js"; +import type { + SpeechProviderOverrides, + TtsDirectiveOverrides, + TtsDirectiveParseResult, +} from "./provider-types.js"; import { resolveTtsProvider } from "./tts-provider-resolution.js"; import { resolveTtsConfig, resolveTtsPrefsPath, resolveTtsRuntimeConfig } from "./tts-settings.js"; -export type PreparedTtsRequest = { +type PreparedTtsRequest = { cfg: OpenClawConfig; directives: TtsDirectiveParseResult; }; diff --git a/src/tts/tts-runtime-fallbacks.test.ts b/src/tts/tts-runtime-fallbacks.test.ts new file mode 100644 index 000000000000..e6364038f95c --- /dev/null +++ b/src/tts/tts-runtime-fallbacks.test.ts @@ -0,0 +1,462 @@ +import { rmSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + clearRuntimeConfigSnapshot, + createMockSpeechProvider, + createTtsConfig, + expectTtsPayloadResult, + installSpeechProviders, + maybeApplyTtsToPayload, + maybeApplyTtsToPayloadCore, + prefsPathFor, + prepareSynthesisMock, + requireFirstSynthesisRequest, + requireRecord, + setSummarizationEnabled, + setTtsMachinePrefsPathResolver, + setTtsMaxLength, + synthesizeMock, + synthesizeSpeech, + testApi, + transcodeAudioBufferMock, + type OpenClawConfig, +} from "./tts-runtime.test-support.js"; + +describe("TTS runtime provider fallback and delivery behavior", () => { + afterEach(() => { + setTtsMachinePrefsPathResolver(); + clearRuntimeConfigSnapshot(); + delete (Object.prototype as Record).polluted; + synthesizeMock.mockClear(); + prepareSynthesisMock.mockClear(); + transcodeAudioBufferMock.mockClear(); + installSpeechProviders([createMockSpeechProvider()]); + }); + + it("ignores voiceModel refs that are not speech models", async () => { + installSpeechProviders([ + createMockSpeechProvider("openai", { + autoSelectOrder: 10, + defaultModel: "gpt-4o-mini-tts", + models: ["gpt-4o-mini-tts"], + resolveConfig: ({ rawConfig }) => { + const providers = requireRecord(rawConfig.providers, "raw provider configs"); + return { + model: "gpt-4o-mini-tts", + modelId: "gpt-4o-mini-tts", + ...requireRecord(providers.openai, "raw openai provider config"), + }; + }, + }), + ]); + + const result = await synthesizeSpeech({ + text: "Use speech provider default for unsupported realtime model.", + cfg: { + agents: { + defaults: { + voiceModel: { primary: "openai/gpt-realtime-2" }, + }, + }, + tts: { + enabled: true, + provider: "openai", + prefsPath: "/tmp/openclaw-speech-core-realtime-voice-model-ignored-test.json", + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("openai"); + expect(result.providerModel).toBe("gpt-4o-mini-tts"); + const request = requireFirstSynthesisRequest("speech model fallback request"); + expect(request.providerConfig).toMatchObject({ + model: "gpt-4o-mini-tts", + modelId: "gpt-4o-mini-tts", + }); + }); + + it("uses the first speech-supported voiceModel fallback as the default provider", async () => { + installSpeechProviders([ + createMockSpeechProvider("openai", { + autoSelectOrder: 1, + models: ["gpt-4o-mini-tts"], + }), + createMockSpeechProvider("elevenlabs", { + autoSelectOrder: 99, + models: ["eleven_multilingual_v2"], + }), + ]); + + const result = await synthesizeSpeech({ + text: "Use first speech-supported voice model.", + cfg: { + agents: { + defaults: { + voiceModel: { + primary: "openai/gpt-realtime-2", + fallbacks: ["elevenlabs/eleven_multilingual_v2"], + }, + }, + }, + tts: { + enabled: true, + prefsPath: "/tmp/openclaw-speech-core-supported-voice-model-provider-test.json", + }, + } as OpenClawConfig, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("elevenlabs"); + expect(result.providerModel).toBe("eleven_multilingual_v2"); + expect(result.attemptedProviders).toEqual(["elevenlabs"]); + }); + + it("maps speakerVoice provider config to provider-compatible voice fields", async () => { + const result = await synthesizeSpeech({ + text: "Use the configured speaker.", + cfg: { + tts: { + enabled: true, + provider: "mock", + providers: { + mock: { + speakerVoice: "cedar", + speakerVoiceId: "voice-123", + voice: "legacy-voice", + voiceName: "legacy-name", + voiceId: "legacy-id", + }, + }, + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + expect(result.providerVoice).toBe("voice-123"); + const request = requireFirstSynthesisRequest("speaker voice synthesis request"); + expect(request.providerConfig).toMatchObject({ + speakerVoice: "cedar", + voice: "cedar", + voiceName: "cedar", + speakerVoiceId: "voice-123", + voiceId: "voice-123", + }); + }); + + it("preserves alias-keyed provider config when resolving canonical providers", async () => { + installSpeechProviders([ + createMockSpeechProvider("xiaomi", { + aliases: ["mimo"], + resolveConfig: ({ rawConfig }) => { + const providers = requireRecord(rawConfig.providers, "raw provider configs"); + return requireRecord(providers.xiaomi ?? providers.mimo, "raw xiaomi provider config"); + }, + }), + ]); + + const result = await synthesizeSpeech({ + text: "Use alias provider config.", + cfg: { + tts: { + enabled: true, + provider: "xiaomi", + providers: { + mimo: { apiKey: "fake" }, + }, + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("alias provider synthesis request"); + expect(request.providerConfig).toMatchObject({ apiKey: "fake" }); + }); + + it("maps speakerVoice persona provider config to provider-compatible voice fields", async () => { + const result = await synthesizeSpeech({ + text: "Use the persona speaker.", + cfg: { + tts: { + enabled: true, + provider: "mock", + persona: "narrator", + personas: { + narrator: { + providers: { + mock: { + speakerVoice: "marin", + }, + }, + }, + }, + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + expect(result.providerVoice).toBe("marin"); + const request = requireFirstSynthesisRequest("persona speaker voice synthesis request"); + expect(request.providerConfig).toMatchObject({ + speakerVoice: "marin", + voice: "marin", + voiceName: "marin", + }); + }); + + it.each(["feishu", "whatsapp"] as const)( + "marks %s voice-note TTS for channel-side transcoding when provider returns mp3", + async (channel) => { + expect(testApi.supportsTranscodedVoiceNoteTts(channel)).toBe(true); + await expectTtsPayloadResult({ + channel, + prefsName: `openclaw-speech-core-tts-${channel}-mp3-test`, + text: `This ${channel} reply should be transcoded by the channel.`, + target: "voice-note", + audioAsVoice: true, + mediaExtension: "mp3", + providerResult: { + audioBuffer: Buffer.from("mp3"), + outputFormat: "mp3", + fileExtension: ".mp3", + voiceCompatible: false, + }, + }); + }, + ); + + it("keeps non-native voice-note channels as regular audio files", async () => { + await expectTtsPayloadResult({ + channel: "slack", + prefsName: "openclaw-speech-core-tts-slack-test", + text: "Slack replies should be delivered as regular audio attachments.", + target: "audio-file", + audioAsVoice: undefined, + }); + }); + + it("preserves the text reply when auto-TTS audio persistence fails", async () => { + const payload = { text: "This text must still be delivered when media storage rejects audio." }; + const result = await maybeApplyTtsToPayloadCore( + { + payload, + cfg: createTtsConfig("openclaw-speech-core-auto-persistence-failure-test"), + channel: "slack", + kind: "final", + }, + async () => { + throw new Error("Media exceeds configured limit"); + }, + ); + + expect(result).toBe(payload); + }); + + it("normalizes voice-note Markdown once before synthesis", async () => { + const text = + 'This short explanation keeps the fenced literal below from becoming code-heavy.\n\n```md\nconst literal = "[x](y)";\n```'; + let mediaDir: string | undefined; + try { + const result = await maybeApplyTtsToPayload({ + payload: { text }, + cfg: createTtsConfig("openclaw-speech-core-once-normalized-markdown-test"), + channel: "telegram", + kind: "final", + }); + + const request = requireFirstSynthesisRequest("once-normalized voice-note synthesis request"); + expect(request.text).toBe( + 'This short explanation keeps the fenced literal below from becoming code-heavy.\n\nconst literal = "[x](y)";', + ); + expect(result.text).toBe(text); + mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; + } finally { + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } + }); + + it("skips channel auto-TTS audio for code-heavy replies", async () => { + const text = "```ts\nexport function answer() {\n return 42;\n}\n```"; + const result = await maybeApplyTtsToPayload({ + payload: { text }, + cfg: createTtsConfig("openclaw-speech-core-code-heavy-voice-note-test"), + channel: "telegram", + kind: "final", + }); + + expect(synthesizeMock).not.toHaveBeenCalled(); + expect(result).toEqual({ text }); + }); + + it("synthesizes code-heavy explicitly tagged hidden TTS text", async () => { + const cfg = createTtsConfig("openclaw-speech-core-code-heavy-hidden-tts-test"); + let mediaDir: string | undefined; + try { + const result = await maybeApplyTtsToPayload({ + payload: { + text: '[[tts:text]]```ts\nconst detailedAnswer = "this code should still be spoken";\n```[[/tts:text]]', + audioAsVoice: true, + }, + cfg, + channel: "telegram", + kind: "final", + }); + + expect(synthesizeMock).toHaveBeenCalled(); + const request = requireFirstSynthesisRequest("code-heavy hidden TTS request"); + expect(request.text).toBe('const detailedAnswer = "this code should still be spoken";'); + expect(result.text).toBeUndefined(); + mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; + } finally { + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } + }); + + it("synthesizes explicitly tagged short hidden TTS text", async () => { + const cfg = createTtsConfig("openclaw-speech-core-short-hidden-tts-test"); + let mediaDir: string | undefined; + try { + const result = await maybeApplyTtsToPayload({ + payload: { + text: "[[tts:text]]hello[[/tts:text]]", + audioAsVoice: true, + }, + cfg, + channel: "telegram", + kind: "final", + }); + + expect(synthesizeMock).toHaveBeenCalled(); + const request = requireFirstSynthesisRequest("hidden TTS request"); + expect(request.text).toBe("hello"); + expect(result.mediaUrl).toMatch(/voice---[a-f0-9-]+\.ogg$/); + expect(result.audioAsVoice).toBe(true); + expect(result.text).toBeUndefined(); + expect(result.ttsSupplement).toBeUndefined(); + mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; + } finally { + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } + }); + + it("truncates long TTS text on a UTF-16 boundary", async () => { + const prefsName = "openclaw-speech-core-utf16-truncate-test"; + const prefsPath = prefsPathFor(prefsName); + const cfg = createTtsConfig(prefsName); + setTtsMaxLength(prefsPath, 11); + setSummarizationEnabled(prefsPath, false); + let mediaDir: string | undefined; + try { + const result = await maybeApplyTtsToPayload({ + payload: { text: `${"a".repeat(7)}😀tail long enough for TTS` }, + cfg, + channel: "telegram", + kind: "final", + }); + + expect(synthesizeMock).toHaveBeenCalled(); + const request = requireFirstSynthesisRequest("utf16 truncated TTS request"); + const spokenText = String(request.text); + expect(spokenText).toBe(`${"a".repeat(7)}...`); + expect(result.spokenText).toBe(spokenText); + mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; + } finally { + rmSync(prefsPath, { force: true }); + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } + }); + + it("skips block delivery kind in final mode (accumulated final tail synthesizes instead)", async () => { + synthesizeMock.mockClear(); + const cfg = createTtsConfig("openclaw-speech-core-block-kind-tts-test"); + const result = await maybeApplyTtsToPayload({ + payload: { text: "WebChat block stream chunks defer TTS to the final tail." }, + cfg, + channel: "webchat", + kind: "block", + }); + + expect(synthesizeMock).not.toHaveBeenCalled(); + expect((result as { trustedLocalMedia?: boolean }).trustedLocalMedia).toBeUndefined(); + expect(result.text).toBe("WebChat block stream chunks defer TTS to the final tail."); + }); + + it("skips tool delivery kind in final mode", async () => { + synthesizeMock.mockClear(); + const cfg = createTtsConfig("openclaw-speech-core-tool-kind-tts-test"); + const result = await maybeApplyTtsToPayload({ + payload: { text: "Intermediate tool output should not be spoken." }, + cfg, + channel: "webchat", + kind: "tool", + }); + + expect(synthesizeMock).not.toHaveBeenCalled(); + expect((result as { trustedLocalMedia?: boolean }).trustedLocalMedia).toBeUndefined(); + expect(result.text).toBe("Intermediate tool output should not be spoken."); + }); + + it("keeps skipping untagged short TTS text", async () => { + const cfg = createTtsConfig("openclaw-speech-core-short-plain-tts-test"); + const result = await maybeApplyTtsToPayload({ + payload: { + text: "hello", + audioAsVoice: true, + }, + cfg, + channel: "telegram", + kind: "final", + }); + + expect(synthesizeMock).not.toHaveBeenCalled(); + expect(result).toEqual({ + text: "hello", + audioAsVoice: true, + }); + }); + + it("skips auto TTS for legacy final media directives", async () => { + synthesizeMock.mockClear(); + const cfg = createTtsConfig("openclaw-speech-core-media-directive-tts-test"); + const result = await maybeApplyTtsToPayload({ + payload: { text: "Here is the render.\nMEDIA:/tmp/render.png" }, + cfg, + channel: "telegram", + kind: "final", + }); + + expect(synthesizeMock).not.toHaveBeenCalled(); + expect(result).toEqual({ text: "Here is the render.\nMEDIA:/tmp/render.png" }); + }); + + it("keeps skipping explicit tagged TTS text that strips to empty markdown", async () => { + const cfg = createTtsConfig("openclaw-speech-core-empty-hidden-tts-test"); + const result = await maybeApplyTtsToPayload({ + payload: { + text: "[[tts:text]]***[[/tts:text]]", + audioAsVoice: true, + }, + cfg, + channel: "telegram", + kind: "final", + }); + + expect(synthesizeMock).not.toHaveBeenCalled(); + expect(result).toEqual({ + audioAsVoice: true, + }); + }); +}); diff --git a/src/tts/tts-runtime-models.test.ts b/src/tts/tts-runtime-models.test.ts new file mode 100644 index 000000000000..c90d13ddaab7 --- /dev/null +++ b/src/tts/tts-runtime-models.test.ts @@ -0,0 +1,341 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + MAX_TIMER_TIMEOUT_MS, + clearRuntimeConfigSnapshot, + createMockSpeechProvider, + installSpeechProviders, + prepareSynthesisMock, + requireAttempt, + requireFirstSynthesisRequest, + requireRecord, + setTtsMachinePrefsPathResolver, + synthesizeMock, + synthesizeSpeech, + textToSpeechStream, + transcodeAudioBufferMock, + type OpenClawConfig, + type SpeechSynthesisRequest, +} from "./tts-runtime.test-support.js"; + +describe("TTS runtime voice model and streaming behavior", () => { + afterEach(() => { + setTtsMachinePrefsPathResolver(); + clearRuntimeConfigSnapshot(); + delete (Object.prototype as Record).polluted; + synthesizeMock.mockClear(); + prepareSynthesisMock.mockClear(); + transcodeAudioBufferMock.mockClear(); + installSpeechProviders([createMockSpeechProvider()]); + }); + + it("caps oversized voice model TTS timeouts before synthesis", async () => { + installSpeechProviders([ + createMockSpeechProvider("mock", { autoSelectOrder: 1, models: ["mock-tts"] }), + ]); + + const result = await synthesizeSpeech({ + text: "Use capped explicit timeout.", + cfg: { + agents: { + defaults: { + voiceModel: { primary: "mock/mock-tts", timeoutMs: Number.MAX_SAFE_INTEGER }, + }, + }, + tts: { + enabled: true, + provider: "mock", + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("voice model capped timeout request"); + expect(request.timeoutMs).toBe(MAX_TIMER_TIMEOUT_MS); + }); + + it("uses agents.defaults.voiceModel as the default speech provider and model", async () => { + installSpeechProviders([ + createMockSpeechProvider("mock", { autoSelectOrder: 1 }), + createMockSpeechProvider("openai", { + autoSelectOrder: 10, + models: ["gpt-4o-mini-tts"], + resolveConfig: ({ rawConfig }) => { + const providers = requireRecord(rawConfig.providers, "raw provider configs"); + return { + model: "provider-default-model", + modelId: "provider-default-model", + ...requireRecord(providers.openai, "raw openai provider config"), + }; + }, + }), + ]); + + const result = await synthesizeSpeech({ + text: "Use configured voice model.", + cfg: { + agents: { + defaults: { + voiceModel: { primary: "openai/gpt-4o-mini-tts", timeoutMs: 12_345 }, + }, + }, + tts: { + enabled: true, + prefsPath: "/tmp/openclaw-speech-core-voice-model-default-test.json", + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("openai"); + expect(result.providerModel).toBe("gpt-4o-mini-tts"); + const request = requireFirstSynthesisRequest("voice model synthesis request"); + expect(request.providerConfig).toMatchObject({ + model: "gpt-4o-mini-tts", + modelId: "gpt-4o-mini-tts", + }); + expect(request.timeoutMs).toBe(12_345); + }); + + it("keeps explicit provider model aliases ahead of voiceModel defaults", async () => { + installSpeechProviders([ + createMockSpeechProvider("openrouter", { + models: ["explicit-model", "default-model"], + resolveConfig: ({ rawConfig }) => { + const providers = requireRecord(rawConfig.providers, "raw provider configs"); + return requireRecord(providers.openrouter, "raw openrouter provider config"); + }, + }), + ]); + + const result = await synthesizeSpeech({ + text: "Prefer explicit model alias.", + cfg: { + agents: { + defaults: { + voiceModel: { primary: "openrouter/default-model" }, + }, + }, + tts: { + enabled: true, + provider: "openrouter", + prefsPath: "/tmp/openclaw-speech-core-explicit-model-alias-test.json", + providers: { + openrouter: { + modelId: "explicit-model", + }, + }, + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("explicit model alias synthesis request"); + const providerConfig = requireRecord(request.providerConfig, "provider config"); + expect(providerConfig).toMatchObject({ + modelId: "explicit-model", + }); + expect(providerConfig.model).toBeUndefined(); + }); + + it("tries voiceModel fallbacks before auto-selected speech providers", async () => { + installSpeechProviders([ + createMockSpeechProvider("mock", { autoSelectOrder: 1 }), + createMockSpeechProvider("openai", { + autoSelectOrder: 10, + models: ["gpt-4o-mini-tts"], + isConfigured: () => false, + }), + createMockSpeechProvider("elevenlabs", { + autoSelectOrder: 99, + models: ["eleven_multilingual_v2"], + }), + ]); + + const result = await synthesizeSpeech({ + text: "Use configured voice model fallback.", + cfg: { + agents: { + defaults: { + voiceModel: { + primary: "openai/gpt-4o-mini-tts", + fallbacks: ["elevenlabs/eleven_multilingual_v2"], + }, + }, + }, + tts: { + enabled: true, + prefsPath: "/tmp/openclaw-speech-core-voice-model-fallback-test.json", + }, + } as OpenClawConfig, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("elevenlabs"); + expect(result.fallbackFrom).toBe("openai"); + expect(result.providerModel).toBe("eleven_multilingual_v2"); + }); + + it("tries same-provider voiceModel fallbacks as separate model attempts", async () => { + const synthesize = vi.fn(async (request: SpeechSynthesisRequest) => { + if (request.providerConfig.model === "bad-tts") { + throw new Error("unavailable model"); + } + return { + audioBuffer: Buffer.from("voice"), + fileExtension: ".ogg", + outputFormat: "ogg", + voiceCompatible: request.target === "voice-note", + }; + }); + installSpeechProviders([ + createMockSpeechProvider("openai", { + autoSelectOrder: 10, + models: ["bad-tts", "good-tts"], + synthesize, + }), + ]); + + const result = await synthesizeSpeech({ + text: "Use same-provider fallback model.", + cfg: { + agents: { + defaults: { + voiceModel: { + primary: "openai/bad-tts", + fallbacks: ["openai/good-tts"], + }, + }, + }, + tts: { + enabled: true, + prefsPath: "/tmp/openclaw-speech-core-same-provider-voice-model-fallback-test.json", + }, + } as OpenClawConfig, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("openai"); + expect(result.providerModel).toBe("good-tts"); + expect(result.attemptedProviders).toEqual(["openai", "openai"]); + expect(synthesize.mock.calls.map(([request]) => request.providerConfig.model)).toEqual([ + "bad-tts", + "good-tts", + ]); + }); + + it("skips non-streaming providers before using a streaming fallback", async () => { + const release = vi.fn(async () => {}); + const streamSynthesize = vi.fn(async () => ({ + audioStream: new ReadableStream({ + start(controller) { + controller.close(); + }, + }), + fileExtension: ".pcm", + outputFormat: "pcm", + voiceCompatible: false, + release, + })); + installSpeechProviders([ + createMockSpeechProvider("buffered", { autoSelectOrder: 1 }), + createMockSpeechProvider("streaming", { + autoSelectOrder: 2, + streamSynthesize, + }), + ]); + + const result = await textToSpeechStream({ + text: "Use streaming fallback.", + cfg: { + tts: { + enabled: true, + provider: "buffered", + prefsPath: "/tmp/openclaw-speech-core-streaming-fallback-test.json", + }, + } as OpenClawConfig, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("streaming"); + expect(result.fallbackFrom).toBe("buffered"); + expect(result.attemptedProviders).toEqual(["buffered", "streaming"]); + expect(result.outputFormat).toBe("pcm"); + expect(result.fileExtension).toBe(".pcm"); + expect(result.target).toBe("audio-file"); + expect(result.release).toBe(release); + const skippedAttempt = requireAttempt(result.attempts, 0); + expect(skippedAttempt).toMatchObject({ + provider: "buffered", + outcome: "skipped", + reasonCode: "unsupported_for_streaming", + personaBinding: "none", + error: "buffered does not support streaming TTS", + }); + expect(skippedAttempt).not.toHaveProperty("latencyMs"); + expect(requireAttempt(result.attempts, 1)).toMatchObject({ + provider: "streaming", + outcome: "success", + reasonCode: "success", + }); + expect(streamSynthesize).toHaveBeenCalledOnce(); + }); + + it("classifies streaming timeouts before falling back with raw text", async () => { + const timeoutStreamSynthesize = vi.fn(async () => { + const error = new Error("stalled"); + error.name = "AbortError"; + throw error; + }); + const fallbackStreamSynthesize = vi.fn(async () => ({ + audioStream: new ReadableStream({ + start(controller) { + controller.close(); + }, + }), + fileExtension: ".pcm", + outputFormat: "pcm", + voiceCompatible: false, + })); + installSpeechProviders([ + createMockSpeechProvider("primary", { + autoSelectOrder: 1, + streamSynthesize: timeoutStreamSynthesize, + }), + createMockSpeechProvider("fallback", { + autoSelectOrder: 2, + streamSynthesize: fallbackStreamSynthesize, + }), + ]); + const text = "## Keep [streaming Markdown](https://example.com) raw!!!!!"; + + const result = await textToSpeechStream({ + text, + cfg: { + tts: { + enabled: true, + provider: "primary", + prefsPath: "/tmp/openclaw-speech-core-streaming-timeout-test.json", + }, + } as OpenClawConfig, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("fallback"); + expect(result.fallbackFrom).toBe("primary"); + expect(requireAttempt(result.attempts, 0)).toMatchObject({ + provider: "primary", + outcome: "failed", + reasonCode: "timeout", + error: "primary: request timed out", + }); + expect(requireAttempt(result.attempts, 1)).toMatchObject({ + provider: "fallback", + outcome: "success", + reasonCode: "success", + }); + expect(fallbackStreamSynthesize).toHaveBeenCalledWith(expect.objectContaining({ text })); + }); +}); diff --git a/src/tts/tts-runtime-personas.test.ts b/src/tts/tts-runtime-personas.test.ts new file mode 100644 index 000000000000..f66aff76b9b9 --- /dev/null +++ b/src/tts/tts-runtime-personas.test.ts @@ -0,0 +1,496 @@ +import { rmSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + clearRuntimeConfigSnapshot, + createMockSpeechProvider, + getTtsPersona, + getTtsProvider, + installSpeechProviders, + isTtsProviderConfigured, + maybeApplyTtsToPayload, + prepareSynthesisMock, + requireAttempt, + requireFirstCallParam, + requireFirstSynthesisRequest, + requireRecord, + resolveTtsConfig, + setTtsMachinePrefsPathResolver, + synthesizeMock, + synthesizeSpeech, + textToSpeechTelephony, + transcodeAudioBufferMock, + type OpenClawConfig, + type ReplyPayload, + type SpeechTelephonySynthesisRequest, +} from "./tts-runtime.test-support.js"; + +describe("TTS runtime persona behavior", () => { + afterEach(() => { + setTtsMachinePrefsPathResolver(); + clearRuntimeConfigSnapshot(); + delete (Object.prototype as Record).polluted; + synthesizeMock.mockClear(); + prepareSynthesisMock.mockClear(); + transcodeAudioBufferMock.mockClear(); + installSpeechProviders([createMockSpeechProvider()]); + }); + + it("selects persona preferred provider before config fallback", () => { + const cfg: OpenClawConfig = { + tts: { + enabled: true, + provider: "other", + persona: "alfred", + personas: { + alfred: { + label: "Alfred", + provider: "mock", + providers: { + mock: { + voice: "Algieba", + }, + }, + }, + }, + }, + }; + const config = resolveTtsConfig(cfg); + const prefsPath = "/tmp/openclaw-speech-core-persona-provider.json"; + + expect(getTtsPersona(config, prefsPath)?.id).toBe("alfred"); + expect(getTtsProvider(config, prefsPath)).toBe("mock"); + }); + + it("treats provider configuration errors as unconfigured", () => { + installSpeechProviders([ + createMockSpeechProvider("broken", { + resolveConfig: () => { + throw new Error("invalid provider URL"); + }, + }), + ]); + const prefsPath = "/tmp/openclaw-speech-core-invalid-provider.json"; + setTtsMachinePrefsPathResolver(() => prefsPath); + const cfg = { + tts: { + providers: { broken: {} }, + }, + } as OpenClawConfig; + const config = resolveTtsConfig(cfg); + + expect(isTtsProviderConfigured(config, "broken", cfg)).toBe(false); + expect(getTtsProvider(config, prefsPath)).toBe(""); + }); + + it("merges active persona provider binding into synthesis config", async () => { + setTtsMachinePrefsPathResolver(() => "/tmp/openclaw-speech-core-persona-merge.json"); + const cfg: OpenClawConfig = { + tts: { + enabled: true, + provider: "mock", + providers: { + mock: { + model: "base-model", + voice: "base-voice", + }, + }, + persona: "alfred", + personas: { + alfred: { + provider: "mock", + providers: { + mock: { + voice: "persona-voice", + style: "dry", + }, + }, + }, + }, + }, + }; + + const payload: ReplyPayload = { + text: "This reply should use persona-specific provider configuration.", + }; + + let mediaDir: string | undefined; + try { + const result = await maybeApplyTtsToPayload({ + payload, + cfg, + channel: "slack", + kind: "final", + }); + + expect(synthesizeMock).toHaveBeenCalled(); + const request = requireFirstSynthesisRequest("persona synthesis request"); + const providerConfig = requireRecord(request.providerConfig, "persona provider config"); + expect(providerConfig.model).toBe("base-model"); + expect(providerConfig.voice).toBe("persona-voice"); + expect(providerConfig.style).toBe("dry"); + expect(result.mediaUrl).toMatch(/voice---[a-f0-9-]+\.ogg$/); + + mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; + } finally { + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } + }); + + it("does not mark skipped unregistered providers as missing persona bindings", async () => { + const result = await synthesizeSpeech({ + text: "Use fallback provider.", + cfg: { + tts: { + enabled: true, + provider: "missing", + persona: "alfred", + personas: { + alfred: { + providers: { + missing: { + voice: "configured-but-unregistered", + }, + }, + }, + }, + }, + }, + }); + + expect(result.success).toBe(true); + const attempt = requireAttempt(result.attempts, 0); + expect(attempt.provider).toBe("missing"); + expect(attempt.outcome).toBe("skipped"); + expect(attempt.reasonCode).toBe("no_provider_registered"); + expect(attempt.persona).toBe("alfred"); + expect(attempt).not.toHaveProperty("personaBinding"); + }); + + it("does not mark skipped telephony providers as missing persona bindings", async () => { + const result = await textToSpeechTelephony({ + text: "Use telephony provider.", + cfg: { + tts: { + enabled: true, + provider: "mock", + persona: "alfred", + personas: { + alfred: { + providers: { + mock: { + voice: "persona-voice", + }, + }, + }, + }, + }, + }, + }); + + expect(result.success).toBe(false); + const attempt = requireAttempt(result.attempts, 0); + expect(attempt.provider).toBe("mock"); + expect(attempt.outcome).toBe("skipped"); + expect(attempt.reasonCode).toBe("unsupported_for_telephony"); + expect(attempt.persona).toBe("alfred"); + expect(attempt).not.toHaveProperty("personaBinding"); + }); + + it("passes directive overrides to telephony synthesis providers", async () => { + const synthesizeTelephonyMock = vi.fn(async (_request: SpeechTelephonySynthesisRequest) => ({ + audioBuffer: Buffer.from("voice"), + outputFormat: "pcm", + sampleRate: 24_000, + })); + installSpeechProviders([ + createMockSpeechProvider("mock", { + synthesizeTelephony: synthesizeTelephonyMock, + }), + ]); + + const text = "## Keep [telephony Markdown](https://example.com) raw!!!!!"; + const result = await textToSpeechTelephony({ + text, + cfg: { + tts: { + enabled: true, + provider: "mock", + providers: { + mock: { + modelId: "telephony-model", + voiceId: "default-voice", + }, + }, + }, + }, + overrides: { + providerOverrides: { + mock: { + speakerVoice: "directed-voice", + speed: 1.5, + }, + }, + }, + }); + + expect(result.success).toBe(true); + expect(result.providerModel).toBe("telephony-model"); + expect(result.providerVoice).toBe("directed-voice"); + expect(synthesizeTelephonyMock).toHaveBeenCalledOnce(); + const telephonyRequest = requireRecord( + requireFirstCallParam(synthesizeTelephonyMock.mock.calls, "telephony synthesis"), + "telephony synthesis request", + ); + expect(telephonyRequest.providerOverrides).toEqual({ + speakerVoice: "directed-voice", + speed: 1.5, + }); + expect(telephonyRequest.text).toBe(text); + expect(telephonyRequest).not.toHaveProperty("target"); + }); + + it("uses provider defaults when fallback policy allows missing persona bindings", async () => { + await synthesizeSpeech({ + text: "Use neutral provider defaults.", + cfg: { + tts: { + enabled: true, + provider: "mock", + persona: "alfred", + personas: { + alfred: { + fallbackPolicy: "provider-defaults", + }, + }, + }, + }, + }); + + expect(prepareSynthesisMock).toHaveBeenCalledOnce(); + const prepareContext = requireRecord( + requireFirstCallParam(prepareSynthesisMock.mock.calls, "prepare synthesis"), + "prepare synthesis context", + ); + expect(prepareContext.persona).toBeUndefined(); + expect(prepareContext.personaProviderConfig).toBeUndefined(); + }); + + it("preserves persona metadata by default when provider bindings are missing", async () => { + await synthesizeSpeech({ + text: "Use persona prompt.", + cfg: { + tts: { + enabled: true, + provider: "mock", + persona: "alfred", + personas: { + alfred: { + label: "Alfred", + }, + }, + }, + }, + }); + + expect(prepareSynthesisMock).toHaveBeenCalledOnce(); + const prepareContext = requireRecord( + requireFirstCallParam(prepareSynthesisMock.mock.calls, "prepare synthesis"), + "prepare synthesis context", + ); + const persona = requireRecord(prepareContext.persona, "prepare synthesis persona"); + expect(persona.id).toBe("alfred"); + expect(prepareContext.personaProviderConfig).toBeUndefined(); + }); + + it("skips unbound providers under fail policy while allowing bound fallbacks", async () => { + installSpeechProviders([ + createMockSpeechProvider("mock", { autoSelectOrder: 1 }), + createMockSpeechProvider("fallback", { autoSelectOrder: 2 }), + ]); + + const result = await synthesizeSpeech({ + text: "Use the first persona-bound provider.", + cfg: { + tts: { + enabled: true, + provider: "mock", + persona: "alfred", + personas: { + alfred: { + fallbackPolicy: "fail", + providers: { + fallback: { + voice: "fallback-voice", + }, + }, + }, + }, + }, + }, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("fallback"); + expect(result.fallbackFrom).toBe("mock"); + const skippedAttempt = requireAttempt(result.attempts, 0); + expect(skippedAttempt.provider).toBe("mock"); + expect(skippedAttempt.outcome).toBe("skipped"); + expect(skippedAttempt.reasonCode).toBe("not_configured"); + expect(skippedAttempt.persona).toBe("alfred"); + expect(skippedAttempt.personaBinding).toBe("missing"); + expect(skippedAttempt.error).toBe("mock: persona alfred has no provider binding"); + const successAttempt = requireAttempt(result.attempts, 1); + expect(successAttempt.provider).toBe("fallback"); + expect(successAttempt.outcome).toBe("success"); + expect(successAttempt.persona).toBe("alfred"); + expect(successAttempt.personaBinding).toBe("applied"); + }); +}); + +describe("TTS runtime per-agent config", () => { + it("deep-merges the active agent TTS override over tts", () => { + const cfg = { + tts: { + enabled: true, + provider: "openai", + providers: { + openai: { + apiKey: "example", + voice: "coral", + speed: 1, + }, + }, + }, + agents: { + list: [ + { + id: "reader", + tts: { + provider: "openai", + providers: { + openai: { + voice: "nova", + }, + }, + }, + }, + ], + }, + } satisfies OpenClawConfig; + + const resolved = resolveTtsConfig(cfg, "reader"); + + const rawConfig = requireRecord(resolved.rawConfig, "resolved raw TTS config"); + expect(rawConfig.enabled).toBe(true); + expect(rawConfig.provider).toBe("openai"); + const providers = requireRecord(rawConfig.providers, "resolved raw TTS providers"); + const openai = requireRecord(providers.openai, "resolved OpenAI TTS provider config"); + expect(openai.apiKey).toBe("example"); + expect(openai.voice).toBe("nova"); + expect(openai.speed).toBe(1); + }); + + it("composes per-agent TTS overrides with active persona bindings", async () => { + const cfg = { + tts: { + enabled: true, + provider: "mock", + providers: { + mock: { + model: "base-model", + voice: "base-voice", + }, + }, + persona: "alfred", + personas: { + alfred: { + provider: "mock", + providers: { + mock: { + voice: "alfred-voice", + }, + }, + }, + jarvis: { + provider: "mock", + providers: { + mock: { + style: "jarvis-style", + }, + }, + }, + }, + }, + agents: { + list: [ + { + id: "reader", + tts: { + persona: "jarvis", + providers: { + mock: { + voice: "agent-voice", + }, + }, + }, + }, + ], + }, + } satisfies OpenClawConfig; + + let mediaDir: string | undefined; + try { + const result = await maybeApplyTtsToPayload({ + payload: { text: "This agent reply should use the composed persona config." }, + cfg, + channel: "slack", + kind: "final", + agentId: "reader", + }); + + expect(synthesizeMock).toHaveBeenCalled(); + const request = requireFirstSynthesisRequest("agent persona synthesis request"); + const providerConfig = requireRecord(request.providerConfig, "agent persona provider config"); + expect(providerConfig.model).toBe("base-model"); + expect(providerConfig.voice).toBe("agent-voice"); + expect(providerConfig.style).toBe("jarvis-style"); + expect(result.mediaUrl).toMatch(/voice---[a-f0-9-]+\.ogg$/); + mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; + } finally { + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } + }); + + it("ignores prototype-pollution keys in agent TTS overrides", () => { + const cfg = { + tts: { + provider: "openai", + providers: { + openai: { + voice: "coral", + }, + }, + }, + agents: { + list: [ + { + id: "reader", + tts: JSON.parse( + '{"providers":{"openai":{"voice":"nova","__proto__":{"polluted":true}}}}', + ), + }, + ], + }, + } as OpenClawConfig; + + const resolved = resolveTtsConfig(cfg, "reader"); + + expect(resolved.rawConfig?.providers?.openai).toEqual({ voice: "nova" }); + expect(({} as Record).polluted).toBeUndefined(); + }); +}); diff --git a/src/tts/tts-runtime-routing.test.ts b/src/tts/tts-runtime-routing.test.ts new file mode 100644 index 000000000000..18b74d854a59 --- /dev/null +++ b/src/tts/tts-runtime-routing.test.ts @@ -0,0 +1,447 @@ +import { rmSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + CODE_HEAVY_SPOKEN_FALLBACK, + MAX_TIMER_TIMEOUT_MS, + buildTtsSystemPromptHint, + clearRuntimeConfigSnapshot, + createMockSpeechProvider, + createTtsConfig, + expectTtsPayloadResult, + installSpeechProviders, + listSpeechVoices, + nativeVoiceNoteChannels, + prefsPathFor, + prepareSynthesisMock, + prepareTtsRequest, + requireFirstCallParam, + requireFirstSynthesisRequest, + requireRecord, + resolveTtsConfig, + resolveTtsPrefsPath, + setRuntimeConfigSnapshot, + setTtsMachinePrefsPathResolver, + synthesizeMock, + synthesizeSpeech, + testApi, + textToSpeech, + textToSpeechCore, + transcodeAudioBufferMock, + type OpenClawConfig, + type SpeechListVoicesRequest, + type TtsConfig, +} from "./tts-runtime.test-support.js"; + +describe("TTS runtime native voice-note routing", () => { + afterEach(() => { + setTtsMachinePrefsPathResolver(); + clearRuntimeConfigSnapshot(); + delete (Object.prototype as Record).polluted; + synthesizeMock.mockClear(); + prepareSynthesisMock.mockClear(); + transcodeAudioBufferMock.mockClear(); + installSpeechProviders([createMockSpeechProvider()]); + }); + + it("prefers the environment preference path over migrated machine state", () => { + const previousEnvPath = process.env.OPENCLAW_TTS_PREFS; + const envPath = prefsPathFor("env-override"); + setTtsMachinePrefsPathResolver(() => prefsPathFor("machine-state")); + process.env.OPENCLAW_TTS_PREFS = envPath; + try { + expect(resolveTtsPrefsPath(resolveTtsConfig({}))).toBe(envPath); + } finally { + if (previousEnvPath === undefined) { + delete process.env.OPENCLAW_TTS_PREFS; + } else { + process.env.OPENCLAW_TTS_PREFS = previousEnvPath; + } + } + }); + + it("resolves voice delivery support from channel capabilities", () => { + for (const channel of nativeVoiceNoteChannels) { + expect(testApi.supportsNativeVoiceNoteTts(channel)).toBe(true); + expect(testApi.supportsNativeVoiceNoteTts(channel.toUpperCase())).toBe(true); + } + expect(testApi.supportsNativeVoiceNoteTts("slack")).toBe(false); + expect(testApi.supportsNativeVoiceNoteTts(undefined)).toBe(false); + }); + + it("tells generic TTS guidance to defer to MEMORY voice-delivery instructions", () => { + const hint = buildTtsSystemPromptHint(createTtsConfig("openclaw-speech-core-tts-hint-test")); + + expect(hint).toContain("Voice (TTS) is enabled."); + expect(hint).toContain( + "If workspace context (especially MEMORY.md) tells you not to use [[tts:...]] or to use a local/non-tagged voice workflow, follow that workspace instruction instead.", + ); + expect(hint).toContain( + "Use [[tts:...]] and optional [[tts:text]]...[[/tts:text]] to control voice/expressiveness.", + ); + }); + + it("prepares deep-merged surface config and directive inputs", () => { + const cfg: OpenClawConfig = { + tts: { + provider: "mock", + modelOverrides: { allowProvider: false }, + providers: { + mock: { + model: "base-model", + voiceSettings: { stability: 0.4 }, + }, + }, + }, + }; + + const prepared = prepareTtsRequest({ + cfg, + override: { + modelOverrides: { allowProvider: true }, + providers: { + mock: { + voice: "surface-voice", + voiceSettings: { speed: 1.1 }, + }, + }, + }, + text: "Hello [[tts:text]]Speak this instead[[/tts:text]] caller", + }); + + expect(prepared.cfg).not.toBe(cfg); + expect(prepared.cfg.tts?.providers?.mock).toEqual({ + model: "base-model", + voice: "surface-voice", + voiceSettings: { stability: 0.4, speed: 1.1 }, + }); + expect(prepared.cfg.tts?.modelOverrides?.allowProvider).toBe(true); + expect(prepared.directives).toEqual({ + cleanedText: "Hello caller", + hasDirective: true, + overrides: { + ttsText: "Speak this instead", + }, + ttsText: "Speak this instead", + warnings: [], + }); + expect(cfg.tts?.providers?.mock).toEqual({ + model: "base-model", + voiceSettings: { stability: 0.4 }, + }); + }); + + it("sanitizes blocked override keys while preparing TTS config", () => { + const prepared = prepareTtsRequest({ + cfg: { + tts: { + provider: "mock", + providers: { mock: { model: "base-model" } }, + }, + }, + override: JSON.parse( + '{"__proto__":{"polluted":"top"},"providers":{"mock":{"voice":"safe","__proto__":{"polluted":"nested"}}}}', + ) as TtsConfig, + text: "[[tts:text]]Speak this instead[[/tts:text]]", + }); + + expect((Object.prototype as Record).polluted).toBeUndefined(); + expect(prepared.cfg.tts).not.toHaveProperty("polluted"); + expect(prepared.cfg.tts?.providers?.mock).toEqual({ + model: "base-model", + voice: "safe", + }); + expect(prepared.directives.cleanedText).toBe(""); + expect(prepared.directives.ttsText).toBe("Speak this instead"); + }); + + it("marks Discord auto TTS replies as native voice messages", async () => { + await expectTtsPayloadResult({ + channel: "discord", + prefsName: "openclaw-speech-core-tts-test", + text: "This Discord reply should be delivered as a native voice note.", + target: "voice-note", + audioAsVoice: true, + }); + }); + + it("keeps compatible audio-file synthesis deliverable as a voice memo", async () => { + await expectTtsPayloadResult({ + channel: "voice-memo-chat", + prefsName: "openclaw-speech-core-tts-voice-memo-mp3-test", + text: "This reply should be delivered as a native voice memo.", + target: "audio-file", + audioAsVoice: true, + mediaExtension: "mp3", + providerResult: { + audioBuffer: Buffer.from("mp3"), + outputFormat: "mp3", + fileExtension: ".mp3", + voiceCompatible: false, + }, + }); + }); + + it("does not mark unsupported audio-file output as a voice memo", async () => { + await expectTtsPayloadResult({ + channel: "voice-memo-chat", + prefsName: "openclaw-speech-core-tts-voice-memo-ogg-test", + text: "This reply should stay a regular audio attachment.", + target: "audio-file", + audioAsVoice: undefined, + }); + }); + + it("pre-transcodes synthesized mp3 to opus-in-CAF when the host can satisfy preferAudioFileFormat", async () => { + transcodeAudioBufferMock.mockResolvedValueOnce({ + ok: true, + buffer: Buffer.from("transcoded-caf"), + }); + await expectTtsPayloadResult({ + channel: "voice-memo-chat", + prefsName: "openclaw-speech-core-tts-voice-memo-caf-transcode-test", + text: "This reply should be pre-transcoded to a native voice-memo CAF.", + target: "audio-file", + audioAsVoice: true, + mediaExtension: "caf", + providerResult: { + audioBuffer: Buffer.from("mp3"), + outputFormat: "mp3", + fileExtension: ".mp3", + voiceCompatible: false, + }, + }); + expect(transcodeAudioBufferMock).toHaveBeenCalledOnce(); + const transcodeRequest = requireRecord( + requireFirstCallParam(transcodeAudioBufferMock.mock.calls as unknown[][], "transcode"), + "transcode request", + ); + expect(transcodeRequest.sourceExtension).toBe("mp3"); + expect(transcodeRequest.targetExtension).toBe("caf"); + }); + + it("falls back to the original mp3 buffer when the host transcoder fails", async () => { + transcodeAudioBufferMock.mockResolvedValueOnce({ + ok: false, + reason: "transcoder-failed", + detail: "exit-1", + }); + // Even though the transcode failed, the original mp3 still satisfies the + // channel audioFileFormats list, so the channel still flips audioAsVoice. + // The user gets a voice memo bubble, possibly with bad duration, instead + // of a regression. The failure is logged via the call site in tts.ts. + await expectTtsPayloadResult({ + channel: "voice-memo-chat", + prefsName: "openclaw-speech-core-tts-voice-memo-caf-fallback-test", + text: "This reply should fall back to the original mp3.", + target: "audio-file", + audioAsVoice: true, + mediaExtension: "mp3", + providerResult: { + audioBuffer: Buffer.from("mp3"), + outputFormat: "mp3", + fileExtension: ".mp3", + voiceCompatible: false, + }, + }); + }); + + it("uses the active runtime snapshot when source config still contains TTS SecretRefs", async () => { + const sourceConfig = { + tts: { + enabled: true, + provider: "mock", + providers: { + mock: { + apiKey: { source: "exec", provider: "mockexec", id: "minimax/tts/apiKey" }, + }, + }, + }, + } as unknown as OpenClawConfig; + const runtimeConfig = { + tts: { + enabled: true, + provider: "mock", + providers: { + mock: { + apiKey: "test-key", + }, + }, + }, + } as unknown as OpenClawConfig; + installSpeechProviders([ + createMockSpeechProvider("mock", { + isConfigured: ({ providerConfig }) => providerConfig.apiKey === "test-key", + resolveConfig: ({ rawConfig }) => { + const providers = rawConfig.providers as Record | undefined; + return providers?.mock ?? {}; + }, + }), + ]); + setRuntimeConfigSnapshot(runtimeConfig, sourceConfig); + + const result = await synthesizeSpeech({ + text: "Runtime snapshot TTS SecretRef", + cfg: sourceConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + expect(synthesizeMock).toHaveBeenCalled(); + const request = requireFirstSynthesisRequest("runtime snapshot synthesis request"); + expect(request.cfg).toBe(runtimeConfig); + const providerConfig = requireRecord(request.providerConfig, "provider config"); + expect(providerConfig.apiKey).toBe("test-key"); + }); + + it("uses provider default TTS timeout when the call and config omit timeoutMs", async () => { + installSpeechProviders([createMockSpeechProvider("mock", { defaultTimeoutMs: 600_000 })]); + + const result = await synthesizeSpeech({ + text: "Use provider timeout.", + cfg: { + tts: { + enabled: true, + provider: "mock", + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("provider default timeout synthesis request"); + expect(request.timeoutMs).toBe(600_000); + }); + + it("normalizes non-streaming synthesis text before calling the provider", async () => { + const result = await synthesizeSpeech({ + text: "## Update\n\nRead the [guide](https://example.com/guide)!!!!!", + cfg: createTtsConfig("openclaw-speech-core-talk-markdown-test"), + disableFallback: true, + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("normalized talk synthesis request"); + expect(request.text).toBe("Update\n\nRead the guide!"); + }); + + it("speaks stripped code through the explicit textToSpeech conversion path", async () => { + let mediaDir: string | undefined; + try { + const result = await textToSpeech({ + text: "```ts\nconst answer = 42;\n```", + cfg: createTtsConfig("openclaw-speech-core-code-convert-test"), + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("explicit code conversion request"); + expect(request.text).toBe("const answer = 42;"); + expect(request.text).not.toBe(CODE_HEAVY_SPOKEN_FALLBACK); + mediaDir = result.audioPath ? path.dirname(result.audioPath) : undefined; + } finally { + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } + }); + + it("returns a normal TTS failure when audio persistence rejects", async () => { + const result = await textToSpeechCore( + { + text: "Store this synthesized reply.", + cfg: createTtsConfig("openclaw-speech-core-persistence-failure-test"), + }, + async () => { + throw new Error("Media exceeds configured limit"); + }, + ); + + expect(result).toMatchObject({ + success: false, + error: "TTS audio persistence failed", + provider: "mock", + }); + }); + + it("resolves the configured timeout for voice listing", async () => { + const listVoicesMock = vi.fn(async (_request: SpeechListVoicesRequest) => []); + installSpeechProviders([ + createMockSpeechProvider("mock", { + defaultTimeoutMs: 60_000, + listVoices: listVoicesMock, + }), + ]); + + await listSpeechVoices({ + provider: "mock", + cfg: { + tts: { + enabled: true, + provider: "mock", + timeoutMs: 45_000, + }, + } as OpenClawConfig, + }); + + expect(listVoicesMock).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 45_000 })); + }); + + it("caps oversized provider default TTS timeouts before synthesis", async () => { + installSpeechProviders([ + createMockSpeechProvider("mock", { defaultTimeoutMs: Number.MAX_SAFE_INTEGER }), + ]); + + const result = await synthesizeSpeech({ + text: "Use capped provider timeout.", + cfg: { + tts: { + enabled: true, + provider: "mock", + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("provider default capped timeout request"); + expect(request.timeoutMs).toBe(MAX_TIMER_TIMEOUT_MS); + }); + + it("ignores nonpositive provider default TTS timeouts", async () => { + installSpeechProviders([createMockSpeechProvider("mock", { defaultTimeoutMs: 0 })]); + + const result = await synthesizeSpeech({ + text: "Use fallback timeout.", + cfg: { + tts: { + enabled: true, + provider: "mock", + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("provider default fallback timeout request"); + expect(request.timeoutMs).toBe(30_000); + }); + + it("keeps explicit TTS config timeout ahead of provider default timeout", async () => { + installSpeechProviders([createMockSpeechProvider("mock", { defaultTimeoutMs: 600_000 })]); + + await synthesizeSpeech({ + text: "Use configured timeout.", + cfg: { + tts: { + enabled: true, + provider: "mock", + timeoutMs: 45_000, + }, + } as OpenClawConfig, + disableFallback: true, + }); + + const request = requireFirstSynthesisRequest("configured timeout synthesis request"); + expect(request.timeoutMs).toBe(45_000); + }); +}); diff --git a/packages/speech-core/src/tts-types.ts b/src/tts/tts-runtime-types.ts similarity index 100% rename from packages/speech-core/src/tts-types.ts rename to src/tts/tts-runtime-types.ts diff --git a/src/tts/tts-runtime.test-support.ts b/src/tts/tts-runtime.test-support.ts new file mode 100644 index 000000000000..577e04d895c3 --- /dev/null +++ b/src/tts/tts-runtime.test-support.ts @@ -0,0 +1,297 @@ +// TTS runtime tests cover speech synthesis behavior. +import crypto from "node:crypto"; +import { mkdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { OpenClawConfig, TtsConfig } from "openclaw/plugin-sdk/config-contracts"; +import { MAX_TIMER_TIMEOUT_MS as MAX_TIMER_TIMEOUT_MS_CORE } from "openclaw/plugin-sdk/number-runtime"; +import type { ReplyPayload } from "openclaw/plugin-sdk/reply-payload"; +import { + clearRuntimeConfigSnapshot as clearRuntimeConfigSnapshotCore, + setRuntimeConfigSnapshot as setRuntimeConfigSnapshotCore, +} from "openclaw/plugin-sdk/runtime-config-snapshot"; +import type { + SpeechListVoicesRequest, + SpeechProviderPlugin, + SpeechProviderPrepareSynthesisContext, + SpeechSynthesisRequest, + SpeechTelephonySynthesisRequest, +} from "openclaw/plugin-sdk/speech-core"; +import { expect, vi } from "vitest"; +import { CODE_HEAVY_SPOKEN_FALLBACK as CODE_HEAVY_SPOKEN_FALLBACK_CORE } from "./speech-text.js"; +import type { TtsAudioPersistence } from "./tts-synthesis.js"; + +type MockSpeechSynthesisResult = Awaited>; + +const synthesizeMock = vi.hoisted(() => + vi.fn( + async (request: SpeechSynthesisRequest): Promise => ({ + audioBuffer: Buffer.from("voice"), + fileExtension: ".ogg", + outputFormat: "ogg", + voiceCompatible: request.target === "voice-note", + }), + ), +); +const prepareSynthesisMock = vi.hoisted(() => + vi.fn(async (_ctx: SpeechProviderPrepareSynthesisContext) => undefined), +); + +const listSpeechProvidersMock = vi.hoisted(() => vi.fn()); +const getSpeechProviderMock = vi.hoisted(() => vi.fn()); +const transcodeAudioBufferMock = vi.hoisted(() => + // Default off: most tests rely on the synthesized buffer reaching the + // channel unchanged. Tests that exercise the pre-transcode branch override + // per-call via `transcodeAudioBufferMock.mockResolvedValueOnce(...)`. + // Typed as the helper's full return shape so per-call overrides aren't + // narrowed to the default's literal. + vi.fn< + () => Promise< + | { ok: true; buffer: Buffer } + | { + ok: false; + reason: + | "platform-unsupported" + | "invalid-extension" + | "noop-same-container" + | "no-recipe" + | "transcoder-failed"; + detail?: string; + } + > + >(async () => ({ ok: false, reason: "platform-unsupported" })), +); + +vi.mock("../media/media-services.js", () => ({ + transcodeAudioBuffer: transcodeAudioBufferMock, +})); + +vi.mock("../channels/plugins/tts-capabilities.js", () => ({ + normalizeChannelId: (channel: string | undefined) => channel?.trim().toLowerCase() ?? null, + resolveChannelTtsVoiceDelivery: (channel: string | undefined) => { + const normalized = channel?.trim().toLowerCase(); + if (normalized === "voice-memo-chat") { + return { + synthesisTarget: "audio-file", + audioFileFormats: ["mp3", "caf", "audio/mpeg", "audio/x-caf"], + preferAudioFileFormat: "caf", + }; + } + if (normalized === "feishu" || normalized === "whatsapp") { + return { synthesisTarget: "voice-note", transcodesAudio: true }; + } + if (normalized === "discord" || normalized === "matrix" || normalized === "telegram") { + return { synthesisTarget: "voice-note" }; + } + return undefined; + }, +})); + +vi.mock("./provider-registry.js", async () => { + const actual = + await vi.importActual("./provider-registry.js"); + const mockProvider: SpeechProviderPlugin = { + id: "mock", + label: "Mock", + autoSelectOrder: 1, + isConfigured: () => true, + prepareSynthesis: prepareSynthesisMock, + synthesize: synthesizeMock, + }; + listSpeechProvidersMock.mockImplementation(() => [mockProvider]); + getSpeechProviderMock.mockImplementation((providerId: string) => + providerId === "mock" ? mockProvider : null, + ); + return { + ...actual, + canonicalizeSpeechProviderId: (providerId: string | undefined) => + providerId?.trim().toLowerCase() || undefined, + normalizeSpeechProviderId: (providerId: string | undefined) => + providerId?.trim().toLowerCase() || undefined, + getSpeechProvider: getSpeechProviderMock, + listSpeechProviders: listSpeechProvidersMock, + }; +}); + +vi.mock("./tts-core.js", async () => { + const actual = await vi.importActual("./tts-core.js"); + return { ...actual, scheduleCleanup: vi.fn() }; +}); + +export const { + testApi, + buildTtsSystemPromptHint, + getTtsPersona, + getTtsProvider, + isTtsProviderConfigured, + listSpeechVoices, + prepareTtsRequest, + resolveTtsConfig, + resolveTtsPrefsPath, + setTtsMachinePrefsPathResolver, + setSummarizationEnabled, + setTtsMaxLength, + synthesizeSpeech, + textToSpeechStream, + textToSpeechTelephony, +} = await import("./runtime-api.js"); +export const { maybeApplyTtsToPayload: maybeApplyTtsToPayloadCore } = + await import("./tts-payload.js"); +export const { textToSpeech: textToSpeechCore } = await import("./tts-synthesis.js"); + +export const CODE_HEAVY_SPOKEN_FALLBACK = CODE_HEAVY_SPOKEN_FALLBACK_CORE; +export const MAX_TIMER_TIMEOUT_MS = MAX_TIMER_TIMEOUT_MS_CORE; +export function clearRuntimeConfigSnapshot(): void { + clearRuntimeConfigSnapshotCore(); +} +export const setRuntimeConfigSnapshot = ( + ...args: Parameters +) => setRuntimeConfigSnapshotCore(...args); + +export const nativeVoiceNoteChannels = [ + "discord", + "feishu", + "matrix", + "telegram", + "whatsapp", +] as const; + +export function createMockSpeechProvider( + id = "mock", + options: Partial = {}, +): SpeechProviderPlugin { + return { + id, + label: id, + autoSelectOrder: id === "mock" ? 1 : 2, + isConfigured: () => true, + prepareSynthesis: prepareSynthesisMock, + synthesize: synthesizeMock, + ...options, + }; +} + +export function installSpeechProviders(providers: SpeechProviderPlugin[]): void { + listSpeechProvidersMock.mockImplementation(() => providers); + getSpeechProviderMock.mockImplementation( + (providerId: string) => providers.find((provider) => provider.id === providerId) ?? null, + ); +} + +// macOS os.tmpdir() is a /var -> /private/var symlink and fs-safe rejects +// symlinked store roots; resolve the canonical dir before writing prefs. +const PREFS_TMP_DIR = realpathSync(os.tmpdir()); + +async function persistTestTtsAudio({ + audioBuffer, + fileExtension, +}: Parameters[0]): Promise { + const dir = path.join(PREFS_TMP_DIR, `openclaw-speech-core-media-${crypto.randomUUID()}`); + mkdirSync(dir, { recursive: true }); + const audioPath = path.join(dir, `voice---${crypto.randomUUID()}${fileExtension}`); + writeFileSync(audioPath, audioBuffer); + return audioPath; +} + +export function textToSpeech(params: Parameters[0]) { + return textToSpeechCore(params, persistTestTtsAudio); +} + +export function maybeApplyTtsToPayload(params: Parameters[0]) { + return maybeApplyTtsToPayloadCore(params, persistTestTtsAudio); +} + +export function prefsPathFor(prefsName: string): string { + return path.join(PREFS_TMP_DIR, `${prefsName}.json`); +} + +export function createTtsConfig(prefsName: string): OpenClawConfig { + setTtsMachinePrefsPathResolver(() => prefsPathFor(prefsName)); + return { + tts: { + enabled: true, + provider: "mock", + }, + }; +} + +export function requireRecord(value: unknown, label: string): Record { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`expected ${label} to be a record`); + } + return value as Record; +} + +export function requireFirstCallParam(calls: ReadonlyArray, label: string) { + const call = calls[0]; + if (!call) { + throw new Error(`expected ${label} call`); + } + return call[0]; +} + +export function requireFirstSynthesisRequest(label: string): Record { + return requireRecord(requireFirstCallParam(synthesizeMock.mock.calls, label), label); +} + +export function requireAttempt(attempts: unknown[] | undefined, index: number) { + if (!attempts) { + throw new Error("expected synthesis attempts"); + } + return requireRecord(attempts[index], `synthesis attempt ${index}`); +} + +export async function expectTtsPayloadResult(params: { + channel: string; + prefsName: string; + text: string; + target: "voice-note" | "audio-file"; + audioAsVoice: true | undefined; + providerResult?: MockSpeechSynthesisResult; + mediaExtension?: string; + kind?: "tool" | "block" | "final"; +}) { + if (params.providerResult) { + synthesizeMock.mockResolvedValueOnce(params.providerResult); + } + const cfg = createTtsConfig(params.prefsName); + let mediaDir: string | undefined; + try { + const result = await maybeApplyTtsToPayload({ + payload: { text: params.text }, + cfg, + channel: params.channel, + kind: params.kind ?? "final", + }); + + expect(synthesizeMock).toHaveBeenCalled(); + const request = requireRecord( + synthesizeMock.mock.calls.at(-1)?.[0], + "latest synthesis request", + ); + expect(request.target).toBe(params.target); + expect(result.audioAsVoice).toBe(params.audioAsVoice); + expect(result.mediaUrl).toMatch( + new RegExp(`voice---[a-f0-9-]+\\.${params.mediaExtension ?? "ogg"}$`), + ); + expect(result.spokenText).toBe(params.text); + expect(result.ttsSupplement).toEqual({ spokenText: params.text }); + expect((result as { trustedLocalMedia?: boolean }).trustedLocalMedia).toBe(true); + + mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; + } finally { + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } +} + +export { prepareSynthesisMock, synthesizeMock, transcodeAudioBufferMock }; +export type { + OpenClawConfig, + ReplyPayload, + SpeechListVoicesRequest, + SpeechSynthesisRequest, + SpeechTelephonySynthesisRequest, + TtsConfig, +}; diff --git a/packages/speech-core/src/tts-settings-writes.ts b/src/tts/tts-settings-writes.ts similarity index 87% rename from packages/speech-core/src/tts-settings-writes.ts rename to src/tts/tts-settings-writes.ts index 01188251a771..41d93562b395 100644 --- a/packages/speech-core/src/tts-settings-writes.ts +++ b/src/tts/tts-settings-writes.ts @@ -1,8 +1,8 @@ // TTS preference mutations stay off the agent prompt's read-only import path. import path from "node:path"; -import type { TtsAutoMode, TtsProvider } from "openclaw/plugin-sdk/config-contracts"; -import { privateFileStoreSync } from "openclaw/plugin-sdk/security-runtime"; -import { canonicalizeSpeechProviderId } from "openclaw/plugin-sdk/speech-core"; +import type { TtsAutoMode, TtsProvider } from "../config/types.js"; +import { privateFileStoreSync } from "../infra/private-file-store.js"; +import { canonicalizeSpeechProviderId } from "./provider-registry.js"; import { normalizeTtsPersonaId, readTtsPrefs, type TtsUserPrefs } from "./tts-settings.js"; function updateTtsPrefs(prefsPath: string, update: (prefs: TtsUserPrefs) => void): void { diff --git a/src/tts/tts-settings.ts b/src/tts/tts-settings.ts index a30c9c550262..b2b93f03bfb5 100644 --- a/src/tts/tts-settings.ts +++ b/src/tts/tts-settings.ts @@ -1,5 +1,397 @@ -// Lightweight core facade for TTS settings used by agent and status hot paths. -export { - buildTtsSystemPromptHint, - resolveTtsSettingsSnapshot, -} from "../../packages/speech-core/src/tts-settings.js"; +// Lightweight TTS settings resolution shared by agent prompts, status, and speech runtime. +import { existsSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { + normalizeOptionalLowercaseString, + normalizeOptionalString, +} from "../../packages/normalization-core/src/string-coerce.js"; +import { + getRuntimeConfigSnapshot, + getRuntimeConfigSourceSnapshot, + selectApplicableRuntimeConfig, +} from "../config/runtime-snapshot.js"; +import type { + OpenClawConfig, + ResolvedTtsPersona, + TtsAutoMode, + TtsConfig, + TtsModelOverrideConfig, + TtsProvider, +} from "../config/types.js"; +import { resolveConfigDir, resolveUserPath } from "../utils.js"; +import { normalizeSpeechProviderId } from "./provider-registry-core.js"; +import type { SpeechProviderConfig } from "./provider-types.js"; +import { withSpeakerSelectionCompat } from "./speaker.js"; +import { normalizeTtsAutoMode } from "./tts-auto-mode.js"; +import { resolveEffectiveTtsConfig, type TtsConfigResolutionContext } from "./tts-config.js"; +import type { ResolvedTtsConfig, ResolvedTtsModelOverrides } from "./tts-types.js"; + +export type { ResolvedTtsConfig, ResolvedTtsModelOverrides }; + +export const DEFAULT_TTS_TIMEOUT_MS = 30_000; +const DEFAULT_TTS_MAX_LENGTH = 1500; +const DEFAULT_TTS_SUMMARIZE = true; +const DEFAULT_MAX_TEXT_LENGTH = 4096; +let machinePrefsPathResolver: () => string | undefined = () => undefined; + +export function setTtsMachinePrefsPathResolver(resolver?: () => string | undefined): void { + machinePrefsPathResolver = resolver ?? (() => undefined); +} + +export type TtsUserPrefs = { + tts?: { + auto?: TtsAutoMode; + enabled?: boolean; + provider?: TtsProvider; + persona?: string | null; + maxLength?: number; + summarize?: boolean; + }; +}; + +function resolveConfiguredTtsAutoMode(raw: TtsConfig): TtsAutoMode { + return normalizeTtsAutoMode(raw.auto) ?? (raw.enabled ? "always" : "off"); +} + +export function normalizeConfiguredSpeechProviderId( + providerId: string | undefined, +): TtsProvider | undefined { + const normalized = normalizeSpeechProviderId(providerId); + if (!normalized) { + return undefined; + } + return normalized === "edge" ? "microsoft" : normalized; +} + +export function normalizeTtsPersonaId(personaId: string | null | undefined): string | undefined { + return normalizeOptionalLowercaseString(personaId ?? undefined); +} + +function resolveTtsPrefsPathValue(prefsPath: string | undefined): string { + // Scoped agent paths must win over the migrated machine-wide default. + if (prefsPath?.trim()) { + return resolveUserPath(prefsPath.trim()); + } + const envPath = process.env.OPENCLAW_TTS_PREFS?.trim(); + if (envPath) { + return resolveUserPath(envPath); + } + const machinePath = machinePrefsPathResolver()?.trim(); + if (machinePath) { + return resolveUserPath(machinePath); + } + return path.join(resolveConfigDir(process.env), "settings", "tts.json"); +} + +export function resolveModelOverridePolicy( + overrides: TtsModelOverrideConfig | undefined, +): ResolvedTtsModelOverrides { + const enabled = overrides?.enabled ?? true; + if (!enabled) { + return { + enabled: false, + allowText: false, + allowProvider: false, + allowVoice: false, + allowModelId: false, + allowVoiceSettings: false, + allowNormalization: false, + allowSeed: false, + }; + } + const allow = (value: boolean | undefined, defaultValue = true) => value ?? defaultValue; + return { + enabled: true, + allowText: allow(overrides?.allowText), + allowProvider: allow(overrides?.allowProvider, false), + allowVoice: allow(overrides?.allowVoice), + allowModelId: allow(overrides?.allowModelId), + allowVoiceSettings: allow(overrides?.allowVoiceSettings), + allowNormalization: allow(overrides?.allowNormalization), + allowSeed: allow(overrides?.allowSeed), + }; +} + +export function resolveTtsRuntimeConfig(cfg: OpenClawConfig): OpenClawConfig { + return ( + selectApplicableRuntimeConfig({ + inputConfig: cfg, + runtimeConfig: getRuntimeConfigSnapshot(), + runtimeSourceConfig: getRuntimeConfigSourceSnapshot(), + }) ?? cfg + ); +} + +export function asProviderConfig(value: unknown): SpeechProviderConfig { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? withSpeakerSelectionCompat(value as SpeechProviderConfig) + : {}; +} + +export function asProviderConfigMap(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : {}; +} + +export function hasOwnProperty(value: object, key: string): boolean { + return Object.hasOwn(value, key); +} + +function normalizeProviderConfigMap( + value: unknown, +): Record | undefined { + const rawMap = asProviderConfigMap(value); + if (Object.keys(rawMap).length === 0) { + return undefined; + } + const next: Record = {}; + for (const [providerId, providerConfig] of Object.entries(rawMap)) { + const normalized = normalizeConfiguredSpeechProviderId(providerId) ?? providerId; + next[normalized] = asProviderConfig(providerConfig); + } + return next; +} + +function collectTtsPersonas(raw: TtsConfig): Record { + const rawPersonas = asProviderConfigMap(raw.personas); + const personas: Record = {}; + for (const [id, value] of Object.entries(rawPersonas)) { + const normalizedId = normalizeTtsPersonaId(id); + if (!normalizedId || typeof value !== "object" || value === null || Array.isArray(value)) { + continue; + } + const persona = value as Omit; + personas[normalizedId] = { + ...persona, + id: normalizedId, + provider: normalizeConfiguredSpeechProviderId(persona.provider) ?? persona.provider, + providers: normalizeProviderConfigMap(persona.providers), + }; + } + return personas; +} + +function collectDirectProviderConfigEntries(raw: TtsConfig): Record { + const entries: Record = {}; + const rawProviders = asProviderConfigMap(raw.providers); + for (const [providerId, value] of Object.entries(rawProviders)) { + const normalized = normalizeConfiguredSpeechProviderId(providerId) ?? providerId; + entries[normalized] = asProviderConfig(value); + } + const reservedKeys = new Set([ + "auto", + "enabled", + "maxTextLength", + "mode", + "modelOverrides", + "persona", + "personas", + "prefsPath", + "provider", + "providers", + "summaryModel", + "timeoutMs", + ]); + for (const [key, value] of Object.entries(raw as Record)) { + if (reservedKeys.has(key)) { + continue; + } + if (typeof value !== "object" || value === null || Array.isArray(value)) { + continue; + } + const normalized = normalizeConfiguredSpeechProviderId(key) ?? key; + entries[normalized] ??= asProviderConfig(value); + } + return entries; +} + +export function resolveTtsConfig( + cfgInput: OpenClawConfig, + contextOrAgentId?: string | TtsConfigResolutionContext, +): ResolvedTtsConfig { + const cfg = resolveTtsRuntimeConfig(cfgInput); + const raw: TtsConfig = resolveEffectiveTtsConfig(cfg, contextOrAgentId); + const providerSource = raw.provider ? "config" : "default"; + const timeoutMs = raw.timeoutMs ?? DEFAULT_TTS_TIMEOUT_MS; + const timeoutMsSource = raw.timeoutMs === undefined ? "default" : "config"; + return { + auto: resolveConfiguredTtsAutoMode(raw), + mode: raw.mode ?? "final", + provider: + normalizeConfiguredSpeechProviderId(raw.provider) ?? + (providerSource === "config" ? (normalizeOptionalLowercaseString(raw.provider) ?? "") : ""), + providerSource, + persona: normalizeTtsPersonaId(raw.persona), + personas: collectTtsPersonas(raw), + summaryModel: normalizeOptionalString(raw.summaryModel), + modelOverrides: resolveModelOverridePolicy(raw.modelOverrides), + providerConfigs: collectDirectProviderConfigEntries(raw), + prefsPath: (raw as TtsConfig & { prefsPath?: string }).prefsPath, + maxTextLength: raw.maxTextLength ?? DEFAULT_MAX_TEXT_LENGTH, + timeoutMs, + timeoutMsSource, + rawConfig: raw, + sourceConfig: cfg, + }; +} + +export function resolveTtsPrefsPath(config: ResolvedTtsConfig): string { + return resolveTtsPrefsPathValue(config.prefsPath); +} + +export function readTtsPrefs(prefsPath: string): TtsUserPrefs { + try { + if (!existsSync(prefsPath)) { + return {}; + } + const parsed: unknown = JSON.parse(readFileSync(prefsPath, "utf8")); + return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as TtsUserPrefs) + : {}; + } catch { + return {}; + } +} + +function resolveTtsAutoModeFromPrefs(prefs: TtsUserPrefs): TtsAutoMode | undefined { + const auto = normalizeTtsAutoMode(prefs.tts?.auto); + if (auto) { + return auto; + } + if (typeof prefs.tts?.enabled === "boolean") { + return prefs.tts.enabled ? "always" : "off"; + } + return undefined; +} + +export function resolveTtsAutoMode(params: { + config: ResolvedTtsConfig; + prefsPath: string; + sessionAuto?: string; +}): TtsAutoMode { + const sessionAuto = normalizeTtsAutoMode(params.sessionAuto); + if (sessionAuto) { + return sessionAuto; + } + return resolveTtsAutoModeFromPrefs(readTtsPrefs(params.prefsPath)) ?? params.config.auto; +} + +function resolveTtsPersonaIdFromPrefs( + config: ResolvedTtsConfig, + prefs: TtsUserPrefs, +): string | undefined { + if (prefs.tts && hasOwnProperty(prefs.tts, "persona")) { + return normalizeTtsPersonaId(prefs.tts.persona); + } + return normalizeTtsPersonaId(config.persona); +} + +export function resolveTtsPersonaFromPrefs( + config: ResolvedTtsConfig, + prefs: TtsUserPrefs, +): ResolvedTtsPersona | undefined { + const personaId = resolveTtsPersonaIdFromPrefs(config, prefs); + return personaId ? config.personas[personaId] : undefined; +} + +type ResolvedTtsSettingsSnapshot = { + autoMode: TtsAutoMode; + config: ResolvedTtsConfig; + maxLength: number; + persona?: ResolvedTtsPersona; + personaId?: string; + preferredProvider?: TtsProvider; + prefsPath: string; + summarize: boolean; +}; + +export function resolveTtsSettingsSnapshot(params: { + cfg: OpenClawConfig; + sessionAuto?: string; + agentId?: string; + channelId?: string; + accountId?: string; +}): ResolvedTtsSettingsSnapshot { + const config = resolveTtsConfig(params.cfg, { + agentId: params.agentId, + channelId: params.channelId, + accountId: params.accountId, + }); + const prefsPath = resolveTtsPrefsPath(config); + const prefs = readTtsPrefs(prefsPath); + const personaId = resolveTtsPersonaIdFromPrefs(config, prefs); + const persona = personaId ? config.personas[personaId] : undefined; + const preferredProvider = + normalizeConfiguredSpeechProviderId(prefs.tts?.provider) ?? + normalizeConfiguredSpeechProviderId(persona?.provider) ?? + (config.providerSource === "config" + ? (normalizeConfiguredSpeechProviderId(config.provider) ?? config.provider) + : undefined); + return { + autoMode: + normalizeTtsAutoMode(params.sessionAuto) ?? resolveTtsAutoModeFromPrefs(prefs) ?? config.auto, + config, + maxLength: prefs.tts?.maxLength ?? DEFAULT_TTS_MAX_LENGTH, + ...(persona ? { persona } : {}), + ...(personaId ? { personaId } : {}), + ...(preferredProvider ? { preferredProvider } : {}), + prefsPath, + summarize: prefs.tts?.summarize ?? DEFAULT_TTS_SUMMARIZE, + }; +} + +export function buildTtsSystemPromptHint( + cfg: OpenClawConfig, + agentId?: string, +): string | undefined { + const settings = resolveTtsSettingsSnapshot({ cfg, agentId }); + if (settings.autoMode === "off") { + return undefined; + } + const autoHint = + settings.autoMode === "inbound" + ? "Only use TTS when the user's last message includes audio/voice." + : settings.autoMode === "tagged" + ? "Only use TTS when you include [[tts:key=value]] directives or a [[tts:text]]...[[/tts:text]] block." + : undefined; + return [ + "Voice (TTS) is enabled.", + autoHint, + settings.persona + ? `Active TTS persona: ${settings.persona.label ?? settings.persona.id}${settings.persona.description ? ` - ${settings.persona.description}` : ""}.` + : undefined, + `Keep spoken text ≤${settings.maxLength} chars to avoid auto-summary (summary ${settings.summarize ? "on" : "off"}).`, + "If workspace context (especially MEMORY.md) tells you not to use [[tts:...]] or to use a local/non-tagged voice workflow, follow that workspace instruction instead.", + "Use [[tts:...]] and optional [[tts:text]]...[[/tts:text]] to control voice/expressiveness.", + ] + .filter(Boolean) + .join("\n"); +} + +export function isTtsEnabled( + config: ResolvedTtsConfig, + prefsPath: string, + sessionAuto?: string, +): boolean { + return resolveTtsAutoMode({ config, prefsPath, sessionAuto }) !== "off"; +} + +export function getTtsPersona( + config: ResolvedTtsConfig, + prefsPath: string, +): ResolvedTtsPersona | undefined { + return resolveTtsPersonaFromPrefs(config, readTtsPrefs(prefsPath)); +} + +export function listTtsPersonas(config: ResolvedTtsConfig): ResolvedTtsPersona[] { + return Object.values(config.personas).toSorted((left, right) => left.id.localeCompare(right.id)); +} + +export function getTtsMaxLength(prefsPath: string): number { + return readTtsPrefs(prefsPath).tts?.maxLength ?? DEFAULT_TTS_MAX_LENGTH; +} + +export function isSummarizationEnabled(prefsPath: string): boolean { + return readTtsPrefs(prefsPath).tts?.summarize ?? DEFAULT_TTS_SUMMARIZE; +} diff --git a/packages/speech-core/src/tts-streaming.ts b/src/tts/tts-streaming.ts similarity index 94% rename from packages/speech-core/src/tts-streaming.ts rename to src/tts/tts-streaming.ts index ad8092680f61..2b3613f3d5ec 100644 --- a/packages/speech-core/src/tts-streaming.ts +++ b/src/tts/tts-streaming.ts @@ -1,9 +1,9 @@ -import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import type { TtsDirectiveOverrides } from "openclaw/plugin-sdk/speech-core"; +import type { OpenClawConfig } from "../config/types.js"; +import type { TtsDirectiveOverrides } from "./provider-types.js"; import { assertSpeechRuntimeAvailable } from "./runtime-availability.js"; +import type { TtsStreamResult, TtsSynthesisStreamResult } from "./tts-runtime-types.js"; import { executeTtsProviderAttempts, resolveTtsRequestSetup } from "./tts-synthesis-support.js"; import { resolveTtsSynthesisTarget } from "./tts-synthesis.js"; -import type { TtsStreamResult, TtsSynthesisStreamResult } from "./tts-types.js"; export async function streamSpeech(params: { text: string; diff --git a/packages/speech-core/src/tts-synthesis-support.ts b/src/tts/tts-synthesis-support.ts similarity index 95% rename from packages/speech-core/src/tts-synthesis-support.ts rename to src/tts/tts-synthesis-support.ts index 5095db7b7c3d..079856db39b0 100644 --- a/packages/speech-core/src/tts-synthesis-support.ts +++ b/src/tts/tts-synthesis-support.ts @@ -1,18 +1,9 @@ -import type { - OpenClawConfig, - ResolvedTtsPersona, - TtsProvider, -} from "openclaw/plugin-sdk/config-contracts"; -import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; -import { redactSensitiveText } from "openclaw/plugin-sdk/logging-core"; -import { logVerbose } from "openclaw/plugin-sdk/runtime-env"; -import { - canonicalizeSpeechProviderId, - getSpeechProvider, - type SpeechProviderConfig, - type SpeechProviderOverrides, -} from "openclaw/plugin-sdk/speech-core"; -import type { VoiceModelRef, VoiceProviderCandidate } from "../voice-models.js"; +import type { OpenClawConfig, ResolvedTtsPersona, TtsProvider } from "../config/types.js"; +import { logVerbose } from "../globals.js"; +import { formatErrorMessage } from "../infra/errors.js"; +import { redactSensitiveText } from "../logging/redact.js"; +import { canonicalizeSpeechProviderId, getSpeechProvider } from "./provider-registry.js"; +import type { SpeechProviderConfig, SpeechProviderOverrides } from "./provider-types.js"; import { getResolvedSpeechProviderConfigForVoiceModel, mergeProviderConfigWithPersona, @@ -22,6 +13,7 @@ import { resolveTtsProvider, resolveTtsProviderCandidates, } from "./tts-provider-resolution.js"; +import type { TtsProviderAttempt } from "./tts-runtime-types.js"; import { getTtsPersona, resolveTtsConfig, @@ -29,7 +21,7 @@ import { resolveTtsRuntimeConfig, type ResolvedTtsConfig, } from "./tts-settings.js"; -import type { TtsProviderAttempt } from "./tts-types.js"; +import type { VoiceModelRef, VoiceProviderCandidate } from "./voice-models.js"; export function formatTtsProviderError(provider: TtsProvider, err: unknown): string { const error = err instanceof Error ? err : new Error(String(err)); diff --git a/packages/speech-core/src/tts-synthesis.ts b/src/tts/tts-synthesis.ts similarity index 94% rename from packages/speech-core/src/tts-synthesis.ts rename to src/tts/tts-synthesis.ts index 659e8cfa131c..57d72f41bfd8 100644 --- a/packages/speech-core/src/tts-synthesis.ts +++ b/src/tts/tts-synthesis.ts @@ -1,16 +1,16 @@ -import { resolveChannelTtsVoiceDelivery } from "openclaw/plugin-sdk/channel-targets"; -import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import { transcodeAudioBuffer } from "openclaw/plugin-sdk/media-runtime"; -import { logVerbose } from "openclaw/plugin-sdk/runtime-env"; -import type { TtsDirectiveOverrides } from "openclaw/plugin-sdk/speech-core"; +import { resolveChannelTtsVoiceDelivery } from "../channels/plugins/tts-capabilities.js"; +import type { OpenClawConfig } from "../config/types.js"; +import { logVerbose } from "../globals.js"; +import { transcodeAudioBuffer } from "../media/media-services.js"; +import type { TtsDirectiveOverrides } from "./provider-types.js"; import { assertSpeechRuntimeAvailable } from "./runtime-availability.js"; import { normalizeSpeechText } from "./speech-text.js"; +import type { TtsResult, TtsSynthesisResult } from "./tts-runtime-types.js"; import { executeTtsProviderAttempts, resolveTtsRequestSetup, sanitizeTtsErrorForLog, } from "./tts-synthesis-support.js"; -import type { TtsResult, TtsSynthesisResult } from "./tts-types.js"; export type TtsAudioPersistence = (params: { audioBuffer: Buffer; diff --git a/packages/speech-core/src/tts-telephony.ts b/src/tts/tts-telephony.ts similarity index 89% rename from packages/speech-core/src/tts-telephony.ts rename to src/tts/tts-telephony.ts index 76e00fe937fc..3441b2ea147e 100644 --- a/packages/speech-core/src/tts-telephony.ts +++ b/src/tts/tts-telephony.ts @@ -1,8 +1,8 @@ -import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import type { TtsDirectiveOverrides } from "openclaw/plugin-sdk/speech-core"; +import type { OpenClawConfig } from "../config/types.js"; +import type { TtsDirectiveOverrides } from "./provider-types.js"; import { assertSpeechRuntimeAvailable } from "./runtime-availability.js"; +import type { TtsTelephonyResult } from "./tts-runtime-types.js"; import { executeTtsProviderAttempts, resolveTtsRequestSetup } from "./tts-synthesis-support.js"; -import type { TtsTelephonyResult } from "./tts-types.js"; export async function textToSpeechTelephony(params: { text: string; diff --git a/src/tts/tts.test.ts b/src/tts/tts.test.ts index b9e27841d946..b1bf6f510a46 100644 --- a/src/tts/tts.test.ts +++ b/src/tts/tts.test.ts @@ -1,41 +1,12 @@ -// TTS integration tests cover text-to-speech command behavior. -import { readFileSync } from "node:fs"; +// TTS integration tests cover host runtime availability behavior. import { afterEach, describe, expect, it } from "vitest"; import { setActiveDegradedSecretOwners } from "../secrets/runtime-degraded-state.js"; -function readSource(relativePath: string): string { - return readFileSync(new URL(relativePath, import.meta.url), "utf8"); -} - describe("tts runtime facade", () => { afterEach(() => { setActiveDegradedSecretOwners([]); }); - it("routes public TTS helpers through the core speech package", () => { - const publicFacadeSource = readSource("./tts.ts"); - const runtimeFacadeSource = readSource("../plugin-sdk/tts-runtime.ts"); - - expect(publicFacadeSource).toContain('} from "../plugin-sdk/tts-runtime.js";'); - expect(publicFacadeSource).toContain("setSpeechRuntimeAvailabilityGuard"); - expect(runtimeFacadeSource).toContain('from "../../packages/speech-core/runtime-api.js";'); - expect(runtimeFacadeSource).not.toContain('dirName: "speech-core"'); - }); - - it("keeps agent prompt TTS settings off the synthesis runtime chain", () => { - const agentConfigSource = readSource("../agents/system-prompt-config.ts"); - const settingsFacadeSource = readSource("./tts-settings.ts"); - const packageSettingsSource = readSource("../../packages/speech-core/src/tts-settings.ts"); - - expect(agentConfigSource).toContain('from "../tts/tts-settings.js";'); - expect(settingsFacadeSource).toContain( - 'from "../../packages/speech-core/src/tts-settings.js";', - ); - expect(settingsFacadeSource).not.toContain("tts-runtime"); - expect(packageSettingsSource).toContain('from "openclaw/plugin-sdk/speech-settings";'); - expect(packageSettingsSource).not.toContain("plugin-sdk/media-runtime"); - }); - it("blocks explicit synthesis but preserves text delivery when TTS is cold", async () => { setActiveDegradedSecretOwners([ { diff --git a/src/tts/tts.ts b/src/tts/tts.ts index face3ad7dd58..930ed326435c 100644 --- a/src/tts/tts.ts +++ b/src/tts/tts.ts @@ -1,10 +1,13 @@ /** Public TTS runtime barrel exposed to core callers and plugin SDK facades. */ +import { assertSecretOwnerAvailable } from "../secrets/runtime-degraded-state.js"; +import { readConfigMachineState } from "../state/config-machine-state.js"; import { setSpeechRuntimeAvailabilityGuard, setTtsMachinePrefsPathResolver, -} from "../../packages/speech-core/runtime-api.js"; -import { assertSecretOwnerAvailable } from "../secrets/runtime-degraded-state.js"; -import { readConfigMachineState } from "../state/config-machine-state.js"; +} from "./runtime-api.js"; +import { persistTtsAudioToMediaStore } from "./tts-audio-store.js"; +import { maybeApplyTtsToPayload as maybeApplyTtsToPayloadCore } from "./tts-payload.js"; +import { textToSpeech as textToSpeechCore } from "./tts-synthesis.js"; setSpeechRuntimeAvailabilityGuard(() => { assertSecretOwnerAvailable("capability", "tts"); @@ -12,6 +15,14 @@ setSpeechRuntimeAvailabilityGuard(() => { setTtsMachinePrefsPathResolver(() => readConfigMachineState("tts.prefsPath")); +export function textToSpeech(params: Parameters[0]) { + return textToSpeechCore(params, persistTtsAudioToMediaStore); +} + +export function maybeApplyTtsToPayload(params: Parameters[0]) { + return maybeApplyTtsToPayloadCore(params, persistTtsAudioToMediaStore); +} + export { getLastTtsAttempt, getResolvedSpeechProviderConfig, @@ -23,7 +34,6 @@ export { isTtsProviderConfigured, listSpeechVoices, listTtsPersonas, - maybeApplyTtsToPayload, resolveExplicitTtsOverrides, resolveTtsAutoMode, resolveTtsConfig, @@ -36,7 +46,6 @@ export { setTtsPersona, setTtsProvider, synthesizeSpeech, - textToSpeech, type ResolvedTtsConfig, type TtsDirectiveOverrides, -} from "../plugin-sdk/tts-runtime.js"; +} from "./runtime-api.js"; diff --git a/packages/speech-core/voice-models.ts b/src/tts/voice-models.ts similarity index 98% rename from packages/speech-core/voice-models.ts rename to src/tts/voice-models.ts index d998c5100607..f01dd1f76e9d 100644 --- a/packages/speech-core/voice-models.ts +++ b/src/tts/voice-models.ts @@ -1,7 +1,7 @@ // Voice model catalog helpers shared by TTS and realtime voice plugins. import { parseModelCatalogRef } from "@openclaw/model-catalog-core/model-catalog-refs"; -export type VoiceModelCapability = "tts" | "realtime_transcription" | "realtime_voice"; +type VoiceModelCapability = "tts" | "realtime_transcription" | "realtime_voice"; /** Capability flags advertised by a voice model catalog entry. */ export type VoiceModelCapabilities = Partial>; @@ -23,7 +23,7 @@ export type VoiceModelProvider = { }; /** Synthesized voice model catalog row exposed to provider/model selection. */ -export type VoiceModelCatalogEntry = { +type VoiceModelCatalogEntry = { kind: "voice"; provider: string; model: string; diff --git a/test/e2e/qa-lab/runtime/media-talk-gateway.ts b/test/e2e/qa-lab/runtime/media-talk-gateway.ts index 3085256c3636..b27c2ef94fbc 100644 --- a/test/e2e/qa-lab/runtime/media-talk-gateway.ts +++ b/test/e2e/qa-lab/runtime/media-talk-gateway.ts @@ -50,7 +50,7 @@ const SCENARIOS = { docsRefs: ["docs/tools/tts.md", "docs/tools/media-overview.md"], codeRefs: [ SOURCE_PATH, - "packages/speech-core/src/tts.ts", + "src/tts/runtime-api.ts", "src/gateway/managed-image-attachments.ts", "src/gateway/server-methods/artifacts.ts", ], diff --git a/test/vitest-scoped-config.test.ts b/test/vitest-scoped-config.test.ts index 687a121b703b..a674dc84e047 100644 --- a/test/vitest-scoped-config.test.ts +++ b/test/vitest-scoped-config.test.ts @@ -247,13 +247,13 @@ describe("createScopedVitestConfig", () => { it("keeps broad package scoped cli directory filters aligned with repo-root include patterns", () => { const config = createScopedVitestConfig(["packages/**/*.test.ts"], { - argv: ["vitest", "run", "packages/speech-core"], + argv: ["vitest", "run", "packages/normalization-core"], dir: "packages", env: {}, passWithNoTests: true, }); - expect(requireTestConfig(config).include).toEqual(["speech-core/**/*.test.*"]); + expect(requireTestConfig(config).include).toEqual(["normalization-core/**/*.test.*"]); }); it("relativizes scoped include and exclude patterns to the configured dir", () => { diff --git a/tsconfig.json b/tsconfig.json index c59dae092ee1..a42c3a7ee9a6 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -250,11 +250,6 @@ "@openclaw/net-policy/url-protocol": ["./packages/net-policy/src/url-protocol.ts"], "@openclaw/net-policy/url-userinfo": ["./packages/net-policy/src/url-userinfo.ts"], "@openclaw/net-policy/*": ["./packages/net-policy/src/*"], - "@openclaw/speech-core": ["./packages/speech-core/runtime-api.ts"], - "@openclaw/speech-core/runtime-api": ["./packages/speech-core/runtime-api.ts"], - "@openclaw/speech-core/speaker": ["./packages/speech-core/speaker.ts"], - "@openclaw/speech-core/voice-models": ["./packages/speech-core/voice-models.ts"], - "@openclaw/speech-core/*": ["./packages/speech-core/*"], "@openclaw/sdk": ["./packages/sdk/src/index.ts"], "@openclaw/plugin-sdk/*": ["./src/plugin-sdk/*.ts"], "openclaw/plugin-sdk/account-id": ["./src/plugin-sdk/account-id.ts"], diff --git a/tsdown.config.ts b/tsdown.config.ts index 2dcafbe5956d..303e18e0acb6 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -410,14 +410,6 @@ function buildPackageDistEntriesFromExports(packageDir: string): Record a.localeCompare(b))); } -function buildSpeechCoreDistEntries(): Record { - return { - "runtime-api": "packages/speech-core/runtime-api.ts", - speaker: "packages/speech-core/speaker.ts", - "voice-models": "packages/speech-core/voice-models.ts", - }; -} - function buildLlmCoreDistEntries(): Record { return { index: "packages/llm-core/src/index.ts", @@ -458,10 +450,6 @@ function shouldExternalizeNetPolicyDependency(id: string): boolean { return id === "ipaddr.js" || id.startsWith("ipaddr.js/"); } -function shouldExternalizeSpeechCoreDependency(id: string): boolean { - return id === "openclaw" || id.startsWith("openclaw/"); -} - function shouldExternalizeLlmCoreDependency(id: string): boolean { return id === "typebox" || id.startsWith("typebox/"); } @@ -665,12 +653,6 @@ const configs = [ neverBundle: shouldExternalizeTerminalCoreDependency, }, }), - nodeWorkspacePackageBuildConfig("speech-core", { - entry: buildSpeechCoreDistEntries(), - deps: { - neverBundle: shouldExternalizeSpeechCoreDependency, - }, - }), nodeWorkspacePackageBuildConfig("llm-core", { entry: buildLlmCoreDistEntries(), deps: { From a0a189360570f4ec4ef3963bb1c40c3ec8cc9805 Mon Sep 17 00:00:00 2001 From: ralf003 <23394662@qq.com> Date: Mon, 3 Aug 2026 17:36:20 +0800 Subject: [PATCH 05/57] fix(read): explain directory paths (#114601) Co-authored-by: ralf003 <23394662@qq.com> --- ...tools.create-openclaw-coding-tools.test.ts | 19 +++++++++++++++++++ src/agents/agent-tools.read.ts | 3 +++ src/agents/sessions/tools/read.test.ts | 11 +++++++++++ src/agents/sessions/tools/read.ts | 14 ++++++++++++-- 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/agents/agent-tools.create-openclaw-coding-tools.test.ts b/src/agents/agent-tools.create-openclaw-coding-tools.test.ts index 42dd5b1b7a03..19cd9e8e5fb6 100644 --- a/src/agents/agent-tools.create-openclaw-coding-tools.test.ts +++ b/src/agents/agent-tools.create-openclaw-coding-tools.test.ts @@ -9,6 +9,7 @@ import path from "node:path"; import type { AgentTool, AgentToolResult } from "openclaw/plugin-sdk/agent-core"; import { Type } from "typebox"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; import type { SessionEntry } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; @@ -57,6 +58,7 @@ const tinyPngBuffer = Buffer.from( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO2f7z8AAAAASUVORK5CYII=", "base64", ); +const tempDirs = useAutoCleanupTempDirTracker(afterEach); const XAI_UNSUPPORTED_SCHEMA_KEYWORDS = new Set(["minContains", "maxContains"]); function collectActionValues(schema: unknown, values: Set): void { if (!schema || typeof schema !== "object") { @@ -2573,6 +2575,23 @@ describe("createOpenClawCodingTools read behavior", () => { } }); + it("rejects sandbox directory reads before calling the bridge read operation", async () => { + const tmpDir = tempDirs.make("openclaw-sbx-directory-"); + const directoryName = "notes"; + await fs.mkdir(path.join(tmpDir, directoryName)); + const hostBridge = createHostSandboxFsBridge(tmpDir); + const readFile = vi.fn(hostBridge.readFile.bind(hostBridge)); + const readTool = createSandboxedReadTool({ + root: tmpDir, + bridge: { ...hostBridge, readFile }, + }); + + await expect(readTool.execute("sandbox-directory", { path: directoryName })).rejects.toThrow( + `Read requires a file path, but ${directoryName} is a directory. List the directory, then read a specific file.`, + ); + expect(readFile).not.toHaveBeenCalled(); + }); + it("auto-pages read output across chunks when context window budget allows", async () => { const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-read-autopage-")); const filePath = path.join(tmpDir, "big.txt"); diff --git a/src/agents/agent-tools.read.ts b/src/agents/agent-tools.read.ts index 53568473102b..7262400fea60 100644 --- a/src/agents/agent-tools.read.ts +++ b/src/agents/agent-tools.read.ts @@ -1090,6 +1090,9 @@ async function assertSandboxFileExists(params: SandboxToolParams, absolutePath: if (!stat) { throw createFsAccessError("ENOENT", absolutePath); } + if (stat.type === "directory") { + throw createFsAccessError("EISDIR", absolutePath); + } } function expandTildeToOsHome(filePath: string): string { diff --git a/src/agents/sessions/tools/read.test.ts b/src/agents/sessions/tools/read.test.ts index 73dc65b5dd04..7170290075e7 100644 --- a/src/agents/sessions/tools/read.test.ts +++ b/src/agents/sessions/tools/read.test.ts @@ -119,6 +119,17 @@ describe("read tool", () => { ); }); + it("explains that directory paths must be listed before reading a file", async () => { + const tempDir = tempDirs.make("openclaw-read-directory-"); + const tool = createReadToolDefinition(tempDir); + + await expect( + tool.execute("call-directory", { path: "." }, undefined, undefined, {} as never), + ).rejects.toThrow( + "Read requires a file path, but . is a directory. List the directory, then read a specific file.", + ); + }); + it("shell-quotes the long-first-line fallback path", async () => { // The fallback command is shown to the model; quote the path so suggested // follow-up commands cannot execute path text as shell syntax. diff --git a/src/agents/sessions/tools/read.ts b/src/agents/sessions/tools/read.ts index 0ce2e909fc3f..f196f08ada6a 100644 --- a/src/agents/sessions/tools/read.ts +++ b/src/agents/sessions/tools/read.ts @@ -3,7 +3,7 @@ import { access as fsAccess, readFile as fsReadFile } from "node:fs/promises"; import { basename, dirname, isAbsolute, relative, resolve as resolvePath, sep } from "node:path"; import { Text } from "@earendil-works/pi-tui"; import { Type } from "typebox"; -import { toErrorObject } from "../../../infra/errors.js"; +import { hasErrnoCode, toErrorObject } from "../../../infra/errors.js"; import { decodeWindowsTextFileBuffer } from "../../../infra/windows-encoding.js"; import type { ImageContent, Model, TextContent } from "../../../llm/types.js"; import { @@ -119,6 +119,16 @@ function createReadDetails( } return { kind: "text", content: text }; } + +function normalizeReadError(error: unknown, filePath: string): Error { + if (hasErrnoCode(error, "EISDIR")) { + return new Error( + `Read requires a file path, but ${filePath} is a directory. List the directory, then read a specific file.`, + ); + } + return toErrorObject(error, "Non-Error rejection"); +} + interface CompactReadClassification { kind: "docs" | "resource" | "skill"; label: string; @@ -481,7 +491,7 @@ export function createReadToolDefinition( } catch (error: unknown) { signal?.removeEventListener("abort", onAbort); if (!aborted) { - reject(toErrorObject(error, "Non-Error rejection")); + reject(normalizeReadError(error, path)); } } })(); From 0fbddda540e171d2390d257006303a45919ed5bc Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 02:37:59 -0700 Subject: [PATCH 06/57] improve(qa): replace terminal reply timing waits (#118600) * test(qa): replace terminal reply timing waits * test(qa): keep terminal contract in channel suite * test(qa): scope terminal settlement per worker * test(qa): bind settlement to child session --------- Co-authored-by: Peter Steinberger --- .../mock-openai/mock-openai-contracts.ts | 1 - .../mock-openai-responses-websocket.ts | 2 + .../src/providers/mock-openai/server.test.ts | 62 ++++++++++ .../src/providers/mock-openai/server.ts | 110 ++++++++++++++++-- .../src/scenario-catalog-channels.test.ts | 30 +++++ .../subagent-completion-direct-fallback.yaml | 62 ++++++---- 6 files changed, 236 insertions(+), 31 deletions(-) diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts index d5d857f5694c..7132d99f3346 100644 --- a/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts @@ -270,7 +270,6 @@ export const QA_SUBAGENT_TERMINAL_MARKERS = { fallback: "QA-SUBAGENT-TERMINAL-FALLBACK-OK", } as const; export const QA_SUBAGENT_TERMINAL_METADATA_SENTINEL = "QA-SUBAGENT-TERMINAL-INTERNAL-MUST-NOT-LEAK"; -export const QA_SUBAGENT_TERMINAL_WORKER_DELAY_MS = 5_000; export const QA_NATIVE_STOP_DELAY_PROMPT_RE = /subagent recovery worker native command target proof\.\s*wait until stopped\./i; export const QA_NATIVE_STOP_DELAY_MS = 180_000; diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.ts index 4918f436b1f6..22439a4eb028 100644 --- a/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.ts +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.ts @@ -10,6 +10,7 @@ export type QaMockResponsesDispatchResult = { type: string; message: string; }; + onResponseSent?: () => void; previewPauseMs?: number; }; @@ -232,6 +233,7 @@ export function attachQaMockResponsesWebSocketServer(params: { } sendEvent(event); } + dispatched.onResponseSent?.(); }) .catch(() => { cachedResponse = undefined; diff --git a/extensions/qa-lab/src/providers/mock-openai/server.test.ts b/extensions/qa-lab/src/providers/mock-openai/server.test.ts index 7b2793c98ad4..5548c2744463 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.test.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.test.ts @@ -2688,6 +2688,68 @@ describe("qa mock openai server", () => { expect(outputText(payload)).toBe(expected); }); + it("binds crossed same-case parent responses to their matching workers", async () => { + const server = await startMockServer(); + const firstChildSessionKey = "agent:qa:subagent:child-1"; + const secondChildSessionKey = "agent:qa:subagent:child-2"; + const startChild = (runtimeSessionId: string, childSessionKey: string) => + postNonStreamingResponses(server, { + model: "gpt-5.6-luna", + instructions: [ + `Runtime: embedded | sessionId=${runtimeSessionId}`, + `- Your session: ${childSessionKey}.`, + ].join("\n"), + input: [makeUserInput("Subagent terminal reply QA worker: visible.")], + }); + const settleParent = async ( + runtimeSessionId: string, + childSessionKey: string, + callId: string, + ) => { + const parent = await expectNonStreamingResponsesJson(server, { + model: "gpt-5.6-luna", + instructions: `Runtime: embedded | sessionId=${runtimeSessionId}`, + tools: [SESSIONS_SPAWN_TOOL, SESSIONS_YIELD_TOOL], + input: [ + makeUserInput("Subagent terminal reply QA check: visible."), + makeToolOutputWithCallId( + callId, + JSON.stringify({ status: "accepted", childSessionKey, runId: `run-${callId}` }), + ), + ], + }); + expect(outputText(parent)).toBe("NO_REPLY"); + }; + + const firstChildResponse = startChild("qa-terminal-child-1", firstChildSessionKey); + const secondChildResponse = startChild("qa-terminal-child-2", secondChildSessionKey); + let firstChildSettled = false; + let secondChildSettled = false; + void firstChildResponse.then(() => { + firstChildSettled = true; + }); + void secondChildResponse.then(() => { + secondChildSettled = true; + }); + + await expect + .poll(async () => { + const inflight = await getJson(server, "/debug/inflight-requests"); + return inflight.length; + }) + .toBe(2); + + await settleParent("qa-terminal-parent-2", secondChildSessionKey, "call_spawn_2"); + const secondChild = await (await expectOk(secondChildResponse)).json(); + expect(outputText(secondChild)).toBe("QA-SUBAGENT-TERMINAL-VISIBLE-OK"); + expect(secondChildSettled).toBe(true); + expect(firstChildSettled).toBe(false); + + await settleParent("qa-terminal-parent-1", firstChildSessionKey, "call_spawn_1"); + const firstChild = await (await expectOk(firstChildResponse)).json(); + expect(outputText(firstChild)).toBe("QA-SUBAGENT-TERMINAL-VISIBLE-OK"); + }); + it("keeps the empty terminal worker empty across retry prompts", async () => { const server = await startMockServer(); const payload = await expectNonStreamingResponsesJson(server, { diff --git a/extensions/qa-lab/src/providers/mock-openai/server.ts b/extensions/qa-lab/src/providers/mock-openai/server.ts index b6d8d043e1d0..05f7472309c5 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.ts @@ -64,7 +64,6 @@ import { QA_SUBAGENT_DIRECT_FALLBACK_MARKER, QA_SUBAGENT_TERMINAL_MARKERS, QA_SUBAGENT_TERMINAL_METADATA_SENTINEL, - QA_SUBAGENT_TERMINAL_WORKER_DELAY_MS, QA_NATIVE_STOP_DELAY_PROMPT_RE, QA_NATIVE_STOP_DELAY_MS, QA_IMAGE_GENERATION_PROMPT_RE, @@ -468,9 +467,76 @@ function extractScenarioPlannedTool(events: StreamEvent[]) { : { name: wireName, args: wireArgs, wireName }; } +type TerminalRequesterSettleGate = { + markSettled: (caseName: string, childSessionKey: string) => void; + waitUntilSettled: (caseName: string, childSessionKey: string) => Promise; +}; + +function createTerminalRequesterSettleGate(): TerminalRequesterSettleGate { + const settledChildren = new Set(); + const waiterPromises = new Map>(); + const waiters = new Map void>(); + const childKey = (caseName: string, childSessionKey: string) => `${caseName}\n${childSessionKey}`; + return { + markSettled(caseName, childSessionKey) { + const key = childKey(caseName, childSessionKey); + settledChildren.add(key); + waiters.get(key)?.(); + }, + async waitUntilSettled(caseName, childSessionKey) { + const key = childKey(caseName, childSessionKey); + if (settledChildren.has(key)) { + return; + } + const existing = waiterPromises.get(key); + if (existing) { + return await existing; + } + const promise = new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + waiters.delete(key); + waiterPromises.delete(key); + reject(new Error(`terminal requester did not settle: ${caseName} (${childSessionKey})`)); + }, 30_000); + const finish = () => { + clearTimeout(timeout); + waiters.delete(key); + waiterPromises.delete(key); + resolve(); + }; + waiters.set(key, finish); + }); + waiterPromises.set(key, promise); + await promise; + }, + }; +} + +function resolveQaRuntimeSessionId(input: ResponsesInputItem[], body: Record) { + return /\bRuntime:\s*[^\n]*\bsessionId=([^\s|]+)/u.exec(extractAllRequestTexts(input, body))?.[1]; +} + +function resolveQaChildSessionKey(input: ResponsesInputItem[], body: Record) { + const systemPrompt = extractAllRequestTexts( + input.filter((item) => item.role === "developer" || item.role === "system"), + body, + ); + return /^- Your session:\s*(.+?)\.\s*$/mu.exec(systemPrompt)?.[1]?.trim(); +} + +function resolveAcceptedChildSessionKey(input: ResponsesInputItem[]) { + const output = parseToolOutputJson(extractToolOutput(input)); + return output?.status === "accepted" && typeof output.childSessionKey === "string" + ? output.childSessionKey.trim() || undefined + : undefined; +} + async function buildResponsesPayload( body: Record, scenarioState: MockScenarioState, + options: { + waitForTerminalRequesterSettled?: (caseName: string, childSessionKey: string) => Promise; + } = {}, ) { const providerVariant = resolveProviderVariant( typeof body.model === "string" ? body.model : undefined, @@ -778,7 +844,10 @@ async function buildResponsesPayload( .at(-1)?.[1] ?.toLowerCase(); if (terminalWorkerCase) { - await sleep(QA_SUBAGENT_TERMINAL_WORKER_DELAY_MS); + const childSessionKey = resolveQaChildSessionKey(input, body); + if (options.waitForTerminalRequesterSettled && childSessionKey) { + await options.waitForTerminalRequesterSettled(terminalWorkerCase, childSessionKey); + } } if (terminalWorkerCase === "silent") { return buildAssistantEvents("NO_REPLY"); @@ -1931,6 +2000,7 @@ export async function startQaMockOpenAiServer(params?: { }) { const host = params?.host ?? "127.0.0.1"; const finalOnlyMarkerPauseMs = params?.finalOnlyMarkerPauseMs ?? 1_500; + const terminalRequesterSettleGate = createTerminalRequesterSettleGate(); const scenarioStates = new Map(); const scenarioStateFor = (body: Record): MockScenarioState => { const input = Array.isArray(body.input) @@ -1939,12 +2009,8 @@ export async function startQaMockOpenAiServer(params?: { system: body.system as AnthropicMessagesRequest["system"], messages: [], }); - const systemPrompt = extractAllRequestTexts( - input.filter((item) => item.role === "developer" || item.role === "system"), - body, - ); const sessionId = - /\bRuntime:\s*[^\n]*\bsessionId=([^\s|]+)/u.exec(systemPrompt)?.[1] ?? + resolveQaRuntimeSessionId(input, body) ?? (body.client_metadata as { session_id?: unknown } | undefined)?.session_id; const key = typeof sessionId === "string" ? sessionId : ""; // Runtime session identity survives provider switches and cache-boundary changes. @@ -1989,12 +2055,29 @@ export async function startQaMockOpenAiServer(params?: { inflightRequests.set(inflightRequestId, { prompt, allInputText }); let events: StreamEvent[]; try { - events = await buildResponsesPayload(request.body, scenarioStateFor(request.body)); + events = await buildResponsesPayload(request.body, scenarioStateFor(request.body), { + waitForTerminalRequesterSettled: terminalRequesterSettleGate.waitUntilSettled, + }); } finally { inflightRequests.delete(inflightRequestId); } const resolvedModel = typeof request.body.model === "string" ? request.body.model : ""; const plannedTool = extractScenarioPlannedTool(events); + const terminalRequesterCase = extractLastMatchingUserTurn( + input, + QA_SUBAGENT_TERMINAL_MATRIX_PROMPT_RE, + ) + ?.text.match(QA_SUBAGENT_TERMINAL_MATRIX_PROMPT_RE)?.[1] + ?.toLowerCase(); + const settledTerminalRequester = + terminalRequesterCase && resolveQaRuntimeSessionId(input, request.body) + ? { + caseName: terminalRequesterCase, + childSessionKey: resolveAcceptedChildSessionKey(input), + } + : undefined; + const settledTerminalCaseName = settledTerminalRequester?.caseName; + const settledChildSessionKey = settledTerminalRequester?.childSessionKey; recordRequest({ raw: request.raw, body: request.body, @@ -2016,6 +2099,15 @@ export async function startQaMockOpenAiServer(params?: { }); return { events, + ...(settledTerminalCaseName && settledChildSessionKey + ? { + onResponseSent: () => + terminalRequesterSettleGate.markSettled( + settledTerminalCaseName, + settledChildSessionKey, + ), + } + : {}), ...(QA_PROVIDER_HTTP_503_AFTER_TOOL_PROMPT_RE.test(allInputText) && hasToolOutput(input) ? { failure: { @@ -2189,6 +2281,7 @@ export async function startQaMockOpenAiServer(params?: { return; } writeJson(res, 200, completion.response); + dispatched.onResponseSent?.(); return; } if (dispatched.previewPauseMs !== undefined) { @@ -2196,6 +2289,7 @@ export async function startQaMockOpenAiServer(params?: { } else { writeSse(res, events); } + dispatched.onResponseSent?.(); return; } if (req.method === "POST" && url.pathname === "/v1/messages") { diff --git a/extensions/qa-lab/src/scenario-catalog-channels.test.ts b/extensions/qa-lab/src/scenario-catalog-channels.test.ts index e457162cf2dd..731504955b26 100644 --- a/extensions/qa-lab/src/scenario-catalog-channels.test.ts +++ b/extensions/qa-lab/src/scenario-catalog-channels.test.ts @@ -143,6 +143,36 @@ describe("qa scenario catalog channel contracts", () => { expect(flow).not.toContain('"value":"subagent-1: ok\\nsubagent-2: ok"'); }); + it("settles terminal-reply scenarios from durable task facts instead of sleeps", () => { + const scenario = requireFlowScenario(readQaScenarioById("subagent-completion-direct-fallback")); + const flow = JSON.stringify(scenario.execution.flow); + const config = scenario.execution.config as + | { cases?: Array<{ name?: string; marker?: string; expectedSendCount?: number }> } + | undefined; + + expect(config?.cases).toEqual([ + { + name: "visible", + marker: "QA-SUBAGENT-TERMINAL-VISIBLE-OK", + expectedSendCount: 1, + }, + { name: "silent", marker: "NO_REPLY", expectedSendCount: 0 }, + { + name: "fallback", + marker: "QA-SUBAGENT-TERMINAL-FALLBACK-OK", + expectedSendCount: 1, + }, + ]); + expect(flow).toContain("env.gateway.call('tasks.list'"); + expect(flow).toContain("task.title === `qa-terminal-${caseName}`"); + expect(flow).toContain("task.status === 'completed'"); + expect(flow).toContain("task.deliveryStatus === 'delivered'"); + expect(flow).toContain("readSettledTerminalTask('restart')"); + expect(flow).toContain("readSettledTerminalTask('empty')"); + expect(flow).toContain("verdicts.length === 5"); + expect(flow).not.toContain('"call":"sleep"'); + }); + it("keeps channel streaming evidence portable across QA Channel and Crabline Telegram", () => { const scenario = requireFlowScenario(readQaScenarioById("channel-message-flows")); diff --git a/qa/scenarios/agents/subagent-completion-direct-fallback.yaml b/qa/scenarios/agents/subagent-completion-direct-fallback.yaml index b0f3543c7e72..8cbbf191d2c0 100644 --- a/qa/scenarios/agents/subagent-completion-direct-fallback.yaml +++ b/qa/scenarios/agents/subagent-completion-direct-fallback.yaml @@ -68,6 +68,15 @@ flow: - set: verdicts value: expr: "[]" + # The task ledger records terminal delivery after the subagent lifecycle + # owner settles it. Use that fact instead of guessing with wall-clock sleeps. + - set: readSettledTerminalTask + value: + lambda: + params: + - caseName + async: true + expr: "(await env.gateway.call('tasks.list', { status: 'completed', agentId: 'qa', limit: 100 }, { timeoutMs: 10000 })).tasks?.find((task) => task.title === `qa-terminal-${caseName}` && task.status === 'completed' && task.deliveryStatus === 'delivered')" - forEach: items: expr: config.cases @@ -93,13 +102,12 @@ flow: senderName: QA Terminal Reply Operator text: expr: "`Subagent terminal reply QA check: ${terminalCase.name}. Spawn one native worker, then finish the parent turn without waiting. Do not use ACP.`" - - call: sleep - args: - - 20000 - call: waitForCondition + saveAs: terminalTask args: - lambda: - expr: "terminalCase.expectedSendCount === 0 || state.getSnapshot().messages.slice(startIndex).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === conversationId && String(candidate.text ?? '').trim() === terminalCase.marker).length >= terminalCase.expectedSendCount" + async: true + expr: "(async () => { const task = await readSettledTerminalTask(terminalCase.name); if (!task) return undefined; const matchingCount = state.getSnapshot().messages.slice(startIndex).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === conversationId && String(candidate.text ?? '').trim() === terminalCase.marker).length; return terminalCase.expectedSendCount === 0 || matchingCount >= terminalCase.expectedSendCount ? task : undefined; })().catch(() => undefined)" - 60000 - 250 - set: caseOutbound @@ -139,14 +147,15 @@ flow: expr: "!caseRequests.some((request) => request.plannedToolName === 'sessions_yield')" message: expr: "`terminal ${terminalCase.name}: parent did not end before direct fallback; requests=${JSON.stringify(caseRequests)}`" + - assert: + expr: "terminalTask.title === `qa-terminal-${terminalCase.name}` && terminalTask.status === 'completed' && terminalTask.deliveryStatus === 'delivered'" + message: + expr: "`terminal ${terminalCase.name}: task lifecycle did not settle authoritatively; task=${JSON.stringify(terminalTask)}`" - set: appendVerdict value: - expr: "verdicts.push({ case: terminalCase.name, conversationId, inputDisposition: terminalCase.name, representation: terminalCase.name === 'silent' ? 'no terminal channel payload' : 'exact terminal text', restart: false, fallback: terminalCase.name === 'fallback', expectedTerminalSendCount: terminalCase.expectedSendCount, actualTerminalSendCount: matchingOutbound.length, capturedTerminalPayloads: matchingOutbound.map((message) => String(message.text ?? '')), auxiliaryChannelEvents: caseOutbound.filter((message) => !matchingOutbound.includes(message)).map((message) => String(message.text ?? '')), silenceTokenLeaked: caseOutbound.some((message) => String(message.text ?? '').trim() === 'NO_REPLY'), internalMetadataLeak: caseOutbound.some((message) => String(message.text ?? '').includes(config.metadataSentinel)), pass: true })" - # The direct platform send commits before transcript mirroring and - # requester cleanup. Restart only after those post-send owners settle. - - call: sleep - args: - - 15000 + expr: "verdicts.push({ case: terminalCase.name, conversationId, taskId: terminalTask.taskId, taskDeliveryStatus: terminalTask.deliveryStatus, inputDisposition: terminalCase.name, representation: terminalCase.name === 'silent' ? 'no terminal channel payload' : 'exact terminal text', restart: false, fallback: terminalCase.name === 'fallback', expectedTerminalSendCount: terminalCase.expectedSendCount, actualTerminalSendCount: matchingOutbound.length, capturedTerminalPayloads: matchingOutbound.map((message) => String(message.text ?? '')), auxiliaryChannelEvents: caseOutbound.filter((message) => !matchingOutbound.includes(message)).map((message) => String(message.text ?? '')), silenceTokenLeaked: caseOutbound.some((message) => String(message.text ?? '').trim() === 'NO_REPLY'), internalMetadataLeak: caseOutbound.some((message) => String(message.text ?? '').includes(config.metadataSentinel)), pass: true })" + # Every prior task is now terminal and delivery-settled, so restart from + # the lifecycle boundary rather than waiting an arbitrary grace period. - set: preRestartOutbound value: expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound' && verdicts.some((verdict) => verdict.conversationId === message.conversation.id)).map((message) => ({ id: message.id, conversationId: message.conversation.id, text: String(message.text ?? '') }))" @@ -170,9 +179,12 @@ flow: args: - ref: env - 180000 - - call: sleep + - call: waitForCondition args: - - 3000 + - lambda: + expr: "state.getSnapshot().messages.find((message) => message.direction === 'outbound' && verdicts.some((verdict) => verdict.conversationId === message.conversation.id) && !preRestartOutbound.some((before) => before.id === message.id) && String(message.text ?? '').includes('interrupted by a gateway restart'))" + - 60000 + - 250 - set: postRestartOutbound value: expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound' && verdicts.some((verdict) => verdict.conversationId === message.conversation.id)).map((message) => ({ id: message.id, conversationId: message.conversation.id, text: String(message.text ?? '') }))" @@ -206,13 +218,12 @@ flow: ref: restartConversationId senderName: QA Restart Operator text: "Subagent terminal reply QA check: restart. Spawn one native worker, then finish the parent turn without waiting. Do not use ACP." - - call: sleep - args: - - 20000 - call: waitForCondition + saveAs: restartTask args: - lambda: - expr: "state.getSnapshot().messages.slice(restartStartIndex).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === restartConversationId && String(candidate.text ?? '').trim() === config.restartMarker).length >= 1" + async: true + expr: "(async () => { const task = await readSettledTerminalTask('restart'); if (!task) return undefined; const delivered = state.getSnapshot().messages.slice(restartStartIndex).some((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === restartConversationId && String(candidate.text ?? '').trim() === config.restartMarker); return delivered ? task : undefined; })().catch(() => undefined)" - 60000 - 250 - set: restartMatches @@ -226,9 +237,13 @@ flow: expr: "state.getSnapshot().messages.slice(restartStartIndex).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === restartConversationId).every((candidate) => !String(candidate.text ?? '').includes(`Agent couldn't generate a response`))" message: expr: "`restart completion produced a failure diagnostic: outbound=${recentOutboundSummary(state)}`" + - assert: + expr: "restartTask.title === 'qa-terminal-restart' && restartTask.status === 'completed' && restartTask.deliveryStatus === 'delivered'" + message: + expr: "`restart completion task lifecycle did not settle authoritatively; task=${JSON.stringify(restartTask)}`" - set: appendRestartVerdict value: - expr: "verdicts.push({ case: 'restart', conversationId: restartConversationId, inputDisposition: 'visible', restart: true, fallback: true, preRestartTerminalMessageCount: preRestartTerminalPayloads.length, postRestartTerminalPayloadCount: postRestartTerminalPayloads.length, priorTerminalPayloadReplayCount: postRestartTerminalPayloads.length - preRestartTerminalPayloads.length, interruptedHandoffRepresentationCount: restartInterruptionPayloads.length, interruptedHandoffPayloads: restartInterruptionPayloads.map((message) => message.text), expectedTerminalSendCount: 1, actualTerminalSendCount: restartMatches.length, capturedTerminalPayloads: restartMatches.map((message) => String(message.text ?? '')), silenceTokenLeaked: false, internalMetadataLeak: false, pass: true })" + expr: "verdicts.push({ case: 'restart', conversationId: restartConversationId, taskId: restartTask.taskId, taskDeliveryStatus: restartTask.deliveryStatus, inputDisposition: 'visible', restart: true, fallback: true, preRestartTerminalMessageCount: preRestartTerminalPayloads.length, postRestartTerminalPayloadCount: postRestartTerminalPayloads.length, priorTerminalPayloadReplayCount: postRestartTerminalPayloads.length - preRestartTerminalPayloads.length, interruptedHandoffRepresentationCount: restartInterruptionPayloads.length, interruptedHandoffPayloads: restartInterruptionPayloads.map((message) => message.text), expectedTerminalSendCount: 1, actualTerminalSendCount: restartMatches.length, capturedTerminalPayloads: restartMatches.map((message) => String(message.text ?? '')), silenceTokenLeaked: false, internalMetadataLeak: false, pass: true })" - set: emptyStartIndex value: expr: state.getSnapshot().messages.length @@ -250,13 +265,12 @@ flow: text: "Subagent terminal reply QA check: empty. Spawn one native worker, then finish the parent turn without waiting. Do not use ACP." # Empty output after one side effect is terminal and must surface one # explicit representation without leaking the protected raw result. - - call: sleep - args: - - 45000 - call: waitForCondition + saveAs: emptyTask args: - lambda: - expr: "state.getSnapshot().messages.slice(emptyStartIndex).some((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === emptyConversationId)" + async: true + expr: "(async () => { const task = await readSettledTerminalTask('empty'); if (!task) return undefined; const represented = state.getSnapshot().messages.slice(emptyStartIndex).some((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === emptyConversationId && String(candidate.text ?? '').trim() === 'QA-SUBAGENT-TERMINAL-EMPTY-REPRESENTED'); return represented ? task : undefined; })().catch(() => undefined)" - 60000 - 250 - set: emptyOutbound @@ -280,9 +294,13 @@ flow: expr: "emptyRequests.some((request) => request.plannedToolName === 'sessions_spawn' && request.plannedToolArgs?.label === 'qa-terminal-empty') && emptyRequests.some((request) => request.plannedToolName === 'write' && request.plannedToolArgs?.path === 'qa-terminal-empty-side-effect.txt') && emptyRequests.some((request) => request.plannedToolName === 'message' && request.plannedToolArgs?.action === 'send' && request.plannedToolArgs?.message === 'QA-SUBAGENT-TERMINAL-EMPTY-REPRESENTED') && !emptyRequests.some((request) => request.plannedToolName === 'sessions_yield')" message: expr: "`empty completion did not exercise native spawn/direct-fallback: ${JSON.stringify(emptyRequests)}`" + - assert: + expr: "emptyTask.title === 'qa-terminal-empty' && emptyTask.status === 'completed' && emptyTask.deliveryStatus === 'delivered'" + message: + expr: "`empty completion task lifecycle did not settle authoritatively; task=${JSON.stringify(emptyTask)}`" - set: appendEmptyVerdict value: - expr: "verdicts.push({ case: 'empty', conversationId: emptyConversationId, inputDisposition: 'empty-after-side-effect', representation: 'visible ambiguity warning for producer-empty result', restart: false, fallback: false, expectedTerminalSendCount: 1, actualTerminalSendCount: emptyRepresentation.length, capturedTerminalPayloads: emptyRepresentation.map((message) => String(message.text ?? '')), auxiliaryChannelEvents: emptyOutbound.filter((message) => !emptyRepresentation.includes(message)).map((message) => String(message.text ?? '')), silenceTokenLeaked: false, internalMetadataLeak: false, pass: true })" + expr: "verdicts.push({ case: 'empty', conversationId: emptyConversationId, taskId: emptyTask.taskId, taskDeliveryStatus: emptyTask.deliveryStatus, inputDisposition: 'empty-after-side-effect', representation: 'visible ambiguity warning for producer-empty result', restart: false, fallback: false, expectedTerminalSendCount: 1, actualTerminalSendCount: emptyRepresentation.length, capturedTerminalPayloads: emptyRepresentation.map((message) => String(message.text ?? '')), auxiliaryChannelEvents: emptyOutbound.filter((message) => !emptyRepresentation.includes(message)).map((message) => String(message.text ?? '')), silenceTokenLeaked: false, internalMetadataLeak: false, pass: true })" - assert: expr: "verdicts.length === 5 && verdicts.every((verdict) => verdict.pass === true)" message: From f6540b0a1f6cd6cdc99f43bb373b4e788a0268c9 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 02:41:12 -0700 Subject: [PATCH 07/57] test(agents): reuse prepared plugin metadata (#118613) Co-authored-by: Peter Steinberger --- src/agents/model-fallback.test.ts | 31 ++++++------------ src/agents/model-runtime-aliases.test.ts | 40 +++++++++++++++++------- 2 files changed, 38 insertions(+), 33 deletions(-) diff --git a/src/agents/model-fallback.test.ts b/src/agents/model-fallback.test.ts index 5d6dd54ad9d8..6f48416f2280 100644 --- a/src/agents/model-fallback.test.ts +++ b/src/agents/model-fallback.test.ts @@ -2,7 +2,7 @@ import crypto from "node:crypto"; import path from "node:path"; import { expectDefined } from "@openclaw/normalization-core"; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { TranscriptNotContinuableError } from "../../packages/agent-core/src/errors.js"; import type { OpenClawConfig } from "../config/config.js"; import { createAgentRunStaleLifecycleError } from "../infra/agent-lifecycle-error.js"; @@ -13,9 +13,6 @@ import { } from "../infra/diagnostic-events.js"; import { resetLogger, setLoggerOverride } from "../logging/logger.js"; import { createWarnLogCapture } from "../logging/test-helpers/warn-log-capture.js"; -import { setCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-snapshot.js"; -import { clearCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-state.js"; -import { loadPluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; import { GatewayDrainingError } from "../process/gateway-work-admission.js"; import { AgentRunTerminalOutcomeError } from "./agent-run-terminal-error.js"; import { AUTH_STORE_VERSION } from "./auth-profiles/constants.js"; @@ -50,13 +47,19 @@ import { resolveSessionSuspensionReason } from "./session-suspension.js"; import { SessionWriteLockTimeoutError } from "./session-write-lock-error.js"; import { makeModelFallbackCfg } from "./test-helpers/model-fallback-config-fixture.js"; +const emptyManifestPlugins = [] as const; + +function resolveFallbackCandidateRoutes(params: Parameters[0]) { + return resolveModelCandidateChain({ manifestPlugins: emptyManifestPlugins, ...params }); +} + function resolveFallbackCandidateRefs(params: Parameters[0]) { - return resolveModelCandidateChain(params).map(({ provider, model }) => ({ provider, model })); + return resolveFallbackCandidateRoutes(params).map(({ provider, model }) => ({ provider, model })); } const testing = { resolveFallbackCandidates: resolveFallbackCandidateRefs, - resolveFallbackCandidateRoutes: resolveModelCandidateChain, + resolveFallbackCandidateRoutes, resolveSessionSuspensionReason, shouldDiscardDeferredSessionSuspension, }; @@ -215,7 +218,6 @@ vi.mock("./auth-profiles.runtime.js", () => authRuntimeMock.runtime); const makeCfg = makeModelFallbackCfg; let authTempRoot = ""; let authTempCounter = 0; -const emptyManifestPlugins = [] as const; function registerFallbackHarness(id: string): void { registerAgentHarness( @@ -242,14 +244,6 @@ function createHarnessScopedPreflightError(harnessId: string): AgentHarnessPrefl const runWithModelFallback: typeof runWithModelFallbackBase = (params) => runWithModelFallbackBase({ manifestPlugins: emptyManifestPlugins, ...params }); -beforeAll(() => { - setDefaultPluginMetadataSnapshot(); -}); - -afterAll(() => { - clearCurrentPluginMetadataSnapshot(); -}); - function resetModelFallbackTestState(): void { // Fallback state has process-level caches for skip markers, harnesses, auth, // and plugin normalization. Reset every surface between tests. @@ -265,13 +259,6 @@ function resetModelFallbackTestState(): void { resetDiagnosticEventsForTest(); } -function setDefaultPluginMetadataSnapshot(): void { - setCurrentPluginMetadataSnapshot(loadPluginMetadataSnapshot({ config: {}, env: process.env }), { - config: {}, - env: process.env, - }); -} - afterEach(() => { resetModelFallbackTestState(); cliBackendsTesting.resetDepsForTest(); diff --git a/src/agents/model-runtime-aliases.test.ts b/src/agents/model-runtime-aliases.test.ts index 82b6ce53a5ca..99e81dd69162 100644 --- a/src/agents/model-runtime-aliases.test.ts +++ b/src/agents/model-runtime-aliases.test.ts @@ -9,9 +9,36 @@ import { import { areRuntimeModelRefsEquivalent, isCliRuntimeProvider, - resolveCliRuntimeExecutionProvider, + resolveCliRuntimeExecutionProvider as resolveCliRuntimeExecutionProviderBase, } from "./model-runtime-aliases.js"; +const anthropicAuthAliasMetadata = { + plugins: [ + { + id: "anthropic", + origin: "bundled", + providerAuthChoices: [ + { + provider: "anthropic", + method: "cli", + choiceId: "anthropic-cli", + deprecatedChoiceIds: ["claude-cli"], + choiceLabel: "Anthropic Claude CLI", + }, + ], + }, + ], +} as never; + +function resolveCliRuntimeExecutionProvider( + params: Omit[0], "metadataSnapshot">, +) { + return resolveCliRuntimeExecutionProviderBase({ + ...params, + metadataSnapshot: anthropicAuthAliasMetadata, + }); +} + function createAnthropicAuthConfig(params: { order?: string[]; models?: NonNullable["defaults"]>["models"]; @@ -103,22 +130,13 @@ describe("resolveCliRuntimeExecutionProvider", () => { ).toBe("claude-cli"); }); - it("uses caller-provided plugin auth aliases without metadata discovery", () => { + it("uses prepared Anthropic auth choice aliases without metadata discovery", () => { expect( resolveCliRuntimeExecutionProvider({ authProfileId: "anthropic:claude-cli", cfg: createAnthropicAuthConfig({ order: ["anthropic:api"] }), provider: "anthropic", modelId: "opus-4.7", - metadataSnapshot: { - plugins: [ - { - id: "anthropic", - origin: "bundled", - providerAuthAliases: { "claude-cli": "anthropic" }, - }, - ], - } as never, }), ).toBe("claude-cli"); }); From 96b4258734eea8094beede6e3f7ae3021706533a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 02:41:47 -0700 Subject: [PATCH 08/57] fix(gateway): reject destroyed watch response sockets (#118606) Co-authored-by: Peter Steinberger --- src/gateway/watch-node-http.test.ts | 88 ++++++++++++++++++++++++++++- src/gateway/watch-node-http.ts | 3 +- 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/src/gateway/watch-node-http.test.ts b/src/gateway/watch-node-http.test.ts index 12732f0384c7..55356a142993 100644 --- a/src/gateway/watch-node-http.test.ts +++ b/src/gateway/watch-node-http.test.ts @@ -1,4 +1,10 @@ -import { createServer, request as httpRequest, type ClientRequest, type Server } from "node:http"; +import { + createServer, + request as httpRequest, + type ClientRequest, + type Server, + type ServerResponse, +} from "node:http"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { @@ -108,6 +114,7 @@ async function startRuntime( abortConnectResponse?: boolean; config?: OpenClawConfig; now?: () => number; + onPollReady?: (response: ServerResponse) => void; }, ) { const nodeRegistry = new NodeRegistry({ @@ -153,6 +160,9 @@ async function startRuntime( res.statusCode = 404; res.end(); } + if (req.url === "/api/nodes/watch/poll" && !res.writableEnded) { + options?.onPollReady?.(res); + } }) .finally(() => { if (isConnect) { @@ -427,6 +437,82 @@ describe("watch node HTTP transport", () => { expect(disconnectedNodes).toHaveLength(1); }); + it.each([ + { destroyTarget: "socket", delivery: "event" }, + { destroyTarget: "socket", delivery: "raw" }, + { destroyTarget: "response", delivery: "event" }, + ] as const)( + "rejects $delivery delivery when the active watch poll $destroyTarget is destroyed", + async ({ destroyTarget, delivery }) => { + let resolvePollReady: (response: ServerResponse) => void = () => undefined; + const pollReady = new Promise((resolve) => { + resolvePollReady = resolve; + }); + const { identity, issued, nodeRegistry, disconnectedNodes, runtime, baseUrl } = + await createWatchNodeFixture("openclaw-watch-node-destroyed-poll-", { + onPollReady: resolvePollReady, + }); + const connectResponse = await connectWatchNode({ + baseUrl, + identity, + bootstrapToken: issued.token, + }); + expect(connectResponse.status).toBe(200); + const { sessionToken } = await readJson(connectResponse); + const authorization = `Bearer ${String(sessionToken)}`; + const pollFailure = new Promise((resolve, reject) => { + const request = httpRequest( + `${baseUrl}/poll`, + { method: "POST", headers: { authorization } }, + (response) => { + response.resume(); + reject(new Error(`unexpected poll response: ${response.statusCode}`)); + }, + ); + request.once("error", (error: NodeJS.ErrnoException) => { + resolve(error.code ?? error.message); + }); + request.end(); + }); + try { + const response = await pollReady; + const socket = response.socket; + expect(socket).not.toBeNull(); + if (destroyTarget === "socket") { + socket!.destroy(); + expect(response.destroyed).toBe(false); + } else { + response.destroy(); + expect(response.destroyed).toBe(true); + } + expect(socket!.destroyed).toBe(true); + expect(response.writableEnded).toBe(false); + + const payload = { id: "lost" }; + const delivered = + delivery === "raw" + ? nodeRegistry.sendEventRaw( + identity.deviceId, + "node.invoke.request", + serializeEventPayload(payload), + ) + : nodeRegistry.sendEvent(identity.deviceId, "node.invoke.request", payload); + expect(delivered).toBe(false); + expect(nodeRegistry.get(identity.deviceId)).toBeUndefined(); + expect(disconnectedNodes).toEqual([ + { nodeId: identity.deviceId, reason: "event delivery failed" }, + ]); + await expect(pollFailure).resolves.toBe("ECONNRESET"); + expect(nodeRegistry.sendEvent(identity.deviceId, "node.invoke.request", payload)).toBe( + false, + ); + } finally { + runtime.close(); + } + expect(disconnectedNodes).toHaveLength(1); + }, + ); + it("rejects an HTTP node session after an external reapproval changes its generation", async () => { const { baseDir, identity, issued, nodeRegistry, disconnectedNodes, runtime, baseUrl } = await createWatchNodeFixture("openclaw-watch-node-reapproval-"); diff --git a/src/gateway/watch-node-http.ts b/src/gateway/watch-node-http.ts index 55679416a6e0..5decec38825e 100644 --- a/src/gateway/watch-node-http.ts +++ b/src/gateway/watch-node-http.ts @@ -348,7 +348,8 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions) }; const sendQueuedEvent = (res: ServerResponse, queued: QueuedNodeEvent): boolean => { - if (res.writableEnded) { + // The socket can be destroyed before its response receives the close event. + if (res.destroyed || res.socket?.destroyed || res.writableEnded) { return false; } try { From 188957a18af8e075c4ea5e971d09dad34be0bbf6 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 02:52:29 -0700 Subject: [PATCH 09/57] fix(gateway): require explicit credentials for probe URL overrides (#118614) --- src/commands/gateway-status.test.ts | 129 ++++++++++++++++++++++++++++ src/commands/gateway-status.ts | 7 ++ 2 files changed, 136 insertions(+) diff --git a/src/commands/gateway-status.test.ts b/src/commands/gateway-status.test.ts index 193b18683184..256bd9b18346 100644 --- a/src/commands/gateway-status.test.ts +++ b/src/commands/gateway-status.test.ts @@ -315,6 +315,8 @@ async function runGatewayStatus( json?: boolean; port?: unknown; url?: string; + token?: string; + password?: string; ssh?: string; sshAuto?: boolean; sshIdentity?: string; @@ -447,6 +449,7 @@ describe("gateway-status command", () => { timeout: "1000", json: true, url: "wss://remote.example:18789", + token: "explicit-remote-token", }); expect(inspectWindowsGatewayFirewall).not.toHaveBeenCalled(); @@ -458,6 +461,132 @@ describe("gateway-status command", () => { ); }); + it.each([ + { + source: "configured token", + auth: { mode: "token", token: "configured-local-token" }, + env: {}, + options: {}, + }, + { + source: "configured password", + auth: { mode: "password", password: "configured-local-password" }, + env: {}, + options: {}, + }, + { + source: "environment token", + auth: { mode: "token" }, + env: { OPENCLAW_GATEWAY_TOKEN: "ambient-local-token" }, + options: {}, + }, + { + source: "environment password", + auth: { mode: "password" }, + env: { OPENCLAW_GATEWAY_PASSWORD: "ambient-local-password" }, + options: {}, + }, + { + source: "whitespace token", + auth: { mode: "token", token: "configured-local-token" }, + env: {}, + options: { token: " " }, + }, + { + source: "whitespace password", + auth: { mode: "password", password: "configured-local-password" }, + env: {}, + options: { password: " " }, + }, + { + source: "explicit loopback URL", + auth: { mode: "token", token: "configured-local-token" }, + env: {}, + options: { url: "ws://127.0.0.1:18991" }, + }, + ])( + "rejects a local $source before probing an explicit Gateway URL", + async ({ auth, env, options }) => { + const configuredGateway = { gateway: { mode: "local", auth } }; + + await withEnvAsync( + { + OPENCLAW_GATEWAY_TOKEN: undefined, + OPENCLAW_GATEWAY_PASSWORD: undefined, + ...env, + }, + async () => { + await readBestEffortConfig.withImplementation( + async () => configuredGateway as never, + async () => { + const { runtime } = createRuntimeCapture(); + await expect( + runGatewayStatus(runtime, { + timeout: "1000", + json: true, + url: "wss://attacker.example:18789", + ...options, + }), + ).rejects.toMatchObject({ + name: "GatewayExplicitAuthRequiredError", + message: expect.stringContaining( + "gateway url override requires explicit credentials", + ), + }); + + expect(readBestEffortConfig).not.toHaveBeenCalled(); + expect(discoverGatewayBeacons).not.toHaveBeenCalled(); + expect(startSshPortForward).not.toHaveBeenCalled(); + expect(probeGateway).not.toHaveBeenCalled(); + }, + ); + }, + ); + }, + ); + + it.each([ + { + credential: "token", + options: { token: "explicit-remote-token" }, + expectedAuth: { token: "explicit-remote-token", password: undefined }, + }, + { + credential: "password", + options: { password: "explicit-remote-password" }, + expectedAuth: { token: undefined, password: "explicit-remote-password" }, + }, + ])( + "honors an explicit $credential for an explicit Gateway URL", + async ({ options, expectedAuth }) => { + const explicitUrl = "wss://attacker.example:18789"; + readBestEffortConfig.mockResolvedValueOnce({ + gateway: { + mode: "local", + auth: { mode: "token", token: "configured-local-token" }, + }, + } as never); + + await withEnvAsync( + { + OPENCLAW_GATEWAY_TOKEN: "ambient-local-token", + OPENCLAW_GATEWAY_PASSWORD: "ambient-local-password", + }, + async () => { + const { runtime } = createRuntimeCapture(); + await runGatewayStatus(runtime, { + timeout: "1000", + json: true, + url: explicitUrl, + ...options, + }); + + expect(requireProbeCall(explicitUrl).auth).toEqual(expectedAuth); + }, + ); + }, + ); + it("includes diagnostic next steps when no gateway is reachable or discoverable", async () => { const { runtime, runtimeLogs, runtimeErrors } = createRuntimeCapture(); const defaultProbeGateway = probeGateway.getMockImplementation(); diff --git a/src/commands/gateway-status.ts b/src/commands/gateway-status.ts index d35e3ac589dc..322d3badf317 100644 --- a/src/commands/gateway-status.ts +++ b/src/commands/gateway-status.ts @@ -3,6 +3,7 @@ import { isRich } from "../../packages/terminal-core/src/theme.js"; import { parseGatewayPortOption } from "../cli/gateway-port-option.js"; import { withProgress } from "../cli/progress.js"; import { readBestEffortConfig, resolveGatewayPort } from "../config/config.js"; +import { ensureExplicitGatewayAuth, resolveExplicitGatewayAuth } from "../gateway/call.js"; import { resolveWideAreaDiscoveryDomain } from "../infra/widearea-dns.js"; import type { RuntimeEnv } from "../runtime.js"; import { createLazyImportLoader } from "../shared/lazy-promise.js"; @@ -52,6 +53,12 @@ export async function gatewayStatusCommand( }, runtime: RuntimeEnv, ) { + ensureExplicitGatewayAuth({ + urlOverride: opts.url?.trim(), + urlOverrideSource: "cli", + explicitAuth: resolveExplicitGatewayAuth(opts), + errorHint: "Fix: pass --token or --password with --url.", + }); const startedAt = Date.now(); const cfg = await readBestEffortConfig(); const rich = isRich() && opts.json !== true; From deb682abfee018fa55e6e3c270012f8d3ebd3dbb Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 02:56:43 -0700 Subject: [PATCH 10/57] refactor(plugins): consolidate extension runtime helpers (#118509) * refactor(plugins): consolidate extension runtime helpers * fix(ci): satisfy extension type and lint checks * chore(plugin-sdk): regenerate API baseline for #118509 --- config/max-lines-baseline.txt | 1 - .../.generated/plugin-sdk-api-baseline.sha256 | 4 +- docs/plugins/sdk-subpaths.md | 4 +- extensions/canvas/runtime-api.ts | 16 +- extensions/canvas/src/capability.ts | 34 - extensions/canvas/src/host/server.test.ts | 28 +- extensions/canvas/src/host/server.ts | 126 +--- .../src/service-attributes.ts | 65 +- .../diagnostics-otel/src/service-constants.ts | 1 - .../diagnostics-otel/src/service-exporter.ts | 6 +- .../src/service-genai-attributes.ts | 12 +- .../src/service-recorders-harness.ts | 35 +- .../src/service-recorders-model.ts | 16 +- .../src/service-recorders-operations.ts | 35 +- .../src/service-recorders-tools.ts | 44 +- .../src/service-recorders-usage.ts | 44 +- .../diagnostics-prometheus/src/service.ts | 208 +++--- .../onepassword/onepassword-op-path.d.ts | 2 - extensions/onepassword/onepassword-op-path.js | 2 - .../onepassword/src/secret-ref-cli.test.ts | 565 +++++------------ extensions/onepassword/src/secret-ref-cli.ts | 450 ++----------- extensions/qa-lab/web/src/app.browser.test.ts | 14 +- extensions/qa-lab/web/src/app.ts | 4 +- .../qa-lab/web/src/ui-conversation-key.ts | 16 +- .../qa-lab/web/src/ui-render-content.ts | 25 +- extensions/qa-lab/web/src/ui-render.test.ts | 10 + extensions/qa-lab/web/src/ui-types.ts | 67 +- extensions/vault/src/cli.test.ts | 34 +- extensions/vault/src/cli.ts | 328 +--------- extensions/voice-call/index.test.ts | 82 ++- extensions/voice-call/index.ts | 594 +++++------------- extensions/voice-call/src/command-service.ts | 171 +++++ .../src/gateway-continue-operation.test.ts | 5 +- .../src/gateway-continue-operation.ts | 18 +- scripts/plugin-sdk-surface-report.mjs | 8 +- src/plugin-sdk/diagnostic-runtime.ts | 31 + src/plugin-sdk/secret-ref-runtime.ts | 346 ++++++++++ 37 files changed, 1339 insertions(+), 2112 deletions(-) delete mode 100644 extensions/canvas/src/capability.ts create mode 100644 extensions/voice-call/src/command-service.ts diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index e40c65b6a56b..7dcf1d6660e3 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -277,7 +277,6 @@ extensions/telegram/src/thread-bindings.ts extensions/telegram/src/webhook.test.ts extensions/tlon/src/monitor/index.ts extensions/voice-call/index.test.ts -extensions/voice-call/index.ts extensions/voice-call/src/cli.ts extensions/voice-call/src/media-stream.ts extensions/voice-call/src/webhook.test.ts diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index fc1b08e20761..e3d19002f9a8 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -61,7 +61,7 @@ c1ea9510dfda047609a99d5d2cd1f1560f5d469a36e6b695766213d695c25b0f module/convers dea96213010cbc816345b1229534f1acb8b0bcfdbd7814c62eefc77030fa9cd8 module/core 4af19d59c2f18674e7d7f7dc1b358b644dc707e6bd601dc47168bd9e4a669940 module/dedupe-runtime f70c93d28053ca2e8353e45e6515ce7acef188097c6117d1545965d0699c8004 module/device-bootstrap -6215d3af5923bf5a616d73062534968b69f448e3e30adc64ae9caebdd1a46d71 module/diagnostic-runtime +68c726280f6585af96c071758ba383e55480b4fc913eae7c26aea1af331dedf3 module/diagnostic-runtime ea81ef06956c1bc0853fa00afbbc2b5a4019116aaf8a436e1b27d06f7a2c9e88 module/directory-runtime e8adcff47c1b677cd2c01a2130fe4d226ac30ab3c88a3ceb4df5a8660316c4a9 module/discord f65408d85477bb362ebe6ed9148c1bb6b9eb7839733e5e3119f4ff8f1cfd0567 module/error-runtime @@ -120,7 +120,7 @@ b6b8edc50ecab8386c9acd8f374a207212b5a99c8f518538bbcf0c458dda3881 module/runtime aa8a411ad37c1d1143b67376bf2d20255b9eedff61d80815f42e4f8ed7bd8e58 module/secret-file 44adc2205f926172fcd3762ca8a96c1485beabcb1bef8b9acfd2233cefea2a6a module/secret-input 57dcb1462d4c4f9a98d934c4ca975b163d704758af9821a64001ff3ac05637c3 module/secret-input-runtime -e576b537880f63b3a91f3608f7e84c873bce6c6a3d9a0ba98c247f46de788d25 module/secret-ref-runtime +dc0ee07d392a85c218939000b28c0138f139215da00f5592b34a68ba8e29a25d module/secret-ref-runtime 62ccaafc8e0677e850339f4a4333f9f16ae9fed979bcef003890b2a47507147f module/security-runtime 673c64502fdffb2d6361a7cf2ad0c33ffe15707b5e5027de1d88701ce3d8ade1 module/session-catalog 50f5e344f98c27570b7a30e32a906b612e2383d21f102e88cd93e1d5425a6de9 module/session-discussion diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index 396bf54104b6..4ab07f3d618a 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -202,7 +202,7 @@ usage endpoint failed or returned no usable usage data. | `plugin-sdk/channel-secret-runtime` | Deprecated broad secret-contract surface (`collectSimpleChannelFieldAssignments`, `getChannelSurface`, `pushAssignment`, secret target types); prefer the focused subpaths below | | `plugin-sdk/channel-secret-basic-runtime` | Narrow secret-contract exports and target-registry builders for non-TTS channel/plugin secret surfaces | | `plugin-sdk/channel-secret-tts-runtime` | Private-local after July 2026; Narrow nested channel TTS secret assignment helpers | - | `plugin-sdk/secret-ref-runtime` | Narrow SecretRef typing, resolution, and shared setup-plan construction for plugin-owned secret providers | + | `plugin-sdk/secret-ref-runtime` | Narrow SecretRef typing, resolution, setup-plan construction, and setup CLI scaffolding for plugin-owned secret providers | | `plugin-sdk/security-runtime` | Deprecated broad barrel for trust, DM gating, root-bounded file/path helpers including create-only writes, sync/async atomic file replacement, sibling temp writes, cross-device move fallback, private file-store helpers, symlink-parent guards, external-content, sensitive text redaction, constant-time secret comparison, and secret-collection helpers; prefer focused security/SSRF/secret subpaths | | `plugin-sdk/ssrf-policy` | Host allowlist and private-network SSRF policy helpers | | `plugin-sdk/ssrf-dispatcher` | Private-local after July 2026; Narrow pinned-dispatcher helpers without the broad infra runtime surface | @@ -304,7 +304,7 @@ Use `isLoopbackHost(host)` when a plugin must accept only the local machine. It | `plugin-sdk/exec-approvals-runtime` | Private-local after July 2026; Exec approval policy file helpers without the broad infra-runtime barrel | | `plugin-sdk/infra-runtime` | Deprecated compatibility shim; use the focused runtime subpaths above | | `plugin-sdk/collection-runtime` | Small bounded cache helpers | - | `plugin-sdk/diagnostic-runtime` | Diagnostic flag, event, and trace-context helpers | + | `plugin-sdk/diagnostic-runtime` | Diagnostic flag, event, trace-context, and low-cardinality dimension normalization helpers | | `plugin-sdk/error-runtime` | Error graph, formatting, unknown-value coercion, shared error classification helpers, `PlatformMessageNotDispatchedError`, `isApprovalNotFoundError` | | `plugin-sdk/fetch-runtime` | Private-local after July 2026; Wrapped fetch, proxy, EnvHttpProxyAgent option, and pinned lookup helpers | | `plugin-sdk/runtime-fetch` | Private-local after July 2026; Dispatcher-aware runtime fetch without proxy/guarded-fetch imports | diff --git a/extensions/canvas/runtime-api.ts b/extensions/canvas/runtime-api.ts index 87c0f733ccbb..f3c2fc0e885c 100644 --- a/extensions/canvas/runtime-api.ts +++ b/extensions/canvas/runtime-api.ts @@ -1,4 +1,4 @@ -/** Runtime API exports for Canvas plugin host, CLI, and capability helpers. */ +/** Runtime API exports for Canvas plugin host and CLI helpers. */ export { canvasConfigSchema, isCanvasHostEnabled, @@ -14,23 +14,11 @@ export { CANVAS_WS_PATH, handleA2uiHttpRequest, } from "./src/host/a2ui.js"; -export { - createCanvasHostHandler, - startCanvasHost, - type CanvasHostHandler, - type CanvasHostServer, -} from "./src/host/server.js"; +export { createCanvasHostHandler, type CanvasHostHandler } from "./src/host/server.js"; export { registerNodesCanvasCommands, type CanvasCliDependencies, type CanvasNodesRpcOpts, } from "./src/cli.js"; export { canvasSnapshotTempPath, parseCanvasSnapshotPayload } from "./src/cli-helpers.js"; -export { - buildCanvasScopedHostUrl, - CANVAS_CAPABILITY_PATH_PREFIX, - CANVAS_CAPABILITY_TTL_MS, - mintCanvasCapabilityToken, - normalizeCanvasScopedUrl, -} from "./src/capability.js"; export { resolveCanvasHostUrl } from "./src/host-url.js"; diff --git a/extensions/canvas/src/capability.ts b/extensions/canvas/src/capability.ts deleted file mode 100644 index cf2841471792..000000000000 --- a/extensions/canvas/src/capability.ts +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Canvas capability-token helpers for scoped hosted node URLs. - */ -import { - buildPluginNodeCapabilityScopedHostUrl, - DEFAULT_PLUGIN_NODE_CAPABILITY_TTL_MS, - mintPluginNodeCapabilityToken, - normalizePluginNodeCapabilityScopedUrl, - PLUGIN_NODE_CAPABILITY_PATH_PREFIX, - type NormalizedPluginNodeCapabilityUrl, -} from "openclaw/plugin-sdk/gateway-runtime"; - -/** Path prefix used for Canvas capability-scoped gateway routes. */ -export const CANVAS_CAPABILITY_PATH_PREFIX = PLUGIN_NODE_CAPABILITY_PATH_PREFIX; -/** Default Canvas capability token TTL in milliseconds. */ -export const CANVAS_CAPABILITY_TTL_MS = DEFAULT_PLUGIN_NODE_CAPABILITY_TTL_MS; - -/** Normalized Canvas capability-scoped URL shape. */ -type NormalizedCanvasScopedUrl = NormalizedPluginNodeCapabilityUrl; - -/** Creates a new opaque Canvas capability token. */ -export function mintCanvasCapabilityToken(): string { - return mintPluginNodeCapabilityToken(); -} - -/** Builds a Canvas host URL scoped by the supplied capability token. */ -export function buildCanvasScopedHostUrl(baseUrl: string, capability: string): string | undefined { - return buildPluginNodeCapabilityScopedHostUrl(baseUrl, capability); -} - -/** Normalizes and validates a Canvas capability-scoped URL. */ -export function normalizeCanvasScopedUrl(rawUrl: string): NormalizedCanvasScopedUrl { - return normalizePluginNodeCapabilityScopedUrl(rawUrl); -} diff --git a/extensions/canvas/src/host/server.test.ts b/extensions/canvas/src/host/server.test.ts index 1834bc74cff9..6ec1ce481307 100644 --- a/extensions/canvas/src/host/server.test.ts +++ b/extensions/canvas/src/host/server.test.ts @@ -197,7 +197,6 @@ describe("canvas host", () => { log: (..._args: Parameters) => {}, }; let createCanvasHostHandler: typeof import("./server.js").createCanvasHostHandler; - let startCanvasHost: typeof import("./server.js").startCanvasHost; let WebSocketServerClass: typeof import("ws").WebSocketServer; let watcherState: ReturnType; let fixtureRoot = ""; @@ -226,7 +225,6 @@ describe("canvas host", () => { }); beforeAll(async () => { - vi.doUnmock("undici"); vi.doMock("node:timers", async (importOriginal) => { const actual = await importOriginal(); return { @@ -241,7 +239,7 @@ describe("canvas host", () => { }); vi.resetModules(); const serverModule = await import("./server.js"); - ({ createCanvasHostHandler, startCanvasHost } = serverModule); + ({ createCanvasHostHandler } = serverModule); const wsModule = await vi.importActual("ws"); WebSocketServerClass = wsModule.WebSocketServer; fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-canvas-fixtures-")); @@ -502,15 +500,12 @@ describe("canvas host", () => { } }); - it("serves canvas content from the mounted base path and reuses handlers without double close", async () => { + it("serves canvas content from the mounted base path", async () => { const dir = await createCaseDir(); await fs.writeFile(path.join(dir, "index.html"), "v1", "utf8"); const handler = await createTestCanvasHostHandler(dir); - const originalClose = handler.close; - const closeSpy = vi.fn(async () => originalClose()); - try { const response = await captureHandlerResponse(handler, `${CANVAS_HOST_PATH}/`); expect(response.status).toBe(200); @@ -523,25 +518,8 @@ describe("canvas host", () => { const miss = await captureHandlerResponse(handler, "/"); expect(miss.handled).toBe(false); - - handler.close = closeSpy; - const hosted = await startCanvasHost({ - runtime: quietRuntime, - handler, - ownsHandler: false, - port: 0, - listenHost: "127.0.0.1", - allowInTests: true, - }); - - try { - expect(hosted.port).toBeGreaterThan(0); - } finally { - await hosted.close(); - expect(closeSpy).not.toHaveBeenCalled(); - } } finally { - await originalClose(); + await handler.close(); } }); diff --git a/extensions/canvas/src/host/server.ts b/extensions/canvas/src/host/server.ts index 9951d69a94c6..6610006644dd 100644 --- a/extensions/canvas/src/host/server.ts +++ b/extensions/canvas/src/host/server.ts @@ -2,7 +2,7 @@ * Canvas host server and static-file/live-reload handler implementation. */ import fs from "node:fs/promises"; -import http, { type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import type { IncomingMessage, ServerResponse } from "node:http"; import type { Socket } from "node:net"; import path from "node:path"; import type { Duplex } from "node:stream"; @@ -14,47 +14,14 @@ import chokidar from "chokidar"; import { detectMime } from "openclaw/plugin-sdk/media-mime"; import { isTruthyEnvValue, type RuntimeEnv } from "openclaw/plugin-sdk/runtime-env"; import { resolveStateDir } from "openclaw/plugin-sdk/state-paths"; -import { - lowercasePreservingWhitespace, - normalizeOptionalString, -} from "openclaw/plugin-sdk/string-coerce-runtime"; +import { lowercasePreservingWhitespace } from "openclaw/plugin-sdk/string-coerce-runtime"; import { ensureDir, resolveUserPath } from "openclaw/plugin-sdk/text-utility-runtime"; import { WebSocketServer } from "ws"; -import { - CANVAS_HOST_PATH, - CANVAS_WS_PATH, - injectCanvasRuntime, - isA2uiPath, -} from "./a2ui-shared.js"; +import { CANVAS_HOST_PATH, CANVAS_WS_PATH, injectCanvasRuntime } from "./a2ui-shared.js"; import { normalizeUrlPath, resolveFileWithinRoot } from "./file-resolver.js"; const CANVAS_LIVE_RELOAD_MAX_INBOUND_MESSAGE_BYTES = 64 * 1024; -/** Options for Canvas host creation. */ -type CanvasHostOpts = { - runtime: RuntimeEnv; - rootDir?: string; - port?: number; - listenHost?: string; - allowInTests?: boolean; - liveReload?: boolean; - watchFactory?: typeof chokidar.watch; - webSocketServerClass?: typeof WebSocketServer; -}; - -/** Options for starting a standalone Canvas host HTTP server. */ -type CanvasHostServerOpts = CanvasHostOpts & { - handler?: CanvasHostHandler; - ownsHandler?: boolean; -}; - -/** Running Canvas host server handle. */ -export type CanvasHostServer = { - port: number; - rootDir: string; - close: () => Promise; -}; - /** Options for creating only the Canvas host request handler. */ type CanvasHostHandlerOpts = { runtime: RuntimeEnv; @@ -435,90 +402,3 @@ export async function createCanvasHostHandler( }, }; } - -/** Starts a standalone loopback Canvas host HTTP server. */ -export async function startCanvasHost(opts: CanvasHostServerOpts): Promise { - if (isDisabledByEnv() && opts.allowInTests !== true) { - return { port: 0, rootDir: "", close: async () => {} }; - } - - const handler = - opts.handler ?? - (await createCanvasHostHandler({ - runtime: opts.runtime, - rootDir: opts.rootDir, - basePath: CANVAS_HOST_PATH, - allowInTests: opts.allowInTests, - liveReload: opts.liveReload, - watchFactory: opts.watchFactory, - webSocketServerClass: opts.webSocketServerClass, - })); - const ownsHandler = opts.ownsHandler ?? opts.handler === undefined; - - const bindHost = normalizeOptionalString(opts.listenHost) || "127.0.0.1"; - const server: Server = http.createServer((req, res) => { - if (lowercasePreservingWhitespace(req.headers.upgrade ?? "") === "websocket") { - return; - } - void (async () => { - if (req.url && isA2uiPath(new URL(req.url, "http://localhost").pathname)) { - const { handleA2uiHttpRequest } = await import("./a2ui.js"); - if (await handleA2uiHttpRequest(req, res)) { - return; - } - } - if (await handler.handleHttpRequest(req, res)) { - return; - } - res.statusCode = 404; - res.setHeader("Content-Type", "text/plain; charset=utf-8"); - res.end("Not Found"); - })().catch((err: unknown) => { - opts.runtime.error(`Canvas host request failed: ${String(err)}`); - res.statusCode = 500; - res.setHeader("Content-Type", "text/plain; charset=utf-8"); - res.end("error"); - }); - }); - server.on("upgrade", (req, socket, head) => { - if (handler.handleUpgrade(req, socket, head)) { - return; - } - socket.destroy(); - }); - - const listenPort = - typeof opts.port === "number" && Number.isFinite(opts.port) && opts.port > 0 ? opts.port : 0; - await new Promise((resolve, reject) => { - const onError = (err: NodeJS.ErrnoException) => { - server.off("listening", onListening); - reject(err); - }; - const onListening = () => { - server.off("error", onError); - resolve(); - }; - server.once("error", onError); - server.once("listening", onListening); - server.listen(listenPort, bindHost); - }); - - const addr = server.address(); - const boundPort = typeof addr === "object" && addr ? addr.port : 0; - opts.runtime.log( - `canvas host listening on http://${bindHost}:${boundPort} (root ${handler.rootDir})`, - ); - - return { - port: boundPort, - rootDir: handler.rootDir, - close: async () => { - if (ownsHandler) { - await handler.close(); - } - await new Promise((resolve, reject) => { - server.close((err) => (err ? reject(err) : resolve())); - }); - }, - }; -} diff --git a/extensions/diagnostics-otel/src/service-attributes.ts b/extensions/diagnostics-otel/src/service-attributes.ts index 84ec1a9a6a31..a12cee4255e9 100644 --- a/extensions/diagnostics-otel/src/service-attributes.ts +++ b/extensions/diagnostics-otel/src/service-attributes.ts @@ -1,10 +1,10 @@ import type { LogRecord } from "@opentelemetry/api-logs"; +import { normalizeDiagnosticValue } from "openclaw/plugin-sdk/diagnostic-runtime"; import type { DiagnosticEventPayload, DiagnosticTraceContext } from "../api.js"; import { redactSensitiveText } from "../api.js"; import { BLOCKED_OTEL_LOG_ATTRIBUTE_KEYS, DROPPED_OTEL_ATTRIBUTE_KEYS, - LOW_CARDINALITY_VALUE_RE, MAX_OTEL_LOG_ATTRIBUTE_COUNT, MAX_OTEL_LOG_ATTRIBUTE_VALUE_CHARS, OTEL_LOG_ATTRIBUTE_KEY_RE, @@ -26,18 +26,6 @@ export function redactOtelAttributes(attributes: Record= 0 ? redacted.slice(0, scopedLaneIndex) : redacted; - return LOW_CARDINALITY_VALUE_RE.test(lane) ? lane : fallback; -} - export function shouldCaptureOtelLogBody(policy: OtelContentCapturePolicy): boolean { return policy.logBodies; } @@ -187,7 +158,7 @@ function assignOtelSecurityEventAttributes( assignOtelLogAttribute( attributes, `openclaw.security.attribute.${key}`, - typeof value === "string" ? lowCardinalityAttr(value) : value, + typeof value === "string" ? normalizeDiagnosticValue(value) : value, ); } } @@ -216,11 +187,19 @@ export function assignOtelSecurityAttributes( ): void { assignOtelLogAttribute(attributes, "openclaw.security.event_id", evt.eventId); assignOtelLogAttribute(attributes, "openclaw.security.category", evt.category); - assignOtelLogAttribute(attributes, "openclaw.security.action", lowCardinalityAttr(evt.action)); + assignOtelLogAttribute( + attributes, + "openclaw.security.action", + normalizeDiagnosticValue(evt.action), + ); assignOtelLogAttribute(attributes, "openclaw.security.outcome", evt.outcome); assignOtelLogAttribute(attributes, "openclaw.security.severity", evt.severity); if (evt.reason) { - assignOtelLogAttribute(attributes, "openclaw.security.reason", lowCardinalityAttr(evt.reason)); + assignOtelLogAttribute( + attributes, + "openclaw.security.reason", + normalizeDiagnosticValue(evt.reason), + ); } if (evt.actor) { assignOtelLogAttribute(attributes, "openclaw.security.actor.kind", evt.actor.kind); @@ -228,35 +207,35 @@ export function assignOtelSecurityAttributes( assignOtelLogAttribute( attributes, "openclaw.security.actor.id_hash", - lowCardinalityAttr(evt.actor.idHash), + normalizeDiagnosticValue(evt.actor.idHash), ); } if (evt.actor.deviceIdHash) { assignOtelLogAttribute( attributes, "openclaw.security.actor.device_id_hash", - lowCardinalityAttr(evt.actor.deviceIdHash), + normalizeDiagnosticValue(evt.actor.deviceIdHash), ); } if (evt.actor.channel) { assignOtelLogAttribute( attributes, "openclaw.security.actor.channel", - lowCardinalityAttr(evt.actor.channel), + normalizeDiagnosticValue(evt.actor.channel), ); } if (evt.actor.role) { assignOtelLogAttribute( attributes, "openclaw.security.actor.role", - lowCardinalityAttr(evt.actor.role), + normalizeDiagnosticValue(evt.actor.role), ); } if (evt.actor.scopes?.length) { assignOtelLogAttribute( attributes, "openclaw.security.actor.scopes", - evt.actor.scopes.map((scope) => lowCardinalityAttr(scope)).join(","), + evt.actor.scopes.map((scope) => normalizeDiagnosticValue(scope)).join(","), ); } } @@ -266,7 +245,7 @@ export function assignOtelSecurityAttributes( assignOtelLogAttribute( attributes, "openclaw.security.target.id_hash", - lowCardinalityAttr(evt.target.idHash), + normalizeDiagnosticValue(evt.target.idHash), ); } if (evt.target.name) { @@ -280,7 +259,7 @@ export function assignOtelSecurityAttributes( assignOtelLogAttribute( attributes, "openclaw.security.target.owner", - lowCardinalityAttr(evt.target.owner), + normalizeDiagnosticValue(evt.target.owner), ); } } @@ -289,7 +268,7 @@ export function assignOtelSecurityAttributes( assignOtelLogAttribute( attributes, "openclaw.security.policy.id", - lowCardinalityAttr(evt.policy.id), + normalizeDiagnosticValue(evt.policy.id), ); } if (evt.policy.decision) { @@ -299,7 +278,7 @@ export function assignOtelSecurityAttributes( assignOtelLogAttribute( attributes, "openclaw.security.policy.reason", - lowCardinalityAttr(evt.policy.reason), + normalizeDiagnosticValue(evt.policy.reason), ); } } @@ -308,7 +287,7 @@ export function assignOtelSecurityAttributes( assignOtelLogAttribute( attributes, "openclaw.security.control.id", - lowCardinalityAttr(evt.control.id), + normalizeDiagnosticValue(evt.control.id), ); } if (evt.control.family) { diff --git a/extensions/diagnostics-otel/src/service-constants.ts b/extensions/diagnostics-otel/src/service-constants.ts index adf3de450ec2..c6b40c4a1022 100644 --- a/extensions/diagnostics-otel/src/service-constants.ts +++ b/extensions/diagnostics-otel/src/service-constants.ts @@ -21,7 +21,6 @@ export const DROPPED_OTEL_ATTRIBUTE_KEYS = new Set([ "openclaw.traceId", "openclaw.trace_id", ]); -export const LOW_CARDINALITY_VALUE_RE = /^[A-Za-z0-9_.:-]{1,120}$/u; export const SECURITY_TARGET_NAME_VALUE_RE = /^[A-Za-z0-9@/_.:-]{1,256}$/u; export const MAX_OTEL_LOG_BODY_CHARS = 4 * 1024; export const MAX_OTEL_LOG_ATTRIBUTE_COUNT = 64; diff --git a/extensions/diagnostics-otel/src/service-exporter.ts b/extensions/diagnostics-otel/src/service-exporter.ts index f9ac91a92377..d01f35397fa9 100644 --- a/extensions/diagnostics-otel/src/service-exporter.ts +++ b/extensions/diagnostics-otel/src/service-exporter.ts @@ -1,7 +1,7 @@ import { readFileSync } from "node:fs"; import nodePath from "node:path"; +import { normalizeDiagnosticValue } from "openclaw/plugin-sdk/diagnostic-runtime"; import { createNodeProxyAgent } from "openclaw/plugin-sdk/fetch-runtime"; -import { lowCardinalityAttr } from "./service-attributes.js"; import { OTEL_EXPORTER_OTLP_CERTIFICATE_ENV, OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE_ENV, @@ -153,9 +153,9 @@ export function formatError(err: unknown): string { export function errorCategory(err: unknown): string { try { if (err instanceof Error && typeof err.name === "string" && err.name.trim()) { - return lowCardinalityAttr(err.name, "Error"); + return normalizeDiagnosticValue(err.name, "Error"); } - return lowCardinalityAttr(typeof err, "unknown"); + return normalizeDiagnosticValue(typeof err, "unknown"); } catch { return "unknown"; } diff --git a/extensions/diagnostics-otel/src/service-genai-attributes.ts b/extensions/diagnostics-otel/src/service-genai-attributes.ts index e5126a2333ed..e87bc4c141ab 100644 --- a/extensions/diagnostics-otel/src/service-genai-attributes.ts +++ b/extensions/diagnostics-otel/src/service-genai-attributes.ts @@ -1,8 +1,8 @@ import { SpanKind } from "@opentelemetry/api"; import { GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT } from "@opentelemetry/semantic-conventions/incubating"; +import { normalizeDiagnosticValue } from "openclaw/plugin-sdk/diagnostic-runtime"; import type { DiagnosticEventPayload } from "../api.js"; import { redactSensitiveText } from "../api.js"; -import { lowCardinalityAttr } from "./service-attributes.js"; import { GEN_AI_LATEST_EXPERIMENTAL_OPT_IN, OTEL_SEMCONV_STABILITY_OPT_IN_ENV, @@ -161,13 +161,13 @@ export function assignGenAiSpanIdentityAttrs( }, ): void { if (emitLatestGenAiSemconv()) { - attrs["gen_ai.provider.name"] = lowCardinalityAttr(input.provider); + attrs["gen_ai.provider.name"] = normalizeDiagnosticValue(input.provider); } else { - attrs["gen_ai.system"] = lowCardinalityAttr(input.provider); + attrs["gen_ai.system"] = normalizeDiagnosticValue(input.provider); } if (input.model) { // Span attributes carry the full model id; only metric labels need bounded cardinality - // (the gen_ai metrics below still use lowCardinalityAttr). The low-cardinality allowlist + // (the gen_ai metrics below still use normalizeDiagnosticValue). The low-cardinality allowlist // regex rejects "/", so provider-qualified ids like "anthropic/claude-sonnet-4.6" collapse // to "unknown" on the SPAN — breaking model attribution in trace backends (e.g. Langfuse // reads gen_ai.request.model). Keep the redacted raw model on the span. @@ -206,7 +206,7 @@ export function modelCallSpanName(evt: { const operationName = genAiOperationName(evt.api, evt.observationUnit); return operationName === GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT ? operationName - : `${operationName} ${lowCardinalityAttr(evt.model)}`; + : `${operationName} ${normalizeDiagnosticValue(evt.model)}`; } export function modelCallSpanKind(): SpanKind | undefined { @@ -220,7 +220,7 @@ export function addUpstreamRequestIdSpanEvent( if (!upstreamRequestIdHash) { return; } - const boundedHash = lowCardinalityAttr(upstreamRequestIdHash); + const boundedHash = normalizeDiagnosticValue(upstreamRequestIdHash); if (boundedHash === "unknown") { return; } diff --git a/extensions/diagnostics-otel/src/service-recorders-harness.ts b/extensions/diagnostics-otel/src/service-recorders-harness.ts index e83381ab90a1..f532c19e1b91 100644 --- a/extensions/diagnostics-otel/src/service-recorders-harness.ts +++ b/extensions/diagnostics-otel/src/service-recorders-harness.ts @@ -1,10 +1,13 @@ import { SpanStatusCode } from "@opentelemetry/api"; +import { + normalizeDiagnosticValue, + normalizeDiagnosticLane, +} from "openclaw/plugin-sdk/diagnostic-runtime"; import type { DiagnosticEventMetadata, DiagnosticEventPayload, DiagnosticEventPrivateData, } from "../api.js"; -import { lowCardinalityAttr, lowCardinalityQueueLaneAttr } from "./service-attributes.js"; import { normalizeOtelErrorMessage } from "./service-content-normalization.js"; import type { DiagnosticsRecorderRuntime } from "./service-recorder-runtime.js"; import type { HarnessRunDiagnosticEvent, ModelFailoverDiagnosticEvent } from "./service-types.js"; @@ -25,16 +28,16 @@ export function createHarnessRecorders(runtime: DiagnosticsRecorderRuntime) { } = runtime; const harnessRunMetricAttrs = (evt: HarnessRunDiagnosticEvent) => ({ - "openclaw.harness.id": lowCardinalityAttr(evt.harnessId, "unknown"), - "openclaw.harness.plugin": lowCardinalityAttr(evt.pluginId), + "openclaw.harness.id": normalizeDiagnosticValue(evt.harnessId, "unknown"), + "openclaw.harness.plugin": normalizeDiagnosticValue(evt.pluginId), ...(evt.type === "harness.run.started" ? {} : { "openclaw.outcome": evt.type === "harness.run.error" ? "error" : evt.outcome, }), - "openclaw.provider": lowCardinalityAttr(evt.provider, "unknown"), - "openclaw.model": lowCardinalityAttr(evt.model, "unknown"), - ...(evt.channel ? { "openclaw.channel": lowCardinalityAttr(evt.channel) } : {}), + "openclaw.provider": normalizeDiagnosticValue(evt.provider, "unknown"), + "openclaw.model": normalizeDiagnosticValue(evt.model, "unknown"), + ...(evt.channel ? { "openclaw.channel": normalizeDiagnosticValue(evt.channel) } : {}), }); const recordHarnessRunStarted = ( @@ -67,7 +70,7 @@ export function createHarnessRecorders(runtime: DiagnosticsRecorderRuntime) { ...harnessRunMetricAttrs(evt), }; if (evt.resultClassification) { - spanAttrs["openclaw.harness.result_classification"] = lowCardinalityAttr( + spanAttrs["openclaw.harness.result_classification"] = normalizeDiagnosticValue( evt.resultClassification, ); } @@ -113,7 +116,7 @@ export function createHarnessRecorders(runtime: DiagnosticsRecorderRuntime) { metadata: DiagnosticEventMetadata, privateData: DiagnosticEventPrivateData, ) => { - const errorType = lowCardinalityAttr(evt.errorCategory, "other"); + const errorType = normalizeDiagnosticValue(evt.errorCategory, "other"); const attrs = { ...harnessRunMetricAttrs(evt), "openclaw.harness.phase": evt.phase, @@ -190,21 +193,21 @@ export function createHarnessRecorders(runtime: DiagnosticsRecorderRuntime) { metadata: DiagnosticEventMetadata, ) => { const metricAttrs: Record = { - "openclaw.failover.reason": lowCardinalityAttr(evt.reason, "unknown"), + "openclaw.failover.reason": normalizeDiagnosticValue(evt.reason, "unknown"), "openclaw.failover.suspended": evt.suspended === undefined ? "unknown" : String(evt.suspended), - "openclaw.lane": lowCardinalityQueueLaneAttr(evt.lane, "unknown"), - "openclaw.model": lowCardinalityAttr(evt.fromModel), - "openclaw.provider": lowCardinalityAttr(evt.fromProvider), - "openclaw.failover.to_model": lowCardinalityAttr(evt.toModel), - "openclaw.failover.to_provider": lowCardinalityAttr(evt.toProvider), + "openclaw.lane": normalizeDiagnosticLane(evt.lane, "unknown"), + "openclaw.model": normalizeDiagnosticValue(evt.fromModel), + "openclaw.provider": normalizeDiagnosticValue(evt.fromProvider), + "openclaw.failover.to_model": normalizeDiagnosticValue(evt.toModel), + "openclaw.failover.to_provider": normalizeDiagnosticValue(evt.toProvider), }; modelFailoverCounter.add(1, metricAttrs); if (!tracesEnabled) { return; } const spanAttrs: Record = { - "openclaw.failover.reason": lowCardinalityAttr(evt.reason, "unknown"), + "openclaw.failover.reason": normalizeDiagnosticValue(evt.reason, "unknown"), }; if (evt.fromProvider) { spanAttrs["openclaw.provider"] = evt.fromProvider; @@ -219,7 +222,7 @@ export function createHarnessRecorders(runtime: DiagnosticsRecorderRuntime) { spanAttrs["openclaw.failover.to_model"] = evt.toModel; } if (evt.lane) { - spanAttrs["openclaw.lane"] = lowCardinalityQueueLaneAttr(evt.lane, "unknown"); + spanAttrs["openclaw.lane"] = normalizeDiagnosticLane(evt.lane, "unknown"); } if (evt.suspended !== undefined) { spanAttrs["openclaw.failover.suspended"] = evt.suspended; diff --git a/extensions/diagnostics-otel/src/service-recorders-model.ts b/extensions/diagnostics-otel/src/service-recorders-model.ts index c8ebdab3f5b5..f2801d3ea228 100644 --- a/extensions/diagnostics-otel/src/service-recorders-model.ts +++ b/extensions/diagnostics-otel/src/service-recorders-model.ts @@ -1,7 +1,7 @@ import { SpanStatusCode } from "@opentelemetry/api"; +import { normalizeDiagnosticValue } from "openclaw/plugin-sdk/diagnostic-runtime"; import { redactSensitiveText } from "../api.js"; import type { DiagnosticEventMetadata, DiagnosticEventPayload } from "../api.js"; -import { lowCardinalityAttr } from "./service-attributes.js"; import { addUpstreamRequestIdSpanEvent, assignGenAiModelCallAttrs, @@ -38,8 +38,8 @@ export function createModelRecorders(runtime: DiagnosticsRecorderRuntime) { const modelCallMetricAttrs = (evt: ModelCallLifecycleDiagnosticEvent) => ({ "openclaw.provider": evt.provider, "openclaw.model": evt.model, - "openclaw.api": lowCardinalityAttr(evt.api), - "openclaw.transport": lowCardinalityAttr(evt.transport), + "openclaw.api": normalizeDiagnosticValue(evt.api), + "openclaw.transport": normalizeDiagnosticValue(evt.transport), "openclaw.model_call.observation_unit": modelCallObservationUnit(evt), }); const genAiModelCallMetricAttrs = ( @@ -47,8 +47,8 @@ export function createModelRecorders(runtime: DiagnosticsRecorderRuntime) { errorType?: string, ) => ({ "gen_ai.operation.name": genAiOperationName(evt.api, evt.observationUnit), - "gen_ai.provider.name": lowCardinalityAttr(evt.provider), - "gen_ai.request.model": lowCardinalityAttr(evt.model), + "gen_ai.provider.name": normalizeDiagnosticValue(evt.provider), + "gen_ai.request.model": normalizeDiagnosticValue(evt.model), ...(errorType ? { "error.type": errorType } : {}), }); const recordGenAiModelCallDuration = ( @@ -152,12 +152,12 @@ export function createModelRecorders(runtime: DiagnosticsRecorderRuntime) { metadata: DiagnosticEventMetadata, modelContent?: OtelModelCallContent, ) => { - const errorType = lowCardinalityAttr(evt.errorCategory, "other"); + const errorType = normalizeDiagnosticValue(evt.errorCategory, "other"); const metricAttrs = { ...modelCallMetricAttrs(evt), "openclaw.errorCategory": errorType, ...(evt.failureKind - ? { "openclaw.failureKind": lowCardinalityAttr(evt.failureKind, "other") } + ? { "openclaw.failureKind": normalizeDiagnosticValue(evt.failureKind, "other") } : {}), }; modelCallDurationHistogram.record(evt.durationMs, metricAttrs); @@ -173,7 +173,7 @@ export function createModelRecorders(runtime: DiagnosticsRecorderRuntime) { "error.type": errorType, }; if (evt.failureKind) { - spanAttrs["openclaw.failureKind"] = lowCardinalityAttr(evt.failureKind, "other"); + spanAttrs["openclaw.failureKind"] = normalizeDiagnosticValue(evt.failureKind, "other"); } assignGenAiModelCallAttrs(spanAttrs, evt); if (evt.api) { diff --git a/extensions/diagnostics-otel/src/service-recorders-operations.ts b/extensions/diagnostics-otel/src/service-recorders-operations.ts index dd779d4c5f4a..8f4b7f89ce05 100644 --- a/extensions/diagnostics-otel/src/service-recorders-operations.ts +++ b/extensions/diagnostics-otel/src/service-recorders-operations.ts @@ -1,11 +1,14 @@ import { SpanStatusCode } from "@opentelemetry/api"; +import { + normalizeDiagnosticValue, + normalizeDiagnosticLane, +} from "openclaw/plugin-sdk/diagnostic-runtime"; import { redactSensitiveText } from "../api.js"; import type { DiagnosticEventMetadata, DiagnosticEventPayload, DiagnosticEventPrivateData, } from "../api.js"; -import { lowCardinalityAttr, lowCardinalityQueueLaneAttr } from "./service-attributes.js"; import { normalizeOtelErrorMessage } from "./service-content-normalization.js"; import type { DiagnosticsRecorderRuntime } from "./service-recorder-runtime.js"; import type { SessionRecoveryDiagnosticEvent, TalkDiagnosticEvent } from "./service-types.js"; @@ -50,7 +53,7 @@ export function createOperationsRecorders(runtime: DiagnosticsRecorderRuntime) { const recordLaneEnqueue = ( evt: Extract, ) => { - const attrs = { "openclaw.lane": lowCardinalityQueueLaneAttr(evt.lane) }; + const attrs = { "openclaw.lane": normalizeDiagnosticLane(evt.lane) }; laneEnqueueCounter.add(1, attrs); queueDepthHistogram.record(evt.queueSize, attrs); }; @@ -58,7 +61,7 @@ export function createOperationsRecorders(runtime: DiagnosticsRecorderRuntime) { const recordLaneDequeue = ( evt: Extract, ) => { - const attrs = { "openclaw.lane": lowCardinalityQueueLaneAttr(evt.lane) }; + const attrs = { "openclaw.lane": normalizeDiagnosticLane(evt.lane) }; laneDequeueCounter.add(1, attrs); queueDepthHistogram.record(evt.queueSize, attrs); if (typeof evt.waitMs === "number") { @@ -78,8 +81,8 @@ export function createOperationsRecorders(runtime: DiagnosticsRecorderRuntime) { evt: Extract, ) => { sessionTurnCreatedCounter.add(1, { - "openclaw.agent": lowCardinalityAttr(evt.agentId, "unknown"), - "openclaw.channel": lowCardinalityAttr(evt.channel, "unknown"), + "openclaw.agent": normalizeDiagnosticValue(evt.agentId, "unknown"), + "openclaw.channel": normalizeDiagnosticValue(evt.channel, "unknown"), "openclaw.trigger": evt.trigger, }); }; @@ -126,7 +129,7 @@ export function createOperationsRecorders(runtime: DiagnosticsRecorderRuntime) { ) => { const attrs = sessionRecoveryAttrs(evt); attrs["openclaw.status"] = evt.status; - attrs["openclaw.action"] = lowCardinalityAttr(evt.action, "unknown"); + attrs["openclaw.action"] = normalizeDiagnosticValue(evt.action, "unknown"); if (evt.outcomeReason) { attrs["openclaw.reason"] = redactSensitiveText(evt.outcomeReason); } @@ -135,11 +138,11 @@ export function createOperationsRecorders(runtime: DiagnosticsRecorderRuntime) { }; const talkEventAttrs = (evt: TalkDiagnosticEvent): Record => ({ - "openclaw.talk.brain": lowCardinalityAttr(evt.brain), - "openclaw.talk.event_type": lowCardinalityAttr(evt.talkEventType), - "openclaw.talk.mode": lowCardinalityAttr(evt.mode), - "openclaw.talk.provider": lowCardinalityAttr(evt.provider), - "openclaw.talk.transport": lowCardinalityAttr(evt.transport), + "openclaw.talk.brain": normalizeDiagnosticValue(evt.brain), + "openclaw.talk.event_type": normalizeDiagnosticValue(evt.talkEventType), + "openclaw.talk.mode": normalizeDiagnosticValue(evt.mode), + "openclaw.talk.provider": normalizeDiagnosticValue(evt.provider), + "openclaw.talk.transport": normalizeDiagnosticValue(evt.transport), }); const recordTalkEvent = (evt: TalkDiagnosticEvent, metadata: DiagnosticEventMetadata) => { @@ -163,13 +166,13 @@ export function createOperationsRecorders(runtime: DiagnosticsRecorderRuntime) { const toolLoopAttrs = ( evt: Extract, ): Record => ({ - "openclaw.toolName": lowCardinalityAttr(evt.toolName, "tool"), + "openclaw.toolName": normalizeDiagnosticValue(evt.toolName, "tool"), "openclaw.loop.level": evt.level, "openclaw.loop.action": evt.action, "openclaw.loop.detector": evt.detector, "openclaw.loop.count": evt.count, ...(evt.pairedToolName - ? { "openclaw.loop.paired_tool": lowCardinalityAttr(evt.pairedToolName, "tool") } + ? { "openclaw.loop.paired_tool": normalizeDiagnosticValue(evt.pairedToolName, "tool") } : {}), }); @@ -285,7 +288,7 @@ export function createOperationsRecorders(runtime: DiagnosticsRecorderRuntime) { attrs["openclaw.channel"] = evt.channel; } if (evt.blockedBy) { - attrs["openclaw.blocked_by"] = lowCardinalityAttr(evt.blockedBy, "unknown"); + attrs["openclaw.blocked_by"] = normalizeDiagnosticValue(evt.blockedBy, "unknown"); } durationHistogram.record(evt.durationMs, attrs); if (!tracesEnabled) { @@ -296,10 +299,10 @@ export function createOperationsRecorders(runtime: DiagnosticsRecorderRuntime) { }; addRunAttrs(spanAttrs, evt); if (evt.blockedBy) { - spanAttrs["openclaw.blocked_by"] = lowCardinalityAttr(evt.blockedBy, "unknown"); + spanAttrs["openclaw.blocked_by"] = normalizeDiagnosticValue(evt.blockedBy, "unknown"); } if (evt.errorCategory) { - spanAttrs["openclaw.errorCategory"] = lowCardinalityAttr(evt.errorCategory, "other"); + spanAttrs["openclaw.errorCategory"] = normalizeDiagnosticValue(evt.errorCategory, "other"); } // Redacted message goes on the span only, never the low-cardinality metric attrs. const redactedError = normalizeOtelErrorMessage(privateData.errorMessage); diff --git a/extensions/diagnostics-otel/src/service-recorders-tools.ts b/extensions/diagnostics-otel/src/service-recorders-tools.ts index dd0cc6ea9997..83c23275669c 100644 --- a/extensions/diagnostics-otel/src/service-recorders-tools.ts +++ b/extensions/diagnostics-otel/src/service-recorders-tools.ts @@ -1,7 +1,7 @@ import { SpanStatusCode } from "@opentelemetry/api"; +import { normalizeDiagnosticValue } from "openclaw/plugin-sdk/diagnostic-runtime"; import { redactSensitiveText } from "../api.js"; import type { DiagnosticEventMetadata, DiagnosticEventPayload } from "../api.js"; -import { lowCardinalityAttr } from "./service-attributes.js"; import { positiveFiniteNumber } from "./service-genai-attributes.js"; import { assignOtelToolContentAttributes, @@ -51,20 +51,22 @@ export function createToolAndSystemRecorders(runtime: DiagnosticsRecorderRuntime >, ): Record => ({ "openclaw.toolName": evt.toolName, - "openclaw.tool.source": lowCardinalityAttr(evt.toolSource, "core"), + "openclaw.tool.source": normalizeDiagnosticValue(evt.toolSource, "core"), "gen_ai.tool.name": evt.toolName, - ...(evt.toolOwner ? { "openclaw.tool.owner": lowCardinalityAttr(evt.toolOwner) } : {}), + ...(evt.toolOwner ? { "openclaw.tool.owner": normalizeDiagnosticValue(evt.toolOwner) } : {}), ...paramsSummaryAttrs(evt.paramsSummary), }); const skillUsedAttrs = ( evt: Extract, ): Record => ({ - "openclaw.skill.name": lowCardinalityAttr(evt.skillName, "skill"), - "openclaw.skill.source": lowCardinalityAttr(evt.skillSource), - "openclaw.skill.activation": lowCardinalityAttr(evt.activation), - ...(evt.agentId ? { "openclaw.agent": lowCardinalityAttr(evt.agentId) } : {}), - ...(evt.toolName ? { "openclaw.toolName": lowCardinalityAttr(evt.toolName, "tool") } : {}), + "openclaw.skill.name": normalizeDiagnosticValue(evt.skillName, "skill"), + "openclaw.skill.source": normalizeDiagnosticValue(evt.skillSource), + "openclaw.skill.activation": normalizeDiagnosticValue(evt.activation), + ...(evt.agentId ? { "openclaw.agent": normalizeDiagnosticValue(evt.agentId) } : {}), + ...(evt.toolName + ? { "openclaw.toolName": normalizeDiagnosticValue(evt.toolName, "tool") } + : {}), }); const recordSkillUsed = ( @@ -139,7 +141,7 @@ export function createToolAndSystemRecorders(runtime: DiagnosticsRecorderRuntime ) => { const attrs = { ...toolExecutionBaseAttrs(evt), - "openclaw.errorCategory": lowCardinalityAttr(evt.errorCategory, "other"), + "openclaw.errorCategory": normalizeDiagnosticValue(evt.errorCategory, "other"), }; toolExecutionDurationHistogram.record(evt.durationMs, attrs); if (!tracesEnabled) { @@ -149,7 +151,7 @@ export function createToolAndSystemRecorders(runtime: DiagnosticsRecorderRuntime addRunAttrs(spanAttrs, evt); assignOtelToolIdentityAttributes(spanAttrs, evt); if (evt.errorCode) { - spanAttrs["openclaw.errorCode"] = lowCardinalityAttr(evt.errorCode, "other"); + spanAttrs["openclaw.errorCode"] = normalizeDiagnosticValue(evt.errorCode, "other"); } assignOtelToolContentAttributes(spanAttrs, toolContent, contentCapturePolicy); const span = @@ -172,7 +174,7 @@ export function createToolAndSystemRecorders(runtime: DiagnosticsRecorderRuntime ) => { toolExecutionBlockedCounter.add(1, { ...toolExecutionBaseAttrs(evt), - "openclaw.deniedReason": lowCardinalityAttr(evt.deniedReason, "other"), + "openclaw.deniedReason": normalizeDiagnosticValue(evt.deniedReason, "other"), }); if (!tracesEnabled) { return; @@ -180,7 +182,7 @@ export function createToolAndSystemRecorders(runtime: DiagnosticsRecorderRuntime const spanAttrs: Record = { ...toolExecutionBaseAttrs(evt), "openclaw.outcome": "blocked", - "openclaw.deniedReason": lowCardinalityAttr(evt.deniedReason, "other"), + "openclaw.deniedReason": normalizeDiagnosticValue(evt.deniedReason, "other"), }; addRunAttrs(spanAttrs, evt); assignOtelToolIdentityAttributes(spanAttrs, evt); @@ -195,10 +197,10 @@ export function createToolAndSystemRecorders(runtime: DiagnosticsRecorderRuntime const recordPayloadLarge = (evt: Extract) => { const attrs = { "openclaw.payload.action": evt.action, - "openclaw.payload.surface": lowCardinalityAttr(evt.surface, "unknown"), - "openclaw.channel": lowCardinalityAttr(evt.channel, "none"), - "openclaw.plugin": lowCardinalityAttr(evt.pluginId, "none"), - "openclaw.reason": lowCardinalityAttr(evt.reason, "none"), + "openclaw.payload.surface": normalizeDiagnosticValue(evt.surface, "unknown"), + "openclaw.channel": normalizeDiagnosticValue(evt.channel, "none"), + "openclaw.plugin": normalizeDiagnosticValue(evt.pluginId, "none"), + "openclaw.reason": normalizeDiagnosticValue(evt.reason, "none"), }; payloadLargeCounter.add(1, attrs); const bytes = positiveFiniteNumber(evt.bytes); @@ -232,7 +234,7 @@ export function createToolAndSystemRecorders(runtime: DiagnosticsRecorderRuntime spanAttrs["openclaw.exec.exit_code"] = evt.exitCode; } if (evt.exitSignal) { - spanAttrs["openclaw.exec.exit_signal"] = lowCardinalityAttr(evt.exitSignal, "other"); + spanAttrs["openclaw.exec.exit_signal"] = normalizeDiagnosticValue(evt.exitSignal, "other"); } if (evt.timedOut !== undefined) { spanAttrs["openclaw.exec.timed_out"] = evt.timedOut; @@ -267,7 +269,7 @@ export function createToolAndSystemRecorders(runtime: DiagnosticsRecorderRuntime ) => { const reason = evt.reasons.join(":"); const attrs = { - "openclaw.liveness.reason": lowCardinalityAttr(reason, "unknown"), + "openclaw.liveness.reason": normalizeDiagnosticValue(reason, "unknown"), }; livenessWarningCounter.add(1, attrs); queueDepthHistogram.record(evt.queued, { "openclaw.channel": "liveness" }); @@ -327,7 +329,7 @@ export function createToolAndSystemRecorders(runtime: DiagnosticsRecorderRuntime return; } const spanAttrs: Record = { - "openclaw.phase": lowCardinalityAttr(evt.name, "unknown"), + "openclaw.phase": normalizeDiagnosticValue(evt.name, "unknown"), ...(evt.cpuUserMs !== undefined ? { "openclaw.phase.cpu_user_ms": evt.cpuUserMs } : {}), ...(evt.cpuSystemMs !== undefined ? { "openclaw.phase.cpu_system_ms": evt.cpuSystemMs } : {}), ...(evt.cpuTotalMs !== undefined ? { "openclaw.phase.cpu_total_ms": evt.cpuTotalMs } : {}), @@ -353,12 +355,12 @@ export function createToolAndSystemRecorders(runtime: DiagnosticsRecorderRuntime return; } telemetryExporterCounter.add(1, { - "openclaw.exporter": lowCardinalityAttr(evt.exporter, "unknown"), + "openclaw.exporter": normalizeDiagnosticValue(evt.exporter, "unknown"), "openclaw.signal": evt.signal, "openclaw.status": evt.status, ...(evt.reason ? { "openclaw.reason": evt.reason } : {}), ...(evt.errorCategory - ? { "openclaw.errorCategory": lowCardinalityAttr(evt.errorCategory, "other") } + ? { "openclaw.errorCategory": normalizeDiagnosticValue(evt.errorCategory, "other") } : {}), }); }; diff --git a/extensions/diagnostics-otel/src/service-recorders-usage.ts b/extensions/diagnostics-otel/src/service-recorders-usage.ts index 232c9ed06be0..3b962d98a6f8 100644 --- a/extensions/diagnostics-otel/src/service-recorders-usage.ts +++ b/extensions/diagnostics-otel/src/service-recorders-usage.ts @@ -1,7 +1,7 @@ import { SpanStatusCode } from "@opentelemetry/api"; +import { normalizeDiagnosticValue } from "openclaw/plugin-sdk/diagnostic-runtime"; import { redactSensitiveText } from "../api.js"; import type { DiagnosticEventMetadata, DiagnosticEventPayload } from "../api.js"; -import { lowCardinalityAttr } from "./service-attributes.js"; import { assignGenAiSpanIdentityAttrs, assignPositiveNumberAttr, @@ -54,14 +54,14 @@ export function createUsageRecorders(runtime: DiagnosticsRecorderRuntime) { ) => { const attrs = { "openclaw.channel": evt.channel ?? "unknown", - "openclaw.agent": lowCardinalityAttr(evt.agentId), + "openclaw.agent": normalizeDiagnosticValue(evt.agentId), "openclaw.provider": evt.provider ?? "unknown", "openclaw.model": evt.model ?? "unknown", }; const genAiAttrs: Record = { "gen_ai.operation.name": "chat", - "gen_ai.provider.name": lowCardinalityAttr(evt.provider), - "gen_ai.request.model": lowCardinalityAttr(evt.model), + "gen_ai.provider.name": normalizeDiagnosticValue(evt.provider), + "gen_ai.request.model": normalizeDiagnosticValue(evt.model), }; const usage = evt.usage; @@ -155,8 +155,8 @@ export function createUsageRecorders(runtime: DiagnosticsRecorderRuntime) { evt: Extract, ) => { const attrs = { - "openclaw.channel": lowCardinalityAttr(evt.channel), - "openclaw.webhook": lowCardinalityAttr(evt.updateType), + "openclaw.channel": normalizeDiagnosticValue(evt.channel), + "openclaw.webhook": normalizeDiagnosticValue(evt.updateType), }; if (typeof evt.durationMs === "number") { webhookDurationHistogram.record(evt.durationMs, attrs); @@ -171,8 +171,8 @@ export function createUsageRecorders(runtime: DiagnosticsRecorderRuntime) { const recordWebhookError = (evt: Extract) => { const attrs = { - "openclaw.channel": lowCardinalityAttr(evt.channel), - "openclaw.webhook": lowCardinalityAttr(evt.updateType), + "openclaw.channel": normalizeDiagnosticValue(evt.channel), + "openclaw.webhook": normalizeDiagnosticValue(evt.updateType), }; webhookErrorCounter.add(1, attrs); if (!tracesEnabled) { @@ -194,8 +194,8 @@ export function createUsageRecorders(runtime: DiagnosticsRecorderRuntime) { evt: Extract, ) => { const attrs = { - "openclaw.channel": lowCardinalityAttr(evt.channel), - "openclaw.source": lowCardinalityAttr(evt.source), + "openclaw.channel": normalizeDiagnosticValue(evt.channel), + "openclaw.source": normalizeDiagnosticValue(evt.source), }; messageQueuedCounter.add(1, attrs); if (typeof evt.queueDepth === "number") { @@ -207,8 +207,8 @@ export function createUsageRecorders(runtime: DiagnosticsRecorderRuntime) { evt: Extract, ) => { messageReceivedCounter.add(1, { - "openclaw.channel": lowCardinalityAttr(evt.channel), - "openclaw.source": lowCardinalityAttr(evt.source), + "openclaw.channel": normalizeDiagnosticValue(evt.channel), + "openclaw.source": normalizeDiagnosticValue(evt.source), }); }; @@ -217,8 +217,8 @@ export function createUsageRecorders(runtime: DiagnosticsRecorderRuntime) { metadata: DiagnosticEventMetadata, ) => { const attrs = { - "openclaw.channel": lowCardinalityAttr(evt.channel), - "openclaw.source": lowCardinalityAttr(evt.source), + "openclaw.channel": normalizeDiagnosticValue(evt.channel), + "openclaw.source": normalizeDiagnosticValue(evt.source), }; messageDispatchStartedCounter.add(1, attrs); if (!tracesEnabled) { @@ -242,10 +242,10 @@ export function createUsageRecorders(runtime: DiagnosticsRecorderRuntime) { evt: Extract, ) => { const attrs = { - "openclaw.channel": lowCardinalityAttr(evt.channel), + "openclaw.channel": normalizeDiagnosticValue(evt.channel), "openclaw.outcome": evt.outcome, - "openclaw.reason": lowCardinalityAttr(evt.reason, "none"), - "openclaw.source": lowCardinalityAttr(evt.source), + "openclaw.reason": normalizeDiagnosticValue(evt.reason, "none"), + "openclaw.source": normalizeDiagnosticValue(evt.source), }; messageDispatchCompletedCounter.add(1, attrs); messageDispatchDurationHistogram.record(evt.durationMs, attrs); @@ -256,7 +256,7 @@ export function createUsageRecorders(runtime: DiagnosticsRecorderRuntime) { metadata: DiagnosticEventMetadata, ) => { const attrs = { - "openclaw.channel": lowCardinalityAttr(evt.channel), + "openclaw.channel": normalizeDiagnosticValue(evt.channel), "openclaw.outcome": evt.outcome ?? "unknown", }; messageProcessedCounter.add(1, attrs); @@ -268,7 +268,7 @@ export function createUsageRecorders(runtime: DiagnosticsRecorderRuntime) { } const spanAttrs: Record = { ...attrs }; if (evt.reason) { - spanAttrs["openclaw.reason"] = lowCardinalityAttr(evt.reason, "unknown"); + spanAttrs["openclaw.reason"] = normalizeDiagnosticValue(evt.reason, "unknown"); } const trackedSpan = getTrackedInternalOrTrustedSpan(evt, metadata); const span = @@ -290,8 +290,8 @@ export function createUsageRecorders(runtime: DiagnosticsRecorderRuntime) { }; const messageDeliveryAttrs = (evt: MessageDeliveryDiagnosticEvent): Record => ({ - "openclaw.channel": lowCardinalityAttr(evt.channel), - "openclaw.delivery.kind": lowCardinalityAttr(evt.deliveryKind, "other"), + "openclaw.channel": normalizeDiagnosticValue(evt.channel), + "openclaw.delivery.kind": normalizeDiagnosticValue(evt.deliveryKind, "other"), }); const recordMessageDeliveryStarted = ( @@ -331,7 +331,7 @@ export function createUsageRecorders(runtime: DiagnosticsRecorderRuntime) { const attrs = { ...messageDeliveryAttrs(evt), "openclaw.outcome": "error", - "openclaw.errorCategory": lowCardinalityAttr(evt.errorCategory, "other"), + "openclaw.errorCategory": normalizeDiagnosticValue(evt.errorCategory, "other"), }; messageDeliveryDurationHistogram.record(evt.durationMs, attrs); if (!tracesEnabled) { diff --git a/extensions/diagnostics-prometheus/src/service.ts b/extensions/diagnostics-prometheus/src/service.ts index 23edbb1dfd77..e27b13376dc7 100644 --- a/extensions/diagnostics-prometheus/src/service.ts +++ b/extensions/diagnostics-prometheus/src/service.ts @@ -1,5 +1,9 @@ // Diagnostics Prometheus plugin module implements service behavior. import type { IncomingMessage, ServerResponse } from "node:http"; +import { + normalizeDiagnosticValue, + normalizeDiagnosticLane, +} from "openclaw/plugin-sdk/diagnostic-runtime"; import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; import type { DiagnosticEventMetadata, @@ -49,34 +53,8 @@ const BYTE_BUCKETS = [ 4294967296, 17179869184, ]; const RATIO_BUCKETS = [0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 2, 4, 8, 16]; -const LOW_CARDINALITY_VALUE_RE = /^[A-Za-z0-9_.:-]{1,120}$/u; const MAX_PROMETHEUS_SERIES = 2048; const DROPPED_SERIES_COUNTER_NAME = "openclaw_prometheus_series_dropped_total"; -function lowCardinalityLabel(value: string | undefined, fallback = "unknown"): string { - if (!value) { - return fallback; - } - const redacted = redactSensitiveText(value.trim()); - const redactedLower = redacted.toLowerCase(); - if (redactedLower.startsWith("agent:") || redactedLower.includes(":agent:")) { - return fallback; - } - return LOW_CARDINALITY_VALUE_RE.test(redacted) ? redacted : fallback; -} - -function lowCardinalityQueueLaneLabel(value: string | undefined, fallback = "unknown"): string { - if (!value) { - return fallback; - } - const redacted = redactSensitiveText(value.trim()); - const redactedLower = redacted.toLowerCase(); - if (redactedLower.startsWith("agent:")) { - return fallback; - } - const scopedLaneIndex = redacted.indexOf(":"); - const lane = scopedLaneIndex >= 0 ? redacted.slice(0, scopedLaneIndex) : redacted; - return LOW_CARDINALITY_VALUE_RE.test(lane) ? lane : fallback; -} function numericValue(value: number | undefined): number | undefined { return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined; @@ -310,12 +288,12 @@ function runLabels(evt: { trigger?: string; }): LabelSet { return { - ...(evt.blockedBy ? { blocked_by: lowCardinalityLabel(evt.blockedBy) } : {}), - channel: lowCardinalityLabel(evt.channel), - model: lowCardinalityLabel(evt.model), - outcome: lowCardinalityLabel(evt.outcome, "unknown"), - provider: lowCardinalityLabel(evt.provider), - trigger: lowCardinalityLabel(evt.trigger), + ...(evt.blockedBy ? { blocked_by: normalizeDiagnosticValue(evt.blockedBy) } : {}), + channel: normalizeDiagnosticValue(evt.channel), + model: normalizeDiagnosticValue(evt.model), + outcome: normalizeDiagnosticValue(evt.outcome, "unknown"), + provider: normalizeDiagnosticValue(evt.provider), + trigger: normalizeDiagnosticValue(evt.trigger), }; } @@ -329,14 +307,16 @@ function modelCallLabels(evt: { type: string; }): LabelSet { return { - api: lowCardinalityLabel(evt.api), + api: normalizeDiagnosticValue(evt.api), error_category: - evt.type === "model.call.error" ? lowCardinalityLabel(evt.errorCategory, "other") : "none", - model: lowCardinalityLabel(evt.model), + evt.type === "model.call.error" + ? normalizeDiagnosticValue(evt.errorCategory, "other") + : "none", + model: normalizeDiagnosticValue(evt.model), observation_unit: evt.observationUnit === "turn" ? "turn" : "request", outcome: evt.type === "model.call.error" ? "error" : "completed", - provider: lowCardinalityLabel(evt.provider), - transport: lowCardinalityLabel(evt.transport), + provider: normalizeDiagnosticValue(evt.provider), + transport: normalizeDiagnosticValue(evt.transport), }; } @@ -344,13 +324,13 @@ function modelFailoverLabels( evt: Extract, ): LabelSet { return { - from_model: lowCardinalityLabel(evt.fromModel), - from_provider: lowCardinalityLabel(evt.fromProvider), - lane: lowCardinalityQueueLaneLabel(evt.lane), - reason: lowCardinalityLabel(evt.reason, "other"), + from_model: normalizeDiagnosticValue(evt.fromModel), + from_provider: normalizeDiagnosticValue(evt.fromProvider), + lane: normalizeDiagnosticLane(evt.lane), + reason: normalizeDiagnosticValue(evt.reason, "other"), suspended: evt.suspended === undefined ? "unknown" : String(evt.suspended), - to_model: lowCardinalityLabel(evt.toModel), - to_provider: lowCardinalityLabel(evt.toProvider), + to_model: normalizeDiagnosticValue(evt.toModel), + to_provider: normalizeDiagnosticValue(evt.toProvider), }; } @@ -365,13 +345,13 @@ function toolExecutionLabels(evt: { return { error_category: evt.type === "tool.execution.error" - ? lowCardinalityLabel(evt.errorCategory, "other") + ? normalizeDiagnosticValue(evt.errorCategory, "other") : "none", outcome: evt.type === "tool.execution.error" ? "error" : "completed", - params_kind: lowCardinalityLabel(evt.paramsSummary?.kind), - tool: lowCardinalityLabel(evt.toolName, "tool"), - tool_owner: lowCardinalityLabel(evt.toolOwner, "none"), - tool_source: lowCardinalityLabel(evt.toolSource, "core"), + params_kind: normalizeDiagnosticValue(evt.paramsSummary?.kind), + tool: normalizeDiagnosticValue(evt.toolName, "tool"), + tool_owner: normalizeDiagnosticValue(evt.toolOwner, "none"), + tool_source: normalizeDiagnosticValue(evt.toolSource, "core"), }; } @@ -379,11 +359,11 @@ function toolExecutionBlockedLabels( evt: Extract, ): LabelSet { return { - denied_reason: lowCardinalityLabel(evt.deniedReason, "other"), - params_kind: lowCardinalityLabel(evt.paramsSummary?.kind), - tool: lowCardinalityLabel(evt.toolName, "tool"), - tool_owner: lowCardinalityLabel(evt.toolOwner, "none"), - tool_source: lowCardinalityLabel(evt.toolSource, "core"), + denied_reason: normalizeDiagnosticValue(evt.deniedReason, "other"), + params_kind: normalizeDiagnosticValue(evt.paramsSummary?.kind), + tool: normalizeDiagnosticValue(evt.toolName, "tool"), + tool_owner: normalizeDiagnosticValue(evt.toolOwner, "none"), + tool_source: normalizeDiagnosticValue(evt.toolSource, "core"), }; } @@ -394,10 +374,10 @@ function skillLabels(evt: { skillSource?: string; }): LabelSet { return { - activation: lowCardinalityLabel(evt.activation, "unknown"), - agent: lowCardinalityLabel(evt.agentId), - skill: lowCardinalityLabel(evt.skillName, "skill"), - source: lowCardinalityLabel(evt.skillSource), + activation: normalizeDiagnosticValue(evt.activation, "unknown"), + agent: normalizeDiagnosticValue(evt.agentId), + skill: normalizeDiagnosticValue(evt.skillName, "skill"), + source: normalizeDiagnosticValue(evt.skillSource), }; } @@ -413,15 +393,17 @@ function harnessLabels(evt: { type: string; }): LabelSet { return { - channel: lowCardinalityLabel(evt.channel), + channel: normalizeDiagnosticValue(evt.channel), error_category: - evt.type === "harness.run.error" ? lowCardinalityLabel(evt.errorCategory, "other") : "none", - harness: lowCardinalityLabel(evt.harnessId), - model: lowCardinalityLabel(evt.model), - outcome: evt.type === "harness.run.error" ? "error" : lowCardinalityLabel(evt.outcome), - phase: evt.type === "harness.run.error" ? lowCardinalityLabel(evt.phase) : "none", - plugin: lowCardinalityLabel(evt.pluginId), - provider: lowCardinalityLabel(evt.provider), + evt.type === "harness.run.error" + ? normalizeDiagnosticValue(evt.errorCategory, "other") + : "none", + harness: normalizeDiagnosticValue(evt.harnessId), + model: normalizeDiagnosticValue(evt.model), + outcome: evt.type === "harness.run.error" ? "error" : normalizeDiagnosticValue(evt.outcome), + phase: evt.type === "harness.run.error" ? normalizeDiagnosticValue(evt.phase) : "none", + plugin: normalizeDiagnosticValue(evt.pluginId), + provider: normalizeDiagnosticValue(evt.provider), }; } @@ -432,8 +414,8 @@ function webhookLabels( >, ): LabelSet { return { - channel: lowCardinalityLabel(evt.channel), - webhook: lowCardinalityLabel(evt.updateType), + channel: normalizeDiagnosticValue(evt.channel), + webhook: normalizeDiagnosticValue(evt.updateType), }; } @@ -441,7 +423,7 @@ function sessionStuckLabels( evt: Extract, ): LabelSet { return { - reason: lowCardinalityLabel(evt.reason, "none"), + reason: normalizeDiagnosticValue(evt.reason, "none"), state: evt.state, }; } @@ -455,11 +437,11 @@ function sessionRecoveryLabels( return { action: evt.type === "session.recovery.completed" - ? lowCardinalityLabel(evt.action, "unknown") + ? normalizeDiagnosticValue(evt.action, "unknown") : evt.allowActiveAbort ? "abort" : "recover", - active_work_kind: lowCardinalityLabel(evt.activeWorkKind, "none"), + active_work_kind: normalizeDiagnosticValue(evt.activeWorkKind, "none"), state: evt.state, status: evt.type === "session.recovery.completed" ? evt.status : "requested", }; @@ -469,7 +451,7 @@ function livenessLabels( evt: Extract, ): LabelSet { return { - reason: lowCardinalityLabel(evt.reasons.join(":"), "unknown"), + reason: normalizeDiagnosticValue(evt.reasons.join(":"), "unknown"), }; } @@ -478,20 +460,20 @@ function payloadLargeLabels( ): LabelSet { return { action: evt.action, - channel: lowCardinalityLabel(evt.channel, "none"), - plugin: lowCardinalityLabel(evt.pluginId, "none"), - reason: lowCardinalityLabel(evt.reason, "none"), - surface: lowCardinalityLabel(evt.surface, "unknown"), + channel: normalizeDiagnosticValue(evt.channel, "none"), + plugin: normalizeDiagnosticValue(evt.pluginId, "none"), + reason: normalizeDiagnosticValue(evt.reason, "none"), + surface: normalizeDiagnosticValue(evt.surface, "unknown"), }; } function talkLabels(evt: Extract): LabelSet { return { - brain: lowCardinalityLabel(evt.brain), - event_type: lowCardinalityLabel(evt.talkEventType), - mode: lowCardinalityLabel(evt.mode), - provider: lowCardinalityLabel(evt.provider), - transport: lowCardinalityLabel(evt.transport), + brain: normalizeDiagnosticValue(evt.brain), + event_type: normalizeDiagnosticValue(evt.talkEventType), + mode: normalizeDiagnosticValue(evt.mode), + provider: normalizeDiagnosticValue(evt.provider), + transport: normalizeDiagnosticValue(evt.transport), }; } @@ -500,10 +482,10 @@ function recordModelUsage( evt: Extract, ) { const labels = { - agent: lowCardinalityLabel(evt.agentId), - channel: lowCardinalityLabel(evt.channel), - model: lowCardinalityLabel(evt.model), - provider: lowCardinalityLabel(evt.provider), + agent: normalizeDiagnosticValue(evt.agentId), + channel: normalizeDiagnosticValue(evt.channel), + model: normalizeDiagnosticValue(evt.model), + provider: normalizeDiagnosticValue(evt.provider), }; const usage = evt.usage; const recordTokens = (tokenType: string, value: number | undefined) => { @@ -643,17 +625,17 @@ function recordDiagnosticEvent( return; case "message.processed": store.counter("openclaw_message_processed_total", "Inbound messages processed by outcome.", { - channel: lowCardinalityLabel(evt.channel), + channel: normalizeDiagnosticValue(evt.channel), outcome: evt.outcome, - reason: lowCardinalityLabel(evt.reason, "none"), + reason: normalizeDiagnosticValue(evt.reason, "none"), }); store.histogram( "openclaw_message_processed_duration_seconds", "Inbound message processing duration in seconds.", { - channel: lowCardinalityLabel(evt.channel), + channel: normalizeDiagnosticValue(evt.channel), outcome: evt.outcome, - reason: lowCardinalityLabel(evt.reason, "none"), + reason: normalizeDiagnosticValue(evt.reason, "none"), }, seconds(evt.durationMs), ); @@ -685,15 +667,15 @@ function recordDiagnosticEvent( "openclaw_message_delivery_started_total", "Outbound message delivery attempts started.", { - channel: lowCardinalityLabel(evt.channel), - delivery_kind: lowCardinalityLabel(evt.deliveryKind, "other"), + channel: normalizeDiagnosticValue(evt.channel), + delivery_kind: normalizeDiagnosticValue(evt.deliveryKind, "other"), }, ); return; case "message.received": store.counter("openclaw_message_received_total", "Inbound messages received by channel.", { - channel: lowCardinalityLabel(evt.channel), - source: lowCardinalityLabel(evt.source), + channel: normalizeDiagnosticValue(evt.channel), + source: normalizeDiagnosticValue(evt.source), }); return; case "message.dispatch.started": @@ -701,8 +683,8 @@ function recordDiagnosticEvent( "openclaw_message_dispatch_started_total", "Inbound message dispatch attempts started by channel.", { - channel: lowCardinalityLabel(evt.channel), - source: lowCardinalityLabel(evt.source), + channel: normalizeDiagnosticValue(evt.channel), + source: normalizeDiagnosticValue(evt.source), }, ); return; @@ -711,20 +693,20 @@ function recordDiagnosticEvent( "openclaw_message_dispatch_completed_total", "Inbound message dispatch attempts completed by outcome.", { - channel: lowCardinalityLabel(evt.channel), + channel: normalizeDiagnosticValue(evt.channel), outcome: evt.outcome, - reason: lowCardinalityLabel(evt.reason, "none"), - source: lowCardinalityLabel(evt.source), + reason: normalizeDiagnosticValue(evt.reason, "none"), + source: normalizeDiagnosticValue(evt.source), }, ); store.histogram( "openclaw_message_dispatch_duration_seconds", "Inbound message dispatch duration in seconds.", { - channel: lowCardinalityLabel(evt.channel), + channel: normalizeDiagnosticValue(evt.channel), outcome: evt.outcome, - reason: lowCardinalityLabel(evt.reason, "none"), - source: lowCardinalityLabel(evt.source), + reason: normalizeDiagnosticValue(evt.reason, "none"), + source: normalizeDiagnosticValue(evt.source), }, seconds(evt.durationMs), ); @@ -735,11 +717,11 @@ function recordDiagnosticEvent( "openclaw_message_delivery_total", "Outbound message delivery attempts by outcome.", { - channel: lowCardinalityLabel(evt.channel), - delivery_kind: lowCardinalityLabel(evt.deliveryKind, "other"), + channel: normalizeDiagnosticValue(evt.channel), + delivery_kind: normalizeDiagnosticValue(evt.deliveryKind, "other"), error_category: evt.type === "message.delivery.error" - ? lowCardinalityLabel(evt.errorCategory, "other") + ? normalizeDiagnosticValue(evt.errorCategory, "other") : "none", outcome: evt.type === "message.delivery.error" ? "error" : "completed", }, @@ -748,11 +730,11 @@ function recordDiagnosticEvent( "openclaw_message_delivery_duration_seconds", "Outbound message delivery duration in seconds.", { - channel: lowCardinalityLabel(evt.channel), - delivery_kind: lowCardinalityLabel(evt.deliveryKind, "other"), + channel: normalizeDiagnosticValue(evt.channel), + delivery_kind: normalizeDiagnosticValue(evt.deliveryKind, "other"), error_category: evt.type === "message.delivery.error" - ? lowCardinalityLabel(evt.errorCategory, "other") + ? normalizeDiagnosticValue(evt.errorCategory, "other") : "none", outcome: evt.type === "message.delivery.error" ? "error" : "completed", }, @@ -795,7 +777,7 @@ function recordDiagnosticEvent( "openclaw_queue_lane_size", "Current diagnostic queue lane size.", { - lane: lowCardinalityQueueLaneLabel(evt.lane), + lane: normalizeDiagnosticLane(evt.lane), }, numericValue(evt.queueSize), ); @@ -803,14 +785,14 @@ function recordDiagnosticEvent( store.histogram( "openclaw_queue_lane_wait_seconds", "Queue lane wait time in seconds.", - { lane: lowCardinalityQueueLaneLabel(evt.lane) }, + { lane: normalizeDiagnosticLane(evt.lane) }, seconds(evt.waitMs), ); } return; case "session.state": store.counter("openclaw_session_state_total", "Session state observations.", { - reason: lowCardinalityLabel(evt.reason, "none"), + reason: normalizeDiagnosticValue(evt.reason, "none"), state: evt.state, }); if (evt.queueDepth !== undefined) { @@ -839,8 +821,8 @@ function recordDiagnosticEvent( return; case "session.turn.created": store.counter("openclaw_session_turn_created_total", "Agent session turns created.", { - agent: lowCardinalityLabel(evt.agentId), - channel: lowCardinalityLabel(evt.channel), + agent: normalizeDiagnosticValue(evt.agentId), + channel: normalizeDiagnosticValue(evt.channel), trigger: evt.trigger, }); return; @@ -974,8 +956,8 @@ function recordDiagnosticEvent( break; case "telemetry.exporter": store.counter("openclaw_telemetry_exporter_total", "Telemetry exporter lifecycle events.", { - exporter: lowCardinalityLabel(evt.exporter), - reason: lowCardinalityLabel(evt.reason, "none"), + exporter: normalizeDiagnosticValue(evt.exporter), + reason: normalizeDiagnosticValue(evt.reason, "none"), signal: evt.signal, status: evt.status, }); diff --git a/extensions/onepassword/onepassword-op-path.d.ts b/extensions/onepassword/onepassword-op-path.d.ts index e276f383a656..b9108789c3e4 100644 --- a/extensions/onepassword/onepassword-op-path.d.ts +++ b/extensions/onepassword/onepassword-op-path.d.ts @@ -1,5 +1,3 @@ -export function resolveTrustedOnePasswordDirectoryPath(targetPath: string): Promise; - export function resolveTrustedOnePasswordCli(options?: { configuredPath?: string; pathEnv?: string; diff --git a/extensions/onepassword/onepassword-op-path.js b/extensions/onepassword/onepassword-op-path.js index 4571d43744a1..86ca8f1ea65d 100644 --- a/extensions/onepassword/onepassword-op-path.js +++ b/extensions/onepassword/onepassword-op-path.js @@ -6,8 +6,6 @@ function errorCode(error) { } const resolveTrustedExecutablePath = pluginSecretRefSetup.resolveTrustedExecutablePath; -export const resolveTrustedOnePasswordDirectoryPath = - pluginSecretRefSetup.resolveTrustedDirectoryPath; export async function resolveTrustedOnePasswordCli(options = {}) { const configuredPath = options.configuredPath?.trim(); diff --git a/extensions/onepassword/src/secret-ref-cli.test.ts b/extensions/onepassword/src/secret-ref-cli.test.ts index f4243fa228e5..48384976eec5 100644 --- a/extensions/onepassword/src/secret-ref-cli.test.ts +++ b/extensions/onepassword/src/secret-ref-cli.test.ts @@ -1,13 +1,17 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { inspectPathPermissions } from "@openclaw/fs-safe/permissions"; import { Command } from "commander"; import type { OpenClawConfig } from "openclaw/plugin-sdk/plugin-entry"; import { afterEach, describe, expect, it, vi } from "vitest"; import { encodeOnePasswordSecretId } from "../onepassword-secret-id.js"; import { registerOnePasswordSecretRefCommands, testing } from "./secret-ref-cli.js"; +type OnePasswordPlan = { + providerUpserts: Record; + targets: Array>; +}; + function captureStdout() { let output = ""; vi.spyOn(process.stdout, "write").mockImplementation((chunk) => { @@ -17,7 +21,7 @@ function captureStdout() { return () => output; } -function createProgram(config: OpenClawConfig): Command { +function createProgram(config: OpenClawConfig = {}): Command { const program = new Command().exitOverride(); const onepassword = program.command("onepassword"); registerOnePasswordSecretRefCommands({ @@ -36,19 +40,29 @@ async function runStatus( const output = captureStdout(); await createProgram(config).parseAsync( ["onepassword", "secretref", "status", "--json", ...args], - { - from: "user", - }, + { from: "user" }, ); return JSON.parse(output()) as Record; } -function createOpenAiPlan() { - return testing.buildPlan({ - providerAlias: "onepassword", - providerConfig: testing.buildProviderConfig(), - providerSecrets: [{ providerId: "openai", secretId: "op://openclaw/OpenAI/credential" }], - }); +async function runSetup(planPath: string, args: string[]): Promise { + const output = captureStdout(); + await createProgram().parseAsync( + ["onepassword", "secretref", "setup", "--plan-out", planPath, ...args], + { from: "user" }, + ); + return output(); +} + +async function createSetupPlan(args: string[]): Promise { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-onepassword-cli-")); + const planPath = path.join(dir, "plan.json"); + try { + await runSetup(planPath, args); + return JSON.parse(await fs.readFile(planPath, "utf8")) as OnePasswordPlan; + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } } afterEach(() => { @@ -56,215 +70,180 @@ afterEach(() => { vi.unstubAllEnvs(); }); -describe("1Password CLI helpers", () => { - it("builds a secrets apply plan for model provider API keys", () => { - const plan = testing.buildPlan({ - providerAlias: "onepassword", - providerConfig: testing.buildProviderConfig(), - providerSecrets: [ - { - providerId: "anthropic", - secretId: "op://openclaw/Anthropic/credential", - }, - { - providerId: "openrouter", - secretId: "openclaw/OpenRouter/credential", - }, - ], - }); +describe("1Password SecretRef setup", () => { + it("builds provider config and model API-key targets", async () => { + const plan = await createSetupPlan([ + "--anthropic-id", + "op://openclaw/Anthropic/credential", + "--openrouter-id", + "openclaw/OpenRouter/credential", + "--provider-key", + "xai=op://openclaw/xAI/credential", + ]); expect(plan.providerUpserts.onepassword).toEqual({ source: "exec", - pluginIntegration: { - pluginId: "onepassword", - integrationId: "onepassword", - }, + pluginIntegration: { pluginId: "onepassword", integrationId: "onepassword" }, }); expect(plan.targets).toEqual([ - { + expect.objectContaining({ type: "models.providers.apiKey", - path: "models.providers.anthropic.apiKey", - pathSegments: ["models", "providers", "anthropic", "apiKey"], providerId: "anthropic", - ref: { - source: "exec", - provider: "onepassword", - id: "op://openclaw/Anthropic/credential", - }, - }, - { - type: "models.providers.apiKey", - path: "models.providers.openrouter.apiKey", - pathSegments: ["models", "providers", "openrouter", "apiKey"], - providerId: "openrouter", - ref: { - source: "exec", - provider: "onepassword", - id: "openclaw/OpenRouter/credential", - }, - }, + ref: { source: "exec", provider: "onepassword", id: "op://openclaw/Anthropic/credential" }, + }), + expect.objectContaining({ providerId: "openrouter" }), + expect.objectContaining({ providerId: "xai" }), ]); }); - it("builds a secrets apply plan for arbitrary known openclaw secret targets", () => { - const plan = testing.buildPlan({ - providerAlias: "onepassword", - providerConfig: testing.buildProviderConfig(), - providerSecrets: [], - configTargetSecrets: testing.parseConfigTargetMappings([ - "channels.telegram.botToken=op://openclaw/Telegram/botToken", - "models.providers.openai.headers.x-api-key=op://openclaw/OpenAI/proxyKey", - "auth-profiles:main:profiles.openai.key=op://openclaw/OpenAI/credential", - ]), - }); + it("builds arbitrary known OpenClaw and auth-profile targets", async () => { + const plan = await createSetupPlan([ + "--target", + "channels.telegram.botToken=op://openclaw/Telegram/botToken", + "--target", + "models.providers.openai.headers.x-api-key=op://openclaw/OpenAI/proxyKey", + "--target", + "auth-profiles:main:profiles.openai.key=op://openclaw/OpenAI/credential", + ]); expect(plan.targets).toEqual([ - { + expect.objectContaining({ type: "channels.telegram.botToken", path: "channels.telegram.botToken", - pathSegments: ["channels", "telegram", "botToken"], - ref: { - source: "exec", - provider: "onepassword", - id: "op://openclaw/Telegram/botToken", - }, - }, - { + }), + expect.objectContaining({ type: "models.providers.headers", - path: "models.providers.openai.headers.x-api-key", - pathSegments: ["models", "providers", "openai", "headers", "x-api-key"], providerId: "openai", - ref: { - source: "exec", - provider: "onepassword", - id: "op://openclaw/OpenAI/proxyKey", - }, - }, - { + }), + expect.objectContaining({ type: "auth-profiles.api_key.key", path: "profiles.openai.key", - pathSegments: ["profiles", "openai", "key"], agentId: "main", - ref: { - source: "exec", - provider: "onepassword", - id: "op://openclaw/OpenAI/credential", - }, - }, + }), ]); }); - it("parses custom provider mappings", () => { - expect(testing.parseProviderKeyMappings(["xai=op://openclaw/xAI/credential"])).toEqual([ - { - providerId: "xai", - secretId: "op://openclaw/xAI/credential", - }, - ]); - }); - - it("accepts native 1Password refs with spaces and encoded selectors", () => { + it("encodes native 1Password refs with spaces and selectors", async () => { const nativeRef = "op://Personal/OpenClaw QA API Key/password?attribute=value%20one"; - expect(testing.parseProviderKeyMappings([`openai=${nativeRef}`])).toEqual([ - { - providerId: "openai", - secretId: encodeOnePasswordSecretId(nativeRef), - }, - ]); + const plan = await createSetupPlan(["--provider-key", `openai=${nativeRef}`]); + expect(plan.targets[0]).toMatchObject({ + providerId: "openai", + ref: { id: encodeOnePasswordSecretId(nativeRef) }, + }); }); it.each([ - ["posix", "/tmp/plan.json", "/tmp/plan.json"], - ["posix", "/tmp/plan with spaces.json", "'/tmp/plan with spaces.json'"], - ["posix", "/tmp/plan'$(touch pwn).json", "'/tmp/plan'\\''$(touch pwn).json'"], - ["powershell", String.raw`C:\$env:TEMP\plan';.json`, String.raw`'C:\$env:TEMP\plan'';.json'`], - ["cmd", String.raw`C:\Users\Jane Doe\plan.json`, String.raw`"C:\Users\Jane Doe\plan.json"`], - ] satisfies Array<["cmd" | "posix" | "powershell", string, string]>)( - "shell-quotes %s command arguments for %j", - (shell, value, expected) => { - expect(testing.quoteCliArg(value, shell)).toBe(expected); + [ + "duplicate providers", + [ + "--openai-id", + "op://openclaw/OpenAI/credential", + "--provider-key", + "OpenAI=op://openclaw/OpenAI/other", + ], + "Duplicate model provider id", + ], + [ + "non-canonical auth-profile agent ids", + ["--target", "auth-profiles:../main:profiles.openai.key=op://openclaw/OpenAI/credential"], + "Invalid --target auth-profiles target for 1Password", + ], + [ + "traversal secret ids", + ["--provider-key", "openai=op://openclaw/../credential"], + "Invalid --provider-key openai 1Password SecretRef id", + ], + [ + "unsupported targets", + ["--target", "secrets.github_pat=op://openclaw/GitHub/pat"], + "Unknown or unsupported 1Password setup target path", + ], + [ + "duplicate target paths", + [ + "--openai-id", + "op://openclaw/OpenAI/credential", + "--target", + "models.providers.openai.apiKey=op://openclaw/OpenAI/other", + ], + "Duplicate secret target path", + ], + ["empty plans", [], "No SecretRef targets selected"], + ])("rejects %s", async (_label, args, message) => { + await expect(createSetupPlan(args)).rejects.toThrow(message); + }); + + it.each(["/absolute/path", "op://openclaw\\OpenAI\\credential", "op://vault/clé"])( + "rejects invalid 1Password ref %s", + async (id) => { + await expect(createSetupPlan(["--provider-key", `openai=${id}`])).rejects.toThrow( + "Invalid --provider-key openai 1Password SecretRef id", + ); }, ); - it("rejects line breaks in generated command arguments", () => { - expect(() => testing.quoteCliArg("plan.json\nopenclaw secrets reload", "posix")).toThrow( - /cannot contain CR or LF/, - ); - expect(() => testing.quoteCliArg("plan.json\r& whoami", "cmd")).toThrow( - /cannot contain CR or LF/, - ); + it("prints a quoted canonical plan path after the readiness command", async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-1password-setup-test-")); + const planPath = path.join(tempDir, "plan with spaces.json"); + const canonicalPlanPath = path.join(await fs.realpath(tempDir), "plan with spaces.json"); + try { + const output = await runSetup(planPath, ["--openai-id", "op://openclaw/OpenAI/credential"]); + expect(output).toContain("openclaw onepassword secretref status"); + expect(output).toContain( + `openclaw secrets apply --from '${canonicalPlanPath}' --dry-run --allow-exec`, + ); + expect(output).toContain(`openclaw secrets apply --from '${canonicalPlanPath}' --allow-exec`); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } }); - it("renders native follow-up commands for both Windows shells", () => { - expect(testing.renderApplyCommands(String.raw`C:\Users\Jane Doe\plan;.json`, "win32")).toEqual([ - "PowerShell:", - String.raw` openclaw secrets apply --from 'C:\Users\Jane Doe\plan;.json' --dry-run --allow-exec`, - String.raw` openclaw secrets apply --from 'C:\Users\Jane Doe\plan;.json' --allow-exec`, - "Command Prompt:", - String.raw` openclaw secrets apply --from "C:\Users\Jane Doe\plan;.json" --dry-run --allow-exec`, - String.raw` openclaw secrets apply --from "C:\Users\Jane Doe\plan;.json" --allow-exec`, - ]); - }); + it.skipIf(process.platform === "win32")( + "rejects plan output in a directory writable by another account", + async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-secret-plan-test-")); + const planPath = path.join(tempDir, "plan.json"); + try { + await fs.chmod(tempDir, 0o777); + await expect( + runSetup(planPath, ["--openai-id", "op://openclaw/OpenAI/credential"]), + ).rejects.toThrow("path is writable by another user"); + await expect(fs.stat(planPath)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await fs.chmod(tempDir, 0o700); + await fs.rm(tempDir, { recursive: true, force: true }); + } + }, + ); - it("omits unsafe interactive Command Prompt commands", () => { - const commands = testing.renderApplyCommands(String.raw`C:\%TEMP%\plan!.json`, "win32"); - expect(commands).toContain( - "Command Prompt: unavailable for paths containing % or !; use PowerShell.", - ); - expect(commands.filter((command) => command.includes("openclaw secrets apply"))).toHaveLength( - 2, - ); - expect(() => testing.quoteCliArg(String.raw`C:\%TEMP%\plan!.json`, "cmd")).toThrow( - /cannot safely quote/, - ); - }); - - it("parses config target mappings", () => { - expect( - testing.parseConfigTargetMappings([ - "channels.telegram.botToken=op://openclaw/Telegram/botToken", - "auth-profiles:main:profiles.openai.key=op://openclaw/OpenAI/credential", - ]), - ).toEqual([ - { - path: "channels.telegram.botToken", - secretId: "op://openclaw/Telegram/botToken", - }, - { - path: "profiles.openai.key", - agentId: "main", - secretId: "op://openclaw/OpenAI/credential", - }, - ]); - }); - - it("rejects non-canonical auth-profile agent ids", () => { - expect(() => - testing.parseConfigTargetMappings([ - "auth-profiles:../main:profiles.openai.key=op://openclaw/OpenAI/credential", - ]), - ).toThrow("Invalid --target auth-profiles target for 1Password"); - }); - - it("rejects duplicate model providers", () => { - expect(() => - testing.collectProviderSecrets({ - openaiId: "op://openclaw/OpenAI/credential", - providerKey: ["openai=op://openclaw/OpenAI/other"], - }), - ).toThrow("Duplicate model provider id in 1Password setup: openai"); - }); - - it("rejects setup plans without targets", () => { - expect(() => - testing.buildPlan({ - providerAlias: "onepassword", - providerConfig: testing.buildProviderConfig(), - providerSecrets: [], - }), - ).toThrow("No SecretRef targets selected"); - }); + it.skipIf(process.platform === "win32")( + "writes through the canonical directory instead of a replaceable alias", + async () => { + const trustedDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-secret-plan-trusted-")); + const aliasParent = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-secret-plan-alias-")); + const aliasDir = path.join(aliasParent, "output"); + const canonicalPlanPath = path.join(await fs.realpath(trustedDir), "plan.json"); + try { + await fs.symlink(trustedDir, aliasDir); + await fs.chmod(aliasParent, 0o777); + const output = await runSetup(path.join(aliasDir, "plan.json"), [ + "--openai-id", + "op://openclaw/OpenAI/credential", + ]); + expect(output).toContain(`Plan written to ${canonicalPlanPath}`); + expect(JSON.parse(await fs.readFile(canonicalPlanPath, "utf8"))).toMatchObject({ + version: 1, + }); + } finally { + await fs.chmod(aliasParent, 0o700); + await fs.rm(aliasParent, { recursive: true, force: true }); + await fs.rm(trustedDir, { recursive: true, force: true }); + } + }, + ); +}); +describe("1Password readiness", () => { it("reports trusted executable and token prerequisites without exposing the token", async () => { const resolveTrustedCli = vi.fn(async () => "/trusted/op"); const readTokenFile = vi.fn(() => "not-a-real-service-account-token"); @@ -296,10 +275,7 @@ describe("1Password CLI helpers", () => { it("reports untrusted op and unsafe token prerequisites", async () => { await expect( testing.inspectSecretRefReadiness( - { - env: { CLAW_1PASSWORD_OP: "op", PATH: "/bin" }, - tokenFile: "/missing-token", - }, + { env: { CLAW_1PASSWORD_OP: "op", PATH: "/bin" }, tokenFile: "/missing-token" }, { resolveTrustedCli: async () => { throw new Error("unsafe path detail"); @@ -318,208 +294,6 @@ describe("1Password CLI helpers", () => { prerequisitesReady: false, }); }); - - it("rejects traversal segments in SecretRef ids", () => { - expect(() => testing.parseProviderKeyMappings(["openai=op://openclaw/../credential"])).toThrow( - "Invalid --provider-key openai 1Password SecretRef id", - ); - }); - - it("rejects invalid 1Password references before encoding", () => { - for (const id of ["/absolute/path", "op://openclaw\\OpenAI\\credential", "op://vault/clé"]) { - expect(() => testing.parseProviderKeyMappings([`openai=${id}`])).toThrow( - "Invalid --provider-key openai 1Password SecretRef id", - ); - } - }); - - it("rejects unsupported config target paths", () => { - expect(() => - testing.buildPlan({ - providerAlias: "onepassword", - providerConfig: testing.buildProviderConfig(), - providerSecrets: [], - configTargetSecrets: [ - { - path: "secrets.github_pat", - secretId: "op://openclaw/GitHub/pat", - }, - ], - }), - ).toThrow("Unknown or unsupported 1Password setup target path: secrets.github_pat"); - }); - - it("rejects duplicate config target paths", () => { - expect(() => - testing.buildPlan({ - providerAlias: "onepassword", - providerConfig: testing.buildProviderConfig(), - providerSecrets: [ - { - providerId: "openai", - secretId: "op://openclaw/OpenAI/credential", - }, - ], - configTargetSecrets: [ - { - path: "models.providers.openai.apiKey", - secretId: "op://openclaw/OpenAI/other", - }, - ], - }), - ).toThrow("Duplicate secret target path in 1Password setup: models.providers.openai.apiKey"); - }); - - it("creates plan files exclusively with owner-only permissions", async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-1password-plan-test-")); - const planPath = path.join(tempDir, "plan.json"); - const plan = createOpenAiPlan(); - try { - await testing.writePlanFile(plan, planPath); - if (process.platform !== "win32") { - expect((await fs.stat(planPath)).mode & 0o777).toBe(0o600); - } else { - const permissions = await inspectPathPermissions(planPath); - expect(permissions).toMatchObject({ - ok: true, - source: "windows-acl", - ownerTrusted: true, - groupReadable: false, - groupWritable: false, - worldReadable: false, - worldWritable: false, - }); - } - await expect(testing.writePlanFile(plan, planPath)).rejects.toThrow( - "Plan path already exists", - ); - - const symlinkPath = path.join(tempDir, "symlink.json"); - await fs.symlink(planPath, symlinkPath); - await expect(testing.writePlanFile(plan, symlinkPath)).rejects.toThrow( - "Plan path already exists", - ); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } - }); - - it.skipIf(process.platform === "win32")( - "rejects plan output in a directory writable by another account", - async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-1password-plan-test-")); - const planPath = path.join(tempDir, "plan.json"); - const plan = createOpenAiPlan(); - try { - await fs.chmod(tempDir, 0o777); - await expect(testing.writePlanFile(plan, planPath)).rejects.toThrow( - "path is writable by another user", - ); - await expect(fs.stat(planPath)).rejects.toMatchObject({ code: "ENOENT" }); - } finally { - await fs.chmod(tempDir, 0o700); - await fs.rm(tempDir, { recursive: true, force: true }); - } - }, - ); - - it.skipIf(process.platform === "win32")( - "writes through the canonical directory instead of a replaceable alias", - async () => { - const trustedDir = await fs.mkdtemp( - path.join(os.tmpdir(), "openclaw-1password-plan-trusted-"), - ); - const aliasParent = await fs.mkdtemp( - path.join(os.tmpdir(), "openclaw-1password-plan-alias-"), - ); - const aliasDir = path.join(aliasParent, "output"); - const canonicalPlanPath = path.join(await fs.realpath(trustedDir), "plan.json"); - const plan = createOpenAiPlan(); - try { - await fs.symlink(trustedDir, aliasDir); - await fs.chmod(aliasParent, 0o777); - await expect(testing.writePlanFile(plan, path.join(aliasDir, "plan.json"))).resolves.toBe( - canonicalPlanPath, - ); - expect(JSON.parse(await fs.readFile(canonicalPlanPath, "utf8"))).toMatchObject({ - version: 1, - }); - } finally { - await fs.chmod(aliasParent, 0o700); - await fs.rm(aliasParent, { recursive: true, force: true }); - await fs.rm(trustedDir, { recursive: true, force: true }); - } - }, - ); - - it.skipIf(process.platform === "win32")( - "rejects unrenderable plan paths before creating a file", - async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-1password-plan-test-")); - const planPath = path.join(tempDir, "plan\n.json"); - const plan = createOpenAiPlan(); - try { - await expect(testing.writePlanFile(plan, planPath)).rejects.toThrow( - "Command argument cannot contain CR or LF", - ); - await expect(fs.stat(planPath)).rejects.toMatchObject({ code: "ENOENT" }); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } - }, - ); - - it("writes a Windows plan through the atomic private-file primitive", async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-1password-plan-test-")); - const planPath = path.join(tempDir, "plan.json"); - const plan = createOpenAiPlan(); - const createPrivateWindowsFile = vi.fn(async (filePath: string, content: string) => { - await fs.writeFile(filePath, content, { flag: "wx" }); - }); - const resolveTrustedPlanDirectory = vi.fn(async (directoryPath: string) => directoryPath); - try { - await testing.writePlanFile(plan, planPath, { - platform: "win32", - createPrivateWindowsFile, - resolveTrustedPlanDirectory, - }); - expect(resolveTrustedPlanDirectory).toHaveBeenCalledWith(path.resolve(tempDir)); - expect(createPrivateWindowsFile).toHaveBeenCalledWith(planPath, expect.any(String)); - expect(JSON.parse(await fs.readFile(planPath, "utf8"))).toMatchObject({ version: 1 }); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } - }); - - it("prints the readiness check before plan application", async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-1password-setup-test-")); - const planPath = path.join(tempDir, "plan with spaces.json"); - const canonicalPlanPath = path.join(await fs.realpath(tempDir), "plan with spaces.json"); - const output = captureStdout(); - try { - await createProgram({}).parseAsync( - [ - "onepassword", - "secretref", - "setup", - "--openai-id", - "op://openclaw/OpenAI/credential", - "--plan-out", - planPath, - ], - { from: "user" }, - ); - expect(output()).toContain("openclaw onepassword secretref status"); - expect(output()).toContain( - `openclaw secrets apply --from '${canonicalPlanPath}' --dry-run --allow-exec`, - ); - expect(output()).toContain( - `openclaw secrets apply --from '${canonicalPlanPath}' --allow-exec`, - ); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } - }); }); describe("1Password CLI status", () => { @@ -534,8 +308,8 @@ describe("1Password CLI status", () => { }, }, }); - expect(result.providerAlias).toBe("corp-onepassword"); expect(result).toMatchObject({ + providerAlias: "corp-onepassword", providerReady: true, opStatus: "not-found", tokenFileStatus: "missing-or-unsafe", @@ -557,8 +331,7 @@ describe("1Password CLI status", () => { }, }, }); - expect(result.providerAlias).toBe("corp-onepassword"); - expect(result.providerReady).toBe(true); + expect(result).toMatchObject({ providerAlias: "corp-onepassword", providerReady: true }); }); it("requires an explicit alias when multiple providers are configured", async () => { diff --git a/extensions/onepassword/src/secret-ref-cli.ts b/extensions/onepassword/src/secret-ref-cli.ts index 560cdfe6f001..7d3ec38c7bd1 100644 --- a/extensions/onepassword/src/secret-ref-cli.ts +++ b/extensions/onepassword/src/secret-ref-cli.ts @@ -1,52 +1,46 @@ import { randomUUID } from "node:crypto"; import path from "node:path"; -import { createInterface } from "node:readline/promises"; -import { isRecord } from "openclaw/plugin-sdk/channel-secret-basic-runtime"; import type { OpenClawConfig } from "openclaw/plugin-sdk/plugin-entry"; import { DEFAULT_SECRET_FILE_MAX_BYTES, tryReadSecretFileSync, } from "openclaw/plugin-sdk/secret-file-runtime"; -import { pluginSecretRefSetup } from "openclaw/plugin-sdk/secret-ref-runtime"; +import { createPluginSecretRefSetupCli } from "openclaw/plugin-sdk/secret-ref-runtime"; +import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; -import { - resolveTrustedOnePasswordCli, - resolveTrustedOnePasswordDirectoryPath, -} from "../onepassword-op-path.js"; +import { resolveTrustedOnePasswordCli } from "../onepassword-op-path.js"; import { encodeOnePasswordSecretId } from "../onepassword-secret-id.js"; -type CommandLike = { - command(name: string): CommandLike; - description(value: string): CommandLike; - option( - flags: string, - description: string, - defaultValueOrParser?: string | ((value: string, previous?: string[]) => string[]), - defaultValue?: string[], - ): CommandLike; - action(fn: (options: TOptions) => void | Promise): CommandLike; -}; +const ONEPASSWORD_PROVIDER_ALIAS = "onepassword"; -type OnePasswordExecProviderConfig = { - source: "exec"; +function normalizeOnePasswordSecretId(label: string, value: string): string { + try { + return encodeOnePasswordSecretId(value); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error(`Invalid ${label} 1Password SecretRef id: ${detail}`, { cause: error }); + } +} + +const onePasswordSecretRefSetupCli = createPluginSecretRefSetupCli({ + productName: "1Password", + secretIdLabel: "1Password SecretRef id", + secretIdPlaceholder: "1password-secret-id", + defaultProviderAlias: ONEPASSWORD_PROVIDER_ALIAS, pluginIntegration: { - pluginId: "onepassword"; - integrationId: "onepassword"; - }; -}; + pluginId: "onepassword", + integrationId: "onepassword", + }, + normalizeSecretId: normalizeOnePasswordSecretId, + defaultPlanPath: () => + path.join(resolvePreferredOpenClawTmpDir(), `openclaw-1password-secrets-${randomUUID()}.json`), + beforeApplyCommands: [ + "openclaw plugins enable onepassword", + "openclaw onepassword secretref status", + ], +}); -type ProviderSecretMapping = { - providerId: string; - secretId: string; -}; - -type ConfigTargetSecretMapping = { - path: string; - agentId?: string; - secretId: string; -}; - -type SecretsApplyPlan = ReturnType; +type CommandLike = Parameters[0]; type RegisterOnePasswordSecretRefCommandsParams = { command: CommandLike; @@ -60,26 +54,6 @@ type StatusOptions = { providerAlias?: string; }; -type SetupOptions = { - planOut?: string; - providerAlias?: string; - openaiId?: string; - anthropicId?: string; - openrouterId?: string; - providerKey?: string[]; - target?: string[]; -}; - -type ProviderStatus = { - configured: boolean; - source?: string; - command?: string; - pluginIntegration?: { - pluginId: string; - integrationId: string; - }; -}; - type SecretRefReadiness = { opCommand: string; opBinaryPath: string | null; @@ -94,14 +68,6 @@ type ReadinessDependencies = { readTokenFile?: (filePath: string) => string | undefined; }; -type WritePlanFileDependencies = { - platform?: NodeJS.Platform; - createPrivateWindowsFile?: (filePath: string, content: string) => Promise; - resolveTrustedPlanDirectory?: typeof resolveTrustedOnePasswordDirectoryPath; -}; - -const ONEPASSWORD_PROVIDER_ALIAS = "onepassword"; - function writeLine(message = ""): void { process.stdout.write(`${message}\n`); } @@ -110,123 +76,6 @@ function writeJson(value: unknown): void { process.stdout.write(`${JSON.stringify(value, null, 2)}\n`); } -function normalizeOptionalString(value: unknown): string | undefined { - return typeof value === "string" && value.trim() ? value.trim() : undefined; -} - -type CommandShell = "cmd" | "posix" | "powershell"; - -function quoteCliArg(value: string, shell: CommandShell): string { - if (/\r|\n/u.test(value)) { - throw new Error("Command argument cannot contain CR or LF"); - } - if (shell === "cmd") { - if (/[%!]/u.test(value)) { - throw new Error("Interactive Command Prompt cannot safely quote paths containing % or !"); - } - const escaped = value.replaceAll('"', '\\"'); - return /[ \t"&|<>^()]/u.test(value) ? `"${escaped}"` : escaped || '""'; - } - if (shell === "powershell") { - return `'${value.replaceAll("'", "''")}'`; - } - if (/^[A-Za-z0-9_/:=.,@%+-]+$/.test(value)) { - return value; - } - return `'${value.replaceAll("'", "'\\''")}'`; -} - -function renderApplyCommands( - planPath: string, - platform: NodeJS.Platform = process.platform, -): string[] { - const render = (shell: CommandShell, extraIndent = "") => { - const quotedPlanPath = quoteCliArg(planPath, shell); - return [ - `${extraIndent}openclaw secrets apply --from ${quotedPlanPath} --dry-run --allow-exec`, - `${extraIndent}openclaw secrets apply --from ${quotedPlanPath} --allow-exec`, - ]; - }; - if (platform !== "win32") { - return render("posix"); - } - // Windows cannot reveal which parent shell will receive these copy-paste commands. - // Print native variants instead of emitting syntax that is unsafe in the other shell. - const powershellCommands = ["PowerShell:", ...render("powershell", " ")]; - if (/[%!]/u.test(planPath)) { - return [ - ...powershellCommands, - "Command Prompt: unavailable for paths containing % or !; use PowerShell.", - ]; - } - return [...powershellCommands, "Command Prompt:", ...render("cmd", " ")]; -} - -function assertValidProviderAlias(value: string): void { - pluginSecretRefSetup.assertValidProviderAlias(value); -} - -function normalizeOnePasswordSecretId(label: string, value: string): string { - try { - return encodeOnePasswordSecretId(value); - } catch (error) { - const detail = error instanceof Error ? error.message : String(error); - throw new Error(`Invalid ${label} 1Password SecretRef id: ${detail}`, { cause: error }); - } -} - -function readProviderStatus(config: OpenClawConfig, providerAlias: string): ProviderStatus { - const provider = config.secrets?.providers?.[providerAlias]; - if (!isRecord(provider)) { - return { configured: false }; - } - const base = { - configured: true, - source: normalizeOptionalString(provider.source), - }; - if (provider.source !== "exec") { - return base; - } - if ("pluginIntegration" in provider) { - return { - ...base, - pluginIntegration: provider.pluginIntegration as ProviderStatus["pluginIntegration"], - }; - } - return { - ...base, - command: normalizeOptionalString(provider.command), - }; -} - -function isOnePasswordIntegrationProvider(value: unknown): boolean { - if (!isRecord(value) || value.source !== "exec" || !isRecord(value.pluginIntegration)) { - return false; - } - return ( - value.pluginIntegration.pluginId === "onepassword" && - value.pluginIntegration.integrationId === "onepassword" - ); -} - -function resolveStatusProviderAlias(config: OpenClawConfig, requestedAlias?: string): string { - const explicitAlias = normalizeOptionalString(requestedAlias); - if (explicitAlias) { - assertValidProviderAlias(explicitAlias); - return explicitAlias; - } - const configuredAliases = Object.entries(config.secrets?.providers ?? {}) - .filter(([, provider]) => isOnePasswordIntegrationProvider(provider)) - .map(([alias]) => alias) - .toSorted(); - if (configuredAliases.length > 1) { - throw new Error( - `Multiple 1Password provider aliases are configured (${configuredAliases.join(", ")}). Use --provider-alias .`, - ); - } - return configuredAliases[0] ?? ONEPASSWORD_PROVIDER_ALIAS; -} - async function inspectSecretRefReadiness( params: { env: NodeJS.ProcessEnv; tokenFile: string }, dependencies: ReadinessDependencies = {}, @@ -275,152 +124,13 @@ async function inspectSecretRefReadiness( }; } -function buildProviderConfig(): OnePasswordExecProviderConfig { - return { - source: "exec", - pluginIntegration: { - pluginId: "onepassword", - integrationId: "onepassword", - }, - }; -} - -function parseTargetSpecifier(value: string): { - path: string; - agentId?: string; -} { - return pluginSecretRefSetup.parseTargetSpecifier("1Password", value); -} - -function parseProviderKeyMappings(values: string[] | undefined): ProviderSecretMapping[] { - return (values ?? []).map((value) => { - const separator = value.indexOf("="); - if (separator <= 0 || separator === value.length - 1) { - throw new Error( - `Invalid --provider-key value "${value}". Use =<1password-secret-id>.`, - ); - } - const providerId = value.slice(0, separator).trim(); - pluginSecretRefSetup.assertValidModelProviderId("--provider-key", providerId); - const secretId = normalizeOnePasswordSecretId( - `--provider-key ${providerId}`, - value.slice(separator + 1).trim(), - ); - return { providerId, secretId }; - }); -} - -function parseConfigTargetMappings(values: string[] | undefined): ConfigTargetSecretMapping[] { - return (values ?? []).map((value) => { - const separator = value.indexOf("="); - if (separator <= 0 || separator === value.length - 1) { - throw new Error( - `Invalid --target value "${value}". Use =<1password-secret-id>.`, - ); - } - const target = parseTargetSpecifier(value.slice(0, separator).trim()); - const secretId = normalizeOnePasswordSecretId( - `--target ${target.path}`, - value.slice(separator + 1).trim(), - ); - return Object.assign( - { path: target.path, secretId }, - target.agentId ? { agentId: target.agentId } : {}, - ); - }); -} - -function collectProviderSecrets(options: { - openaiId?: string; - anthropicId?: string; - openrouterId?: string; - providerKey?: string[]; -}): ProviderSecretMapping[] { - const providerSecrets: ProviderSecretMapping[] = []; - if (options.openaiId) { - providerSecrets.push({ providerId: "openai", secretId: options.openaiId }); - } - if (options.anthropicId) { - providerSecrets.push({ providerId: "anthropic", secretId: options.anthropicId }); - } - if (options.openrouterId) { - providerSecrets.push({ providerId: "openrouter", secretId: options.openrouterId }); - } - providerSecrets.push(...parseProviderKeyMappings(options.providerKey)); - - const seen = new Set(); - for (const entry of providerSecrets) { - const normalized = entry.providerId.toLowerCase(); - if (seen.has(normalized)) { - throw new Error(`Duplicate model provider id in 1Password setup: ${entry.providerId}`); - } - seen.add(normalized); - } - return providerSecrets; -} - -function buildPlan(params: { - providerAlias: string; - providerConfig: OnePasswordExecProviderConfig; - providerSecrets: ProviderSecretMapping[]; - configTargetSecrets?: ConfigTargetSecretMapping[]; -}): SecretsApplyPlan { - const plan = pluginSecretRefSetup.buildPlan({ productName: "1Password", ...params }); - if (plan.targets.length === 0) { - throw new Error( - "No SecretRef targets selected. Pass --openai-id, --anthropic-id, --openrouter-id, --provider-key, or --target.", - ); - } - return plan; -} - -async function promptOptionalSecretId(label: string): Promise { - if (!process.stdin.isTTY || !process.stdout.isTTY) { - return undefined; - } - const rl = createInterface({ input: process.stdin, output: process.stdout }); - try { - return normalizeOptionalString( - await rl.question(`${label} 1Password SecretRef id (blank to skip): `), - ); - } finally { - rl.close(); - } -} - -async function promptProviderSecrets(options: SetupOptions): Promise { - const openaiId = - normalizeOptionalString(options.openaiId) ?? (await promptOptionalSecretId("OpenAI")); - const anthropicId = - normalizeOptionalString(options.anthropicId) ?? (await promptOptionalSecretId("Anthropic")); - const openrouterId = - normalizeOptionalString(options.openrouterId) ?? (await promptOptionalSecretId("OpenRouter")); - const normalizedOpenaiId = openaiId - ? normalizeOnePasswordSecretId("OpenAI", openaiId) - : undefined; - const normalizedAnthropicId = anthropicId - ? normalizeOnePasswordSecretId("Anthropic", anthropicId) - : undefined; - const normalizedOpenrouterId = openrouterId - ? normalizeOnePasswordSecretId("OpenRouter", openrouterId) - : undefined; - return collectProviderSecrets({ - ...(normalizedOpenaiId ? { openaiId: normalizedOpenaiId } : {}), - ...(normalizedAnthropicId ? { anthropicId: normalizedAnthropicId } : {}), - ...(normalizedOpenrouterId ? { openrouterId: normalizedOpenrouterId } : {}), - providerKey: options.providerKey, - }); -} - async function runStatus( params: RegisterOnePasswordSecretRefCommandsParams, options: StatusOptions, ): Promise { - const config = params.config; - const providerAlias = resolveStatusProviderAlias(config, options.providerAlias); - const provider = readProviderStatus(config, providerAlias); - const providerReady = isOnePasswordIntegrationProvider( - config.secrets?.providers?.[providerAlias], + const { providerAlias, provider, providerReady } = onePasswordSecretRefSetupCli.inspectProvider( + params.config, + options.providerAlias, ); const readiness = await inspectSecretRefReadiness({ env: params.env ?? process.env, @@ -470,7 +180,7 @@ async function runStatus( if (issues.length === 0) { return; } - writeLine(""); + writeLine(); writeLine("Next actions:"); if (!providerReady) { writeLine(" Generate and apply a 1Password SecretRef setup plan."); @@ -485,60 +195,6 @@ async function runStatus( } } -async function writePlanFile( - plan: SecretsApplyPlan, - requestedPath?: string, - dependencies: WritePlanFileDependencies = {}, -): Promise { - const requestedPlanPath = - normalizeOptionalString(requestedPath) ?? - path.join(resolvePreferredOpenClawTmpDir(), `openclaw-1password-secrets-${randomUUID()}.json`); - const content = `${JSON.stringify(plan, null, 2)}\n`; - const requestedPlanPathAbsolute = path.resolve(requestedPlanPath); - const planDirectory = await ( - dependencies.resolveTrustedPlanDirectory ?? resolveTrustedOnePasswordDirectoryPath - )(path.dirname(requestedPlanPathAbsolute)); - // Write through the canonical directory returned by the trust check. Reusing the requested - // alias would let another local account retarget a writable parent symlink after validation. - const planPath = path.join(planDirectory, path.basename(requestedPlanPathAbsolute)); - const platform = dependencies.platform ?? process.platform; - // Validate the exact canonical path before the exclusive write. Follow-up command rendering - // must not fail after leaving a plan behind that the next setup attempt cannot overwrite. - renderApplyCommands(planPath, platform); - await pluginSecretRefSetup.writePlanFile({ - planPath, - content, - platform, - createPrivateWindowsFile: dependencies.createPrivateWindowsFile, - }); - return planPath; -} - -async function runSetup(options: SetupOptions): Promise { - const providerAlias = - normalizeOptionalString(options.providerAlias) ?? ONEPASSWORD_PROVIDER_ALIAS; - assertValidProviderAlias(providerAlias); - const providerSecrets = await promptProviderSecrets(options); - const plan = buildPlan({ - providerAlias, - providerConfig: buildProviderConfig(), - providerSecrets, - configTargetSecrets: parseConfigTargetMappings(options.target), - }); - const planPath = await writePlanFile(plan, options.planOut); - writeLine(`Plan written to ${planPath}`); - writeLine(`Targets: ${plan.targets.length}`); - writeLine(""); - writeLine("Next steps:"); - writeLine(" openclaw plugins enable onepassword"); - writeLine(" openclaw onepassword secretref status"); - for (const command of renderApplyCommands(planPath)) { - writeLine(` ${command}`); - } - writeLine(" openclaw secrets audit --check --allow-exec"); - writeLine(" openclaw secrets reload"); -} - export function registerOnePasswordSecretRefCommands( params: RegisterOnePasswordSecretRefCommandsParams, ): void { @@ -549,41 +205,7 @@ export function registerOnePasswordSecretRefCommands( .option("--json", "Print JSON status") .option("--provider-alias ", "Secret provider alias to inspect") .action((options: StatusOptions) => runStatus(params, options)); - secretRef - .command("setup") - .description("Create a 1Password SecretRef setup plan") - .option("--plan-out ", "Write the generated secrets apply plan to a path") - .option( - "--provider-alias ", - "Secret provider alias to configure", - ONEPASSWORD_PROVIDER_ALIAS, - ) - .option("--openai-id ", "1Password SecretRef id for models.providers.openai.apiKey") - .option("--anthropic-id ", "1Password SecretRef id for models.providers.anthropic.apiKey") - .option("--openrouter-id ", "1Password SecretRef id for models.providers.openrouter.apiKey") - .option( - "--provider-key ", - "1Password SecretRef id for any models.providers..apiKey target", - (value: string, previous: string[] = []) => [...previous, value], - [], - ) - .option( - "--target ", - "1Password SecretRef id for any known SecretRef target path", - (value: string, previous: string[] = []) => [...previous, value], - [], - ) - .action((options: SetupOptions) => runSetup(options)); + onePasswordSecretRefSetupCli.registerSetupCommand(secretRef); } -export const testing = { - buildPlan, - buildProviderConfig, - collectProviderSecrets, - parseConfigTargetMappings, - parseProviderKeyMappings, - quoteCliArg, - renderApplyCommands, - inspectSecretRefReadiness, - writePlanFile, -}; +export const testing = { inspectSecretRefReadiness }; diff --git a/extensions/qa-lab/web/src/app.browser.test.ts b/extensions/qa-lab/web/src/app.browser.test.ts index 0ab4dd5c950d..e43f6914067f 100644 --- a/extensions/qa-lab/web/src/app.browser.test.ts +++ b/extensions/qa-lab/web/src/app.browser.test.ts @@ -1,8 +1,9 @@ /* @vitest-environment jsdom */ import { readFileSync } from "node:fs"; import path from "node:path"; +import type { QaBusStateSnapshot } from "openclaw/plugin-sdk/qa-channel-protocol"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { Bootstrap, RunnerSelection, Snapshot } from "./ui-types.js"; +import type { Bootstrap, RunnerSelection } from "./ui-types.js"; const httpMock = vi.hoisted(() => { class QaLabHttpError extends Error { @@ -105,7 +106,13 @@ function createBootstrap(selection: RunnerSelection): Bootstrap { async function mountRunner( selection: RunnerSelection, - snapshot: Snapshot = { conversations: [], events: [], messages: [], threads: [] }, + snapshot: QaBusStateSnapshot = { + conversations: [], + cursor: 0, + events: [], + messages: [], + threads: [], + }, ) { let bootstrap = createBootstrap(selection); httpMock.getJson.mockImplementation(async (url: string) => { @@ -215,12 +222,15 @@ describe("QA Lab runner browser interactions", () => { }, { conversations: [{ accountId: "default", id: "qa-room", kind: "channel" }], + cursor: 0, events: [], messages: [], threads: [ { accountId: "default", conversationId: "qa-room", + createdAt: 0, + createdBy: "qa-operator", id: "owned-thread", title: "Owned thread", }, diff --git a/extensions/qa-lab/web/src/app.ts b/extensions/qa-lab/web/src/app.ts index 56cc8c7ac539..7dec06f54e1a 100644 --- a/extensions/qa-lab/web/src/app.ts +++ b/extensions/qa-lab/web/src/app.ts @@ -1,4 +1,5 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import type { QaBusStateSnapshot } from "openclaw/plugin-sdk/qa-channel-protocol"; // Qa Lab plugin module implements app behavior. import { defaultQaModelForMode, isQaFastModeEnabled } from "../../model-selection.js"; import { normalizeCaptureSavedView, normalizeCaptureSavedViews } from "./capture-saved-view.js"; @@ -11,7 +12,6 @@ import { type ReportEnvelope, type RunnerResolvedPlan, type RunnerSelection, - type Snapshot, type TabId, type CaptureEventsEnvelope, type CaptureCoverageEnvelope, @@ -342,7 +342,7 @@ export async function createQaLabApp(root: HTMLDivElement) { try { const [bootstrap, snapshot, report, outcomes] = await Promise.all([ getJson("/api/bootstrap"), - getJson("/api/state"), + getJson("/api/state"), getJson("/api/report"), getJson("/api/outcomes"), ]); diff --git a/extensions/qa-lab/web/src/ui-conversation-key.ts b/extensions/qa-lab/web/src/ui-conversation-key.ts index 2b14a249ff2c..c7a46878de01 100644 --- a/extensions/qa-lab/web/src/ui-conversation-key.ts +++ b/extensions/qa-lab/web/src/ui-conversation-key.ts @@ -1,6 +1,10 @@ -import type { Conversation, Message, Thread } from "./ui-types.js"; +import type { + QaBusMessage, + QaBusSnapshotConversation, + QaBusThread, +} from "openclaw/plugin-sdk/qa-channel-protocol"; -type ConversationIdentity = Pick; +type ConversationIdentity = Pick; // Raw ids can collide across accounts and conversation kinds. Keep one key // shape for sidebar selection, transcript filtering, and thread navigation. @@ -9,9 +13,9 @@ export function conversationSelectionKey(identity: ConversationIdentity): string } export function findConversationBySelectionKey( - conversations: Conversation[], + conversations: QaBusSnapshotConversation[], selectionKey: string | null, -): Conversation | undefined { +): QaBusSnapshotConversation | undefined { if (!selectionKey) { return undefined; } @@ -20,7 +24,7 @@ export function findConversationBySelectionKey( ); } -export function messageConversationSelectionKey(message: Message): string { +export function messageConversationSelectionKey(message: QaBusMessage): string { return conversationSelectionKey({ accountId: message.accountId, id: message.conversation.id, @@ -28,7 +32,7 @@ export function messageConversationSelectionKey(message: Message): string { }); } -export function threadConversationSelectionKey(thread: Thread): string { +export function threadConversationSelectionKey(thread: QaBusThread): string { // QA bus thread records come only from channel-scoped createThread; direct // message thread ids do not create sidebar thread records. return conversationSelectionKey({ diff --git a/extensions/qa-lab/web/src/ui-render-content.ts b/extensions/qa-lab/web/src/ui-render-content.ts index 8f4d83b02dd3..3db5751c175e 100644 --- a/extensions/qa-lab/web/src/ui-render-content.ts +++ b/extensions/qa-lab/web/src/ui-render-content.ts @@ -1,3 +1,8 @@ +import type { + QaBusAttachment, + QaBusMessage, + QaBusSnapshotConversation, +} from "openclaw/plugin-sdk/qa-channel-protocol"; import { conversationSelectionKey, findConversationBySelectionKey, @@ -6,9 +11,9 @@ import { } from "./ui-conversation-key.js"; import { findScenarioOutcome } from "./ui-render-scenario.js"; import { badgeHtml, esc, formatIso, formatTime } from "./ui-render-utils.js"; -import type { Attachment, Conversation, Message, SeedScenario, UiState } from "./ui-types.js"; +import type { SeedScenario, UiState } from "./ui-types.js"; -function attachmentSourceUrl(attachment: Attachment): string | null { +function attachmentSourceUrl(attachment: QaBusAttachment): string | null { if (attachment.url?.trim()) { return attachment.url; } @@ -18,7 +23,7 @@ function attachmentSourceUrl(attachment: Attachment): string | null { return null; } -function renderMessageAttachments(message: Message): string { +function renderMessageAttachments(message: QaBusMessage): string { const attachments = message.attachments ?? []; if (attachments.length === 0) { return ""; @@ -91,8 +96,8 @@ function filteredMessages(state: UiState) { } function formatConversationLabel( - conversation: Conversation, - conversations: Conversation[], + conversation: QaBusSnapshotConversation, + conversations: QaBusSnapshotConversation[], ): string { const label = conversation.title || conversation.id; const sidebarCollisions = conversations.filter( @@ -239,14 +244,14 @@ export function renderChatView(state: UiState): string { `; } -function messageAvatar(m: Message): { emoji: string; bg: string; role: string } { +function messageAvatar(m: QaBusMessage): { emoji: string; bg: string; role: string } { if (m.direction === "outbound") { return { emoji: "\uD83E\uDD80", bg: "#7c6cff", role: "Claw" }; // 🦀 } return { emoji: "\uD83E\uDD9E", bg: "#d97706", role: "Clawfather" }; // 🦞 } -function renderMessage(m: Message): string { +function renderMessage(m: QaBusMessage): string { const name = m.senderName || m.senderId; const avatar = messageAvatar(m); const dirClass = m.direction === "inbound" ? "msg-direction-inbound" : "msg-direction-outbound"; @@ -288,7 +293,7 @@ function recentInspectorMessages(state: UiState, limit = 18) { return (state.snapshot?.messages ?? []).slice(-limit).toReversed(); } -function renderInspectorLiveMessage(message: Message): string { +function renderInspectorLiveMessage(message: QaBusMessage): string { const avatar = messageAvatar(message); const conversationLabel = message.conversation.title || message.conversation.id; const threadLabel = message.threadTitle || message.threadId; @@ -483,9 +488,7 @@ export function renderEventsView(state: UiState): string { const detail = "thread" in e ? `${e.thread.conversationId}/${e.thread.id}` - : e.message - ? `${e.message.senderId}: ${e.message.text}` - : ""; + : `${e.message.senderId}: ${e.message.text}`; return `
${esc(e.kind)} diff --git a/extensions/qa-lab/web/src/ui-render.test.ts b/extensions/qa-lab/web/src/ui-render.test.ts index ca85a0b2dde8..193c8a700c26 100644 --- a/extensions/qa-lab/web/src/ui-render.test.ts +++ b/extensions/qa-lab/web/src/ui-render.test.ts @@ -97,6 +97,7 @@ describe("QA Lab UI evidence render", () => { { accountId: "account-b", id: "shared", kind: "channel" }, { accountId: "account-a", id: "shared", kind: "direct" }, ], + cursor: 0, events: [], messages: [ { @@ -134,12 +135,16 @@ describe("QA Lab UI evidence render", () => { { accountId: "account-a", conversationId: "shared", + createdAt: 0, + createdBy: "openclaw", id: "selected-thread", title: "Selected thread", }, { accountId: "account-b", conversationId: "shared", + createdAt: 0, + createdBy: "openclaw", id: "foreign-thread", title: "Foreign thread", }, @@ -167,6 +172,7 @@ describe("QA Lab UI evidence render", () => { { accountId: "account-a", id: "shared", kind: "group" }, { accountId: "account-b", id: "shared", kind: "channel" }, ], + cursor: 0, events: [], messages: [], threads: [], @@ -197,6 +203,7 @@ describe("QA Lab UI evidence render", () => { { accountId: "account-a", id: "shared", kind: "channel" }, { accountId: "account-a", id: "shared", kind: "direct" }, ], + cursor: 0, events: [], messages: [ { @@ -251,6 +258,7 @@ describe("QA Lab UI evidence render", () => { const selectedConversationKey = JSON.stringify(["default", "channel", "qa-room"]); const snapshot: NonNullable = { conversations: [{ accountId: "default", id: "qa-room", kind: "channel" }], + cursor: 0, events: [], messages: [ { @@ -290,6 +298,8 @@ describe("QA Lab UI evidence render", () => { { accountId: "default", conversationId: "qa-room", + createdAt: 0, + createdBy: "openclaw", id: "owned-thread", title: "Owned thread", }, diff --git a/extensions/qa-lab/web/src/ui-types.ts b/extensions/qa-lab/web/src/ui-types.ts index 5c86096abecf..4b72a27ffe3f 100644 --- a/extensions/qa-lab/web/src/ui-types.ts +++ b/extensions/qa-lab/web/src/ui-types.ts @@ -1,3 +1,7 @@ +import type { + QaBusConversationKind, + QaBusStateSnapshot, +} from "openclaw/plugin-sdk/qa-channel-protocol"; import type { QaLabExecutionKind, QaLabResolvedRunPlan, @@ -13,65 +17,6 @@ import type { QaEvidenceProducerContextFile, } from "../../shared/evidence-gallery-types.js"; -/* ===== Shared types (unchanged from the bus protocol) ===== */ - -export type Conversation = { - accountId: string; - id: string; - kind: "direct" | "channel" | "group"; - title?: string; -}; - -export type Attachment = { - id: string; - kind: "image" | "video" | "audio" | "file"; - mimeType: string; - fileName?: string; - inline?: boolean; - url?: string; - contentBase64?: string; - width?: number; - height?: number; - durationMs?: number; - altText?: string; - transcript?: string; -}; - -export type Thread = { - accountId: string; - id: string; - conversationId: string; - title: string; -}; - -export type Message = { - accountId: string; - id: string; - direction: "inbound" | "outbound"; - conversation: Omit; - senderId: string; - senderName?: string; - text: string; - timestamp: number; - threadId?: string; - threadTitle?: string; - deleted?: boolean; - editedAt?: number; - attachments?: Attachment[]; - reactions: Array<{ emoji: string; senderId: string }>; -}; - -type BusEvent = - | { cursor: number; kind: "thread-created"; thread: Thread } - | { cursor: number; kind: string; message?: Message; emoji?: string }; - -export type Snapshot = { - conversations: Conversation[]; - threads: Thread[]; - messages: Message[]; - events: BusEvent[]; -}; - export type ReportEnvelope = { report: null | { outputPath: string; @@ -300,7 +245,7 @@ export type TabId = "chat" | "results" | "report" | "events" | "capture" | "evid export type UiState = { theme: "light" | "dark"; bootstrap: Bootstrap | null; - snapshot: Snapshot | null; + snapshot: QaBusStateSnapshot | null; latestReport: ReportEnvelope["report"]; scenarioRun: ScenarioRun | null; captureSessions: CaptureSessionSummary[]; @@ -371,7 +316,7 @@ export type UiState = { runnerDraftDirty: boolean; runnerPlanOverride: RunnerResolvedPlan | null; composer: { - conversationKind: "direct" | "channel" | "group"; + conversationKind: QaBusConversationKind; conversationId: string; senderId: string; senderName: string; diff --git a/extensions/vault/src/cli.test.ts b/extensions/vault/src/cli.test.ts index 8b8fadd8c000..56109cb7f3b4 100644 --- a/extensions/vault/src/cli.test.ts +++ b/extensions/vault/src/cli.test.ts @@ -40,12 +40,13 @@ async function createSetupPlan(args: string[]): Promise { } } -async function runSetup(planPath: string, args: string[]): Promise { +async function runSetup(planPath: string, args: string[]): Promise { const stdout = captureStdout(); try { await createProgram().parseAsync(["vault", "setup", "--plan-out", planPath, ...args], { from: "user", }); + return stdout.output(); } finally { stdout.restore(); } @@ -163,6 +164,7 @@ describe("vault CLI setup plan", () => { }); it.each([ + ["empty plans", [], "No SecretRef targets selected"], [ "duplicate providers", ["--openai-id", "providers/openai/apiKey", "--provider-key", "OpenAI=providers/openai/other"], @@ -197,6 +199,21 @@ describe("vault CLI setup plan", () => { await expect(createSetupPlan(args)).rejects.toThrow(message); }); + it("prints shell-safe commands using the canonical plan path", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-vault-command-")); + const planPath = path.join(dir, "plan with spaces.json"); + const canonicalPlanPath = path.join(await fs.realpath(dir), "plan with spaces.json"); + try { + const output = await runSetup(planPath, setupArgs); + expect(output).toContain( + `openclaw secrets apply --from '${canonicalPlanPath}' --dry-run --allow-exec`, + ); + expect(output).toContain(`openclaw secrets apply --from '${canonicalPlanPath}' --allow-exec`); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + }); + it.each([ "providers/openai/apiKey/", "/providers/openai/apiKey", @@ -224,6 +241,21 @@ describe("vault CLI status", () => { expect(result.providerAlias).toBe("corp-vault"); }); + it("prefers the managed integration when the default alias is unrelated", async () => { + const result = await runStatus({ + secrets: { + providers: { + vault: { source: "exec", command: "/legacy/resolver" }, + "corp-vault": { + source: "exec", + pluginIntegration: { pluginId: "vault", integrationId: "vault" }, + }, + }, + }, + }); + expect(result.providerAlias).toBe("corp-vault"); + }); + it("requires an explicit alias when multiple Vault providers are configured", async () => { const config = { secrets: { diff --git a/extensions/vault/src/cli.ts b/extensions/vault/src/cli.ts index 7eac6d67dea5..ab5cb769d593 100644 --- a/extensions/vault/src/cli.ts +++ b/extensions/vault/src/cli.ts @@ -1,43 +1,38 @@ import path from "node:path"; -import { createInterface } from "node:readline/promises"; import { fileURLToPath } from "node:url"; -import { isRecord } from "openclaw/plugin-sdk/channel-secret-basic-runtime"; import type { OpenClawConfig } from "openclaw/plugin-sdk/plugin-entry"; -import { pluginSecretRefSetup } from "openclaw/plugin-sdk/secret-ref-runtime"; +import { createPluginSecretRefSetupCli } from "openclaw/plugin-sdk/secret-ref-runtime"; import { pathExists } from "openclaw/plugin-sdk/security-runtime"; +import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; import { parseVaultSecretId } from "../vault-secret-id.js"; -type CommandLike = { - command(name: string): CommandLike; - description(value: string): CommandLike; - option( - flags: string, - description: string, - defaultValueOrParser?: string | ((value: string, previous?: string[]) => string[]), - defaultValue?: string[], - ): CommandLike; - action(fn: (options: TOptions) => void | Promise): CommandLike; -}; +const VAULT_PROVIDER_ALIAS = "vault"; -type VaultExecProviderConfig = { - source: "exec"; +function normalizeVaultSecretId(label: string, value: string): string { + try { + parseVaultSecretId(value); + return value; + } catch { + throw new Error(`Invalid ${label} Vault secret id: ${value}`); + } +} + +const vaultSecretRefSetupCli = createPluginSecretRefSetupCli({ + productName: "Vault", + secretIdLabel: "Vault secret id", + secretIdPlaceholder: "vault-secret-id", + defaultProviderAlias: VAULT_PROVIDER_ALIAS, pluginIntegration: { - pluginId: "vault"; - integrationId: "vault"; - }; -}; + pluginId: "vault", + integrationId: "vault", + }, + normalizeSecretId: normalizeVaultSecretId, + defaultPlanPath: () => + path.join(resolvePreferredOpenClawTmpDir(), `openclaw-vault-secrets-${process.pid}.json`), +}); -type ProviderSecretMapping = { - providerId: string; - secretId: string; -}; - -type ConfigTargetSecretMapping = { - path: string; - agentId?: string; - secretId: string; -}; +type CommandLike = Parameters[0]; type RegisterVaultCommandsParams = { program: CommandLike; @@ -49,28 +44,6 @@ type StatusOptions = { providerAlias?: string; }; -type SetupOptions = { - planOut?: string; - providerAlias?: string; - openaiId?: string; - anthropicId?: string; - openrouterId?: string; - providerKey?: string[]; - target?: string[]; -}; - -type ProviderStatus = { - configured: boolean; - source?: string; - command?: string; - pluginIntegration?: { - pluginId: string; - integrationId: string; - }; -}; - -const VAULT_PROVIDER_ALIAS = "vault"; - function writeLine(message = ""): void { process.stdout.write(`${message}\n`); } @@ -79,77 +52,6 @@ function writeJson(value: unknown): void { process.stdout.write(`${JSON.stringify(value, null, 2)}\n`); } -function normalizeOptionalString(value: unknown): string | undefined { - return typeof value === "string" && value.trim() ? value.trim() : undefined; -} - -function assertValidProviderAlias(value: string): void { - pluginSecretRefSetup.assertValidProviderAlias(value); -} - -function assertValidVaultSecretId(label: string, value: string): void { - try { - parseVaultSecretId(value); - } catch { - throw new Error(`Invalid ${label} Vault secret id: ${value}`); - } -} - -function readProviderStatus(config: OpenClawConfig, providerAlias: string): ProviderStatus { - const provider = config.secrets?.providers?.[providerAlias]; - if (!isRecord(provider)) { - return { configured: false }; - } - const base = { - configured: true, - source: normalizeOptionalString(provider.source), - }; - if (provider.source !== "exec") { - return base; - } - if ("pluginIntegration" in provider) { - return { - ...base, - pluginIntegration: provider.pluginIntegration, - }; - } - return { - ...base, - command: normalizeOptionalString(provider.command), - }; -} - -function isVaultIntegrationProvider(value: unknown): boolean { - if (!isRecord(value) || value.source !== "exec" || !isRecord(value.pluginIntegration)) { - return false; - } - return ( - value.pluginIntegration.pluginId === "vault" && - value.pluginIntegration.integrationId === "vault" - ); -} - -function resolveStatusProviderAlias(config: OpenClawConfig, requestedAlias?: string): string { - const explicitAlias = normalizeOptionalString(requestedAlias); - if (explicitAlias) { - assertValidProviderAlias(explicitAlias); - return explicitAlias; - } - if (readProviderStatus(config, VAULT_PROVIDER_ALIAS).configured) { - return VAULT_PROVIDER_ALIAS; - } - const configuredAliases = Object.entries(config.secrets?.providers ?? {}) - .filter(([, provider]) => isVaultIntegrationProvider(provider)) - .map(([alias]) => alias) - .toSorted(); - if (configuredAliases.length > 1) { - throw new Error( - `Multiple Vault provider aliases are configured (${configuredAliases.join(", ")}). Use --provider-alias .`, - ); - } - return configuredAliases[0] ?? VAULT_PROVIDER_ALIAS; -} - function resolverScriptPathCandidates(baseUrl: string): [string, string] { return [ fileURLToPath(new URL("../vault-secret-ref-resolver.js", baseUrl)), @@ -170,134 +72,11 @@ async function resolveResolverScriptPath( return candidates[0]; } -function buildProviderConfig(): VaultExecProviderConfig { - return { - source: "exec", - pluginIntegration: { - pluginId: "vault", - integrationId: "vault", - }, - }; -} - -function parseTargetSpecifier(value: string): { - path: string; - agentId?: string; -} { - return pluginSecretRefSetup.parseTargetSpecifier("Vault", value); -} - -function parseProviderKeyMappings(values: string[] | undefined): ProviderSecretMapping[] { - return (values ?? []).map((value) => { - const separator = value.indexOf("="); - if (separator <= 0 || separator === value.length - 1) { - throw new Error( - `Invalid --provider-key value "${value}". Use =.`, - ); - } - const providerId = value.slice(0, separator).trim(); - const secretId = value.slice(separator + 1).trim(); - pluginSecretRefSetup.assertValidModelProviderId("--provider-key", providerId); - assertValidVaultSecretId(`--provider-key ${providerId}`, secretId); - return { providerId, secretId }; - }); -} - -function parseConfigTargetMappings(values: string[] | undefined): ConfigTargetSecretMapping[] { - return (values ?? []).map((value) => { - const separator = value.indexOf("="); - if (separator <= 0 || separator === value.length - 1) { - throw new Error( - `Invalid --target value "${value}". Use =.`, - ); - } - const target = parseTargetSpecifier(value.slice(0, separator).trim()); - const secretId = value.slice(separator + 1).trim(); - assertValidVaultSecretId(`--target ${target.path}`, secretId); - return Object.assign( - { path: target.path, secretId }, - target.agentId ? { agentId: target.agentId } : {}, - ); - }); -} - -function collectProviderSecrets(options: { - openaiId?: string; - anthropicId?: string; - openrouterId?: string; - providerKey?: string[]; -}): ProviderSecretMapping[] { - const providerSecrets: ProviderSecretMapping[] = []; - if (options.openaiId) { - providerSecrets.push({ providerId: "openai", secretId: options.openaiId }); - } - if (options.anthropicId) { - providerSecrets.push({ providerId: "anthropic", secretId: options.anthropicId }); - } - if (options.openrouterId) { - providerSecrets.push({ providerId: "openrouter", secretId: options.openrouterId }); - } - providerSecrets.push(...parseProviderKeyMappings(options.providerKey)); - - const seen = new Set(); - for (const entry of providerSecrets) { - const normalized = entry.providerId.toLowerCase(); - if (seen.has(normalized)) { - throw new Error(`Duplicate model provider id in Vault setup: ${entry.providerId}`); - } - seen.add(normalized); - } - return providerSecrets; -} - -function buildPlan(params: { - providerAlias: string; - providerConfig: VaultExecProviderConfig; - providerSecrets: ProviderSecretMapping[]; - configTargetSecrets?: ConfigTargetSecretMapping[]; -}) { - return pluginSecretRefSetup.buildPlan({ productName: "Vault", ...params }); -} - -async function promptOptionalSecretId(label: string): Promise { - if (!process.stdin.isTTY || !process.stdout.isTTY) { - return undefined; - } - const rl = createInterface({ input: process.stdin, output: process.stdout }); - try { - return normalizeOptionalString(await rl.question(`${label} Vault secret id (blank to skip): `)); - } finally { - rl.close(); - } -} - -async function promptProviderSecrets(options: SetupOptions): Promise { - const openaiId = - normalizeOptionalString(options.openaiId) ?? (await promptOptionalSecretId("OpenAI")); - const anthropicId = - normalizeOptionalString(options.anthropicId) ?? (await promptOptionalSecretId("Anthropic")); - const openrouterId = - normalizeOptionalString(options.openrouterId) ?? (await promptOptionalSecretId("OpenRouter")); - if (openaiId) { - assertValidVaultSecretId("OpenAI", openaiId); - } - if (anthropicId) { - assertValidVaultSecretId("Anthropic", anthropicId); - } - if (openrouterId) { - assertValidVaultSecretId("OpenRouter", openrouterId); - } - return collectProviderSecrets({ - ...(openaiId ? { openaiId } : {}), - ...(anthropicId ? { anthropicId } : {}), - ...(openrouterId ? { openrouterId } : {}), - providerKey: options.providerKey, - }); -} - async function runStatus(config: OpenClawConfig, options: StatusOptions): Promise { - const providerAlias = resolveStatusProviderAlias(config, options.providerAlias); - const provider = readProviderStatus(config, providerAlias); + const { providerAlias, provider } = vaultSecretRefSetupCli.inspectProvider( + config, + options.providerAlias, + ); const authMethod = normalizeOptionalString(process.env.OPENCLAW_VAULT_AUTH_METHOD) ?? "token"; const result = { providerAlias, @@ -343,33 +122,6 @@ async function runStatus(config: OpenClawConfig, options: StatusOptions): Promis writeLine(`KV version: ${result.kvVersion}`); } -async function runSetup(options: SetupOptions): Promise { - const providerAlias = normalizeOptionalString(options.providerAlias) ?? VAULT_PROVIDER_ALIAS; - assertValidProviderAlias(providerAlias); - const providerSecrets = await promptProviderSecrets(options); - const plan = buildPlan({ - providerAlias, - providerConfig: buildProviderConfig(), - providerSecrets, - configTargetSecrets: parseConfigTargetMappings(options.target), - }); - const planPath = - normalizeOptionalString(options.planOut) ?? - path.join(resolvePreferredOpenClawTmpDir(), `openclaw-vault-secrets-${process.pid}.json`); - await pluginSecretRefSetup.writePlanFile({ - planPath, - content: `${JSON.stringify(plan, null, 2)}\n`, - }); - writeLine(`Plan written to ${planPath}`); - writeLine(`Targets: ${plan.targets.length}`); - writeLine(""); - writeLine("Next steps:"); - writeLine(` openclaw secrets apply --from ${planPath} --dry-run --allow-exec`); - writeLine(` openclaw secrets apply --from ${planPath} --allow-exec`); - writeLine(" openclaw secrets audit --check --allow-exec"); - writeLine(" openclaw secrets reload"); -} - export function registerVaultCommands(params: RegisterVaultCommandsParams): void { const vault = params.program.command("vault").description("Manage Vault SecretRefs"); vault @@ -378,25 +130,5 @@ export function registerVaultCommands(params: RegisterVaultCommandsParams): void .option("--json", "Print JSON status") .option("--provider-alias ", "Secret provider alias to inspect") .action((options: StatusOptions) => runStatus(params.config, options)); - vault - .command("setup") - .description("Create a Vault SecretRef setup plan") - .option("--plan-out ", "Write the generated secrets apply plan to a path") - .option("--provider-alias ", "Secret provider alias to configure", VAULT_PROVIDER_ALIAS) - .option("--openai-id ", "Vault secret id for models.providers.openai.apiKey") - .option("--anthropic-id ", "Vault secret id for models.providers.anthropic.apiKey") - .option("--openrouter-id ", "Vault secret id for models.providers.openrouter.apiKey") - .option( - "--provider-key ", - "Vault secret id for any models.providers..apiKey target", - (value: string, previous: string[] = []) => [...previous, value], - [], - ) - .option( - "--target ", - "Vault secret id for any known SecretRef target path", - (value: string, previous: string[] = []) => [...previous, value], - [], - ) - .action((options: SetupOptions) => runSetup(options)); + vaultSecretRefSetupCli.registerSetupCommand(vault); } diff --git a/extensions/voice-call/index.test.ts b/extensions/voice-call/index.test.ts index d29c851a4383..8dffe592d08d 100644 --- a/extensions/voice-call/index.test.ts +++ b/extensions/voice-call/index.test.ts @@ -696,6 +696,47 @@ describe("voice-call plugin", () => { ]); }); + it("routes tool speech through the active realtime bridge", async () => { + runtimeStub.config.realtime.enabled = true; + runtimeStub.manager.getCall = vi.fn(() => undefined); + runtimeStub.manager.getCallByProviderCallId = vi.fn(() => + createCallRecord({ callId: "call-1", providerCallId: "CA123" }), + ); + runtimeStub.webhookServer.speakRealtime = vi.fn(() => ({ success: true })); + const { tools } = setup({ provider: "mock" }); + const tool = tools[0] as { + execute: (id: string, params: unknown) => Promise; + }; + + const result = (await tool.execute("id", { + action: "speak_to_user", + callId: "CA123", + message: "hello", + })) as { details: { success?: boolean } }; + + expect(runtimeStub.webhookServer["speakRealtime"]).toHaveBeenCalledWith("call-1", "hello"); + expect(runtimeStub.manager["speak"]).not.toHaveBeenCalled(); + expect(result.details.success).toBe(true); + }); + + it("keeps the tool's classic speech fallback when no realtime bridge is active", async () => { + runtimeStub.config.realtime.enabled = true; + const { tools } = setup({ provider: "mock" }); + const tool = tools[0] as { + execute: (id: string, params: unknown) => Promise; + }; + + const result = (await tool.execute("id", { + action: "speak_to_user", + callId: "call-1", + message: "hello", + })) as { details: { success?: boolean } }; + + expect(runtimeStub.webhookServer["speakRealtime"]).toHaveBeenCalledWith("call-1", "hello"); + expect(runtimeStub.manager["speak"]).toHaveBeenCalledWith("call-1", "hello"); + expect(result.details.success).toBe(true); + }); + it("reports ended call history when speaking to a stale call", async () => { runtimeStub.manager.getCall = vi.fn(() => undefined); runtimeStub.manager.getCallByProviderCallId = vi.fn(() => undefined); @@ -1059,7 +1100,7 @@ describe("voice-call plugin", () => { } }); - it("gateway continue operations return pending then completed results", async () => { + it("gateway continue operations return pending, completed, and failed results", async () => { let finishContinue: ((value: { success: true; transcript: string }) => void) | undefined; const continuePromise = new Promise<{ success: true; transcript: string }>((resolve) => { finishContinue = resolve; @@ -1111,18 +1152,41 @@ describe("voice-call plugin", () => { finishContinue?.({ success: true, transcript: "gateway hello" }); await continuePromise; - await Promise.resolve(); - - const completedRespond = vi.fn(); - await result?.({ - params: { operationId: startPayload?.operationId }, - respond: completedRespond, + const completedCall = await vi.waitFor(async () => { + const respond = vi.fn(); + await result?.({ params: { operationId: startPayload?.operationId }, respond }); + const call = firstRespondCall(respond); + const payload = call[1] as { status?: unknown } | undefined; + expect(payload?.status).toBe("completed"); + return call; }); - const completedCall = firstRespondCall(completedRespond); const completedPayload = completedCall[1] as { status?: unknown; result?: unknown } | undefined; expect(completedCall[0]).toBe(true); - expect(completedPayload?.status).toBe("completed"); expect(completedPayload?.result).toEqual({ success: true, transcript: "gateway hello" }); + + runtimeStub.manager.continueCall = vi.fn(async () => ({ + success: false, + error: "turn failed", + })) as VoiceCallRuntime["manager"]["continueCall"]; + const failedStartRespond = vi.fn(); + await start?.({ + params: { callId: "call-1", message: "Try again" }, + respond: failedStartRespond, + }); + const failedOperationId = ( + firstRespondCall(failedStartRespond)[1] as { operationId?: string } | undefined + )?.operationId; + + const failedCall = await vi.waitFor(async () => { + const respond = vi.fn(); + await result?.({ params: { operationId: failedOperationId }, respond }); + const call = firstRespondCall(respond); + const payload = call[1] as { status?: unknown } | undefined; + expect(payload?.status).toBe("failed"); + return call; + }); + expect(failedCall[0]).toBe(true); + expect(failedCall[1]).toMatchObject({ status: "failed", error: "turn failed" }); }); it("CLI setup prints human-readable checks by default", async () => { diff --git a/extensions/voice-call/index.ts b/extensions/voice-call/index.ts index 083d311511e2..3f62497040e7 100644 --- a/extensions/voice-call/index.ts +++ b/extensions/voice-call/index.ts @@ -1,7 +1,6 @@ // Voice Call plugin entrypoint registers its OpenClaw integration. import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { ErrorCodes, errorShape } from "openclaw/plugin-sdk/gateway-runtime"; -import { timestampMsToIsoString } from "openclaw/plugin-sdk/number-runtime"; import { normalizeAgentId, parseAgentSessionKey } from "openclaw/plugin-sdk/routing"; import { asOptionalRecord, @@ -17,6 +16,10 @@ import { import { VOICE_CALL_CLI_DESCRIPTOR } from "./cli-output-mode.js"; import { createVoiceCallRuntime, type VoiceCallRuntime } from "./runtime-entry.js"; import { registerVoiceCallCli } from "./src/cli.js"; +import { + createVoiceCallCommandService, + VoiceCallCommandInputError, +} from "./src/command-service.js"; import { VoiceCallConfigSchema, resolveVoiceCallConfig, @@ -25,7 +28,6 @@ import { } from "./src/config.js"; import type { CoreConfig } from "./src/core-bridge.js"; import { createVoiceCallContinueOperationStore } from "./src/gateway-continue-operation.js"; -import type { CallRecord } from "./src/types.js"; const VOICE_CALL_WRITE_METHOD_SCOPE = { scope: "operator.write" as const }; const VOICE_CALL_READ_METHOD_SCOPE = { scope: "operator.read" as const }; @@ -234,33 +236,6 @@ function isCliOnlyProcess(): boolean { return process.env.OPENCLAW_CLI === "1" && !process.argv.slice(2).includes("gateway"); } -type VoiceCallStatus = Pick< - CallRecord, - | "callId" - | "providerCallId" - | "provider" - | "direction" - | "state" - | "startedAt" - | "answeredAt" - | "endedAt" - | "endReason" ->; - -function toVoiceCallStatus(call: CallRecord): VoiceCallStatus { - return { - callId: call.callId, - ...(call.providerCallId !== undefined ? { providerCallId: call.providerCallId } : {}), - provider: call.provider, - direction: call.direction, - state: call.state, - startedAt: call.startedAt, - ...(call.answeredAt !== undefined ? { answeredAt: call.answeredAt } : {}), - ...(call.endedAt !== undefined ? { endedAt: call.endedAt } : {}), - ...(call.endReason !== undefined ? { endReason: call.endReason } : {}), - }; -} - const VOICE_CALL_RUNTIME_KEY = Symbol.for("openclaw.voice-call.runtime"); const VOICE_CALL_RUNTIME_PROMISE_KEY = Symbol.for("openclaw.voice-call.runtimePromise"); const VOICE_CALL_RUNTIME_STOP_PROMISE_KEY = Symbol.for("openclaw.voice-call.runtimeStopPromise"); @@ -349,363 +324,150 @@ export default definePluginEntry({ } }; - const respondError = ( - respond: GatewayRequestHandlerOptions["respond"], - message: string, - code: (typeof ErrorCodes)[keyof typeof ErrorCodes] = ErrorCodes.UNAVAILABLE, + const commands = createVoiceCallCommandService(ensureRuntime); + const registerGatewayCommand = ( + method: string, + handler: (options: GatewayRequestHandlerOptions) => unknown, + scope: typeof VOICE_CALL_WRITE_METHOD_SCOPE | typeof VOICE_CALL_READ_METHOD_SCOPE, ) => { - respond(false, undefined, errorShape(code, message)); - }; - - const sendError = (respond: GatewayRequestHandlerOptions["respond"], err: unknown) => { - respondError(respond, formatErrorMessage(err)); - }; - - const describeHistoricalCall = async (rt: VoiceCallRuntime, callId: string) => { - const call = await rt.manager.getCallFromMemoryOrStore(callId); - if (!call) { - return undefined; - } - const endedAt = timestampMsToIsoString(call.endedAt); - const details = [ - `last state=${call.state}`, - call.endReason ? `endReason=${call.endReason}` : undefined, - endedAt ? `endedAt=${endedAt}` : undefined, - ].filter(Boolean); - return `call is not active (${details.join(", ")})`; - }; - - const resolveCallMessageRequest = async (params: GatewayRequestHandlerOptions["params"]) => { - const callId = normalizeOptionalString(params?.callId) ?? ""; - const message = normalizeOptionalString(params?.message) ?? ""; - if (!callId || !message) { - return { error: "callId and message required" } as const; - } - const rt = await ensureRuntime(); - const activeCall = rt.manager.getCall(callId) ?? rt.manager.getCallByProviderCallId(callId); - if (activeCall) { - return { rt, callId: activeCall.callId, message } as const; - } - return { error: (await describeHistoricalCall(rt, callId)) ?? "Call not found" } as const; - }; - - const initiateCallAndRespond = async (params: { - rt: VoiceCallRuntime; - respond: GatewayRequestHandlerOptions["respond"]; - to: string; - message?: string; - mode?: "notify" | "conversation"; - dtmfSequence?: string; - sessionKey?: string; - requesterSessionKey?: string; - agentId?: string; - }) => { - const result = await params.rt.manager.initiateCall(params.to, params.sessionKey, { - message: params.message, - mode: params.mode, - dtmfSequence: params.dtmfSequence, - ...(params.requesterSessionKey ? { requesterSessionKey: params.requesterSessionKey } : {}), - ...(params.agentId ? { agentId: params.agentId } : {}), - }); - if (!result.success) { - respondError(params.respond, result.error || "initiate failed"); - return; - } - params.respond(true, { callId: result.callId, initiated: true }); - }; - - const respondToCallMessageAction = async (params: { - requestParams: GatewayRequestHandlerOptions["params"]; - respond: GatewayRequestHandlerOptions["respond"]; - action: ( - request: Exclude>, { error: string }>, - ) => Promise<{ - success: boolean; - error?: string; - transcript?: string; - }>; - failure: string; - includeTranscript?: boolean; - }) => { - const request = await resolveCallMessageRequest(params.requestParams); - if ("error" in request) { - respondError( - params.respond, - request.error ?? "callId and message required", - ErrorCodes.INVALID_REQUEST, - ); - return; - } - const result = await params.action(request); - if (!result.success) { - respondError(params.respond, result.error || params.failure); - return; - } - params.respond( - true, - params.includeTranscript - ? { success: true, transcript: result.transcript } - : { success: true }, + api.registerGatewayMethod( + method, + async (options: GatewayRequestHandlerOptions) => { + try { + options.respond(true, await handler(options)); + } catch (err) { + const code = + err instanceof VoiceCallCommandInputError + ? ErrorCodes.INVALID_REQUEST + : ErrorCodes.UNAVAILABLE; + options.respond(false, undefined, errorShape(code, formatErrorMessage(err))); + } + }, + scope, ); }; - api.registerGatewayMethod( + registerGatewayCommand( "voicecall.initiate", - async ({ params, respond }: GatewayRequestHandlerOptions) => { - try { - const message = normalizeOptionalString(params?.message) ?? ""; - if (!message) { - respondError(respond, "message required", ErrorCodes.INVALID_REQUEST); - return; - } - const rt = await ensureRuntime(); - const to = normalizeOptionalString(params?.to) ?? rt.config.toNumber; - if (!to) { - respondError(respond, "to required", ErrorCodes.INVALID_REQUEST); - return; - } - const mode = - params?.mode === "notify" || params?.mode === "conversation" ? params.mode : undefined; - await initiateCallAndRespond({ - rt, - respond, - to, - message, - mode, - sessionKey: normalizeOptionalString(params?.sessionKey), - requesterSessionKey: normalizeOptionalString(params?.requesterSessionKey), - }); - } catch (err) { - sendError(respond, err); + async ({ params }) => { + const message = normalizeOptionalString(params?.message); + if (!message) { + throw new VoiceCallCommandInputError("message required"); } + return await commands.initiate({ + to: normalizeOptionalString(params?.to), + message, + mode: + params?.mode === "notify" || params?.mode === "conversation" ? params.mode : undefined, + sessionKey: normalizeOptionalString(params?.sessionKey), + requesterSessionKey: normalizeOptionalString(params?.requesterSessionKey), + }); }, VOICE_CALL_WRITE_METHOD_SCOPE, ); - api.registerGatewayMethod( + registerGatewayCommand( "voicecall.continue", - async ({ params, respond }: GatewayRequestHandlerOptions) => { - try { - await respondToCallMessageAction({ - requestParams: params, - respond, - action: (request) => request.rt.manager.continueCall(request.callId, request.message), - failure: "continue failed", - includeTranscript: true, - }); - } catch (err) { - sendError(respond, err); - } - }, + ({ params }) => + commands.continueCall( + normalizeOptionalString(params?.callId), + normalizeOptionalString(params?.message), + ), VOICE_CALL_WRITE_METHOD_SCOPE, ); - api.registerGatewayMethod( + registerGatewayCommand( "voicecall.continue.start", - async ({ params, respond }: GatewayRequestHandlerOptions) => { - try { - const request = await resolveCallMessageRequest(params); - if ("error" in request) { - respondError( - respond, - request.error ?? "callId and message required", - ErrorCodes.INVALID_REQUEST, - ); - return; - } - respond(true, continueOperationStore.start(request)); - } catch (err) { - sendError(respond, err); - } - }, + async ({ params }) => + continueOperationStore.start( + await commands.prepareContinue( + normalizeOptionalString(params?.callId), + normalizeOptionalString(params?.message), + ), + ), VOICE_CALL_WRITE_METHOD_SCOPE, ); - api.registerGatewayMethod( + registerGatewayCommand( "voicecall.continue.result", - async ({ params, respond }: GatewayRequestHandlerOptions) => { - try { - const operationId = normalizeOptionalString(params?.operationId) ?? ""; - if (!operationId) { - respondError(respond, "operationId required", ErrorCodes.INVALID_REQUEST); - return; - } - const operation = continueOperationStore.read(operationId); - if (!operation.ok) { - respondError(respond, operation.error, ErrorCodes.INVALID_REQUEST); - return; - } - respond(true, operation.payload); - } catch (err) { - sendError(respond, err); + ({ params }) => { + const operationId = normalizeOptionalString(params?.operationId); + if (!operationId) { + throw new VoiceCallCommandInputError("operationId required"); } + const operation = continueOperationStore.read(operationId); + if (!operation.ok) { + throw new VoiceCallCommandInputError(operation.error); + } + return operation.payload; }, VOICE_CALL_READ_METHOD_SCOPE, ); - api.registerGatewayMethod( + registerGatewayCommand( "voicecall.speak", - async ({ params, respond }: GatewayRequestHandlerOptions) => { - try { - const request = await resolveCallMessageRequest(params); - if ("error" in request) { - respondError( - respond, - request.error ?? "callId and message required", - ErrorCodes.INVALID_REQUEST, - ); - return; - } - if (request.rt.config.realtime.enabled) { - const realtimeResult = request.rt.webhookServer.speakRealtime( - request.callId, - request.message, - ); - if (realtimeResult.success) { - respond(true, { success: true }); - return; - } - if (params?.allowTwimlFallback === false) { - respond(true, { - success: false, - error: realtimeResult.error ?? "Realtime bridge is not active", - }); - return; - } - } - const result = await request.rt.manager.speak(request.callId, request.message); - if (!result.success) { - respondError(respond, result.error || "speak failed"); - return; - } - respond(true, { success: true }); - } catch (err) { - sendError(respond, err); - } - }, + ({ params }) => + commands.speak({ + callId: normalizeOptionalString(params?.callId), + message: normalizeOptionalString(params?.message), + allowTwimlFallback: params?.allowTwimlFallback !== false, + }), VOICE_CALL_WRITE_METHOD_SCOPE, ); - api.registerGatewayMethod( + registerGatewayCommand( "voicecall.dtmf", - async ({ params, respond }: GatewayRequestHandlerOptions) => { - try { - const callId = normalizeOptionalString(params?.callId) ?? ""; - const digits = normalizeOptionalString(params?.digits) ?? ""; - if (!callId || !digits) { - respondError(respond, "callId and digits required", ErrorCodes.INVALID_REQUEST); - return; - } - const rt = await ensureRuntime(); - const result = await rt.manager.sendDtmf(callId, digits); - if (!result.success) { - respondError(respond, result.error || "dtmf failed"); - return; - } - respond(true, { success: true }); - } catch (err) { - sendError(respond, err); - } - }, + ({ params }) => + commands.sendDtmf( + normalizeOptionalString(params?.callId), + normalizeOptionalString(params?.digits), + ), VOICE_CALL_WRITE_METHOD_SCOPE, ); - api.registerGatewayMethod( + registerGatewayCommand( "voicecall.end", - async ({ params, respond }: GatewayRequestHandlerOptions) => { - try { - const callId = normalizeOptionalString(params?.callId) ?? ""; - if (!callId) { - respondError(respond, "callId required", ErrorCodes.INVALID_REQUEST); - return; - } - const rt = await ensureRuntime(); - const result = await rt.manager.endCall(callId); - if (!result.success) { - respondError(respond, result.error || "end failed"); - return; - } - respond(true, { success: true }); - } catch (err) { - sendError(respond, err); - } - }, + ({ params }) => commands.endCall(normalizeOptionalString(params?.callId)), VOICE_CALL_WRITE_METHOD_SCOPE, ); - api.registerGatewayMethod( + registerGatewayCommand( "voicecall.status", - async ({ params, respond }: GatewayRequestHandlerOptions) => { - try { - const raw = - normalizeOptionalString(params?.callId) ?? normalizeOptionalString(params?.sid) ?? ""; - const rt = await ensureRuntime(); - if (!raw) { - respond(true, { - found: true, - calls: rt.manager.getActiveCalls().map(toVoiceCallStatus), - }); - return; - } - const call = await rt.manager.getCallFromMemoryOrStore(raw); - if (!call) { - respond(true, { found: false }); - return; - } - respond(true, { found: true, call: toVoiceCallStatus(call) }); - } catch (err) { - sendError(respond, err); - } - }, + ({ params }) => + commands.status( + normalizeOptionalString(params?.callId) ?? normalizeOptionalString(params?.sid), + ), VOICE_CALL_READ_METHOD_SCOPE, ); - api.registerGatewayMethod( + registerGatewayCommand( "voicecall.start", - async ({ params, client, respond }: GatewayRequestHandlerOptions) => { - try { - const to = normalizeOptionalString(params?.to) ?? ""; - const message = normalizeOptionalString(params?.message) ?? ""; - const dtmfSequence = normalizeOptionalString(params?.dtmfSequence); - const sessionKey = normalizeOptionalString(params?.sessionKey); - const requesterSessionKey = normalizeOptionalString(params?.requesterSessionKey); - const requestedAgentId = normalizeOptionalString(params?.agentId); - const normalizedAgentId = requestedAgentId - ? normalizeAgentId(requestedAgentId) - : undefined; - const pluginOwnerId = normalizeOptionalString(client?.internal?.pluginRuntimeOwnerId); - if ( - requestedAgentId && - (!pluginOwnerId || normalizedAgentId !== requestedAgentId.toLowerCase()) - ) { - respondError( - respond, - "agentId requires a trusted plugin caller and a valid agent id", - ErrorCodes.INVALID_REQUEST, - ); - return; - } - if (!to) { - respondError(respond, "to required", ErrorCodes.INVALID_REQUEST); - return; - } - const mode = - params?.mode === "notify" || params?.mode === "conversation" ? params.mode : undefined; - const rt = await ensureRuntime(); - await initiateCallAndRespond({ - rt, - respond, - to, - message: message || undefined, - mode, - dtmfSequence, - sessionKey, - ...(requesterSessionKey ? { requesterSessionKey } : {}), - ...(normalizedAgentId ? { agentId: normalizedAgentId } : {}), - }); - } catch (err) { - sendError(respond, err); + async ({ params, client }) => { + const to = normalizeOptionalString(params?.to); + const requestedAgentId = normalizeOptionalString(params?.agentId); + const normalizedAgentId = requestedAgentId ? normalizeAgentId(requestedAgentId) : undefined; + const pluginOwnerId = normalizeOptionalString(client?.internal?.pluginRuntimeOwnerId); + if ( + requestedAgentId && + (!pluginOwnerId || normalizedAgentId !== requestedAgentId.toLowerCase()) + ) { + throw new VoiceCallCommandInputError( + "agentId requires a trusted plugin caller and a valid agent id", + ); } + if (!to) { + throw new VoiceCallCommandInputError("to required"); + } + return await commands.initiate({ + to, + message: normalizeOptionalString(params?.message), + mode: + params?.mode === "notify" || params?.mode === "conversation" ? params.mode : undefined, + dtmfSequence: normalizeOptionalString(params?.dtmfSequence), + sessionKey: normalizeOptionalString(params?.sessionKey), + requesterSessionKey: normalizeOptionalString(params?.requesterSessionKey), + agentId: normalizedAgentId, + }); }, VOICE_CALL_WRITE_METHOD_SCOPE, ); @@ -725,94 +487,59 @@ export default definePluginEntry({ parseAgentSessionKey(requesterSessionKey)?.agentId; const agentId = contextAgentId ? normalizeAgentId(contextAgentId) : undefined; try { - const rt = await ensureRuntime(); - + // Preserve tool error precedence: runtime availability is checked before model input. + await ensureRuntime(); if (typeof rawParams.action === "string") { switch (rawParams.action) { case "initiate_call": { - const message = normalizeOptionalString(rawParams.message) ?? ""; + const message = normalizeOptionalString(rawParams.message); if (!message) { - throw new Error("message required"); + throw new VoiceCallCommandInputError("message required"); } - const to = normalizeOptionalString(rawParams.to) ?? rt.config.toNumber; - if (!to) { - throw new Error("to required"); - } - const result = await rt.manager.initiateCall( - to, - normalizeOptionalString(rawParams.sessionKey), - { + return json( + await commands.initiate({ + to: normalizeOptionalString(rawParams.to), message, dtmfSequence: normalizeOptionalString(rawParams.dtmfSequence), mode: rawParams.mode === "notify" || rawParams.mode === "conversation" ? rawParams.mode : undefined, - ...(agentId ? { agentId } : {}), - ...(requesterSessionKey ? { requesterSessionKey } : {}), - }, + sessionKey: normalizeOptionalString(rawParams.sessionKey), + agentId, + requesterSessionKey, + }), ); - if (!result.success) { - throw new Error(result.error || "initiate failed"); - } - return json({ callId: result.callId, initiated: true }); } - case "continue_call": { - const callId = normalizeOptionalString(rawParams.callId) ?? ""; - const message = normalizeOptionalString(rawParams.message) ?? ""; - if (!callId || !message) { - throw new Error("callId and message required"); - } - const result = await rt.manager.continueCall(callId, message); - if (!result.success) { - throw new Error(result.error || "continue failed"); - } - return json({ success: true, transcript: result.transcript }); - } - case "speak_to_user": { - const callId = normalizeOptionalString(rawParams.callId) ?? ""; - const message = normalizeOptionalString(rawParams.message) ?? ""; - if (!callId || !message) { - throw new Error("callId and message required"); - } - const result = await rt.manager.speak(callId, message); - if (!result.success) { - throw new Error(result.error || "speak failed"); - } - return json({ success: true }); - } - case "send_dtmf": { - const callId = normalizeOptionalString(rawParams.callId) ?? ""; - const digits = normalizeOptionalString(rawParams.digits) ?? ""; - if (!callId || !digits) { - throw new Error("callId and digits required"); - } - const result = await rt.manager.sendDtmf(callId, digits); - if (!result.success) { - throw new Error(result.error || "dtmf failed"); - } - return json({ success: true }); - } - case "end_call": { - const callId = normalizeOptionalString(rawParams.callId) ?? ""; - if (!callId) { - throw new Error("callId required"); - } - const result = await rt.manager.endCall(callId); - if (!result.success) { - throw new Error(result.error || "end failed"); - } - return json({ success: true }); - } - case "get_status": { - const callId = normalizeOptionalString(rawParams.callId) ?? ""; - if (!callId) { - throw new Error("callId required"); - } - const call = await rt.manager.getCallFromMemoryOrStore(callId); + case "continue_call": return json( - call ? { found: true, call: toVoiceCallStatus(call) } : { found: false }, + await commands.continueCall( + normalizeOptionalString(rawParams.callId), + normalizeOptionalString(rawParams.message), + ), ); + case "speak_to_user": + return json( + await commands.speak({ + callId: normalizeOptionalString(rawParams.callId), + message: normalizeOptionalString(rawParams.message), + }), + ); + case "send_dtmf": + return json( + await commands.sendDtmf( + normalizeOptionalString(rawParams.callId), + normalizeOptionalString(rawParams.digits), + ), + ); + case "end_call": + return json(await commands.endCall(normalizeOptionalString(rawParams.callId))); + case "get_status": { + const callId = normalizeOptionalString(rawParams.callId); + if (!callId) { + throw new VoiceCallCommandInputError("callId required"); + } + return json(await commands.status(callId)); } } } @@ -823,28 +550,22 @@ export default definePluginEntry({ if (!sid) { throw new Error("sid required for status"); } - const call = await rt.manager.getCallFromMemoryOrStore(sid); - return json(call ? { found: true, call: toVoiceCallStatus(call) } : { found: false }); + return json(await commands.status(sid)); } - const to = normalizeOptionalString(rawParams.to) ?? rt.config.toNumber; - if (!to) { - throw new Error("to required for call"); - } - const result = await rt.manager.initiateCall( - to, - normalizeOptionalString(rawParams.sessionKey), - { - dtmfSequence: normalizeOptionalString(rawParams.dtmfSequence), - message: normalizeOptionalString(rawParams.message), - ...(agentId ? { agentId } : {}), - ...(requesterSessionKey ? { requesterSessionKey } : {}), - }, + return json( + await commands.initiate( + { + to: normalizeOptionalString(rawParams.to), + dtmfSequence: normalizeOptionalString(rawParams.dtmfSequence), + message: normalizeOptionalString(rawParams.message), + sessionKey: normalizeOptionalString(rawParams.sessionKey), + agentId, + requesterSessionKey, + }, + "to required for call", + ), ); - if (!result.success) { - throw new Error(result.error || "initiate failed"); - } - return json({ callId: result.callId, initiated: true }); } catch (err) { return json({ error: formatErrorMessage(err), @@ -912,4 +633,3 @@ export default definePluginEntry({ }); }, }); -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/extensions/voice-call/src/command-service.ts b/extensions/voice-call/src/command-service.ts new file mode 100644 index 000000000000..46e4e6258cf1 --- /dev/null +++ b/extensions/voice-call/src/command-service.ts @@ -0,0 +1,171 @@ +// Voice Call command service owns operations shared by gateway and model-tool adapters. +import { timestampMsToIsoString } from "openclaw/plugin-sdk/number-runtime"; +import type { CallMode } from "./config.js"; +import type { VoiceCallRuntime } from "./runtime.js"; +import type { CallRecord } from "./types.js"; + +type VoiceCallStatus = Pick< + CallRecord, + | "callId" + | "providerCallId" + | "provider" + | "direction" + | "state" + | "startedAt" + | "answeredAt" + | "endedAt" + | "endReason" +>; + +export class VoiceCallCommandInputError extends Error {} + +function toVoiceCallStatus(call: CallRecord): VoiceCallStatus { + return { + callId: call.callId, + ...(call.providerCallId !== undefined ? { providerCallId: call.providerCallId } : {}), + provider: call.provider, + direction: call.direction, + state: call.state, + startedAt: call.startedAt, + ...(call.answeredAt !== undefined ? { answeredAt: call.answeredAt } : {}), + ...(call.endedAt !== undefined ? { endedAt: call.endedAt } : {}), + ...(call.endReason !== undefined ? { endReason: call.endReason } : {}), + }; +} + +function requireInput(value: string | undefined, message: string): string { + if (!value) { + throw new VoiceCallCommandInputError(message); + } + return value; +} + +function requireSuccess(result: { success: boolean; error?: string }, fallback: string): void { + if (!result.success) { + throw new Error(result.error || fallback); + } +} + +export function createVoiceCallCommandService(ensureRuntime: () => Promise) { + const describeHistoricalCall = async (rt: VoiceCallRuntime, callId: string) => { + const call = await rt.manager.getCallFromMemoryOrStore(callId); + if (!call) { + return undefined; + } + const endedAt = timestampMsToIsoString(call.endedAt); + const details = [ + `last state=${call.state}`, + call.endReason ? `endReason=${call.endReason}` : undefined, + endedAt ? `endedAt=${endedAt}` : undefined, + ].filter(Boolean); + return `call is not active (${details.join(", ")})`; + }; + + const resolveCallMessage = async (callId?: string, message?: string) => { + const resolvedCallId = requireInput(callId, "callId and message required"); + const resolvedMessage = requireInput(message, "callId and message required"); + const rt = await ensureRuntime(); + const activeCall = + rt.manager.getCall(resolvedCallId) ?? rt.manager.getCallByProviderCallId(resolvedCallId); + if (!activeCall) { + throw new VoiceCallCommandInputError( + (await describeHistoricalCall(rt, resolvedCallId)) ?? "Call not found", + ); + } + return { rt, callId: activeCall.callId, message: resolvedMessage }; + }; + + const prepareContinue = async (callId?: string, message?: string) => { + const request = await resolveCallMessage(callId, message); + return { + rt: request.rt, + callId: request.callId, + run: async () => { + const result = await request.rt.manager.continueCall(request.callId, request.message); + requireSuccess(result, "continue failed"); + return { success: true as const, transcript: result.transcript }; + }, + }; + }; + + return { + prepareContinue, + + async initiate( + params: { + to?: string; + message?: string; + mode?: CallMode; + sessionKey?: string; + dtmfSequence?: string; + requesterSessionKey?: string; + agentId?: string; + }, + missingToMessage = "to required", + ) { + const rt = await ensureRuntime(); + const to = requireInput(params.to ?? rt.config.toNumber, missingToMessage); + const result = await rt.manager.initiateCall(to, params.sessionKey, { + message: params.message, + mode: params.mode, + dtmfSequence: params.dtmfSequence, + ...(params.requesterSessionKey ? { requesterSessionKey: params.requesterSessionKey } : {}), + ...(params.agentId ? { agentId: params.agentId } : {}), + }); + requireSuccess(result, "initiate failed"); + return { callId: result.callId, initiated: true }; + }, + + async continueCall(callId?: string, message?: string) { + return await (await prepareContinue(callId, message)).run(); + }, + + async speak(params: { callId?: string; message?: string; allowTwimlFallback?: boolean }) { + const request = await resolveCallMessage(params.callId, params.message); + if (request.rt.config.realtime.enabled) { + const realtimeResult = request.rt.webhookServer.speakRealtime( + request.callId, + request.message, + ); + if (realtimeResult.success) { + return { success: true }; + } + if (params.allowTwimlFallback === false) { + return { + success: false, + error: realtimeResult.error ?? "Realtime bridge is not active", + }; + } + } + const result = await request.rt.manager.speak(request.callId, request.message); + requireSuccess(result, "speak failed"); + return { success: true }; + }, + + async sendDtmf(callId?: string, digits?: string) { + const resolvedCallId = requireInput(callId, "callId and digits required"); + const resolvedDigits = requireInput(digits, "callId and digits required"); + const rt = await ensureRuntime(); + const result = await rt.manager.sendDtmf(resolvedCallId, resolvedDigits); + requireSuccess(result, "dtmf failed"); + return { success: true }; + }, + + async endCall(callId?: string) { + const resolvedCallId = requireInput(callId, "callId required"); + const rt = await ensureRuntime(); + const result = await rt.manager.endCall(resolvedCallId); + requireSuccess(result, "end failed"); + return { success: true }; + }, + + async status(callId?: string) { + const rt = await ensureRuntime(); + if (!callId) { + return { found: true, calls: rt.manager.getActiveCalls().map(toVoiceCallStatus) }; + } + const call = await rt.manager.getCallFromMemoryOrStore(callId); + return call ? { found: true, call: toVoiceCallStatus(call) } : { found: false }; + }, + }; +} diff --git a/extensions/voice-call/src/gateway-continue-operation.test.ts b/extensions/voice-call/src/gateway-continue-operation.test.ts index cc17c145dabe..64db928a396b 100644 --- a/extensions/voice-call/src/gateway-continue-operation.test.ts +++ b/extensions/voice-call/src/gateway-continue-operation.test.ts @@ -15,13 +15,10 @@ describe("voice-call gateway continue operation store", () => { const started = store.start({ callId: "call-1", - message: "hello", rt: { config: {}, - manager: { - continueCall: async () => new Promise(() => {}), - }, } as never, + run: async () => await new Promise(() => {}), }); expect(started.pollTimeoutMs).toBe(MAX_TIMER_TIMEOUT_MS); diff --git a/extensions/voice-call/src/gateway-continue-operation.ts b/extensions/voice-call/src/gateway-continue-operation.ts index 02c66809bcc9..d3478495524a 100644 --- a/extensions/voice-call/src/gateway-continue-operation.ts +++ b/extensions/voice-call/src/gateway-continue-operation.ts @@ -69,7 +69,7 @@ type VoiceCallContinueOperationResultPayload = type VoiceCallContinueOperationRequest = { rt: VoiceCallRuntime; callId: string; - message: string; + run: () => Promise<{ success: true; transcript?: string }>; }; /** Create a process-local operation store for gateway continue-call polling. */ @@ -115,25 +115,13 @@ export function createVoiceCallContinueOperationStore(params: { pollTimeoutMs, }); - void request.rt.manager - .continueCall(request.callId, request.message) + void request + .run() .then((result) => { const current = operations.get(operationId); if (!current || current.status !== "pending") { return; } - if (!result.success) { - operations.set(operationId, { - operationId, - status: "failed", - callId: request.callId, - startedAtMs, - completedAtMs: Date.now(), - pollTimeoutMs, - error: result.error || "continue failed", - }); - return; - } operations.set(operationId, { operationId, status: "completed", diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index 9c6204d8856c..7e8e4b3753da 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -189,6 +189,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +1: typed owner-required error for session store path resolution. // +1: native approval messaging target resolver. // +1: shared plugin SecretRef setup plan helper. + // +2: shared low-cardinality diagnostic dimension normalizers. + // +1: shared plugin SecretRef setup CLI factory. // +1: shared multi-claim ingress lifecycle fan-in. // +3: channel prompt-context entry/compat types and channel metadata builder. // +4: focused CLI root-option constants and parsers. @@ -210,7 +212,7 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +45: restore typed session-catalog and tool-results exports promised to plugins. // +1: forwarding-routed approver-restricted native approval capability factory. // +1: shared inbound-event delivery correlation factory for channel plugins. - 4822, + 4825, env, ), publicFunctionExports: readPluginSdkSurfaceBudgetEnv( @@ -234,6 +236,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +2: focused media-local-roots helpers. // +3: channel DM policy factory and its account/patch callbacks. // +1: native approval messaging target resolver. + // +2: shared low-cardinality diagnostic dimension normalizers. + // +1: shared plugin SecretRef setup CLI factory. // +1: shared multi-claim ingress lifecycle fan-in. // +1: channel metadata builder. // +3: focused CLI root-option parsers. @@ -252,7 +256,7 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +14: restore callable session-catalog and tool-results helpers promised to plugins. // +1: forwarding-routed approver-restricted native approval capability factory. // +1: shared inbound-event delivery correlation factory for channel plugins. - 2899, + 2902, env, ), publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv( diff --git a/src/plugin-sdk/diagnostic-runtime.ts b/src/plugin-sdk/diagnostic-runtime.ts index fada4d6d2fc0..e744fe0f4ca6 100644 --- a/src/plugin-sdk/diagnostic-runtime.ts +++ b/src/plugin-sdk/diagnostic-runtime.ts @@ -1,5 +1,36 @@ // Diagnostic flag/event helpers for plugins that want narrow runtime gating. +import { redactSensitiveText } from "../logging/redact.js"; + +const LOW_CARDINALITY_DIAGNOSTIC_VALUE_RE = /^[A-Za-z0-9_.:-]{1,120}$/u; + +export function normalizeDiagnosticValue(value: string | undefined, fallback = "unknown"): string { + if (!value) { + return fallback; + } + const redacted = redactSensitiveText(value.trim()); + const redactedLower = redacted.toLowerCase(); + // Session-shaped agent identifiers are unbounded and must never become exporter dimensions. + if (redactedLower.startsWith("agent:") || redactedLower.includes(":agent:")) { + return fallback; + } + return LOW_CARDINALITY_DIAGNOSTIC_VALUE_RE.test(redacted) ? redacted : fallback; +} + +export function normalizeDiagnosticLane(value: string | undefined, fallback = "unknown"): string { + if (!value) { + return fallback; + } + const redacted = redactSensitiveText(value.trim()); + if (redacted.toLowerCase().startsWith("agent:")) { + return fallback; + } + // Scoped lane suffixes carry session identity; exporters group only by the stable lane prefix. + const scopedLaneIndex = redacted.indexOf(":"); + const lane = scopedLaneIndex >= 0 ? redacted.slice(0, scopedLaneIndex) : redacted; + return LOW_CARDINALITY_DIAGNOSTIC_VALUE_RE.test(lane) ? lane : fallback; +} + export { isDiagnosticFlagEnabled } from "../infra/diagnostic-flags.js"; export type { DiagnosticEventMetadata, diff --git a/src/plugin-sdk/secret-ref-runtime.ts b/src/plugin-sdk/secret-ref-runtime.ts index a29db11c63d8..63b1f2e838f5 100644 --- a/src/plugin-sdk/secret-ref-runtime.ts +++ b/src/plugin-sdk/secret-ref-runtime.ts @@ -1,6 +1,12 @@ // Narrow shared secret-ref helpers for plugin config and secret-contract paths. import fs from "node:fs/promises"; +import path from "node:path"; +import { createInterface } from "node:readline/promises"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { PluginIntegrationSecretProviderConfig } from "../config/types.secrets.js"; import { sameFileIdentity } from "../infra/fs-safe-advanced.js"; import { assertValidPluginModelProviderId, @@ -17,6 +23,63 @@ import { type PlanFileIdentity = { dev: bigint; ino: bigint }; +type SecretRefSetupCommand = { + command(name: string): SecretRefSetupCommand; + description(value: string): SecretRefSetupCommand; + option( + flags: string, + description: string, + defaultValueOrParser?: string | ((value: string, previous?: string[]) => string[]), + defaultValue?: string[], + ): SecretRefSetupCommand; + action(fn: (options: TOptions) => void | Promise): SecretRefSetupCommand; +}; + +type SecretRefSetupOptions = { + planOut?: string; + providerAlias?: string; + openaiId?: string; + anthropicId?: string; + openrouterId?: string; + providerKey?: string[]; + target?: string[]; +}; + +type SecretRefProviderStatus = { + configured: boolean; + source?: string; + command?: string; + pluginIntegration?: { + pluginId: string; + integrationId: string; + }; +}; + +type SecretRefProviderMapping = { + providerId: string; + secretId: string; +}; + +type SecretRefConfigTargetMapping = { + path: string; + agentId?: string; + secretId: string; +}; + +type PluginSecretRefSetupCliParams = { + productName: string; + secretIdLabel: string; + secretIdPlaceholder: string; + defaultProviderAlias: string; + pluginIntegration: { + pluginId: string; + integrationId: string; + }; + normalizeSecretId: (label: string, value: string) => string; + defaultPlanPath: () => string; + beforeApplyCommands?: readonly string[]; +}; + function throwPlanFileError(error: unknown, planPath: string): never { if ((error as NodeJS.ErrnoException)?.code === "EEXIST") { throw new Error(`Plan path already exists; choose a new --plan-out path: ${planPath}`, { @@ -82,6 +145,289 @@ async function writeSecretPlanFile(params: { } } +type CommandShell = "cmd" | "posix" | "powershell"; + +function quoteSecretRefCliArg(value: string, shell: CommandShell): string { + if (/\r|\n/u.test(value)) { + throw new Error("Command argument cannot contain CR or LF"); + } + if (shell === "cmd") { + if (/[%!]/u.test(value)) { + throw new Error("Interactive Command Prompt cannot safely quote paths containing % or !"); + } + const escaped = value.replaceAll('"', '\\"'); + return /[ \t"&|<>^()]/u.test(value) ? `"${escaped}"` : escaped || '""'; + } + if (shell === "powershell") { + return `'${value.replaceAll("'", "''")}'`; + } + if (/^[A-Za-z0-9_/:=.,@%+-]+$/.test(value)) { + return value; + } + return `'${value.replaceAll("'", "'\\''")}'`; +} + +function renderSecretRefApplyCommands( + planPath: string, + platform: NodeJS.Platform = process.platform, +): string[] { + const render = (shell: CommandShell, indent = "") => { + const quotedPlanPath = quoteSecretRefCliArg(planPath, shell); + return [ + `${indent}openclaw secrets apply --from ${quotedPlanPath} --dry-run --allow-exec`, + `${indent}openclaw secrets apply --from ${quotedPlanPath} --allow-exec`, + ]; + }; + if (platform !== "win32") { + return render("posix"); + } + // The parent shell is unknown, so emit native variants instead of unsafe hybrid syntax. + const powershellCommands = ["PowerShell:", ...render("powershell", " ")]; + if (/[%!]/u.test(planPath)) { + return [ + ...powershellCommands, + "Command Prompt: unavailable for paths containing % or !; use PowerShell.", + ]; + } + return [...powershellCommands, "Command Prompt:", ...render("cmd", " ")]; +} + +function readSecretRefProviderStatus( + config: OpenClawConfig, + providerAlias: string, +): SecretRefProviderStatus { + const provider = config.secrets?.providers?.[providerAlias]; + if (!isRecord(provider)) { + return { configured: false }; + } + const base = { + configured: true, + source: normalizeOptionalString(provider.source), + }; + if (provider.source !== "exec") { + return base; + } + if ("pluginIntegration" in provider) { + return { + ...base, + pluginIntegration: provider.pluginIntegration as SecretRefProviderStatus["pluginIntegration"], + }; + } + return { + ...base, + command: normalizeOptionalString(provider.command), + }; +} + +function writeSecretRefCliLine(message = ""): void { + process.stdout.write(`${message}\n`); +} + +/** Build the canonical setup/status adapter shared by plugin-owned SecretRef CLIs. */ +export function createPluginSecretRefSetupCli(params: PluginSecretRefSetupCliParams) { + const isIntegrationProvider = (value: unknown): boolean => + isRecord(value) && + value.source === "exec" && + isRecord(value.pluginIntegration) && + value.pluginIntegration.pluginId === params.pluginIntegration.pluginId && + value.pluginIntegration.integrationId === params.pluginIntegration.integrationId; + + const inspectProvider = (config: OpenClawConfig, requestedAlias?: string) => { + const explicitAlias = normalizeOptionalString(requestedAlias); + let providerAlias: string; + if (explicitAlias) { + assertValidPluginSecretProviderAlias(explicitAlias); + providerAlias = explicitAlias; + } else { + const configuredAliases = Object.entries(config.secrets?.providers ?? {}) + .filter(([, provider]) => isIntegrationProvider(provider)) + .map(([alias]) => alias) + .toSorted(); + if (configuredAliases.length > 1) { + throw new Error( + `Multiple ${params.productName} provider aliases are configured (${configuredAliases.join(", ")}). Use --provider-alias .`, + ); + } + providerAlias = configuredAliases[0] ?? params.defaultProviderAlias; + } + return { + providerAlias, + provider: readSecretRefProviderStatus(config, providerAlias), + providerReady: isIntegrationProvider(config.secrets?.providers?.[providerAlias]), + }; + }; + + const parseProviderKeyMappings = (values: string[] | undefined): SecretRefProviderMapping[] => + (values ?? []).map((value) => { + const separator = value.indexOf("="); + if (separator <= 0 || separator === value.length - 1) { + throw new Error( + `Invalid --provider-key value "${value}". Use =<${params.secretIdPlaceholder}>.`, + ); + } + const providerId = value.slice(0, separator).trim(); + assertValidPluginModelProviderId("--provider-key", providerId); + return { + providerId, + secretId: params.normalizeSecretId( + `--provider-key ${providerId}`, + value.slice(separator + 1).trim(), + ), + }; + }); + + const parseConfigTargetMappings = ( + values: string[] | undefined, + ): SecretRefConfigTargetMapping[] => + (values ?? []).map((value) => { + const separator = value.indexOf("="); + if (separator <= 0 || separator === value.length - 1) { + throw new Error( + `Invalid --target value "${value}". Use =<${params.secretIdPlaceholder}>.`, + ); + } + const target = parsePluginSecretTargetSpecifier( + params.productName, + value.slice(0, separator).trim(), + ); + const secretId = params.normalizeSecretId( + `--target ${target.path}`, + value.slice(separator + 1).trim(), + ); + return Object.assign( + { path: target.path, secretId }, + target.agentId ? { agentId: target.agentId } : {}, + ); + }); + + const promptOptionalSecretId = async (label: string): Promise => { + if (!process.stdin.isTTY || !process.stdout.isTTY) { + return undefined; + } + const readline = createInterface({ input: process.stdin, output: process.stdout }); + try { + return normalizeOptionalString( + await readline.question(`${label} ${params.secretIdLabel} (blank to skip): `), + ); + } finally { + readline.close(); + } + }; + + const collectProviderSecrets = async ( + options: SecretRefSetupOptions, + ): Promise => { + const commonProviders = [ + { providerId: "openai", label: "OpenAI", value: options.openaiId }, + { providerId: "anthropic", label: "Anthropic", value: options.anthropicId }, + { providerId: "openrouter", label: "OpenRouter", value: options.openrouterId }, + ] as const; + const providerSecrets: SecretRefProviderMapping[] = []; + for (const provider of commonProviders) { + const value = + normalizeOptionalString(provider.value) ?? (await promptOptionalSecretId(provider.label)); + if (value) { + providerSecrets.push({ + providerId: provider.providerId, + secretId: params.normalizeSecretId(provider.label, value), + }); + } + } + providerSecrets.push(...parseProviderKeyMappings(options.providerKey)); + const seen = new Set(); + for (const entry of providerSecrets) { + const normalized = entry.providerId.toLowerCase(); + if (seen.has(normalized)) { + throw new Error( + `Duplicate model provider id in ${params.productName} setup: ${entry.providerId}`, + ); + } + seen.add(normalized); + } + return providerSecrets; + }; + + const runSetup = async (options: SecretRefSetupOptions): Promise => { + const providerAlias = + normalizeOptionalString(options.providerAlias) ?? params.defaultProviderAlias; + assertValidPluginSecretProviderAlias(providerAlias); + const providerConfig: PluginIntegrationSecretProviderConfig = { + source: "exec", + pluginIntegration: params.pluginIntegration, + }; + const plan = buildPluginSecretRefSetupPlan({ + productName: params.productName, + providerAlias, + providerConfig, + providerSecrets: await collectProviderSecrets(options), + configTargetSecrets: parseConfigTargetMappings(options.target), + }); + if (plan.targets.length === 0) { + throw new Error( + "No SecretRef targets selected. Pass --openai-id, --anthropic-id, --openrouter-id, --provider-key, or --target.", + ); + } + const requestedPlanPath = normalizeOptionalString(options.planOut) ?? params.defaultPlanPath(); + const absolutePlanPath = path.resolve(requestedPlanPath); + const planDirectory = await resolveTrustedPlanDirectoryPath(path.dirname(absolutePlanPath)); + // Use the verified canonical parent for both the write and copy-paste commands. + const planPath = path.join(planDirectory, path.basename(absolutePlanPath)); + const applyCommands = renderSecretRefApplyCommands(planPath); + await writeSecretPlanFile({ + planPath, + content: `${JSON.stringify(plan, null, 2)}\n`, + }); + writeSecretRefCliLine(`Plan written to ${planPath}`); + writeSecretRefCliLine(`Targets: ${plan.targets.length}`); + writeSecretRefCliLine(); + writeSecretRefCliLine("Next steps:"); + for (const command of params.beforeApplyCommands ?? []) { + writeSecretRefCliLine(` ${command}`); + } + for (const command of applyCommands) { + writeSecretRefCliLine(` ${command}`); + } + writeSecretRefCliLine(" openclaw secrets audit --check --allow-exec"); + writeSecretRefCliLine(" openclaw secrets reload"); + }; + + const registerSetupCommand = (command: SecretRefSetupCommand): void => { + command + .command("setup") + .description(`Create a ${params.productName} SecretRef setup plan`) + .option("--plan-out ", "Write the generated secrets apply plan to a path") + .option( + "--provider-alias ", + "Secret provider alias to configure", + params.defaultProviderAlias, + ) + .option("--openai-id ", `${params.secretIdLabel} for models.providers.openai.apiKey`) + .option( + "--anthropic-id ", + `${params.secretIdLabel} for models.providers.anthropic.apiKey`, + ) + .option( + "--openrouter-id ", + `${params.secretIdLabel} for models.providers.openrouter.apiKey`, + ) + .option( + "--provider-key ", + `${params.secretIdLabel} for any models.providers..apiKey target`, + (value: string, previous: string[] = []) => [...previous, value], + [], + ) + .option( + "--target ", + `${params.secretIdLabel} for any known SecretRef target path`, + (value: string, previous: string[] = []) => [...previous, value], + [], + ) + .action((options: SecretRefSetupOptions) => runSetup(options)); + }; + + return { inspectProvider, registerSetupCommand }; +} + export { coerceSecretRef } from "../config/types.secrets.js"; export type { SecretInput, SecretRef } from "../config/types.secrets.js"; export { resolveSecretRefValues } from "../secrets/resolve.js"; From a7396255d02300f9cae05c8a4178bfbe4aff1457 Mon Sep 17 00:00:00 2001 From: Sukhdeep <46626869+sukhdeepjohar@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:59:08 +0800 Subject: [PATCH 11/57] fix(litellm): send image edits as multipart/form-data (#118562) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The litellm image provider built edit requests as a JSON body with `images: [{image_url}]`, but LiteLLM's POST /v1/images/edits is a multipart endpoint following OpenAI's edits schema: the reference image must be an uploaded file part named `image` (or `image[]`). LiteLLM never finds an `image` key in a JSON body and fails before contacting the upstream provider: HTTP 500 aimage_edit() missing 1 required positional argument: 'image' so every image edit through this provider fails. Plain generation is unaffected. No JSON variant works. Verified against LiteLLM v1.82.3 with a real 512x512 PNG: plural `images` 500s as above; singular `image` as a data URL or bare base64, string or array, returns HTTP 400 "Invalid image file or mode". Only multipart succeeds. Note that singular `image` clears the 500 and reaches the provider, which looks like progress but never delivers usable bytes. Switch buildEditRequest to return { kind: "multipart", form }, which createOpenAiCompatibleImageGenerationProvider already routes to postMultipartRequest — matching the built-in openai and deepinfra providers. A single reference uses the `image` part name and multiple use repeated `image[]` parts; LiteLLM accepts either as List[UploadFile] and merges them, erroring only when both appear in one request. Also drops the now-unused imageToDataUrl helper and its imports. Tests: the existing edit test asserted the buggy JSON shape, so it now asserts a multipart request, that postJsonRequest is not called, and the single-image part name; a new test covers repeated image[] parts for multiple references. Verified end-to-end against LiteLLM v1.82.3 with a live agent: single-reference and multi-image edits both return real images; both failed with the 500 before. --- .../litellm/image-generation-provider.test.ts | 50 +++++++++++++++++-- .../litellm/image-generation-provider.ts | 41 ++++++++------- 2 files changed, 68 insertions(+), 23 deletions(-) diff --git a/extensions/litellm/image-generation-provider.test.ts b/extensions/litellm/image-generation-provider.test.ts index 269a6b354547..77edb314ec80 100644 --- a/extensions/litellm/image-generation-provider.test.ts +++ b/extensions/litellm/image-generation-provider.test.ts @@ -31,6 +31,15 @@ function mockGeneratedPngResponse() { }); } +function mockEditedPngResponse() { + postMultipartRequestMock.mockResolvedValue({ + response: jsonResponse({ + data: [{ b64_json: Buffer.from("png-bytes").toString("base64") }], + }), + release: vi.fn(async () => {}), + }); +} + function mockObjectArg(mock: unknown, index = -1): Record { const calls = (mock as { mock?: { calls?: Array> } }).mock?.calls ?? []; const call = index < 0 ? calls.at(index) : calls[index]; @@ -147,8 +156,8 @@ describe("litellm image generation provider", () => { }); }); - it("routes to the edit endpoint when input images are provided", async () => { - mockGeneratedPngResponse(); + it("routes to the edit endpoint as multipart when input images are provided", async () => { + mockEditedPngResponse(); const provider = buildLitellmImageGenerationProvider(); await provider.generateImage({ @@ -164,9 +173,40 @@ describe("litellm image generation provider", () => { ], }); - expect(mockObjectArg(postJsonRequestMock).url).toBe("http://localhost:4000/images/edits"); - const call = postJsonRequestMock.mock.calls[0]?.[0] as { body: { images: unknown[] } }; - expect(call.body.images).toHaveLength(1); + // Edits must be multipart, never JSON: LiteLLM's /images/edits maps onto + // `aimage_edit(image=...)` and rejects a JSON body outright. + expect(postJsonRequestMock).not.toHaveBeenCalled(); + expect(mockObjectArg(postMultipartRequestMock).url).toBe("http://localhost:4000/images/edits"); + + const form = mockObjectArg(postMultipartRequestMock).body as FormData; + expect(form.get("model")).toBe("gpt-image-2"); + expect(form.get("prompt")).toBe("refine the hero"); + // A single reference uses the singular `image` part name. + expect(form.getAll("image")).toHaveLength(1); + expect(form.getAll("image[]")).toHaveLength(0); + expect(form.get("image")).toBeInstanceOf(Blob); + }); + + it("sends multiple reference images as repeated image[] parts", async () => { + mockEditedPngResponse(); + + const provider = buildLitellmImageGenerationProvider(); + await provider.generateImage({ + provider: "litellm", + model: "gpt-image-2", + prompt: "merge these", + cfg: {}, + inputImages: [ + { buffer: Buffer.from("first"), mimeType: "image/png" }, + { buffer: Buffer.from("second"), mimeType: "image/jpeg" }, + ], + }); + + const form = mockObjectArg(postMultipartRequestMock).body as FormData; + // Both names are accepted by OpenAI-compatible edit endpoints, but only one + // may be present per request — sending both is an error. + expect(form.getAll("image[]")).toHaveLength(2); + expect(form.getAll("image")).toHaveLength(0); }); it("throws a clear error when the API key is missing", async () => { diff --git a/extensions/litellm/image-generation-provider.ts b/extensions/litellm/image-generation-provider.ts index 00b5d32bab46..08d8c472270a 100644 --- a/extensions/litellm/image-generation-provider.ts +++ b/extensions/litellm/image-generation-provider.ts @@ -3,9 +3,8 @@ import { isIP } from "node:net"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { createOpenAiCompatibleImageGenerationProvider, + imageSourceUploadFileName, type ImageGenerationProvider, - type ImageGenerationSourceImage, - toImageDataUrl, } from "openclaw/plugin-sdk/image-generation"; import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; import { LITELLM_BASE_URL } from "./onboard.js"; @@ -41,10 +40,6 @@ function resolveConfiguredLitellmBaseUrl(cfg: OpenClawConfig | undefined): strin return normalizeOptionalString(resolveLitellmProviderConfig(cfg)?.baseUrl) ?? LITELLM_BASE_URL; } -function imageToDataUrl(image: ImageGenerationSourceImage): string { - return toImageDataUrl({ buffer: image.buffer, mimeType: image.mimeType }); -} - // LiteLLM's default proxy is loopback. Auto-enable private-network access only // for loopback-style hosts; LAN/custom private endpoints should use the // explicit models.providers.litellm.request.allowPrivateNetwork opt-in. @@ -124,18 +119,28 @@ export function buildLitellmImageGenerationProvider(): ImageGenerationProvider { size: req.size ?? DEFAULT_SIZE, }, }), - buildEditRequest: ({ req, inputImages, model, count }) => ({ - kind: "json", - body: { - model, - prompt: req.prompt, - n: count, - size: req.size ?? DEFAULT_SIZE, - images: inputImages.map((image) => ({ - image_url: imageToDataUrl(image), - })), - }, - }), + // LiteLLM's /v1/images/edits is multipart (OpenAI's edits schema): the + // reference image must be an uploaded file part, not a JSON field — a JSON + // body fails before the request reaches the provider. + buildEditRequest: ({ req, inputImages, model, count }) => { + const form = new FormData(); + form.set("model", model); + form.set("prompt", req.prompt); + form.set("n", String(count)); + form.set("size", req.size ?? DEFAULT_SIZE); + // OpenAI-compatible edits take repeated `image[]` parts when more than one + // reference is supplied, and a single `image` part otherwise. + const partName = inputImages.length > 1 ? "image[]" : "image"; + for (const [index, image] of inputImages.entries()) { + const mimeType = normalizeOptionalString(image.mimeType) ?? "image/png"; + form.append( + partName, + new Blob([new Uint8Array(image.buffer)], { type: mimeType }), + imageSourceUploadFileName({ image, index }), + ); + } + return { kind: "multipart", form }; + }, missingApiKeyError: "LiteLLM API key missing", failureLabels: { generate: "LiteLLM image generation failed", From cc940b5ff5f99f00f178b22aa4ee4cabf6a22504 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 02:59:47 -0700 Subject: [PATCH 12/57] test(agents): reuse managed worktree fixtures (#118619) Co-authored-by: Peter Steinberger --- src/agents/worktrees/run-lease.test.ts | 1 + src/agents/worktrees/service.naming.test.ts | 29 +- .../worktrees/service.provisioned.test.ts | 46 ++- .../worktrees/service.remove-lease.test.ts | 1 + src/agents/worktrees/service.test-support.ts | 68 ++++ src/agents/worktrees/service.test.ts | 294 +++++++++--------- 6 files changed, 275 insertions(+), 164 deletions(-) create mode 100644 src/agents/worktrees/service.test-support.ts diff --git a/src/agents/worktrees/run-lease.test.ts b/src/agents/worktrees/run-lease.test.ts index 8bd363601f13..947a3d0e6870 100644 --- a/src/agents/worktrees/run-lease.test.ts +++ b/src/agents/worktrees/run-lease.test.ts @@ -77,6 +77,7 @@ describe("worktree run lease", () => { const created = await service.create({ repoRoot: repo, name: "run-lease-session", + baseRef: "HEAD", ownerKind: "session", ownerId: "agent:main:run-lease", }); diff --git a/src/agents/worktrees/service.naming.test.ts b/src/agents/worktrees/service.naming.test.ts index b3db34e6aa7d..9b7798208b49 100644 --- a/src/agents/worktrees/service.naming.test.ts +++ b/src/agents/worktrees/service.naming.test.ts @@ -42,9 +42,13 @@ describe("ManagedWorktreeService naming", () => { }); it("uses readable defaults and numbers colliding inferred names", async () => { - const fallback = await service.create({ repoRoot: repo }); - await service.create({ repoRoot: repo, name: "release-planning" }); - const second = await service.create({ repoRoot: repo, suggestedName: "release-planning" }); + const fallback = await service.create({ repoRoot: repo, baseRef: "HEAD" }); + await service.create({ repoRoot: repo, name: "release-planning", baseRef: "HEAD" }); + const second = await service.create({ + repoRoot: repo, + suggestedName: "release-planning", + baseRef: "HEAD", + }); expect(fallback.name).toMatch( /^[a-z]+-(?:barnacle|claw|crab|crayfish|krill|langoustine|lobster|prawn|shrimp|shell)$/, @@ -53,19 +57,23 @@ describe("ManagedWorktreeService naming", () => { }); it("numbers inferred names around unmanaged Git and filesystem collisions", async () => { - const anchor = await service.create({ repoRoot: repo, name: "anchor" }); + const anchor = await service.create({ repoRoot: repo, name: "anchor", baseRef: "HEAD" }); await git(repo, "branch", "openclaw/release-planning"); await fs.mkdir(path.join(path.dirname(anchor.path), "release-planning-2")); - const created = await service.create({ repoRoot: repo, suggestedName: "release-planning" }); + const created = await service.create({ + repoRoot: repo, + suggestedName: "release-planning", + baseRef: "HEAD", + }); expect(created.name).toBe("release-planning-3"); }); it("serializes concurrent inferred-name creation", async () => { const created = await Promise.all([ - service.create({ repoRoot: repo, suggestedName: "concurrent-task" }), - service.create({ repoRoot: repo, suggestedName: "concurrent-task" }), + service.create({ repoRoot: repo, suggestedName: "concurrent-task", baseRef: "HEAD" }), + service.create({ repoRoot: repo, suggestedName: "concurrent-task", baseRef: "HEAD" }), ]); expect(created.map((record) => record.name).toSorted()).toEqual([ @@ -77,6 +85,7 @@ describe("ManagedWorktreeService naming", () => { it("reuses concurrent inferred names for the same owner", async () => { const owner = { repoRoot: repo, + baseRef: "HEAD", ownerKind: "session" as const, ownerId: "agent:main:session-1", }; @@ -93,11 +102,11 @@ describe("ManagedWorktreeService naming", () => { }); it("serializes overlapping numeric suffix families", async () => { - await service.create({ repoRoot: repo, name: "task" }); + await service.create({ repoRoot: repo, name: "task", baseRef: "HEAD" }); const created = await Promise.all([ - service.create({ repoRoot: repo, suggestedName: "task" }), - service.create({ repoRoot: repo, suggestedName: "task-2" }), + service.create({ repoRoot: repo, suggestedName: "task", baseRef: "HEAD" }), + service.create({ repoRoot: repo, suggestedName: "task-2", baseRef: "HEAD" }), ]); const names = created.map((record) => record.name); diff --git a/src/agents/worktrees/service.provisioned.test.ts b/src/agents/worktrees/service.provisioned.test.ts index 34837b6bc80c..561c53e6686a 100644 --- a/src/agents/worktrees/service.provisioned.test.ts +++ b/src/agents/worktrees/service.provisioned.test.ts @@ -83,7 +83,7 @@ describe("ManagedWorktreeService provisioned state", () => { await fs.writeFile(path.join(repo, "large.local"), source); await addRemote(root, repo); - const created = await service.create({ repoRoot: repo, name: "large-local" }); + const created = await service.create({ repoRoot: repo, name: "large-local", baseRef: "HEAD" }); await service.acquire(created.id); const copyPath = path.join(created.path, "large.local"); const copy = Buffer.from(source); @@ -105,9 +105,21 @@ describe("ManagedWorktreeService provisioned state", () => { await fs.writeFile(path.join(repo, "settings.local"), "theme=source\n"); await addRemote(root, repo); - const manifestRemoved = await service.create({ repoRoot: repo, name: "manifest-removed" }); - const patternRemoved = await service.create({ repoRoot: repo, name: "pattern-removed" }); - const restorable = await service.create({ repoRoot: repo, name: "manifest-restorable" }); + const manifestRemoved = await service.create({ + repoRoot: repo, + name: "manifest-removed", + baseRef: "HEAD", + }); + const patternRemoved = await service.create({ + repoRoot: repo, + name: "pattern-removed", + baseRef: "HEAD", + }); + const restorable = await service.create({ + repoRoot: repo, + name: "manifest-restorable", + baseRef: "HEAD", + }); await service.acquire(manifestRemoved.id); await service.acquire(patternRemoved.id); await service.acquire(restorable.id); @@ -177,7 +189,11 @@ describe("ManagedWorktreeService provisioned state", () => { await git(repo, "commit", "-m", "configure worktree provisioning"); await fs.writeFile(path.join(repo, ".env.local"), "value=source\n"); - const tracked = await service.create({ repoRoot: repo, name: "tracked-provisioned" }); + const tracked = await service.create({ + repoRoot: repo, + name: "tracked-provisioned", + baseRef: "HEAD", + }); await git(tracked.path, "add", "-f", ".env.local"); await git(tracked.path, "commit", "-m", "track provisioned file"); await expect(service.remove({ id: tracked.id, reason: "manual" })).rejects.toThrow( @@ -188,7 +204,11 @@ describe("ManagedWorktreeService provisioned state", () => { "provisioned path is tracked at HEAD", ); - const unignored = await service.create({ repoRoot: repo, name: "unignored-provisioned" }); + const unignored = await service.create({ + repoRoot: repo, + name: "unignored-provisioned", + baseRef: "HEAD", + }); await fs.writeFile(path.join(unignored.path, ".gitignore"), ""); await expect(service.remove({ id: unignored.id, reason: "manual" })).rejects.toThrow( "provisioned path is no longer ignored", @@ -212,7 +232,11 @@ describe("ManagedWorktreeService provisioned state", () => { await fs.writeFile(path.join(repo, wildcardName), "wildcard source\n"); await fs.writeFile(path.join(repo, backslashName), "backslash source\n"); - const created = await service.create({ repoRoot: repo, name: "literal-paths" }); + const created = await service.create({ + repoRoot: repo, + name: "literal-paths", + baseRef: "HEAD", + }); await fs.writeFile(path.join(created.path, wildcardName), "wildcard local\n"); await fs.writeFile(path.join(created.path, backslashName), "backslash local\n"); await service.remove({ id: created.id, reason: "test" }); @@ -228,7 +252,11 @@ describe("ManagedWorktreeService provisioned state", () => { ); it("snapshots deleted skip-worktree files still included by sparse rules", async () => { - const created = await service.create({ repoRoot: repo, name: "stale-sparse-bit" }); + const created = await service.create({ + repoRoot: repo, + name: "stale-sparse-bit", + baseRef: "HEAD", + }); await git(created.path, "sparse-checkout", "set", "--no-cone", "/*"); await git(created.path, "update-index", "--skip-worktree", "README.md"); await fs.rm(path.join(created.path, "README.md")); @@ -250,7 +278,7 @@ describe("ManagedWorktreeService provisioned state", () => { await git(repo, "add", "-A"); await git(repo, "commit", "-m", "add raw path"); - const created = await service.create({ repoRoot: repo, name: "raw-path" }); + const created = await service.create({ repoRoot: repo, name: "raw-path", baseRef: "HEAD" }); const worktreePath = Buffer.concat([ Buffer.from(created.path), Buffer.from(path.sep), diff --git a/src/agents/worktrees/service.remove-lease.test.ts b/src/agents/worktrees/service.remove-lease.test.ts index 23187384cd61..1e470b0de8f0 100644 --- a/src/agents/worktrees/service.remove-lease.test.ts +++ b/src/agents/worktrees/service.remove-lease.test.ts @@ -55,6 +55,7 @@ describe("ManagedWorktreeService removal against a live run lease", () => { const created = await service.create({ repoRoot: repo, name: "removal-session", + baseRef: "HEAD", ownerKind: "session", ownerId: "agent:main:removal", }); diff --git a/src/agents/worktrees/service.test-support.ts b/src/agents/worktrees/service.test-support.ts new file mode 100644 index 000000000000..cb0eb61b2bb4 --- /dev/null +++ b/src/agents/worktrees/service.test-support.ts @@ -0,0 +1,68 @@ +import { execFile } from "node:child_process"; +import { constants as fsConstants } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; +import { insertRegistryWorktree } from "./registry.js"; +import type { ManagedWorktreeOwnerKind, ManagedWorktreeRecord } from "./types.js"; + +const execFileAsync = promisify(execFile); + +async function git(cwd: string, ...args: string[]): Promise { + await execFileAsync("git", ["-C", cwd, ...args]); +} + +async function copyProvisionedFiles(params: { + repoRoot: string; + worktreePath: string; + provisionedPaths: readonly string[]; +}): Promise { + for (const provisionedPath of params.provisionedPaths) { + const source = path.join(params.repoRoot, provisionedPath); + const target = path.join(params.worktreePath, provisionedPath); + const sourceStat = await fs.lstat(source); + await fs.mkdir(path.dirname(target), { recursive: true }); + await fs.copyFile(source, target, fsConstants.COPYFILE_FICLONE); + if (process.platform !== "win32") { + await fs.chmod(target, sourceStat.mode & 0o7777); + } + } +} + +export async function materializeManagedWorktreeFixture(params: { + env: NodeJS.ProcessEnv; + name: string; + now: number; + ownerKind?: ManagedWorktreeOwnerKind; + ownerId?: string; + provisionedPaths?: readonly string[]; + repoRoot: string; + stateDir: string; +}): Promise { + const repoFingerprint = "downstream-fixture"; + const worktreePath = path.join(params.stateDir, "worktrees", repoFingerprint, params.name); + const branch = `openclaw/${params.name}`; + await fs.mkdir(path.dirname(worktreePath), { recursive: true }); + await git(params.repoRoot, "worktree", "add", "-b", branch, "--", worktreePath, "HEAD"); + const provisionedPaths = params.provisionedPaths ?? []; + await copyProvisionedFiles({ + repoRoot: params.repoRoot, + worktreePath, + provisionedPaths, + }); + const record: ManagedWorktreeRecord = { + id: `fixture-${params.name}`, + name: params.name, + repoFingerprint, + repoRoot: params.repoRoot, + path: worktreePath, + branch, + baseRef: "HEAD", + ownerKind: params.ownerKind ?? "manual", + ...(params.ownerId ? { ownerId: params.ownerId } : {}), + createdAt: params.now, + lastActiveAt: params.now, + }; + insertRegistryWorktree(params.env, record, { provisionedPaths }); + return record; +} diff --git a/src/agents/worktrees/service.test.ts b/src/agents/worktrees/service.test.ts index de077c87595b..4a6a998887fb 100644 --- a/src/agents/worktrees/service.test.ts +++ b/src/agents/worktrees/service.test.ts @@ -21,6 +21,7 @@ import { resolveWorktreeCleanupLimits, SNAPSHOT_RETENTION_MS, } from "./service.js"; +import { materializeManagedWorktreeFixture } from "./service.test-support.js"; const execFileAsync = promisify(execFile); @@ -79,6 +80,34 @@ describe("ManagedWorktreeService", () => { let env: NodeJS.ProcessEnv; let now: number; let service: ManagedWorktreeService; + let caseOrdinal = 0; + + // Snapshot/removal/GC tests need the real Git-worktree + registry boundary, + // while create policy and provisioning composition stay covered above. + async function materializeDownstreamFixture( + name: string, + params: { + ownerKind?: "manual" | "session" | "workboard"; + ownerId?: string; + provisionedPaths?: readonly string[]; + repoRoot?: string; + } = {}, + ) { + return await materializeManagedWorktreeFixture({ + env, + name, + now, + repoRoot: params.repoRoot ?? repo, + stateDir, + ...params, + }); + } + + const materializeRunOwnedFixture = ( + name: string, + ownerKind: "session" | "workboard", + ownerId?: string, + ) => materializeDownstreamFixture(name, { ownerKind, ownerId }); beforeAll(async () => { const tempRoot = await fs.realpath(os.tmpdir()); @@ -97,8 +126,8 @@ describe("ManagedWorktreeService", () => { }); beforeEach(async () => { - const tempRoot = await fs.realpath(os.tmpdir()); - root = await fs.mkdtemp(path.join(tempRoot, "openclaw-managed-worktrees-")); + root = path.join(templateRoot, `case-${caseOrdinal++}`); + await fs.mkdir(root); repo = path.join(root, "repo"); await fs.cp(templateRepo, repo, { mode: fsConstants.COPYFILE_FICLONE, @@ -116,7 +145,6 @@ describe("ManagedWorktreeService", () => { deleteRegistryWorktree(env, record.id); } await fs.rm(path.join(stateDir, "worktrees"), { recursive: true, force: true }); - await fs.rm(root, { recursive: true, force: true }); }); it("creates from origin HEAD and returns the existing live named worktree", async () => { @@ -135,6 +163,7 @@ describe("ManagedWorktreeService", () => { const created = await service.create({ repoRoot: repo, name: "session-owned", + baseRef: "HEAD", ownerKind: "session", ownerId: "session-1", }); @@ -322,6 +351,7 @@ describe("ManagedWorktreeService", () => { await service.create({ repoRoot: repo, name: "shared-name", + baseRef: "HEAD", ownerKind: "session", ownerId: "agent:main:dashboard:one", }); @@ -329,17 +359,19 @@ describe("ManagedWorktreeService", () => { service.create({ repoRoot: repo, name: "shared-name", + baseRef: "HEAD", ownerKind: "session", ownerId: "agent:main:dashboard:two", }), ).rejects.toThrow(/already in use by session/); - await expect(service.create({ repoRoot: repo, name: "shared-name" })).rejects.toThrow( - /already in use by session/, - ); + await expect( + service.create({ repoRoot: repo, name: "shared-name", baseRef: "HEAD" }), + ).rejects.toThrow(/already in use by session/); // The rightful owner still reuses its record. const reused = await service.create({ repoRoot: repo, name: "shared-name", + baseRef: "HEAD", ownerKind: "session", ownerId: "agent:main:dashboard:one", }); @@ -377,7 +409,11 @@ describe("ManagedWorktreeService", () => { const linked = path.join(root, "linked-source"); await git(repo, "worktree", "add", "-b", "linked-source", linked, "HEAD"); const linkedRoot = await fs.realpath(linked); - const created = await service.create({ repoRoot: linkedRoot, name: "linked-task" }); + const created = await service.create({ + repoRoot: linkedRoot, + name: "linked-task", + baseRef: "HEAD", + }); expect(created.repoRoot).toBe(repo); await git(repo, "worktree", "remove", "--force", linkedRoot); @@ -431,7 +467,7 @@ describe("ManagedWorktreeService", () => { await fs.writeFile(path.join(outsideDir, "escape.txt"), "outside\n"); await fs.symlink(outsideDir, path.join(repo, "linked-dir")); - const created = await service.create({ repoRoot: repo, name: "includes" }); + const created = await service.create({ repoRoot: repo, name: "includes", baseRef: "HEAD" }); const copied = path.join(created.path, "cache", "keep.txt"); expect(await fs.readFile(copied, "utf8")).toBe("keep\n"); expect((await fs.stat(copied)).mode & 0o777).toBe(0o744); @@ -478,7 +514,7 @@ describe("ManagedWorktreeService", () => { '#!/bin/sh\nprintf "%s\\n%s\\n" "$OPENCLAW_SOURCE_TREE_PATH" "$OPENCLAW_WORKTREE_PATH" > setup-paths.txt\n', { mode: 0o755 }, ); - const created = await service.create({ repoRoot: repo, name: "setup" }); + const created = await service.create({ repoRoot: repo, name: "setup", baseRef: "HEAD" }); expect( (await fs.readFile(path.join(created.path, "setup-paths.txt"), "utf8")).split("\n"), ).toEqual([repo, created.path, ""]); @@ -499,7 +535,12 @@ describe("ManagedWorktreeService", () => { { mode: 0o755 }, ); - await service.create({ repoRoot: repo, name: "no-repo-code", runSetupScript: false }); + await service.create({ + repoRoot: repo, + name: "no-repo-code", + baseRef: "HEAD", + runSetupScript: false, + }); await expect(fs.access(hookMarker)).rejects.toMatchObject({ code: "ENOENT" }); await expect(fs.access(setupMarker)).rejects.toMatchObject({ code: "ENOENT" }); @@ -509,9 +550,9 @@ describe("ManagedWorktreeService", () => { await fs.mkdir(path.join(repo, ".openclaw")); const script = path.join(repo, ".openclaw", "worktree-setup.sh"); await fs.writeFile(script, "#!/bin/sh\necho setup-broke >&2\nexit 9\n", { mode: 0o755 }); - await expect(service.create({ repoRoot: repo, name: "broken-setup" })).rejects.toThrow( - "setup-broke", - ); + await expect( + service.create({ repoRoot: repo, name: "broken-setup", baseRef: "HEAD" }), + ).rejects.toThrow("setup-broke"); expect(await git(repo, "worktree", "list", "--porcelain")).not.toContain("broken-setup"); expect(await git(repo, "branch", "--list", "openclaw/broken-setup")).toBe(""); }); @@ -523,7 +564,9 @@ describe("ManagedWorktreeService", () => { await git(repo, "commit", "-m", "configure worktree provisioning"); await fs.writeFile(path.join(repo, "provisioned.env"), "source value\n"); const mode = (await fs.stat(path.join(repo, "provisioned.env"))).mode & 0o7777; - const created = await service.create({ repoRoot: repo, name: "roundtrip" }); + const created = await materializeDownstreamFixture("roundtrip", { + provisionedPaths: ["provisioned.env"], + }); const originalHead = await git(created.path, "rev-parse", "HEAD"); await fs.writeFile(path.join(created.path, "README.md"), "changed\n"); await fs.writeFile(path.join(created.path, "untracked.txt"), "untracked\n"); @@ -583,7 +626,7 @@ describe("ManagedWorktreeService", () => { await git(repo, "commit", "-m", "add executable"); await git(repo, "config", "core.filemode", "false"); - const created = await service.create({ repoRoot: repo, name: "filemode" }); + const created = await materializeDownstreamFixture("filemode"); await fs.chmod(path.join(created.path, "tool.sh"), 0o644); await fs.writeFile(path.join(created.path, "README.md"), "changed\n"); const removed = await service.remove({ id: created.id, reason: "test" }); @@ -592,7 +635,7 @@ describe("ManagedWorktreeService", () => { }); it("snapshots modified tracked files marked assume-unchanged", async () => { - const created = await service.create({ repoRoot: repo, name: "assume-unchanged" }); + const created = await materializeDownstreamFixture("assume-unchanged"); await git(created.path, "update-index", "--assume-unchanged", "README.md"); await fs.writeFile(path.join(created.path, "README.md"), "hidden local change\n"); expect(await git(created.path, "status", "--porcelain")).toBe(""); @@ -606,7 +649,7 @@ describe("ManagedWorktreeService", () => { }); it("snapshots materialized tracked files marked skip-worktree", async () => { - const created = await service.create({ repoRoot: repo, name: "skip-worktree" }); + const created = await materializeDownstreamFixture("skip-worktree"); await git(created.path, "update-index", "--skip-worktree", "README.md"); await fs.writeFile(path.join(created.path, "README.md"), "hidden sparse change\n"); expect(await git(created.path, "status", "--porcelain")).toBe(""); @@ -620,7 +663,7 @@ describe("ManagedWorktreeService", () => { }); it("snapshots deletions hidden by skip-worktree outside sparse checkout", async () => { - const created = await service.create({ repoRoot: repo, name: "skip-worktree-deleted" }); + const created = await materializeDownstreamFixture("skip-worktree-deleted"); await git(created.path, "update-index", "--skip-worktree", "README.md"); await fs.rm(path.join(created.path, "README.md")); expect(await git(created.path, "status", "--porcelain")).toBe(""); @@ -634,7 +677,7 @@ describe("ManagedWorktreeService", () => { }); it("refuses to overwrite a branch recreated before restore", async () => { - const created = await service.create({ repoRoot: repo, name: "restore-collision" }); + const created = await materializeDownstreamFixture("restore-collision"); await service.remove({ id: created.id, reason: "test" }); await git(repo, "branch", created.branch, "HEAD"); const branchTip = await git(repo, "rev-parse", created.branch); @@ -655,7 +698,7 @@ describe("ManagedWorktreeService", () => { await fs.writeFile(path.join(repo, "tracked", "outer.txt"), "tracked\n"); await git(repo, "add", "tracked/outer.txt"); await git(repo, "commit", "-m", "add tracked parent"); - const created = await service.create({ repoRoot: repo, name: "nested-repository" }); + const created = await materializeDownstreamFixture("nested-repository"); const nested = await initializeRepository( path.join(created.path, "tracked"), gitTemplate, @@ -684,6 +727,7 @@ describe("ManagedWorktreeService", () => { const created = await service.create({ repoRoot: repo, name: "wb-card", + baseRef: "HEAD", ownerKind: "workboard", ownerId: "card", }); @@ -706,11 +750,11 @@ describe("ManagedWorktreeService", () => { it("removes lossless run-end worktrees but keeps dirty and unpushed work", async () => { await addRemote(root, repo); - const clean = await service.create({ repoRoot: repo, name: "clean" }); + const clean = await materializeDownstreamFixture("clean"); await service.acquire(clean.id); expect(await service.removeIfLossless(clean.id)).toBe(true); - const dirty = await service.create({ repoRoot: repo, name: "dirty" }); + const dirty = await materializeDownstreamFixture("dirty"); await service.acquire(dirty.id); await fs.writeFile(path.join(dirty.path, "dirty.txt"), "dirty\n"); expect(await service.removeIfLossless(dirty.id)).toBe(false); @@ -718,7 +762,7 @@ describe("ManagedWorktreeService", () => { (await service.list()).find((entry) => entry.id === dirty.id)?.removedAt, ).toBeUndefined(); - const committed = await service.create({ repoRoot: repo, name: "committed" }); + const committed = await materializeDownstreamFixture("committed"); await service.acquire(committed.id); await fs.writeFile(path.join(committed.path, "commit.txt"), "commit\n"); await git(committed.path, "add", "commit.txt"); @@ -734,7 +778,9 @@ describe("ManagedWorktreeService", () => { await fs.writeFile(path.join(repo, ".env.local"), "value=old-source\n"); await addRemote(root, repo); - const rotated = await service.create({ repoRoot: repo, name: "rotated-local" }); + const rotated = await materializeDownstreamFixture("rotated-local", { + provisionedPaths: [".env.local"], + }); await service.acquire(rotated.id); expect(await fs.readFile(path.join(rotated.path, ".env.local"), "utf8")).toBe( "value=old-source\n", @@ -748,13 +794,17 @@ describe("ManagedWorktreeService", () => { "value=rotated-only-copy\n", ); - const rebuildable = await service.create({ repoRoot: repo, name: "rebuildable" }); + const rebuildable = await materializeDownstreamFixture("rebuildable", { + provisionedPaths: [".env.local"], + }); await service.acquire(rebuildable.id); await fs.mkdir(path.join(rebuildable.path, "node_modules"), { recursive: true }); await fs.writeFile(path.join(rebuildable.path, "node_modules", "cache.js"), "cache\n"); expect(await service.removeIfLossless(rebuildable.id)).toBe(true); - const deleted = await service.create({ repoRoot: repo, name: "deleted-local" }); + const deleted = await materializeDownstreamFixture("deleted-local", { + provisionedPaths: [".env.local"], + }); await service.acquire(deleted.id); const deletedCopy = path.join(deleted.path, ".env.local"); await fs.rm(deletedCopy); @@ -776,7 +826,9 @@ describe("ManagedWorktreeService", () => { await fs.writeFile(sourcePath, "value=source\n", { mode: 0o644 }); await addRemote(root, repo); - const executable = await service.create({ repoRoot: repo, name: "executable-local" }); + const executable = await materializeDownstreamFixture("executable-local", { + provisionedPaths: [".env.local"], + }); await service.acquire(executable.id); const executableCopy = path.join(executable.path, ".env.local"); await fs.chmod(executableCopy, 0o755); @@ -787,7 +839,9 @@ describe("ManagedWorktreeService", () => { 0o755, ); - const specialMode = await service.create({ repoRoot: repo, name: "special-mode-local" }); + const specialMode = await materializeDownstreamFixture("special-mode-local", { + provisionedPaths: [".env.local"], + }); await service.acquire(specialMode.id); const specialModeCopy = path.join(specialMode.path, ".env.local"); await fs.chmod(specialModeCopy, 0o1644); @@ -797,7 +851,9 @@ describe("ManagedWorktreeService", () => { (await fs.lstat(path.join(restoredSpecialMode.path, ".env.local"))).mode & 0o7777, ).toBe(0o1644); - const linked = await service.create({ repoRoot: repo, name: "linked-local" }); + const linked = await materializeDownstreamFixture("linked-local", { + provisionedPaths: [".env.local"], + }); await service.acquire(linked.id); const linkedCopy = path.join(linked.path, ".env.local"); await fs.rm(linkedCopy); @@ -806,7 +862,9 @@ describe("ManagedWorktreeService", () => { expect(await service.removeIfLossless(linked.id)).toBe(false); expect((await fs.lstat(linkedCopy)).isSymbolicLink()).toBe(true); - const sourceLinked = await service.create({ repoRoot: repo, name: "source-linked-local" }); + const sourceLinked = await materializeDownstreamFixture("source-linked-local", { + provisionedPaths: [".env.local"], + }); await service.acquire(sourceLinked.id); const outside = path.join(root, "same-local-value"); await fs.writeFile(outside, "value=source\n"); @@ -822,12 +880,8 @@ describe("ManagedWorktreeService", () => { ); it("exempts manual worktrees and garbage collects idle run-owned worktrees", async () => { - const manual = await service.create({ repoRoot: repo, name: "manual-idle" }); - const created = await service.create({ - repoRoot: repo, - name: "idle-dead", - ownerKind: "workboard", - }); + const manual = await materializeDownstreamFixture("manual-idle"); + const created = await materializeRunOwnedFixture("idle-dead", "workboard"); await git(repo, "worktree", "lock", "--reason", "openclaw pid=999999", created.path); now += IDLE_GC_MS + 1; @@ -845,10 +899,9 @@ describe("ManagedWorktreeService", () => { await git(repo, "commit", "-m", "configure worktree provisioning"); await fs.writeFile(path.join(repo, ".env.local"), "value=old-source\n"); - const created = await service.create({ - repoRoot: repo, - name: "idle-rotated", + const created = await materializeDownstreamFixture("idle-rotated", { ownerKind: "workboard", + provisionedPaths: [".env.local"], }); await fs.rm(path.join(repo, ".worktreeinclude")); await fs.writeFile(path.join(created.path, ".env.local"), "value=rotated-only-copy\n"); @@ -863,18 +916,16 @@ describe("ManagedWorktreeService", () => { }); it("uses owner activity to protect only active idle session worktrees", async () => { - const active = await service.create({ - repoRoot: repo, - name: "active-session", - ownerKind: "session", - ownerId: "agent:main:active", - }); - const inactive = await service.create({ - repoRoot: repo, - name: "inactive-session", - ownerKind: "session", - ownerId: "agent:main:inactive", - }); + const active = await materializeRunOwnedFixture( + "active-session", + "session", + "agent:main:active", + ); + const inactive = await materializeRunOwnedFixture( + "inactive-session", + "session", + "agent:main:inactive", + ); now += IDLE_GC_MS + 1; const shouldProtectOwner = vi.fn( (_ownerKind: string, ownerId: string) => ownerId === "agent:main:active", @@ -890,11 +941,7 @@ describe("ManagedWorktreeService", () => { }); it("protects foreign locks during idle garbage collection", async () => { - const created = await service.create({ - repoRoot: repo, - name: "foreign-lock", - ownerKind: "session", - }); + const created = await materializeRunOwnedFixture("foreign-lock", "session"); await git(repo, "worktree", "lock", "--reason", "other-tool", created.path); now += IDLE_GC_MS + 1; @@ -903,17 +950,9 @@ describe("ManagedWorktreeService", () => { }); it("continues garbage collection after one worktree cannot be snapshotted", async () => { - const removable = await service.create({ - repoRoot: repo, - name: "removable", - ownerKind: "workboard", - }); + const removable = await materializeRunOwnedFixture("removable", "workboard"); now += 1; - const nestedRecord = await service.create({ - repoRoot: repo, - name: "nested-idle", - ownerKind: "workboard", - }); + const nestedRecord = await materializeRunOwnedFixture("nested-idle", "workboard"); await initializeRepository(nestedRecord.path, gitTemplate, "nested"); now += IDLE_GC_MS + 1; @@ -926,15 +965,12 @@ describe("ManagedWorktreeService", () => { it("continues garbage collection when one repository control path is missing", async () => { const otherRepo = await initializeRepository(root, gitTemplate, "other-repo"); - const removable = await service.create({ + const removable = await materializeDownstreamFixture("other-removable", { repoRoot: otherRepo, - name: "other-removable", ownerKind: "session", }); now += 1; - const broken = await service.create({ - repoRoot: repo, - name: "missing-control", + const broken = await materializeDownstreamFixture("missing-control", { ownerKind: "session", }); await fs.rename(repo, path.join(root, "moved-repo")); @@ -962,27 +998,12 @@ describe("ManagedWorktreeService", () => { }); it("evicts the least recently active run-owned worktrees over the count limit", async () => { - const manual = await service.create({ repoRoot: repo, name: "manual-kept" }); - const oldest = await service.create({ - repoRoot: repo, - name: "count-oldest", - ownerKind: "session", - ownerId: "agent:main:oldest", - }); + const manual = await materializeDownstreamFixture("manual-kept"); + const oldest = await materializeRunOwnedFixture("count-oldest", "session", "agent:main:oldest"); now += 1; - const middle = await service.create({ - repoRoot: repo, - name: "count-middle", - ownerKind: "workboard", - ownerId: "card-middle", - }); + const middle = await materializeRunOwnedFixture("count-middle", "workboard", "card-middle"); now += 1; - const newest = await service.create({ - repoRoot: repo, - name: "count-newest", - ownerKind: "session", - ownerId: "agent:main:newest", - }); + const newest = await materializeRunOwnedFixture("count-newest", "session", "agent:main:newest"); const result = await service.gc({ limits: { maxCount: 2 } }); @@ -994,19 +1015,13 @@ describe("ManagedWorktreeService", () => { }); it("skips active owners during count-limit eviction", async () => { - const activeOldest = await service.create({ - repoRoot: repo, - name: "limit-active", - ownerKind: "session", - ownerId: "agent:main:active", - }); + const activeOldest = await materializeRunOwnedFixture( + "limit-active", + "session", + "agent:main:active", + ); now += 1; - const idle = await service.create({ - repoRoot: repo, - name: "limit-idle", - ownerKind: "session", - ownerId: "agent:main:idle", - }); + const idle = await materializeRunOwnedFixture("limit-idle", "session", "agent:main:idle"); const shouldProtectOwner = vi.fn( (_ownerKind: string, ownerId: string) => ownerId === "agent:main:active", ); @@ -1018,20 +1033,18 @@ describe("ManagedWorktreeService", () => { }); it("evicts oldest worktrees until total size fits the size limit", async () => { - const oldest = await service.create({ - repoRoot: repo, - name: "size-oldest", - ownerKind: "session", - ownerId: "agent:main:size-old", - }); + const oldest = await materializeRunOwnedFixture( + "size-oldest", + "session", + "agent:main:size-old", + ); await fs.writeFile(path.join(oldest.path, "blob.bin"), Buffer.alloc(10_000)); now += 1; - const newest = await service.create({ - repoRoot: repo, - name: "size-newest", - ownerKind: "session", - ownerId: "agent:main:size-new", - }); + const newest = await materializeRunOwnedFixture( + "size-newest", + "session", + "agent:main:size-new", + ); const result = await service.gc({ limits: { maxTotalSizeBytes: 6_000 } }); @@ -1044,12 +1057,11 @@ describe("ManagedWorktreeService", () => { if (process.getuid?.() === 0) { return; // chmod-based EACCES cannot be simulated as root } - const unreadable = await service.create({ - repoRoot: repo, - name: "size-unreadable", - ownerKind: "session", - ownerId: "agent:main:size-unreadable", - }); + const unreadable = await materializeRunOwnedFixture( + "size-unreadable", + "session", + "agent:main:size-unreadable", + ); await fs.writeFile(path.join(unreadable.path, "blob.bin"), Buffer.alloc(10_000)); const locked = path.join(unreadable.path, "locked"); await fs.mkdir(locked); @@ -1066,26 +1078,23 @@ describe("ManagedWorktreeService", () => { }); it("counts a competing removal instead of evicting an extra worktree", async () => { - const oldest = await service.create({ - repoRoot: repo, - name: "race-oldest", - ownerKind: "session", - ownerId: "agent:main:race-old", - }); + const oldest = await materializeRunOwnedFixture( + "race-oldest", + "session", + "agent:main:race-old", + ); now += 1; - const middle = await service.create({ - repoRoot: repo, - name: "race-middle", - ownerKind: "session", - ownerId: "agent:main:race-mid", - }); + const middle = await materializeRunOwnedFixture( + "race-middle", + "session", + "agent:main:race-mid", + ); now += 1; - const newest = await service.create({ - repoRoot: repo, - name: "race-newest", - ownerKind: "session", - ownerId: "agent:main:race-new", - }); + const newest = await materializeRunOwnedFixture( + "race-newest", + "session", + "agent:main:race-new", + ); const realRemove = service.remove.bind(service); const removeSpy = vi .spyOn(service, "remove") @@ -1107,12 +1116,7 @@ describe("ManagedWorktreeService", () => { }); it("leaves everything in place when limits are not exceeded", async () => { - const created = await service.create({ - repoRoot: repo, - name: "under-limit", - ownerKind: "session", - ownerId: "agent:main:under", - }); + const created = await materializeRunOwnedFixture("under-limit", "session", "agent:main:under"); const result = await service.gc({ limits: { maxCount: 5, maxTotalSizeBytes: 1024 ** 3 }, @@ -1127,7 +1131,7 @@ describe("ManagedWorktreeService", () => { }); it("prunes expired snapshot refs and registry rows", async () => { - const created = await service.create({ repoRoot: repo, name: "expired" }); + const created = await materializeDownstreamFixture("expired"); const removed = await service.remove({ id: created.id, reason: "retention" }); now += SNAPSHOT_RETENTION_MS + 1; From c53e594aae79690d7bc809157c904a5ecd745190 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 03:00:17 -0700 Subject: [PATCH 13/57] fix(slack): stop partially started native streams after fallback (#118617) --- extensions/slack/src/streaming.test.ts | 58 ++++++++++++++++++++++++++ extensions/slack/src/streaming.ts | 2 +- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/extensions/slack/src/streaming.test.ts b/extensions/slack/src/streaming.test.ts index 3015e1077398..69aba8b86f92 100644 --- a/extensions/slack/src/streaming.test.ts +++ b/extensions/slack/src/streaming.test.ts @@ -370,6 +370,64 @@ describe("stopSlackStream finalize error handling", () => { expect(alreadyDelivered.stopped).toBe(false); }); + it("finalizes a stream started during failed stop after fallback delivery", async () => { + const streamTs = "1700000000.500300"; + const startStream = vi.fn(async () => ({ ok: true, ts: streamTs })); + const stopStream = vi + .fn() + .mockRejectedValueOnce(slackApiError("user_not_found")) + .mockResolvedValueOnce({ ok: true, ts: streamTs }); + const client = { + chat: { + startStream, + appendStream: vi.fn(async () => ({ ok: true })), + stopStream, + }, + }; + const streamer = new ChatStreamer( + client as never, + { debug: vi.fn() } as never, + { + channel: "C123", + thread_ts: "1700000000.000100", + }, + { buffer_size: 256 }, + ); + const session: SlackStreamSession = { + streamer, + channel: "C123", + threadTs: "1700000000.000100", + stopped: false, + delivered: false, + pendingText: "", + }; + const metadata = { event_type: "openclaw.reply", event_payload: { turn: "qa" } }; + + await appendSlackStream({ session, text: "short buffered reply" }); + await expect(stopSlackStream({ session, metadata })).rejects.toBeInstanceOf( + SlackStreamNotDeliveredError, + ); + expect(streamer.ts).toBe(streamTs); + expect(session.delivered).toBe(false); + + markSlackStreamFallbackDelivered(session); + expect(session.stopped).toBe(false); + await expect(stopSlackStream({ session, metadata })).resolves.toEqual({ messageId: streamTs }); + + expect(startStream).toHaveBeenCalledOnce(); + expect(stopStream).toHaveBeenCalledTimes(2); + expect(stopStream).toHaveBeenNthCalledWith(2, { + token: undefined, + channel: "C123", + ts: streamTs, + chunks: [], + metadata, + }); + expect(session.stopped).toBe(true); + expect(session.delivered).toBe(true); + expect(session.pendingText).toBe(""); + }); + it("clears the SDK buffer before finalizing an already-visible fallback stream", async () => { const startStream = vi.fn(async () => ({ ok: true, ts: "1700000000.500300" })); const stopStream = vi.fn(async () => ({ ok: true, ts: "1700000000.500300" })); diff --git a/extensions/slack/src/streaming.ts b/extensions/slack/src/streaming.ts index b1284b422681..72893aad00a8 100644 --- a/extensions/slack/src/streaming.ts +++ b/extensions/slack/src/streaming.ts @@ -371,7 +371,7 @@ function extractSlackErrorCode(err: unknown): string | undefined { } export function markSlackStreamFallbackDelivered(session: SlackStreamSession): void { - const nativeStreamWasStarted = session.delivered; + const nativeStreamWasStarted = session.delivered || Boolean(session.streamer.ts); session.pendingText = ""; // @slack/web-api 7.16.0 retains its private buffer after a failed flush. // Clear fallback-owned text before retrying stop(), or the SDK resends it. From f9df87ce6885e715134c5dd1b8bf849e1856add3 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 03:04:30 -0700 Subject: [PATCH 14/57] fix(agents): preserve successful sessions_yield handoffs (#118584) * fix(agents): preserve successful sessions_yield handoffs * fix(agents): require visible replies after yielded subagents * fix(agents): reject progress notices as yielded final replies --- src/agents/agent-tools.abort.ts | 20 +- src/agents/agent-tools.runtime.test.ts | 152 +++++++++++ src/agents/subagent-announce-delivery.test.ts | 239 +++++++++++++++++- src/agents/subagent-announce-delivery.ts | 101 ++++++-- ...ent-announce.requester-settle-wake.test.ts | 59 +++++ ...subagent-announce.requester-settle-wake.ts | 15 +- 6 files changed, 551 insertions(+), 35 deletions(-) diff --git a/src/agents/agent-tools.abort.ts b/src/agents/agent-tools.abort.ts index e31275799620..55c1891a3d9a 100644 --- a/src/agents/agent-tools.abort.ts +++ b/src/agents/agent-tools.abort.ts @@ -20,10 +20,27 @@ function throwAbortError(): never { * Tool settlements pass through untouched to preserve tool error semantics, * including non-Error rejections. */ -function raceWithAbortSignal(promise: Promise, signal: AbortSignal): Promise { +function raceWithAbortSignal( + promise: Promise, + signal: AbortSignal, + yieldRunSignal?: AbortSignal, +): Promise { return new Promise((resolve, reject) => { const onAbort = () => { signal.removeEventListener("abort", onAbort); + const reason = yieldRunSignal?.reason as + | { code?: unknown; turnHandoff?: unknown } + | undefined; + // Only the initiating tool may finish its run owner's deliberate handoff; + // caller-authored aborts and concurrent sibling tools must still cancel. + if ( + yieldRunSignal?.aborted && + signal.reason === reason && + reason?.code === "sessions_yield" && + reason.turnHandoff === true + ) { + return; + } reject(createAbortError("Aborted")); }; signal.addEventListener("abort", onAbort, { once: true }); @@ -67,6 +84,7 @@ export function wrapToolWithAbortSignal( return await raceWithAbortSignal( execute(toolCallId, params, combinedSignal, onUpdate), combinedSignal, + tool.name === "sessions_yield" ? abortSignal : undefined, ); }, }; diff --git a/src/agents/agent-tools.runtime.test.ts b/src/agents/agent-tools.runtime.test.ts index f56ea17a8023..ca90d8d3b04d 100644 --- a/src/agents/agent-tools.runtime.test.ts +++ b/src/agents/agent-tools.runtime.test.ts @@ -14,6 +14,7 @@ import { getToolTerminalPresentation, setToolTerminalPresentation, } from "./tool-terminal-presentation.js"; +import { createSessionsYieldTool } from "./tools/sessions-yield-tool.js"; type ExecuteMock = ReturnType; @@ -84,6 +85,157 @@ describe("wrapToolWithAbortSignal", () => { await flushMicrotasks(); }); + it("preserves the successful result when sessions_yield intentionally aborts its own run", async () => { + const runAbort = new AbortController(); + const handoffReason = { code: "sessions_yield", turnHandoff: true } as const; + const beforeYield = vi.fn(); + const wrapped = wrapToolWithAbortSignal( + createSessionsYieldTool({ + sessionId: "requester", + onBeforeYield: beforeYield, + onYield: () => { + runAbort.abort(handoffReason); + }, + }), + runAbort.signal, + ); + + await expect(wrapped.execute("call-yield", {})).resolves.toMatchObject({ + details: { status: "yielded", message: "Turn yielded." }, + }); + expect(beforeYield).toHaveBeenCalledOnce(); + expect(runAbort.signal.reason).toBe(handoffReason); + }); + + it("still aborts a concurrent sibling when sessions_yield hands off the run", async () => { + const runAbort = new AbortController(); + const handoffReason = { code: "sessions_yield", turnHandoff: true } as const; + const sibling = wrapToolWithAbortSignal( + asAgentTool({ name: "wedged", execute: vi.fn(() => new Promise(() => {})) }), + runAbort.signal, + ); + const siblingAborted = expect(sibling.execute("call-sibling", {})).rejects.toMatchObject({ + name: "AbortError", + message: "Aborted", + }); + const yieldTool = wrapToolWithAbortSignal( + createSessionsYieldTool({ + sessionId: "requester", + onYield: () => { + runAbort.abort(handoffReason); + }, + }), + runAbort.signal, + ); + + await expect(yieldTool.execute("call-yield", {})).resolves.toMatchObject({ + details: { status: "yielded" }, + }); + await siblingAborted; + }); + + it("preserves the handoff when distinct run and per-call signals both yield", async () => { + const runAbort = new AbortController(); + const callAbort = new AbortController(); + const handoffReason = { code: "sessions_yield", turnHandoff: true } as const; + const wrapped = wrapToolWithAbortSignal( + createSessionsYieldTool({ + sessionId: "requester", + onYield: () => { + runAbort.abort(handoffReason); + callAbort.abort(handoffReason); + }, + }), + runAbort.signal, + ); + + await expect(wrapped.execute("call-yield", {}, callAbort.signal)).resolves.toMatchObject({ + details: { status: "yielded" }, + }); + expect(runAbort.signal.reason).toBe(handoffReason); + expect(callAbort.signal.reason).toBe(handoffReason); + }); + + it.each([ + { name: "ordinary caller cancellation", reason: new Error("operator cancelled") }, + { + name: "a caller-authored lookalike handoff", + reason: { code: "sessions_yield", turnHandoff: true }, + }, + ])("rejects sessions_yield for $name without an owner-authored handoff", async ({ reason }) => { + const runAbort = new AbortController(); + const callAbort = new AbortController(); + const execute = vi.fn(() => new Promise(() => {})); + const wrapped = wrapToolWithAbortSignal( + asAgentTool({ name: "sessions_yield", execute }), + runAbort.signal, + ); + + const executePromise = wrapped.execute("call-yield", {}, callAbort.signal); + callAbort.abort(reason); + + await expect(executePromise).rejects.toMatchObject({ + name: "AbortError", + message: "Aborted", + }); + expect(runAbort.signal.aborted).toBe(false); + }); + + it.each([ + { name: "ordinary cancellation", reason: new Error("operator cancelled") }, + { name: "a missing handoff flag", reason: { code: "sessions_yield" } }, + { name: "a disabled handoff flag", reason: { code: "sessions_yield", turnHandoff: false } }, + { name: "a different handoff owner", reason: { code: "different", turnHandoff: true } }, + ])("rejects sessions_yield when its run owner aborts with $name", async ({ reason }) => { + const runAbort = new AbortController(); + const execute = vi.fn(async () => { + runAbort.abort(reason); + return textResult("late"); + }); + const wrapped = wrapToolWithAbortSignal( + asAgentTool({ name: "sessions_yield", execute }), + runAbort.signal, + ); + + await expect(wrapped.execute("call-yield", {})).rejects.toMatchObject({ + name: "AbortError", + message: "Aborted", + }); + }); + + it("does not start sessions_yield when the run was already handed off", async () => { + const runAbort = new AbortController(); + runAbort.abort({ code: "sessions_yield", turnHandoff: true }); + const onYield = vi.fn(); + const wrapped = wrapToolWithAbortSignal( + createSessionsYieldTool({ sessionId: "requester", onYield }), + runAbort.signal, + ); + + await expect(wrapped.execute("call-yield", {})).rejects.toMatchObject({ + name: "AbortError", + message: "Aborted", + }); + expect(onYield).not.toHaveBeenCalled(); + }); + + it("preserves an actual sessions_yield failure after its owner starts the handoff", async () => { + const runAbort = new AbortController(); + const yieldError = new Error("yield bookkeeping failed"); + const wrapped = wrapToolWithAbortSignal( + createSessionsYieldTool({ + sessionId: "requester", + onYield: () => { + runAbort.abort({ code: "sessions_yield", turnHandoff: true }); + throw yieldError; + }, + }), + runAbort.signal, + ); + + await expect(wrapped.execute("call-yield", {})).rejects.toBe(yieldError); + }); + it("rejects with AbortError when the per-call signal aborts through the combined signal", async () => { const runAbort = new AbortController(); const callAbort = new AbortController(); diff --git a/src/agents/subagent-announce-delivery.test.ts b/src/agents/subagent-announce-delivery.test.ts index f7f9692a8906..dd52ebf0ab69 100644 --- a/src/agents/subagent-announce-delivery.test.ts +++ b/src/agents/subagent-announce-delivery.test.ts @@ -3131,8 +3131,240 @@ describe("deliverSubagentAnnouncement completion delivery", () => { }); }); - it("directly delivers settle synthesis even when a direct-message requester turn is active", async () => { - const callGateway = createGatewayMock(); + const requesterSettleSourceTarget = { + tool: "message", + provider: "discord", + accountId: "acct-1", + to: "dm:U123", + text: "the consolidated answer", + } as const; + const deliveredRequesterFinal = { delivered: true, path: "direct" } as const; + const missingRequesterFinal = { + delivered: false, + path: "direct", + reason: "visible_reply_missing", + } as const; + + it.each([ + { + name: "preserves an ordinary non-yielded direct settle turn", + response: {}, + requireVisibleReply: false, + expected: deliveredRequesterFinal, + }, + { + name: "preserves an intentional silent non-yielded settle turn", + response: { result: { payloads: [{ text: "NO_REPLY" }] } }, + requireVisibleReply: false, + expected: deliveredRequesterFinal, + }, + { + name: "accepts a yielded requester's visible final answer", + response: { result: { payloads: [{ text: "The consolidated answer." }] } }, + requireVisibleReply: true, + expected: deliveredRequesterFinal, + }, + { + name: "rejects a yielded turn without a result", + response: {}, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "rejects a yielded turn with no response payloads", + response: { result: { payloads: [] } }, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "rejects a yielded turn that emits only an error", + response: { result: { payloads: [{ text: "tool failed", isError: true }] } }, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "rejects a yielded turn that emits only private reasoning", + response: { result: { payloads: [{ text: "thinking", isReasoning: true }] } }, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "rejects pre-tool commentary instead of a final answer", + response: { result: { payloads: [{ text: "working on it", isCommentary: true }] } }, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "rejects a compaction notice instead of a final answer", + response: { result: { payloads: [{ text: "compacting", isCompactionNotice: true }] } }, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "rejects a provider-fallback notice instead of a final answer", + response: { result: { payloads: [{ text: "switching providers", isFallbackNotice: true }] } }, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "rejects a transient status notice instead of a final answer", + response: { result: { payloads: [{ text: "still working", isStatusNotice: true }] } }, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "rejects an explicitly hidden assistant payload", + response: { result: { payloads: [{ text: "not user visible", visible: false }] } }, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "rejects a yielded turn that emits only the silent reply token", + response: { result: { payloads: [{ text: "NO_REPLY" }] } }, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "rejects a visible final whose external delivery was suppressed", + response: { + result: { + payloads: [{ text: "never delivered" }], + deliveryStatus: { status: "suppressed", succeeded: true, resultCount: 0 }, + }, + }, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "rejects a messaging-tool flag without a committed source receipt", + response: { result: { payloads: [], didSendViaMessagingTool: true } }, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "rejects messaging aggregates without a source-matched receipt", + response: { + result: { + payloads: [], + didSendViaMessagingTool: true, + messagingToolSentTexts: ["sent somewhere else"], + }, + }, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "rejects an accepted subagent spawn without a final reply", + response: { + result: { + payloads: [], + acceptedSessionSpawns: [{ runId: "run-child", childSessionKey: "agent:main:child" }], + }, + }, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "rejects a cron side effect without a final reply", + response: { result: { payloads: [], successfulCronAdds: 1 } }, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "rejects a source-matched messaging progress update", + response: { + result: { + payloads: [], + didSendViaMessagingTool: true, + messagingToolSentTargets: [{ ...requesterSettleSourceTarget, sourceReplyFinal: false }], + }, + }, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "rejects a final message sent to another recipient", + response: { + result: { + payloads: [], + didSendViaMessagingTool: true, + messagingToolSentTargets: [ + { ...requesterSettleSourceTarget, to: "dm:OTHER", sourceReplyFinal: true }, + ], + }, + }, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "does not let an off-target final upgrade source progress", + response: { + result: { + payloads: [], + didSendViaMessagingTool: true, + messagingToolSentTargets: [ + { ...requesterSettleSourceTarget, sourceReplyFinal: false }, + { ...requesterSettleSourceTarget, to: "dm:OTHER", sourceReplyFinal: true }, + ], + }, + }, + requireVisibleReply: true, + expected: missingRequesterFinal, + }, + { + name: "accepts an explicit source-matched final messaging delivery", + response: { + result: { + payloads: [{ text: "NO_REPLY" }], + didSendViaMessagingTool: true, + messagingToolSentTargets: [{ ...requesterSettleSourceTarget, sourceReplyFinal: true }], + }, + }, + requireVisibleReply: true, + expected: deliveredRequesterFinal, + }, + { + name: "accepts an automatic source-matched final without legacy intent markers", + response: { + result: { + payloads: [{ text: "NO_REPLY" }], + didSendViaMessagingTool: true, + messagingToolSentTargets: [requesterSettleSourceTarget], + }, + }, + requireVisibleReply: true, + expected: deliveredRequesterFinal, + }, + { + name: "accepts a source final after source progress in the same turn", + response: { + result: { + payloads: [], + didSendViaMessagingTool: true, + messagingToolSentTargets: [ + { ...requesterSettleSourceTarget, sourceReplyFinal: false }, + { ...requesterSettleSourceTarget, sourceReplyFinal: true }, + ], + }, + }, + requireVisibleReply: true, + expected: deliveredRequesterFinal, + }, + { + name: "accepts a committed source final when automatic delivery was suppressed", + response: { + result: { + payloads: [{ text: "NO_REPLY" }], + deliveryStatus: { status: "suppressed", succeeded: true, resultCount: 0 }, + didSendViaMessagingTool: true, + messagingToolSentTargets: [{ ...requesterSettleSourceTarget, sourceReplyFinal: true }], + }, + }, + requireVisibleReply: true, + expected: deliveredRequesterFinal, + }, + ])("$name", async ({ response, requireVisibleReply, expected }) => { + const callGateway = createGatewayMock(response); const queueEmbeddedAgentMessageWithOutcome = createQueueOutcomeMock(true); const origin = { channel: "discord", @@ -3160,11 +3392,12 @@ describe("deliverSubagentAnnouncement completion delivery", () => { requesterIsSubagent: false, expectsCompletionMessage: false, requireDirectDelivery: true, + ...(requireVisibleReply ? { requireVisibleReply: true } : {}), directIdempotencyKey: "announce-requester-settle-direct", sourceTool: "subagent_announce", }); - expectDeliveryPath(result, "direct"); + expect(result).toMatchObject(expected); expect(queueEmbeddedAgentMessageWithOutcome).not.toHaveBeenCalled(); const agentParams = expectGatewayAgentParams(callGateway, { deliver: true, diff --git a/src/agents/subagent-announce-delivery.ts b/src/agents/subagent-announce-delivery.ts index d27348aaf75b..9d94d8a861be 100644 --- a/src/agents/subagent-announce-delivery.ts +++ b/src/agents/subagent-announce-delivery.ts @@ -42,6 +42,7 @@ import { hasMessagingToolDeliveryEvidence, hasPayloadOutcomeSendEvidence, hasUnaccountedMessagingToolAggregateEvidence, + resolveExplicitFinalSourceReplyDeliveryEvidence, } from "./embedded-agent-runner/delivery-evidence.js"; import { hasIntentionalSilentAgentPayload, @@ -796,7 +797,43 @@ function hasMessagingToolDeliveryToSource( messagingToolSourceReplyPayloads?: unknown; }, deliveryTarget: Parameters[1], + options?: { requireFinalReply?: boolean }, ): boolean { + const targets = Array.isArray(result.messagingToolSentTargets) + ? result.messagingToolSentTargets + : []; + const sourceTargets = targets.filter((target) => { + if ( + !target || + typeof target !== "object" || + Array.isArray(target) || + !deliveryTarget.channel || + !deliveryTarget.to + ) { + return false; + } + const record = target as Parameters[0]; + // Older source receipts omit `to`; explicit off-target sends must never satisfy it. + const sourceTarget = + typeof record.to === "string" && record.to.trim() + ? record + : { ...record, to: deliveryTarget.to }; + return sourceDeliveryTargetsMatch(sourceTarget, deliveryTarget); + }); + if (options?.requireFinalReply) { + const hasCommittedSourceDelivery = + hasCommittedSourceReplyDeliveryEvidence(result) || + (hasMessagingToolDeliveryEvidence(result) && sourceTargets.length > 0); + // Only current-source final markers count; another target's final cannot + // turn a source progress update into the owed requester reply. + return ( + hasCommittedSourceDelivery && + resolveExplicitFinalSourceReplyDeliveryEvidence({ + messagingToolSentTargets: sourceTargets, + messagingToolSourceReplyPayloads: result.messagingToolSourceReplyPayloads, + }) !== false + ); + } if ( hasCommittedSourceReplyDeliveryEvidence(result) || hasUnaccountedMessagingToolAggregateEvidence({ ...result, didSendViaMessagingTool: false }) @@ -804,28 +841,11 @@ function hasMessagingToolDeliveryToSource( return true; } - const targets = Array.isArray(result.messagingToolSentTargets) - ? result.messagingToolSentTargets - : []; if (targets.length === 0 || !deliveryTarget.channel || !deliveryTarget.to) { return hasMessagingToolDeliveryEvidence(result); } - return ( - hasMessagingToolDeliveryEvidence(result) && - targets.some((target) => { - if (!target || typeof target !== "object" || Array.isArray(target)) { - return false; - } - const record = target as Parameters[0]; - // Older current-source receipts omit `to`; explicit off-target sends must never satisfy it. - const sourceTarget = - typeof record.to === "string" && record.to.trim() - ? record - : { ...record, to: deliveryTarget.to }; - return sourceDeliveryTargetsMatch(sourceTarget, deliveryTarget); - }) - ); + return hasMessagingToolDeliveryEvidence(result) && sourceTargets.length > 0; } async function sendSubagentAnnounceDirectly(params: { @@ -834,6 +854,7 @@ async function sendSubagentAnnounceDirectly(params: { triggerMessage: string; internalEvents?: AgentInternalEvent[]; expectsCompletionMessage: boolean; + requireVisibleReply?: boolean; bestEffortDeliver?: boolean; directIdempotencyKey: string; completionDirectOrigin?: DeliveryContext; @@ -1199,11 +1220,32 @@ async function sendSubagentAnnounceDirectly(params: { } const hasVisibleCompletionReply = Boolean( directAnnounceResult && - (hasMessagingToolDelivery || - hasVisibleAgentPayload(directAnnounceResult, { - ...completionPayloadVisibility, - includeSilentReplyPayloads: false, - })), + ((params.requireVisibleReply + ? hasMessagingToolDeliveryToSource(directAnnounceResult, deliveryTarget, { + requireFinalReply: true, + }) + : hasMessagingToolDelivery) || + (hasVisibleAgentPayload( + params.requireVisibleReply + ? { + payloads: Array.isArray(directAnnounceResult.payloads) + ? directAnnounceResult.payloads.filter((payload) => { + const flags = payload as Record; + return ( + flags?.isCommentary !== true && + flags?.isCompactionNotice !== true && + flags?.isFallbackNotice !== true && + flags?.isStatusNotice !== true && + flags?.visible !== false + ); + }) + : [], + } + : directAnnounceResult, + { ...completionPayloadVisibility, includeSilentReplyPayloads: false }, + ) && + (!params.requireVisibleReply || + directAnnounceResult.deliveryStatus?.status !== "suppressed"))), ); const hasCompletionSideEffect = Boolean( directAnnounceResult && hasCommittedOutboundDeliveryEvidence(directAnnounceResult), @@ -1211,12 +1253,13 @@ async function sendSubagentAnnounceDirectly(params: { const acceptsIntentionalSilentCompletion = hasIntentionalSilentCompletionReply && !isSubagentCompletion; if ( - params.expectsCompletionMessage && - !shouldDeliverAgentFinal && - !requiresMessageToolDelivery && !hasVisibleCompletionReply && - !hasCompletionSideEffect && - !acceptsIntentionalSilentCompletion + (params.requireVisibleReply || + (params.expectsCompletionMessage && + !shouldDeliverAgentFinal && + !requiresMessageToolDelivery && + !hasCompletionSideEffect && + !acceptsIntentionalSilentCompletion)) ) { return { delivered: false, @@ -1279,6 +1322,7 @@ export async function deliverSubagentAnnouncement(params: { requesterIsSubagent: boolean; expectsCompletionMessage: boolean; requireDirectDelivery?: boolean; + requireVisibleReply?: boolean; bestEffortDeliver?: boolean; directIdempotencyKey: string; onDeliveryResult?: (delivery: SubagentAnnounceDeliveryResult) => void; @@ -1432,6 +1476,7 @@ export async function deliverSubagentAnnouncement(params: { isSourceSessionEffectsAllowed: params.isSourceSessionEffectsAllowed, requesterIsSubagent: params.requesterIsSubagent, expectsCompletionMessage: params.expectsCompletionMessage, + requireVisibleReply: params.requireVisibleReply, onDeliveryResult: params.onDeliveryResult, signal: params.signal, bestEffortDeliver: params.bestEffortDeliver, diff --git a/src/agents/subagent-announce.requester-settle-wake.test.ts b/src/agents/subagent-announce.requester-settle-wake.test.ts index dccedea65a01..b65c50d2a190 100644 --- a/src/agents/subagent-announce.requester-settle-wake.test.ts +++ b/src/agents/subagent-announce.requester-settle-wake.test.ts @@ -188,11 +188,13 @@ describe("maybeWakeRequesterAfterAllChildrenSettled", () => { expect(call.requesterIsSubagent).toBe(false); expect(call.expectsCompletionMessage).toBe(false); expect(call.requireDirectDelivery).toBe(true); + expect(call.requireVisibleReply).toBeUndefined(); expect(call.directIdempotencyKey).toBe(`announce:requester-settle:${REQUESTER}:run-a,run-b`); const message = String(call.triggerMessage); expect(message).toContain("settled"); expect(message).toContain("social findings"); expect(message).toContain("network findings"); + expect(message).toContain("NO_REPLY"); expect(registryRuntimeMock.hasDescendantRunAwaitingSettle).toHaveBeenCalledWith( REQUESTER, "run-b", @@ -425,6 +427,10 @@ describe("maybeWakeRequesterAfterAllChildrenSettled", () => { expect(woke).toBe(true); expect(deliverSpy).toHaveBeenCalledOnce(); + expect(deliveredCallArg().requireVisibleReply).toBe(true); + const message = String(deliveredCallArg().triggerMessage); + expect(message).not.toContain("NO_REPLY"); + expect(message).toContain("original user request still requires your visible final answer"); expect(deliveredCallArg().directIdempotencyKey).toBe( `announce:requester-settle:${REQUESTER}:run-b:yield-1`, ); @@ -452,6 +458,10 @@ describe("maybeWakeRequesterAfterAllChildrenSettled", () => { expect(woke).toBe(true); expect(deliverSpy).toHaveBeenCalledOnce(); + expect(deliveredCallArg().requireVisibleReply).toBe(true); + const message = String(deliveredCallArg().triggerMessage); + expect(message).not.toContain("NO_REPLY"); + expect(message).toContain("original user request still requires your visible final answer"); expect(deliveredCallArg().directIdempotencyKey).toBe( `announce:requester-settle:${REQUESTER}:run-b:yield-1`, ); @@ -543,6 +553,55 @@ describe("maybeWakeRequesterAfterAllChildrenSettled", () => { } }); + it("retains a yielded wake after a silent final and retries its visible reply", async () => { + const child = makeSettledChild({ + runId: "run-b", + delivery: { status: "delivered" }, + requesterSettleWake: { + status: "pending", + attemptCount: 0, + batchRunIds: ["run-b"], + requesterYieldBatch: true, + rearmGeneration: 1, + }, + }); + registryRuntimeMock.listSubagentRunsForRequester.mockReturnValue([child]); + deliverSpy.mockResolvedValueOnce({ + delivered: false, + path: "direct", + reason: "visible_reply_missing", + }); + + vi.useFakeTimers(); + vi.setSystemTime(0); + try { + await expect( + maybeWakeRequesterAfterAllChildrenSettled(wakeParams({ settledEntry: child })), + ).resolves.toBe(false); + expect(completeBatchSpy).not.toHaveBeenCalled(); + expect(child.requesterSettleWake).toMatchObject({ + status: "pending", + attemptCount: 1, + nextAttemptAt: 30_000, + requesterYieldBatch: true, + rearmGeneration: 1, + lastError: "visible_reply_missing", + }); + + await vi.advanceTimersByTimeAsync(30_000); + await expect( + maybeWakeRequesterAfterAllChildrenSettled(wakeParams({ settledEntry: child })), + ).resolves.toBe(true); + expect(deliverSpy.mock.calls.map(([arg]) => arg.directIdempotencyKey)).toEqual([ + `announce:requester-settle:${REQUESTER}:run-b:yield-1`, + `announce:requester-settle:${REQUESTER}:run-b:yield-1:retry-1`, + ]); + expect(completeBatchSpy).toHaveBeenCalledWith(["run-b"], 1); + } finally { + vi.useRealTimers(); + } + }); + it("replays an ambiguous transport failure with the same idempotency key", async () => { const firstChild = makeSettledChild({ runId: "run-a" }); const secondChild = makeSettledChild({ runId: "run-b" }); diff --git a/src/agents/subagent-announce.requester-settle-wake.ts b/src/agents/subagent-announce.requester-settle-wake.ts index de29dbd4a452..35bff5f94b71 100644 --- a/src/agents/subagent-announce.requester-settle-wake.ts +++ b/src/agents/subagent-announce.requester-settle-wake.ts @@ -59,12 +59,17 @@ const REQUESTER_SETTLE_WAKE_MAX_AMBIGUOUS_REPLAYS = 3; const REQUESTER_SETTLE_WAKE_RETRY_DELAYS_MS = [30_000, 120_000] as const; const activeRequesterSettleWakeBatches = new Set(); -function buildRequesterSettleWakeMessage(params: { findings?: string }): string { +function buildRequesterSettleWakeMessage(params: { + findings?: string; + requireVisibleReply: boolean; +}): string { return [ "[Subagent Context] Every subagent spawned from this session has now settled — none are still running or awaiting completion delivery.", "[Subagent Context] Do not keep waiting or call sessions_yield again for this batch; no further completion events will arrive.", "[Subagent Context] Review the completion results and send your consolidated final answer to the user now.", - `[Subagent Context] Reply ONLY: ${SILENT_REPLY_TOKEN} only if you already delivered the consolidated final answer for this batch.`, + params.requireVisibleReply + ? "[Subagent Context] Child completion delivery is internal; the original user request still requires your visible final answer." + : `[Subagent Context] Reply ONLY: ${SILENT_REPLY_TOKEN} only if you already delivered the consolidated final answer for this batch.`, "", params.findings ?? "(each child result was announced individually in earlier completion events)", @@ -317,7 +322,10 @@ export async function maybeWakeRequesterAfterAllChildrenSettled(params: { }), ), ); - const wakeMessage = buildRequesterSettleWakeMessage({ findings }); + const wakeMessage = buildRequesterSettleWakeMessage({ + findings, + requireVisibleReply: requesterYieldedAfterDelivery, + }); const requesterSessionOrigin = normalizeDeliveryContext(params.requesterOrigin); const directOrigin = resolveAnnounceOrigin(requesterEntry, requesterSessionOrigin); const wakeKeyBase = [ @@ -400,6 +408,7 @@ export async function maybeWakeRequesterAfterAllChildrenSettled(params: { requesterIsSubagent: false, expectsCompletionMessage: false, requireDirectDelivery: true, + ...(requesterYieldedAfterDelivery ? { requireVisibleReply: true } : {}), directIdempotencyKey: buildAnnounceIdempotencyKey( attemptIndex === 0 ? wakeKeyBase : `${wakeKeyBase}:retry-${attemptIndex}`, ), From 3f3e59b54d1c2f5eac033cb2e82b6208a6bb22d6 Mon Sep 17 00:00:00 2001 From: qingminlong Date: Mon, 3 Aug 2026 18:07:46 +0800 Subject: [PATCH 15/57] fix(cron): defer auto-disable notifications until persistence (#118384) * fix(cron): defer auto-disable notifications until persistence * fix(cron): emit disable notifications only after durable roster commits --------- Co-authored-by: Peter Steinberger --- src/cron/service/jobs-scheduling.ts | 6 ++ src/cron/service/ops-mutations.ts | 11 ++- src/cron/service/ops-run-preparation.ts | 19 ++-- src/cron/service/ops-shared.ts | 12 ++- src/cron/service/ops.test.ts | 117 +++++++++++++++++++++++- src/cron/service/timer-catchup.ts | 42 ++++++--- src/cron/service/timer-scheduler.ts | 21 +++-- 7 files changed, 196 insertions(+), 32 deletions(-) diff --git a/src/cron/service/jobs-scheduling.ts b/src/cron/service/jobs-scheduling.ts index 9c277011a638..f5c716de9229 100644 --- a/src/cron/service/jobs-scheduling.ts +++ b/src/cron/service/jobs-scheduling.ts @@ -535,6 +535,7 @@ function recomputeJobNextRunAtMs(params: { job: CronJob; nowMs: number; deferredNotifications?: DeferredCronNotifications; + skipScheduleErrorHandling?: boolean; }) { let changed = false; try { @@ -562,6 +563,9 @@ function recomputeJobNextRunAtMs(params: { changed = true; } } catch (err) { + if (params.skipScheduleErrorHandling) { + return false; + } if ( recordScheduleComputeError({ state: params.state, @@ -611,6 +615,7 @@ export function recomputeNextRunsForMaintenance( repairFutureCronNextRunAtMs?: boolean; preserveExpiredPacedNextRunJobId?: string; deferredNotifications?: DeferredCronNotifications; + skipScheduleErrorHandling?: boolean; }, ): boolean { const recomputeExpired = opts?.recomputeExpired ?? false; @@ -621,6 +626,7 @@ export function recomputeNextRunsForMaintenance( job, nowMs, deferredNotifications: opts?.deferredNotifications, + skipScheduleErrorHandling: opts?.skipScheduleErrorHandling, }); return walkSchedulableJobs( state, diff --git a/src/cron/service/ops-mutations.ts b/src/cron/service/ops-mutations.ts index bfabd0c00bda..93c0d8e62231 100644 --- a/src/cron/service/ops-mutations.ts +++ b/src/cron/service/ops-mutations.ts @@ -504,13 +504,19 @@ export async function removeAgentJobsTransactional( state.store.jobs = state.store.jobs.filter( (job) => resolveEffectiveJobAgentId(job, defaultAgentId) !== id, ); - recomputeNextRunsForMaintenance(state); + const postPersistNotifications: DeferredCronNotifications = []; + recomputeNextRunsForMaintenance(state, { deferredNotifications: postPersistNotifications }); + // Cron is durable first, but notifications stay speculative until the roster commits. await persistOrRestore(state, snapshot); let result: T; try { result = await commit(); } catch (error) { if (error instanceof AgentDeletionCommitUncertainError) { + // Uncertain roster writes intentionally keep the cron deletion durable. + for (const notify of postPersistNotifications) { + notify(); + } armTimer(state); for (const job of removedJobs) { noteActiveCronJobRemoval(job.id); @@ -534,6 +540,9 @@ export async function removeAgentJobsTransactional( } throw error; } + for (const notify of postPersistNotifications) { + notify(); + } for (const job of removedJobs) { noteActiveCronJobRemoval(job.id); try { diff --git a/src/cron/service/ops-run-preparation.ts b/src/cron/service/ops-run-preparation.ts index 630eec278d43..df9473eacd31 100644 --- a/src/cron/service/ops-run-preparation.ts +++ b/src/cron/service/ops-run-preparation.ts @@ -207,6 +207,15 @@ async function skipInvalidPersistedManualRun(params: { armTimer(params.state); } +function recomputeManualRunPreflight(state: CronServiceState, id: string, mode?: "due" | "force") { + // Preflight is advisory and may be called by read-shaped queue checks. Do not + // let a schedule error turn that check into an auto-disable transition. + return recomputeNextRunsForMaintenance(state, { + ...(mode === "force" ? { preserveExpiredPacedNextRunJobId: id } : {}), + skipScheduleErrorHandling: true, + }); +} + async function inspectManualRunPreflight( state: CronServiceState, id: string, @@ -228,10 +237,7 @@ async function inspectManualRunPreflight( // Normalize job tick state (clears stale runningAtMs markers) before // checking if already running, so a stale marker from a crashed Phase-1 // persist does not block manual triggers for up to STUCK_RUN_MS (#17554). - recomputeNextRunsForMaintenance( - state, - mode === "force" ? { preserveExpiredPacedNextRunJobId: id } : undefined, - ); + recomputeManualRunPreflight(state, id, mode); const job = findJobOrThrow(state, id); if (!admitsStreamSourceRun(job, streamScheduleKey, streamSourceIdentity)) { return { ok: true, ran: false, reason: "not-due" } as const; @@ -308,10 +314,7 @@ export async function prepareManualRun( // The initial preflight is advisory. A command-lane wait or another cron // run can change this job before its reservation is persisted. await ensureLoaded(state, { skipRecompute: true }); - recomputeNextRunsForMaintenance( - state, - mode === "force" ? { preserveExpiredPacedNextRunJobId: id } : undefined, - ); + recomputeManualRunPreflight(state, id, mode); const job = findJobOrThrow(state, id); if (!admitsStreamSourceRun(job, opts?.streamScheduleKey, opts?.streamSourceIdentity)) { return { ok: true, ran: false, reason: "not-due" as const }; diff --git a/src/cron/service/ops-shared.ts b/src/cron/service/ops-shared.ts index af39cb929e6f..40641f5523f9 100644 --- a/src/cron/service/ops-shared.ts +++ b/src/cron/service/ops-shared.ts @@ -5,8 +5,8 @@ import { cronStreamScheduleKey } from "../stream-schedule.js"; import type { CronJob } from "../types.js"; import { recomputeNextRunsForMaintenance } from "./jobs.js"; import { normalizeOptionalAgentId } from "./normalize.js"; -import type { CronServiceState } from "./state.js"; -import { ensureLoaded, persist } from "./store.js"; +import type { CronServiceState, DeferredCronNotifications } from "./state.js"; +import { ensureLoaded, persistOrRestore, snapshotStoreForRollback } from "./store.js"; import { type IsolatedAgentSetupTimeoutSignal, maybeNotifyIsolatedAgentSetupTimeout, @@ -74,9 +74,13 @@ export async function ensureLoadedForRead(state: CronServiceState) { } // Use the maintenance-only version so that read-only operations never // advance a past-due nextRunAtMs without executing the job (#16156). - const changed = recomputeNextRunsForMaintenance(state); + const rollbackSnapshot = snapshotStoreForRollback(state); + const postPersistNotifications: DeferredCronNotifications = []; + const changed = recomputeNextRunsForMaintenance(state, { + deferredNotifications: postPersistNotifications, + }); if (changed) { - await persist(state); + await persistOrRestore(state, rollbackSnapshot, { postPersistNotifications }); } } diff --git a/src/cron/service/ops.test.ts b/src/cron/service/ops.test.ts index d0ce61aa3f79..0c37b5915555 100644 --- a/src/cron/service/ops.test.ts +++ b/src/cron/service/ops.test.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { AgentDeletionCommitUncertainError } from "../../agents/agent-lifecycle-registry.js"; import { runOpenClawStateWriteTransaction } from "../../state/openclaw-state-db.js"; import * as taskExecutor from "../../tasks/task-executor.js"; import { findTaskByRunId, listTaskRecordsUnsorted } from "../../tasks/task-registry.js"; @@ -16,8 +17,15 @@ import { loadCronJobsStoreWithConfigJobs, loadCronStore } from "../store.js"; import { cronStoreKey } from "../store/key.js"; import type { CronJob } from "../types.js"; import { start, stop } from "./ops-lifecycle.js"; -import { add, remove, removeStaleJobFamily, update } from "./ops-mutations.js"; +import { + add, + remove, + removeAgentJobsTransactional, + removeStaleJobFamily, + update, +} from "./ops-mutations.js"; import { list } from "./ops-read.js"; +import { inspectManualRunDisposition } from "./ops-run-preparation.js"; import { run } from "./ops-run.js"; import { createCronServiceState, type CronEvent } from "./state.js"; import { tryCreateCronTaskRun, tryFinishCronTaskRun } from "./task-runs.js"; @@ -1738,5 +1746,112 @@ describe("cron service ops persist rollback", () => { expect(enqueueSystemEvent).toHaveBeenCalledTimes(1); expect(requestHeartbeat).toHaveBeenCalledTimes(1); }); + + it.each(["failed", "committed", "uncertain"] as const)( + "publishes agent-removal auto-disable notifications only after a %s roster outcome", + async (outcome) => { + const { storePath } = await makeStorePath(); + const now = Date.parse("2026-06-09T00:00:00.000Z"); + const state = createOkIsolatedCronState({ storePath, now }); + const removed = await add(state, { + ...makeCreateInput("deleted agent job"), + agentId: "doomed", + }); + const malformed = await add(state, { + ...makeCreateInput("malformed surviving job"), + agentId: "survivor", + schedule: { kind: "cron", expr: "0 1 * * *" }, + }); + if (state.timer) { + clearTimeout(state.timer); + } + malformed.state.nextRunAtMs = undefined; + malformed.state.scheduleErrorCount = 2; + const enqueueSystemEvent = vi.mocked(state.deps.enqueueSystemEvent); + const requestHeartbeat = vi.mocked(state.deps.requestHeartbeat); + enqueueSystemEvent.mockClear(); + requestHeartbeat.mockClear(); + const computeNextRunAtMs = cronSchedule.computeNextRunAtMs; + vi.spyOn(cronSchedule, "computeNextRunAtMs").mockImplementation((schedule, nowMs) => { + if (schedule.kind === "cron" && schedule.expr === "0 1 * * *") { + throw new Error("simulated schedule failure"); + } + return computeNextRunAtMs(schedule, nowMs); + }); + + const commit = vi.fn(async () => { + expect(enqueueSystemEvent).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); + const persisted = await loadCronStore(storePath); + expect(persisted.jobs.find((job) => job.id === removed.id)).toBeUndefined(); + expect(persisted.jobs.find((job) => job.id === malformed.id)?.enabled).toBe(false); + if (outcome === "failed") { + throw new Error("roster commit failed"); + } + if (outcome === "uncertain") { + throw new AgentDeletionCommitUncertainError(new Error("roster commit uncertain")); + } + return "roster committed"; + }); + const transaction = removeAgentJobsTransactional(state, "doomed", commit); + if (outcome === "committed") { + await expect(transaction).resolves.toBe("roster committed"); + } else if (outcome === "uncertain") { + await expect(transaction).rejects.toBeInstanceOf(AgentDeletionCommitUncertainError); + } else { + await expect(transaction).rejects.toThrow("roster commit failed"); + } + if (state.timer) { + clearTimeout(state.timer); + } + + const rolledBack = outcome === "failed"; + const notificationCount = rolledBack ? 0 : 1; + expect(commit).toHaveBeenCalledOnce(); + expect(enqueueSystemEvent).toHaveBeenCalledTimes(notificationCount); + expect(requestHeartbeat).toHaveBeenCalledTimes(notificationCount); + expect(state.store?.jobs.some((job) => job.id === removed.id)).toBe(rolledBack); + expect(state.store?.jobs.find((job) => job.id === malformed.id)?.enabled).toBe(rolledBack); + const persisted = await loadCronStore(storePath); + expect(persisted.jobs.some((job) => job.id === removed.id)).toBe(rolledBack); + expect(persisted.jobs.find((job) => job.id === malformed.id)?.enabled).toBe(rolledBack); + }, + ); + + it("does not auto-disable a job during manual-run preflight", async () => { + const { storePath } = await makeStorePath(); + const now = Date.parse("2026-06-09T00:00:00.000Z"); + const state = createOkIsolatedCronState({ storePath, now }); + const job = await add(state, { + ...makeCreateInput("preflight schedule failure"), + schedule: { kind: "cron", expr: "0 1 * * *" }, + }); + if (state.timer) { + clearTimeout(state.timer); + } + job.state.nextRunAtMs = undefined; + job.state.scheduleErrorCount = 2; + const enqueueSystemEvent = vi.mocked(state.deps.enqueueSystemEvent); + const requestHeartbeat = vi.mocked(state.deps.requestHeartbeat); + enqueueSystemEvent.mockClear(); + requestHeartbeat.mockClear(); + const computeSpy = vi.spyOn(cronSchedule, "computeNextRunAtMs").mockImplementation(() => { + throw new Error("simulated preflight schedule failure"); + }); + + try { + await expect(inspectManualRunDisposition(state, job.id)).resolves.toEqual({ + ok: true, + ran: false, + reason: "not-due", + }); + expect(job.enabled).toBe(true); + expect(job.state.scheduleErrorCount).toBe(2); + expect(enqueueSystemEvent).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); + } finally { + computeSpy.mockRestore(); + } + }); }); /* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/cron/service/timer-catchup.ts b/src/cron/service/timer-catchup.ts index 5e8770b2a006..8f3fb0cbd0d2 100644 --- a/src/cron/service/timer-catchup.ts +++ b/src/cron/service/timer-catchup.ts @@ -15,7 +15,7 @@ import { runWithCronAdmission, updateQueuedCronRunReservationMarker, } from "./run-admission.js"; -import { type CronServiceState, emit } from "./state.js"; +import { type CronServiceState, type DeferredCronNotifications, emit } from "./state.js"; import { ensureLoaded, persist, persistOrRestore, snapshotStoreForRollback } from "./store.js"; import { tryCreateCronTaskRun } from "./task-runs.js"; import { @@ -88,8 +88,12 @@ async function releaseStartupCatchupReservationsAfterFailure( if (pendingReleases.length === 0) { return; } - recomputeNextRunsForMaintenance(state, { repairFutureCronNextRunAtMs: false }); - await persistOrRestore(state, rollbackSnapshot); + const postPersistNotifications: DeferredCronNotifications = []; + recomputeNextRunsForMaintenance(state, { + repairFutureCronNextRunAtMs: false, + deferredNotifications: postPersistNotifications, + }); + await persistOrRestore(state, rollbackSnapshot, { postPersistNotifications }); for (const pending of pendingReleases) { releaseQueuedCronRun(state, pending.jobId, pending.reservationIdentity); } @@ -309,8 +313,12 @@ async function executeStartupCatchupPlan( ) { const rollbackSnapshot = snapshotStoreForRollback(state); delete job.state.queuedAtMs; - recomputeNextRunsForMaintenance(state, { repairFutureCronNextRunAtMs: false }); - await persistOrRestore(state, rollbackSnapshot); + const postPersistNotifications: DeferredCronNotifications = []; + recomputeNextRunsForMaintenance(state, { + repairFutureCronNextRunAtMs: false, + deferredNotifications: postPersistNotifications, + }); + await persistOrRestore(state, rollbackSnapshot, { postPersistNotifications }); releaseQueuedCronRun(state, candidate.jobId, candidate.reservationIdentity); return undefined; } @@ -437,8 +445,12 @@ async function applyStartupCatchupOutcomes( const rollbackSnapshot = snapshotStoreForRollback(state); const pendingReleases = clearUnstartedStartupCatchupReservationMarkers(state, plan, outcomes); if (pendingReleases.length > 0) { - recomputeNextRunsForMaintenance(state, { repairFutureCronNextRunAtMs: false }); - await persistOrRestore(state, rollbackSnapshot); + const postPersistNotifications: DeferredCronNotifications = []; + recomputeNextRunsForMaintenance(state, { + repairFutureCronNextRunAtMs: false, + deferredNotifications: postPersistNotifications, + }); + await persistOrRestore(state, rollbackSnapshot, { postPersistNotifications }); for (const pending of pendingReleases) { releaseQueuedCronRun(state, pending.jobId, pending.reservationIdentity); } @@ -450,8 +462,12 @@ async function applyStartupCatchupOutcomes( const pendingReleases = clearUnstartedStartupCatchupReservationMarkers(state, plan, outcomes); if (outcomes.length === 0 && plan.deferredJobs.length === 0) { if (pendingReleases.length > 0) { - recomputeNextRunsForMaintenance(state, { repairFutureCronNextRunAtMs: false }); - await persistOrRestore(state, rollbackSnapshot); + const postPersistNotifications: DeferredCronNotifications = []; + recomputeNextRunsForMaintenance(state, { + repairFutureCronNextRunAtMs: false, + deferredNotifications: postPersistNotifications, + }); + await persistOrRestore(state, rollbackSnapshot, { postPersistNotifications }); for (const pending of pendingReleases) { releaseQueuedCronRun(state, pending.jobId, pending.reservationIdentity); } @@ -484,8 +500,12 @@ async function applyStartupCatchupOutcomes( // Startup overflow owns these staggered wake times; repairing future // schedules here would silently move a deferred run to its natural slot. - recomputeNextRunsForMaintenance(state, { repairFutureCronNextRunAtMs: false }); - await persistOrRestore(state, rollbackSnapshot); + const postPersistNotifications: DeferredCronNotifications = []; + recomputeNextRunsForMaintenance(state, { + repairFutureCronNextRunAtMs: false, + deferredNotifications: postPersistNotifications, + }); + await persistOrRestore(state, rollbackSnapshot, { postPersistNotifications }); for (const pending of pendingReleases) { releaseQueuedCronRun(state, pending.jobId, pending.reservationIdentity); } diff --git a/src/cron/service/timer-scheduler.ts b/src/cron/service/timer-scheduler.ts index 140b55e79746..e51e7c4b8f56 100644 --- a/src/cron/service/timer-scheduler.ts +++ b/src/cron/service/timer-scheduler.ts @@ -28,8 +28,8 @@ import { runWithCronAdmission, updateQueuedCronRunReservationMarker, } from "./run-admission.js"; -import { type CronServiceState, emit } from "./state.js"; -import { ensureLoaded, persist, persistOrRestore, snapshotStoreForRollback } from "./store.js"; +import { type CronServiceState, type DeferredCronNotifications, emit } from "./state.js"; +import { ensureLoaded, persistOrRestore, snapshotStoreForRollback } from "./store.js"; import { tryCreateCronTaskRun } from "./task-runs.js"; import { resolveCronJobTimeoutMs } from "./timeout-policy.js"; import { @@ -208,12 +208,15 @@ async function onAdmittedTimer(state: CronServiceState) { // Use maintenance-only recompute to avoid advancing past-due nextRunAtMs // values without execution. This prevents jobs from being silently skipped // when the timer wakes up but findDueJobs returns empty (see #13992). + const rollbackSnapshot = snapshotStoreForRollback(state); + const postPersistNotifications: DeferredCronNotifications = []; const changed = recomputeNextRunsForMaintenance(state, { recomputeExpired: true, nowMs: dueCheckNow, + deferredNotifications: postPersistNotifications, }); if (changed) { - await persist(state); + await persistOrRestore(state, rollbackSnapshot, { postPersistNotifications }); } return []; } @@ -262,8 +265,11 @@ async function onAdmittedTimer(state: CronServiceState) { releaseQueuedCronRun(state, candidate.id, candidate.reservationIdentity); } } - recomputeNextRunsForMaintenance(state); - await persistOrRestore(state, rollbackSnapshot); + const postPersistNotifications: DeferredCronNotifications = []; + recomputeNextRunsForMaintenance(state, { + deferredNotifications: postPersistNotifications, + }); + await persistOrRestore(state, rollbackSnapshot, { postPersistNotifications }); for (const candidate of pendingReleases) { releaseQueuedCronRun(state, candidate.id, candidate.reservationIdentity); } @@ -380,8 +386,9 @@ async function onAdmittedTimer(state: CronServiceState) { releaseQueuedCronRun(state, due.id, due.reservationIdentity); } } - recomputeNextRunsForMaintenance(state); - await persistOrRestore(state, rollbackSnapshot); + const postPersistNotifications: DeferredCronNotifications = []; + recomputeNextRunsForMaintenance(state, { deferredNotifications: postPersistNotifications }); + await persistOrRestore(state, rollbackSnapshot, { postPersistNotifications }); for (const due of pendingReleases) { releaseQueuedCronRun(state, due.id, due.reservationIdentity); } From 9c22a4349d8034ac38b60e5e7c21409bdd04c18f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 03:17:25 -0700 Subject: [PATCH 16/57] chore: speed up gateway CLI test startup (#118620) * test(cli): avoid unrelated startup graphs * test(cli): preserve startup coverage --------- Co-authored-by: Peter Steinberger --- src/cli/gateway-cli.coverage.test.ts | 159 +----------------- .../gateway-cli/run.option-collisions.test.ts | 26 ++- 2 files changed, 27 insertions(+), 158 deletions(-) diff --git a/src/cli/gateway-cli.coverage.test.ts b/src/cli/gateway-cli.coverage.test.ts index 694eaa2d7487..99e1622f26a1 100644 --- a/src/cli/gateway-cli.coverage.test.ts +++ b/src/cli/gateway-cli.coverage.test.ts @@ -3,9 +3,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { Command } from "commander"; -import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import { withEnvOverride } from "../config/test-helpers.js"; -import { GatewayLockError } from "../infra/gateway-lock.js"; import { registerGatewayCli } from "./gateway-cli.js"; type GatewayCliDependencies = Parameters[1]; @@ -18,26 +17,11 @@ const callGateway = vi.fn<(opts: unknown) => Promise>(defaultCallGatewa const formatGatewayAuthErrorJson = vi.fn(); const formatGatewayClientRequestErrorJson = vi.fn(); const formatGatewayTransportErrorJson = vi.fn(); -const startGatewayServer = vi.fn< - (port: number, opts?: unknown) => Promise<{ close: () => Promise }> ->(async () => ({ - close: vi.fn(async () => {}), -})); const setVerbose = vi.fn(); -const forceFreePortAndWait = vi.fn< - (port: number) => Promise<{ killed: unknown[]; waitedMs: number; escalatedToSigkill: boolean }> ->(async () => ({ - killed: [], - waitedMs: 0, - escalatedToSigkill: false, -})); -const serviceIsLoaded = vi.fn().mockResolvedValue(true); const discoverGatewayBeacons = vi.fn<(opts: unknown) => Promise>( async () => [], ); const gatewayStatusCommand = vi.fn<(opts: unknown) => Promise>(async () => {}); -const inspectPortUsage = vi.fn(async (_port: number) => ({ status: "free" as const })); -const formatPortDiagnostics = vi.fn((_diagnostics: unknown) => [] as string[]); const mocks = await vi.hoisted(async () => { const { createCliRuntimeMock } = await import("./test-runtime-mock.js"); @@ -68,10 +52,6 @@ vi.mock( }), ); -vi.mock("../gateway/server.js", () => ({ - startGatewayServer: (port: number, opts?: unknown) => startGatewayServer(port, opts), -})); - vi.mock("../globals.js", () => ({ info: (msg: string) => msg, isVerbose: () => false, @@ -83,10 +63,6 @@ vi.mock("../runtime.js", async () => ({ defaultRuntime: mocks.defaultRuntime, })); -vi.mock("./ports.js", () => ({ - forceFreePortAndWait: (port: number) => forceFreePortAndWait(port), -})); - vi.mock("../daemon/service.js", () => ({ resolveGatewayService: () => ({ label: "LaunchAgent", @@ -97,7 +73,7 @@ vi.mock("../daemon/service.js", () => ({ uninstall: vi.fn(), stop: vi.fn(), restart: vi.fn(), - isLoaded: serviceIsLoaded, + isLoaded: vi.fn().mockResolvedValue(true), readCommand: vi.fn(), readRuntime: vi.fn().mockResolvedValue({ status: "running" }), }), @@ -120,11 +96,6 @@ vi.mock("../commands/gateway-status.js", () => ({ gatewayStatusCommand: (opts: unknown) => gatewayStatusCommand(opts), })); -vi.mock("../infra/ports.js", () => ({ - inspectPortUsage: (port: number) => inspectPortUsage(port), - formatPortDiagnostics: (diagnostics: unknown) => formatPortDiagnostics(diagnostics), -})); - let gatewayProgram: Command; function createGatewayProgram(deps?: GatewayCliDependencies) { @@ -151,13 +122,6 @@ function firstMockArg(mock: { mock: { calls: ReadonlyArray { - beforeAll(async () => { - // Gateway startup intentionally primes this large graph before installing - // signal handlers. Load it as suite setup so failure-path timings measure - // the lifecycle behavior rather than the one-time module parse. - await import("./gateway-cli/lifecycle.runtime.js"); - }); - beforeEach(() => { gatewayProgram = createGatewayProgram(); callGateway.mockReset(); @@ -169,9 +133,6 @@ describe("gateway-cli coverage", () => { defaultRuntime.writeStdout.mockClear(); defaultRuntime.writeJson.mockClear(); defaultRuntime.exit.mockClear(); - startGatewayServer.mockClear(); - inspectPortUsage.mockClear(); - formatPortDiagnostics.mockClear(); formatGatewayAuthErrorJson.mockReset(); formatGatewayAuthErrorJson.mockReturnValue(null); formatGatewayClientRequestErrorJson.mockReset(); @@ -673,120 +634,4 @@ describe("gateway-cli coverage", () => { expect(callGateway).not.toHaveBeenCalled(); expect(runtimeErrors.join("\n")).toContain("Invalid --timeout"); }); - - it("validates gateway ports before starting", async () => { - await expectGatewayExit(["gateway", "--port", "0", "--token", "test-token"]); - }); - - it("reports force-free port failures", async () => { - forceFreePortAndWait.mockImplementationOnce(async () => { - throw new Error("boom"); - }); - await expectGatewayExit([ - "gateway", - "--port", - "18789", - "--token", - "test-token", - "--force", - "--allow-unconfigured", - ]); - }); - - it("reports gateway start failures without leaking signal listeners", async () => { - startGatewayServer.mockRejectedValueOnce(new Error("nope")); - const beforeSigterm = new Set(process.listeners("SIGTERM")); - const beforeSigint = new Set(process.listeners("SIGINT")); - await expectGatewayExit([ - "gateway", - "--port", - "18789", - "--token", - "test-token", - "--allow-unconfigured", - ]); - for (const listener of process.listeners("SIGTERM")) { - if (!beforeSigterm.has(listener)) { - process.removeListener("SIGTERM", listener); - } - } - for (const listener of process.listeners("SIGINT")) { - if (!beforeSigint.has(listener)) { - process.removeListener("SIGINT", listener); - } - } - }); - - it("prints stop hints on an already-running GatewayLockError", async () => { - await withEnvOverride( - { - LAUNCH_JOB_LABEL: undefined, - LAUNCH_JOB_NAME: undefined, - XPC_SERVICE_NAME: undefined, - OPENCLAW_LAUNCHD_LABEL: undefined, - OPENCLAW_SYSTEMD_UNIT: undefined, - INVOCATION_ID: undefined, - SYSTEMD_EXEC_PID: undefined, - JOURNAL_STREAM: undefined, - OPENCLAW_WINDOWS_TASK_NAME: undefined, - OPENCLAW_SERVICE_MARKER: undefined, - OPENCLAW_SERVICE_KIND: undefined, - }, - async () => { - serviceIsLoaded.mockResolvedValue(true); - startGatewayServer.mockRejectedValueOnce( - new GatewayLockError("another gateway instance is already listening"), - ); - await expect( - runGatewayCommand(["gateway", "--token", "test-token", "--allow-unconfigured"]), - ).rejects.toThrow(/__exit__:[01]/); - - expect(startGatewayServer).toHaveBeenCalledTimes(1); - expect(runtimeErrors.join("\n")).toContain("Gateway failed to start:"); - expect(runtimeErrors.join("\n")).toContain("gateway stop"); - }, - ); - }); - - it("keeps exit 1 for gateway bind failures wrapped as GatewayLockError", async () => { - runtimeLogs.length = 0; - runtimeErrors.length = 0; - serviceIsLoaded.mockResolvedValue(true); - startGatewayServer.mockRejectedValueOnce( - new GatewayLockError("failed to bind gateway socket on ws://127.0.0.1:18789: Error: boom"), - ); - - await expectGatewayExit(["gateway", "--token", "test-token", "--allow-unconfigured"]); - - expect(runtimeErrors.join("\n")).toContain("failed to bind gateway socket"); - }); - - it("keeps exit 1 for gateway lock acquisition failures", async () => { - runtimeLogs.length = 0; - runtimeErrors.length = 0; - serviceIsLoaded.mockResolvedValue(true); - startGatewayServer.mockRejectedValueOnce( - new GatewayLockError("failed to acquire gateway lock at /tmp/openclaw/gateway.lock"), - ); - - await expectGatewayExit(["gateway", "--token", "test-token", "--allow-unconfigured"]); - - expect(runtimeErrors.join("\n")).toContain("failed to acquire gateway lock"); - }); - - it("uses env/config port when --port is omitted", async () => { - await withEnvOverride({ OPENCLAW_GATEWAY_PORT: "19001" }, async () => { - runtimeLogs.length = 0; - runtimeErrors.length = 0; - startGatewayServer.mockClear(); - - startGatewayServer.mockRejectedValueOnce(new Error("nope")); - await expectGatewayExit(["gateway", "--token", "test-token", "--allow-unconfigured"]); - - expect(startGatewayServer).toHaveBeenCalledTimes(1); - const startCall = startGatewayServer.mock.calls[0]; - expect(startCall?.[0]).toBe(19001); - expect(typeof startCall?.[1]).toBe("object"); - }); - }); }); diff --git a/src/cli/gateway-cli/run.option-collisions.test.ts b/src/cli/gateway-cli/run.option-collisions.test.ts index 667965641759..881d97c5db06 100644 --- a/src/cli/gateway-cli/run.option-collisions.test.ts +++ b/src/cli/gateway-cli/run.option-collisions.test.ts @@ -485,7 +485,7 @@ describe("gateway run option collisions", () => { expect(gatewayStartOptions().auth?.mode).toBe(mode); } - it("runs the fast-path bootstrap hook before gateway startup", async () => { + it("composes gateway run registration through startup after the fast-path bootstrap", async () => { normalizeStateDirEnv.mockImplementation((_env?: NodeJS.ProcessEnv) => { callOrder.push("normalize"); }); @@ -500,6 +500,15 @@ describe("gateway run option collisions", () => { expect(callOrder).toEqual(["bootstrap", "normalize", "normalize", "start"]); }); + it("rejects invalid gateway ports before startup", async () => { + await expect( + runGatewayCli(["gateway", "--port", "0", "--token", "test-token"]), + ).rejects.toThrow("__exit__:1"); + + expect(startGatewayServer).not.toHaveBeenCalled(); + expect(runtimeErrors.join("\n")).toContain("Invalid --port. Use a port number from 1 to 65535"); + }); + it("suppresses ambient channel triggers for dev gateways by default", async () => { await runGatewayCli(["gateway", "run", "--allow-unconfigured", "--dev"]); @@ -1031,6 +1040,18 @@ describe("gateway run option collisions", () => { expect(runtimeErrors.join("\n")).toContain("--profile with a free port"); }); + it("reports forced port cleanup failures before startup", async () => { + forceFreePortAndWait.mockRejectedValueOnce(new Error("boom")); + + await expect( + runGatewayCli(["gateway", "run", "--allow-unconfigured", "--force"]), + ).rejects.toThrow("__exit__:1"); + + expect(startGatewayServer).not.toHaveBeenCalled(); + expect(runtimeErrors.join("\n")).toContain("Could not free port 18789: boom"); + expect(runtimeErrors.join("\n")).toContain("openclaw gateway status --deep"); + }); + it("marks service-mode gateway descendants with the live gateway pid", async () => { await withEnvAsync( { @@ -1671,6 +1692,9 @@ describe("gateway run option collisions", () => { }); expect(writeDiagnosticStabilityBundleForFailureSync).not.toHaveBeenCalled(); + expect(startGatewayServer).toHaveBeenCalledWith(port, expect.any(Object)); + expect(runtimeErrors.join("\n")).toContain(`gateway already running on port ${port}`); + expect(runtimeErrors.join("\n")).toContain("gateway stop"); }); it("exits 78 and parks launchd for a repairable shared-state schema", async () => { From a9da10ea9ed6048e0ef4c5170eae0b67133d4112 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 03:20:38 -0700 Subject: [PATCH 17/57] fix(gateway): bind response history to verified proxy identity (#118607) --- src/gateway/http-auth-utils.ts | 2 + .../http-utils.authorize-request.test.ts | 1 + src/gateway/openresponses-http.test.ts | 50 +++++++++++++++++++ src/gateway/openresponses-http.ts | 16 +++--- 4 files changed, 62 insertions(+), 7 deletions(-) diff --git a/src/gateway/http-auth-utils.ts b/src/gateway/http-auth-utils.ts index dfcb4f05963e..2d87af53dfb9 100644 --- a/src/gateway/http-auth-utils.ts +++ b/src/gateway/http-auth-utils.ts @@ -52,6 +52,7 @@ export function getBearerToken(req: IncomingMessage): string | undefined { type SharedSecretGatewayAuth = Pick; export type AuthorizedGatewayHttpRequest = { authMethod?: GatewayAuthResult["method"]; + user?: string; trustDeclaredOperatorScopes: boolean; controlUiPluginGrants?: ControlUiPluginTabAuthGrant[]; controlUiPluginGrant?: ControlUiPluginTabAuthGrant; @@ -258,6 +259,7 @@ async function checkGatewayHttpRequestAuthWith( ok: true, requestAuth: { authMethod: authResult.method, + ...(authResult.user ? { user: authResult.user } : {}), // Shared-secret bearer auth proves possession of the gateway secret, but it // does not prove a narrower per-request operator identity. HTTP endpoints // must opt in explicitly if they want to treat that shared-secret path as a diff --git a/src/gateway/http-utils.authorize-request.test.ts b/src/gateway/http-utils.authorize-request.test.ts index e4e693098025..fa1ce32bf789 100644 --- a/src/gateway/http-utils.authorize-request.test.ts +++ b/src/gateway/http-utils.authorize-request.test.ts @@ -88,6 +88,7 @@ describe("authorizeGatewayHttpRequestOrReply", () => { }), ).resolves.toEqual({ authMethod: "trusted-proxy", + user: "operator", trustDeclaredOperatorScopes: true, }); }); diff --git a/src/gateway/openresponses-http.test.ts b/src/gateway/openresponses-http.test.ts index a66de2e569dc..f08f64dd5a27 100644 --- a/src/gateway/openresponses-http.test.ts +++ b/src/gateway/openresponses-http.test.ts @@ -1865,6 +1865,56 @@ describe("OpenResponses HTTP API (e2e)", () => { } } + agentCommand.mockClear(); + agentCommand.mockResolvedValue({ payloads: [{ text: "hello" }] } as never); + const forwardedHeaders = { + "x-forwarded-proto": "https", + authorization: "Bearer forwarded-untrusted", + }; + const aliceResponse = await postResponses( + port, + { model: "openclaw", user: "alice", input: "private alice history" }, + { ...forwardedHeaders, "x-forwarded-user": "Alice@example.com" }, + ); + expect(aliceResponse.status).toBe(200); + const aliceResponseId = ((await aliceResponse.json()) as { id: string }).id; + const aliceSessionKey = requireSessionKey( + firstAgentOpts().sessionKey as string | undefined, + "Alice trusted-proxy response", + ); + + const aliceContinuation = await postResponses( + port, + { + model: "openclaw", + user: "alice", + previous_response_id: aliceResponseId, + input: "continue alice history", + }, + { + ...forwardedHeaders, + authorization: "Bearer different-forwarded-untrusted", + "x-forwarded-user": "Alice@example.com", + }, + ); + expect(aliceContinuation.status).toBe(200); + await ensureResponseConsumed(aliceContinuation); + + const bobContinuation = await postResponses( + port, + { + model: "openclaw", + user: "bob", + previous_response_id: aliceResponseId, + input: "attempt alice history", + }, + { ...forwardedHeaders, "x-forwarded-user": "bob@example.com" }, + ); + expect(bobContinuation.status).toBe(200); + await ensureResponseConsumed(bobContinuation); + expect(firstAgentOpts(2).sessionKey).not.toBe(aliceSessionKey); + expect(firstAgentOpts(1).sessionKey).toBe(aliceSessionKey); + agentCommand.mockClear(); const unauthorized = await postResponses( port, diff --git a/src/gateway/openresponses-http.ts b/src/gateway/openresponses-http.ts index b2ecac13e4e3..8fe72b9cc219 100644 --- a/src/gateway/openresponses-http.ts +++ b/src/gateway/openresponses-http.ts @@ -52,6 +52,7 @@ import { } from "./http-common.js"; import { handleGatewayPostJsonEndpoint } from "./http-endpoint-helpers.js"; import { + type AuthorizedGatewayHttpRequest, authorizeOpenAiCompatibleHttpModelOverride, getBearerToken, getHeader, @@ -126,27 +127,27 @@ function normalizeResponseSessionScope(scope: ResponseSessionScope): ResponseSes function resolveResponseSessionAuthSubject(params: { req: IncomingMessage; auth: ResolvedGatewayAuth; + requestAuth: AuthorizedGatewayHttpRequest; }): string { + // Proxy-verified identity owns continuation; forwarded bearers are unverified. + if (params.requestAuth.authMethod === "trusted-proxy") { + return `trusted-proxy:${params.requestAuth.user}`; + } const bearer = getBearerToken(params.req); if (bearer) { return `bearer:${createHash("sha256").update(bearer).digest("hex")}`; } - if (params.auth.mode === "trusted-proxy" && params.auth.trustedProxy?.userHeader) { - const user = getHeader(params.req, params.auth.trustedProxy.userHeader)?.trim(); - if (user) { - return `trusted-proxy:${user}`; - } - } return `gateway-auth:${params.auth.mode}`; } function createResponseSessionScope(params: { req: IncomingMessage; auth: ResolvedGatewayAuth; + requestAuth: AuthorizedGatewayHttpRequest; agentId: string; }): ResponseSessionScope { return normalizeResponseSessionScope({ - authSubject: resolveResponseSessionAuthSubject({ req: params.req, auth: params.auth }), + authSubject: resolveResponseSessionAuthSubject(params), agentId: params.agentId, requestedSessionKey: getHeader(params.req, "x-openclaw-session-key"), }); @@ -647,6 +648,7 @@ export async function handleOpenResponsesHttpRequest( const responseSessionScope = createResponseSessionScope({ req, auth: opts.auth, + requestAuth: handled.requestAuth, agentId: resolved.agentId, }); // Resolve session key: reuse previous_response_id only when it matches the From 3832074cd982ff5fc931970663aa1aceea2afd7d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 03:21:55 -0700 Subject: [PATCH 18/57] fix(mattermost): preserve channel lookup failures (#118622) Co-authored-by: Peter Steinberger --- .../mattermost/src/mattermost/client.ts | 14 +++ .../mattermost/src/mattermost/send.test.ts | 116 +++++++++++++++++- extensions/mattermost/src/mattermost/send.ts | 7 +- .../src/mattermost/target-resolution.test.ts | 4 +- .../src/mattermost/target-resolution.ts | 14 +-- 5 files changed, 138 insertions(+), 17 deletions(-) diff --git a/extensions/mattermost/src/mattermost/client.ts b/extensions/mattermost/src/mattermost/client.ts index 63358cf832b3..71ab707e044b 100644 --- a/extensions/mattermost/src/mattermost/client.ts +++ b/extensions/mattermost/src/mattermost/client.ts @@ -92,6 +92,20 @@ type MattermostFileInfo = { size?: number | null; }; +export function parseMattermostApiStatus(error: unknown): number | undefined { + if (!error || typeof error !== "object") { + return undefined; + } + const message = "message" in error && typeof error.message === "string" ? error.message : ""; + // Read only the provider's status prefix; upstream details can mention other HTTP statuses. + const match = /Mattermost API (\d{3})\b/.exec(message); + if (!match) { + return undefined; + } + const status = Number(match[1]); + return Number.isFinite(status) ? status : undefined; +} + export function normalizeMattermostBaseUrl(raw?: string | null): string | undefined { const trimmed = raw?.trim(); if (!trimmed) { diff --git a/extensions/mattermost/src/mattermost/send.test.ts b/extensions/mattermost/src/mattermost/send.test.ts index 302c69bd561e..3f2d8fc6bc7d 100644 --- a/extensions/mattermost/src/mattermost/send.test.ts +++ b/extensions/mattermost/src/mattermost/send.test.ts @@ -110,6 +110,34 @@ function directChannelRetryCall() { ) as [unknown, unknown, MattermostDirectRetryOptions?]; } +async function createMattermostProviderFailure( + status: number, + statusText: string, + message: string, +): Promise { + const { createMattermostClient } = + await vi.importActual("./client.js"); + const client = createMattermostClient({ + baseUrl: "https://mattermost.example.com", + botToken: "test-bot-token", + fetchImpl: async () => + new Response(JSON.stringify({ message }), { + status, + statusText, + headers: { "content-type": "application/json" }, + }), + }); + try { + await client.request("/teams/team-first/channels/name/release-alerts"); + } catch (error) { + if (error instanceof Error) { + return error; + } + throw error; + } + throw new Error("Expected the Mattermost provider request to fail"); +} + vi.mock("../../runtime-api.js", () => ({ loadOutboundMediaFromUrl: mockState.loadOutboundMediaFromUrl, })); @@ -162,7 +190,9 @@ vi.mock("./accounts.js", () => ({ resolveMattermostAccount: mockState.resolveMattermostAccount, })); -vi.mock("./client.js", () => ({ +vi.mock("./client.js", async () => ({ + parseMattermostApiStatus: (await vi.importActual("./client.js")) + .parseMattermostApiStatus, createMattermostClient: mockState.createMattermostClient, createMattermostDirectChannelWithRetry: mockState.createMattermostDirectChannelWithRetry, createMattermostPost: mockState.createMattermostPost, @@ -263,6 +293,90 @@ describe("sendMessageMattermost", () => { }); }); + it("continues searching later teams only when a channel is genuinely absent", async () => { + mockState.fetchMattermostUserTeams.mockResolvedValueOnce([ + { id: "team-first" }, + { id: "team-second" }, + ]); + mockState.fetchMattermostChannelByName + .mockRejectedValueOnce(await createMattermostProviderFailure(404, "Not Found", "missing")) + .mockResolvedValueOnce({ id: "channel-second" }); + + const result = await sendMessageMattermost("#release-alerts", "hello", { cfg: TEST_CFG }); + + expect(result.channelId).toBe("channel-second"); + expect(mockState.fetchMattermostChannelByName).toHaveBeenNthCalledWith( + 1, + {}, + "team-first", + "release-alerts", + ); + expect(mockState.fetchMattermostChannelByName).toHaveBeenNthCalledWith( + 2, + {}, + "team-second", + "release-alerts", + ); + expect(mockState.createMattermostPost).toHaveBeenCalledOnce(); + }); + + it("reports a missing named channel after every team returns not found", async () => { + mockState.fetchMattermostUserTeams.mockResolvedValueOnce([ + { id: "team-first" }, + { id: "team-second" }, + ]); + mockState.fetchMattermostChannelByName.mockRejectedValue( + await createMattermostProviderFailure(404, "Not Found", "missing channel"), + ); + + await expect( + sendMessageMattermost("#release-alerts", "hello", { cfg: TEST_CFG }), + ).rejects.toThrow('Mattermost channel "#release-alerts" not found in any team'); + + expect(mockState.fetchMattermostChannelByName).toHaveBeenCalledTimes(2); + expect(mockState.createMattermostPost).not.toHaveBeenCalled(); + }); + + it.each([ + { + name: "an expired bot token", + createError: () => createMattermostProviderFailure(401, "Unauthorized", "bot token expired"), + }, + { + name: "missing channel permissions", + createError: () => createMattermostProviderFailure(403, "Forbidden", "access denied"), + }, + { + name: "provider rate limiting", + createError: () => createMattermostProviderFailure(429, "Too Many Requests", "retry later"), + }, + { + name: "an outage whose detail mentions a missing resource", + createError: () => + createMattermostProviderFailure(503, "Service Unavailable", "upstream returned 404"), + }, + { + name: "a network failure", + createError: async () => new Error("connect ECONNRESET 192.0.2.12:443"), + }, + ])("preserves $name while resolving a named channel", async ({ createError }) => { + const error = await createError(); + mockState.fetchMattermostUserTeams.mockResolvedValueOnce([ + { id: "team-first" }, + { id: "team-second" }, + ]); + mockState.fetchMattermostChannelByName + .mockRejectedValueOnce(error) + .mockResolvedValueOnce({ id: "channel-second" }); + + await expect(sendMessageMattermost("#release-alerts", "hello", { cfg: TEST_CFG })).rejects.toBe( + error, + ); + + expect(mockState.fetchMattermostChannelByName).toHaveBeenCalledOnce(); + expect(mockState.createMattermostPost).not.toHaveBeenCalled(); + }); + it.each(MATTERMOST_MARKDOWN_GOLDENS)("$name", async ({ input, before, after }) => { expect(convertMarkdownTables(input, "code")).toBe(before); diff --git a/extensions/mattermost/src/mattermost/send.ts b/extensions/mattermost/src/mattermost/send.ts index 31f289c6db4d..880ee1f17487 100644 --- a/extensions/mattermost/src/mattermost/send.ts +++ b/extensions/mattermost/src/mattermost/send.ts @@ -26,6 +26,7 @@ import { fetchMattermostUserByUsername, fetchMattermostUserTeams, normalizeMattermostBaseUrl, + parseMattermostApiStatus, uploadMattermostFile, type MattermostUser, type CreateDmChannelRetryOptions, @@ -233,8 +234,10 @@ async function resolveChannelIdByName(params: { ); return channel.id; } - } catch { - // Channel not found in this team, try next + } catch (error) { + if (parseMattermostApiStatus(error) !== 404) { + throw error; + } } } throw new Error(`Mattermost channel "#${name}" not found in any team the bot belongs to`); diff --git a/extensions/mattermost/src/mattermost/target-resolution.test.ts b/extensions/mattermost/src/mattermost/target-resolution.test.ts index fc550f92b5bf..73a751dff814 100644 --- a/extensions/mattermost/src/mattermost/target-resolution.test.ts +++ b/extensions/mattermost/src/mattermost/target-resolution.test.ts @@ -11,7 +11,9 @@ vi.mock("./accounts.js", () => ({ resolveMattermostAccount, })); -vi.mock("./client.js", () => ({ +vi.mock("./client.js", async () => ({ + parseMattermostApiStatus: (await vi.importActual("./client.js")) + .parseMattermostApiStatus, createMattermostClient, fetchMattermostUser, fetchMattermostChannel, diff --git a/extensions/mattermost/src/mattermost/target-resolution.ts b/extensions/mattermost/src/mattermost/target-resolution.ts index 53d70bf76a2f..23b1324533c0 100644 --- a/extensions/mattermost/src/mattermost/target-resolution.ts +++ b/extensions/mattermost/src/mattermost/target-resolution.ts @@ -11,6 +11,7 @@ import { fetchMattermostChannel, fetchMattermostUser, normalizeMattermostBaseUrl, + parseMattermostApiStatus, } from "./client.js"; import { resolveMattermostTrustedChatKind } from "./monitor-auth.js"; import type { OpenClawConfig } from "./runtime-api.js"; @@ -138,19 +139,6 @@ function isExplicitMattermostTarget(raw: string): boolean { ); } -function parseMattermostApiStatus(err: unknown): number | undefined { - if (!err || typeof err !== "object") { - return undefined; - } - const msg = "message" in err && typeof err.message === "string" ? err.message : ""; - const match = /Mattermost API (\d{3})\b/.exec(msg); - if (!match) { - return undefined; - } - const code = Number(match[1]); - return Number.isFinite(code) ? code : undefined; -} - export async function resolveMattermostOpaqueTarget(params: { input: string; cfg?: OpenClawConfig; From 06957a69d8674024f44e20b2262403f17bb40d7e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 03:34:05 -0700 Subject: [PATCH 19/57] fix(discord): keep acknowledged interaction replies private (#118611) --- .../agent-components.plugin-interactive.ts | 9 ++-- .../discord/src/monitor/monitor.test.ts | 42 +++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/extensions/discord/src/monitor/agent-components.plugin-interactive.ts b/extensions/discord/src/monitor/agent-components.plugin-interactive.ts index c54650f51c90..fbba2ecc84f9 100644 --- a/extensions/discord/src/monitor/agent-components.plugin-interactive.ts +++ b/extensions/discord/src/monitor/agent-components.plugin-interactive.ts @@ -66,10 +66,11 @@ export async function dispatchPluginDiscordInteractiveEvent(params: { }, reply: async ({ text, ephemeral = true }: { text: string; ephemeral?: boolean }) => { responded = true; - await params.interaction.reply({ - content: text, - ephemeral, - }); + const payload = { content: text, ephemeral }; + // Deferred component replies edit the public source; follow-ups preserve reply visibility. + await (acknowledged + ? params.interaction.followUp(payload) + : params.interaction.reply(payload)); }, followUp: async ({ text, ephemeral = true }: { text: string; ephemeral?: boolean }) => { responded = true; diff --git a/extensions/discord/src/monitor/monitor.test.ts b/extensions/discord/src/monitor/monitor.test.ts index 0cf6d3bd1cea..b2e4c85f8d62 100644 --- a/extensions/discord/src/monitor/monitor.test.ts +++ b/extensions/discord/src/monitor/monitor.test.ts @@ -919,6 +919,48 @@ describe("discord component interactions", () => { expect(dispatchReplyMock).not.toHaveBeenCalled(); }); + it.each([ + { visibility: "private", ephemeral: true }, + { visibility: "public", ephemeral: false }, + { visibility: "default-private", ephemeral: undefined }, + ])( + "sends $visibility plugin replies as new messages after component acknowledgment", + async ({ ephemeral }) => { + registerDiscordComponentEntries({ + entries: [createButtonEntry({ callbackData: "codex:approve" })], + modals: [], + }); + dispatchPluginInteractiveHandlerMock.mockImplementation(async (params: unknown) => { + const typedParams = params as { + onMatched: () => Promise; + respond: { reply: (payload: { text: string; ephemeral?: boolean }) => Promise }; + }; + await typedParams.onMatched(); + await typedParams.respond.reply({ + text: "Plugin result", + ...(ephemeral === undefined ? {} : { ephemeral }), + }); + return { matched: true, handled: true, duplicate: false }; + }); + + const acknowledge = vi.fn().mockResolvedValue(undefined); + const followUp = vi.fn().mockResolvedValue(undefined); + const reply = vi.fn().mockResolvedValue(undefined); + const button = createDiscordComponentButton(createComponentContext()); + const { interaction } = createComponentButtonInteraction({ acknowledge, followUp, reply }); + + await button.run(interaction, { cid: "btn_1" } as ComponentData); + + expect(acknowledge).toHaveBeenCalledTimes(1); + expect(followUp).toHaveBeenCalledWith({ + content: "Plugin result", + ephemeral: ephemeral ?? true, + }); + expect(reply).not.toHaveBeenCalled(); + expect(dispatchReplyMock).not.toHaveBeenCalled(); + }, + ); + it("lets plugin Discord interactions clear components after acknowledging", async () => { registerDiscordComponentEntries({ entries: [createButtonEntry({ callbackData: "codex:approve" })], From 56f02c495600a16dbedfb702bc41a1a8a0a2b6f0 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 03:37:58 -0700 Subject: [PATCH 20/57] fix(otel): fail closed when configured proxies are invalid (#118612) --- .../diagnostics-otel/src/service-exporter.ts | 5 +- .../diagnostics-otel/src/service.test.ts | 93 +++++++++++++++---- extensions/diagnostics-otel/src/service.ts | 4 +- 3 files changed, 78 insertions(+), 24 deletions(-) diff --git a/extensions/diagnostics-otel/src/service-exporter.ts b/extensions/diagnostics-otel/src/service-exporter.ts index d01f35397fa9..fa48ea8199d7 100644 --- a/extensions/diagnostics-otel/src/service-exporter.ts +++ b/extensions/diagnostics-otel/src/service-exporter.ts @@ -119,10 +119,7 @@ export function resolveOtelHttpAgentOptions(params: { const agent = createNodeProxyAgent({ mode: "env", targetUrl: url, agentOptions }); return agent ? () => agent : undefined; } catch { - logger.warn( - `diagnostics-otel: env proxy agent unavailable for OTLP ${signalIdentifier.toLowerCase()} exporter; falling back to default Node agent`, - ); - return undefined; + throw new Error("Configured telemetry proxy is invalid or unsupported; refusing direct export"); } } diff --git a/extensions/diagnostics-otel/src/service.test.ts b/extensions/diagnostics-otel/src/service.test.ts index ab2b7431b66d..65f279eb62df 100644 --- a/extensions/diagnostics-otel/src/service.test.ts +++ b/extensions/diagnostics-otel/src/service.test.ts @@ -1869,32 +1869,89 @@ describe("diagnostics-otel service", () => { } }); - test("falls back to default OTLP agents when env proxy agent creation fails", async () => { + test.each([ + ["traces", { traces: true }, "unsupported proxy protocol"], + ["metrics", { metrics: true }, "invalid proxy URL"], + ["logs", { logs: true }, "unsupported proxy protocol"], + ] as const)( + "refuses direct %s export when the configured proxy cannot initialize", + async (_signal, signals, errorMessage) => { + createNodeProxyAgentMock.mockImplementation(() => { + throw new Error(errorMessage); + }); + + await expect( + startOtelService({ endpoint: "https://collector.example.com/otlp", ...signals }), + ).rejects.toThrow( + "Configured telemetry proxy is invalid or unsupported; refusing direct export", + ); + + expect(traceExporterCtor).not.toHaveBeenCalled(); + expect(metricExporterCtor).not.toHaveBeenCalled(); + expect(logExporterCtor).not.toHaveBeenCalled(); + }, + ); + + test("redacts proxy credentials from telemetry startup failures", async () => { + const proxyPassword = "qa-otel-proxy-password-sentinel"; createNodeProxyAgentMock.mockImplementation(() => { - throw new Error("unsupported proxy protocol"); + throw new Error(`Invalid proxy URL: "https://operator:${proxyPassword}@proxy.example.com"`); }); - const { ctx } = await startOtelService({ + const failure = await startOtelService({ endpoint: "https://collector.example.com/otlp", traces: true, - metrics: true, - logs: true, - }); + }).catch((error: unknown) => error); - expect(firstExporterOptions(traceExporterCtor).httpAgentOptions).toBeUndefined(); - expect(firstExporterOptions(metricExporterCtor).httpAgentOptions).toBeUndefined(); - expect(firstExporterOptions(logExporterCtor).httpAgentOptions).toBeUndefined(); - expect(ctx.logger.warn).toHaveBeenCalledWith( - "diagnostics-otel: env proxy agent unavailable for OTLP traces exporter; falling back to default Node agent", - ); - expect(ctx.logger.warn).toHaveBeenCalledWith( - "diagnostics-otel: env proxy agent unavailable for OTLP metrics exporter; falling back to default Node agent", - ); - expect(ctx.logger.warn).toHaveBeenCalledWith( - "diagnostics-otel: env proxy agent unavailable for OTLP logs exporter; falling back to default Node agent", - ); + expect(failure).toBeInstanceOf(Error); + expect(failure).toMatchObject({ + message: "Configured telemetry proxy is invalid or unsupported; refusing direct export", + }); + expect(failure).not.toHaveProperty("cause"); + expect(String(failure)).not.toContain(proxyPassword); + expect(traceExporterCtor).not.toHaveBeenCalled(); }); + test.each([ + { + disabledSignal: "traces", + enabledSignal: "metrics", + disabledEndpoint: "tracesEndpoint", + signals: { traces: false, metrics: true }, + }, + { + disabledSignal: "metrics", + enabledSignal: "traces", + disabledEndpoint: "metricsEndpoint", + signals: { traces: true, metrics: false }, + }, + ] as const)( + "does not resolve proxy settings for disabled $disabledSignal export", + async ({ disabledSignal, enabledSignal, disabledEndpoint, signals }) => { + createNodeProxyAgentMock.mockImplementation(({ targetUrl }: { targetUrl: string }) => { + if (targetUrl.includes(`disabled-${disabledSignal}.example.com`)) { + throw new Error("invalid disabled-signal proxy"); + } + return nodeProxyAgent; + }); + + await startOtelService({ + endpoint: "https://collector.example.com/otlp", + ...signals, + configure: (ctx) => { + ctx.config.diagnostics!.otel![disabledEndpoint] = + `https://disabled-${disabledSignal}.example.com/otlp`; + }, + }); + + expect(createNodeProxyAgentCalls()).toEqual([ + expect.objectContaining({ + targetUrl: `https://collector.example.com/otlp/v1/${enabledSignal}`, + }), + ]); + }, + ); + test("leaves OTLP HTTP exporters on their default agents when env proxy is bypassed", async () => { await startOtelService({ endpoint: "https://collector.example.com/otlp", diff --git a/extensions/diagnostics-otel/src/service.ts b/extensions/diagnostics-otel/src/service.ts index 178b2d53d5ad..d0808565ed14 100644 --- a/extensions/diagnostics-otel/src/service.ts +++ b/extensions/diagnostics-otel/src/service.ts @@ -175,12 +175,12 @@ export function createDiagnosticsOtelService(): OpenClawPluginService { path: "v1/metrics", }); const traceHttpAgentOptions = resolveOtelHttpAgentOptions({ - url: traceUrl, + url: tracesEnabled ? traceUrl : undefined, signalIdentifier: "TRACES", logger: ctx.logger, }); const metricHttpAgentOptions = resolveOtelHttpAgentOptions({ - url: metricUrl, + url: metricsEnabled ? metricUrl : undefined, signalIdentifier: "METRICS", logger: ctx.logger, }); From 6d8c1ed30db4cc9b22cda96670d6bd110ad0e4ed Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 03:38:02 -0700 Subject: [PATCH 21/57] fix(voice-call): prevent unrelated session recall in fast context (#118498) * fix(talk): authorize fast session context * test(plugins): preserve memory runtime mock types * fix(plugins): preserve memory authorizer binding * fix(memory): forward search authorization in lazy runtime * test(memory): verify lazy authorizer host binding * docs(plugins): define memory search authorization contract --- docs/plugins/sdk-overview.md | 8 ++ docs/plugins/voice-call.md | 2 +- extensions/memory-core/index.test.ts | 38 ++++++++ extensions/memory-core/index.ts | 8 ++ .../memory-core/src/runtime-provider.test.ts | 41 ++++++++ .../memory-core/src/runtime-provider.ts | 5 + .../src/session-search-visibility.test.ts | 43 +++++++++ src/plugins/memory-runtime.test.ts | 96 ++++++++++++++++++- src/plugins/memory-runtime.ts | 22 +++++ src/plugins/registry-contribution-types.ts | 10 +- src/talk/fast-context-runtime.test.ts | 64 ++++++++++++- src/talk/fast-context-runtime.ts | 30 +++--- 12 files changed, 348 insertions(+), 19 deletions(-) diff --git a/docs/plugins/sdk-overview.md b/docs/plugins/sdk-overview.md index 60e0c9de9d1a..ba1a83c73f5a 100644 --- a/docs/plugins/sdk-overview.md +++ b/docs/plugins/sdk-overview.md @@ -635,6 +635,14 @@ For an end-to-end authoring guide, see artifacts still use `listActiveMemoryPublicArtifacts(...)` from the retained `openclaw/plugin-sdk/memory-host-core` facade until a focused public consumer API exists; they must not reach into another plugin's private layout. +- A memory runtime that can return session-transcript hits should implement + `runtime.authorizeSearchHits(...)`. The host calls this hook before raw search + hits reach caller-visible surfaces and supplies the requesting agent, session + key, and sandbox state. Return only hits the requester may observe. If the hook + is absent, OpenClaw fails closed by withholding session-source hits while + retaining ordinary memory hits. Keep transcript identity and visibility + policy in the owning memory plugin; callers must not infer authorization from + paths or duplicate plugin-specific rules. - `MemoryFlushPlan.model` can pin the flush turn to an exact `provider/model` reference, such as `ollama/qwen3:8b`, without inheriting the active fallback chain. diff --git a/docs/plugins/voice-call.md b/docs/plugins/voice-call.md index 0eaf1203c53c..10c0c8e696fb 100644 --- a/docs/plugins/voice-call.md +++ b/docs/plugins/voice-call.md @@ -264,7 +264,7 @@ Current runtime behavior: - Voice Call exposes the shared `openclaw_agent_consult` realtime tool by default. The realtime model can call it when the caller asks for deeper reasoning, current information, or normal OpenClaw tools. - `realtime.consultPolicy` optionally adds guidance for when the realtime model should call `openclaw_agent_consult`. - `realtime.agentContext.enabled` is default-off. When enabled, Voice Call injects a bounded agent identity and selected workspace-file capsule into the realtime provider instructions at session setup. -- `realtime.fastContext.enabled` is default-off. When enabled, Voice Call first searches indexed memory/session context for the consult question and returns those snippets to the realtime model within `realtime.fastContext.timeoutMs` before falling back to the full consult agent only if `realtime.fastContext.fallbackToConsult` is true. +- `realtime.fastContext.enabled` is default-off. When enabled, Voice Call first searches indexed memory/session context for the consult question and returns authorized snippets to the realtime model within `realtime.fastContext.timeoutMs` before falling back to the full consult agent only if `realtime.fastContext.fallbackToConsult` is true. The active memory plugin authorizes session-transcript hits; plugins without that capability fail closed for session hits while ordinary memory hits remain available. - If `realtime.provider` points at an unregistered provider, or no realtime voice provider is registered at all, Voice Call logs a warning and skips realtime media instead of failing the whole plugin. - `inboundPolicy` must not be `"disabled"` when `realtime.enabled` is true; `validateProviderConfig` rejects that combination. - Consult session keys reuse the stored call session when available, then fall back to the configured `sessionScope` (`per-phone` by default, or `per-call` for isolated calls). diff --git a/extensions/memory-core/index.test.ts b/extensions/memory-core/index.test.ts index 28331697554f..a474e5bd6179 100644 --- a/extensions/memory-core/index.test.ts +++ b/extensions/memory-core/index.test.ts @@ -10,8 +10,10 @@ import { buildPromptSection } from "./src/prompt-section.js"; const closeMemorySearchManagerMock = vi.hoisted(() => vi.fn(async () => {})); const getMemorySearchManagerMock = vi.hoisted(() => vi.fn(async () => null)); +const authorizeSearchHitsMock = vi.hoisted(() => vi.fn(async ({ hits }) => hits)); const createMemoryRuntimeMock = vi.hoisted(() => vi.fn((_host: MemoryCoreRuntimeHost = {}) => ({ + authorizeSearchHits: authorizeSearchHitsMock, closeAllMemorySearchManagers: vi.fn(async () => {}), closeMemorySearchManager: closeMemorySearchManagerMock, getMemorySearchManager: getMemorySearchManagerMock, @@ -319,6 +321,42 @@ describe("memory-core plugin runtime registration", () => { }); }); + it("forwards search-hit authorization through the registered memory runtime", async () => { + const runtime = registerMemoryCoreRuntime(); + const cfg = {} as OpenClawConfig; + const hits = [ + { + source: "sessions" as const, + path: "sessions/private.jsonl", + startLine: 1, + endLine: 1, + score: 1, + snippet: "private", + }, + ]; + + await expect( + runtime.authorizeSearchHits?.({ + cfg, + agentId: "main", + requesterSessionKey: "agent:main:voice:15550001234", + sandboxed: false, + hits, + }), + ).resolves.toEqual(hits); + expect(authorizeSearchHitsMock).toHaveBeenCalledWith({ + cfg, + agentId: "main", + requesterSessionKey: "agent:main:voice:15550001234", + sandboxed: false, + hits, + }); + expect(createMemoryRuntimeMock).toHaveBeenCalledWith({ + acquireLocalService: hostRuntime.llm.acquireLocalService, + withLease: expect.any(Function), + }); + }); + it("binds the host SQLite lease hook to tools and CLI runtime", async () => { const runtime = registerMemoryCoreRuntime(); const cfg = {} as OpenClawConfig; diff --git a/extensions/memory-core/index.ts b/extensions/memory-core/index.ts index 2820001f2672..eb66da5e7dc3 100644 --- a/extensions/memory-core/index.ts +++ b/extensions/memory-core/index.ts @@ -252,6 +252,14 @@ function createLazyMemoryRuntime(host: MemoryCoreRuntimeHost): MemoryPluginRunti const { createMemoryRuntime } = await loadRuntimeProviderModule(); return await createMemoryRuntime(host).getMemorySearchManager(params); }, + async authorizeSearchHits(params) { + const { createMemoryRuntime } = await loadRuntimeProviderModule(); + const runtime = createMemoryRuntime(host); + if (!runtime.authorizeSearchHits) { + throw new Error("memory-core runtime search authorization is unavailable"); + } + return await runtime.authorizeSearchHits(params); + }, resolveMemoryBackendConfig(params) { return resolveMemoryBackendConfig(params); }, diff --git a/extensions/memory-core/src/runtime-provider.test.ts b/extensions/memory-core/src/runtime-provider.test.ts index bb4e13ee19d1..878ad9f61466 100644 --- a/extensions/memory-core/src/runtime-provider.test.ts +++ b/extensions/memory-core/src/runtime-provider.test.ts @@ -1,5 +1,6 @@ // Memory Core provider tests cover plugin runtime integration. import type { OpenClawConfig } from "openclaw/plugin-sdk/memory-core-host-runtime-core"; +import type { MemorySearchResult } from "openclaw/plugin-sdk/memory-core-host-runtime-files"; import { describe, expect, it, vi } from "vitest"; const managerDebug = { @@ -17,6 +18,7 @@ const getMemorySearchManagerMock = vi.hoisted(() => error: undefined, })), ); +const filterMemorySearchHitsBySessionVisibilityMock = vi.hoisted(() => vi.fn()); vi.mock("./memory/index.js", () => ({ closeAllMemorySearchManagers: vi.fn(async () => {}), @@ -24,6 +26,10 @@ vi.mock("./memory/index.js", () => ({ getMemorySearchManager: getMemorySearchManagerMock, })); +vi.mock("./session-search-visibility.js", () => ({ + filterMemorySearchHitsBySessionVisibility: filterMemorySearchHitsBySessionVisibilityMock, +})); + import { createMemoryRuntime, memoryRuntime } from "./runtime-provider.js"; describe("memoryRuntime", () => { @@ -97,4 +103,39 @@ describe("memoryRuntime", () => { withLease: secondLease, }); }); + + it("delegates raw-hit authorization to the canonical session visibility filter", async () => { + const cfg = {} as OpenClawConfig; + const hits: MemorySearchResult[] = [ + { + source: "sessions", + path: "sessions/private.jsonl", + startLine: 1, + endLine: 1, + score: 1, + snippet: "private", + }, + ]; + filterMemorySearchHitsBySessionVisibilityMock.mockResolvedValue([]); + if (!memoryRuntime.authorizeSearchHits) { + throw new Error("memory runtime search authorizer is unavailable"); + } + + await expect( + memoryRuntime.authorizeSearchHits({ + cfg, + agentId: "main", + requesterSessionKey: "agent:main:voice:15550001234", + sandboxed: false, + hits, + }), + ).resolves.toEqual([]); + expect(filterMemorySearchHitsBySessionVisibilityMock).toHaveBeenCalledWith({ + cfg, + agentId: "main", + requesterSessionKey: "agent:main:voice:15550001234", + sandboxed: false, + hits, + }); + }); }); diff --git a/extensions/memory-core/src/runtime-provider.ts b/extensions/memory-core/src/runtime-provider.ts index b9648e3e718d..35d00e8bd5ec 100644 --- a/extensions/memory-core/src/runtime-provider.ts +++ b/extensions/memory-core/src/runtime-provider.ts @@ -25,6 +25,11 @@ export function createMemoryRuntime(host: MemoryCoreRuntimeHost = {}): MemoryPlu resolveMemoryBackendConfig(params) { return resolveMemoryBackendConfig(params); }, + async authorizeSearchHits(params) { + const { filterMemorySearchHitsBySessionVisibility } = + await import("./session-search-visibility.js"); + return await filterMemorySearchHitsBySessionVisibility(params); + }, async closeAllMemorySearchManagers() { await closeAllMemorySearchManagers(); }, diff --git a/extensions/memory-core/src/session-search-visibility.test.ts b/extensions/memory-core/src/session-search-visibility.test.ts index 294e179259af..b9c65f0dea9d 100644 --- a/extensions/memory-core/src/session-search-visibility.test.ts +++ b/extensions/memory-core/src/session-search-visibility.test.ts @@ -168,6 +168,49 @@ describe("filterMemorySearchHitsBySessionVisibility", () => { expect(filtered).toEqual(hits); }); + it("keeps memory but hides an unrelated same-agent session from a voice requester", async () => { + combinedSessionStore = { + "agent:main:voice:15550001111": { + sessionId: "voice", + updatedAt: 2, + sessionFile: "/tmp/sessions/voice.jsonl", + chatType: "direct", + }, + "agent:main:telegram:direct:owner": { + sessionId: "private", + updatedAt: 1, + sessionFile: "/tmp/sessions/private.jsonl", + chatType: "direct", + }, + }; + const memoryHit: MemorySearchResult = { + path: "memory/allowed.md", + source: "memory", + score: 1, + snippet: "Visible memory", + startLine: 1, + endLine: 2, + }; + const sessionHit: MemorySearchResult = { + path: "sessions/private.jsonl", + source: "sessions", + score: 1, + snippet: "Private session secret", + startLine: 1, + endLine: 2, + }; + + const filtered = await filterMemorySearchHitsBySessionVisibility({ + cfg: asOpenClawConfig({}), + agentId: "main", + requesterSessionKey: "agent:main:voice:15550001111", + sandboxed: false, + hits: [memoryHit, sessionHit], + }); + + expect(filtered).toEqual([memoryHit]); + }); + it("allows another same-agent private transcript through trusted conversation recall", async () => { combinedSessionStore = { "agent:main:telegram:direct:owner": { diff --git a/src/plugins/memory-runtime.test.ts b/src/plugins/memory-runtime.test.ts index 6d3926274a8d..e215509be1b9 100644 --- a/src/plugins/memory-runtime.test.ts +++ b/src/plugins/memory-runtime.test.ts @@ -1,8 +1,12 @@ /** Covers non-activating memory registry handles and requesting-agent workspace ownership. */ import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { MemorySearchResult } from "../memory-host-sdk/host/types.js"; +import type { MemoryPluginRuntime } from "./registry-contribution-types.js"; import { createEmptyPluginRegistry } from "./registry-empty.js"; import { getPluginRuntimeGatewayRequestScope } from "./runtime/gateway-request-scope.js"; +type AuthorizeSearchHits = NonNullable; + const mocks = vi.hoisted(() => ({ getMemoryRuntime: vi.fn(), loadPluginRegistryHandle: vi.fn(), @@ -25,6 +29,7 @@ vi.mock("./memory-state.js", async (importOriginal) => { }); import { + authorizeActiveMemorySearchHits, closeActiveMemorySearchManager, closeActiveMemorySearchManagers, getActiveMemorySearchManager, @@ -35,14 +40,24 @@ import { hasMemoryRuntime } from "./memory-state.js"; function createRuntime() { return { + authorizeSearchHits: vi.fn(async ({ hits }) => hits), getMemorySearchManager: vi.fn(async () => ({ manager: null, error: "no index" })), resolveMemoryBackendConfig: vi.fn(() => ({ backend: "builtin" as const })), closeMemorySearchManager: vi.fn(async () => {}), closeAllMemorySearchManagers: vi.fn(async () => {}), - }; + } satisfies MemoryPluginRuntime; } -function createRegistry(runtime = createRuntime()) { +type TestRegistry = { + registry: ReturnType; + runtime: T; +}; + +function createRegistry(): TestRegistry>; +function createRegistry(runtime: T): TestRegistry; +function createRegistry( + runtime: MemoryPluginRuntime = createRuntime(), +): TestRegistry { const registry = createEmptyPluginRegistry(); registry.memoryCapabilities.push({ pluginId: "memory-core", capability: { runtime } }); return { registry, runtime }; @@ -212,6 +227,83 @@ describe("memory runtime handles", () => { expect(mocks.loadPluginRegistryHandle).not.toHaveBeenCalled(); }); + it("authorizes raw hits inside the selected plugin runtime scope", async () => { + const { registry, runtime } = createRegistry(); + runtime.authorizeSearchHits.mockImplementationOnce(async ({ hits }) => { + expect(getPluginRuntimeGatewayRequestScope()?.pluginRegistry).toBe(registry); + return hits.filter((hit) => hit.source === "memory"); + }); + mocks.loadPluginRegistryHandle.mockReturnValue(registry); + const hits: MemorySearchResult[] = [ + { + source: "memory", + path: "memory.md", + startLine: 1, + endLine: 1, + score: 1, + snippet: "memory", + }, + { + source: "sessions", + path: "sessions/private.jsonl", + startLine: 1, + endLine: 1, + score: 1, + snippet: "private", + }, + ]; + + await expect( + authorizeActiveMemorySearchHits({ + cfg: memoryConfig, + agentId: "main", + requesterSessionKey: "agent:main:voice:15550001234", + sandboxed: false, + hits, + }), + ).resolves.toEqual([hits[0]]); + }); + + it("fails closed on session hits when a memory runtime has no authorizer", async () => { + const runtimeWithoutAuthorizer = { + getMemorySearchManager: vi.fn(async () => ({ manager: null, error: "no index" })), + resolveMemoryBackendConfig: vi.fn(() => ({ backend: "builtin" as const })), + closeMemorySearchManager: vi.fn(async () => {}), + closeAllMemorySearchManagers: vi.fn(async () => {}), + } satisfies MemoryPluginRuntime; + mocks.loadPluginRegistryHandle.mockReturnValue( + createRegistry(runtimeWithoutAuthorizer).registry, + ); + const hits: MemorySearchResult[] = [ + { + source: "memory", + path: "memory.md", + startLine: 1, + endLine: 1, + score: 1, + snippet: "memory", + }, + { + source: "sessions", + path: "sessions/private.jsonl", + startLine: 1, + endLine: 1, + score: 1, + snippet: "private", + }, + ]; + + await expect( + authorizeActiveMemorySearchHits({ + cfg: memoryConfig, + agentId: "main", + requesterSessionKey: "agent:main:voice:15550001234", + sandboxed: false, + hits, + }), + ).resolves.toEqual([hits[0]]); + }); + it("closes managers through current and retired workspace handles without reloading", async () => { const main = createRegistry(); const research = createRegistry(); diff --git a/src/plugins/memory-runtime.ts b/src/plugins/memory-runtime.ts index 252c7f1d984c..8c2e4c9d54e1 100644 --- a/src/plugins/memory-runtime.ts +++ b/src/plugins/memory-runtime.ts @@ -9,12 +9,16 @@ import { resolveMemoryCapabilityRegistration, setStandaloneMemoryManagerActive, } from "./memory-state.js"; +import type { MemoryPluginRuntime } from "./registry-contribution-types.js"; import type { PluginRegistry } from "./registry-types.js"; import { withPluginRuntimeRegistryScope } from "./runtime/gateway-request-scope.js"; type MemoryRuntime = NonNullable< PluginRegistry["memoryCapabilities"][number]["capability"]["runtime"] >; +type MemorySearchAuthorization = Parameters< + NonNullable +>[0]; type MemoryRuntimeOwner = { runtime: MemoryRuntime; registry?: PluginRegistry }; let standaloneMemoryRegistrySlot: | { key: string; registry: PluginRegistry; retiredRuntimes: Map } @@ -134,6 +138,24 @@ export async function getActiveMemorySearchManager(params: { ); } +/** Applies the selected memory plugin's authorization policy to raw search hits. */ +export async function authorizeActiveMemorySearchHits( + params: MemorySearchAuthorization, +): Promise { + const owner = ensureMemoryRuntime(params); + if (!owner) { + // Session artifacts need plugin-owned identity mapping before they are safe + // to expose. Runtimes without that capability may still return memory hits. + return params.hits.filter((hit) => hit.source !== "sessions"); + } + return await withMemoryRuntimeOwner(owner, async (runtime) => { + if (!runtime.authorizeSearchHits) { + return params.hits.filter((hit) => hit.source !== "sessions"); + } + return await runtime.authorizeSearchHits(params); + }); +} + /** Resolves current memory backend config without constructing a manager. */ export function resolveActiveMemoryBackendConfig(params: { cfg: OpenClawConfig; agentId: string }) { const owner = ensureMemoryRuntime(params); diff --git a/src/plugins/registry-contribution-types.ts b/src/plugins/registry-contribution-types.ts index 7d5bc0c87fe6..bb30be36a52b 100644 --- a/src/plugins/registry-contribution-types.ts +++ b/src/plugins/registry-contribution-types.ts @@ -3,7 +3,7 @@ import type { EmbeddingInput } from "../../packages/memory-host-sdk/src/engine-e import type { MemoryCitationsMode } from "../config/types.memory.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { ContextEngine } from "../context-engine/types.js"; -import type { MemorySearchManager } from "../memory-host-sdk/host/types.js"; +import type { MemorySearchManager, MemorySearchResult } from "../memory-host-sdk/host/types.js"; import type { EmbeddingProvider, EmbeddingProviderAdapter, @@ -290,6 +290,14 @@ export type MemoryPluginRuntime = { cfg: OpenClawConfig; agentId: string; }): MemoryRuntimeBackendConfig; + /** Authorize raw hits before caller-visible use; absent runtimes must not expose session hits. */ + authorizeSearchHits?(params: { + cfg: OpenClawConfig; + agentId: string; + requesterSessionKey: string | undefined; + sandboxed: boolean; + hits: MemorySearchResult[]; + }): Promise; closeMemorySearchManager?(params: { cfg: OpenClawConfig; agentId: string }): Promise; closeAllMemorySearchManagers?(): Promise; }; diff --git a/src/talk/fast-context-runtime.test.ts b/src/talk/fast-context-runtime.test.ts index 1b00903c89aa..05e64296cf81 100644 --- a/src/talk/fast-context-runtime.test.ts +++ b/src/talk/fast-context-runtime.test.ts @@ -1,22 +1,28 @@ // Fast context runtime tests cover timeout and fast context generation behavior. import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ + authorizeActiveMemorySearchHits: vi.fn(), getActiveMemorySearchManager: vi.fn(), })); vi.mock("../plugins/memory-runtime.js", () => ({ + authorizeActiveMemorySearchHits: mocks.authorizeActiveMemorySearchHits, getActiveMemorySearchManager: mocks.getActiveMemorySearchManager, })); import { resolveRealtimeVoiceFastContextConsult } from "./fast-context-runtime.js"; describe("resolveRealtimeVoiceFastContextConsult", () => { + beforeEach(() => { + mocks.authorizeActiveMemorySearchHits.mockReset().mockImplementation(async ({ hits }) => hits); + mocks.getActiveMemorySearchManager.mockReset(); + }); + afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); - mocks.getActiveMemorySearchManager.mockReset(); }); it("caps oversized fast-context timeouts before scheduling Node timers", async () => { @@ -119,4 +125,58 @@ describe("resolveRealtimeVoiceFastContextConsult", () => { }, }); }); + + it("removes unauthorized session hits before building caller context", async () => { + const cfg = {}; + const hits = [ + { + path: "memory/allowed.md", + startLine: 1, + endLine: 1, + snippet: "Visible memory", + source: "memory" as const, + score: 1, + }, + { + path: "sessions/private.jsonl", + startLine: 1, + endLine: 1, + snippet: "Private session secret", + source: "sessions" as const, + score: 1, + }, + ]; + mocks.getActiveMemorySearchManager.mockResolvedValue({ + manager: { search: vi.fn().mockResolvedValue(hits) }, + }); + mocks.authorizeActiveMemorySearchHits.mockResolvedValue([hits[0]]); + + const result = await resolveRealtimeVoiceFastContextConsult({ + cfg, + agentId: "main", + sessionKey: "agent:main:voice:15550001234", + config: { + enabled: true, + timeoutMs: 1_000, + maxResults: 2, + sources: ["memory", "sessions"], + fallbackToConsult: false, + }, + args: { question: "What do you remember?" }, + logger: {}, + }); + + expect(mocks.authorizeActiveMemorySearchHits).toHaveBeenCalledWith({ + cfg, + agentId: "main", + requesterSessionKey: "agent:main:voice:15550001234", + sandboxed: false, + hits, + }); + expect(result).toEqual({ + handled: true, + result: { text: expect.stringContaining("Visible memory") }, + }); + expect(result.handled && result.result.text).not.toContain("Private session secret"); + }); }); diff --git a/src/talk/fast-context-runtime.ts b/src/talk/fast-context-runtime.ts index 0871ac6344de..eeb33a39f576 100644 --- a/src/talk/fast-context-runtime.ts +++ b/src/talk/fast-context-runtime.ts @@ -9,7 +9,11 @@ import { resolveTimerTimeoutMs } from "@openclaw/normalization-core/number-coerc import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { formatErrorMessage } from "../infra/errors.js"; -import { getActiveMemorySearchManager } from "../plugins/memory-runtime.js"; +import type { MemorySearchResult } from "../memory-host-sdk/host/types.js"; +import { + authorizeActiveMemorySearchHits, + getActiveMemorySearchManager, +} from "../plugins/memory-runtime.js"; import { withTimeout } from "../utils/with-timeout.js"; import type { RealtimeVoiceAgentConsultResult } from "./agent-consult-runtime.js"; import { parseRealtimeVoiceAgentConsultArgs } from "./agent-consult-tool.js"; @@ -18,15 +22,6 @@ type Logger = { debug?: (message: string) => void; }; -type MemorySearchHit = { - path: string; - startLine: number; - endLine: number; - snippet: string; - source: "memory" | "sessions"; - score: number; -}; - /** Fast-context lookup policy for realtime voice consult shortcuts. */ export type RealtimeVoiceFastContextConfig = { enabled: boolean; @@ -48,7 +43,7 @@ export type RealtimeVoiceFastContextLabels = { type FastContextLookupResult = | { status: "unavailable"; error?: string } - | { status: "hits"; hits: MemorySearchHit[] }; + | { status: "hits"; hits: MemorySearchResult[] }; export type RealtimeVoiceFastContextConsultResult = | { handled: false } @@ -89,7 +84,7 @@ function resolveLabels( function buildContextText(params: { query: string; - hits: MemorySearchHit[]; + hits: MemorySearchResult[]; labels: RealtimeVoiceFastContextLabels; }): string { const hits = params.hits @@ -133,11 +128,20 @@ async function lookupFastContext(params: { error: memory.error ?? "no active memory manager", }; } - const hits = await memory.manager.search(params.query, { + const rawHits = await memory.manager.search(params.query, { maxResults: params.config.maxResults, sessionKey: params.sessionKey, sources: params.config.sources, }); + // This shortcut runs before an agent sandbox exists, but it still carries + // the voice session identity needed for ordinary session-history visibility. + const hits = await authorizeActiveMemorySearchHits({ + cfg: params.cfg, + agentId: params.agentId, + requesterSessionKey: params.sessionKey, + sandboxed: false, + hits: rawHits, + }); return { status: "hits", hits }; } From 0e1a3220ea6ed6766a49ccd8f3785213351d21cb Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 03:40:22 -0700 Subject: [PATCH 22/57] fix(agents): prevent hosted tools during conversation compaction (#118618) --- src/agents/embedded-agent-runner/compact.hooks.test.ts | 2 ++ .../embedded-agent-runner/compaction-session-agent.ts | 8 ++++---- .../embedded-agent-runner/compaction-session-execution.ts | 1 - 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/agents/embedded-agent-runner/compact.hooks.test.ts b/src/agents/embedded-agent-runner/compact.hooks.test.ts index 4b1195c4288d..b48fee2262ca 100644 --- a/src/agents/embedded-agent-runner/compact.hooks.test.ts +++ b/src/agents/embedded-agent-runner/compact.hooks.test.ts @@ -919,6 +919,8 @@ describe("compactEmbeddedAgentSessionDirect hooks", () => { expectRecordFields(mockCallArg(applyExtraParamsToAgentMock, 0, 11), { nativeWebSearchPolicyContext: { sessionKey: undefined, + webSearchEnabled: false, + runtimeToolAllowlist: [], sandboxToolPolicy: undefined, messageProvider: undefined, agentAccountId: undefined, diff --git a/src/agents/embedded-agent-runner/compaction-session-agent.ts b/src/agents/embedded-agent-runner/compaction-session-agent.ts index 24497b069727..de5ff1882dab 100644 --- a/src/agents/embedded-agent-runner/compaction-session-agent.ts +++ b/src/agents/embedded-agent-runner/compaction-session-agent.ts @@ -42,7 +42,6 @@ export async function prepareCompactionSessionAgent(params: { senderName?: string | null; senderUsername?: string | null; senderE164?: string | null; - webSearchEnabled?: boolean; }) { const authStorage = params.authStorage && @@ -107,10 +106,11 @@ export async function prepareCompactionSessionAgent(params: { { ...(preparedRuntimeExtraParams ? { preparedExtraParams: preparedRuntimeExtraParams } : {}), nativeWebSearchPolicyContext: { - // Compaction rebuilds the stream wrapper, so preserve the session policy - // inputs that can suppress provider-native search. + // Summaries have no tool loop; provider-hosted tools must not inherit + // the originating conversation's broader web-search authority. sessionKey: params.sessionKey, - webSearchEnabled: params.webSearchEnabled, + webSearchEnabled: false, + runtimeToolAllowlist: [], sandboxToolPolicy: params.sandboxToolPolicy, messageProvider: params.messageProvider, agentAccountId: params.agentAccountId, diff --git a/src/agents/embedded-agent-runner/compaction-session-execution.ts b/src/agents/embedded-agent-runner/compaction-session-execution.ts index ce8934b54b80..441196629111 100644 --- a/src/agents/embedded-agent-runner/compaction-session-execution.ts +++ b/src/agents/embedded-agent-runner/compaction-session-execution.ts @@ -284,7 +284,6 @@ export async function executePreparedCompactionSession(runtime: PreparedCompacti senderName: params.senderName, senderUsername: params.senderUsername, senderE164: params.senderE164, - webSearchEnabled: params.toolOverrides?.webSearch !== false, }); session.agent.streamFn = wrapStreamFnWithDiagnosticModelCallEvents( session.agent.streamFn, From 0dbf9b6faef9592462a47872450980313776d251 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 03:43:23 -0700 Subject: [PATCH 23/57] fix(ui): surface fenced-code clipboard failures (#118608) * fix(ui): surface fenced-code clipboard failures * fix(ui): ignore stale clipboard attempt completions --- .../components/markdown-code-blocks.test.ts | 172 ++++++++++++++++++ ui/src/components/markdown-code-blocks.ts | 24 ++- ui/src/e2e/chat-flow.clipboard.e2e.test.ts | 135 ++++++++++---- 3 files changed, 291 insertions(+), 40 deletions(-) create mode 100644 ui/src/components/markdown-code-blocks.test.ts diff --git a/ui/src/components/markdown-code-blocks.test.ts b/ui/src/components/markdown-code-blocks.test.ts new file mode 100644 index 000000000000..e24b4c95a09b --- /dev/null +++ b/ui/src/components/markdown-code-blocks.test.ts @@ -0,0 +1,172 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { handleMarkdownCodeBlockCopy } from "./markdown-code-blocks.ts"; +import { toSanitizedMarkdownHtml } from "./markdown.ts"; + +const originalExecCommand = Object.getOwnPropertyDescriptor(document, "execCommand"); + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + if (originalExecCommand) { + Object.defineProperty(document, "execCommand", originalExecCommand); + } else { + Reflect.deleteProperty(document, "execCommand"); + } + document.body.innerHTML = ""; +}); + +function renderCodeCopyButton(): HTMLButtonElement { + document.body.innerHTML = toSanitizedMarkdownHtml("```ts\nconst answer = 42;\n```"); + const button = document.querySelector(".code-block-copy"); + if (!button) { + throw new Error("Expected Markdown code-copy button"); + } + button.addEventListener("click", handleMarkdownCodeBlockCopy); + return button; +} + +describe("Markdown code-block clipboard feedback", () => { + it("visibly reports both denied clipboard paths and restores the idle labels", async () => { + vi.useFakeTimers(); + const writeText = vi.fn(async () => { + throw new DOMException("Clipboard access denied", "NotAllowedError"); + }); + const execCommand = vi.fn(() => false); + vi.stubGlobal("navigator", { clipboard: { writeText } }); + Object.defineProperty(document, "execCommand", { + configurable: true, + value: execCommand, + }); + const button = renderCodeCopyButton(); + + button.click(); + await vi.advanceTimersByTimeAsync(0); + + expect(writeText).toHaveBeenCalledWith("const answer = 42;"); + expect(execCommand).toHaveBeenCalledWith("copy"); + expect(button.querySelector(".code-block-copy__idle")?.textContent).toBe("Copy failed"); + expect(button.getAttribute("aria-label")).toBe("Copy failed"); + expect(button.classList.contains("copied")).toBe(false); + + await vi.advanceTimersByTimeAsync(2_000); + + expect(button.querySelector(".code-block-copy__idle")?.textContent).toBe("Copy"); + expect(button.getAttribute("aria-label")).toBe("Copy code"); + }); + + it("preserves successful copy feedback and restores its accessible label", async () => { + vi.useFakeTimers(); + const writeText = vi.fn(async () => undefined); + vi.stubGlobal("navigator", { clipboard: { writeText } }); + const button = renderCodeCopyButton(); + + button.click(); + await vi.advanceTimersByTimeAsync(0); + + expect(writeText).toHaveBeenCalledWith("const answer = 42;"); + expect(button.classList.contains("copied")).toBe(true); + expect(button.getAttribute("aria-label")).toBe("Copied!"); + + await vi.advanceTimersByTimeAsync(1_500); + + expect(button.classList.contains("copied")).toBe(false); + expect(button.getAttribute("aria-label")).toBe("Copy code"); + }); + + it("ignores an older clipboard attempt that finishes after the latest denied copy", async () => { + vi.useFakeTimers(); + let resolveFirstWrite!: () => void; + const firstWrite = new Promise((resolve) => { + resolveFirstWrite = resolve; + }); + const writeText = vi + .fn() + .mockReturnValueOnce(firstWrite) + .mockRejectedValueOnce(new DOMException("Clipboard access denied", "NotAllowedError")); + vi.stubGlobal("navigator", { clipboard: { writeText } }); + Object.defineProperty(document, "execCommand", { + configurable: true, + value: () => false, + }); + const button = renderCodeCopyButton(); + + button.click(); + button.click(); + await vi.advanceTimersByTimeAsync(0); + + expect(button.getAttribute("aria-label")).toBe("Copy failed"); + + resolveFirstWrite(); + await vi.advanceTimersByTimeAsync(0); + + expect(button.querySelector(".code-block-copy__idle")?.textContent).toBe("Copy failed"); + expect(button.getAttribute("aria-label")).toBe("Copy failed"); + expect(button.classList.contains("copied")).toBe(false); + + await vi.advanceTimersByTimeAsync(2_000); + + expect(button.getAttribute("aria-label")).toBe("Copy code"); + }); + + it.each([ + { name: "a previous denied copy", firstSucceeds: false, firstResetAtMs: 2_000 }, + { name: "a previous successful copy", firstSucceeds: true, firstResetAtMs: 1_500 }, + ])("keeps the latest denied-copy feedback after $name", async (scenario) => { + vi.useFakeTimers(); + const writeText = vi + .fn() + .mockRejectedValue(new DOMException("Clipboard access denied", "NotAllowedError")); + if (scenario.firstSucceeds) { + writeText.mockResolvedValueOnce(undefined); + } + vi.stubGlobal("navigator", { clipboard: { writeText } }); + Object.defineProperty(document, "execCommand", { + configurable: true, + value: () => false, + }); + const button = renderCodeCopyButton(); + + button.click(); + await vi.advanceTimersByTimeAsync(1_000); + button.click(); + await vi.advanceTimersByTimeAsync(scenario.firstResetAtMs - 1_000); + + expect(button.querySelector(".code-block-copy__idle")?.textContent).toBe("Copy failed"); + expect(button.getAttribute("aria-label")).toBe("Copy failed"); + + await vi.advanceTimersByTimeAsync(3_000 - scenario.firstResetAtMs); + + expect(button.querySelector(".code-block-copy__idle")?.textContent).toBe("Copy"); + expect(button.getAttribute("aria-label")).toBe("Copy code"); + }); + + it("keeps independent reset deadlines for different code-copy buttons", async () => { + vi.useFakeTimers(); + vi.stubGlobal("navigator", { + clipboard: { + writeText: vi.fn().mockRejectedValue(new DOMException("Clipboard access denied")), + }, + }); + Object.defineProperty(document, "execCommand", { + configurable: true, + value: () => false, + }); + const first = renderCodeCopyButton(); + const second = first.cloneNode(true) as HTMLButtonElement; + second.addEventListener("click", handleMarkdownCodeBlockCopy); + document.body.append(second); + + first.click(); + await vi.advanceTimersByTimeAsync(500); + second.click(); + await vi.advanceTimersByTimeAsync(1_500); + + expect(first.getAttribute("aria-label")).toBe("Copy code"); + expect(second.getAttribute("aria-label")).toBe("Copy failed"); + + second.remove(); + await vi.advanceTimersByTimeAsync(500); + + expect(second.getAttribute("aria-label")).toBe("Copy code"); + }); +}); diff --git a/ui/src/components/markdown-code-blocks.ts b/ui/src/components/markdown-code-blocks.ts index 9b87c9352155..e065f81e0d2e 100644 --- a/ui/src/components/markdown-code-blocks.ts +++ b/ui/src/components/markdown-code-blocks.ts @@ -20,6 +20,8 @@ import { escapeMarkdownHtml, isMarkdownBlockArtText } from "./markdown-text.ts"; const blockArtCopyPayloadPrefix = "openclaw:block-art-code:"; const blockArtCodeBlockCopyPayloadEncoding = "block-art-json"; +const codeBlockCopyAttempts = new WeakMap(); +const codeBlockCopyResetTimers = new WeakMap>(); for (const [language, definition] of Object.entries({ bash, @@ -74,12 +76,28 @@ export function handleMarkdownCodeBlockCopy(event: Event): void { return; } const code = decodeCodeBlockCopyPayload(button.dataset.code ?? "", button.dataset.codeEncoding); + const attempt = (codeBlockCopyAttempts.get(button) ?? 0) + 1; + codeBlockCopyAttempts.set(button, attempt); void copyToClipboard(code).then((copied) => { - if (!copied) { + // Clipboard writes can finish out of click order; older attempts must not own feedback. + if (codeBlockCopyAttempts.get(button) !== attempt) { return; } - button.classList.add("copied"); - setTimeout(() => button.classList.remove("copied"), 1500); + const idleLabel = button.querySelector(".code-block-copy__idle"); + idleLabel?.replaceChildren(t(copied ? "common.copy" : "common.copyFailed")); + button.classList.toggle("copied", copied); + button.setAttribute("aria-label", t(copied ? "common.copied" : "common.copyFailed")); + clearTimeout(codeBlockCopyResetTimers.get(button)); + const resetTimer = setTimeout( + () => { + button.classList.remove("copied"); + idleLabel?.replaceChildren(t("common.copy")); + button.setAttribute("aria-label", t("common.copyCode")); + codeBlockCopyResetTimers.delete(button); + }, + copied ? 1500 : 2000, + ); + codeBlockCopyResetTimers.set(button, resetTimer); }); } diff --git a/ui/src/e2e/chat-flow.clipboard.e2e.test.ts b/ui/src/e2e/chat-flow.clipboard.e2e.test.ts index cdc824d01dad..5e8a82caa3aa 100644 --- a/ui/src/e2e/chat-flow.clipboard.e2e.test.ts +++ b/ui/src/e2e/chat-flow.clipboard.e2e.test.ts @@ -1,10 +1,51 @@ import { mkdir } from "node:fs/promises"; import path from "node:path"; +import type { Page } from "playwright"; import { expect, it } from "vitest"; import { createChatFlowE2eSuite, installMockGateway } from "./chat-flow.test-support.ts"; const suite = createChatFlowE2eSuite(); +type ClipboardFailureProof = { + asyncAttempts: number; + legacyAttempts: number; + value: string; +}; + +async function installDeniedClipboard(page: Page): Promise { + await page.addInitScript(() => { + const proof = { asyncAttempts: 0, legacyAttempts: 0, value: "" }; + Object.defineProperty(globalThis, "clipboardFailureProof", { + configurable: true, + value: proof, + }); + Object.defineProperty(navigator, "clipboard", { + configurable: true, + value: { + writeText: async (text: string) => { + proof.asyncAttempts += 1; + proof.value = text; + throw new DOMException("Clipboard access denied", "NotAllowedError"); + }, + }, + }); + document.execCommand = ((command: string) => { + if (command === "copy") { + proof.legacyAttempts += 1; + } + return false; + }) as typeof document.execCommand; + }); +} + +async function readClipboardFailureProof(page: Page): Promise { + return page.evaluate( + () => + (globalThis as typeof globalThis & { clipboardFailureProof: ClipboardFailureProof }) + .clipboardFailureProof, + ); +} + suite.define(() => { it.each([ { action: "copy-path", label: "Copy path", value: "/workspace" }, @@ -18,29 +59,7 @@ suite.define(() => { viewport: { height: 900, width: 1280 }, }); const page = await context.newPage(); - await page.addInitScript(() => { - const proof = { asyncAttempts: 0, legacyAttempts: 0, value: "" }; - Object.defineProperty(globalThis, "clipboardFailureProof", { - configurable: true, - value: proof, - }); - Object.defineProperty(navigator, "clipboard", { - configurable: true, - value: { - writeText: async (text: string) => { - proof.asyncAttempts += 1; - proof.value = text; - throw new DOMException("Clipboard access denied", "NotAllowedError"); - }, - }, - }); - document.execCommand = ((command: string) => { - if (command === "copy") { - proof.legacyAttempts += 1; - } - return false; - }) as typeof document.execCommand; - }); + await installDeniedClipboard(page); const gateway = await installMockGateway(page, { workspace: "/workspace", workspaceGit: true, @@ -57,20 +76,11 @@ suite.define(() => { const alert = page.getByRole("alert").filter({ hasText: "Copy failed" }); await alert.waitFor({ state: "visible", timeout: 10_000 }); - expect( - await page.evaluate( - () => - ( - globalThis as typeof globalThis & { - clipboardFailureProof: { - asyncAttempts: number; - legacyAttempts: number; - value: string; - }; - } - ).clipboardFailureProof, - ), - ).toEqual({ asyncAttempts: 1, legacyAttempts: 1, value }); + expect(await readClipboardFailureProof(page)).toEqual({ + asyncAttempts: 1, + legacyAttempts: 1, + value, + }); expect(await gateway.getRequests("chat.send")).toHaveLength(0); const artifactDir = process.env.OPENCLAW_UI_E2E_ARTIFACT_DIR?.trim(); @@ -86,4 +96,55 @@ suite.define(() => { } }, ); + + it("shows and resets a visible accessible failure when assistant code cannot be copied", async () => { + const context = await suite.newBrowserContext({ + locale: "en-US", + serviceWorkers: "block", + viewport: { height: 900, width: 1280 }, + }); + const page = await context.newPage(); + await installDeniedClipboard(page); + const code = "const answer = 42;"; + const gateway = await installMockGateway(page, { + historyMessages: [ + { + content: [{ text: `\`\`\`ts\n${code}\n\`\`\``, type: "text" }], + role: "assistant", + timestamp: Date.now(), + }, + ], + }); + + try { + await page.goto(`${suite.server.baseUrl}chat`); + const button = page.locator(".code-block-copy"); + await button.click(); + + await expect.poll(() => button.getAttribute("aria-label")).toBe("Copy failed"); + await expect + .poll(() => button.locator(".code-block-copy__idle").textContent()) + .toBe("Copy failed"); + expect(await readClipboardFailureProof(page)).toEqual({ + asyncAttempts: 1, + legacyAttempts: 1, + value: code, + }); + expect(await gateway.getRequests("chat.send")).toHaveLength(0); + + const artifactDir = process.env.OPENCLAW_UI_E2E_ARTIFACT_DIR?.trim(); + if (artifactDir) { + await mkdir(artifactDir, { recursive: true }); + await page.screenshot({ + fullPage: true, + path: path.join(artifactDir, "clipboard-assistant-code-failure.png"), + }); + } + + await expect.poll(() => button.getAttribute("aria-label")).toBe("Copy code"); + await expect.poll(() => button.locator(".code-block-copy__idle").textContent()).toBe("Copy"); + } finally { + await suite.closeBrowserContext(context); + } + }); }); From fe6fa891ea4ef3bb152d02df944a8def6518fdd1 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 03:54:41 -0700 Subject: [PATCH 24/57] fix(whatsapp): clear and migrate every Baileys credential class (#118610) * fix(whatsapp): clear and migrate every auth credential class * test(whatsapp): await nullable legacy migration detector --- extensions/whatsapp/setup-entry.test.ts | 54 +++++++++++++++++++++ extensions/whatsapp/src/auth-store.test.ts | 50 +++++++++++++++++++ extensions/whatsapp/src/auth-store.ts | 18 ++----- extensions/whatsapp/src/creds-files.ts | 26 ++++++++++ extensions/whatsapp/src/state-migrations.ts | 13 +---- 5 files changed, 135 insertions(+), 26 deletions(-) diff --git a/extensions/whatsapp/setup-entry.test.ts b/extensions/whatsapp/setup-entry.test.ts index 4c0de0b07469..d0fc62f9fb97 100644 --- a/extensions/whatsapp/setup-entry.test.ts +++ b/extensions/whatsapp/setup-entry.test.ts @@ -1,4 +1,7 @@ // Whatsapp tests cover setup entry plugin behavior. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import * as legacySessionSurfaceApi from "./legacy-session-surface-api.js"; import * as legacyStateMigrationsApi from "./legacy-state-migrations-api.js"; @@ -68,6 +71,57 @@ describe("whatsapp setup entry", () => { expect(legacySessionSurface.isLegacyGroupSessionKey).toBeTypeOf("function"); }); + it("plans migration for every Baileys auth category while preserving other shared-root files", async () => { + const oauthDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-wa-legacy-migration-")); + const authFiles = [ + "creds.json", + "creds.json.bak", + "pre-key-1.json", + "session-contact.json", + "sender-key-group.json", + "sender-key-memory-group.json", + "app-state-sync-key-contact.json", + "app-state-sync-version-contact.json", + "lid-mapping-15551234567.json", + "device-list-15551234567.json", + "tctoken-15551234567.json", + "identity-key-15551234567.json", + ]; + + try { + for (const file of [...authFiles, "oauth.json", "google-oauth.json", "notes.txt"]) { + fs.writeFileSync(path.join(oauthDir, file), "{}", "utf-8"); + } + fs.mkdirSync(path.join(oauthDir, "nested")); + fs.writeFileSync(path.join(oauthDir, "nested", "session-keep.json"), "{}", "utf-8"); + fs.symlinkSync(path.join(oauthDir, "notes.txt"), path.join(oauthDir, "session-linked.json")); + + const detectLegacyStateMigrations = + setupEntry.loadLegacyStateMigrationDetector?.(setupEntryLoadOptions); + if (!detectLegacyStateMigrations) { + throw new Error("expected WhatsApp legacy state migration detector"); + } + const migrations = + (await detectLegacyStateMigrations({ + cfg: {}, + env: {}, + oauthDir, + stateDir: oauthDir, + })) ?? []; + + expect(migrations.map((migration) => path.basename(migration.sourcePath)).toSorted()).toEqual( + authFiles.toSorted(), + ); + for (const migration of migrations) { + expect(migration.targetPath).toBe( + path.join(oauthDir, "whatsapp", "default", path.basename(migration.sourcePath)), + ); + } + } finally { + fs.rmSync(oauthDir, { recursive: true, force: true }); + } + }); + it("loads the delegated setup wizard without importing runtime dependencies", async () => { const { whatsappSetupWizard } = await import("./src/setup-surface.js"); diff --git a/extensions/whatsapp/src/auth-store.test.ts b/extensions/whatsapp/src/auth-store.test.ts index c52ca100ef4c..54f54ce87f47 100644 --- a/extensions/whatsapp/src/auth-store.test.ts +++ b/extensions/whatsapp/src/auth-store.test.ts @@ -351,6 +351,56 @@ describe("auth-store", () => { } }); + it("clears every Baileys auth category from the shared legacy root without touching other files", async () => { + const authDir = createTempAuthDir("openclaw-wa-auth-legacy-categories"); + const previousOAuthDir = hoisted.oauthDir; + const authFiles = [ + "creds.json", + "creds.json.bak", + "pre-key-1.json", + "session-contact.json", + "sender-key-group.json", + "sender-key-memory-group.json", + "app-state-sync-key-contact.json", + "app-state-sync-version-contact.json", + "lid-mapping-15551234567.json", + "device-list-15551234567.json", + "tctoken-15551234567.json", + "identity-key-15551234567.json", + ]; + const unrelatedFiles = ["oauth.json", "google-oauth.json", "notes.txt"]; + const nestedAuthFile = path.join(authDir, "nested", "session-keep.json"); + hoisted.oauthDir = authDir; + + try { + for (const file of [...authFiles, ...unrelatedFiles]) { + fsSync.writeFileSync(path.join(authDir, file), "{}", "utf-8"); + } + fsSync.mkdirSync(path.dirname(nestedAuthFile)); + fsSync.writeFileSync(nestedAuthFile, "keep", "utf-8"); + fsSync.symlinkSync( + path.join(authDir, "notes.txt"), + path.join(authDir, "session-linked.json"), + ); + + await expect(logoutWeb({ authDir, isLegacyAuthDir: true })).resolves.toBe(true); + + for (const file of authFiles) { + expect(fsSync.existsSync(path.join(authDir, file)), file).toBe(false); + } + for (const file of unrelatedFiles) { + expect(fsSync.existsSync(path.join(authDir, file)), file).toBe(true); + } + expect(fsSync.readFileSync(nestedAuthFile, "utf-8")).toBe("keep"); + expect(fsSync.lstatSync(path.join(authDir, "session-linked.json")).isSymbolicLink()).toBe( + true, + ); + } finally { + hoisted.oauthDir = previousOAuthDir; + fsSync.rmSync(authDir, { recursive: true, force: true }); + } + }); + it("clears auth state even when directory enumeration fails", async () => { await withOwnedOAuthAuthDir("openclaw-wa-auth-readdir", async (authDir) => { fsSync.writeFileSync(path.join(authDir, "creds.json"), "{}", "utf-8"); diff --git a/extensions/whatsapp/src/auth-store.ts b/extensions/whatsapp/src/auth-store.ts index a268456872c8..a8c67258629a 100644 --- a/extensions/whatsapp/src/auth-store.ts +++ b/extensions/whatsapp/src/auth-store.ts @@ -10,6 +10,7 @@ import { resolveOAuthDir } from "./auth-store.runtime.js"; import { assertWebCredsPathRegularFileOrMissing, hasWebCredsSync, + isWhatsAppBaileysAuthFileName, readWebCredsJsonRaw, readWebCredsJsonRawSync, resolveWebCredsBackupPath, @@ -225,19 +226,6 @@ export async function readWebAuthSnapshotBestEffort(authDir: string = resolveDef } as const; } -function isBaileysAuthFileName(name: string): boolean { - if (name === "oauth.json") { - return false; - } - if (name === "creds.json" || name === "creds.json.bak") { - return true; - } - if (!name.endsWith(".json")) { - return false; - } - return /^(app-state-sync|session|sender-key|pre-key)-/.test(name); -} - async function clearBaileysAuthFiles( authDir: string, beforeCredentialPersistence?: () => Promise, @@ -248,7 +236,7 @@ async function clearBaileysAuthFiles( } const entries = await fs.readdir(authDir, { withFileTypes: true }); const credentialFiles = entries.filter( - (entry) => entry.isFile() && isBaileysAuthFileName(entry.name), + (entry) => entry.isFile() && isWhatsAppBaileysAuthFileName(entry.name), ); if (credentialFiles.length === 0) { return; @@ -273,7 +261,7 @@ async function shouldClearOnLogout(authDir: string, isLegacyAuthDir: boolean): P if (!entry.isFile()) { return false; } - return isBaileysAuthFileName(entry.name); + return isWhatsAppBaileysAuthFileName(entry.name); }); } const credsStats = await fs.lstat(resolveWebCredsPath(authDir)).catch(() => null); diff --git a/extensions/whatsapp/src/creds-files.ts b/extensions/whatsapp/src/creds-files.ts index cf41011a450c..bc6e555dc838 100644 --- a/extensions/whatsapp/src/creds-files.ts +++ b/extensions/whatsapp/src/creds-files.ts @@ -1,5 +1,6 @@ // Whatsapp plugin module implements creds files behavior. import path from "node:path"; +import type { SignalDataTypeMap } from "baileys"; import { assertNoSymlinkParents, assertNoSymlinkParentsSync, @@ -9,6 +10,31 @@ import { statRegularFileSync, } from "openclaw/plugin-sdk/security-runtime"; +// The legacy OAuth root is shared; keep its exact WhatsApp namespaces aligned +// with Baileys without importing the provider into setup discovery. +const BAILEYS_SIGNAL_AUTH_CATEGORIES = { + "app-state-sync-key": true, + "app-state-sync-version": true, + "device-list": true, + "identity-key": true, + "lid-mapping": true, + "pre-key": true, + "sender-key": true, + "sender-key-memory": true, + session: true, + tctoken: true, +} satisfies Record; + +export function isWhatsAppBaileysAuthFileName(name: string): boolean { + if (name === "creds.json" || name === "creds.json.bak") { + return true; + } + return ( + name.endsWith(".json") && + Object.keys(BAILEYS_SIGNAL_AUTH_CATEGORIES).some((category) => name.startsWith(`${category}-`)) + ); +} + export function resolveWebCredsPath(authDir: string): string { return path.join(authDir, "creds.json"); } diff --git a/extensions/whatsapp/src/state-migrations.ts b/extensions/whatsapp/src/state-migrations.ts index deaf2043b0cd..93a2cd183277 100644 --- a/extensions/whatsapp/src/state-migrations.ts +++ b/extensions/whatsapp/src/state-migrations.ts @@ -4,16 +4,7 @@ import path from "node:path"; import { DEFAULT_ACCOUNT_ID } from "openclaw/plugin-sdk/account-id"; import type { ChannelLegacyStateMigrationPlan } from "openclaw/plugin-sdk/channel-contract"; import { fileExists } from "openclaw/plugin-sdk/security-runtime"; - -function isLegacyWhatsAppAuthFile(name: string): boolean { - if (name === "creds.json" || name === "creds.json.bak") { - return true; - } - if (!name.endsWith(".json")) { - return false; - } - return /^(app-state-sync|session|sender-key|pre-key)-/.test(name); -} +import { isWhatsAppBaileysAuthFileName } from "./creds-files.js"; export function detectWhatsAppLegacyStateMigrations(params: { oauthDir: string; @@ -28,7 +19,7 @@ export function detectWhatsAppLegacyStateMigrations(params: { })(); return entries.flatMap((entry) => { - if (!entry.isFile() || entry.name === "oauth.json" || !isLegacyWhatsAppAuthFile(entry.name)) { + if (!entry.isFile() || !isWhatsAppBaileysAuthFileName(entry.name)) { return []; } const sourcePath = path.join(params.oauthDir, entry.name); From a6f9da8bdb659abdf761f41bbd2b5c705c617b25 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 04:08:54 -0700 Subject: [PATCH 25/57] refactor: centralize cron, doctor, and TUI ownership (#118515) * refactor(cron): centralize queued run activation * refactor(doctor): share session SQLite report scaffolding * refactor(tui): unify slash command registry * fix(tui): satisfy command registry lint * fix(tui): align shared command usage help * fix(btw): preserve outbound usage placeholder --- docs/web/tui.md | 4 +- .../doctor-session-sqlite-recover-report.ts | 67 +- .../doctor-session-sqlite-restore-report.ts | 51 +- src/commands/doctor-session-sqlite-types.ts | 51 ++ src/commands/doctor-session-sqlite.test.ts | 5 + src/commands/doctor-session-sqlite.ts | 72 +- src/cron/service/jobs-scheduling.ts | 23 +- src/cron/service/ops-run-preparation.ts | 41 +- .../service/ops.run-admission-cleanup.test.ts | 6 + src/cron/service/run-admission.ts | 77 +- src/cron/service/timer-catchup.ts | 28 +- src/cron/service/timer-scheduler.ts | 32 +- src/tui/commands.test.ts | 19 + src/tui/commands.ts | 295 ++++--- src/tui/tui-command-handlers.test.ts | 2 +- src/tui/tui-command-handlers.ts | 766 +++++++++--------- src/tui/tui-pty-harness.e2e.test.ts | 5 + src/tui/tui-pty-local.e2e.test.ts | 2 +- 18 files changed, 778 insertions(+), 768 deletions(-) diff --git a/docs/web/tui.md b/docs/web/tui.md index 583e1e5a0187..f4d725f1bd9d 100644 --- a/docs/web/tui.md +++ b/docs/web/tui.md @@ -115,7 +115,8 @@ Session controls: - `/trace ` - `/reasoning ` - `/usage ` (`reset`/`inherit`/`clear`/`default` clears the session override) -- `/goal [status] | /goal start | /goal edit | /goal pause|resume|complete|block|clear` +- `/goal | /goal [status] | /goal start | /goal edit | /goal pause|resume|complete|block|clear` +- `/btw ` (alias: `/side`; asks without changing future session context) - `/elevated ` (alias: `/elev`) - `/activation ` - `/queue [debounce:] [cap:] [drop:]` @@ -126,6 +127,7 @@ Session lifecycle: - `/new` (spawn a fresh, isolated session under a new key; does not affect other TUI clients on the old session) - `/reset` (reset the current session key in place) - `/abort` (abort the active run) +- `/stop` (stop the active or queued run) - `/settings` - `/exit` (or `/quit`) diff --git a/src/commands/doctor-session-sqlite-recover-report.ts b/src/commands/doctor-session-sqlite-recover-report.ts index 65e567b8f3a4..a8c0de77d329 100644 --- a/src/commands/doctor-session-sqlite-recover-report.ts +++ b/src/commands/doctor-session-sqlite-recover-report.ts @@ -23,10 +23,13 @@ import { type SessionSqliteMigrationTargetInput, } from "./doctor-session-sqlite-migration-run.js"; import { resolveTargetSqlitePath } from "./doctor-session-sqlite-readers.js"; -import type { - DoctorSessionSqliteOptions, - DoctorSessionSqliteReport, - DoctorSessionSqliteTargetReport, +import { + createDoctorSessionSqliteTotals, + createDoctorSessionSqliteTargetReport, + sumDoctorSessionSqliteTargets, + type DoctorSessionSqliteOptions, + type DoctorSessionSqliteReport, + type DoctorSessionSqliteTargetReport, } from "./doctor-session-sqlite-types.js"; type SessionSqliteRecoverTargetValidator = ( @@ -383,70 +386,38 @@ function createSyntheticRecoverTargetReport( env: NodeJS.ProcessEnv, message: string, ): DoctorSessionSqliteTargetReport { - return { + return createDoctorSessionSqliteTargetReport({ agentId: "recover", - archivedTranscriptFiles: [], - archivedUnreferencedJsonlFiles: [], - importedEntries: 0, - importedTranscriptEvents: 0, issues: [{ code: "recover_manifest_missing", message }], - legacyEntries: 0, - referencedTranscriptFiles: 0, - sqliteEntries: 0, sqlitePath: "", storePath: resolveSessionSqliteMigrationRunsDir(env), - unreferencedJsonlFiles: [], - validatedEntries: 0, - validatedTranscriptEvents: 0, - }; + }); } function createEmptyRecoverTargetReport( target: SessionStoreTarget, sqlitePath: string, ): DoctorSessionSqliteTargetReport { - return { + return createDoctorSessionSqliteTargetReport({ agentId: target.agentId, - archivedTranscriptFiles: [], - archivedUnreferencedJsonlFiles: [], - importedEntries: 0, - importedTranscriptEvents: 0, - issues: [], - legacyEntries: 0, - referencedTranscriptFiles: 0, - sqliteEntries: 0, sqlitePath, storePath: target.storePath, - unreferencedJsonlFiles: [], - validatedEntries: 0, - validatedTranscriptEvents: 0, - }; + }); } function summarizeRecoverReport( targets: DoctorSessionSqliteTargetReport[], ): DoctorSessionSqliteReport { + const sum = (value: (target: DoctorSessionSqliteTargetReport) => number) => + sumDoctorSessionSqliteTargets(targets, value); return { mode: "recover", targets, - totals: { - archivedTranscriptFiles: 0, - archivedUnreferencedJsonlFiles: 0, - importedEntries: 0, - importedTranscriptEvents: 0, - issues: targets.reduce((total, target) => total + target.issues.length, 0), - legacyEntries: targets.reduce((total, target) => total + target.legacyEntries, 0), - sqliteEntries: targets.reduce((total, target) => total + target.sqliteEntries, 0), - targets: targets.length, - unreferencedJsonlFiles: targets.reduce( - (total, target) => total + target.unreferencedJsonlFiles.length, - 0, - ), - validatedEntries: targets.reduce((total, target) => total + target.validatedEntries, 0), - validatedTranscriptEvents: targets.reduce( - (total, target) => total + target.validatedTranscriptEvents, - 0, - ), - }, + totals: createDoctorSessionSqliteTotals(targets, { + legacyEntries: sum((target) => target.legacyEntries), + unreferencedJsonlFiles: sum((target) => target.unreferencedJsonlFiles.length), + validatedEntries: sum((target) => target.validatedEntries), + validatedTranscriptEvents: sum((target) => target.validatedTranscriptEvents), + }), }; } diff --git a/src/commands/doctor-session-sqlite-restore-report.ts b/src/commands/doctor-session-sqlite-restore-report.ts index ed9e14487b13..8acf20460933 100644 --- a/src/commands/doctor-session-sqlite-restore-report.ts +++ b/src/commands/doctor-session-sqlite-restore-report.ts @@ -5,9 +5,11 @@ import { restoreSessionSqliteMigrationRuns, } from "./doctor-session-sqlite-migration-run.js"; import { readSqliteEntryCount, resolveTargetSqlitePath } from "./doctor-session-sqlite-readers.js"; -import type { - DoctorSessionSqliteReport, - DoctorSessionSqliteTargetReport, +import { + createDoctorSessionSqliteTargetReport, + createDoctorSessionSqliteTotals, + type DoctorSessionSqliteReport, + type DoctorSessionSqliteTargetReport, } from "./doctor-session-sqlite-types.js"; export async function restoreDoctorSessionSqliteTargets(params: { @@ -40,44 +42,23 @@ export async function restoreDoctorSessionSqliteTargets(params: { } function createEmptyTargetReport(target: SessionStoreTarget): DoctorSessionSqliteTargetReport { - return { + return createDoctorSessionSqliteTargetReport({ agentId: target.agentId, - archivedTranscriptFiles: [], - archivedUnreferencedJsonlFiles: [], - importedEntries: 0, - importedTranscriptEvents: 0, - issues: [], - legacyEntries: 0, - referencedTranscriptFiles: 0, sqliteEntries: readSqliteEntryCount(target), sqlitePath: resolveTargetSqlitePath(target), storePath: target.storePath, - unreferencedJsonlFiles: [], - validatedEntries: 0, - validatedTranscriptEvents: 0, - }; + }); } function createSyntheticRestoreTargetReport( env: NodeJS.ProcessEnv, manifestPath: string, ): DoctorSessionSqliteTargetReport { - return { + return createDoctorSessionSqliteTargetReport({ agentId: "restore", - archivedTranscriptFiles: [], - archivedUnreferencedJsonlFiles: [], - importedEntries: 0, - importedTranscriptEvents: 0, - issues: [], - legacyEntries: 0, - referencedTranscriptFiles: 0, - sqliteEntries: 0, sqlitePath: "", storePath: manifestPath || resolveSessionSqliteMigrationRunsDir(env), - unreferencedJsonlFiles: [], - validatedEntries: 0, - validatedTranscriptEvents: 0, - }; + }); } function summarizeRestoreReport( @@ -86,18 +67,6 @@ function summarizeRestoreReport( return { mode: "restore", targets, - totals: { - archivedTranscriptFiles: 0, - archivedUnreferencedJsonlFiles: 0, - importedEntries: 0, - importedTranscriptEvents: 0, - issues: targets.reduce((total, target) => total + target.issues.length, 0), - legacyEntries: 0, - sqliteEntries: targets.reduce((total, target) => total + target.sqliteEntries, 0), - targets: targets.length, - unreferencedJsonlFiles: 0, - validatedEntries: 0, - validatedTranscriptEvents: 0, - }, + totals: createDoctorSessionSqliteTotals(targets), }; } diff --git a/src/commands/doctor-session-sqlite-types.ts b/src/commands/doctor-session-sqlite-types.ts index fe20617fddee..53689c1f805d 100644 --- a/src/commands/doctor-session-sqlite-types.ts +++ b/src/commands/doctor-session-sqlite-types.ts @@ -116,6 +116,26 @@ export type DoctorSessionSqliteTargetReport = { restore?: DoctorSessionSqliteRestoreReport; }; +export function createDoctorSessionSqliteTargetReport( + values: Pick & + Partial>, +): DoctorSessionSqliteTargetReport { + return { + archivedTranscriptFiles: [], + archivedUnreferencedJsonlFiles: [], + importedEntries: 0, + importedTranscriptEvents: 0, + issues: [], + legacyEntries: 0, + referencedTranscriptFiles: 0, + sqliteEntries: 0, + unreferencedJsonlFiles: [], + validatedEntries: 0, + validatedTranscriptEvents: 0, + ...values, + }; +} + export type DoctorSessionSqliteReport = { migrationRun?: { failureReportJsonPath?: string; @@ -142,3 +162,34 @@ export type DoctorSessionSqliteReport = { validatedTranscriptEvents: number; }; }; + +export function sumDoctorSessionSqliteTargets( + targets: DoctorSessionSqliteTargetReport[], + value: (target: DoctorSessionSqliteTargetReport) => number, +): number { + return targets.reduce((total, target) => total + value(target), 0); +} + +export function createDoctorSessionSqliteTotals( + targets: DoctorSessionSqliteTargetReport[], + values: Partial< + Omit + > = {}, +): DoctorSessionSqliteReport["totals"] { + const { archivedLegacyStoreFiles, reclaimedBytes } = values; + return { + ...(archivedLegacyStoreFiles === undefined ? {} : { archivedLegacyStoreFiles }), + archivedTranscriptFiles: values.archivedTranscriptFiles ?? 0, + archivedUnreferencedJsonlFiles: values.archivedUnreferencedJsonlFiles ?? 0, + importedEntries: values.importedEntries ?? 0, + importedTranscriptEvents: values.importedTranscriptEvents ?? 0, + issues: sumDoctorSessionSqliteTargets(targets, (target) => target.issues.length), + legacyEntries: values.legacyEntries ?? 0, + ...(reclaimedBytes === undefined ? {} : { reclaimedBytes }), + sqliteEntries: sumDoctorSessionSqliteTargets(targets, (target) => target.sqliteEntries), + targets: targets.length, + unreferencedJsonlFiles: values.unreferencedJsonlFiles ?? 0, + validatedEntries: values.validatedEntries ?? 0, + validatedTranscriptEvents: values.validatedTranscriptEvents ?? 0, + }; +} diff --git a/src/commands/doctor-session-sqlite.test.ts b/src/commands/doctor-session-sqlite.test.ts index 78910159596b..953ae54d842f 100644 --- a/src/commands/doctor-session-sqlite.test.ts +++ b/src/commands/doctor-session-sqlite.test.ts @@ -1246,6 +1246,8 @@ describe("runDoctorSessionSqlite", () => { }); expect(restore.totals.issues).toBe(0); + expect(restore.totals).not.toHaveProperty("archivedLegacyStoreFiles"); + expect(restore.totals).not.toHaveProperty("reclaimedBytes"); expect(restore.targets[0]?.restore).toMatchObject({ conflicts: [], restoredFiles: expect.arrayContaining(sourcePaths), @@ -2256,6 +2258,8 @@ describe("runDoctorSessionSqlite", () => { }); expect(recover.mode).toBe("recover"); + expect(recover.totals).not.toHaveProperty("archivedLegacyStoreFiles"); + expect(recover.totals).not.toHaveProperty("reclaimedBytes"); expect(recover.targets[0]?.issues).toMatchObject([ { code: "active_sqlite_transcript_jsonl", sessionKey: "agent:main:main" }, ]); @@ -2691,6 +2695,7 @@ describe("runDoctorSessionSqlite", () => { issues: 0, sqliteEntries: 2, }); + expect(report.totals).toHaveProperty("reclaimedBytes"); const manifest = readMigrationManifest(report.migrationRun?.manifestPath); for (const target of manifest.targets) { expect(target.completedMoves.some((move) => move.kind === "legacy-store")).toBe(true); diff --git a/src/commands/doctor-session-sqlite.ts b/src/commands/doctor-session-sqlite.ts index 2c01230a400c..1ba02652f490 100644 --- a/src/commands/doctor-session-sqlite.ts +++ b/src/commands/doctor-session-sqlite.ts @@ -56,7 +56,10 @@ import { import { recoverDoctorSessionSqliteTargets } from "./doctor-session-sqlite-recover-report.js"; import { restoreDoctorSessionSqliteTargets } from "./doctor-session-sqlite-restore-report.js"; import { + createDoctorSessionSqliteTotals, + createDoctorSessionSqliteTargetReport, isSessionSqliteMigrationWarning, + sumDoctorSessionSqliteTargets, type DoctorSessionSqliteIssue, type DoctorSessionSqliteMode, type DoctorSessionSqliteOptions, @@ -295,13 +298,9 @@ async function inspectOrMigrateTarget(params: { const referencedTranscriptFiles = new Set( allRecords.flatMap((record) => (record.transcriptPath ? [record.transcriptPath] : [])), ); - const report: DoctorSessionSqliteTargetReport = { + const report = createDoctorSessionSqliteTargetReport({ agentId: params.target.agentId, archivedLegacyStoreFiles: [], - archivedTranscriptFiles: [], - archivedUnreferencedJsonlFiles: [], - importedEntries: 0, - importedTranscriptEvents: 0, issues, legacyEntries: records.length, referencedTranscriptFiles: referencedTranscriptFiles.size, @@ -311,9 +310,7 @@ async function inspectOrMigrateTarget(params: { unreferencedJsonlFiles: listUnreferencedJsonlFiles(params.target.storePath, [ ...referencedTranscriptFiles, ]), - validatedEntries: 0, - validatedTranscriptEvents: 0, - }; + }); if (params.mode === "inspect") { report.sqliteEntries = readSqliteEntryCount(params.target); appendSqliteDbStats(params.target, report); @@ -1332,6 +1329,8 @@ function summarizeDoctorSessionSqliteReport( targets: DoctorSessionSqliteTargetReport[], activeRun?: ActiveSessionSqliteMigrationRun, ): DoctorSessionSqliteReport { + const sum = (value: (target: DoctorSessionSqliteTargetReport) => number) => + sumDoctorSessionSqliteTargets(targets, value); return { ...(activeRun ? { @@ -1349,51 +1348,18 @@ function summarizeDoctorSessionSqliteReport( : {}), mode, targets, - totals: { - archivedLegacyStoreFiles: targets.reduce( - (total, target) => total + (target.archivedLegacyStoreFiles?.length ?? 0), - 0, - ), - archivedTranscriptFiles: targets.reduce( - (total, target) => total + target.archivedTranscriptFiles.length, - 0, - ), - archivedUnreferencedJsonlFiles: targets.reduce( - (total, target) => total + target.archivedUnreferencedJsonlFiles.length, - 0, - ), - importedEntries: sumTargets(targets, "importedEntries"), - importedTranscriptEvents: sumTargets(targets, "importedTranscriptEvents"), - issues: targets.reduce((total, target) => total + target.issues.length, 0), - legacyEntries: sumTargets(targets, "legacyEntries"), - reclaimedBytes: targets.reduce( - (total, target) => total + (target.compact?.reclaimedBytes ?? 0), - 0, - ), - sqliteEntries: sumTargets(targets, "sqliteEntries"), - targets: targets.length, - unreferencedJsonlFiles: targets.reduce( - (total, target) => total + target.unreferencedJsonlFiles.length, - 0, - ), - validatedEntries: sumTargets(targets, "validatedEntries"), - validatedTranscriptEvents: sumTargets(targets, "validatedTranscriptEvents"), - }, + totals: createDoctorSessionSqliteTotals(targets, { + archivedLegacyStoreFiles: sum((target) => target.archivedLegacyStoreFiles?.length ?? 0), + archivedTranscriptFiles: sum((target) => target.archivedTranscriptFiles.length), + archivedUnreferencedJsonlFiles: sum((target) => target.archivedUnreferencedJsonlFiles.length), + importedEntries: sum((target) => target.importedEntries), + importedTranscriptEvents: sum((target) => target.importedTranscriptEvents), + legacyEntries: sum((target) => target.legacyEntries), + reclaimedBytes: sum((target) => target.compact?.reclaimedBytes ?? 0), + unreferencedJsonlFiles: sum((target) => target.unreferencedJsonlFiles.length), + validatedEntries: sum((target) => target.validatedEntries), + validatedTranscriptEvents: sum((target) => target.validatedTranscriptEvents), + }), }; } - -function sumTargets( - targets: DoctorSessionSqliteTargetReport[], - key: keyof Pick< - DoctorSessionSqliteTargetReport, - | "importedEntries" - | "importedTranscriptEvents" - | "legacyEntries" - | "sqliteEntries" - | "validatedEntries" - | "validatedTranscriptEvents" - >, -): number { - return targets.reduce((total, target) => total + target[key], 0); -} /* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/cron/service/jobs-scheduling.ts b/src/cron/service/jobs-scheduling.ts index f5c716de9229..3215288be3f8 100644 --- a/src/cron/service/jobs-scheduling.ts +++ b/src/cron/service/jobs-scheduling.ts @@ -14,13 +14,26 @@ import { createCronStreamSourceIdentity, resolveCronStreamBatching } from "../st import type { CronJob, CronSchedule } from "../types.js"; import { autoDisableCronJob } from "./auto-disable.js"; import { normalizePayloadToSystemText } from "./normalize.js"; -import { isQueuedCronRun, isQueuedForceCronRun } from "./run-admission.js"; import type { CronServiceState, DeferredCronNotifications } from "./state.js"; const STUCK_RUN_MS = 2 * 60 * 60 * 1000; const STAGGER_OFFSET_CACHE_MAX = 4096; const staggerOffsetCache = new Map(); +// A matching process reservation keeps its durable queued/running marker live; +// disabled jobs additionally require force-run ownership. +function ownsCronRunMarker( + state: CronServiceState, + jobId: string, + markerAtMs: number, + requireForce = false, +): boolean { + const reservation = state.queuedRunReservationsByJobId.get(jobId); + return ( + reservation?.markerAtMs === markerAtMs && (!requireForce || reservation.preserveWhenDisabled) + ); +} + export function normalizeStreamScheduleBounds(schedule: CronSchedule): CronSchedule { if (schedule.kind !== "stream") { return schedule; @@ -439,14 +452,14 @@ function normalizeJobTickState(params: { state: CronServiceState; job: CronJob; } if ( job.state.queuedAtMs !== undefined && - !isQueuedForceCronRun(state, job.id, job.state.queuedAtMs) + !ownsCronRunMarker(state, job.id, job.state.queuedAtMs, true) ) { job.state.queuedAtMs = undefined; changed = true; } if ( job.state.runningAtMs !== undefined && - !isQueuedForceCronRun(state, job.id, job.state.runningAtMs) && + !ownsCronRunMarker(state, job.id, job.state.runningAtMs, true) && !isCronJobActive(job.id) ) { job.state.runningAtMs = undefined; @@ -474,7 +487,7 @@ function normalizeJobTickState(params: { state: CronServiceState; job: CronJob; if ( typeof queuedAt === "number" && Math.abs(nowMs - queuedAt) > STUCK_RUN_MS && - !isQueuedCronRun(state, job.id, queuedAt) + !ownsCronRunMarker(state, job.id, queuedAt) ) { state.deps.log.warn( { jobId: job.id, queuedAtMs: queuedAt }, @@ -488,7 +501,7 @@ function normalizeJobTickState(params: { state: CronServiceState; job: CronJob; if ( typeof runningAt === "number" && Math.abs(nowMs - runningAt) > STUCK_RUN_MS && - !isQueuedCronRun(state, job.id, runningAt) + !ownsCronRunMarker(state, job.id, runningAt) ) { state.deps.log.warn( { jobId: job.id, runningAtMs: runningAt }, diff --git a/src/cron/service/ops-run-preparation.ts b/src/cron/service/ops-run-preparation.ts index df9473eacd31..c8b1589cb032 100644 --- a/src/cron/service/ops-run-preparation.ts +++ b/src/cron/service/ops-run-preparation.ts @@ -15,12 +15,12 @@ import { import { locked } from "./locked.js"; import { markManualCronJobActive, ownsStreamSource } from "./ops-shared.js"; import { + activateQueuedCronRun, clearQueuedCronRunReservationMarker, isQueuedCronRunReservationCurrent, isQueuedCronRunReservationMarkerCurrent, releaseQueuedCronRun, reserveQueuedCronRun, - updateQueuedCronRunReservationMarker, } from "./run-admission.js"; import type { CronEvent, CronServiceState, DeferredCronNotifications } from "./state.js"; import { emit } from "./state.js"; @@ -468,39 +468,18 @@ export async function activatePreparedManualRun( return { ok: true, ran: false, reason: "invalid-spec" } as const; } - const startedAt = state.deps.nowMs(); - const previousLastError = job.state.lastError; - const activationRollbackSnapshot = snapshotStoreForRollback(state); - delete job.state.queuedAtMs; - job.state.runningAtMs = startedAt; - job.state.lastError = undefined; - // A failed write restores the durable reservation; run() owns releasing - // that queued claim for every activation failure before it propagates. - await persistOrRestore(state, activationRollbackSnapshot); - updateQueuedCronRunReservationMarker( + const activation = await activateQueuedCronRun({ state, - prepared.jobId, - prepared.reservationIdentity, - startedAt, - previousLastError, - ); - if (state.stopped || state.restartRecoveryPending) { - job.state.lastError = previousLastError; - const rollbackSnapshot = snapshotStoreForRollback(state); - delete job.state.runningAtMs; - try { - await persistOrRestore(state, rollbackSnapshot); - } catch (error) { + job, + reservationIdentity: prepared.reservationIdentity, + onUnavailableRollbackError: async () => { await releasePreparedManualReservationWithRetry(state, prepared); - throw error; - } - releaseQueuedCronRun(state, prepared.jobId, prepared.reservationIdentity); - return { - ok: true, - ran: false, - reason: state.stopped ? "stopped" : "restart-recovery-pending", - } as const; + }, + }); + if (activation.kind === "unavailable") { + return { ok: true, ran: false, reason: activation.reason } as const; } + const { startedAt } = activation; emit(state, { jobId: job.id, action: "started", job, runAtMs: startedAt }); const taskRunId = tryCreateCronTaskRun({ state, diff --git a/src/cron/service/ops.run-admission-cleanup.test.ts b/src/cron/service/ops.run-admission-cleanup.test.ts index 9d0120c01210..b6d389966075 100644 --- a/src/cron/service/ops.run-admission-cleanup.test.ts +++ b/src/cron/service/ops.run-admission-cleanup.test.ts @@ -152,6 +152,7 @@ describe("cron service run admission cleanup", () => { }); const realSave = cronStoreModule.saveCronJobsStore; let reservationPersisted = false; + const markerTransitions: Array<"queued" | "running" | "idle"> = []; const saveSpy = vi .spyOn(cronStoreModule, "saveCronJobsStore") .mockImplementation(async (storePath, nextStore, opts) => { @@ -161,9 +162,13 @@ describe("cron service run admission cleanup", () => { await realSave(storePath, nextStore, opts); if (!reservationPersisted && queuedAtMs === dueAt) { reservationPersisted = true; + markerTransitions.push("queued"); now = dueAt + 1; } else if (reservationPersisted && runningAtMs === dueAt + 1) { + markerTransitions.push("running"); stop(state); + } else if (markerTransitions.length === 2 && !queuedAtMs && !runningAtMs) { + markerTransitions.push("idle"); } }); @@ -184,6 +189,7 @@ describe("cron service run admission cleanup", () => { } expect(runIsolatedAgentJob).not.toHaveBeenCalled(); + expect(markerTransitions).toEqual(["queued", "running", "idle"]); expect(state.queuedRunReservationsByJobId.has(job.id)).toBe(false); const persistedJob = (await loadCronStore(store.storePath)).jobs.find( (entry) => entry.id === job.id, diff --git a/src/cron/service/run-admission.ts b/src/cron/service/run-admission.ts index fdf6a2547381..27a1dd22624d 100644 --- a/src/cron/service/run-admission.ts +++ b/src/cron/service/run-admission.ts @@ -1,6 +1,8 @@ // Shared execution admission for scheduled, manual, and on-exit cron runs. import { DEFAULT_CRON_MAX_CONCURRENT_RUNS } from "../../config/cron-limits.js"; +import type { CronJob } from "../types.js"; import type { CronServiceState } from "./state.js"; +import { persistOrRestore, snapshotStoreForRollback } from "./store.js"; export function resolveRunConcurrency(): number { return DEFAULT_CRON_MAX_CONCURRENT_RUNS; @@ -93,22 +95,6 @@ export function isQueuedCronRunReservationCurrent( return state.queuedRunReservationsByJobId.get(jobId)?.identity === identity; } -export function updateQueuedCronRunReservationMarker( - state: CronServiceState, - jobId: string, - identity: object, - runningAtMs: number, - previousLastError: string | undefined, -): boolean { - const reservation = state.queuedRunReservationsByJobId.get(jobId); - if (reservation?.identity !== identity) { - return false; - } - reservation.markerAtMs = runningAtMs; - reservation.activationPreviousLastError = { value: previousLastError }; - return true; -} - export function restoreQueuedCronRunReservationLastError( state: CronServiceState, jobId: string, @@ -157,23 +143,50 @@ export function isQueuedCronRunReservationMarkerCurrent( return reservation?.identity === identity && reservation.markerAtMs === runningAtMs; } -/** A matching process-local record means this durable queued or running marker is still owned. */ -export function isQueuedCronRun( - state: CronServiceState, - jobId: string, - queuedAtMs: number, -): boolean { - return state.queuedRunReservationsByJobId.get(jobId)?.markerAtMs === queuedAtMs; -} +export async function activateQueuedCronRun(params: { + state: CronServiceState; + job: CronJob; + reservationIdentity: object; + onUnavailable?: () => void; + onUnavailableRollbackError?: () => Promise; +}): Promise< + | { kind: "activated"; startedAt: number } + | { kind: "unavailable"; reason: "stopped" | "restart-recovery-pending" } +> { + const { state, job, reservationIdentity } = params; + const startedAt = state.deps.nowMs(); + const previousLastError = job.state.lastError; + const activationRollbackSnapshot = snapshotStoreForRollback(state); + delete job.state.queuedAtMs; + job.state.runningAtMs = startedAt; + job.state.lastError = undefined; + // Persist running ownership before execution. A failed write restores the + // durable queued marker so the caller can release or recover that claim. + await persistOrRestore(state, activationRollbackSnapshot); + const reservation = state.queuedRunReservationsByJobId.get(job.id); + if (reservation?.identity === reservationIdentity) { + reservation.markerAtMs = startedAt; + reservation.activationPreviousLastError = { value: previousLastError }; + } + if (!state.stopped && !state.restartRecoveryPending) { + return { kind: "activated", startedAt }; + } -/** A disabled job can retain only a force reservation that predated the disabled state. */ -export function isQueuedForceCronRun( - state: CronServiceState, - jobId: string, - markerAtMs: number, -): boolean { - const reservation = state.queuedRunReservationsByJobId.get(jobId); - return reservation?.markerAtMs === markerAtMs && reservation.preserveWhenDisabled; + params.onUnavailable?.(); + job.state.lastError = previousLastError; + const rollbackSnapshot = snapshotStoreForRollback(state); + delete job.state.runningAtMs; + try { + await persistOrRestore(state, rollbackSnapshot); + } catch (error) { + await params.onUnavailableRollbackError?.(); + throw error; + } + releaseQueuedCronRun(state, job.id, reservationIdentity); + return { + kind: "unavailable", + reason: state.stopped ? "stopped" : "restart-recovery-pending", + }; } /** diff --git a/src/cron/service/timer-catchup.ts b/src/cron/service/timer-catchup.ts index 8f3fb0cbd0d2..9827d56fe2e5 100644 --- a/src/cron/service/timer-catchup.ts +++ b/src/cron/service/timer-catchup.ts @@ -9,11 +9,11 @@ import { } from "./jobs.js"; import { locked } from "./locked.js"; import { + activateQueuedCronRun, isQueuedCronRunReservationCurrent, releaseQueuedCronRun, reserveQueuedCronRun, runWithCronAdmission, - updateQueuedCronRunReservationMarker, } from "./run-admission.js"; import { type CronServiceState, type DeferredCronNotifications, emit } from "./state.js"; import { ensureLoaded, persist, persistOrRestore, snapshotStoreForRollback } from "./store.js"; @@ -322,29 +322,15 @@ async function executeStartupCatchupPlan( releaseQueuedCronRun(state, candidate.jobId, candidate.reservationIdentity); return undefined; } - const startedAt = state.deps.nowMs(); - const previousLastError = job.state.lastError; - const activationRollbackSnapshot = snapshotStoreForRollback(state); - delete job.state.queuedAtMs; - job.state.runningAtMs = startedAt; - job.state.lastError = undefined; - await persistOrRestore(state, activationRollbackSnapshot); - updateQueuedCronRunReservationMarker( + const activation = await activateQueuedCronRun({ state, - candidate.jobId, - candidate.reservationIdentity, - startedAt, - previousLastError, - ); - if (state.stopped || state.restartRecoveryPending) { - job.state.lastError = previousLastError; - const rollbackSnapshot = snapshotStoreForRollback(state); - delete job.state.runningAtMs; - await persistOrRestore(state, rollbackSnapshot); - releaseQueuedCronRun(state, candidate.jobId, candidate.reservationIdentity); + job, + reservationIdentity: candidate.reservationIdentity, + }); + if (activation.kind === "unavailable") { return undefined; } - return { ...candidate, job, startedAt }; + return { ...candidate, job, startedAt: activation.startedAt }; }); if (!startedCandidate) { return undefined; diff --git a/src/cron/service/timer-scheduler.ts b/src/cron/service/timer-scheduler.ts index e51e7c4b8f56..af643412dba8 100644 --- a/src/cron/service/timer-scheduler.ts +++ b/src/cron/service/timer-scheduler.ts @@ -18,6 +18,7 @@ import { } from "./jobs.js"; import { locked } from "./locked.js"; import { + activateQueuedCronRun, clearQueuedCronRunReservationMarker, isQueuedCronRunReservationCurrent, isQueuedCronRunReservationMarkerCurrent, @@ -26,7 +27,6 @@ import { resolveRunConcurrency, restoreQueuedCronRunReservationLastError, runWithCronAdmission, - updateQueuedCronRunReservationMarker, } from "./run-admission.js"; import { type CronServiceState, type DeferredCronNotifications, emit } from "./state.js"; import { ensureLoaded, persistOrRestore, snapshotStoreForRollback } from "./store.js"; @@ -457,30 +457,18 @@ async function onAdmittedTimer(state: CronServiceState) { releaseQueuedCronRun(state, due.id, due.reservationIdentity); return undefined; } - const startedAt = state.deps.nowMs(); - const previousLastError = job.state.lastError; - const activationRollbackSnapshot = snapshotStoreForRollback(state); - delete job.state.queuedAtMs; - job.state.runningAtMs = startedAt; - job.state.lastError = undefined; - await persistOrRestore(state, activationRollbackSnapshot); - updateQueuedCronRunReservationMarker( + const activation = await activateQueuedCronRun({ state, - due.id, - due.reservationIdentity, - startedAt, - previousLastError, - ); - if (state.stopped || state.restartRecoveryPending) { - stopAdmittingDueJobs = true; - job.state.lastError = previousLastError; - const rollbackSnapshot = snapshotStoreForRollback(state); - delete job.state.runningAtMs; - await persistOrRestore(state, rollbackSnapshot); - releaseQueuedCronRun(state, due.id, due.reservationIdentity); + job, + reservationIdentity: due.reservationIdentity, + onUnavailable: () => { + stopAdmittingDueJobs = true; + }, + }); + if (activation.kind === "unavailable") { return undefined; } - return { ...due, job, startedAt }; + return { ...due, job, startedAt: activation.startedAt }; }); if (!currentDueJob) { return pMapSkip; diff --git a/src/tui/commands.test.ts b/src/tui/commands.test.ts index 10282b24533c..1cad9ce14fa0 100644 --- a/src/tui/commands.test.ts +++ b/src/tui/commands.test.ts @@ -202,6 +202,25 @@ describe("helpText", () => { expect(output).toContain("/openclaw [request]"); }); + it.each(["goal", "btw", "queue", "stop"])( + "keeps /%s visible in completion and help across TUI modes", + (name) => { + for (const options of [{}, { local: true }]) { + expect(getSlashCommands(options).map((command) => command.name)).toContain(name); + expect(helpText(options)).toContain(`/${name}`); + } + }, + ); + + it.each([{}, { local: true }])("shows required arguments in shared command help", (options) => { + const output = helpText(options); + + expect(output).toContain("/goal start "); + expect(output).toContain("/goal edit "); + expect(output).toContain("/btw "); + expect(output).not.toContain("/btw [side question]"); + }); + it("does not advertise Gateway-owned commands in local mode", () => { const output = helpText({ local: true }); diff --git a/src/tui/commands.ts b/src/tui/commands.ts index c8b8f289da0a..d284e1814d37 100644 --- a/src/tui/commands.ts +++ b/src/tui/commands.ts @@ -38,16 +38,6 @@ type SlashCommandOptions = { dynamicCommands?: CommandEntry[]; }; -const COMMAND_ALIASES: Record = { - crestodian: "openclaw", // hidden alias - gwstatus: "gateway-status", -}; - -// These shared commands have explicit local TUI routing but no same-named -// built-in autocomplete entry. Other shared commands require the Gateway and -// must stay out of local autocomplete and model prompts. -const LOCAL_TUI_ROUTED_SHARED_COMMANDS = new Set(["btw", "goal", "queue", "stop"]); - function createLevelCompletion( levels: string[], ): NonNullable { @@ -66,6 +56,138 @@ export function formatTuiLevelCommandUsage(command: "verbose" | "reasoning"): st return `/${command} <${levels.join("|")}>`; } +type TuiCommandDescriptor = { + name: string; + description?: string; + aliases?: readonly { name: string; description?: string; hidden?: boolean }[]; + scope?: "both" | "local" | "remote"; + shared?: boolean; + handler?: true; + help?: string | readonly string[]; + completions?: readonly string[] | "thinking"; +}; + +type TuiCommandRow = readonly [ + name: string, + description?: string, + help?: string | readonly string[], + completions?: readonly string[] | "thinking", + options?: Pick & { handler?: false }, +]; + +const TUI_COMMAND_ROWS = [ + ["help", "Show slash command help", "/help"], + [ + "commands", + undefined, + "/commands", + undefined, + { scope: "remote", shared: true, handler: false }, + ], + ["status", undefined, "/status", undefined, { scope: "remote", shared: true, handler: false }], + [ + "gateway-status", + "Show gateway status summary", + ["/gateway-status", "/gwstatus"], + undefined, + { aliases: [{ name: "gwstatus", description: "Alias for /gateway-status" }] }, + ], + ["auth", "Run provider auth/login flow", "/auth [provider]", undefined, { scope: "local" }], + ["agent", "Switch agent (or open picker)", "/agent (or /agents)"], + ["agents", "Open agent picker"], + [ + "openclaw", + "Return to OpenClaw", + "/openclaw [request]", + undefined, + { aliases: [{ name: "crestodian", hidden: true }] }, + ], + ["session", "Switch session (or open picker)", "/session (or /sessions)"], + ["sessions", "Open session picker"], + ["model", "Set model (or open picker)", "/model (or /models)"], + ["models", "Open model picker"], + ["think", "Set thinking level", "/think <{thinkingLevels}>", "thinking"], + ["fast", "Set fast mode auto/on/off", "/fast ", FAST_LEVELS], + [ + "verbose", + `Set verbose ${VERBOSE_LEVELS.join("/")}`, + formatTuiLevelCommandUsage("verbose"), + VERBOSE_LEVELS, + ], + ["trace", "Set trace on/off", "/trace ", TRACE_LEVELS], + [ + "reasoning", + `Set reasoning ${REASONING_LEVELS.join("/")}`, + formatTuiLevelCommandUsage("reasoning"), + REASONING_LEVELS, + ], + [ + "usage", + "Toggle per-response usage line", + "/usage ", + USAGE_FOOTER_LEVELS, + ], + [ + "elevated", + "Set elevated on/off/ask/full", + ["/elevated ", "/elev "], + ELEVATED_LEVELS, + { aliases: [{ name: "elev", description: "Alias for /elevated" }] }, + ], + ["activation", "Set group activation", "/activation ", ACTIVATION_LEVELS], + ["context", undefined, undefined, undefined, { scope: "remote", shared: true }], + [ + "goal", + undefined, + "/goal | /goal [status] | /goal start | /goal edit | /goal pause|resume|complete|block|clear", + undefined, + { shared: true }, + ], + ["btw", undefined, "/btw ", undefined, { shared: true }], + ["queue", undefined, "/queue [mode]", undefined, { shared: true }], + ["stop", undefined, "/stop", undefined, { shared: true }], + ["new", "Spawn a new isolated session", "/new or /reset"], + ["reset", "Reset the current session"], + ["abort", "Abort active run", "/abort"], + ["settings", "Open settings", "/settings"], + [ + "exit", + "Exit the TUI", + "/exit", + undefined, + { aliases: [{ name: "quit", description: "Exit the TUI" }] }, + ], +] as const satisfies readonly TuiCommandRow[]; + +const TUI_COMMAND_ROW_VALUES: readonly TuiCommandRow[] = TUI_COMMAND_ROWS; +const TUI_COMMAND_DESCRIPTORS: readonly TuiCommandDescriptor[] = TUI_COMMAND_ROW_VALUES.map( + ([name, description, help, completions, options]) => { + const descriptor: TuiCommandDescriptor = { name, description, help, completions }; + descriptor.aliases = options?.aliases; + descriptor.scope = options?.scope; + descriptor.shared = options?.shared; + if (options?.handler !== false) { + descriptor.handler = true; + } + return descriptor; + }, +); + +export type TuiCommandHandlerName = Exclude< + (typeof TUI_COMMAND_ROWS)[number][0], + "commands" | "status" +>; + +export function resolveTuiCommandDescriptor(name: string): TuiCommandDescriptor | undefined { + return TUI_COMMAND_DESCRIPTORS.find( + (command) => command.name === name || command.aliases?.some((alias) => alias.name === name), + ); +} + +function commandIsVisible(command: TuiCommandDescriptor, local: boolean): boolean { + return command.scope !== (local ? "remote" : "local"); +} + function normalizeSlashCommandName(value: string): string { return value.replace(/^\//, "").trim(); } @@ -75,13 +197,14 @@ function appendSlashCommand( seen: Set, name: string, description: string, + getArgumentCompletions?: SlashCommand["getArgumentCompletions"], ) { const normalizedName = normalizeSlashCommandName(name); if (!normalizedName || seen.has(normalizedName)) { return; } seen.add(normalizedName); - commands.push({ name: normalizedName, description }); + commands.push({ name: normalizedName, description, getArgumentCompletions }); } export function parseCommand(input: string): ParsedCommand { @@ -98,8 +221,9 @@ export function parseCommand(input: string): ParsedCommand { } const [name, ...rest] = trimmed.split(/\s+/); const normalized = normalizeLowercaseStringOrEmpty(name); + const descriptor = resolveTuiCommandDescriptor(normalized); return { - name: COMMAND_ALIASES[normalized] ?? normalized, + name: descriptor?.name ?? normalized, args: rest.join(" ").trim(), }; } @@ -113,92 +237,43 @@ export function getSlashCommands(options: SlashCommandOptions = {}): SlashComman const thinkLevels = options.thinkingLevels?.length ? options.thinkingLevels.map((level) => level.label) : listThinkingLevelLabels(options.provider, options.model, undefined, options.agentRuntime); - const verboseCompletions = createLevelCompletion(VERBOSE_LEVELS); - const traceCompletions = createLevelCompletion(TRACE_LEVELS); - const fastCompletions = createLevelCompletion(FAST_LEVELS); - const reasoningCompletions = createLevelCompletion(REASONING_LEVELS); - const usageCompletions = createLevelCompletion(USAGE_FOOTER_LEVELS); - const elevatedCompletions = createLevelCompletion(ELEVATED_LEVELS); - const activationCompletions = createLevelCompletion(ACTIVATION_LEVELS); - const commands: SlashCommand[] = [ - { name: "help", description: "Show slash command help" }, - { name: "gateway-status", description: "Show gateway status summary" }, - { name: "gwstatus", description: "Alias for /gateway-status" }, - ...(options.local ? [{ name: "auth", description: "Run provider auth/login flow" }] : []), - { name: "agent", description: "Switch agent (or open picker)" }, - { name: "agents", description: "Open agent picker" }, - { name: "openclaw", description: "Return to OpenClaw" }, - { name: "session", description: "Switch session (or open picker)" }, - { name: "sessions", description: "Open session picker" }, - { - name: "model", - description: "Set model (or open picker)", - }, - { name: "models", description: "Open model picker" }, - { - name: "think", - description: "Set thinking level", - getArgumentCompletions: (prefix) => - thinkLevels - .filter((v) => v.startsWith(normalizeLowercaseStringOrEmpty(prefix))) - .map((value) => ({ value, label: value })), - }, - { - name: "fast", - description: "Set fast mode auto/on/off", - getArgumentCompletions: fastCompletions, - }, - { - name: "verbose", - description: `Set verbose ${VERBOSE_LEVELS.join("/")}`, - getArgumentCompletions: verboseCompletions, - }, - { - name: "trace", - description: "Set trace on/off", - getArgumentCompletions: traceCompletions, - }, - { - name: "reasoning", - description: `Set reasoning ${REASONING_LEVELS.join("/")}`, - getArgumentCompletions: reasoningCompletions, - }, - { - name: "usage", - description: "Toggle per-response usage line", - getArgumentCompletions: usageCompletions, - }, - { - name: "elevated", - description: "Set elevated on/off/ask/full", - getArgumentCompletions: elevatedCompletions, - }, - { - name: "elev", - description: "Alias for /elevated", - getArgumentCompletions: elevatedCompletions, - }, - { - name: "activation", - description: "Set group activation", - getArgumentCompletions: activationCompletions, - }, - { name: "abort", description: "Abort active run" }, - { name: "new", description: "Spawn a new isolated session" }, - { name: "reset", description: "Reset the current session" }, - { name: "settings", description: "Open settings" }, - { name: "exit", description: "Exit the TUI" }, - { name: "quit", description: "Exit the TUI" }, - ]; + const commands: SlashCommand[] = []; + const seen = new Set(); + for (const command of TUI_COMMAND_DESCRIPTORS) { + if ( + command.shared || + !command.description || + !commandIsVisible(command, options.local === true) + ) { + continue; + } + const completions = + command.completions === "thinking" + ? createLevelCompletion(thinkLevels) + : command.completions + ? createLevelCompletion([...command.completions]) + : undefined; + appendSlashCommand(commands, seen, command.name, command.description, completions); + for (const alias of command.aliases ?? []) { + if (!alias.hidden) { + appendSlashCommand( + commands, + seen, + alias.name, + alias.description ?? command.description, + completions, + ); + } + } + } - const seen = new Set(commands.map((command) => command.name)); const gatewayCommands = options.cfg ? listChatCommandsForConfig(options.cfg) : listChatCommands(); for (const command of gatewayCommands) { - if ( - options.local && - !seen.has(command.key) && - !LOCAL_TUI_ROUTED_SHARED_COMMANDS.has(command.key) - ) { + const descriptor = resolveTuiCommandDescriptor(command.key); + if (options.local && !seen.has(command.key) && !descriptor?.shared) { + continue; + } + if (options.local && descriptor && !commandIsVisible(descriptor, true)) { continue; } const aliases = command.textAliases.length > 0 ? command.textAliases : [`/${command.key}`]; @@ -247,30 +322,16 @@ export function helpText(options: SlashCommandOptions = {}): string { undefined, options.agentRuntime, ); + const commandHelp = TUI_COMMAND_DESCRIPTORS.flatMap((command) => { + if (!command.help || !commandIsVisible(command, options.local === true)) { + return []; + } + const lines = typeof command.help === "string" ? [command.help] : command.help; + return lines.map((line) => line.replace("{thinkingLevels}", thinkLevels)); + }); return [ "Slash commands:", - "/help", - ...(options.local ? [] : ["/commands", "/status"]), - "/gateway-status", - "/gwstatus", - ...(options.local ? ["/auth [provider]"] : []), - "/agent (or /agents)", - "/openclaw [request]", - "/session (or /sessions)", - "/model (or /models)", - `/think <${thinkLevels}>`, - "/fast ", - formatTuiLevelCommandUsage("verbose"), - "/trace ", - formatTuiLevelCommandUsage("reasoning"), - "/usage ", - "/elevated ", - "/elev ", - "/activation ", - "/new or /reset", - "/abort", - "/settings", - "/exit", + ...commandHelp, "", "Keyboard shortcuts:", "Enter: send message", diff --git a/src/tui/tui-command-handlers.test.ts b/src/tui/tui-command-handlers.test.ts index 12691f08392e..544fd84d3dfe 100644 --- a/src/tui/tui-command-handlers.test.ts +++ b/src/tui/tui-command-handlers.test.ts @@ -571,7 +571,7 @@ describe("tui command handlers", () => { const emptySide = createHarness({ opts: { local: true } }); await emptySide.handleCommand("/side"); expect(emptySide.sendChat).not.toHaveBeenCalled(); - expect(emptySide.addSystem).toHaveBeenCalledWith("Usage: /btw [side question]"); + expect(emptySide.addSystem).toHaveBeenCalledWith("Usage: /btw "); const side = createHarness({ opts: { local: true } }); await side.handleCommand("/side check this"); diff --git a/src/tui/tui-command-handlers.ts b/src/tui/tui-command-handlers.ts index c7c617d076a9..bb0a0a59b00a 100644 --- a/src/tui/tui-command-handlers.ts +++ b/src/tui/tui-command-handlers.ts @@ -20,6 +20,8 @@ import { helpText, isSharedTextCommand, parseCommand, + resolveTuiCommandDescriptor, + type TuiCommandHandlerName, } from "./commands.js"; import type { ChatLog } from "./components/chat-log.js"; import { @@ -490,412 +492,386 @@ export function createCommandHandlers(context: CommandHandlerContext) { tui.requestRender(); }; + type CommandHandler = (args: string, raw: string) => void | Promise; + const commandHandlers = { + help: () => { + chatLog.addSystem( + helpText({ + local: opts.local, + provider: state.sessionInfo.modelProvider, + model: state.sessionInfo.model, + agentRuntime: state.sessionInfo.agentRuntime?.id, + }), + ); + }, + auth: async (args) => { + if (!runAuthFlow) { + chatLog.addSystem("auth login is only available in local embedded mode"); + return; + } + if (state.activeChatRunId || hasPendingSubmit(state)) { + chatLog.addSystem("abort the current run before /auth"); + return; + } + const provider = args.trim() || state.sessionInfo.modelProvider || undefined; + chatLog.addSystem( + provider + ? `opening auth flow for ${provider}; TUI will resume when it exits` + : "opening auth flow; TUI will resume when it exits", + ); + tui.requestRender(); + setActivityStatus("auth"); + try { + const result = await runAuthFlow({ provider }); + await refreshSessionInfo(); + if (result.exitCode === 0 && !result.signal) { + chatLog.addSystem(provider ? `auth flow finished for ${provider}` : "auth flow finished"); + setActivityStatus("idle"); + } else { + const failureSuffix = result.signal + ? ` (signal ${result.signal})` + : typeof result.exitCode === "number" + ? ` (exit ${String(result.exitCode)})` + : ""; + chatLog.addSystem(`auth flow failed${failureSuffix}`); + setActivityStatus("error"); + } + } catch (err) { + chatLog.addSystem(`auth flow failed: ${formatTuiErrorMessage(err)}`); + setActivityStatus("error"); + } + }, + "gateway-status": async () => { + try { + const status = await client.getGatewayStatus(); + if (typeof status === "string") { + chatLog.addSystem(status); + return; + } + if (status && typeof status === "object") { + const lines = formatStatusSummary(status as GatewayStatusSummary); + for (const line of lines) { + chatLog.addSystem(line); + } + return; + } + chatLog.addSystem("status: unknown response"); + } catch (err) { + chatLog.addSystem(`status failed: ${formatTuiErrorMessage(err)}`); + } + }, + agent: async (args) => { + if (!args) { + await openAgentSelector(); + } else { + await setAgent(args); + } + }, + agents: async () => await openAgentSelector(), + context: async (args, raw) => { + if (opts.local) { + addUnsupportedLocalCommand("context"); + } else if (!args) { + openContextModeSelector(); + } else { + await sendMessage(raw); + } + }, + goal: async (_args, raw) => { + if (opts.local === true && client.runGoalCommand) { + try { + const result = await client.runGoalCommand({ + sessionKey: state.currentSessionKey, + agentId: state.currentAgentId, + command: raw, + }); + chatLog.addSystem(result.text); + await refreshSessionInfo(); + if (result.continuationPrompt) { + await sendMessage(result.continuationPrompt); + } + } catch (err) { + chatLog.addSystem(`goal failed: ${formatTuiErrorMessage(err)}`); + } + } else { + await sendMessage(raw); + } + }, + btw: async (args, raw) => { + if (args) { + await sendMessage(raw); + } else { + chatLog.addSystem("Usage: /btw "); + } + }, + queue: async (_args, raw) => await sendMessage(raw), + openclaw: (args) => { + chatLog.addSystem( + args ? `returning to OpenClaw with request: ${args}` : "returning to OpenClaw", + ); + requestExit({ + exitReason: "return-to-system-agent", + ...(args ? { systemAgentMessage: args } : {}), + }); + }, + session: async (args) => { + if (!args) { + await openSessionSelector(); + } else { + await setSession(args); + } + }, + sessions: async () => await openSessionSelector(), + model: async (args, raw) => { + if (shouldForwardModelCommandToServer(args)) { + await sendMessage(raw); + } else if (!args) { + await openModelSelector(); + } else { + await applySessionSetting( + { model: args }, + (result) => { + const resolvedModel = result.resolved?.model; + const resolvedProvider = result.resolved?.modelProvider; + const resolvedModelRef = resolvedModel + ? resolvedProvider + ? modelKey(resolvedProvider, resolvedModel) + : resolvedModel + : args; + return `model set to ${resolvedModelRef}`; + }, + "model set failed", + ); + } + }, + models: async () => await openModelSelector(), + think: async (args) => { + if (!args) { + const levels = + state.sessionInfo.thinkingLevels?.map((level) => level.label).join("|") || + formatThinkingLevels( + state.sessionInfo.modelProvider, + state.sessionInfo.model, + "|", + undefined, + state.sessionInfo.agentRuntime?.id, + ); + chatLog.addSystem(`usage: /think <${levels}>`); + return; + } + await applySessionSetting({ thinkingLevel: args }, `thinking set to ${args}`, "think failed"); + }, + verbose: async (args) => { + if (!args) { + chatLog.addSystem(`usage: ${formatTuiLevelCommandUsage("verbose")}`); + return; + } + await applySessionSetting( + { verboseLevel: args }, + `verbose set to ${args}`, + "verbose failed", + async () => { + if (args === "off") { + chatLog.clearTools(); + await refreshSessionInfo(); + } else { + await loadHistory(); + } + }, + ); + }, + trace: async (args) => { + if (!args) { + chatLog.addSystem("usage: /trace "); + return; + } + await applySessionSetting({ traceLevel: args }, `trace set to ${args}`, "trace failed"); + }, + fast: async (args) => { + if (!args || args === "status") { + chatLog.addSystem(`fast mode: ${formatTuiFastMode(state.sessionInfo.fastMode)}`); + return; + } + if (args !== "auto" && args !== "on" && args !== "off") { + chatLog.addSystem("usage: /fast "); + return; + } + await applySessionSetting( + { fastMode: args === "auto" ? "auto" : args === "on" }, + `fast mode set to ${args}`, + "fast failed", + ); + }, + reasoning: async (args) => { + if (!args) { + chatLog.addSystem(`usage: ${formatTuiLevelCommandUsage("reasoning")}`); + return; + } + await applySessionSetting( + { reasoningLevel: args }, + `reasoning set to ${args}`, + "reasoning failed", + ); + }, + usage: async (args) => { + const isReset = args ? isSessionDefaultDirectiveValue(args) : false; + const normalized = args && !isReset ? normalizeUsageDisplay(args) : undefined; + if (args && !normalized && !isReset) { + chatLog.addSystem("usage: /usage "); + return; + } + if (isReset) { + await applySessionSetting( + { responseUsage: null }, + "usage footer: reset to default", + "usage failed", + async () => { + delete state.sessionInfo.responseUsage; + delete state.sessionInfo.effectiveResponseUsage; + await refreshSessionInfo(); + }, + ); + return; + } + const current = + state.sessionInfo.effectiveResponseUsage ?? + resolveResponseUsageMode(state.sessionInfo.responseUsage); + const next = + normalized ?? (current === "off" ? "tokens" : current === "tokens" ? "full" : "off"); + await applySessionSetting({ responseUsage: next }, `usage footer: ${next}`, "usage failed"); + }, + elevated: async (args) => { + if (!args) { + chatLog.addSystem("usage: /elevated "); + return; + } + if (!["on", "off", "ask", "full"].includes(args)) { + chatLog.addSystem("usage: /elevated "); + return; + } + await applySessionSetting( + { elevatedLevel: args }, + `elevated set to ${args}`, + "elevated failed", + ); + }, + activation: async (args) => { + if (!args) { + chatLog.addSystem("usage: /activation "); + return; + } + const activation = normalizeGroupActivation(args); + if (!activation) { + chatLog.addSystem("usage: /activation "); + return; + } + await applySessionSetting( + { groupActivation: activation }, + `activation set to ${activation}`, + "activation failed", + ); + }, + new: async () => { + if (rejectUnsafeSessionRollover("new")) { + return; + } + const finishSessionTransition = beginSessionTransition("new"); + try { + // Clear token counts immediately to avoid stale display (#1523) + state.sessionInfo.inputTokens = null; + state.sessionInfo.outputTokens = null; + state.sessionInfo.totalTokens = null; + tui.requestRender(); + + const uniqueKey = `tui-${randomUUID()}`; + const result = await client.createSession({ + key: uniqueKey, + agentId: state.currentAgentId, + ...(state.currentSessionId + ? { parentSessionKey: state.currentSessionKey, succeedsParent: true } + : {}), + }); + if (!result.key) { + throw new Error("sessions.create returned no session key"); + } + await setSession(result.key); + chatLog.addSystem(`new session: ${result.key}`); + } catch (err) { + chatLog.addSystem(`new session failed: ${formatTuiErrorMessage(err)}`); + } finally { + finishSessionTransition(); + } + }, + reset: async () => { + if (rejectUnsafeSessionRollover("reset")) { + return; + } + const resetSelection = captureSessionSelection(); + let resetResultSelection = resetSelection; + const finishSessionTransition = beginSessionTransition("reset"); + try { + // Clear token counts immediately to avoid stale display (#1523) + state.sessionInfo.inputTokens = null; + state.sessionInfo.outputTokens = null; + state.sessionInfo.totalTokens = null; + tui.requestRender(); + + const result = await client.resetSession( + resetSelection.sessionKey, + "reset", + resetSelection.sessionKey === "global" ? { agentId: resetSelection.agentId } : undefined, + ); + if (!isCurrentSessionSelection(resetSelection)) { + return; + } + if (applySessionMutationResult(result, resetSelection)) { + resetResultSelection = captureSessionSelection(); + await refreshSessionInfo(); + } else { + await loadHistory(); + } + if (!isCurrentSessionSelection(resetResultSelection)) { + return; + } + chatLog.addSystem(`session ${state.currentSessionKey} reset`); + } catch (err) { + if (!isCurrentSessionSelection(resetResultSelection)) { + return; + } + chatLog.addSystem(`reset failed: ${formatTuiErrorMessage(err)}`); + } finally { + finishSessionTransition(); + } + }, + abort: async () => await abortActive(), + stop: async () => { + // Queued client runs can terminalize before the followup executes, so + // local run ids are not a complete stop target inventory. + await abortActive({ preferActive: true }); + }, + settings: () => openSettings(), + exit: () => requestExit(), + } satisfies Record; + const handleCommand = async (raw: string) => { const { name, args } = parseCommand(raw); if (!name) { return; } - if (sessionTransition.active && name !== "exit" && name !== "quit") { + const descriptor = resolveTuiCommandDescriptor(name); + if (sessionTransition.active && descriptor?.name !== "exit") { chatLog.addSystem( `session change in progress; wait for /${sessionTransition.active} to finish`, ); tui.requestRender(); return; } - switch (name) { - case "help": - chatLog.addSystem( - helpText({ - local: opts.local, - provider: state.sessionInfo.modelProvider, - model: state.sessionInfo.model, - agentRuntime: state.sessionInfo.agentRuntime?.id, - }), - ); - break; - case "auth": { - if (!runAuthFlow) { - chatLog.addSystem("auth login is only available in local embedded mode"); - break; - } - if (state.activeChatRunId || hasPendingSubmit(state)) { - chatLog.addSystem("abort the current run before /auth"); - break; - } - const provider = args.trim() || state.sessionInfo.modelProvider || undefined; - chatLog.addSystem( - provider - ? `opening auth flow for ${provider}; TUI will resume when it exits` - : "opening auth flow; TUI will resume when it exits", - ); - tui.requestRender(); - setActivityStatus("auth"); - try { - const result = await runAuthFlow({ provider }); - await refreshSessionInfo(); - if (result.exitCode === 0 && !result.signal) { - chatLog.addSystem( - provider ? `auth flow finished for ${provider}` : "auth flow finished", - ); - setActivityStatus("idle"); - } else { - const failureSuffix = result.signal - ? ` (signal ${result.signal})` - : typeof result.exitCode === "number" - ? ` (exit ${String(result.exitCode)})` - : ""; - chatLog.addSystem(`auth flow failed${failureSuffix}`); - setActivityStatus("error"); - } - } catch (err) { - chatLog.addSystem(`auth flow failed: ${formatTuiErrorMessage(err)}`); - setActivityStatus("error"); - } - break; - } - case "gateway-status": - try { - const status = await client.getGatewayStatus(); - if (typeof status === "string") { - chatLog.addSystem(status); - break; - } - if (status && typeof status === "object") { - const lines = formatStatusSummary(status as GatewayStatusSummary); - for (const line of lines) { - chatLog.addSystem(line); - } - break; - } - chatLog.addSystem("status: unknown response"); - } catch (err) { - chatLog.addSystem(`status failed: ${formatTuiErrorMessage(err)}`); - } - break; - case "agent": - if (!args) { - await openAgentSelector(); - } else { - await setAgent(args); - } - break; - case "agents": - await openAgentSelector(); - break; - case "context": - if (opts.local) { - addUnsupportedLocalCommand(name); - } else if (!args) { - openContextModeSelector(); - } else { - await sendMessage(raw); - } - break; - case "goal": - if (opts.local === true && client.runGoalCommand) { - try { - const result = await client.runGoalCommand({ - sessionKey: state.currentSessionKey, - agentId: state.currentAgentId, - command: raw, - }); - chatLog.addSystem(result.text); - await refreshSessionInfo(); - if (result.continuationPrompt) { - await sendMessage(result.continuationPrompt); - } - } catch (err) { - chatLog.addSystem(`goal failed: ${formatTuiErrorMessage(err)}`); - } - } else { - await sendMessage(raw); - } - break; - case "btw": - if (args) { - await sendMessage(raw); - } else { - chatLog.addSystem("Usage: /btw [side question]"); - } - break; - case "queue": - await sendMessage(raw); - break; - case "openclaw": - chatLog.addSystem( - args ? `returning to OpenClaw with request: ${args}` : "returning to OpenClaw", - ); - requestExit({ - exitReason: "return-to-system-agent", - ...(args ? { systemAgentMessage: args } : {}), - }); - break; - case "session": - if (!args) { - await openSessionSelector(); - } else { - await setSession(args); - } - break; - case "sessions": - await openSessionSelector(); - break; - case "model": - if (shouldForwardModelCommandToServer(args)) { - await sendMessage(raw); - } else if (!args) { - await openModelSelector(); - } else { - await applySessionSetting( - { model: args }, - (result) => { - const resolvedModel = result.resolved?.model; - const resolvedProvider = result.resolved?.modelProvider; - const resolvedModelRef = resolvedModel - ? resolvedProvider - ? modelKey(resolvedProvider, resolvedModel) - : resolvedModel - : args; - return `model set to ${resolvedModelRef}`; - }, - "model set failed", - ); - } - break; - case "models": - await openModelSelector(); - break; - case "think": - if (!args) { - const levels = - state.sessionInfo.thinkingLevels?.map((level) => level.label).join("|") || - formatThinkingLevels( - state.sessionInfo.modelProvider, - state.sessionInfo.model, - "|", - undefined, - state.sessionInfo.agentRuntime?.id, - ); - chatLog.addSystem(`usage: /think <${levels}>`); - break; - } - await applySessionSetting( - { thinkingLevel: args }, - `thinking set to ${args}`, - "think failed", - ); - break; - case "verbose": - if (!args) { - chatLog.addSystem(`usage: ${formatTuiLevelCommandUsage("verbose")}`); - break; - } - await applySessionSetting( - { verboseLevel: args }, - `verbose set to ${args}`, - "verbose failed", - async () => { - if (args === "off") { - chatLog.clearTools(); - await refreshSessionInfo(); - } else { - await loadHistory(); - } - }, - ); - break; - case "trace": - if (!args) { - chatLog.addSystem("usage: /trace "); - break; - } - await applySessionSetting({ traceLevel: args }, `trace set to ${args}`, "trace failed"); - break; - case "fast": - if (!args || args === "status") { - chatLog.addSystem(`fast mode: ${formatTuiFastMode(state.sessionInfo.fastMode)}`); - break; - } - if (args !== "auto" && args !== "on" && args !== "off") { - chatLog.addSystem("usage: /fast "); - break; - } - await applySessionSetting( - { fastMode: args === "auto" ? "auto" : args === "on" }, - `fast mode set to ${args}`, - "fast failed", - ); - break; - case "reasoning": - if (!args) { - chatLog.addSystem(`usage: ${formatTuiLevelCommandUsage("reasoning")}`); - break; - } - await applySessionSetting( - { reasoningLevel: args }, - `reasoning set to ${args}`, - "reasoning failed", - ); - break; - case "usage": { - const isReset = args ? isSessionDefaultDirectiveValue(args) : false; - const normalized = args && !isReset ? normalizeUsageDisplay(args) : undefined; - if (args && !normalized && !isReset) { - chatLog.addSystem("usage: /usage "); - break; - } - if (isReset) { - await applySessionSetting( - { responseUsage: null }, - "usage footer: reset to default", - "usage failed", - async () => { - delete state.sessionInfo.responseUsage; - delete state.sessionInfo.effectiveResponseUsage; - await refreshSessionInfo(); - }, - ); - break; - } - const current = - state.sessionInfo.effectiveResponseUsage ?? - resolveResponseUsageMode(state.sessionInfo.responseUsage); - const next = - normalized ?? (current === "off" ? "tokens" : current === "tokens" ? "full" : "off"); - await applySessionSetting({ responseUsage: next }, `usage footer: ${next}`, "usage failed"); - break; - } - case "elevated": - if (!args) { - chatLog.addSystem("usage: /elevated "); - break; - } - if (!["on", "off", "ask", "full"].includes(args)) { - chatLog.addSystem("usage: /elevated "); - break; - } - await applySessionSetting( - { elevatedLevel: args }, - `elevated set to ${args}`, - "elevated failed", - ); - break; - case "activation": { - if (!args) { - chatLog.addSystem("usage: /activation "); - break; - } - const activation = normalizeGroupActivation(args); - if (!activation) { - chatLog.addSystem("usage: /activation "); - break; - } - await applySessionSetting( - { groupActivation: activation }, - `activation set to ${activation}`, - "activation failed", - ); - break; - } - case "new": { - if (rejectUnsafeSessionRollover("new")) { - break; - } - const finishSessionTransition = beginSessionTransition("new"); - try { - // Clear token counts immediately to avoid stale display (#1523) - state.sessionInfo.inputTokens = null; - state.sessionInfo.outputTokens = null; - state.sessionInfo.totalTokens = null; - tui.requestRender(); - - const uniqueKey = `tui-${randomUUID()}`; - const result = await client.createSession({ - key: uniqueKey, - agentId: state.currentAgentId, - ...(state.currentSessionId - ? { parentSessionKey: state.currentSessionKey, succeedsParent: true } - : {}), - }); - if (!result.key) { - throw new Error("sessions.create returned no session key"); - } - await setSession(result.key); - chatLog.addSystem(`new session: ${result.key}`); - } catch (err) { - chatLog.addSystem(`new session failed: ${formatTuiErrorMessage(err)}`); - } finally { - finishSessionTransition(); - } - break; - } - case "reset": { - if (rejectUnsafeSessionRollover("reset")) { - break; - } - const resetSelection = captureSessionSelection(); - let resetResultSelection = resetSelection; - const finishSessionTransition = beginSessionTransition("reset"); - try { - // Clear token counts immediately to avoid stale display (#1523) - state.sessionInfo.inputTokens = null; - state.sessionInfo.outputTokens = null; - state.sessionInfo.totalTokens = null; - tui.requestRender(); - - const result = await client.resetSession( - resetSelection.sessionKey, - name, - resetSelection.sessionKey === "global" - ? { agentId: resetSelection.agentId } - : undefined, - ); - if (!isCurrentSessionSelection(resetSelection)) { - return; - } - if (applySessionMutationResult(result, resetSelection)) { - resetResultSelection = captureSessionSelection(); - await refreshSessionInfo(); - } else { - await loadHistory(); - } - if (!isCurrentSessionSelection(resetResultSelection)) { - return; - } - chatLog.addSystem(`session ${state.currentSessionKey} reset`); - } catch (err) { - if (!isCurrentSessionSelection(resetResultSelection)) { - return; - } - chatLog.addSystem(`reset failed: ${formatTuiErrorMessage(err)}`); - } finally { - finishSessionTransition(); - } - break; - } - case "abort": - await abortActive(); - break; - case "stop": - // Queued client runs can terminalize before the followup executes, so - // local run ids are not a complete stop target inventory. - await abortActive({ preferActive: true }); - break; - case "settings": - openSettings(); - break; - case "exit": - case "quit": - requestExit(); - break; - default: { - if (opts.local && isSharedTextCommand(raw)) { - addUnsupportedLocalCommand(name); - break; - } - await sendMessage(raw); - break; - } + if (descriptor?.handler) { + await commandHandlers[descriptor.name as TuiCommandHandlerName](args, raw); + } else if (opts.local && isSharedTextCommand(raw)) { + addUnsupportedLocalCommand(name); + } else { + await sendMessage(raw); } tui.requestRender(); }; diff --git a/src/tui/tui-pty-harness.e2e.test.ts b/src/tui/tui-pty-harness.e2e.test.ts index dabc249365f4..a0fe30ffa5c5 100644 --- a/src/tui/tui-pty-harness.e2e.test.ts +++ b/src/tui/tui-pty-harness.e2e.test.ts @@ -872,6 +872,11 @@ describe.sequential("TUI PTY harness", () => { await fixture.run.waitForOutput("/help"); await fixture.run.waitForOutput("/verbose "); await fixture.run.waitForOutput("/reasoning "); + await fixture.run.waitForOutput("/goal"); + await fixture.run.waitForOutput("/goal start "); + await fixture.run.waitForOutput("/btw "); + await fixture.run.waitForOutput("/queue"); + await fixture.run.waitForOutput("/stop"); await fixture.run.waitForOutput("/exit"); }, TEST_TIMEOUT_MS, diff --git a/src/tui/tui-pty-local.e2e.test.ts b/src/tui/tui-pty-local.e2e.test.ts index 3233ad4f04fd..da6624361288 100644 --- a/src/tui/tui-pty-local.e2e.test.ts +++ b/src/tui/tui-pty-local.e2e.test.ts @@ -876,7 +876,7 @@ describe("TUI PTY real backends", () => { ); } await fixture.run.write("/side\r"); - await fixture.run.waitForOutput("Usage: /btw [side question]"); + await fixture.run.waitForOutput("Usage: /btw "); expect(fixture.mockModel.requests()).toHaveLength(0); await fixture.run.write("slow local parent\r"); From 5b7272f9b55de03871eb2166f29f7a77be340bc2 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 04:10:41 -0700 Subject: [PATCH 26/57] fix(pdf): surface image-render failures when documents have no text (#118629) --- .../document-extractor.test.ts | 24 ++++++++++++++++++- .../document-extract/document-extractor.ts | 3 +++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/extensions/document-extract/document-extractor.test.ts b/extensions/document-extract/document-extractor.test.ts index 9155474a1bb6..e25169a2d28e 100644 --- a/extensions/document-extract/document-extractor.test.ts +++ b/extensions/document-extract/document-extractor.test.ts @@ -159,8 +159,9 @@ describe("PDF document extractor", () => { .mockResolvedValueOnce({ text: "", images: [] }); const extractor = createPdfDocumentExtractor(); - await extractor.extract(request({ pageNumbers: [3, 2, 0, 1], maxPages: 2 })); + const result = await extractor.extract(request({ pageNumbers: [3, 2, 0, 1], maxPages: 2 })); + expect(result).toEqual({ text: "", images: [] }); expect(pdfDocument.extract).toHaveBeenNthCalledWith( 1, expect.objectContaining({ mode: "text", pages: [2, 1] }), @@ -189,4 +190,25 @@ describe("PDF document extractor", () => { expect(onImageExtractionError).toHaveBeenCalledWith(failure); expect(pdfDocument.destroy).toHaveBeenCalledTimes(1); }); + + it.each([ + { label: "empty", text: "", reportError: true }, + { label: "whitespace-only", text: " \t\n", reportError: false }, + ])("surfaces image fallback failures for $label PDF text", async ({ text, reportError }) => { + const { PdfBudgetError } = await vi.importActual("clawpdf"); + const onImageExtractionError = vi.fn(); + const failure = new PdfBudgetError("renderPixels", 100); + pdfDocument.extract.mockResolvedValueOnce({ text, images: [] }).mockRejectedValueOnce(failure); + const overrides = reportError ? { onImageExtractionError } : {}; + + await expect(createPdfDocumentExtractor().extract(request(overrides))).rejects.toMatchObject({ + message: "PDF image extraction failed with no extractable text.", + cause: failure, + }); + expect(onImageExtractionError).toHaveBeenCalledTimes(reportError ? 1 : 0); + if (reportError) { + expect(onImageExtractionError).toHaveBeenCalledWith(failure); + } + expect(pdfDocument.destroy).toHaveBeenCalledTimes(1); + }); }); diff --git a/extensions/document-extract/document-extractor.ts b/extensions/document-extract/document-extractor.ts index 809c9d70b284..e2cb7dc0ac30 100644 --- a/extensions/document-extract/document-extractor.ts +++ b/extensions/document-extract/document-extractor.ts @@ -116,6 +116,9 @@ async function extractPdfContent( return { text, images }; } catch (err) { request.onImageExtractionError?.(err); + if (!text.trim()) { + throw new Error("PDF image extraction failed with no extractable text.", { cause: err }); + } return { text, images: [] }; } } finally { From e87fb689b533c2e78f03087675a2d6021d3126fc Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 04:12:18 -0700 Subject: [PATCH 27/57] fix(channels): preserve thread participation expiry across restarts (#118630) Co-authored-by: Peter Steinberger --- .../mattermost/thread-participation.test.ts | 17 ++++++++++++-- .../src/mattermost/thread-participation.ts | 2 ++ .../slack/src/sent-thread-cache.test.ts | 23 +++++++++++++++---- extensions/slack/src/sent-thread-cache.ts | 2 ++ 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/extensions/mattermost/src/mattermost/thread-participation.test.ts b/extensions/mattermost/src/mattermost/thread-participation.test.ts index 7ac0f89974cc..c29af517d1ee 100644 --- a/extensions/mattermost/src/mattermost/thread-participation.test.ts +++ b/extensions/mattermost/src/mattermost/thread-participation.test.ts @@ -49,6 +49,7 @@ describe("mattermost thread participation", () => { afterEach(() => { threadParticipationMemory.clear(); resetPluginStateStoreForTests(); + vi.restoreAllMocks(); }); it("remembers a thread the bot replied in", async () => { @@ -85,10 +86,13 @@ describe("mattermost thread participation", () => { ).resolves.toBe(false); }); - it("recovers participation from the persistent store after the in-memory cache is lost", async () => { + it("restores participation after a restart without extending its original expiry", async () => { + const repliedAt = 1_711_406_400_000; + const now = vi.spyOn(Date, "now").mockReturnValue(repliedAt); recordMattermostThreadParticipation("acct", "chan", "root-1"); await flush(); - // Simulate a restart: in-memory cache cleared, persistent SQLite store intact. + now.mockReturnValue(repliedAt + 7 * 24 * 60 * 60 * 1000 - 1000); + // Simulate a restart near expiry: memory is lost, but the SQLite row is still valid. threadParticipationMemory.clear(); await expect( hasMattermostThreadParticipationWithPersistence({ @@ -97,6 +101,15 @@ describe("mattermost thread participation", () => { threadRootId: "root-1", }), ).resolves.toBe(true); + + now.mockReturnValue(repliedAt + 7 * 24 * 60 * 60 * 1000 + 1000); + await expect( + hasMattermostThreadParticipationWithPersistence({ + accountId: "acct", + channelId: "chan", + threadRootId: "root-1", + }), + ).resolves.toBe(false); }); it("degrades to in-memory only when the persistent store fails", async () => { diff --git a/extensions/mattermost/src/mattermost/thread-participation.ts b/extensions/mattermost/src/mattermost/thread-participation.ts index 8f43bd1a9cd5..3443bc33ccaa 100644 --- a/extensions/mattermost/src/mattermost/thread-participation.ts +++ b/extensions/mattermost/src/mattermost/thread-participation.ts @@ -37,6 +37,8 @@ const threadParticipation = createPersistentDedupeCache repliedAt, }, }); diff --git a/extensions/slack/src/sent-thread-cache.test.ts b/extensions/slack/src/sent-thread-cache.test.ts index 9c3fe8e4441d..1291ce71fd37 100644 --- a/extensions/slack/src/sent-thread-cache.test.ts +++ b/extensions/slack/src/sent-thread-cache.test.ts @@ -93,9 +93,14 @@ describe("slack sent-thread-cache", () => { expect(hasSlackThreadParticipation("A1", "C123", "1700000000.005000")).toBe(true); }); - it("writes and reads persistent thread participation when runtime state is available", async () => { + it("restores persistent thread participation without extending its original expiry", async () => { + const repliedAt = 1_711_406_400_000; + const ttlMs = 24 * 60 * 60 * 1000; + const now = vi.spyOn(Date, "now").mockReturnValue(repliedAt); const register = vi.fn().mockResolvedValue(undefined); - const lookup = vi.fn().mockResolvedValue({ repliedAt: 123 }); + const lookup = vi + .fn() + .mockImplementation(async () => (Date.now() < repliedAt + ttlMs ? { repliedAt } : undefined)); const openKeyedStore = vi.fn(() => ({ register, lookup, @@ -109,14 +114,14 @@ describe("slack sent-thread-cache", () => { logging: { getChildLogger: () => ({ warn: vi.fn() }) }, } as never); - vi.spyOn(Date, "now").mockReturnValue(1_711_406_400_000); recordSlackThreadParticipation("A1", "C123", "1700000000.000002"); await vi.waitFor(() => expect(register).toHaveBeenCalledTimes(1)); expect(register).toHaveBeenCalledWith("A1:C123:1700000000.000002", { - repliedAt: 1_711_406_400_000, + repliedAt, }); + now.mockReturnValue(repliedAt + ttlMs - 1000); clearSlackThreadParticipationCache(); await expect( hasSlackThreadParticipationWithPersistence({ @@ -137,6 +142,16 @@ describe("slack sent-thread-cache", () => { }), ).resolves.toBe(true); expect(lookup).not.toHaveBeenCalled(); + + now.mockReturnValue(repliedAt + ttlMs + 1000); + await expect( + hasSlackThreadParticipationWithPersistence({ + accountId: "A1", + channelId: "C123", + threadTs: "1700000000.000002", + }), + ).resolves.toBe(false); + expect(lookup).toHaveBeenCalledWith("A1:C123:1700000000.000002"); }); it("falls back to in-memory thread participation when persistent state cannot open", async () => { diff --git a/extensions/slack/src/sent-thread-cache.ts b/extensions/slack/src/sent-thread-cache.ts index b11178968fbb..a06c27ea609d 100644 --- a/extensions/slack/src/sent-thread-cache.ts +++ b/extensions/slack/src/sent-thread-cache.ts @@ -37,6 +37,8 @@ const threadParticipation = createPersistentDedupeCache repliedAt, }, }); From 3c085c18728bcc0c5d71412546f7b3b54eed6744 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 04:17:03 -0700 Subject: [PATCH 28/57] test(gateway): isolate session patch provider policy (#118633) Co-authored-by: Peter Steinberger --- src/gateway/sessions-patch.test.ts | 37 +++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/gateway/sessions-patch.test.ts b/src/gateway/sessions-patch.test.ts index 3ed958b6254c..2e6449a53b9c 100644 --- a/src/gateway/sessions-patch.test.ts +++ b/src/gateway/sessions-patch.test.ts @@ -1,6 +1,6 @@ // Session patch tests cover model/provider edits, subagent patching, provider // aliases, model catalog validation, and rejected invalid patch payloads. -import { afterEach, describe, expect, test, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import type { SessionCreatedActor } from "../../packages/gateway-protocol/src/index.js"; import { resetProviderAuthAliasMapCacheForTest } from "../agents/provider-auth-aliases.test-support.js"; import type { OpenClawConfig } from "../config/config.js"; @@ -16,11 +16,20 @@ import { applySessionsPatchToStore } from "./sessions-patch.js"; const acpSessionMetaMocks = vi.hoisted(() => ({ readAcpSessionMetaForEntry: vi.fn(), })); +const providerThinkingMocks = vi.hoisted(() => ({ + resolveProviderThinkingProfile: + vi.fn(), +})); vi.mock("../acp/runtime/session-meta.js", () => ({ readAcpSessionMetaForEntry: acpSessionMetaMocks.readAcpSessionMetaForEntry, })); +// This suite owns patch projection; provider policy artifacts have dedicated contract coverage. +vi.mock("../plugins/provider-thinking.js", () => ({ + resolveProviderThinkingProfile: providerThinkingMocks.resolveProviderThinkingProfile, +})); + const SUBAGENT_MODEL = "synthetic/hf:moonshotai/Kimi-K2.7-Code"; const KIMI_SUBAGENT_KEY = "agent:kimi:subagent:child"; const MAIN_SESSION_KEY = "agent:main:main"; @@ -256,6 +265,32 @@ function createAllowlistedAnthropicModelCfg(): OpenClawConfig { } describe("gateway sessions patch", () => { + beforeEach(() => { + providerThinkingMocks.resolveProviderThinkingProfile.mockReset(); + providerThinkingMocks.resolveProviderThinkingProfile.mockImplementation( + ({ provider, context }) => { + if (provider !== "openai") { + return undefined; + } + if (context.modelId === "gpt-5.5") { + return { + levels: (["off", "minimal", "low", "medium", "high", "xhigh"] as const).map((id) => ({ + id, + })), + }; + } + if (context.modelId === "gpt-5.6-luna") { + const levels = + context.agentRuntime === "openclaw" + ? (["off", "minimal", "low", "medium", "high", "max", "ultra"] as const) + : (["off", "minimal", "low", "medium", "high", "max"] as const); + return { levels: levels.map((id) => ({ id })) }; + } + return undefined; + }, + ); + }); + afterEach(() => { acpSessionMetaMocks.readAcpSessionMetaForEntry.mockReset(); resetProviderAuthAliasMapCacheForTest(); From 1af8bdd9ee221177ce512442e219bc072e5ca6a1 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 04:18:57 -0700 Subject: [PATCH 29/57] test(system-agent): reuse setup fixtures (#118636) Co-authored-by: Peter Steinberger --- src/system-agent/assistant.configured.test.ts | 26 ++++++------------- src/system-agent/operations.setup.test.ts | 12 ++++++++- src/system-agent/setup-inference.test.ts | 13 ++++++++++ src/system-agent/verified-inference.test.ts | 4 +++ 4 files changed, 36 insertions(+), 19 deletions(-) diff --git a/src/system-agent/assistant.configured.test.ts b/src/system-agent/assistant.configured.test.ts index 52d2875611e8..65dafe7c07c7 100644 --- a/src/system-agent/assistant.configured.test.ts +++ b/src/system-agent/assistant.configured.test.ts @@ -1,6 +1,5 @@ // Configured OpenClaw assistant tests cover route-owned, tool-free planning. import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { testing as cliBackendsTesting } from "../agents/cli-backends.test-support.js"; import type { RunCliAgentParams } from "../agents/cli-runner/types.js"; import { fingerprintResolvedProviderAuth } from "../agents/execution-auth-binding.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; @@ -8,7 +7,10 @@ import { planSystemAgentCommandWithConfiguredModel } from "./assistant.js"; import { SystemAgentInferenceUnavailableError } from "./inference-error.js"; import { resolveSystemAgentConfiguredRouteFromConfig } from "./inference-route.js"; import type { SystemAgentOverview } from "./overview.js"; -import { createSystemAgentVerifiedInferenceTestFixture } from "./system-agent.test-helpers.js"; +import { + createSystemAgentVerifiedInferenceTestFixture, + installSystemAgentClaudeCliBackendTestFixture, +} from "./system-agent.test-helpers.js"; import { createSystemAgentVerifiedInferenceBinding, type SystemAgentVerifiedInferenceBinding, @@ -55,26 +57,14 @@ function useFastVerifiedInference( return binding; } +let restoreCliBackendFixture: (() => void) | undefined; + beforeAll(() => { - cliBackendsTesting.setDepsForTest({ - resolvePluginSetupCliBackend: () => undefined, - resolvePluginSetupRegistry: () => ({ cliBackends: [] }) as never, - resolveRuntimeCliBackends: () => [ - { - id: "claude-cli", - pluginId: "anthropic", - modelProvider: "anthropic", - bundleMcp: true, - bundleMcpMode: "claude-config-file", - config: { command: "claude" }, - sideQuestionToolMode: "disabled", - }, - ], - }); + restoreCliBackendFixture = installSystemAgentClaudeCliBackendTestFixture(); }); afterAll(() => { - cliBackendsTesting.resetDepsForTest(); + restoreCliBackendFixture?.(); }); function overview(defaultModel?: string): SystemAgentOverview { diff --git a/src/system-agent/operations.setup.test.ts b/src/system-agent/operations.setup.test.ts index b47147c3a40d..329b0a2264e5 100644 --- a/src/system-agent/operations.setup.test.ts +++ b/src/system-agent/operations.setup.test.ts @@ -1,7 +1,7 @@ // OpenClaw operation tests cover rescue operation planning and execution. import fs from "node:fs/promises"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import { listAgentEntries } from "../agents/agent-scope-config.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; @@ -13,6 +13,7 @@ import { createSystemAgentTestRuntime, expectSystemAgentAuditRecord as expectAuditRecord, expectTestRecordFields as expectRecordFields, + installSystemAgentClaudeCliBackendTestFixture, readLastSystemAgentAuditEntry as readLastAuditEntry, requireTestRecord as requireRecord, } from "./system-agent.test-helpers.js"; @@ -146,6 +147,15 @@ vi.mock("../state/local-onboarding-state.js", () => ({ })); const opTempDirs = useAutoCleanupTempDirTracker(afterEach); +let restoreCliBackendFixture: (() => void) | undefined; + +beforeAll(() => { + restoreCliBackendFixture = installSystemAgentClaudeCliBackendTestFixture(); +}); + +afterAll(() => { + restoreCliBackendFixture?.(); +}); describe("parseSystemAgentOperation", () => { let stateDirSnapshot: ReturnType | undefined; diff --git a/src/system-agent/setup-inference.test.ts b/src/system-agent/setup-inference.test.ts index 6c319ea0ee42..0d0f8e556b96 100644 --- a/src/system-agent/setup-inference.test.ts +++ b/src/system-agent/setup-inference.test.ts @@ -133,11 +133,16 @@ const suiteTempRootTracker = createSuiteTempRootTracker({ prefix: "setup-inference-test-", }); let pluginMetadataSnapshot: SystemAgentPluginMetadataTestSnapshot | undefined; +let preparedPluginMetadataSnapshot: ReturnType | undefined; beforeAll(async () => { pluginMetadataSnapshot = installSystemAgentPluginMetadataTestSnapshot( materializedMainRuntimeConfig, ); + preparedPluginMetadataSnapshot = resolvePluginMetadataSnapshot({ + config: materializedMainRuntimeConfig, + env: process.env, + }); cliBackendsTesting.setDepsForTest({ resolvePluginSetupCliBackend: () => undefined, resolvePluginSetupRegistry: () => ({ cliBackends: [] }) as never, @@ -461,6 +466,13 @@ function mockCodexRuntimeInstall(installRecord?: PluginInstallRecord) { })) as never; } +function requirePreparedPluginMetadataSnapshot() { + if (!preparedPluginMetadataSnapshot) { + throw new Error("setup inference plugin metadata fixture was not initialized"); + } + return preparedPluginMetadataSnapshot; +} + function activateCodexSetup(params: Omit) { return activateSetupInference({ kind: "codex-cli", @@ -469,6 +481,7 @@ function activateCodexSetup(params: Omit {}) as never, + resolvePluginMetadataSnapshot: requirePreparedPluginMetadataSnapshot, ...params.deps, }, }); diff --git a/src/system-agent/verified-inference.test.ts b/src/system-agent/verified-inference.test.ts index 50f9b7223424..4ee13ec00f1d 100644 --- a/src/system-agent/verified-inference.test.ts +++ b/src/system-agent/verified-inference.test.ts @@ -14,6 +14,7 @@ import type { PluginOrigin } from "../plugins/types.js"; import { resolveSystemAgentConfiguredRouteFromConfig } from "./inference-route.js"; import { resolvePersistentApplyInference } from "./setup-inference.js"; import { + installSystemAgentClaudeCliBackendTestFixture, installSystemAgentPluginMetadataTestSnapshot, type SystemAgentPluginMetadataTestSnapshot, } from "./system-agent.test-helpers.js"; @@ -81,12 +82,15 @@ const profile = { const runtime = { log: () => {}, error: () => {}, exit: () => {} } as never; let pluginMetadataSnapshot: SystemAgentPluginMetadataTestSnapshot | undefined; +let restoreCliBackendFixture: (() => void) | undefined; beforeAll(() => { pluginMetadataSnapshot = installSystemAgentPluginMetadataTestSnapshot(config()); + restoreCliBackendFixture = installSystemAgentClaudeCliBackendTestFixture(); }); afterAll(() => { + restoreCliBackendFixture?.(); pluginMetadataSnapshot?.restore(); }); From 629bf6f2d3172b8e1d203fc00d25217051e443fc Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 04:24:53 -0700 Subject: [PATCH 30/57] fix(otel): redact invalid collector URLs and suppress implicit exporters (#118635) --- .../diagnostics-otel/src/service-exporter.ts | 43 ++- .../src/service.otlp-export.test.ts | 250 +++++++++++++++++- .../diagnostics-otel/src/service.test.ts | 123 +++++++++ extensions/diagnostics-otel/src/service.ts | 63 +++-- 4 files changed, 434 insertions(+), 45 deletions(-) diff --git a/extensions/diagnostics-otel/src/service-exporter.ts b/extensions/diagnostics-otel/src/service-exporter.ts index fa48ea8199d7..775b4315da29 100644 --- a/extensions/diagnostics-otel/src/service-exporter.ts +++ b/extensions/diagnostics-otel/src/service-exporter.ts @@ -28,14 +28,10 @@ function resolveOtelUrl(endpoint: string | undefined, path: string): string | un return endpoint; } if (/[?#]/u.test(endpoint)) { - try { - const url = new URL(endpoint); - const basePath = url.pathname.replace(/\/+$/u, ""); - url.pathname = `${basePath}/${path}`; - return url.toString(); - } catch { - // Fall back to the historical concatenation path for non-URL test doubles. - } + const url = new URL(endpoint); + const basePath = url.pathname.replace(/\/+$/u, ""); + url.pathname = `${basePath}/${path}`; + return url.toString(); } return `${endpoint}/${path}`; } @@ -43,13 +39,36 @@ function resolveOtelUrl(endpoint: string | undefined, path: string): string | un export function resolveSignalOtelUrl(params: { signalEndpoint?: string; signalEnvEndpoint?: string; + sharedEnvEndpoint?: string; endpoint?: string; path: string; }): string | undefined { - return resolveOtelUrl( - normalizeEndpoint(params.signalEndpoint ?? params.signalEnvEndpoint) ?? params.endpoint, - params.path, - ); + const endpoint = + normalizeEndpoint(params.signalEndpoint ?? params.signalEnvEndpoint) ?? params.endpoint; + // OTLP parses nonblank env values verbatim even when explicit config takes precedence. + const signalEnvEndpoint = params.signalEnvEndpoint?.trim() ? params.signalEnvEndpoint : undefined; + const sharedEnvEndpoint = params.sharedEnvEndpoint?.trim() ? params.sharedEnvEndpoint : undefined; + const consumedSharedEnvEndpoint = signalEnvEndpoint ? undefined : sharedEnvEndpoint; + const appendedSharedEnvEndpoint = consumedSharedEnvEndpoint + ? `${consumedSharedEnvEndpoint}${consumedSharedEnvEndpoint.endsWith("/") ? "" : "/"}${params.path}` + : undefined; + const resolvedEndpoint = + endpoint && URL.canParse(endpoint) ? resolveOtelUrl(endpoint, params.path) : endpoint; + + for (const candidate of [ + endpoint, + signalEnvEndpoint ?? sharedEnvEndpoint, + appendedSharedEnvEndpoint, + resolvedEndpoint, + ]) { + if (candidate && !URL.canParse(candidate)) { + throw new Error( + "Configured OpenTelemetry collector endpoint is invalid; check the collector URL", + ); + } + } + + return resolvedEndpoint; } function readOtelEnvFile(params: { diff --git a/extensions/diagnostics-otel/src/service.otlp-export.test.ts b/extensions/diagnostics-otel/src/service.otlp-export.test.ts index 5790b7e67b01..0fc82b92e897 100644 --- a/extensions/diagnostics-otel/src/service.otlp-export.test.ts +++ b/extensions/diagnostics-otel/src/service.otlp-export.test.ts @@ -5,11 +5,11 @@ // test feeds in, collapsing the diagnostic and OTel id spaces into one value. That hides // a parent lookup keyed by one id space and queried with the other. // -// It drives the service through the OPENCLAW_OTEL_PRELOADED seam so the plugin uses this -// file's tracer provider instead of starting its own NodeSDK. trace.disable() in teardown -// then fully releases the global API slot; a NodeSDK cannot be unregistered, and the -// leftover dead provider would make any later real-SDK test export nothing. -import { trace } from "@opentelemetry/api"; +// Trace cases use the OPENCLAW_OTEL_PRELOADED seam to retain this file's tracer provider. +// Collector-boundary cases start the real NodeSDK, so teardown restores every global SDK +// registration; otherwise a shutdown provider would poison later real-SDK cases. +import { context, diag, DiagLogLevel, metrics, propagation, trace } from "@opentelemetry/api"; +import { logs } from "@opentelemetry/api-logs"; import { BasicTracerProvider, InMemorySpanExporter, @@ -23,17 +23,58 @@ import { resetDiagnosticEventsForTest, waitForDiagnosticEventsDrained, } from "openclaw/plugin-sdk/diagnostic-runtime"; -import { afterEach, beforeEach, expect, test } from "vitest"; -import { startOtelService, stopStartedOtelServices } from "./service.test-helpers.js"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { createDiagnosticsOtelService } from "./service.js"; +import { + createOtelContext, + startOtelService, + stopStartedOtelServices, +} from "./service.test-helpers.js"; const PRELOAD_ENV = "OPENCLAW_OTEL_PRELOADED"; +const ENDPOINT_ENV_KEYS = [ + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", + "OTEL_LOG_LEVEL", +] as const; +const OTEL_GLOBAL_API_KEY = Symbol.for("opentelemetry.js.api.1"); +const OTEL_GLOBAL_LOGS_KEY = Symbol.for("io.opentelemetry.js.api.logs"); + +type OtelGlobalRegistrations = { + context?: Parameters[0]; + diag?: Parameters[0]; + metrics?: Parameters[0]; + propagation?: Parameters[0]; + trace?: Parameters[0]; +}; let exporter: InMemorySpanExporter; let provider: BasicTracerProvider; let originalPreloaded: string | undefined; +let originalEndpointEnv: Record<(typeof ENDPOINT_ENV_KEYS)[number], string | undefined>; +let originalOtelGlobals: OtelGlobalRegistrations; +let originalLogsProvider: ReturnType | undefined; + +function registeredOtelGlobals(): OtelGlobalRegistrations | undefined { + return (globalThis as unknown as Record)[ + OTEL_GLOBAL_API_KEY + ]; +} beforeEach(() => { originalPreloaded = process.env[PRELOAD_ENV]; + originalEndpointEnv = Object.fromEntries( + ENDPOINT_ENV_KEYS.map((key) => [key, process.env[key]]), + ) as Record<(typeof ENDPOINT_ENV_KEYS)[number], string | undefined>; + for (const key of ENDPOINT_ENV_KEYS) { + delete process.env[key]; + } + originalOtelGlobals = { ...registeredOtelGlobals() }; + originalLogsProvider = Object.hasOwn(globalThis, OTEL_GLOBAL_LOGS_KEY) + ? logs.getLoggerProvider() + : undefined; process.env[PRELOAD_ENV] = "1"; exporter = new InMemorySpanExporter(); provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); @@ -43,13 +84,58 @@ beforeEach(() => { afterEach(async () => { await stopStartedOtelServices(); await provider.shutdown(); - trace.disable(); + const currentGlobals = registeredOtelGlobals(); + if (currentGlobals?.context !== originalOtelGlobals.context) { + context.disable(); + if (originalOtelGlobals.context) { + context.setGlobalContextManager(originalOtelGlobals.context); + } + } + if (currentGlobals?.propagation !== originalOtelGlobals.propagation) { + propagation.disable(); + if (originalOtelGlobals.propagation) { + propagation.setGlobalPropagator(originalOtelGlobals.propagation); + } + } + if (currentGlobals?.metrics !== originalOtelGlobals.metrics) { + metrics.disable(); + if (originalOtelGlobals.metrics) { + metrics.setGlobalMeterProvider(originalOtelGlobals.metrics); + } + } + if (currentGlobals?.trace !== originalOtelGlobals.trace) { + trace.disable(); + if (originalOtelGlobals.trace) { + trace.setGlobalTracerProvider(originalOtelGlobals.trace); + } + } + if (Object.hasOwn(globalThis, OTEL_GLOBAL_LOGS_KEY) || originalLogsProvider) { + logs.disable(); + if (originalLogsProvider) { + logs.setGlobalLoggerProvider(originalLogsProvider); + } + } exporter.reset(); if (originalPreloaded === undefined) { delete process.env[PRELOAD_ENV]; } else { process.env[PRELOAD_ENV] = originalPreloaded; } + for (const key of ENDPOINT_ENV_KEYS) { + const value = originalEndpointEnv[key]; + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + diag.disable(); + if (originalOtelGlobals.diag) { + diag.setLogger(originalOtelGlobals.diag, { + logLevel: DiagLogLevel.ALL, + suppressOverrideMessage: true, + }); + } resetDiagnosticEventsForTest(); }); @@ -60,6 +146,24 @@ function spanNamed(spans: ReadableSpan[], name: string) { return spans.find((span) => span.name === name); } +function captureOtelDiagnostics(): string[] { + const messages: string[] = []; + const capture = (...args: unknown[]) => { + messages.push(args.map((value) => String(value)).join(" ")); + }; + diag.setLogger( + { + debug: () => {}, + error: capture, + info: () => {}, + verbose: () => {}, + warn: capture, + }, + { logLevel: DiagLogLevel.ALL, suppressOverrideMessage: true }, + ); + return messages; +} + // Covers all three completeTrackedLifecycleSpan owners: run.completed, // harness.run.completed, and message.processed. The mocked suite cannot tell the two id // spaces apart, so a regression at any one of them is only visible here. @@ -237,3 +341,133 @@ test("leaves exec spans parentless rather than naming a span nobody exported", a expect(execSpan).toBeDefined(); expect(execSpan?.parentSpanContext).toBeUndefined(); }, 30_000); + +const OTEL_ENDPOINT_SIGNAL_CASES = [ + { + signal: "traces", + envKey: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + configKey: "tracesEndpoint", + flags: { traces: true, metrics: false, logs: false }, + }, + { + signal: "metrics", + envKey: "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", + configKey: "metricsEndpoint", + flags: { traces: false, metrics: true, logs: false }, + }, + { + signal: "logs", + envKey: "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", + configKey: "logsEndpoint", + flags: { traces: false, metrics: false, logs: true }, + }, +] as const; + +const OTEL_ENDPOINT_SECURITY_CASES = OTEL_ENDPOINT_SIGNAL_CASES.flatMap((signal) => + ( + [ + "shared configuration", + "signal configuration", + "signal environment", + "shared environment", + "Unicode-prefixed signal environment", + "Unicode-prefixed shared environment", + "path-concatenated shared environment", + "path-concatenated shared configuration", + "path-concatenated signal configuration", + ] as const + ).map((source) => Object.assign({ source }, signal)), +); + +test.each(OTEL_ENDPOINT_SECURITY_CASES)( + "rejects malformed $signal collector $source before the real SDK can expose credentials", + async ({ signal, envKey, configKey, flags, source }) => { + process.env[PRELOAD_ENV] = "0"; + const credential = `qa-otel-${signal}-endpoint-password-sentinel`; + const malformedEndpoint = source.startsWith("Unicode-prefixed") + ? `\u00a0https://operator:${credential}@collector.example.com/otlp` + : source === "path-concatenated shared environment" + ? `https://operator:${credential}@collector.example.com: ` + : source.startsWith("path-concatenated") + ? `https://operator:${credential}@collector.example.com /` + : `https://operator:${credential}@[`; + const configuredEndpoint = source.endsWith("shared configuration") + ? malformedEndpoint + : "https://collector.example.com/otlp"; + if (source.endsWith("signal environment")) { + process.env[envKey] = malformedEndpoint; + } else if (source.endsWith("shared environment")) { + process.env.OTEL_EXPORTER_OTLP_ENDPOINT = malformedEndpoint; + } + + const diagnostics = captureOtelDiagnostics(); + const ctx = createOtelContext(configuredEndpoint, flags); + if (source.endsWith("signal configuration") || source.endsWith("signal environment")) { + ctx.config.diagnostics!.otel![configKey] = source.endsWith("signal configuration") + ? malformedEndpoint + : "https://signal.example.com/otlp"; + } + ctx.internalDiagnostics!.emit = () => {}; + const service = createDiagnosticsOtelService(); + let failure: unknown; + try { + await service.start(ctx); + } catch (error) { + failure = error; + } finally { + await service.stop?.(ctx); + } + + expect(diagnostics.join("\n")).not.toContain(credential); + expect(failure).toBeInstanceOf(Error); + const startupError = failure as Error; + expect(startupError.message).toBe( + "Configured OpenTelemetry collector endpoint is invalid; check the collector URL", + ); + expect(startupError.stack).not.toContain(credential); + expect(startupError).not.toHaveProperty("cause"); + expect(JSON.stringify(vi.mocked(ctx.logger.error).mock.calls)).not.toContain(credential); + expect(JSON.stringify(vi.mocked(ctx.logger.warn).mock.calls)).not.toContain(credential); + }, +); + +test.each([ + { + disabledSignal: "metrics", + envKey: "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", + flags: { traces: true, metrics: false, logs: false }, + }, + { + disabledSignal: "traces", + envKey: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + flags: { traces: false, metrics: true, logs: false }, + }, + { + disabledSignal: "logs", + envKey: "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", + flags: { traces: true, metrics: false, logs: false }, + }, + { + disabledSignal: "stdout-only logs", + envKey: "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", + flags: { traces: true, metrics: false, logs: true, logsExporter: "stdout" }, + }, +] as const)( + "does not auto-create an undeclared $disabledSignal OTLP exporter", + async ({ disabledSignal, envKey, flags }) => { + process.env[PRELOAD_ENV] = "0"; + const credential = `qa-otel-${disabledSignal.replaceAll(" ", "-")}-disabled-password`; + process.env[envKey] = `https://operator:${credential}@[`; + const diagnostics = captureOtelDiagnostics(); + const ctx = createOtelContext("https://collector.example.com/otlp", flags); + ctx.internalDiagnostics!.emit = () => {}; + const service = createDiagnosticsOtelService(); + + try { + await service.start(ctx); + expect(diagnostics.join("\n")).not.toContain(credential); + } finally { + await service.stop?.(ctx); + } + }, +); diff --git a/extensions/diagnostics-otel/src/service.test.ts b/extensions/diagnostics-otel/src/service.test.ts index 65f279eb62df..2ebe1ce9f56d 100644 --- a/extensions/diagnostics-otel/src/service.test.ts +++ b/extensions/diagnostics-otel/src/service.test.ts @@ -57,6 +57,7 @@ const telemetryState = vi.hoisted(() => { const sdkStart = vi.hoisted(() => vi.fn().mockResolvedValue(undefined)); const sdkShutdown = vi.hoisted(() => vi.fn().mockResolvedValue(undefined)); +const sdkCtor = vi.hoisted(() => vi.fn()); const logEmit = vi.hoisted(() => vi.fn()); const logShutdown = vi.hoisted(() => vi.fn().mockResolvedValue(undefined)); const traceExporterCtor = vi.hoisted(() => vi.fn()); @@ -107,6 +108,10 @@ vi.mock("@opentelemetry/api", () => ({ vi.mock("@opentelemetry/sdk-node", () => ({ NodeSDK: class { + constructor(options?: unknown) { + sdkCtor(options); + } + start = sdkStart; shutdown = sdkShutdown; }, @@ -220,7 +225,9 @@ const LATE_CHILD_ELAPSED_MS = 30 * 60_000 + 1_000; const PROTO_KEY = "__proto__"; const MAX_TEST_OTEL_CONTENT_ATTRIBUTE_CHARS = 128 * 1024; const OTEL_TRUNCATED_SUFFIX_MAX_CHARS = 20; +const OTEL_TEST_USERINFO = ["operator", "example-fixture"].join(":"); const ORIGINAL_OPENCLAW_OTEL_PRELOADED = process.env.OPENCLAW_OTEL_PRELOADED; +const ORIGINAL_OTEL_EXPORTER_OTLP_ENDPOINT = process.env.OTEL_EXPORTER_OTLP_ENDPOINT; const ORIGINAL_OTEL_EXPORTER_OTLP_PROTOCOL = process.env.OTEL_EXPORTER_OTLP_PROTOCOL; const ORIGINAL_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT; const ORIGINAL_OTEL_EXPORTER_OTLP_METRICS_ENDPOINT = @@ -584,6 +591,7 @@ describe("diagnostics-otel service", () => { telemetryState.tracer.setSpanContext.mockClear(); telemetryState.meter.createCounter.mockClear(); telemetryState.meter.createHistogram.mockClear(); + sdkCtor.mockClear(); sdkStart.mockClear(); sdkShutdown.mockClear(); logEmit.mockReset(); @@ -597,6 +605,7 @@ describe("diagnostics-otel service", () => { createNodeProxyAgentMock.mockReturnValue(undefined); unhandledRejectionHandlerState.reset(); unhandledRejectionHandlerState.register.mockClear(); + delete process.env.OTEL_EXPORTER_OTLP_ENDPOINT; delete process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT; delete process.env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT; delete process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT; @@ -613,6 +622,11 @@ describe("diagnostics-otel service", () => { } else { process.env.OPENCLAW_OTEL_PRELOADED = ORIGINAL_OPENCLAW_OTEL_PRELOADED; } + if (ORIGINAL_OTEL_EXPORTER_OTLP_ENDPOINT === undefined) { + delete process.env.OTEL_EXPORTER_OTLP_ENDPOINT; + } else { + process.env.OTEL_EXPORTER_OTLP_ENDPOINT = ORIGINAL_OTEL_EXPORTER_OTLP_ENDPOINT; + } if (ORIGINAL_OTEL_EXPORTER_OTLP_PROTOCOL === undefined) { delete process.env.OTEL_EXPORTER_OTLP_PROTOCOL; } else { @@ -1687,6 +1701,16 @@ describe("diagnostics-otel service", () => { "https://collector.example.com/otlp#tenant-a", "https://collector.example.com/otlp/v1/traces#tenant-a", ], + [ + "preserves valid collector credentials and query parameters", + `https://${OTEL_TEST_USERINFO}@collector.example.com/otlp?tenant=red`, + `https://${OTEL_TEST_USERINFO}@collector.example.com/otlp/v1/traces?tenant=red`, + ], + [ + "preserves parseable non-HTTP collector URL schemes", + "custom+otel://collector.example.com/otlp", + "custom+otel://collector.example.com/otlp/v1/traces", + ], [ "keeps signal-qualified endpoint unchanged when signal path casing differs", "https://collector.example.com/v1/Traces", @@ -1763,6 +1787,105 @@ describe("diagnostics-otel service", () => { expect(logOptions.url).toBe("https://log-env.example.com/otlp/v1/logs"); }); + test("ignores malformed shared OTLP env when valid signal endpoints shadow it", async () => { + process.env.OTEL_EXPORTER_OTLP_ENDPOINT = "https://operator:qa-ignored-shared-password@["; + process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = "https://trace-env.example.com/v1/traces"; + process.env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT = "https://metric-env.example.com/v1/metrics"; + process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT = "https://log-env.example.com/v1/logs"; + + await startOtelService({ traces: true, metrics: true, logs: true }); + + expect(firstExporterOptions(traceExporterCtor).url).toBe( + "https://trace-env.example.com/v1/traces", + ); + expect(firstExporterOptions(metricExporterCtor).url).toBe( + "https://metric-env.example.com/v1/metrics", + ); + expect(firstExporterOptions(logExporterCtor).url).toBe("https://log-env.example.com/v1/logs"); + }); + + test("treats whitespace-only OTLP environment endpoints as unset", async () => { + process.env.OTEL_EXPORTER_OTLP_ENDPOINT = " \u00a0 "; + process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = " \t "; + process.env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT = "\u2000"; + process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT = "\ufeff"; + + await startOtelService({ traces: true, metrics: true, logs: true }); + + expect(firstExporterOptions(traceExporterCtor).url).toBe(`${OTEL_TEST_ENDPOINT}/v1/traces`); + expect(firstExporterOptions(metricExporterCtor).url).toBe(`${OTEL_TEST_ENDPOINT}/v1/metrics`); + expect(firstExporterOptions(logExporterCtor).url).toBe(`${OTEL_TEST_ENDPOINT}/v1/logs`); + }); + + test.each([ + { + enabledSignal: "traces", + flags: { traces: true, metrics: false, logs: false }, + metricReaderCount: 0, + tracesDisabled: false, + }, + { + enabledSignal: "metrics", + flags: { traces: false, metrics: true, logs: false }, + metricReaderCount: 1, + tracesDisabled: true, + }, + { + enabledSignal: "traces and metrics", + flags: { traces: true, metrics: true, logs: false }, + metricReaderCount: 1, + tracesDisabled: false, + }, + ] as const)( + "keeps NodeSDK exporter ownership explicit for $enabledSignal", + async ({ flags, metricReaderCount, tracesDisabled }) => { + await startOtelService(flags); + + const options = mockCallArg(sdkCtor, 0) as { + logRecordProcessors?: unknown[]; + metricReaders?: unknown[]; + spanProcessors?: unknown[]; + }; + expect(options.logRecordProcessors).toEqual([]); + expect(options.metricReaders).toHaveLength(metricReaderCount); + expect(options).not.toHaveProperty("metricReader"); + if (tracesDisabled) { + expect(options.spanProcessors).toEqual([]); + } + }, + ); + + test("ignores malformed collector endpoints for preloaded traces and metrics", async () => { + process.env.OPENCLAW_OTEL_PRELOADED = "1"; + process.env.OTEL_EXPORTER_OTLP_ENDPOINT = "https://operator:qa-preloaded-shared-password@["; + process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = + "https://operator:qa-preloaded-trace-password@["; + process.env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT = + "https://operator:qa-preloaded-metric-password@["; + + await startOtelService({ traces: true, metrics: true, logs: false }); + + expect(sdkCtor).not.toHaveBeenCalled(); + expect(traceExporterCtor).not.toHaveBeenCalled(); + expect(metricExporterCtor).not.toHaveBeenCalled(); + }); + + test("ignores malformed collector endpoints for stdout-only diagnostics", async () => { + process.env.OTEL_EXPORTER_OTLP_ENDPOINT = "https://operator:qa-stdout-shared-password@["; + process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT = "https://operator:qa-stdout-log-password@["; + + await startOtelService({ + endpoint: "https://operator:qa-stdout-config-password@[", + traces: false, + metrics: false, + logs: true, + logsExporter: "stdout", + }); + + expect(sdkCtor).not.toHaveBeenCalled(); + expect(logExporterCtor).not.toHaveBeenCalled(); + }); + test("passes env proxy agents to OTLP HTTP exporters", async () => { createNodeProxyAgentMock.mockReturnValue(nodeProxyAgent); diff --git a/extensions/diagnostics-otel/src/service.ts b/extensions/diagnostics-otel/src/service.ts index d0808565ed14..3e6ebac4ac0d 100644 --- a/extensions/diagnostics-otel/src/service.ts +++ b/extensions/diagnostics-otel/src/service.ts @@ -141,9 +141,8 @@ export function createDiagnosticsOtelService(): OpenClawPluginService { return; } - const endpoint = normalizeEndpoint( - otel.endpoint ?? process.env[OTEL_EXPORTER_OTLP_ENDPOINT_ENV], - ); + const sharedEnvEndpoint = process.env[OTEL_EXPORTER_OTLP_ENDPOINT_ENV]; + const endpoint = normalizeEndpoint(otel.endpoint ?? sharedEnvEndpoint); const headers = otel.headers ?? undefined; const serviceName = otel.serviceName?.trim() || process.env.OTEL_SERVICE_NAME || DEFAULT_SERVICE_NAME; @@ -155,32 +154,41 @@ export function createDiagnosticsOtelService(): OpenClawPluginService { [ATTR_SERVICE_NAME]: serviceName, }); - const logUrl = resolveSignalOtelUrl({ - signalEndpoint: otel.logsEndpoint, - signalEnvEndpoint: process.env[OTEL_EXPORTER_OTLP_LOGS_ENDPOINT_ENV], - endpoint, - path: "v1/logs", - }); + const logUrl = logsToOtlp + ? resolveSignalOtelUrl({ + signalEndpoint: otel.logsEndpoint, + signalEnvEndpoint: process.env[OTEL_EXPORTER_OTLP_LOGS_ENDPOINT_ENV], + sharedEnvEndpoint, + endpoint, + path: "v1/logs", + }) + : undefined; if (!sdkPreloaded && (tracesEnabled || metricsEnabled)) { - const traceUrl = resolveSignalOtelUrl({ - signalEndpoint: otel.tracesEndpoint, - signalEnvEndpoint: process.env[OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_ENV], - endpoint, - path: "v1/traces", - }); - const metricUrl = resolveSignalOtelUrl({ - signalEndpoint: otel.metricsEndpoint, - signalEnvEndpoint: process.env[OTEL_EXPORTER_OTLP_METRICS_ENDPOINT_ENV], - endpoint, - path: "v1/metrics", - }); + const traceUrl = tracesEnabled + ? resolveSignalOtelUrl({ + signalEndpoint: otel.tracesEndpoint, + signalEnvEndpoint: process.env[OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_ENV], + sharedEnvEndpoint, + endpoint, + path: "v1/traces", + }) + : undefined; + const metricUrl = metricsEnabled + ? resolveSignalOtelUrl({ + signalEndpoint: otel.metricsEndpoint, + signalEnvEndpoint: process.env[OTEL_EXPORTER_OTLP_METRICS_ENDPOINT_ENV], + sharedEnvEndpoint, + endpoint, + path: "v1/metrics", + }) + : undefined; const traceHttpAgentOptions = resolveOtelHttpAgentOptions({ - url: tracesEnabled ? traceUrl : undefined, + url: traceUrl, signalIdentifier: "TRACES", logger: ctx.logger, }); const metricHttpAgentOptions = resolveOtelHttpAgentOptions({ - url: metricsEnabled ? metricUrl : undefined, + url: metricUrl, signalIdentifier: "METRICS", logger: ctx.logger, }); @@ -219,8 +227,13 @@ export function createDiagnosticsOtelService(): OpenClawPluginService { sdk = new NodeSDK({ resource, - ...(spanProcessors ? { spanProcessors } : traceExporter ? { traceExporter } : {}), - ...(metricReader ? { metricReader } : {}), + ...(spanProcessors + ? { spanProcessors } + : traceExporter + ? { traceExporter } + : { spanProcessors: [] }), + metricReaders: metricReader ? [metricReader] : [], + logRecordProcessors: [], ...(sampleRate !== undefined ? { sampler: new ParentBasedSampler({ From 8d1fe4f091064246a6f78e57464e784b5e7dd9e8 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 04:28:17 -0700 Subject: [PATCH 31/57] fix(pdf): reject requested pages outside the document range (#118639) --- .../document-extractor.test.ts | 18 ++++++++++++++++++ .../document-extract/document-extractor.ts | 3 +++ 2 files changed, 21 insertions(+) diff --git a/extensions/document-extract/document-extractor.test.ts b/extensions/document-extract/document-extractor.test.ts index e25169a2d28e..da6555a34c28 100644 --- a/extensions/document-extract/document-extractor.test.ts +++ b/extensions/document-extract/document-extractor.test.ts @@ -176,6 +176,24 @@ describe("PDF document extractor", () => { ); }); + it("rejects selected pages outside the PDF page count before extraction", async () => { + pdfDocument.pageCount = 1; + pdfDocument.extract.mockResolvedValueOnce({ text: "", images: [] }); + const extractor = createPdfDocumentExtractor(); + + await expect(extractor.extract(request({ pageNumbers: [2] }))).rejects.toThrow( + "No requested PDF pages exist in this 1-page document.", + ); + expect(pdfDocument.extract).not.toHaveBeenCalled(); + expect(pdfDocument.destroy).toHaveBeenCalledTimes(1); + + await expect(extractor.extract(request({ pageNumbers: [] }))).resolves.toEqual({ + text: "", + images: [], + }); + expect(pdfDocument.destroy).toHaveBeenCalledTimes(2); + }); + it("reports image fallback failures and returns extracted text", async () => { const onImageExtractionError = vi.fn(); const failure = new Error("render failed"); diff --git a/extensions/document-extract/document-extractor.ts b/extensions/document-extract/document-extractor.ts index e2cb7dc0ac30..1c65c0072a18 100644 --- a/extensions/document-extract/document-extractor.ts +++ b/extensions/document-extract/document-extractor.ts @@ -70,6 +70,9 @@ async function extractPdfContent( .filter((p) => Number.isInteger(p) && p >= 1 && p <= pdf.pageCount) .slice(0, request.maxPages) : undefined; + if (request.pageNumbers?.length && pages?.length === 0) { + throw new Error(`No requested PDF pages exist in this ${pdf.pageCount}-page document.`); + } const pageSelection = pages ? { pages } : { maxPages: request.maxPages }; const textResult = await pdf.extract({ From 2a8f2e6756d2333b39c6c7c6681241f1cb5b9436 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 04:33:46 -0700 Subject: [PATCH 32/57] fix(matrix): block plaintext attachments in encrypted rooms (#118609) * fix(matrix): block plaintext uploads to encrypted rooms * test(matrix): spy on room encryption detection safely * fix(matrix): enforce encrypted room confidentiality at outbound owner * test(matrix): keep delivery trace client aligned with send owner --- extensions/matrix/src/delivery-trace.test.ts | 1 + extensions/matrix/src/matrix/sdk.test.ts | 443 +++++++++++++++++- .../matrix/src/matrix/sdk/client-base.ts | 5 +- .../matrix/src/matrix/sdk/client-core.ts | 86 +++- .../src/matrix/sdk/crypto-facade.test.ts | 57 ++- .../matrix/src/matrix/sdk/crypto-facade.ts | 20 +- extensions/matrix/src/matrix/send.test.ts | 172 ++++++- extensions/matrix/src/matrix/send.ts | 13 +- extensions/matrix/src/matrix/send/media.ts | 69 +-- 9 files changed, 753 insertions(+), 113 deletions(-) diff --git a/extensions/matrix/src/delivery-trace.test.ts b/extensions/matrix/src/delivery-trace.test.ts index c8e515b775fd..20838a36d66e 100644 --- a/extensions/matrix/src/delivery-trace.test.ts +++ b/extensions/matrix/src/delivery-trace.test.ts @@ -84,6 +84,7 @@ function createRecordingMatrixClient(recorder: WireRecorder): Partial = { getUserId: async () => BOT_USER_ID, + prepareRoomForMessageSend: async () => "m.room.message", sendMessage: async (roomId: string, content: Record) => { const eventId = mintEventId(); // Snapshot before recording: edit flows reuse content structures, and the diff --git a/extensions/matrix/src/matrix/sdk.test.ts b/extensions/matrix/src/matrix/sdk.test.ts index 8371c8987518..2e237bd54582 100644 --- a/extensions/matrix/src/matrix/sdk.test.ts +++ b/extensions/matrix/src/matrix/sdk.test.ts @@ -6,6 +6,9 @@ import os from "node:os"; import path from "node:path"; import { CryptoEvent } from "matrix-js-sdk/lib/crypto-api/CryptoEvent.js"; import type { DecryptionFailureCode as DecryptionFailureCodeValue } from "matrix-js-sdk/lib/crypto-api/index.js"; +import { MatrixError } from "matrix-js-sdk/lib/http-api/errors.js"; +import { MsgType } from "matrix-js-sdk/lib/matrix.js"; +import { EventStatus } from "matrix-js-sdk/lib/models/event-status.js"; import { resetPluginStateStoreForTests } from "openclaw/plugin-sdk/plugin-state-test-runtime"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { installMatrixTestRuntime } from "../test-runtime.js"; @@ -247,6 +250,7 @@ type MatrixJsClientStub = { setAccountData: ReturnType; getRoomIdForAlias: ReturnType; sendMessage: ReturnType; + resendEvent: ReturnType; sendEvent: ReturnType; sendStateEvent: ReturnType; redactEvent: ReturnType; @@ -278,12 +282,18 @@ function createMatrixJsClientStub(): MatrixJsClientStub { client.getDeviceId = vi.fn(() => "DEVICE123"); client.getJoinedRooms = vi.fn(async () => ({ joined_rooms: [] })); client.getJoinedRoomMembers = vi.fn(async () => ({ joined: {} })); - client.getStateEvent = vi.fn(async () => ({})); + client.getStateEvent = vi.fn(async (_roomId: string, eventType: string) => { + if (eventType === "m.room.encryption") { + throw new MatrixError({ errcode: "M_NOT_FOUND", error: "State event not found" }, 404); + } + return {}; + }); client.getAccountData = vi.fn(() => undefined); client.getAccountDataFromServer = vi.fn(async () => null); client.setAccountData = vi.fn(async () => {}); client.getRoomIdForAlias = vi.fn(async () => ({ room_id: "!resolved:example.org" })); client.sendMessage = vi.fn(async () => ({ event_id: "$sent" })); + client.resendEvent = vi.fn(async () => ({ event_id: "$resent" })); client.sendEvent = vi.fn(async () => ({ event_id: "$sent-event" })); client.sendStateEvent = vi.fn(async () => ({ event_id: "$state" })); client.redactEvent = vi.fn(async () => ({ event_id: "$redact" })); @@ -409,6 +419,425 @@ describe("MatrixClient request hardening", () => { await expect(first.getTransactionScopeId()).resolves.toBe(await first.getTransactionScopeId()); }); + it.each([null, { hasEncryptionStateEvent: () => false }])( + "detects authoritative room encryption when the synced room cache is incomplete", + async (room) => { + matrixJsClient.getRoom.mockReturnValue(room); + matrixJsClient.getStateEvent.mockResolvedValue({ algorithm: "m.megolm.v1.aes-sha2" }); + const client = new MatrixClient("https://matrix.example.org", "token"); + + await expect(client.getMessageWireEventType("!room:example.org")).resolves.toBe( + "m.room.encrypted", + ); + expect(matrixJsClient.getStateEvent).toHaveBeenCalledWith( + "!room:example.org", + "m.room.encryption", + "", + ); + }, + ); + + it("treats an existing malformed room-encryption state as encrypted", async () => { + matrixJsClient.getRoom.mockReturnValue(null); + matrixJsClient.getStateEvent.mockResolvedValue({}); + const client = new MatrixClient("https://matrix.example.org", "token"); + + await expect(client.getMessageWireEventType("!room:example.org")).resolves.toBe( + "m.room.encrypted", + ); + }); + + it("trusts cached encrypted room state without probing the homeserver", async () => { + matrixJsClient.getRoom.mockReturnValue({ hasEncryptionStateEvent: () => true }); + matrixJsClient.getStateEvent.mockRejectedValue(new Error("state unavailable")); + const client = new MatrixClient("https://matrix.example.org", "token"); + + await expect(client.getMessageWireEventType("!room:example.org")).resolves.toBe( + "m.room.encrypted", + ); + expect(matrixJsClient.getStateEvent).not.toHaveBeenCalled(); + }); + + it("preserves persisted encryption settings when the homeserver no longer exposes room state", async () => { + matrixJsClient.getRoom.mockReturnValue({ hasEncryptionStateEvent: () => false }); + matrixJsClient.getCrypto.mockReturnValue({ + isEncryptionEnabledInRoom: vi.fn(async () => true), + }); + matrixJsClient.getStateEvent.mockRejectedValue( + new MatrixError({ errcode: "M_NOT_FOUND", error: "State event not found" }, 404), + ); + const client = new MatrixClient("https://matrix.example.org", "token", { encryption: true }); + + await expect(client.getMessageWireEventType("!room:example.org")).resolves.toBe( + "m.room.encrypted", + ); + await expect( + client.sendMessage("!room:example.org", { msgtype: "m.text", body: "secret" }), + ).resolves.toBe("$sent"); + expect(matrixJsClient.getStateEvent).not.toHaveBeenCalled(); + }); + + it("accepts plaintext only when the homeserver explicitly reports missing encryption state", async () => { + matrixJsClient.getRoom.mockReturnValue(null); + matrixJsClient.getStateEvent.mockRejectedValue( + new MatrixError({ errcode: "M_NOT_FOUND", error: "State event not found" }, 404), + ); + const client = new MatrixClient("https://matrix.example.org", "token"); + + await expect(client.getMessageWireEventType("!room:example.org")).resolves.toBe( + "m.room.message", + ); + await expect( + client.sendMessage("!room:example.org", { msgtype: "m.text", body: "hello" }), + ).resolves.toBe("$sent"); + expect(matrixJsClient.getStateEvent).toHaveBeenCalled(); + }); + + it.each([ + new MatrixError({ errcode: "M_UNRECOGNIZED", error: "Endpoint not found" }, 404), + new MatrixError({ errcode: "M_NOT_FOUND", error: "Malformed proxy response" }, 503), + new MatrixError({ errcode: "M_UNKNOWN_TOKEN", error: "Access token not found" }, 401), + new Error("Matrix state endpoint unavailable"), + ])("fails closed when authoritative room encryption state cannot be verified", async (error) => { + matrixJsClient.getRoom.mockReturnValue(null); + matrixJsClient.getStateEvent.mockRejectedValue(error); + const client = new MatrixClient("https://matrix.example.org", "token"); + + await expect(client.getMessageWireEventType("!room:example.org")).rejects.toBe(error); + await expect( + client.sendMessage("!room:example.org", { msgtype: "m.text", body: "secret" }), + ).rejects.toBe(error); + expect(matrixJsClient.sendMessage).not.toHaveBeenCalled(); + }); + + it.each(["message", "poll"])( + "blocks %s before plaintext dispatch when the room is encrypted and crypto is disabled", + async (kind) => { + matrixJsClient.getRoom.mockReturnValue(null); + matrixJsClient.getStateEvent.mockResolvedValue({ algorithm: "m.megolm.v1.aes-sha2" }); + const client = new MatrixClient("https://matrix.example.org", "token"); + const operation = + kind === "message" + ? client.sendMessage("!room:example.org", { msgtype: "m.text", body: "secret" }) + : client.sendEvent("!room:example.org", "m.poll.start", { "m.text": "secret" }); + + await expect(operation).rejects.toThrow(/enable encryption/i); + expect(matrixJsClient.sendMessage).not.toHaveBeenCalled(); + expect(matrixJsClient.sendEvent).not.toHaveBeenCalled(); + }, + ); + + it("blocks encrypted sends until the SDK has a room object available for encryption", async () => { + matrixJsClient.getRoom.mockReturnValue(null); + matrixJsClient.getCrypto.mockReturnValue({ + isEncryptionEnabledInRoom: vi.fn(async () => true), + }); + const client = new MatrixClient("https://matrix.example.org", "token", { encryption: true }); + + await expect( + client.sendMessage("!room:example.org", { msgtype: "m.text", body: "secret" }), + ).rejects.toThrow(/sync/i); + expect(matrixJsClient.sendMessage).not.toHaveBeenCalled(); + }); + + it("blocks encrypted sends when cached room state and the crypto backend both miss encryption", async () => { + matrixJsClient.getRoom.mockReturnValue({ hasEncryptionStateEvent: () => false }); + matrixJsClient.getCrypto.mockReturnValue({ + isEncryptionEnabledInRoom: vi.fn(async () => false), + }); + matrixJsClient.getStateEvent.mockResolvedValue({ algorithm: "m.megolm.v1.aes-sha2" }); + const client = new MatrixClient("https://matrix.example.org", "token", { encryption: true }); + + await expect( + client.sendMessage("!room:example.org", { msgtype: "m.text", body: "secret" }), + ).rejects.toThrow(/sync/i); + expect(matrixJsClient.sendMessage).not.toHaveBeenCalled(); + }); + + it("does not treat an initialized crypto facade as a working SDK encryption backend", async () => { + matrixJsClient.getRoom.mockReturnValue({ hasEncryptionStateEvent: () => true }); + const client = new MatrixClient("https://matrix.example.org", "token", { encryption: true }); + (client as { crypto?: object }).crypto = {}; + + await expect( + client.sendMessage("!room:example.org", { msgtype: "m.text", body: "secret" }), + ).rejects.toThrow(/enable encryption/i); + expect(matrixJsClient.sendMessage).not.toHaveBeenCalled(); + }); + + it.each([ + { + label: "primary attachment", + content: { + msgtype: "m.image", + body: "photo.png", + url: "mxc://example/plain-primary", + }, + }, + { + label: "audio attachment", + content: { + msgtype: "m.audio", + body: "recording.mp3", + url: "mxc://example/plain-audio", + }, + }, + { + label: "video attachment", + content: { + msgtype: "m.video", + body: "recording.mp4", + url: "mxc://example/plain-video", + }, + }, + { + label: "file attachment", + content: { + msgtype: "m.file", + body: "report.pdf", + url: "mxc://example/plain-file", + }, + }, + { + label: "thumbnail", + content: { + msgtype: "m.image", + body: "photo.png", + info: { thumbnail_url: "mxc://example/plain-thumbnail" }, + }, + }, + { + label: "location thumbnail", + content: { + msgtype: MsgType.Location, + body: "Current location", + geo_uri: "geo:1,2", + info: { thumbnail_url: "mxc://example/plain-location-thumbnail" }, + }, + }, + ])("rejects an unencrypted $label in an encrypted room", async ({ content }) => { + matrixJsClient.getRoom.mockReturnValue({ hasEncryptionStateEvent: () => true }); + matrixJsClient.getCrypto.mockReturnValue({ + isEncryptionEnabledInRoom: vi.fn(async () => true), + }); + const client = new MatrixClient("https://matrix.example.org", "token", { encryption: true }); + + await expect(client.sendMessage("!room:example.org", content)).rejects.toThrow( + /unencrypted media.*retry/i, + ); + expect(matrixJsClient.sendMessage).not.toHaveBeenCalled(); + }); + + it.each([ + { + label: "primary attachment", + content: { + msgtype: "m.image", + body: "photo.png", + url: "mxc://example/plain-primary", + }, + }, + { + label: "thumbnail", + content: { + msgtype: "m.image", + body: "photo.png", + info: { thumbnail_url: "mxc://example/plain-thumbnail" }, + }, + }, + { + label: "location thumbnail", + content: { + msgtype: MsgType.Location, + body: "Current location", + geo_uri: "geo:1,2", + info: { thumbnail_url: "mxc://example/plain-location-thumbnail" }, + }, + }, + ])("rejects an unencrypted $label sent through the generic event owner", async ({ content }) => { + matrixJsClient.getRoom.mockReturnValue({ hasEncryptionStateEvent: () => true }); + matrixJsClient.getCrypto.mockReturnValue({ + isEncryptionEnabledInRoom: vi.fn(async () => true), + }); + const client = new MatrixClient("https://matrix.example.org", "token", { encryption: true }); + + await expect(client.sendEvent("!room:example.org", "m.room.message", content)).rejects.toThrow( + /unencrypted media.*retry/i, + ); + expect(matrixJsClient.sendEvent).not.toHaveBeenCalled(); + }); + + it("allows fully encrypted attachments and preserves text URLs in encrypted rooms", async () => { + matrixJsClient.getRoom.mockReturnValue({ hasEncryptionStateEvent: () => true }); + matrixJsClient.getCrypto.mockReturnValue({ + isEncryptionEnabledInRoom: vi.fn(async () => true), + }); + const client = new MatrixClient("https://matrix.example.org", "token", { encryption: true }); + const encryptedFile = { + url: "mxc://example/encrypted", + key: { alg: "A256CTR", key_ops: ["encrypt", "decrypt"], kty: "oct", k: "key", ext: true }, + iv: "iv", + hashes: { sha256: "hash" }, + v: "v2", + }; + + await expect( + client.sendMessage("!room:example.org", { + msgtype: "m.image", + body: "photo.png", + file: encryptedFile, + info: { thumbnail_file: encryptedFile }, + }), + ).resolves.toBe("$sent"); + await expect( + client.sendMessage("!room:example.org", { + msgtype: MsgType.Location, + body: "Current location", + geo_uri: "geo:1,2", + info: { thumbnail_file: encryptedFile }, + }), + ).resolves.toBe("$sent"); + await expect( + client.sendMessage("!room:example.org", { + msgtype: "m.text", + body: "custom link", + url: "https://example.org/custom-text-field", + }), + ).resolves.toBe("$sent"); + await expect( + client.sendEvent("!room:example.org", "m.room.message", { + msgtype: "m.image", + body: "photo.png", + file: encryptedFile, + info: { thumbnail_file: encryptedFile }, + }), + ).resolves.toBe("$sent-event"); + await expect( + client.sendEvent("!room:example.org", "m.room.message", { + msgtype: MsgType.Location, + body: "Current location", + geo_uri: "geo:1,2", + info: { thumbnail_file: encryptedFile }, + }), + ).resolves.toBe("$sent-event"); + await expect( + client.sendEvent("!room:example.org", "m.room.message", { + msgtype: "m.text", + body: "custom link", + url: "https://example.org/custom-text-field", + }), + ).resolves.toBe("$sent-event"); + await expect( + client.sendEvent("!room:example.org", "m.poll.start", { "m.text": "Lunch?" }), + ).resolves.toBe("$sent-event"); + }); + + it.each([ + { eventType: "m.room.encrypted", message: /encrypted wire events.*sdk/i }, + { eventType: "m.room.redaction", message: /redaction wire events.*redactEvent/i }, + ])("rejects caller-supplied reserved $eventType wire events", async ({ eventType, message }) => { + matrixJsClient.getRoom.mockReturnValue({ hasEncryptionStateEvent: () => true }); + matrixJsClient.getCrypto.mockReturnValue({ + isEncryptionEnabledInRoom: vi.fn(async () => true), + }); + const client = new MatrixClient("https://matrix.example.org", "token", { encryption: true }); + + await expect( + client.sendEvent("!room:example.org", eventType, { body: "secret" }), + ).rejects.toThrow(message); + expect(matrixJsClient.sendEvent).not.toHaveBeenCalled(); + }); + + it("preserves the dedicated Matrix room-event redaction owner", async () => { + const client = new MatrixClient("https://matrix.example.org", "token"); + + await expect(client.redactEvent("!room:example.org", "$target")).resolves.toBe("$redact"); + expect(matrixJsClient.redactEvent).toHaveBeenCalledWith( + "!room:example.org", + "$target", + undefined, + undefined, + ); + }); + + it("preserves the Matrix protocol exemption for unencrypted reactions", async () => { + matrixJsClient.getRoom.mockReturnValue(null); + matrixJsClient.getStateEvent.mockRejectedValue(new Error("state unavailable")); + const client = new MatrixClient("https://matrix.example.org", "token"); + + await expect( + client.sendEvent("!room:example.org", "m.reaction", { + "m.relates_to": { event_id: "$target", key: "👍", rel_type: "m.annotation" }, + }), + ).resolves.toBe("$sent-event"); + expect(matrixJsClient.getStateEvent).not.toHaveBeenCalled(); + }); + + it("returns an already-sent durable event without probing current room state", async () => { + matrixJsClient.getRoom.mockReturnValue({ + hasEncryptionStateEvent: () => false, + getEventForTxnId: () => ({ status: EventStatus.SENT, getId: () => "$already-sent" }), + }); + matrixJsClient.getStateEvent.mockRejectedValue(new Error("state unavailable")); + const client = new MatrixClient("https://matrix.example.org", "token"); + + await expect( + client.sendMessage( + "!room:example.org", + { msgtype: "m.text", body: "already delivered" }, + "oc_already_sent", + ), + ).resolves.toBe("$already-sent"); + expect(matrixJsClient.getStateEvent).not.toHaveBeenCalled(); + expect(matrixJsClient.sendMessage).not.toHaveBeenCalled(); + }); + + it("checks encrypted-room readiness before retrying an unsent durable event", async () => { + matrixJsClient.getRoom.mockReturnValue({ + hasEncryptionStateEvent: () => false, + getEventForTxnId: () => ({ + status: EventStatus.NOT_SENT, + getId: () => "~pending", + getContent: () => ({ msgtype: "m.text", body: "secret" }), + }), + }); + matrixJsClient.getStateEvent.mockResolvedValue({ algorithm: "m.megolm.v1.aes-sha2" }); + const client = new MatrixClient("https://matrix.example.org", "token"); + + await expect( + client.sendMessage("!room:example.org", { msgtype: "m.text", body: "secret" }, "oc_retry"), + ).rejects.toThrow(/enable encryption/i); + expect(matrixJsClient.resendEvent).not.toHaveBeenCalled(); + }); + + it("rejects unencrypted attachment references before retrying an unsent encrypted event", async () => { + matrixJsClient.getRoom.mockReturnValue({ + hasEncryptionStateEvent: () => true, + getEventForTxnId: () => ({ + status: EventStatus.NOT_SENT, + getId: () => "~pending", + getContent: () => ({ + msgtype: "m.image", + body: "photo.png", + url: "mxc://example/plain-primary", + }), + }), + }); + matrixJsClient.getCrypto.mockReturnValue({ + isEncryptionEnabledInRoom: vi.fn(async () => true), + }); + const client = new MatrixClient("https://matrix.example.org", "token", { encryption: true }); + + await expect( + client.sendMessage( + "!room:example.org", + { msgtype: "m.text", body: "caller content cannot replace the pending event" }, + "oc_retry", + ), + ).rejects.toThrow(/unencrypted media.*retry/i); + expect(matrixJsClient.resendEvent).not.toHaveBeenCalled(); + }); + it("passes stable transaction ids into matrix-js-sdk timeline sends", async () => { const client = new MatrixClient("https://matrix.example.org", "token"); @@ -661,9 +1090,9 @@ describe("MatrixClient request hardening", () => { "m.relates_to": { event_id: "$target", key: "👍", rel_type: "m.annotation" }, }); - await Promise.resolve(); - await Promise.resolve(); - expect(started).toEqual(["message"]); + await vi.waitFor(() => { + expect(started).toEqual(["message"]); + }); expect(matrixJsClient.sendEvent).not.toHaveBeenCalled(); releaseFirst?.(); @@ -696,9 +1125,9 @@ describe("MatrixClient request hardening", () => { body: "b", }); - await Promise.resolve(); - await Promise.resolve(); - expect(started).toEqual(["!room-a:example.org", "!room-b:example.org"]); + await vi.waitFor(() => { + expect(started).toEqual(["!room-a:example.org", "!room-b:example.org"]); + }); releaseFirst?.(); diff --git a/extensions/matrix/src/matrix/sdk/client-base.ts b/extensions/matrix/src/matrix/sdk/client-base.ts index 29345252f144..ad8bf86f21e9 100644 --- a/extensions/matrix/src/matrix/sdk/client-base.ts +++ b/extensions/matrix/src/matrix/sdk/client-base.ts @@ -101,6 +101,7 @@ export abstract class MatrixClientBase { eventType: string, stateKey?: string, ): Promise>; + abstract getMessageWireEventType(roomId: string): Promise<"m.room.message" | "m.room.encrypted">; abstract downloadContent( mxcUrl: string, opts?: { allowRemote?: boolean; maxBytes?: number; readIdleTimeoutMs?: number }, @@ -352,8 +353,8 @@ export abstract class MatrixClientBase { client: this.client, verificationManager: this.verificationManager, recoveryKeyStore: this.recoveryKeyStore, - getRoomStateEvent: (roomId, eventType, stateKey = "") => - this.getRoomStateEvent(roomId, eventType, stateKey), + isRoomEncrypted: async (roomId) => + (await this.getMessageWireEventType(roomId)) === "m.room.encrypted", downloadContent: (mxcUrl, opts) => this.downloadContent(mxcUrl, opts), }); } diff --git a/extensions/matrix/src/matrix/sdk/client-core.ts b/extensions/matrix/src/matrix/sdk/client-core.ts index 67aba693e0bf..239617966a1d 100644 --- a/extensions/matrix/src/matrix/sdk/client-core.ts +++ b/extensions/matrix/src/matrix/sdk/client-core.ts @@ -1,8 +1,16 @@ import { createHash } from "node:crypto"; -import { MatrixEventEvent, Preset, type MatrixEvent } from "matrix-js-sdk/lib/matrix.js"; +import { + EventType, + MatrixError, + MatrixEventEvent, + MsgType, + Preset, + type MatrixEvent, +} from "matrix-js-sdk/lib/matrix.js"; import { EventStatus } from "matrix-js-sdk/lib/models/event-status.js"; import type { Direction } from "matrix-js-sdk/lib/models/event-timeline.js"; import { formatMatrixErrorReason } from "../errors.js"; +import { MATRIX_REACTION_EVENT_TYPE } from "../reaction-common.js"; import { MatrixClientBase, type MatrixMessageWireDispatch } from "./client-base.js"; import { matrixEventToRaw, parseMxc } from "./event-helpers.js"; import { noop } from "./logger.js"; @@ -201,6 +209,7 @@ export abstract class MatrixClientCore extends MatrixClientBase { return existingId; } if (existing.status === EventStatus.NOT_SENT && room) { + await this.prepareRoomForMessageSend(roomId, existing.getContent()); const resent = await this.client.resendEvent(existing, room); return resent.event_id; } @@ -209,6 +218,7 @@ export abstract class MatrixClientCore extends MatrixClientBase { ); } } + await this.prepareRoomForMessageSend(roomId, content); const sent = await this.client.sendMessage(roomId, content as never, transactionId); return sent.event_id; }, @@ -221,9 +231,62 @@ export abstract class MatrixClientCore extends MatrixClientBase { return "m.room.encrypted"; } const crypto = this.client.getCrypto(); - return crypto && (await crypto.isEncryptionEnabledInRoom(roomId)) - ? "m.room.encrypted" - : "m.room.message"; + if (crypto && (await crypto.isEncryptionEnabledInRoom(roomId))) { + return "m.room.encrypted"; + } + try { + // A missing local room/state is unknown; only the homeserver can prove + // that encryption was never enabled before plaintext leaves the client. + await this.getRoomStateEvent(roomId, "m.room.encryption", ""); + return "m.room.encrypted"; + } catch (error) { + if ( + error instanceof MatrixError && + error.httpStatus === 404 && + error.errcode === "M_NOT_FOUND" + ) { + return "m.room.message"; + } + throw error; + } + } + + async prepareRoomForMessageSend( + roomId: string, + content?: MessageEventContent, + ): Promise<"m.room.message" | "m.room.encrypted"> { + if ((await this.getMessageWireEventType(roomId)) === "m.room.message") { + return "m.room.message"; + } + const crypto = this.client.getCrypto(); + if (!crypto) { + throw new Error("Encrypted Matrix room: enable encryption before sending messages"); + } + const room = this.client.getRoom(roomId); + // matrix-js-sdk skips encryption for unknown rooms; authoritative state + // alone does not hydrate its Room or configure the crypto backend. + if ( + !room || + (!room.hasEncryptionStateEvent() && !(await crypto.isEncryptionEnabledInRoom(roomId))) + ) { + throw new Error("Encrypted Matrix room is not ready: wait for room sync before sending"); + } + if ( + content && + (((content.msgtype === MsgType.Image || + content.msgtype === MsgType.Audio || + content.msgtype === MsgType.Video || + content.msgtype === MsgType.File) && + typeof content.url === "string") || + (content.info && + "thumbnail_url" in content.info && + typeof content.info.thumbnail_url === "string")) + ) { + // Room encryption can change after media uploads; never reference a + // plaintext primary or thumbnail from a newly encrypted room event. + throw new Error("Encrypted Matrix room contains unencrypted media; retry the send"); + } + return "m.room.encrypted"; } async sendEvent( @@ -232,6 +295,21 @@ export abstract class MatrixClientCore extends MatrixClientBase { content: Record, ): Promise { return await this.runSerializedRoomSend(roomId, async () => { + // SDK encryption trusts these wire event types without inspecting their + // payload; only SDK encryption and the dedicated redaction owner may emit them. + if ( + eventType === EventType.RoomMessageEncrypted.toString() || + eventType === EventType.RoomRedaction.toString() + ) { + throw new Error( + eventType === EventType.RoomRedaction.toString() + ? "Matrix redaction wire events must use redactEvent" + : "Matrix encrypted wire events must be generated by the SDK", + ); + } + if (eventType !== MATRIX_REACTION_EVENT_TYPE) { + await this.prepareRoomForMessageSend(roomId, content); + } const sent = await this.client.sendEvent(roomId, eventType as never, content as never); return sent.event_id; }); diff --git a/extensions/matrix/src/matrix/sdk/crypto-facade.test.ts b/extensions/matrix/src/matrix/sdk/crypto-facade.test.ts index 1fb5c2208775..25c2b8e6c973 100644 --- a/extensions/matrix/src/matrix/sdk/crypto-facade.test.ts +++ b/extensions/matrix/src/matrix/sdk/crypto-facade.test.ts @@ -39,55 +39,64 @@ function createFacadeHarness(params?: { client?: Partial; verificationManager?: Partial; recoveryKeySummary?: ReturnType; - getRoomStateEvent?: MatrixCryptoFacadeDeps["getRoomStateEvent"]; + isRoomEncrypted?: MatrixCryptoFacadeDeps["isRoomEncrypted"]; downloadContent?: MatrixCryptoFacadeDeps["downloadContent"]; }) { - const getRoomStateEvent: MatrixCryptoFacadeDeps["getRoomStateEvent"] = - params?.getRoomStateEvent ?? (async () => ({})); + const isRoomEncrypted: MatrixCryptoFacadeDeps["isRoomEncrypted"] = + params?.isRoomEncrypted ?? (async () => false); const downloadContent: MatrixCryptoFacadeDeps["downloadContent"] = params?.downloadContent ?? (async () => Buffer.alloc(0)); const facade = createMatrixCryptoFacade({ client: { - getRoom: params?.client?.getRoom ?? (() => null), getCrypto: params?.client?.getCrypto ?? (() => undefined), getUserId: params?.client?.getUserId ?? (() => "@bot:example.org"), }, verificationManager: createVerificationManagerMock(params?.verificationManager), recoveryKeyStore: createRecoveryKeyStoreMock(params?.recoveryKeySummary ?? null), - getRoomStateEvent, + isRoomEncrypted, downloadContent, }); - return { facade, getRoomStateEvent, downloadContent }; + return { facade, isRoomEncrypted, downloadContent }; } describe("createMatrixCryptoFacade", () => { - it("detects encrypted rooms from cached room state", async () => { + it("delegates encrypted-room classification to the canonical client owner", async () => { + const isRoomEncrypted = vi.fn(async () => true); const { facade } = createFacadeHarness({ - client: { - getRoom: () => ({ - hasEncryptionStateEvent: () => true, - }), - }, + isRoomEncrypted, + }); + + await expect(facade.isRoomEncrypted("!room:example.org")).resolves.toBe(true); + expect(isRoomEncrypted).toHaveBeenCalledWith("!room:example.org"); + }); + + it("preserves authoritative plaintext-room classification", async () => { + const isRoomEncrypted = vi.fn(async () => false); + const { facade } = createFacadeHarness({ + isRoomEncrypted, + }); + + await expect(facade.isRoomEncrypted("!room:example.org")).resolves.toBe(false); + expect(isRoomEncrypted).toHaveBeenCalledWith("!room:example.org"); + }); + + it("never downgrades an existing malformed encryption event to plaintext", async () => { + const { facade } = createFacadeHarness({ + isRoomEncrypted: async () => true, }); await expect(facade.isRoomEncrypted("!room:example.org")).resolves.toBe(true); }); - it("falls back to server room state when room cache has no encryption event", async () => { - const getRoomStateEvent = vi.fn(async () => ({ - algorithm: "m.megolm.v1.aes-sha2", - })); + it("propagates authoritative room-state failures without permitting plaintext", async () => { + const error = new Error("Matrix room state authorization failed"); const { facade } = createFacadeHarness({ - client: { - getRoom: () => ({ - hasEncryptionStateEvent: () => false, - }), + isRoomEncrypted: async () => { + throw error; }, - getRoomStateEvent, }); - await expect(facade.isRoomEncrypted("!room:example.org")).resolves.toBe(true); - expect(getRoomStateEvent).toHaveBeenCalledWith("!room:example.org", "m.room.encryption", ""); + await expect(facade.isRoomEncrypted("!room:example.org")).rejects.toBe(error); }); it("forwards verification requests and uses client crypto API", async () => { @@ -110,7 +119,6 @@ describe("createMatrixCryptoFacade", () => { })); const { facade } = createFacadeHarness({ client: { - getRoom: () => null, getCrypto: () => crypto, }, verificationManager: { @@ -176,7 +184,6 @@ describe("createMatrixCryptoFacade", () => { }; const { facade } = createFacadeHarness({ client: { - getRoom: () => null, getCrypto: () => crypto, }, verificationManager: { diff --git a/extensions/matrix/src/matrix/sdk/crypto-facade.ts b/extensions/matrix/src/matrix/sdk/crypto-facade.ts index 9ab745691be4..b420d341db61 100644 --- a/extensions/matrix/src/matrix/sdk/crypto-facade.ts +++ b/extensions/matrix/src/matrix/sdk/crypto-facade.ts @@ -11,7 +11,6 @@ import type { } from "./verification-manager.js"; type MatrixCryptoFacadeClient = { - getRoom: (roomId: string) => { hasEncryptionStateEvent: () => boolean } | null; getCrypto: () => unknown; getUserId: () => string | null; }; @@ -106,11 +105,7 @@ export function createMatrixCryptoFacade(deps: { client: MatrixCryptoFacadeClient; verificationManager: MatrixVerificationManager; recoveryKeyStore: MatrixRecoveryKeyStore; - getRoomStateEvent: ( - roomId: string, - eventType: string, - stateKey?: string, - ) => Promise>; + isRoomEncrypted: (roomId: string) => Promise; downloadContent: ( mxcUrl: string, opts?: { maxBytes?: number; readIdleTimeoutMs?: number }, @@ -129,18 +124,7 @@ export function createMatrixCryptoFacade(deps: { ) => { // compatibility no-op }, - isRoomEncrypted: async (roomId: string): Promise => { - const room = deps.client.getRoom(roomId); - if (room?.hasEncryptionStateEvent()) { - return true; - } - try { - const event = await deps.getRoomStateEvent(roomId, "m.room.encryption", ""); - return typeof event.algorithm === "string" && event.algorithm.length > 0; - } catch { - return false; - } - }, + isRoomEncrypted: deps.isRoomEncrypted, requestOwnUserVerification: async () => { const crypto = deps.client.getCrypto() as MatrixVerificationCryptoApi | undefined; return await deps.verificationManager.requestOwnUserVerification(crypto); diff --git a/extensions/matrix/src/matrix/send.test.ts b/extensions/matrix/src/matrix/send.test.ts index 931415d02252..dc034251111e 100644 --- a/extensions/matrix/src/matrix/send.test.ts +++ b/extensions/matrix/src/matrix/send.test.ts @@ -118,12 +118,14 @@ const makeClient = () => { const getEvent = vi.fn(); const getJoinedRoomMembers = vi.fn().mockResolvedValue([]); const uploadContent = vi.fn().mockResolvedValue("mxc://example/file"); + const prepareRoomForMessageSend = vi.fn(); const client = { sendMessage, sendEvent, getEvent, getJoinedRoomMembers, uploadContent, + prepareRoomForMessageSend, getTransactionScopeId: vi.fn().mockResolvedValue("scope-1"), getMessageWireEventType: vi.fn().mockResolvedValue("m.room.message"), getUserId: vi.fn().mockResolvedValue("@bot:example.org"), @@ -132,6 +134,24 @@ const makeClient = () => { stop: vi.fn(() => undefined), stopAndPersist: vi.fn(async () => undefined), } as unknown as import("./sdk.js").MatrixClient; + prepareRoomForMessageSend.mockImplementation( + async (roomId: string, content?: import("./sdk.js").MessageEventContent) => { + const eventType = await client.getMessageWireEventType(roomId); + if (eventType === "m.room.encrypted" && !client.crypto) { + throw new Error("Encrypted Matrix room: enable encryption before sending messages"); + } + if ( + eventType === "m.room.encrypted" && + (typeof content?.url === "string" || + (content?.info && + "thumbnail_url" in content.info && + typeof content.info.thumbnail_url === "string")) + ) { + throw new Error("Encrypted Matrix room contains unencrypted media; retry the send"); + } + return eventType; + }, + ); return { client, sendMessage, sendEvent, getEvent, getJoinedRoomMembers, uploadContent }; }; @@ -141,6 +161,7 @@ function makeEncryptedMediaClient() { isRoomEncrypted: vi.fn().mockResolvedValue(true), encryptMedia: vi.fn().mockResolvedValue(createEncryptedMediaPayload()), }; + vi.spyOn(result.client, "getMessageWireEventType").mockResolvedValue("m.room.encrypted"); return result; } @@ -582,6 +603,7 @@ describe("sendMessageMatrix media", () => { const uploadArg = mockCallArg(uploadContent, "uploadContent", 0); expect(Buffer.isBuffer(uploadArg)).toBe(true); expect(uploadArg).toEqual(Buffer.from("media")); + expect(uploadContent).toHaveBeenCalledWith(Buffer.from("media"), "image/png", "photo.png"); const content = sentContent(sendMessage) as { url?: string; @@ -595,6 +617,104 @@ describe("sendMessageMatrix media", () => { expect(content.url).toBe("mxc://example/file"); }); + it("rejects encrypted-room media before upload when encryption is unavailable", async () => { + const { client, sendMessage, uploadContent } = makeClient(); + vi.spyOn(client, "getMessageWireEventType").mockResolvedValue("m.room.encrypted"); + + await expect( + sendMessageMatrix("room:!room:example", "caption", { + client, + cfg: {} as never, + mediaUrl: "file:///tmp/photo.png", + }), + ).rejects.toThrow(/enable encryption/i); + + expect(uploadContent).not.toHaveBeenCalled(); + expect(sendMessage).not.toHaveBeenCalled(); + }); + + it.each(["text", "media"])( + "rejects encrypted-room %s before reporting a platform dispatch when encryption is disabled", + async (kind) => { + const { client, sendMessage, uploadContent } = makeClient(); + vi.spyOn(client, "getMessageWireEventType").mockResolvedValue("m.room.encrypted"); + const onPlatformSendDispatch = vi.fn(); + + await expect( + sendMessageMatrix("room:!room:example", "secret", { + client, + cfg: {} as never, + ...(kind === "media" ? { mediaUrl: "file:///tmp/photo.png" } : {}), + onPlatformSendDispatch, + }), + ).rejects.toThrow(/enable encryption/i); + + expect(onPlatformSendDispatch).not.toHaveBeenCalled(); + expect(uploadContent).not.toHaveBeenCalled(); + expect(sendMessage).not.toHaveBeenCalled(); + if (kind === "media") { + expect(loadOutboundMediaFromUrlMock).not.toHaveBeenCalled(); + } + }, + ); + + it("rejects uploads when a room becomes encrypted while media is loading", async () => { + const { client, sendMessage, uploadContent } = makeClient(); + const onPlatformSendDispatch = vi.fn(); + loadOutboundMediaFromUrlMock.mockImplementationOnce(async () => { + vi.spyOn(client, "getMessageWireEventType").mockResolvedValue("m.room.encrypted"); + return { + buffer: Buffer.from("secret media"), + fileName: "secret.png", + contentType: "image/png", + kind: "image", + }; + }); + + await expect( + sendMessageMatrix("room:!room:example", "secret", { + client, + cfg: {} as never, + mediaUrl: "file:///tmp/secret.png", + onPlatformSendDispatch, + }), + ).rejects.toThrow(/enable encryption/i); + + expect(uploadContent).not.toHaveBeenCalled(); + expect(sendMessage).not.toHaveBeenCalled(); + expect(onPlatformSendDispatch).not.toHaveBeenCalled(); + }); + + it("encrypts uploads when a room becomes encrypted while media is loading", async () => { + const { client, sendMessage, uploadContent } = makeClient(); + (client as { crypto?: object }).crypto = { + encryptMedia: vi.fn().mockResolvedValue(createEncryptedMediaPayload()), + }; + loadOutboundMediaFromUrlMock.mockImplementationOnce(async () => { + vi.spyOn(client, "getMessageWireEventType").mockResolvedValue("m.room.encrypted"); + return { + buffer: Buffer.from("secret media"), + fileName: "secret.png", + contentType: "image/png", + kind: "image", + }; + }); + + await sendMessageMatrix("room:!room:example", "secret", { + client, + cfg: {} as never, + mediaUrl: "file:///tmp/secret.png", + }); + + expect(uploadContent).toHaveBeenCalledWith( + Buffer.from("encrypted"), + "application/octet-stream", + ); + const content = sentContent(sendMessage); + expect(content.file).toBeDefined(); + expect(content.url).toBeUndefined(); + }); + it("records each media and overflow event with its actual kind and reply relation", async () => { const { client, sendMessage } = makeClient(); resolveTextChunkLimitMock.mockReturnValue(6); @@ -646,18 +766,26 @@ describe("sendMessageMatrix media", () => { expect(uploadArg instanceof Uint8Array ? Buffer.from(uploadArg).toString() : undefined).toBe( "encrypted", ); + expect(uploadContent).toHaveBeenCalledWith( + Buffer.from("encrypted"), + "application/octet-stream", + ); const content = sentContent(sendMessage) as { url?: string; file?: { url?: string }; + filename?: string; + info?: { mimetype?: string }; }; expect(content.url).toBeUndefined(); expect(content.file?.url).toBe("mxc://example/file"); + expect(content.filename).toBe("photo.png"); + expect(content.info?.mimetype).toBe("image/png"); }); it("encrypts thumbnail via thumbnail_file when room is encrypted", async () => { const { client, sendMessage, uploadContent } = makeClient(); - const isRoomEncrypted = vi.fn().mockResolvedValue(true); + vi.spyOn(client, "getMessageWireEventType").mockResolvedValue("m.room.encrypted"); const encryptMedia = vi.fn().mockResolvedValue({ buffer: Buffer.from("encrypted-thumb"), file: { @@ -668,7 +796,6 @@ describe("sendMessageMatrix media", () => { }, }); (client as { crypto?: object }).crypto = { - isRoomEncrypted, encryptMedia, }; // Return image metadata so thumbnail generation is triggered (image > 800px) @@ -688,8 +815,11 @@ describe("sendMessageMatrix media", () => { }); // encryptMedia called twice: once for main media, once for thumbnail - expect(isRoomEncrypted).toHaveBeenCalledTimes(1); expect(encryptMedia).toHaveBeenCalledTimes(2); + expect(uploadContent.mock.calls).toEqual([ + [Buffer.from("encrypted-thumb"), "application/octet-stream"], + [Buffer.from("encrypted-thumb"), "application/octet-stream"], + ]); const content = sentContent(sendMessage) as { url?: string; @@ -778,7 +908,10 @@ describe("sendMessageMatrix media", () => { mediaUrl: "file:///tmp/photo.png", }); - expect(uploadContent).toHaveBeenCalledTimes(2); + expect(uploadContent.mock.calls).toEqual([ + [Buffer.from("media"), "image/png", "photo.png"], + [Buffer.from("thumb"), "image/jpeg", "thumbnail.jpg"], + ]); const content = sentContent(sendMessage) as { info?: { thumbnail_url?: string; @@ -801,6 +934,37 @@ describe("sendMessageMatrix media", () => { }); }); + it("rejects mixed attachments when a room becomes encrypted while an image is resized", async () => { + const { client, sendMessage, uploadContent } = makeClient(); + const onPlatformSendDispatch = vi.fn(); + (client as { crypto?: object }).crypto = { + encryptMedia: vi.fn().mockResolvedValue(createEncryptedMediaPayload()), + }; + getImageMetadataMock + .mockResolvedValueOnce({ width: 1600, height: 1200 }) + .mockResolvedValueOnce({ width: 800, height: 600 }); + resizeToJpegMock.mockImplementationOnce(async () => { + vi.spyOn(client, "getMessageWireEventType").mockResolvedValue("m.room.encrypted"); + return Buffer.from("secret thumbnail"); + }); + + await expect( + sendMessageMatrix("room:!room:example", "caption", { + client, + cfg: {} as never, + mediaUrl: "file:///tmp/photo.png", + onPlatformSendDispatch, + }), + ).rejects.toThrow(/unencrypted media.*retry/i); + + expect(uploadContent.mock.calls).toEqual([ + [Buffer.from("media"), "image/png", "photo.png"], + [Buffer.from("encrypted"), "application/octet-stream"], + ]); + expect(sendMessage).not.toHaveBeenCalled(); + expect(onPlatformSendDispatch).not.toHaveBeenCalled(); + }); + it("uses explicit cfg for media sends instead of runtime loadConfig fallbacks", async () => { const { client } = makeClient(); const explicitCfg = { diff --git a/extensions/matrix/src/matrix/send.ts b/extensions/matrix/src/matrix/send.ts index 4b5fe64a6050..1d7a135c9d62 100644 --- a/extensions/matrix/src/matrix/send.ts +++ b/extensions/matrix/src/matrix/send.ts @@ -38,7 +38,7 @@ import { buildMediaContent, prepareImageInfo, resolveMediaDurationMs, - uploadMediaMaybeEncrypted, + uploadMediaWithEncryption, } from "./send/media.js"; import { normalizeThreadId, resolveMatrixRoomId } from "./send/targets.js"; import { @@ -212,12 +212,10 @@ export async function sendMessageMatrix( }, async (client) => { const roomId = await resolveMatrixRoomId(client, to); + const wireEventType = await client.prepareRoomForMessageSend(roomId); const cfg = requireRuntimeConfig(opts.cfg, "Matrix send") as CoreConfig; const threadId = normalizeThreadId(opts.threadId); const transactionScopeId = durableIdentity ? await client.getTransactionScopeId() : undefined; - const wireEventType = durableIdentity - ? await client.getMessageWireEventType(roomId) - : undefined; const storedPlan = durableIdentity ? await loadMatrixDeliveryPlan({ identity: durableIdentity, @@ -258,7 +256,7 @@ export async function sendMessageMatrix( mediaLocalRoots: opts.mediaLocalRoots, mediaReadFile: opts.mediaReadFile, }); - const uploaded = await uploadMediaMaybeEncrypted(client, roomId, media.buffer, { + const uploaded = await uploadMediaWithEncryption(client, roomId, media.buffer, { contentType: media.contentType, filename: media.fileName, }); @@ -281,7 +279,7 @@ export async function sendMessageMatrix( ? await prepareImageInfo({ buffer: media.buffer, client, - encrypted: Boolean(uploaded.file), + roomId, }) : undefined; const [firstChunk, ...rest] = chunks; @@ -345,6 +343,9 @@ export async function sendMessageMatrix( })); } + if (opts.mediaUrl) { + await client.prepareRoomForMessageSend(roomId, plannedEvents[0]?.content); + } let platformDispatchStarted = false; if (!durableIdentity) { await opts.onPlatformSendDispatch?.(); diff --git a/extensions/matrix/src/matrix/send/media.ts b/extensions/matrix/src/matrix/send/media.ts index 905c25050016..4470430eeba0 100644 --- a/extensions/matrix/src/matrix/send/media.ts +++ b/extensions/matrix/src/matrix/send/media.ts @@ -170,7 +170,7 @@ function resolveAifcIma4DurationSeconds(buffer: Buffer, sampleRate?: number): nu export async function prepareImageInfo(params: { buffer: Buffer; client: MatrixClient; - encrypted?: boolean; + roomId: string; }): Promise { const meta = await getCore() .media.getImageMetadata(params.buffer) @@ -191,10 +191,9 @@ export async function prepareImageInfo(params: { const thumbMeta = await getCore() .media.getImageMetadata(thumbBuffer) .catch(() => null); - const result = await uploadMediaWithEncryption(params.client, thumbBuffer, { + const result = await uploadMediaWithEncryption(params.client, params.roomId, thumbBuffer, { contentType: "image/jpeg", filename: "thumbnail.jpg", - encrypted: params.encrypted === true, }); if (result.file) { imageInfo.thumbnail_file = result.file; @@ -250,44 +249,7 @@ export async function resolveMediaDurationMs(params: { return undefined; } -async function uploadFile( - client: MatrixClient, - file: Buffer, - params: { - contentType?: string; - filename?: string; - }, -): Promise { - return await client.uploadContent(file, params.contentType, params.filename); -} - -async function uploadMediaWithEncryption( - client: MatrixClient, - buffer: Buffer, - params: { - contentType?: string; - filename?: string; - encrypted: boolean; - }, -): Promise<{ url: string; file?: EncryptedFile }> { - if (params.encrypted && client.crypto) { - const encrypted = await client.crypto.encryptMedia(buffer); - const mxc = await client.uploadContent(encrypted.buffer, params.contentType, params.filename); - const file: EncryptedFile = { url: mxc, ...encrypted.file }; - return { - url: mxc, - file, - }; - } - - const mxc = await uploadFile(client, buffer, params); - return { url: mxc }; -} - -/** - * Upload media with optional encryption for E2EE rooms. - */ -export async function uploadMediaMaybeEncrypted( +export async function uploadMediaWithEncryption( client: MatrixClient, roomId: string, buffer: Buffer, @@ -296,10 +258,23 @@ export async function uploadMediaMaybeEncrypted( filename?: string; }, ): Promise<{ url: string; file?: EncryptedFile }> { - // Check if room is encrypted and crypto is available - const isEncrypted = Boolean(client.crypto && (await client.crypto.isRoomEncrypted(roomId))); - return await uploadMediaWithEncryption(client, buffer, { - ...params, - encrypted: isEncrypted, - }); + // Downloads and thumbnail generation can yield while encryption changes; + // resolve room policy at the upload boundary instead of reusing stale facts. + if ((await client.prepareRoomForMessageSend(roomId)) === "m.room.encrypted") { + if (!client.crypto) { + throw new Error("Encrypted Matrix room: enable encryption before uploading media"); + } + const encrypted = await client.crypto.encryptMedia(buffer); + // Upload URLs and headers are visible; keep real media metadata inside + // the encrypted room event instead of exposing it with the ciphertext. + const mxc = await client.uploadContent(encrypted.buffer, "application/octet-stream"); + const file: EncryptedFile = { url: mxc, ...encrypted.file }; + return { + url: mxc, + file, + }; + } + + const mxc = await client.uploadContent(buffer, params.contentType, params.filename); + return { url: mxc }; } From f5cb0f2f76dd81aca3e624fa0a0fc500a54fd8e8 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 04:36:03 -0700 Subject: [PATCH 33/57] test(memory): isolate index suite plugin discovery (#118641) --- extensions/memory-core/src/memory/index.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/extensions/memory-core/src/memory/index.test.ts b/extensions/memory-core/src/memory/index.test.ts index 170cd50e95c2..89d8e50996d9 100644 --- a/extensions/memory-core/src/memory/index.test.ts +++ b/extensions/memory-core/src/memory/index.test.ts @@ -35,6 +35,7 @@ import { closeMemoryIndexManagersForAgent, MemoryIndexManager as RuntimeMemoryIndexManager, } from "./manager.js"; +import { isolateMemoryManagerTestConfig } from "./test-config-helpers.js"; // This suite performs real sqlite/media indexing and can exceed the global // timeout when it shares a packed CI extension shard. @@ -417,7 +418,7 @@ describe("memory index", () => { temporalDecay?: { enabled: boolean }; }; }): TestCfg { - return { + return isolateMemoryManagerTestConfig({ memory: { search: { ...(params.provider !== undefined ? { provider: params.provider } : {}), @@ -449,7 +450,7 @@ describe("memory index", () => { list: [{ id: "main", default: true }], }, models: params.providerAliases ? { providers: params.providerAliases } : undefined, - }; + }); } async function seedMemoryIndexSessionTranscript(params: { From 757d22020362db03b73924a0ab85159ca1297948 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 04:36:28 -0700 Subject: [PATCH 34/57] fix(ui): surface sidebar clipboard failures across reconnects (#118632) * fix(ui): surface sidebar clipboard failures across reconnects * test(ui): type sidebar clipboard lifecycle fixtures correctly --- .../chat/components/chat-sidebar.test.ts | 279 +++++++++++++++++- ui/src/pages/chat/components/chat-sidebar.ts | 132 ++++++--- 2 files changed, 361 insertions(+), 50 deletions(-) diff --git a/ui/src/pages/chat/components/chat-sidebar.test.ts b/ui/src/pages/chat/components/chat-sidebar.test.ts index 7922fae88c94..0368abcbb388 100644 --- a/ui/src/pages/chat/components/chat-sidebar.test.ts +++ b/ui/src/pages/chat/components/chat-sidebar.test.ts @@ -1,6 +1,6 @@ /* @vitest-environment jsdom */ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { openEditor } from "../../../lib/editor-links.ts"; import { hasUniformLineEndings } from "./chat-sidebar.ts"; @@ -217,3 +217,280 @@ describe("markdown sidebar", () => { panel.remove(); }); }); + +describe("file sidebar clipboard feedback", () => { + const originalExecCommand = Object.getOwnPropertyDescriptor(document, "execCommand"); + const copyActions = [ + { label: "Copy path", value: "src/example.ts" }, + { label: "Copy file contents", value: "const answer = 42;" }, + ]; + + type FilePanel = HTMLElement & { + content: unknown; + ensureFileEditor: () => Promise; + updateComplete: Promise; + }; + + async function mountFilePanel(): Promise { + const panel = document.createElement("openclaw-chat-detail-panel") as FilePanel; + panel.content = { + kind: "file", + path: "src/example.ts", + name: "example.ts", + content: "const answer = 42;", + }; + vi.spyOn(panel, "ensureFileEditor").mockResolvedValue(); + document.body.append(panel); + await panel.updateComplete; + return panel; + } + + function findCopyButton(panel: FilePanel, label: string): HTMLButtonElement { + const button = Array.from(panel.querySelectorAll("button")).find( + (candidate) => candidate.getAttribute("aria-label") === label, + ); + if (!button) { + throw new Error(`Missing sidebar button: ${label}`); + } + return button; + } + + function denyClipboard() { + const writeText = vi.fn().mockRejectedValue(new DOMException("Clipboard access denied")); + const execCommand = vi.fn(() => false); + vi.stubGlobal("navigator", { clipboard: { writeText } }); + Object.defineProperty(document, "execCommand", { configurable: true, value: execCommand }); + return { execCommand, writeText }; + } + + function captureFeedbackTimers() { + const schedule = vi.spyOn(globalThis, "setTimeout"); + return { + schedule, + run(delay: number, index = 0) { + const timerIndex = schedule.mock.calls + .map(([, timeout], callIndex) => (timeout === delay ? callIndex : -1)) + .filter((callIndex) => callIndex >= 0)[index]; + if (timerIndex === undefined) { + throw new Error(`Missing sidebar clipboard reset timer after ${delay}ms`); + } + const reset = schedule.mock.calls[timerIndex]?.[0]; + if (typeof reset !== "function") { + throw new Error(`Expected sidebar clipboard reset timer after ${delay}ms`); + } + globalThis.clearTimeout(schedule.mock.results[timerIndex]?.value); + reset(); + }, + }; + } + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + if (originalExecCommand) { + Object.defineProperty(document, "execCommand", originalExecCommand); + } else { + Reflect.deleteProperty(document, "execCommand"); + } + document.body.replaceChildren(); + }); + + it.each(copyActions)( + "shows and resets a visible accessible error when $label fails", + async ({ label, value }) => { + const { execCommand, writeText } = denyClipboard(); + const panel = await mountFilePanel(); + const button = findCopyButton(panel, label); + const timers = captureFeedbackTimers(); + + button.click(); + await vi.waitFor(() => expect(button.getAttribute("aria-label")).toBe("Copy failed")); + + expect(writeText).toHaveBeenCalledWith(value); + expect(execCommand).toHaveBeenCalledWith("copy"); + expect(panel.querySelector('[role="alert"]')?.textContent).toContain("Copy failed"); + + timers.run(2_000); + await panel.updateComplete; + + expect(button.getAttribute("aria-label")).toBe(label); + expect(panel.querySelector('[role="alert"]')).toBeNull(); + }, + ); + + it.each(copyActions)( + "preserves and resets successful $label feedback", + async ({ label, value }) => { + const writeText = vi.fn().mockResolvedValue(undefined); + vi.stubGlobal("navigator", { clipboard: { writeText } }); + const panel = await mountFilePanel(); + const button = findCopyButton(panel, label); + const timers = captureFeedbackTimers(); + + button.click(); + await vi.waitFor(() => expect(button.getAttribute("aria-label")).toBe("Copied!")); + + expect(writeText).toHaveBeenCalledWith(value); + expect(button.classList.contains("copied")).toBe(true); + expect(panel.querySelector('[role="alert"]')).toBeNull(); + + timers.run(1_500); + await panel.updateComplete; + + expect(button.getAttribute("aria-label")).toBe(label); + expect(button.classList.contains("copied")).toBe(false); + }, + ); + + it.each(copyActions)( + "ignores an older successful $label attempt after a failed retry", + async ({ label }) => { + const { writeText } = denyClipboard(); + let finishFirstCopy = () => {}; + writeText.mockReturnValueOnce( + new Promise((resolve) => { + finishFirstCopy = resolve; + }), + ); + const panel = await mountFilePanel(); + const button = findCopyButton(panel, label); + + button.click(); + button.click(); + await vi.waitFor(() => expect(button.getAttribute("aria-label")).toBe("Copy failed")); + finishFirstCopy(); + await Promise.resolve(); + await Promise.resolve(); + await panel.updateComplete; + + expect(writeText).toHaveBeenCalledTimes(2); + expect(button.getAttribute("aria-label")).toBe("Copy failed"); + expect(panel.querySelector('[role="alert"]')?.textContent).toContain("Copy failed"); + }, + ); + + it("keeps path and contents feedback reset timers independent", async () => { + denyClipboard(); + const panel = await mountFilePanel(); + const pathButton = findCopyButton(panel, "Copy path"); + const contentsButton = findCopyButton(panel, "Copy file contents"); + const timers = captureFeedbackTimers(); + + pathButton.click(); + contentsButton.click(); + await vi.waitFor(() => { + expect(pathButton.getAttribute("aria-label")).toBe("Copy failed"); + expect(contentsButton.getAttribute("aria-label")).toBe("Copy failed"); + }); + + timers.run(2_000); + await panel.updateComplete; + expect(pathButton.getAttribute("aria-label")).toBe("Copy path"); + expect(contentsButton.getAttribute("aria-label")).toBe("Copy failed"); + expect(panel.querySelector('[role="alert"]')?.textContent).toContain("Copy failed"); + + timers.run(2_000, 1); + await panel.updateComplete; + expect(contentsButton.getAttribute("aria-label")).toBe("Copy file contents"); + expect(panel.querySelector('[role="alert"]')).toBeNull(); + }); + + it.each(["file selection", "disconnection"])( + "ignores a delayed successful copy after %s changes its owner", + async (change) => { + let finishCopy = () => {}; + const writeText = vi.fn( + () => + new Promise((resolve) => { + finishCopy = resolve; + }), + ); + vi.stubGlobal("navigator", { clipboard: { writeText } }); + const panel = await mountFilePanel(); + const button = findCopyButton(panel, "Copy file contents"); + const timers = captureFeedbackTimers(); + + button.click(); + if (change === "file selection") { + panel.content = { + kind: "file", + path: "src/next.ts", + name: "next.ts", + content: "const next = true;", + }; + await panel.updateComplete; + } else { + panel.remove(); + } + finishCopy(); + await Promise.resolve(); + await Promise.resolve(); + await panel.updateComplete; + + expect(timers.schedule.mock.calls.some(([, delay]) => delay === 1_500)).toBe(false); + expect(button.getAttribute("aria-label")).toBe("Copy file contents"); + expect(panel.querySelector('[role="alert"]')).toBeNull(); + }, + ); + + it.each([ + { label: "Copy path", failed: true }, + { label: "Copy file contents", failed: false }, + ])( + "restores idle $label feedback when the same sidebar reconnects", + async ({ label, failed }) => { + if (failed) { + denyClipboard(); + } else { + vi.stubGlobal("navigator", { + clipboard: { writeText: vi.fn().mockResolvedValue(undefined) }, + }); + } + const panel = await mountFilePanel(); + const button = findCopyButton(panel, label); + + button.click(); + await vi.waitFor(() => + expect(button.getAttribute("aria-label")).toBe(failed ? "Copy failed" : "Copied!"), + ); + + panel.remove(); + document.body.append(panel); + await panel.updateComplete; + + expect(findCopyButton(panel, label)).toBe(button); + expect(button.classList.contains("copied")).toBe(false); + expect(panel.querySelector('[role="alert"]')).toBeNull(); + }, + ); + + it.each(copyActions)( + "ignores an older $label completion after sidebar reconnection", + async ({ label }) => { + let finishCopy = () => {}; + const writeText = vi.fn( + () => + new Promise((resolve) => { + finishCopy = resolve; + }), + ); + vi.stubGlobal("navigator", { clipboard: { writeText } }); + const panel = await mountFilePanel(); + const button = findCopyButton(panel, label); + const timers = captureFeedbackTimers(); + + button.click(); + panel.remove(); + document.body.append(panel); + await panel.updateComplete; + finishCopy(); + await Promise.resolve(); + await Promise.resolve(); + await panel.updateComplete; + + expect(button.getAttribute("aria-label")).toBe(label); + expect(timers.schedule.mock.calls.some(([, delay]) => delay === 1_500)).toBe(false); + expect(panel.querySelector('[role="alert"]')).toBeNull(); + }, + ); +}); diff --git a/ui/src/pages/chat/components/chat-sidebar.ts b/ui/src/pages/chat/components/chat-sidebar.ts index cbfa66e29523..0fe6841c68df 100644 --- a/ui/src/pages/chat/components/chat-sidebar.ts +++ b/ui/src/pages/chat/components/chat-sidebar.ts @@ -245,8 +245,12 @@ function absoluteFilePath(content: FileSidebarContent): string | null { return `${content.root.replace(/[\\/]+$/, "")}/${content.path.replace(/^[\\/]+/, "")}`; } +type FileCopyAction = "path" | "contents"; +type FileCopyFeedback = Partial>; +const noFileCopyFeedback: FileCopyFeedback = {}; + type FileViewControls = { - copied: boolean; + copyFeedback: FileCopyFeedback; currentMatchIndex: number; dirty: boolean; editorMenuOpen: boolean; @@ -258,7 +262,7 @@ type FileViewControls = { saveNotice: { kind: "conflict" } | { kind: "error"; message: string } | null; saving: boolean; searchOpen: boolean; - onCopyContents: () => void; + onCopy: (action: FileCopyAction) => void; onDiscard: () => void; onEdit: () => void; onNextMatch: () => void; @@ -274,6 +278,31 @@ type FileViewControls = { onToggleSearch: () => void; }; +function renderFileCopyButton(action: FileCopyAction, controls?: FileViewControls) { + const feedback = controls?.copyFeedback[action]; + const label = t( + feedback === "failed" + ? "common.copyFailed" + : feedback === "copied" + ? "common.copied" + : action === "path" + ? "chat.detailPanel.copyPath" + : "chat.detailPanel.copyContents", + ); + return html` + + + + `; +} + function renderFileSidebarContent( content: FileSidebarContent, onViewRawText: () => void, @@ -286,16 +315,7 @@ function renderFileSidebarContent(
${content.path} - - - + ${renderFileCopyButton("path", controls)}
${controls ? html` @@ -366,25 +386,15 @@ function renderFileSidebarContent( onOpenChange: controls.onEditorMenuOpenChange, onOpenEditor: controls.onOpenEditor, })} - - - + ${renderFileCopyButton("contents", controls)} `}
` : nothing}
+ ${Object.values(controls?.copyFeedback ?? {}).includes("failed") + ? html`` + : nothing} ${controls?.searchOpen ? html`
@@ -696,7 +706,7 @@ class ChatDetailPanel extends OpenClawLightDomElement { @state() private fileSearchQuery = ""; @state() private fileSearchMatchIndex = 0; @state() private fileEditorMenuOpen = false; - @state() private fileContentsCopied = false; + @state() private fileCopyFeedback = noFileCopyFeedback; @state() private fileEditorLoading = false; @state() private fileEditing = false; @state() private fileDirty = false; @@ -715,20 +725,22 @@ class ChatDetailPanel extends OpenClawLightDomElement { private fileDraftContent: string | null = null; private fileSavedContent = ""; private fileHash = ""; - private copyFeedbackTimer: ReturnType | null = null; + private readonly copyAttempts = new Map(); + private readonly copyFeedbackTimers = new Map< + FileCopyAction, + ReturnType + >(); override connectedCallback() { super.connectedCallback(); + this.fileCopyFeedback = noFileCopyFeedback; document.addEventListener("pointerdown", this.handleDocumentPointerDown); } override disconnectedCallback() { document.removeEventListener("pointerdown", this.handleDocumentPointerDown); this.destroyFileEditor(); - if (this.copyFeedbackTimer) { - globalThis.clearTimeout(this.copyFeedbackTimer); - this.copyFeedbackTimer = null; - } + this.clearFileCopyFeedback(); super.disconnectedCallback(); } @@ -744,7 +756,8 @@ class ChatDetailPanel extends OpenClawLightDomElement { this.fileSearchQuery = ""; this.fileSearchMatchIndex = 0; this.fileEditorMenuOpen = false; - this.fileContentsCopied = false; + this.clearFileCopyFeedback(); + this.fileCopyFeedback = noFileCopyFeedback; this.fileOperationVersion += 1; this.fileEditing = false; this.fileDirty = false; @@ -771,9 +784,16 @@ class ChatDetailPanel extends OpenClawLightDomElement { this.fileDirty = Boolean(restoredDraft); this.fileEditorLoading = this.content?.kind === "file"; this.destroyFileEditor(); - if (this.copyFeedbackTimer) { - globalThis.clearTimeout(this.copyFeedbackTimer); - this.copyFeedbackTimer = null; + } + + private clearFileCopyFeedback() { + for (const timer of this.copyFeedbackTimers.values()) { + globalThis.clearTimeout(timer); + } + this.copyFeedbackTimers.clear(); + // Keep attempt tokens monotonic so old work cannot become current after reconnection. + for (const [action, attempt] of this.copyAttempts) { + this.copyAttempts.set(action, attempt + 1); } } @@ -977,23 +997,37 @@ class ChatDetailPanel extends OpenClawLightDomElement { openEditor(editor, absPath, content.line); }; - private readonly copyFileContents = () => { + private readonly copyFileValue = (action: FileCopyAction) => { const content = this.visibleContent; if (content?.kind !== "file") { return; } - void copyToClipboard(content.content).then((copied) => { - if (!copied) { + const attempt = (this.copyAttempts.get(action) ?? 0) + 1; + this.copyAttempts.set(action, attempt); + void copyToClipboard(action === "path" ? content.path : content.content).then((copied) => { + // A newer copy or file selection owns feedback; stale completions must stay invisible. + if ( + this.copyAttempts.get(action) !== attempt || + this.visibleContent !== content || + !this.isConnected + ) { return; } - this.fileContentsCopied = true; - if (this.copyFeedbackTimer) { - globalThis.clearTimeout(this.copyFeedbackTimer); - } - this.copyFeedbackTimer = globalThis.setTimeout(() => { - this.copyFeedbackTimer = null; - this.fileContentsCopied = false; - }, 1500); + this.fileCopyFeedback = { + ...this.fileCopyFeedback, + [action]: copied ? "copied" : "failed", + }; + globalThis.clearTimeout(this.copyFeedbackTimers.get(action)); + this.copyFeedbackTimers.set( + action, + globalThis.setTimeout( + () => { + this.copyFeedbackTimers.delete(action); + this.fileCopyFeedback = { ...this.fileCopyFeedback, [action]: undefined }; + }, + copied ? 1500 : 2000, + ), + ); }); }; @@ -1312,7 +1346,7 @@ class ChatDetailPanel extends OpenClawLightDomElement { content: this.visibleContent, error: this.error, fileView: { - copied: this.fileContentsCopied, + copyFeedback: this.fileCopyFeedback, currentMatchIndex, dirty: this.fileDirty, editorMenuOpen: this.fileEditorMenuOpen, @@ -1324,7 +1358,7 @@ class ChatDetailPanel extends OpenClawLightDomElement { saveNotice: this.fileSaveNotice, saving: this.fileSaving, searchOpen: this.fileSearchOpen, - onCopyContents: this.copyFileContents, + onCopy: this.copyFileValue, onDiscard: this.discardFileEdits, onEdit: this.editFile, onNextMatch: () => this.moveFileSearch(1), From ea2c6a63c954d32abb37eddf4a819f65a56a26ed Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 04:40:30 -0700 Subject: [PATCH 35/57] refactor(scripts): adopt shared scaffolding (#118514) * refactor(scripts): adopt shared scaffolding * fix(scripts): satisfy strict tooling checks * fix(scripts): preserve scaffolding contracts --- scripts/android-release-signing.mjs | 78 +++++----- scripts/audit-seams.mjs | 4 +- scripts/bundled-plugin-assets.mjs | 6 +- scripts/check-channel-agnostic-boundaries.mjs | 2 +- scripts/check-cli-startup-memory.mjs | 6 +- .../check-control-ui-precompressed-assets.mjs | 5 +- .../check-database-first-legacy-stores.mjs | 3 +- scripts/check-deprecated-jsdoc.mjs | 4 +- scripts/check-docker-e2e-boundaries.mjs | 5 +- scripts/check-duplicates.mjs | 5 +- scripts/check-dynamic-import-warts.mjs | 8 +- .../check-extension-package-tsc-boundary.mjs | 3 +- .../check-extension-plugin-sdk-boundary.mjs | 3 +- .../check-extension-wildcard-reexports.mjs | 4 +- scripts/check-kysely-guardrails.mjs | 2 +- ...check-plugin-extension-import-boundary.mjs | 3 +- scripts/check-plugin-sdk-subpath-exports.mjs | 5 +- .../check-plugin-sdk-wildcard-reexports.mjs | 4 +- scripts/check-protocol-registry.mjs | 6 +- scripts/check-protocol-since.mjs | 4 +- scripts/check-runtime-sidecar-loaders.mjs | 2 +- scripts/check-session-accessor-boundary.mjs | 2 +- ...eck-session-transcript-reader-boundary.mjs | 2 +- scripts/check-sqlite-transaction-boundary.mjs | 2 +- .../check-telegram-grammy-types-imports.mjs | 4 +- scripts/check-tsgo-core-boundary.mjs | 5 +- .../check-web-fetch-provider-boundaries.mjs | 6 +- .../check-web-search-provider-boundaries.mjs | 5 +- scripts/check.mjs | 38 +++-- scripts/docs-sync-publish.mjs | 6 +- scripts/e2e-sandbox-bind-conflict.mjs | 5 +- scripts/ensure-cli-startup-build.mjs | 6 +- scripts/ensure-extension-memory-build.mjs | 6 +- scripts/ensure-playwright-chromium.mjs | 5 +- scripts/format-docs.mjs | 4 +- scripts/full-release-validation-at-sha.d.mts | 7 - scripts/full-release-validation-at-sha.mjs | 51 ++++--- .../generate-dependency-release-evidence.mjs | 91 ++++-------- ...enerate-host-env-security-policy-swift.mjs | 5 +- scripts/ios-release-signing.mjs | 67 +++++---- scripts/ios-write-swift-filelist.mjs | 4 +- scripts/lib/arg-utils.d.mts | 18 ++- scripts/lib/arg-utils.mjs | 38 +++-- scripts/lib/callsite-guard.mjs | 7 +- .../lib/extension-import-boundary-checker.mjs | 7 +- scripts/lib/pairing-guard-context.mjs | 3 +- scripts/lib/plain-gh.d.mts | 26 ++++ scripts/lib/plain-gh.mjs | 15 +- scripts/lib/repo-root.d.mts | 2 + scripts/lib/repo-root.mjs | 20 +++ scripts/lib/report-cli-helpers.mjs | 64 +++----- scripts/lib/ts-guard-utils.d.mts | 4 - scripts/lib/ts-guard-utils.mjs | 21 +-- scripts/plugin-sdk-surface-report.mjs | 34 +++-- ...e-extension-package-boundary-artifacts.mjs | 4 +- scripts/profile-tsgo.mjs | 4 +- scripts/publish-model-catalog.mjs | 5 +- scripts/release-beta-smoke.ts | 87 ++++++----- scripts/release-candidate-checklist.mjs | 139 +++++++----------- .../resolve-openclaw-package-candidate.mjs | 98 ++++++------ scripts/run-android-gradle.mjs | 4 +- scripts/run-vitest.mjs | 4 +- scripts/runtime-postbuild.mjs | 5 +- scripts/sync-native-a2ui.mjs | 6 +- scripts/test-built-plugin-singleton.mjs | 5 +- scripts/verify-pr-hosted-gates.mjs | 88 +++++------ scripts/verify.mjs | 24 +-- scripts/watch-pr-ci.mjs | 31 ++-- test/scripts/arg-utils.test.ts | 25 ++++ test/scripts/check.test.ts | 14 +- .../full-release-validation-at-sha.test.ts | 26 +--- ...nerate-dependency-release-evidence.test.ts | 12 +- test/scripts/plain-gh.test.ts | 55 +++++++ ...lugin-npm-extended-stable-workflow.test.ts | 5 + .../scripts/plugin-sdk-surface-report.test.ts | 26 ++-- test/scripts/test-force.test.ts | 8 +- test/scripts/test-projects.test.ts | 1 + test/scripts/ts-guard-utils.test.ts | 20 ++- test/scripts/verify.test.ts | 16 +- 79 files changed, 742 insertions(+), 712 deletions(-) create mode 100644 scripts/lib/repo-root.d.mts create mode 100644 scripts/lib/repo-root.mjs diff --git a/scripts/android-release-signing.mjs b/scripts/android-release-signing.mjs index a5be21037df0..34bcba588605 100644 --- a/scripts/android-release-signing.mjs +++ b/scripts/android-release-signing.mjs @@ -2,10 +2,10 @@ import fs from "node:fs"; import path from "node:path"; import process from "node:process"; -import { fileURLToPath } from "node:url"; import { runAndroidSigningCommandSync } from "./lib/android-release-signing-process.mjs"; - -const rootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +import { parseFlagArgs, stringFlag } from "./lib/arg-utils.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const rootDir = resolveRepoRoot(import.meta.url); const defaultManifestPath = path.join(rootDir, "apps", "android", "Config", "ReleaseSigning.json"); const requiredPropertyNames = [ "OPENCLAW_ANDROID_STORE_FILE", @@ -46,33 +46,43 @@ function parseArgs(argv) { keystorePath: process.env.OPENCLAW_ANDROID_UPLOAD_KEYSTORE || "", propertiesPath: process.env.OPENCLAW_ANDROID_SIGNING_PROPERTIES || "", }; - - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - if (arg === "--mode") { - options.mode = readOptionValue(argv, index, arg); - index += 1; - } else if (arg === "--manifest") { - options.manifestPath = path.resolve(readOptionValue(argv, index, arg)); - index += 1; - } else if (arg === "--workspace") { - options.workspace = path.resolve(readOptionValue(argv, index, arg)); - index += 1; - } else if (arg === "--materialized-dir") { - options.materializedDir = path.resolve(readOptionValue(argv, index, arg)); - index += 1; - } else if (arg === "--keystore") { - options.keystorePath = path.resolve(readOptionValue(argv, index, arg)); - index += 1; - } else if (arg === "--properties") { - options.propertiesPath = path.resolve(readOptionValue(argv, index, arg)); - index += 1; - } else if (arg === "-h" || arg === "--help") { - usage(); - process.exit(0); - } else { - throw new Error(`Unknown argument: ${arg}`); - } + const helpIndex = argv.findIndex((arg) => arg === "-h" || arg === "--help"); + parseFlagArgs( + helpIndex === -1 ? argv : argv.slice(0, helpIndex), + options, + [ + stringFlag("--mode", "mode", { + allowInline: false, + missingValueMessage: "Missing value for --mode.", + rejectShortOptions: true, + repeatable: true, + }), + ...[ + ["--manifest", "manifestPath"], + ["--workspace", "workspace"], + ["--materialized-dir", "materializedDir"], + ["--keystore", "keystorePath"], + ["--properties", "propertiesPath"], + ].map(([flag, key]) => + stringFlag(flag, key, { + allowInline: false, + missingValueMessage: `Missing value for ${flag}.`, + rejectShortOptions: true, + repeatable: true, + transform: path.resolve, + }), + ), + ], + { + ignoreDoubleDash: false, + onUnhandledArg(arg) { + throw new Error(`Unknown argument: ${arg}`); + }, + }, + ); + if (helpIndex !== -1) { + usage(); + process.exit(0); } if (!options.mode) { @@ -82,14 +92,6 @@ function parseArgs(argv) { return options; } -function readOptionValue(argv, index, option) { - const value = argv[index + 1] ?? ""; - if (!value || value.startsWith("-")) { - throw new Error(`Missing value for ${option}.`); - } - return value; -} - function requireString(value, key) { if (typeof value !== "string" || value.trim() === "") { throw new Error(`Android release signing manifest missing ${key}.`); diff --git a/scripts/audit-seams.mjs b/scripts/audit-seams.mjs index 2416da640d04..27dc80e9d21a 100644 --- a/scripts/audit-seams.mjs +++ b/scripts/audit-seams.mjs @@ -12,9 +12,9 @@ import { import { visitModuleSpecifiers } from "./lib/guard-inventory-utils.mjs"; import { optionalBundledClusterSet } from "./lib/optional-bundled-clusters.mjs"; import { escapeRegExp } from "./lib/regexp.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { toLine } from "./lib/ts-guard-utils.mjs"; - -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const repoRoot = resolveRepoRoot(import.meta.url); const srcRoot = path.join(repoRoot, "src"); const extensionsRoot = path.join(repoRoot, BUNDLED_PLUGIN_ROOT_DIR); const testRoot = path.join(repoRoot, "test"); diff --git a/scripts/bundled-plugin-assets.mjs b/scripts/bundled-plugin-assets.mjs index 9147bd94f1b1..b70b50a32cfb 100644 --- a/scripts/bundled-plugin-assets.mjs +++ b/scripts/bundled-plugin-assets.mjs @@ -4,12 +4,12 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs/promises"; import path from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; +import { pathToFileURL } from "node:url"; import { runManagedCommand } from "./lib/managed-child-process.mjs"; import { assertRealOutputRoot } from "./lib/output-root-guard.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { listGeneratedExtensionAssetSources } from "./lib/static-extension-assets.mjs"; - -const rootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const rootDir = resolveRepoRoot(import.meta.url); const VALID_PHASES = new Set(["build", "copy"]); // Each complete bundled-plugin asset generator gets the same 10-minute build ceiling. const BUNDLED_PLUGIN_ASSET_HOOK_TIMEOUT_MS = 600_000; diff --git a/scripts/check-channel-agnostic-boundaries.mjs b/scripts/check-channel-agnostic-boundaries.mjs index b3a7be44cf54..682abe6128e7 100644 --- a/scripts/check-channel-agnostic-boundaries.mjs +++ b/scripts/check-channel-agnostic-boundaries.mjs @@ -5,10 +5,10 @@ import { promises as fs } from "node:fs"; import path from "node:path"; import ts from "typescript"; import { visitModuleSpecifiers } from "./lib/guard-inventory-utils.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { collectTypeScriptFiles, getPropertyNameText, - resolveRepoRoot, runAsScript, toLine, } from "./lib/ts-guard-utils.mjs"; diff --git a/scripts/check-cli-startup-memory.mjs b/scripts/check-cli-startup-memory.mjs index 8b547ca7aafd..03d45d5a3b4d 100644 --- a/scripts/check-cli-startup-memory.mjs +++ b/scripts/check-cli-startup-memory.mjs @@ -5,9 +5,9 @@ import { spawnSync as defaultSpawnSync } from "node:child_process"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; - -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +import { pathToFileURL } from "node:url"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const repoRoot = resolveRepoRoot(import.meta.url); const tmpDir = process.env.TMPDIR || process.env.TEMP || process.env.TMP || os.tmpdir(); const MAX_RSS_MARKER = "__OPENCLAW_MAX_RSS_KB__="; const DEFAULT_COMMAND_TIMEOUT_MS = 60_000; diff --git a/scripts/check-control-ui-precompressed-assets.mjs b/scripts/check-control-ui-precompressed-assets.mjs index 38c9dca2a54b..f2fe536c9be5 100644 --- a/scripts/check-control-ui-precompressed-assets.mjs +++ b/scripts/check-control-ui-precompressed-assets.mjs @@ -2,10 +2,9 @@ // Verifies each generated Control UI sidecar encodes the final emitted asset bytes. import fs from "node:fs"; import path from "node:path"; -import { fileURLToPath } from "node:url"; import { brotliDecompressSync, gunzipSync } from "node:zlib"; - -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const repoRoot = resolveRepoRoot(import.meta.url); const assetsDir = path.join(repoRoot, "dist", "control-ui", "assets"); const errors = []; let checked = 0; diff --git a/scripts/check-database-first-legacy-stores.mjs b/scripts/check-database-first-legacy-stores.mjs index f492afa39961..e57277f94849 100644 --- a/scripts/check-database-first-legacy-stores.mjs +++ b/scripts/check-database-first-legacy-stores.mjs @@ -12,7 +12,8 @@ import { mergeLegacyObjectPropertyValues, mergeLegacyPathBranchAssignments, } from "./lib/legacy-store-path-domain.mjs"; -import { resolveRepoRoot, runAsScript, toLine, unwrapExpression } from "./lib/ts-guard-utils.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +import { runAsScript, toLine, unwrapExpression } from "./lib/ts-guard-utils.mjs"; const databaseFirstLegacyStoreSourceRoots = ["src", "extensions", "packages"]; const databaseFirstNativeSourceRoots = ["apps/macos/Sources/OpenClaw"]; diff --git a/scripts/check-deprecated-jsdoc.mjs b/scripts/check-deprecated-jsdoc.mjs index c1a7d86e72ec..52bf9242df78 100644 --- a/scripts/check-deprecated-jsdoc.mjs +++ b/scripts/check-deprecated-jsdoc.mjs @@ -3,12 +3,12 @@ import fs from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; -import { fileURLToPath } from "node:url"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; const require = createRequire(import.meta.url); const ts = require("typescript"); -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const repoRoot = resolveRepoRoot(import.meta.url); const SCAN_ROOTS = ["src", "extensions", "packages"]; const SOURCE_FILE_RE = /\.(?:ts|tsx)$/; const SKIP_PATH_RE = diff --git a/scripts/check-docker-e2e-boundaries.mjs b/scripts/check-docker-e2e-boundaries.mjs index 659e9700b2c5..246c279297a1 100644 --- a/scripts/check-docker-e2e-boundaries.mjs +++ b/scripts/check-docker-e2e-boundaries.mjs @@ -4,7 +4,6 @@ // the source checkout copied or mounted as the app under test. import fs from "node:fs"; import path from "node:path"; -import { fileURLToPath } from "node:url"; import { laneResources, laneWeight } from "./lib/docker-e2e-plan.mjs"; import { allReleasePathLanes, @@ -12,8 +11,8 @@ import { publicInstallerLanes, tailLanes, } from "./lib/docker-e2e-scenarios.mjs"; - -const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const ROOT_DIR = resolveRepoRoot(import.meta.url); const errors = []; const packageJson = JSON.parse(readText("package.json")); const packageScripts = new Set(Object.keys(packageJson.scripts ?? {})); diff --git a/scripts/check-duplicates.mjs b/scripts/check-duplicates.mjs index 892ccc59a7c6..460e14031fcc 100644 --- a/scripts/check-duplicates.mjs +++ b/scripts/check-duplicates.mjs @@ -2,9 +2,8 @@ // Runs duplicate-code detection with repo-specific excludes. import { spawnSync } from "node:child_process"; import path from "node:path"; -import { fileURLToPath } from "node:url"; - -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const repoRoot = resolveRepoRoot(import.meta.url); const jscpdBin = path.join(repoRoot, "node_modules", "jscpd", "bin", "jscpd"); const targets = [ diff --git a/scripts/check-dynamic-import-warts.mjs b/scripts/check-dynamic-import-warts.mjs index 35adcab82e9a..691659580c0a 100644 --- a/scripts/check-dynamic-import-warts.mjs +++ b/scripts/check-dynamic-import-warts.mjs @@ -4,12 +4,8 @@ import { promises as fs } from "node:fs"; import path from "node:path"; import ts from "typescript"; -import { - collectTypeScriptFilesFromRoots, - resolveRepoRoot, - runAsScript, - toLine, -} from "./lib/ts-guard-utils.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +import { collectTypeScriptFilesFromRoots, runAsScript, toLine } from "./lib/ts-guard-utils.mjs"; const repoRoot = resolveRepoRoot(import.meta.url); const defaultRoots = [path.join(repoRoot, "src"), path.join(repoRoot, "extensions")]; diff --git a/scripts/check-extension-package-tsc-boundary.mjs b/scripts/check-extension-package-tsc-boundary.mjs index f5c2a1c7e4ae..f489db0bbbe3 100644 --- a/scripts/check-extension-package-tsc-boundary.mjs +++ b/scripts/check-extension-package-tsc-boundary.mjs @@ -16,6 +16,7 @@ import os from "node:os"; import path, { dirname, join, resolve } from "node:path"; import pMap from "p-map"; import { parsePositiveInt } from "./lib/numeric-options.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { forwardSignalToVitestProcessGroup, installVitestProcessGroupCleanup, @@ -23,7 +24,7 @@ import { } from "./vitest-process-group.mjs"; const require = createRequire(import.meta.url); -const repoRoot = resolve(import.meta.dirname, ".."); +const repoRoot = resolveRepoRoot(import.meta.url); const tscBin = require.resolve("typescript/bin/tsc"); const nativePreviewPackageJsonPath = require.resolve("@typescript/native-preview/package.json"); const nativePreviewPackageJson = JSON.parse(readFileSync(nativePreviewPackageJsonPath, "utf8")); diff --git a/scripts/check-extension-plugin-sdk-boundary.mjs b/scripts/check-extension-plugin-sdk-boundary.mjs index a10d3188575b..9e172c90d2de 100644 --- a/scripts/check-extension-plugin-sdk-boundary.mjs +++ b/scripts/check-extension-plugin-sdk-boundary.mjs @@ -15,8 +15,9 @@ import { resolveRepoSpecifier, writeLine, } from "./lib/guard-inventory-utils.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { listGeneratedExtensionAssetSources } from "./lib/static-extension-assets.mjs"; -import { resolveRepoRoot, runAsScript } from "./lib/ts-guard-utils.mjs"; +import { runAsScript } from "./lib/ts-guard-utils.mjs"; const repoRoot = resolveRepoRoot(import.meta.url); // Generated bundles are validated at their build owner; they are not bounded authored source. diff --git a/scripts/check-extension-wildcard-reexports.mjs b/scripts/check-extension-wildcard-reexports.mjs index 3b9bbebe5000..b6511f7594d1 100644 --- a/scripts/check-extension-wildcard-reexports.mjs +++ b/scripts/check-extension-wildcard-reexports.mjs @@ -4,8 +4,8 @@ import fs from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; - -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const repoRoot = resolveRepoRoot(import.meta.url); const LOCAL_WILDCARD_REEXPORT_PATTERN = /^\s*export\s+(?:type\s+)?\*\s+from\s+["'](?:\.{1,2}\/)/u; diff --git a/scripts/check-kysely-guardrails.mjs b/scripts/check-kysely-guardrails.mjs index de91253a36cf..08bea26594ff 100644 --- a/scripts/check-kysely-guardrails.mjs +++ b/scripts/check-kysely-guardrails.mjs @@ -4,10 +4,10 @@ import { promises as fs } from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { collectTypeScriptFilesFromRoots, getPropertyNameText, - resolveRepoRoot, runAsScript, toLine, unwrapExpression, diff --git a/scripts/check-plugin-extension-import-boundary.mjs b/scripts/check-plugin-extension-import-boundary.mjs index b824675d854d..9931e36f3565 100644 --- a/scripts/check-plugin-extension-import-boundary.mjs +++ b/scripts/check-plugin-extension-import-boundary.mjs @@ -11,7 +11,8 @@ import { runBaselineInventoryCheck, resolveRepoSpecifier, } from "./lib/guard-inventory-utils.mjs"; -import { resolveRepoRoot, runAsScript } from "./lib/ts-guard-utils.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +import { runAsScript } from "./lib/ts-guard-utils.mjs"; const repoRoot = resolveRepoRoot(import.meta.url); const baselinePath = path.join( diff --git a/scripts/check-plugin-sdk-subpath-exports.mjs b/scripts/check-plugin-sdk-subpath-exports.mjs index 1a09eb8e157d..0474d003919f 100644 --- a/scripts/check-plugin-sdk-subpath-exports.mjs +++ b/scripts/check-plugin-sdk-subpath-exports.mjs @@ -3,16 +3,15 @@ // Verifies plugin SDK subpath exports and generated entrypoint metadata. import { readFileSync } from "node:fs"; import path from "node:path"; -import { fileURLToPath } from "node:url"; import ts from "typescript"; import { normalizeRepoPath, visitModuleSpecifiers } from "./lib/guard-inventory-utils.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { collectTypeScriptFilesFromRoots, resolveSourceRoots, toLine, } from "./lib/ts-guard-utils.mjs"; - -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const repoRoot = resolveRepoRoot(import.meta.url); const scanRoots = resolveSourceRoots(repoRoot, [ "src", "packages", diff --git a/scripts/check-plugin-sdk-wildcard-reexports.mjs b/scripts/check-plugin-sdk-wildcard-reexports.mjs index 70c520bde4ee..7839b2b8f86c 100644 --- a/scripts/check-plugin-sdk-wildcard-reexports.mjs +++ b/scripts/check-plugin-sdk-wildcard-reexports.mjs @@ -4,8 +4,8 @@ import fs from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; - -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const repoRoot = resolveRepoRoot(import.meta.url); const extensionsRoot = path.join(repoRoot, "extensions"); const WILDCARD_PLUGIN_SDK_REEXPORT_PATTERN = diff --git a/scripts/check-protocol-registry.mjs b/scripts/check-protocol-registry.mjs index da0f8d7c6825..b74854d3e9d8 100644 --- a/scripts/check-protocol-registry.mjs +++ b/scripts/check-protocol-registry.mjs @@ -1,8 +1,8 @@ import fs from "node:fs"; import path from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; - -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +import { pathToFileURL } from "node:url"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const repoRoot = resolveRepoRoot(import.meta.url); const schemaDir = path.join(repoRoot, "packages/gateway-protocol/src/schema"); const failures = []; const read = (relativePath) => fs.readFileSync(path.join(repoRoot, relativePath), "utf8"); diff --git a/scripts/check-protocol-since.mjs b/scripts/check-protocol-since.mjs index ad08083f5dc9..23b474804df4 100644 --- a/scripts/check-protocol-since.mjs +++ b/scripts/check-protocol-since.mjs @@ -4,12 +4,12 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; -import { fileURLToPath } from "node:url"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; const require = createRequire(import.meta.url); const ts = require("typescript"); -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const repoRoot = resolveRepoRoot(import.meta.url); const descriptorPath = "src/gateway/methods/core-descriptors.ts"; function runGit(args) { diff --git a/scripts/check-runtime-sidecar-loaders.mjs b/scripts/check-runtime-sidecar-loaders.mjs index 5e8bcd5d9cb0..1f8684f60f05 100644 --- a/scripts/check-runtime-sidecar-loaders.mjs +++ b/scripts/check-runtime-sidecar-loaders.mjs @@ -4,9 +4,9 @@ import { promises as fs } from "node:fs"; import path from "node:path"; import ts from "typescript"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { collectTypeScriptFilesFromRoots, - resolveRepoRoot, runAsScript, toLine, unwrapExpression, diff --git a/scripts/check-session-accessor-boundary.mjs b/scripts/check-session-accessor-boundary.mjs index 372c15425be2..7b45b6072c5f 100644 --- a/scripts/check-session-accessor-boundary.mjs +++ b/scripts/check-session-accessor-boundary.mjs @@ -3,10 +3,10 @@ import fs from "node:fs/promises"; import path from "node:path"; import ts from "typescript"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { collectFileViolations, getPropertyNameText, - resolveRepoRoot, resolveSourceRoots, runAsScript, toLine, diff --git a/scripts/check-session-transcript-reader-boundary.mjs b/scripts/check-session-transcript-reader-boundary.mjs index 42d996872aa0..254b25c72ada 100644 --- a/scripts/check-session-transcript-reader-boundary.mjs +++ b/scripts/check-session-transcript-reader-boundary.mjs @@ -2,9 +2,9 @@ import path from "node:path"; import ts from "typescript"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { collectFileViolations, - resolveRepoRoot, resolveSourceRoots, runAsScript, toLine, diff --git a/scripts/check-sqlite-transaction-boundary.mjs b/scripts/check-sqlite-transaction-boundary.mjs index 183f7cfaa6de..c4f7c112c2c2 100644 --- a/scripts/check-sqlite-transaction-boundary.mjs +++ b/scripts/check-sqlite-transaction-boundary.mjs @@ -1,9 +1,9 @@ #!/usr/bin/env node import ts from "typescript"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { collectFileViolations, - resolveRepoRoot, resolveSourceRoots, runAsScript, toLine, diff --git a/scripts/check-telegram-grammy-types-imports.mjs b/scripts/check-telegram-grammy-types-imports.mjs index f3cbd323eef5..76e7d6dd64c0 100644 --- a/scripts/check-telegram-grammy-types-imports.mjs +++ b/scripts/check-telegram-grammy-types-imports.mjs @@ -2,8 +2,8 @@ // Prevents Telegram runtime imports from grammy type-only modules. import { readdirSync, readFileSync } from "node:fs"; import path from "node:path"; - -const repoRoot = path.resolve(import.meta.dirname, ".."); +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const repoRoot = resolveRepoRoot(import.meta.url); const telegramRoot = path.join(repoRoot, "extensions/telegram"); const importSpecifierPatterns = [ /\bimport\s+(?:type\s+)?[\s\S]*?\bfrom\s*["']([^"']+)["']/gu, diff --git a/scripts/check-tsgo-core-boundary.mjs b/scripts/check-tsgo-core-boundary.mjs index bb834e4d0dbe..2bc1fe296c42 100644 --- a/scripts/check-tsgo-core-boundary.mjs +++ b/scripts/check-tsgo-core-boundary.mjs @@ -2,11 +2,10 @@ // Enforces core tsgo project boundaries and sparse-checkout safety. import { spawnSync } from "node:child_process"; -import path from "node:path"; import { resolveRepoToolBinPath } from "./lib/local-heavy-check-runtime.mjs"; import { createManagedCommandInvocation } from "./lib/managed-child-process.mjs"; - -const repoRoot = path.resolve(import.meta.dirname, ".."); +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const repoRoot = resolveRepoRoot(import.meta.url); const tsgoPath = resolveRepoToolBinPath("tsgo", { cwd: repoRoot }); const coreGraphs = [ diff --git a/scripts/check-web-fetch-provider-boundaries.mjs b/scripts/check-web-fetch-provider-boundaries.mjs index e1f5e6eaa01f..170dddbed428 100644 --- a/scripts/check-web-fetch-provider-boundaries.mjs +++ b/scripts/check-web-fetch-provider-boundaries.mjs @@ -1,12 +1,10 @@ #!/usr/bin/env node // Checks core web-fetch surfaces for provider-owned Firecrawl coupling. -import path from "node:path"; -import { fileURLToPath } from "node:url"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { collectSourceFileContents } from "./lib/source-file-scan-cache.mjs"; import { runAsScript } from "./lib/ts-guard-utils.mjs"; - -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const repoRoot = resolveRepoRoot(import.meta.url); const scanExtensions = new Set([".ts", ".js", ".mjs", ".cjs"]); const ignoredDirNames = new Set([ ".artifacts", diff --git a/scripts/check-web-search-provider-boundaries.mjs b/scripts/check-web-search-provider-boundaries.mjs index d7de4fa51e02..9eecd8acba15 100644 --- a/scripts/check-web-search-provider-boundaries.mjs +++ b/scripts/check-web-search-provider-boundaries.mjs @@ -3,12 +3,11 @@ // Inventories core web-search surfaces that still mention bundled providers. import { promises as fs } from "node:fs"; import path from "node:path"; -import { fileURLToPath } from "node:url"; import { diffInventoryEntries, runBaselineInventoryCheck } from "./lib/guard-inventory-utils.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { collectSourceFileContents } from "./lib/source-file-scan-cache.mjs"; import { runAsScript } from "./lib/ts-guard-utils.mjs"; - -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const repoRoot = resolveRepoRoot(import.meta.url); const baselinePath = path.join( repoRoot, "test", diff --git a/scripts/check.mjs b/scripts/check.mjs index 62454a548b36..c239e526940a 100644 --- a/scripts/check.mjs +++ b/scripts/check.mjs @@ -1,5 +1,6 @@ // Runs the repository check lanes selected by CLI arguments. import { performance } from "node:perf_hooks"; +import { booleanFlag, parseFlagArgs } from "./lib/arg-utils.mjs"; import { printTimingSummary } from "./lib/check-timing-summary.mjs"; import { runManagedCommand } from "./lib/managed-child-process.mjs"; @@ -24,26 +25,23 @@ export function usage() { * Parses aggregate check runner arguments. */ function parseCheckArgs(argv) { - const args = { - help: false, - includeArchitecture: false, - includeTestTypes: false, - timed: false, - }; - for (const arg of argv) { - if (arg === "--timed") { - args.timed = true; - } else if (arg === "--include-architecture") { - args.includeArchitecture = true; - } else if (arg === "--include-test-types") { - args.includeTestTypes = true; - } else if (arg === "--help" || arg === "-h") { - args.help = true; - } else { - throw new Error(`unknown argument: ${arg}\n\n${usage()}`); - } - } - return args; + return parseFlagArgs( + argv, + { help: false, includeArchitecture: false, includeTestTypes: false, timed: false }, + [ + booleanFlag("--timed", "timed", true, { repeatable: true }), + booleanFlag("--include-architecture", "includeArchitecture", true, { repeatable: true }), + booleanFlag("--include-test-types", "includeTestTypes", true, { repeatable: true }), + booleanFlag("--help", "help", true, { repeatable: true }), + booleanFlag("-h", "help", true, { repeatable: true }), + ], + { + ignoreDoubleDash: false, + onUnhandledArg(arg) { + throw new Error(`unknown argument: ${arg}\n\n${usage()}`); + }, + }, + ); } /** diff --git a/scripts/docs-sync-publish.mjs b/scripts/docs-sync-publish.mjs index bab5500734ef..b99dd3dad4ee 100644 --- a/scripts/docs-sync-publish.mjs +++ b/scripts/docs-sync-publish.mjs @@ -4,12 +4,12 @@ import { execFileSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; +import { pathToFileURL } from "node:url"; import { renderDocsHeadingMap } from "./docs-list.js"; import { repairMintlifyAccordionIndentation } from "./lib/mintlify-accordion.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; -const HERE = path.dirname(fileURLToPath(import.meta.url)); -const ROOT = path.resolve(HERE, ".."); +const ROOT = resolveRepoRoot(import.meta.url); const SOURCE_DOCS_DIR = path.join(ROOT, "docs"); const SOURCE_CONFIG_PATH = path.join(SOURCE_DOCS_DIR, "docs.json"); const INTERNAL_DOCS_DIRS = ["internal"]; diff --git a/scripts/e2e-sandbox-bind-conflict.mjs b/scripts/e2e-sandbox-bind-conflict.mjs index 38dc2783e93d..2ee73867244d 100644 --- a/scripts/e2e-sandbox-bind-conflict.mjs +++ b/scripts/e2e-sandbox-bind-conflict.mjs @@ -12,10 +12,9 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { fileURLToPath } from "node:url"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; -const scriptDir = path.dirname(fileURLToPath(import.meta.url)); -const repoRoot = path.resolve(scriptDir, ".."); +const repoRoot = resolveRepoRoot(import.meta.url); const engine = process.env.OPENCLAW_SANDBOX_E2E_ENGINE?.trim() || "docker"; const image = process.env.OPENCLAW_SANDBOX_E2E_IMAGE?.trim() || "e2e-sleep:latest"; const useSudo = process.env.OPENCLAW_SANDBOX_E2E_SUDO === "1"; diff --git a/scripts/ensure-cli-startup-build.mjs b/scripts/ensure-cli-startup-build.mjs index 879960b67491..cff45f1e7b05 100644 --- a/scripts/ensure-cli-startup-build.mjs +++ b/scripts/ensure-cli-startup-build.mjs @@ -4,10 +4,10 @@ import { spawnSync } from "node:child_process"; import { existsSync } from "node:fs"; import path from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; +import { pathToFileURL } from "node:url"; import { readPositiveEnvInt } from "./lib/numeric-options.mjs"; - -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const repoRoot = resolveRepoRoot(import.meta.url); const entryCandidates = ["dist/entry.js", "dist/entry.mjs"]; const startupMetadataPath = "dist/cli-startup-metadata.json"; const DEFAULT_BUILD_TIMEOUT_MS = 10 * 60 * 1000; diff --git a/scripts/ensure-extension-memory-build.mjs b/scripts/ensure-extension-memory-build.mjs index 533623d4da9c..b12dd4d37315 100644 --- a/scripts/ensure-extension-memory-build.mjs +++ b/scripts/ensure-extension-memory-build.mjs @@ -4,14 +4,14 @@ import { spawnSync } from "node:child_process"; import { existsSync, readdirSync } from "node:fs"; import path from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; +import { pathToFileURL } from "node:url"; import { collectBundledPluginBuildEntries, NON_PACKAGED_BUNDLED_PLUGIN_DIRS, } from "./lib/bundled-plugin-build-entries.mjs"; import { readPositiveEnvInt } from "./lib/numeric-options.mjs"; - -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const repoRoot = resolveRepoRoot(import.meta.url); const DEFAULT_BUILD_TIMEOUT_MS = 10 * 60 * 1000; /** diff --git a/scripts/ensure-playwright-chromium.mjs b/scripts/ensure-playwright-chromium.mjs index 4c2f0e23c800..9d64b8f783c1 100644 --- a/scripts/ensure-playwright-chromium.mjs +++ b/scripts/ensure-playwright-chromium.mjs @@ -2,12 +2,13 @@ // Ensures Playwright Chromium is installed or a usable system browser is available. import { spawnSync as spawnSyncImpl } from "node:child_process"; import { existsSync as existsSyncImpl, realpathSync } from "node:fs"; -import { dirname, resolve } from "node:path"; +import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { chromium } from "playwright"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { resolvePnpmRunner } from "./pnpm-runner.mjs"; -const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const repoRoot = resolveRepoRoot(import.meta.url); const playwrightInstallBaseArgs = ["--dir", "ui", "exec", "playwright", "install"]; const executableOverrideEnvKey = "PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH"; const chromiumPackageNames = ["chromium-browser", "chromium"]; diff --git a/scripts/format-docs.mjs b/scripts/format-docs.mjs index d802eee5f057..11d99ba5b2ce 100644 --- a/scripts/format-docs.mjs +++ b/scripts/format-docs.mjs @@ -9,9 +9,9 @@ import { pathToFileURL } from "node:url"; import { resolveRepoToolBinPath } from "./lib/local-heavy-check-runtime.mjs"; import { repairMintlifyAccordionIndentation } from "./lib/mintlify-accordion.mjs"; import { outputTail } from "./lib/output-tail.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { buildCmdExeCommandLine, resolveWindowsCmdExePath } from "./windows-cmd-helpers.mjs"; - -const ROOT = path.resolve(import.meta.dirname, ".."); +const ROOT = resolveRepoRoot(import.meta.url); const CHECK = process.argv.includes("--check"); const DOCS_FORMAT_MAX_BUFFER_BYTES = 1024 * 1024 * 16; const DOCS_FORMAT_MAX_COMMAND_LINE_BYTES = 24 * 1024; diff --git a/scripts/full-release-validation-at-sha.d.mts b/scripts/full-release-validation-at-sha.d.mts index c3de6e8bc6df..938401db2cb6 100644 --- a/scripts/full-release-validation-at-sha.d.mts +++ b/scripts/full-release-validation-at-sha.d.mts @@ -19,13 +19,6 @@ export function releaseProfileForTarget( readPackageJson?: (sha: string) => string, ): "beta" | "stable"; export function releaseEvidenceVerificationArgs(parentRunId: unknown): string[]; -export function runGhRead( - args: string[], - params?: { - execFileSyncImpl?: (...args: unknown[]) => unknown; - timeoutMs?: number; - }, -): string; export function shouldDeleteTemporaryWorkflowRef(params: { keepBranch: boolean; dryRun: boolean; diff --git a/scripts/full-release-validation-at-sha.mjs b/scripts/full-release-validation-at-sha.mjs index 3179fd261e10..bc5fd214ca69 100755 --- a/scripts/full-release-validation-at-sha.mjs +++ b/scripts/full-release-validation-at-sha.mjs @@ -5,6 +5,7 @@ import { existsSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; +import { execGhRead } from "./lib/plain-gh.mjs"; const WORKFLOW = "full-release-validation.yml"; const TRUSTED_WORKFLOW_PATH = `.github/workflows/${WORKFLOW}`; @@ -13,6 +14,12 @@ const RELEASE_EVIDENCE_VERIFIER_PATHS = [ ".agents/skills/release-openclaw-ci/scripts/release-ci-summary.mjs", ]; const GH_READ_TIMEOUT_MS = 60_000; +const GH_READ_OPTIONS = { + encoding: "utf8", + killSignal: "SIGKILL", + stdio: ["ignore", "pipe", "inherit"], + timeout: GH_READ_TIMEOUT_MS, +}; const RELEASE_BRANCH_PATTERN = /^(?:release\/[0-9]{4}\.[0-9]+\.[0-9]+|extended-stable\/[0-9]{4}\.[0-9]+\.33)$/u; const RELEASE_TAG_PATTERN = /^v[0-9]{4}\.[0-9]+\.[0-9]+(?:-(?:alpha|beta)\.[0-9]+)?$/u; @@ -62,17 +69,6 @@ function runStatus(command, args, options = {}) { }); } -export function runGhRead(args, params = {}) { - const execFileSyncImpl = params.execFileSyncImpl ?? execFileSync; - const output = execFileSyncImpl("gh", args, { - encoding: "utf8", - killSignal: "SIGKILL", - stdio: ["ignore", "pipe", "inherit"], - timeout: params.timeoutMs ?? GH_READ_TIMEOUT_MS, - }); - return typeof output === "string" ? output.trim() : ""; -} - function readOptionValue(argv, index, optionName) { const value = argv[index + 1]; if (value === undefined || value === "" || value.startsWith("-")) { @@ -264,20 +260,23 @@ function collectRunId(dispatchOutput) { } function findLatestRunId(branch, sha) { - const json = runGhRead([ - "run", - "list", - "--workflow", - WORKFLOW, - "--branch", - branch, - "--event", - "workflow_dispatch", - "--limit", - "20", - "--json", - "databaseId,headSha,createdAt", - ]); + const json = execGhRead( + [ + "run", + "list", + "--workflow", + WORKFLOW, + "--branch", + branch, + "--event", + "workflow_dispatch", + "--limit", + "20", + "--json", + "databaseId,headSha,createdAt", + ], + GH_READ_OPTIONS, + ); const runs = JSON.parse(json); const match = runs.find((runItem) => runItem.headSha === sha); return match?.databaseId ? String(match.databaseId) : ""; @@ -288,7 +287,7 @@ function readWorkflowRun(parentRunId, workflowSha) { throw new Error("parent run ID must be a positive decimal"); } const workflowRun = JSON.parse( - runGhRead(["api", `repos/openclaw/openclaw/actions/runs/${parentRunId}`]), + execGhRead(["api", `repos/openclaw/openclaw/actions/runs/${parentRunId}`], GH_READ_OPTIONS), ); if (workflowRun.head_sha !== workflowSha) { throw new Error( diff --git a/scripts/generate-dependency-release-evidence.mjs b/scripts/generate-dependency-release-evidence.mjs index e6e867d1486b..13fa3791d853 100644 --- a/scripts/generate-dependency-release-evidence.mjs +++ b/scripts/generate-dependency-release-evidence.mjs @@ -5,6 +5,7 @@ import { execFileSync } from "node:child_process"; import { appendFile, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import process from "node:process"; +import { parseFlagArgs, stringFlag } from "./lib/arg-utils.mjs"; /** * Dependency evidence reports generated for release artifacts. @@ -379,14 +380,6 @@ async function generateDependencyReleaseEvidence({ return { manifest, counts, outputDir }; } -function readOptionValue(argv, index, optionName, { allowEmpty = false } = {}) { - const value = argv[index + 1]; - if (value === undefined || value.startsWith("-") || (!allowEmpty && value === "")) { - throw new Error(`Expected ${optionName} .`); - } - return value; -} - function usage() { return `Usage: node scripts/generate-dependency-release-evidence.mjs --output-dir --release-ref --npm-dist-tag [options] @@ -414,60 +407,34 @@ export function parseArgs(argv) { githubOutput: process.env.GITHUB_OUTPUT, githubStepSummary: process.env.GITHUB_STEP_SUMMARY, }; - const seen = new Set(); - const setOnce = (flag, key, value) => { - if (seen.has(flag)) { - throw new Error(`${flag} was provided more than once.`); - } - seen.add(flag); - options[key] = value; - }; - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - if (arg === "--") { - continue; - } - if (arg === "-h" || arg === "--help") { - return { ...options, help: true }; - } - if (arg === "--root") { - setOnce(arg, "rootDir", readOptionValue(argv, index, arg)); - index += 1; - continue; - } - if (arg === "--output-dir") { - setOnce(arg, "outputDir", readOptionValue(argv, index, arg)); - index += 1; - continue; - } - if (arg === "--release-ref") { - setOnce(arg, "releaseRef", readOptionValue(argv, index, arg)); - index += 1; - continue; - } - if (arg === "--npm-dist-tag") { - setOnce(arg, "npmDistTag", readOptionValue(argv, index, arg)); - index += 1; - continue; - } - if (arg === "--base-ref") { - setOnce(arg, "baseRef", readOptionValue(argv, index, arg)); - index += 1; - continue; - } - if (arg === "--github-output") { - setOnce(arg, "githubOutput", readOptionValue(argv, index, arg, { allowEmpty: true })); - index += 1; - continue; - } - if (arg === "--github-step-summary") { - setOnce(arg, "githubStepSummary", readOptionValue(argv, index, arg, { allowEmpty: true })); - index += 1; - continue; - } - throw new Error(`Unsupported argument: ${arg}`); - } - return options; + const helpIndex = argv.findIndex((arg) => arg === "-h" || arg === "--help"); + const parsed = parseFlagArgs( + helpIndex === -1 ? argv : argv.slice(0, helpIndex), + options, + [ + ["--root", "rootDir", false], + ["--output-dir", "outputDir", false], + ["--release-ref", "releaseRef", false], + ["--npm-dist-tag", "npmDistTag", false], + ["--base-ref", "baseRef", false], + ["--github-output", "githubOutput", true], + ["--github-step-summary", "githubStepSummary", true], + ].map(([flag, key, allowEmpty]) => + stringFlag(flag, key, { + allowEmpty, + allowInline: false, + missingValueMessage: `Expected ${flag} .`, + rejectShortOptions: true, + }), + ), + { + duplicateOptionMessage: (flag) => `${flag} was provided more than once.`, + onUnhandledArg(arg) { + throw new Error(`Unsupported argument: ${arg}`); + }, + }, + ); + return helpIndex === -1 ? parsed : { ...parsed, help: true }; } /** diff --git a/scripts/generate-host-env-security-policy-swift.mjs b/scripts/generate-host-env-security-policy-swift.mjs index 7d2f7404edc7..daa2375df6b6 100644 --- a/scripts/generate-host-env-security-policy-swift.mjs +++ b/scripts/generate-host-env-security-policy-swift.mjs @@ -2,8 +2,8 @@ // Generates Swift constants for the host environment security policy. import fs from "node:fs"; import path from "node:path"; -import { fileURLToPath } from "node:url"; import { loadHostEnvSecurityPolicy } from "../src/infra/host-env-security-policy.js"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; const args = new Set(process.argv.slice(2)); const checkOnly = args.has("--check"); @@ -14,8 +14,7 @@ if (checkOnly && args.has("--write")) { process.exit(1); } -const here = path.dirname(fileURLToPath(import.meta.url)); -const repoRoot = path.resolve(here, ".."); +const repoRoot = resolveRepoRoot(import.meta.url); const policyPath = path.join(repoRoot, "src", "infra", "host-env-security-policy.json"); const outputPath = path.join( repoRoot, diff --git a/scripts/ios-release-signing.mjs b/scripts/ios-release-signing.mjs index 42f980e7bf7d..8487932c3409 100755 --- a/scripts/ios-release-signing.mjs +++ b/scripts/ios-release-signing.mjs @@ -2,9 +2,9 @@ import fs from "node:fs"; import path from "node:path"; import process from "node:process"; -import { fileURLToPath } from "node:url"; - -const rootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +import { parseFlagArgs, stringFlag } from "./lib/arg-utils.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const rootDir = resolveRepoRoot(import.meta.url); const defaultManifestPath = path.join(rootDir, "apps", "ios", "Config", "AppStoreSigning.json"); function validateAppGroupId(value, context) { @@ -31,38 +31,41 @@ validates the checked-in manifest and renders local release xcconfig settings. } function parseArgs(argv) { - let mode = ""; - let manifestPath = defaultManifestPath; - - for (let i = 0; i < argv.length; i += 1) { - const arg = argv[i]; - if (arg === "--mode") { - mode = readOptionValue(argv, i, arg); - i += 1; - } else if (arg === "--manifest") { - manifestPath = path.resolve(readOptionValue(argv, i, arg)); - i += 1; - } else if (arg === "-h" || arg === "--help") { - usage(); - process.exit(0); - } else { - throw new Error(`Unknown argument: ${arg}`); - } + const options = { manifestPath: defaultManifestPath, mode: "" }; + const helpIndex = argv.findIndex((arg) => arg === "-h" || arg === "--help"); + parseFlagArgs( + helpIndex === -1 ? argv : argv.slice(0, helpIndex), + options, + [ + stringFlag("--mode", "mode", { + allowInline: false, + missingValueMessage: "Missing value for --mode.", + rejectShortOptions: true, + repeatable: true, + }), + stringFlag("--manifest", "manifestPath", { + allowInline: false, + missingValueMessage: "Missing value for --manifest.", + rejectShortOptions: true, + repeatable: true, + transform: path.resolve, + }), + ], + { + ignoreDoubleDash: false, + onUnhandledArg(arg) { + throw new Error(`Unknown argument: ${arg}`); + }, + }, + ); + if (helpIndex !== -1) { + usage(); + process.exit(0); } - - if (!mode) { + if (!options.mode) { throw new Error("Missing required --mode."); } - - return { mode, manifestPath }; -} - -function readOptionValue(argv, index, option) { - const value = argv[index + 1] ?? ""; - if (!value || value.startsWith("-")) { - throw new Error(`Missing value for ${option}.`); - } - return value; + return options; } function readManifest(manifestPath) { diff --git a/scripts/ios-write-swift-filelist.mjs b/scripts/ios-write-swift-filelist.mjs index 005576415bd6..1be8e371b839 100644 --- a/scripts/ios-write-swift-filelist.mjs +++ b/scripts/ios-write-swift-filelist.mjs @@ -1,8 +1,8 @@ #!/usr/bin/env node import { existsSync, lstatSync, mkdirSync, readdirSync, writeFileSync } from "node:fs"; import path from "node:path"; - -const repoRoot = path.resolve(import.meta.dirname, ".."); +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const repoRoot = resolveRepoRoot(import.meta.url); const iosRoot = path.join(repoRoot, "apps", "ios"); const outputPath = path.join(iosRoot, "SwiftSources.input.xcfilelist"); diff --git a/scripts/lib/arg-utils.d.mts b/scripts/lib/arg-utils.d.mts index e0245a00bfec..249d461ba747 100644 --- a/scripts/lib/arg-utils.d.mts +++ b/scripts/lib/arg-utils.d.mts @@ -17,12 +17,24 @@ export function stripLeadingPackageManagerSeparator(argv: string[]): string[]; export function stringFlag( flag: string, key: string, - options?: { rejectShortOptions?: boolean }, + options?: { + allowEmpty?: boolean; + allowInline?: boolean; + missingValueMessage?: string; + rejectShortOptions?: boolean; + repeatable?: boolean; + transform?: (value: string) => unknown; + }, ): FlagSpec; export function stringListFlag( flag: string, key: string, - options?: { rejectShortOptions?: boolean }, + options?: { + allowEmpty?: boolean; + allowInline?: boolean; + missingValueMessage?: string; + rejectShortOptions?: boolean; + }, ): FlagSpec; export function intFlag( flag: string, @@ -33,6 +45,7 @@ export function booleanFlag( flag: string, key: string, value?: unknown, + options?: { repeatable?: boolean }, ): FlagSpec; export function parseFlagArgs( argv: readonly string[], @@ -40,6 +53,7 @@ export function parseFlagArgs( specs: readonly FlagSpec[], options?: { allowUnknownOptions?: boolean; + duplicateOptionMessage?: (flag: string) => string; ignoreDoubleDash?: boolean; onUnhandledArg?: (arg: string, args: T) => "handled" | void; }, diff --git a/scripts/lib/arg-utils.mjs b/scripts/lib/arg-utils.mjs index 6bd038273ed0..032dbf29b793 100644 --- a/scripts/lib/arg-utils.mjs +++ b/scripts/lib/arg-utils.mjs @@ -1,4 +1,7 @@ // Shared argument parsing helpers for repository scripts. +function failFlagParse(message) { + throw new Error(message); +} /** * Read a flag value from `--flag value` or `--flag=value` arguments. * @internal Shared repository-script contract. @@ -25,7 +28,7 @@ export function stripLeadingPackageManagerSeparator(argv) { } function isMissingStringFlagValue(value, options = {}) { - if (!value) { + if (value === undefined || (!value && options.allowEmpty !== true)) { return true; } if (value.startsWith("--")) { @@ -35,10 +38,10 @@ function isMissingStringFlagValue(value, options = {}) { } function consumeStringFlag(argv, index, flag, options = {}) { - const inlineValue = readInlineFlagValue(argv[index], flag); + const inlineValue = options.allowInline === false ? null : readInlineFlagValue(argv[index], flag); if (inlineValue !== null) { if (isMissingStringFlagValue(inlineValue, options)) { - throw new Error(`${flag} requires a value`); + failFlagParse(options.missingValueMessage ?? `${flag} requires a value`); } return { nextIndex: index, @@ -50,7 +53,7 @@ function consumeStringFlag(argv, index, flag, options = {}) { } const value = argv[index + 1]; if (isMissingStringFlagValue(value, options)) { - throw new Error(`${flag} requires a value`); + failFlagParse(options.missingValueMessage ?? `${flag} requires a value`); } return { nextIndex: index + 1, @@ -66,7 +69,7 @@ function consumeIntFlag(argv, index, flag, options = {}) { const parsed = parseIntegerFlagValue(raw.value, flag); const min = options.min ?? Number.NEGATIVE_INFINITY; if (parsed < min) { - throw new Error(`${flag} must be at least ${min}`); + failFlagParse(`${flag} must be at least ${min}`); } return { nextIndex: raw.nextIndex, @@ -83,7 +86,7 @@ function readFlagOptionValue(argv, index, flag) { const inlineValue = readInlineFlagValue(argv[index], flag); if (inlineValue !== null) { if (!inlineValue) { - throw new Error(`${flag} requires a value`); + failFlagParse(`${flag} requires a value`); } return { nextIndex: index, value: inlineValue }; } @@ -92,7 +95,7 @@ function readFlagOptionValue(argv, index, flag) { } const value = argv[index + 1]; if (!value || value.startsWith("--")) { - throw new Error(`${flag} requires a value`); + failFlagParse(`${flag} requires a value`); } return { nextIndex: index + 1, value }; } @@ -100,11 +103,11 @@ function readFlagOptionValue(argv, index, flag) { function parseIntegerFlagValue(raw, flag) { const text = String(raw).trim(); if (!/^-?\d+$/u.test(text)) { - throw new Error(`${flag} must be an integer`); + failFlagParse(`${flag} must be an integer`); } const parsed = Number(text); if (!Number.isSafeInteger(parsed)) { - throw new Error(`${flag} must be a safe integer`); + failFlagParse(`${flag} must be a safe integer`); } return parsed; } @@ -120,9 +123,9 @@ export function stringFlag(flag, key, options = {}) { return { flag, nextIndex: option.nextIndex, - repeatable: false, + repeatable: options.repeatable === true, apply(target) { - target[key] = option.value; + target[key] = options.transform ? options.transform(option.value) : option.value; }, }; }, @@ -184,7 +187,7 @@ export function intFlag(flag, key, options) { } /** Create a flag spec that assigns a fixed boolean-like value when present. */ -export function booleanFlag(flag, key, value = true) { +export function booleanFlag(flag, key, value = true, options = {}) { return { consume(argv, index) { if (argv[index] !== flag) { @@ -193,7 +196,7 @@ export function booleanFlag(flag, key, value = true) { return { flag, nextIndex: index, - repeatable: false, + repeatable: options.repeatable === true, apply(target) { target[key] = value; }, @@ -218,11 +221,14 @@ export function parseFlagArgs(argv, args, specs, options = {}) { continue; } if (typeof option.flag !== "string" || !option.flag) { - throw new Error("parseFlagArgs specs must declare a flag for consumed options"); + failFlagParse("parseFlagArgs specs must declare a flag for consumed options"); } if (option.repeatable !== true) { if (seenFlags.has(option.flag)) { - throw new Error(`${option.flag} was provided more than once`); + failFlagParse( + options.duplicateOptionMessage?.(option.flag) ?? + `${option.flag} was provided more than once`, + ); } seenFlags.add(option.flag); } @@ -239,7 +245,7 @@ export function parseFlagArgs(argv, args, specs, options = {}) { continue; } if (!options.allowUnknownOptions && arg.startsWith("-")) { - throw new Error(`Unknown option: ${arg}`); + failFlagParse(`Unknown option: ${arg}`); } } return args; diff --git a/scripts/lib/callsite-guard.mjs b/scripts/lib/callsite-guard.mjs index 68f248ffd56b..b84472958a2a 100644 --- a/scripts/lib/callsite-guard.mjs +++ b/scripts/lib/callsite-guard.mjs @@ -1,11 +1,8 @@ // Shared scanner for guard scripts that reject disallowed source callsites. import { promises as fs } from "node:fs"; import path from "node:path"; -import { - collectTypeScriptFilesFromRoots, - resolveRepoRoot, - resolveSourceRoots, -} from "./ts-guard-utils.mjs"; +import { resolveRepoRoot } from "./repo-root.mjs"; +import { collectTypeScriptFilesFromRoots, resolveSourceRoots } from "./ts-guard-utils.mjs"; /** Run a callsite guard over TypeScript roots and exit non-zero on violations. */ export async function runCallsiteGuard(params) { diff --git a/scripts/lib/extension-import-boundary-checker.mjs b/scripts/lib/extension-import-boundary-checker.mjs index 2d42ea90ccef..9c19d74b62e4 100644 --- a/scripts/lib/extension-import-boundary-checker.mjs +++ b/scripts/lib/extension-import-boundary-checker.mjs @@ -10,11 +10,8 @@ import { resolveRepoSpecifier, writeLine, } from "./guard-inventory-utils.mjs"; -import { - collectTypeScriptFilesFromRoots, - resolveRepoRoot, - resolveSourceRoots, -} from "./ts-guard-utils.mjs"; +import { resolveRepoRoot } from "./repo-root.mjs"; +import { collectTypeScriptFilesFromRoots, resolveSourceRoots } from "./ts-guard-utils.mjs"; const repoRoot = resolveRepoRoot(import.meta.url); const DEFAULT_BOUNDARY_SOURCE_MAX_BYTES = 2 * 1024 * 1024; diff --git a/scripts/lib/pairing-guard-context.mjs b/scripts/lib/pairing-guard-context.mjs index 71baf24a04fc..e9d5189811a0 100644 --- a/scripts/lib/pairing-guard-context.mjs +++ b/scripts/lib/pairing-guard-context.mjs @@ -1,6 +1,7 @@ // Builds shared repo/source-root context for pairing guard scripts. import path from "node:path"; -import { resolveRepoRoot, resolveSourceRoots } from "./ts-guard-utils.mjs"; +import { resolveRepoRoot } from "./repo-root.mjs"; +import { resolveSourceRoots } from "./ts-guard-utils.mjs"; /** Create repo root and source root helpers for pairing guard scanners. */ export function createPairingGuardContext(importMetaUrl) { diff --git a/scripts/lib/plain-gh.d.mts b/scripts/lib/plain-gh.d.mts index 5753b212e9a8..dbbaa164b132 100644 --- a/scripts/lib/plain-gh.d.mts +++ b/scripts/lib/plain-gh.d.mts @@ -4,6 +4,12 @@ import type { ExecFileSyncOptionsWithStringEncoding, } from "node:child_process"; +type ExecGhReadImpl = ( + command: string, + args: readonly string[], + options: ExecFileSyncOptions, +) => string | Uint8Array; + export function plainGhEnv(env?: NodeJS.ProcessEnv): { [key: string]: string | undefined; }; @@ -20,6 +26,26 @@ export function execPlainGh( args: readonly string[], options?: ExecFileSyncOptions, ): string | Uint8Array; +export function execGhRead( + args: readonly string[], + options: ExecFileSyncOptionsWithStringEncoding, + params?: { execFileSyncImpl?: ExecGhReadImpl }, +): string; +export function execGhRead( + args: readonly string[], + options?: ExecFileSyncOptionsWithBufferEncoding, + params?: { execFileSyncImpl?: ExecGhReadImpl }, +): Uint8Array; +export function execGhRead( + args: readonly string[], + options?: ExecFileSyncOptions, + params?: { execFileSyncImpl?: ExecGhReadImpl }, +): string | Uint8Array; +export function execGhJson( + args: readonly string[], + options?: ExecFileSyncOptions, + params?: { execFileSyncImpl?: ExecGhReadImpl }, +): unknown; export function execGhApiRead( endpoint: string, options: ExecFileSyncOptionsWithStringEncoding, diff --git a/scripts/lib/plain-gh.mjs b/scripts/lib/plain-gh.mjs index 2f26b43bedc2..5cbee81081a8 100644 --- a/scripts/lib/plain-gh.mjs +++ b/scripts/lib/plain-gh.mjs @@ -88,13 +88,22 @@ export function execPlainGh(args, options = {}) { }); } -export function execGhApiRead(endpoint, options = {}) { +export function execGhRead(args, options = {}, params = {}) { const env = plainGhEnv(options.env ?? process.env); - // Keep reads on the normal PATH shim; OPENCLAW_GH_BIN pins maintainer writes. + // Reads stay on the cache-aware PATH shim; the explicit binary is reserved for writes. delete env.OPENCLAW_GH_BIN; - return execFileSync("gh", ["api", endpoint, "--method", "GET"], { + const execFileSyncImpl = params.execFileSyncImpl ?? execFileSync; + return execFileSyncImpl("gh", args, { ...options, env, maxBuffer: options.maxBuffer ?? PLAIN_GH_MAX_BUFFER_BYTES, }); } + +export function execGhJson(args, options = {}, params = {}) { + return JSON.parse(execGhRead(args, { ...options, encoding: "utf8" }, params)); +} + +export function execGhApiRead(endpoint, options = {}) { + return execGhRead(["api", endpoint, "--method", "GET"], options); +} diff --git a/scripts/lib/repo-root.d.mts b/scripts/lib/repo-root.d.mts new file mode 100644 index 000000000000..e8342a6a2839 --- /dev/null +++ b/scripts/lib/repo-root.d.mts @@ -0,0 +1,2 @@ +/** Resolves the repository root by walking upward from the caller module. */ +export function resolveRepoRoot(importMetaUrl: string): string; diff --git a/scripts/lib/repo-root.mjs b/scripts/lib/repo-root.mjs new file mode 100644 index 000000000000..5b0677eda87b --- /dev/null +++ b/scripts/lib/repo-root.mjs @@ -0,0 +1,20 @@ +import { existsSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +/** Resolves the repository root by walking upward from the caller module. */ +export function resolveRepoRoot(importMetaUrl) { + let dir = path.dirname(fileURLToPath(importMetaUrl)); + const { root } = path.parse(dir); + while (dir !== root) { + if ( + existsSync(path.join(dir, ".git")) || + (existsSync(path.join(dir, "package.json")) && + existsSync(path.join(dir, "pnpm-workspace.yaml"))) + ) { + return dir; + } + dir = path.dirname(dir); + } + return path.resolve(path.dirname(fileURLToPath(importMetaUrl)), "..", ".."); +} diff --git a/scripts/lib/report-cli-helpers.mjs b/scripts/lib/report-cli-helpers.mjs index d934b0604df1..cda4a504c3ff 100644 --- a/scripts/lib/report-cli-helpers.mjs +++ b/scripts/lib/report-cli-helpers.mjs @@ -1,17 +1,7 @@ // Parses report CLI output arguments and writes optional artifacts. import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; - -/** - * Parses shared `--root`, `--json`, and `--markdown` flags for report scripts. - */ -function readReportOptionValue(argv, index, optionName) { - const value = argv[index + 1]; - if (value === undefined || value === "" || value.startsWith("-")) { - throw new Error(`Expected ${optionName} .`); - } - return value; -} +import { parseFlagArgs, stringFlag } from "./arg-utils.mjs"; export function parseReportCliArgs(argv) { const options = { @@ -19,37 +9,27 @@ export function parseReportCliArgs(argv) { jsonPath: null, markdownPath: null, }; - const seen = new Set(); - const setOnce = (flag, key, value) => { - if (seen.has(flag)) { - throw new Error(`${flag} was provided more than once.`); - } - seen.add(flag); - options[key] = value; - }; - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - if (arg === "--") { - continue; - } - if (arg === "--root") { - setOnce(arg, "rootDir", readReportOptionValue(argv, index, arg)); - index += 1; - continue; - } - if (arg === "--json") { - setOnce(arg, "jsonPath", readReportOptionValue(argv, index, arg)); - index += 1; - continue; - } - if (arg === "--markdown") { - setOnce(arg, "markdownPath", readReportOptionValue(argv, index, arg)); - index += 1; - continue; - } - throw new Error(`Unsupported argument: ${arg}`); - } - return options; + return parseFlagArgs( + argv, + options, + [ + ["--root", "rootDir"], + ["--json", "jsonPath"], + ["--markdown", "markdownPath"], + ].map(([flag, key]) => + stringFlag(flag, key, { + allowInline: false, + missingValueMessage: `Expected ${flag} .`, + rejectShortOptions: true, + }), + ), + { + duplicateOptionMessage: (flag) => `${flag} was provided more than once.`, + onUnhandledArg(arg) { + throw new Error(`Unsupported argument: ${arg}`); + }, + }, + ); } /** diff --git a/scripts/lib/ts-guard-utils.d.mts b/scripts/lib/ts-guard-utils.d.mts index 7e66bbd319a8..69cf294f449e 100644 --- a/scripts/lib/ts-guard-utils.d.mts +++ b/scripts/lib/ts-guard-utils.d.mts @@ -1,7 +1,3 @@ -/** - * Resolves the repository root by walking upward from the caller module. - */ -export function resolveRepoRoot(importMetaUrl: string): string; /** * Converts repo-relative source roots into absolute paths. */ diff --git a/scripts/lib/ts-guard-utils.mjs b/scripts/lib/ts-guard-utils.mjs index e355f0d5e160..623bd40cf9af 100644 --- a/scripts/lib/ts-guard-utils.mjs +++ b/scripts/lib/ts-guard-utils.mjs @@ -1,5 +1,5 @@ // Shared TypeScript AST and source-file helpers for guard scripts. -import { existsSync, promises as fs } from "node:fs"; +import { promises as fs } from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -14,25 +14,6 @@ function getTypeScript() { const baseTestSuffixes = [".test.ts", ".test-utils.ts", ".test-harness.ts", ".e2e-harness.ts"]; -/** - * Resolves the repository root by walking upward from the caller module. - */ -export function resolveRepoRoot(importMetaUrl) { - // Walk up from the caller's directory until we find the repo root (.git). - // This handles callers at any depth (scripts/*.mjs, scripts/lib/*.mjs, etc.) - // instead of assuming a fixed number of parent traversals. - let dir = path.dirname(fileURLToPath(importMetaUrl)); - const { root } = path.parse(dir); - while (dir !== root) { - if (existsSync(path.join(dir, ".git"))) { - return dir; - } - dir = path.dirname(dir); - } - // Fallback: two levels up (original behavior). - return path.resolve(path.dirname(fileURLToPath(importMetaUrl)), "..", ".."); -} - /** * Converts repo-relative source roots into absolute paths. */ diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index 7e8e4b3753da..df3e682c535d 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -4,7 +4,8 @@ import fs from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; +import { pathToFileURL } from "node:url"; +import { booleanFlag, parseFlagArgs } from "./lib/arg-utils.mjs"; import { deprecatedBarrelPluginSdkEntrypoints, deprecatedPublicPluginSdkEntrypoints, @@ -13,8 +14,9 @@ import { privateLocalOnlyPluginSdkEntrypoints, publicPluginSdkEntrypoints, } from "./lib/plugin-sdk-entries.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const repoRoot = resolveRepoRoot(import.meta.url); const require = createRequire(import.meta.url); let ts; @@ -30,19 +32,21 @@ Options: } function parsePluginSdkSurfaceReportArgs(argv) { - const args = { check: false, help: false }; - for (const arg of argv) { - if (arg === "--check") { - args.check = true; - continue; - } - if (arg === "--help" || arg === "-h") { - args.help = true; - continue; - } - throw new Error(`Unknown plugin SDK surface report option: ${arg}`); - } - return args; + return parseFlagArgs( + argv, + { check: false, help: false }, + [ + booleanFlag("--check", "check", true, { repeatable: true }), + booleanFlag("--help", "help", true, { repeatable: true }), + booleanFlag("-h", "help", true, { repeatable: true }), + ], + { + ignoreDoubleDash: false, + onUnhandledArg(arg) { + throw new Error(`Unknown plugin SDK surface report option: ${arg}`); + }, + }, + ); } const publicEntrypointSet = new Set(publicPluginSdkEntrypoints); const localOnlyEntrypointSet = new Set(privateLocalOnlyPluginSdkEntrypoints); diff --git a/scripts/prepare-extension-package-boundary-artifacts.mjs b/scripts/prepare-extension-package-boundary-artifacts.mjs index 573770f57b08..0f23499b6569 100644 --- a/scripts/prepare-extension-package-boundary-artifacts.mjs +++ b/scripts/prepare-extension-package-boundary-artifacts.mjs @@ -12,9 +12,9 @@ import { } from "./lib/local-heavy-check-runtime.mjs"; import { parsePositiveInt } from "./lib/numeric-options.mjs"; import { pluginSdkEntrypoints, productionPluginSdkEntrypoints } from "./lib/plugin-sdk-entries.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { resolveWindowsTaskkillPath } from "./lib/windows-taskkill.mjs"; - -const repoRoot = resolve(import.meta.dirname, ".."); +const repoRoot = resolveRepoRoot(import.meta.url); const runTsgoScript = path.join(repoRoot, "scripts/run-tsgo.mjs"); const TYPE_INPUT_EXTENSIONS = new Set([".ts", ".tsx", ".d.ts", ".js", ".mjs", ".json"]); const VALID_MODES = new Set(["all", "package-boundary"]); diff --git a/scripts/profile-tsgo.mjs b/scripts/profile-tsgo.mjs index 1ccb5fbd1702..b88d571a5fb8 100644 --- a/scripts/profile-tsgo.mjs +++ b/scripts/profile-tsgo.mjs @@ -12,8 +12,8 @@ import { shouldAcquireLocalHeavyCheckLockForTsgo, } from "./lib/local-heavy-check-runtime.mjs"; import { createManagedCommandInvocation } from "./lib/managed-child-process.mjs"; - -const repoRoot = path.resolve(import.meta.dirname, ".."); +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const repoRoot = resolveRepoRoot(import.meta.url); const artifactRoot = path.resolve(repoRoot, ".artifacts/tsgo-profile"); const tsgoPath = resolveRepoToolBinPath("tsgo", { cwd: repoRoot }); diff --git a/scripts/publish-model-catalog.mjs b/scripts/publish-model-catalog.mjs index 4f9a63ea47fa..f770d0423d35 100644 --- a/scripts/publish-model-catalog.mjs +++ b/scripts/publish-model-catalog.mjs @@ -1,7 +1,8 @@ import { execFileSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; +import { pathToFileURL } from "node:url"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; const MODEL_CATALOG_MIN_VERSION = "2026.7.0"; export const MODEL_CATALOG_MIN_MODELS = 200; @@ -14,7 +15,7 @@ const PRICING_FETCH_TIMEOUT_MS = 60_000; const MAX_PRICING_CATALOG_BYTES = 5 * 1024 * 1024; const BUNDLE_SIZE_WARNING_BYTES = 2 * 1024 * 1024; const CLIENT_BUNDLE_LIMIT_BYTES = 4 * 1024 * 1024; -const defaultRootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const defaultRootDir = resolveRepoRoot(import.meta.url); function requireOptionValue(args, index, flag) { const value = args[index + 1]?.trim(); diff --git a/scripts/release-beta-smoke.ts b/scripts/release-beta-smoke.ts index a7d33d089975..f1d23d2e55f7 100644 --- a/scripts/release-beta-smoke.ts +++ b/scripts/release-beta-smoke.ts @@ -4,8 +4,14 @@ import { spawnSync } from "node:child_process"; import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; import path from "node:path"; import { pathToFileURL } from "node:url"; +import { + booleanFlag, + parseFlagArgs, + stringFlag, + stripLeadingPackageManagerSeparator, +} from "./lib/arg-utils.mjs"; -interface Options { +type Options = { beta: string; model: string; providerMode: string; @@ -13,7 +19,7 @@ interface Options { repo: string; skipParallels: boolean; skipTelegram: boolean; -} +}; export type RunOptions = { capture?: boolean; @@ -52,6 +58,8 @@ Options: export function parseArgs(argv: string[]): Options { const args = stripLeadingPackageManagerSeparator(argv); + const terminatorIndex = args.indexOf("--"); + const cliArgs = terminatorIndex === -1 ? args : args.slice(0, terminatorIndex); const options: Options = { beta: "beta", model: "openai/gpt-5.4", @@ -61,39 +69,38 @@ export function parseArgs(argv: string[]): Options { skipParallels: false, skipTelegram: false, }; - parseArgv: for (let i = 0; i < args.length; i++) { - const arg = args[i]; - switch (arg) { - case "--": - break parseArgv; - case "--beta": - options.beta = requireValue(args, ++i, arg); - break; - case "--model": - options.model = requireValue(args, ++i, arg); - break; - case "--provider-mode": - options.providerMode = requireValue(args, ++i, arg); - break; - case "--ref": - options.ref = requireValue(args, ++i, arg); - break; - case "--repo": - options.repo = requireValue(args, ++i, arg); - break; - case "--skip-parallels": - options.skipParallels = true; - break; - case "--skip-telegram": - options.skipTelegram = true; - break; - case "-h": - case "--help": - process.stdout.write(usage()); - process.exit(0); - default: + const helpIndex = cliArgs.findIndex((arg) => arg === "-h" || arg === "--help"); + parseFlagArgs( + helpIndex === -1 ? cliArgs : cliArgs.slice(0, helpIndex), + options, + [ + ...( + [ + ["--beta", "beta"], + ["--model", "model"], + ["--provider-mode", "providerMode"], + ["--ref", "ref"], + ["--repo", "repo"], + ] as const + ).map(([flag, key]) => + stringFlag(flag, key, { + allowInline: false, + rejectShortOptions: true, + repeatable: true, + }), + ), + booleanFlag("--skip-parallels", "skipParallels", true, { repeatable: true }), + booleanFlag("--skip-telegram", "skipTelegram", true, { repeatable: true }), + ], + { + onUnhandledArg(arg) { throw new Error(`unknown option: ${arg}`); - } + }, + }, + ); + if (helpIndex !== -1) { + process.stdout.write(usage()); + process.exit(0); } if (options.skipParallels && options.skipTelegram) { throw new Error("--skip-parallels and --skip-telegram cannot be used together"); @@ -101,18 +108,6 @@ export function parseArgs(argv: string[]): Options { return options; } -function stripLeadingPackageManagerSeparator(argv: string[]): string[] { - return argv[0] === "--" ? argv.slice(1) : argv; -} - -function requireValue(argv: string[], index: number, flag: string): string { - const value = argv[index]; - if (!value || value.startsWith("-")) { - throw new Error(`${flag} requires a value`); - } - return value; -} - const CAPTURE_MAX_BUFFER_BYTES = 32 * 1024 * 1024; const DEFAULT_COMMAND_TIMEOUT_MS = readPositiveInt( process.env.OPENCLAW_RELEASE_BETA_SMOKE_COMMAND_MS, diff --git a/scripts/release-candidate-checklist.mjs b/scripts/release-candidate-checklist.mjs index 9e38ad071ed2..a4d19d1f764e 100644 --- a/scripts/release-candidate-checklist.mjs +++ b/scripts/release-candidate-checklist.mjs @@ -17,7 +17,13 @@ import { tmpdir } from "node:os"; import { basename, dirname, join, resolve as resolvePath } from "node:path"; import { fileURLToPath } from "node:url"; import { isDeepStrictEqual } from "node:util"; -import { stripLeadingPackageManagerSeparator } from "./lib/arg-utils.mjs"; +import { + booleanFlag, + parseFlagArgs, + stringFlag, + stringListFlag, + stripLeadingPackageManagerSeparator, +} from "./lib/arg-utils.mjs"; import { readBoundedResponseText } from "./lib/bounded-response.mjs"; import { dedicatedSectionVersionForTag, @@ -109,14 +115,6 @@ Options: `; } -function requireValue(argv, index, flag) { - const value = argv[index]; - if (!value || value.startsWith("-")) { - throw new Error(`${flag} requires a value`); - } - return value; -} - export function releaseBranchForTag(tag) { if (tag.includes("-alpha.")) { return ""; @@ -130,6 +128,8 @@ export function releaseBranchForTag(tag) { */ export function parseArgs(argv) { const args = stripLeadingPackageManagerSeparator(argv); + const terminatorIndex = args.indexOf("--"); + const cliArgs = terminatorIndex === -1 ? args : args.slice(0, terminatorIndex); const options = { repo: DEFAULT_REPO, provider: DEFAULT_PROVIDER, @@ -154,86 +154,49 @@ export function parseArgs(argv) { windowsNodeInstallerDigests: "", outputDir: "", }; - const seen = new Set(); - const setOnce = (flag, key, value) => { - if (seen.has(flag)) { - throw new Error(`${flag} was provided more than once`); - } - seen.add(flag); - options[key] = value; - }; - parseArgv: for (let index = 0; index < args.length; index += 1) { - const arg = args[index]; - switch (arg) { - case "--": - break parseArgv; - case "--tag": - setOnce(arg, "tag", requireValue(args, ++index, arg)); - break; - case "--target-sha": - setOnce(arg, "targetSha", requireValue(args, ++index, arg)); - break; - case "--workflow-ref": - setOnce(arg, "workflowRef", requireValue(args, ++index, arg)); - break; - case "--repo": - setOnce(arg, "repo", requireValue(args, ++index, arg)); - break; - case "--full-release-run": - setOnce(arg, "fullReleaseRunId", requireValue(args, ++index, arg)); - break; - case "--npm-preflight-run": - setOnce(arg, "npmPreflightRunId", requireValue(args, ++index, arg)); - break; - case "--windows-node-tag": - setOnce(arg, "windowsNodeTag", requireValue(args, ++index, arg)); - break; - case "--skip-dispatch": - setOnce(arg, "skipDispatch", true); - break; - case "--skip-local-generated-check": - setOnce(arg, "skipLocalGeneratedCheck", true); - break; - case "--skip-parallels": - setOnce(arg, "skipParallels", true); - break; - case "--parallels-registry-package-artifact": - options.parallelsRegistryPackageArtifactDirs.push(requireValue(args, ++index, arg)); - break; - case "--skip-telegram": - setOnce(arg, "skipTelegram", true); - break; - case "--telegram-provider-mode": - setOnce(arg, "telegramProviderMode", requireValue(args, ++index, arg)); - break; - case "--provider": - setOnce(arg, "provider", requireValue(args, ++index, arg)); - break; - case "--mode": - setOnce(arg, "mode", requireValue(args, ++index, arg)); - break; - case "--release-profile": - setOnce(arg, "releaseProfile", requireValue(args, ++index, arg)); - break; - case "--npm-dist-tag": - setOnce(arg, "npmDistTag", requireValue(args, ++index, arg)); - break; - case "--plugin-publish-scope": - setOnce(arg, "pluginPublishScope", requireValue(args, ++index, arg)); - break; - case "--plugins": - setOnce(arg, "plugins", requireValue(args, ++index, arg)); - break; - case "--output-dir": - setOnce(arg, "outputDir", requireValue(args, ++index, arg)); - break; - case "-h": - case "--help": - process.stdout.write(usage()); - process.exit(0); - default: + const helpIndex = cliArgs.findIndex((arg) => arg === "-h" || arg === "--help"); + parseFlagArgs( + helpIndex === -1 ? cliArgs : cliArgs.slice(0, helpIndex), + options, + [ + ...[ + ["--tag", "tag"], + ["--target-sha", "targetSha"], + ["--workflow-ref", "workflowRef"], + ["--repo", "repo"], + ["--full-release-run", "fullReleaseRunId"], + ["--npm-preflight-run", "npmPreflightRunId"], + ["--windows-node-tag", "windowsNodeTag"], + ["--telegram-provider-mode", "telegramProviderMode"], + ["--provider", "provider"], + ["--mode", "mode"], + ["--release-profile", "releaseProfile"], + ["--npm-dist-tag", "npmDistTag"], + ["--plugin-publish-scope", "pluginPublishScope"], + ["--plugins", "plugins"], + ["--output-dir", "outputDir"], + ].map(([flag, key]) => + stringFlag(flag, key, { allowInline: false, rejectShortOptions: true }), + ), + stringListFlag( + "--parallels-registry-package-artifact", + "parallelsRegistryPackageArtifactDirs", + { allowInline: false, rejectShortOptions: true }, + ), + booleanFlag("--skip-dispatch", "skipDispatch"), + booleanFlag("--skip-local-generated-check", "skipLocalGeneratedCheck"), + booleanFlag("--skip-parallels", "skipParallels"), + booleanFlag("--skip-telegram", "skipTelegram"), + ], + { + onUnhandledArg(arg) { throw new Error(`unknown option: ${arg}`); - } + }, + }, + ); + if (helpIndex !== -1) { + process.stdout.write(usage()); + process.exit(0); } if (!options.tag) { throw new Error("--tag is required"); diff --git a/scripts/resolve-openclaw-package-candidate.mjs b/scripts/resolve-openclaw-package-candidate.mjs index 1d71325814f7..cbd1e91f54f2 100644 --- a/scripts/resolve-openclaw-package-candidate.mjs +++ b/scripts/resolve-openclaw-package-candidate.mjs @@ -13,11 +13,13 @@ import os from "node:os"; import path from "node:path"; import { pipeline } from "node:stream/promises"; import { fileURLToPath } from "node:url"; +import { booleanFlag, parseFlagArgs, stringFlag } from "./lib/arg-utils.mjs"; import { resolveNpmJsonEntries } from "./lib/npm-json-output.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { resolveWindowsTaskkillPath } from "./lib/windows-taskkill.mjs"; import { resolveNpmRunner } from "./npm-runner.mjs"; -const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const ROOT_DIR = resolveRepoRoot(import.meta.url); const DEFAULT_OUTPUT_NAME = "openclaw-current.tgz"; const PACKAGE_URL_DOWNLOAD_TIMEOUT_MS = 60_000; const PACKAGE_URL_MAX_BYTES = 250 * 1024 * 1024; @@ -98,57 +100,49 @@ export function parseArgs(argv) { trustedSourceId: "", trustedSourcePolicy: TRUSTED_PACKAGE_SOURCE_POLICY, }; - const seen = new Set(); - const setOnce = (flag, key, value) => { - if (seen.has(flag)) { - throw new Error(`${flag} was provided more than once`); - } - seen.add(flag); - options[key] = value; - }; - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - const readValue = (name, readOptions = {}) => { - const value = argv[(index += 1)]; - if ( - value === undefined || - (!readOptions.allowEmpty && value === "") || - value.startsWith("-") - ) { - throw new Error(`${name} requires a value`); - } - return value; - }; - if (arg === "--artifact-dir") { - setOnce(arg, "artifactDir", readValue(arg)); - } else if (arg === "--github-output") { - setOnce(arg, "githubOutput", readValue(arg)); - } else if (arg === "--metadata") { - setOnce(arg, "metadata", readValue(arg)); - } else if (arg === "--output-dir") { - setOnce(arg, "outputDir", readValue(arg)); - } else if (arg === "--output-name") { - setOnce(arg, "outputName", readValue(arg)); - } else if (arg === "--package-sha256") { - setOnce(arg, "packageSha256", readValue(arg, { allowEmpty: true }).toLowerCase()); - } else if (arg === "--package-ref") { - setOnce(arg, "packageRef", readValue(arg, { allowEmpty: true })); - } else if (arg === "--package-spec") { - setOnce(arg, "packageSpec", readValue(arg, { allowEmpty: true })); - } else if (arg === "--package-url") { - setOnce(arg, "packageUrl", readValue(arg, { allowEmpty: true })); - } else if (arg === "--source") { - setOnce(arg, "source", readValue(arg)); - } else if (arg === "--trusted-source-id") { - setOnce(arg, "trustedSourceId", readValue(arg, { allowEmpty: true })); - } else if (arg === "--trusted-source-policy") { - setOnce(arg, "trustedSourcePolicy", readValue(arg)); - } else if (arg === "--help" || arg === "-h") { - options.help = true; - } else { - throw new Error(`unknown argument: ${arg}`); - } - } + parseFlagArgs( + argv, + options, + [ + ...[ + ["--artifact-dir", "artifactDir"], + ["--github-output", "githubOutput"], + ["--metadata", "metadata"], + ["--output-dir", "outputDir"], + ["--output-name", "outputName"], + ["--source", "source"], + ["--trusted-source-policy", "trustedSourcePolicy"], + ].map(([flag, key]) => + stringFlag(flag, key, { allowInline: false, rejectShortOptions: true }), + ), + ...[ + ["--package-ref", "packageRef"], + ["--package-spec", "packageSpec"], + ["--package-url", "packageUrl"], + ["--trusted-source-id", "trustedSourceId"], + ].map(([flag, key]) => + stringFlag(flag, key, { + allowEmpty: true, + allowInline: false, + rejectShortOptions: true, + }), + ), + stringFlag("--package-sha256", "packageSha256", { + allowEmpty: true, + allowInline: false, + rejectShortOptions: true, + transform: (value) => value.toLowerCase(), + }), + booleanFlag("--help", "help", true, { repeatable: true }), + booleanFlag("-h", "help", true, { repeatable: true }), + ], + { + ignoreDoubleDash: false, + onUnhandledArg(arg) { + throw new Error(`unknown argument: ${arg}`); + }, + }, + ); validateOutputName(options.outputName); return options; } diff --git a/scripts/run-android-gradle.mjs b/scripts/run-android-gradle.mjs index ecaf841fb535..4d4c89a016d9 100644 --- a/scripts/run-android-gradle.mjs +++ b/scripts/run-android-gradle.mjs @@ -4,9 +4,9 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; -const scriptDir = path.dirname(fileURLToPath(import.meta.url)); -const repoRoot = path.resolve(scriptDir, ".."); +const repoRoot = resolveRepoRoot(import.meta.url); const androidDir = path.join(repoRoot, "apps", "android"); const isMain = process.argv[1] ? path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) diff --git a/scripts/run-vitest.mjs b/scripts/run-vitest.mjs index b1d1f6a0e30c..4f8b6d1abd0c 100644 --- a/scripts/run-vitest.mjs +++ b/scripts/run-vitest.mjs @@ -4,13 +4,13 @@ import { spawn } from "node:child_process"; import fs from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; -import { fileURLToPath } from "node:url"; import { embeddedAgentVitestProjectOwners } from "../test/vitest/vitest.agents-paths.mjs"; import { toolingIsolatedTestFiles } from "../test/vitest/vitest.tooling-isolated-paths.mjs"; import { isUiTestTarget } from "../test/vitest/vitest.ui-paths.mjs"; import { boundaryTestFiles } from "../test/vitest/vitest.unit-paths.mjs"; import { runWithFailedTrailer, writeFailedTrailer } from "./lib/failed-trailer.mjs"; import { signalExitCode } from "./lib/managed-child-process.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { resolveLocalVitestEnv } from "./lib/vitest-local-scheduling.mjs"; import { spawnPnpmRunner } from "./pnpm-runner.mjs"; import { @@ -127,7 +127,7 @@ const VITEST_DOTTED_OPTIONS_WITH_VALUE_PREFIXES = [ "--typecheck.", ]; const require = createRequire(import.meta.url); -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const repoRoot = resolveRepoRoot(import.meta.url); const testProjectsRunnerPath = path.join(repoRoot, "scripts", "test-projects.mjs"); function isTruthyEnvValue(value) { diff --git a/scripts/runtime-postbuild.mjs b/scripts/runtime-postbuild.mjs index 70c93d741feb..a9ce1dafd58b 100644 --- a/scripts/runtime-postbuild.mjs +++ b/scripts/runtime-postbuild.mjs @@ -4,11 +4,12 @@ import fs from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; import { performance } from "node:perf_hooks"; -import { fileURLToPath, pathToFileURL } from "node:url"; +import { pathToFileURL } from "node:url"; import { buildSync } from "esbuild"; import { copyBundledPluginMetadata } from "./copy-bundled-plugin-metadata.mjs"; import { assertRealOutputRoot } from "./lib/output-root-guard.mjs"; import { escapeRegExp } from "./lib/regexp.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { copyStaticExtensionAssets, copyStaticExtensionAssetsToRuntimeOverlay, @@ -21,7 +22,7 @@ import { writeOfficialChannelCatalog } from "./write-official-channel-catalog.mj /** @internal Shared repository-script contract. */ export { listStaticExtensionAssetOutputs }; -const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const ROOT = resolveRepoRoot(import.meta.url); const ROOT_RUNTIME_ALIAS_PATTERN = /^(?.+\.(?:runtime|contract))-[A-Za-z0-9_-]+\.js$/u; const ROOT_STABLE_RUNTIME_ALIAS_PATTERN = /^.+\.(?:runtime|contract)\.js$/u; const ROOT_RUNTIME_IMPORT_SPECIFIER_PATTERN = diff --git a/scripts/sync-native-a2ui.mjs b/scripts/sync-native-a2ui.mjs index 097cf4ea444f..71c1a03ba077 100644 --- a/scripts/sync-native-a2ui.mjs +++ b/scripts/sync-native-a2ui.mjs @@ -5,9 +5,9 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; - -const rootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +import { pathToFileURL } from "node:url"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const rootDir = resolveRepoRoot(import.meta.url); const REQUIRED_RESOURCE_FILES = ["a2ui.bundle.js", "index.html"]; export function getNativeA2uiResourcePaths(repoRoot = rootDir) { diff --git a/scripts/test-built-plugin-singleton.mjs b/scripts/test-built-plugin-singleton.mjs index 55f004613378..8eee009a43f7 100644 --- a/scripts/test-built-plugin-singleton.mjs +++ b/scripts/test-built-plugin-singleton.mjs @@ -3,13 +3,14 @@ import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; +import { pathToFileURL } from "node:url"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; import { installProcessWarningFilter } from "./process-warning-filter.mjs"; import { stageBundledPluginRuntime } from "./stage-bundled-plugin-runtime.mjs"; installProcessWarningFilter(); -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const repoRoot = resolveRepoRoot(import.meta.url); const smokeEntryPath = path.join(repoRoot, "dist", "plugins", "build-smoke-entry.js"); assert.ok(fs.existsSync(smokeEntryPath), `missing build output: ${smokeEntryPath}`); diff --git a/scripts/verify-pr-hosted-gates.mjs b/scripts/verify-pr-hosted-gates.mjs index 35fad2404d81..bb223193f68c 100644 --- a/scripts/verify-pr-hosted-gates.mjs +++ b/scripts/verify-pr-hosted-gates.mjs @@ -2,6 +2,7 @@ import { execFileSync, spawnSync } from "node:child_process"; import { mkdirSync, writeFileSync } from "node:fs"; import path from "node:path"; +import { booleanFlag, parseFlagArgs, stringFlag } from "./lib/arg-utils.mjs"; import { isDirectRunUrl } from "./lib/direct-run.mjs"; import { execGhApiRead, plainGhEnv } from "./lib/plain-gh.mjs"; @@ -31,14 +32,6 @@ const MAX_CI_REUSE_CANDIDATES = 5; const CI_REUSE_RUN_LIST_LIMIT = 50; const GIT_MAX_BUFFER_BYTES = 64 * 1024 * 1024; -function readOptionValue(argv, index, optionName) { - const value = argv[index + 1]; - if (!value || value.startsWith("-")) { - throw new Error(`Expected ${optionName} .`); - } - return value; -} - export function parseArgs(argv) { const args = { repo: "", @@ -48,49 +41,44 @@ export function parseArgs(argv) { output: "", changelogOnly: false, }; - const seen = new Set(); - const setOnce = (flag, key, value) => { - if (seen.has(flag)) { - throw new Error(`${flag} was provided more than once.`); - } - seen.add(flag); - args[key] = value; - }; - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - switch (arg) { - case "--repo": - setOnce(arg, "repo", readOptionValue(argv, index, arg)); - index += 1; - break; - case "--sha": - setOnce(arg, "sha", readOptionValue(argv, index, arg)); - index += 1; - break; - case "--pr": { - const value = Number(readOptionValue(argv, index, arg)); - if (!Number.isSafeInteger(value) || value <= 0) { - throw new Error("Expected --pr ."); - } - setOnce(arg, "pr", value); - index += 1; - break; - } - case "--recent-sha": - setOnce(arg, "recentSha", readOptionValue(argv, index, arg)); - index += 1; - break; - case "--output": - setOnce(arg, "output", readOptionValue(argv, index, arg)); - index += 1; - break; - case "--changelog-only": - setOnce(arg, "changelogOnly", true); - break; - default: + parseFlagArgs( + argv, + args, + [ + ...[ + ["--repo", "repo"], + ["--sha", "sha"], + ["--recent-sha", "recentSha"], + ["--output", "output"], + ].map(([flag, key]) => + stringFlag(flag, key, { + allowInline: false, + missingValueMessage: `Expected ${flag} .`, + rejectShortOptions: true, + }), + ), + stringFlag("--pr", "pr", { + allowInline: false, + missingValueMessage: "Expected --pr .", + rejectShortOptions: true, + transform(value) { + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error("Expected --pr ."); + } + return parsed; + }, + }), + booleanFlag("--changelog-only", "changelogOnly"), + ], + { + duplicateOptionMessage: (flag) => `${flag} was provided more than once.`, + ignoreDoubleDash: false, + onUnhandledArg(arg) { throw new Error(`Unknown option: ${arg}`); - } - } + }, + }, + ); if (!args.repo || !args.sha || !args.pr || !args.output) { throw new Error( "Usage: node scripts/verify-pr-hosted-gates.mjs --repo --sha --pr [--recent-sha ] --output ", diff --git a/scripts/verify.mjs b/scripts/verify.mjs index 4b61524b7099..20e5b151e499 100644 --- a/scripts/verify.mjs +++ b/scripts/verify.mjs @@ -1,5 +1,6 @@ // Runs the broad verification graph used by Crabbox/Testbox: check then test. import { performance } from "node:perf_hooks"; +import { booleanFlag, parseFlagArgs } from "./lib/arg-utils.mjs"; import { formatMs, printTimingSummary } from "./lib/check-timing-summary.mjs"; import { runManagedCommand } from "./lib/managed-child-process.mjs"; @@ -26,15 +27,20 @@ function usage() { * Parses verify wrapper CLI args. */ function parseVerifyArgs(argv) { - const args = { help: false }; - for (const arg of argv) { - if (arg === "--help" || arg === "-h") { - args.help = true; - } else { - throw new Error(`unknown argument: ${arg}\n\n${usage()}`); - } - } - return args; + return parseFlagArgs( + argv, + { help: false }, + [ + booleanFlag("--help", "help", true, { repeatable: true }), + booleanFlag("-h", "help", true, { repeatable: true }), + ], + { + ignoreDoubleDash: false, + onUnhandledArg(arg) { + throw new Error(`unknown argument: ${arg}\n\n${usage()}`); + }, + }, + ); } async function runStage(stage) { diff --git a/scripts/watch-pr-ci.mjs b/scripts/watch-pr-ci.mjs index 598ca6ecd999..36ac82d90635 100644 --- a/scripts/watch-pr-ci.mjs +++ b/scripts/watch-pr-ci.mjs @@ -1,8 +1,8 @@ #!/usr/bin/env node -import { execFileSync } from "node:child_process"; import { setTimeout as sleep } from "node:timers/promises"; import { parseArgs as parseNodeArgs } from "node:util"; import { isDirectRunUrl } from "./lib/direct-run.mjs"; +import { execGhJson } from "./lib/plain-gh.mjs"; const USAGE = "Usage: node scripts/watch-pr-ci.mjs [--repo owner/repo] [--after run-id] [--attach-timeout 900] [--timeout 3600] [--interval 120]"; @@ -15,6 +15,10 @@ const FAILURE_CONCLUSIONS = new Set([ "TIMED_OUT", ]); const ROLLUP_QUERY = `query($owner:String!,$name:String!,$pr:Int!,$cursor:String){repository(owner:$owner,name:$name){pullRequest(number:$pr){state mergeable headRefOid statusCheckRollup{state contexts(first:100,after:$cursor){totalCount pageInfo{hasNextPage endCursor} nodes{kind:__typename ... on CheckRun{name status conclusion databaseId checkSuite{workflowRun{databaseId workflow{databaseId}}}} ... on StatusContext{context state}}}}}}}`; +const GH_READ_OPTIONS = { + stdio: ["ignore", "pipe", "pipe"], + timeout: 60_000, +}; // Adapted from Node's MIT-licensed util.stripVTControlCharacters implementation. const ANSI_ESCAPE_SEQUENCE = new RegExp( "[\\u001B\\u009B][[\\]()#;?]*" + @@ -207,18 +211,11 @@ export function classifyRollup(rollup) { return { verdict: "PENDING", pendingCount, failingNames: [], supersededCount }; } -function ghJson(...args) { - return JSON.parse( - execFileSync("gh", args, { - encoding: "utf8", - stdio: ["ignore", "pipe", "pipe"], - timeout: 60_000, - }), - ); -} - const readPr = (pr, repo) => - ghJson(...`pr view ${pr} --repo ${repo} --json state,mergeable,headRefOid`.split(" ")); + execGhJson( + `pr view ${pr} --repo ${repo} --json state,mergeable,headRefOid`.split(" "), + GH_READ_OPTIONS, + ); export const buildFindRunArgs = (repo, sha) => [ "run", "list", @@ -237,9 +234,13 @@ export const buildFindRunArgs = (repo, sha) => [ ]; export const selectRunAfter = (runs, after) => runs.find((run) => after === undefined || run.databaseId > after); -const findRun = (repo, sha, after) => selectRunAfter(ghJson(...buildFindRunArgs(repo, sha)), after); +const findRun = (repo, sha, after) => + selectRunAfter(execGhJson(buildFindRunArgs(repo, sha), GH_READ_OPTIONS), after); const readRun = (repo, runId) => - ghJson(...`run view ${runId} --repo ${repo} --json status,conclusion`.split(" ")); + execGhJson( + `run view ${runId} --repo ${repo} --json status,conclusion`.split(" "), + GH_READ_OPTIONS, + ); export function classifyRunAttachment(runId, run, after) { if (run.conclusion === "skipped") { @@ -317,7 +318,7 @@ function readRollup(pr, repo) { if (cursor !== null) { queryArgs.push("-f", `cursor=${cursor}`); } - return ghJson(...queryArgs).data?.repository?.pullRequest; + return execGhJson(queryArgs, GH_READ_OPTIONS).data?.repository?.pullRequest; }); } diff --git a/test/scripts/arg-utils.test.ts b/test/scripts/arg-utils.test.ts index f2480af0333d..07b873032c47 100644 --- a/test/scripts/arg-utils.test.ts +++ b/test/scripts/arg-utils.test.ts @@ -38,6 +38,31 @@ describe("scripts/lib/arg-utils parseFlagArgs", () => { expect(parsed.match).toEqual(["alpha", "beta"]); }); + it("supports split-only, empty, transformed, and last-value-wins string contracts", () => { + expect(() => + parseFlagArgs(["--value=inline"], { value: "" }, [ + stringFlag("--value", "value", { allowInline: false }), + ]), + ).toThrow("Unknown option: --value=inline"); + expect( + parseFlagArgs(["--value", "", "--value", "SECOND"], { value: "" }, [ + stringFlag("--value", "value", { + allowEmpty: true, + repeatable: true, + transform: (value) => value.toLowerCase(), + }), + ]).value, + ).toBe("second"); + }); + + it("supports idempotent boolean flags", () => { + expect( + parseFlagArgs(["--verbose", "--verbose"], { verbose: false }, [ + booleanFlag("--verbose", "verbose", true, { repeatable: true }), + ]).verbose, + ).toBe(true); + }); + it("rejects duplicate single-value flags", () => { expect(() => parseFlagArgs(["--label", "first", "--label=second"], { label: "" }, [ diff --git a/test/scripts/check.test.ts b/test/scripts/check.test.ts index 0c6c82819bcb..e942ac2d851c 100644 --- a/test/scripts/check.test.ts +++ b/test/scripts/check.test.ts @@ -21,13 +21,15 @@ describe("scripts/check", () => { }); it("rejects unknown args before running check stages", () => { - const result = runCheck("--bogus"); + for (const args of [["--bogus"], ["bogus", "--help"]]) { + const result = runCheck(...args); - expect(result.status).toBe(2); - expect(result.stdout).toBe(""); - expect(result.stderr).toContain("unknown argument: --bogus"); - expect(result.stderr).toContain("Usage: node scripts/check.mjs"); - expect(result.stderr).not.toContain("[check]"); + expect(result.status).toBe(2); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain(`unknown argument: ${args[0]}`); + expect(result.stderr).toContain("Usage: node scripts/check.mjs"); + expect(result.stderr).not.toContain("[check]"); + } }); it("runs pnpm commands through the managed child runner", async () => { diff --git a/test/scripts/full-release-validation-at-sha.test.ts b/test/scripts/full-release-validation-at-sha.test.ts index 31af38cecab2..26049aea147c 100644 --- a/test/scripts/full-release-validation-at-sha.test.ts +++ b/test/scripts/full-release-validation-at-sha.test.ts @@ -9,7 +9,6 @@ import { releaseEvidenceVerificationArgs, releaseEvidenceVerifierPath, resolveRemoteTargetRefSha, - runGhRead, shouldDeleteTemporaryWorkflowRef, } from "../../scripts/full-release-validation-at-sha.mjs"; @@ -163,30 +162,9 @@ describe("full-release-validation-at-sha", () => { }); it("bounds GitHub reads without applying a timeout to workflow dispatch", () => { - const calls: unknown[][] = []; - expect( - runGhRead(["api", "repos/openclaw/openclaw/actions/runs/123"], { - execFileSyncImpl: (...args: unknown[]) => { - calls.push(args); - return " result "; - }, - }), - ).toBe("result"); - expect(calls).toEqual([ - [ - "gh", - ["api", "repos/openclaw/openclaw/actions/runs/123"], - expect.objectContaining({ - killSignal: "SIGKILL", - timeout: 60_000, - }), - ], - ]); - const source = readFileSync("scripts/full-release-validation-at-sha.mjs", "utf8"); - expect(source).toContain( - 'runGhRead(["api", `repos/openclaw/openclaw/actions/runs/${parentRunId}`])', - ); + expect(source).toContain("timeout: GH_READ_TIMEOUT_MS"); + expect(source.match(/GH_READ_OPTIONS/gu)).toHaveLength(3); expect(source).toContain('const dispatchOutput = run("gh", dispatchArgs'); }); diff --git a/test/scripts/generate-dependency-release-evidence.test.ts b/test/scripts/generate-dependency-release-evidence.test.ts index 163b4f183df4..f8ec23c1ae3c 100644 --- a/test/scripts/generate-dependency-release-evidence.test.ts +++ b/test/scripts/generate-dependency-release-evidence.test.ts @@ -206,12 +206,14 @@ describe("generate-dependency-release-evidence", () => { }); it("reports CLI argument errors without a Node stack trace", () => { - const result = runCli("--wat"); + for (const args of [["--wat"], ["wat", "--help"]]) { + const result = runCli(...args); - expect(result.status).toBe(1); - expect(result.stdout).toBe(""); - expect(result.stderr.trim()).toBe("Unsupported argument: --wat"); - expectNoNodeStack(result.stderr); + expect(result.status).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr.trim()).toBe(`Unsupported argument: ${args[0]}`); + expectNoNodeStack(result.stderr); + } }); it("falls back to fetching tags when local previous-release resolution misses", () => { diff --git a/test/scripts/plain-gh.test.ts b/test/scripts/plain-gh.test.ts index 5767a43c5dc4..cdce4cd52462 100644 --- a/test/scripts/plain-gh.test.ts +++ b/test/scripts/plain-gh.test.ts @@ -6,6 +6,8 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { execGhApiRead, + execGhJson, + execGhRead, execPlainGh, plainGhEnv, PLAIN_GH_SYSTEM_CANDIDATES, @@ -114,6 +116,59 @@ describe("plain gh helpers", () => { expect(output).toContain("OPENCLAW_GH_BIN_SET="); }); + it("shares bounded PATH-shim reads and JSON parsing", () => { + const calls: unknown[][] = []; + const execFileSyncImpl = (...args: unknown[]) => { + calls.push(args); + return '{"ok":true}'; + }; + + expect( + execGhJson( + ["api", "repos/openclaw/openclaw"], + { + killSignal: "SIGKILL", + stdio: ["ignore", "pipe", "inherit"], + timeout: 60_000, + }, + { execFileSyncImpl }, + ), + ).toEqual({ ok: true }); + expect(calls).toEqual([ + [ + "gh", + ["api", "repos/openclaw/openclaw"], + expect.objectContaining({ + encoding: "utf8", + killSignal: "SIGKILL", + maxBuffer: 32 * 1024 * 1024, + stdio: ["ignore", "pipe", "inherit"], + timeout: 60_000, + }), + ], + ]); + expect( + execGhRead( + ["api", "rate_limit"], + { encoding: "utf8" }, + { execFileSyncImpl: () => " result " }, + ), + ).toBe(" result "); + + const failure = new Error("gh read failed"); + expect(() => + execGhRead( + ["api", "rate_limit"], + {}, + { + execFileSyncImpl: () => { + throw failure; + }, + }, + ), + ).toThrow(failure); + }); + it("runs the shell helper with color disabled", () => { const ghPath = makeFakeGh(); const outputPath = path.join(path.dirname(path.dirname(ghPath)), "output.txt"); diff --git a/test/scripts/plugin-npm-extended-stable-workflow.test.ts b/test/scripts/plugin-npm-extended-stable-workflow.test.ts index 23e9d4694a27..7a01cecb29a6 100644 --- a/test/scripts/plugin-npm-extended-stable-workflow.test.ts +++ b/test/scripts/plugin-npm-extended-stable-workflow.test.ts @@ -107,6 +107,11 @@ describe("plugin npm extended-stable workflow", () => { it("overlays the complete trusted packaging helper dependency set", () => { const parsed = workflow(); + const lockGenerator = readFileSync("scripts/generate-npm-package-lock.mjs", "utf8"); + expect(lockGenerator).toContain( + 'path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")', + ); + expect(lockGenerator).not.toContain("./lib/repo-root.mjs"); const preflightCheckout = step( parsed.jobs?.preview_plugin_pack, "Checkout trusted packaging helper", diff --git a/test/scripts/plugin-sdk-surface-report.test.ts b/test/scripts/plugin-sdk-surface-report.test.ts index 0def49d33993..8bfc57cb8444 100644 --- a/test/scripts/plugin-sdk-surface-report.test.ts +++ b/test/scripts/plugin-sdk-surface-report.test.ts @@ -59,19 +59,21 @@ describe("plugin SDK surface report", () => { }); it("rejects unknown CLI options before collecting SDK stats", () => { - const result = spawnSync( - process.execPath, - ["scripts/plugin-sdk-surface-report.mjs", "--chekc"], - { - cwd: process.cwd(), - encoding: "utf8", - }, - ); + for (const args of [["--chekc"], ["chekc", "--help"]]) { + const result = spawnSync( + process.execPath, + ["scripts/plugin-sdk-surface-report.mjs", ...args], + { + cwd: process.cwd(), + encoding: "utf8", + }, + ); - expect(result.status).toBe(1); - expect(result.stdout).toBe(""); - expect(result.stderr.trim()).toBe("Unknown plugin SDK surface report option: --chekc"); - expect(result.stderr).not.toContain("at "); + expect(result.status).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr.trim()).toBe(`Unknown plugin SDK surface report option: ${args[0]}`); + expect(result.stderr).not.toContain("at "); + } }); it("prints help before collecting SDK stats", () => { diff --git a/test/scripts/test-force.test.ts b/test/scripts/test-force.test.ts index 140bb2c9d2b9..d0cb4d4ba49e 100644 --- a/test/scripts/test-force.test.ts +++ b/test/scripts/test-force.test.ts @@ -4,7 +4,7 @@ import { testForceTesting } from "../../scripts/test-force.js"; describe("scripts/test-force.ts", () => { it("prints help without clearing ports or running tests", () => { - const args = testForceTesting.parseArgs(["--help"]); + const args = testForceTesting.parseArgs(["--help", "--bogus"]); expect(args).toEqual({ help: true }); expect(testForceTesting.usage()).toContain("Usage: node --import tsx scripts/test-force.ts"); @@ -16,5 +16,11 @@ describe("scripts/test-force.ts", () => { expect(() => testForceTesting.parseArgs(["--bogus"])).toThrow( /unknown argument: --bogus[\s\S]*Usage: node --import tsx scripts\/test-force\.ts/u, ); + expect(() => testForceTesting.parseArgs(["bogus"])).toThrow( + /unknown argument: bogus[\s\S]*Usage: node --import tsx scripts\/test-force\.ts/u, + ); + expect(() => testForceTesting.parseArgs(["bogus", "--help"])).toThrow( + /unknown argument: bogus[\s\S]*Usage: node --import tsx scripts\/test-force\.ts/u, + ); }); }); diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts index 7867231d50f2..51315d9c9134 100644 --- a/test/scripts/test-projects.test.ts +++ b/test/scripts/test-projects.test.ts @@ -1931,6 +1931,7 @@ describe("scripts/test-projects changed-target routing", () => { "scripts/lib/ts-topology/analyze.ts": ["test/scripts/ts-topology.test.ts"], "scripts/lib/ts-topology/reports.ts": ["test/scripts/ts-topology.test.ts"], "scripts/lib/ts-topology/scope.ts": ["test/scripts/ts-topology.test.ts"], + "scripts/lib/repo-root.mjs": ["test/scripts/ts-guard-utils.test.ts"], "scripts/lib/ts-guard-utils.mjs": ["test/scripts/ts-guard-utils.test.ts"], "scripts/lib/tsgo-sparse-guard.mjs": [ "test/scripts/run-tsgo.test.ts", diff --git a/test/scripts/ts-guard-utils.test.ts b/test/scripts/ts-guard-utils.test.ts index 3a50e99b9721..c7849aac6fb6 100644 --- a/test/scripts/ts-guard-utils.test.ts +++ b/test/scripts/ts-guard-utils.test.ts @@ -1,9 +1,10 @@ // Ts Guard Utils tests cover ts guard utils script behavior. -import { existsSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; import { describe, expect, it } from "vitest"; -import { resolveRepoRoot } from "../../scripts/lib/ts-guard-utils.mjs"; +import { resolveRepoRoot } from "../../scripts/lib/repo-root.mjs"; /** * Regression tests for resolveRepoRoot(). @@ -49,4 +50,19 @@ describe("resolveRepoRoot", () => { expect(fromLib).toBe(fromScripts); expect(fromScripts).toBe(fromExtension); }); + + it("resolves an unpacked workspace without git metadata", () => { + const root = mkdtempSync(path.join(tmpdir(), "openclaw-repo-root-")); + try { + mkdirSync(path.join(root, "scripts", "nested"), { recursive: true }); + writeFileSync(path.join(root, "package.json"), '{"name":"openclaw"}\n'); + writeFileSync(path.join(root, "pnpm-workspace.yaml"), "packages: []\n"); + + expect( + resolveRepoRoot(pathToFileURL(path.join(root, "scripts", "nested", "tool.mjs")).href), + ).toBe(root); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); diff --git a/test/scripts/verify.test.ts b/test/scripts/verify.test.ts index 15faa7f97f7f..03a024f1e210 100644 --- a/test/scripts/verify.test.ts +++ b/test/scripts/verify.test.ts @@ -21,13 +21,15 @@ describe("scripts/verify", () => { }); it("rejects unknown args before running verify stages", () => { - const result = runVerify("--bogus"); + for (const args of [["--bogus"], ["bogus", "--help"]]) { + const result = runVerify(...args); - expect(result.status).toBe(2); - expect(result.stdout).toBe(""); - expect(result.stderr).toContain("unknown argument: --bogus"); - expect(result.stderr).toContain("Usage: node scripts/verify.mjs"); - expect(result.stderr).not.toContain("CRABBOX_PHASE:"); - expect(result.stderr).not.toContain("[verify]"); + expect(result.status).toBe(2); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain(`unknown argument: ${args[0]}`); + expect(result.stderr).toContain("Usage: node scripts/verify.mjs"); + expect(result.stderr).not.toContain("CRABBOX_PHASE:"); + expect(result.stderr).not.toContain("[verify]"); + } }); }); From 8895b0c28858caf9c45f4924bc27db281f672522 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Mon, 3 Aug 2026 19:44:58 +0800 Subject: [PATCH 36/57] fix(codex): preserve yielded subagent handoff (#118644) --- .../src/app-server/dynamic-tools.test.ts | 46 +++++++++++++++++++ .../codex/src/app-server/dynamic-tools.ts | 29 +++++++++++- .../event-projector.commentary.test.ts | 16 ++++++- .../codex/src/app-server/event-projector.ts | 5 +- 4 files changed, 91 insertions(+), 5 deletions(-) diff --git a/extensions/codex/src/app-server/dynamic-tools.test.ts b/extensions/codex/src/app-server/dynamic-tools.test.ts index c4cac0e3d4c9..4dabfaabb60a 100644 --- a/extensions/codex/src/app-server/dynamic-tools.test.ts +++ b/extensions/codex/src/app-server/dynamic-tools.test.ts @@ -444,6 +444,51 @@ describe("createCodexDynamicToolBridge", () => { expect(onAgentToolResult).toHaveBeenCalledWith( expect.objectContaining({ toolName: "sessions_spawn", isError: false }), ); + expect(bridge.telemetry.acceptedSessionSpawns).toEqual([ + { runId: "run_5f3a9c", childSessionKey: "child-7b21" }, + ]); + }); + + it("preserves an accepted sessions_spawn after result middleware strips its details", async () => { + const registry = createEmptyPluginRegistry(); + const handler = vi.fn(async (event: { result: AgentToolResult }) => ({ + result: { + ...event.result, + content: [{ type: "text" as const, text: "Child launch recorded." }], + details: {}, + }, + })); + registry.agentToolResultMiddlewares.push({ + pluginId: "result-compactor", + pluginName: "Result Compactor", + rawHandler: handler, + handler, + runtimes: ["codex"], + source: "test", + }); + setActivePluginRegistry(registry); + const bridge = createBridgeWithToolResult( + "sessions_spawn", + textToolResult("Accepted: launching child session.", { + status: "accepted", + runId: "run_compacted", + childSessionKey: "child-compacted", + }), + ); + + const result = await bridge.handleToolCall({ + threadId: "thread-1", + turnId: "turn-1", + callId: "call-compacted", + namespace: null, + tool: "sessions_spawn", + arguments: { task: "scan logs" }, + }); + + expect(result).toEqual(expectInputText("Child launch recorded.")); + expect(bridge.telemetry.acceptedSessionSpawns).toEqual([ + { runId: "run_compacted", childSessionKey: "child-compacted" }, + ]); }); it("retains only MCP App preview details for OpenClaw transcript projection", async () => { @@ -500,6 +545,7 @@ describe("createCodexDynamicToolBridge", () => { }); expect(result.success).toBe(false); + expect(bridge.telemetry.acceptedSessionSpawns).toEqual([]); }); it("treats accepted goal tool statuses (created / updated) as successful dynamic tool calls", async () => { diff --git a/extensions/codex/src/app-server/dynamic-tools.ts b/extensions/codex/src/app-server/dynamic-tools.ts index 8944c60a96ec..8b1d85fc08d2 100644 --- a/extensions/codex/src/app-server/dynamic-tools.ts +++ b/extensions/codex/src/app-server/dynamic-tools.ts @@ -45,7 +45,11 @@ import { emitTrustedDiagnosticEvent } from "openclaw/plugin-sdk/diagnostic-runti import { expectDefined } from "openclaw/plugin-sdk/expect-runtime"; import type { ImageContent, TextContent } from "openclaw/plugin-sdk/llm"; import { normalizeOpenAIToolSchemas } from "openclaw/plugin-sdk/provider-tools"; -import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { + asOptionalRecord, + isRecord, + normalizeOptionalString, +} from "openclaw/plugin-sdk/string-coerce-runtime"; import { DEFAULT_MAX_LIVE_TOOL_RESULT_CHARS, estimateToolResultTextChars, @@ -381,10 +385,24 @@ export type CodexDynamicToolBridge = { toolMediaUrls: string[]; toolAudioAsVoice: boolean; successfulCronAdds?: number; + acceptedSessionSpawns: Array<{ runId: string; childSessionKey: string }>; quarantinedTools: CodexDynamicToolSchemaQuarantine[]; }; }; +function normalizeAcceptedSessionSpawn(result: unknown): { + runId: string; + childSessionKey: string; +} | null { + const details = asOptionalRecord(asOptionalRecord(result)?.details); + if (!details || details.status !== "accepted") { + return null; + } + const runId = normalizeOptionalString(details.runId); + const childSessionKey = normalizeOptionalString(details.childSessionKey); + return runId && childSessionKey ? { runId, childSessionKey } : null; +} + /** Namespace attached to OpenClaw-owned dynamic tools exposed to Codex. */ const CODEX_OPENCLAW_DYNAMIC_TOOL_NAMESPACE = "openclaw"; @@ -490,6 +508,7 @@ export function createCodexDynamicToolBridge(params: { messagingToolSourceReplyPayloads: [], toolMediaUrls: [], toolAudioAsVoice: false, + acceptedSessionSpawns: [], quarantinedTools, }; const middlewareRunner = createAgentToolResultMiddlewareRunner({ @@ -654,6 +673,14 @@ export function createCodexDynamicToolBridge(params: { result: middlewareResult, }); const resultIsError = rawIsError || isToolResultError(result); + // A successful spawn is durable before presentation middleware can rewrite details. + const acceptedSessionSpawn = + toolName === "sessions_spawn" && !rawIsError + ? normalizeAcceptedSessionSpawn(telemetryRawResult) + : null; + if (acceptedSessionSpawn) { + telemetry.acceptedSessionSpawns.push(acceptedSessionSpawn); + } const finalResultFailureKind = resolveToolResultFailureKind(result); const resultFailureKind = rawResultFailureKind ?? finalResultFailureKind; const observerResult = diff --git a/extensions/codex/src/app-server/event-projector.commentary.test.ts b/extensions/codex/src/app-server/event-projector.commentary.test.ts index f9acff07f43c..8a493017f79b 100644 --- a/extensions/codex/src/app-server/event-projector.commentary.test.ts +++ b/extensions/codex/src/app-server/event-projector.commentary.test.ts @@ -647,7 +647,7 @@ describe("CodexAppServerEventProjector commentary projection", () => { expect(result.lastAssistant).toBeUndefined(); }); - it("preserves sessions_yield detection in attempt results", () => { + it("preserves accepted session spawns as yield continuation evidence", () => { const projector = new CodexAppServerEventProjector( { prompt: "hello", @@ -664,8 +664,20 @@ describe("CodexAppServerEventProjector commentary projection", () => { TURN_ID, ); - const result = projector.buildResult(buildEmptyToolTelemetry(), { yieldDetected: true }); + const result = projector.buildResult( + { + ...buildEmptyToolTelemetry(), + acceptedSessionSpawns: [ + { runId: "child-run", childSessionKey: "agent:main:subagent:child" }, + ], + }, + { yieldDetected: true }, + ); expect(result.yieldDetected).toBe(true); + expect(result.acceptedSessionSpawns).toEqual([ + { runId: "child-run", childSessionKey: "agent:main:subagent:child" }, + ]); + expect(result.replayMetadata).toEqual({ hadPotentialSideEffects: true, replaySafe: false }); }); }); diff --git a/extensions/codex/src/app-server/event-projector.ts b/extensions/codex/src/app-server/event-projector.ts index a95e2b11e91b..7090958aced2 100644 --- a/extensions/codex/src/app-server/event-projector.ts +++ b/extensions/codex/src/app-server/event-projector.ts @@ -73,7 +73,7 @@ type CodexAppServerToolTelemetry = { toolMediaUrls?: string[]; toolAudioAsVoice?: boolean; successfulCronAdds?: number; -}; +} & Pick; export class CodexAppServerEventProjector { private readonly assistantProjection: CodexAssistantProjection; @@ -403,7 +403,7 @@ export class CodexAppServerEventProjector { const toolMetas = this.toolProgressProjection.toolMetas; const hadPotentialSideEffects = toolTelemetry.didSendViaMessagingTool || - (toolTelemetry.successfulCronAdds ?? 0) > 0 || + Boolean(toolTelemetry.successfulCronAdds || toolTelemetry.acceptedSessionSpawns?.length) || this.generatedMediaProjection.hasGeneratedMedia() || this.toolProgressProjection.hasPotentialSideEffects; return { @@ -439,6 +439,7 @@ export class CodexAppServerEventProjector { hostOwnedToolMediaUrls: this.generatedMediaProjection.buildHostOwnedMediaUrls(toolTelemetry), toolAudioAsVoice: toolTelemetry.toolAudioAsVoice, successfulCronAdds: toolTelemetry.successfulCronAdds, + acceptedSessionSpawns: toolTelemetry.acceptedSessionSpawns, cloudCodeAssistFormatError: false, attemptUsage: projectedUsage, ...(this.completedCompactionCount > 0 From 6addeae3ffe64f3ad994dc0cda581b6c6479e3e3 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 04:52:39 -0700 Subject: [PATCH 37/57] test(agents): reuse attempt execution SQLite fixtures (#118646) * test(agents): reuse attempt execution sqlite fixtures * test(agents): track attempt fixture temp roots --------- Co-authored-by: Peter Steinberger --- .../command/attempt-execution.cli.test.ts | 138 ++++++++++++------ 1 file changed, 94 insertions(+), 44 deletions(-) diff --git a/src/agents/command/attempt-execution.cli.test.ts b/src/agents/command/attempt-execution.cli.test.ts index 79b703349d1e..10fd2d639e50 100644 --- a/src/agents/command/attempt-execution.cli.test.ts +++ b/src/agents/command/attempt-execution.cli.test.ts @@ -2,7 +2,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { SessionEntry } from "../../config/sessions.js"; import { formatSqliteSessionFileMarker, @@ -19,11 +19,19 @@ import { clearSessionStoreCacheForTest } from "../../config/sessions/store-write import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { createUserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js"; import { createTestUserTurnTranscriptTarget } from "../../sessions/user-turn-transcript.test-support.js"; -import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js"; +import { + disposeOpenClawAgentDatabaseByPath, + listOpenClawAgentDatabasesForTest, + runOpenClawAgentWriteTransaction, +} from "../../state/openclaw-agent-db.js"; import { registerGeneratedMediaTaskActivity } from "../../tasks/generated-media-task-activity.js"; import { resetGeneratedMediaTaskActivityForTests } from "../../tasks/task-runtime.test-helpers.js"; +import { createSuiteTempRootTracker } from "../../test-helpers/temp-dir.js"; import { captureEnv, setTestEnvValue } from "../../test-utils/env.js"; -import { saveAuthProfileStore } from "../auth-profiles/store.js"; +import { + clearRuntimeAuthProfileStoreSnapshots, + saveAuthProfileStore, +} from "../auth-profiles/store.js"; import { testing as cliBackendsTesting } from "../cli-backends.test-support.js"; import type { EmbeddedAgentRunResult } from "../embedded-agent.js"; import { FailoverError } from "../failover-error.js"; @@ -501,10 +509,20 @@ function firstEmbeddedAgentArg(callIndex = 0) { } describe("CLI attempt execution", () => { + const fixtureRoot = createSuiteTempRootTracker({ prefix: "openclaw-cli-attempt-suite-" }); + let suiteRoot: string; + let agentDir: string; let tmpDir: string; let storePath: string; let homeEnvSnapshot: ReturnType | undefined; + beforeAll(async () => { + suiteRoot = await fixtureRoot.setup(); + agentDir = path.join(suiteRoot, "agents", "main", "agent"); + storePath = path.join(suiteRoot, "sessions.json"); + await fs.mkdir(agentDir, { recursive: true }); + }); + async function runOpenClawEmbeddedAttemptForTest(overrides?: { opts?: Partial; config?: OpenClawConfig; @@ -585,7 +603,7 @@ describe("CLI attempt execution", () => { messageChannel: "telegram", skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: providerOverride, sessionStore, @@ -599,8 +617,8 @@ describe("CLI attempt execution", () => { beforeEach(async () => { homeEnvSnapshot = captureEnv(["HOME", "OPENCLAW_STATE_DIR"]); - tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-cli-attempt-")); - storePath = path.join(tmpDir, "sessions.json"); + setTestEnvValue("OPENCLAW_STATE_DIR", suiteRoot); + tmpDir = await fixtureRoot.make(); runCliAgentMock.mockReset(); runEmbeddedAgentMock.mockReset(); resetGeneratedMediaTaskActivityForTests(); @@ -633,7 +651,6 @@ describe("CLI attempt execution", () => { for (const [sessionKey, entry] of Object.entries(sessionStore)) { await replaceSessionEntry({ sessionKey, storePath }, entry); } - closeOpenClawAgentDatabasesForTest(); } function createSubagentAnnounceSessionStore( @@ -668,10 +685,43 @@ describe("CLI attempt execution", () => { afterEach(async () => { vi.useRealTimers(); cliBackendsTesting.resetDepsForTest(); - closeOpenClawAgentDatabasesForTest(); + clearRuntimeAuthProfileStoreSnapshots(); + clearSessionStoreCacheForTest(); + for (const database of listOpenClawAgentDatabasesForTest()) { + if (!database.path.startsWith(`${suiteRoot}${path.sep}`)) { + continue; + } + runOpenClawAgentWriteTransaction( + (fixture) => { + fixture.db.exec(` + DELETE FROM session_transcript_fts; + DELETE FROM session_nodes; + DELETE FROM conversations; + DELETE FROM auth_profile_store; + DELETE FROM auth_profile_state; + DELETE FROM cache_entries; + DELETE FROM state_leases; + `); + }, + database, + { operationLabel: "test.attempt-execution.reset" }, + ); + } + await fs.rm(tmpDir, { recursive: true, force: true }); + await fs.rm(storePath, { force: true }); homeEnvSnapshot?.restore(); homeEnvSnapshot = undefined; - await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + afterAll(async () => { + for (const database of listOpenClawAgentDatabasesForTest()) { + if (database.path.startsWith(`${suiteRoot}${path.sep}`)) { + disposeOpenClawAgentDatabaseByPath(database.path, { + env: { OPENCLAW_STATE_DIR: suiteRoot }, + }); + } + } + await fixtureRoot.cleanup(); }); it("forwards explicit local-agent timeouts while preserving the default when omitted", async () => { @@ -742,7 +792,7 @@ describe("CLI attempt execution", () => { messageChannel: undefined, skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "claude-cli", sessionStore: params.sessionStore, @@ -881,7 +931,7 @@ describe("CLI attempt execution", () => { messageChannel: undefined, skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "claude-cli", sessionStore, @@ -1278,7 +1328,7 @@ describe("CLI attempt execution", () => { messageChannel: undefined, skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "claude-cli", sessionHasHistory: false, @@ -1556,7 +1606,7 @@ describe("CLI attempt execution", () => { messageChannel: undefined, skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "openai", sessionStore, @@ -1608,7 +1658,7 @@ describe("CLI attempt execution", () => { messageChannel: undefined, skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "openai-codex", sessionStore, @@ -1641,7 +1691,7 @@ describe("CLI attempt execution", () => { }, }, }, - tmpDir, + agentDir, { filterExternalAuthProfiles: false, syncExternalCli: false }, ); runCliAgentMock.mockResolvedValueOnce(makeCliResult("gemini cli response")); @@ -1683,7 +1733,7 @@ describe("CLI attempt execution", () => { messageChannel: undefined, skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "google", sessionStore, @@ -1716,7 +1766,7 @@ describe("CLI attempt execution", () => { }, }, }, - tmpDir, + agentDir, { filterExternalAuthProfiles: false, syncExternalCli: false }, ); runCliAgentMock.mockResolvedValueOnce(makeCliResult("gemini cli api-key response")); @@ -1753,7 +1803,7 @@ describe("CLI attempt execution", () => { messageChannel: undefined, skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "google", sessionStore, @@ -1786,7 +1836,7 @@ describe("CLI attempt execution", () => { }, }, }, - tmpDir, + agentDir, { filterExternalAuthProfiles: false, syncExternalCli: false }, ); expect(() => @@ -1822,7 +1872,7 @@ describe("CLI attempt execution", () => { messageChannel: undefined, skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "google", sessionStore, @@ -1861,7 +1911,7 @@ describe("CLI attempt execution", () => { }, }, }, - tmpDir, + agentDir, { filterExternalAuthProfiles: false, syncExternalCli: false }, ); runCliAgentMock.mockResolvedValueOnce(makeCliResult("gemini cli api-key response")); @@ -1903,7 +1953,7 @@ describe("CLI attempt execution", () => { messageChannel: undefined, skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "google", sessionStore, @@ -1934,7 +1984,7 @@ describe("CLI attempt execution", () => { }, }, }, - tmpDir, + agentDir, { filterExternalAuthProfiles: false, syncExternalCli: false }, ); runCliAgentMock.mockResolvedValueOnce(makeCliResult("gemini cli api-key response")); @@ -1976,7 +2026,7 @@ describe("CLI attempt execution", () => { messageChannel: undefined, skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "google", sessionStore, @@ -1997,7 +2047,7 @@ describe("CLI attempt execution", () => { const sessionKey = `agent:main:internal-session-effects:${visibleSessionId}`; setTestEnvValue("HOME", tmpDir); setTestEnvValue("OPENCLAW_STATE_DIR", path.join(tmpDir, "state")); - const internalStorePath = path.join(tmpDir, "sessions.json"); + const internalStorePath = storePath; const internalSessionFile = formatSqliteSessionFileMarker({ agentId: "main", sessionId, @@ -2643,7 +2693,7 @@ describe("CLI attempt execution", () => { messageChannel: "discord", skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "claude-cli", sessionStore, @@ -2700,7 +2750,7 @@ describe("CLI attempt execution", () => { messageChannel: "discord", skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "claude-cli", sessionStore, @@ -2752,7 +2802,7 @@ describe("CLI attempt execution", () => { messageChannel: undefined, skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "claude-cli", sessionStore, @@ -2782,7 +2832,7 @@ describe("CLI attempt execution", () => { }, }, }, - tmpDir, + agentDir, { filterExternalAuthProfiles: false, syncExternalCli: false }, ); runCliAgentMock.mockResolvedValueOnce(makeCliResult("configured claude cli")); @@ -2835,7 +2885,7 @@ describe("CLI attempt execution", () => { messageChannel: undefined, skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "anthropic", sessionStore, @@ -2898,7 +2948,7 @@ describe("CLI attempt execution", () => { messageChannel: "discord", skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "claude-cli", sessionStore, @@ -2975,7 +3025,7 @@ describe("CLI attempt execution", () => { messageChannel: "telegram", skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "claude-cli", sessionStore, @@ -3129,7 +3179,7 @@ describe("CLI attempt execution", () => { messageChannel: "telegram", skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "claude-cli", sessionStore, @@ -3190,7 +3240,7 @@ describe("CLI attempt execution", () => { messageChannel: "discord", skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "claude-cli", sessionStore, @@ -3251,7 +3301,7 @@ describe("CLI attempt execution", () => { messageChannel: "telegram", skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "anthropic", sessionStore, @@ -3319,7 +3369,7 @@ describe("CLI attempt execution", () => { messageChannel: "telegram", skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "anthropic", sessionStore, @@ -3388,7 +3438,7 @@ describe("CLI attempt execution", () => { messageChannel: "telegram", skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "openai", sessionStore, @@ -3751,7 +3801,7 @@ describe("CLI attempt execution", () => { }, }, }, - tmpDir, + agentDir, { filterExternalAuthProfiles: false, syncExternalCli: false }, ); runEmbeddedAgentMock.mockResolvedValueOnce({ @@ -3780,7 +3830,7 @@ describe("CLI attempt execution", () => { messageChannel: undefined, skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "fixture", sessionStore, @@ -3827,7 +3877,7 @@ describe("CLI attempt execution", () => { }, }, }, - tmpDir, + agentDir, { filterExternalAuthProfiles: false, syncExternalCli: false }, ); clearAgentHarnesses(); @@ -3864,7 +3914,7 @@ describe("CLI attempt execution", () => { messageChannel: undefined, skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "openai", sessionStore, @@ -3932,7 +3982,7 @@ describe("CLI attempt execution", () => { messageChannel: "discord", skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "anthropic", sessionStore, @@ -3996,7 +4046,7 @@ describe("CLI attempt execution", () => { messageChannel: "telegram", skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "openai", sessionStore, @@ -4046,7 +4096,7 @@ describe("CLI attempt execution", () => { messageChannel: undefined, skillsSnapshot: undefined, resolvedVerboseLevel: undefined, - agentDir: tmpDir, + agentDir, onAgentEvent: vi.fn(), authProfileProvider: "claude-cli", sessionStore, From a30adedc9b10d3cece669da551d41e820f69df28 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 04:52:42 -0700 Subject: [PATCH 38/57] fix(cli): preserve gateway request errors in health JSON (#118645) Co-authored-by: Peter Steinberger --- src/commands/health.test.ts | 65 +++++++++++++++++++++++++++++++++++-- src/commands/health.ts | 6 +++- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/src/commands/health.test.ts b/src/commands/health.test.ts index 7ef9da29747e..b14e988c53bb 100644 --- a/src/commands/health.test.ts +++ b/src/commands/health.test.ts @@ -1,5 +1,6 @@ // Health command tests cover gateway health probes, JSON output, and status formatting. import { beforeEach, describe, expect, it, vi } from "vitest"; +import { GatewayClientRequestError } from "../../packages/gateway-client/src/index.js"; import { stripAnsi } from "../../packages/terminal-core/src/ansi.js"; import { buildCredentialsRequiredHealthDiagnostic, @@ -80,6 +81,7 @@ const buildGatewayProbeConnectionDetailsMock = vi.fn(() => ({ url: TEST_GATEWAY_URL, })); const formatGatewayAuthErrorJsonMock = vi.fn(); +const formatGatewayClientRequestErrorJsonMock = vi.fn(); const formatGatewayTransportErrorJsonMock = vi.fn(); const probeGatewayStatusMock = vi.fn(); vi.mock("../gateway/call.js", () => ({ @@ -89,6 +91,8 @@ vi.mock("../gateway/call.js", () => ({ buildGatewayProbeConnectionDetails: (...args: [unknown, ...unknown[]]) => Reflect.apply(buildGatewayProbeConnectionDetailsMock, undefined, args), formatGatewayAuthErrorJson: (...args: unknown[]) => formatGatewayAuthErrorJsonMock(...args), + formatGatewayClientRequestErrorJson: (...args: unknown[]) => + formatGatewayClientRequestErrorJsonMock(...args), formatGatewayTransportErrorJson: (...args: unknown[]) => formatGatewayTransportErrorJsonMock(...args), isGatewayCredentialsRequiredError: (value: unknown) => @@ -145,9 +149,14 @@ describe("healthCommand", () => { tlsFingerprint: TEST_TLS_FINGERPRINT, url: TEST_GATEWAY_URL, }); - formatGatewayAuthErrorJsonMock.mockReset(); - formatGatewayAuthErrorJsonMock.mockReturnValue(null); - formatGatewayTransportErrorJsonMock.mockReturnValue(null); + for (const formatterMock of [ + formatGatewayAuthErrorJsonMock, + formatGatewayClientRequestErrorJsonMock, + formatGatewayTransportErrorJsonMock, + ]) { + formatterMock.mockReset(); + formatterMock.mockReturnValue(null); + } isGatewayCredentialsRequiredErrorMock.mockReturnValue(false); isGatewaySecretRefUnavailableErrorMock.mockReturnValue(false); probeGatewayStatusMock.mockReset(); @@ -409,6 +418,56 @@ describe("healthCommand", () => { expect(JSON.parse(requireFirstRuntimeLog())).toEqual(payload); }); + it("keeps Gateway health request failures machine-readable in JSON mode", async () => { + const error = new GatewayClientRequestError({ + code: "UNAVAILABLE", + message: "health snapshot unavailable", + details: { operation: "refresh" }, + retryable: true, + retryAfterMs: 250, + }); + const payload = { + ok: false, + error: { + type: "gateway_request_error", + code: "UNAVAILABLE", + message: "health snapshot unavailable", + details: { operation: "refresh" }, + retryable: true, + retryAfterMs: 250, + }, + }; + callGatewayMock.mockRejectedValueOnce(error); + formatGatewayClientRequestErrorJsonMock.mockReturnValueOnce(payload); + + await healthCommand({ json: true, timeoutMs: 5000, config: {} }, runtime as never); + + expect(formatGatewayAuthErrorJsonMock).toHaveBeenCalledWith(error); + expect(formatGatewayClientRequestErrorJsonMock).toHaveBeenCalledWith(error); + expect(formatGatewayTransportErrorJsonMock).not.toHaveBeenCalled(); + expect(runtime.log).toHaveBeenCalledTimes(1); + expect(runtime.error).not.toHaveBeenCalled(); + expect(runtime.exit).toHaveBeenCalledWith(1); + expect(JSON.parse(requireFirstRuntimeLog())).toEqual(payload); + }); + + it("preserves Gateway health request failures in human-readable mode", async () => { + const error = new GatewayClientRequestError({ + code: "UNAVAILABLE", + message: "health snapshot unavailable", + retryable: true, + }); + callGatewayMock.mockRejectedValueOnce(error); + + await expect( + healthCommand({ json: false, timeoutMs: 5000, config: {} }, runtime as never), + ).rejects.toBe(error); + + expect(formatGatewayAuthErrorJsonMock).not.toHaveBeenCalled(); + expect(formatGatewayClientRequestErrorJsonMock).not.toHaveBeenCalled(); + expect(formatGatewayTransportErrorJsonMock).not.toHaveBeenCalled(); + }); + it.each([ { json: true, expectedLogs: 1 }, { json: undefined, expectedLogs: 2 }, diff --git a/src/commands/health.ts b/src/commands/health.ts index 59b08df4e5d4..146aa35341e7 100644 --- a/src/commands/health.ts +++ b/src/commands/health.ts @@ -13,6 +13,7 @@ import { buildGatewayProbeConnectionDetails, callGateway, formatGatewayAuthErrorJson, + formatGatewayClientRequestErrorJson, formatGatewayTransportErrorJson, isGatewayCredentialsRequiredError, } from "../gateway/call.js"; @@ -249,7 +250,10 @@ export async function healthCommand( return; } if (opts.json) { - const payload = formatGatewayAuthErrorJson(error) ?? formatGatewayTransportErrorJson(error); + const payload = + formatGatewayAuthErrorJson(error) ?? + formatGatewayClientRequestErrorJson(error) ?? + formatGatewayTransportErrorJson(error); if (payload) { writeRuntimeJson(runtime, payload); runtime.exit(1); From 35c9c76b8c76c5d79f8f06b4bdce982518e61449 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 04:52:52 -0700 Subject: [PATCH 39/57] fix(otel): fail closed when configured TLS material is invalid (#118648) --- .../diagnostics-otel/src/service-exporter.ts | 48 +++--- .../diagnostics-otel/src/service-logs.ts | 11 +- .../src/service.otlp-export.test.ts | 156 ++++++++++++++++++ .../diagnostics-otel/src/service.test.ts | 129 +++++++++++++++ extensions/diagnostics-otel/src/service.ts | 31 ++-- 5 files changed, 335 insertions(+), 40 deletions(-) diff --git a/extensions/diagnostics-otel/src/service-exporter.ts b/extensions/diagnostics-otel/src/service-exporter.ts index 775b4315da29..5f832c2617da 100644 --- a/extensions/diagnostics-otel/src/service-exporter.ts +++ b/extensions/diagnostics-otel/src/service-exporter.ts @@ -10,7 +10,6 @@ import { import type { OtelHttpAgentFactory, OtelHttpAgentOptions, - OtelLogger, OtelSignalIdentifier, } from "./service-types.js"; @@ -75,8 +74,7 @@ function readOtelEnvFile(params: { signalIdentifier: OtelSignalIdentifier; signalSuffix: "CERTIFICATE" | "CLIENT_CERTIFICATE" | "CLIENT_KEY"; sharedEnvName: string; - logger: OtelLogger; - warning: string; + label: string; }): Buffer | undefined { const signalEnvName = `OTEL_EXPORTER_OTLP_${params.signalIdentifier}_${params.signalSuffix}`; const filePath = @@ -86,48 +84,53 @@ function readOtelEnvFile(params: { return undefined; } try { - return readFileSync(nodePath.resolve(process.cwd(), filePath)); + const material = readFileSync(nodePath.resolve(process.cwd(), filePath)); + if (material.length > 0) { + return material; + } } catch { - params.logger.warn(`diagnostics-otel: ${params.warning}`); - return undefined; + // Never expose certificate paths or silently downgrade to system trust. } + throw new Error( + `Configured OpenTelemetry ${params.label} file is missing, empty, or unreadable; refusing insecure export`, + ); } function normalizeOtelEnvValue(value: string | undefined): string | undefined { - const trimmed = value?.trim(); - return trimmed ? trimmed : undefined; + return value?.trim() ? value : undefined; } export function resolveOtelHttpAgentOptions(params: { url: string | undefined; signalIdentifier: OtelSignalIdentifier; - logger: OtelLogger; -}): OtelHttpAgentFactory | undefined { - const { url, signalIdentifier, logger } = params; - if (!url) { - return undefined; - } +}): OtelHttpAgentFactory | OtelHttpAgentOptions | undefined { + const { url, signalIdentifier } = params; const ca = readOtelEnvFile({ signalIdentifier, signalSuffix: "CERTIFICATE", sharedEnvName: OTEL_EXPORTER_OTLP_CERTIFICATE_ENV, - logger, - warning: "failed to read root certificate file", + label: "TLS root certificate", }); const cert = readOtelEnvFile({ signalIdentifier, signalSuffix: "CLIENT_CERTIFICATE", sharedEnvName: OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE_ENV, - logger, - warning: "failed to read client certificate chain file", + label: "mTLS client certificate", }); const key = readOtelEnvFile({ signalIdentifier, signalSuffix: "CLIENT_KEY", sharedEnvName: OTEL_EXPORTER_OTLP_CLIENT_KEY_ENV, - logger, - warning: "failed to read client certificate private key file", + label: "mTLS client private key", }); + if ((cert === undefined) !== (key === undefined)) { + throw new Error( + "Configured OpenTelemetry mTLS requires both a client certificate and private key; refusing insecure export", + ); + } + if (!url) { + return undefined; + } const agentOptions: OtelHttpAgentOptions = { keepAlive: true, ...(ca !== undefined ? { ca } : {}), @@ -136,10 +139,13 @@ export function resolveOtelHttpAgentOptions(params: { }; try { const agent = createNodeProxyAgent({ mode: "env", targetUrl: url, agentOptions }); - return agent ? () => agent : undefined; + if (agent) { + return () => agent; + } } catch { throw new Error("Configured telemetry proxy is invalid or unsupported; refusing direct export"); } + return (ca || cert || key) && new URL(url).protocol === "https:" ? agentOptions : undefined; } export function resolveSampleRate(value: number | undefined): number | undefined { diff --git a/extensions/diagnostics-otel/src/service-logs.ts b/extensions/diagnostics-otel/src/service-logs.ts index 44812bd15360..89f126e16b04 100644 --- a/extensions/diagnostics-otel/src/service-logs.ts +++ b/extensions/diagnostics-otel/src/service-logs.ts @@ -20,7 +20,7 @@ import { normalizeOtelLogString, type OtelContentCapturePolicy, } from "./service-content-normalization.js"; -import { errorCategory, formatError, resolveOtelHttpAgentOptions } from "./service-exporter.js"; +import { errorCategory, formatError } from "./service-exporter.js"; import { addTraceAttributes, contextForTrustedTraceContext, @@ -28,6 +28,8 @@ import { } from "./service-trace-context.js"; import type { BuiltOtelLogRecord, + OtelHttpAgentFactory, + OtelHttpAgentOptions, OtelLogger, TelemetryExporterDiagnosticEvent, } from "./service-types.js"; @@ -50,6 +52,7 @@ export function createDiagnosticsLogExporter(params: { logsEnabled: boolean; logsToOtlp: boolean; logsToStdout: boolean; + logHttpAgentOptions?: OtelHttpAgentFactory | OtelHttpAgentOptions; logUrl?: string; resource: Resource; serviceName: string; @@ -63,6 +66,7 @@ export function createDiagnosticsLogExporter(params: { logsEnabled, logsToOtlp, logsToStdout, + logHttpAgentOptions, logUrl, resource, serviceName, @@ -86,11 +90,6 @@ export function createDiagnosticsLogExporter(params: { let otelLogger: { emit: (logRecord: LogRecord) => void } | undefined; if (logsToOtlp) { - const logHttpAgentOptions = resolveOtelHttpAgentOptions({ - url: logUrl, - signalIdentifier: "LOGS", - logger, - }); const logExporter = new OTLPLogExporter({ ...(logUrl ? { url: logUrl } : {}), ...(headers ? { headers } : {}), diff --git a/extensions/diagnostics-otel/src/service.otlp-export.test.ts b/extensions/diagnostics-otel/src/service.otlp-export.test.ts index 0fc82b92e897..31b228bbeb64 100644 --- a/extensions/diagnostics-otel/src/service.otlp-export.test.ts +++ b/extensions/diagnostics-otel/src/service.otlp-export.test.ts @@ -8,6 +8,9 @@ // Trace cases use the OPENCLAW_OTEL_PRELOADED seam to retain this file's tracer provider. // Collector-boundary cases start the real NodeSDK, so teardown restores every global SDK // registration; otherwise a shutdown provider would poison later real-SDK cases. +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { context, diag, DiagLogLevel, metrics, propagation, trace } from "@opentelemetry/api"; import { logs } from "@opentelemetry/api-logs"; import { @@ -37,6 +40,18 @@ const ENDPOINT_ENV_KEYS = [ "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", + "OTEL_EXPORTER_OTLP_CERTIFICATE", + "OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE", + "OTEL_EXPORTER_OTLP_CLIENT_KEY", + "OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", + "OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE", + "OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY", + "OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE", + "OTEL_EXPORTER_OTLP_METRICS_CLIENT_CERTIFICATE", + "OTEL_EXPORTER_OTLP_METRICS_CLIENT_KEY", + "OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE", + "OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE", + "OTEL_EXPORTER_OTLP_LOGS_CLIENT_KEY", "OTEL_LOG_LEVEL", ] as const; const OTEL_GLOBAL_API_KEY = Symbol.for("opentelemetry.js.api.1"); @@ -471,3 +486,144 @@ test.each([ } }, ); + +const OTEL_TLS_MATERIAL_CASES = [ + { suffix: "CERTIFICATE", label: "TLS root certificate" }, + { suffix: "CLIENT_CERTIFICATE", label: "mTLS client certificate" }, + { suffix: "CLIENT_KEY", label: "mTLS client private key" }, +] as const; + +const OTEL_TLS_FILE_SECURITY_CASES = OTEL_ENDPOINT_SIGNAL_CASES.flatMap((signal) => + OTEL_TLS_MATERIAL_CASES.flatMap((material) => + (["shared", "signal"] as const).map((scope) => Object.assign({ scope }, signal, material)), + ), +); + +test.each(OTEL_TLS_FILE_SECURITY_CASES)( + "refuses the real $signal exporter when its $scope $label file cannot be read", + async ({ signal, suffix, label, scope, flags }) => { + process.env[PRELOAD_ENV] = "0"; + const pathSentinel = `qa-otel-${signal}-${suffix.toLowerCase()}-file-sentinel`; + const missingPath = `/definitely-missing/${pathSentinel}.pem`; + const envKey = + scope === "shared" + ? `OTEL_EXPORTER_OTLP_${suffix}` + : `OTEL_EXPORTER_OTLP_${signal.toUpperCase()}_${suffix}`; + process.env[envKey] = missingPath; + + const diagnostics = captureOtelDiagnostics(); + const ctx = createOtelContext("https://collector.example.com/otlp", flags); + ctx.internalDiagnostics!.emit = () => {}; + const service = createDiagnosticsOtelService(); + let failure: unknown; + try { + await service.start(ctx); + } catch (error) { + failure = error; + } finally { + await service.stop?.(ctx); + } + + expect(failure).toBeInstanceOf(Error); + const startupError = failure as Error; + expect(startupError.message).toBe( + `Configured OpenTelemetry ${label} file is missing, empty, or unreadable; refusing insecure export`, + ); + expect(startupError.stack).not.toContain(pathSentinel); + expect(startupError).not.toHaveProperty("cause"); + expect(diagnostics.join("\n")).not.toContain(pathSentinel); + expect(JSON.stringify(vi.mocked(ctx.logger.error).mock.calls)).not.toContain(pathSentinel); + expect(JSON.stringify(vi.mocked(ctx.logger.warn).mock.calls)).not.toContain(pathSentinel); + }, +); + +test.each(OTEL_ENDPOINT_SIGNAL_CASES)( + "refuses invalid TLS material before the real default $signal exporter is constructed", + async ({ signal, flags }) => { + process.env[PRELOAD_ENV] = "0"; + process.env[`OTEL_EXPORTER_OTLP_${signal.toUpperCase()}_CERTIFICATE`] = + "/definitely-missing/qa-otel-default-root.pem"; + const ctx = createOtelContext("", flags); + ctx.internalDiagnostics!.emit = () => {}; + const service = createDiagnosticsOtelService(); + + try { + await expect(service.start(ctx)).rejects.toThrow( + "Configured OpenTelemetry TLS root certificate file is missing, empty, or unreadable; refusing insecure export", + ); + } finally { + await service.stop?.(ctx); + } + }, +); + +test.each( + OTEL_ENDPOINT_SIGNAL_CASES.flatMap((signal) => + OTEL_TLS_MATERIAL_CASES.map((material) => Object.assign({}, signal, material)), + ), +)( + "rejects an empty $signal $label file before the SDK can silently downgrade trust", + async ({ signal, suffix, label, flags }) => { + process.env[PRELOAD_ENV] = "0"; + const certDir = mkdtempSync(path.join(tmpdir(), "openclaw-otel-empty-tls-")); + const emptyMaterialPath = path.join(certDir, "empty.pem"); + writeFileSync(emptyMaterialPath, ""); + process.env[`OTEL_EXPORTER_OTLP_${signal.toUpperCase()}_${suffix}`] = emptyMaterialPath; + const ctx = createOtelContext("https://collector.example.com/otlp", flags); + ctx.internalDiagnostics!.emit = () => {}; + const service = createDiagnosticsOtelService(); + + try { + await expect(service.start(ctx)).rejects.toThrow( + `Configured OpenTelemetry ${label} file is missing, empty, or unreadable; refusing insecure export`, + ); + } finally { + await service.stop?.(ctx); + rmSync(certDir, { force: true, recursive: true }); + } + }, +); + +test.each(OTEL_ENDPOINT_SIGNAL_CASES)( + "rejects the raw whitespace-padded $signal TLS certificate path the SDK cannot read", + async ({ signal, flags }) => { + process.env[PRELOAD_ENV] = "0"; + process.env[`OTEL_EXPORTER_OTLP_${signal.toUpperCase()}_CERTIFICATE`] = ` ${process.execPath} `; + const ctx = createOtelContext("https://collector.example.com/otlp", flags); + ctx.internalDiagnostics!.emit = () => {}; + const service = createDiagnosticsOtelService(); + + try { + await expect(service.start(ctx)).rejects.toThrow( + "Configured OpenTelemetry TLS root certificate file is missing, empty, or unreadable; refusing insecure export", + ); + } finally { + await service.stop?.(ctx); + } + }, +); + +test.each( + OTEL_ENDPOINT_SIGNAL_CASES.flatMap((signal) => + (["CLIENT_CERTIFICATE", "CLIENT_KEY"] as const).map((suffix) => + Object.assign({ suffix }, signal), + ), + ), +)( + "rejects the real $signal exporter when only $suffix mTLS material is configured", + async ({ signal, suffix, flags }) => { + process.env[PRELOAD_ENV] = "0"; + process.env[`OTEL_EXPORTER_OTLP_${signal.toUpperCase()}_${suffix}`] = process.execPath; + const ctx = createOtelContext("https://collector.example.com/otlp", flags); + ctx.internalDiagnostics!.emit = () => {}; + const service = createDiagnosticsOtelService(); + + try { + await expect(service.start(ctx)).rejects.toThrow( + "Configured OpenTelemetry mTLS requires both a client certificate and private key; refusing insecure export", + ); + } finally { + await service.stop?.(ctx); + } + }, +); diff --git a/extensions/diagnostics-otel/src/service.test.ts b/extensions/diagnostics-otel/src/service.test.ts index 2ebe1ce9f56d..a2ac02eba47d 100644 --- a/extensions/diagnostics-otel/src/service.test.ts +++ b/extensions/diagnostics-otel/src/service.test.ts @@ -1928,11 +1928,14 @@ describe("diagnostics-otel service", () => { try { const rootCertificatePath = path.join(certDir, "root.pem"); const clientCertificatePath = path.join(certDir, "client.pem"); + const sharedClientCertificatePath = path.join(certDir, "shared-client.pem"); const clientKeyPath = path.join(certDir, "client-key.pem"); writeFileSync(rootCertificatePath, "root-certificate"); writeFileSync(clientCertificatePath, "trace-client-certificate"); + writeFileSync(sharedClientCertificatePath, "shared-client-certificate"); writeFileSync(clientKeyPath, "client-key"); process.env.OTEL_EXPORTER_OTLP_CERTIFICATE = rootCertificatePath; + process.env.OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE = sharedClientCertificatePath; process.env.OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE = clientCertificatePath; process.env.OTEL_EXPORTER_OTLP_CLIENT_KEY = clientKeyPath; createNodeProxyAgentMock.mockReturnValue(nodeProxyAgent); @@ -1959,6 +1962,7 @@ describe("diagnostics-otel service", () => { expect(metricCall.agentOptions).toEqual({ keepAlive: true, ca: Buffer.from("root-certificate"), + cert: Buffer.from("shared-client-certificate"), key: Buffer.from("client-key"), }); } finally { @@ -1992,6 +1996,131 @@ describe("diagnostics-otel service", () => { } }); + test("pins validated collector TLS material on direct HTTPS exporter agents", async () => { + const certDir = mkdtempSync(path.join(tmpdir(), "openclaw-otel-direct-tls-")); + try { + const rootCertificatePath = path.join(certDir, "root.pem"); + writeFileSync(rootCertificatePath, "explicit-root-certificate"); + process.env.OTEL_EXPORTER_OTLP_CERTIFICATE = rootCertificatePath; + + await startOtelService({ + endpoint: "https://collector.example.com/otlp", + traces: true, + }); + + expect(firstExporterOptions(traceExporterCtor).httpAgentOptions).toEqual({ + keepAlive: true, + ca: Buffer.from("explicit-root-certificate"), + }); + } finally { + rmSync(certDir, { force: true, recursive: true }); + } + }); + + test("validates log TLS before constructing any trace, metric, or SDK owner", async () => { + process.env.OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE = + "/definitely-missing/qa-otel-log-root-atomic.pem"; + + await expect( + startOtelService({ + endpoint: "https://collector.example.com/otlp", + traces: true, + metrics: true, + logs: true, + }), + ).rejects.toThrow( + "Configured OpenTelemetry TLS root certificate file is missing, empty, or unreadable; refusing insecure export", + ); + + expect(traceExporterCtor).not.toHaveBeenCalled(); + expect(metricExporterCtor).not.toHaveBeenCalled(); + expect(logExporterCtor).not.toHaveBeenCalled(); + expect(sdkCtor).not.toHaveBeenCalled(); + expect(sdkStart).not.toHaveBeenCalled(); + }); + + test("never falls back from an unreadable signal TLS file to readable shared trust", async () => { + process.env.OTEL_EXPORTER_OTLP_CERTIFICATE = process.execPath; + process.env.OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE = + "/definitely-missing/qa-otel-signal-override.pem"; + + await expect(startOtelService({ traces: true })).rejects.toThrow( + "Configured OpenTelemetry TLS root certificate file is missing, empty, or unreadable; refusing insecure export", + ); + expect(traceExporterCtor).not.toHaveBeenCalled(); + }); + + test("lets a readable signal TLS file shadow an unreadable shared trust file", async () => { + process.env.OTEL_EXPORTER_OTLP_CERTIFICATE = + "/definitely-missing/qa-otel-shadowed-shared-root.pem"; + process.env.OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE = process.execPath; + + await startOtelService({ traces: true }); + + expect(traceExporterCtor).toHaveBeenCalledTimes(1); + }); + + test("keeps valid ambient TLS material compatible with plain HTTP collectors", async () => { + process.env.OTEL_EXPORTER_OTLP_CERTIFICATE = process.execPath; + + await startOtelService({ endpoint: "http://collector.example.com/otlp", traces: true }); + + expect(traceExporterCtor).toHaveBeenCalledTimes(1); + expect(firstExporterOptions(traceExporterCtor).httpAgentOptions).toBeUndefined(); + }); + + test("does not validate TLS material owned by a preloaded SDK", async () => { + process.env.OPENCLAW_OTEL_PRELOADED = "1"; + process.env.OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE = + "/definitely-missing/qa-otel-preloaded-traces-root.pem"; + process.env.OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE = + "/definitely-missing/qa-otel-preloaded-metrics-root.pem"; + + await startOtelService({ traces: true, metrics: true, logs: false }); + + expect(sdkCtor).not.toHaveBeenCalled(); + }); + + test("still validates plugin-owned OTLP logs when a trace SDK is preloaded", async () => { + process.env.OPENCLAW_OTEL_PRELOADED = "1"; + process.env.OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE = + "/definitely-missing/qa-otel-preloaded-log-root.pem"; + + await expect(startOtelService({ traces: true, logs: true })).rejects.toThrow( + "Configured OpenTelemetry TLS root certificate file is missing, empty, or unreadable; refusing insecure export", + ); + expect(logExporterCtor).not.toHaveBeenCalled(); + }); + + test.each([ + { + signal: "disabled traces", + envKey: "OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", + flags: { traces: false, metrics: true, logs: false }, + }, + { + signal: "disabled metrics", + envKey: "OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE", + flags: { traces: true, metrics: false, logs: false }, + }, + { + signal: "disabled logs", + envKey: "OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE", + flags: { traces: true, metrics: false, logs: false }, + }, + { + signal: "stdout-only logs", + envKey: "OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE", + flags: { traces: true, metrics: false, logs: true, logsExporter: "stdout" }, + }, + ] as const)("does not read TLS files for $signal", async ({ envKey, flags }) => { + process.env[envKey] = "/definitely-missing/qa-otel-inactive-signal-root.pem"; + + await startOtelService(flags); + + expect(sdkCtor).toHaveBeenCalledTimes(1); + }); + test.each([ ["traces", { traces: true }, "unsupported proxy protocol"], ["metrics", { metrics: true }, "invalid proxy URL"], diff --git a/extensions/diagnostics-otel/src/service.ts b/extensions/diagnostics-otel/src/service.ts index 3e6ebac4ac0d..c4240d20e3a7 100644 --- a/extensions/diagnostics-otel/src/service.ts +++ b/extensions/diagnostics-otel/src/service.ts @@ -163,8 +163,8 @@ export function createDiagnosticsOtelService(): OpenClawPluginService { path: "v1/logs", }) : undefined; - if (!sdkPreloaded && (tracesEnabled || metricsEnabled)) { - const traceUrl = tracesEnabled + const traceUrl = + !sdkPreloaded && tracesEnabled ? resolveSignalOtelUrl({ signalEndpoint: otel.tracesEndpoint, signalEnvEndpoint: process.env[OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_ENV], @@ -173,7 +173,8 @@ export function createDiagnosticsOtelService(): OpenClawPluginService { path: "v1/traces", }) : undefined; - const metricUrl = metricsEnabled + const metricUrl = + !sdkPreloaded && metricsEnabled ? resolveSignalOtelUrl({ signalEndpoint: otel.metricsEndpoint, signalEnvEndpoint: process.env[OTEL_EXPORTER_OTLP_METRICS_ENDPOINT_ENV], @@ -182,16 +183,19 @@ export function createDiagnosticsOtelService(): OpenClawPluginService { path: "v1/metrics", }) : undefined; - const traceHttpAgentOptions = resolveOtelHttpAgentOptions({ - url: traceUrl, - signalIdentifier: "TRACES", - logger: ctx.logger, - }); - const metricHttpAgentOptions = resolveOtelHttpAgentOptions({ - url: metricUrl, - signalIdentifier: "METRICS", - logger: ctx.logger, - }); + // Validate every owned signal before any SDK can export with downgraded TLS trust. + const logHttpAgentOptions = logsToOtlp + ? resolveOtelHttpAgentOptions({ url: logUrl, signalIdentifier: "LOGS" }) + : undefined; + const traceHttpAgentOptions = + !sdkPreloaded && tracesEnabled + ? resolveOtelHttpAgentOptions({ url: traceUrl, signalIdentifier: "TRACES" }) + : undefined; + const metricHttpAgentOptions = + !sdkPreloaded && metricsEnabled + ? resolveOtelHttpAgentOptions({ url: metricUrl, signalIdentifier: "METRICS" }) + : undefined; + if (!sdkPreloaded && (tracesEnabled || metricsEnabled)) { const traceExporter = tracesEnabled ? new OTLPTraceExporter({ ...(traceUrl ? { url: traceUrl } : {}), @@ -281,6 +285,7 @@ export function createDiagnosticsOtelService(): OpenClawPluginService { logsEnabled, logsToOtlp, logsToStdout, + logHttpAgentOptions, logUrl, resource, serviceName, From a5160c1294521bc04e40d39f7fe4ec06cad5fad6 Mon Sep 17 00:00:00 2001 From: Goutam Adwant <8672451+goutamadwant@users.noreply.github.com> Date: Mon, 3 Aug 2026 04:57:28 -0700 Subject: [PATCH 40/57] fix(xai): classify exhausted credits as billing (#118615) --- extensions/xai/index.test.ts | 25 +++++++++++++++++++ extensions/xai/index.ts | 2 +- ...errors-provider-structured-signals.test.ts | 21 ++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/extensions/xai/index.test.ts b/extensions/xai/index.test.ts index 360fb6927f84..022c78d0f825 100644 --- a/extensions/xai/index.test.ts +++ b/extensions/xai/index.test.ts @@ -471,6 +471,31 @@ describe("xai provider plugin", () => { ).toBeUndefined(); }); + it("classifies exhausted Grok credits and subscription requirements as billing", async () => { + const provider = await registerSingleProviderPlugin(plugin); + + expect( + provider.classifyFailoverReason?.({ + errorMessage: '403 {"error":"You have run out of credits"}', + }), + ).toBe("billing"); + expect( + provider.classifyFailoverReason?.({ + errorMessage: '403 {"error":"You need a Grok subscription"}', + }), + ).toBe("billing"); + expect( + provider.classifyFailoverReason?.({ + errorMessage: "403 Forbidden", + }), + ).toBeUndefined(); + expect( + provider.classifyFailoverReason?.({ + errorMessage: "429 Too Many Requests", + }), + ).toBe("rate_limit"); + }); + it("registers xAI speech providers for batch and streaming STT", async () => { const { mediaProviders, realtimeTranscriptionProviders } = await registerProviderPlugin({ plugin, diff --git a/extensions/xai/index.ts b/extensions/xai/index.ts index ba50572ff912..bd9e5854a769 100644 --- a/extensions/xai/index.ts +++ b/extensions/xai/index.ts @@ -57,7 +57,7 @@ import { const PROVIDER_ID = "xai"; const XAI_CREDIT_OR_SPENDING_LIMIT_RE = - /\b(?:used all available credits|monthly spending limit|purchase more credits|raise your spending limit)\b/i; + /\b(?:used all available credits|run out of credits|monthly spending limit|purchase more credits|raise your spending limit|need a Grok subscription)\b/i; const XAI_RATE_LIMIT_RE = /\b(?:rate limit exceeded|too many requests)\b/i; const loadCodeExecutionModule = createLazyRuntimeModule(() => import("./code-execution.js")); diff --git a/src/agents/embedded-agent-helpers/errors-provider-structured-signals.test.ts b/src/agents/embedded-agent-helpers/errors-provider-structured-signals.test.ts index afeebf06329b..a1ef7520bdca 100644 --- a/src/agents/embedded-agent-helpers/errors-provider-structured-signals.test.ts +++ b/src/agents/embedded-agent-helpers/errors-provider-structured-signals.test.ts @@ -69,6 +69,27 @@ describe("provider failover hook structured signals", () => { ).toEqual({ kind: "reason", reason: "auth" }); }); + it("lets provider billing text override a leading 403 in assistant failures", () => { + providerRuntimeMocks.classifyProviderPluginError.mockImplementation((context) => { + return context.provider === "demo-provider" && + context.errorMessage.includes("quota exhausted") + ? "billing" + : undefined; + }); + + const errorMessage = '403 {"error":"Account quota exhausted"}'; + expect( + classifyAssistantFailoverReason( + makeAssistantMessageFixture({ provider: "demo-provider", errorMessage }), + ), + ).toBe("billing"); + expect( + classifyAssistantFailoverReason( + makeAssistantMessageFixture({ provider: "other-provider", errorMessage }), + ), + ).toBe("auth"); + }); + it("does not call the direct provider hook for unstructured classified messages", () => { // Plain message classifiers run first; provider hooks only see structured // descriptors where a plugin can make a reliable decision. From dfb13780435f807f6004be5167384bbe6f2fdaf0 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 05:04:05 -0700 Subject: [PATCH 41/57] chore(tui): stabilize queued-turn admission in Gateway PTY test (#118638) * test(tui): wait for queued turn admission before releasing parent * test(tui): clarify queued admission event ordering --- src/tui/tui-pty-local.e2e.test.ts | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/tui/tui-pty-local.e2e.test.ts b/src/tui/tui-pty-local.e2e.test.ts index da6624361288..405b998c7deb 100644 --- a/src/tui/tui-pty-local.e2e.test.ts +++ b/src/tui/tui-pty-local.e2e.test.ts @@ -1323,15 +1323,31 @@ describe("TUI PTY real backends", () => { }); try { let queueClientConnected = false; + const admittedRunIds = new Set(); queueClient.onConnected = () => { queueClientConnected = true; }; + // Retain admission events that arrive before both chat.send ACKs settle. + queueClient.onEvent = ({ event, payload }) => { + if (event !== "chat" || !payload || typeof payload !== "object") { + return; + } + const chatEvent = payload as { runId?: unknown; sessionKey?: unknown; state?: unknown }; + if ( + chatEvent.state === "final" && + chatEvent.sessionKey === fixture.sessionKey && + typeof chatEvent.runId === "string" + ) { + admittedRunIds.add(chatEvent.runId); + } + }; queueClient.start(); await waitFor({ timeoutMs: LOCAL_STARTUP_TIMEOUT_MS, read: () => (queueClientConnected ? true : null), onTimeout: () => new Error("TUI Gateway client did not connect"), }); + await queueClient.subscribeSessionEvents(); await fixture.run.write("/queue collect debounce:250ms\r", { delay: false }); await fixture.waitForOutput("Queue mode set to collect."); await fixture.run.write("slow collect parent\r"); @@ -1345,16 +1361,23 @@ describe("TUI PTY real backends", () => { sessionKey: fixture.sessionKey, message: "collect prompt alpha", }); - await sleep(50); const betaSend = queueClient.sendChat({ sessionKey: fixture.sessionKey, message: "collect prompt beta", }); const sendResults = await Promise.all([alphaSend, betaSend]); expect(sendResults.map((result) => result.status)).toEqual(["started", "started"]); - // Let both Gateway submissions reach the active-turn queue before the - // parent response opens the collect debounce window. - await sleep(1_000); + const expectedRunIds = sendResults.map(({ runId }) => runId); + await waitFor({ + timeoutMs: LOCAL_OUTPUT_TIMEOUT_MS, + read: () => (expectedRunIds.every((runId) => admittedRunIds.has(runId)) ? true : null), + onTimeout: () => + new Error( + `queued prompts were not admitted: expected ${expectedRunIds.join(", ")}; ` + + `observed ${[...admittedRunIds].join(", ")}\n${fixture.gateway.logs()}\n` + + fixture.run.output(), + ), + }); fixture.mockModel.releaseFirstResponse(); await waitFor({ timeoutMs: LOCAL_OUTPUT_TIMEOUT_MS, From a8efce28ee3c0548c7086dd59bc301a3a82041f8 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 05:16:33 -0700 Subject: [PATCH 42/57] test(ui): reuse responsive browser fixtures (#118655) Co-authored-by: Peter Steinberger --- .../chat/chat-responsive.browser.test.ts | 473 +++++++++--------- 1 file changed, 230 insertions(+), 243 deletions(-) diff --git a/ui/src/pages/chat/chat-responsive.browser.test.ts b/ui/src/pages/chat/chat-responsive.browser.test.ts index 313c493238f7..dec420839266 100644 --- a/ui/src/pages/chat/chat-responsive.browser.test.ts +++ b/ui/src/pages/chat/chat-responsive.browser.test.ts @@ -1,5 +1,5 @@ // Control UI tests cover chat responsive behavior. -import { chromium, type Browser, type Page } from "playwright"; +import { chromium, type Browser, type BrowserContext, type Page } from "playwright"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { readStyleSheet } from "../../../../test/helpers/ui-style-fixtures.js"; import { @@ -21,14 +21,12 @@ const VIEWPORTS = [ [1440, 900], ] as const; const TOUCH_TARGET_MIN_PX = 43.5; -// Real-app cases boot through a cold Vite dev server that transforms the whole -// Control UI module graph on first request; with 6 Vitest workers sharing an -// 8vCPU CI runner that first render can starve well past 10s. Budget the -// first-render waits for contention while staying inside the 60s testTimeout. +// The shared real-app page still cold-boots Vite's full Control UI graph once; +// under CI contention that first render can starve well past 10s. const APP_FIRST_RENDER_TIMEOUT_MS = 30_000; const FULL_APP_TEST_OPTIONS = { - // These cases cold-boot Vite through one shared Chromium/server pair. Keep them - // as a sequential barrier so concurrent layout pages cannot starve navigation. + // Shared-page interactions mutate viewport, pointer, and composer state. Keep + // each as a sequential barrier so they cannot overlap one another. concurrent: false, timeout: 60_000, } as const; @@ -42,7 +40,17 @@ const describeBrowserLayout = canRunPlaywrightChromium(chromiumExecutablePath) : describe.skip; let sharedBrowser: Browser | null = null; +let sharedLayoutContext: BrowserContext | null = null; +let sharedAppPage: Page | null = null; +let sharedAppPagePromise: Promise | null = null; +const sharedAppPageErrors: string[] = []; let realChatServer: ControlUiE2eServer | null = null; +let cachedUiCss: string | null = null; + +const SHARED_APP_CONTEXT_TEXT = "Context hover regression fixture."; +const SHARED_APP_SLASH_TEXT = "Short landscape slash command keyboard regression fixture."; +const SHARED_APP_IMAGE_URL = "https://cdn.example/render%2Epng?download=1"; +const SHARED_APP_VIDEO_URL = "https://cdn.example/clip%2Emp4?download=1"; function installResponsiveChatGateway(page: Page, scenario: ControlUiMockGatewayScenario = {}) { return installMockGateway(page, { @@ -51,6 +59,62 @@ function installResponsiveChatGateway(page: Page, scenario: ControlUiMockGateway }); } +async function getSharedAppPage(): Promise { + sharedAppPagePromise ??= createSharedAppPage(); + return await sharedAppPagePromise; +} + +async function createSharedAppPage(): Promise { + if (!realChatServer) { + throw new Error("Expected the Control UI server to be ready"); + } + // The five app assertions use disjoint fixture messages and reset mutable + // page state, so one lazy boot preserves coverage without five graph loads. + const page = await openBrowserPage(1366, 900, { isolated: true }); + try { + page.on("pageerror", (error) => sharedAppPageErrors.push(error.message)); + await page.route("https://cdn.example/**", (route) => route.abort()); + await installResponsiveChatGateway(page, { + assistantName: "Claw", + historyMessages: [ + { + content: [{ text: SHARED_APP_CONTEXT_TEXT, type: "text" }], + model: "openai/gpt-5.5", + role: "assistant", + timestamp: Date.UTC(2026, 6, 5, 9, 51), + usage: { cacheRead: 2_400, input: 19_600, output: 126 }, + }, + { + content: `MEDIA:${SHARED_APP_IMAGE_URL}`, + role: "assistant", + timestamp: Date.UTC(2026, 6, 9, 10, 0), + }, + { + content: "Encoded transcript video", + __openclaw: { media: [{ url: SHARED_APP_VIDEO_URL, contentType: "video/mp4" }] }, + role: "user", + timestamp: Date.UTC(2026, 6, 9, 10, 1), + }, + { + content: [{ text: SHARED_APP_SLASH_TEXT, type: "text" }], + role: "assistant", + timestamp: Date.UTC(2026, 6, 9, 10, 2), + }, + ], + }); + await page.goto(`${realChatServer.baseUrl}chat/main`, { + waitUntil: "domcontentloaded", + timeout: APP_FIRST_RENDER_TIMEOUT_MS, + }); + await page.getByText(SHARED_APP_SLASH_TEXT).waitFor({ timeout: APP_FIRST_RENDER_TIMEOUT_MS }); + sharedAppPage = page; + return page; + } catch (error) { + await closeBrowserPage(page); + throw error; + } +} + type ControlRect = { x: number; y: number; @@ -96,6 +160,9 @@ function expectControlRect(rect: ControlRect | null, label: string): ControlRect } function readUiCss(): string { + if (cachedUiCss !== null) { + return cachedUiCss; + } const files = [ "ui/src/styles/base.css", "ui/src/styles/layout.css", @@ -108,7 +175,8 @@ function readUiCss(): string { "ui/src/styles/chat/question-card.css", "ui/src/styles/chat/sidebar.css", ]; - return files.map((file) => readStyleSheet(file)).join("\n"); + cachedUiCss = files.map((file) => readStyleSheet(file)).join("\n"); + return cachedUiCss; } function iconSvg() { @@ -445,12 +513,24 @@ async function openFixture(width: number, height: number, opts: ChatFixtureOptio } } -async function openBrowserPage(width: number, height: number): Promise { +async function openBrowserPage( + width: number, + height: number, + options: { isolated?: boolean } = {}, +): Promise { sharedBrowser ??= await chromium.launch({ executablePath: chromiumExecutablePath, headless: true, }); - return await sharedBrowser.newPage({ viewport: { width, height } }); + if (options.isolated) { + return await sharedBrowser.newPage({ viewport: { width, height } }); + } + // Static setContent fixtures do not mutate context-owned storage or routes, + // so they can share one context while their pages remain concurrent. + sharedLayoutContext ??= await sharedBrowser.newContext(); + const page = await sharedLayoutContext.newPage(); + await page.setViewportSize({ width, height }); + return page; } async function closeBrowserPage(page: Page): Promise { @@ -544,12 +624,18 @@ describeBrowserLayout.concurrent("chat responsive browser layout", () => { executablePath: chromiumExecutablePath, headless: true, }); + sharedLayoutContext = await sharedBrowser.newContext(); realChatServer = await startControlUiE2eServer(); }); afterAll(async () => { + await sharedAppPage?.close(); + sharedAppPage = null; + sharedAppPagePromise = null; await realChatServer?.close(); realChatServer = null; + await sharedLayoutContext?.close(); + sharedLayoutContext = null; await sharedBrowser?.close(); sharedBrowser = null; }); @@ -558,122 +644,98 @@ describeBrowserLayout.concurrent("chat responsive browser layout", () => { "does not replay a consumed session rail open generation after round trips or remounts", FULL_APP_TEST_OPTIONS, async () => { - if (!realChatServer) { - throw new Error("Expected the Control UI server to be ready"); - } - const page = await openBrowserPage(900, 700); - try { - await page.goto(realChatServer.baseUrl, { - waitUntil: "domcontentloaded", - timeout: APP_FIRST_RENDER_TIMEOUT_MS, - }); - await page.evaluate( - () => - new Promise((resolve, reject) => { - const script = document.createElement("script"); - script.type = "module"; - script.src = "/src/pages/chat/components/chat-session-rail.ts"; - script.addEventListener("load", () => resolve(), { once: true }); - script.addEventListener( - "error", - () => reject(new Error("Session rail module failed")), - { - once: true, - }, - ); - document.head.append(script); - }), - ); - const result = await page.evaluate(async () => { - localStorage.setItem("openclaw.chat.observerHud.display", "pill"); - type Rail = HTMLElement & { - companion: { - exchanges: []; - pendingQuestion: null; - failedQuestion: null; - hint: null; - draft: string; - }; - connected: boolean; - consumedOpenRequest: number; - onOpenRequestConsumed: (openRequest: number) => void; - onVisibilityChange: (visible: boolean) => void; - openRequest: number; - sessionKey: string; - updateComplete: Promise; + const page = await getSharedAppPage(); + const result = await page.evaluate(async () => { + await customElements.whenDefined("openclaw-chat-session-rail"); + localStorage.setItem("openclaw.chat.observerHud.display", "pill"); + type Rail = HTMLElement & { + companion: { + exchanges: []; + pendingQuestion: null; + failedQuestion: null; + hint: null; + draft: string; }; - const createRail = () => document.createElement("openclaw-chat-session-rail") as Rail; - let rail = createRail(); - let consumedOpenRequest = 0; - let visibleReports = 0; - const configureRail = (nextRail: Rail) => { - nextRail.companion = { - exchanges: [], - pendingQuestion: null, - failedQuestion: null, - hint: null, - draft: "What changed?", - }; - nextRail.connected = true; - nextRail.consumedOpenRequest = consumedOpenRequest; - nextRail.onOpenRequestConsumed = (openRequest) => { - consumedOpenRequest = openRequest; - }; - nextRail.onVisibilityChange = (visible) => { - if (visible) { - visibleReports += 1; - } - }; + connected: boolean; + consumedOpenRequest: number; + onOpenRequestConsumed: (openRequest: number) => void; + onVisibilityChange: (visible: boolean) => void; + openRequest: number; + sessionKey: string; + updateComplete: Promise; + }; + const createRail = () => document.createElement("openclaw-chat-session-rail") as Rail; + let rail = createRail(); + let consumedOpenRequest = 0; + let visibleReports = 0; + const configureRail = (nextRail: Rail) => { + nextRail.companion = { + exchanges: [], + pendingQuestion: null, + failedQuestion: null, + hint: null, + draft: "What changed?", }; - configureRail(rail); - rail.sessionKey = "agent:main:a"; - document.body.replaceChildren(rail); + nextRail.connected = true; + nextRail.consumedOpenRequest = consumedOpenRequest; + nextRail.onOpenRequestConsumed = (openRequest) => { + consumedOpenRequest = openRequest; + }; + nextRail.onVisibilityChange = (visible) => { + if (visible) { + visibleReports += 1; + } + }; + }; + configureRail(rail); + rail.sessionKey = "agent:main:a"; + document.body.append(rail); + await rail.updateComplete; + const mode = () => + rail.querySelector(".chat-session-rail--expanded") ? "expanded" : "pill"; + const update = async (sessionKey: string, openRequest: number) => { + rail.sessionKey = sessionKey; + rail.openRequest = openRequest; + rail.consumedOpenRequest = consumedOpenRequest; await rail.updateComplete; - const mode = () => - rail.querySelector(".chat-session-rail--expanded") ? "expanded" : "pill"; - const update = async (sessionKey: string, openRequest: number) => { - rail.sessionKey = sessionKey; - rail.openRequest = openRequest; - rail.consumedOpenRequest = consumedOpenRequest; - await rail.updateComplete; - return mode(); - }; + return mode(); + }; - const sameElementModes = [ - await update("agent:main:a", 1), - await update("agent:main:b", 0), - await update("agent:main:a", 1), - await update("agent:main:a", 2), - ]; - rail.remove(); - rail = createRail(); - configureRail(rail); - rail.sessionKey = "agent:main:a"; - rail.openRequest = 2; - document.body.append(rail); - await rail.updateComplete; - const remountMode = mode(); - const nextGenerationMode = await update("agent:main:a", 3); + const sameElementModes = [ + await update("agent:main:a", 1), + await update("agent:main:b", 0), + await update("agent:main:a", 1), + await update("agent:main:a", 2), + ]; + rail.remove(); + rail = createRail(); + configureRail(rail); + rail.sessionKey = "agent:main:a"; + rail.openRequest = 2; + document.body.append(rail); + await rail.updateComplete; + const remountMode = mode(); + const nextGenerationMode = await update("agent:main:a", 3); - return { - sameElementModes, - remountMode, - nextGenerationMode, - storedPreference: localStorage.getItem("openclaw.chat.observerHud.display"), - visibleReports, - }; - }); + const snapshot = { + sameElementModes, + remountMode, + nextGenerationMode, + storedPreference: localStorage.getItem("openclaw.chat.observerHud.display"), + visibleReports, + }; + rail.remove(); + localStorage.removeItem("openclaw.chat.observerHud.display"); + return snapshot; + }); - expect(result).toEqual({ - sameElementModes: ["expanded", "pill", "pill", "expanded"], - remountMode: "pill", - nextGenerationMode: "expanded", - storedPreference: "pill", - visibleReports: 3, - }); - } finally { - await closeBrowserPage(page); - } + expect(result).toEqual({ + sameElementModes: ["expanded", "pill", "pill", "expanded"], + remountMode: "pill", + nextGenerationMode: "expanded", + storedPreference: "pill", + visibleReports: 3, + }); }, ); @@ -1104,18 +1166,10 @@ describeBrowserLayout.concurrent("chat responsive browser layout", () => { "remeasures a populated composer when the viewport width changes", FULL_APP_TEST_OPTIONS, async () => { - if (!realChatServer) { - throw new Error("Expected the Control UI server to be ready"); - } - const page = await openBrowserPage(900, 800); - const pageErrors: string[] = []; - page.on("pageerror", (error) => pageErrors.push(error.message)); + const page = await getSharedAppPage(); + const errorStart = sharedAppPageErrors.length; try { - await installResponsiveChatGateway(page); - await page.goto(`${realChatServer.baseUrl}chat/main`, { - waitUntil: "domcontentloaded", - timeout: APP_FIRST_RENDER_TIMEOUT_MS, - }); + await page.setViewportSize({ width: 900, height: 800 }); const textarea = page.locator(".agent-chat__composer-combobox > textarea"); await textarea.waitFor({ timeout: APP_FIRST_RENDER_TIMEOUT_MS }); await textarea.fill( @@ -1143,9 +1197,14 @@ describeBrowserLayout.concurrent("chat responsive browser layout", () => { ); return element !== null && element.getBoundingClientRect().height < previousHeight - 1; }, narrowHeight); - expect(pageErrors.filter((message) => message.includes("ResizeObserver loop"))).toEqual([]); + expect( + sharedAppPageErrors + .slice(errorStart) + .filter((message) => message.includes("ResizeObserver loop")), + ).toEqual([]); } finally { - await closeBrowserPage(page); + await page.locator(".agent-chat__composer-combobox > textarea").fill(""); + await page.setViewportSize({ width: 1366, height: 900 }); } }, ); @@ -1154,60 +1213,40 @@ describeBrowserLayout.concurrent("chat responsive browser layout", () => { "reveals, pins, and dismisses message context from the timestamp", FULL_APP_TEST_OPTIONS, async () => { - if (!realChatServer) { - throw new Error("Expected the Control UI server to be ready"); - } - const page = await openBrowserPage(1366, 900); + const page = await getSharedAppPage(); try { - await installResponsiveChatGateway(page, { - assistantName: "Claw", - historyMessages: [ - { - content: [{ text: "Context hover regression fixture.", type: "text" }], - model: "openai/gpt-5.5", - role: "assistant", - timestamp: Date.UTC(2026, 6, 5, 9, 51), - usage: { cacheRead: 2_400, input: 19_600, output: 126 }, - }, - ], - }); - await page.goto(`${realChatServer.baseUrl}chat/main`, { - waitUntil: "domcontentloaded", - timeout: APP_FIRST_RENDER_TIMEOUT_MS, - }); - await page - .getByText("Context hover regression fixture.") - .waitFor({ timeout: APP_FIRST_RENDER_TIMEOUT_MS }); - - const details = page.locator("details.msg-meta"); - const context = page.locator(".msg-meta__details"); - const initialLayout = await page.evaluate(() => { - const footer = document.querySelector(".chat-group-footer")!; - const group = document.querySelector(".chat-group")!; + await page.setViewportSize({ width: 1366, height: 900 }); + const group = page.locator(".chat-group").filter({ hasText: SHARED_APP_CONTEXT_TEXT }); + const details = group.locator("details.msg-meta"); + const context = details.locator(".msg-meta__details"); + const summary = details.locator(".msg-meta__summary"); + const messageText = group.locator(".chat-text").first(); + await messageText.waitFor({ timeout: APP_FIRST_RENDER_TIMEOUT_MS }); + const initialLayout = await group.evaluate((node) => { + const footer = node.querySelector(".chat-group-footer")!; return { footerHeight: footer.getBoundingClientRect().height, - groupHeight: group.getBoundingClientRect().height, + groupHeight: (node as HTMLElement).getBoundingClientRect().height, }; }); expect(await context.isVisible()).toBe(false); // Travel like a real pointer: the footer overlay is pointer-gated until // the group is hovered, so enter through the message body first. - await page.locator(".chat-text").first().hover(); - await page.locator(".msg-meta__summary").hover(); + await messageText.hover(); + await summary.hover(); // The reveal is state-driven, so the re-render can lag the hover event // under CPU contention; poll instead of a one-shot visibility read. await context.waitFor({ state: "visible", timeout: 10_000 }); - const hoverLayout = await page.evaluate(() => { - const footer = document.querySelector(".chat-group-footer")!; - const group = document.querySelector(".chat-group")!; - const summary = document.querySelector(".msg-meta__summary")!; - const detailsOverlay = document.querySelector(".msg-meta__details")!; + const hoverLayout = await group.evaluate((node) => { + const footer = node.querySelector(".chat-group-footer")!; + const summaryNode = node.querySelector(".msg-meta__summary")!; + const detailsOverlay = node.querySelector(".msg-meta__details")!; return { contextBottom: detailsOverlay.getBoundingClientRect().bottom, footerHeight: footer.getBoundingClientRect().height, - groupHeight: group.getBoundingClientRect().height, - summaryTop: summary.getBoundingClientRect().top, + groupHeight: (node as HTMLElement).getBoundingClientRect().height, + summaryTop: summaryNode.getBoundingClientRect().top, }; }); expect(hoverLayout.footerHeight).toBeCloseTo(initialLayout.footerHeight, 2); @@ -1217,12 +1256,12 @@ describeBrowserLayout.concurrent("chat responsive browser layout", () => { await page.mouse.move(0, 0); await context.waitFor({ state: "hidden", timeout: 10_000 }); - await page.locator(".chat-text").first().hover(); - await page.locator(".msg-meta__summary").hover(); + await messageText.hover(); + await summary.hover(); // Escape only owns pinned disclosures; it must not corrupt an active // hover preview before the click converts that preview into a pin. await page.keyboard.press("Escape"); - await page.locator(".msg-meta__summary").click(); + await summary.click(); await page.mouse.move(0, 0); // Click-to-open must survive the pointer leaving the message group. await context.waitFor({ state: "visible", timeout: 10_000 }); @@ -1232,14 +1271,15 @@ describeBrowserLayout.concurrent("chat responsive browser layout", () => { await context.waitFor({ state: "hidden", timeout: 10_000 }); expect(await details.getAttribute("open")).toBeNull(); - await page.locator(".chat-text").first().hover(); - await page.locator(".msg-meta__summary").click(); + await messageText.hover(); + await summary.click(); await context.waitFor({ state: "visible", timeout: 10_000 }); await page.keyboard.press("Escape"); await context.waitFor({ state: "hidden", timeout: 10_000 }); expect(await details.getAttribute("open")).toBeNull(); } finally { - await closeBrowserPage(page); + await page.keyboard.press("Escape"); + await page.mouse.move(0, 0); } }, ); @@ -1248,45 +1288,13 @@ describeBrowserLayout.concurrent("chat responsive browser layout", () => { "renders encoded media extensions from assistant output and transcript fields", FULL_APP_TEST_OPTIONS, async () => { - if (!realChatServer) { - throw new Error("Expected the Control UI server to be ready"); - } - const imageUrl = "https://cdn.example/render%2Epng?download=1"; - const videoUrl = "https://cdn.example/clip%2Emp4?download=1"; - const page = await openBrowserPage(1366, 900); - try { - await page.route("https://cdn.example/**", (route) => route.abort()); - await installResponsiveChatGateway(page, { - historyMessages: [ - { - content: `MEDIA:${imageUrl}`, - role: "assistant", - timestamp: Date.UTC(2026, 6, 9, 10, 0), - }, - { - content: "Encoded transcript video", - __openclaw: { media: [{ url: videoUrl, contentType: "video/mp4" }] }, - role: "user", - timestamp: Date.UTC(2026, 6, 9, 10, 1), - }, - ], - }); - await page.goto(`${realChatServer.baseUrl}chat/main`, { - waitUntil: "domcontentloaded", - timeout: APP_FIRST_RENDER_TIMEOUT_MS, - }); - - const image = page.locator("img.chat-message-image"); - const video = page.locator("video"); - // First wait absorbs the cold-app render; both elements land in the same - // history render pass. Video stays behind its placeholder until metadata loads. - await image.waitFor({ timeout: APP_FIRST_RENDER_TIMEOUT_MS }); - await video.waitFor({ state: "attached", timeout: 10_000 }); - expect(await image.getAttribute("src")).toBe(imageUrl); - expect(await video.getAttribute("src")).toBe(videoUrl); - } finally { - await closeBrowserPage(page); - } + const page = await getSharedAppPage(); + const image = page.locator(`img.chat-message-image[src="${SHARED_APP_IMAGE_URL}"]`); + const video = page.locator(`video[src="${SHARED_APP_VIDEO_URL}"]`); + await image.waitFor({ timeout: APP_FIRST_RENDER_TIMEOUT_MS }); + await video.waitFor({ state: "attached", timeout: 10_000 }); + expect(await image.getAttribute("src")).toBe(SHARED_APP_IMAGE_URL); + expect(await video.getAttribute("src")).toBe(SHARED_APP_VIDEO_URL); }, ); @@ -2449,38 +2457,17 @@ describeBrowserLayout.concurrent("chat responsive browser layout", () => { let page: Page; beforeAll(async () => { - if (!realChatServer) { - throw new Error("Expected the Control UI server to be ready"); - } - page = await openBrowserPage(568, 320); - await installResponsiveChatGateway(page, { - historyMessages: [ - { - content: [ - { - text: "Short landscape slash command keyboard regression fixture.", - type: "text", - }, - ], - role: "assistant", - timestamp: Date.now(), - }, - ], - }); - await page.goto(`${realChatServer.baseUrl}chat/main`, { - waitUntil: "domcontentloaded", - timeout: APP_FIRST_RENDER_TIMEOUT_MS, - }); - await page - .getByText("Short landscape slash command keyboard regression fixture.") - .waitFor({ timeout: APP_FIRST_RENDER_TIMEOUT_MS }); + page = await getSharedAppPage(); + await page.setViewportSize({ width: 568, height: 320 }); + await page.getByText(SHARED_APP_SLASH_TEXT).waitFor({ timeout: APP_FIRST_RENDER_TIMEOUT_MS }); const textarea = page.locator(".agent-chat__composer-combobox > textarea"); await textarea.fill("/"); await textarea.focus(); }); afterAll(async () => { - await closeBrowserPage(page); + await page.locator(".agent-chat__composer-combobox > textarea").fill(""); + await page.setViewportSize({ width: 1366, height: 900 }); }); it("scrolls the keyboard-active slash option into view in short landscape", async () => { From 68be0e3378368e024e111c5c5241141a896bb679 Mon Sep 17 00:00:00 2001 From: Jesse Merhi <79823012+jesse-merhi@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:18:27 +1000 Subject: [PATCH 43/57] feat: enable rich setup controls in custodian chat (#114631) * feat(protocol): carry the awaited wizard step on the chat result * fix(system-agent): strip sensitive wizard prefill from chat results * fix(wizard): keep setup secrets server-side * test(wizard): preserve prompt mock typing * fix(ui): add reveal toggle to wizard secrets * refactor(ui): adopt Carapace sensitive input * fix(ui): hide revealed sensitive input mask * test(twitch): cover environment-only setup * feat(custodian): render rich wizard steps * fix(custodian): validate wizard text replies * feat(custodian): submit typed wizard answers * refactor(gateway): isolate custodian chat turns * fix(gateway): accept session engine adapter * fix(ui): narrow wizard control values * refactor: simplify rich wizard answer flow * fix(custodian): recover evicted wizard sessions * docs: note custodian rich setup controls * test(gateway): split wizard answer coverage --- CHANGELOG.md | 1 + .../OpenClawProtocol/GatewayModels.swift | 10 +- extensions/synology-chat/src/core.test.ts | 54 ++++ extensions/synology-chat/src/setup-surface.ts | 3 +- extensions/tlon/src/core.test.ts | 49 +++ extensions/tlon/src/setup-core.ts | 2 + extensions/twitch/src/setup-surface.test.ts | 87 +++++- extensions/twitch/src/setup-surface.ts | 55 +++- packages/gateway-protocol/CHANGELOG.md | 4 +- .../src/openclaw.schema.test.ts | 131 ++++++++ .../src/schema/openclaw.test.ts | 15 + .../gateway-protocol/src/schema/openclaw.ts | 10 +- .../gateway-protocol/src/schema/wizard.ts | 3 +- src/channels/plugins/setup-wizard-types.ts | 4 +- src/channels/plugins/setup-wizard.ts | 41 ++- src/gateway/server-methods/shared-types.ts | 7 + .../system-agent-chat-turn.test.ts | 88 ++++++ .../server-methods/system-agent-chat-turn.ts | 71 +++++ .../system-agent-session-ownership.test.ts | 63 +++- src/gateway/server-methods/system-agent.ts | 91 +++--- src/gateway/server-methods/wizard.test.ts | 81 +++++ src/gateway/server-methods/wizard.ts | 23 +- src/system-agent/chat-engine.test.ts | 287 ++++++++++++++++++ src/system-agent/chat-engine.ts | 117 ++++++- src/wizard/i18n/locales/en.ts | 5 +- src/wizard/i18n/locales/zh-CN.ts | 5 +- src/wizard/i18n/locales/zh-TW.ts | 5 +- src/wizard/session.ts | 10 + .../custodian/custodian-panel.test.ts | 19 +- .../components/form-controls.browser.test.ts | 36 +++ ui/src/components/sensitive-input.test.ts | 92 ++++++ ui/src/components/sensitive-input.ts | 101 ++++++ ui/src/components/wizard-step-controls.ts | 58 ++-- ui/src/e2e/custodian-event-nudge.e2e.test.ts | 121 ++++++++ ui/src/pages/channels/channels-page.ts | 4 + ui/src/pages/channels/view.pairing.test.ts | 4 + ui/src/pages/channels/view.test.ts | 4 + ui/src/pages/channels/view.ts | 4 + ui/src/pages/channels/view.types.ts | 4 + ui/src/pages/channels/wizard-host.ts | 26 +- .../pages/channels/wizard-view.busy.test.ts | 31 +- ui/src/pages/channels/wizard-view.test.ts | 78 +++++ ui/src/pages/channels/wizard-view.ts | 18 +- .../custodian-page.session-lifecycle.test.ts | 54 ++++ ui/src/pages/custodian/custodian-page.test.ts | 124 +++++++- .../custodian/custodian-session-store.ts | 98 ++++-- ui/src/pages/custodian/custodian-surface.ts | 111 ++++--- .../custodian/custodian-wizard-step.test.ts | 60 ++++ .../pages/custodian/custodian-wizard-step.ts | 62 ++++ ui/src/pages/custodian/transcript.ts | 32 ++ ui/src/styles/channels.css | 14 + ui/src/styles/components.css | 130 ++++++++ ui/src/styles/custodian.css | 15 + 53 files changed, 2398 insertions(+), 224 deletions(-) create mode 100644 packages/gateway-protocol/src/openclaw.schema.test.ts create mode 100644 src/gateway/server-methods/system-agent-chat-turn.test.ts create mode 100644 src/gateway/server-methods/system-agent-chat-turn.ts create mode 100644 ui/src/components/sensitive-input.test.ts create mode 100644 ui/src/components/sensitive-input.ts create mode 100644 ui/src/pages/custodian/custodian-wizard-step.test.ts create mode 100644 ui/src/pages/custodian/custodian-wizard-step.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 260255abd777..b082ec07227b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ Docs: https://docs.openclaw.ai - **Gateway TTS playback:** add an operator-scoped `tts.speak` RPC that returns configured-provider speech as inline whole-clip audio for remote clients. (#100708, #100770) - **Workboard dispatch cap:** add a request-scoped `--max-starts` override while preserving the default cap, sequential starts, and one-card-per-owner guard. (#100174) Thanks @souvikDevloper. - **Plugin install provenance warnings:** require explicit `--force` acknowledgement for arbitrary executable plugin sources in CLI and chat installs, keep trusted ClawHub, bundled, official-catalog, and tracked-update flows frictionless, and restrict Crestodian installs to trusted sources. (#102197) Thanks @jesse-merhi. +- **Custodian rich setup controls:** render the Gateway's sanitized wizard steps as native selects, multiselects, text fields, and masked secret inputs while preserving text-only chat compatibility. (#114631) Thanks @jesse-merhi. ### Fixes diff --git a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift index 5eb9d086b09e..4e54818e7068 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift @@ -9297,6 +9297,7 @@ public struct ConfigSchemaLookupResult: Codable, Sendable { public struct SystemAgentChatParams: Codable, Sendable { public let sessionid: String public let message: String? + public let wizardanswer: [String: AnyCodable]? public let welcomevariant: AnyCodable? public let reset: Bool? public let context: [String: AnyCodable]? @@ -9305,6 +9306,7 @@ public struct SystemAgentChatParams: Codable, Sendable { public init( sessionid: String, message: String? = nil, + wizardanswer: [String: AnyCodable]? = nil, welcomevariant: AnyCodable? = nil, reset: Bool? = nil, context: [String: AnyCodable]? = nil, @@ -9312,6 +9314,7 @@ public struct SystemAgentChatParams: Codable, Sendable { { self.sessionid = sessionid self.message = message + self.wizardanswer = wizardanswer self.welcomevariant = welcomevariant self.reset = reset self.context = context @@ -9321,6 +9324,7 @@ public struct SystemAgentChatParams: Codable, Sendable { private enum CodingKeys: String, CodingKey { case sessionid = "sessionId" case message + case wizardanswer = "wizardAnswer" case welcomevariant = "welcomeVariant" case reset case context @@ -9339,6 +9343,7 @@ public struct SystemAgentChatResult: Codable, Sendable { public let needsapproval: Bool? public let proposalid: String? public let question: [String: AnyCodable]? + public let step: WizardStep? public init( sessionid: String, @@ -9350,7 +9355,8 @@ public struct SystemAgentChatResult: Codable, Sendable { agentid: String? = nil, needsapproval: Bool? = nil, proposalid: String? = nil, - question: [String: AnyCodable]? = nil) + question: [String: AnyCodable]? = nil, + step: WizardStep? = nil) { self.sessionid = sessionid self.reply = reply @@ -9362,6 +9368,7 @@ public struct SystemAgentChatResult: Codable, Sendable { self.needsapproval = needsapproval self.proposalid = proposalid self.question = question + self.step = step } private enum CodingKeys: String, CodingKey { @@ -9375,6 +9382,7 @@ public struct SystemAgentChatResult: Codable, Sendable { case needsapproval = "needsApproval" case proposalid = "proposalId" case question + case step } } diff --git a/extensions/synology-chat/src/core.test.ts b/extensions/synology-chat/src/core.test.ts index e5b0dea0df39..7ccd82f711da 100644 --- a/extensions/synology-chat/src/core.test.ts +++ b/extensions/synology-chat/src/core.test.ts @@ -146,6 +146,60 @@ describe("synology-chat core", () => { ); }); + it("never sends an existing token-bearing incoming URL back through setup prompts", async () => { + const existingIncomingUrl = + "https://nas.example.com/webapi/entry.cgi?api=SYNO.Chat.External&token=existing-secret"; + const replacementIncomingUrl = + "https://nas.example.com/webapi/entry.cgi?api=SYNO.Chat.External&token=replacement"; + const text = vi.fn(async ({ message }: { message: string }) => { + if (message === "Incoming webhook URL") { + return replacementIncomingUrl; + } + if (message === "Outgoing webhook path (optional)") { + return ""; + } + throw new Error(`Unexpected prompt: ${message}`); + }); + const confirm = vi.fn(async ({ message }: { message: string }) => { + if (message === "Synology Chat webhook token already configured. Keep it?") { + return true; + } + if (message.startsWith("Incoming webhook URL")) { + return false; + } + throw new Error(`Unexpected confirmation: ${message}`); + }); + const prompter = createTestWizardPrompter({ + text: text as WizardPrompter["text"], + confirm, + }); + + const result = await runSetupWizardConfigure({ + configure: synologyChatConfigure, + cfg: { + channels: { + "synology-chat": { + enabled: true, + token: "existing-outgoing-token", + incomingUrl: existingIncomingUrl, + }, + }, + } as OpenClawConfig, + prompter, + options: { secretInputMode: "plaintext" as const }, + }); + + expect(result.cfg.channels?.["synology-chat"]?.incomingUrl).toBe(replacementIncomingUrl); + expect(JSON.stringify({ confirms: confirm.mock.calls, texts: text.mock.calls })).not.toContain( + existingIncomingUrl, + ); + const urlPrompt = text.mock.calls.find( + ([args]) => args.message === "Incoming webhook URL", + )?.[0]; + expect(urlPrompt).toMatchObject({ sensitive: true }); + expect(urlPrompt).not.toHaveProperty("initialValue"); + }); + it("records allowed user ids when setup forces allowFrom", async () => { const prompter = createSynologySetupPrompter({ allowedUserIds: "123456, synology-chat:789012", diff --git a/extensions/synology-chat/src/setup-surface.ts b/extensions/synology-chat/src/setup-surface.ts index 1c76818b8c33..ee28adf9c519 100644 --- a/extensions/synology-chat/src/setup-surface.ts +++ b/extensions/synology-chat/src/setup-surface.ts @@ -294,8 +294,9 @@ export const synologyChatSetupWizard: ChannelSetupWizard = { t("wizard.synologyChat.incomingWebhookHelpUseUrl"), t("wizard.synologyChat.incomingWebhookHelpReplies"), ], + sensitive: true, currentValue: ({ cfg, accountId }) => getRawAccountConfig(cfg, accountId).incomingUrl?.trim(), - keepPrompt: (value) => t("wizard.synologyChat.incomingWebhookKeep", { value }), + keepPrompt: t("wizard.synologyChat.incomingWebhookKeep"), validate: ({ value }) => validateWebhookUrl(value), applySet: async ({ cfg, accountId, value }) => patchSynologyChatAccountConfig({ diff --git a/extensions/tlon/src/core.test.ts b/extensions/tlon/src/core.test.ts index cf4ec5824e8a..02aec3ad67af 100644 --- a/extensions/tlon/src/core.test.ts +++ b/extensions/tlon/src/core.test.ts @@ -205,6 +205,55 @@ describe("tlon core", () => { expect(result.cfg.channels?.tlon?.network?.dangerouslyAllowPrivateNetwork).toBe(false); }); + it("never sends an existing login code back through setup prompts", async () => { + const existingCode = "lidlut-existing-secret-code"; + const text = vi.fn(async ({ message }: { message: string }) => { + if (message === "Login code") { + return "lidlut-replacement-code"; + } + throw new Error(`Unexpected prompt: ${message}`); + }); + const confirm = vi.fn(async ({ message }: { message: string }) => { + if (message.startsWith("Ship name") || message.startsWith("Ship URL")) { + return true; + } + if (message.startsWith("Login code")) { + return false; + } + if (message === "Enable auto-discovery of group channels?") { + return true; + } + return false; + }); + const prompter = createTestWizardPrompter({ + text: text as WizardPrompter["text"], + confirm, + }); + + const result = await runSetupWizardConfigure({ + configure: tlonConfigure, + cfg: { + channels: { + tlon: { + ship: "~sampel-palnet", + url: "https://urbit.example.com", + code: existingCode, + }, + }, + } as OpenClawConfig, + prompter, + options: {}, + }); + + expect(result.cfg.channels?.tlon?.code).toBe("lidlut-replacement-code"); + expect(JSON.stringify({ confirms: confirm.mock.calls, texts: text.mock.calls })).not.toContain( + existingCode, + ); + const codePrompt = text.mock.calls.find(([args]) => args.message === "Login code")?.[0]; + expect(codePrompt).toMatchObject({ sensitive: true }); + expect(codePrompt).not.toHaveProperty("initialValue"); + }); + it("resolves dm targets to normalized ships", () => { expect(resolveTlonOutboundTarget("dm/sampel-palnet")).toEqual({ ok: true, diff --git a/extensions/tlon/src/setup-core.ts b/extensions/tlon/src/setup-core.ts index f7187452c640..afa776b66ca8 100644 --- a/extensions/tlon/src/setup-core.ts +++ b/extensions/tlon/src/setup-core.ts @@ -113,6 +113,8 @@ export function createTlonSetupWizardBase(params: TlonSetupWizardBaseParams): Ch inputKey: "code", message: t("wizard.tlon.loginCodePrompt"), placeholder: "lidlut-tabwed-pillex-ridrup", + sensitive: true, + keepPrompt: t("wizard.tlon.loginCodeKeep"), currentValue: ({ cfg, accountId }) => resolveTlonAccount(cfg, accountId).code ?? undefined, validate: ({ value }) => normalizeStringifiedOptionalString(value) ? undefined : "Required", diff --git a/extensions/twitch/src/setup-surface.test.ts b/extensions/twitch/src/setup-surface.test.ts index 74d65d6c67d4..716549af72f9 100644 --- a/extensions/twitch/src/setup-surface.test.ts +++ b/extensions/twitch/src/setup-surface.test.ts @@ -43,10 +43,16 @@ const mockAccount: TwitchAccountConfig = { clientId: "test-client-id", channel: "#testchannel", }; +const mockRefreshAccount: TwitchAccountConfig = { + ...mockAccount, + clientSecret: "existing-secret", + refreshToken: "existing-refresh", +}; function requireFirstTextPromptArgs(): { message?: string; initialValue?: string; + sensitive?: boolean; validate?: (value: string) => string | undefined; } { const [call] = mockPromptText.mock.calls; @@ -56,6 +62,7 @@ function requireFirstTextPromptArgs(): { return call[0] as { message?: string; initialValue?: string; + sensitive?: boolean; validate?: (value: string) => string | undefined; }; } @@ -78,7 +85,7 @@ describe("setup surface helpers", () => { it("should return existing token when user confirms to keep it", async () => { mockPromptConfirm.mockResolvedValue(true); - const result = await promptToken(mockPrompter, mockAccount, undefined); + const result = await promptToken(mockPrompter, mockAccount); expect(result).toBe("oauth:test123"); expect(mockPromptConfirm).toHaveBeenCalledWith({ @@ -88,8 +95,7 @@ describe("setup surface helpers", () => { expect(mockPromptText).not.toHaveBeenCalled(); }); - it("should validate token format", async () => { - // Set up mocks - user doesn't want to keep existing token + it("should use a sensitive prompt when replacing a configured token", async () => { mockPromptConfirm.mockResolvedValueOnce(false); // Track how many times promptText is called @@ -106,11 +112,15 @@ describe("setup surface helpers", () => { }); // Call promptToken - const result = await promptToken(mockPrompter, mockAccount, undefined); + const result = await promptToken(mockPrompter, mockAccount); // Verify promptText was called expect(promptTextCallCount).toBe(1); expect(result).toBe("oauth:test123"); + expect(requireFirstTextPromptArgs()).toMatchObject({ + sensitive: true, + }); + expect(requireFirstTextPromptArgs()).not.toHaveProperty("initialValue"); // Test the validate function if (!capturedValidate) { @@ -180,18 +190,42 @@ describe("setup surface helpers", () => { it("should prompt for credentials when user accepts", async () => { mockPromptConfirm - .mockResolvedValueOnce(true) // First call: useRefresh - .mockResolvedValueOnce("secret123") // clientSecret - .mockResolvedValueOnce("refresh123"); // refreshToken + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(false) + .mockResolvedValueOnce(false); mockPromptText.mockResolvedValueOnce("secret123").mockResolvedValueOnce("refresh123"); - const result = await promptRefreshTokenSetup(mockPrompter, null); + const result = await promptRefreshTokenSetup(mockPrompter, mockRefreshAccount); expect(result).toEqual({ clientSecret: "secret123", refreshToken: "refresh123", }); + expect(mockPromptText).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + sensitive: true, + }), + ); + expect(mockPromptText.mock.calls[0]?.[0]).not.toHaveProperty("initialValue"); + expect(mockPromptText).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + sensitive: true, + }), + ); + expect(mockPromptText.mock.calls[1]?.[0]).not.toHaveProperty("initialValue"); + }); + + it("should keep existing credentials without opening masked replacement prompts", async () => { + mockPromptConfirm.mockResolvedValue(true); + + await expect(promptRefreshTokenSetup(mockPrompter, mockRefreshAccount)).resolves.toEqual({ + clientSecret: "existing-secret", + refreshToken: "existing-refresh", + }); + expect(mockPromptText).not.toHaveBeenCalled(); }); }); @@ -333,6 +367,18 @@ describe("setup surface helpers", () => { describe("setup wizard account routing", () => { type FinalizeArgs = Parameters>[0]; + async function finalizeDefaultTwitchSetup(cfg: FinalizeArgs["cfg"]) { + return await twitchSetupWizard.finalize?.({ + cfg, + accountId: "default", + credentialValues: {}, + runtime: {} as FinalizeArgs["runtime"], + prompter: mockPrompter, + options: {}, + forceAllowFrom: false, + }); + } + async function finalizeTwitchSetupForAccount(cfg: FinalizeArgs["cfg"]) { return await twitchSetupWizard.finalize?.({ cfg, @@ -345,6 +391,31 @@ describe("setup surface helpers", () => { }); } + it("uses an environment-only token without sending it to wizard prompts", async () => { + const envToken = "oauth:environment-only"; + process.env.OPENCLAW_TWITCH_ACCESS_TOKEN = envToken; + mockPromptConfirm.mockReset().mockResolvedValueOnce(true as never); + mockPromptText + .mockReset() + .mockResolvedValueOnce("env-bot" as never) + .mockResolvedValueOnce("env-client" as never); + + const result = await finalizeDefaultTwitchSetup({}); + + expect(result?.cfg?.channels?.twitch?.accounts?.default).toMatchObject({ + username: "env-bot", + accessToken: envToken, + clientId: "env-client", + }); + expect(mockPromptConfirm).toHaveBeenCalledWith({ + message: "Twitch env var OPENCLAW_TWITCH_ACCESS_TOKEN detected. Use env token?", + initialValue: true, + }); + expect(mockPromptText).toHaveBeenCalledTimes(2); + expect(JSON.stringify(mockPromptConfirm.mock.calls)).not.toContain(envToken); + expect(JSON.stringify(mockPromptText.mock.calls)).not.toContain(envToken); + }); + it("rejects reserved account ids before using them as config keys", () => { expect(() => setTwitchAccount( diff --git a/extensions/twitch/src/setup-surface.ts b/extensions/twitch/src/setup-surface.ts index 21c4fa92a0b3..b1ea7775b01f 100644 --- a/extensions/twitch/src/setup-surface.ts +++ b/extensions/twitch/src/setup-surface.ts @@ -112,11 +112,10 @@ async function noteTwitchSetupHelp(prompter: WizardPrompter): Promise { export async function promptToken( prompter: WizardPrompter, account: TwitchAccountConfig | null, - envToken: string | undefined, ): Promise { const existingToken = account?.accessToken ?? ""; - if (existingToken && !envToken) { + if (existingToken) { const keepToken = await prompter.confirm({ message: t("wizard.twitch.accessTokenKeep"), initialValue: true, @@ -129,7 +128,7 @@ export async function promptToken( return ( await prompter.text({ message: t("wizard.twitch.oauthTokenPrompt"), - initialValue: envToken ?? "", + sensitive: true, validate: (value) => { const raw = value?.trim() ?? ""; if (!raw) { @@ -191,6 +190,30 @@ export async function promptChannelName( ); } +async function promptRefreshCredential(params: { + prompter: WizardPrompter; + existingValue: string | undefined; + keepMessage: string; + inputMessage: string; +}): Promise { + const existingValue = params.existingValue?.trim(); + if (existingValue) { + const keep = await params.prompter.confirm({ + message: params.keepMessage, + initialValue: true, + }); + if (keep) { + return existingValue; + } + } + const value = await params.prompter.text({ + message: params.inputMessage, + sensitive: true, + validate: (input) => (input?.trim() ? undefined : "Required"), + }); + return value.trim() || undefined; +} + export async function promptRefreshTokenSetup( prompter: WizardPrompter, account: TwitchAccountConfig | null, @@ -204,18 +227,18 @@ export async function promptRefreshTokenSetup( return {}; } - const clientSecret = - (await promptRequiredTwitchAccountValue( - prompter, - t("wizard.twitch.clientSecretPrompt"), - account?.clientSecret, - )) || undefined; - const refreshToken = - (await promptRequiredTwitchAccountValue( - prompter, - t("wizard.twitch.refreshTokenInputPrompt"), - account?.refreshToken, - )) || undefined; + const clientSecret = await promptRefreshCredential({ + prompter, + existingValue: account?.clientSecret, + keepMessage: t("wizard.twitch.clientSecretKeep"), + inputMessage: t("wizard.twitch.clientSecretPrompt"), + }); + const refreshToken = await promptRefreshCredential({ + prompter, + existingValue: account?.refreshToken, + keepMessage: t("wizard.twitch.refreshTokenKeep"), + inputMessage: t("wizard.twitch.refreshTokenInputPrompt"), + }); return { clientSecret, refreshToken }; } @@ -466,7 +489,7 @@ export const twitchSetupWizard: ChannelSetupWizard = { } const username = await promptUsername(prompter, account); - const token = await promptToken(prompter, account, envToken); + const token = await promptToken(prompter, account); const clientId = await promptClientId(prompter, account); const channelName = await promptChannelName(prompter, account); const { clientSecret, refreshToken } = await promptRefreshTokenSetup(prompter, account); diff --git a/packages/gateway-protocol/CHANGELOG.md b/packages/gateway-protocol/CHANGELOG.md index 494060b0f27b..5340db811e21 100644 --- a/packages/gateway-protocol/CHANGELOG.md +++ b/packages/gateway-protocol/CHANGELOG.md @@ -11,6 +11,7 @@ version and the additive schema surface. Dates are authoring dates (2026). - Rename structured-question item `id` to `questionId` and flatten keyed answer arrays. - Slim worker and session-catalog payloads to the active wire contract. - Remove dead protocol surfaces and add since-vintage metadata to retained schemas and methods. +- Add optional `step` on `SystemAgentChatResult` carrying the full awaited wizard step. ## Protocol v4 (current) @@ -126,7 +127,8 @@ Enhancement-only month (no new schema modules): - Add cron event triggers via polled condition-watcher scripts (#101195) and native mobile Automations parity (#106355). - Add system-agent conversational onboarding (#99935); rename `crestodian.*` methods to - `openclaw.chat` / `openclaw.setup.*` (2026-07-14, `a6a0716`). + `openclaw.chat` / `openclaw.setup.*` (2026-07-14, `a6a0716`); add typed hosted-wizard + steps and answers to `openclaw.chat` (#114631). - Add typed structured questions / `ask_user` with live option cards (#109922, #110242) and the questions schema module. - Add ui-command / screen-tool Control UI layout control and capability-gated diff --git a/packages/gateway-protocol/src/openclaw.schema.test.ts b/packages/gateway-protocol/src/openclaw.schema.test.ts new file mode 100644 index 000000000000..96b1513aec79 --- /dev/null +++ b/packages/gateway-protocol/src/openclaw.schema.test.ts @@ -0,0 +1,131 @@ +// Gateway Protocol tests cover openclaw.schema behavior. +import { Compile } from "typebox/compile"; +import { describe, expect, it } from "vitest"; +import { SystemAgentChatResultSchema } from "./schema/openclaw.js"; +import type { WizardStep } from "./schema/wizard.js"; + +/** + * The chat result carries the awaited wizard step verbatim so control-capable + * clients can render it. Every step type has to survive the wire, including the + * fields the card-shaped `question` projection drops. + */ +describe("SystemAgentChatResultSchema", () => { + const validate = Compile(SystemAgentChatResultSchema); + const base = { sessionId: "chat-1", reply: "Bot token", action: "none" }; + + const steps: Array<{ name: string; step: WizardStep }> = [ + { + // No initialValue: the schema still permits one (wizard.start/next carry + // prefill for editable prompts), but the chat engine strips it from a + // sensitive step before serializing, so this is the shape that ships here. + name: "sensitive text carrying placeholder but no prefilled secret", + step: { + id: "step-text", + type: "text", + message: "Bot token", + placeholder: "123:abc", + sensitive: true, + executor: "client", + }, + }, + { + name: "non-sensitive text carrying a prefilled value", + step: { + id: "step-text-prefill", + type: "text", + message: "Display name", + initialValue: "openclaw-bot", + executor: "client", + }, + }, + { + name: "select with options", + step: { + id: "step-select", + type: "select", + message: "DM mode", + options: [ + { value: "alpha", label: "Alpha", hint: "First" }, + { value: "beta", label: "Beta" }, + ], + initialValue: "beta", + executor: "client", + }, + }, + { + name: "multiselect with options", + step: { + id: "step-multiselect", + type: "multiselect", + message: "Features", + options: [ + { value: "alerts", label: "Alerts" }, + { value: "logs", label: "Logs" }, + ], + initialValue: ["alerts"], + executor: "client", + }, + }, + { + name: "confirm", + step: { + id: "step-confirm", + type: "confirm", + message: "Enable delegated auth?", + initialValue: false, + executor: "client", + }, + }, + { + // The engine auto-answers notes, so this shape only reaches clients on the + // wizard methods; the chat result still has to accept it losslessly. + name: "note carrying a device code and an external URL", + step: { + id: "step-note", + type: "note", + title: "Sign in", + message: "Enter this one-time code on the provider's sign-in page.", + format: "plain", + externalUrl: "https://example.com/auth", + deviceCode: { + code: "ABCD-EFGH", + expiresInMinutes: 15, + message: "Never share this code.", + }, + executor: "client", + }, + }, + { + name: "progress", + step: { + id: "step-progress", + type: "progress", + message: "Linking your account", + executor: "gateway", + }, + }, + { + name: "action executed by the client", + step: { + id: "step-action", + type: "action", + title: "Authorize", + message: "Approve the app in your browser.", + externalUrl: "https://example.com/authorize", + executor: "client", + }, + }, + ]; + + it.each(steps)("accepts a chat result carrying a $name step", ({ step }) => { + expect(validate.Check({ ...base, step })).toBe(true); + }); + + it("stays optional for replies with no awaited step", () => { + expect(validate.Check(base)).toBe(true); + }); + + it("rejects a step outside the wizard step contract", () => { + expect(validate.Check({ ...base, step: { id: "step-bogus", type: "freeform" } })).toBe(false); + }); +}); diff --git a/packages/gateway-protocol/src/schema/openclaw.test.ts b/packages/gateway-protocol/src/schema/openclaw.test.ts index ca0cd76d790e..171ae394a36c 100644 --- a/packages/gateway-protocol/src/schema/openclaw.test.ts +++ b/packages/gateway-protocol/src/schema/openclaw.test.ts @@ -23,6 +23,21 @@ describe("OpenClaw chat params protocol", () => { ).toBe(true); }); + it("accepts a typed wizard answer and rejects unknown answer fields", () => { + expect( + validateSystemAgentChatParams({ + sessionId: "session-1", + wizardAnswer: { stepId: "channel", value: "twitch" }, + }), + ).toBe(true); + expect( + validateSystemAgentChatParams({ + sessionId: "session-1", + wizardAnswer: { stepId: "channel", value: "twitch", display: "Twitch" }, + }), + ).toBe(false); + }); + it("rejects unsafe page ids and unknown context fields", () => { expect(validateSystemAgentChatParams({ ...base, context: { page: "channels?tab=all" } })).toBe( false, diff --git a/packages/gateway-protocol/src/schema/openclaw.ts b/packages/gateway-protocol/src/schema/openclaw.ts index b5dc01ebb8b3..d0d5c9553847 100644 --- a/packages/gateway-protocol/src/schema/openclaw.ts +++ b/packages/gateway-protocol/src/schema/openclaw.ts @@ -3,7 +3,7 @@ import type { Static } from "typebox"; import { Type } from "typebox"; import { closedObject } from "./closed-object.js"; import { NonEmptyString } from "./primitives.js"; -import { WizardStartResultSchema } from "./wizard.js"; +import { WizardAnswerSchema, WizardStartResultSchema, WizardStepSchema } from "./wizard.js"; /** * OpenClaw chat lets clients (macOS app onboarding, future UIs) hold the @@ -13,7 +13,10 @@ import { WizardStartResultSchema } from "./wizard.js"; */ export const SystemAgentChatParamsSchema = closedObject({ sessionId: NonEmptyString, + /** Free-text input for conversational and text-only clients. */ message: Type.Optional(Type.String()), + /** Typed answer from a client rendering the current `WizardStep`. */ + wizardAnswer: Type.Optional(WizardAnswerSchema), /** Seeds a purpose-specific first greeting for a fresh conversation. */ welcomeVariant: Type.Optional( Type.Union([Type.Literal("onboarding"), Type.Literal("new-agent")]), @@ -90,6 +93,11 @@ export const SystemAgentChatResultSchema = closedObject({ needsApproval: Type.Optional(Type.Boolean()), proposalId: Type.Optional(NonEmptyString), question: Type.Optional(SystemAgentChatQuestionSchema), + /** + * The awaited wizard step in full. `question` above is a lossy card projection + * of the same step, so control-capable clients render this instead. + */ + step: Type.Optional(WizardStepSchema), }); export const SystemAgentChatHistoryParamsSchema = closedObject({ diff --git a/packages/gateway-protocol/src/schema/wizard.ts b/packages/gateway-protocol/src/schema/wizard.ts index 6f803155335d..71aca05dd9f6 100644 --- a/packages/gateway-protocol/src/schema/wizard.ts +++ b/packages/gateway-protocol/src/schema/wizard.ts @@ -25,7 +25,7 @@ export const WizardStartParamsSchema = closedObject({ }); /** Client answer payload for the current wizard step. */ -const WizardAnswerSchema = closedObject({ +export const WizardAnswerSchema = closedObject({ stepId: NonEmptyString, value: Type.Optional(Type.Unknown()), }); @@ -122,6 +122,7 @@ export const WizardStatusResultSchema = closedObject({ // Wire types derive directly from local schema consts so public d.ts graphs never // pull in the ProtocolSchemas registry. export type WizardStartParams = Static; +export type WizardAnswer = Static; export type WizardNextParams = Static; export type WizardCancelParams = Static; export type WizardStatusParams = Static; diff --git a/src/channels/plugins/setup-wizard-types.ts b/src/channels/plugins/setup-wizard-types.ts index f4a6c91c51a0..35c2b6ef1e9f 100644 --- a/src/channels/plugins/setup-wizard-types.ts +++ b/src/channels/plugins/setup-wizard-types.ts @@ -120,12 +120,14 @@ export type ChannelSetupWizardCredential = { }) => OpenClawConfig | Promise; }; -/** Declarative non-secret text step that can depend on resolved credentials. */ +/** Declarative text step that can depend on resolved credentials. */ export type ChannelSetupWizardTextInput = { /** Plugin-owned key written into the runtime setup input. */ inputKey: string; message: string; placeholder?: string; + /** Mask input and keep any configured value server-side. */ + sensitive?: boolean; required?: boolean; applyEmptyValue?: boolean; helpTitle?: string; diff --git a/src/channels/plugins/setup-wizard.ts b/src/channels/plugins/setup-wizard.ts index 08218003995a..916b43d7b442 100644 --- a/src/channels/plugins/setup-wizard.ts +++ b/src/channels/plugins/setup-wizard.ts @@ -232,6 +232,21 @@ async function applyWizardTextInputValue(params: { }).cfg; } +function resolveTextInputKeepMessage( + input: ChannelSetupWizardTextInput, + currentValue: string, +): string { + if (input.sensitive === true) { + // Never pass a configured secret to plugin-owned presentation code. + return typeof input.keepPrompt === "string" + ? input.keepPrompt + : `${input.message} already configured. Keep it?`; + } + return typeof input.keepPrompt === "function" + ? input.keepPrompt(currentValue) + : (input.keepPrompt ?? `${input.message} set (${currentValue}). Keep it?`); +} + export function buildChannelSetupWizardAdapterFromSetupWizard(params: { plugin: ChannelSetupWizardPlugin; wizard: ChannelSetupWizard; @@ -511,11 +526,7 @@ export function buildChannelSetupWizardAdapterFromSetupWizard(params: { if (currentValue && textInput.confirmCurrentValue !== false) { const keep = await prompter.confirm({ - message: - typeof textInput.keepPrompt === "function" - ? textInput.keepPrompt(currentValue) - : (textInput.keepPrompt ?? - `${textInput.message} set (${currentValue}). Keep it?`), + message: resolveTextInputKeepMessage(textInput, currentValue), initialValue: true, }); if (keep) { @@ -533,17 +544,21 @@ export function buildChannelSetupWizardAdapterFromSetupWizard(params: { } } - const initialValue = normalizeOptionalString( - (await textInput.initialValue?.({ - cfg: next, - accountId, - credentialValues, - })) ?? currentValue, - ); + const initialValue = + textInput.sensitive === true + ? undefined + : normalizeOptionalString( + (await textInput.initialValue?.({ + cfg: next, + accountId, + credentialValues, + })) ?? currentValue, + ); const rawValue = await prompter.text({ message: textInput.message, - initialValue, placeholder: textInput.placeholder, + ...(textInput.sensitive === true ? {} : { initialValue }), + ...(textInput.sensitive === true ? { sensitive: true } : {}), validate: (value) => { const trimmed = normalizeOptionalString(value) ?? ""; if (!trimmed && textInput.required !== false) { diff --git a/src/gateway/server-methods/shared-types.ts b/src/gateway/server-methods/shared-types.ts index 541292f2012a..5a4d48a4b0eb 100644 --- a/src/gateway/server-methods/shared-types.ts +++ b/src/gateway/server-methods/shared-types.ts @@ -1,6 +1,7 @@ import type { SessionApprovalReplay, SystemAgentChatQuestion, + WizardAnswer, } from "../../../packages/gateway-protocol/src/index.js"; // Shared server-method types define the client, context, response, and handler // contracts used by every gateway RPC method module. @@ -136,6 +137,12 @@ type GatewaySystemAgentSession = { sensitive?: boolean; question?: SystemAgentChatQuestion; }>; + answerWizard: (answer: WizardAnswer) => Promise<{ + text: string; + action: "none" | "exit" | "open-tui" | "open-setup"; + sensitive?: boolean; + question?: SystemAgentChatQuestion; + }>; seedHistory: (turns: readonly SystemAgentHistoryTurn[]) => void; historyLength: () => number; historySince: (index: number) => SystemAgentHistoryTurn[]; diff --git a/src/gateway/server-methods/system-agent-chat-turn.test.ts b/src/gateway/server-methods/system-agent-chat-turn.test.ts new file mode 100644 index 000000000000..f9ffeab6bb67 --- /dev/null +++ b/src/gateway/server-methods/system-agent-chat-turn.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it, vi } from "vitest"; +import { + buildSystemAgentChatResult, + getSystemAgentChatInputError, + runSystemAgentChatInput, +} from "./system-agent-chat-turn.js"; + +function makeEngine() { + const handle = vi.fn(); + const answerWizard = vi.fn(); + return { + answerWizard, + handle, + engine: { answerWizard, handle }, + }; +} + +describe("system-agent chat input", () => { + it.each([ + { + input: { + sessionId: "s1", + message: "5", + wizardAnswer: { stepId: "channel", value: "twitch" }, + }, + error: "Send either message or wizardAnswer, not both.", + }, + { + input: { + sessionId: "s1", + wizardAnswer: { stepId: "secret", value: "not-forwarded" }, + delegation: { agentId: "main", sessionKey: "agent:main:main" }, + }, + error: "Delegated OpenClaw sessions cannot submit structured wizard answers.", + }, + { + input: { + sessionId: "s1", + wizardAnswer: { stepId: "channel", value: "twitch" }, + reset: true, + }, + error: "A wizard answer cannot reset its OpenClaw chat session.", + }, + ])("rejects invalid mixed input: $error", ({ input, error }) => { + expect(getSystemAgentChatInputError(input)).toBe(error); + }); + + it("routes a structured wizard answer through the typed engine seam", async () => { + const { engine, answerWizard, handle } = makeEngine(); + answerWizard.mockResolvedValue({ text: "Next step.", action: "none" }); + + await expect( + runSystemAgentChatInput({ + engine, + input: { + sessionId: "s1", + wizardAnswer: { stepId: "channel", value: "twitch" }, + }, + }), + ).resolves.toEqual({ text: "Next step.", action: "none" }); + + expect(answerWizard).toHaveBeenCalledWith({ stepId: "channel", value: "twitch" }); + expect(handle).not.toHaveBeenCalled(); + }); + + it("preserves the enriched wizard step in the gateway result", () => { + expect( + buildSystemAgentChatResult({ + sessionId: "s1", + reply: { + text: "Choose a channel.", + action: "none", + step: { + id: "channel", + type: "select", + message: "Channel", + options: [{ label: "Twitch", value: "twitch" }], + }, + }, + }), + ).toMatchObject({ + sessionId: "s1", + reply: "Choose a channel.", + action: "none", + step: { id: "channel", type: "select" }, + }); + }); +}); diff --git a/src/gateway/server-methods/system-agent-chat-turn.ts b/src/gateway/server-methods/system-agent-chat-turn.ts new file mode 100644 index 000000000000..643f308470ea --- /dev/null +++ b/src/gateway/server-methods/system-agent-chat-turn.ts @@ -0,0 +1,71 @@ +import type { + SystemAgentChatParams, + SystemAgentChatResult, +} from "../../../packages/gateway-protocol/src/index.js"; +import type { SystemAgentChatEngine } from "../../system-agent/chat-engine.js"; + +type SystemAgentChatReply = Awaited>; +type SystemAgentChatEngineInput = Pick; + +export function getSystemAgentChatInputError(params: SystemAgentChatParams): string | undefined { + if (params.message !== undefined && params.wizardAnswer !== undefined) { + return "Send either message or wizardAnswer, not both."; + } + if (params.wizardAnswer !== undefined && params.delegation !== undefined) { + return "Delegated OpenClaw sessions cannot submit structured wizard answers."; + } + if (params.wizardAnswer !== undefined && params.reset === true) { + return "A wizard answer cannot reset its OpenClaw chat session."; + } + return undefined; +} + +export async function runSystemAgentChatInput(params: { + engine: SystemAgentChatEngineInput; + input: SystemAgentChatParams; +}): Promise { + if (params.input.wizardAnswer !== undefined) { + return await params.engine.answerWizard(params.input.wizardAnswer); + } + if (params.input.message === undefined) { + return undefined; + } + return params.input.delegation === undefined && params.input.context + ? await params.engine.handle(params.input.message, { uiContext: params.input.context }) + : await params.engine.handle(params.input.message); +} + +export function buildSystemAgentChatResult(params: { + sessionId: string; + reply: SystemAgentChatReply; + proposalId?: string; +}): SystemAgentChatResult { + const action = + params.reply.action === "open-tui" + ? "open-agent" + : params.reply.action === "open-setup" + ? "none" + : params.reply.action; + return { + sessionId: params.sessionId, + reply: + params.reply.text || + (action === "open-agent" + ? "Setup here is done — continue with your agent." + : "Nothing to change."), + action, + ...(action === "open-agent" && params.reply.agentDraft + ? { agentDraft: params.reply.agentDraft } + : {}), + ...(action === "open-agent" && + params.reply.handoff?.kind === "open-tui" && + params.reply.handoff.agentId + ? { agentId: params.reply.handoff.agentId } + : {}), + ...(params.reply.sensitive === true ? { sensitive: true } : {}), + ...(params.reply.wizardInputPending === true ? { wizardInputPending: true } : {}), + ...(params.reply.question ? { question: params.reply.question } : {}), + ...(params.reply.step ? { step: params.reply.step } : {}), + ...(params.proposalId ? { needsApproval: true, proposalId: params.proposalId } : {}), + }; +} diff --git a/src/gateway/server-methods/system-agent-session-ownership.test.ts b/src/gateway/server-methods/system-agent-session-ownership.test.ts index 0bc162be5427..b61aadebc7fc 100644 --- a/src/gateway/server-methods/system-agent-session-ownership.test.ts +++ b/src/gateway/server-methods/system-agent-session-ownership.test.ts @@ -3,6 +3,7 @@ import { expectDefined } from "@openclaw/normalization-core"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { resetCommandQueueStateForTest } from "../../process/command-queue.test-support.js"; +import { SystemAgentWizardAnswerError } from "../../system-agent/chat-engine.js"; import { systemAgentHandlers, type SystemAgentChatSession } from "./system-agent.js"; import type { GatewayClient, GatewayRequestContext, RespondFn } from "./types.js"; @@ -10,6 +11,11 @@ const setupInferenceMocks = vi.hoisted(() => ({ verifySetupInference: vi.fn() }) const delegatedInferenceMocks = vi.hoisted(() => ({ verifySystemAgentInferenceWithFallback: vi.fn(), })); +const transcriptStoreMocks = vi.hoisted(() => ({ + appendTranscriptReset: vi.fn(), + appendTranscriptTurn: vi.fn(), + readTranscriptTail: vi.fn(() => []), +})); vi.mock("../../system-agent/setup-inference.js", () => ({ verifySetupInference: setupInferenceMocks.verifySetupInference, @@ -18,11 +24,7 @@ vi.mock("../../system-agent/inference-fallback.js", () => ({ verifySystemAgentInferenceWithFallback: delegatedInferenceMocks.verifySystemAgentInferenceWithFallback, })); -vi.mock("../../system-agent/transcript-store.js", () => ({ - appendTranscriptReset: vi.fn(), - appendTranscriptTurn: vi.fn(), - readTranscriptTail: vi.fn(() => []), -})); +vi.mock("../../system-agent/transcript-store.js", () => transcriptStoreMocks); // Ownership tests exercise fresh-session creation; keep the caretaker greeting // deterministic so identity behavior is the only variable under test. vi.mock("../../system-agent/greeting.js", () => ({ @@ -38,6 +40,7 @@ vi.mock("../../system-agent/greeting.js", () => ({ })); type FakeEngine = { + answerWizard: ReturnType; handle: ReturnType; seedHistory: ReturnType; historyLength: ReturnType; @@ -51,6 +54,9 @@ type FakeEngine = { function makeEngine(): FakeEngine { return { + answerWizard: vi.fn(async () => { + throw new SystemAgentWizardAnswerError("No hosted wizard is awaiting an answer."); + }), handle: vi.fn(async () => ({ text: "did the thing", action: "none" })), seedHistory: vi.fn(), historyLength: vi.fn(() => 0), @@ -65,13 +71,17 @@ function makeEngine(): FakeEngine { const createdEngines = vi.hoisted(() => [] as FakeEngine[]); -vi.mock("../../system-agent/chat-engine.js", () => ({ - SystemAgentChatEngine: function FakeSystemAgentChatEngine(this: FakeEngine) { - const engine = makeEngine(); - createdEngines.push(engine); - Object.assign(this, engine); - }, -})); +vi.mock("../../system-agent/chat-engine.js", () => { + class FakeSystemAgentWizardAnswerError extends Error {} + return { + SystemAgentWizardAnswerError: FakeSystemAgentWizardAnswerError, + SystemAgentChatEngine: function FakeSystemAgentChatEngine(this: FakeEngine) { + const engine = makeEngine(); + createdEngines.push(engine); + Object.assign(this, engine); + }, + }; +}); vi.mock("../../system-agent/overview.js", () => ({ formatSystemAgentStartupMessage: vi.fn(() => "welcome text"), })); @@ -315,6 +325,35 @@ describe("openclaw.chat session responses", () => { expect(call.payload).toMatchObject({ reply: "did the thing", action: "none" }); }); + it("rejects a structured answer without an active chat session", async () => { + const call = await callChat(makeContext(new Map()), { + sessionId: "missing", + wizardAnswer: { stepId: "channel", value: "twitch" }, + }); + + expect(call).toMatchObject({ + ok: false, + error: { + code: "INVALID_REQUEST", + details: { code: "system_agent_session_invalidated" }, + }, + }); + expect(setupInferenceMocks.verifySetupInference).not.toHaveBeenCalled(); + }); + + it("rejects a structured answer when the active session has no hosted wizard", async () => { + const engine = makeEngine(); + const sessions = new Map([["s1", seededSession({ engine })]]); + + const call = await callChat(makeContext(sessions), { + sessionId: "s1", + wizardAnswer: { stepId: "stale", value: "twitch" }, + }); + + expect(call).toMatchObject({ ok: false, error: { code: "INVALID_REQUEST" } }); + expect(transcriptStoreMocks.appendTranscriptTurn).not.toHaveBeenCalled(); + }); + it("forwards sensitive-input metadata", async () => { const engine = makeEngine(); engine.handle.mockResolvedValue({ diff --git a/src/gateway/server-methods/system-agent.ts b/src/gateway/server-methods/system-agent.ts index 7f2b26998a90..cfc72e904013 100644 --- a/src/gateway/server-methods/system-agent.ts +++ b/src/gateway/server-methods/system-agent.ts @@ -23,7 +23,10 @@ import { enqueueCommandInLane, setCommandLaneConcurrency } from "../../process/c import { runWithGatewayIndependentRootWorkContinuation } from "../../process/gateway-work-admission.js"; import { CommandLane } from "../../process/lanes.js"; import { defaultRuntime } from "../../runtime.js"; -import { SystemAgentChatEngine } from "../../system-agent/chat-engine.js"; +import { + SystemAgentChatEngine, + SystemAgentWizardAnswerError, +} from "../../system-agent/chat-engine.js"; import { resolveSystemAgentDelegationKey } from "../../system-agent/delegation-session.js"; import { acknowledgeSystemAgentGreetingDelivery, @@ -48,6 +51,11 @@ import { listVisiblePendingApprovalRequests, } from "./approval-shared.js"; import { sanitizeSystemAgentChatParams } from "./system-agent-chat-params.js"; +import { + buildSystemAgentChatResult, + getSystemAgentChatInputError, + runSystemAgentChatInput, +} from "./system-agent-chat-turn.js"; import type { GatewayClient, GatewayRequestContext, GatewayRequestHandlers } from "./types.js"; import { assertValidParams } from "./validation.js"; @@ -503,6 +511,11 @@ export const systemAgentHandlers: GatewayRequestHandlers = { if (!assertValidParams(params, validateSystemAgentChatParams, "openclaw.chat", respond)) { return; } + const inputError = getSystemAgentChatInputError(params); + if (inputError) { + respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, inputError)); + return; + } await runSystemAgentGatewayTask(async () => { const sessions = context.systemAgentSessions; const sessionId = params.sessionId; @@ -543,8 +556,22 @@ export const systemAgentHandlers: GatewayRequestHandlers = { await existing?.engine.dispose(); } let session = sessions.get(sessionId); + if (params.wizardAnswer !== undefined && !session) { + respond( + false, + undefined, + errorShape( + ErrorCodes.INVALID_REQUEST, + "No active OpenClaw chat session is awaiting that wizard answer.", + { details: buildSystemAgentSessionInvalidatedErrorDetails() }, + ), + ); + return; + } let greetingAuditSequence: number | undefined; - const welcomeOnly = params.message === undefined || !params.message.trim(); + const welcomeOnly = + params.wizardAnswer === undefined && + (params.message === undefined || !params.message.trim()); if (!session) { const inference = params.delegation ? await import("../../system-agent/inference-fallback.js").then( @@ -654,7 +681,10 @@ export const systemAgentHandlers: GatewayRequestHandlers = { } session.lastUsedAt = Date.now(); // Inline check (not `welcomeOnly`) so TS narrows params.message below. - if (params.message === undefined || !params.message.trim()) { + if ( + params.wizardAnswer === undefined && + (params.message === undefined || !params.message.trim()) + ) { respond( true, { @@ -671,12 +701,25 @@ export const systemAgentHandlers: GatewayRequestHandlers = { const historyStart = session.engine.historyLength(); let reply: Awaited>; try { - reply = - params.delegation === undefined && params.context - ? await session.engine.handle(params.message, { uiContext: params.context }) - : await session.engine.handle(params.message); + const turnReply = await runSystemAgentChatInput({ + engine: session.engine, + input: params, + }); + if (!turnReply) { + respond( + false, + undefined, + errorShape(ErrorCodes.INVALID_REQUEST, "OpenClaw chat input is missing."), + ); + return; + } + reply = turnReply; } catch (error) { persistEngineHistory(session.engine, historyStart); + if (error instanceof SystemAgentWizardAnswerError) { + respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, error.message)); + return; + } if (!isSystemAgentInferenceUnavailableError(error)) { throw error; } @@ -702,14 +745,6 @@ export const systemAgentHandlers: GatewayRequestHandlers = { return; } persistEngineHistory(session.engine, historyStart); - // The TUI-only "open-tui" handoff becomes a client-visible "open-agent" - // signal: the app should move the user to their normal agent chat. - const action = - reply.action === "open-tui" - ? "open-agent" - : reply.action === "open-setup" - ? "none" - : reply.action; const delegation = params.delegation; let proposalId: string | undefined; if (delegation) { @@ -725,31 +760,7 @@ export const systemAgentHandlers: GatewayRequestHandlers = { }); } } - respond( - true, - { - sessionId, - reply: - reply.text || - (action === "open-agent" - ? "Setup here is done — continue with your agent." - : "Nothing to change."), - action, - ...(action === "open-agent" && reply.agentDraft - ? { agentDraft: reply.agentDraft } - : {}), - ...(action === "open-agent" && - reply.handoff?.kind === "open-tui" && - reply.handoff.agentId - ? { agentId: reply.handoff.agentId } - : {}), - ...(reply.sensitive === true ? { sensitive: true } : {}), - ...(reply.wizardInputPending === true ? { wizardInputPending: true } : {}), - ...(reply.question ? { question: reply.question } : {}), - ...(proposalId ? { needsApproval: true, proposalId } : {}), - }, - undefined, - ); + respond(true, buildSystemAgentChatResult({ sessionId, reply, proposalId }), undefined); }); }); }, diff --git a/src/gateway/server-methods/wizard.test.ts b/src/gateway/server-methods/wizard.test.ts index 3653f960d28f..4b5f0346ddb0 100644 --- a/src/gateway/server-methods/wizard.test.ts +++ b/src/gateway/server-methods/wizard.test.ts @@ -8,6 +8,37 @@ import { createWizardSessionTracker } from "../server-wizard-sessions.js"; import type { GatewayRequestHandlerOptions } from "./types.js"; import { type SetupWizardRunner, wizardHandlers } from "./wizard.js"; +function createWizardContext( + wizardRunner: NonNullable["wizardRunner"], +) { + const wizardSessions = new Map(); + return { + wizardSessions, + wizardRunner, + findRunningWizard: () => undefined, + purgeWizardSession: (sessionId: string) => wizardSessions.delete(sessionId), + }; +} + +function readSuccessfulResponse(respond: ReturnType): Record { + expect(respond).toHaveBeenCalledOnce(); + const [ok, result] = respond.mock.calls[0] ?? []; + expect(ok).toBe(true); + expect(result).toBeDefined(); + return result as Record; +} + +async function invokeWizard( + method: "wizard.start" | "wizard.next", + params: Record, + context: ReturnType, +): Promise> { + const respond = vi.fn(); + const handler = expectDefined(wizardHandlers[method], `wizardHandlers[${method}] test invariant`); + await handler({ params, respond, context } as never); + return readSuccessfulResponse(respond); +} + describe("wizard session lookup", () => { it.each([ { method: "wizard.next", params: { sessionId: "expired" } }, @@ -211,3 +242,53 @@ describe("wizard setup ownership", () => { } }); }); + +describe("wizard step serialization", () => { + it("strips a sensitive initial value from wizard.start", async () => { + const context = createWizardContext(async (_opts, _runtime, prompter) => { + await prompter.text({ + message: "Bot token", + sensitive: true, + initialValue: "123456:REAL-SECRET", + }); + }); + const result = await invokeWizard("wizard.start", {}, context); + expect(result.step).toMatchObject({ sensitive: true }); + expect(result.step).not.toHaveProperty("initialValue"); + for (const session of context.wizardSessions.values()) { + session.cancel(); + } + }); + + it("keeps a plain default but strips the next sensitive one from wizard.next", async () => { + const context = createWizardContext(async (_opts, _runtime, prompter) => { + await prompter.text({ + message: "Display name", + initialValue: "OpenClaw", + }); + await prompter.text({ + message: "Bot token", + sensitive: true, + initialValue: "123456:REAL-SECRET", + }); + }); + const startResult = await invokeWizard("wizard.start", {}, context); + expect(startResult.step).toMatchObject({ initialValue: "OpenClaw" }); + const sessionId = startResult.sessionId; + expect(typeof sessionId).toBe("string"); + + const params = { + sessionId, + answer: { + stepId: (startResult.step as { id: string }).id, + value: "Renamed", + }, + }; + const nextResult = await invokeWizard("wizard.next", params, context); + expect(nextResult.step).toMatchObject({ sensitive: true }); + expect(nextResult.step).not.toHaveProperty("initialValue"); + for (const session of context.wizardSessions.values()) { + session.cancel(); + } + }); +}); diff --git a/src/gateway/server-methods/wizard.ts b/src/gateway/server-methods/wizard.ts index 2979338a8caf..ba2561e885a8 100644 --- a/src/gateway/server-methods/wizard.ts +++ b/src/gateway/server-methods/wizard.ts @@ -14,7 +14,11 @@ import { import type { OnboardOptions } from "../../commands/onboard-types.js"; import { createNonExitingRuntime, ExitError, type RuntimeEnv } from "../../runtime.js"; import type { WizardPrompter } from "../../wizard/prompts.js"; -import { WizardSession } from "../../wizard/session.js"; +import { + sanitizeWizardStepForClient, + WizardSession, + type WizardStep, +} from "../../wizard/session.js"; import { formatForLog } from "../ws-log.js"; import type { GatewayRequestContext, GatewayRequestHandlers, RespondFn } from "./types.js"; import { assertValidParams } from "./validation.js"; @@ -65,6 +69,10 @@ function readWizardStatus(session: WizardSession) { }; } +function sanitizeWizardResultForClient(result: T): T { + return result.step ? { ...result, step: sanitizeWizardStepForClient(result.step) } : result; +} + /** Resolves a live wizard session or sends the public not-found error. */ function findWizardSessionOrRespond(params: { context: GatewayRequestContext; @@ -135,7 +143,7 @@ export const wizardHandlers: GatewayRequestHandlers = { // clients get a clean not-found response for stale session ids. context.purgeWizardSession(sessionId); } - respond(true, { sessionId, ...result }, undefined); + respond(true, { sessionId, ...sanitizeWizardResultForClient(result) }, undefined); }, "wizard.next": async ({ params, respond, context }) => { if (!assertValidParams(params, validateWizardNextParams, "wizard.next", respond)) { @@ -155,7 +163,14 @@ export const wizardHandlers: GatewayRequestHandlers = { try { const validationError = await session.answer(answer.stepId ?? "", answer.value); if (validationError) { - respond(true, { ...(await session.next()), error: validationError }, undefined); + respond( + true, + { + ...sanitizeWizardResultForClient(await session.next()), + error: validationError, + }, + undefined, + ); return; } } catch (err) { @@ -169,7 +184,7 @@ export const wizardHandlers: GatewayRequestHandlers = { // wizard.start's immediate-completion path. context.purgeWizardSession(sessionId); } - respond(true, result, undefined); + respond(true, sanitizeWizardResultForClient(result), undefined); }, "wizard.cancel": ({ params, respond, context }) => { if (!assertValidParams(params, validateWizardCancelParams, "wizard.cancel", respond)) { diff --git a/src/system-agent/chat-engine.test.ts b/src/system-agent/chat-engine.test.ts index a542bd0ab49d..b6f1c177fa7c 100644 --- a/src/system-agent/chat-engine.test.ts +++ b/src/system-agent/chat-engine.test.ts @@ -17,6 +17,7 @@ import { runSystemAgentTurnWithDeps } from "./agent-turn.test-support.js"; import { classifySystemAgentApprovalText } from "./approval-intent.js"; import { SystemAgentChatEngine as RuntimeSystemAgentChatEngine, + SystemAgentWizardAnswerError, type SystemAgentChatEngineOptions, } from "./chat-engine.js"; import { SystemAgentInferenceUnavailableError } from "./inference-error.js"; @@ -3263,6 +3264,292 @@ describe("OpenClaw agent loop backends", () => { }); }); +describe("OpenClaw chat wizard step payload", () => { + // `action` is missing on purpose: no production path or prompter method emits + // a step of that type, so it is unreachable through this seam. The protocol + // round-trip test in packages/gateway-protocol covers it instead. + const cases: Array<{ + name: string; + run: (prompter: WizardPrompter) => Promise; + /** Undefined means no step awaits an answer when the reply is built. */ + step: Record | undefined; + }> = [ + { + name: "text", + // openUrl binds to the next created step, so this proves the fields the + // card projection drops (placeholder/initialValue/sensitive/externalUrl). + // It is optional on the prompter contract; a prompter without it would + // fail the step assertion below on the missing externalUrl. + run: async (prompter) => { + await prompter.openUrl?.("https://example.com/auth"); + await prompter.text({ + message: "Bot token", + initialValue: "seed-token", + placeholder: "123:abc", + sensitive: true, + }); + }, + // initialValue is absent on purpose: the prompt below seeds one, but a + // sensitive step's prefilled value is the secret and must not cross to + // chat-result consumers. Everything else survives verbatim. + step: { + id: expect.any(String), + type: "text", + message: "Bot token", + placeholder: "123:abc", + sensitive: true, + executor: "client", + externalUrl: "https://example.com/auth", + }, + }, + { + name: "select", + run: async (prompter) => { + // Option values avoid "telegram" so tryAutoSelectChannel cannot answer + // this step for us and null the bridge's awaited step. + await prompter.select({ + message: "DM mode", + options: [ + { value: "alpha", label: "Alpha", hint: "First" }, + { value: "beta", label: "Beta" }, + ], + initialValue: "beta", + }); + }, + step: { + id: expect.any(String), + type: "select", + message: "DM mode", + options: [ + { value: "alpha", label: "Alpha", hint: "First" }, + { value: "beta", label: "Beta" }, + ], + initialValue: "beta", + executor: "client", + }, + }, + { + name: "confirm", + run: async (prompter) => { + await prompter.confirm({ message: "Enable delegated auth?", initialValue: false }); + }, + step: { + id: expect.any(String), + type: "confirm", + message: "Enable delegated auth?", + initialValue: false, + executor: "client", + }, + }, + { + name: "multiselect", + run: async (prompter) => { + await prompter.multiselect({ + message: "Features", + options: [ + { value: "alerts", label: "Alerts" }, + { value: "logs", label: "Logs" }, + ], + }); + }, + step: { + id: expect.any(String), + type: "multiselect", + message: "Features", + options: [ + { value: "alerts", label: "Alerts" }, + { value: "logs", label: "Logs" }, + ], + executor: "client", + }, + }, + { + // Informational steps are auto-answered by the pump before the reply is + // built, so they render as prose with no control. Absent `step` here is + // the contract, not a gap. + name: "note", + run: async (prompter) => { + await prompter.note("Open the provider console first."); + }, + step: undefined, + }, + { + name: "progress", + run: async (prompter) => { + prompter.progress("Linking your account"); + }, + step: undefined, + }, + ]; + + it.each(cases)("carries the awaited $name step on the chat reply", async ({ run, step }) => { + useTempStateDir(); + const engine = new SystemAgentChatEngine({ + surface: "gateway", + runAgentTurn: async () => null, + planWithAssistant: async () => null, + deps: { loadOverview: fakeOverviewLoader() }, + runChannelSetupWizard: async (_channel: string, prompter: WizardPrompter) => { + await run(prompter); + }, + }); + + const reply = await engine.handle("connect telegram"); + + if (step) { + expect(reply.step).toEqual(step); + } else { + expect(reply.step).toBeUndefined(); + } + }); + + it("strips a sensitive step's prefilled value but keeps a plain one", async () => { + useTempStateDir(); + const makeEngine = (sensitive: boolean) => + new SystemAgentChatEngine({ + surface: "gateway", + runAgentTurn: async () => null, + planWithAssistant: async () => null, + deps: { loadOverview: fakeOverviewLoader() }, + runChannelSetupWizard: async (_channel: string, prompter: WizardPrompter) => { + await prompter.text({ + message: "Bot token", + initialValue: "123456:REAL-SECRET", + ...(sensitive ? { sensitive: true } : {}), + }); + }, + }); + + const secret = await makeEngine(true).handle("connect telegram"); + expect(secret.step?.sensitive).toBe(true); + expect(secret.step).not.toHaveProperty("initialValue"); + expect(JSON.stringify(secret)).not.toContain("REAL-SECRET"); + + // Redaction is scoped to sensitive steps; ordinary prefill still reaches + // clients, otherwise every edit-in-place prompt would lose its default. + const plain = await makeEngine(false).handle("connect telegram"); + expect(plain.step?.initialValue).toBe("123456:REAL-SECRET"); + }); + + it("omits the wizard step outside an awaiting hosted wizard", async () => { + useTempStateDir(); + const engine = new SystemAgentChatEngine({ + surface: "gateway", + runAgentTurn: async () => ({ text: "*click* Everything looks healthy." }), + planWithAssistant: async () => null, + deps: { loadOverview: fakeOverviewLoader() }, + runChannelSetupWizard: async (_channel: string, prompter: WizardPrompter) => { + await prompter.text({ message: "Bot token" }); + }, + }); + + const ordinary = await engine.handle("how is my setup looking?"); + expect(ordinary.step).toBeUndefined(); + + const awaiting = await engine.handle("connect telegram"); + expect(awaiting.step?.type).toBe("text"); + + const done = await engine.handle("123:abc"); + expect(done.text).toContain("telegram is configured"); + expect(done.step).toBeUndefined(); + }); + + it("submits a typed answer directly and records the server-owned option label", async () => { + useTempStateDir(); + let selected: unknown; + const engine = new SystemAgentChatEngine({ + surface: "gateway", + runAgentTurn: async () => null, + planWithAssistant: async () => null, + deps: { loadOverview: fakeOverviewLoader() }, + runChannelSetupWizard: async (_channel: string, prompter: WizardPrompter) => { + selected = await prompter.select({ + message: "Choose one", + options: [ + { value: "alpha", label: "Alpha" }, + { value: "beta", label: "Beta" }, + ], + }); + }, + }); + + const prompt = await engine.handle("connect telegram"); + const stepId = expectDefined(prompt.step?.id, "expected an active wizard step"); + await engine.answerWizard({ stepId, value: "beta" }); + + expect(selected).toBe("beta"); + expect(engine.historySince(0)).toContainEqual({ role: "user", text: "Beta" }); + }); + + it("rejects a stale structured answer without changing the active step", async () => { + useTempStateDir(); + const engine = new SystemAgentChatEngine({ + surface: "gateway", + runAgentTurn: async () => null, + planWithAssistant: async () => null, + deps: { loadOverview: fakeOverviewLoader() }, + runChannelSetupWizard: async (_channel: string, prompter: WizardPrompter) => { + await prompter.text({ message: "Bot token" }); + }, + }); + + const prompt = await engine.handle("connect telegram"); + await expect( + engine.answerWizard({ stepId: "stale-step", value: "ignored" }), + ).rejects.toBeInstanceOf(SystemAgentWizardAnswerError); + const stepId = expectDefined(prompt.step?.id, "expected an active wizard step"); + const done = await engine.answerWizard({ stepId, value: "123:abc" }); + + expect(done.step).toBeUndefined(); + expect(JSON.stringify(engine.historySince(0))).not.toContain("ignored"); + }); + + it("redacts a sensitive structured answer from engine history", async () => { + useTempStateDir(); + const engine = new SystemAgentChatEngine({ + surface: "gateway", + runAgentTurn: async () => null, + planWithAssistant: async () => null, + deps: { loadOverview: fakeOverviewLoader() }, + runChannelSetupWizard: async (_channel: string, prompter: WizardPrompter) => { + await prompter.text({ message: "Bot token", sensitive: true }); + }, + }); + + const prompt = await engine.handle("connect telegram"); + const stepId = expectDefined(prompt.step?.id, "expected an active wizard step"); + await engine.answerWizard({ stepId, value: "raw-secret-value" }); + + expect(engine.historySince(0)).toContainEqual({ role: "user", text: "" }); + expect(JSON.stringify(engine.historySince(0))).not.toContain("raw-secret-value"); + }); + + it("keeps the numbered text grammar for text-only wizard clients", async () => { + useTempStateDir(); + let selected: unknown; + const engine = new SystemAgentChatEngine({ + surface: "gateway", + runAgentTurn: async () => null, + planWithAssistant: async () => null, + deps: { loadOverview: fakeOverviewLoader() }, + runChannelSetupWizard: async (_channel: string, prompter: WizardPrompter) => { + selected = await prompter.select({ + message: "Choose one", + options: [ + { value: "alpha", label: "Alpha" }, + { value: "beta", label: "Beta" }, + ], + }); + }, + }); + + await engine.handle("connect telegram"); + await engine.handle("2"); + + expect(selected).toBe("beta"); + }); +}); + function fakeOverviewLoader( overrides: { defaultModel?: string; claudeFound?: boolean; codexFound?: boolean } = {}, ) { diff --git a/src/system-agent/chat-engine.ts b/src/system-agent/chat-engine.ts index db8fff76174e..b5305a77aa98 100644 --- a/src/system-agent/chat-engine.ts +++ b/src/system-agent/chat-engine.ts @@ -1,10 +1,18 @@ // OpenClaw chat engine: transport-agnostic conversation over typed operations. -import type { SystemAgentChatQuestion } from "../../packages/gateway-protocol/src/index.js"; +import type { + SystemAgentChatQuestion, + WizardAnswer, +} from "../../packages/gateway-protocol/src/index.js"; import { isSensitiveConfigPath } from "../config/sensitive-paths.js"; import { formatErrorMessage } from "../infra/errors.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import type { RuntimeEnv } from "../runtime.js"; -import { WizardSession, wizardStepAwaitsInput, type WizardStep } from "../wizard/session.js"; +import { + sanitizeWizardStepForClient, + WizardSession, + wizardStepAwaitsInput, + type WizardStep, +} from "../wizard/session.js"; import type { MemoryImportProviderOutcome, SetupMemoryImportOutcome, @@ -132,6 +140,8 @@ type SystemAgentChatReply = { handoff?: SystemAgentOperation; /** Structured choice mirroring the awaited wizard step for card-capable clients. */ question?: SystemAgentChatQuestion; + /** The awaited wizard step in full; `question` is its lossy card projection. */ + step?: WizardStep; }; type WizardPrompterLike = import("../wizard/prompts.js").WizardPrompter; @@ -599,6 +609,48 @@ function parseWizardAnswer(step: WizardStep, text: string): { value: unknown } | return { value: step.type === "action" ? true : undefined }; } +function formatStructuredWizardAnswerForHistory(step: WizardStep, value: unknown): string { + if (step.sensitive === true) { + return ""; + } + if (step.type === "text") { + if ( + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" || + typeof value === "bigint" + ) { + return String(value); + } + return ""; + } + if (step.type === "confirm") { + return typeof value === "boolean" ? (value ? "Yes" : "No") : ""; + } + if (step.type === "select") { + return ( + step.options?.find((option) => Object.is(option.value, value))?.label ?? "" + ); + } + if (step.type === "multiselect") { + if (!Array.isArray(value)) { + return ""; + } + if (value.length === 0) { + return "None"; + } + const labels = value.map( + (entry) => step.options?.find((option) => Object.is(option.value, entry))?.label, + ); + return labels.every((label): label is string => label !== undefined) + ? labels.join(", ") + : ""; + } + return "Continue"; +} + +export class SystemAgentWizardAnswerError extends Error {} + function formatOperationError(error: unknown): string { const message = error instanceof Error ? error.message : String(error); return `That did not go through: ${message}`; @@ -736,6 +788,12 @@ export class SystemAgentChatEngine { return await turn; } + async answerWizard(answer: WizardAnswer): Promise { + const turn = this.turnQueue.then(() => this.answerWizardSerialized(answer)); + this.turnQueue = turn.catch(() => undefined); + return await turn; + } + private async handleSerialized( text: string, options?: SystemAgentChatTurnOptions, @@ -744,30 +802,57 @@ export class SystemAgentChatEngine { // Snapshot before resolving: wizard answers to sensitive steps (tokens, // passwords) must never enter the AI-visible history. const sensitiveTurn = this.wizardBridge?.step?.sensitive === true; - const resolved = await this.resolveTurn(text, options); + const reply = await this.resolveTurn(text, options); + return this.completeTurn( + reply, + sensitiveTurn ? "" : redactSensitiveCommandText(text), + ); + } + + private async answerWizardSerialized(answer: WizardAnswer): Promise { + await this.requireVerifiedInference(); + const bridge = this.wizardBridge; + const step = bridge?.step; + if (!bridge || !step) { + throw new SystemAgentWizardAnswerError("No hosted wizard is awaiting an answer."); + } + if (answer.stepId !== step.id) { + throw new SystemAgentWizardAnswerError("The hosted wizard answer targets a stale step."); + } + const validationError = await bridge.session.answer(step.id, answer.value); + const text = validationError + ? [validationError, renderWizardStep(step)].join("\n\n") + : await this.pumpWizardBridge(); + return this.completeTurn( + { text, action: "none" }, + formatStructuredWizardAnswerForHistory(step, answer.value), + ); + } + + private completeTurn(reply: SystemAgentChatReply, userHistoryText: string): SystemAgentChatReply { // The hint belongs to the outgoing message, not to each rendered step: one // turn can concatenate several auto-answered notes, and a wizard that just // ended must not offer a cancel that can no longer happen. const awaitedStep = this.wizardBridge?.step; - const reply: SystemAgentChatReply = - resolved.text && awaitedStep && wizardStepAwaitsInput(awaitedStep) - ? { ...resolved, text: `${resolved.text}\n${WIZARD_CANCEL_HINT}` } - : resolved; - this.history.push({ - role: "user", - text: sensitiveTurn ? "" : redactSensitiveCommandText(text), - }); - if (reply.text) { - this.history.push({ role: "assistant", text: reply.text }); + const completedReply: SystemAgentChatReply = + reply.text && awaitedStep && wizardStepAwaitsInput(awaitedStep) + ? { ...reply, text: `${reply.text}\n${WIZARD_CANCEL_HINT}` } + : reply; + this.history.push({ role: "user", text: userHistoryText }); + if (completedReply.text) { + this.history.push({ role: "assistant", text: completedReply.text }); } // While a hosted wizard awaits a step, every turn routes to it, so the // awaited step is always the question this reply asks. - const question = wizardStepChatQuestion(this.wizardBridge?.step ?? null); + const step = this.wizardBridge?.step ?? null; + const question = wizardStepChatQuestion(step); + const clientStep = step ? sanitizeWizardStepForClient(step) : null; return { - ...reply, - ...(this.wizardBridge?.step?.sensitive === true ? { sensitive: true } : {}), + ...completedReply, + ...(step?.sensitive === true ? { sensitive: true } : {}), ...(this.wizardBridge ? { wizardInputPending: true } : {}), ...(question ? { question } : {}), + ...(clientStep ? { step: clientStep } : {}), }; } diff --git a/src/wizard/i18n/locales/en.ts b/src/wizard/i18n/locales/en.ts index 5f9ac1258c39..2a7b8e8d83ab 100644 --- a/src/wizard/i18n/locales/en.ts +++ b/src/wizard/i18n/locales/en.ts @@ -899,6 +899,7 @@ export const en = { helpNeedsUrlCode: "You need your Urbit ship URL and login code.", helpPrivateNetwork: "If your ship URL is on a private network (LAN/localhost), you must explicitly allow it during setup.", + loginCodeKeep: "Login code already configured. Keep it?", loginCodePrompt: "Login code", privateNetworkPrompt: "Ship URL looks like a private/internal host. Allow private network access? (SSRF risk)", @@ -921,7 +922,7 @@ export const en = { helpPointWebhook: "3) Point the outgoing webhook to https://{path}", incomingWebhookHelpReplies: "This is the URL OpenClaw uses to send replies back to Chat.", incomingWebhookHelpUseUrl: "Use the incoming webhook URL from Synology Chat integrations.", - incomingWebhookKeep: "Incoming webhook URL set ({value}). Keep it?", + incomingWebhookKeep: "Incoming webhook URL already configured. Keep it?", incomingWebhookTitle: "Synology Chat incoming webhook", incomingWebhookUrlPrompt: "Incoming webhook URL", multipleEntries: "Multiple entries: comma-separated.", @@ -1014,6 +1015,7 @@ export const en = { botUsernamePrompt: "Twitch bot username", channelJoinPrompt: "Channel to join", clientIdPrompt: "Twitch Client ID", + clientSecretKeep: "Client secret already configured. Keep it?", clientSecretPrompt: "Twitch Client Secret (for token refresh)", envPrompt: "Twitch env var OPENCLAW_TWITCH_ACCESS_TOKEN detected. Use env token?", helpCopyToken: "3. Copy the token (starts with 'oauth:') and Client ID", @@ -1024,6 +1026,7 @@ export const en = { helpTokenTools: " Use https://twitchtokengenerator.com/ or https://twitchapps.com/tmi/", oauthTokenPrompt: "Twitch OAuth token (oauth:...)", refreshTokenInputPrompt: "Twitch Refresh Token", + refreshTokenKeep: "Refresh token already configured. Keep it?", refreshTokenPrompt: "Enable automatic token refresh (requires client secret and refresh token)?", setupTitle: "Twitch setup", diff --git a/src/wizard/i18n/locales/zh-CN.ts b/src/wizard/i18n/locales/zh-CN.ts index 07b713660eae..e03455673d1d 100644 --- a/src/wizard/i18n/locales/zh-CN.ts +++ b/src/wizard/i18n/locales/zh-CN.ts @@ -871,6 +871,7 @@ export const zh_CN = { helpExampleUrl: "URL 示例：https://your-ship-host", helpNeedsUrlCode: "需要你的 Urbit ship URL 和登录码。", helpPrivateNetwork: "如果 ship URL 位于私有网络（LAN/localhost），设置时必须明确允许。", + loginCodeKeep: "登录码已配置。保留当前值？", loginCodePrompt: "登录码", privateNetworkPrompt: "Ship URL 看起来是私有/内部 host。允许私有网络访问？（SSRF 风险）", restrictDmsPrompt: "使用允许列表限制 DM？", @@ -892,7 +893,7 @@ export const zh_CN = { helpPointWebhook: "3) 将 outgoing webhook 指向 https://{path}", incomingWebhookHelpReplies: "这是 OpenClaw 用来向 Chat 发送回复的 URL。", incomingWebhookHelpUseUrl: "使用 Synology Chat 集成里的 incoming webhook URL。", - incomingWebhookKeep: "Incoming webhook URL 已设置（{value}）。保留？", + incomingWebhookKeep: "Incoming webhook URL 已配置。保留当前值？", incomingWebhookTitle: "Synology Chat incoming webhook", incomingWebhookUrlPrompt: "Incoming webhook URL", multipleEntries: "多个条目请用逗号分隔。", @@ -982,6 +983,7 @@ export const zh_CN = { botUsernamePrompt: "Twitch bot 用户名", channelJoinPrompt: "要加入的频道", clientIdPrompt: "Twitch Client ID", + clientSecretKeep: "Client secret 已配置。保留当前值？", clientSecretPrompt: "Twitch Client Secret（用于 token 刷新）", envPrompt: "检测到 Twitch 环境变量 OPENCLAW_TWITCH_ACCESS_TOKEN。使用环境 token？", helpCopyToken: "3. 复制 token（以 'oauth:' 开头）和 Client ID", @@ -992,6 +994,7 @@ export const zh_CN = { helpTokenTools: " 可使用 https://twitchtokengenerator.com/ 或 https://twitchapps.com/tmi/", oauthTokenPrompt: "Twitch OAuth token（oauth:...）", refreshTokenInputPrompt: "Twitch Refresh Token", + refreshTokenKeep: "Refresh token 已配置。保留当前值？", refreshTokenPrompt: "启用自动 token 刷新？（需要 client secret 和 refresh token）", setupTitle: "Twitch 设置", }, diff --git a/src/wizard/i18n/locales/zh-TW.ts b/src/wizard/i18n/locales/zh-TW.ts index cb397aae6739..fb541dd217e1 100644 --- a/src/wizard/i18n/locales/zh-TW.ts +++ b/src/wizard/i18n/locales/zh-TW.ts @@ -872,6 +872,7 @@ export const zh_TW = { helpExampleUrl: "URL 範例：https://your-ship-host", helpNeedsUrlCode: "需要你的 Urbit ship URL 和登入碼。", helpPrivateNetwork: "如果 ship URL 位於私有網路（LAN/localhost），設定時必須明確允許。", + loginCodeKeep: "登入碼已設定。保留目前值？", loginCodePrompt: "登入碼", privateNetworkPrompt: "Ship URL 看起來是私有/內部 host。允許私有網路存取？（SSRF 風險）", restrictDmsPrompt: "使用允許清單限制 DM？", @@ -893,7 +894,7 @@ export const zh_TW = { helpPointWebhook: "3) 將 outgoing webhook 指向 https://{path}", incomingWebhookHelpReplies: "這是 OpenClaw 用來向 Chat 傳送回覆的 URL。", incomingWebhookHelpUseUrl: "使用 Synology Chat 整合裡的 incoming webhook URL。", - incomingWebhookKeep: "Incoming webhook URL 已設定（{value}）。保留？", + incomingWebhookKeep: "Incoming webhook URL 已設定。保留目前值？", incomingWebhookTitle: "Synology Chat incoming webhook", incomingWebhookUrlPrompt: "Incoming webhook URL", multipleEntries: "多個項目請用逗號分隔。", @@ -983,6 +984,7 @@ export const zh_TW = { botUsernamePrompt: "Twitch bot 使用者名稱", channelJoinPrompt: "要加入的頻道", clientIdPrompt: "Twitch Client ID", + clientSecretKeep: "Client secret 已設定。保留目前值？", clientSecretPrompt: "Twitch Client Secret（用於 token 更新）", envPrompt: "偵測到 Twitch 環境變數 OPENCLAW_TWITCH_ACCESS_TOKEN。使用環境 token？", helpCopyToken: "3. 複製 token（以 'oauth:' 開頭）和 Client ID", @@ -993,6 +995,7 @@ export const zh_TW = { helpTokenTools: " 可使用 https://twitchtokengenerator.com/ 或 https://twitchapps.com/tmi/", oauthTokenPrompt: "Twitch OAuth token（oauth:...）", refreshTokenInputPrompt: "Twitch Refresh Token", + refreshTokenKeep: "Refresh token 已設定。保留目前值？", refreshTokenPrompt: "啟用自動 token 更新？（需要 client secret 和 refresh token）", setupTitle: "Twitch 設定", }, diff --git a/src/wizard/session.ts b/src/wizard/session.ts index 185d1f3b5e52..b2c96367b80c 100644 --- a/src/wizard/session.ts +++ b/src/wizard/session.ts @@ -35,6 +35,16 @@ export function wizardStepAwaitsInput(step: WizardStep): boolean { return unhandledRequirement; } +/** Remove secret prefill before a wizard step crosses a client boundary. */ +export function sanitizeWizardStepForClient(step: WizardStep): WizardStep { + if (step.sensitive !== true || step.initialValue === undefined) { + return step; + } + const safe = { ...step }; + delete safe.initialValue; + return safe; +} + type WizardSessionStatus = "running" | "done" | "cancelled" | "error"; type WizardNextResult = { diff --git a/ui/src/components/custodian/custodian-panel.test.ts b/ui/src/components/custodian/custodian-panel.test.ts index bceb7540c936..1c13100e91be 100644 --- a/ui/src/components/custodian/custodian-panel.test.ts +++ b/ui/src/components/custodian/custodian-panel.test.ts @@ -62,8 +62,8 @@ describe("custodian panel", () => { store.connect(context, "caretaker"); await vi.waitFor(() => expect(request).toHaveBeenCalledOnce()); store.messages = [ - { id: 1, role: "assistant", text: "Ready.", at: 1, question: null }, - { id: 2, role: "user", text: "Check this system", at: 2, question: null }, + { id: 1, role: "assistant", text: "Ready.", at: 1, question: null, step: null }, + { id: 2, role: "user", text: "Check this system", at: 2, question: null, step: null }, ]; panel.suppressed = false; @@ -110,7 +110,9 @@ describe("custodian panel", () => { const { context, panel, request, store } = await mountPanel(); store.connect(context, "caretaker"); await vi.waitFor(() => expect(request).toHaveBeenCalledOnce()); - store.messages = [{ id: 1, role: "user", text: "Check this system", at: 1, question: null }]; + store.messages = [ + { id: 1, role: "user", text: "Check this system", at: 1, question: null, step: null }, + ]; panel.available = false; panel.suppressed = false; panel.minimizeRequestId = 1; @@ -127,8 +129,15 @@ describe("custodian panel", () => { store.connect(context, "onboarding"); await vi.waitFor(() => expect(request).toHaveBeenCalledOnce()); store.messages = [ - { id: 1, role: "assistant", text: "Set up your system", at: 1, question: null }, - { id: 2, role: "user", text: "Continue setup", at: 2, question: null }, + { + id: 1, + role: "assistant", + text: "Set up your system", + at: 1, + question: null, + step: null, + }, + { id: 2, role: "user", text: "Continue setup", at: 2, question: null, step: null }, ]; panel.suppressed = false; diff --git a/ui/src/components/form-controls.browser.test.ts b/ui/src/components/form-controls.browser.test.ts index 2e0353e4d87b..bb61a4ac73b6 100644 --- a/ui/src/components/form-controls.browser.test.ts +++ b/ui/src/components/form-controls.browser.test.ts @@ -59,6 +59,23 @@ function controlsHtml() { `; } +function revealedSensitiveInputHtml() { + return ` + + + + + + `; +} + function mediaDeviceRowsHtml() { return `
@@ -130,6 +147,25 @@ afterAll(async () => { await browser?.close().catch(() => {}); }); +describeBrowserLayout("sensitive input visibility", () => { + it("removes the mask layer from layout when the value is revealed", async () => { + const page = await desktopContext.newPage(); + try { + await page.setContent( + `${revealedSensitiveInputHtml()}`, + ); + + const state = await page.locator("[data-sensitive-mask]").evaluate((mask) => ({ + hidden: (mask as HTMLElement).hidden, + display: getComputedStyle(mask).display, + })); + expect(state).toEqual({ hidden: true, display: "none" }); + } finally { + await page.close().catch(() => {}); + } + }); +}); + describeBrowserLayout("settings media device controls", () => { it("keeps paired selectors the same width across device labels and viewports", async () => { const page = await desktopContext.newPage(); diff --git a/ui/src/components/sensitive-input.test.ts b/ui/src/components/sensitive-input.test.ts new file mode 100644 index 000000000000..d42ee9019803 --- /dev/null +++ b/ui/src/components/sensitive-input.test.ts @@ -0,0 +1,92 @@ +/* @vitest-environment jsdom */ + +import { nothing, render } from "lit"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { renderSensitiveInput } from "./sensitive-input.ts"; + +describe("renderSensitiveInput", () => { + afterEach(() => { + for (const container of document.body.querySelectorAll("div")) { + render(nothing, container); + } + document.body.replaceChildren(); + }); + + it("conceals the value and keeps the Carapace mask in sync", () => { + const container = document.createElement("div"); + const onInput = vi.fn(); + document.body.append(container); + render( + renderSensitiveInput({ + id: "secret", + name: "secret", + value: "secret", + revealed: false, + revealLabel: "Show API key", + hideLabel: "Hide API key", + onInput, + onToggle: vi.fn(), + }), + container, + ); + + const input = container.querySelector("[data-sensitive-value]"); + const mask = container.querySelector("[data-sensitive-mask]"); + const maskText = container.querySelector("[data-sensitive-mask-text]"); + const toggle = container.querySelector(".oc-sensitive-toggle"); + expect(input?.type).toBe("password"); + expect(mask?.hidden).toBe(false); + expect(maskText?.textContent).toBe("******"); + expect(toggle?.dataset.sensitiveIcon).toBe("eye"); + expect(toggle?.getAttribute("aria-label")).toBe("Show API key"); + expect(toggle?.getAttribute("aria-pressed")).toBe("false"); + + if (input) { + input.value = "longer-key"; + input.dispatchEvent(new Event("input", { bubbles: true })); + input.scrollLeft = 12; + input.dispatchEvent(new Event("scroll")); + } + expect(onInput).toHaveBeenCalledWith("longer-key"); + expect(maskText?.textContent).toBe("**********"); + expect(maskText?.style.transform).toBe("translateX(-12px)"); + + if (input) { + input.value = "👨‍👩‍👧‍👦x"; + input.dispatchEvent(new Event("input", { bubbles: true })); + } + expect(maskText?.textContent).toBe("**"); + }); + + it("reveals the value and presents the hide action", () => { + const container = document.createElement("div"); + const onToggle = vi.fn(); + document.body.append(container); + render( + renderSensitiveInput({ + id: "secret", + value: "secret", + revealed: true, + revealLabel: "Show API key", + hideLabel: "Hide API key", + disabled: false, + onInput: vi.fn(), + onToggle, + }), + container, + ); + + const input = container.querySelector("[data-sensitive-value]"); + const mask = container.querySelector(".oc-sensitive-mask"); + const toggle = container.querySelector(".oc-sensitive-toggle"); + expect(input?.type).toBe("text"); + expect(input?.value).toBe("secret"); + expect(mask?.hidden).toBe(true); + expect(toggle?.dataset.sensitiveIcon).toBe("eye-off"); + expect(toggle?.getAttribute("aria-label")).toBe("Hide API key"); + expect(toggle?.getAttribute("aria-pressed")).toBe("true"); + + toggle?.click(); + expect(onToggle).toHaveBeenCalledOnce(); + }); +}); diff --git a/ui/src/components/sensitive-input.ts b/ui/src/components/sensitive-input.ts new file mode 100644 index 000000000000..2a78fb92d5bb --- /dev/null +++ b/ui/src/components/sensitive-input.ts @@ -0,0 +1,101 @@ +// Control UI adapter for Carapace's framework-neutral Sensitive Input pattern. +import { html, nothing, type TemplateResult } from "lit"; +import { icons } from "./icons.ts"; +import "./tooltip.ts"; + +type SensitiveInputProps = { + id: string; + name?: string; + value: string; + revealed: boolean; + revealLabel: string; + hideLabel: string; + className?: string; + inputClassName?: string; + placeholder?: string; + disabled?: boolean; + onInput: (value: string) => void; + onToggle: () => void; +}; + +const graphemeSegmenter = new Intl.Segmenter(undefined, { granularity: "grapheme" }); + +function maskedValue(value: string): string { + return "*".repeat(Array.from(graphemeSegmenter.segment(value)).length); +} + +function syncMask(input: HTMLInputElement): void { + const control = input.closest("[data-sensitive-input]"); + const maskText = control?.querySelector("[data-sensitive-mask-text]"); + if (!maskText) { + return; + } + maskText.textContent = maskedValue(input.value); + maskText.style.transform = `translateX(${-input.scrollLeft}px)`; +} + +export function renderSensitiveInput(props: SensitiveInputProps): TemplateResult { + const visibilityLabel = props.revealed ? props.hideLabel : props.revealLabel; + const className = props.className + ? `oc-sensitive-input ${props.className}` + : "oc-sensitive-input"; + const handleInput = (event: Event) => { + const input = event.currentTarget as HTMLInputElement; + syncMask(input); + props.onInput(input.value); + }; + const handleMaskSync = (event: Event) => { + syncMask(event.currentTarget as HTMLInputElement); + }; + + return html` + + + + + + + + `; +} diff --git a/ui/src/components/wizard-step-controls.ts b/ui/src/components/wizard-step-controls.ts index 4e49dea3dff5..cd81dfb03c1a 100644 --- a/ui/src/components/wizard-step-controls.ts +++ b/ui/src/components/wizard-step-controls.ts @@ -2,6 +2,7 @@ import { html, nothing, type TemplateResult } from "lit"; import type { WizardStep } from "../api/types.ts"; import { t } from "../i18n/index.ts"; import { copyToClipboard } from "../lib/clipboard.ts"; +import { renderSensitiveInput } from "./sensitive-input.ts"; import "../styles/wizard-step-controls.css"; type WizardStepOption = NonNullable[number]; @@ -16,10 +17,12 @@ type WizardStepControlsProps = { // so two step controls in one document cannot capture each other's label. inputId: string; onValueChange: (value: unknown) => void; - onAnswer: (value: unknown, includeValue?: boolean) => void; + onAnswer: (value: unknown) => void; presentation?: "channels"; answerLabel?: string; confirmAffirmativeLabel?: string; + sensitiveRevealed?: boolean; + onToggleSensitiveVisibility?: () => void; }; function stepClass(props: WizardStepControlsProps, name: string): string { @@ -129,7 +132,7 @@ function renderOption( return html`
` : nothing} - - props.presentation !== "channels" && - props.onValueChange((event.currentTarget as HTMLInputElement).value)} - /> - ${renderAnswerButton(props, t("modelSetup.wizard.submit"))} + ${input} ${renderAnswerButton(props, t("modelSetup.wizard.submit"))} `; } diff --git a/ui/src/e2e/custodian-event-nudge.e2e.test.ts b/ui/src/e2e/custodian-event-nudge.e2e.test.ts index 534aa0a22422..e79c94407fd3 100644 --- a/ui/src/e2e/custodian-event-nudge.e2e.test.ts +++ b/ui/src/e2e/custodian-event-nudge.e2e.test.ts @@ -303,6 +303,127 @@ describeControlUiE2e("Control UI custodian event nudge mocked Gateway E2E", () = } }); + it("renders rich wizard controls and sends typed answers", async () => { + const context = await browser.newContext({ + colorScheme: "dark", + locale: "en-US", + serviceWorkers: "block", + viewport: { height: 900, width: 1280 }, + }); + const page = await context.newPage(); + const gateway = await installMockGateway(page, { + featureMethods: ["chat.metadata", "chat.startup", "openclaw.chat"], + methodResponses: { + "openclaw.chat": { + sessionId: "e2e-rich-wizard", + reply: "Choose a channel.", + action: "none", + wizardInputPending: true, + step: { + id: "channel", + type: "select", + message: "Which channel?", + options: ["Discord", "Slack", "Telegram", "WhatsApp", "Twitch"].map((label) => ({ + label, + value: label.toLowerCase(), + })), + }, + }, + }, + }); + + try { + await page.goto(`${server.baseUrl}custodian`); + await page.getByLabel("Twitch").waitFor(); + expect(await page.locator("openclaw-option-card").count()).toBe(0); + expect(await page.locator(".agent-chat__composer-shell").count()).toBe(0); + + await gateway.setMethodResponse("openclaw.chat", { + sessionId: "e2e-rich-wizard", + reply: "Choose features.", + action: "none", + wizardInputPending: true, + step: { + id: "features", + type: "multiselect", + message: "Which features?", + options: [ + { label: "Chat", value: "chat" }, + { label: "Moderation", value: "moderation" }, + { label: "Announcements", value: "announcements" }, + ], + }, + }); + await page.getByLabel("Twitch").check(); + await page.getByRole("button", { name: "Continue" }).click(); + await page.getByLabel("Announcements").waitFor(); + + await gateway.setMethodResponse("openclaw.chat", { + sessionId: "e2e-rich-wizard", + reply: "Enter the secret.", + action: "none", + sensitive: true, + wizardInputPending: true, + step: { + id: "secret", + type: "text", + message: "Twitch client secret", + sensitive: true, + }, + }); + await page.getByLabel("Chat").check(); + await page.getByLabel("Announcements").check(); + await page.getByRole("button", { name: "Continue" }).click(); + const secretInput = page.getByRole("textbox", { + name: "Twitch client secret", + }); + await secretInput.waitFor(); + expect(await secretInput.getAttribute("type")).toBe("password"); + await page.getByRole("button", { name: "Reveal value" }).click(); + expect(await secretInput.getAttribute("type")).toBe("text"); + await page.getByRole("button", { name: "Hide value" }).click(); + expect(await secretInput.getAttribute("type")).toBe("password"); + + await gateway.setMethodResponse("openclaw.chat", { + sessionId: "e2e-rich-wizard", + reply: "Setup complete.", + action: "none", + }); + await secretInput.fill("fake-client-secret"); + await page.getByRole("button", { name: "Submit" }).click(); + await page.getByText("Setup complete.").waitFor(); + + const requests = await gateway.getRequests("openclaw.chat"); + expect(requests.map((request) => request.params)).toEqual([ + expect.objectContaining({ sessionId: expect.any(String) }), + expect.objectContaining({ + wizardAnswer: { stepId: "channel", value: "twitch" }, + }), + expect.objectContaining({ + wizardAnswer: { stepId: "features", value: ["chat", "announcements"] }, + }), + expect.objectContaining({ + wizardAnswer: { stepId: "secret", value: "fake-client-secret" }, + }), + ]); + expect( + requests + .slice(1) + .every( + (request) => + typeof request.params === "object" && + request.params !== null && + !Object.hasOwn(request.params, "message"), + ), + ).toBe(true); + expect(await page.getByText("Sensitive reply sent").count()).toBe(1); + expect(await page.getByText("fake-client-secret").count()).toBe(0); + expect(await page.locator(".agent-chat__composer-shell").count()).toBe(1); + } finally { + await context.close(); + } + }); + it("stays silent during onboarding", async () => { const context = await browser.newContext({ locale: "en-US", diff --git a/ui/src/pages/channels/channels-page.ts b/ui/src/pages/channels/channels-page.ts index 5134f98f7ec1..ce5834ab2a13 100644 --- a/ui/src/pages/channels/channels-page.ts +++ b/ui/src/pages/channels/channels-page.ts @@ -675,6 +675,8 @@ class ChannelsPage extends OpenClawLightDomElement { selectedChannel: this.selectedChannel, wizard: this.wizardHost.state, wizardMultiselect: this.wizardHost.multiselect, + wizardTextValue: this.wizardHost.textValue, + wizardSecretVisible: this.wizardHost.secretVisible, setupBlockedByDirtyConfig: this.wizardHost.blockedByDirtyConfig, onShowDetail: (channelId) => { this.selectedChannel = channelId; @@ -685,6 +687,8 @@ class ChannelsPage extends OpenClawLightDomElement { onStartSetup: (channelId) => this.wizardHost.startSetup(channelId), onWizardAnswer: (value) => this.wizardHost.answer(value), onWizardToggleMultiselect: (value) => this.wizardHost.toggleMultiselect(value), + onWizardTextInput: (value) => this.wizardHost.setTextValue(value), + onWizardToggleSecretVisibility: () => this.wizardHost.toggleSecretVisibility(), onWizardClose: () => this.wizardHost.close(), onRefresh: (probe) => void context.channels.refresh(probe), onPairingRefresh: () => void context.channels.refreshPairing(), diff --git a/ui/src/pages/channels/view.pairing.test.ts b/ui/src/pages/channels/view.pairing.test.ts index 37060621f553..775d2b94b58a 100644 --- a/ui/src/pages/channels/view.pairing.test.ts +++ b/ui/src/pages/channels/view.pairing.test.ts @@ -70,12 +70,16 @@ function createProps(overrides: Partial = {}): ChannelsProps { selectedChannel: null, wizard: { phase: "idle" }, wizardMultiselect: [], + wizardTextValue: "", + wizardSecretVisible: false, setupBlockedByDirtyConfig: false, onShowDetail: () => undefined, onCloseDetail: () => undefined, onStartSetup: () => undefined, onWizardAnswer: () => undefined, onWizardToggleMultiselect: () => undefined, + onWizardTextInput: () => undefined, + onWizardToggleSecretVisibility: () => undefined, onWizardClose: () => undefined, onRefresh: () => undefined, onPairingRefresh: () => undefined, diff --git a/ui/src/pages/channels/view.test.ts b/ui/src/pages/channels/view.test.ts index 5285dfea8a94..ae436334787f 100644 --- a/ui/src/pages/channels/view.test.ts +++ b/ui/src/pages/channels/view.test.ts @@ -51,12 +51,16 @@ function createProps(snapshot: ChannelsProps["snapshot"]): ChannelsProps { selectedChannel: null, wizard: { phase: "idle" }, wizardMultiselect: [], + wizardTextValue: "", + wizardSecretVisible: false, setupBlockedByDirtyConfig: false, onShowDetail: () => {}, onCloseDetail: () => {}, onStartSetup: () => {}, onWizardAnswer: () => {}, onWizardToggleMultiselect: () => {}, + onWizardTextInput: () => {}, + onWizardToggleSecretVisibility: () => {}, onWizardClose: () => {}, onRefresh: () => {}, onPairingRefresh: () => {}, diff --git a/ui/src/pages/channels/view.ts b/ui/src/pages/channels/view.ts index 0e9caaf5c992..73ad0c801d25 100644 --- a/ui/src/pages/channels/view.ts +++ b/ui/src/pages/channels/view.ts @@ -117,6 +117,10 @@ export function renderChannels(props: ChannelsProps) { channelLabel: (channelId) => resolveChannelLabel(props.snapshot, channelId), multiselectValues: props.wizardMultiselect, onToggleMultiselect: props.onWizardToggleMultiselect, + textValue: props.wizardTextValue, + secretVisible: props.wizardSecretVisible, + onTextInput: props.onWizardTextInput, + onToggleSecretVisibility: props.onWizardToggleSecretVisibility, onAnswer: props.onWizardAnswer, onClose: props.onWizardClose, whatsappQrDataUrl: props.whatsappQrDataUrl, diff --git a/ui/src/pages/channels/view.types.ts b/ui/src/pages/channels/view.types.ts index 41b815b4ca28..2616ab525512 100644 --- a/ui/src/pages/channels/view.types.ts +++ b/ui/src/pages/channels/view.types.ts @@ -60,12 +60,16 @@ export type ChannelsProps = { selectedChannel: string | null; wizard: ChannelWizardState; wizardMultiselect: readonly unknown[]; + wizardTextValue: string; + wizardSecretVisible: boolean; setupBlockedByDirtyConfig: boolean; onShowDetail: (channelId: string) => void; onCloseDetail: () => void; onStartSetup: (channelId: string | null) => void; onWizardAnswer: (value: unknown) => void; onWizardToggleMultiselect: (value: unknown) => void; + onWizardTextInput: (value: string) => void; + onWizardToggleSecretVisibility: () => void; onWizardClose: () => void; onRefresh: (probe: boolean) => void; onPairingRefresh: () => void; diff --git a/ui/src/pages/channels/wizard-host.ts b/ui/src/pages/channels/wizard-host.ts index 0e095aaf3c3b..44aaeeba05d7 100644 --- a/ui/src/pages/channels/wizard-host.ts +++ b/ui/src/pages/channels/wizard-host.ts @@ -1,5 +1,5 @@ // Page-side host for the channel setup wizard: owns the RPC controller, -// per-step multiselect state, dirty-config guarding, and completion effects +// per-step form state, dirty-config guarding, and completion effects // (config resync + WhatsApp QR handoff) so the page element stays thin. import type { ApplicationContext } from "../../app/context.ts"; import { t } from "../../i18n/index.ts"; @@ -14,8 +14,11 @@ type WizardHostDeps = { export class ChannelWizardHost { multiselect: unknown[] = []; + textValue = ""; + secretVisible = false; blockedByDirtyConfig = false; private multiselectStepId: string | null = null; + private textStepId: string | null = null; private lastPhase = "idle"; private readonly controller: ChannelWizardController; @@ -78,8 +81,17 @@ export class ChannelWizardHost { this.deps.requestUpdate(); } + setTextValue(value: string): void { + this.textValue = value; + } + + toggleSecretVisibility(): void { + this.secretVisible = !this.secretVisible; + this.deps.requestUpdate(); + } + private handleControllerChange(): void { - // Pending multiselect toggles survive busy re-renders but reset per step. + // Pending input state survives unrelated page re-renders but resets per step. const wizard = this.controller.state; const stepId = wizard.phase === "step" ? wizard.step.id : null; if (stepId !== this.multiselectStepId) { @@ -89,6 +101,16 @@ export class ChannelWizardHost { ? [...wizard.step.initialValue] : []; } + if (stepId !== this.textStepId) { + this.textStepId = stepId; + this.textValue = + wizard.phase === "step" && + wizard.step.type === "text" && + typeof wizard.step.initialValue === "string" + ? wizard.step.initialValue + : ""; + this.secretVisible = false; + } if (wizard.phase === "done" && this.lastPhase !== "done") { void this.handleCompleted(wizard.accounts); } diff --git a/ui/src/pages/channels/wizard-view.busy.test.ts b/ui/src/pages/channels/wizard-view.busy.test.ts index d9bf4cb34693..ef77296a2e20 100644 --- a/ui/src/pages/channels/wizard-view.busy.test.ts +++ b/ui/src/pages/channels/wizard-view.busy.test.ts @@ -6,7 +6,11 @@ import { i18n } from "../../i18n/index.ts"; import type { ChannelWizardStep } from "./wizard-controller.ts"; import { renderChannelWizard } from "./wizard-view.ts"; -function renderStep(step: ChannelWizardStep, busy = true) { +function renderStep( + step: ChannelWizardStep, + busy = true, + textValue = typeof step.initialValue === "string" ? step.initialValue : "", +) { const container = document.createElement("div"); const onAnswer = vi.fn(); const onClose = vi.fn(); @@ -25,6 +29,10 @@ function renderStep(step: ChannelWizardStep, busy = true) { channelLabel: (channelId) => channelId, multiselectValues: ["alpha"], onToggleMultiselect, + textValue, + secretVisible: false, + onTextInput: vi.fn(), + onToggleSecretVisibility: vi.fn(), onAnswer, onClose, whatsappQrDataUrl: null, @@ -136,16 +144,23 @@ describe("renderChannelWizard busy controls", () => { }); it("disables text editing and submission while a step is running", () => { - const text = renderStep({ - id: "text", - type: "text", - message: "Enter a value", - initialValue: "original", - }); + const text = renderStep( + { + id: "text", + type: "text", + message: "Enter a value", + sensitive: true, + }, + true, + "replacement", + ); const input = text.container.querySelector('input[name="wizard-text"]'); const submit = text.container.querySelector('button[type="submit"]'); + const toggle = text.container.querySelector(".oc-sensitive-toggle"); expect(input?.disabled).toBe(true); - expect(input?.value).toBe("original"); + expect(input?.type).toBe("password"); + expect(input?.value).toBe("replacement"); + expect(toggle?.disabled).toBe(true); expect(submit?.disabled).toBe(true); submit?.click(); expect(text.onAnswer).not.toHaveBeenCalled(); diff --git a/ui/src/pages/channels/wizard-view.test.ts b/ui/src/pages/channels/wizard-view.test.ts index ce0083e7670d..53d9b2c15acd 100644 --- a/ui/src/pages/channels/wizard-view.test.ts +++ b/ui/src/pages/channels/wizard-view.test.ts @@ -45,6 +45,10 @@ describe("renderChannelWizard", () => { channelLabel: (channelId) => channelId, multiselectValues: [], onToggleMultiselect: vi.fn(), + textValue: "", + secretVisible: false, + onTextInput: vi.fn(), + onToggleSecretVisibility: vi.fn(), onAnswer: vi.fn(), onClose: vi.fn(), whatsappQrDataUrl: null, @@ -64,9 +68,79 @@ describe("renderChannelWizard", () => { expect(label?.textContent).toBe("New Matrix account id"); expect(input?.type).toBe(expectedType); expect(input?.labels).toContain(label); + if (sensitive) { + expect(container.querySelector(".oc-sensitive-toggle")).not.toBeNull(); + } else { + expect(container.querySelector(".oc-sensitive-toggle")).toBeNull(); + } }, ); + it("reveals only the replacement value entered in a sensitive step", () => { + const container = document.createElement("div"); + const onTextInput = vi.fn(); + const onToggleSecretVisibility = vi.fn(); + document.body.append(container); + const renderSensitiveStep = (secretVisible: boolean, textValue: string) => + render( + renderChannelWizard({ + wizard: { + phase: "step", + channel: "twitch", + step: { + id: "client-secret", + type: "text", + message: "Twitch Client Secret", + sensitive: true, + }, + stepIndex: 1, + busy: false, + validationError: null, + }, + channelLabel: (channelId) => channelId, + multiselectValues: [], + onToggleMultiselect: vi.fn(), + textValue, + secretVisible, + onTextInput, + onToggleSecretVisibility, + onAnswer: vi.fn(), + onClose: vi.fn(), + whatsappQrDataUrl: null, + whatsappMessage: null, + whatsappConnected: null, + whatsappBusy: false, + onWhatsAppStart: vi.fn(), + onWhatsAppWait: vi.fn(), + }), + container, + ); + + renderSensitiveStep(false, ""); + const hiddenInput = container.querySelector("#channel-wizard-text-input"); + const toggle = container.querySelector(".oc-sensitive-toggle"); + expect(hiddenInput?.type).toBe("password"); + expect(hiddenInput?.value).toBe(""); + expect(toggle?.getAttribute("aria-label")).toBe("Reveal value"); + expect(toggle?.dataset.sensitiveIcon).toBe("eye"); + if (hiddenInput) { + hiddenInput.value = "new-secret"; + hiddenInput.dispatchEvent(new Event("input", { bubbles: true })); + } + toggle?.click(); + expect(onTextInput).toHaveBeenCalledWith("new-secret"); + expect(onToggleSecretVisibility).toHaveBeenCalledOnce(); + + renderSensitiveStep(true, "new-secret"); + const revealedInput = container.querySelector("#channel-wizard-text-input"); + const hideToggle = container.querySelector(".oc-sensitive-toggle"); + expect(revealedInput?.type).toBe("text"); + expect(revealedInput?.value).toBe("new-secret"); + expect(hideToggle?.getAttribute("aria-label")).toBe("Hide value"); + expect(hideToggle?.getAttribute("aria-pressed")).toBe("true"); + expect(hideToggle?.dataset.sensitiveIcon).toBe("eye-off"); + }); + it("copies setup text through the plain-HTTP clipboard fallback", async () => { vi.stubGlobal("navigator", {}); let copiedText: string | undefined; @@ -94,6 +168,10 @@ describe("renderChannelWizard", () => { channelLabel: (channelId) => channelId, multiselectValues: [], onToggleMultiselect: vi.fn(), + textValue: "", + secretVisible: false, + onTextInput: vi.fn(), + onToggleSecretVisibility: vi.fn(), onAnswer: vi.fn(), onClose: vi.fn(), whatsappQrDataUrl: null, diff --git a/ui/src/pages/channels/wizard-view.ts b/ui/src/pages/channels/wizard-view.ts index a5867ec19f3d..0b5307ec302e 100644 --- a/ui/src/pages/channels/wizard-view.ts +++ b/ui/src/pages/channels/wizard-view.ts @@ -16,6 +16,10 @@ type ChannelWizardViewProps = { // Pending multiselect toggles live in page state so re-renders keep them. multiselectValues: readonly unknown[]; onToggleMultiselect: (value: unknown) => void; + textValue: string; + secretVisible: boolean; + onTextInput: (value: string) => void; + onToggleSecretVisibility: () => void; onAnswer: (value: unknown) => void; onClose: () => void; // WhatsApp QR linking phase (wizard done + channel === whatsapp). @@ -84,13 +88,23 @@ function renderStepBody(step: ChannelWizardStep, props: ChannelWizardViewProps) } return renderWizardStepControls({ step, - value: step.type === "multiselect" ? props.multiselectValues : step.initialValue, + value: + step.type === "multiselect" + ? props.multiselectValues + : step.type === "text" + ? props.textValue + : step.initialValue, busy: stepIsBusy(props), inputId: "channel-wizard-text-input", presentation: "channels", answerLabel: t("channels.setup.continue"), - onValueChange: props.onToggleMultiselect, + sensitiveRevealed: props.secretVisible, + onValueChange: + step.type === "text" + ? (value) => props.onTextInput(typeof value === "string" ? value : "") + : props.onToggleMultiselect, onAnswer: props.onAnswer, + onToggleSensitiveVisibility: props.onToggleSecretVisibility, }); } diff --git a/ui/src/pages/custodian/custodian-page.session-lifecycle.test.ts b/ui/src/pages/custodian/custodian-page.session-lifecycle.test.ts index 1cc8bac23816..37f45698332c 100644 --- a/ui/src/pages/custodian/custodian-page.session-lifecycle.test.ts +++ b/ui/src/pages/custodian/custodian-page.session-lifecycle.test.ts @@ -59,6 +59,60 @@ describe("custodian page session lifecycle", () => { expect(page.textContent).toContain("started a fresh session"); }); + it("starts fresh after the gateway evicts a typed wizard session", async () => { + const request = vi + .fn() + .mockResolvedValueOnce({ + sessionId: "evicted-wizard-session", + reply: "Choose a channel.", + action: "none", + wizardInputPending: true, + step: { + id: "channel", + type: "select", + message: "Which channel?", + options: [ + { label: "Slack", value: "slack" }, + { label: "Twitch", value: "twitch" }, + ], + }, + }) + .mockRejectedValueOnce( + new GatewayProtocolRequestError({ + code: "INVALID_REQUEST", + message: "No active OpenClaw chat session is awaiting that wizard answer.", + details: buildSystemAgentSessionInvalidatedErrorDetails(), + }), + ) + .mockResolvedValueOnce({ + sessionId: "replacement-session", + reply: "Fresh session ready.", + action: "none", + }); + const { context } = createContext(request); + const { page } = await mountPage(context); + await waitForFast(() => + expect(page.querySelectorAll('.custodian__wizard-step input[type="radio"]')).toHaveLength(2), + ); + + page + .querySelectorAll('.custodian__wizard-step input[type="radio"]')[1]! + .click(); + await page.updateComplete; + page.querySelector(".custodian__wizard-step .btn.primary")!.click(); + + await waitForFast(() => expect(request).toHaveBeenCalledTimes(3)); + expect(request.mock.calls[1]?.[1]).toMatchObject({ + sessionId: "evicted-wizard-session", + wizardAnswer: { stepId: "channel", value: "twitch" }, + }); + expect(request.mock.calls[2]?.[1]).not.toHaveProperty("message"); + expect(request.mock.calls[2]?.[1]).not.toHaveProperty("wizardAnswer"); + expect(request.mock.calls[2]?.[1]?.sessionId).not.toBe("evicted-wizard-session"); + await waitForFast(() => expect(page.textContent).toContain("Fresh session ready.")); + expect(page.querySelector(".custodian__wizard-step")).toBeNull(); + }); + it("keeps the live session after an error that does not invalidate it", async () => { const request = vi .fn() diff --git a/ui/src/pages/custodian/custodian-page.test.ts b/ui/src/pages/custodian/custodian-page.test.ts index 69cc89071dd9..10cab0bef496 100644 --- a/ui/src/pages/custodian/custodian-page.test.ts +++ b/ui/src/pages/custodian/custodian-page.test.ts @@ -70,7 +70,7 @@ describe("custodian page", () => { await page.updateComplete; expect(request.mock.calls[0]?.[0]).toBe("openclaw.chat"); expect(request.mock.calls[0]?.[1]).toMatchObject({ welcomeVariant: "onboarding" }); - // The engine receives the parseable reply text; the transcript shows the label. + // LLM-authored option cards remain chat messages; wizard controls use wizardAnswer below. expect(request.mock.calls[1]?.[1]).toMatchObject({ welcomeVariant: "onboarding", message: "connect whatsapp", @@ -80,6 +80,128 @@ describe("custodian page", () => { expect(connectOption.disabled).toBe(true); }); + it("renders and answers rich select, multiselect, and sensitive text wizard steps", async () => { + const request = vi + .fn() + .mockResolvedValueOnce({ + sessionId: "rich-wizard-session", + reply: "Choose a channel.", + action: "none", + wizardInputPending: true, + step: { + id: "channel", + type: "select", + message: "Which channel?", + options: ["Discord", "Slack", "Telegram", "WhatsApp", "Twitch"].map((label) => ({ + label, + value: label.toLowerCase(), + })), + }, + }) + .mockResolvedValueOnce({ + sessionId: "rich-wizard-session", + reply: "Choose features.", + action: "none", + wizardInputPending: true, + step: { + id: "features", + type: "multiselect", + message: "Which features?", + options: [ + { label: "Chat", value: "chat" }, + { label: "Moderation", value: "moderation" }, + { label: "Announcements", value: "announcements" }, + ], + }, + }) + .mockResolvedValueOnce({ + sessionId: "rich-wizard-session", + reply: "Enter the secret.", + action: "none", + sensitive: true, + wizardInputPending: true, + step: { + id: "secret", + type: "text", + message: "Twitch client secret", + sensitive: true, + }, + }) + .mockResolvedValueOnce({ + sessionId: "rich-wizard-session", + reply: "Setup complete.", + action: "none", + }); + const { context } = createContext(request); + const { page } = await mountPage(context); + + await waitForFast(() => + expect(page.querySelectorAll('.custodian__wizard-step input[type="radio"]')).toHaveLength(5), + ); + expect(page.querySelector("openclaw-option-card")).toBeNull(); + expect(page.querySelector(".agent-chat__composer-shell")).toBeNull(); + page + .querySelectorAll('.custodian__wizard-step input[type="radio"]')[4]! + .click(); + await page.updateComplete; + page.querySelector(".custodian__wizard-step .btn.primary")!.click(); + + await waitForFast(() => expect(request).toHaveBeenCalledTimes(2)); + await waitForFast(() => + expect(page.querySelectorAll('.custodian__wizard-step input[type="checkbox"]')).toHaveLength( + 3, + ), + ); + expect(request.mock.calls[1]?.[1]).toMatchObject({ + wizardAnswer: { stepId: "channel", value: "twitch" }, + }); + expect(request.mock.calls[1]?.[1]).not.toHaveProperty("message"); + page + .querySelectorAll('.custodian__wizard-step input[type="checkbox"]')[0]! + .click(); + await page.updateComplete; + page + .querySelectorAll('.custodian__wizard-step input[type="checkbox"]')[2]! + .click(); + await page.updateComplete; + page.querySelector(".custodian__wizard-step .btn.primary")!.click(); + + await waitForFast(() => expect(request).toHaveBeenCalledTimes(3)); + const secretInput = await waitForFast(() => { + const input = page.querySelector("#custodian-wizard-input-5"); + expect(input).not.toBeNull(); + return input!; + }); + expect(request.mock.calls[2]?.[1]).toMatchObject({ + wizardAnswer: { stepId: "features", value: ["chat", "announcements"] }, + }); + expect(secretInput.type).toBe("password"); + const revealSecret = page.querySelector( + '.custodian__wizard-step button[aria-label="Reveal value"]', + ); + expect(revealSecret).not.toBeNull(); + revealSecret!.click(); + await page.updateComplete; + const revealedInput = page.querySelector("#custodian-wizard-input-5")!; + expect(revealedInput.type).toBe("text"); + revealedInput.value = "fake-client-secret"; + revealedInput.dispatchEvent(new Event("input", { bubbles: true })); + await page.updateComplete; + page.querySelector(".custodian__wizard-step .btn.primary")!.click(); + + await waitForFast(() => expect(request).toHaveBeenCalledTimes(4)); + await waitForFast(() => expect(page.textContent).toContain("Setup complete.")); + expect(request.mock.calls[3]?.[1]).toMatchObject({ + wizardAnswer: { stepId: "secret", value: "fake-client-secret" }, + }); + expect(request.mock.calls[3]?.[1]).not.toHaveProperty("message"); + expect(page.textContent).toContain("Twitch"); + expect(page.textContent).toContain("Chat, Announcements"); + expect(page.textContent).toContain("Sensitive reply sent"); + expect(page.textContent).not.toContain("fake-client-secret"); + expect(page.querySelector(".agent-chat__composer-shell")).not.toBeNull(); + }); + it("collapses an empty transcript around a blocking startup error", async () => { const request = vi .fn() diff --git a/ui/src/pages/custodian/custodian-session-store.ts b/ui/src/pages/custodian/custodian-session-store.ts index 6fecad285497..900f47de43eb 100644 --- a/ui/src/pages/custodian/custodian-session-store.ts +++ b/ui/src/pages/custodian/custodian-session-store.ts @@ -4,12 +4,14 @@ import { type SystemAgentChatResult, } from "@openclaw/gateway-protocol"; import type { GatewayBrowserClient } from "../../api/gateway.ts"; +import type { WizardStep } from "../../api/types.ts"; import { selectApplicationSession } from "../../app/agent-selection.ts"; import type { ApplicationContext } from "../../app/context.ts"; import { t } from "../../i18n/index.ts"; import { isGatewayMethodAdvertised } from "../../lib/gateway-methods.ts"; import { buildAgentMainSessionKey, normalizeAgentId } from "../../lib/sessions/session-key.ts"; import { pathForCustodianAgentHandoff } from "./custodian-navigation.ts"; +import { custodianWizardSubmission, initialCustodianWizardValue } from "./custodian-wizard-step.ts"; import * as eventNudgeState from "./event-nudge.ts"; import { custodianChatParams, @@ -30,6 +32,10 @@ import { const SYSTEM_AGENT_CHAT_TIMEOUT_MS = 190_000; const SILENT_REPLY_PATTERN = /^\s*NO_REPLY\s*$/; +function hasCustodianUserInput(params: SystemAgentChatParams): boolean { + return params.message !== undefined || params.wizardAnswer !== undefined; +} + type StoreListener = () => void; type ConfiguredInferenceState = "unresolved" | "required" | "ready"; type CustodianSetupIssue = "missing" | "unavailable"; @@ -41,6 +47,8 @@ export class CustodianSessionStore { sending = false; sensitive = false; wizardInputPending = false; + wizardValue: unknown; + wizardSecretVisible = false; questionReplyUncertain = false; error: string | null = null; setupIssue: CustodianSetupIssue | null = null; @@ -118,6 +126,16 @@ export class CustodianSessionStore { this.emit(); } + setWizardValue(value: unknown): void { + this.wizardValue = value; + this.emit(); + } + + toggleWizardSecretVisibility(): void { + this.wizardSecretVisible = !this.wizardSecretVisible; + this.emit(); + } + hasRealUserTurn(): boolean { return this.messages.some((message) => message.role === "user"); } @@ -137,7 +155,7 @@ export class CustodianSessionStore { } canRetry(): boolean { - return this.retryParams !== null && this.retryParams.message === undefined; + return this.retryParams !== null && !hasCustodianUserInput(this.retryParams); } get setupRequired(): boolean { @@ -147,7 +165,7 @@ export class CustodianSessionStore { retry(): void { const client = this.activeClient; const params = this.retryParams; - if (client && params && params.message === undefined && this.chatAvailable && !this.sending) { + if (client && params && !hasCustodianUserInput(params) && this.chatAvailable && !this.sending) { void this.initializeSession(client, params); } } @@ -160,15 +178,32 @@ export class CustodianSessionStore { // Trim decides emptiness only; sensitive values may carry meaningful whitespace. const message = this.sensitive ? text : text.trim(); const client = this.activeClient; - const questionState = [this.answeredQuestions, this.questionReplyUncertain] as const; - if (questionReply) { - this.questionReplyUncertain = true; - } if (!message.trim() || !client || !this.chatAvailable || this.sending || this.setupRequired) { this.emit(); return "rejected"; } const displayText = this.sensitive ? t("custodian.sensitiveReply") : (display ?? message); + return await this.sendUserTurn( + client, + { + sessionId: this.sessionId, + ...custodianChatParams(this.variant, message), + }, + displayText, + questionReply, + ); + } + + private async sendUserTurn( + client: GatewayBrowserClient, + params: SystemAgentChatParams, + displayText: string, + questionReply: boolean, + ): Promise { + const questionState = [this.answeredQuestions, this.questionReplyUncertain] as const; + if (questionReply) { + this.questionReplyUncertain = true; + } this.abandonedTurnOutcomeUnknown = false; this.answeredQuestions = retireCustodianQuestions(this.messages, this.answeredQuestions); this.messages = [ @@ -179,14 +214,12 @@ export class CustodianSessionStore { text: displayText, at: Date.now(), question: null, + step: null, }, ]; this.input = ""; this.emit(); - const reply = this.requestReply(client, { - sessionId: this.sessionId, - ...custodianChatParams(this.variant, message), - }); + const reply = this.requestReply(client, params); const replyEpoch = this.requestEpoch; const outcome = await reply; if (questionReply && this.requestEpoch === replyEpoch) { @@ -263,6 +296,25 @@ export class CustodianSessionStore { void this.send(option?.reply ?? label, label, true); } + answerWizardStep(message: CustodianMessage, value: unknown): void { + if (!message.step || !this.wizardInputPending) { + return; + } + const submission = custodianWizardSubmission(message.step, value); + const client = this.activeClient; + if (!submission || !client || !this.chatAvailable || this.sending || this.setupRequired) { + this.emit(); + return; + } + const displayText = message.step.sensitive ? t("custodian.sensitiveReply") : submission.display; + void this.sendUserTurn( + client, + { sessionId: this.sessionId, wizardAnswer: submission.answer }, + displayText, + true, + ); + } + exitSetup(): void { this.context?.navigate("chat"); } @@ -323,6 +375,8 @@ export class CustodianSessionStore { this.answeredQuestions = retireCustodianQuestions(this.messages, this.answeredQuestions); this.retryParams = null; this.input = ""; + this.wizardValue = undefined; + this.wizardSecretVisible = false; this.sensitive = this.wizardInputPending = this.questionReplyUncertain = false; this.error = null; this.setupIssue = null; @@ -495,11 +549,17 @@ export class CustodianSessionStore { this.error = null; this.setupIssue = null; this.input = ""; + this.wizardValue = undefined; + this.wizardSecretVisible = false; this.sensitive = this.wizardInputPending = this.questionReplyUncertain = false; this.earlierBoundaryAfterId = null; } - private appendAssistant(reply: string, question: CustodianStructuredQuestion | null): void { + private appendAssistant( + reply: string, + question: CustodianStructuredQuestion | null, + step: WizardStep | null, + ): void { this.messages = [ ...this.messages, { @@ -508,6 +568,7 @@ export class CustodianSessionStore { text: reply, at: Date.now(), question, + step, }, ]; } @@ -524,7 +585,7 @@ export class CustodianSessionStore { let delivery: eventNudgeState.CustodianSendDelivery = "unsent"; this.sending = true; this.error = null; - if (params.message !== undefined) { + if (hasCustodianUserInput(params)) { this.setupIssue = null; } this.retryParams = params; @@ -543,10 +604,13 @@ export class CustodianSessionStore { this.wizardInputPending = result.wizardInputPending === true; this.retryParams = null; this.setupIssue = null; - const question = parseCustodianQuestion(result.question); + const step = result.step ?? null; + const question = step ? null : parseCustodianQuestion(result.question); + this.wizardValue = step ? initialCustodianWizardValue(step) : undefined; + this.wizardSecretVisible = false; const silentReply = SILENT_REPLY_PATTERN.test(result.reply); - if (!silentReply || question) { - this.appendAssistant(silentReply ? "" : result.reply, question); + if (!silentReply || question || step) { + this.appendAssistant(silentReply ? "" : result.reply, question, step); } if (result.action === "open-agent") { let sessionKey = context.gateway.snapshot.sessionKey?.trim(); @@ -589,13 +653,13 @@ export class CustodianSessionStore { ? "missing" : "unavailable" : null; - if (params.message !== undefined && isCustodianSessionInvalidatedError(error)) { + if (hasCustodianUserInput(params) && isCustodianSessionInvalidatedError(error)) { // Retained transcript rows are display context only; the next turn needs a fresh id. this.rotateVolatileSession(client, this.currentSessionVariant()); this.error = t("custodian.sessionRestarted", { error: custodianErrorMessage(error) }); } } - if (params.message !== undefined && this.retryParams === params) { + if (hasCustodianUserInput(params) && this.retryParams === params) { // User turns have no idempotency key and are never replayed after an ambiguous failure. this.retryParams = null; } diff --git a/ui/src/pages/custodian/custodian-surface.ts b/ui/src/pages/custodian/custodian-surface.ts index 57fc18159043..7e421790d1bf 100644 --- a/ui/src/pages/custodian/custodian-surface.ts +++ b/ui/src/pages/custodian/custodian-surface.ts @@ -133,6 +133,9 @@ class CustodianSurface extends OpenClawLightDomElement { `; } const emptyError = store.messages.length === 0 && store.error !== null && !store.sending; + const activeWizardMessage = store.wizardInputPending + ? store.messages.findLast((message) => message.step !== null) + : undefined; return html`
store.answerQuestion(message, label), onSkip: () => void store.dismissQuestion(message), + showWizardStep: message === activeWizardMessage, + wizardValue: store.wizardValue, + wizardDisabled: store.sending || !store.chatAvailable, + wizardSecretVisible: store.wizardSecretVisible, + onWizardValueChange: (value) => store.setWizardValue(value), + onWizardAnswer: (value) => store.answerWizardStep(message, value), + onToggleWizardSecretVisibility: () => store.toggleWizardSecretVisibility(), }); })} ${store.sending @@ -204,60 +214,61 @@ class CustodianSurface extends OpenClawLightDomElement { ${this.historyContent} - -
-
-
-
- ${store.sensitive - ? html` +
+
+
+ ${store.sensitive + ? html` + store.setInput((event.target as HTMLInputElement).value)} + @keydown=${(event: KeyboardEvent) => this.handleComposerKeydown(event)} + />` + : html``} +
+
+ +
+
-
- -
-
-
-
+
`}
`; } diff --git a/ui/src/pages/custodian/custodian-wizard-step.test.ts b/ui/src/pages/custodian/custodian-wizard-step.test.ts new file mode 100644 index 000000000000..015860a78836 --- /dev/null +++ b/ui/src/pages/custodian/custodian-wizard-step.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import type { WizardStep } from "../../api/types.ts"; +import { custodianWizardSubmission, initialCustodianWizardValue } from "./custodian-wizard-step.ts"; + +const options = [ + { label: "Discord", value: "discord" }, + { label: "Slack", value: "slack" }, + { label: "Twitch", value: "twitch" }, +]; + +function step(patch: Partial): WizardStep { + return { id: "step", type: "select", options, ...patch }; +} + +describe("Custodian rich wizard answers", () => { + it("preserves typed select and multiselect values", () => { + expect(custodianWizardSubmission(step({}), "twitch")).toEqual({ + answer: { stepId: "step", value: "twitch" }, + display: "Twitch", + }); + expect(custodianWizardSubmission(step({ type: "multiselect" }), ["discord", "twitch"])).toEqual( + { + answer: { stepId: "step", value: ["discord", "twitch"] }, + display: "Discord, Twitch", + }, + ); + expect(custodianWizardSubmission(step({ type: "multiselect" }), [])).toEqual({ + answer: { stepId: "step", value: [] }, + display: "none", + }); + }); + + it("builds confirm, text, and continue submissions", () => { + expect(custodianWizardSubmission(step({ type: "confirm" }), true)).toEqual({ + answer: { stepId: "step", value: true }, + display: "Yes", + }); + expect(custodianWizardSubmission(step({ type: "text" }), "secret")).toEqual({ + answer: { stepId: "step", value: "secret" }, + display: "secret", + }); + expect(custodianWizardSubmission(step({ type: "action" }), undefined)).toEqual({ + answer: { stepId: "step" }, + display: "Continue", + }); + }); + + it("copies multiselect defaults and rejects values outside the step", () => { + const initialValue = ["discord"]; + const value = initialCustodianWizardValue( + step({ type: "multiselect", initialValue }), + ) as unknown[]; + value.push("twitch"); + + expect(initialValue).toEqual(["discord"]); + expect(custodianWizardSubmission(step({}), "unknown")).toBeNull(); + expect(custodianWizardSubmission(step({ type: "text" }), { secret: true })).toBeNull(); + expect(custodianWizardSubmission(step({ type: "multiselect" }), ["unknown"])).toBeNull(); + }); +}); diff --git a/ui/src/pages/custodian/custodian-wizard-step.ts b/ui/src/pages/custodian/custodian-wizard-step.ts new file mode 100644 index 000000000000..0fb1b04f3001 --- /dev/null +++ b/ui/src/pages/custodian/custodian-wizard-step.ts @@ -0,0 +1,62 @@ +import type { WizardAnswer } from "@openclaw/gateway-protocol"; +import type { WizardStep } from "../../api/types.ts"; +import { t } from "../../i18n/index.ts"; + +type CustodianWizardSubmission = { + answer: WizardAnswer; + display: string; +}; + +function findOption(step: WizardStep, value: unknown) { + return step.options?.find((option) => Object.is(option.value, value)); +} + +/** Build the typed answer sent by a client rendering the current wizard step. */ +export function custodianWizardSubmission( + step: WizardStep, + value: unknown, +): CustodianWizardSubmission | null { + if (step.type === "note" || step.type === "action" || step.type === "progress") { + return { answer: { stepId: step.id }, display: t("common.continue") }; + } + if (step.type === "text") { + return typeof value === "string" + ? { answer: { stepId: step.id, value }, display: value } + : null; + } + if (step.type === "confirm") { + if (typeof value !== "boolean") { + return null; + } + return { + answer: { stepId: step.id, value }, + display: t(value ? "common.yes" : "common.no"), + }; + } + if (step.type === "select") { + const option = findOption(step, value); + return option ? { answer: { stepId: step.id, value }, display: option.label } : null; + } + if (!Array.isArray(value)) { + return null; + } + if (value.length === 0) { + return { answer: { stepId: step.id, value: [] }, display: t("common.none") }; + } + const labels = value.map((entry) => findOption(step, entry)?.label); + if (!labels.every((label): label is string => label !== undefined)) { + return null; + } + return { + answer: { stepId: step.id, value }, + display: labels.join(", "), + }; +} + +export function initialCustodianWizardValue(step: WizardStep): unknown { + return step.type === "multiselect" + ? Array.isArray(step.initialValue) + ? [...step.initialValue] + : [] + : step.initialValue; +} diff --git a/ui/src/pages/custodian/transcript.ts b/ui/src/pages/custodian/transcript.ts index c4eeb29962a6..19031725b288 100644 --- a/ui/src/pages/custodian/transcript.ts +++ b/ui/src/pages/custodian/transcript.ts @@ -4,6 +4,8 @@ import type { } from "@openclaw/gateway-protocol"; import { html, nothing } from "lit"; import type { GatewayBrowserClient } from "../../api/gateway.ts"; +import type { WizardStep } from "../../api/types.ts"; +import { renderWizardStepControls } from "../../components/wizard-step-controls.ts"; import { t } from "../../i18n/index.ts"; import type { MessageGroup } from "../../lib/chat/chat-types.ts"; import { renderChatDivider } from "../chat/components/chat-divider.ts"; @@ -19,6 +21,7 @@ export type CustodianMessage = { text: string; at: number; question: CustodianStructuredQuestion | null; + step: WizardStep | null; }; export function hasUnresolvedCustodianQuestion( @@ -121,6 +124,7 @@ export function createCustodianTranscriptMessages( : turn.text, at: turn.at, question: null, + step: null, })); return { messages, nextMessageId }; } @@ -142,10 +146,18 @@ export function renderCustodianTranscriptEntry(params: { assistantAvatar: string; showQuestion: boolean; questionDisabled: boolean; + showWizardStep: boolean; + wizardValue: unknown; + wizardDisabled: boolean; + wizardSecretVisible: boolean; onSelect: (label: string) => void; onSkip: () => void; + onWizardValueChange: (value: unknown) => void; + onWizardAnswer: (value: unknown) => void; + onToggleWizardSecretVisibility: () => void; }) { const question = params.message.question; + const step = params.message.step; return html` ${params.message.text ? renderMessageGroup(toCustodianMessageGroup(params.message), { @@ -164,5 +176,25 @@ export function renderCustodianTranscriptEntry(params: { onSkip: params.onSkip, }) : nothing} + ${params.showWizardStep && step + ? html`
+ ${step.title + ? html`${step.title}` + : nothing} + ${renderWizardStepControls({ + step, + value: params.wizardValue, + busy: params.wizardDisabled, + inputId: `custodian-wizard-input-${params.message.id}`, + sensitiveRevealed: params.wizardSecretVisible, + onValueChange: params.onWizardValueChange, + onAnswer: params.onWizardAnswer, + onToggleSensitiveVisibility: params.onToggleWizardSecretVisibility, + })} +
` + : nothing} `; } diff --git a/ui/src/styles/channels.css b/ui/src/styles/channels.css index e2b795247a2f..48346c7564fe 100644 --- a/ui/src/styles/channels.css +++ b/ui/src/styles/channels.css @@ -294,6 +294,20 @@ line-height: 1.4; } +.channels-wizard__text, +.channels-wizard__secret { + width: 100%; + margin-top: 10px; +} + +.channels-wizard__text { + display: flex; +} + +.channels-wizard__text > .input { + min-height: 44px; +} + .channels-wizard__options { display: grid; gap: var(--space-2); diff --git a/ui/src/styles/components.css b/ui/src/styles/components.css index c4fd870f7f56..75c186718fb8 100644 --- a/ui/src/styles/components.css +++ b/ui/src/styles/components.css @@ -1128,6 +1128,136 @@ openclaw-session-owner-chip { box-shadow: var(--focus-ring); } +/* Carapace Sensitive Input adapted to the Control UI token contract. */ +.oc-sensitive-input { + position: relative; + display: flex; + width: 100%; + min-width: 0; + min-height: 44px; + align-items: stretch; + border: 1px solid var(--input); + border-radius: var(--radius-md); + background: var(--card); + box-shadow: inset 0 1px 0 var(--card-highlight); + transition: + border-color var(--duration-fast) var(--ease-out), + box-shadow var(--duration-fast) var(--ease-out); +} + +.oc-sensitive-input:hover:not(:has(input:disabled)) { + border-color: var(--border-strong); +} + +.oc-sensitive-input:focus-within { + border-color: var(--ring); + box-shadow: var(--focus-ring); +} + +.oc-sensitive-input > input { + width: 100%; + min-width: 0; + padding: 8px 12px; + border: 0; + border-radius: var(--radius-md) 0 0 var(--radius-md); + outline: none; + background: transparent; + color: var(--text); + box-shadow: none; +} + +.oc-sensitive-input > input:disabled { + cursor: not-allowed; + opacity: 0.55; +} + +.oc-sensitive-input > input::placeholder { + color: var(--muted); +} + +.oc-sensitive-mask { + position: absolute; + z-index: 1; + inset: 0 44px 0 0; + display: flex; + overflow: hidden; + align-items: center; + padding: 0 12px; + color: var(--text); + font-family: var(--font-mono); + font-size: 13px; + letter-spacing: 0.08em; + line-height: 1; + pointer-events: none; + white-space: nowrap; +} + +.oc-sensitive-mask[hidden] { + display: none; +} + +.oc-sensitive-mask > span { + display: block; + will-change: transform; +} + +.oc-sensitive-input[data-sensitive-mask-ready="true"][data-revealed="false"] > input { + color: transparent; + caret-color: var(--text); +} + +.oc-sensitive-input[data-sensitive-mask-ready="true"][data-revealed="false"] > input::selection { + color: transparent; +} + +.oc-sensitive-toggle { + position: relative; + z-index: 2; + display: inline-grid; + width: 44px; + min-width: 44px; + height: 100%; + min-height: 44px; + place-items: center; + padding: 0; + border: 0; + border-left: 1px solid var(--input); + border-radius: 0 var(--radius-md) var(--radius-md) 0; + background: transparent; + color: var(--muted); + cursor: pointer; + touch-action: manipulation; +} + +.oc-sensitive-toggle:hover:not(:disabled) { + color: var(--text); +} + +.oc-sensitive-toggle:disabled { + cursor: not-allowed; + opacity: 0.55; +} + +.oc-sensitive-toggle:focus-visible { + outline: 2px solid var(--ring); + outline-offset: -3px; +} + +.oc-sensitive-toggle svg { + width: 16px; + height: 16px; +} + +@media (forced-colors: active) { + .oc-sensitive-mask { + display: none; + } + + .oc-sensitive-input[data-sensitive-mask-ready="true"][data-revealed="false"] > input { + color: CanvasText; + } +} + .field select { appearance: none; padding-right: 36px; diff --git a/ui/src/styles/custodian.css b/ui/src/styles/custodian.css index bb58adc22200..3aba580992cb 100644 --- a/ui/src/styles/custodian.css +++ b/ui/src/styles/custodian.css @@ -256,6 +256,21 @@ openclaw-custodian-page { margin: -12px 16px 14px 46px; } +.custodian__wizard-step { + display: grid; + gap: 12px; + margin: -12px 16px 14px 46px; + padding: 16px; + border: 1px solid var(--border); + border-radius: var(--radius-lg); + background: var(--bg-elevated); +} + +.custodian__wizard-title { + color: var(--text-strong); + font-size: 14px; +} + .custodian__thinking { display: flex; flex-direction: row; From e402606c713f7cf8b9c0f55030923714408a61c0 Mon Sep 17 00:00:00 2001 From: "clawsweeper[bot]" <274271284+clawsweeper[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 05:20:38 -0700 Subject: [PATCH 44/57] fix(line): clear default access token when removing account (#118055) * fix(line): clear default access token when removing account * fix(line): clear default access token when removing account --------- Co-authored-by: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com> --- extensions/line/src/config-adapter.test.ts | 51 ++++++++++++++++++++++ extensions/line/src/config-adapter.ts | 2 +- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 extensions/line/src/config-adapter.test.ts diff --git a/extensions/line/src/config-adapter.test.ts b/extensions/line/src/config-adapter.test.ts new file mode 100644 index 000000000000..b907b13d4293 --- /dev/null +++ b/extensions/line/src/config-adapter.test.ts @@ -0,0 +1,51 @@ +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { listLineAccountIds } from "./accounts.js"; +import { lineConfigAdapter } from "./config-adapter.js"; + +describe("LINE config adapter", () => { + beforeEach(() => { + vi.stubEnv("LINE_CHANNEL_ACCESS_TOKEN", ""); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("clears default account credentials while preserving named accounts", () => { + const cfg = { + channels: { + line: { + channelAccessToken: "default-token", + channelSecret: "default-secret", + tokenFile: "/tmp/default-token", + secretFile: "/tmp/default-secret", + name: "Default LINE", + accounts: { + alerts: { + channelAccessToken: "alerts-token", + channelSecret: "alerts-secret", + }, + }, + }, + }, + } satisfies OpenClawConfig; + + const nextCfg = lineConfigAdapter.deleteAccount!({ cfg, accountId: "default" }); + + expect(nextCfg.channels?.line).toMatchObject({ + accounts: { + alerts: { + channelAccessToken: "alerts-token", + channelSecret: "alerts-secret", + }, + }, + }); + expect(nextCfg.channels?.line?.channelAccessToken).toBeUndefined(); + expect(nextCfg.channels?.line?.channelSecret).toBeUndefined(); + expect(nextCfg.channels?.line?.tokenFile).toBeUndefined(); + expect(nextCfg.channels?.line?.secretFile).toBeUndefined(); + expect(nextCfg.channels?.line?.name).toBeUndefined(); + expect(listLineAccountIds(nextCfg)).toEqual(["alerts"]); + }); +}); diff --git a/extensions/line/src/config-adapter.ts b/extensions/line/src/config-adapter.ts index eb31c18dda5a..4ec5a443e924 100644 --- a/extensions/line/src/config-adapter.ts +++ b/extensions/line/src/config-adapter.ts @@ -21,7 +21,7 @@ export const lineConfigAdapter = createScopedChannelConfigAdapter< resolveAccount: (cfg, accountId) => resolveLineAccount({ cfg, accountId: accountId ?? undefined }), defaultAccountId: resolveDefaultLineAccountId, - clearBaseFields: ["channelSecret", "tokenFile", "secretFile"], + clearBaseFields: ["channelAccessToken", "channelSecret", "tokenFile", "secretFile", "name"], resolveAllowFrom: (account) => account.config.allowFrom, formatAllowFrom: (allowFrom) => normalizeStringEntries(allowFrom).map(normalizeLineAllowFrom), }); From 65a0db95ddb3d57456db0a5fc59f5ef61c4080cd Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 05:26:59 -0700 Subject: [PATCH 45/57] test(memory): isolate wiki plugin fixtures (#118654) --- extensions/memory-wiki/src/agent-vault-isolation.test.ts | 2 ++ extensions/memory-wiki/src/corpus-supplement.visibility.test.ts | 2 ++ 2 files changed, 4 insertions(+) diff --git a/extensions/memory-wiki/src/agent-vault-isolation.test.ts b/extensions/memory-wiki/src/agent-vault-isolation.test.ts index ab3f608caadf..70b325b1e705 100644 --- a/extensions/memory-wiki/src/agent-vault-isolation.test.ts +++ b/extensions/memory-wiki/src/agent-vault-isolation.test.ts @@ -82,6 +82,8 @@ describe("agent-scoped memory-wiki tools", () => { it("keeps apply, search, and get behavior isolated by configured agent", async () => { const vaultParent = await createTempDir("memory-wiki-agent-vaults-"); const appConfig = { + // This suite registers memory-core directly; runtime discovery would load unrelated plugins. + plugins: { enabled: false }, agents: { list: [{ id: "support", default: true }, { id: "marketing" }], }, diff --git a/extensions/memory-wiki/src/corpus-supplement.visibility.test.ts b/extensions/memory-wiki/src/corpus-supplement.visibility.test.ts index 9c743eb3c206..0a6903789b97 100644 --- a/extensions/memory-wiki/src/corpus-supplement.visibility.test.ts +++ b/extensions/memory-wiki/src/corpus-supplement.visibility.test.ts @@ -18,6 +18,8 @@ import { createMemoryWikiTestHarness } from "./test-helpers.js"; const { createVault } = createMemoryWikiTestHarness(); const appConfig = { + // This suite registers memory-core directly; runtime discovery would load unrelated plugins. + plugins: { enabled: false }, agents: { list: [{ id: "main", default: true }, { id: "secondary" }] }, } as OpenClawConfig; From e1b1d4ba7c3f14fa1d37811d3a182400e2e15068 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 3 Aug 2026 14:38:17 +0200 Subject: [PATCH 46/57] fix(config): stop plugin schemas rejecting the channel key core writes (#117992) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(config): stop plugin schemas rejecting the channel key core writes A published channel plugin declares a closed config schema listing only the keys it knew about when that version shipped. Core's doctor migration writes heartbeatVisibility into channels. and its accounts for every channel, so a plugin predating that key turns config core itself produced into a gateway that refuses to start with 78/CONFIG — and doctor --fix reports it without repairing it, leaving no in-product recovery. This took the live gateway down after the last deploy: @openclaw/signal 2026.7.2-beta.3 ships a 44-property closed schema with no heartbeatVisibility, while core reads all three precedence layers (src/infra/heartbeat-visibility.ts) and documents them (docs/gateway/heartbeat.md). Channel schemas built from core's shape may omit common fields on purpose — buildCommonChannelAccountShape takes an `omit` list — so widening every common key would silently accept config that can never take effect. The fix stays on the keys core writes: core wrote it, so core accepts it. Everything else is untouched, including unknown-key rejection, which is covered by a test and verified end to end. Applied once when the channel schema map is built, so the AJV cache key still maps to exactly one schema per channel, and unchanged schemas are returned by identity. Verified against the real failure: with the live plugin set loaded, the config that crashed the gateway now validates at both channel and per-account level, while an unknown key on the same channel is still rejected. * fix(config): normalize external channel schemas at metadata owner * fix(config): normalize core-owned channel visibility schemas * style(config): format channel metadata regression coverage * refactor(config): simplify channel schema normalization coverage --------- Co-authored-by: Peter Steinberger --- src/config/channel-config-metadata.ts | 94 ++++++- src/config/runtime-schema.test.ts | 122 +++++++++ .../validation.channel-metadata.test.ts | 255 ++++++++++++++++++ 3 files changed, 470 insertions(+), 1 deletion(-) diff --git a/src/config/channel-config-metadata.ts b/src/config/channel-config-metadata.ts index 32a3d3f07884..b7b09c0cef2c 100644 --- a/src/config/channel-config-metadata.ts +++ b/src/config/channel-config-metadata.ts @@ -2,9 +2,11 @@ * Converts plugin manifest metadata into deterministic config UI metadata for docs, validation, and runtime schema. * When multiple plugin origins expose the same id/channel, the closest origin owns the surfaced schema. */ +import { isRecord } from "@openclaw/normalization-core/record-coerce"; import type { PluginManifestRegistry } from "../plugins/manifest-registry.js"; import type { PluginOrigin } from "../plugins/plugin-origin.types.js"; import type { ChannelUiMetadata, PluginUiMetadata } from "./schema.js"; +import { ChannelHeartbeatVisibilitySchema } from "./zod-schema.channels.js"; type ChannelSchemaMetadataWithOwnership = ChannelUiMetadata & { schemaPluginId?: string; @@ -34,6 +36,92 @@ const PLUGIN_ORIGIN_RANK: Readonly> = { bundled: 3, }; +const CHANNEL_HEARTBEAT_VISIBILITY_JSON_SCHEMA = + ChannelHeartbeatVisibilitySchema.unwrap().toJSONSchema({ target: "draft-07" }); + +function normalizeCoreOwnedChannelSchema(schema: Record): Record { + const normalized = structuredClone(schema); + let changed = false; + const normalizeNode = ( + node: Record, + accountMap = false, + rootScope = true, + ): void => { + let withinRootScope = rootScope && (node === normalized || typeof node.$id !== "string"); + if (typeof node.$ref === "string") { + const match = withinRootScope + ? /^#\/(\$defs|definitions)\/([A-Za-z0-9_.-]+)$/.exec(node.$ref) + : null; + const definitions = match?.[1] ? normalized[match[1]] : undefined; + const target = isRecord(definitions) && match?.[2] ? definitions[match[2]] : undefined; + if ( + !isRecord(target) || + Object.keys(node).some( + (key) => !["$ref", "$defs", "definitions", "$id", "$schema"].includes(key), + ) || + ["$id", "$anchor", "$dynamicAnchor", "$recursiveAnchor", "$schema", "$ref"].some((key) => + Object.hasOwn(target, key), + ) + ) { + return; + } + // Inline only this owner; changing shared definitions would affect unrelated consumers. + const owner = { ...node }; + Object.assign(node, structuredClone(target), owner); + delete node.$ref; + changed = true; + withinRootScope = node === normalized; + } + + for (const key of ["allOf", "anyOf", "oneOf"] as const) { + const variants = node[key]; + for (const variant of Array.isArray(variants) ? variants : []) { + if (isRecord(variant)) { + normalizeNode(variant, accountMap, withinRootScope); + } + } + } + + if (accountMap) { + if (node.additionalProperties === true) { + node.additionalProperties = {}; + changed = true; + } + const entries = [ + node.additionalProperties, + ...Object.values(isRecord(node.patternProperties) ? node.patternProperties : {}), + ]; + for (const entry of entries) { + if (isRecord(entry)) { + normalizeNode(entry, false, withinRootScope); + } + } + return; + } + + const properties = isRecord(node.properties) ? node.properties : {}; + if ( + JSON.stringify(properties.heartbeatVisibility) !== + JSON.stringify(CHANNEL_HEARTBEAT_VISIBILITY_JSON_SCHEMA) + ) { + node.properties = { + ...properties, + heartbeatVisibility: CHANNEL_HEARTBEAT_VISIBILITY_JSON_SCHEMA, + }; + changed = true; + } + + // Account maps are containers; only each account entry owns heartbeat visibility. + const accounts = properties.accounts; + if (isRecord(accounts)) { + normalizeNode(accounts, true, withinRootScope); + } + }; + + normalizeNode(normalized); + return changed ? normalized : schema; +} + /** Collects plugin config UI metadata with deterministic origin precedence and output ordering. */ export function collectPluginSchemaMetadata(registry: PluginManifestRegistry): PluginUiMetadata[] { const deduped = new Map< @@ -109,7 +197,11 @@ export function collectChannelSchemaMetadataWithOwnership( id: channelId, label: channelConfig.label ?? rootLabel ?? current?.label, description: channelConfig.description ?? rootDescription ?? current?.description, - configSchema: channelConfig.schema, + // Installed plugin schemas can lag core; bundled schemas share its release and identity. + configSchema: + record.origin === "bundled" || channelConfig.schema === undefined + ? channelConfig.schema + : normalizeCoreOwnedChannelSchema(channelConfig.schema), configUiHints: channelConfig.uiHints as ChannelUiMetadata["configUiHints"], schemaPluginId: channelConfig.schema === undefined ? undefined : record.id, schemaPluginOrigin: channelConfig.schema === undefined ? undefined : record.origin, diff --git a/src/config/runtime-schema.test.ts b/src/config/runtime-schema.test.ts index 2c38f2e513c0..6b882029ee78 100644 --- a/src/config/runtime-schema.test.ts +++ b/src/config/runtime-schema.test.ts @@ -303,6 +303,128 @@ describe("loadGatewayRuntimeConfigSchema", () => { expect(channelProps).toHaveProperty("matrix"); }); + it("projects strict heartbeat visibility for external channels and their accounts", () => { + mockLoadPluginManifestRegistry.mockReturnValue({ + diagnostics: [], + plugins: [ + { + id: "external-chat", + origin: "workspace", + channels: ["external-chat"], + channelConfigs: { + "external-chat": { + schema: { + type: "object", + properties: { + endpoint: { type: "string" }, + accounts: { + type: "object", + additionalProperties: { + type: "object", + properties: { endpoint: { type: "string" } }, + additionalProperties: false, + }, + }, + }, + additionalProperties: false, + }, + }, + }, + }, + ], + }); + + const result = loadGatewayRuntimeConfigSchema(); + const schema = result.schema as { properties?: Record }; + const channels = schema.properties?.channels as { properties?: Record }; + const heartbeatVisibility = { + type: "object", + properties: { + showOk: { type: "boolean" }, + showAlerts: { type: "boolean" }, + useIndicator: { type: "boolean" }, + }, + additionalProperties: false, + }; + + expect(channels.properties?.["external-chat"]).toMatchObject({ + additionalProperties: false, + properties: { + heartbeatVisibility, + accounts: { + additionalProperties: { + additionalProperties: false, + properties: { heartbeatVisibility }, + }, + }, + }, + }); + }); + + it("projects canonical heartbeats into composed schemas and referenced open accounts", () => { + mockLoadPluginManifestRegistry.mockReturnValue({ + diagnostics: [], + plugins: [ + { + id: "external-chat", + origin: "workspace", + channels: ["external-chat"], + channelConfigs: { + "external-chat": { + schema: { + $defs: { Account: {} }, + anyOf: [ + { type: "object", additionalProperties: true }, + { + type: "object", + properties: { + accounts: { + type: "object", + additionalProperties: { $ref: "#/$defs/Account" }, + }, + }, + additionalProperties: false, + }, + ], + }, + }, + }, + }, + ], + }); + + const result = loadGatewayRuntimeConfigSchema(); + const schema = result.schema as { properties?: Record }; + const channels = schema.properties?.channels as { properties?: Record }; + const heartbeatVisibility = { + type: "object", + additionalProperties: false, + properties: { + showOk: { type: "boolean" }, + showAlerts: { type: "boolean" }, + useIndicator: { type: "boolean" }, + }, + }; + + const projected = channels.properties?.["external-chat"] as Record; + expect(projected).toMatchObject({ + properties: { heartbeatVisibility }, + anyOf: [ + { additionalProperties: true, properties: { heartbeatVisibility } }, + { + additionalProperties: false, + properties: { + heartbeatVisibility, + accounts: { + additionalProperties: { properties: { heartbeatVisibility } }, + }, + }, + }, + ], + }); + expect(projected.$defs).toEqual({ Account: {} }); + }); + it("reuses the current gateway plugin metadata snapshot for config schema requests", () => { mockGetCurrentPluginMetadataSnapshot.mockReturnValueOnce({ manifestRegistry: { diff --git a/src/config/validation.channel-metadata.test.ts b/src/config/validation.channel-metadata.test.ts index ed9764f81d33..ec3e38cf2c32 100644 --- a/src/config/validation.channel-metadata.test.ts +++ b/src/config/validation.channel-metadata.test.ts @@ -93,6 +93,14 @@ function createExternalFeishuSchemaRegistry(): PluginManifestRegistry { appSecret: { type: "string" }, replyMode: { type: "string", enum: ["thread", "direct"] }, footer: { type: "string" }, + accounts: { + type: "object", + additionalProperties: { + type: "object", + properties: { appId: { type: "string" } }, + additionalProperties: false, + }, + }, }, required: ["appId", "appSecret"], additionalProperties: false, @@ -105,6 +113,20 @@ function createExternalFeishuSchemaRegistry(): PluginManifestRegistry { }; } +function requireExternalFeishuChannelSchema(registry: PluginManifestRegistry) { + return expectDefined( + registry.plugins[0]?.channelConfigs?.feishu?.schema, + "external Feishu channel schema", + ); +} + +function requireExternalFeishuChannelProperties(registry: PluginManifestRegistry) { + return expectDefined( + requireExternalFeishuChannelSchema(registry).properties as Record | undefined, + "external Feishu channel schema properties", + ); +} + function createExternalFeishuSchemaWithCloserMetadataRegistry(): PluginManifestRegistry { const registry = createExternalFeishuSchemaRegistry(); return { @@ -593,6 +615,239 @@ describe("validateConfigObjectRawWithPlugins channel metadata", () => { expect(result.ok).toBe(true); }); + it("accepts core-owned heartbeat visibility in closed channel and account schemas", () => { + mockLoadPluginManifestRegistry.mockReturnValue(createExternalFeishuSchemaRegistry()); + + const result = validateConfigObjectRawWithPlugins({ + channels: { + feishu: { + appId: "app-id", + appSecret: "secret", + heartbeatVisibility: { showAlerts: false, useIndicator: true }, + accounts: { + work: { heartbeatVisibility: { showOk: true } }, + }, + }, + }, + }); + + expect(result.ok).toBe(true); + }); + + it.each([ + { label: "a scalar", value: "enabled" }, + { label: "a non-boolean visibility flag", value: { showAlerts: 0 } }, + { label: "an unknown visibility field", value: { showOk: true, unexpected: true } }, + ])("rejects $label at channel and account heartbeat visibility scopes", ({ value }) => { + mockLoadPluginManifestRegistry.mockReturnValue(createExternalFeishuSchemaRegistry()); + + for (const config of [ + { appId: "app-id", appSecret: "secret", heartbeatVisibility: value }, + { + appId: "app-id", + appSecret: "secret", + accounts: { work: { heartbeatVisibility: value } }, + }, + ]) { + const result = validateConfigObjectRawWithPlugins({ channels: { feishu: config } }); + + expect(result.ok).toBe(false); + if (!result.ok) { + const hasHeartbeatVisibilityIssue = result.issues.some((issue) => + issue.path.includes("heartbeatVisibility"), + ); + expect(hasHeartbeatVisibilityIssue).toBe(true); + } + } + }); + + it.each(["anyOf", "oneOf"] as const)( + "accepts heartbeat visibility in %s channel branches and their accounts", + (composition) => { + const registry = createExternalFeishuSchemaRegistry(); + const plugin = expectDefined(registry.plugins[0], "external Feishu plugin manifest"); + const channel = expectDefined( + plugin.channelConfigs?.feishu, + "external Feishu channel config", + ); + channel.schema = { + [composition]: [ + { + type: "object", + properties: { appId: { type: "string" } }, + required: ["appId"], + additionalProperties: false, + }, + { + type: "object", + properties: { + accounts: { + type: "object", + additionalProperties: { + type: "object", + properties: { appId: { type: "string" } }, + additionalProperties: false, + }, + }, + }, + required: ["accounts"], + additionalProperties: false, + }, + ], + } as typeof channel.schema; + mockLoadPluginManifestRegistry.mockReturnValue(registry); + + for (const config of [ + { appId: "app-id", heartbeatVisibility: { showOk: true } }, + { + heartbeatVisibility: { useIndicator: false }, + accounts: { work: { appId: "app-id", heartbeatVisibility: { showAlerts: false } } }, + }, + ]) { + expect(validateConfigObjectRawWithPlugins({ channels: { feishu: config } }).ok).toBe(true); + } + }, + ); + + it.each(["patterned", "composed"] as const)( + "accepts core-owned heartbeat visibility for %s accounts", + (shape) => { + const registry = createExternalFeishuSchemaRegistry(); + const properties = requireExternalFeishuChannelProperties(registry); + const account = (properties.accounts as Record).additionalProperties; + properties.accounts = + shape === "patterned" + ? { + type: "object", + patternProperties: { "^work$": account }, + additionalProperties: false, + } + : { allOf: [{ type: "object", additionalProperties: account }] }; + mockLoadPluginManifestRegistry.mockReturnValue(registry); + + const result = validateConfigObjectRawWithPlugins({ + channels: { + feishu: { + appId: "app-id", + appSecret: "secret", + accounts: { work: { heartbeatVisibility: { showOk: true } } }, + }, + }, + }); + expect(result.ok).toBe(true); + }, + ); + + it.each(["root", "account", "composed"] as const)( + "normalizes %s local schema references without changing shared definitions", + (scope) => { + const registry = createExternalFeishuSchemaRegistry(); + const channel = expectDefined(registry.plugins[0]?.channelConfigs?.feishu, "Feishu channel"); + const schema = channel.schema; + const accounts = requireExternalFeishuChannelProperties(registry).accounts as Record< + string, + unknown + >; + const account = accounts.additionalProperties as Record; + const definitions = [schema, account]; + + if (scope === "root") { + channel.schema = { + $id: "https://example.com/external-feishu", + $schema: "http://json-schema.org/draft-07/schema#", + $ref: "#/$defs/Channel", + $defs: { Channel: schema }, + }; + } else if (scope === "account") { + schema.definitions = { Account: account }; + accounts.additionalProperties = { $ref: "#/definitions/Account" }; + } else { + const root = { anyOf: [schema] }; + accounts.additionalProperties = { $ref: "#/$defs/Account" }; + channel.schema = { $ref: "#/$defs/Root", $defs: { Root: root, Account: account } }; + definitions.push(root); + } + mockLoadPluginManifestRegistry.mockReturnValue(registry); + + const config = { + appId: "app-id", + appSecret: "secret", + heartbeatVisibility: { showOk: true }, + accounts: { work: { heartbeatVisibility: { showAlerts: false } } }, + }; + expect(validateConfigObjectRawWithPlugins({ channels: { feishu: config } }).ok).toBe(true); + expect( + validateConfigObjectRawWithPlugins({ + channels: { + feishu: { ...config, accounts: { work: { heartbeatVisibility: { showAlerts: 0 } } } }, + }, + }).ok, + ).toBe(false); + for (const definition of definitions) { + expect(definition).not.toHaveProperty("properties.heartbeatVisibility"); + } + }, + ); + + it.each([{}, true])( + "validates open channel/account heartbeat settings without rejecting custom fields (%j)", + (accountSchema) => { + const registry = createExternalFeishuSchemaRegistry(); + const schema = requireExternalFeishuChannelSchema(registry); + schema.additionalProperties = true; + const properties = requireExternalFeishuChannelProperties(registry); + properties.accounts = { type: "object", additionalProperties: accountSchema }; + mockLoadPluginManifestRegistry.mockReturnValue(registry); + + const base = { + appId: "app-id", + appSecret: "secret", + customChannelField: true, + heartbeatVisibility: { showOk: true }, + accounts: { + work: { customAccountField: true, heartbeatVisibility: { showAlerts: false } }, + }, + }; + expect(validateConfigObjectRawWithPlugins({ channels: { feishu: base } }).ok).toBe(true); + + for (const config of [ + { ...base, heartbeatVisibility: "enabled" }, + { ...base, accounts: { work: { heartbeatVisibility: { showOk: "yes" } } } }, + ]) { + expect(validateConfigObjectRawWithPlugins({ channels: { feishu: config } }).ok).toBe(false); + } + }, + ); + + it.each([ + { label: "an empty schema", declaration: {} }, + { label: "a boolean schema", declaration: true }, + { label: "an open object schema", declaration: { type: "object", additionalProperties: true } }, + { label: "a stale disabled schema", declaration: false }, + { + label: "an overly strict schema", + declaration: { + type: "object", + properties: { showAlerts: { const: true } }, + additionalProperties: false, + }, + }, + ])("keeps canonical heartbeat validation when a plugin declares $label", ({ declaration }) => { + const registry = createExternalFeishuSchemaRegistry(); + requireExternalFeishuChannelProperties(registry).heartbeatVisibility = declaration; + mockLoadPluginManifestRegistry.mockReturnValue(registry); + + for (const [value, accepted] of [ + [{ showAlerts: false }, true], + [{ showAlerts: 0 }, false], + ] as const) { + const result = validateConfigObjectRawWithPlugins({ + channels: { feishu: { appId: "app-id", appSecret: "secret", heartbeatVisibility: value } }, + }); + expect(result.ok).toBe(accepted); + } + }); + it("names the external plugin owner for unsupported channel properties", () => { mockLoadPluginManifestRegistry.mockReturnValue(createExternalFeishuSchemaRegistry()); From e99f6dc4c3e24f5f006d3040f9c74a09701293ce Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 05:53:28 -0700 Subject: [PATCH 47/57] fix(ui): surface onboarding and pairing clipboard failures (#118651) * fix(ui): surface onboarding and pairing clipboard failures * fix(ui): keep translated clipboard labels after feedback resets --- ui/src/app/app-host-pairing-access.test.ts | 57 ++++++++++++- ui/src/app/app-shell-view.ts | 2 - ui/src/components/connect-command.ts | 13 ++- ui/src/components/copy-button.ts | 94 +++++++++++----------- ui/src/components/login-gate.test.ts | 70 ++++++++++++++++ ui/src/components/wizard-step-controls.ts | 7 +- ui/src/pages/channels/wizard-view.test.ts | 73 +++++++++++++++++ ui/src/pages/channels/wizard-view.ts | 8 +- ui/src/pages/model-setup/view.test.ts | 22 ++--- ui/src/pages/nodes/view-pairing.ts | 8 +- 10 files changed, 276 insertions(+), 78 deletions(-) diff --git a/ui/src/app/app-host-pairing-access.test.ts b/ui/src/app/app-host-pairing-access.test.ts index 413fb8e67c72..df6fe250dd3c 100644 --- a/ui/src/app/app-host-pairing-access.test.ts +++ b/ui/src/app/app-host-pairing-access.test.ts @@ -19,7 +19,11 @@ type PairingSidebar = HTMLElement & { type PairingAuth = { role: string; scopes?: string[] }; -function createPairingShell(params: { auth: PairingAuth | null; connected?: boolean }) { +function createPairingShell(params: { + auth: PairingAuth | null; + connected?: boolean; + setupCode?: string; +}) { const snapshot: ApplicationGatewaySnapshot = { client: { request: vi.fn(async () => ({})) } as unknown as GatewayBrowserClient, phase: params.connected === false ? "stopped" : "connected", @@ -47,10 +51,17 @@ function createPairingShell(params: { auth: PairingAuth | null; connected?: bool approvalErrors: new Map(), approvalNowMs: 0, approvalBusy: false, - devicePairSetupOpen: false, + devicePairSetupOpen: Boolean(params.setupCode), devicePairSetupLoading: false, devicePairSetupError: null, - devicePairSetup: null, + devicePairSetup: params.setupCode + ? { + setupCode: params.setupCode, + gatewayUrl: "wss://gateway.example.test", + auth: "token", + urlSource: "test", + } + : null, devicePairSetupAccess: "full", devicePairPendingCount: 0, updateAvailable: null, @@ -82,12 +93,14 @@ function createPairingShell(params: { auth: PairingAuth | null; connected?: bool return sidebar; }; - return { snapshot, openDevicePairSetup, renderSidebar }; + return { snapshot, openDevicePairSetup, renderSidebar, container }; } afterEach(() => { document.body.replaceChildren(); + vi.unstubAllGlobals(); vi.restoreAllMocks(); + Reflect.deleteProperty(document, "execCommand"); }); describe("application shell pairing access", () => { @@ -137,4 +150,40 @@ describe("application shell pairing access", () => { expect(renderSidebar().canPairDevice).toBe(false); }); + + it("shows a visible accessible error when a mobile setup code cannot be copied", async () => { + const writeText = vi.fn().mockRejectedValue(new DOMException("Clipboard access denied")); + const execCommand = vi.fn(() => false); + vi.stubGlobal("navigator", { clipboard: { writeText } }); + Object.defineProperty(document, "execCommand", { configurable: true, value: execCommand }); + const schedule = vi.spyOn(window, "setTimeout"); + const { container, renderSidebar } = createPairingShell({ + auth: { role: "operator", scopes: ["operator.pairing"] }, + setupCode: "pair-mobile-secret", + }); + renderSidebar(); + const pairing = container.querySelector(".device-pair-setup"); + if (!pairing) { + throw new Error("Expected the application shell to render its mobile pairing dialog"); + } + document.body.append(pairing); + const button = pairing.querySelector(".device-pair-setup__actions button"); + + button?.click(); + + await vi.waitFor(() => expect(button?.textContent?.trim()).toBe("Copy failed")); + expect(button?.getAttribute("aria-label")).toBe("Copy failed"); + expect(button?.querySelector("svg")).not.toBeNull(); + expect(writeText).toHaveBeenCalledWith("pair-mobile-secret"); + expect(execCommand).toHaveBeenCalledWith("copy"); + + const reset = schedule.mock.calls.find(([, delay]) => delay === 2_000)?.[0]; + if (typeof reset !== "function") { + throw new Error("Expected the failed copy feedback to schedule its reset"); + } + reset(); + + expect(button?.textContent?.trim()).toBe("Copy setup code"); + expect(button?.getAttribute("aria-label")).toBe("Copy setup code"); + }); }); diff --git a/ui/src/app/app-shell-view.ts b/ui/src/app/app-shell-view.ts index e09d33b7c2d6..e468afd52832 100644 --- a/ui/src/app/app-shell-view.ts +++ b/ui/src/app/app-shell-view.ts @@ -10,7 +10,6 @@ import { icons } from "../components/icons.ts"; import { renderSettingsSidebar } from "../components/settings-sidebar.ts"; import type { ThemeModeChangeDetail } from "../components/theme-mode-toggle.ts"; import { t } from "../i18n/index.ts"; -import { copyToClipboard } from "../lib/clipboard.ts"; import { isGatewayMethodAdvertised } from "../lib/gateway-methods.ts"; import { readSessionMethodAccess } from "../lib/session-method-access.ts"; import { normalizeAgentId } from "../lib/sessions/session-key.ts"; @@ -489,7 +488,6 @@ export function renderApplicationShell(host: ShellViewHost) { onRefresh: () => void context.overlays.refreshDevicePairSetup(), onAccessChange: (access) => void context.overlays.setDevicePairSetupAccess(access), onClose: () => context.overlays.closeDevicePairSetup(), - onCopy: (setupCode) => void copyToClipboard(setupCode), onManageDevices: () => { context.overlays.closeDevicePairSetup(); host.navigate("nodes"); diff --git a/ui/src/components/connect-command.ts b/ui/src/components/connect-command.ts index 889ddbbd113c..3289ba6357dd 100644 --- a/ui/src/components/connect-command.ts +++ b/ui/src/components/connect-command.ts @@ -1,12 +1,11 @@ // Control UI component renders a copyable gateway connection command. import { html } from "lit"; import { t } from "../i18n/index.ts"; -import { copyToClipboard } from "../lib/clipboard.ts"; import { renderCopyButton } from "./copy-button.ts"; import "./tooltip.ts"; -async function copyCommand(command: string) { - await copyToClipboard(command); +function copyCommand(event: Event) { + (event.currentTarget as HTMLElement).querySelector(".chat-copy-btn")?.click(); } export function renderConnectCommand(command: string) { @@ -18,18 +17,18 @@ export function renderConnectCommand(command: string) { role="button" tabindex="0" aria-label=${t("connection.help.copyCommandAria", { command })} - @click=${async (event: Event) => { + @click=${(event: Event) => { if ((event.target as HTMLElement | null)?.closest(".chat-copy-btn")) { return; } - await copyCommand(command); + copyCommand(event); }} - @keydown=${async (event: KeyboardEvent) => { + @keydown=${(event: KeyboardEvent) => { if (event.key !== "Enter" && event.key !== " ") { return; } event.preventDefault(); - await copyCommand(command); + copyCommand(event); }} > ${command} diff --git a/ui/src/components/copy-button.ts b/ui/src/components/copy-button.ts index d2ef250cd2b6..44e50eedff81 100644 --- a/ui/src/components/copy-button.ts +++ b/ui/src/components/copy-button.ts @@ -21,6 +21,54 @@ type CopyButtonOptions = { function setButtonLabel(button: HTMLButtonElement, label: string) { button.setAttribute("aria-label", label); + // Preserve Lit's marker nodes so a later locale change can rerender this label. + const visibleLabel = button.querySelector("[data-copy-label]")?.lastChild; + if (visibleLabel?.nodeType === Node.TEXT_NODE) { + visibleLabel.nodeValue = label; + } +} + +export async function handleCopyButton(event: Event, text: string, idleLabel: string) { + const button = event.currentTarget as HTMLButtonElement | null; + if (!button || button.dataset.copying === "1") { + return; + } + + // Older reset timers must not replace feedback from a newer copy attempt. + const attempt = String(Number(button.dataset.copyAttempt ?? "0") + 1); + button.dataset.copyAttempt = attempt; + button.dataset.copying = "1"; + button.setAttribute("aria-busy", "true"); + button.disabled = true; + + const copied = await copyToClipboard(text); + delete button.dataset.copying; + button.removeAttribute("aria-busy"); + button.disabled = false; + if (!button.isConnected || button.dataset.copyAttempt !== attempt) { + return; + } + + const feedback = copied ? "copied" : "error"; + delete button.dataset[copied ? "error" : "copied"]; + button.dataset[feedback] = "1"; + const feedbackLabel = t(copied ? "common.copied" : "common.copyFailed"); + setButtonLabel(button, feedbackLabel); + + const duration = copied ? COPIED_FOR_MS : ERROR_FOR_MS; + window.setTimeout(() => { + if (!button.isConnected || button.dataset.copyAttempt !== attempt) { + return; + } + delete button.dataset[feedback]; + // A locale rerender can replace the idle label while feedback is still active. + const renderedLabel = + button.querySelector("[data-copy-label]")?.textContent ?? button.getAttribute("aria-label"); + setButtonLabel( + button, + renderedLabel && renderedLabel !== feedbackLabel ? renderedLabel : idleLabel, + ); + }, duration); } function createCopyButton(options: CopyButtonOptions): TemplateResult { @@ -31,51 +79,7 @@ function createCopyButton(options: CopyButtonOptions): TemplateResult { class=${options.bare ? "chat-copy-btn" : "btn btn--xs chat-copy-btn"} type="button" aria-label=${idleLabel} - @click=${async (e: Event) => { - const btn = e.currentTarget as HTMLButtonElement | null; - - if (!btn || btn.dataset.copying === "1") { - return; - } - - btn.dataset.copying = "1"; - btn.setAttribute("aria-busy", "true"); - btn.disabled = true; - - const copied = await copyToClipboard(options.text()); - if (!btn.isConnected) { - return; - } - - delete btn.dataset.copying; - btn.removeAttribute("aria-busy"); - btn.disabled = false; - - if (!copied) { - btn.dataset.error = "1"; - setButtonLabel(btn, t("common.copyFailed")); - - window.setTimeout(() => { - if (!btn.isConnected) { - return; - } - delete btn.dataset.error; - setButtonLabel(btn, idleLabel); - }, ERROR_FOR_MS); - return; - } - - btn.dataset.copied = "1"; - setButtonLabel(btn, t("common.copied")); - - window.setTimeout(() => { - if (!btn.isConnected) { - return; - } - delete btn.dataset.copied; - setButtonLabel(btn, idleLabel); - }, COPIED_FOR_MS); - }} + @click=${(event: Event) => void handleCopyButton(event, options.text(), idleLabel)} >