From 9b90e104c53377a349b92bc14c7c9984d02b68a9 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 12 Aug 2026 21:18:15 +0800 Subject: [PATCH] fix(tts): preserve hidden-only tagged fallback text (#122608) Punchcard-Session: golden-brook-lantern-zw --- src/auto-reply/reply/dispatch-acp.test.ts | 55 +++++++++++++++++++ src/auto-reply/reply/dispatch-acp.ts | 8 +++ ...from-config.delivery-and-tts.test-utils.ts | 51 +++++++++++++++++ .../reply/dispatch-from-config.finalize.ts | 16 ++++++ 4 files changed, 130 insertions(+) diff --git a/src/auto-reply/reply/dispatch-acp.test.ts b/src/auto-reply/reply/dispatch-acp.test.ts index 2de35d9b07fa..727fb84ff65d 100644 --- a/src/auto-reply/reply/dispatch-acp.test.ts +++ b/src/auto-reply/reply/dispatch-acp.test.ts @@ -3086,6 +3086,61 @@ describe("tryDispatchAcpReplyCore", () => { expect(dispatcherCall(dispatcher.sendFinalReply).text).toBe("Visible. Done."); }); + it.each([ + { + expectedText: "Private ACP speech.", + ttsReply: { text: "Private ACP speech." }, + finalReply: {}, + streamedText: "[[tts:text]]Private ACP speech.[[/tts:text]]", + }, + { + expectedText: undefined, + ttsReply: { + text: "Private ACP speech.", + mediaUrl: "/tmp/openclaw-media/acp-tts.ogg", + audioAsVoice: true, + }, + finalReply: { + mediaUrl: "/tmp/openclaw-media/acp-tts.ogg", + audioAsVoice: true, + }, + streamedText: "[[tts:text]]Private ACP speech.[[/tts:text]]", + }, + { + expectedText: "Visible ACP answer. ", + ttsReply: { text: "Visible ACP answer." }, + finalReply: undefined, + streamedText: "Visible ACP answer. [[tts:text]]Private speech.[[/tts:text]]", + }, + ])("keeps tagged ACP TTS delivery single for $streamedText", async (testCase) => { + setReadyAcpResolution(); + queueTtsReplies(testCase.ttsReply as MockTtsReply); + mockVisibleTextTurn(testCase.streamedText); + const { dispatcher } = createDispatcher(); + + await runDispatch({ + bodyForAgent: "reply", + cfg: createAcpTestConfig({ + acp: { enabled: true, stream: { deliveryMode: "live" } }, + tts: { auto: "tagged" }, + }), + dispatcher, + ctxOverrides: { Provider: "telegram", Surface: "telegram" }, + }); + + const blockReply = vi.mocked(dispatcher.sendBlockReply).mock.calls[0]?.[0]; + const deliveredPayload = testCase.finalReply + ? dispatcherCall(dispatcher.sendFinalReply) + : blockReply; + expect(deliveredPayload?.text).toBe(testCase.expectedText); + if (testCase.finalReply) { + expect(dispatcher.sendFinalReply).toHaveBeenCalledTimes(1); + expect(deliveredPayload).toMatchObject(testCase.finalReply); + } else { + expect(dispatcher.sendFinalReply).not.toHaveBeenCalled(); + } + }); + it("falls back to Telegram ACP text when a routed captioned voice is suppressed", async () => { setReadyAcpResolution(); ttsCapabilityMocks.captionedFinalText = true; diff --git a/src/auto-reply/reply/dispatch-acp.ts b/src/auto-reply/reply/dispatch-acp.ts index e08c2249a560..0735ebcc3536 100644 --- a/src/auto-reply/reply/dispatch-acp.ts +++ b/src/auto-reply/reply/dispatch-acp.ts @@ -56,6 +56,7 @@ import { createAcpDispatchDeliveryCoordinator, type AcpDispatchDeliveryCoordinator, } from "./dispatch-acp-delivery.js"; +import { needsTtsFallback } from "./dispatch-from-config.finalize.js"; import { appendRecentHistoryImageContext } from "./history-media.js"; import { hasInboundMediaForUnderstanding } from "./inbound-media.js"; import type { ReplyDispatchKind, ReplyDispatcher } from "./reply-dispatcher.types.js"; @@ -368,6 +369,13 @@ async function finalizeAcpTurnOutput(params: { { skipTts: true }, ); queuedFinal = queuedFinal || delivered; + } else if (needsTtsFallback(true, accumulatedVisibleBlockText, ttsSyntheticReply.text)) { + const delivered = await params.delivery.deliver( + "final", + { text: ttsSyntheticReply.text }, + { skipTts: true }, + ); + queuedFinal = queuedFinal || delivered; } } catch (err) { logVerbose(`dispatch-acp: accumulated ACP block TTS failed: ${formatErrorMessage(err)}`); diff --git a/src/auto-reply/reply/dispatch-from-config.delivery-and-tts.test-utils.ts b/src/auto-reply/reply/dispatch-from-config.delivery-and-tts.test-utils.ts index 005a53da5443..bc2ad313e3af 100644 --- a/src/auto-reply/reply/dispatch-from-config.delivery-and-tts.test-utils.ts +++ b/src/auto-reply/reply/dispatch-from-config.delivery-and-tts.test-utils.ts @@ -15,6 +15,7 @@ import { createTestRegistry } from "../../test-utils/channel-plugins.js"; import { getReplyPayloadMetadata, setReplyPayloadMetadata } from "../reply-payload.js"; import type { MsgContext } from "../templating.js"; import type { GetReplyOptions, ReplyPayload } from "../types.js"; +import { needsTtsFallback } from "./dispatch-from-config.finalize.js"; import { createDispatcher, diagnosticMocks, @@ -2223,5 +2224,55 @@ describe("dispatchReplyFromConfig", () => { expect(dispatcher.sendBlockReply).toHaveBeenCalledWith({ text: "Plain tagged text." }); expect(dispatcher.sendFinalReply).not.toHaveBeenCalled(); }); + + it.each([ + { + expectedText: "Private speech.", + ttsReply: { text: "Private speech." }, + finalReply: {}, + streamedText: "[[tts:text]]Private speech.[[/tts:text]]", + }, + { + expectedText: undefined, + ttsReply: { text: "Private speech.", mediaUrl: "https://x/tts.opus", audioAsVoice: true }, + finalReply: { mediaUrl: "https://x/tts.opus", audioAsVoice: true }, + streamedText: "[[tts:text]]Private speech.[[/tts:text]]", + }, + { + expectedText: "Visible answer.", + ttsReply: { text: "Visible answer." }, + finalReply: undefined, + streamedText: "Visible answer. [[tts:text]]Private speech.[[/tts:text]]", + }, + ])("keeps tagged TTS delivery single for $streamedText", async (testCase) => { + setNoAbort(); + ttsMocks.state.statusSnapshot.autoMode = "tagged"; + ttsMocks.maybeApplyTtsToPayload.mockResolvedValueOnce(testCase.ttsReply); + const dispatcher = createDispatcher(); + const replyResolver = async (_ctx: MsgContext, opts?: GetReplyOptions) => { + await opts?.onBlockReply?.({ text: testCase.streamedText }); + return undefined; + }; + + await dispatchReplyFromConfig({ + ctx: buildTestCtx({ Provider: "telegram", Surface: "telegram" }), + cfg: emptyConfig, + dispatcher, + replyResolver, + }); + + const blockReply = vi.mocked(dispatcher.sendBlockReply).mock.calls[0]?.[0]; + const deliveredPayload = testCase.finalReply ? firstFinalReplyPayload(dispatcher) : blockReply; + expect(deliveredPayload?.text?.trim()).toBe(testCase.expectedText); + if (testCase.finalReply) { + expect(dispatcher.sendFinalReply).toHaveBeenCalledTimes(1); + expect(deliveredPayload).toMatchObject(testCase.finalReply); + } else { + expect(dispatcher.sendFinalReply).not.toHaveBeenCalled(); + } + }); + + it("skips fallback when directives stay visible", () => + expect(needsTtsFallback(false, "[[tts:text]]x", "x")).toBe(false)); }); /* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/auto-reply/reply/dispatch-from-config.finalize.ts b/src/auto-reply/reply/dispatch-from-config.finalize.ts index 188d3f22cb92..aa8e48f3528f 100644 --- a/src/auto-reply/reply/dispatch-from-config.finalize.ts +++ b/src/auto-reply/reply/dispatch-from-config.finalize.ts @@ -31,6 +31,9 @@ type ExecuteDispatchReadyState = Extract< { status: "ready" } >["state"]; +export const needsTtsFallback = (clean: boolean, visible: string, fallback?: string) => + clean && !visible.trim() && Boolean(fallback?.trim()); + export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState) { const { cfg, @@ -233,6 +236,19 @@ export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState) }); queuedFinal = finalReply.queuedFinal || queuedFinal; routedFinalCount += finalReply.routedFinalCount; + } else if ( + needsTtsFallback( + Boolean(state.cleanBlockTtsDirectiveText), + cleanDeferredFinalText(deferredTtsTextPending), + ttsSyntheticReply.text, + ) + ) { + const finalReply = await state.sendFinalPayload(ttsSyntheticReply, { + abortSignal: getDispatchAbortSignal(), + skipTts: true, + }); + queuedFinal = finalReply.queuedFinal || queuedFinal; + routedFinalCount += finalReply.routedFinalCount; } } catch (err) { if (isDispatchReplyOperationAbortedError(err)) {