From a77c0770ffe64c48672d05fc2185efa9b68a9f0d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 21:47:06 -0700 Subject: [PATCH] refactor(speech-core): share TTS request pre-resolution across discord and voice-call (#109640) --- .../discord/src/voice/manager.e2e.test.ts | 32 +- extensions/discord/src/voice/tts.ts | 79 +---- extensions/voice-call/src/runtime.ts | 4 +- .../voice-call/src/telephony-tts.test.ts | 273 +++++++++--------- extensions/voice-call/src/telephony-tts.ts | 199 ++----------- packages/speech-core/runtime-api.ts | 2 + packages/speech-core/src/tts.test.ts | 82 +++++- packages/speech-core/src/tts.ts | 30 ++ .../test-helpers/plugin-runtime-mock.ts | 1 + src/plugin-sdk/tts-runtime.types.ts | 23 +- src/plugins/runtime/index.ts | 3 + src/plugins/runtime/runtime-tts-request.ts | 2 + src/plugins/runtime/types-core.ts | 2 + 13 files changed, 335 insertions(+), 397 deletions(-) create mode 100644 src/plugins/runtime/runtime-tts-request.ts diff --git a/extensions/discord/src/voice/manager.e2e.test.ts b/extensions/discord/src/voice/manager.e2e.test.ts index ae1b5bcd1f59..46d92b6c1f5e 100644 --- a/extensions/discord/src/voice/manager.e2e.test.ts +++ b/extensions/discord/src/voice/manager.e2e.test.ts @@ -22,6 +22,7 @@ const { agentCommandMock, resolveRealtimeBootstrapContextInstructionsMock, transcribeAudioFileMock, + prepareTtsRequestMock, textToSpeechStreamMock, textToSpeechMock, logVerboseMock, @@ -149,6 +150,15 @@ const { (...args: unknown[]) => Promise >(async () => undefined), transcribeAudioFileMock: vi.fn(async () => ({ text: "hello from voice" })), + prepareTtsRequestMock: vi.fn(async ({ cfg, text }: { cfg: unknown; text: string }) => ({ + cfg, + directives: { + cleanedText: text, + hasDirective: false, + overrides: {}, + warnings: [], + }, + })), textToSpeechStreamMock: vi.fn( async (): Promise => ({ success: false, error: "stream unavailable" }), ), @@ -219,13 +229,7 @@ vi.mock("openclaw/plugin-sdk/agent-runtime", async () => { return { ...actual, agentCommandFromIngress: agentCommandMock, - getTtsProvider: vi.fn(() => "openai"), resolveAgentDir: vi.fn(() => "/tmp/openclaw-agent"), - resolveTtsConfig: vi.fn(() => ({ - modelOverrides: {}, - providerConfigs: {}, - })), - resolveTtsPrefsPath: vi.fn(() => "/tmp/openclaw-tts.json"), }; }); @@ -286,6 +290,7 @@ vi.mock("../runtime.js", () => ({ transcribeAudioFile: transcribeAudioFileMock, }, tts: { + prepareTtsRequest: prepareTtsRequestMock, textToSpeechStream: textToSpeechStreamMock, textToSpeech: textToSpeechMock, }, @@ -371,6 +376,18 @@ describe("DiscordVoiceManager", () => { resolveRealtimeBootstrapContextInstructionsMock.mockResolvedValue(undefined); transcribeAudioFileMock.mockReset(); transcribeAudioFileMock.mockResolvedValue({ text: "hello from voice" }); + prepareTtsRequestMock.mockReset(); + prepareTtsRequestMock.mockImplementation( + async ({ cfg, text }: { cfg: unknown; text: string }) => ({ + cfg, + directives: { + cleanedText: text, + hasDirective: false, + overrides: {}, + warnings: [], + }, + }), + ); textToSpeechStreamMock.mockReset(); textToSpeechStreamMock.mockResolvedValue({ success: false, error: "stream unavailable" }); textToSpeechMock.mockReset(); @@ -6612,6 +6629,9 @@ describe("DiscordVoiceManager", () => { expect(commandArgs?.messageProvider).toBe("discord-voice"); expect(commandArgs?.message).toContain("Do not call the tts tool"); expect(commandArgs?.message).toContain("repair obvious transcription artifacts"); + expect(prepareTtsRequestMock).toHaveBeenCalledWith( + expect.objectContaining({ text: "hello back" }), + ); expect(lastTtsArgs().channel).toBe("discord"); expect(lastTtsArgs().text).toBe("hello back"); }); diff --git a/extensions/discord/src/voice/tts.ts b/extensions/discord/src/voice/tts.ts index 2bc5f49905aa..fea06068c5ec 100644 --- a/extensions/discord/src/voice/tts.ts +++ b/extensions/discord/src/voice/tts.ts @@ -1,14 +1,7 @@ // Discord plugin module implements tts behavior. -import { - getTtsProvider, - resolveAgentDir, - resolveTtsConfig, - resolveTtsPrefsPath, - type ResolvedTtsConfig, -} from "openclaw/plugin-sdk/agent-runtime"; +import { resolveAgentDir } from "openclaw/plugin-sdk/agent-runtime"; import type { OpenClawConfig, TtsConfig } from "openclaw/plugin-sdk/config-contracts"; -import { parseTtsDirectives } from "openclaw/plugin-sdk/speech"; -import { normalizeOptionalString, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; import { getDiscordRuntime } from "../runtime.js"; import { sanitizeVoiceReplyTextForSpeech } from "./sanitize.js"; @@ -34,58 +27,6 @@ type VoiceReplyAudioResult = error?: string; }; -function mergeTtsConfig(base: TtsConfig, override?: TtsConfig): TtsConfig { - if (!override) { - return base; - } - const baseProviders = base.providers ?? {}; - const overrideProviders = override.providers ?? {}; - const mergedProviders = Object.fromEntries( - uniqueStrings([...Object.keys(baseProviders), ...Object.keys(overrideProviders)]).map( - (providerId) => { - const baseProvider = baseProviders[providerId] ?? {}; - const overrideProvider = overrideProviders[providerId] ?? {}; - return [ - providerId, - { - ...baseProvider, - ...overrideProvider, - }, - ]; - }, - ), - ); - return { - ...base, - ...override, - modelOverrides: { - ...base.modelOverrides, - ...override.modelOverrides, - }, - ...(Object.keys(mergedProviders).length === 0 ? {} : { providers: mergedProviders }), - }; -} - -function resolveVoiceTtsConfig(params: { cfg: OpenClawConfig; override?: TtsConfig }): { - cfg: OpenClawConfig; - resolved: ResolvedTtsConfig; -} { - if (!params.override) { - return { cfg: params.cfg, resolved: resolveTtsConfig(params.cfg) }; - } - const base = params.cfg.messages?.tts ?? {}; - const merged = mergeTtsConfig(base, params.override); - const messages = params.cfg.messages ?? {}; - const cfg = { - ...params.cfg, - messages: { - ...messages, - tts: merged, - }, - }; - return { cfg, resolved: resolveTtsConfig(cfg) }; -} - export async function transcribeVoiceAudio(params: { cfg: OpenClawConfig; agentId: string; @@ -106,25 +47,21 @@ export async function synthesizeVoiceReplyAudio(params: { replyText: string; speakerLabel: string; }): Promise { - const { cfg: ttsCfg, resolved: ttsConfig } = resolveVoiceTtsConfig({ + const runtime = getDiscordRuntime(); + const prepared = await runtime.tts.prepareTtsRequest({ cfg: params.cfg, override: params.override, + text: params.replyText, }); - const directive = parseTtsDirectives(params.replyText, ttsConfig.modelOverrides, { - cfg: ttsCfg, - providerConfigs: ttsConfig.providerConfigs, - preferredProviderId: getTtsProvider(ttsConfig, resolveTtsPrefsPath(ttsConfig)), - }); + const directive = prepared.directives; const rawSpeakText = directive.overrides.ttsText ?? directive.cleanedText.trim(); const speakText = sanitizeVoiceReplyTextForSpeech(rawSpeakText, params.speakerLabel); if (!speakText) { return { status: "empty" }; } - - const runtime = getDiscordRuntime(); const streamResult = await runtime.tts.textToSpeechStream?.({ text: speakText, - cfg: ttsCfg, + cfg: prepared.cfg, channel: "discord", overrides: directive.overrides, disableFallback: true, @@ -141,7 +78,7 @@ export async function synthesizeVoiceReplyAudio(params: { const result = await runtime.tts.textToSpeech({ text: speakText, - cfg: ttsCfg, + cfg: prepared.cfg, channel: "discord", overrides: directive.overrides, }); diff --git a/extensions/voice-call/src/runtime.ts b/extensions/voice-call/src/runtime.ts index 8d8aaf134957..0b0dfcd507f7 100644 --- a/extensions/voice-call/src/runtime.ts +++ b/extensions/voice-call/src/runtime.ts @@ -521,8 +521,8 @@ export async function createVoiceCallRuntime(params: { const twilioProvider = provider as TwilioProvider; if (ttsRuntime?.textToSpeechTelephony) { try { - const ttsProvider = createTelephonyTtsProvider({ - coreConfig, + const ttsProvider = await createTelephonyTtsProvider({ + coreConfig: cfg, ttsOverride: config.tts, runtime: ttsRuntime, logger: log, diff --git a/extensions/voice-call/src/telephony-tts.test.ts b/extensions/voice-call/src/telephony-tts.test.ts index 02d85c93920a..c87777022d0b 100644 --- a/extensions/voice-call/src/telephony-tts.test.ts +++ b/extensions/voice-call/src/telephony-tts.test.ts @@ -1,11 +1,11 @@ +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; // Voice Call tests cover telephony tts plugin behavior. import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import type { VoiceCallTtsConfig } from "./config.js"; -import type { CoreConfig } from "./core-bridge.js"; -import { createTelephonyTtsProvider } from "./telephony-tts.js"; +import { createTelephonyTtsProvider, type TelephonyTtsRuntime } from "./telephony-tts.js"; -function createCoreConfig(): CoreConfig { +function createCoreConfig(): OpenClawConfig { const tts: VoiceCallTtsConfig = { provider: "openai", providers: { @@ -18,98 +18,82 @@ function createCoreConfig(): CoreConfig { return { messages: { tts } }; } -function requireMergedTtsConfig(mergedConfig: CoreConfig | undefined) { - const tts = mergedConfig?.messages?.tts; - if (!tts) { - throw new Error("telephony TTS runtime did not receive merged TTS config"); - } - return tts as Record; -} - -function requireOpenAIProviderConfig(tts: Record): Record { - const providers = - tts.providers && typeof tts.providers === "object" - ? (tts.providers as Record) - : null; - const openai = providers?.openai; - if (!openai || typeof openai !== "object") { - throw new Error("merged TTS config did not preserve providers.openai"); - } - return openai as Record; -} - -async function mergeOverride(override: unknown): Promise> { - let mergedConfig: CoreConfig | undefined; - const provider = createTelephonyTtsProvider({ - coreConfig: createCoreConfig(), - ttsOverride: override as VoiceCallTtsConfig, - runtime: { - textToSpeechTelephony: async ({ cfg }) => { - mergedConfig = cfg; - return { - success: true, - audioBuffer: Buffer.alloc(2), - sampleRate: 8000, - }; - }, - }, - }); - - await provider.synthesizeForTelephony("hello"); - return requireMergedTtsConfig(mergedConfig); -} - -afterEach(() => { - delete (Object.prototype as Record).polluted; +const passthroughPreparation: TelephonyTtsRuntime["prepareTtsRequest"] = async ({ cfg, text }) => ({ + cfg, + directives: { + cleanedText: text, + hasDirective: false, + overrides: {}, + warnings: [], + }, }); -describe("createTelephonyTtsProvider deepMerge hardening", () => { - it("merges safe nested overrides", async () => { - const tts = await mergeOverride({ - providers: { openai: { voice: "coral" } }, +function createRuntime( + textToSpeechTelephony: TelephonyTtsRuntime["textToSpeechTelephony"], + prepareTtsRequest: TelephonyTtsRuntime["prepareTtsRequest"] = passthroughPreparation, +): TelephonyTtsRuntime { + return { prepareTtsRequest, textToSpeechTelephony }; +} + +describe("createTelephonyTtsProvider", () => { + it("uses shared preparation for the surface override and request text", async () => { + const effectiveConfig: OpenClawConfig = { + messages: { tts: { provider: "openai", timeoutMs: 15_000 } }, + }; + const prepareTtsRequest = vi.fn( + async ({ cfg, override, text }) => ({ + cfg: override ? effectiveConfig : cfg, + directives: { + cleanedText: text, + hasDirective: false, + overrides: {}, + warnings: [], + }, + }), + ); + const textToSpeechTelephony = vi.fn(async () => ({ + success: true, + audioBuffer: Buffer.alloc(2), + sampleRate: 8000, + })); + const override: VoiceCallTtsConfig = { timeoutMs: 15_000 }; + const provider = await createTelephonyTtsProvider({ + coreConfig: createCoreConfig(), + ttsOverride: override, + runtime: createRuntime(textToSpeechTelephony, prepareTtsRequest), }); - const openai = requireOpenAIProviderConfig(tts); - expect(openai.voice).toBe("coral"); - expect(openai.model).toBe("gpt-4o-mini-tts"); - }); + await provider.synthesizeForTelephony("hello"); - it("blocks top-level __proto__ keys", async () => { - const tts = await mergeOverride( - JSON.parse('{"__proto__":{"polluted":"top"},"providers":{"openai":{"voice":"coral"}}}'), - ); - const openai = requireOpenAIProviderConfig(tts); - - expect((Object.prototype as Record).polluted).toBeUndefined(); - expect(tts.polluted).toBeUndefined(); - expect(openai.voice).toBe("coral"); - }); - - it("blocks nested __proto__ keys", async () => { - const tts = await mergeOverride( - JSON.parse('{"providers":{"openai":{"model":"safe","__proto__":{"polluted":"nested"}}}}'), - ); - const openai = requireOpenAIProviderConfig(tts); - - expect((Object.prototype as Record).polluted).toBeUndefined(); - expect(openai.polluted).toBeUndefined(); - expect(openai.model).toBe("safe"); + expect(provider.synthesisTimeoutMs).toBe(15_000); + expect(prepareTtsRequest).toHaveBeenNthCalledWith(1, { + cfg: createCoreConfig(), + override, + text: "", + }); + expect(prepareTtsRequest).toHaveBeenNthCalledWith(2, { + cfg: effectiveConfig, + text: "hello", + }); + expect(textToSpeechTelephony).toHaveBeenCalledWith({ + text: "hello", + cfg: effectiveConfig, + overrides: {}, + }); }); it("logs fallback metadata when telephony TTS uses a fallback provider", async () => { const warn = vi.fn(); - const provider = createTelephonyTtsProvider({ + const provider = await createTelephonyTtsProvider({ coreConfig: createCoreConfig(), - runtime: { - textToSpeechTelephony: async () => ({ - success: true, - audioBuffer: Buffer.alloc(2), - sampleRate: 8000, - provider: "microsoft", - fallbackFrom: "elevenlabs", - attemptedProviders: ["elevenlabs", "microsoft"], - }), - }, + runtime: createRuntime(async () => ({ + success: true, + audioBuffer: Buffer.alloc(2), + sampleRate: 8000, + provider: "microsoft", + fallbackFrom: "elevenlabs", + attemptedProviders: ["elevenlabs", "microsoft"], + })), logger: { warn }, }); @@ -119,95 +103,100 @@ describe("createTelephonyTtsProvider deepMerge hardening", () => { ); }); - it("strips telephony TTS directive tags before synthesis", async () => { - let requestText: string | undefined; - const provider = createTelephonyTtsProvider({ + it("uses prepared directive-stripped text for synthesis", async () => { + const textToSpeechTelephony = vi.fn(async () => ({ + success: true, + audioBuffer: Buffer.alloc(2), + sampleRate: 8000, + })); + const provider = await createTelephonyTtsProvider({ coreConfig: createCoreConfig(), - runtime: { - textToSpeechTelephony: async ({ text }) => { - requestText = text; - return { - success: true, - audioBuffer: Buffer.alloc(2), - sampleRate: 8000, - }; + runtime: createRuntime(textToSpeechTelephony, async ({ cfg, text }) => ({ + cfg, + directives: { + cleanedText: text ? "Hello caller" : "", + hasDirective: text.length > 0, + overrides: {}, + warnings: [], }, - }, + })), }); await provider.synthesizeForTelephony("[[tts]]Hello caller[[/tts]]"); - expect(requestText).toBe("Hello caller"); + expect(textToSpeechTelephony).toHaveBeenCalledWith( + expect.objectContaining({ text: "Hello caller" }), + ); }); - it("uses hidden telephony TTS directive text for synthesis", async () => { - let requestText: string | undefined; - let requestOverrides: unknown; - const provider = createTelephonyTtsProvider({ + it("uses prepared hidden directive text and overrides for synthesis", async () => { + const textToSpeechTelephony = vi.fn(async () => ({ + success: true, + audioBuffer: Buffer.alloc(2), + sampleRate: 8000, + })); + const provider = await createTelephonyTtsProvider({ coreConfig: createCoreConfig(), - runtime: { - textToSpeechTelephony: async ({ text, overrides }) => { - requestText = text; - requestOverrides = overrides; - return { - success: true, - audioBuffer: Buffer.alloc(2), - sampleRate: 8000, - }; + runtime: createRuntime(textToSpeechTelephony, async ({ cfg, text }) => ({ + cfg, + directives: { + cleanedText: text ? "Visible text " : "", + ttsText: text ? "Speak this instead" : undefined, + hasDirective: text.length > 0, + overrides: text ? { ttsText: "Speak this instead" } : {}, + warnings: [], }, - }, + })), }); await provider.synthesizeForTelephony( "Visible text [[tts:text]]Speak this instead[[/tts:text]]", ); - expect(requestText).toBe("Speak this instead"); - expect(requestOverrides).toStrictEqual({ ttsText: "Speak this instead" }); + expect(textToSpeechTelephony).toHaveBeenCalledWith( + expect.objectContaining({ + text: "Speak this instead", + overrides: { ttsText: "Speak this instead" }, + }), + ); }); - it("exposes configured timeoutMs as synthesisTimeoutMs", () => { - const provider = createTelephonyTtsProvider({ + it("exposes configured timeoutMs as synthesisTimeoutMs", async () => { + const provider = await createTelephonyTtsProvider({ coreConfig: { messages: { tts: { provider: "openai", timeoutMs: 15000 } } }, - runtime: { - textToSpeechTelephony: async () => ({ - success: true, - audioBuffer: Buffer.alloc(2), - sampleRate: 8000, - }), - }, + runtime: createRuntime(async () => ({ + success: true, + audioBuffer: Buffer.alloc(2), + sampleRate: 8000, + })), }); expect(provider.synthesisTimeoutMs).toBe(15000); }); - it("clamps oversized configured timeoutMs", () => { - const provider = createTelephonyTtsProvider({ + it("clamps oversized configured timeoutMs", async () => { + const provider = await createTelephonyTtsProvider({ coreConfig: { messages: { tts: { provider: "openai", timeoutMs: Number.MAX_SAFE_INTEGER } }, }, - runtime: { - textToSpeechTelephony: async () => ({ - success: true, - audioBuffer: Buffer.alloc(2), - sampleRate: 8000, - }), - }, + runtime: createRuntime(async () => ({ + success: true, + audioBuffer: Buffer.alloc(2), + sampleRate: 8000, + })), }); expect(provider.synthesisTimeoutMs).toBe(MAX_TIMER_TIMEOUT_MS); }); - it("keeps the telephony timeout default when timeoutMs is not configured", () => { - const provider = createTelephonyTtsProvider({ + it("keeps the telephony timeout default when timeoutMs is not configured", async () => { + const provider = await createTelephonyTtsProvider({ coreConfig: createCoreConfig(), - runtime: { - textToSpeechTelephony: async () => ({ - success: true, - audioBuffer: Buffer.alloc(2), - sampleRate: 8000, - }), - }, + runtime: createRuntime(async () => ({ + success: true, + audioBuffer: Buffer.alloc(2), + sampleRate: 8000, + })), }); expect(provider.synthesisTimeoutMs).toBe(8000); diff --git a/extensions/voice-call/src/telephony-tts.ts b/extensions/voice-call/src/telephony-tts.ts index 016e2ee4e468..7ae34245739b 100644 --- a/extensions/voice-call/src/telephony-tts.ts +++ b/extensions/voice-call/src/telephony-tts.ts @@ -1,23 +1,25 @@ // Voice Call plugin module implements telephony tts behavior. +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { resolveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime"; -import { mergeDeep } from "openclaw/plugin-sdk/plugin-config-runtime"; -import { - parseTtsDirectives, - type SpeechModelOverridePolicy, - type SpeechProviderConfig, - type TtsDirectiveOverrides, -} from "openclaw/plugin-sdk/speech"; +import type { TtsDirectiveOverrides, TtsDirectiveParseResult } from "openclaw/plugin-sdk/speech"; import type { VoiceCallTtsConfig } from "./config.js"; -import type { CoreConfig } from "./core-bridge.js"; import { convertPcmToMulaw8k } from "./telephony-audio.js"; // Telephony TTS adapter that applies voice-call overrides and emits 8kHz mulaw audio. /** Core runtime TTS API used by the telephony adapter. */ export type TelephonyTtsRuntime = { + prepareTtsRequest: (params: { + cfg: OpenClawConfig; + override?: VoiceCallTtsConfig; + text: string; + }) => Promise<{ + cfg: OpenClawConfig; + directives: TtsDirectiveParseResult; + }>; textToSpeechTelephony: (params: { text: string; - cfg: CoreConfig; + cfg: OpenClawConfig; prefsPath?: string; overrides?: TtsDirectiveOverrides; }) => Promise<{ @@ -40,48 +42,34 @@ export type TelephonyTtsProvider = { /** Default timeout for one telephony synthesis request. */ export const TELEPHONY_DEFAULT_TTS_TIMEOUT_MS = 8000; -/** Voice-call override policy for inline TTS model directives. */ -type TelephonyModelOverrideConfig = { - enabled?: boolean; - allowText?: boolean; - allowProvider?: boolean; - allowVoice?: boolean; - allowModelId?: boolean; - allowVoiceSettings?: boolean; - allowNormalization?: boolean; - allowSeed?: boolean; -}; - /** Create a TTS provider that honors voice-call overrides and converts PCM to mulaw. */ -export function createTelephonyTtsProvider(params: { - coreConfig: CoreConfig; +export async function createTelephonyTtsProvider(params: { + coreConfig: OpenClawConfig; ttsOverride?: VoiceCallTtsConfig; runtime: TelephonyTtsRuntime; logger?: { warn?: (message: string) => void; }; -}): TelephonyTtsProvider { +}): Promise { const { coreConfig, ttsOverride, runtime, logger } = params; - const mergedConfig = applyTtsOverride(coreConfig, ttsOverride); - const ttsConfig = mergedConfig.messages?.tts; - const modelOverrides = resolveTelephonyModelOverridePolicy( - readTelephonyModelOverrides(ttsConfig), - ); - const providerConfigs = collectTelephonyProviderConfigs(ttsConfig); - const activeProvider = normalizeProviderId(ttsConfig?.provider); + const preparedConfig = await runtime.prepareTtsRequest({ + cfg: coreConfig, + override: ttsOverride, + text: "", + }); const synthesisTimeoutMs = resolveTimerTimeoutMs( - mergedConfig.messages?.tts?.timeoutMs, + preparedConfig.cfg.messages?.tts?.timeoutMs, TELEPHONY_DEFAULT_TTS_TIMEOUT_MS, ); return { synthesisTimeoutMs, synthesizeForTelephony: async (text: string) => { - const directives = parseTtsDirectives(text, modelOverrides, { - cfg: mergedConfig, - providerConfigs, - preferredProviderId: activeProvider, + const prepared = await runtime.prepareTtsRequest({ + cfg: preparedConfig.cfg, + text, }); + const directives = prepared.directives; if (directives.warnings.length > 0) { logger?.warn?.( `[voice-call] Ignored telephony TTS directive overrides (${directives.warnings.join("; ")})`, @@ -92,7 +80,7 @@ export function createTelephonyTtsProvider(params: { : text; const result = await runtime.textToSpeechTelephony({ text: cleanText, - cfg: mergedConfig, + cfg: prepared.cfg, overrides: directives.overrides, }); @@ -114,140 +102,3 @@ export function createTelephonyTtsProvider(params: { }, }; } - -/** Apply voice-call TTS overrides to core config without mutating the original object. */ -function applyTtsOverride(coreConfig: CoreConfig, override?: VoiceCallTtsConfig): CoreConfig { - if (!override) { - return coreConfig; - } - - const base = coreConfig.messages?.tts; - const merged = mergeTtsConfig(base, override); - if (!merged) { - return coreConfig; - } - - return { - ...coreConfig, - messages: { - ...coreConfig.messages, - tts: merged, - }, - }; -} - -/** Merge core and voice-call TTS config, keeping undefined override fields out. */ -function mergeTtsConfig( - base?: VoiceCallTtsConfig, - override?: VoiceCallTtsConfig, -): VoiceCallTtsConfig | undefined { - if (!base && !override) { - return undefined; - } - if (!override) { - return base; - } - if (!base) { - return override; - } - return mergeDeep(base, override) as VoiceCallTtsConfig; -} - -/** Resolve directive override policy for telephony synthesis. */ -function resolveTelephonyModelOverridePolicy( - overrides: TelephonyModelOverrideConfig | undefined, -): SpeechModelOverridePolicy { - const enabled = overrides?.enabled ?? true; - if (!enabled) { - return { - enabled: false, - allowText: false, - allowProvider: false, - allowVoice: false, - allowModelId: false, - allowVoiceSettings: false, - allowNormalization: false, - allowSeed: false, - }; - } - const allow = (value: boolean | undefined, defaultValue = true) => value ?? defaultValue; - return { - enabled: true, - allowText: allow(overrides?.allowText), - allowProvider: allow(overrides?.allowProvider, false), - allowVoice: allow(overrides?.allowVoice), - allowModelId: allow(overrides?.allowModelId), - allowVoiceSettings: allow(overrides?.allowVoiceSettings), - allowNormalization: allow(overrides?.allowNormalization), - allowSeed: allow(overrides?.allowSeed), - }; -} - -/** Read model override policy from TTS config when present. */ -function readTelephonyModelOverrides( - ttsConfig: VoiceCallTtsConfig | undefined, -): TelephonyModelOverrideConfig | undefined { - const value = (ttsConfig as Record | undefined)?.modelOverrides; - return value && typeof value === "object" && !Array.isArray(value) - ? (value as TelephonyModelOverrideConfig) - : undefined; -} - -/** Normalize provider ids for config lookup. */ -function normalizeProviderId(value: unknown): string | undefined { - return typeof value === "string" ? value.trim().toLowerCase() || undefined : undefined; -} - -/** Coerce provider config objects while rejecting arrays and primitives. */ -function asProviderConfig(value: unknown): SpeechProviderConfig { - return value && typeof value === "object" && !Array.isArray(value) - ? (value as SpeechProviderConfig) - : {}; -} - -/** Collect named provider configs from canonical and legacy TTS config shapes. */ -function collectTelephonyProviderConfigs( - ttsConfig: VoiceCallTtsConfig | undefined, -): Record { - if (!ttsConfig) { - return {}; - } - const entries: Record = {}; - const rawProviders = - ttsConfig.providers && - typeof ttsConfig.providers === "object" && - !Array.isArray(ttsConfig.providers) - ? (ttsConfig.providers as Record) - : {}; - for (const [providerId, value] of Object.entries(rawProviders)) { - const normalized = normalizeProviderId(providerId) ?? providerId; - entries[normalized] = asProviderConfig(value); - } - const reservedKeys = new Set([ - "auto", - "enabled", - "maxTextLength", - "mode", - "modelOverrides", - "persona", - "personas", - "prefsPath", - "provider", - "providers", - "summaryModel", - "timeoutMs", - ]); - for (const [key, value] of Object.entries(ttsConfig as Record)) { - if ( - reservedKeys.has(key) || - typeof value !== "object" || - value === null || - Array.isArray(value) - ) { - continue; - } - const normalized = normalizeProviderId(key) ?? key; - entries[normalized] ??= asProviderConfig(value); - } - return entries; -} diff --git a/packages/speech-core/runtime-api.ts b/packages/speech-core/runtime-api.ts index 5c2419796236..9cbc8874a5db 100644 --- a/packages/speech-core/runtime-api.ts +++ b/packages/speech-core/runtime-api.ts @@ -29,6 +29,7 @@ export { isTtsProviderConfigured, listSpeechVoices, maybeApplyTtsToPayload, + prepareTtsRequest, resolveExplicitTtsOverrides, resolveTtsProviderOrder, setLastTtsAttempt, @@ -41,6 +42,7 @@ export { testApi, type TtsDirectiveOverrides, type TtsDirectiveParseResult, + type PreparedTtsRequest, type TtsResult, type TtsSynthesisResult, type TtsSynthesisStreamResult, diff --git a/packages/speech-core/src/tts.test.ts b/packages/speech-core/src/tts.test.ts index 76cd5d08e11f..e70b5de4c967 100644 --- a/packages/speech-core/src/tts.test.ts +++ b/packages/speech-core/src/tts.test.ts @@ -2,7 +2,7 @@ import { realpathSync, rmSync } from "node:fs"; import os from "node:os"; import path from "node:path"; -import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import type { OpenClawConfig, TtsConfig } from "openclaw/plugin-sdk/config-contracts"; import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime"; import type { ReplyPayload } from "openclaw/plugin-sdk/reply-payload"; import { @@ -117,6 +117,7 @@ const { getTtsProvider, listSpeechVoices, maybeApplyTtsToPayload, + prepareTtsRequest, resolveTtsConfig, setSummarizationEnabled, setTtsMaxLength, @@ -240,6 +241,7 @@ async function expectTtsPayloadResult(params: { describe("speech-core native voice-note routing", () => { afterEach(() => { clearRuntimeConfigSnapshot(); + delete (Object.prototype as Record).polluted; synthesizeMock.mockClear(); prepareSynthesisMock.mockClear(); transcodeAudioBufferMock.mockClear(); @@ -267,6 +269,84 @@ describe("speech-core native voice-note routing", () => { ); }); + it("prepares deep-merged surface config and directive inputs", () => { + const cfg: OpenClawConfig = { + messages: { + tts: { + provider: "mock", + modelOverrides: { allowProvider: false }, + providers: { + mock: { + model: "base-model", + voiceSettings: { stability: 0.4 }, + }, + }, + }, + }, + }; + + const prepared = prepareTtsRequest({ + cfg, + override: { + modelOverrides: { allowProvider: true }, + providers: { + mock: { + voice: "surface-voice", + voiceSettings: { speed: 1.1 }, + }, + }, + }, + text: "Hello [[tts:text]]Speak this instead[[/tts:text]] caller", + }); + + expect(prepared.cfg).not.toBe(cfg); + expect(prepared.cfg.messages?.tts?.providers?.mock).toEqual({ + model: "base-model", + voice: "surface-voice", + voiceSettings: { stability: 0.4, speed: 1.1 }, + }); + expect(prepared.cfg.messages?.tts?.modelOverrides?.allowProvider).toBe(true); + expect(prepared.directives).toEqual({ + cleanedText: "Hello caller", + hasDirective: true, + overrides: { + ttsText: "Speak this instead", + }, + ttsText: "Speak this instead", + warnings: [], + }); + expect(cfg.messages?.tts?.providers?.mock).toEqual({ + model: "base-model", + voiceSettings: { stability: 0.4 }, + }); + }); + + it("sanitizes blocked override keys while preparing TTS config", () => { + const prepared = prepareTtsRequest({ + cfg: { + messages: { + tts: { + provider: "mock", + providers: { mock: { model: "base-model" } }, + }, + }, + }, + override: JSON.parse( + '{"__proto__":{"polluted":"top"},"providers":{"mock":{"voice":"safe","__proto__":{"polluted":"nested"}}}}', + ) as TtsConfig, + text: "[[tts:text]]Speak this instead[[/tts:text]]", + }); + + expect((Object.prototype as Record).polluted).toBeUndefined(); + expect(prepared.cfg.messages?.tts).not.toHaveProperty("polluted"); + expect(prepared.cfg.messages?.tts?.providers?.mock).toEqual({ + model: "base-model", + voice: "safe", + }); + expect(prepared.directives.cleanedText).toBe(""); + expect(prepared.directives.ttsText).toBe("Speak this instead"); + }); + it("marks Discord auto TTS replies as native voice messages", async () => { await expectTtsPayloadResult({ channel: "discord", diff --git a/packages/speech-core/src/tts.ts b/packages/speech-core/src/tts.ts index cf4b1dd26ea1..60be9f625fc1 100644 --- a/packages/speech-core/src/tts.ts +++ b/packages/speech-core/src/tts.ts @@ -10,6 +10,7 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { redactSensitiveText } from "openclaw/plugin-sdk/logging-core"; import { transcodeAudioBuffer } from "openclaw/plugin-sdk/media-runtime"; import { clampTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime"; +import { mergeDeep } from "openclaw/plugin-sdk/plugin-config-runtime"; import { markReplyPayloadAsTtsSupplement, resolveSendableOutboundReplyParts, @@ -467,6 +468,35 @@ export function getTtsProvider(config: ResolvedTtsConfig, prefsPath: string): Tt return config.provider; } +export type PreparedTtsRequest = { + cfg: OpenClawConfig; + directives: TtsDirectiveParseResult; +}; + +/** Merge a surface TTS override and resolve its inline synthesis directives. */ +export function prepareTtsRequest(params: { + cfg: OpenClawConfig; + override?: TtsConfig; + text: string; +}): PreparedTtsRequest { + const cfg = params.override + ? { + ...params.cfg, + messages: { + ...params.cfg.messages, + tts: mergeDeep(params.cfg.messages?.tts ?? {}, params.override) as TtsConfig, + }, + } + : params.cfg; + const config = resolveTtsConfig(cfg); + const directives = parseTtsDirectives(params.text, config.modelOverrides, { + cfg, + providerConfigs: config.providerConfigs, + preferredProviderId: getTtsProvider(config, resolveTtsPrefsPath(config)), + }); + return { cfg, directives }; +} + export function resolveExplicitTtsOverrides(params: { cfg: OpenClawConfig; prefsPath?: string; diff --git a/src/plugin-sdk/test-helpers/plugin-runtime-mock.ts b/src/plugin-sdk/test-helpers/plugin-runtime-mock.ts index 4b0f93898cf3..7b5ee87d217d 100644 --- a/src/plugin-sdk/test-helpers/plugin-runtime-mock.ts +++ b/src/plugin-sdk/test-helpers/plugin-runtime-mock.ts @@ -534,6 +534,7 @@ export function createPluginRuntimeMock(overrides: DeepPartial = resizeToJpeg: vi.fn() as unknown as PluginRuntime["media"]["resizeToJpeg"], }, tts: { + prepareTtsRequest: vi.fn() as unknown as PluginRuntime["tts"]["prepareTtsRequest"], textToSpeech: vi.fn() as unknown as PluginRuntime["tts"]["textToSpeech"], textToSpeechStream: vi.fn() as unknown as PluginRuntime["tts"]["textToSpeechStream"], textToSpeechTelephony: vi.fn() as unknown as PluginRuntime["tts"]["textToSpeechTelephony"], diff --git a/src/plugin-sdk/tts-runtime.types.ts b/src/plugin-sdk/tts-runtime.types.ts index 26c8f84e2cdf..e39c3337d25d 100644 --- a/src/plugin-sdk/tts-runtime.types.ts +++ b/src/plugin-sdk/tts-runtime.types.ts @@ -1,6 +1,11 @@ // TTS runtime types define plugin-facing text-to-speech synthesis hooks and results. import type { OpenClawConfig } from "../config/types.openclaw.js"; -import type { ResolvedTtsPersona, TtsAutoMode, TtsProvider } from "../config/types.tts.js"; +import type { + ResolvedTtsPersona, + TtsAutoMode, + TtsConfig, + TtsProvider, +} from "../config/types.tts.js"; import type { SpeechProviderConfig, SpeechVoiceOption, @@ -94,6 +99,22 @@ export type TtsRequestParams = { accountId?: string; }; +/** Inputs for surface-specific config merge and directive pre-resolution. */ +export type PrepareTtsRequestParams = { + cfg: OpenClawConfig; + override?: TtsConfig; + text: string; +}; + +/** Effective synthesis inputs returned before choosing file, stream, or telephony output. */ +export type PreparedTtsRequest = { + cfg: OpenClawConfig; + directives: TtsDirectiveParseResult; +}; + +/** Shared surface-specific TTS request preparation contract. */ +export type PrepareTtsRequest = (params: PrepareTtsRequestParams) => Promise; + /** Telephony-specific synthesis request where output format is constrained by the caller. */ export type TtsTelephonyRequestParams = { text: string; diff --git a/src/plugins/runtime/index.ts b/src/plugins/runtime/index.ts index 4030f65e3b0a..d1b8ce260556 100644 --- a/src/plugins/runtime/index.ts +++ b/src/plugins/runtime/index.ts @@ -37,6 +37,7 @@ import { createRuntimeTasks } from "./runtime-tasks.js"; import type { CreatePluginRuntimeOptions, PluginRuntime } from "./types.js"; const loadTtsRuntime = createLazyRuntimeModule(() => import("../../plugin-sdk/tts-runtime.js")); +const loadTtsRequestRuntime = createLazyRuntimeModule(() => import("./runtime-tts-request.js")); const loadMediaUnderstandingRuntime = createLazyRuntimeModule( () => import("../../media-understanding/runtime.js"), ); @@ -62,7 +63,9 @@ function createRuntimeGateway(): PluginRuntime["gateway"] { function createRuntimeTts(): PluginRuntime["tts"] { const bindTtsRuntime = createLazyRuntimeMethodBinder(loadTtsRuntime); + const bindTtsRequestRuntime = createLazyRuntimeMethodBinder(loadTtsRequestRuntime); return { + prepareTtsRequest: bindTtsRequestRuntime((runtime) => runtime.prepareTtsRequest), textToSpeech: bindTtsRuntime((runtime) => runtime.textToSpeech), textToSpeechStream: bindTtsRuntime((runtime) => runtime.textToSpeechStream), textToSpeechTelephony: bindTtsRuntime((runtime) => runtime.textToSpeechTelephony), diff --git a/src/plugins/runtime/runtime-tts-request.ts b/src/plugins/runtime/runtime-tts-request.ts new file mode 100644 index 000000000000..e735cd4a6a25 --- /dev/null +++ b/src/plugins/runtime/runtime-tts-request.ts @@ -0,0 +1,2 @@ +// Lazy runtime bridge for speech-core request pre-resolution. +export { prepareTtsRequest } from "../../../packages/speech-core/runtime-api.js"; diff --git a/src/plugins/runtime/types-core.ts b/src/plugins/runtime/types-core.ts index e79025d04600..c04120aafbbb 100644 --- a/src/plugins/runtime/types-core.ts +++ b/src/plugins/runtime/types-core.ts @@ -8,6 +8,7 @@ import type { LogLevel } from "../../logging/levels.js"; import type { MediaUnderstandingRuntime } from "../../media-understanding/runtime-types.js"; import type { ListSpeechVoices, + PrepareTtsRequest, TextToSpeech, TextToSpeechStream, TextToSpeechTelephony, @@ -361,6 +362,7 @@ export type PluginRuntimeCore = { resizeToJpeg: typeof import("../../media/media-services.js").resizeToJpeg; }; tts: { + prepareTtsRequest: PrepareTtsRequest; textToSpeech: TextToSpeech; textToSpeechStream: TextToSpeechStream; textToSpeechTelephony: TextToSpeechTelephony;