diff --git a/packages/speech-core/src/tts-streaming.ts b/packages/speech-core/src/tts-streaming.ts index 737d212c9583..ad8092680f61 100644 --- a/packages/speech-core/src/tts-streaming.ts +++ b/packages/speech-core/src/tts-streaming.ts @@ -1,21 +1,9 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import { logVerbose } from "openclaw/plugin-sdk/runtime-env"; import type { TtsDirectiveOverrides } from "openclaw/plugin-sdk/speech-core"; import { assertSpeechRuntimeAvailable } from "./runtime-availability.js"; -import { resolveSpeechProviderTimeoutMs } from "./tts-provider-resolution.js"; -import { - buildTtsFailureResult, - formatTtsProviderError, - prepareSpeechSynthesis, - resolvePersonaBinding, - resolveReadySpeechProvider, - resolveTtsRequestSetup, - resolveTtsResultModel, - resolveTtsResultVoice, - sanitizeTtsErrorForLog, -} from "./tts-synthesis-support.js"; +import { executeTtsProviderAttempts, resolveTtsRequestSetup } from "./tts-synthesis-support.js"; import { resolveTtsSynthesisTarget } from "./tts-synthesis.js"; -import type { TtsProviderAttempt, TtsStreamResult, TtsSynthesisStreamResult } from "./tts-types.js"; +import type { TtsStreamResult, TtsSynthesisStreamResult } from "./tts-types.js"; export async function streamSpeech(params: { text: string; @@ -45,136 +33,48 @@ export async function streamSpeech(params: { const { cfg, config, persona, providers } = setup; const target = resolveTtsSynthesisTarget(params.channel); - const errors: string[] = []; - const attemptedProviders: string[] = []; - const attempts: TtsProviderAttempt[] = []; - const primaryProvider = providers[0]?.provider; - logVerbose( - `TTS stream: starting with provider ${primaryProvider}, fallbacks: ${ - providers - .slice(1) - .map((entry) => entry.provider) - .join(", ") || "none" - }`, - ); - - for (const { provider, voiceModel } of providers) { - attemptedProviders.push(provider); - const providerStart = Date.now(); - try { - const resolvedProvider = resolveReadySpeechProvider({ - provider, - cfg, - config, - persona, - voiceModel, - }); - if (resolvedProvider.kind === "skip") { - errors.push(resolvedProvider.message); - attempts.push({ - provider, - outcome: "skipped", - reasonCode: resolvedProvider.reasonCode, - persona: persona?.id, - ...(resolvedProvider.personaBinding - ? { personaBinding: resolvedProvider.personaBinding } - : {}), - error: resolvedProvider.message, - }); - logVerbose(`TTS stream: provider ${provider} skipped (${resolvedProvider.message})`); - continue; - } + return await executeTtsProviderAttempts({ + cfg, + config, + persona, + providers, + synthesisText: params.text, + providerOverrides: params.overrides?.providerOverrides, + timeoutMs: params.timeoutMs, + target, + logLabel: "TTS stream", + selectOperation: ({ provider, resolvedProvider }) => { if (!resolvedProvider.provider.streamSynthesize) { - const message = `${provider} does not support streaming TTS`; - errors.push(message); - attempts.push({ - provider, - outcome: "skipped", + return { + kind: "skip", reasonCode: "unsupported_for_streaming", - persona: persona?.id, - personaBinding: resolvedProvider.personaBinding, - error: message, - }); - logVerbose(`TTS stream: provider ${provider} skipped (${message})`); - continue; + message: `${provider} does not support streaming TTS`, + }; } - const timeoutMs = resolveSpeechProviderTimeoutMs({ - timeoutMs: params.timeoutMs ?? voiceModel?.timeoutMs, - config, - provider: resolvedProvider.provider, - }); - const prepared = await prepareSpeechSynthesis({ - provider: resolvedProvider.provider, - text: params.text, - cfg, - providerConfig: resolvedProvider.providerConfig, - providerOverrides: params.overrides?.providerOverrides?.[resolvedProvider.provider.id], - persona: resolvedProvider.synthesisPersona, - personaProviderConfig: resolvedProvider.personaProviderConfig, - target, - timeoutMs, - }); - const synthesis = await resolvedProvider.provider.streamSynthesize({ - text: prepared.text, - cfg, - providerConfig: prepared.providerConfig, - target, - providerOverrides: prepared.providerOverrides, - timeoutMs, - }); - const latencyMs = Date.now() - providerStart; - attempts.push({ - provider, - outcome: "success", - reasonCode: "success", - persona: persona?.id, - personaBinding: resolvedProvider.personaBinding, - latencyMs, - }); return { - success: true, - audioStream: synthesis.audioStream, - latencyMs, - provider, - providerModel: resolveTtsResultModel(prepared.providerConfig, prepared.providerOverrides), - providerVoice: resolveTtsResultVoice(prepared.providerConfig, prepared.providerOverrides), - persona: persona?.id, - fallbackFrom: provider !== primaryProvider ? primaryProvider : undefined, - attemptedProviders, - attempts, - outputFormat: synthesis.outputFormat, - voiceCompatible: synthesis.voiceCompatible, - fileExtension: synthesis.fileExtension, - target, - release: synthesis.release, + kind: "ready", + synthesize: ({ prepared, cfg: runtimeCfg, target: synthesisTarget, timeoutMs }) => + resolvedProvider.provider.streamSynthesize!({ + text: prepared.text, + cfg: runtimeCfg, + providerConfig: prepared.providerConfig, + target: synthesisTarget, + providerOverrides: prepared.providerOverrides, + timeoutMs, + }), }; - } catch (err) { - const errorMsg = formatTtsProviderError(provider, err); - const latencyMs = Date.now() - providerStart; - errors.push(errorMsg); - attempts.push({ - provider, - outcome: "failed", - reasonCode: - err instanceof Error && err.name === "AbortError" ? "timeout" : "provider_error", - latencyMs, - persona: persona?.id, - personaBinding: resolvePersonaBinding(persona, provider), - error: errorMsg, - }); - const rawError = sanitizeTtsErrorForLog(err); - if (provider === primaryProvider) { - const hasFallbacks = providers.length > 1; - logVerbose( - `TTS stream: primary provider ${provider} failed (${rawError})${hasFallbacks ? "; trying fallback providers." : "; no fallback providers configured."}`, - ); - } else { - logVerbose(`TTS stream: ${provider} failed (${rawError}); trying next provider.`); - } - } - } - - return buildTtsFailureResult(errors, attemptedProviders, attempts, persona?.id); + }, + buildSuccess: ({ synthesis, ...metadata }) => ({ + success: true, + ...metadata, + audioStream: synthesis.audioStream, + outputFormat: synthesis.outputFormat, + voiceCompatible: synthesis.voiceCompatible, + fileExtension: synthesis.fileExtension, + target, + release: synthesis.release, + }), + }); } export async function textToSpeechStream(params: { diff --git a/packages/speech-core/src/tts-synthesis-support.ts b/packages/speech-core/src/tts-synthesis-support.ts index 39e046833a96..5095db7b7c3d 100644 --- a/packages/speech-core/src/tts-synthesis-support.ts +++ b/packages/speech-core/src/tts-synthesis-support.ts @@ -5,6 +5,7 @@ import type { } from "openclaw/plugin-sdk/config-contracts"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { redactSensitiveText } from "openclaw/plugin-sdk/logging-core"; +import { logVerbose } from "openclaw/plugin-sdk/runtime-env"; import { canonicalizeSpeechProviderId, getSpeechProvider, @@ -43,7 +44,7 @@ export function sanitizeTtsErrorForLog(err: unknown): string { return redactSensitiveText(raw).replace(/\r/g, "\\r").replace(/\n/g, "\\n").replace(/\t/g, "\\t"); } -export function buildTtsFailureResult( +function buildTtsFailureResult( errors: string[], attemptedProviders?: string[], attempts?: TtsProviderAttempt[], @@ -80,7 +81,7 @@ type TtsProviderReadyResolution = personaBinding?: "missing"; }; -export function resolveReadySpeechProvider(params: { +function resolveReadySpeechProvider(params: { provider: TtsProvider; cfg: OpenClawConfig; config: ResolvedTtsConfig; @@ -151,7 +152,7 @@ export function resolveReadySpeechProvider(params: { }; } -export async function prepareSpeechSynthesis(params: { +async function prepareSpeechSynthesis(params: { provider: NonNullable>; text: string; cfg: OpenClawConfig; @@ -238,11 +239,187 @@ export function resolveTtsRequestSetup(params: { }; } +type ReadySpeechProvider = Extract; +type PreparedSpeechSynthesis = Awaited>; +type TtsProviderOperation = + | { + kind: "ready"; + synthesize: (params: { + prepared: PreparedSpeechSynthesis; + cfg: OpenClawConfig; + target: "audio-file" | "voice-note" | "telephony"; + timeoutMs: number; + }) => Promise; + } + | { + kind: "skip"; + reasonCode: TtsProviderAttempt["reasonCode"]; + message: string; + }; +type TtsProviderSuccess = { + synthesis: TSynthesis; + latencyMs: number; + provider: string; + providerModel?: string; + providerVoice?: string; + persona?: string; + fallbackFrom?: string; + attemptedProviders: string[]; + attempts: TtsProviderAttempt[]; +}; + +export async function executeTtsProviderAttempts(params: { + cfg: OpenClawConfig; + config: ResolvedTtsConfig; + persona?: ResolvedTtsPersona; + providers: VoiceProviderCandidate[]; + synthesisText: string; + providerOverrides?: Record; + timeoutMs?: number; + target: "audio-file" | "voice-note" | "telephony"; + logLabel: string; + requireTelephony?: boolean; + selectOperation: (params: { + provider: TtsProvider; + resolvedProvider: ReadySpeechProvider; + }) => TtsProviderOperation; + buildSuccess: (params: TtsProviderSuccess) => TResult; +}): Promise> { + const { cfg, config, persona, providers } = params; + const errors: string[] = []; + const attemptedProviders: string[] = []; + const attempts: TtsProviderAttempt[] = []; + const primaryProvider = providers[0]?.provider; + logVerbose( + `${params.logLabel}: starting with provider ${primaryProvider}, fallbacks: ${ + providers + .slice(1) + .map((entry) => entry.provider) + .join(", ") || "none" + }`, + ); + + for (const { provider, voiceModel } of providers) { + attemptedProviders.push(provider); + const providerStart = Date.now(); + try { + const resolvedProvider = resolveReadySpeechProvider({ + provider, + cfg, + config, + persona, + voiceModel, + requireTelephony: params.requireTelephony, + }); + if (resolvedProvider.kind === "skip") { + errors.push(resolvedProvider.message); + attempts.push({ + provider, + outcome: "skipped", + reasonCode: resolvedProvider.reasonCode, + persona: persona?.id, + ...(resolvedProvider.personaBinding + ? { personaBinding: resolvedProvider.personaBinding } + : {}), + error: resolvedProvider.message, + }); + logVerbose( + `${params.logLabel}: provider ${provider} skipped (${resolvedProvider.message})`, + ); + continue; + } + + const operation = params.selectOperation({ provider, resolvedProvider }); + if (operation.kind === "skip") { + errors.push(operation.message); + attempts.push({ + provider, + outcome: "skipped", + reasonCode: operation.reasonCode, + persona: persona?.id, + personaBinding: resolvedProvider.personaBinding, + error: operation.message, + }); + logVerbose(`${params.logLabel}: provider ${provider} skipped (${operation.message})`); + continue; + } + + const timeoutMs = resolveSpeechProviderTimeoutMs({ + timeoutMs: params.timeoutMs ?? voiceModel?.timeoutMs, + config, + provider: resolvedProvider.provider, + }); + const prepared = await prepareSpeechSynthesis({ + provider: resolvedProvider.provider, + text: params.synthesisText, + cfg, + providerConfig: resolvedProvider.providerConfig, + providerOverrides: params.providerOverrides?.[resolvedProvider.provider.id], + persona: resolvedProvider.synthesisPersona, + personaProviderConfig: resolvedProvider.personaProviderConfig, + target: params.target, + timeoutMs, + }); + const synthesis = await operation.synthesize({ + prepared, + cfg, + target: params.target, + timeoutMs, + }); + const latencyMs = Date.now() - providerStart; + attempts.push({ + provider, + outcome: "success", + reasonCode: "success", + persona: persona?.id, + personaBinding: resolvedProvider.personaBinding, + latencyMs, + }); + return params.buildSuccess({ + synthesis, + latencyMs, + provider, + providerModel: resolveTtsResultModel(prepared.providerConfig, prepared.providerOverrides), + providerVoice: resolveTtsResultVoice(prepared.providerConfig, prepared.providerOverrides), + persona: persona?.id, + fallbackFrom: provider !== primaryProvider ? primaryProvider : undefined, + attemptedProviders, + attempts, + }); + } catch (err) { + const errorMsg = formatTtsProviderError(provider, err); + const latencyMs = Date.now() - providerStart; + errors.push(errorMsg); + attempts.push({ + provider, + outcome: "failed", + reasonCode: + err instanceof Error && err.name === "AbortError" ? "timeout" : "provider_error", + latencyMs, + persona: persona?.id, + personaBinding: resolvePersonaBinding(persona, provider), + error: errorMsg, + }); + const rawError = sanitizeTtsErrorForLog(err); + if (provider === primaryProvider) { + const hasFallbacks = providers.length > 1; + logVerbose( + `${params.logLabel}: primary provider ${provider} failed (${rawError})${hasFallbacks ? "; trying fallback providers." : "; no fallback providers configured."}`, + ); + } else { + logVerbose(`${params.logLabel}: ${provider} failed (${rawError}); trying next provider.`); + } + } + } + + return buildTtsFailureResult(errors, attemptedProviders, attempts, persona?.id); +} + function readTtsResultString(value: unknown): string | undefined { return typeof value === "string" && value.trim() ? value.trim() : undefined; } -export function resolveTtsResultModel( +function resolveTtsResultModel( providerConfig: SpeechProviderConfig, providerOverrides?: SpeechProviderOverrides, ): string | undefined { @@ -254,7 +431,7 @@ export function resolveTtsResultModel( ); } -export function resolveTtsResultVoice( +function resolveTtsResultVoice( providerConfig: SpeechProviderConfig, providerOverrides?: SpeechProviderOverrides, ): string | undefined { @@ -272,7 +449,7 @@ export function resolveTtsResultVoice( ); } -export function resolvePersonaBinding( +function resolvePersonaBinding( persona: ResolvedTtsPersona | undefined, provider: string, ): "applied" | "missing" | "none" { diff --git a/packages/speech-core/src/tts-synthesis.ts b/packages/speech-core/src/tts-synthesis.ts index 65b8dd7f9388..d427febb89cf 100644 --- a/packages/speech-core/src/tts-synthesis.ts +++ b/packages/speech-core/src/tts-synthesis.ts @@ -6,19 +6,8 @@ import { tempWorkspaceSync, resolvePreferredOpenClawTmpDir } from "openclaw/plug import { scheduleCleanup, type TtsDirectiveOverrides } from "openclaw/plugin-sdk/speech-core"; import { assertSpeechRuntimeAvailable } from "./runtime-availability.js"; import { normalizeSpeechText } from "./speech-text.js"; -import { resolveSpeechProviderTimeoutMs } from "./tts-provider-resolution.js"; -import { - buildTtsFailureResult, - formatTtsProviderError, - prepareSpeechSynthesis, - resolvePersonaBinding, - resolveReadySpeechProvider, - resolveTtsRequestSetup, - resolveTtsResultModel, - resolveTtsResultVoice, - sanitizeTtsErrorForLog, -} from "./tts-synthesis-support.js"; -import type { TtsProviderAttempt, TtsResult, TtsSynthesisResult } from "./tts-types.js"; +import { executeTtsProviderAttempts, resolveTtsRequestSetup } from "./tts-synthesis-support.js"; +import type { TtsResult, TtsSynthesisResult } from "./tts-types.js"; export function supportsNativeVoiceNoteTts(channel: string | undefined): boolean { return resolveChannelTtsVoiceDelivery(channel) !== undefined; @@ -214,122 +203,37 @@ export async function synthesizeSpeech(params: { } const { cfg, config, persona, providers } = setup; - const textForSynthesis = normalizeSpeechText(params.text); const target = resolveTtsSynthesisTarget(params.channel); - - const errors: string[] = []; - const attemptedProviders: string[] = []; - const attempts: TtsProviderAttempt[] = []; - const primaryProvider = providers[0]?.provider; - logVerbose( - `TTS: starting with provider ${primaryProvider}, fallbacks: ${ - providers - .slice(1) - .map((entry) => entry.provider) - .join(", ") || "none" - }`, - ); - - for (const { provider, voiceModel } of providers) { - attemptedProviders.push(provider); - const providerStart = Date.now(); - try { - const resolvedProvider = resolveReadySpeechProvider({ - provider, - cfg, - config, - persona, - voiceModel, - }); - if (resolvedProvider.kind === "skip") { - errors.push(resolvedProvider.message); - attempts.push({ - provider, - outcome: "skipped", - reasonCode: resolvedProvider.reasonCode, - persona: persona?.id, - ...(resolvedProvider.personaBinding - ? { personaBinding: resolvedProvider.personaBinding } - : {}), - error: resolvedProvider.message, - }); - logVerbose(`TTS: provider ${provider} skipped (${resolvedProvider.message})`); - continue; - } - const timeoutMs = resolveSpeechProviderTimeoutMs({ - timeoutMs: params.timeoutMs ?? voiceModel?.timeoutMs, - config, - provider: resolvedProvider.provider, - }); - const prepared = await prepareSpeechSynthesis({ - provider: resolvedProvider.provider, - text: textForSynthesis, - cfg, - providerConfig: resolvedProvider.providerConfig, - providerOverrides: params.overrides?.providerOverrides?.[resolvedProvider.provider.id], - persona: resolvedProvider.synthesisPersona, - personaProviderConfig: resolvedProvider.personaProviderConfig, - target, - timeoutMs, - }); - const synthesis = await resolvedProvider.provider.synthesize({ - text: prepared.text, - cfg, - providerConfig: prepared.providerConfig, - target, - providerOverrides: prepared.providerOverrides, - timeoutMs, - }); - const latencyMs = Date.now() - providerStart; - attempts.push({ - provider, - outcome: "success", - reasonCode: "success", - persona: persona?.id, - personaBinding: resolvedProvider.personaBinding, - latencyMs, - }); - return { - success: true, - audioBuffer: synthesis.audioBuffer, - latencyMs, - provider, - providerModel: resolveTtsResultModel(prepared.providerConfig, prepared.providerOverrides), - providerVoice: resolveTtsResultVoice(prepared.providerConfig, prepared.providerOverrides), - persona: persona?.id, - fallbackFrom: provider !== primaryProvider ? primaryProvider : undefined, - attemptedProviders, - attempts, - outputFormat: synthesis.outputFormat, - voiceCompatible: synthesis.voiceCompatible, - fileExtension: synthesis.fileExtension, - target, - }; - } catch (err) { - const errorMsg = formatTtsProviderError(provider, err); - const latencyMs = Date.now() - providerStart; - errors.push(errorMsg); - attempts.push({ - provider, - outcome: "failed", - reasonCode: - err instanceof Error && err.name === "AbortError" ? "timeout" : "provider_error", - latencyMs, - persona: persona?.id, - personaBinding: resolvePersonaBinding(persona, provider), - error: errorMsg, - }); - const rawError = sanitizeTtsErrorForLog(err); - if (provider === primaryProvider) { - const hasFallbacks = providers.length > 1; - logVerbose( - `TTS: primary provider ${provider} failed (${rawError})${hasFallbacks ? "; trying fallback providers." : "; no fallback providers configured."}`, - ); - } else { - logVerbose(`TTS: ${provider} failed (${rawError}); trying next provider.`); - } - } - } - - return buildTtsFailureResult(errors, attemptedProviders, attempts, persona?.id); + return await executeTtsProviderAttempts({ + cfg, + config, + persona, + providers, + synthesisText: normalizeSpeechText(params.text), + providerOverrides: params.overrides?.providerOverrides, + timeoutMs: params.timeoutMs, + target, + logLabel: "TTS", + selectOperation: ({ resolvedProvider }) => ({ + kind: "ready", + synthesize: ({ prepared, cfg: runtimeCfg, target: synthesisTarget, timeoutMs }) => + resolvedProvider.provider.synthesize({ + text: prepared.text, + cfg: runtimeCfg, + providerConfig: prepared.providerConfig, + target: synthesisTarget, + providerOverrides: prepared.providerOverrides, + timeoutMs, + }), + }), + buildSuccess: ({ synthesis, ...metadata }) => ({ + success: true, + ...metadata, + audioBuffer: synthesis.audioBuffer, + outputFormat: synthesis.outputFormat, + voiceCompatible: synthesis.voiceCompatible, + fileExtension: synthesis.fileExtension, + target, + }), + }); } diff --git a/packages/speech-core/src/tts-telephony.ts b/packages/speech-core/src/tts-telephony.ts index 9dfcc940e75b..76e00fe937fc 100644 --- a/packages/speech-core/src/tts-telephony.ts +++ b/packages/speech-core/src/tts-telephony.ts @@ -1,20 +1,8 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import { logVerbose } from "openclaw/plugin-sdk/runtime-env"; import type { TtsDirectiveOverrides } from "openclaw/plugin-sdk/speech-core"; import { assertSpeechRuntimeAvailable } from "./runtime-availability.js"; -import { resolveSpeechProviderTimeoutMs } from "./tts-provider-resolution.js"; -import { - buildTtsFailureResult, - formatTtsProviderError, - prepareSpeechSynthesis, - resolvePersonaBinding, - resolveReadySpeechProvider, - resolveTtsRequestSetup, - resolveTtsResultModel, - resolveTtsResultVoice, - sanitizeTtsErrorForLog, -} from "./tts-synthesis-support.js"; -import type { TtsProviderAttempt, TtsTelephonyResult } from "./tts-types.js"; +import { executeTtsProviderAttempts, resolveTtsRequestSetup } from "./tts-synthesis-support.js"; +import type { TtsTelephonyResult } from "./tts-types.js"; export async function textToSpeechTelephony(params: { text: string; @@ -35,121 +23,39 @@ export async function textToSpeechTelephony(params: { } const { cfg, config, persona, providers } = setup; - const errors: string[] = []; - const attemptedProviders: string[] = []; - const attempts: TtsProviderAttempt[] = []; - const primaryProvider = providers[0]?.provider; - logVerbose( - `TTS telephony: starting with provider ${primaryProvider}, fallbacks: ${ - providers - .slice(1) - .map((entry) => entry.provider) - .join(", ") || "none" - }`, - ); - - for (const { provider, voiceModel } of providers) { - attemptedProviders.push(provider); - const providerStart = Date.now(); - try { - const resolvedProvider = resolveReadySpeechProvider({ - provider, - cfg, - config, - persona, - voiceModel, - requireTelephony: true, - }); - if (resolvedProvider.kind === "skip") { - errors.push(resolvedProvider.message); - attempts.push({ - provider, - outcome: "skipped", - reasonCode: resolvedProvider.reasonCode, - persona: persona?.id, - ...(resolvedProvider.personaBinding - ? { personaBinding: resolvedProvider.personaBinding } - : {}), - error: resolvedProvider.message, - }); - logVerbose(`TTS telephony: provider ${provider} skipped (${resolvedProvider.message})`); - continue; - } - const timeoutMs = resolveSpeechProviderTimeoutMs({ - timeoutMs: params.timeoutMs ?? voiceModel?.timeoutMs, - config, - provider: resolvedProvider.provider, - }); + return await executeTtsProviderAttempts({ + cfg, + config, + persona, + providers, + synthesisText: params.text, + providerOverrides: params.overrides?.providerOverrides, + timeoutMs: params.timeoutMs, + target: "telephony", + logLabel: "TTS telephony", + requireTelephony: true, + selectOperation: ({ resolvedProvider }) => { const synthesizeTelephony = resolvedProvider.provider.synthesizeTelephony as NonNullable< typeof resolvedProvider.provider.synthesizeTelephony >; - const prepared = await prepareSpeechSynthesis({ - provider: resolvedProvider.provider, - text: params.text, - cfg, - providerConfig: resolvedProvider.providerConfig, - providerOverrides: params.overrides?.providerOverrides?.[resolvedProvider.provider.id], - persona: resolvedProvider.synthesisPersona, - personaProviderConfig: resolvedProvider.personaProviderConfig, - target: "telephony", - timeoutMs, - }); - const synthesis = await synthesizeTelephony({ - text: prepared.text, - cfg, - providerConfig: prepared.providerConfig, - providerOverrides: prepared.providerOverrides, - timeoutMs, - }); - const latencyMs = Date.now() - providerStart; - attempts.push({ - provider, - outcome: "success", - reasonCode: "success", - persona: persona?.id, - personaBinding: resolvedProvider.personaBinding, - latencyMs, - }); - return { - success: true, - audioBuffer: synthesis.audioBuffer, - latencyMs, - provider, - providerModel: resolveTtsResultModel(prepared.providerConfig, prepared.providerOverrides), - providerVoice: resolveTtsResultVoice(prepared.providerConfig, prepared.providerOverrides), - persona: persona?.id, - fallbackFrom: provider !== primaryProvider ? primaryProvider : undefined, - attemptedProviders, - attempts, - outputFormat: synthesis.outputFormat, - sampleRate: synthesis.sampleRate, + kind: "ready", + synthesize: ({ prepared, cfg: runtimeCfg, timeoutMs }) => + synthesizeTelephony({ + text: prepared.text, + cfg: runtimeCfg, + providerConfig: prepared.providerConfig, + providerOverrides: prepared.providerOverrides, + timeoutMs, + }), }; - } catch (err) { - const errorMsg = formatTtsProviderError(provider, err); - const latencyMs = Date.now() - providerStart; - errors.push(errorMsg); - attempts.push({ - provider, - outcome: "failed", - reasonCode: - err instanceof Error && err.name === "AbortError" ? "timeout" : "provider_error", - latencyMs, - persona: persona?.id, - personaBinding: resolvePersonaBinding(persona, provider), - error: errorMsg, - }); - const rawError = sanitizeTtsErrorForLog(err); - if (provider === primaryProvider) { - const hasFallbacks = providers.length > 1; - logVerbose( - `TTS telephony: primary provider ${provider} failed (${rawError})${hasFallbacks ? "; trying fallback providers." : "; no fallback providers configured."}`, - ); - } else { - logVerbose(`TTS telephony: ${provider} failed (${rawError}); trying next provider.`); - } - } - } - - return buildTtsFailureResult(errors, attemptedProviders, attempts, persona?.id); + }, + buildSuccess: ({ synthesis, ...metadata }) => ({ + success: true, + ...metadata, + audioBuffer: synthesis.audioBuffer, + outputFormat: synthesis.outputFormat, + sampleRate: synthesis.sampleRate, + }), + }); } diff --git a/packages/speech-core/src/tts.test.ts b/packages/speech-core/src/tts.test.ts index 2da23566e011..0a665d6c2251 100644 --- a/packages/speech-core/src/tts.test.ts +++ b/packages/speech-core/src/tts.test.ts @@ -127,6 +127,7 @@ const { setTtsMaxLength, synthesizeSpeech, textToSpeech, + textToSpeechStream, textToSpeechTelephony, } = await import("../runtime-api.js"); @@ -835,6 +836,119 @@ describe("speech-core native voice-note routing", () => { ]); }); + it("skips non-streaming providers before using a streaming fallback", async () => { + const release = vi.fn(async () => {}); + const streamSynthesize = vi.fn(async () => ({ + audioStream: new ReadableStream({ + start(controller) { + controller.close(); + }, + }), + fileExtension: ".pcm", + outputFormat: "pcm", + voiceCompatible: false, + release, + })); + installSpeechProviders([ + createMockSpeechProvider("buffered", { autoSelectOrder: 1 }), + createMockSpeechProvider("streaming", { + autoSelectOrder: 2, + streamSynthesize, + }), + ]); + + const result = await textToSpeechStream({ + text: "Use streaming fallback.", + cfg: { + tts: { + enabled: true, + provider: "buffered", + prefsPath: "/tmp/openclaw-speech-core-streaming-fallback-test.json", + }, + } as OpenClawConfig, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("streaming"); + expect(result.fallbackFrom).toBe("buffered"); + expect(result.attemptedProviders).toEqual(["buffered", "streaming"]); + expect(result.outputFormat).toBe("pcm"); + expect(result.fileExtension).toBe(".pcm"); + expect(result.target).toBe("audio-file"); + expect(result.release).toBe(release); + const skippedAttempt = requireAttempt(result.attempts, 0); + expect(skippedAttempt).toMatchObject({ + provider: "buffered", + outcome: "skipped", + reasonCode: "unsupported_for_streaming", + personaBinding: "none", + error: "buffered does not support streaming TTS", + }); + expect(skippedAttempt).not.toHaveProperty("latencyMs"); + expect(requireAttempt(result.attempts, 1)).toMatchObject({ + provider: "streaming", + outcome: "success", + reasonCode: "success", + }); + expect(streamSynthesize).toHaveBeenCalledOnce(); + }); + + it("classifies streaming timeouts before falling back with raw text", async () => { + const timeoutStreamSynthesize = vi.fn(async () => { + const error = new Error("stalled"); + error.name = "AbortError"; + throw error; + }); + const fallbackStreamSynthesize = vi.fn(async () => ({ + audioStream: new ReadableStream({ + start(controller) { + controller.close(); + }, + }), + fileExtension: ".pcm", + outputFormat: "pcm", + voiceCompatible: false, + })); + installSpeechProviders([ + createMockSpeechProvider("primary", { + autoSelectOrder: 1, + streamSynthesize: timeoutStreamSynthesize, + }), + createMockSpeechProvider("fallback", { + autoSelectOrder: 2, + streamSynthesize: fallbackStreamSynthesize, + }), + ]); + const text = "## Keep [streaming Markdown](https://example.com) raw!!!!!"; + + const result = await textToSpeechStream({ + text, + cfg: { + tts: { + enabled: true, + provider: "primary", + prefsPath: "/tmp/openclaw-speech-core-streaming-timeout-test.json", + }, + } as OpenClawConfig, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("fallback"); + expect(result.fallbackFrom).toBe("primary"); + expect(requireAttempt(result.attempts, 0)).toMatchObject({ + provider: "primary", + outcome: "failed", + reasonCode: "timeout", + error: "primary: request timed out", + }); + expect(requireAttempt(result.attempts, 1)).toMatchObject({ + provider: "fallback", + outcome: "success", + reasonCode: "success", + }); + expect(fallbackStreamSynthesize).toHaveBeenCalledWith(expect.objectContaining({ text })); + }); + it("ignores voiceModel refs that are not speech models", async () => { installSpeechProviders([ createMockSpeechProvider("openai", { @@ -1419,8 +1533,9 @@ describe("speech-core native voice-note routing", () => { }), ]); + const text = "## Keep [telephony Markdown](https://example.com) raw!!!!!"; const result = await textToSpeechTelephony({ - text: "Use a directed telephony voice.", + text, cfg: { tts: { enabled: true, @@ -1455,6 +1570,8 @@ describe("speech-core native voice-note routing", () => { speakerVoice: "directed-voice", speed: 1.5, }); + expect(telephonyRequest.text).toBe(text); + expect(telephonyRequest).not.toHaveProperty("target"); }); it("uses provider defaults when fallback policy allows missing persona bindings", async () => {