diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index 63c85bd1d2f4..2909224de28d 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -364,7 +364,6 @@ packages/model-catalog-core/src/model-catalog-normalize.ts packages/sdk/src/client.ts packages/sdk/src/index.test.ts packages/speech-core/src/tts.test.ts -packages/speech-core/src/tts.ts packages/tool-call-repair/src/stream-normalizer.test.ts packages/tool-call-repair/src/stream-normalizer.ts src/acp/control-plane/manager.test.ts diff --git a/packages/speech-core/src/tts-payload.ts b/packages/speech-core/src/tts-payload.ts new file mode 100644 index 000000000000..9eda0c20ebdd --- /dev/null +++ b/packages/speech-core/src/tts-payload.ts @@ -0,0 +1,270 @@ +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { + markReplyPayloadAsTtsSupplement, + resolveSendableOutboundReplyParts, + type ReplyPayload, +} from "openclaw/plugin-sdk/reply-payload"; +import { isVerbose, logVerbose } from "openclaw/plugin-sdk/runtime-env"; +import { + canonicalizeSpeechProviderId, + getSpeechProvider, + parseTtsDirectives, + summarizeText, + type SpeechVoiceOption, +} from "openclaw/plugin-sdk/speech-core"; +import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; +import { assertSpeechRuntimeAvailable, isSpeechRuntimeAvailable } from "./runtime-availability.js"; +import { isCodeHeavySpeechText, normalizeSpeechText } from "./speech-text.js"; +import { + getResolvedSpeechProviderConfig, + resolveSpeechProviderTimeoutMs, + resolveTtsProvider, +} from "./tts-provider-resolution.js"; +import { + getTtsMaxLength, + isSummarizationEnabled, + resolveTtsConfig, + resolveTtsRuntimeConfig, + resolveTtsSettingsSnapshot, + type ResolvedTtsConfig, +} from "./tts-settings.js"; +import { textToSpeech } from "./tts-synthesis.js"; +import type { TtsStatusEntry } from "./tts-types.js"; + +let lastTtsAttempt: TtsStatusEntry | undefined; + +export function getLastTtsAttempt(): TtsStatusEntry | undefined { + return lastTtsAttempt; +} + +export function setLastTtsAttempt(entry: TtsStatusEntry | undefined): void { + lastTtsAttempt = entry; +} + +export async function listSpeechVoices(params: { + provider: string; + cfg?: OpenClawConfig; + config?: ResolvedTtsConfig; + apiKey?: string; + baseUrl?: string; +}): Promise { + assertSpeechRuntimeAvailable(); + const cfg = params.cfg ? resolveTtsRuntimeConfig(params.cfg) : undefined; + const provider = canonicalizeSpeechProviderId(params.provider, cfg); + if (!provider) { + throw new Error("speech provider id is required"); + } + const config = params.config ?? (cfg ? resolveTtsConfig(cfg) : undefined); + if (!config) { + throw new Error(`speech provider ${provider} requires cfg or resolved config`); + } + const resolvedProvider = getSpeechProvider(provider, cfg); + if (!resolvedProvider) { + throw new Error(`speech provider ${provider} is not registered`); + } + if (!resolvedProvider.listVoices) { + throw new Error(`speech provider ${provider} does not support voice listing`); + } + const timeoutMs = resolveSpeechProviderTimeoutMs({ + config, + provider: resolvedProvider, + }); + return await resolvedProvider.listVoices({ + cfg, + providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, cfg), + apiKey: params.apiKey, + baseUrl: params.baseUrl, + timeoutMs, + }); +} + +function hasLegacyFinalMediaDirective(text: string): boolean { + return /(?:^|\n)\s*MEDIA\s*:/i.test(text); +} + +export async function maybeApplyTtsToPayload(params: { + payload: ReplyPayload; + cfg: OpenClawConfig; + channel?: string; + kind?: "tool" | "block" | "final"; + inboundAudio?: boolean; + ttsAuto?: string; + agentId?: string; + accountId?: string; +}): Promise { + if (!isSpeechRuntimeAvailable()) { + return params.payload; + } + if (params.payload.isCompactionNotice) { + return params.payload; + } + const cfg = resolveTtsRuntimeConfig(params.cfg); + const { autoMode, config, prefsPath } = resolveTtsSettingsSnapshot({ + cfg, + sessionAuto: params.ttsAuto, + agentId: params.agentId, + channelId: params.channel, + accountId: params.accountId, + }); + if (autoMode === "off") { + return params.payload; + } + const activeProvider = resolveTtsProvider(config, prefsPath); + + const reply = resolveSendableOutboundReplyParts(params.payload); + const text = reply.text; + const directives = parseTtsDirectives(text, config.modelOverrides, { + cfg, + providerConfigs: config.providerConfigs, + preferredProviderId: activeProvider, + }); + if (directives.warnings.length > 0) { + logVerbose(`TTS: ignored directive overrides (${directives.warnings.join("; ")})`); + } + + if (isVerbose()) { + const effectiveProvider = directives.overrides?.provider + ? (canonicalizeSpeechProviderId(directives.overrides.provider, cfg) ?? activeProvider) + : activeProvider; + logVerbose( + `TTS: auto mode enabled (${autoMode}), channel=${params.channel}, selected provider=${effectiveProvider}, config.provider=${config.provider}, config.providerSource=${config.providerSource}`, + ); + } + + const cleanedText = directives.cleanedText; + const trimmedCleaned = cleanedText.trim(); + const visibleText = trimmedCleaned.length > 0 ? trimmedCleaned : ""; + const explicitTtsText = directives.ttsText?.trim() || ""; + const ttsText = explicitTtsText || visibleText; + + const nextPayload = + visibleText === text.trim() + ? params.payload + : { + ...params.payload, + text: visibleText.length > 0 ? visibleText : undefined, + }; + + if (autoMode === "tagged" && !directives.hasDirective) { + return nextPayload; + } + if (autoMode === "inbound" && params.inboundAudio !== true) { + return nextPayload; + } + + const mode = config.mode ?? "final"; + if (mode === "final" && params.kind && params.kind !== "final") { + return nextPayload; + } + + if (!ttsText.trim()) { + return nextPayload; + } + if (reply.hasMedia || hasLegacyFinalMediaDirective(text)) { + return nextPayload; + } + if (!explicitTtsText && ttsText.trim().length < 10) { + return nextPayload; + } + + const maxLength = getTtsMaxLength(prefsPath); + let textForAudio = ttsText.trim(); + let wasSummarized = false; + + if (!explicitTtsText && isCodeHeavySpeechText(textForAudio)) { + // The visible reply already carries code-heavy detail. Skip noisy voice-note audio instead of + // telling channel users to look at a screen they may not have. + return nextPayload; + } + + if (textForAudio.length > maxLength) { + if (!isSummarizationEnabled(prefsPath)) { + logVerbose( + `TTS: truncating long text (${textForAudio.length} > ${maxLength}), summarization disabled.`, + ); + textForAudio = `${truncateUtf16Safe(textForAudio, maxLength - 3)}...`; + } else { + try { + const summary = await summarizeText({ + text: textForAudio, + targetLength: maxLength, + cfg, + config, + timeoutMs: config.timeoutMs, + }); + textForAudio = summary.summary; + wasSummarized = true; + if (textForAudio.length > config.maxTextLength) { + logVerbose( + `TTS: summary exceeded hard limit (${textForAudio.length} > ${config.maxTextLength}); truncating.`, + ); + textForAudio = `${truncateUtf16Safe(textForAudio, config.maxTextLength - 3)}...`; + } + } catch (err) { + const error = err as Error; + logVerbose(`TTS: summarization failed, truncating instead: ${error.message}`); + textForAudio = `${truncateUtf16Safe(textForAudio, maxLength - 3)}...`; + } + } + } + + const normalizedTextForAudio = normalizeSpeechText(textForAudio); + if (!normalizedTextForAudio) { + return nextPayload; + } + if (!explicitTtsText && normalizedTextForAudio.length < 10) { + return nextPayload; + } + + const ttsStart = Date.now(); + const result = await textToSpeech({ + text: textForAudio, + cfg, + prefsPath, + channel: params.channel, + overrides: directives.overrides, + agentId: params.agentId, + accountId: params.accountId, + }); + + if (result.success && result.audioPath) { + lastTtsAttempt = { + timestamp: Date.now(), + success: true, + textLength: text.length, + summarized: wasSummarized, + provider: result.provider, + persona: result.persona, + fallbackFrom: result.fallbackFrom, + attemptedProviders: result.attemptedProviders, + attempts: result.attempts, + latencyMs: result.latencyMs, + }; + + const payloadWithAudio = { + ...nextPayload, + mediaUrl: result.audioPath, + audioAsVoice: result.audioAsVoice || params.payload.audioAsVoice, + spokenText: textForAudio, + trustedLocalMedia: true, + } as ReplyPayload; + return nextPayload.text?.trim() + ? markReplyPayloadAsTtsSupplement(payloadWithAudio) + : payloadWithAudio; + } + + lastTtsAttempt = { + timestamp: Date.now(), + success: false, + textLength: text.length, + summarized: wasSummarized, + persona: result.persona, + attemptedProviders: result.attemptedProviders, + attempts: result.attempts, + error: result.error, + }; + + const latency = Date.now() - ttsStart; + logVerbose(`TTS: conversion failed after ${latency}ms (${result.error ?? "unknown"}).`); + return nextPayload; +} diff --git a/packages/speech-core/src/tts-provider-resolution.ts b/packages/speech-core/src/tts-provider-resolution.ts new file mode 100644 index 000000000000..d5c811e78bb4 --- /dev/null +++ b/packages/speech-core/src/tts-provider-resolution.ts @@ -0,0 +1,394 @@ +import type { + OpenClawConfig, + ResolvedTtsPersona, + TtsConfig, + TtsProvider, +} from "openclaw/plugin-sdk/config-contracts"; +import { clampTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime"; +import { + canonicalizeSpeechProviderId, + getSpeechProvider, + listSpeechProviders, + normalizeSpeechProviderId, + type SpeechProviderConfig, + type SpeechProviderPlugin, +} from "openclaw/plugin-sdk/speech-core"; +import { + normalizeLowercaseStringOrEmpty, + normalizeOptionalString, +} from "openclaw/plugin-sdk/string-coerce-runtime"; +import { withSpeakerSelectionCompat } from "../speaker.js"; +import { + resolvePrimaryVoiceProviderCandidate, + resolveSupportedVoiceModelRefs, + resolveVoiceModelRefs, + resolveVoiceProviderCandidates, + voiceProviderSupportsModel, + type VoiceModelProvider, + type VoiceModelRef, + type VoiceProviderCandidate, +} from "../voice-models.js"; +import { + DEFAULT_TTS_TIMEOUT_MS, + asProviderConfig, + asProviderConfigMap, + hasOwnProperty, + normalizeConfiguredSpeechProviderId, + readTtsPrefs as readPrefs, + resolveTtsPersonaFromPrefs, + resolveTtsRuntimeConfig, + type ResolvedTtsConfig, +} from "./tts-settings.js"; + +function resolvePositiveTimeoutMs(timeoutMs: number | undefined): number | undefined { + return typeof timeoutMs === "number" && Number.isFinite(timeoutMs) && timeoutMs > 0 + ? clampTimerTimeoutMs(timeoutMs) + : undefined; +} + +export function resolveSpeechProviderTimeoutMs(params: { + timeoutMs?: number; + config: ResolvedTtsConfig; + provider: Pick; +}): number { + if (params.timeoutMs !== undefined) { + return resolvePositiveTimeoutMs(params.timeoutMs) ?? params.config.timeoutMs; + } + if (params.config.timeoutMsSource !== "default") { + return resolvePositiveTimeoutMs(params.config.timeoutMs) ?? DEFAULT_TTS_TIMEOUT_MS; + } + return resolvePositiveTimeoutMs(params.provider.defaultTimeoutMs) ?? params.config.timeoutMs; +} + +function sortSpeechProvidersForAutoSelection(cfg?: OpenClawConfig) { + return listSpeechProviders(cfg).toSorted((left, right) => { + const leftOrder = left.autoSelectOrder ?? Number.MAX_SAFE_INTEGER; + const rightOrder = right.autoSelectOrder ?? Number.MAX_SAFE_INTEGER; + if (leftOrder !== rightOrder) { + return leftOrder - rightOrder; + } + return left.id.localeCompare(right.id); + }); +} + +function resolveConfiguredSpeechVoiceModelRefs(cfg: OpenClawConfig | undefined): VoiceModelRef[] { + const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : undefined; + return resolveSupportedVoiceModelRefs({ + config: effectiveCfg?.agents?.defaults?.voiceModel, + providers: sortSpeechProvidersForAutoSelection(effectiveCfg), + }); +} + +function resolveConfiguredSpeechVoiceModelForProvider(params: { + cfg: OpenClawConfig | undefined; + providerId: string; + provider?: VoiceModelProvider; + voiceModel?: VoiceModelRef; +}): VoiceModelRef | undefined { + const provider = params.provider ?? getSpeechProvider(params.providerId, params.cfg); + if (params.voiceModel) { + return voiceProviderSupportsModel(provider, params.voiceModel.model) + ? params.voiceModel + : undefined; + } + return resolveSupportedVoiceModelRefs({ + config: params.cfg?.agents?.defaults?.voiceModel, + providers: provider ? [provider] : [], + providerId: params.providerId, + })[0]; +} + +function applyVoiceModelToSpeechProviderConfig(params: { + cfg: OpenClawConfig | undefined; + providerId: string; + providerConfig: SpeechProviderConfig; + provider?: VoiceModelProvider; + voiceModel?: VoiceModelRef; +}): SpeechProviderConfig { + const voiceModel = resolveConfiguredSpeechVoiceModelForProvider({ + cfg: params.cfg, + providerId: params.providerId, + provider: params.provider, + voiceModel: params.voiceModel, + }); + if (!voiceModel) { + return params.providerConfig; + } + const hasExplicitModel = + normalizeOptionalString(params.providerConfig.model) || + normalizeOptionalString(params.providerConfig.modelId); + if (hasExplicitModel) { + return params.providerConfig; + } + return { + ...params.providerConfig, + model: voiceModel.model, + modelId: voiceModel.model, + }; +} + +export function resolvePersonaProviderConfig( + persona: ResolvedTtsPersona | undefined, + providerId: string, +): SpeechProviderConfig | undefined { + if (!persona?.providers) { + return undefined; + } + const normalized = normalizeConfiguredSpeechProviderId(providerId) ?? providerId; + if (hasOwnProperty(persona.providers, normalized)) { + return persona.providers[normalized]; + } + if (hasOwnProperty(persona.providers, providerId)) { + return persona.providers[providerId]; + } + return undefined; +} + +export function mergeProviderConfigWithPersona(params: { + providerConfig: SpeechProviderConfig; + persona?: ResolvedTtsPersona; + providerId: string; +}): { + providerConfig: SpeechProviderConfig; + personaProviderConfig?: SpeechProviderConfig; + personaBinding: "applied" | "missing" | "none"; +} { + if (!params.persona) { + return { providerConfig: params.providerConfig, personaBinding: "none" }; + } + const personaProviderConfig = resolvePersonaProviderConfig(params.persona, params.providerId); + if (!personaProviderConfig) { + return { providerConfig: params.providerConfig, personaBinding: "missing" }; + } + return { + providerConfig: { + ...params.providerConfig, + ...personaProviderConfig, + }, + personaProviderConfig, + personaBinding: "applied", + }; +} + +function resolveRawProviderConfig( + raw: TtsConfig | undefined, + providerId: string, +): SpeechProviderConfig { + if (!raw) { + return {}; + } + const rawProviders = asProviderConfigMap(raw.providers); + const direct = rawProviders[providerId] ?? (raw as Record)[providerId]; + return withSpeakerSelectionCompat(asProviderConfig(direct)); +} + +function resolveLazyProviderConfig( + config: ResolvedTtsConfig, + providerId: string, + cfg?: OpenClawConfig, + voiceModel?: VoiceModelRef, +): SpeechProviderConfig { + const canonical = + normalizeConfiguredSpeechProviderId(providerId) ?? normalizeLowercaseStringOrEmpty(providerId); + const existing = voiceModel ? undefined : config.providerConfigs[canonical]; + const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : config.sourceConfig; + if (existing && !effectiveCfg) { + return existing; + } + const rawConfig = resolveRawProviderConfig(config.rawConfig, canonical); + const rawBaseConfig = config.rawConfig as Record | undefined; + const rawProviders = asProviderConfigMap(config.rawConfig?.providers); + const resolvedProvider = getSpeechProvider(canonical, effectiveCfg); + let hasRawProviderConfig = + Object.hasOwn(rawProviders, canonical) || + (rawBaseConfig ? Object.hasOwn(rawBaseConfig, canonical) : false); + let rawProviderConfig = rawProviders[canonical] ?? rawBaseConfig?.[canonical]; + if (!hasRawProviderConfig) { + for (const alias of resolvedProvider?.aliases ?? []) { + const normalizedAlias = normalizeSpeechProviderId(alias); + if (!normalizedAlias) { + continue; + } + if (Object.hasOwn(rawProviders, normalizedAlias)) { + hasRawProviderConfig = true; + rawProviderConfig = rawProviders[normalizedAlias]; + break; + } + if (rawBaseConfig && Object.hasOwn(rawBaseConfig, normalizedAlias)) { + hasRawProviderConfig = true; + rawProviderConfig = rawBaseConfig[normalizedAlias]; + break; + } + } + } + const compatRawProviderConfig = applyVoiceModelToSpeechProviderConfig({ + cfg: effectiveCfg, + providerId: canonical, + providerConfig: withSpeakerSelectionCompat(asProviderConfig(rawProviderConfig)), + provider: resolvedProvider, + voiceModel, + }); + const shouldInjectCanonicalProviderConfig = + hasRawProviderConfig || Boolean(voiceModel) || Object.keys(rawProviders).length === 0; + const rawConfigForProvider = { + ...rawBaseConfig, + providers: shouldInjectCanonicalProviderConfig + ? { + ...rawProviders, + [canonical]: compatRawProviderConfig, + } + : rawProviders, + ...(shouldInjectCanonicalProviderConfig ? { [canonical]: compatRawProviderConfig } : {}), + }; + const next = withSpeakerSelectionCompat( + effectiveCfg && resolvedProvider?.resolveConfig + ? resolvedProvider.resolveConfig({ + cfg: effectiveCfg, + rawConfig: rawConfigForProvider, + timeoutMs: resolveSpeechProviderTimeoutMs({ config, provider: resolvedProvider }), + }) + : applyVoiceModelToSpeechProviderConfig({ + cfg: effectiveCfg, + providerId: canonical, + providerConfig: rawConfig, + provider: resolvedProvider, + voiceModel, + }), + ); + if (!voiceModel) { + config.providerConfigs[canonical] = next; + } + return next; +} + +export function getResolvedSpeechProviderConfig( + config: ResolvedTtsConfig, + providerId: string, + cfg?: OpenClawConfig, +): SpeechProviderConfig { + const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : config.sourceConfig; + const canonical = + canonicalizeSpeechProviderId(providerId, effectiveCfg) ?? + normalizeConfiguredSpeechProviderId(providerId) ?? + normalizeLowercaseStringOrEmpty(providerId); + return resolveLazyProviderConfig(config, canonical, effectiveCfg); +} + +export function getResolvedSpeechProviderConfigForVoiceModel(params: { + config: ResolvedTtsConfig; + providerId: string; + cfg: OpenClawConfig; + voiceModel?: VoiceModelRef; +}): SpeechProviderConfig { + if (!params.voiceModel) { + return getResolvedSpeechProviderConfig(params.config, params.providerId, params.cfg); + } + const effectiveCfg = resolveTtsRuntimeConfig(params.cfg); + const canonical = + canonicalizeSpeechProviderId(params.providerId, effectiveCfg) ?? + normalizeConfiguredSpeechProviderId(params.providerId) ?? + normalizeLowercaseStringOrEmpty(params.providerId); + return resolveLazyProviderConfig(params.config, canonical, effectiveCfg, params.voiceModel); +} + +export function resolveTtsProvider(config: ResolvedTtsConfig, prefsPath: string): TtsProvider { + const prefs = readPrefs(prefsPath); + const prefsProvider = + canonicalizeSpeechProviderId(prefs.tts?.provider) ?? + normalizeConfiguredSpeechProviderId(prefs.tts?.provider); + if (prefsProvider) { + return prefsProvider; + } + const activePersona = resolveTtsPersonaFromPrefs(config, prefs); + const personaProvider = + canonicalizeSpeechProviderId(activePersona?.provider, config.sourceConfig) ?? + normalizeConfiguredSpeechProviderId(activePersona?.provider); + if (personaProvider && getSpeechProvider(personaProvider, config.sourceConfig)) { + return personaProvider; + } + if (config.providerSource === "config") { + return normalizeConfiguredSpeechProviderId(config.provider) ?? config.provider; + } + const configuredVoiceProvider = resolveConfiguredSpeechVoiceModelRefs(config.sourceConfig)[0] + ?.provider; + if (configuredVoiceProvider && getSpeechProvider(configuredVoiceProvider, config.sourceConfig)) { + return configuredVoiceProvider; + } + + const effectiveCfg = config.sourceConfig; + for (const provider of sortSpeechProvidersForAutoSelection(effectiveCfg)) { + if (isTtsProviderConfigured(config, provider.id, effectiveCfg)) { + return provider.id; + } + } + return config.provider; +} + +export function resolveTtsProviderOrder(primary: TtsProvider, cfg?: OpenClawConfig): TtsProvider[] { + const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : undefined; + const normalizedPrimary = canonicalizeSpeechProviderId(primary, effectiveCfg) ?? primary; + const ordered = new Set([normalizedPrimary]); + for (const ref of resolveVoiceModelRefs(effectiveCfg?.agents?.defaults?.voiceModel)) { + const provider = canonicalizeSpeechProviderId(ref.provider, effectiveCfg) ?? ref.provider; + if (provider !== normalizedPrimary) { + ordered.add(provider); + } + } + for (const provider of sortSpeechProvidersForAutoSelection(effectiveCfg)) { + const normalized = provider.id; + if (normalized !== normalizedPrimary) { + ordered.add(normalized); + } + } + return [...ordered]; +} + +export function resolveTtsProviderCandidates( + primary: TtsProvider, + cfg?: OpenClawConfig, +): VoiceProviderCandidate[] { + const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : undefined; + const normalizedPrimary = canonicalizeSpeechProviderId(primary, effectiveCfg) ?? primary; + return resolveVoiceProviderCandidates({ + primaryProvider: normalizedPrimary, + providers: sortSpeechProvidersForAutoSelection(effectiveCfg), + voiceModelConfig: effectiveCfg?.agents?.defaults?.voiceModel, + }); +} + +export function resolvePrimaryTtsProviderCandidate( + primary: TtsProvider, + cfg?: OpenClawConfig, +): VoiceProviderCandidate { + const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : undefined; + return resolvePrimaryVoiceProviderCandidate({ + primaryProvider: canonicalizeSpeechProviderId(primary, effectiveCfg) ?? primary, + providers: sortSpeechProvidersForAutoSelection(effectiveCfg), + voiceModelConfig: effectiveCfg?.agents?.defaults?.voiceModel, + }); +} + +export function isTtsProviderConfigured( + config: ResolvedTtsConfig, + provider: TtsProvider, + cfg?: OpenClawConfig, +): boolean { + try { + const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : config.sourceConfig; + const resolvedProvider = getSpeechProvider(provider, effectiveCfg); + if (!resolvedProvider) { + return false; + } + return ( + resolvedProvider.isConfigured({ + cfg: effectiveCfg, + providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, effectiveCfg), + timeoutMs: resolveSpeechProviderTimeoutMs({ config, provider: resolvedProvider }), + }) ?? false + ); + } catch { + // Configuration probes drive provider selection and status catalogs. A + // malformed provider config must not hide other usable providers. + return false; + } +} diff --git a/packages/speech-core/src/tts-request.ts b/packages/speech-core/src/tts-request.ts new file mode 100644 index 000000000000..8466a0ad7506 --- /dev/null +++ b/packages/speech-core/src/tts-request.ts @@ -0,0 +1,106 @@ +import type { OpenClawConfig, TtsConfig } from "openclaw/plugin-sdk/config-contracts"; +import { mergeDeep } from "openclaw/plugin-sdk/plugin-config-runtime"; +import { + canonicalizeSpeechProviderId, + getSpeechProvider, + parseTtsDirectives, + type SpeechProviderOverrides, + type TtsDirectiveOverrides, + type TtsDirectiveParseResult, +} from "openclaw/plugin-sdk/speech-core"; +import { resolveTtsProvider } from "./tts-provider-resolution.js"; +import { resolveTtsConfig, resolveTtsPrefsPath, resolveTtsRuntimeConfig } from "./tts-settings.js"; + +export type PreparedTtsRequest = { + cfg: OpenClawConfig; + directives: TtsDirectiveParseResult; +}; + +/** Merge a surface TTS override and resolve its inline synthesis directives. */ +export function prepareTtsRequest(params: { + cfg: OpenClawConfig; + override?: TtsConfig; + text: string; +}): PreparedTtsRequest { + const cfg = params.override + ? { + ...params.cfg, + tts: mergeDeep(params.cfg.tts ?? {}, params.override) as TtsConfig, + } + : params.cfg; + const config = resolveTtsConfig(cfg); + const directives = parseTtsDirectives(params.text, config.modelOverrides, { + cfg, + providerConfigs: config.providerConfigs, + preferredProviderId: resolveTtsProvider(config, resolveTtsPrefsPath(config)), + }); + return { cfg, directives }; +} + +export function resolveExplicitTtsOverrides(params: { + cfg: OpenClawConfig; + prefsPath?: string; + provider?: string; + modelId?: string; + voiceId?: string; + agentId?: string; + channelId?: string; + accountId?: string; +}): TtsDirectiveOverrides { + const cfg = resolveTtsRuntimeConfig(params.cfg); + const providerInput = params.provider?.trim(); + const modelId = params.modelId?.trim(); + const voiceId = params.voiceId?.trim(); + const config = resolveTtsConfig(cfg, { + agentId: params.agentId, + channelId: params.channelId, + accountId: params.accountId, + }); + const prefsPath = params.prefsPath ?? resolveTtsPrefsPath(config); + const selectedProvider = + canonicalizeSpeechProviderId(providerInput, cfg) ?? + (modelId || voiceId ? resolveTtsProvider(config, prefsPath) : undefined); + + if (providerInput && !selectedProvider) { + throw new Error(`Unknown TTS provider "${providerInput}".`); + } + + if (!modelId && !voiceId) { + return selectedProvider ? { provider: selectedProvider } : {}; + } + + if (!selectedProvider) { + throw new Error("TTS model or voice overrides require a resolved provider."); + } + + const provider = getSpeechProvider(selectedProvider, cfg); + if (!provider) { + throw new Error(`speech provider ${selectedProvider} is not registered`); + } + if (!provider.resolveTalkOverrides) { + throw new Error( + `TTS provider "${selectedProvider}" does not support model or voice overrides.`, + ); + } + + const providerOverrides = provider.resolveTalkOverrides({ + talkProviderConfig: {}, + params: { + ...(voiceId ? { voiceId } : {}), + ...(modelId ? { modelId } : {}), + }, + }); + if ((voiceId || modelId) && (!providerOverrides || Object.keys(providerOverrides).length === 0)) { + throw new Error( + `TTS provider "${selectedProvider}" ignored the requested model or voice overrides.`, + ); + } + + const overridesRecord = providerOverrides as SpeechProviderOverrides; + return { + provider: selectedProvider, + providerOverrides: { + [provider.id]: overridesRecord, + }, + }; +} diff --git a/packages/speech-core/src/tts-streaming.ts b/packages/speech-core/src/tts-streaming.ts new file mode 100644 index 000000000000..737d212c9583 --- /dev/null +++ b/packages/speech-core/src/tts-streaming.ts @@ -0,0 +1,202 @@ +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { logVerbose } from "openclaw/plugin-sdk/runtime-env"; +import type { TtsDirectiveOverrides } from "openclaw/plugin-sdk/speech-core"; +import { assertSpeechRuntimeAvailable } from "./runtime-availability.js"; +import { resolveSpeechProviderTimeoutMs } from "./tts-provider-resolution.js"; +import { + buildTtsFailureResult, + formatTtsProviderError, + prepareSpeechSynthesis, + resolvePersonaBinding, + resolveReadySpeechProvider, + resolveTtsRequestSetup, + resolveTtsResultModel, + resolveTtsResultVoice, + sanitizeTtsErrorForLog, +} from "./tts-synthesis-support.js"; +import { resolveTtsSynthesisTarget } from "./tts-synthesis.js"; +import type { TtsProviderAttempt, TtsStreamResult, TtsSynthesisStreamResult } from "./tts-types.js"; + +export async function streamSpeech(params: { + text: string; + cfg: OpenClawConfig; + prefsPath?: string; + channel?: string; + overrides?: TtsDirectiveOverrides; + disableFallback?: boolean; + timeoutMs?: number; + agentId?: string; + accountId?: string; +}): Promise { + assertSpeechRuntimeAvailable(); + const setup = resolveTtsRequestSetup({ + text: params.text, + cfg: params.cfg, + prefsPath: params.prefsPath, + providerOverride: params.overrides?.provider, + disableFallback: params.disableFallback, + agentId: params.agentId, + channelId: params.channel, + accountId: params.accountId, + }); + if ("error" in setup) { + return { success: false, error: setup.error }; + } + + const { cfg, config, persona, providers } = setup; + const target = resolveTtsSynthesisTarget(params.channel); + const errors: string[] = []; + const attemptedProviders: string[] = []; + const attempts: TtsProviderAttempt[] = []; + const primaryProvider = providers[0]?.provider; + logVerbose( + `TTS stream: starting with provider ${primaryProvider}, fallbacks: ${ + providers + .slice(1) + .map((entry) => entry.provider) + .join(", ") || "none" + }`, + ); + + for (const { provider, voiceModel } of providers) { + attemptedProviders.push(provider); + const providerStart = Date.now(); + try { + const resolvedProvider = resolveReadySpeechProvider({ + provider, + cfg, + config, + persona, + voiceModel, + }); + if (resolvedProvider.kind === "skip") { + errors.push(resolvedProvider.message); + attempts.push({ + provider, + outcome: "skipped", + reasonCode: resolvedProvider.reasonCode, + persona: persona?.id, + ...(resolvedProvider.personaBinding + ? { personaBinding: resolvedProvider.personaBinding } + : {}), + error: resolvedProvider.message, + }); + logVerbose(`TTS stream: provider ${provider} skipped (${resolvedProvider.message})`); + continue; + } + if (!resolvedProvider.provider.streamSynthesize) { + const message = `${provider} does not support streaming TTS`; + errors.push(message); + attempts.push({ + provider, + outcome: "skipped", + reasonCode: "unsupported_for_streaming", + persona: persona?.id, + personaBinding: resolvedProvider.personaBinding, + error: message, + }); + logVerbose(`TTS stream: provider ${provider} skipped (${message})`); + continue; + } + const timeoutMs = resolveSpeechProviderTimeoutMs({ + timeoutMs: params.timeoutMs ?? voiceModel?.timeoutMs, + config, + provider: resolvedProvider.provider, + }); + const prepared = await prepareSpeechSynthesis({ + provider: resolvedProvider.provider, + text: params.text, + cfg, + providerConfig: resolvedProvider.providerConfig, + providerOverrides: params.overrides?.providerOverrides?.[resolvedProvider.provider.id], + persona: resolvedProvider.synthesisPersona, + personaProviderConfig: resolvedProvider.personaProviderConfig, + target, + timeoutMs, + }); + const synthesis = await resolvedProvider.provider.streamSynthesize({ + text: prepared.text, + cfg, + providerConfig: prepared.providerConfig, + target, + providerOverrides: prepared.providerOverrides, + timeoutMs, + }); + const latencyMs = Date.now() - providerStart; + attempts.push({ + provider, + outcome: "success", + reasonCode: "success", + persona: persona?.id, + personaBinding: resolvedProvider.personaBinding, + latencyMs, + }); + return { + success: true, + audioStream: synthesis.audioStream, + latencyMs, + provider, + providerModel: resolveTtsResultModel(prepared.providerConfig, prepared.providerOverrides), + providerVoice: resolveTtsResultVoice(prepared.providerConfig, prepared.providerOverrides), + persona: persona?.id, + fallbackFrom: provider !== primaryProvider ? primaryProvider : undefined, + attemptedProviders, + attempts, + outputFormat: synthesis.outputFormat, + voiceCompatible: synthesis.voiceCompatible, + fileExtension: synthesis.fileExtension, + target, + release: synthesis.release, + }; + } catch (err) { + const errorMsg = formatTtsProviderError(provider, err); + const latencyMs = Date.now() - providerStart; + errors.push(errorMsg); + attempts.push({ + provider, + outcome: "failed", + reasonCode: + err instanceof Error && err.name === "AbortError" ? "timeout" : "provider_error", + latencyMs, + persona: persona?.id, + personaBinding: resolvePersonaBinding(persona, provider), + error: errorMsg, + }); + const rawError = sanitizeTtsErrorForLog(err); + if (provider === primaryProvider) { + const hasFallbacks = providers.length > 1; + logVerbose( + `TTS stream: primary provider ${provider} failed (${rawError})${hasFallbacks ? "; trying fallback providers." : "; no fallback providers configured."}`, + ); + } else { + logVerbose(`TTS stream: ${provider} failed (${rawError}); trying next provider.`); + } + } + } + + return buildTtsFailureResult(errors, attemptedProviders, attempts, persona?.id); +} + +export async function textToSpeechStream(params: { + text: string; + cfg: OpenClawConfig; + prefsPath?: string; + channel?: string; + overrides?: TtsDirectiveOverrides; + disableFallback?: boolean; + timeoutMs?: number; + agentId?: string; + accountId?: string; +}): Promise { + const synthesis = await streamSpeech(params); + if (!synthesis.success || !synthesis.audioStream || !synthesis.fileExtension) { + return { + success: false, + error: synthesis.error ?? "Streaming TTS conversion failed", + persona: synthesis.persona, + attemptedProviders: synthesis.attemptedProviders, + attempts: synthesis.attempts, + }; + } + return synthesis; +} diff --git a/packages/speech-core/src/tts-synthesis-support.ts b/packages/speech-core/src/tts-synthesis-support.ts new file mode 100644 index 000000000000..39e046833a96 --- /dev/null +++ b/packages/speech-core/src/tts-synthesis-support.ts @@ -0,0 +1,284 @@ +import type { + OpenClawConfig, + ResolvedTtsPersona, + TtsProvider, +} from "openclaw/plugin-sdk/config-contracts"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { redactSensitiveText } from "openclaw/plugin-sdk/logging-core"; +import { + canonicalizeSpeechProviderId, + getSpeechProvider, + type SpeechProviderConfig, + type SpeechProviderOverrides, +} from "openclaw/plugin-sdk/speech-core"; +import type { VoiceModelRef, VoiceProviderCandidate } from "../voice-models.js"; +import { + getResolvedSpeechProviderConfigForVoiceModel, + mergeProviderConfigWithPersona, + resolvePersonaProviderConfig, + resolvePrimaryTtsProviderCandidate, + resolveSpeechProviderTimeoutMs, + resolveTtsProvider, + resolveTtsProviderCandidates, +} from "./tts-provider-resolution.js"; +import { + getTtsPersona, + resolveTtsConfig, + resolveTtsPrefsPath, + resolveTtsRuntimeConfig, + type ResolvedTtsConfig, +} from "./tts-settings.js"; +import type { TtsProviderAttempt } from "./tts-types.js"; + +export function formatTtsProviderError(provider: TtsProvider, err: unknown): string { + const error = err instanceof Error ? err : new Error(String(err)); + if (error.name === "AbortError") { + return `${provider}: request timed out`; + } + return `${provider}: ${redactSensitiveText(error.message)}`; +} + +export function sanitizeTtsErrorForLog(err: unknown): string { + const raw = formatErrorMessage(err); + return redactSensitiveText(raw).replace(/\r/g, "\\r").replace(/\n/g, "\\n").replace(/\t/g, "\\t"); +} + +export function buildTtsFailureResult( + errors: string[], + attemptedProviders?: string[], + attempts?: TtsProviderAttempt[], + persona?: string, +): { + success: false; + error: string; + attemptedProviders?: string[]; + attempts?: TtsProviderAttempt[]; + persona?: string; +} { + return { + success: false, + error: `TTS conversion failed: ${errors.join("; ") || "no providers available"}`, + attemptedProviders, + attempts, + persona, + }; +} + +type TtsProviderReadyResolution = + | { + kind: "ready"; + provider: NonNullable>; + providerConfig: SpeechProviderConfig; + personaProviderConfig?: SpeechProviderConfig; + synthesisPersona?: ResolvedTtsPersona; + personaBinding: "applied" | "missing" | "none"; + } + | { + kind: "skip"; + reasonCode: "no_provider_registered" | "not_configured" | "unsupported_for_telephony"; + message: string; + personaBinding?: "missing"; + }; + +export function resolveReadySpeechProvider(params: { + provider: TtsProvider; + cfg: OpenClawConfig; + config: ResolvedTtsConfig; + persona?: ResolvedTtsPersona; + voiceModel?: VoiceModelRef; + requireTelephony?: boolean; +}): TtsProviderReadyResolution { + const resolvedProvider = getSpeechProvider(params.provider, params.cfg); + if (!resolvedProvider) { + return { + kind: "skip", + reasonCode: "no_provider_registered", + message: `${params.provider}: no provider registered`, + }; + } + const providerConfig = getResolvedSpeechProviderConfigForVoiceModel({ + config: params.config, + providerId: resolvedProvider.id, + cfg: params.cfg, + voiceModel: params.voiceModel, + }); + const merged = mergeProviderConfigWithPersona({ + providerConfig, + persona: params.persona, + providerId: resolvedProvider.id, + }); + if (params.persona?.fallbackPolicy === "fail" && merged.personaBinding === "missing") { + return { + kind: "skip", + reasonCode: "not_configured", + message: `${params.provider}: persona ${params.persona.id} has no provider binding`, + personaBinding: "missing", + }; + } + if ( + !resolvedProvider.isConfigured({ + cfg: params.cfg, + providerConfig: merged.providerConfig, + timeoutMs: resolveSpeechProviderTimeoutMs({ + config: params.config, + provider: resolvedProvider, + }), + }) + ) { + return { + kind: "skip", + reasonCode: "not_configured", + message: `${params.provider}: not configured`, + }; + } + if (params.requireTelephony && !resolvedProvider.synthesizeTelephony) { + return { + kind: "skip", + reasonCode: "unsupported_for_telephony", + message: `${params.provider}: unsupported for telephony`, + }; + } + return { + kind: "ready", + provider: resolvedProvider, + providerConfig: merged.providerConfig, + personaProviderConfig: merged.personaProviderConfig, + synthesisPersona: + params.persona?.fallbackPolicy === "provider-defaults" && merged.personaBinding === "missing" + ? undefined + : params.persona, + personaBinding: merged.personaBinding, + }; +} + +export async function prepareSpeechSynthesis(params: { + provider: NonNullable>; + text: string; + cfg: OpenClawConfig; + providerConfig: SpeechProviderConfig; + providerOverrides?: SpeechProviderOverrides; + persona?: ResolvedTtsPersona; + personaProviderConfig?: SpeechProviderConfig; + target: "audio-file" | "voice-note" | "telephony"; + timeoutMs: number; +}): Promise<{ + text: string; + providerConfig: SpeechProviderConfig; + providerOverrides?: SpeechProviderOverrides; +}> { + if (!params.provider.prepareSynthesis) { + return { + text: params.text, + providerConfig: params.providerConfig, + providerOverrides: params.providerOverrides, + }; + } + const prepared = await params.provider.prepareSynthesis({ + text: params.text, + cfg: params.cfg, + providerConfig: params.providerConfig, + providerOverrides: params.providerOverrides, + persona: params.persona, + personaProviderConfig: params.personaProviderConfig, + target: params.target, + timeoutMs: params.timeoutMs, + }); + return { + text: prepared?.text ?? params.text, + providerConfig: prepared?.providerConfig + ? { ...params.providerConfig, ...prepared.providerConfig } + : params.providerConfig, + providerOverrides: prepared?.providerOverrides + ? { ...params.providerOverrides, ...prepared.providerOverrides } + : params.providerOverrides, + }; +} + +export function resolveTtsRequestSetup(params: { + text: string; + cfg: OpenClawConfig; + prefsPath?: string; + providerOverride?: TtsProvider; + disableFallback?: boolean; + agentId?: string; + channelId?: string; + accountId?: string; +}): + | { + cfg: OpenClawConfig; + config: ResolvedTtsConfig; + persona?: ResolvedTtsPersona; + providers: VoiceProviderCandidate[]; + } + | { + error: string; + } { + const cfg = resolveTtsRuntimeConfig(params.cfg); + const config = resolveTtsConfig(cfg, { + agentId: params.agentId, + channelId: params.channelId, + accountId: params.accountId, + }); + const prefsPath = params.prefsPath ?? resolveTtsPrefsPath(config); + if (params.text.length > config.maxTextLength) { + return { + error: `Text too long (${params.text.length} chars, max ${config.maxTextLength})`, + }; + } + + const userProvider = resolveTtsProvider(config, prefsPath); + const provider = canonicalizeSpeechProviderId(params.providerOverride, cfg) ?? userProvider; + return { + cfg, + config, + persona: getTtsPersona(config, prefsPath), + providers: params.disableFallback + ? [resolvePrimaryTtsProviderCandidate(provider, cfg)] + : resolveTtsProviderCandidates(provider, cfg), + }; +} + +function readTtsResultString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +export function resolveTtsResultModel( + providerConfig: SpeechProviderConfig, + providerOverrides?: SpeechProviderOverrides, +): string | undefined { + return ( + readTtsResultString(providerOverrides?.modelId) ?? + readTtsResultString(providerOverrides?.model) ?? + readTtsResultString(providerConfig.modelId) ?? + readTtsResultString(providerConfig.model) + ); +} + +export function resolveTtsResultVoice( + providerConfig: SpeechProviderConfig, + providerOverrides?: SpeechProviderOverrides, +): string | undefined { + return ( + readTtsResultString(providerOverrides?.speakerVoiceId) ?? + readTtsResultString(providerOverrides?.speakerVoice) ?? + readTtsResultString(providerOverrides?.voiceId) ?? + readTtsResultString(providerOverrides?.voiceName) ?? + readTtsResultString(providerOverrides?.voice) ?? + readTtsResultString(providerConfig.speakerVoiceId) ?? + readTtsResultString(providerConfig.speakerVoice) ?? + readTtsResultString(providerConfig.voiceId) ?? + readTtsResultString(providerConfig.voiceName) ?? + readTtsResultString(providerConfig.voice) + ); +} + +export function resolvePersonaBinding( + persona: ResolvedTtsPersona | undefined, + provider: string, +): "applied" | "missing" | "none" { + return resolvePersonaProviderConfig(persona, provider) != null + ? "applied" + : persona + ? "missing" + : "none"; +} diff --git a/packages/speech-core/src/tts-synthesis.ts b/packages/speech-core/src/tts-synthesis.ts new file mode 100644 index 000000000000..65b8dd7f9388 --- /dev/null +++ b/packages/speech-core/src/tts-synthesis.ts @@ -0,0 +1,335 @@ +import { resolveChannelTtsVoiceDelivery } from "openclaw/plugin-sdk/channel-targets"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { transcodeAudioBuffer } from "openclaw/plugin-sdk/media-runtime"; +import { logVerbose } from "openclaw/plugin-sdk/runtime-env"; +import { tempWorkspaceSync, resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/sandbox"; +import { scheduleCleanup, type TtsDirectiveOverrides } from "openclaw/plugin-sdk/speech-core"; +import { assertSpeechRuntimeAvailable } from "./runtime-availability.js"; +import { normalizeSpeechText } from "./speech-text.js"; +import { resolveSpeechProviderTimeoutMs } from "./tts-provider-resolution.js"; +import { + buildTtsFailureResult, + formatTtsProviderError, + prepareSpeechSynthesis, + resolvePersonaBinding, + resolveReadySpeechProvider, + resolveTtsRequestSetup, + resolveTtsResultModel, + resolveTtsResultVoice, + sanitizeTtsErrorForLog, +} from "./tts-synthesis-support.js"; +import type { TtsProviderAttempt, TtsResult, TtsSynthesisResult } from "./tts-types.js"; + +export function supportsNativeVoiceNoteTts(channel: string | undefined): boolean { + return resolveChannelTtsVoiceDelivery(channel) !== undefined; +} + +export function supportsTranscodedVoiceNoteTts(channel: string | undefined): boolean { + const delivery = resolveChannelTtsVoiceDelivery(channel); + return delivery?.synthesisTarget === "voice-note" && delivery.transcodesAudio === true; +} + +export function resolveTtsSynthesisTarget( + channel: string | undefined, +): "audio-file" | "voice-note" { + return resolveChannelTtsVoiceDelivery(channel)?.synthesisTarget ?? "audio-file"; +} + +function supportsAudioFileVoiceMemoOutput(params: { + fileExtension?: string; + outputFormat?: string; + audioFileFormats?: readonly string[]; +}): boolean { + const formats = new Set(params.audioFileFormats?.map((format) => format.trim().toLowerCase())); + if (formats.size === 0) { + return false; + } + const extension = params.fileExtension?.trim().toLowerCase(); + if (extension && formats.has(extension.replace(/^\./, ""))) { + return true; + } + const outputFormat = params.outputFormat?.trim().toLowerCase(); + return outputFormat ? formats.has(outputFormat) : false; +} + +export function shouldDeliverTtsAsVoice(params: { + channel: string | undefined; + target: "audio-file" | "voice-note" | undefined; + voiceCompatible: boolean | undefined; + fileExtension?: string; + outputFormat?: string; +}): boolean { + const delivery = resolveChannelTtsVoiceDelivery(params.channel); + if (!delivery) { + return false; + } + if (delivery.synthesisTarget === "audio-file") { + return ( + params.target === "audio-file" && + supportsAudioFileVoiceMemoOutput({ + fileExtension: params.fileExtension, + outputFormat: params.outputFormat, + audioFileFormats: delivery.audioFileFormats, + }) + ); + } + if (params.target !== "voice-note") { + return false; + } + return params.voiceCompatible === true || delivery.transcodesAudio === true; +} + +export async function textToSpeech(params: { + text: string; + cfg: OpenClawConfig; + prefsPath?: string; + channel?: string; + overrides?: TtsDirectiveOverrides; + disableFallback?: boolean; + timeoutMs?: number; + agentId?: string; + accountId?: string; +}): Promise { + const synthesis = await synthesizeSpeech(params); + if (!synthesis.success || !synthesis.audioBuffer || !synthesis.fileExtension) { + return { + success: false, + error: synthesis.error ?? "TTS conversion failed", + persona: synthesis.persona, + attemptedProviders: synthesis.attemptedProviders, + attempts: synthesis.attempts, + }; + } + + let audioBuffer = synthesis.audioBuffer; + let fileExtension = synthesis.fileExtension; + let outputFormat = synthesis.outputFormat; + const transcoded = await maybePreTranscodeForVoiceDelivery({ + channel: params.channel, + target: synthesis.target, + audioBuffer, + fileExtension, + outputFormat, + }); + if (transcoded) { + audioBuffer = transcoded.audioBuffer; + fileExtension = transcoded.fileExtension; + outputFormat = transcoded.outputFormat; + } + + const temp = tempWorkspaceSync({ + rootDir: resolvePreferredOpenClawTmpDir(), + prefix: "tts-", + }); + const audioPath = temp.write(`voice-${Date.now()}${fileExtension}`, audioBuffer); + scheduleCleanup(temp.dir); + + return { + success: true, + audioPath, + latencyMs: synthesis.latencyMs, + provider: synthesis.provider, + persona: synthesis.persona, + fallbackFrom: synthesis.fallbackFrom, + attemptedProviders: synthesis.attemptedProviders, + attempts: synthesis.attempts, + outputFormat, + voiceCompatible: synthesis.voiceCompatible, + audioAsVoice: shouldDeliverTtsAsVoice({ + channel: params.channel, + target: synthesis.target, + voiceCompatible: synthesis.voiceCompatible, + fileExtension, + outputFormat, + }), + target: synthesis.target, + }; +} + +async function maybePreTranscodeForVoiceDelivery(params: { + channel: string | undefined; + target: "audio-file" | "voice-note" | undefined; + audioBuffer: Buffer; + fileExtension: string; + outputFormat?: string; +}): Promise<{ audioBuffer: Buffer; fileExtension: string; outputFormat?: string } | undefined> { + if (params.target !== "audio-file") { + return undefined; + } + const delivery = resolveChannelTtsVoiceDelivery(params.channel); + const preferred = delivery?.preferAudioFileFormat?.trim().toLowerCase(); + if (!preferred) { + return undefined; + } + const sourceExt = params.fileExtension.trim().toLowerCase().replace(/^\./, ""); + if (sourceExt === preferred) { + return undefined; + } + const outcome = await transcodeAudioBuffer({ + audioBuffer: params.audioBuffer, + sourceExtension: sourceExt, + targetExtension: preferred, + }); + if (!outcome.ok) { + if (outcome.reason === "transcoder-failed") { + // Surface only the case where the host actually attempted the transcode + // and it broke. The other reasons are by-design skips and would just be log noise. + logVerbose( + `TTS: pre-transcode ${sourceExt}->${preferred} for channel=${params.channel ?? "?"} failed: ${outcome.detail ?? "unknown"}`, + ); + } + return undefined; + } + return { + audioBuffer: outcome.buffer, + fileExtension: `.${preferred}`, + outputFormat: preferred, + }; +} + +export async function synthesizeSpeech(params: { + text: string; + cfg: OpenClawConfig; + prefsPath?: string; + channel?: string; + overrides?: TtsDirectiveOverrides; + disableFallback?: boolean; + timeoutMs?: number; + agentId?: string; + accountId?: string; +}): Promise { + assertSpeechRuntimeAvailable(); + const setup = resolveTtsRequestSetup({ + text: params.text, + cfg: params.cfg, + prefsPath: params.prefsPath, + providerOverride: params.overrides?.provider, + disableFallback: params.disableFallback, + agentId: params.agentId, + channelId: params.channel, + accountId: params.accountId, + }); + if ("error" in setup) { + return { success: false, error: setup.error }; + } + + const { cfg, config, persona, providers } = setup; + const textForSynthesis = normalizeSpeechText(params.text); + const target = resolveTtsSynthesisTarget(params.channel); + + const errors: string[] = []; + const attemptedProviders: string[] = []; + const attempts: TtsProviderAttempt[] = []; + const primaryProvider = providers[0]?.provider; + logVerbose( + `TTS: starting with provider ${primaryProvider}, fallbacks: ${ + providers + .slice(1) + .map((entry) => entry.provider) + .join(", ") || "none" + }`, + ); + + for (const { provider, voiceModel } of providers) { + attemptedProviders.push(provider); + const providerStart = Date.now(); + try { + const resolvedProvider = resolveReadySpeechProvider({ + provider, + cfg, + config, + persona, + voiceModel, + }); + if (resolvedProvider.kind === "skip") { + errors.push(resolvedProvider.message); + attempts.push({ + provider, + outcome: "skipped", + reasonCode: resolvedProvider.reasonCode, + persona: persona?.id, + ...(resolvedProvider.personaBinding + ? { personaBinding: resolvedProvider.personaBinding } + : {}), + error: resolvedProvider.message, + }); + logVerbose(`TTS: provider ${provider} skipped (${resolvedProvider.message})`); + continue; + } + const timeoutMs = resolveSpeechProviderTimeoutMs({ + timeoutMs: params.timeoutMs ?? voiceModel?.timeoutMs, + config, + provider: resolvedProvider.provider, + }); + const prepared = await prepareSpeechSynthesis({ + provider: resolvedProvider.provider, + text: textForSynthesis, + cfg, + providerConfig: resolvedProvider.providerConfig, + providerOverrides: params.overrides?.providerOverrides?.[resolvedProvider.provider.id], + persona: resolvedProvider.synthesisPersona, + personaProviderConfig: resolvedProvider.personaProviderConfig, + target, + timeoutMs, + }); + const synthesis = await resolvedProvider.provider.synthesize({ + text: prepared.text, + cfg, + providerConfig: prepared.providerConfig, + target, + providerOverrides: prepared.providerOverrides, + timeoutMs, + }); + const latencyMs = Date.now() - providerStart; + attempts.push({ + provider, + outcome: "success", + reasonCode: "success", + persona: persona?.id, + personaBinding: resolvedProvider.personaBinding, + latencyMs, + }); + return { + success: true, + audioBuffer: synthesis.audioBuffer, + latencyMs, + provider, + providerModel: resolveTtsResultModel(prepared.providerConfig, prepared.providerOverrides), + providerVoice: resolveTtsResultVoice(prepared.providerConfig, prepared.providerOverrides), + persona: persona?.id, + fallbackFrom: provider !== primaryProvider ? primaryProvider : undefined, + attemptedProviders, + attempts, + outputFormat: synthesis.outputFormat, + voiceCompatible: synthesis.voiceCompatible, + fileExtension: synthesis.fileExtension, + target, + }; + } catch (err) { + const errorMsg = formatTtsProviderError(provider, err); + const latencyMs = Date.now() - providerStart; + errors.push(errorMsg); + attempts.push({ + provider, + outcome: "failed", + reasonCode: + err instanceof Error && err.name === "AbortError" ? "timeout" : "provider_error", + latencyMs, + persona: persona?.id, + personaBinding: resolvePersonaBinding(persona, provider), + error: errorMsg, + }); + const rawError = sanitizeTtsErrorForLog(err); + if (provider === primaryProvider) { + const hasFallbacks = providers.length > 1; + logVerbose( + `TTS: primary provider ${provider} failed (${rawError})${hasFallbacks ? "; trying fallback providers." : "; no fallback providers configured."}`, + ); + } else { + logVerbose(`TTS: ${provider} failed (${rawError}); trying next provider.`); + } + } + } + + return buildTtsFailureResult(errors, attemptedProviders, attempts, persona?.id); +} diff --git a/packages/speech-core/src/tts-telephony.ts b/packages/speech-core/src/tts-telephony.ts new file mode 100644 index 000000000000..9dfcc940e75b --- /dev/null +++ b/packages/speech-core/src/tts-telephony.ts @@ -0,0 +1,155 @@ +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { logVerbose } from "openclaw/plugin-sdk/runtime-env"; +import type { TtsDirectiveOverrides } from "openclaw/plugin-sdk/speech-core"; +import { assertSpeechRuntimeAvailable } from "./runtime-availability.js"; +import { resolveSpeechProviderTimeoutMs } from "./tts-provider-resolution.js"; +import { + buildTtsFailureResult, + formatTtsProviderError, + prepareSpeechSynthesis, + resolvePersonaBinding, + resolveReadySpeechProvider, + resolveTtsRequestSetup, + resolveTtsResultModel, + resolveTtsResultVoice, + sanitizeTtsErrorForLog, +} from "./tts-synthesis-support.js"; +import type { TtsProviderAttempt, TtsTelephonyResult } from "./tts-types.js"; + +export async function textToSpeechTelephony(params: { + text: string; + cfg: OpenClawConfig; + prefsPath?: string; + overrides?: TtsDirectiveOverrides; + timeoutMs?: number; +}): Promise { + assertSpeechRuntimeAvailable(); + const setup = resolveTtsRequestSetup({ + text: params.text, + cfg: params.cfg, + prefsPath: params.prefsPath, + providerOverride: params.overrides?.provider, + }); + if ("error" in setup) { + return { success: false, error: setup.error }; + } + + const { cfg, config, persona, providers } = setup; + const errors: string[] = []; + const attemptedProviders: string[] = []; + const attempts: TtsProviderAttempt[] = []; + const primaryProvider = providers[0]?.provider; + logVerbose( + `TTS telephony: starting with provider ${primaryProvider}, fallbacks: ${ + providers + .slice(1) + .map((entry) => entry.provider) + .join(", ") || "none" + }`, + ); + + for (const { provider, voiceModel } of providers) { + attemptedProviders.push(provider); + const providerStart = Date.now(); + try { + const resolvedProvider = resolveReadySpeechProvider({ + provider, + cfg, + config, + persona, + voiceModel, + requireTelephony: true, + }); + if (resolvedProvider.kind === "skip") { + errors.push(resolvedProvider.message); + attempts.push({ + provider, + outcome: "skipped", + reasonCode: resolvedProvider.reasonCode, + persona: persona?.id, + ...(resolvedProvider.personaBinding + ? { personaBinding: resolvedProvider.personaBinding } + : {}), + error: resolvedProvider.message, + }); + logVerbose(`TTS telephony: provider ${provider} skipped (${resolvedProvider.message})`); + continue; + } + const timeoutMs = resolveSpeechProviderTimeoutMs({ + timeoutMs: params.timeoutMs ?? voiceModel?.timeoutMs, + config, + provider: resolvedProvider.provider, + }); + const synthesizeTelephony = resolvedProvider.provider.synthesizeTelephony as NonNullable< + typeof resolvedProvider.provider.synthesizeTelephony + >; + const prepared = await prepareSpeechSynthesis({ + provider: resolvedProvider.provider, + text: params.text, + cfg, + providerConfig: resolvedProvider.providerConfig, + providerOverrides: params.overrides?.providerOverrides?.[resolvedProvider.provider.id], + persona: resolvedProvider.synthesisPersona, + personaProviderConfig: resolvedProvider.personaProviderConfig, + target: "telephony", + timeoutMs, + }); + const synthesis = await synthesizeTelephony({ + text: prepared.text, + cfg, + providerConfig: prepared.providerConfig, + providerOverrides: prepared.providerOverrides, + timeoutMs, + }); + const latencyMs = Date.now() - providerStart; + attempts.push({ + provider, + outcome: "success", + reasonCode: "success", + persona: persona?.id, + personaBinding: resolvedProvider.personaBinding, + latencyMs, + }); + + return { + success: true, + audioBuffer: synthesis.audioBuffer, + latencyMs, + provider, + providerModel: resolveTtsResultModel(prepared.providerConfig, prepared.providerOverrides), + providerVoice: resolveTtsResultVoice(prepared.providerConfig, prepared.providerOverrides), + persona: persona?.id, + fallbackFrom: provider !== primaryProvider ? primaryProvider : undefined, + attemptedProviders, + attempts, + outputFormat: synthesis.outputFormat, + sampleRate: synthesis.sampleRate, + }; + } catch (err) { + const errorMsg = formatTtsProviderError(provider, err); + const latencyMs = Date.now() - providerStart; + errors.push(errorMsg); + attempts.push({ + provider, + outcome: "failed", + reasonCode: + err instanceof Error && err.name === "AbortError" ? "timeout" : "provider_error", + latencyMs, + persona: persona?.id, + personaBinding: resolvePersonaBinding(persona, provider), + error: errorMsg, + }); + const rawError = sanitizeTtsErrorForLog(err); + if (provider === primaryProvider) { + const hasFallbacks = providers.length > 1; + logVerbose( + `TTS telephony: primary provider ${provider} failed (${rawError})${hasFallbacks ? "; trying fallback providers." : "; no fallback providers configured."}`, + ); + } else { + logVerbose(`TTS telephony: ${provider} failed (${rawError}); trying next provider.`); + } + } + } + + return buildTtsFailureResult(errors, attemptedProviders, attempts, persona?.id); +} diff --git a/packages/speech-core/src/tts-types.ts b/packages/speech-core/src/tts-types.ts new file mode 100644 index 000000000000..1ffddbbfa0de --- /dev/null +++ b/packages/speech-core/src/tts-types.ts @@ -0,0 +1,103 @@ +type TtsAttemptReasonCode = + | "success" + | "no_provider_registered" + | "not_configured" + | "unsupported_for_streaming" + | "unsupported_for_telephony" + | "timeout" + | "provider_error"; + +export type TtsProviderAttempt = { + provider: string; + outcome: "success" | "skipped" | "failed"; + reasonCode: TtsAttemptReasonCode; + persona?: string; + personaBinding?: "applied" | "missing" | "none"; + latencyMs?: number; + error?: string; +}; + +export type TtsResult = { + success: boolean; + audioPath?: string; + error?: string; + latencyMs?: number; + provider?: string; + persona?: string; + fallbackFrom?: string; + attemptedProviders?: string[]; + attempts?: TtsProviderAttempt[]; + outputFormat?: string; + voiceCompatible?: boolean; + audioAsVoice?: boolean; + target?: "audio-file" | "voice-note"; +}; + +export type TtsSynthesisResult = { + success: boolean; + audioBuffer?: Buffer; + error?: string; + latencyMs?: number; + provider?: string; + providerModel?: string; + providerVoice?: string; + persona?: string; + fallbackFrom?: string; + attemptedProviders?: string[]; + attempts?: TtsProviderAttempt[]; + outputFormat?: string; + voiceCompatible?: boolean; + fileExtension?: string; + target?: "audio-file" | "voice-note"; +}; + +export type TtsStreamResult = { + success: boolean; + audioStream?: ReadableStream; + error?: string; + latencyMs?: number; + provider?: string; + providerModel?: string; + providerVoice?: string; + persona?: string; + fallbackFrom?: string; + attemptedProviders?: string[]; + attempts?: TtsProviderAttempt[]; + outputFormat?: string; + voiceCompatible?: boolean; + fileExtension?: string; + target?: "audio-file" | "voice-note"; + release?: () => Promise; +}; + +export type TtsSynthesisStreamResult = TtsStreamResult; + +export type TtsTelephonyResult = { + success: boolean; + audioBuffer?: Buffer; + error?: string; + latencyMs?: number; + provider?: string; + providerModel?: string; + providerVoice?: string; + persona?: string; + fallbackFrom?: string; + attemptedProviders?: string[]; + attempts?: TtsProviderAttempt[]; + outputFormat?: string; + sampleRate?: number; +}; + +export type TtsStatusEntry = { + timestamp: number; + success: boolean; + textLength: number; + summarized: boolean; + provider?: string; + persona?: string; + fallbackFrom?: string; + attemptedProviders?: string[]; + attempts?: TtsProviderAttempt[]; + latencyMs?: number; + error?: string; +}; diff --git a/packages/speech-core/src/tts.ts b/packages/speech-core/src/tts.ts index 0a2c76fc8eaf..bf6242f69721 100644 --- a/packages/speech-core/src/tts.ts +++ b/packages/speech-core/src/tts.ts @@ -1,1760 +1,50 @@ -// Speech Core module implements tts behavior. -import { resolveChannelTtsVoiceDelivery } from "openclaw/plugin-sdk/channel-targets"; -import type { - OpenClawConfig, - ResolvedTtsPersona, - TtsConfig, - TtsProvider, -} from "openclaw/plugin-sdk/config-contracts"; -import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; -import { redactSensitiveText } from "openclaw/plugin-sdk/logging-core"; -import { transcodeAudioBuffer } from "openclaw/plugin-sdk/media-runtime"; -import { clampTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime"; -import { mergeDeep } from "openclaw/plugin-sdk/plugin-config-runtime"; +import type { TtsProvider } from "openclaw/plugin-sdk/config-contracts"; +import { parseTtsDirectives, summarizeText } from "openclaw/plugin-sdk/speech-core"; +import { getResolvedSpeechProviderConfig, resolveTtsProvider } from "./tts-provider-resolution.js"; +import { resolveModelOverridePolicy, type ResolvedTtsConfig } from "./tts-settings.js"; +import { formatTtsProviderError, sanitizeTtsErrorForLog } from "./tts-synthesis-support.js"; import { - markReplyPayloadAsTtsSupplement, - resolveSendableOutboundReplyParts, - type ReplyPayload, -} from "openclaw/plugin-sdk/reply-payload"; -import { isVerbose, logVerbose } from "openclaw/plugin-sdk/runtime-env"; -import { tempWorkspaceSync, resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/sandbox"; -import { - canonicalizeSpeechProviderId, - getSpeechProvider, - listSpeechProviders, - normalizeSpeechProviderId, - parseTtsDirectives, - scheduleCleanup, - summarizeText, - type SpeechProviderPlugin, - type SpeechProviderConfig, - type SpeechProviderOverrides, - type SpeechVoiceOption, - type TtsDirectiveOverrides, - type TtsDirectiveParseResult, + shouldDeliverTtsAsVoice, + supportsNativeVoiceNoteTts, + supportsTranscodedVoiceNoteTts, + resolveTtsSynthesisTarget, +} from "./tts-synthesis.js"; + +export type { + TtsDirectiveOverrides, + TtsDirectiveParseResult, } from "openclaw/plugin-sdk/speech-core"; -import { - normalizeLowercaseStringOrEmpty, - normalizeOptionalString, -} from "openclaw/plugin-sdk/string-coerce-runtime"; -import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; -import { withSpeakerSelectionCompat } from "../speaker.js"; -import { - resolvePrimaryVoiceProviderCandidate, - resolveSupportedVoiceModelRefs, - resolveVoiceModelRefs, - resolveVoiceProviderCandidates, - voiceProviderSupportsModel, - type VoiceModelRef, - type VoiceModelProvider, - type VoiceProviderCandidate, -} from "../voice-models.js"; -import { assertSpeechRuntimeAvailable, isSpeechRuntimeAvailable } from "./runtime-availability.js"; -import { isCodeHeavySpeechText, normalizeSpeechText } from "./speech-text.js"; -import { - DEFAULT_TTS_TIMEOUT_MS, - asProviderConfig, - asProviderConfigMap, - getTtsMaxLength, - getTtsPersona, - hasOwnProperty, - isSummarizationEnabled, - normalizeConfiguredSpeechProviderId, - readTtsPrefs as readPrefs, - resolveModelOverridePolicy, - resolveTtsConfig, - resolveTtsPersonaFromPrefs, - resolveTtsPrefsPath, - resolveTtsRuntimeConfig, - resolveTtsSettingsSnapshot, - type ResolvedTtsConfig, -} from "./tts-settings.js"; - -export type { TtsDirectiveOverrides, TtsDirectiveParseResult }; - -function resolvePositiveTimeoutMs(timeoutMs: number | undefined): number | undefined { - return typeof timeoutMs === "number" && Number.isFinite(timeoutMs) && timeoutMs > 0 - ? clampTimerTimeoutMs(timeoutMs) - : undefined; -} - -function resolveSpeechProviderTimeoutMs(params: { - timeoutMs?: number; - config: ResolvedTtsConfig; - provider: Pick; -}): number { - if (params.timeoutMs !== undefined) { - return resolvePositiveTimeoutMs(params.timeoutMs) ?? params.config.timeoutMs; - } - if (params.config.timeoutMsSource !== "default") { - return resolvePositiveTimeoutMs(params.config.timeoutMs) ?? DEFAULT_TTS_TIMEOUT_MS; - } - return resolvePositiveTimeoutMs(params.provider.defaultTimeoutMs) ?? params.config.timeoutMs; -} - -type TtsAttemptReasonCode = - | "success" - | "no_provider_registered" - | "not_configured" - | "unsupported_for_streaming" - | "unsupported_for_telephony" - | "timeout" - | "provider_error"; - -type TtsProviderAttempt = { - provider: string; - outcome: "success" | "skipped" | "failed"; - reasonCode: TtsAttemptReasonCode; - persona?: string; - personaBinding?: "applied" | "missing" | "none"; - latencyMs?: number; - error?: string; -}; - -export type TtsResult = { - success: boolean; - audioPath?: string; - error?: string; - latencyMs?: number; - provider?: string; - persona?: string; - fallbackFrom?: string; - attemptedProviders?: string[]; - attempts?: TtsProviderAttempt[]; - outputFormat?: string; - voiceCompatible?: boolean; - audioAsVoice?: boolean; - target?: "audio-file" | "voice-note"; -}; - -export type TtsSynthesisResult = { - success: boolean; - audioBuffer?: Buffer; - error?: string; - latencyMs?: number; - provider?: string; - providerModel?: string; - providerVoice?: string; - persona?: string; - fallbackFrom?: string; - attemptedProviders?: string[]; - attempts?: TtsProviderAttempt[]; - outputFormat?: string; - voiceCompatible?: boolean; - fileExtension?: string; - target?: "audio-file" | "voice-note"; -}; - -export type TtsStreamResult = { - success: boolean; - audioStream?: ReadableStream; - error?: string; - latencyMs?: number; - provider?: string; - providerModel?: string; - providerVoice?: string; - persona?: string; - fallbackFrom?: string; - attemptedProviders?: string[]; - attempts?: TtsProviderAttempt[]; - outputFormat?: string; - voiceCompatible?: boolean; - fileExtension?: string; - target?: "audio-file" | "voice-note"; - release?: () => Promise; -}; - -export type TtsSynthesisStreamResult = TtsStreamResult; - -export type TtsTelephonyResult = { - success: boolean; - audioBuffer?: Buffer; - error?: string; - latencyMs?: number; - provider?: string; - providerModel?: string; - providerVoice?: string; - persona?: string; - fallbackFrom?: string; - attemptedProviders?: string[]; - attempts?: TtsProviderAttempt[]; - outputFormat?: string; - sampleRate?: number; -}; - -type TtsStatusEntry = { - timestamp: number; - success: boolean; - textLength: number; - summarized: boolean; - provider?: string; - persona?: string; - fallbackFrom?: string; - attemptedProviders?: string[]; - attempts?: TtsProviderAttempt[]; - latencyMs?: number; - error?: string; -}; - -let lastTtsAttempt: TtsStatusEntry | undefined; - -function resolveConfiguredSpeechVoiceModelRefs(cfg: OpenClawConfig | undefined): VoiceModelRef[] { - const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : undefined; - return resolveSupportedVoiceModelRefs({ - config: effectiveCfg?.agents?.defaults?.voiceModel, - providers: sortSpeechProvidersForAutoSelection(effectiveCfg), - }); -} - -function resolveConfiguredSpeechVoiceModelForProvider(params: { - cfg: OpenClawConfig | undefined; - providerId: string; - provider?: VoiceModelProvider; - voiceModel?: VoiceModelRef; -}): VoiceModelRef | undefined { - const provider = params.provider ?? getSpeechProvider(params.providerId, params.cfg); - if (params.voiceModel) { - return voiceProviderSupportsModel(provider, params.voiceModel.model) - ? params.voiceModel - : undefined; - } - return resolveSupportedVoiceModelRefs({ - config: params.cfg?.agents?.defaults?.voiceModel, - providers: provider ? [provider] : [], - providerId: params.providerId, - })[0]; -} - -function applyVoiceModelToSpeechProviderConfig(params: { - cfg: OpenClawConfig | undefined; - providerId: string; - providerConfig: SpeechProviderConfig; - provider?: VoiceModelProvider; - voiceModel?: VoiceModelRef; -}): SpeechProviderConfig { - const voiceModel = resolveConfiguredSpeechVoiceModelForProvider({ - cfg: params.cfg, - providerId: params.providerId, - provider: params.provider, - voiceModel: params.voiceModel, - }); - if (!voiceModel) { - return params.providerConfig; - } - const hasExplicitModel = - normalizeOptionalString(params.providerConfig.model) || - normalizeOptionalString(params.providerConfig.modelId); - if (hasExplicitModel) { - return params.providerConfig; - } - return { - ...params.providerConfig, - model: voiceModel.model, - modelId: voiceModel.model, - }; -} - -function sortSpeechProvidersForAutoSelection(cfg?: OpenClawConfig) { - return listSpeechProviders(cfg).toSorted((left, right) => { - const leftOrder = left.autoSelectOrder ?? Number.MAX_SAFE_INTEGER; - const rightOrder = right.autoSelectOrder ?? Number.MAX_SAFE_INTEGER; - if (leftOrder !== rightOrder) { - return leftOrder - rightOrder; - } - return left.id.localeCompare(right.id); - }); -} - -function resolvePersonaProviderConfig( - persona: ResolvedTtsPersona | undefined, - providerId: string, -): SpeechProviderConfig | undefined { - if (!persona?.providers) { - return undefined; - } - const normalized = normalizeConfiguredSpeechProviderId(providerId) ?? providerId; - if (hasOwnProperty(persona.providers, normalized)) { - return persona.providers[normalized]; - } - if (hasOwnProperty(persona.providers, providerId)) { - return persona.providers[providerId]; - } - return undefined; -} - -function mergeProviderConfigWithPersona(params: { - providerConfig: SpeechProviderConfig; - persona?: ResolvedTtsPersona; - providerId: string; -}): { - providerConfig: SpeechProviderConfig; - personaProviderConfig?: SpeechProviderConfig; - personaBinding: "applied" | "missing" | "none"; -} { - if (!params.persona) { - return { providerConfig: params.providerConfig, personaBinding: "none" }; - } - const personaProviderConfig = resolvePersonaProviderConfig(params.persona, params.providerId); - if (!personaProviderConfig) { - return { providerConfig: params.providerConfig, personaBinding: "missing" }; - } - return { - providerConfig: { - ...params.providerConfig, - ...personaProviderConfig, - }, - personaProviderConfig, - personaBinding: "applied", - }; -} - -function resolveRawProviderConfig( - raw: TtsConfig | undefined, - providerId: string, -): SpeechProviderConfig { - if (!raw) { - return {}; - } - const rawProviders = asProviderConfigMap(raw.providers); - const direct = rawProviders[providerId] ?? (raw as Record)[providerId]; - return withSpeakerSelectionCompat(asProviderConfig(direct)); -} - -function resolveLazyProviderConfig( - config: ResolvedTtsConfig, - providerId: string, - cfg?: OpenClawConfig, - voiceModel?: VoiceModelRef, -): SpeechProviderConfig { - const canonical = - normalizeConfiguredSpeechProviderId(providerId) ?? normalizeLowercaseStringOrEmpty(providerId); - const existing = voiceModel ? undefined : config.providerConfigs[canonical]; - const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : config.sourceConfig; - if (existing && !effectiveCfg) { - return existing; - } - const rawConfig = resolveRawProviderConfig(config.rawConfig, canonical); - const rawBaseConfig = config.rawConfig as Record | undefined; - const rawProviders = asProviderConfigMap(config.rawConfig?.providers); - const resolvedProvider = getSpeechProvider(canonical, effectiveCfg); - let hasRawProviderConfig = - Object.hasOwn(rawProviders, canonical) || - (rawBaseConfig ? Object.hasOwn(rawBaseConfig, canonical) : false); - let rawProviderConfig = rawProviders[canonical] ?? rawBaseConfig?.[canonical]; - if (!hasRawProviderConfig) { - for (const alias of resolvedProvider?.aliases ?? []) { - const normalizedAlias = normalizeSpeechProviderId(alias); - if (!normalizedAlias) { - continue; - } - if (Object.hasOwn(rawProviders, normalizedAlias)) { - hasRawProviderConfig = true; - rawProviderConfig = rawProviders[normalizedAlias]; - break; - } - if (rawBaseConfig && Object.hasOwn(rawBaseConfig, normalizedAlias)) { - hasRawProviderConfig = true; - rawProviderConfig = rawBaseConfig[normalizedAlias]; - break; - } - } - } - const compatRawProviderConfig = applyVoiceModelToSpeechProviderConfig({ - cfg: effectiveCfg, - providerId: canonical, - providerConfig: withSpeakerSelectionCompat(asProviderConfig(rawProviderConfig)), - provider: resolvedProvider, - voiceModel, - }); - const shouldInjectCanonicalProviderConfig = - hasRawProviderConfig || Boolean(voiceModel) || Object.keys(rawProviders).length === 0; - const rawConfigForProvider = { - ...rawBaseConfig, - providers: shouldInjectCanonicalProviderConfig - ? { - ...rawProviders, - [canonical]: compatRawProviderConfig, - } - : rawProviders, - ...(shouldInjectCanonicalProviderConfig ? { [canonical]: compatRawProviderConfig } : {}), - }; - const next = withSpeakerSelectionCompat( - effectiveCfg && resolvedProvider?.resolveConfig - ? resolvedProvider.resolveConfig({ - cfg: effectiveCfg, - rawConfig: rawConfigForProvider, - timeoutMs: resolveSpeechProviderTimeoutMs({ config, provider: resolvedProvider }), - }) - : applyVoiceModelToSpeechProviderConfig({ - cfg: effectiveCfg, - providerId: canonical, - providerConfig: rawConfig, - provider: resolvedProvider, - voiceModel, - }), - ); - if (!voiceModel) { - config.providerConfigs[canonical] = next; - } - return next; -} - -export function getResolvedSpeechProviderConfig( - config: ResolvedTtsConfig, - providerId: string, - cfg?: OpenClawConfig, -): SpeechProviderConfig { - const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : config.sourceConfig; - const canonical = - canonicalizeSpeechProviderId(providerId, effectiveCfg) ?? - normalizeConfiguredSpeechProviderId(providerId) ?? - normalizeLowercaseStringOrEmpty(providerId); - return resolveLazyProviderConfig(config, canonical, effectiveCfg); -} - -function getResolvedSpeechProviderConfigForVoiceModel(params: { - config: ResolvedTtsConfig; - providerId: string; - cfg: OpenClawConfig; - voiceModel?: VoiceModelRef; -}): SpeechProviderConfig { - if (!params.voiceModel) { - return getResolvedSpeechProviderConfig(params.config, params.providerId, params.cfg); - } - const effectiveCfg = resolveTtsRuntimeConfig(params.cfg); - const canonical = - canonicalizeSpeechProviderId(params.providerId, effectiveCfg) ?? - normalizeConfiguredSpeechProviderId(params.providerId) ?? - normalizeLowercaseStringOrEmpty(params.providerId); - return resolveLazyProviderConfig(params.config, canonical, effectiveCfg, params.voiceModel); -} export function getTtsProvider(config: ResolvedTtsConfig, prefsPath: string): TtsProvider { - const prefs = readPrefs(prefsPath); - const prefsProvider = - canonicalizeSpeechProviderId(prefs.tts?.provider) ?? - normalizeConfiguredSpeechProviderId(prefs.tts?.provider); - if (prefsProvider) { - return prefsProvider; - } - const activePersona = resolveTtsPersonaFromPrefs(config, prefs); - const personaProvider = - canonicalizeSpeechProviderId(activePersona?.provider, config.sourceConfig) ?? - normalizeConfiguredSpeechProviderId(activePersona?.provider); - if (personaProvider && getSpeechProvider(personaProvider, config.sourceConfig)) { - return personaProvider; - } - if (config.providerSource === "config") { - return normalizeConfiguredSpeechProviderId(config.provider) ?? config.provider; - } - const configuredVoiceProvider = resolveConfiguredSpeechVoiceModelRefs(config.sourceConfig)[0] - ?.provider; - if (configuredVoiceProvider && getSpeechProvider(configuredVoiceProvider, config.sourceConfig)) { - return configuredVoiceProvider; - } - - const effectiveCfg = config.sourceConfig; - for (const provider of sortSpeechProvidersForAutoSelection(effectiveCfg)) { - if (isTtsProviderConfigured(config, provider.id, effectiveCfg)) { - return provider.id; - } - } - return config.provider; + return resolveTtsProvider(config, prefsPath); } -export type PreparedTtsRequest = { - cfg: OpenClawConfig; - directives: TtsDirectiveParseResult; -}; - -/** Merge a surface TTS override and resolve its inline synthesis directives. */ -export function prepareTtsRequest(params: { - cfg: OpenClawConfig; - override?: TtsConfig; - text: string; -}): PreparedTtsRequest { - const cfg = params.override - ? { - ...params.cfg, - tts: mergeDeep(params.cfg.tts ?? {}, params.override) as TtsConfig, - } - : params.cfg; - const config = resolveTtsConfig(cfg); - const directives = parseTtsDirectives(params.text, config.modelOverrides, { - cfg, - providerConfigs: config.providerConfigs, - preferredProviderId: getTtsProvider(config, resolveTtsPrefsPath(config)), - }); - return { cfg, directives }; -} - -export function resolveExplicitTtsOverrides(params: { - cfg: OpenClawConfig; - prefsPath?: string; - provider?: string; - modelId?: string; - voiceId?: string; - agentId?: string; - channelId?: string; - accountId?: string; -}): TtsDirectiveOverrides { - const cfg = resolveTtsRuntimeConfig(params.cfg); - const providerInput = params.provider?.trim(); - const modelId = params.modelId?.trim(); - const voiceId = params.voiceId?.trim(); - const config = resolveTtsConfig(cfg, { - agentId: params.agentId, - channelId: params.channelId, - accountId: params.accountId, - }); - const prefsPath = params.prefsPath ?? resolveTtsPrefsPath(config); - const selectedProvider = - canonicalizeSpeechProviderId(providerInput, cfg) ?? - (modelId || voiceId ? getTtsProvider(config, prefsPath) : undefined); - - if (providerInput && !selectedProvider) { - throw new Error(`Unknown TTS provider "${providerInput}".`); - } - - if (!modelId && !voiceId) { - return selectedProvider ? { provider: selectedProvider } : {}; - } - - if (!selectedProvider) { - throw new Error("TTS model or voice overrides require a resolved provider."); - } - - const provider = getSpeechProvider(selectedProvider, cfg); - if (!provider) { - throw new Error(`speech provider ${selectedProvider} is not registered`); - } - if (!provider.resolveTalkOverrides) { - throw new Error( - `TTS provider "${selectedProvider}" does not support model or voice overrides.`, - ); - } - - const providerOverrides = provider.resolveTalkOverrides({ - talkProviderConfig: {}, - params: { - ...(voiceId ? { voiceId } : {}), - ...(modelId ? { modelId } : {}), - }, - }); - if ((voiceId || modelId) && (!providerOverrides || Object.keys(providerOverrides).length === 0)) { - throw new Error( - `TTS provider "${selectedProvider}" ignored the requested model or voice overrides.`, - ); - } - - const overridesRecord = providerOverrides as SpeechProviderOverrides; - return { - provider: selectedProvider, - providerOverrides: { - [provider.id]: overridesRecord, - }, - }; -} - -export function getLastTtsAttempt(): TtsStatusEntry | undefined { - return lastTtsAttempt; -} - -export function setLastTtsAttempt(entry: TtsStatusEntry | undefined): void { - lastTtsAttempt = entry; -} - -function supportsNativeVoiceNoteTts(channel: string | undefined): boolean { - return resolveChannelTtsVoiceDelivery(channel) !== undefined; -} - -function supportsTranscodedVoiceNoteTts(channel: string | undefined): boolean { - const delivery = resolveChannelTtsVoiceDelivery(channel); - return delivery?.synthesisTarget === "voice-note" && delivery.transcodesAudio === true; -} - -function resolveTtsSynthesisTarget(channel: string | undefined): "audio-file" | "voice-note" { - return resolveChannelTtsVoiceDelivery(channel)?.synthesisTarget ?? "audio-file"; -} - -function supportsAudioFileVoiceMemoOutput(params: { - fileExtension?: string; - outputFormat?: string; - audioFileFormats?: readonly string[]; -}): boolean { - const formats = new Set(params.audioFileFormats?.map((format) => format.trim().toLowerCase())); - if (formats.size === 0) { - return false; - } - const extension = params.fileExtension?.trim().toLowerCase(); - if (extension && formats.has(extension.replace(/^\./, ""))) { - return true; - } - const outputFormat = params.outputFormat?.trim().toLowerCase(); - return outputFormat ? formats.has(outputFormat) : false; -} - -function shouldDeliverTtsAsVoice(params: { - channel: string | undefined; - target: "audio-file" | "voice-note" | undefined; - voiceCompatible: boolean | undefined; - fileExtension?: string; - outputFormat?: string; -}): boolean { - const delivery = resolveChannelTtsVoiceDelivery(params.channel); - if (!delivery) { - return false; - } - if (delivery.synthesisTarget === "audio-file") { - return ( - params.target === "audio-file" && - supportsAudioFileVoiceMemoOutput({ - fileExtension: params.fileExtension, - outputFormat: params.outputFormat, - audioFileFormats: delivery.audioFileFormats, - }) - ); - } - if (params.target !== "voice-note") { - return false; - } - return params.voiceCompatible === true || delivery.transcodesAudio === true; -} - -export function resolveTtsProviderOrder(primary: TtsProvider, cfg?: OpenClawConfig): TtsProvider[] { - const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : undefined; - const normalizedPrimary = canonicalizeSpeechProviderId(primary, effectiveCfg) ?? primary; - const ordered = new Set([normalizedPrimary]); - for (const ref of resolveVoiceModelRefs(effectiveCfg?.agents?.defaults?.voiceModel)) { - const provider = canonicalizeSpeechProviderId(ref.provider, effectiveCfg) ?? ref.provider; - if (provider !== normalizedPrimary) { - ordered.add(provider); - } - } - for (const provider of sortSpeechProvidersForAutoSelection(effectiveCfg)) { - const normalized = provider.id; - if (normalized !== normalizedPrimary) { - ordered.add(normalized); - } - } - return [...ordered]; -} - -function resolveTtsProviderCandidates( - primary: TtsProvider, - cfg?: OpenClawConfig, -): VoiceProviderCandidate[] { - const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : undefined; - const normalizedPrimary = canonicalizeSpeechProviderId(primary, effectiveCfg) ?? primary; - return resolveVoiceProviderCandidates({ - primaryProvider: normalizedPrimary, - providers: sortSpeechProvidersForAutoSelection(effectiveCfg), - voiceModelConfig: effectiveCfg?.agents?.defaults?.voiceModel, - }); -} - -function resolvePrimaryTtsProviderCandidate( - primary: TtsProvider, - cfg?: OpenClawConfig, -): VoiceProviderCandidate { - const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : undefined; - return resolvePrimaryVoiceProviderCandidate({ - primaryProvider: canonicalizeSpeechProviderId(primary, effectiveCfg) ?? primary, - providers: sortSpeechProvidersForAutoSelection(effectiveCfg), - voiceModelConfig: effectiveCfg?.agents?.defaults?.voiceModel, - }); -} - -export function isTtsProviderConfigured( - config: ResolvedTtsConfig, - provider: TtsProvider, - cfg?: OpenClawConfig, -): boolean { - try { - const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : config.sourceConfig; - const resolvedProvider = getSpeechProvider(provider, effectiveCfg); - if (!resolvedProvider) { - return false; - } - return ( - resolvedProvider.isConfigured({ - cfg: effectiveCfg, - providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, effectiveCfg), - timeoutMs: resolveSpeechProviderTimeoutMs({ config, provider: resolvedProvider }), - }) ?? false - ); - } catch { - // Configuration probes drive provider selection and status catalogs. A - // malformed provider config must not hide other usable providers. - return false; - } -} - -function formatTtsProviderError(provider: TtsProvider, err: unknown): string { - const error = err instanceof Error ? err : new Error(String(err)); - if (error.name === "AbortError") { - return `${provider}: request timed out`; - } - return `${provider}: ${redactSensitiveText(error.message)}`; -} - -function sanitizeTtsErrorForLog(err: unknown): string { - const raw = formatErrorMessage(err); - return redactSensitiveText(raw).replace(/\r/g, "\\r").replace(/\n/g, "\\n").replace(/\t/g, "\\t"); -} - -function buildTtsFailureResult( - errors: string[], - attemptedProviders?: string[], - attempts?: TtsProviderAttempt[], - persona?: string, -): { - success: false; - error: string; - attemptedProviders?: string[]; - attempts?: TtsProviderAttempt[]; - persona?: string; -} { - return { - success: false, - error: `TTS conversion failed: ${errors.join("; ") || "no providers available"}`, - attemptedProviders, - attempts, - persona, - }; -} - -type TtsProviderReadyResolution = - | { - kind: "ready"; - provider: NonNullable>; - providerConfig: SpeechProviderConfig; - personaProviderConfig?: SpeechProviderConfig; - synthesisPersona?: ResolvedTtsPersona; - personaBinding: "applied" | "missing" | "none"; - } - | { - kind: "skip"; - reasonCode: "no_provider_registered" | "not_configured" | "unsupported_for_telephony"; - message: string; - personaBinding?: "missing"; - }; - -function resolveReadySpeechProvider(params: { - provider: TtsProvider; - cfg: OpenClawConfig; - config: ResolvedTtsConfig; - persona?: ResolvedTtsPersona; - voiceModel?: VoiceModelRef; - requireTelephony?: boolean; -}): TtsProviderReadyResolution { - const resolvedProvider = getSpeechProvider(params.provider, params.cfg); - if (!resolvedProvider) { - return { - kind: "skip", - reasonCode: "no_provider_registered", - message: `${params.provider}: no provider registered`, - }; - } - const providerConfig = getResolvedSpeechProviderConfigForVoiceModel({ - config: params.config, - providerId: resolvedProvider.id, - cfg: params.cfg, - voiceModel: params.voiceModel, - }); - const merged = mergeProviderConfigWithPersona({ - providerConfig, - persona: params.persona, - providerId: resolvedProvider.id, - }); - if (params.persona?.fallbackPolicy === "fail" && merged.personaBinding === "missing") { - return { - kind: "skip", - reasonCode: "not_configured", - message: `${params.provider}: persona ${params.persona.id} has no provider binding`, - personaBinding: "missing", - }; - } - if ( - !resolvedProvider.isConfigured({ - cfg: params.cfg, - providerConfig: merged.providerConfig, - timeoutMs: resolveSpeechProviderTimeoutMs({ - config: params.config, - provider: resolvedProvider, - }), - }) - ) { - return { - kind: "skip", - reasonCode: "not_configured", - message: `${params.provider}: not configured`, - }; - } - if (params.requireTelephony && !resolvedProvider.synthesizeTelephony) { - return { - kind: "skip", - reasonCode: "unsupported_for_telephony", - message: `${params.provider}: unsupported for telephony`, - }; - } - return { - kind: "ready", - provider: resolvedProvider, - providerConfig: merged.providerConfig, - personaProviderConfig: merged.personaProviderConfig, - synthesisPersona: - params.persona?.fallbackPolicy === "provider-defaults" && merged.personaBinding === "missing" - ? undefined - : params.persona, - personaBinding: merged.personaBinding, - }; -} - -async function prepareSpeechSynthesis(params: { - provider: NonNullable>; - text: string; - cfg: OpenClawConfig; - providerConfig: SpeechProviderConfig; - providerOverrides?: SpeechProviderOverrides; - persona?: ResolvedTtsPersona; - personaProviderConfig?: SpeechProviderConfig; - target: "audio-file" | "voice-note" | "telephony"; - timeoutMs: number; -}): Promise<{ - text: string; - providerConfig: SpeechProviderConfig; - providerOverrides?: SpeechProviderOverrides; -}> { - if (!params.provider.prepareSynthesis) { - return { - text: params.text, - providerConfig: params.providerConfig, - providerOverrides: params.providerOverrides, - }; - } - const prepared = await params.provider.prepareSynthesis({ - text: params.text, - cfg: params.cfg, - providerConfig: params.providerConfig, - providerOverrides: params.providerOverrides, - persona: params.persona, - personaProviderConfig: params.personaProviderConfig, - target: params.target, - timeoutMs: params.timeoutMs, - }); - return { - text: prepared?.text ?? params.text, - providerConfig: prepared?.providerConfig - ? { ...params.providerConfig, ...prepared.providerConfig } - : params.providerConfig, - providerOverrides: prepared?.providerOverrides - ? { ...params.providerOverrides, ...prepared.providerOverrides } - : params.providerOverrides, - }; -} - -function resolveTtsRequestSetup(params: { - text: string; - cfg: OpenClawConfig; - prefsPath?: string; - providerOverride?: TtsProvider; - disableFallback?: boolean; - agentId?: string; - channelId?: string; - accountId?: string; -}): - | { - cfg: OpenClawConfig; - config: ResolvedTtsConfig; - persona?: ResolvedTtsPersona; - providers: VoiceProviderCandidate[]; - } - | { - error: string; - } { - const cfg = resolveTtsRuntimeConfig(params.cfg); - const config = resolveTtsConfig(cfg, { - agentId: params.agentId, - channelId: params.channelId, - accountId: params.accountId, - }); - const prefsPath = params.prefsPath ?? resolveTtsPrefsPath(config); - if (params.text.length > config.maxTextLength) { - return { - error: `Text too long (${params.text.length} chars, max ${config.maxTextLength})`, - }; - } - - const userProvider = getTtsProvider(config, prefsPath); - const provider = canonicalizeSpeechProviderId(params.providerOverride, cfg) ?? userProvider; - return { - cfg, - config, - persona: getTtsPersona(config, prefsPath), - providers: params.disableFallback - ? [resolvePrimaryTtsProviderCandidate(provider, cfg)] - : resolveTtsProviderCandidates(provider, cfg), - }; -} - -function readTtsResultString(value: unknown): string | undefined { - return typeof value === "string" && value.trim() ? value.trim() : undefined; -} - -function resolveTtsResultModel( - providerConfig: SpeechProviderConfig, - providerOverrides?: SpeechProviderOverrides, -): string | undefined { - return ( - readTtsResultString(providerOverrides?.modelId) ?? - readTtsResultString(providerOverrides?.model) ?? - readTtsResultString(providerConfig.modelId) ?? - readTtsResultString(providerConfig.model) - ); -} - -function resolveTtsResultVoice( - providerConfig: SpeechProviderConfig, - providerOverrides?: SpeechProviderOverrides, -): string | undefined { - return ( - readTtsResultString(providerOverrides?.speakerVoiceId) ?? - readTtsResultString(providerOverrides?.speakerVoice) ?? - readTtsResultString(providerOverrides?.voiceId) ?? - readTtsResultString(providerOverrides?.voiceName) ?? - readTtsResultString(providerOverrides?.voice) ?? - readTtsResultString(providerConfig.speakerVoiceId) ?? - readTtsResultString(providerConfig.speakerVoice) ?? - readTtsResultString(providerConfig.voiceId) ?? - readTtsResultString(providerConfig.voiceName) ?? - readTtsResultString(providerConfig.voice) - ); -} - -export async function textToSpeech(params: { - text: string; - cfg: OpenClawConfig; - prefsPath?: string; - channel?: string; - overrides?: TtsDirectiveOverrides; - disableFallback?: boolean; - timeoutMs?: number; - agentId?: string; - accountId?: string; -}): Promise { - const synthesis = await synthesizeSpeech(params); - if (!synthesis.success || !synthesis.audioBuffer || !synthesis.fileExtension) { - return { - success: false, - error: synthesis.error ?? "TTS conversion failed", - persona: synthesis.persona, - attemptedProviders: synthesis.attemptedProviders, - attempts: synthesis.attempts, - }; - } - - let audioBuffer = synthesis.audioBuffer; - let fileExtension = synthesis.fileExtension; - let outputFormat = synthesis.outputFormat; - const transcoded = await maybePreTranscodeForVoiceDelivery({ - channel: params.channel, - target: synthesis.target, - audioBuffer, - fileExtension, - outputFormat, - }); - if (transcoded) { - audioBuffer = transcoded.audioBuffer; - fileExtension = transcoded.fileExtension; - outputFormat = transcoded.outputFormat; - } - - const temp = tempWorkspaceSync({ - rootDir: resolvePreferredOpenClawTmpDir(), - prefix: "tts-", - }); - const audioPath = temp.write(`voice-${Date.now()}${fileExtension}`, audioBuffer); - scheduleCleanup(temp.dir); - - return { - success: true, - audioPath, - latencyMs: synthesis.latencyMs, - provider: synthesis.provider, - persona: synthesis.persona, - fallbackFrom: synthesis.fallbackFrom, - attemptedProviders: synthesis.attemptedProviders, - attempts: synthesis.attempts, - outputFormat, - voiceCompatible: synthesis.voiceCompatible, - audioAsVoice: shouldDeliverTtsAsVoice({ - channel: params.channel, - target: synthesis.target, - voiceCompatible: synthesis.voiceCompatible, - fileExtension, - outputFormat, - }), - target: synthesis.target, - }; -} - -async function maybePreTranscodeForVoiceDelivery(params: { - channel: string | undefined; - target: "audio-file" | "voice-note" | undefined; - audioBuffer: Buffer; - fileExtension: string; - outputFormat?: string; -}): Promise<{ audioBuffer: Buffer; fileExtension: string; outputFormat?: string } | undefined> { - if (params.target !== "audio-file") { - return undefined; - } - const delivery = resolveChannelTtsVoiceDelivery(params.channel); - const preferred = delivery?.preferAudioFileFormat?.trim().toLowerCase(); - if (!preferred) { - return undefined; - } - const sourceExt = params.fileExtension.trim().toLowerCase().replace(/^\./, ""); - if (sourceExt === preferred) { - return undefined; - } - const outcome = await transcodeAudioBuffer({ - audioBuffer: params.audioBuffer, - sourceExtension: sourceExt, - targetExtension: preferred, - }); - if (!outcome.ok) { - if (outcome.reason === "transcoder-failed") { - // Surface only the case where the host actually attempted the transcode - // and it broke. The other reasons ("no-recipe", "noop-same-container", - // "platform-unsupported", "invalid-extension") are by-design skips and - // would just be log noise. This is the line that tells you "the channel - // asked for a pre-encode, the host had a recipe for it, and it failed" - // — i.e. the case where #72506 silently regresses. - logVerbose( - `TTS: pre-transcode ${sourceExt}->${preferred} for channel=${params.channel ?? "?"} failed: ${outcome.detail ?? "unknown"}`, - ); - } - return undefined; - } - return { - audioBuffer: outcome.buffer, - fileExtension: `.${preferred}`, - outputFormat: preferred, - }; -} - -export async function synthesizeSpeech(params: { - text: string; - cfg: OpenClawConfig; - prefsPath?: string; - channel?: string; - overrides?: TtsDirectiveOverrides; - disableFallback?: boolean; - timeoutMs?: number; - agentId?: string; - accountId?: string; -}): Promise { - assertSpeechRuntimeAvailable(); - const setup = resolveTtsRequestSetup({ - text: params.text, - cfg: params.cfg, - prefsPath: params.prefsPath, - providerOverride: params.overrides?.provider, - disableFallback: params.disableFallback, - agentId: params.agentId, - channelId: params.channel, - accountId: params.accountId, - }); - if ("error" in setup) { - return { success: false, error: setup.error }; - } - - const { cfg, config, persona, providers } = setup; - const textForSynthesis = normalizeSpeechText(params.text); - const target = resolveTtsSynthesisTarget(params.channel); - - const errors: string[] = []; - const attemptedProviders: string[] = []; - const attempts: TtsProviderAttempt[] = []; - const primaryProvider = providers[0]?.provider; - logVerbose( - `TTS: starting with provider ${primaryProvider}, fallbacks: ${ - providers - .slice(1) - .map((entry) => entry.provider) - .join(", ") || "none" - }`, - ); - - for (const { provider, voiceModel } of providers) { - attemptedProviders.push(provider); - const providerStart = Date.now(); - try { - const resolvedProvider = resolveReadySpeechProvider({ - provider, - cfg, - config, - persona, - voiceModel, - }); - if (resolvedProvider.kind === "skip") { - errors.push(resolvedProvider.message); - attempts.push({ - provider, - outcome: "skipped", - reasonCode: resolvedProvider.reasonCode, - persona: persona?.id, - ...(resolvedProvider.personaBinding - ? { personaBinding: resolvedProvider.personaBinding } - : {}), - error: resolvedProvider.message, - }); - logVerbose(`TTS: provider ${provider} skipped (${resolvedProvider.message})`); - continue; - } - const timeoutMs = resolveSpeechProviderTimeoutMs({ - timeoutMs: params.timeoutMs ?? voiceModel?.timeoutMs, - config, - provider: resolvedProvider.provider, - }); - const prepared = await prepareSpeechSynthesis({ - provider: resolvedProvider.provider, - text: textForSynthesis, - cfg, - providerConfig: resolvedProvider.providerConfig, - providerOverrides: params.overrides?.providerOverrides?.[resolvedProvider.provider.id], - persona: resolvedProvider.synthesisPersona, - personaProviderConfig: resolvedProvider.personaProviderConfig, - target, - timeoutMs, - }); - const synthesis = await resolvedProvider.provider.synthesize({ - text: prepared.text, - cfg, - providerConfig: prepared.providerConfig, - target, - providerOverrides: prepared.providerOverrides, - timeoutMs, - }); - const latencyMs = Date.now() - providerStart; - attempts.push({ - provider, - outcome: "success", - reasonCode: "success", - persona: persona?.id, - personaBinding: resolvedProvider.personaBinding, - latencyMs, - }); - return { - success: true, - audioBuffer: synthesis.audioBuffer, - latencyMs, - provider, - providerModel: resolveTtsResultModel(prepared.providerConfig, prepared.providerOverrides), - providerVoice: resolveTtsResultVoice(prepared.providerConfig, prepared.providerOverrides), - persona: persona?.id, - fallbackFrom: provider !== primaryProvider ? primaryProvider : undefined, - attemptedProviders, - attempts, - outputFormat: synthesis.outputFormat, - voiceCompatible: synthesis.voiceCompatible, - fileExtension: synthesis.fileExtension, - target, - }; - } catch (err) { - const errorMsg = formatTtsProviderError(provider, err); - const latencyMs = Date.now() - providerStart; - errors.push(errorMsg); - attempts.push({ - provider, - outcome: "failed", - reasonCode: - err instanceof Error && err.name === "AbortError" ? "timeout" : "provider_error", - latencyMs, - persona: persona?.id, - personaBinding: - resolvePersonaProviderConfig(persona, provider) != null - ? "applied" - : persona - ? "missing" - : "none", - error: errorMsg, - }); - const rawError = sanitizeTtsErrorForLog(err); - if (provider === primaryProvider) { - const hasFallbacks = providers.length > 1; - logVerbose( - `TTS: primary provider ${provider} failed (${rawError})${hasFallbacks ? "; trying fallback providers." : "; no fallback providers configured."}`, - ); - } else { - logVerbose(`TTS: ${provider} failed (${rawError}); trying next provider.`); - } - } - } - - return buildTtsFailureResult(errors, attemptedProviders, attempts, persona?.id); -} - -export async function streamSpeech(params: { - text: string; - cfg: OpenClawConfig; - prefsPath?: string; - channel?: string; - overrides?: TtsDirectiveOverrides; - disableFallback?: boolean; - timeoutMs?: number; - agentId?: string; - accountId?: string; -}): Promise { - assertSpeechRuntimeAvailable(); - const setup = resolveTtsRequestSetup({ - text: params.text, - cfg: params.cfg, - prefsPath: params.prefsPath, - providerOverride: params.overrides?.provider, - disableFallback: params.disableFallback, - agentId: params.agentId, - channelId: params.channel, - accountId: params.accountId, - }); - if ("error" in setup) { - return { success: false, error: setup.error }; - } - - const { cfg, config, persona, providers } = setup; - const target = resolveTtsSynthesisTarget(params.channel); - const errors: string[] = []; - const attemptedProviders: string[] = []; - const attempts: TtsProviderAttempt[] = []; - const primaryProvider = providers[0]?.provider; - logVerbose( - `TTS stream: starting with provider ${primaryProvider}, fallbacks: ${ - providers - .slice(1) - .map((entry) => entry.provider) - .join(", ") || "none" - }`, - ); - - for (const { provider, voiceModel } of providers) { - attemptedProviders.push(provider); - const providerStart = Date.now(); - try { - const resolvedProvider = resolveReadySpeechProvider({ - provider, - cfg, - config, - persona, - voiceModel, - }); - if (resolvedProvider.kind === "skip") { - errors.push(resolvedProvider.message); - attempts.push({ - provider, - outcome: "skipped", - reasonCode: resolvedProvider.reasonCode, - persona: persona?.id, - ...(resolvedProvider.personaBinding - ? { personaBinding: resolvedProvider.personaBinding } - : {}), - error: resolvedProvider.message, - }); - logVerbose(`TTS stream: provider ${provider} skipped (${resolvedProvider.message})`); - continue; - } - if (!resolvedProvider.provider.streamSynthesize) { - const message = `${provider} does not support streaming TTS`; - errors.push(message); - attempts.push({ - provider, - outcome: "skipped", - reasonCode: "unsupported_for_streaming", - persona: persona?.id, - personaBinding: resolvedProvider.personaBinding, - error: message, - }); - logVerbose(`TTS stream: provider ${provider} skipped (${message})`); - continue; - } - const timeoutMs = resolveSpeechProviderTimeoutMs({ - timeoutMs: params.timeoutMs ?? voiceModel?.timeoutMs, - config, - provider: resolvedProvider.provider, - }); - const prepared = await prepareSpeechSynthesis({ - provider: resolvedProvider.provider, - text: params.text, - cfg, - providerConfig: resolvedProvider.providerConfig, - providerOverrides: params.overrides?.providerOverrides?.[resolvedProvider.provider.id], - persona: resolvedProvider.synthesisPersona, - personaProviderConfig: resolvedProvider.personaProviderConfig, - target, - timeoutMs, - }); - const synthesis = await resolvedProvider.provider.streamSynthesize({ - text: prepared.text, - cfg, - providerConfig: prepared.providerConfig, - target, - providerOverrides: prepared.providerOverrides, - timeoutMs, - }); - const latencyMs = Date.now() - providerStart; - attempts.push({ - provider, - outcome: "success", - reasonCode: "success", - persona: persona?.id, - personaBinding: resolvedProvider.personaBinding, - latencyMs, - }); - return { - success: true, - audioStream: synthesis.audioStream, - latencyMs, - provider, - providerModel: resolveTtsResultModel(prepared.providerConfig, prepared.providerOverrides), - providerVoice: resolveTtsResultVoice(prepared.providerConfig, prepared.providerOverrides), - persona: persona?.id, - fallbackFrom: provider !== primaryProvider ? primaryProvider : undefined, - attemptedProviders, - attempts, - outputFormat: synthesis.outputFormat, - voiceCompatible: synthesis.voiceCompatible, - fileExtension: synthesis.fileExtension, - target, - release: synthesis.release, - }; - } catch (err) { - const errorMsg = formatTtsProviderError(provider, err); - const latencyMs = Date.now() - providerStart; - errors.push(errorMsg); - attempts.push({ - provider, - outcome: "failed", - reasonCode: - err instanceof Error && err.name === "AbortError" ? "timeout" : "provider_error", - latencyMs, - persona: persona?.id, - personaBinding: - resolvePersonaProviderConfig(persona, provider) != null - ? "applied" - : persona - ? "missing" - : "none", - error: errorMsg, - }); - const rawError = sanitizeTtsErrorForLog(err); - if (provider === primaryProvider) { - const hasFallbacks = providers.length > 1; - logVerbose( - `TTS stream: primary provider ${provider} failed (${rawError})${hasFallbacks ? "; trying fallback providers." : "; no fallback providers configured."}`, - ); - } else { - logVerbose(`TTS stream: ${provider} failed (${rawError}); trying next provider.`); - } - } - } - - return buildTtsFailureResult(errors, attemptedProviders, attempts, persona?.id); -} - -export async function textToSpeechStream(params: { - text: string; - cfg: OpenClawConfig; - prefsPath?: string; - channel?: string; - overrides?: TtsDirectiveOverrides; - disableFallback?: boolean; - timeoutMs?: number; - agentId?: string; - accountId?: string; -}): Promise { - const synthesis = await streamSpeech(params); - if (!synthesis.success || !synthesis.audioStream || !synthesis.fileExtension) { - return { - success: false, - error: synthesis.error ?? "Streaming TTS conversion failed", - persona: synthesis.persona, - attemptedProviders: synthesis.attemptedProviders, - attempts: synthesis.attempts, - }; - } - return synthesis; -} - -export async function textToSpeechTelephony(params: { - text: string; - cfg: OpenClawConfig; - prefsPath?: string; - overrides?: TtsDirectiveOverrides; - timeoutMs?: number; -}): Promise { - assertSpeechRuntimeAvailable(); - const setup = resolveTtsRequestSetup({ - text: params.text, - cfg: params.cfg, - prefsPath: params.prefsPath, - providerOverride: params.overrides?.provider, - }); - if ("error" in setup) { - return { success: false, error: setup.error }; - } - - const { cfg, config, persona, providers } = setup; - const errors: string[] = []; - const attemptedProviders: string[] = []; - const attempts: TtsProviderAttempt[] = []; - const primaryProvider = providers[0]?.provider; - logVerbose( - `TTS telephony: starting with provider ${primaryProvider}, fallbacks: ${ - providers - .slice(1) - .map((entry) => entry.provider) - .join(", ") || "none" - }`, - ); - - for (const { provider, voiceModel } of providers) { - attemptedProviders.push(provider); - const providerStart = Date.now(); - try { - const resolvedProvider = resolveReadySpeechProvider({ - provider, - cfg, - config, - persona, - voiceModel, - requireTelephony: true, - }); - if (resolvedProvider.kind === "skip") { - errors.push(resolvedProvider.message); - attempts.push({ - provider, - outcome: "skipped", - reasonCode: resolvedProvider.reasonCode, - persona: persona?.id, - ...(resolvedProvider.personaBinding - ? { personaBinding: resolvedProvider.personaBinding } - : {}), - error: resolvedProvider.message, - }); - logVerbose(`TTS telephony: provider ${provider} skipped (${resolvedProvider.message})`); - continue; - } - const timeoutMs = resolveSpeechProviderTimeoutMs({ - timeoutMs: params.timeoutMs ?? voiceModel?.timeoutMs, - config, - provider: resolvedProvider.provider, - }); - const synthesizeTelephony = resolvedProvider.provider.synthesizeTelephony as NonNullable< - typeof resolvedProvider.provider.synthesizeTelephony - >; - const prepared = await prepareSpeechSynthesis({ - provider: resolvedProvider.provider, - text: params.text, - cfg, - providerConfig: resolvedProvider.providerConfig, - providerOverrides: params.overrides?.providerOverrides?.[resolvedProvider.provider.id], - persona: resolvedProvider.synthesisPersona, - personaProviderConfig: resolvedProvider.personaProviderConfig, - target: "telephony", - timeoutMs, - }); - const synthesis = await synthesizeTelephony({ - text: prepared.text, - cfg, - providerConfig: prepared.providerConfig, - providerOverrides: prepared.providerOverrides, - timeoutMs, - }); - const latencyMs = Date.now() - providerStart; - attempts.push({ - provider, - outcome: "success", - reasonCode: "success", - persona: persona?.id, - personaBinding: resolvedProvider.personaBinding, - latencyMs, - }); - - return { - success: true, - audioBuffer: synthesis.audioBuffer, - latencyMs, - provider, - providerModel: resolveTtsResultModel(prepared.providerConfig, prepared.providerOverrides), - providerVoice: resolveTtsResultVoice(prepared.providerConfig, prepared.providerOverrides), - persona: persona?.id, - fallbackFrom: provider !== primaryProvider ? primaryProvider : undefined, - attemptedProviders, - attempts, - outputFormat: synthesis.outputFormat, - sampleRate: synthesis.sampleRate, - }; - } catch (err) { - const errorMsg = formatTtsProviderError(provider, err); - const latencyMs = Date.now() - providerStart; - errors.push(errorMsg); - attempts.push({ - provider, - outcome: "failed", - reasonCode: - err instanceof Error && err.name === "AbortError" ? "timeout" : "provider_error", - latencyMs, - persona: persona?.id, - personaBinding: - resolvePersonaProviderConfig(persona, provider) != null - ? "applied" - : persona - ? "missing" - : "none", - error: errorMsg, - }); - const rawError = sanitizeTtsErrorForLog(err); - if (provider === primaryProvider) { - const hasFallbacks = providers.length > 1; - logVerbose( - `TTS telephony: primary provider ${provider} failed (${rawError})${hasFallbacks ? "; trying fallback providers." : "; no fallback providers configured."}`, - ); - } else { - logVerbose(`TTS telephony: ${provider} failed (${rawError}); trying next provider.`); - } - } - } - - return buildTtsFailureResult(errors, attemptedProviders, attempts, persona?.id); -} - -export async function listSpeechVoices(params: { - provider: string; - cfg?: OpenClawConfig; - config?: ResolvedTtsConfig; - apiKey?: string; - baseUrl?: string; -}): Promise { - assertSpeechRuntimeAvailable(); - const cfg = params.cfg ? resolveTtsRuntimeConfig(params.cfg) : undefined; - const provider = canonicalizeSpeechProviderId(params.provider, cfg); - if (!provider) { - throw new Error("speech provider id is required"); - } - const config = params.config ?? (cfg ? resolveTtsConfig(cfg) : undefined); - if (!config) { - throw new Error(`speech provider ${provider} requires cfg or resolved config`); - } - const resolvedProvider = getSpeechProvider(provider, cfg); - if (!resolvedProvider) { - throw new Error(`speech provider ${provider} is not registered`); - } - if (!resolvedProvider.listVoices) { - throw new Error(`speech provider ${provider} does not support voice listing`); - } - const timeoutMs = resolveSpeechProviderTimeoutMs({ - config, - provider: resolvedProvider, - }); - return await resolvedProvider.listVoices({ - cfg, - providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, cfg), - apiKey: params.apiKey, - baseUrl: params.baseUrl, - timeoutMs, - }); -} - -function hasLegacyFinalMediaDirective(text: string): boolean { - return /(?:^|\n)\s*MEDIA\s*:/i.test(text); -} - -export async function maybeApplyTtsToPayload(params: { - payload: ReplyPayload; - cfg: OpenClawConfig; - channel?: string; - kind?: "tool" | "block" | "final"; - inboundAudio?: boolean; - ttsAuto?: string; - agentId?: string; - accountId?: string; -}): Promise { - if (!isSpeechRuntimeAvailable()) { - return params.payload; - } - if (params.payload.isCompactionNotice) { - return params.payload; - } - const cfg = resolveTtsRuntimeConfig(params.cfg); - const { autoMode, config, prefsPath } = resolveTtsSettingsSnapshot({ - cfg, - sessionAuto: params.ttsAuto, - agentId: params.agentId, - channelId: params.channel, - accountId: params.accountId, - }); - if (autoMode === "off") { - return params.payload; - } - const activeProvider = getTtsProvider(config, prefsPath); - - const reply = resolveSendableOutboundReplyParts(params.payload); - const text = reply.text; - const directives = parseTtsDirectives(text, config.modelOverrides, { - cfg, - providerConfigs: config.providerConfigs, - preferredProviderId: activeProvider, - }); - if (directives.warnings.length > 0) { - logVerbose(`TTS: ignored directive overrides (${directives.warnings.join("; ")})`); - } - - if (isVerbose()) { - const effectiveProvider = directives.overrides?.provider - ? (canonicalizeSpeechProviderId(directives.overrides.provider, cfg) ?? activeProvider) - : activeProvider; - logVerbose( - `TTS: auto mode enabled (${autoMode}), channel=${params.channel}, selected provider=${effectiveProvider}, config.provider=${config.provider}, config.providerSource=${config.providerSource}`, - ); - } - - const cleanedText = directives.cleanedText; - const trimmedCleaned = cleanedText.trim(); - const visibleText = trimmedCleaned.length > 0 ? trimmedCleaned : ""; - const explicitTtsText = directives.ttsText?.trim() || ""; - const ttsText = explicitTtsText || visibleText; - - const nextPayload = - visibleText === text.trim() - ? params.payload - : { - ...params.payload, - text: visibleText.length > 0 ? visibleText : undefined, - }; - - if (autoMode === "tagged" && !directives.hasDirective) { - return nextPayload; - } - if (autoMode === "inbound" && params.inboundAudio !== true) { - return nextPayload; - } - - const mode = config.mode ?? "final"; - if (mode === "final" && params.kind && params.kind !== "final") { - return nextPayload; - } - - if (!ttsText.trim()) { - return nextPayload; - } - if (reply.hasMedia || hasLegacyFinalMediaDirective(text)) { - return nextPayload; - } - if (!explicitTtsText && ttsText.trim().length < 10) { - return nextPayload; - } - - const maxLength = getTtsMaxLength(prefsPath); - let textForAudio = ttsText.trim(); - let wasSummarized = false; - - if (!explicitTtsText && isCodeHeavySpeechText(textForAudio)) { - // The visible reply already carries code-heavy detail. Skip noisy voice-note audio instead of - // telling channel users to look at a screen they may not have. - return nextPayload; - } - - if (textForAudio.length > maxLength) { - if (!isSummarizationEnabled(prefsPath)) { - logVerbose( - `TTS: truncating long text (${textForAudio.length} > ${maxLength}), summarization disabled.`, - ); - textForAudio = `${truncateUtf16Safe(textForAudio, maxLength - 3)}...`; - } else { - try { - const summary = await summarizeText({ - text: textForAudio, - targetLength: maxLength, - cfg, - config, - timeoutMs: config.timeoutMs, - }); - textForAudio = summary.summary; - wasSummarized = true; - if (textForAudio.length > config.maxTextLength) { - logVerbose( - `TTS: summary exceeded hard limit (${textForAudio.length} > ${config.maxTextLength}); truncating.`, - ); - textForAudio = `${truncateUtf16Safe(textForAudio, config.maxTextLength - 3)}...`; - } - } catch (err) { - const error = err as Error; - logVerbose(`TTS: summarization failed, truncating instead: ${error.message}`); - textForAudio = `${truncateUtf16Safe(textForAudio, maxLength - 3)}...`; - } - } - } - - const normalizedTextForAudio = normalizeSpeechText(textForAudio); - if (!normalizedTextForAudio) { - return nextPayload; - } - if (!explicitTtsText && normalizedTextForAudio.length < 10) { - return nextPayload; - } - - const ttsStart = Date.now(); - const result = await textToSpeech({ - text: textForAudio, - cfg, - prefsPath, - channel: params.channel, - overrides: directives.overrides, - agentId: params.agentId, - accountId: params.accountId, - }); - - if (result.success && result.audioPath) { - lastTtsAttempt = { - timestamp: Date.now(), - success: true, - textLength: text.length, - summarized: wasSummarized, - provider: result.provider, - persona: result.persona, - fallbackFrom: result.fallbackFrom, - attemptedProviders: result.attemptedProviders, - attempts: result.attempts, - latencyMs: result.latencyMs, - }; - - const payloadWithAudio = { - ...nextPayload, - mediaUrl: result.audioPath, - audioAsVoice: result.audioAsVoice || params.payload.audioAsVoice, - spokenText: textForAudio, - trustedLocalMedia: true, - } as ReplyPayload; - return nextPayload.text?.trim() - ? markReplyPayloadAsTtsSupplement(payloadWithAudio) - : payloadWithAudio; - } - - lastTtsAttempt = { - timestamp: Date.now(), - success: false, - textLength: text.length, - summarized: wasSummarized, - persona: result.persona, - attemptedProviders: result.attemptedProviders, - attempts: result.attempts, - error: result.error, - }; - - const latency = Date.now() - ttsStart; - logVerbose(`TTS: conversion failed after ${latency}ms (${result.error ?? "unknown"}).`); - return nextPayload; -} +export { + getLastTtsAttempt, + listSpeechVoices, + maybeApplyTtsToPayload, + setLastTtsAttempt, +} from "./tts-payload.js"; +export { + getResolvedSpeechProviderConfig, + isTtsProviderConfigured, + resolveTtsProviderOrder, +} from "./tts-provider-resolution.js"; +export { + prepareTtsRequest, + resolveExplicitTtsOverrides, + type PreparedTtsRequest, +} from "./tts-request.js"; +export { streamSpeech, textToSpeechStream } from "./tts-streaming.js"; +export { synthesizeSpeech, textToSpeech } from "./tts-synthesis.js"; +export { textToSpeechTelephony } from "./tts-telephony.js"; +export type { + TtsResult, + TtsStreamResult, + TtsSynthesisResult, + TtsSynthesisStreamResult, + TtsTelephonyResult, +} from "./tts-types.js"; export const testApi = { parseTtsDirectives, @@ -1768,4 +58,3 @@ export const testApi = { formatTtsProviderError, sanitizeTtsErrorForLog, }; -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */