mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
refactor(speech-core): split TTS runtime (#112768)
* refactor(speech-core): split TTS runtime * refactor(speech-core): keep helper types private
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
18f70922de
commit
4501da6123
@@ -364,7 +364,6 @@ packages/model-catalog-core/src/model-catalog-normalize.ts
|
||||
packages/sdk/src/client.ts
|
||||
packages/sdk/src/index.test.ts
|
||||
packages/speech-core/src/tts.test.ts
|
||||
packages/speech-core/src/tts.ts
|
||||
packages/tool-call-repair/src/stream-normalizer.test.ts
|
||||
packages/tool-call-repair/src/stream-normalizer.ts
|
||||
src/acp/control-plane/manager.test.ts
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import {
|
||||
markReplyPayloadAsTtsSupplement,
|
||||
resolveSendableOutboundReplyParts,
|
||||
type ReplyPayload,
|
||||
} from "openclaw/plugin-sdk/reply-payload";
|
||||
import { isVerbose, logVerbose } from "openclaw/plugin-sdk/runtime-env";
|
||||
import {
|
||||
canonicalizeSpeechProviderId,
|
||||
getSpeechProvider,
|
||||
parseTtsDirectives,
|
||||
summarizeText,
|
||||
type SpeechVoiceOption,
|
||||
} from "openclaw/plugin-sdk/speech-core";
|
||||
import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
||||
import { assertSpeechRuntimeAvailable, isSpeechRuntimeAvailable } from "./runtime-availability.js";
|
||||
import { isCodeHeavySpeechText, normalizeSpeechText } from "./speech-text.js";
|
||||
import {
|
||||
getResolvedSpeechProviderConfig,
|
||||
resolveSpeechProviderTimeoutMs,
|
||||
resolveTtsProvider,
|
||||
} from "./tts-provider-resolution.js";
|
||||
import {
|
||||
getTtsMaxLength,
|
||||
isSummarizationEnabled,
|
||||
resolveTtsConfig,
|
||||
resolveTtsRuntimeConfig,
|
||||
resolveTtsSettingsSnapshot,
|
||||
type ResolvedTtsConfig,
|
||||
} from "./tts-settings.js";
|
||||
import { textToSpeech } from "./tts-synthesis.js";
|
||||
import type { TtsStatusEntry } from "./tts-types.js";
|
||||
|
||||
let lastTtsAttempt: TtsStatusEntry | undefined;
|
||||
|
||||
export function getLastTtsAttempt(): TtsStatusEntry | undefined {
|
||||
return lastTtsAttempt;
|
||||
}
|
||||
|
||||
export function setLastTtsAttempt(entry: TtsStatusEntry | undefined): void {
|
||||
lastTtsAttempt = entry;
|
||||
}
|
||||
|
||||
export async function listSpeechVoices(params: {
|
||||
provider: string;
|
||||
cfg?: OpenClawConfig;
|
||||
config?: ResolvedTtsConfig;
|
||||
apiKey?: string;
|
||||
baseUrl?: string;
|
||||
}): Promise<SpeechVoiceOption[]> {
|
||||
assertSpeechRuntimeAvailable();
|
||||
const cfg = params.cfg ? resolveTtsRuntimeConfig(params.cfg) : undefined;
|
||||
const provider = canonicalizeSpeechProviderId(params.provider, cfg);
|
||||
if (!provider) {
|
||||
throw new Error("speech provider id is required");
|
||||
}
|
||||
const config = params.config ?? (cfg ? resolveTtsConfig(cfg) : undefined);
|
||||
if (!config) {
|
||||
throw new Error(`speech provider ${provider} requires cfg or resolved config`);
|
||||
}
|
||||
const resolvedProvider = getSpeechProvider(provider, cfg);
|
||||
if (!resolvedProvider) {
|
||||
throw new Error(`speech provider ${provider} is not registered`);
|
||||
}
|
||||
if (!resolvedProvider.listVoices) {
|
||||
throw new Error(`speech provider ${provider} does not support voice listing`);
|
||||
}
|
||||
const timeoutMs = resolveSpeechProviderTimeoutMs({
|
||||
config,
|
||||
provider: resolvedProvider,
|
||||
});
|
||||
return await resolvedProvider.listVoices({
|
||||
cfg,
|
||||
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, cfg),
|
||||
apiKey: params.apiKey,
|
||||
baseUrl: params.baseUrl,
|
||||
timeoutMs,
|
||||
});
|
||||
}
|
||||
|
||||
function hasLegacyFinalMediaDirective(text: string): boolean {
|
||||
return /(?:^|\n)\s*MEDIA\s*:/i.test(text);
|
||||
}
|
||||
|
||||
export async function maybeApplyTtsToPayload(params: {
|
||||
payload: ReplyPayload;
|
||||
cfg: OpenClawConfig;
|
||||
channel?: string;
|
||||
kind?: "tool" | "block" | "final";
|
||||
inboundAudio?: boolean;
|
||||
ttsAuto?: string;
|
||||
agentId?: string;
|
||||
accountId?: string;
|
||||
}): Promise<ReplyPayload> {
|
||||
if (!isSpeechRuntimeAvailable()) {
|
||||
return params.payload;
|
||||
}
|
||||
if (params.payload.isCompactionNotice) {
|
||||
return params.payload;
|
||||
}
|
||||
const cfg = resolveTtsRuntimeConfig(params.cfg);
|
||||
const { autoMode, config, prefsPath } = resolveTtsSettingsSnapshot({
|
||||
cfg,
|
||||
sessionAuto: params.ttsAuto,
|
||||
agentId: params.agentId,
|
||||
channelId: params.channel,
|
||||
accountId: params.accountId,
|
||||
});
|
||||
if (autoMode === "off") {
|
||||
return params.payload;
|
||||
}
|
||||
const activeProvider = resolveTtsProvider(config, prefsPath);
|
||||
|
||||
const reply = resolveSendableOutboundReplyParts(params.payload);
|
||||
const text = reply.text;
|
||||
const directives = parseTtsDirectives(text, config.modelOverrides, {
|
||||
cfg,
|
||||
providerConfigs: config.providerConfigs,
|
||||
preferredProviderId: activeProvider,
|
||||
});
|
||||
if (directives.warnings.length > 0) {
|
||||
logVerbose(`TTS: ignored directive overrides (${directives.warnings.join("; ")})`);
|
||||
}
|
||||
|
||||
if (isVerbose()) {
|
||||
const effectiveProvider = directives.overrides?.provider
|
||||
? (canonicalizeSpeechProviderId(directives.overrides.provider, cfg) ?? activeProvider)
|
||||
: activeProvider;
|
||||
logVerbose(
|
||||
`TTS: auto mode enabled (${autoMode}), channel=${params.channel}, selected provider=${effectiveProvider}, config.provider=${config.provider}, config.providerSource=${config.providerSource}`,
|
||||
);
|
||||
}
|
||||
|
||||
const cleanedText = directives.cleanedText;
|
||||
const trimmedCleaned = cleanedText.trim();
|
||||
const visibleText = trimmedCleaned.length > 0 ? trimmedCleaned : "";
|
||||
const explicitTtsText = directives.ttsText?.trim() || "";
|
||||
const ttsText = explicitTtsText || visibleText;
|
||||
|
||||
const nextPayload =
|
||||
visibleText === text.trim()
|
||||
? params.payload
|
||||
: {
|
||||
...params.payload,
|
||||
text: visibleText.length > 0 ? visibleText : undefined,
|
||||
};
|
||||
|
||||
if (autoMode === "tagged" && !directives.hasDirective) {
|
||||
return nextPayload;
|
||||
}
|
||||
if (autoMode === "inbound" && params.inboundAudio !== true) {
|
||||
return nextPayload;
|
||||
}
|
||||
|
||||
const mode = config.mode ?? "final";
|
||||
if (mode === "final" && params.kind && params.kind !== "final") {
|
||||
return nextPayload;
|
||||
}
|
||||
|
||||
if (!ttsText.trim()) {
|
||||
return nextPayload;
|
||||
}
|
||||
if (reply.hasMedia || hasLegacyFinalMediaDirective(text)) {
|
||||
return nextPayload;
|
||||
}
|
||||
if (!explicitTtsText && ttsText.trim().length < 10) {
|
||||
return nextPayload;
|
||||
}
|
||||
|
||||
const maxLength = getTtsMaxLength(prefsPath);
|
||||
let textForAudio = ttsText.trim();
|
||||
let wasSummarized = false;
|
||||
|
||||
if (!explicitTtsText && isCodeHeavySpeechText(textForAudio)) {
|
||||
// The visible reply already carries code-heavy detail. Skip noisy voice-note audio instead of
|
||||
// telling channel users to look at a screen they may not have.
|
||||
return nextPayload;
|
||||
}
|
||||
|
||||
if (textForAudio.length > maxLength) {
|
||||
if (!isSummarizationEnabled(prefsPath)) {
|
||||
logVerbose(
|
||||
`TTS: truncating long text (${textForAudio.length} > ${maxLength}), summarization disabled.`,
|
||||
);
|
||||
textForAudio = `${truncateUtf16Safe(textForAudio, maxLength - 3)}...`;
|
||||
} else {
|
||||
try {
|
||||
const summary = await summarizeText({
|
||||
text: textForAudio,
|
||||
targetLength: maxLength,
|
||||
cfg,
|
||||
config,
|
||||
timeoutMs: config.timeoutMs,
|
||||
});
|
||||
textForAudio = summary.summary;
|
||||
wasSummarized = true;
|
||||
if (textForAudio.length > config.maxTextLength) {
|
||||
logVerbose(
|
||||
`TTS: summary exceeded hard limit (${textForAudio.length} > ${config.maxTextLength}); truncating.`,
|
||||
);
|
||||
textForAudio = `${truncateUtf16Safe(textForAudio, config.maxTextLength - 3)}...`;
|
||||
}
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logVerbose(`TTS: summarization failed, truncating instead: ${error.message}`);
|
||||
textForAudio = `${truncateUtf16Safe(textForAudio, maxLength - 3)}...`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const normalizedTextForAudio = normalizeSpeechText(textForAudio);
|
||||
if (!normalizedTextForAudio) {
|
||||
return nextPayload;
|
||||
}
|
||||
if (!explicitTtsText && normalizedTextForAudio.length < 10) {
|
||||
return nextPayload;
|
||||
}
|
||||
|
||||
const ttsStart = Date.now();
|
||||
const result = await textToSpeech({
|
||||
text: textForAudio,
|
||||
cfg,
|
||||
prefsPath,
|
||||
channel: params.channel,
|
||||
overrides: directives.overrides,
|
||||
agentId: params.agentId,
|
||||
accountId: params.accountId,
|
||||
});
|
||||
|
||||
if (result.success && result.audioPath) {
|
||||
lastTtsAttempt = {
|
||||
timestamp: Date.now(),
|
||||
success: true,
|
||||
textLength: text.length,
|
||||
summarized: wasSummarized,
|
||||
provider: result.provider,
|
||||
persona: result.persona,
|
||||
fallbackFrom: result.fallbackFrom,
|
||||
attemptedProviders: result.attemptedProviders,
|
||||
attempts: result.attempts,
|
||||
latencyMs: result.latencyMs,
|
||||
};
|
||||
|
||||
const payloadWithAudio = {
|
||||
...nextPayload,
|
||||
mediaUrl: result.audioPath,
|
||||
audioAsVoice: result.audioAsVoice || params.payload.audioAsVoice,
|
||||
spokenText: textForAudio,
|
||||
trustedLocalMedia: true,
|
||||
} as ReplyPayload;
|
||||
return nextPayload.text?.trim()
|
||||
? markReplyPayloadAsTtsSupplement(payloadWithAudio)
|
||||
: payloadWithAudio;
|
||||
}
|
||||
|
||||
lastTtsAttempt = {
|
||||
timestamp: Date.now(),
|
||||
success: false,
|
||||
textLength: text.length,
|
||||
summarized: wasSummarized,
|
||||
persona: result.persona,
|
||||
attemptedProviders: result.attemptedProviders,
|
||||
attempts: result.attempts,
|
||||
error: result.error,
|
||||
};
|
||||
|
||||
const latency = Date.now() - ttsStart;
|
||||
logVerbose(`TTS: conversion failed after ${latency}ms (${result.error ?? "unknown"}).`);
|
||||
return nextPayload;
|
||||
}
|
||||
@@ -0,0 +1,394 @@
|
||||
import type {
|
||||
OpenClawConfig,
|
||||
ResolvedTtsPersona,
|
||||
TtsConfig,
|
||||
TtsProvider,
|
||||
} from "openclaw/plugin-sdk/config-contracts";
|
||||
import { clampTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime";
|
||||
import {
|
||||
canonicalizeSpeechProviderId,
|
||||
getSpeechProvider,
|
||||
listSpeechProviders,
|
||||
normalizeSpeechProviderId,
|
||||
type SpeechProviderConfig,
|
||||
type SpeechProviderPlugin,
|
||||
} from "openclaw/plugin-sdk/speech-core";
|
||||
import {
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
normalizeOptionalString,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { withSpeakerSelectionCompat } from "../speaker.js";
|
||||
import {
|
||||
resolvePrimaryVoiceProviderCandidate,
|
||||
resolveSupportedVoiceModelRefs,
|
||||
resolveVoiceModelRefs,
|
||||
resolveVoiceProviderCandidates,
|
||||
voiceProviderSupportsModel,
|
||||
type VoiceModelProvider,
|
||||
type VoiceModelRef,
|
||||
type VoiceProviderCandidate,
|
||||
} from "../voice-models.js";
|
||||
import {
|
||||
DEFAULT_TTS_TIMEOUT_MS,
|
||||
asProviderConfig,
|
||||
asProviderConfigMap,
|
||||
hasOwnProperty,
|
||||
normalizeConfiguredSpeechProviderId,
|
||||
readTtsPrefs as readPrefs,
|
||||
resolveTtsPersonaFromPrefs,
|
||||
resolveTtsRuntimeConfig,
|
||||
type ResolvedTtsConfig,
|
||||
} from "./tts-settings.js";
|
||||
|
||||
function resolvePositiveTimeoutMs(timeoutMs: number | undefined): number | undefined {
|
||||
return typeof timeoutMs === "number" && Number.isFinite(timeoutMs) && timeoutMs > 0
|
||||
? clampTimerTimeoutMs(timeoutMs)
|
||||
: undefined;
|
||||
}
|
||||
|
||||
export function resolveSpeechProviderTimeoutMs(params: {
|
||||
timeoutMs?: number;
|
||||
config: ResolvedTtsConfig;
|
||||
provider: Pick<SpeechProviderPlugin, "defaultTimeoutMs">;
|
||||
}): number {
|
||||
if (params.timeoutMs !== undefined) {
|
||||
return resolvePositiveTimeoutMs(params.timeoutMs) ?? params.config.timeoutMs;
|
||||
}
|
||||
if (params.config.timeoutMsSource !== "default") {
|
||||
return resolvePositiveTimeoutMs(params.config.timeoutMs) ?? DEFAULT_TTS_TIMEOUT_MS;
|
||||
}
|
||||
return resolvePositiveTimeoutMs(params.provider.defaultTimeoutMs) ?? params.config.timeoutMs;
|
||||
}
|
||||
|
||||
function sortSpeechProvidersForAutoSelection(cfg?: OpenClawConfig) {
|
||||
return listSpeechProviders(cfg).toSorted((left, right) => {
|
||||
const leftOrder = left.autoSelectOrder ?? Number.MAX_SAFE_INTEGER;
|
||||
const rightOrder = right.autoSelectOrder ?? Number.MAX_SAFE_INTEGER;
|
||||
if (leftOrder !== rightOrder) {
|
||||
return leftOrder - rightOrder;
|
||||
}
|
||||
return left.id.localeCompare(right.id);
|
||||
});
|
||||
}
|
||||
|
||||
function resolveConfiguredSpeechVoiceModelRefs(cfg: OpenClawConfig | undefined): VoiceModelRef[] {
|
||||
const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : undefined;
|
||||
return resolveSupportedVoiceModelRefs({
|
||||
config: effectiveCfg?.agents?.defaults?.voiceModel,
|
||||
providers: sortSpeechProvidersForAutoSelection(effectiveCfg),
|
||||
});
|
||||
}
|
||||
|
||||
function resolveConfiguredSpeechVoiceModelForProvider(params: {
|
||||
cfg: OpenClawConfig | undefined;
|
||||
providerId: string;
|
||||
provider?: VoiceModelProvider;
|
||||
voiceModel?: VoiceModelRef;
|
||||
}): VoiceModelRef | undefined {
|
||||
const provider = params.provider ?? getSpeechProvider(params.providerId, params.cfg);
|
||||
if (params.voiceModel) {
|
||||
return voiceProviderSupportsModel(provider, params.voiceModel.model)
|
||||
? params.voiceModel
|
||||
: undefined;
|
||||
}
|
||||
return resolveSupportedVoiceModelRefs({
|
||||
config: params.cfg?.agents?.defaults?.voiceModel,
|
||||
providers: provider ? [provider] : [],
|
||||
providerId: params.providerId,
|
||||
})[0];
|
||||
}
|
||||
|
||||
function applyVoiceModelToSpeechProviderConfig(params: {
|
||||
cfg: OpenClawConfig | undefined;
|
||||
providerId: string;
|
||||
providerConfig: SpeechProviderConfig;
|
||||
provider?: VoiceModelProvider;
|
||||
voiceModel?: VoiceModelRef;
|
||||
}): SpeechProviderConfig {
|
||||
const voiceModel = resolveConfiguredSpeechVoiceModelForProvider({
|
||||
cfg: params.cfg,
|
||||
providerId: params.providerId,
|
||||
provider: params.provider,
|
||||
voiceModel: params.voiceModel,
|
||||
});
|
||||
if (!voiceModel) {
|
||||
return params.providerConfig;
|
||||
}
|
||||
const hasExplicitModel =
|
||||
normalizeOptionalString(params.providerConfig.model) ||
|
||||
normalizeOptionalString(params.providerConfig.modelId);
|
||||
if (hasExplicitModel) {
|
||||
return params.providerConfig;
|
||||
}
|
||||
return {
|
||||
...params.providerConfig,
|
||||
model: voiceModel.model,
|
||||
modelId: voiceModel.model,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolvePersonaProviderConfig(
|
||||
persona: ResolvedTtsPersona | undefined,
|
||||
providerId: string,
|
||||
): SpeechProviderConfig | undefined {
|
||||
if (!persona?.providers) {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = normalizeConfiguredSpeechProviderId(providerId) ?? providerId;
|
||||
if (hasOwnProperty(persona.providers, normalized)) {
|
||||
return persona.providers[normalized];
|
||||
}
|
||||
if (hasOwnProperty(persona.providers, providerId)) {
|
||||
return persona.providers[providerId];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function mergeProviderConfigWithPersona(params: {
|
||||
providerConfig: SpeechProviderConfig;
|
||||
persona?: ResolvedTtsPersona;
|
||||
providerId: string;
|
||||
}): {
|
||||
providerConfig: SpeechProviderConfig;
|
||||
personaProviderConfig?: SpeechProviderConfig;
|
||||
personaBinding: "applied" | "missing" | "none";
|
||||
} {
|
||||
if (!params.persona) {
|
||||
return { providerConfig: params.providerConfig, personaBinding: "none" };
|
||||
}
|
||||
const personaProviderConfig = resolvePersonaProviderConfig(params.persona, params.providerId);
|
||||
if (!personaProviderConfig) {
|
||||
return { providerConfig: params.providerConfig, personaBinding: "missing" };
|
||||
}
|
||||
return {
|
||||
providerConfig: {
|
||||
...params.providerConfig,
|
||||
...personaProviderConfig,
|
||||
},
|
||||
personaProviderConfig,
|
||||
personaBinding: "applied",
|
||||
};
|
||||
}
|
||||
|
||||
function resolveRawProviderConfig(
|
||||
raw: TtsConfig | undefined,
|
||||
providerId: string,
|
||||
): SpeechProviderConfig {
|
||||
if (!raw) {
|
||||
return {};
|
||||
}
|
||||
const rawProviders = asProviderConfigMap(raw.providers);
|
||||
const direct = rawProviders[providerId] ?? (raw as Record<string, unknown>)[providerId];
|
||||
return withSpeakerSelectionCompat(asProviderConfig(direct));
|
||||
}
|
||||
|
||||
function resolveLazyProviderConfig(
|
||||
config: ResolvedTtsConfig,
|
||||
providerId: string,
|
||||
cfg?: OpenClawConfig,
|
||||
voiceModel?: VoiceModelRef,
|
||||
): SpeechProviderConfig {
|
||||
const canonical =
|
||||
normalizeConfiguredSpeechProviderId(providerId) ?? normalizeLowercaseStringOrEmpty(providerId);
|
||||
const existing = voiceModel ? undefined : config.providerConfigs[canonical];
|
||||
const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : config.sourceConfig;
|
||||
if (existing && !effectiveCfg) {
|
||||
return existing;
|
||||
}
|
||||
const rawConfig = resolveRawProviderConfig(config.rawConfig, canonical);
|
||||
const rawBaseConfig = config.rawConfig as Record<string, unknown> | undefined;
|
||||
const rawProviders = asProviderConfigMap(config.rawConfig?.providers);
|
||||
const resolvedProvider = getSpeechProvider(canonical, effectiveCfg);
|
||||
let hasRawProviderConfig =
|
||||
Object.hasOwn(rawProviders, canonical) ||
|
||||
(rawBaseConfig ? Object.hasOwn(rawBaseConfig, canonical) : false);
|
||||
let rawProviderConfig = rawProviders[canonical] ?? rawBaseConfig?.[canonical];
|
||||
if (!hasRawProviderConfig) {
|
||||
for (const alias of resolvedProvider?.aliases ?? []) {
|
||||
const normalizedAlias = normalizeSpeechProviderId(alias);
|
||||
if (!normalizedAlias) {
|
||||
continue;
|
||||
}
|
||||
if (Object.hasOwn(rawProviders, normalizedAlias)) {
|
||||
hasRawProviderConfig = true;
|
||||
rawProviderConfig = rawProviders[normalizedAlias];
|
||||
break;
|
||||
}
|
||||
if (rawBaseConfig && Object.hasOwn(rawBaseConfig, normalizedAlias)) {
|
||||
hasRawProviderConfig = true;
|
||||
rawProviderConfig = rawBaseConfig[normalizedAlias];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
const compatRawProviderConfig = applyVoiceModelToSpeechProviderConfig({
|
||||
cfg: effectiveCfg,
|
||||
providerId: canonical,
|
||||
providerConfig: withSpeakerSelectionCompat(asProviderConfig(rawProviderConfig)),
|
||||
provider: resolvedProvider,
|
||||
voiceModel,
|
||||
});
|
||||
const shouldInjectCanonicalProviderConfig =
|
||||
hasRawProviderConfig || Boolean(voiceModel) || Object.keys(rawProviders).length === 0;
|
||||
const rawConfigForProvider = {
|
||||
...rawBaseConfig,
|
||||
providers: shouldInjectCanonicalProviderConfig
|
||||
? {
|
||||
...rawProviders,
|
||||
[canonical]: compatRawProviderConfig,
|
||||
}
|
||||
: rawProviders,
|
||||
...(shouldInjectCanonicalProviderConfig ? { [canonical]: compatRawProviderConfig } : {}),
|
||||
};
|
||||
const next = withSpeakerSelectionCompat(
|
||||
effectiveCfg && resolvedProvider?.resolveConfig
|
||||
? resolvedProvider.resolveConfig({
|
||||
cfg: effectiveCfg,
|
||||
rawConfig: rawConfigForProvider,
|
||||
timeoutMs: resolveSpeechProviderTimeoutMs({ config, provider: resolvedProvider }),
|
||||
})
|
||||
: applyVoiceModelToSpeechProviderConfig({
|
||||
cfg: effectiveCfg,
|
||||
providerId: canonical,
|
||||
providerConfig: rawConfig,
|
||||
provider: resolvedProvider,
|
||||
voiceModel,
|
||||
}),
|
||||
);
|
||||
if (!voiceModel) {
|
||||
config.providerConfigs[canonical] = next;
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
export function getResolvedSpeechProviderConfig(
|
||||
config: ResolvedTtsConfig,
|
||||
providerId: string,
|
||||
cfg?: OpenClawConfig,
|
||||
): SpeechProviderConfig {
|
||||
const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : config.sourceConfig;
|
||||
const canonical =
|
||||
canonicalizeSpeechProviderId(providerId, effectiveCfg) ??
|
||||
normalizeConfiguredSpeechProviderId(providerId) ??
|
||||
normalizeLowercaseStringOrEmpty(providerId);
|
||||
return resolveLazyProviderConfig(config, canonical, effectiveCfg);
|
||||
}
|
||||
|
||||
export function getResolvedSpeechProviderConfigForVoiceModel(params: {
|
||||
config: ResolvedTtsConfig;
|
||||
providerId: string;
|
||||
cfg: OpenClawConfig;
|
||||
voiceModel?: VoiceModelRef;
|
||||
}): SpeechProviderConfig {
|
||||
if (!params.voiceModel) {
|
||||
return getResolvedSpeechProviderConfig(params.config, params.providerId, params.cfg);
|
||||
}
|
||||
const effectiveCfg = resolveTtsRuntimeConfig(params.cfg);
|
||||
const canonical =
|
||||
canonicalizeSpeechProviderId(params.providerId, effectiveCfg) ??
|
||||
normalizeConfiguredSpeechProviderId(params.providerId) ??
|
||||
normalizeLowercaseStringOrEmpty(params.providerId);
|
||||
return resolveLazyProviderConfig(params.config, canonical, effectiveCfg, params.voiceModel);
|
||||
}
|
||||
|
||||
export function resolveTtsProvider(config: ResolvedTtsConfig, prefsPath: string): TtsProvider {
|
||||
const prefs = readPrefs(prefsPath);
|
||||
const prefsProvider =
|
||||
canonicalizeSpeechProviderId(prefs.tts?.provider) ??
|
||||
normalizeConfiguredSpeechProviderId(prefs.tts?.provider);
|
||||
if (prefsProvider) {
|
||||
return prefsProvider;
|
||||
}
|
||||
const activePersona = resolveTtsPersonaFromPrefs(config, prefs);
|
||||
const personaProvider =
|
||||
canonicalizeSpeechProviderId(activePersona?.provider, config.sourceConfig) ??
|
||||
normalizeConfiguredSpeechProviderId(activePersona?.provider);
|
||||
if (personaProvider && getSpeechProvider(personaProvider, config.sourceConfig)) {
|
||||
return personaProvider;
|
||||
}
|
||||
if (config.providerSource === "config") {
|
||||
return normalizeConfiguredSpeechProviderId(config.provider) ?? config.provider;
|
||||
}
|
||||
const configuredVoiceProvider = resolveConfiguredSpeechVoiceModelRefs(config.sourceConfig)[0]
|
||||
?.provider;
|
||||
if (configuredVoiceProvider && getSpeechProvider(configuredVoiceProvider, config.sourceConfig)) {
|
||||
return configuredVoiceProvider;
|
||||
}
|
||||
|
||||
const effectiveCfg = config.sourceConfig;
|
||||
for (const provider of sortSpeechProvidersForAutoSelection(effectiveCfg)) {
|
||||
if (isTtsProviderConfigured(config, provider.id, effectiveCfg)) {
|
||||
return provider.id;
|
||||
}
|
||||
}
|
||||
return config.provider;
|
||||
}
|
||||
|
||||
export function resolveTtsProviderOrder(primary: TtsProvider, cfg?: OpenClawConfig): TtsProvider[] {
|
||||
const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : undefined;
|
||||
const normalizedPrimary = canonicalizeSpeechProviderId(primary, effectiveCfg) ?? primary;
|
||||
const ordered = new Set<TtsProvider>([normalizedPrimary]);
|
||||
for (const ref of resolveVoiceModelRefs(effectiveCfg?.agents?.defaults?.voiceModel)) {
|
||||
const provider = canonicalizeSpeechProviderId(ref.provider, effectiveCfg) ?? ref.provider;
|
||||
if (provider !== normalizedPrimary) {
|
||||
ordered.add(provider);
|
||||
}
|
||||
}
|
||||
for (const provider of sortSpeechProvidersForAutoSelection(effectiveCfg)) {
|
||||
const normalized = provider.id;
|
||||
if (normalized !== normalizedPrimary) {
|
||||
ordered.add(normalized);
|
||||
}
|
||||
}
|
||||
return [...ordered];
|
||||
}
|
||||
|
||||
export function resolveTtsProviderCandidates(
|
||||
primary: TtsProvider,
|
||||
cfg?: OpenClawConfig,
|
||||
): VoiceProviderCandidate[] {
|
||||
const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : undefined;
|
||||
const normalizedPrimary = canonicalizeSpeechProviderId(primary, effectiveCfg) ?? primary;
|
||||
return resolveVoiceProviderCandidates({
|
||||
primaryProvider: normalizedPrimary,
|
||||
providers: sortSpeechProvidersForAutoSelection(effectiveCfg),
|
||||
voiceModelConfig: effectiveCfg?.agents?.defaults?.voiceModel,
|
||||
});
|
||||
}
|
||||
|
||||
export function resolvePrimaryTtsProviderCandidate(
|
||||
primary: TtsProvider,
|
||||
cfg?: OpenClawConfig,
|
||||
): VoiceProviderCandidate {
|
||||
const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : undefined;
|
||||
return resolvePrimaryVoiceProviderCandidate({
|
||||
primaryProvider: canonicalizeSpeechProviderId(primary, effectiveCfg) ?? primary,
|
||||
providers: sortSpeechProvidersForAutoSelection(effectiveCfg),
|
||||
voiceModelConfig: effectiveCfg?.agents?.defaults?.voiceModel,
|
||||
});
|
||||
}
|
||||
|
||||
export function isTtsProviderConfigured(
|
||||
config: ResolvedTtsConfig,
|
||||
provider: TtsProvider,
|
||||
cfg?: OpenClawConfig,
|
||||
): boolean {
|
||||
try {
|
||||
const effectiveCfg = cfg ? resolveTtsRuntimeConfig(cfg) : config.sourceConfig;
|
||||
const resolvedProvider = getSpeechProvider(provider, effectiveCfg);
|
||||
if (!resolvedProvider) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
resolvedProvider.isConfigured({
|
||||
cfg: effectiveCfg,
|
||||
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, effectiveCfg),
|
||||
timeoutMs: resolveSpeechProviderTimeoutMs({ config, provider: resolvedProvider }),
|
||||
}) ?? false
|
||||
);
|
||||
} catch {
|
||||
// Configuration probes drive provider selection and status catalogs. A
|
||||
// malformed provider config must not hide other usable providers.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import type { OpenClawConfig, TtsConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { mergeDeep } from "openclaw/plugin-sdk/plugin-config-runtime";
|
||||
import {
|
||||
canonicalizeSpeechProviderId,
|
||||
getSpeechProvider,
|
||||
parseTtsDirectives,
|
||||
type SpeechProviderOverrides,
|
||||
type TtsDirectiveOverrides,
|
||||
type TtsDirectiveParseResult,
|
||||
} from "openclaw/plugin-sdk/speech-core";
|
||||
import { resolveTtsProvider } from "./tts-provider-resolution.js";
|
||||
import { resolveTtsConfig, resolveTtsPrefsPath, resolveTtsRuntimeConfig } from "./tts-settings.js";
|
||||
|
||||
export type PreparedTtsRequest = {
|
||||
cfg: OpenClawConfig;
|
||||
directives: TtsDirectiveParseResult;
|
||||
};
|
||||
|
||||
/** Merge a surface TTS override and resolve its inline synthesis directives. */
|
||||
export function prepareTtsRequest(params: {
|
||||
cfg: OpenClawConfig;
|
||||
override?: TtsConfig;
|
||||
text: string;
|
||||
}): PreparedTtsRequest {
|
||||
const cfg = params.override
|
||||
? {
|
||||
...params.cfg,
|
||||
tts: mergeDeep(params.cfg.tts ?? {}, params.override) as TtsConfig,
|
||||
}
|
||||
: params.cfg;
|
||||
const config = resolveTtsConfig(cfg);
|
||||
const directives = parseTtsDirectives(params.text, config.modelOverrides, {
|
||||
cfg,
|
||||
providerConfigs: config.providerConfigs,
|
||||
preferredProviderId: resolveTtsProvider(config, resolveTtsPrefsPath(config)),
|
||||
});
|
||||
return { cfg, directives };
|
||||
}
|
||||
|
||||
export function resolveExplicitTtsOverrides(params: {
|
||||
cfg: OpenClawConfig;
|
||||
prefsPath?: string;
|
||||
provider?: string;
|
||||
modelId?: string;
|
||||
voiceId?: string;
|
||||
agentId?: string;
|
||||
channelId?: string;
|
||||
accountId?: string;
|
||||
}): TtsDirectiveOverrides {
|
||||
const cfg = resolveTtsRuntimeConfig(params.cfg);
|
||||
const providerInput = params.provider?.trim();
|
||||
const modelId = params.modelId?.trim();
|
||||
const voiceId = params.voiceId?.trim();
|
||||
const config = resolveTtsConfig(cfg, {
|
||||
agentId: params.agentId,
|
||||
channelId: params.channelId,
|
||||
accountId: params.accountId,
|
||||
});
|
||||
const prefsPath = params.prefsPath ?? resolveTtsPrefsPath(config);
|
||||
const selectedProvider =
|
||||
canonicalizeSpeechProviderId(providerInput, cfg) ??
|
||||
(modelId || voiceId ? resolveTtsProvider(config, prefsPath) : undefined);
|
||||
|
||||
if (providerInput && !selectedProvider) {
|
||||
throw new Error(`Unknown TTS provider "${providerInput}".`);
|
||||
}
|
||||
|
||||
if (!modelId && !voiceId) {
|
||||
return selectedProvider ? { provider: selectedProvider } : {};
|
||||
}
|
||||
|
||||
if (!selectedProvider) {
|
||||
throw new Error("TTS model or voice overrides require a resolved provider.");
|
||||
}
|
||||
|
||||
const provider = getSpeechProvider(selectedProvider, cfg);
|
||||
if (!provider) {
|
||||
throw new Error(`speech provider ${selectedProvider} is not registered`);
|
||||
}
|
||||
if (!provider.resolveTalkOverrides) {
|
||||
throw new Error(
|
||||
`TTS provider "${selectedProvider}" does not support model or voice overrides.`,
|
||||
);
|
||||
}
|
||||
|
||||
const providerOverrides = provider.resolveTalkOverrides({
|
||||
talkProviderConfig: {},
|
||||
params: {
|
||||
...(voiceId ? { voiceId } : {}),
|
||||
...(modelId ? { modelId } : {}),
|
||||
},
|
||||
});
|
||||
if ((voiceId || modelId) && (!providerOverrides || Object.keys(providerOverrides).length === 0)) {
|
||||
throw new Error(
|
||||
`TTS provider "${selectedProvider}" ignored the requested model or voice overrides.`,
|
||||
);
|
||||
}
|
||||
|
||||
const overridesRecord = providerOverrides as SpeechProviderOverrides;
|
||||
return {
|
||||
provider: selectedProvider,
|
||||
providerOverrides: {
|
||||
[provider.id]: overridesRecord,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { logVerbose } from "openclaw/plugin-sdk/runtime-env";
|
||||
import type { TtsDirectiveOverrides } from "openclaw/plugin-sdk/speech-core";
|
||||
import { assertSpeechRuntimeAvailable } from "./runtime-availability.js";
|
||||
import { resolveSpeechProviderTimeoutMs } from "./tts-provider-resolution.js";
|
||||
import {
|
||||
buildTtsFailureResult,
|
||||
formatTtsProviderError,
|
||||
prepareSpeechSynthesis,
|
||||
resolvePersonaBinding,
|
||||
resolveReadySpeechProvider,
|
||||
resolveTtsRequestSetup,
|
||||
resolveTtsResultModel,
|
||||
resolveTtsResultVoice,
|
||||
sanitizeTtsErrorForLog,
|
||||
} from "./tts-synthesis-support.js";
|
||||
import { resolveTtsSynthesisTarget } from "./tts-synthesis.js";
|
||||
import type { TtsProviderAttempt, TtsStreamResult, TtsSynthesisStreamResult } from "./tts-types.js";
|
||||
|
||||
export async function streamSpeech(params: {
|
||||
text: string;
|
||||
cfg: OpenClawConfig;
|
||||
prefsPath?: string;
|
||||
channel?: string;
|
||||
overrides?: TtsDirectiveOverrides;
|
||||
disableFallback?: boolean;
|
||||
timeoutMs?: number;
|
||||
agentId?: string;
|
||||
accountId?: string;
|
||||
}): Promise<TtsSynthesisStreamResult> {
|
||||
assertSpeechRuntimeAvailable();
|
||||
const setup = resolveTtsRequestSetup({
|
||||
text: params.text,
|
||||
cfg: params.cfg,
|
||||
prefsPath: params.prefsPath,
|
||||
providerOverride: params.overrides?.provider,
|
||||
disableFallback: params.disableFallback,
|
||||
agentId: params.agentId,
|
||||
channelId: params.channel,
|
||||
accountId: params.accountId,
|
||||
});
|
||||
if ("error" in setup) {
|
||||
return { success: false, error: setup.error };
|
||||
}
|
||||
|
||||
const { cfg, config, persona, providers } = setup;
|
||||
const target = resolveTtsSynthesisTarget(params.channel);
|
||||
const errors: string[] = [];
|
||||
const attemptedProviders: string[] = [];
|
||||
const attempts: TtsProviderAttempt[] = [];
|
||||
const primaryProvider = providers[0]?.provider;
|
||||
logVerbose(
|
||||
`TTS stream: starting with provider ${primaryProvider}, fallbacks: ${
|
||||
providers
|
||||
.slice(1)
|
||||
.map((entry) => entry.provider)
|
||||
.join(", ") || "none"
|
||||
}`,
|
||||
);
|
||||
|
||||
for (const { provider, voiceModel } of providers) {
|
||||
attemptedProviders.push(provider);
|
||||
const providerStart = Date.now();
|
||||
try {
|
||||
const resolvedProvider = resolveReadySpeechProvider({
|
||||
provider,
|
||||
cfg,
|
||||
config,
|
||||
persona,
|
||||
voiceModel,
|
||||
});
|
||||
if (resolvedProvider.kind === "skip") {
|
||||
errors.push(resolvedProvider.message);
|
||||
attempts.push({
|
||||
provider,
|
||||
outcome: "skipped",
|
||||
reasonCode: resolvedProvider.reasonCode,
|
||||
persona: persona?.id,
|
||||
...(resolvedProvider.personaBinding
|
||||
? { personaBinding: resolvedProvider.personaBinding }
|
||||
: {}),
|
||||
error: resolvedProvider.message,
|
||||
});
|
||||
logVerbose(`TTS stream: provider ${provider} skipped (${resolvedProvider.message})`);
|
||||
continue;
|
||||
}
|
||||
if (!resolvedProvider.provider.streamSynthesize) {
|
||||
const message = `${provider} does not support streaming TTS`;
|
||||
errors.push(message);
|
||||
attempts.push({
|
||||
provider,
|
||||
outcome: "skipped",
|
||||
reasonCode: "unsupported_for_streaming",
|
||||
persona: persona?.id,
|
||||
personaBinding: resolvedProvider.personaBinding,
|
||||
error: message,
|
||||
});
|
||||
logVerbose(`TTS stream: provider ${provider} skipped (${message})`);
|
||||
continue;
|
||||
}
|
||||
const timeoutMs = resolveSpeechProviderTimeoutMs({
|
||||
timeoutMs: params.timeoutMs ?? voiceModel?.timeoutMs,
|
||||
config,
|
||||
provider: resolvedProvider.provider,
|
||||
});
|
||||
const prepared = await prepareSpeechSynthesis({
|
||||
provider: resolvedProvider.provider,
|
||||
text: params.text,
|
||||
cfg,
|
||||
providerConfig: resolvedProvider.providerConfig,
|
||||
providerOverrides: params.overrides?.providerOverrides?.[resolvedProvider.provider.id],
|
||||
persona: resolvedProvider.synthesisPersona,
|
||||
personaProviderConfig: resolvedProvider.personaProviderConfig,
|
||||
target,
|
||||
timeoutMs,
|
||||
});
|
||||
const synthesis = await resolvedProvider.provider.streamSynthesize({
|
||||
text: prepared.text,
|
||||
cfg,
|
||||
providerConfig: prepared.providerConfig,
|
||||
target,
|
||||
providerOverrides: prepared.providerOverrides,
|
||||
timeoutMs,
|
||||
});
|
||||
const latencyMs = Date.now() - providerStart;
|
||||
attempts.push({
|
||||
provider,
|
||||
outcome: "success",
|
||||
reasonCode: "success",
|
||||
persona: persona?.id,
|
||||
personaBinding: resolvedProvider.personaBinding,
|
||||
latencyMs,
|
||||
});
|
||||
return {
|
||||
success: true,
|
||||
audioStream: synthesis.audioStream,
|
||||
latencyMs,
|
||||
provider,
|
||||
providerModel: resolveTtsResultModel(prepared.providerConfig, prepared.providerOverrides),
|
||||
providerVoice: resolveTtsResultVoice(prepared.providerConfig, prepared.providerOverrides),
|
||||
persona: persona?.id,
|
||||
fallbackFrom: provider !== primaryProvider ? primaryProvider : undefined,
|
||||
attemptedProviders,
|
||||
attempts,
|
||||
outputFormat: synthesis.outputFormat,
|
||||
voiceCompatible: synthesis.voiceCompatible,
|
||||
fileExtension: synthesis.fileExtension,
|
||||
target,
|
||||
release: synthesis.release,
|
||||
};
|
||||
} catch (err) {
|
||||
const errorMsg = formatTtsProviderError(provider, err);
|
||||
const latencyMs = Date.now() - providerStart;
|
||||
errors.push(errorMsg);
|
||||
attempts.push({
|
||||
provider,
|
||||
outcome: "failed",
|
||||
reasonCode:
|
||||
err instanceof Error && err.name === "AbortError" ? "timeout" : "provider_error",
|
||||
latencyMs,
|
||||
persona: persona?.id,
|
||||
personaBinding: resolvePersonaBinding(persona, provider),
|
||||
error: errorMsg,
|
||||
});
|
||||
const rawError = sanitizeTtsErrorForLog(err);
|
||||
if (provider === primaryProvider) {
|
||||
const hasFallbacks = providers.length > 1;
|
||||
logVerbose(
|
||||
`TTS stream: primary provider ${provider} failed (${rawError})${hasFallbacks ? "; trying fallback providers." : "; no fallback providers configured."}`,
|
||||
);
|
||||
} else {
|
||||
logVerbose(`TTS stream: ${provider} failed (${rawError}); trying next provider.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return buildTtsFailureResult(errors, attemptedProviders, attempts, persona?.id);
|
||||
}
|
||||
|
||||
export async function textToSpeechStream(params: {
|
||||
text: string;
|
||||
cfg: OpenClawConfig;
|
||||
prefsPath?: string;
|
||||
channel?: string;
|
||||
overrides?: TtsDirectiveOverrides;
|
||||
disableFallback?: boolean;
|
||||
timeoutMs?: number;
|
||||
agentId?: string;
|
||||
accountId?: string;
|
||||
}): Promise<TtsStreamResult> {
|
||||
const synthesis = await streamSpeech(params);
|
||||
if (!synthesis.success || !synthesis.audioStream || !synthesis.fileExtension) {
|
||||
return {
|
||||
success: false,
|
||||
error: synthesis.error ?? "Streaming TTS conversion failed",
|
||||
persona: synthesis.persona,
|
||||
attemptedProviders: synthesis.attemptedProviders,
|
||||
attempts: synthesis.attempts,
|
||||
};
|
||||
}
|
||||
return synthesis;
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
import type {
|
||||
OpenClawConfig,
|
||||
ResolvedTtsPersona,
|
||||
TtsProvider,
|
||||
} from "openclaw/plugin-sdk/config-contracts";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { redactSensitiveText } from "openclaw/plugin-sdk/logging-core";
|
||||
import {
|
||||
canonicalizeSpeechProviderId,
|
||||
getSpeechProvider,
|
||||
type SpeechProviderConfig,
|
||||
type SpeechProviderOverrides,
|
||||
} from "openclaw/plugin-sdk/speech-core";
|
||||
import type { VoiceModelRef, VoiceProviderCandidate } from "../voice-models.js";
|
||||
import {
|
||||
getResolvedSpeechProviderConfigForVoiceModel,
|
||||
mergeProviderConfigWithPersona,
|
||||
resolvePersonaProviderConfig,
|
||||
resolvePrimaryTtsProviderCandidate,
|
||||
resolveSpeechProviderTimeoutMs,
|
||||
resolveTtsProvider,
|
||||
resolveTtsProviderCandidates,
|
||||
} from "./tts-provider-resolution.js";
|
||||
import {
|
||||
getTtsPersona,
|
||||
resolveTtsConfig,
|
||||
resolveTtsPrefsPath,
|
||||
resolveTtsRuntimeConfig,
|
||||
type ResolvedTtsConfig,
|
||||
} from "./tts-settings.js";
|
||||
import type { TtsProviderAttempt } from "./tts-types.js";
|
||||
|
||||
export function formatTtsProviderError(provider: TtsProvider, err: unknown): string {
|
||||
const error = err instanceof Error ? err : new Error(String(err));
|
||||
if (error.name === "AbortError") {
|
||||
return `${provider}: request timed out`;
|
||||
}
|
||||
return `${provider}: ${redactSensitiveText(error.message)}`;
|
||||
}
|
||||
|
||||
export function sanitizeTtsErrorForLog(err: unknown): string {
|
||||
const raw = formatErrorMessage(err);
|
||||
return redactSensitiveText(raw).replace(/\r/g, "\\r").replace(/\n/g, "\\n").replace(/\t/g, "\\t");
|
||||
}
|
||||
|
||||
export function buildTtsFailureResult(
|
||||
errors: string[],
|
||||
attemptedProviders?: string[],
|
||||
attempts?: TtsProviderAttempt[],
|
||||
persona?: string,
|
||||
): {
|
||||
success: false;
|
||||
error: string;
|
||||
attemptedProviders?: string[];
|
||||
attempts?: TtsProviderAttempt[];
|
||||
persona?: string;
|
||||
} {
|
||||
return {
|
||||
success: false,
|
||||
error: `TTS conversion failed: ${errors.join("; ") || "no providers available"}`,
|
||||
attemptedProviders,
|
||||
attempts,
|
||||
persona,
|
||||
};
|
||||
}
|
||||
|
||||
type TtsProviderReadyResolution =
|
||||
| {
|
||||
kind: "ready";
|
||||
provider: NonNullable<ReturnType<typeof getSpeechProvider>>;
|
||||
providerConfig: SpeechProviderConfig;
|
||||
personaProviderConfig?: SpeechProviderConfig;
|
||||
synthesisPersona?: ResolvedTtsPersona;
|
||||
personaBinding: "applied" | "missing" | "none";
|
||||
}
|
||||
| {
|
||||
kind: "skip";
|
||||
reasonCode: "no_provider_registered" | "not_configured" | "unsupported_for_telephony";
|
||||
message: string;
|
||||
personaBinding?: "missing";
|
||||
};
|
||||
|
||||
export function resolveReadySpeechProvider(params: {
|
||||
provider: TtsProvider;
|
||||
cfg: OpenClawConfig;
|
||||
config: ResolvedTtsConfig;
|
||||
persona?: ResolvedTtsPersona;
|
||||
voiceModel?: VoiceModelRef;
|
||||
requireTelephony?: boolean;
|
||||
}): TtsProviderReadyResolution {
|
||||
const resolvedProvider = getSpeechProvider(params.provider, params.cfg);
|
||||
if (!resolvedProvider) {
|
||||
return {
|
||||
kind: "skip",
|
||||
reasonCode: "no_provider_registered",
|
||||
message: `${params.provider}: no provider registered`,
|
||||
};
|
||||
}
|
||||
const providerConfig = getResolvedSpeechProviderConfigForVoiceModel({
|
||||
config: params.config,
|
||||
providerId: resolvedProvider.id,
|
||||
cfg: params.cfg,
|
||||
voiceModel: params.voiceModel,
|
||||
});
|
||||
const merged = mergeProviderConfigWithPersona({
|
||||
providerConfig,
|
||||
persona: params.persona,
|
||||
providerId: resolvedProvider.id,
|
||||
});
|
||||
if (params.persona?.fallbackPolicy === "fail" && merged.personaBinding === "missing") {
|
||||
return {
|
||||
kind: "skip",
|
||||
reasonCode: "not_configured",
|
||||
message: `${params.provider}: persona ${params.persona.id} has no provider binding`,
|
||||
personaBinding: "missing",
|
||||
};
|
||||
}
|
||||
if (
|
||||
!resolvedProvider.isConfigured({
|
||||
cfg: params.cfg,
|
||||
providerConfig: merged.providerConfig,
|
||||
timeoutMs: resolveSpeechProviderTimeoutMs({
|
||||
config: params.config,
|
||||
provider: resolvedProvider,
|
||||
}),
|
||||
})
|
||||
) {
|
||||
return {
|
||||
kind: "skip",
|
||||
reasonCode: "not_configured",
|
||||
message: `${params.provider}: not configured`,
|
||||
};
|
||||
}
|
||||
if (params.requireTelephony && !resolvedProvider.synthesizeTelephony) {
|
||||
return {
|
||||
kind: "skip",
|
||||
reasonCode: "unsupported_for_telephony",
|
||||
message: `${params.provider}: unsupported for telephony`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
kind: "ready",
|
||||
provider: resolvedProvider,
|
||||
providerConfig: merged.providerConfig,
|
||||
personaProviderConfig: merged.personaProviderConfig,
|
||||
synthesisPersona:
|
||||
params.persona?.fallbackPolicy === "provider-defaults" && merged.personaBinding === "missing"
|
||||
? undefined
|
||||
: params.persona,
|
||||
personaBinding: merged.personaBinding,
|
||||
};
|
||||
}
|
||||
|
||||
export async function prepareSpeechSynthesis(params: {
|
||||
provider: NonNullable<ReturnType<typeof getSpeechProvider>>;
|
||||
text: string;
|
||||
cfg: OpenClawConfig;
|
||||
providerConfig: SpeechProviderConfig;
|
||||
providerOverrides?: SpeechProviderOverrides;
|
||||
persona?: ResolvedTtsPersona;
|
||||
personaProviderConfig?: SpeechProviderConfig;
|
||||
target: "audio-file" | "voice-note" | "telephony";
|
||||
timeoutMs: number;
|
||||
}): Promise<{
|
||||
text: string;
|
||||
providerConfig: SpeechProviderConfig;
|
||||
providerOverrides?: SpeechProviderOverrides;
|
||||
}> {
|
||||
if (!params.provider.prepareSynthesis) {
|
||||
return {
|
||||
text: params.text,
|
||||
providerConfig: params.providerConfig,
|
||||
providerOverrides: params.providerOverrides,
|
||||
};
|
||||
}
|
||||
const prepared = await params.provider.prepareSynthesis({
|
||||
text: params.text,
|
||||
cfg: params.cfg,
|
||||
providerConfig: params.providerConfig,
|
||||
providerOverrides: params.providerOverrides,
|
||||
persona: params.persona,
|
||||
personaProviderConfig: params.personaProviderConfig,
|
||||
target: params.target,
|
||||
timeoutMs: params.timeoutMs,
|
||||
});
|
||||
return {
|
||||
text: prepared?.text ?? params.text,
|
||||
providerConfig: prepared?.providerConfig
|
||||
? { ...params.providerConfig, ...prepared.providerConfig }
|
||||
: params.providerConfig,
|
||||
providerOverrides: prepared?.providerOverrides
|
||||
? { ...params.providerOverrides, ...prepared.providerOverrides }
|
||||
: params.providerOverrides,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveTtsRequestSetup(params: {
|
||||
text: string;
|
||||
cfg: OpenClawConfig;
|
||||
prefsPath?: string;
|
||||
providerOverride?: TtsProvider;
|
||||
disableFallback?: boolean;
|
||||
agentId?: string;
|
||||
channelId?: string;
|
||||
accountId?: string;
|
||||
}):
|
||||
| {
|
||||
cfg: OpenClawConfig;
|
||||
config: ResolvedTtsConfig;
|
||||
persona?: ResolvedTtsPersona;
|
||||
providers: VoiceProviderCandidate[];
|
||||
}
|
||||
| {
|
||||
error: string;
|
||||
} {
|
||||
const cfg = resolveTtsRuntimeConfig(params.cfg);
|
||||
const config = resolveTtsConfig(cfg, {
|
||||
agentId: params.agentId,
|
||||
channelId: params.channelId,
|
||||
accountId: params.accountId,
|
||||
});
|
||||
const prefsPath = params.prefsPath ?? resolveTtsPrefsPath(config);
|
||||
if (params.text.length > config.maxTextLength) {
|
||||
return {
|
||||
error: `Text too long (${params.text.length} chars, max ${config.maxTextLength})`,
|
||||
};
|
||||
}
|
||||
|
||||
const userProvider = resolveTtsProvider(config, prefsPath);
|
||||
const provider = canonicalizeSpeechProviderId(params.providerOverride, cfg) ?? userProvider;
|
||||
return {
|
||||
cfg,
|
||||
config,
|
||||
persona: getTtsPersona(config, prefsPath),
|
||||
providers: params.disableFallback
|
||||
? [resolvePrimaryTtsProviderCandidate(provider, cfg)]
|
||||
: resolveTtsProviderCandidates(provider, cfg),
|
||||
};
|
||||
}
|
||||
|
||||
function readTtsResultString(value: unknown): string | undefined {
|
||||
return typeof value === "string" && value.trim() ? value.trim() : undefined;
|
||||
}
|
||||
|
||||
export function resolveTtsResultModel(
|
||||
providerConfig: SpeechProviderConfig,
|
||||
providerOverrides?: SpeechProviderOverrides,
|
||||
): string | undefined {
|
||||
return (
|
||||
readTtsResultString(providerOverrides?.modelId) ??
|
||||
readTtsResultString(providerOverrides?.model) ??
|
||||
readTtsResultString(providerConfig.modelId) ??
|
||||
readTtsResultString(providerConfig.model)
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveTtsResultVoice(
|
||||
providerConfig: SpeechProviderConfig,
|
||||
providerOverrides?: SpeechProviderOverrides,
|
||||
): string | undefined {
|
||||
return (
|
||||
readTtsResultString(providerOverrides?.speakerVoiceId) ??
|
||||
readTtsResultString(providerOverrides?.speakerVoice) ??
|
||||
readTtsResultString(providerOverrides?.voiceId) ??
|
||||
readTtsResultString(providerOverrides?.voiceName) ??
|
||||
readTtsResultString(providerOverrides?.voice) ??
|
||||
readTtsResultString(providerConfig.speakerVoiceId) ??
|
||||
readTtsResultString(providerConfig.speakerVoice) ??
|
||||
readTtsResultString(providerConfig.voiceId) ??
|
||||
readTtsResultString(providerConfig.voiceName) ??
|
||||
readTtsResultString(providerConfig.voice)
|
||||
);
|
||||
}
|
||||
|
||||
export function resolvePersonaBinding(
|
||||
persona: ResolvedTtsPersona | undefined,
|
||||
provider: string,
|
||||
): "applied" | "missing" | "none" {
|
||||
return resolvePersonaProviderConfig(persona, provider) != null
|
||||
? "applied"
|
||||
: persona
|
||||
? "missing"
|
||||
: "none";
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
import { resolveChannelTtsVoiceDelivery } from "openclaw/plugin-sdk/channel-targets";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { transcodeAudioBuffer } from "openclaw/plugin-sdk/media-runtime";
|
||||
import { logVerbose } from "openclaw/plugin-sdk/runtime-env";
|
||||
import { tempWorkspaceSync, resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/sandbox";
|
||||
import { scheduleCleanup, type TtsDirectiveOverrides } from "openclaw/plugin-sdk/speech-core";
|
||||
import { assertSpeechRuntimeAvailable } from "./runtime-availability.js";
|
||||
import { normalizeSpeechText } from "./speech-text.js";
|
||||
import { resolveSpeechProviderTimeoutMs } from "./tts-provider-resolution.js";
|
||||
import {
|
||||
buildTtsFailureResult,
|
||||
formatTtsProviderError,
|
||||
prepareSpeechSynthesis,
|
||||
resolvePersonaBinding,
|
||||
resolveReadySpeechProvider,
|
||||
resolveTtsRequestSetup,
|
||||
resolveTtsResultModel,
|
||||
resolveTtsResultVoice,
|
||||
sanitizeTtsErrorForLog,
|
||||
} from "./tts-synthesis-support.js";
|
||||
import type { TtsProviderAttempt, TtsResult, TtsSynthesisResult } from "./tts-types.js";
|
||||
|
||||
export function supportsNativeVoiceNoteTts(channel: string | undefined): boolean {
|
||||
return resolveChannelTtsVoiceDelivery(channel) !== undefined;
|
||||
}
|
||||
|
||||
export function supportsTranscodedVoiceNoteTts(channel: string | undefined): boolean {
|
||||
const delivery = resolveChannelTtsVoiceDelivery(channel);
|
||||
return delivery?.synthesisTarget === "voice-note" && delivery.transcodesAudio === true;
|
||||
}
|
||||
|
||||
export function resolveTtsSynthesisTarget(
|
||||
channel: string | undefined,
|
||||
): "audio-file" | "voice-note" {
|
||||
return resolveChannelTtsVoiceDelivery(channel)?.synthesisTarget ?? "audio-file";
|
||||
}
|
||||
|
||||
function supportsAudioFileVoiceMemoOutput(params: {
|
||||
fileExtension?: string;
|
||||
outputFormat?: string;
|
||||
audioFileFormats?: readonly string[];
|
||||
}): boolean {
|
||||
const formats = new Set(params.audioFileFormats?.map((format) => format.trim().toLowerCase()));
|
||||
if (formats.size === 0) {
|
||||
return false;
|
||||
}
|
||||
const extension = params.fileExtension?.trim().toLowerCase();
|
||||
if (extension && formats.has(extension.replace(/^\./, ""))) {
|
||||
return true;
|
||||
}
|
||||
const outputFormat = params.outputFormat?.trim().toLowerCase();
|
||||
return outputFormat ? formats.has(outputFormat) : false;
|
||||
}
|
||||
|
||||
export function shouldDeliverTtsAsVoice(params: {
|
||||
channel: string | undefined;
|
||||
target: "audio-file" | "voice-note" | undefined;
|
||||
voiceCompatible: boolean | undefined;
|
||||
fileExtension?: string;
|
||||
outputFormat?: string;
|
||||
}): boolean {
|
||||
const delivery = resolveChannelTtsVoiceDelivery(params.channel);
|
||||
if (!delivery) {
|
||||
return false;
|
||||
}
|
||||
if (delivery.synthesisTarget === "audio-file") {
|
||||
return (
|
||||
params.target === "audio-file" &&
|
||||
supportsAudioFileVoiceMemoOutput({
|
||||
fileExtension: params.fileExtension,
|
||||
outputFormat: params.outputFormat,
|
||||
audioFileFormats: delivery.audioFileFormats,
|
||||
})
|
||||
);
|
||||
}
|
||||
if (params.target !== "voice-note") {
|
||||
return false;
|
||||
}
|
||||
return params.voiceCompatible === true || delivery.transcodesAudio === true;
|
||||
}
|
||||
|
||||
export async function textToSpeech(params: {
|
||||
text: string;
|
||||
cfg: OpenClawConfig;
|
||||
prefsPath?: string;
|
||||
channel?: string;
|
||||
overrides?: TtsDirectiveOverrides;
|
||||
disableFallback?: boolean;
|
||||
timeoutMs?: number;
|
||||
agentId?: string;
|
||||
accountId?: string;
|
||||
}): Promise<TtsResult> {
|
||||
const synthesis = await synthesizeSpeech(params);
|
||||
if (!synthesis.success || !synthesis.audioBuffer || !synthesis.fileExtension) {
|
||||
return {
|
||||
success: false,
|
||||
error: synthesis.error ?? "TTS conversion failed",
|
||||
persona: synthesis.persona,
|
||||
attemptedProviders: synthesis.attemptedProviders,
|
||||
attempts: synthesis.attempts,
|
||||
};
|
||||
}
|
||||
|
||||
let audioBuffer = synthesis.audioBuffer;
|
||||
let fileExtension = synthesis.fileExtension;
|
||||
let outputFormat = synthesis.outputFormat;
|
||||
const transcoded = await maybePreTranscodeForVoiceDelivery({
|
||||
channel: params.channel,
|
||||
target: synthesis.target,
|
||||
audioBuffer,
|
||||
fileExtension,
|
||||
outputFormat,
|
||||
});
|
||||
if (transcoded) {
|
||||
audioBuffer = transcoded.audioBuffer;
|
||||
fileExtension = transcoded.fileExtension;
|
||||
outputFormat = transcoded.outputFormat;
|
||||
}
|
||||
|
||||
const temp = tempWorkspaceSync({
|
||||
rootDir: resolvePreferredOpenClawTmpDir(),
|
||||
prefix: "tts-",
|
||||
});
|
||||
const audioPath = temp.write(`voice-${Date.now()}${fileExtension}`, audioBuffer);
|
||||
scheduleCleanup(temp.dir);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
audioPath,
|
||||
latencyMs: synthesis.latencyMs,
|
||||
provider: synthesis.provider,
|
||||
persona: synthesis.persona,
|
||||
fallbackFrom: synthesis.fallbackFrom,
|
||||
attemptedProviders: synthesis.attemptedProviders,
|
||||
attempts: synthesis.attempts,
|
||||
outputFormat,
|
||||
voiceCompatible: synthesis.voiceCompatible,
|
||||
audioAsVoice: shouldDeliverTtsAsVoice({
|
||||
channel: params.channel,
|
||||
target: synthesis.target,
|
||||
voiceCompatible: synthesis.voiceCompatible,
|
||||
fileExtension,
|
||||
outputFormat,
|
||||
}),
|
||||
target: synthesis.target,
|
||||
};
|
||||
}
|
||||
|
||||
async function maybePreTranscodeForVoiceDelivery(params: {
|
||||
channel: string | undefined;
|
||||
target: "audio-file" | "voice-note" | undefined;
|
||||
audioBuffer: Buffer;
|
||||
fileExtension: string;
|
||||
outputFormat?: string;
|
||||
}): Promise<{ audioBuffer: Buffer; fileExtension: string; outputFormat?: string } | undefined> {
|
||||
if (params.target !== "audio-file") {
|
||||
return undefined;
|
||||
}
|
||||
const delivery = resolveChannelTtsVoiceDelivery(params.channel);
|
||||
const preferred = delivery?.preferAudioFileFormat?.trim().toLowerCase();
|
||||
if (!preferred) {
|
||||
return undefined;
|
||||
}
|
||||
const sourceExt = params.fileExtension.trim().toLowerCase().replace(/^\./, "");
|
||||
if (sourceExt === preferred) {
|
||||
return undefined;
|
||||
}
|
||||
const outcome = await transcodeAudioBuffer({
|
||||
audioBuffer: params.audioBuffer,
|
||||
sourceExtension: sourceExt,
|
||||
targetExtension: preferred,
|
||||
});
|
||||
if (!outcome.ok) {
|
||||
if (outcome.reason === "transcoder-failed") {
|
||||
// Surface only the case where the host actually attempted the transcode
|
||||
// and it broke. The other reasons are by-design skips and would just be log noise.
|
||||
logVerbose(
|
||||
`TTS: pre-transcode ${sourceExt}->${preferred} for channel=${params.channel ?? "?"} failed: ${outcome.detail ?? "unknown"}`,
|
||||
);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
audioBuffer: outcome.buffer,
|
||||
fileExtension: `.${preferred}`,
|
||||
outputFormat: preferred,
|
||||
};
|
||||
}
|
||||
|
||||
export async function synthesizeSpeech(params: {
|
||||
text: string;
|
||||
cfg: OpenClawConfig;
|
||||
prefsPath?: string;
|
||||
channel?: string;
|
||||
overrides?: TtsDirectiveOverrides;
|
||||
disableFallback?: boolean;
|
||||
timeoutMs?: number;
|
||||
agentId?: string;
|
||||
accountId?: string;
|
||||
}): Promise<TtsSynthesisResult> {
|
||||
assertSpeechRuntimeAvailable();
|
||||
const setup = resolveTtsRequestSetup({
|
||||
text: params.text,
|
||||
cfg: params.cfg,
|
||||
prefsPath: params.prefsPath,
|
||||
providerOverride: params.overrides?.provider,
|
||||
disableFallback: params.disableFallback,
|
||||
agentId: params.agentId,
|
||||
channelId: params.channel,
|
||||
accountId: params.accountId,
|
||||
});
|
||||
if ("error" in setup) {
|
||||
return { success: false, error: setup.error };
|
||||
}
|
||||
|
||||
const { cfg, config, persona, providers } = setup;
|
||||
const textForSynthesis = normalizeSpeechText(params.text);
|
||||
const target = resolveTtsSynthesisTarget(params.channel);
|
||||
|
||||
const errors: string[] = [];
|
||||
const attemptedProviders: string[] = [];
|
||||
const attempts: TtsProviderAttempt[] = [];
|
||||
const primaryProvider = providers[0]?.provider;
|
||||
logVerbose(
|
||||
`TTS: starting with provider ${primaryProvider}, fallbacks: ${
|
||||
providers
|
||||
.slice(1)
|
||||
.map((entry) => entry.provider)
|
||||
.join(", ") || "none"
|
||||
}`,
|
||||
);
|
||||
|
||||
for (const { provider, voiceModel } of providers) {
|
||||
attemptedProviders.push(provider);
|
||||
const providerStart = Date.now();
|
||||
try {
|
||||
const resolvedProvider = resolveReadySpeechProvider({
|
||||
provider,
|
||||
cfg,
|
||||
config,
|
||||
persona,
|
||||
voiceModel,
|
||||
});
|
||||
if (resolvedProvider.kind === "skip") {
|
||||
errors.push(resolvedProvider.message);
|
||||
attempts.push({
|
||||
provider,
|
||||
outcome: "skipped",
|
||||
reasonCode: resolvedProvider.reasonCode,
|
||||
persona: persona?.id,
|
||||
...(resolvedProvider.personaBinding
|
||||
? { personaBinding: resolvedProvider.personaBinding }
|
||||
: {}),
|
||||
error: resolvedProvider.message,
|
||||
});
|
||||
logVerbose(`TTS: provider ${provider} skipped (${resolvedProvider.message})`);
|
||||
continue;
|
||||
}
|
||||
const timeoutMs = resolveSpeechProviderTimeoutMs({
|
||||
timeoutMs: params.timeoutMs ?? voiceModel?.timeoutMs,
|
||||
config,
|
||||
provider: resolvedProvider.provider,
|
||||
});
|
||||
const prepared = await prepareSpeechSynthesis({
|
||||
provider: resolvedProvider.provider,
|
||||
text: textForSynthesis,
|
||||
cfg,
|
||||
providerConfig: resolvedProvider.providerConfig,
|
||||
providerOverrides: params.overrides?.providerOverrides?.[resolvedProvider.provider.id],
|
||||
persona: resolvedProvider.synthesisPersona,
|
||||
personaProviderConfig: resolvedProvider.personaProviderConfig,
|
||||
target,
|
||||
timeoutMs,
|
||||
});
|
||||
const synthesis = await resolvedProvider.provider.synthesize({
|
||||
text: prepared.text,
|
||||
cfg,
|
||||
providerConfig: prepared.providerConfig,
|
||||
target,
|
||||
providerOverrides: prepared.providerOverrides,
|
||||
timeoutMs,
|
||||
});
|
||||
const latencyMs = Date.now() - providerStart;
|
||||
attempts.push({
|
||||
provider,
|
||||
outcome: "success",
|
||||
reasonCode: "success",
|
||||
persona: persona?.id,
|
||||
personaBinding: resolvedProvider.personaBinding,
|
||||
latencyMs,
|
||||
});
|
||||
return {
|
||||
success: true,
|
||||
audioBuffer: synthesis.audioBuffer,
|
||||
latencyMs,
|
||||
provider,
|
||||
providerModel: resolveTtsResultModel(prepared.providerConfig, prepared.providerOverrides),
|
||||
providerVoice: resolveTtsResultVoice(prepared.providerConfig, prepared.providerOverrides),
|
||||
persona: persona?.id,
|
||||
fallbackFrom: provider !== primaryProvider ? primaryProvider : undefined,
|
||||
attemptedProviders,
|
||||
attempts,
|
||||
outputFormat: synthesis.outputFormat,
|
||||
voiceCompatible: synthesis.voiceCompatible,
|
||||
fileExtension: synthesis.fileExtension,
|
||||
target,
|
||||
};
|
||||
} catch (err) {
|
||||
const errorMsg = formatTtsProviderError(provider, err);
|
||||
const latencyMs = Date.now() - providerStart;
|
||||
errors.push(errorMsg);
|
||||
attempts.push({
|
||||
provider,
|
||||
outcome: "failed",
|
||||
reasonCode:
|
||||
err instanceof Error && err.name === "AbortError" ? "timeout" : "provider_error",
|
||||
latencyMs,
|
||||
persona: persona?.id,
|
||||
personaBinding: resolvePersonaBinding(persona, provider),
|
||||
error: errorMsg,
|
||||
});
|
||||
const rawError = sanitizeTtsErrorForLog(err);
|
||||
if (provider === primaryProvider) {
|
||||
const hasFallbacks = providers.length > 1;
|
||||
logVerbose(
|
||||
`TTS: primary provider ${provider} failed (${rawError})${hasFallbacks ? "; trying fallback providers." : "; no fallback providers configured."}`,
|
||||
);
|
||||
} else {
|
||||
logVerbose(`TTS: ${provider} failed (${rawError}); trying next provider.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return buildTtsFailureResult(errors, attemptedProviders, attempts, persona?.id);
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { logVerbose } from "openclaw/plugin-sdk/runtime-env";
|
||||
import type { TtsDirectiveOverrides } from "openclaw/plugin-sdk/speech-core";
|
||||
import { assertSpeechRuntimeAvailable } from "./runtime-availability.js";
|
||||
import { resolveSpeechProviderTimeoutMs } from "./tts-provider-resolution.js";
|
||||
import {
|
||||
buildTtsFailureResult,
|
||||
formatTtsProviderError,
|
||||
prepareSpeechSynthesis,
|
||||
resolvePersonaBinding,
|
||||
resolveReadySpeechProvider,
|
||||
resolveTtsRequestSetup,
|
||||
resolveTtsResultModel,
|
||||
resolveTtsResultVoice,
|
||||
sanitizeTtsErrorForLog,
|
||||
} from "./tts-synthesis-support.js";
|
||||
import type { TtsProviderAttempt, TtsTelephonyResult } from "./tts-types.js";
|
||||
|
||||
export async function textToSpeechTelephony(params: {
|
||||
text: string;
|
||||
cfg: OpenClawConfig;
|
||||
prefsPath?: string;
|
||||
overrides?: TtsDirectiveOverrides;
|
||||
timeoutMs?: number;
|
||||
}): Promise<TtsTelephonyResult> {
|
||||
assertSpeechRuntimeAvailable();
|
||||
const setup = resolveTtsRequestSetup({
|
||||
text: params.text,
|
||||
cfg: params.cfg,
|
||||
prefsPath: params.prefsPath,
|
||||
providerOverride: params.overrides?.provider,
|
||||
});
|
||||
if ("error" in setup) {
|
||||
return { success: false, error: setup.error };
|
||||
}
|
||||
|
||||
const { cfg, config, persona, providers } = setup;
|
||||
const errors: string[] = [];
|
||||
const attemptedProviders: string[] = [];
|
||||
const attempts: TtsProviderAttempt[] = [];
|
||||
const primaryProvider = providers[0]?.provider;
|
||||
logVerbose(
|
||||
`TTS telephony: starting with provider ${primaryProvider}, fallbacks: ${
|
||||
providers
|
||||
.slice(1)
|
||||
.map((entry) => entry.provider)
|
||||
.join(", ") || "none"
|
||||
}`,
|
||||
);
|
||||
|
||||
for (const { provider, voiceModel } of providers) {
|
||||
attemptedProviders.push(provider);
|
||||
const providerStart = Date.now();
|
||||
try {
|
||||
const resolvedProvider = resolveReadySpeechProvider({
|
||||
provider,
|
||||
cfg,
|
||||
config,
|
||||
persona,
|
||||
voiceModel,
|
||||
requireTelephony: true,
|
||||
});
|
||||
if (resolvedProvider.kind === "skip") {
|
||||
errors.push(resolvedProvider.message);
|
||||
attempts.push({
|
||||
provider,
|
||||
outcome: "skipped",
|
||||
reasonCode: resolvedProvider.reasonCode,
|
||||
persona: persona?.id,
|
||||
...(resolvedProvider.personaBinding
|
||||
? { personaBinding: resolvedProvider.personaBinding }
|
||||
: {}),
|
||||
error: resolvedProvider.message,
|
||||
});
|
||||
logVerbose(`TTS telephony: provider ${provider} skipped (${resolvedProvider.message})`);
|
||||
continue;
|
||||
}
|
||||
const timeoutMs = resolveSpeechProviderTimeoutMs({
|
||||
timeoutMs: params.timeoutMs ?? voiceModel?.timeoutMs,
|
||||
config,
|
||||
provider: resolvedProvider.provider,
|
||||
});
|
||||
const synthesizeTelephony = resolvedProvider.provider.synthesizeTelephony as NonNullable<
|
||||
typeof resolvedProvider.provider.synthesizeTelephony
|
||||
>;
|
||||
const prepared = await prepareSpeechSynthesis({
|
||||
provider: resolvedProvider.provider,
|
||||
text: params.text,
|
||||
cfg,
|
||||
providerConfig: resolvedProvider.providerConfig,
|
||||
providerOverrides: params.overrides?.providerOverrides?.[resolvedProvider.provider.id],
|
||||
persona: resolvedProvider.synthesisPersona,
|
||||
personaProviderConfig: resolvedProvider.personaProviderConfig,
|
||||
target: "telephony",
|
||||
timeoutMs,
|
||||
});
|
||||
const synthesis = await synthesizeTelephony({
|
||||
text: prepared.text,
|
||||
cfg,
|
||||
providerConfig: prepared.providerConfig,
|
||||
providerOverrides: prepared.providerOverrides,
|
||||
timeoutMs,
|
||||
});
|
||||
const latencyMs = Date.now() - providerStart;
|
||||
attempts.push({
|
||||
provider,
|
||||
outcome: "success",
|
||||
reasonCode: "success",
|
||||
persona: persona?.id,
|
||||
personaBinding: resolvedProvider.personaBinding,
|
||||
latencyMs,
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
audioBuffer: synthesis.audioBuffer,
|
||||
latencyMs,
|
||||
provider,
|
||||
providerModel: resolveTtsResultModel(prepared.providerConfig, prepared.providerOverrides),
|
||||
providerVoice: resolveTtsResultVoice(prepared.providerConfig, prepared.providerOverrides),
|
||||
persona: persona?.id,
|
||||
fallbackFrom: provider !== primaryProvider ? primaryProvider : undefined,
|
||||
attemptedProviders,
|
||||
attempts,
|
||||
outputFormat: synthesis.outputFormat,
|
||||
sampleRate: synthesis.sampleRate,
|
||||
};
|
||||
} catch (err) {
|
||||
const errorMsg = formatTtsProviderError(provider, err);
|
||||
const latencyMs = Date.now() - providerStart;
|
||||
errors.push(errorMsg);
|
||||
attempts.push({
|
||||
provider,
|
||||
outcome: "failed",
|
||||
reasonCode:
|
||||
err instanceof Error && err.name === "AbortError" ? "timeout" : "provider_error",
|
||||
latencyMs,
|
||||
persona: persona?.id,
|
||||
personaBinding: resolvePersonaBinding(persona, provider),
|
||||
error: errorMsg,
|
||||
});
|
||||
const rawError = sanitizeTtsErrorForLog(err);
|
||||
if (provider === primaryProvider) {
|
||||
const hasFallbacks = providers.length > 1;
|
||||
logVerbose(
|
||||
`TTS telephony: primary provider ${provider} failed (${rawError})${hasFallbacks ? "; trying fallback providers." : "; no fallback providers configured."}`,
|
||||
);
|
||||
} else {
|
||||
logVerbose(`TTS telephony: ${provider} failed (${rawError}); trying next provider.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return buildTtsFailureResult(errors, attemptedProviders, attempts, persona?.id);
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
type TtsAttemptReasonCode =
|
||||
| "success"
|
||||
| "no_provider_registered"
|
||||
| "not_configured"
|
||||
| "unsupported_for_streaming"
|
||||
| "unsupported_for_telephony"
|
||||
| "timeout"
|
||||
| "provider_error";
|
||||
|
||||
export type TtsProviderAttempt = {
|
||||
provider: string;
|
||||
outcome: "success" | "skipped" | "failed";
|
||||
reasonCode: TtsAttemptReasonCode;
|
||||
persona?: string;
|
||||
personaBinding?: "applied" | "missing" | "none";
|
||||
latencyMs?: number;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
export type TtsResult = {
|
||||
success: boolean;
|
||||
audioPath?: string;
|
||||
error?: string;
|
||||
latencyMs?: number;
|
||||
provider?: string;
|
||||
persona?: string;
|
||||
fallbackFrom?: string;
|
||||
attemptedProviders?: string[];
|
||||
attempts?: TtsProviderAttempt[];
|
||||
outputFormat?: string;
|
||||
voiceCompatible?: boolean;
|
||||
audioAsVoice?: boolean;
|
||||
target?: "audio-file" | "voice-note";
|
||||
};
|
||||
|
||||
export type TtsSynthesisResult = {
|
||||
success: boolean;
|
||||
audioBuffer?: Buffer;
|
||||
error?: string;
|
||||
latencyMs?: number;
|
||||
provider?: string;
|
||||
providerModel?: string;
|
||||
providerVoice?: string;
|
||||
persona?: string;
|
||||
fallbackFrom?: string;
|
||||
attemptedProviders?: string[];
|
||||
attempts?: TtsProviderAttempt[];
|
||||
outputFormat?: string;
|
||||
voiceCompatible?: boolean;
|
||||
fileExtension?: string;
|
||||
target?: "audio-file" | "voice-note";
|
||||
};
|
||||
|
||||
export type TtsStreamResult = {
|
||||
success: boolean;
|
||||
audioStream?: ReadableStream<Uint8Array>;
|
||||
error?: string;
|
||||
latencyMs?: number;
|
||||
provider?: string;
|
||||
providerModel?: string;
|
||||
providerVoice?: string;
|
||||
persona?: string;
|
||||
fallbackFrom?: string;
|
||||
attemptedProviders?: string[];
|
||||
attempts?: TtsProviderAttempt[];
|
||||
outputFormat?: string;
|
||||
voiceCompatible?: boolean;
|
||||
fileExtension?: string;
|
||||
target?: "audio-file" | "voice-note";
|
||||
release?: () => Promise<void>;
|
||||
};
|
||||
|
||||
export type TtsSynthesisStreamResult = TtsStreamResult;
|
||||
|
||||
export type TtsTelephonyResult = {
|
||||
success: boolean;
|
||||
audioBuffer?: Buffer;
|
||||
error?: string;
|
||||
latencyMs?: number;
|
||||
provider?: string;
|
||||
providerModel?: string;
|
||||
providerVoice?: string;
|
||||
persona?: string;
|
||||
fallbackFrom?: string;
|
||||
attemptedProviders?: string[];
|
||||
attempts?: TtsProviderAttempt[];
|
||||
outputFormat?: string;
|
||||
sampleRate?: number;
|
||||
};
|
||||
|
||||
export type TtsStatusEntry = {
|
||||
timestamp: number;
|
||||
success: boolean;
|
||||
textLength: number;
|
||||
summarized: boolean;
|
||||
provider?: string;
|
||||
persona?: string;
|
||||
fallbackFrom?: string;
|
||||
attemptedProviders?: string[];
|
||||
attempts?: TtsProviderAttempt[];
|
||||
latencyMs?: number;
|
||||
error?: string;
|
||||
};
|
||||
+41
-1752
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user