From ecc49b5a870b8c35108c656ef619dc528abb149e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 02:25:48 -0700 Subject: [PATCH] refactor(tts): absorb speech core package (#118513) * refactor(tts): absorb speech core package * fix(tts): preserve runtime SDK exports * refactor(tts): remove private package exports * test(tts): align canonical runtime mocks * test(tts): complete canonical settings mocks * chore(plugin-sdk): regenerate API baseline for #118513 --- config/knip.config.ts | 5 - config/max-lines-baseline.txt | 1 - .../.generated/plugin-sdk-api-baseline.sha256 | 6 +- .../messaging/trusted-media-path.test.ts | 2 +- .../speech-core-runtime-api.d.ts | 34 - extensions/xai/tsconfig.json | 3 - packages/speech-core/package.json | 34 - packages/speech-core/runtime-api.ts | 49 - packages/speech-core/src/tts-settings.ts | 401 ---- packages/speech-core/src/tts.test.ts | 1877 ----------------- packages/speech-core/tsconfig.json | 16 - pnpm-lock.yaml | 6 - qa/scenarios/media/webchat-auto-tts.yaml | 2 +- scripts/lib/extension-package-boundary.ts | 1 - scripts/lib/tsdown-output-roots.mjs | 1 - src/agents/cli-runner.reliability.test.ts | 2 + .../cli-runner/helpers.system-prompt.test.ts | 2 + src/agents/cli-runner/prepare.test.ts | 2 + .../attempt.spawn-workspace.test-support.ts | 2 + .../system-prompt.test.ts | 2 + src/agents/system-prompt-config.test.ts | 2 + .../reply/commands-system-prompt.test.ts | 2 + src/channels/plugins/types.core.ts | 2 +- src/gateway/server-methods/talk-shared.ts | 12 +- src/gateway/server-methods/talk.ts | 15 +- src/gateway/server.talk-runtime.test.ts | 2 +- src/infra/outbound/reply-payload-parts.ts | 71 + src/plugin-sdk/facade-runtime.test.ts | 2 +- src/plugin-sdk/reply-payload.ts | 82 +- src/plugin-sdk/tts-runtime.ts | 21 +- src/plugins/capability-provider-runtime.ts | 2 +- src/plugins/model-catalog-registration.ts | 2 +- src/plugins/runtime/runtime-tts-request.ts | 4 +- .../src/tts.ts => src/tts/runtime-api.ts | 54 +- .../src => src/tts}/runtime-availability.ts | 2 +- {packages/speech-core => src/tts}/speaker.ts | 2 +- .../src => src/tts}/speech-text.test.ts | 2 +- .../src => src/tts}/speech-text.ts | 2 +- src/tts/tts-audio-store.ts | 4 +- .../src => src/tts}/tts-payload.ts | 26 +- .../tts}/tts-provider-resolution.ts | 40 +- .../src => src/tts}/tts-request.ts | 21 +- src/tts/tts-runtime-fallbacks.test.ts | 462 ++++ src/tts/tts-runtime-models.test.ts | 341 +++ src/tts/tts-runtime-personas.test.ts | 496 +++++ src/tts/tts-runtime-routing.test.ts | 447 ++++ .../tts/tts-runtime-types.ts | 0 src/tts/tts-runtime.test-support.ts | 297 +++ .../src => src/tts}/tts-settings-writes.ts | 6 +- src/tts/tts-settings.ts | 402 +++- .../src => src/tts}/tts-streaming.ts | 6 +- .../src => src/tts}/tts-synthesis-support.ts | 24 +- .../src => src/tts}/tts-synthesis.ts | 12 +- .../src => src/tts}/tts-telephony.ts | 6 +- src/tts/tts.test.ts | 31 +- src/tts/tts.ts | 21 +- .../speech-core => src/tts}/voice-models.ts | 4 +- test/e2e/qa-lab/runtime/media-talk-gateway.ts | 2 +- test/vitest-scoped-config.test.ts | 4 +- tsconfig.json | 5 - tsdown.config.ts | 18 - 61 files changed, 2693 insertions(+), 2711 deletions(-) delete mode 100644 extensions/xai/.boundary-stubs/speech-core-runtime-api.d.ts delete mode 100644 packages/speech-core/package.json delete mode 100644 packages/speech-core/runtime-api.ts delete mode 100644 packages/speech-core/src/tts-settings.ts delete mode 100644 packages/speech-core/src/tts.test.ts delete mode 100644 packages/speech-core/tsconfig.json create mode 100644 src/infra/outbound/reply-payload-parts.ts rename packages/speech-core/src/tts.ts => src/tts/runtime-api.ts (57%) rename {packages/speech-core/src => src/tts}/runtime-availability.ts (89%) rename {packages/speech-core => src/tts}/speaker.ts (96%) rename {packages/speech-core/src => src/tts}/speech-text.test.ts (98%) rename {packages/speech-core/src => src/tts}/speech-text.ts (98%) rename {packages/speech-core/src => src/tts}/tts-payload.ts (92%) rename {packages/speech-core/src => src/tts}/tts-provider-resolution.ts (96%) rename {packages/speech-core/src => src/tts}/tts-request.ts (87%) create mode 100644 src/tts/tts-runtime-fallbacks.test.ts create mode 100644 src/tts/tts-runtime-models.test.ts create mode 100644 src/tts/tts-runtime-personas.test.ts create mode 100644 src/tts/tts-runtime-routing.test.ts rename packages/speech-core/src/tts-types.ts => src/tts/tts-runtime-types.ts (100%) create mode 100644 src/tts/tts-runtime.test-support.ts rename {packages/speech-core/src => src/tts}/tts-settings-writes.ts (87%) rename {packages/speech-core/src => src/tts}/tts-streaming.ts (94%) rename {packages/speech-core/src => src/tts}/tts-synthesis-support.ts (95%) rename {packages/speech-core/src => src/tts}/tts-synthesis.ts (94%) rename {packages/speech-core/src => src/tts}/tts-telephony.ts (89%) rename {packages/speech-core => src/tts}/voice-models.ts (98%) diff --git a/config/knip.config.ts b/config/knip.config.ts index 17fdce530266..865c7a969a74 100644 --- a/config/knip.config.ts +++ b/config/knip.config.ts @@ -620,11 +620,6 @@ const config = { entry: ["src/*.ts!", "src/host/embeddings-worker-child.ts!"], project: ["src/**/*.ts!"], }, - "packages/speech-core": { - entry: ["runtime-api.ts!", "speaker.ts!", "voice-models.ts!"], - project: ["**/*.ts!"], - ignoreDependencies: ["openclaw"], - }, "packages/*": { entry: ["index.js!", "scripts/postinstall.js!"], project: ["index.js!", "scripts/**/*.js!"], diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index 1d197eaaf706..e40c65b6a56b 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -320,7 +320,6 @@ packages/markdown-core/src/ir.ts packages/memory-host-sdk/src/host/session-files.ts packages/sdk/src/client.ts packages/sdk/src/index.test.ts -packages/speech-core/src/tts.test.ts packages/tool-call-repair/src/stream-normalizer.test.ts packages/tool-call-repair/src/stream-normalizer.ts src/acp/control-plane/manager.test.ts diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index bca39b21bf78..fc1b08e20761 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -6,7 +6,7 @@ e5e67ddf3cab38fcbf9220bc3160715897e2709d9a9ff6ff36f1ecc9453c2367 module/agent-c 74daa746deb548379d3f0d6eac3c4d082df1034c4360cc03bf51fee0f10a2e4d module/agent-harness 95a907e1c33305b9473be64cc8d723e1a12b909eda86b15b94cee91879fb6a89 module/agent-harness-runtime 5168648cd946abad8a92822889f13ceacc87ed502314a66190d0b1eb8ebe76ea module/agent-media-payload -fd54eb654443d646d6430d2be99d1f25c32701f1c45aaa870ffe74aefc7d7f00 module/agent-runtime +6ee8bb70cd7b8a5a976ee84cd0c6e632dbfa5e4a047f616cdc00fa5b27879b27 module/agent-runtime 56b6d5fb6af3d95af1200065aca2e7d4f59e5fa59740505fe6ff433077ef6646 module/allow-from 55cea5390d68839ca7768b4a0cc570b17b65fa0fa3bc4d76130ef0f16cb79ede module/allowlist-config-edit 7ddd81bd5f55de9adf64bf4d92d012f24b37b6da0a72805a3a220d8feff24ca3 module/approval-auth-runtime @@ -29,7 +29,7 @@ c0f910ebfa3dbf283145fb1e3b9c016d03e853ef13e70402b09f3b9d9c2f4ab0 module/channel 9a5aaf650f9242523bb57bdc2556c323ab64e55aa11e25e2685b73b23ee12534 module/channel-dm-policy ba41c40956d6b4565605fa38c2d12f4b8471a0f8afe842798716b9032ee4d74a module/channel-entry-contract 982f29a18e07228e3da82cae67d06ff38249592a29c2fd28f01f0d2016ff80d9 module/channel-feedback -d645d24bcb7a5f68cc46c692ad0d1fbd19be0a99996f31e9479ce9cffce301c1 module/channel-inbound +1d13edd07a8ae8e21ece6c542cb0cde22e5b6e11d3bb9b2bfaa5ebd2a5239bf8 module/channel-inbound 76bb7f531f3702c801e8fe7479e9e499f601fb361a4303afdcb45fc0da440e4b module/channel-inbound-debounce df567ce2f4a4ba8a0937f825c46e83763412a724e36b72ce727dc4203c7dd134 module/channel-ingress-runtime 0e6efb79730fae59bb549ad00d9af2848c139b4bb1762e83b66886284e1c1421 module/channel-lifecycle @@ -107,7 +107,7 @@ aa2a56b4448c8ebdec9d06aac95d809995f533093d42fa32cd75e1d852967245 module/questio 2e09c3181e79e157ed5366b144d116ef8cc06023256ace3fa59b35c43cab513a module/reply-chunking 7994045066b29af1fc6b36ae32068f2a6f277195971af84701cb739cc23d0579 module/reply-dispatch-runtime ac2b199e95c5c8b1e2a65e62bd41d1b6322e531bca294ef4979a297a12640bce module/reply-history -f394fe4d5a7ed9e4d574063ae44e8d6af85c9a0e7d8b329f750ca16b0664325f module/reply-payload +ad69a4a6970cfac86f9379efb927beac898217946d025c3aea55e6f399da08bb module/reply-payload 1f899eb54013f268d6698ce8e6943289ea0e872747e8e868259106829518db86 module/reply-runtime b4043b356372f6af64ee3c26e4d6a6d623b817e4d95346358dc0e64a3b61d1e0 module/root-walk 97fc4ed1ac6e62b7af95b4352cee3893252d8b691fef71a000c2602b3128541a module/routing diff --git a/extensions/qqbot/src/engine/messaging/trusted-media-path.test.ts b/extensions/qqbot/src/engine/messaging/trusted-media-path.test.ts index 26a491c183c3..d3e9e5b95123 100644 --- a/extensions/qqbot/src/engine/messaging/trusted-media-path.test.ts +++ b/extensions/qqbot/src/engine/messaging/trusted-media-path.test.ts @@ -20,7 +20,7 @@ afterEach(() => { }); function makeTtsStyleVoiceFile(): string { - // Mirrors cron auto-TTS: speech-core writes the voice file under the preferred + // Mirrors cron auto-TTS: the TTS runtime writes the voice file under the preferred // OpenClaw temp root, which is outside the QQ Bot media storage tree. const tmpRoot = resolvePreferredOpenClawTmpDir(); const ttsDir = makeTrackedDir(tmpRoot, "tts-"); diff --git a/extensions/xai/.boundary-stubs/speech-core-runtime-api.d.ts b/extensions/xai/.boundary-stubs/speech-core-runtime-api.d.ts deleted file mode 100644 index 1845352dfd8a..000000000000 --- a/extensions/xai/.boundary-stubs/speech-core-runtime-api.d.ts +++ /dev/null @@ -1,34 +0,0 @@ -// Xai type declarations define plugin contracts. -export type ResolvedTtsConfig = unknown; -export type ResolvedTtsModelOverrides = unknown; -export type TtsDirectiveOverrides = unknown; -export type TtsDirectiveParseResult = unknown; -export type TtsResult = unknown; -export type TtsSynthesisResult = unknown; -export type TtsTelephonyResult = unknown; - -export const testApi: unknown; -export { testApi as _test }; -export const buildTtsSystemPromptHint: (...args: unknown[]) => unknown; -export const getLastTtsAttempt: (...args: unknown[]) => unknown; -export const getResolvedSpeechProviderConfig: (...args: unknown[]) => unknown; -export const getTtsMaxLength: (...args: unknown[]) => unknown; -export const getTtsProvider: (...args: unknown[]) => unknown; -export const isSummarizationEnabled: (...args: unknown[]) => unknown; -export const isTtsEnabled: (...args: unknown[]) => unknown; -export const isTtsProviderConfigured: (...args: unknown[]) => unknown; -export const listSpeechVoices: (...args: unknown[]) => unknown; -export const maybeApplyTtsToPayload: (...args: unknown[]) => unknown; -export const resolveTtsAutoMode: (...args: unknown[]) => unknown; -export const resolveTtsConfig: (...args: unknown[]) => unknown; -export const resolveTtsPrefsPath: (...args: unknown[]) => unknown; -export const resolveTtsProviderOrder: (...args: unknown[]) => unknown; -export const setLastTtsAttempt: (...args: unknown[]) => unknown; -export const setSummarizationEnabled: (...args: unknown[]) => unknown; -export const setTtsAutoMode: (...args: unknown[]) => unknown; -export const setTtsEnabled: (...args: unknown[]) => unknown; -export const setTtsMaxLength: (...args: unknown[]) => unknown; -export const setTtsProvider: (...args: unknown[]) => unknown; -export const synthesizeSpeech: (...args: unknown[]) => unknown; -export const textToSpeech: (...args: unknown[]) => unknown; -export const textToSpeechTelephony: (...args: unknown[]) => unknown; diff --git a/extensions/xai/tsconfig.json b/extensions/xai/tsconfig.json index 994c2c99ade7..43db97d95e7f 100644 --- a/extensions/xai/tsconfig.json +++ b/extensions/xai/tsconfig.json @@ -848,9 +848,6 @@ ], "@openclaw/ollama/runtime-api.js": [ "./.boundary-stubs/ollama-runtime-api.d.ts" - ], - "@openclaw/speech-core/runtime-api.js": [ - "./.boundary-stubs/speech-core-runtime-api.d.ts" ] } } diff --git a/packages/speech-core/package.json b/packages/speech-core/package.json deleted file mode 100644 index 1fd094355af6..000000000000 --- a/packages/speech-core/package.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "name": "@openclaw/speech-core", - "version": "2026.5.31", - "private": true, - "description": "OpenClaw speech runtime package", - "type": "module", - "main": "./dist/runtime-api.mjs", - "types": "./dist/runtime-api.d.mts", - "exports": { - ".": { - "types": "./dist/runtime-api.d.mts", - "import": "./dist/runtime-api.mjs", - "default": "./dist/runtime-api.mjs" - }, - "./runtime-api": { - "types": "./dist/runtime-api.d.mts", - "import": "./dist/runtime-api.mjs", - "default": "./dist/runtime-api.mjs" - }, - "./speaker": { - "types": "./dist/speaker.d.mts", - "import": "./dist/speaker.mjs", - "default": "./dist/speaker.mjs" - }, - "./voice-models": { - "types": "./dist/voice-models.d.mts", - "import": "./dist/voice-models.mjs", - "default": "./dist/voice-models.mjs" - } - }, - "dependencies": { - "openclaw": "workspace:*" - } -} diff --git a/packages/speech-core/runtime-api.ts b/packages/speech-core/runtime-api.ts deleted file mode 100644 index b63c16349142..000000000000 --- a/packages/speech-core/runtime-api.ts +++ /dev/null @@ -1,49 +0,0 @@ -// Runtime speech API barrel for TTS preferences, synthesis, streaming, and test -// helpers used by speech-capable plugins. -export { setSpeechRuntimeAvailabilityGuard } from "./src/runtime-availability.js"; -export { - buildTtsSystemPromptHint, - getTtsMaxLength, - getTtsPersona, - isSummarizationEnabled, - isTtsEnabled, - listTtsPersonas, - resolveTtsAutoMode, - resolveTtsConfig, - resolveTtsPrefsPath, - setTtsMachinePrefsPathResolver, - type ResolvedTtsConfig, - type ResolvedTtsModelOverrides, -} from "./src/tts-settings.js"; -export { - setSummarizationEnabled, - setTtsAutoMode, - setTtsEnabled, - setTtsMaxLength, - setTtsPersona, - setTtsProvider, -} from "./src/tts-settings-writes.js"; -export { - getLastTtsAttempt, - getResolvedSpeechProviderConfig, - getTtsProvider, - isTtsProviderConfigured, - listSpeechVoices, - prepareTtsRequest, - resolveExplicitTtsOverrides, - resolveTtsProviderOrder, - setLastTtsAttempt, - synthesizeSpeech, - streamSpeech, - textToSpeechStream, - textToSpeechTelephony, - testApi as _test, - testApi, - type TtsDirectiveOverrides, - type TtsDirectiveParseResult, - type PreparedTtsRequest, - type TtsSynthesisResult, - type TtsSynthesisStreamResult, - type TtsStreamResult, - type TtsTelephonyResult, -} from "./src/tts.js"; diff --git a/packages/speech-core/src/tts-settings.ts b/packages/speech-core/src/tts-settings.ts deleted file mode 100644 index cfd56072a276..000000000000 --- a/packages/speech-core/src/tts-settings.ts +++ /dev/null @@ -1,401 +0,0 @@ -// Lightweight TTS settings resolution shared by agent prompts, status, and speech runtime. -import { existsSync, readFileSync } from "node:fs"; -import path from "node:path"; -import type { - OpenClawConfig, - ResolvedTtsPersona, - TtsAutoMode, - TtsConfig, - TtsModelOverrideConfig, - TtsProvider, -} from "openclaw/plugin-sdk/config-contracts"; -import { - getRuntimeConfigSnapshot, - getRuntimeConfigSourceSnapshot, - selectApplicableRuntimeConfig, -} from "openclaw/plugin-sdk/runtime-config-snapshot"; -import type { SpeechProviderConfig } from "openclaw/plugin-sdk/speech-core"; -import { - normalizeSpeechProviderId, - normalizeTtsAutoMode, - resolveEffectiveTtsConfig, - type ResolvedTtsConfig, - type ResolvedTtsModelOverrides, - type TtsConfigResolutionContext, -} from "openclaw/plugin-sdk/speech-settings"; -import { - normalizeOptionalLowercaseString, - normalizeOptionalString, -} from "openclaw/plugin-sdk/string-coerce-runtime"; -import { resolveConfigDir, resolveUserPath } from "openclaw/plugin-sdk/text-utility-runtime"; -import { withSpeakerSelectionCompat } from "../speaker.js"; - -export type { ResolvedTtsConfig, ResolvedTtsModelOverrides }; - -export const DEFAULT_TTS_TIMEOUT_MS = 30_000; -const DEFAULT_TTS_MAX_LENGTH = 1500; -const DEFAULT_TTS_SUMMARIZE = true; -const DEFAULT_MAX_TEXT_LENGTH = 4096; -let machinePrefsPathResolver: () => string | undefined = () => undefined; - -export function setTtsMachinePrefsPathResolver(resolver?: () => string | undefined): void { - machinePrefsPathResolver = resolver ?? (() => undefined); -} - -export type TtsUserPrefs = { - tts?: { - auto?: TtsAutoMode; - enabled?: boolean; - provider?: TtsProvider; - persona?: string | null; - maxLength?: number; - summarize?: boolean; - }; -}; - -function resolveConfiguredTtsAutoMode(raw: TtsConfig): TtsAutoMode { - return normalizeTtsAutoMode(raw.auto) ?? (raw.enabled ? "always" : "off"); -} - -export function normalizeConfiguredSpeechProviderId( - providerId: string | undefined, -): TtsProvider | undefined { - const normalized = normalizeSpeechProviderId(providerId); - if (!normalized) { - return undefined; - } - return normalized === "edge" ? "microsoft" : normalized; -} - -export function normalizeTtsPersonaId(personaId: string | null | undefined): string | undefined { - return normalizeOptionalLowercaseString(personaId ?? undefined); -} - -function resolveTtsPrefsPathValue(prefsPath: string | undefined): string { - // Scoped agent paths must win over the migrated machine-wide default. - if (prefsPath?.trim()) { - return resolveUserPath(prefsPath.trim()); - } - const envPath = process.env.OPENCLAW_TTS_PREFS?.trim(); - if (envPath) { - return resolveUserPath(envPath); - } - const machinePath = machinePrefsPathResolver()?.trim(); - if (machinePath) { - return resolveUserPath(machinePath); - } - return path.join(resolveConfigDir(process.env), "settings", "tts.json"); -} - -export function resolveModelOverridePolicy( - overrides: TtsModelOverrideConfig | undefined, -): ResolvedTtsModelOverrides { - const enabled = overrides?.enabled ?? true; - if (!enabled) { - return { - enabled: false, - allowText: false, - allowProvider: false, - allowVoice: false, - allowModelId: false, - allowVoiceSettings: false, - allowNormalization: false, - allowSeed: false, - }; - } - const allow = (value: boolean | undefined, defaultValue = true) => value ?? defaultValue; - return { - enabled: true, - allowText: allow(overrides?.allowText), - allowProvider: allow(overrides?.allowProvider, false), - allowVoice: allow(overrides?.allowVoice), - allowModelId: allow(overrides?.allowModelId), - allowVoiceSettings: allow(overrides?.allowVoiceSettings), - allowNormalization: allow(overrides?.allowNormalization), - allowSeed: allow(overrides?.allowSeed), - }; -} - -export function resolveTtsRuntimeConfig(cfg: OpenClawConfig): OpenClawConfig { - return ( - selectApplicableRuntimeConfig({ - inputConfig: cfg, - runtimeConfig: getRuntimeConfigSnapshot(), - runtimeSourceConfig: getRuntimeConfigSourceSnapshot(), - }) ?? cfg - ); -} - -export function asProviderConfig(value: unknown): SpeechProviderConfig { - return typeof value === "object" && value !== null && !Array.isArray(value) - ? withSpeakerSelectionCompat(value as SpeechProviderConfig) - : {}; -} - -export function asProviderConfigMap(value: unknown): Record { - return typeof value === "object" && value !== null && !Array.isArray(value) - ? (value as Record) - : {}; -} - -export function hasOwnProperty(value: object, key: string): boolean { - return Object.hasOwn(value, key); -} - -function normalizeProviderConfigMap( - value: unknown, -): Record | undefined { - const rawMap = asProviderConfigMap(value); - if (Object.keys(rawMap).length === 0) { - return undefined; - } - const next: Record = {}; - for (const [providerId, providerConfig] of Object.entries(rawMap)) { - const normalized = normalizeConfiguredSpeechProviderId(providerId) ?? providerId; - next[normalized] = asProviderConfig(providerConfig); - } - return next; -} - -function collectTtsPersonas(raw: TtsConfig): Record { - const rawPersonas = asProviderConfigMap(raw.personas); - const personas: Record = {}; - for (const [id, value] of Object.entries(rawPersonas)) { - const normalizedId = normalizeTtsPersonaId(id); - if (!normalizedId || typeof value !== "object" || value === null || Array.isArray(value)) { - continue; - } - const persona = value as Omit; - personas[normalizedId] = { - ...persona, - id: normalizedId, - provider: normalizeConfiguredSpeechProviderId(persona.provider) ?? persona.provider, - providers: normalizeProviderConfigMap(persona.providers), - }; - } - return personas; -} - -function collectDirectProviderConfigEntries(raw: TtsConfig): Record { - const entries: Record = {}; - const rawProviders = asProviderConfigMap(raw.providers); - for (const [providerId, value] of Object.entries(rawProviders)) { - const normalized = normalizeConfiguredSpeechProviderId(providerId) ?? providerId; - entries[normalized] = asProviderConfig(value); - } - const reservedKeys = new Set([ - "auto", - "enabled", - "maxTextLength", - "mode", - "modelOverrides", - "persona", - "personas", - "prefsPath", - "provider", - "providers", - "summaryModel", - "timeoutMs", - ]); - for (const [key, value] of Object.entries(raw as Record)) { - if (reservedKeys.has(key)) { - continue; - } - if (typeof value !== "object" || value === null || Array.isArray(value)) { - continue; - } - const normalized = normalizeConfiguredSpeechProviderId(key) ?? key; - entries[normalized] ??= asProviderConfig(value); - } - return entries; -} - -export function resolveTtsConfig( - cfgInput: OpenClawConfig, - contextOrAgentId?: string | TtsConfigResolutionContext, -): ResolvedTtsConfig { - const cfg = resolveTtsRuntimeConfig(cfgInput); - const raw: TtsConfig = resolveEffectiveTtsConfig(cfg, contextOrAgentId); - const providerSource = raw.provider ? "config" : "default"; - const timeoutMs = raw.timeoutMs ?? DEFAULT_TTS_TIMEOUT_MS; - const timeoutMsSource = raw.timeoutMs === undefined ? "default" : "config"; - return { - auto: resolveConfiguredTtsAutoMode(raw), - mode: raw.mode ?? "final", - provider: - normalizeConfiguredSpeechProviderId(raw.provider) ?? - (providerSource === "config" ? (normalizeOptionalLowercaseString(raw.provider) ?? "") : ""), - providerSource, - persona: normalizeTtsPersonaId(raw.persona), - personas: collectTtsPersonas(raw), - summaryModel: normalizeOptionalString(raw.summaryModel), - modelOverrides: resolveModelOverridePolicy(raw.modelOverrides), - providerConfigs: collectDirectProviderConfigEntries(raw), - prefsPath: (raw as TtsConfig & { prefsPath?: string }).prefsPath, - maxTextLength: raw.maxTextLength ?? DEFAULT_MAX_TEXT_LENGTH, - timeoutMs, - timeoutMsSource, - rawConfig: raw, - sourceConfig: cfg, - }; -} - -export function resolveTtsPrefsPath(config: ResolvedTtsConfig): string { - return resolveTtsPrefsPathValue(config.prefsPath); -} - -export function readTtsPrefs(prefsPath: string): TtsUserPrefs { - try { - if (!existsSync(prefsPath)) { - return {}; - } - const parsed: unknown = JSON.parse(readFileSync(prefsPath, "utf8")); - return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) - ? (parsed as TtsUserPrefs) - : {}; - } catch { - return {}; - } -} - -function resolveTtsAutoModeFromPrefs(prefs: TtsUserPrefs): TtsAutoMode | undefined { - const auto = normalizeTtsAutoMode(prefs.tts?.auto); - if (auto) { - return auto; - } - if (typeof prefs.tts?.enabled === "boolean") { - return prefs.tts.enabled ? "always" : "off"; - } - return undefined; -} - -export function resolveTtsAutoMode(params: { - config: ResolvedTtsConfig; - prefsPath: string; - sessionAuto?: string; -}): TtsAutoMode { - const sessionAuto = normalizeTtsAutoMode(params.sessionAuto); - if (sessionAuto) { - return sessionAuto; - } - return resolveTtsAutoModeFromPrefs(readTtsPrefs(params.prefsPath)) ?? params.config.auto; -} - -function resolveTtsPersonaIdFromPrefs( - config: ResolvedTtsConfig, - prefs: TtsUserPrefs, -): string | undefined { - if (prefs.tts && hasOwnProperty(prefs.tts, "persona")) { - return normalizeTtsPersonaId(prefs.tts.persona); - } - return normalizeTtsPersonaId(config.persona); -} - -export function resolveTtsPersonaFromPrefs( - config: ResolvedTtsConfig, - prefs: TtsUserPrefs, -): ResolvedTtsPersona | undefined { - const personaId = resolveTtsPersonaIdFromPrefs(config, prefs); - return personaId ? config.personas[personaId] : undefined; -} - -type ResolvedTtsSettingsSnapshot = { - autoMode: TtsAutoMode; - config: ResolvedTtsConfig; - maxLength: number; - persona?: ResolvedTtsPersona; - personaId?: string; - preferredProvider?: TtsProvider; - prefsPath: string; - summarize: boolean; -}; - -export function resolveTtsSettingsSnapshot(params: { - cfg: OpenClawConfig; - sessionAuto?: string; - agentId?: string; - channelId?: string; - accountId?: string; -}): ResolvedTtsSettingsSnapshot { - const config = resolveTtsConfig(params.cfg, { - agentId: params.agentId, - channelId: params.channelId, - accountId: params.accountId, - }); - const prefsPath = resolveTtsPrefsPath(config); - const prefs = readTtsPrefs(prefsPath); - const personaId = resolveTtsPersonaIdFromPrefs(config, prefs); - const persona = personaId ? config.personas[personaId] : undefined; - const preferredProvider = - normalizeConfiguredSpeechProviderId(prefs.tts?.provider) ?? - normalizeConfiguredSpeechProviderId(persona?.provider) ?? - (config.providerSource === "config" - ? (normalizeConfiguredSpeechProviderId(config.provider) ?? config.provider) - : undefined); - return { - autoMode: - normalizeTtsAutoMode(params.sessionAuto) ?? resolveTtsAutoModeFromPrefs(prefs) ?? config.auto, - config, - maxLength: prefs.tts?.maxLength ?? DEFAULT_TTS_MAX_LENGTH, - ...(persona ? { persona } : {}), - ...(personaId ? { personaId } : {}), - ...(preferredProvider ? { preferredProvider } : {}), - prefsPath, - summarize: prefs.tts?.summarize ?? DEFAULT_TTS_SUMMARIZE, - }; -} - -export function buildTtsSystemPromptHint( - cfg: OpenClawConfig, - agentId?: string, -): string | undefined { - const settings = resolveTtsSettingsSnapshot({ cfg, agentId }); - if (settings.autoMode === "off") { - return undefined; - } - const autoHint = - settings.autoMode === "inbound" - ? "Only use TTS when the user's last message includes audio/voice." - : settings.autoMode === "tagged" - ? "Only use TTS when you include [[tts:key=value]] directives or a [[tts:text]]...[[/tts:text]] block." - : undefined; - return [ - "Voice (TTS) is enabled.", - autoHint, - settings.persona - ? `Active TTS persona: ${settings.persona.label ?? settings.persona.id}${settings.persona.description ? ` - ${settings.persona.description}` : ""}.` - : undefined, - `Keep spoken text ≤${settings.maxLength} chars to avoid auto-summary (summary ${settings.summarize ? "on" : "off"}).`, - "If workspace context (especially MEMORY.md) tells you not to use [[tts:...]] or to use a local/non-tagged voice workflow, follow that workspace instruction instead.", - "Use [[tts:...]] and optional [[tts:text]]...[[/tts:text]] to control voice/expressiveness.", - ] - .filter(Boolean) - .join("\n"); -} - -export function isTtsEnabled( - config: ResolvedTtsConfig, - prefsPath: string, - sessionAuto?: string, -): boolean { - return resolveTtsAutoMode({ config, prefsPath, sessionAuto }) !== "off"; -} - -export function getTtsPersona( - config: ResolvedTtsConfig, - prefsPath: string, -): ResolvedTtsPersona | undefined { - return resolveTtsPersonaFromPrefs(config, readTtsPrefs(prefsPath)); -} - -export function listTtsPersonas(config: ResolvedTtsConfig): ResolvedTtsPersona[] { - return Object.values(config.personas).toSorted((left, right) => left.id.localeCompare(right.id)); -} - -export function getTtsMaxLength(prefsPath: string): number { - return readTtsPrefs(prefsPath).tts?.maxLength ?? DEFAULT_TTS_MAX_LENGTH; -} - -export function isSummarizationEnabled(prefsPath: string): boolean { - return readTtsPrefs(prefsPath).tts?.summarize ?? DEFAULT_TTS_SUMMARIZE; -} diff --git a/packages/speech-core/src/tts.test.ts b/packages/speech-core/src/tts.test.ts deleted file mode 100644 index fc9e6ded0438..000000000000 --- a/packages/speech-core/src/tts.test.ts +++ /dev/null @@ -1,1877 +0,0 @@ -// Speech Core tests cover tts behavior. -import crypto from "node:crypto"; -import { mkdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import type { OpenClawConfig, TtsConfig } from "openclaw/plugin-sdk/config-contracts"; -import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime"; -import type { ReplyPayload } from "openclaw/plugin-sdk/reply-payload"; -import { - clearRuntimeConfigSnapshot, - setRuntimeConfigSnapshot, -} from "openclaw/plugin-sdk/runtime-config-snapshot"; -import type { - SpeechListVoicesRequest, - SpeechProviderPlugin, - SpeechProviderPrepareSynthesisContext, - SpeechSynthesisRequest, - SpeechTelephonySynthesisRequest, -} from "openclaw/plugin-sdk/speech-core"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { CODE_HEAVY_SPOKEN_FALLBACK } from "./speech-text.js"; -import type { TtsAudioPersistence } from "./tts-synthesis.js"; - -type MockSpeechSynthesisResult = Awaited>; - -const synthesizeMock = vi.hoisted(() => - vi.fn( - async (request: SpeechSynthesisRequest): Promise => ({ - audioBuffer: Buffer.from("voice"), - fileExtension: ".ogg", - outputFormat: "ogg", - voiceCompatible: request.target === "voice-note", - }), - ), -); -const prepareSynthesisMock = vi.hoisted(() => - vi.fn(async (_ctx: SpeechProviderPrepareSynthesisContext) => undefined), -); - -const listSpeechProvidersMock = vi.hoisted(() => vi.fn()); -const getSpeechProviderMock = vi.hoisted(() => vi.fn()); -const transcodeAudioBufferMock = vi.hoisted(() => - // Default off: most tests rely on the synthesized buffer reaching the - // channel unchanged. Tests that exercise the pre-transcode branch override - // per-call via `transcodeAudioBufferMock.mockResolvedValueOnce(...)`. - // Typed as the helper's full return shape so per-call overrides aren't - // narrowed to the default's literal. - vi.fn< - () => Promise< - | { ok: true; buffer: Buffer } - | { - ok: false; - reason: - | "platform-unsupported" - | "invalid-extension" - | "noop-same-container" - | "no-recipe" - | "transcoder-failed"; - detail?: string; - } - > - >(async () => ({ ok: false, reason: "platform-unsupported" })), -); - -vi.mock("openclaw/plugin-sdk/media-runtime", () => ({ - transcodeAudioBuffer: transcodeAudioBufferMock, -})); - -vi.mock("openclaw/plugin-sdk/channel-targets", () => ({ - normalizeChannelId: (channel: string | undefined) => channel?.trim().toLowerCase() ?? null, - resolveChannelTtsVoiceDelivery: (channel: string | undefined) => { - const normalized = channel?.trim().toLowerCase(); - if (normalized === "voice-memo-chat") { - return { - synthesisTarget: "audio-file", - audioFileFormats: ["mp3", "caf", "audio/mpeg", "audio/x-caf"], - preferAudioFileFormat: "caf", - }; - } - if (normalized === "feishu" || normalized === "whatsapp") { - return { synthesisTarget: "voice-note", transcodesAudio: true }; - } - if (normalized === "discord" || normalized === "matrix" || normalized === "telegram") { - return { synthesisTarget: "voice-note" }; - } - return undefined; - }, -})); - -vi.mock("openclaw/plugin-sdk/speech-core", async () => { - const actual = await vi.importActual("openclaw/plugin-sdk/speech-core"); - const mockProvider: SpeechProviderPlugin = { - id: "mock", - label: "Mock", - autoSelectOrder: 1, - isConfigured: () => true, - prepareSynthesis: prepareSynthesisMock, - synthesize: synthesizeMock, - }; - listSpeechProvidersMock.mockImplementation(() => [mockProvider]); - getSpeechProviderMock.mockImplementation((providerId: string) => - providerId === "mock" ? mockProvider : null, - ); - return { - ...actual, - canonicalizeSpeechProviderId: (providerId: string | undefined) => - providerId?.trim().toLowerCase() || undefined, - normalizeSpeechProviderId: (providerId: string | undefined) => - providerId?.trim().toLowerCase() || undefined, - getSpeechProvider: getSpeechProviderMock, - listSpeechProviders: listSpeechProvidersMock, - scheduleCleanup: vi.fn(), - }; -}); - -const { - testApi, - buildTtsSystemPromptHint, - getTtsPersona, - getTtsProvider, - isTtsProviderConfigured, - listSpeechVoices, - prepareTtsRequest, - resolveTtsConfig, - resolveTtsPrefsPath, - setTtsMachinePrefsPathResolver, - setSummarizationEnabled, - setTtsMaxLength, - synthesizeSpeech, - textToSpeechStream, - textToSpeechTelephony, -} = await import("../runtime-api.js"); -const { maybeApplyTtsToPayload: maybeApplyTtsToPayloadCore } = await import("./tts-payload.js"); -const { textToSpeech: textToSpeechCore } = await import("./tts-synthesis.js"); - -const nativeVoiceNoteChannels = ["discord", "feishu", "matrix", "telegram", "whatsapp"] as const; - -function createMockSpeechProvider( - id = "mock", - options: Partial = {}, -): SpeechProviderPlugin { - return { - id, - label: id, - autoSelectOrder: id === "mock" ? 1 : 2, - isConfigured: () => true, - prepareSynthesis: prepareSynthesisMock, - synthesize: synthesizeMock, - ...options, - }; -} - -function installSpeechProviders(providers: SpeechProviderPlugin[]): void { - listSpeechProvidersMock.mockImplementation(() => providers); - getSpeechProviderMock.mockImplementation( - (providerId: string) => providers.find((provider) => provider.id === providerId) ?? null, - ); -} - -// macOS os.tmpdir() is a /var -> /private/var symlink and fs-safe rejects -// symlinked store roots; resolve the canonical dir before writing prefs. -const PREFS_TMP_DIR = realpathSync(os.tmpdir()); - -async function persistTestTtsAudio({ - audioBuffer, - fileExtension, -}: Parameters[0]): Promise { - const dir = path.join(PREFS_TMP_DIR, `openclaw-speech-core-media-${crypto.randomUUID()}`); - mkdirSync(dir, { recursive: true }); - const audioPath = path.join(dir, `voice---${crypto.randomUUID()}${fileExtension}`); - writeFileSync(audioPath, audioBuffer); - return audioPath; -} - -function textToSpeech(params: Parameters[0]) { - return textToSpeechCore(params, persistTestTtsAudio); -} - -function maybeApplyTtsToPayload(params: Parameters[0]) { - return maybeApplyTtsToPayloadCore(params, persistTestTtsAudio); -} - -function prefsPathFor(prefsName: string): string { - return path.join(PREFS_TMP_DIR, `${prefsName}.json`); -} - -function createTtsConfig(prefsName: string): OpenClawConfig { - setTtsMachinePrefsPathResolver(() => prefsPathFor(prefsName)); - return { - tts: { - enabled: true, - provider: "mock", - }, - }; -} - -function requireRecord(value: unknown, label: string): Record { - if (value === null || typeof value !== "object" || Array.isArray(value)) { - throw new Error(`expected ${label} to be a record`); - } - return value as Record; -} - -function requireFirstCallParam(calls: ReadonlyArray, label: string) { - const call = calls[0]; - if (!call) { - throw new Error(`expected ${label} call`); - } - return call[0]; -} - -function requireFirstSynthesisRequest(label: string): Record { - return requireRecord(requireFirstCallParam(synthesizeMock.mock.calls, label), label); -} - -function requireAttempt(attempts: unknown[] | undefined, index: number) { - if (!attempts) { - throw new Error("expected synthesis attempts"); - } - return requireRecord(attempts[index], `synthesis attempt ${index}`); -} - -async function expectTtsPayloadResult(params: { - channel: string; - prefsName: string; - text: string; - target: "voice-note" | "audio-file"; - audioAsVoice: true | undefined; - providerResult?: MockSpeechSynthesisResult; - mediaExtension?: string; - kind?: "tool" | "block" | "final"; -}) { - if (params.providerResult) { - synthesizeMock.mockResolvedValueOnce(params.providerResult); - } - const cfg = createTtsConfig(params.prefsName); - let mediaDir: string | undefined; - try { - const result = await maybeApplyTtsToPayload({ - payload: { text: params.text }, - cfg, - channel: params.channel, - kind: params.kind ?? "final", - }); - - expect(synthesizeMock).toHaveBeenCalled(); - const request = requireRecord( - synthesizeMock.mock.calls.at(-1)?.[0], - "latest synthesis request", - ); - expect(request.target).toBe(params.target); - expect(result.audioAsVoice).toBe(params.audioAsVoice); - expect(result.mediaUrl).toMatch( - new RegExp(`voice---[a-f0-9-]+\\.${params.mediaExtension ?? "ogg"}$`), - ); - expect(result.spokenText).toBe(params.text); - expect(result.ttsSupplement).toEqual({ spokenText: params.text }); - expect((result as { trustedLocalMedia?: boolean }).trustedLocalMedia).toBe(true); - - mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; - } finally { - if (mediaDir) { - rmSync(mediaDir, { recursive: true, force: true }); - } - } -} - -describe("speech-core native voice-note routing", () => { - afterEach(() => { - setTtsMachinePrefsPathResolver(); - clearRuntimeConfigSnapshot(); - delete (Object.prototype as Record).polluted; - synthesizeMock.mockClear(); - prepareSynthesisMock.mockClear(); - transcodeAudioBufferMock.mockClear(); - installSpeechProviders([createMockSpeechProvider()]); - }); - - it("prefers the environment preference path over migrated machine state", () => { - const previousEnvPath = process.env.OPENCLAW_TTS_PREFS; - const envPath = prefsPathFor("env-override"); - setTtsMachinePrefsPathResolver(() => prefsPathFor("machine-state")); - process.env.OPENCLAW_TTS_PREFS = envPath; - try { - expect(resolveTtsPrefsPath(resolveTtsConfig({}))).toBe(envPath); - } finally { - if (previousEnvPath === undefined) { - delete process.env.OPENCLAW_TTS_PREFS; - } else { - process.env.OPENCLAW_TTS_PREFS = previousEnvPath; - } - } - }); - - it("resolves voice delivery support from channel capabilities", () => { - for (const channel of nativeVoiceNoteChannels) { - expect(testApi.supportsNativeVoiceNoteTts(channel)).toBe(true); - expect(testApi.supportsNativeVoiceNoteTts(channel.toUpperCase())).toBe(true); - } - expect(testApi.supportsNativeVoiceNoteTts("slack")).toBe(false); - expect(testApi.supportsNativeVoiceNoteTts(undefined)).toBe(false); - }); - - it("tells generic TTS guidance to defer to MEMORY voice-delivery instructions", () => { - const hint = buildTtsSystemPromptHint(createTtsConfig("openclaw-speech-core-tts-hint-test")); - - expect(hint).toContain("Voice (TTS) is enabled."); - expect(hint).toContain( - "If workspace context (especially MEMORY.md) tells you not to use [[tts:...]] or to use a local/non-tagged voice workflow, follow that workspace instruction instead.", - ); - expect(hint).toContain( - "Use [[tts:...]] and optional [[tts:text]]...[[/tts:text]] to control voice/expressiveness.", - ); - }); - - it("prepares deep-merged surface config and directive inputs", () => { - const cfg: OpenClawConfig = { - tts: { - provider: "mock", - modelOverrides: { allowProvider: false }, - providers: { - mock: { - model: "base-model", - voiceSettings: { stability: 0.4 }, - }, - }, - }, - }; - - const prepared = prepareTtsRequest({ - cfg, - override: { - modelOverrides: { allowProvider: true }, - providers: { - mock: { - voice: "surface-voice", - voiceSettings: { speed: 1.1 }, - }, - }, - }, - text: "Hello [[tts:text]]Speak this instead[[/tts:text]] caller", - }); - - expect(prepared.cfg).not.toBe(cfg); - expect(prepared.cfg.tts?.providers?.mock).toEqual({ - model: "base-model", - voice: "surface-voice", - voiceSettings: { stability: 0.4, speed: 1.1 }, - }); - expect(prepared.cfg.tts?.modelOverrides?.allowProvider).toBe(true); - expect(prepared.directives).toEqual({ - cleanedText: "Hello caller", - hasDirective: true, - overrides: { - ttsText: "Speak this instead", - }, - ttsText: "Speak this instead", - warnings: [], - }); - expect(cfg.tts?.providers?.mock).toEqual({ - model: "base-model", - voiceSettings: { stability: 0.4 }, - }); - }); - - it("sanitizes blocked override keys while preparing TTS config", () => { - const prepared = prepareTtsRequest({ - cfg: { - tts: { - provider: "mock", - providers: { mock: { model: "base-model" } }, - }, - }, - override: JSON.parse( - '{"__proto__":{"polluted":"top"},"providers":{"mock":{"voice":"safe","__proto__":{"polluted":"nested"}}}}', - ) as TtsConfig, - text: "[[tts:text]]Speak this instead[[/tts:text]]", - }); - - expect((Object.prototype as Record).polluted).toBeUndefined(); - expect(prepared.cfg.tts).not.toHaveProperty("polluted"); - expect(prepared.cfg.tts?.providers?.mock).toEqual({ - model: "base-model", - voice: "safe", - }); - expect(prepared.directives.cleanedText).toBe(""); - expect(prepared.directives.ttsText).toBe("Speak this instead"); - }); - - it("marks Discord auto TTS replies as native voice messages", async () => { - await expectTtsPayloadResult({ - channel: "discord", - prefsName: "openclaw-speech-core-tts-test", - text: "This Discord reply should be delivered as a native voice note.", - target: "voice-note", - audioAsVoice: true, - }); - }); - - it("keeps compatible audio-file synthesis deliverable as a voice memo", async () => { - await expectTtsPayloadResult({ - channel: "voice-memo-chat", - prefsName: "openclaw-speech-core-tts-voice-memo-mp3-test", - text: "This reply should be delivered as a native voice memo.", - target: "audio-file", - audioAsVoice: true, - mediaExtension: "mp3", - providerResult: { - audioBuffer: Buffer.from("mp3"), - outputFormat: "mp3", - fileExtension: ".mp3", - voiceCompatible: false, - }, - }); - }); - - it("does not mark unsupported audio-file output as a voice memo", async () => { - await expectTtsPayloadResult({ - channel: "voice-memo-chat", - prefsName: "openclaw-speech-core-tts-voice-memo-ogg-test", - text: "This reply should stay a regular audio attachment.", - target: "audio-file", - audioAsVoice: undefined, - }); - }); - - it("pre-transcodes synthesized mp3 to opus-in-CAF when the host can satisfy preferAudioFileFormat", async () => { - transcodeAudioBufferMock.mockResolvedValueOnce({ - ok: true, - buffer: Buffer.from("transcoded-caf"), - }); - await expectTtsPayloadResult({ - channel: "voice-memo-chat", - prefsName: "openclaw-speech-core-tts-voice-memo-caf-transcode-test", - text: "This reply should be pre-transcoded to a native voice-memo CAF.", - target: "audio-file", - audioAsVoice: true, - mediaExtension: "caf", - providerResult: { - audioBuffer: Buffer.from("mp3"), - outputFormat: "mp3", - fileExtension: ".mp3", - voiceCompatible: false, - }, - }); - expect(transcodeAudioBufferMock).toHaveBeenCalledOnce(); - const transcodeRequest = requireRecord( - requireFirstCallParam(transcodeAudioBufferMock.mock.calls as unknown[][], "transcode"), - "transcode request", - ); - expect(transcodeRequest.sourceExtension).toBe("mp3"); - expect(transcodeRequest.targetExtension).toBe("caf"); - }); - - it("falls back to the original mp3 buffer when the host transcoder fails", async () => { - transcodeAudioBufferMock.mockResolvedValueOnce({ - ok: false, - reason: "transcoder-failed", - detail: "exit-1", - }); - // Even though the transcode failed, the original mp3 still satisfies the - // channel audioFileFormats list, so the channel still flips audioAsVoice. - // The user gets a voice memo bubble, possibly with bad duration, instead - // of a regression. The failure is logged via the call site in tts.ts. - await expectTtsPayloadResult({ - channel: "voice-memo-chat", - prefsName: "openclaw-speech-core-tts-voice-memo-caf-fallback-test", - text: "This reply should fall back to the original mp3.", - target: "audio-file", - audioAsVoice: true, - mediaExtension: "mp3", - providerResult: { - audioBuffer: Buffer.from("mp3"), - outputFormat: "mp3", - fileExtension: ".mp3", - voiceCompatible: false, - }, - }); - }); - - it("uses the active runtime snapshot when source config still contains TTS SecretRefs", async () => { - const sourceConfig = { - tts: { - enabled: true, - provider: "mock", - providers: { - mock: { - apiKey: { source: "exec", provider: "mockexec", id: "minimax/tts/apiKey" }, - }, - }, - }, - } as unknown as OpenClawConfig; - const runtimeConfig = { - tts: { - enabled: true, - provider: "mock", - providers: { - mock: { - apiKey: "resolved-minimax-key", - }, - }, - }, - } as unknown as OpenClawConfig; - installSpeechProviders([ - createMockSpeechProvider("mock", { - isConfigured: ({ providerConfig }) => providerConfig.apiKey === "resolved-minimax-key", - resolveConfig: ({ rawConfig }) => { - const providers = rawConfig.providers as Record | undefined; - return { - apiKey: providers?.mock?.apiKey, - }; - }, - }), - ]); - setRuntimeConfigSnapshot(runtimeConfig, sourceConfig); - - const result = await synthesizeSpeech({ - text: "Runtime snapshot TTS SecretRef", - cfg: sourceConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - expect(synthesizeMock).toHaveBeenCalled(); - const request = requireFirstSynthesisRequest("runtime snapshot synthesis request"); - expect(request.cfg).toBe(runtimeConfig); - const providerConfig = requireRecord(request.providerConfig, "provider config"); - expect(providerConfig.apiKey).toBe("resolved-minimax-key"); - }); - - it("uses provider default TTS timeout when the call and config omit timeoutMs", async () => { - installSpeechProviders([createMockSpeechProvider("mock", { defaultTimeoutMs: 600_000 })]); - - const result = await synthesizeSpeech({ - text: "Use provider timeout.", - cfg: { - tts: { - enabled: true, - provider: "mock", - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - const request = requireFirstSynthesisRequest("provider default timeout synthesis request"); - expect(request.timeoutMs).toBe(600_000); - }); - - it("normalizes non-streaming synthesis text before calling the provider", async () => { - const result = await synthesizeSpeech({ - text: "## Update\n\nRead the [guide](https://example.com/guide)!!!!!", - cfg: createTtsConfig("openclaw-speech-core-talk-markdown-test"), - disableFallback: true, - }); - - expect(result.success).toBe(true); - const request = requireFirstSynthesisRequest("normalized talk synthesis request"); - expect(request.text).toBe("Update\n\nRead the guide!"); - }); - - it("speaks stripped code through the explicit textToSpeech conversion path", async () => { - let mediaDir: string | undefined; - try { - const result = await textToSpeech({ - text: "```ts\nconst answer = 42;\n```", - cfg: createTtsConfig("openclaw-speech-core-code-convert-test"), - }); - - expect(result.success).toBe(true); - const request = requireFirstSynthesisRequest("explicit code conversion request"); - expect(request.text).toBe("const answer = 42;"); - expect(request.text).not.toBe(CODE_HEAVY_SPOKEN_FALLBACK); - mediaDir = result.audioPath ? path.dirname(result.audioPath) : undefined; - } finally { - if (mediaDir) { - rmSync(mediaDir, { recursive: true, force: true }); - } - } - }); - - it("returns a normal TTS failure when audio persistence rejects", async () => { - const result = await textToSpeechCore( - { - text: "Store this synthesized reply.", - cfg: createTtsConfig("openclaw-speech-core-persistence-failure-test"), - }, - async () => { - throw new Error("Media exceeds configured limit"); - }, - ); - - expect(result).toMatchObject({ - success: false, - error: "TTS audio persistence failed", - provider: "mock", - }); - }); - - it("resolves the configured timeout for voice listing", async () => { - const listVoicesMock = vi.fn(async (_request: SpeechListVoicesRequest) => []); - installSpeechProviders([ - createMockSpeechProvider("mock", { - defaultTimeoutMs: 60_000, - listVoices: listVoicesMock, - }), - ]); - - await listSpeechVoices({ - provider: "mock", - cfg: { - tts: { - enabled: true, - provider: "mock", - timeoutMs: 45_000, - }, - } as OpenClawConfig, - }); - - expect(listVoicesMock).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 45_000 })); - }); - - it("caps oversized provider default TTS timeouts before synthesis", async () => { - installSpeechProviders([ - createMockSpeechProvider("mock", { defaultTimeoutMs: Number.MAX_SAFE_INTEGER }), - ]); - - const result = await synthesizeSpeech({ - text: "Use capped provider timeout.", - cfg: { - tts: { - enabled: true, - provider: "mock", - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - const request = requireFirstSynthesisRequest("provider default capped timeout request"); - expect(request.timeoutMs).toBe(MAX_TIMER_TIMEOUT_MS); - }); - - it("ignores nonpositive provider default TTS timeouts", async () => { - installSpeechProviders([createMockSpeechProvider("mock", { defaultTimeoutMs: 0 })]); - - const result = await synthesizeSpeech({ - text: "Use fallback timeout.", - cfg: { - tts: { - enabled: true, - provider: "mock", - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - const request = requireFirstSynthesisRequest("provider default fallback timeout request"); - expect(request.timeoutMs).toBe(30_000); - }); - - it("keeps explicit TTS config timeout ahead of provider default timeout", async () => { - installSpeechProviders([createMockSpeechProvider("mock", { defaultTimeoutMs: 600_000 })]); - - await synthesizeSpeech({ - text: "Use configured timeout.", - cfg: { - tts: { - enabled: true, - provider: "mock", - timeoutMs: 45_000, - }, - } as OpenClawConfig, - disableFallback: true, - }); - - const request = requireFirstSynthesisRequest("configured timeout synthesis request"); - expect(request.timeoutMs).toBe(45_000); - }); - - it("caps oversized voice model TTS timeouts before synthesis", async () => { - installSpeechProviders([ - createMockSpeechProvider("mock", { autoSelectOrder: 1, models: ["mock-tts"] }), - ]); - - const result = await synthesizeSpeech({ - text: "Use capped explicit timeout.", - cfg: { - agents: { - defaults: { - voiceModel: { primary: "mock/mock-tts", timeoutMs: Number.MAX_SAFE_INTEGER }, - }, - }, - tts: { - enabled: true, - provider: "mock", - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - const request = requireFirstSynthesisRequest("voice model capped timeout request"); - expect(request.timeoutMs).toBe(MAX_TIMER_TIMEOUT_MS); - }); - - it("uses agents.defaults.voiceModel as the default speech provider and model", async () => { - installSpeechProviders([ - createMockSpeechProvider("mock", { autoSelectOrder: 1 }), - createMockSpeechProvider("openai", { - autoSelectOrder: 10, - models: ["gpt-4o-mini-tts"], - resolveConfig: ({ rawConfig }) => { - const providers = requireRecord(rawConfig.providers, "raw provider configs"); - return { - model: "provider-default-model", - modelId: "provider-default-model", - ...requireRecord(providers.openai, "raw openai provider config"), - }; - }, - }), - ]); - - const result = await synthesizeSpeech({ - text: "Use configured voice model.", - cfg: { - agents: { - defaults: { - voiceModel: { primary: "openai/gpt-4o-mini-tts", timeoutMs: 12_345 }, - }, - }, - tts: { - enabled: true, - prefsPath: "/tmp/openclaw-speech-core-voice-model-default-test.json", - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - expect(result.provider).toBe("openai"); - expect(result.providerModel).toBe("gpt-4o-mini-tts"); - const request = requireFirstSynthesisRequest("voice model synthesis request"); - expect(request.providerConfig).toMatchObject({ - model: "gpt-4o-mini-tts", - modelId: "gpt-4o-mini-tts", - }); - expect(request.timeoutMs).toBe(12_345); - }); - - it("keeps explicit provider model aliases ahead of voiceModel defaults", async () => { - installSpeechProviders([ - createMockSpeechProvider("openrouter", { - models: ["explicit-model", "default-model"], - resolveConfig: ({ rawConfig }) => { - const providers = requireRecord(rawConfig.providers, "raw provider configs"); - return requireRecord(providers.openrouter, "raw openrouter provider config"); - }, - }), - ]); - - const result = await synthesizeSpeech({ - text: "Prefer explicit model alias.", - cfg: { - agents: { - defaults: { - voiceModel: { primary: "openrouter/default-model" }, - }, - }, - tts: { - enabled: true, - provider: "openrouter", - prefsPath: "/tmp/openclaw-speech-core-explicit-model-alias-test.json", - providers: { - openrouter: { - modelId: "explicit-model", - }, - }, - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - const request = requireFirstSynthesisRequest("explicit model alias synthesis request"); - const providerConfig = requireRecord(request.providerConfig, "provider config"); - expect(providerConfig).toMatchObject({ - modelId: "explicit-model", - }); - expect(providerConfig.model).toBeUndefined(); - }); - - it("tries voiceModel fallbacks before auto-selected speech providers", async () => { - installSpeechProviders([ - createMockSpeechProvider("mock", { autoSelectOrder: 1 }), - createMockSpeechProvider("openai", { - autoSelectOrder: 10, - models: ["gpt-4o-mini-tts"], - isConfigured: () => false, - }), - createMockSpeechProvider("elevenlabs", { - autoSelectOrder: 99, - models: ["eleven_multilingual_v2"], - }), - ]); - - const result = await synthesizeSpeech({ - text: "Use configured voice model fallback.", - cfg: { - agents: { - defaults: { - voiceModel: { - primary: "openai/gpt-4o-mini-tts", - fallbacks: ["elevenlabs/eleven_multilingual_v2"], - }, - }, - }, - tts: { - enabled: true, - prefsPath: "/tmp/openclaw-speech-core-voice-model-fallback-test.json", - }, - } as OpenClawConfig, - }); - - expect(result.success).toBe(true); - expect(result.provider).toBe("elevenlabs"); - expect(result.fallbackFrom).toBe("openai"); - expect(result.providerModel).toBe("eleven_multilingual_v2"); - }); - - it("tries same-provider voiceModel fallbacks as separate model attempts", async () => { - const synthesize = vi.fn(async (request: SpeechSynthesisRequest) => { - if (request.providerConfig.model === "bad-tts") { - throw new Error("unavailable model"); - } - return { - audioBuffer: Buffer.from("voice"), - fileExtension: ".ogg", - outputFormat: "ogg", - voiceCompatible: request.target === "voice-note", - }; - }); - installSpeechProviders([ - createMockSpeechProvider("openai", { - autoSelectOrder: 10, - models: ["bad-tts", "good-tts"], - synthesize, - }), - ]); - - const result = await synthesizeSpeech({ - text: "Use same-provider fallback model.", - cfg: { - agents: { - defaults: { - voiceModel: { - primary: "openai/bad-tts", - fallbacks: ["openai/good-tts"], - }, - }, - }, - tts: { - enabled: true, - prefsPath: "/tmp/openclaw-speech-core-same-provider-voice-model-fallback-test.json", - }, - } as OpenClawConfig, - }); - - expect(result.success).toBe(true); - expect(result.provider).toBe("openai"); - expect(result.providerModel).toBe("good-tts"); - expect(result.attemptedProviders).toEqual(["openai", "openai"]); - expect(synthesize.mock.calls.map(([request]) => request.providerConfig.model)).toEqual([ - "bad-tts", - "good-tts", - ]); - }); - - it("skips non-streaming providers before using a streaming fallback", async () => { - const release = vi.fn(async () => {}); - const streamSynthesize = vi.fn(async () => ({ - audioStream: new ReadableStream({ - start(controller) { - controller.close(); - }, - }), - fileExtension: ".pcm", - outputFormat: "pcm", - voiceCompatible: false, - release, - })); - installSpeechProviders([ - createMockSpeechProvider("buffered", { autoSelectOrder: 1 }), - createMockSpeechProvider("streaming", { - autoSelectOrder: 2, - streamSynthesize, - }), - ]); - - const result = await textToSpeechStream({ - text: "Use streaming fallback.", - cfg: { - tts: { - enabled: true, - provider: "buffered", - prefsPath: "/tmp/openclaw-speech-core-streaming-fallback-test.json", - }, - } as OpenClawConfig, - }); - - expect(result.success).toBe(true); - expect(result.provider).toBe("streaming"); - expect(result.fallbackFrom).toBe("buffered"); - expect(result.attemptedProviders).toEqual(["buffered", "streaming"]); - expect(result.outputFormat).toBe("pcm"); - expect(result.fileExtension).toBe(".pcm"); - expect(result.target).toBe("audio-file"); - expect(result.release).toBe(release); - const skippedAttempt = requireAttempt(result.attempts, 0); - expect(skippedAttempt).toMatchObject({ - provider: "buffered", - outcome: "skipped", - reasonCode: "unsupported_for_streaming", - personaBinding: "none", - error: "buffered does not support streaming TTS", - }); - expect(skippedAttempt).not.toHaveProperty("latencyMs"); - expect(requireAttempt(result.attempts, 1)).toMatchObject({ - provider: "streaming", - outcome: "success", - reasonCode: "success", - }); - expect(streamSynthesize).toHaveBeenCalledOnce(); - }); - - it("classifies streaming timeouts before falling back with raw text", async () => { - const timeoutStreamSynthesize = vi.fn(async () => { - const error = new Error("stalled"); - error.name = "AbortError"; - throw error; - }); - const fallbackStreamSynthesize = vi.fn(async () => ({ - audioStream: new ReadableStream({ - start(controller) { - controller.close(); - }, - }), - fileExtension: ".pcm", - outputFormat: "pcm", - voiceCompatible: false, - })); - installSpeechProviders([ - createMockSpeechProvider("primary", { - autoSelectOrder: 1, - streamSynthesize: timeoutStreamSynthesize, - }), - createMockSpeechProvider("fallback", { - autoSelectOrder: 2, - streamSynthesize: fallbackStreamSynthesize, - }), - ]); - const text = "## Keep [streaming Markdown](https://example.com) raw!!!!!"; - - const result = await textToSpeechStream({ - text, - cfg: { - tts: { - enabled: true, - provider: "primary", - prefsPath: "/tmp/openclaw-speech-core-streaming-timeout-test.json", - }, - } as OpenClawConfig, - }); - - expect(result.success).toBe(true); - expect(result.provider).toBe("fallback"); - expect(result.fallbackFrom).toBe("primary"); - expect(requireAttempt(result.attempts, 0)).toMatchObject({ - provider: "primary", - outcome: "failed", - reasonCode: "timeout", - error: "primary: request timed out", - }); - expect(requireAttempt(result.attempts, 1)).toMatchObject({ - provider: "fallback", - outcome: "success", - reasonCode: "success", - }); - expect(fallbackStreamSynthesize).toHaveBeenCalledWith(expect.objectContaining({ text })); - }); - - it("ignores voiceModel refs that are not speech models", async () => { - installSpeechProviders([ - createMockSpeechProvider("openai", { - autoSelectOrder: 10, - defaultModel: "gpt-4o-mini-tts", - models: ["gpt-4o-mini-tts"], - resolveConfig: ({ rawConfig }) => { - const providers = requireRecord(rawConfig.providers, "raw provider configs"); - return { - model: "gpt-4o-mini-tts", - modelId: "gpt-4o-mini-tts", - ...requireRecord(providers.openai, "raw openai provider config"), - }; - }, - }), - ]); - - const result = await synthesizeSpeech({ - text: "Use speech provider default for unsupported realtime model.", - cfg: { - agents: { - defaults: { - voiceModel: { primary: "openai/gpt-realtime-2" }, - }, - }, - tts: { - enabled: true, - provider: "openai", - prefsPath: "/tmp/openclaw-speech-core-realtime-voice-model-ignored-test.json", - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - expect(result.provider).toBe("openai"); - expect(result.providerModel).toBe("gpt-4o-mini-tts"); - const request = requireFirstSynthesisRequest("speech model fallback request"); - expect(request.providerConfig).toMatchObject({ - model: "gpt-4o-mini-tts", - modelId: "gpt-4o-mini-tts", - }); - }); - - it("uses the first speech-supported voiceModel fallback as the default provider", async () => { - installSpeechProviders([ - createMockSpeechProvider("openai", { - autoSelectOrder: 1, - models: ["gpt-4o-mini-tts"], - }), - createMockSpeechProvider("elevenlabs", { - autoSelectOrder: 99, - models: ["eleven_multilingual_v2"], - }), - ]); - - const result = await synthesizeSpeech({ - text: "Use first speech-supported voice model.", - cfg: { - agents: { - defaults: { - voiceModel: { - primary: "openai/gpt-realtime-2", - fallbacks: ["elevenlabs/eleven_multilingual_v2"], - }, - }, - }, - tts: { - enabled: true, - prefsPath: "/tmp/openclaw-speech-core-supported-voice-model-provider-test.json", - }, - } as OpenClawConfig, - }); - - expect(result.success).toBe(true); - expect(result.provider).toBe("elevenlabs"); - expect(result.providerModel).toBe("eleven_multilingual_v2"); - expect(result.attemptedProviders).toEqual(["elevenlabs"]); - }); - - it("maps speakerVoice provider config to provider-compatible voice fields", async () => { - const result = await synthesizeSpeech({ - text: "Use the configured speaker.", - cfg: { - tts: { - enabled: true, - provider: "mock", - providers: { - mock: { - speakerVoice: "cedar", - speakerVoiceId: "voice-123", - voice: "legacy-voice", - voiceName: "legacy-name", - voiceId: "legacy-id", - }, - }, - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - expect(result.providerVoice).toBe("voice-123"); - const request = requireFirstSynthesisRequest("speaker voice synthesis request"); - expect(request.providerConfig).toMatchObject({ - speakerVoice: "cedar", - voice: "cedar", - voiceName: "cedar", - speakerVoiceId: "voice-123", - voiceId: "voice-123", - }); - }); - - it("preserves alias-keyed provider config when resolving canonical providers", async () => { - installSpeechProviders([ - createMockSpeechProvider("xiaomi", { - aliases: ["mimo"], - resolveConfig: ({ rawConfig }) => { - const providers = requireRecord(rawConfig.providers, "raw provider configs"); - return requireRecord(providers.xiaomi ?? providers.mimo, "raw xiaomi provider config"); - }, - }), - ]); - - const result = await synthesizeSpeech({ - text: "Use alias provider config.", - cfg: { - tts: { - enabled: true, - provider: "xiaomi", - providers: { - mimo: { apiKey: "mimo-key" }, - }, - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - const request = requireFirstSynthesisRequest("alias provider synthesis request"); - expect(request.providerConfig).toMatchObject({ apiKey: "mimo-key" }); - }); - - it("maps speakerVoice persona provider config to provider-compatible voice fields", async () => { - const result = await synthesizeSpeech({ - text: "Use the persona speaker.", - cfg: { - tts: { - enabled: true, - provider: "mock", - persona: "narrator", - personas: { - narrator: { - providers: { - mock: { - speakerVoice: "marin", - }, - }, - }, - }, - }, - } as OpenClawConfig, - disableFallback: true, - }); - - expect(result.success).toBe(true); - expect(result.providerVoice).toBe("marin"); - const request = requireFirstSynthesisRequest("persona speaker voice synthesis request"); - expect(request.providerConfig).toMatchObject({ - speakerVoice: "marin", - voice: "marin", - voiceName: "marin", - }); - }); - - it.each(["feishu", "whatsapp"] as const)( - "marks %s voice-note TTS for channel-side transcoding when provider returns mp3", - async (channel) => { - expect(testApi.supportsTranscodedVoiceNoteTts(channel)).toBe(true); - await expectTtsPayloadResult({ - channel, - prefsName: `openclaw-speech-core-tts-${channel}-mp3-test`, - text: `This ${channel} reply should be transcoded by the channel.`, - target: "voice-note", - audioAsVoice: true, - mediaExtension: "mp3", - providerResult: { - audioBuffer: Buffer.from("mp3"), - outputFormat: "mp3", - fileExtension: ".mp3", - voiceCompatible: false, - }, - }); - }, - ); - - it("keeps non-native voice-note channels as regular audio files", async () => { - await expectTtsPayloadResult({ - channel: "slack", - prefsName: "openclaw-speech-core-tts-slack-test", - text: "Slack replies should be delivered as regular audio attachments.", - target: "audio-file", - audioAsVoice: undefined, - }); - }); - - it("preserves the text reply when auto-TTS audio persistence fails", async () => { - const payload = { text: "This text must still be delivered when media storage rejects audio." }; - const result = await maybeApplyTtsToPayloadCore( - { - payload, - cfg: createTtsConfig("openclaw-speech-core-auto-persistence-failure-test"), - channel: "slack", - kind: "final", - }, - async () => { - throw new Error("Media exceeds configured limit"); - }, - ); - - expect(result).toBe(payload); - }); - - it("normalizes voice-note Markdown once before synthesis", async () => { - const text = - 'This short explanation keeps the fenced literal below from becoming code-heavy.\n\n```md\nconst literal = "[x](y)";\n```'; - let mediaDir: string | undefined; - try { - const result = await maybeApplyTtsToPayload({ - payload: { text }, - cfg: createTtsConfig("openclaw-speech-core-once-normalized-markdown-test"), - channel: "telegram", - kind: "final", - }); - - const request = requireFirstSynthesisRequest("once-normalized voice-note synthesis request"); - expect(request.text).toBe( - 'This short explanation keeps the fenced literal below from becoming code-heavy.\n\nconst literal = "[x](y)";', - ); - expect(result.text).toBe(text); - mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; - } finally { - if (mediaDir) { - rmSync(mediaDir, { recursive: true, force: true }); - } - } - }); - - it("skips channel auto-TTS audio for code-heavy replies", async () => { - const text = "```ts\nexport function answer() {\n return 42;\n}\n```"; - const result = await maybeApplyTtsToPayload({ - payload: { text }, - cfg: createTtsConfig("openclaw-speech-core-code-heavy-voice-note-test"), - channel: "telegram", - kind: "final", - }); - - expect(synthesizeMock).not.toHaveBeenCalled(); - expect(result).toEqual({ text }); - }); - - it("synthesizes code-heavy explicitly tagged hidden TTS text", async () => { - const cfg = createTtsConfig("openclaw-speech-core-code-heavy-hidden-tts-test"); - let mediaDir: string | undefined; - try { - const result = await maybeApplyTtsToPayload({ - payload: { - text: '[[tts:text]]```ts\nconst detailedAnswer = "this code should still be spoken";\n```[[/tts:text]]', - audioAsVoice: true, - }, - cfg, - channel: "telegram", - kind: "final", - }); - - expect(synthesizeMock).toHaveBeenCalled(); - const request = requireFirstSynthesisRequest("code-heavy hidden TTS request"); - expect(request.text).toBe('const detailedAnswer = "this code should still be spoken";'); - expect(result.text).toBeUndefined(); - mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; - } finally { - if (mediaDir) { - rmSync(mediaDir, { recursive: true, force: true }); - } - } - }); - - it("synthesizes explicitly tagged short hidden TTS text", async () => { - const cfg = createTtsConfig("openclaw-speech-core-short-hidden-tts-test"); - let mediaDir: string | undefined; - try { - const result = await maybeApplyTtsToPayload({ - payload: { - text: "[[tts:text]]hello[[/tts:text]]", - audioAsVoice: true, - }, - cfg, - channel: "telegram", - kind: "final", - }); - - expect(synthesizeMock).toHaveBeenCalled(); - const request = requireFirstSynthesisRequest("hidden TTS request"); - expect(request.text).toBe("hello"); - expect(result.mediaUrl).toMatch(/voice---[a-f0-9-]+\.ogg$/); - expect(result.audioAsVoice).toBe(true); - expect(result.text).toBeUndefined(); - expect(result.ttsSupplement).toBeUndefined(); - mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; - } finally { - if (mediaDir) { - rmSync(mediaDir, { recursive: true, force: true }); - } - } - }); - - it("truncates long TTS text on a UTF-16 boundary", async () => { - const prefsName = "openclaw-speech-core-utf16-truncate-test"; - const prefsPath = prefsPathFor(prefsName); - const cfg = createTtsConfig(prefsName); - setTtsMaxLength(prefsPath, 11); - setSummarizationEnabled(prefsPath, false); - let mediaDir: string | undefined; - try { - const result = await maybeApplyTtsToPayload({ - payload: { text: `${"a".repeat(7)}😀tail long enough for TTS` }, - cfg, - channel: "telegram", - kind: "final", - }); - - expect(synthesizeMock).toHaveBeenCalled(); - const request = requireFirstSynthesisRequest("utf16 truncated TTS request"); - const spokenText = String(request.text); - expect(spokenText).toBe(`${"a".repeat(7)}...`); - expect(result.spokenText).toBe(spokenText); - mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; - } finally { - rmSync(prefsPath, { force: true }); - if (mediaDir) { - rmSync(mediaDir, { recursive: true, force: true }); - } - } - }); - - it("skips block delivery kind in final mode (accumulated final tail synthesizes instead)", async () => { - synthesizeMock.mockClear(); - const cfg = createTtsConfig("openclaw-speech-core-block-kind-tts-test"); - const result = await maybeApplyTtsToPayload({ - payload: { text: "WebChat block stream chunks defer TTS to the final tail." }, - cfg, - channel: "webchat", - kind: "block", - }); - - expect(synthesizeMock).not.toHaveBeenCalled(); - expect((result as { trustedLocalMedia?: boolean }).trustedLocalMedia).toBeUndefined(); - expect(result.text).toBe("WebChat block stream chunks defer TTS to the final tail."); - }); - - it("skips tool delivery kind in final mode", async () => { - synthesizeMock.mockClear(); - const cfg = createTtsConfig("openclaw-speech-core-tool-kind-tts-test"); - const result = await maybeApplyTtsToPayload({ - payload: { text: "Intermediate tool output should not be spoken." }, - cfg, - channel: "webchat", - kind: "tool", - }); - - expect(synthesizeMock).not.toHaveBeenCalled(); - expect((result as { trustedLocalMedia?: boolean }).trustedLocalMedia).toBeUndefined(); - expect(result.text).toBe("Intermediate tool output should not be spoken."); - }); - - it("keeps skipping untagged short TTS text", async () => { - const cfg = createTtsConfig("openclaw-speech-core-short-plain-tts-test"); - const result = await maybeApplyTtsToPayload({ - payload: { - text: "hello", - audioAsVoice: true, - }, - cfg, - channel: "telegram", - kind: "final", - }); - - expect(synthesizeMock).not.toHaveBeenCalled(); - expect(result).toEqual({ - text: "hello", - audioAsVoice: true, - }); - }); - - it("skips auto TTS for legacy final media directives", async () => { - synthesizeMock.mockClear(); - const cfg = createTtsConfig("openclaw-speech-core-media-directive-tts-test"); - const result = await maybeApplyTtsToPayload({ - payload: { text: "Here is the render.\nMEDIA:/tmp/render.png" }, - cfg, - channel: "telegram", - kind: "final", - }); - - expect(synthesizeMock).not.toHaveBeenCalled(); - expect(result).toEqual({ text: "Here is the render.\nMEDIA:/tmp/render.png" }); - }); - - it("keeps skipping explicit tagged TTS text that strips to empty markdown", async () => { - const cfg = createTtsConfig("openclaw-speech-core-empty-hidden-tts-test"); - const result = await maybeApplyTtsToPayload({ - payload: { - text: "[[tts:text]]***[[/tts:text]]", - audioAsVoice: true, - }, - cfg, - channel: "telegram", - kind: "final", - }); - - expect(synthesizeMock).not.toHaveBeenCalled(); - expect(result).toEqual({ - audioAsVoice: true, - }); - }); - - it("selects persona preferred provider before config fallback", () => { - const cfg: OpenClawConfig = { - tts: { - enabled: true, - provider: "other", - persona: "alfred", - personas: { - alfred: { - label: "Alfred", - provider: "mock", - providers: { - mock: { - voice: "Algieba", - }, - }, - }, - }, - }, - }; - const config = resolveTtsConfig(cfg); - const prefsPath = "/tmp/openclaw-speech-core-persona-provider.json"; - - expect(getTtsPersona(config, prefsPath)?.id).toBe("alfred"); - expect(getTtsProvider(config, prefsPath)).toBe("mock"); - }); - - it("treats provider configuration errors as unconfigured", () => { - installSpeechProviders([ - createMockSpeechProvider("broken", { - resolveConfig: () => { - throw new Error("invalid provider URL"); - }, - }), - ]); - const prefsPath = "/tmp/openclaw-speech-core-invalid-provider.json"; - setTtsMachinePrefsPathResolver(() => prefsPath); - const cfg = { - tts: { - providers: { broken: {} }, - }, - } as OpenClawConfig; - const config = resolveTtsConfig(cfg); - - expect(isTtsProviderConfigured(config, "broken", cfg)).toBe(false); - expect(getTtsProvider(config, prefsPath)).toBe(""); - }); - - it("merges active persona provider binding into synthesis config", async () => { - setTtsMachinePrefsPathResolver(() => "/tmp/openclaw-speech-core-persona-merge.json"); - const cfg: OpenClawConfig = { - tts: { - enabled: true, - provider: "mock", - providers: { - mock: { - model: "base-model", - voice: "base-voice", - }, - }, - persona: "alfred", - personas: { - alfred: { - provider: "mock", - providers: { - mock: { - voice: "persona-voice", - style: "dry", - }, - }, - }, - }, - }, - }; - - const payload: ReplyPayload = { - text: "This reply should use persona-specific provider configuration.", - }; - - let mediaDir: string | undefined; - try { - const result = await maybeApplyTtsToPayload({ - payload, - cfg, - channel: "slack", - kind: "final", - }); - - expect(synthesizeMock).toHaveBeenCalled(); - const request = requireFirstSynthesisRequest("persona synthesis request"); - const providerConfig = requireRecord(request.providerConfig, "persona provider config"); - expect(providerConfig.model).toBe("base-model"); - expect(providerConfig.voice).toBe("persona-voice"); - expect(providerConfig.style).toBe("dry"); - expect(result.mediaUrl).toMatch(/voice---[a-f0-9-]+\.ogg$/); - - mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; - } finally { - if (mediaDir) { - rmSync(mediaDir, { recursive: true, force: true }); - } - } - }); - - it("does not mark skipped unregistered providers as missing persona bindings", async () => { - const result = await synthesizeSpeech({ - text: "Use fallback provider.", - cfg: { - tts: { - enabled: true, - provider: "missing", - persona: "alfred", - personas: { - alfred: { - providers: { - missing: { - voice: "configured-but-unregistered", - }, - }, - }, - }, - }, - }, - }); - - expect(result.success).toBe(true); - const attempt = requireAttempt(result.attempts, 0); - expect(attempt.provider).toBe("missing"); - expect(attempt.outcome).toBe("skipped"); - expect(attempt.reasonCode).toBe("no_provider_registered"); - expect(attempt.persona).toBe("alfred"); - expect(attempt).not.toHaveProperty("personaBinding"); - }); - - it("does not mark skipped telephony providers as missing persona bindings", async () => { - const result = await textToSpeechTelephony({ - text: "Use telephony provider.", - cfg: { - tts: { - enabled: true, - provider: "mock", - persona: "alfred", - personas: { - alfred: { - providers: { - mock: { - voice: "persona-voice", - }, - }, - }, - }, - }, - }, - }); - - expect(result.success).toBe(false); - const attempt = requireAttempt(result.attempts, 0); - expect(attempt.provider).toBe("mock"); - expect(attempt.outcome).toBe("skipped"); - expect(attempt.reasonCode).toBe("unsupported_for_telephony"); - expect(attempt.persona).toBe("alfred"); - expect(attempt).not.toHaveProperty("personaBinding"); - }); - - it("passes directive overrides to telephony synthesis providers", async () => { - const synthesizeTelephonyMock = vi.fn(async (_request: SpeechTelephonySynthesisRequest) => ({ - audioBuffer: Buffer.from("voice"), - outputFormat: "pcm", - sampleRate: 24_000, - })); - installSpeechProviders([ - createMockSpeechProvider("mock", { - synthesizeTelephony: synthesizeTelephonyMock, - }), - ]); - - const text = "## Keep [telephony Markdown](https://example.com) raw!!!!!"; - const result = await textToSpeechTelephony({ - text, - cfg: { - tts: { - enabled: true, - provider: "mock", - providers: { - mock: { - modelId: "telephony-model", - voiceId: "default-voice", - }, - }, - }, - }, - overrides: { - providerOverrides: { - mock: { - speakerVoice: "directed-voice", - speed: 1.5, - }, - }, - }, - }); - - expect(result.success).toBe(true); - expect(result.providerModel).toBe("telephony-model"); - expect(result.providerVoice).toBe("directed-voice"); - expect(synthesizeTelephonyMock).toHaveBeenCalledOnce(); - const telephonyRequest = requireRecord( - requireFirstCallParam(synthesizeTelephonyMock.mock.calls, "telephony synthesis"), - "telephony synthesis request", - ); - expect(telephonyRequest.providerOverrides).toEqual({ - speakerVoice: "directed-voice", - speed: 1.5, - }); - expect(telephonyRequest.text).toBe(text); - expect(telephonyRequest).not.toHaveProperty("target"); - }); - - it("uses provider defaults when fallback policy allows missing persona bindings", async () => { - await synthesizeSpeech({ - text: "Use neutral provider defaults.", - cfg: { - tts: { - enabled: true, - provider: "mock", - persona: "alfred", - personas: { - alfred: { - fallbackPolicy: "provider-defaults", - }, - }, - }, - }, - }); - - expect(prepareSynthesisMock).toHaveBeenCalledOnce(); - const prepareContext = requireRecord( - requireFirstCallParam(prepareSynthesisMock.mock.calls, "prepare synthesis"), - "prepare synthesis context", - ); - expect(prepareContext.persona).toBeUndefined(); - expect(prepareContext.personaProviderConfig).toBeUndefined(); - }); - - it("preserves persona metadata by default when provider bindings are missing", async () => { - await synthesizeSpeech({ - text: "Use persona prompt.", - cfg: { - tts: { - enabled: true, - provider: "mock", - persona: "alfred", - personas: { - alfred: { - label: "Alfred", - }, - }, - }, - }, - }); - - expect(prepareSynthesisMock).toHaveBeenCalledOnce(); - const prepareContext = requireRecord( - requireFirstCallParam(prepareSynthesisMock.mock.calls, "prepare synthesis"), - "prepare synthesis context", - ); - const persona = requireRecord(prepareContext.persona, "prepare synthesis persona"); - expect(persona.id).toBe("alfred"); - expect(prepareContext.personaProviderConfig).toBeUndefined(); - }); - - it("skips unbound providers under fail policy while allowing bound fallbacks", async () => { - installSpeechProviders([ - createMockSpeechProvider("mock", { autoSelectOrder: 1 }), - createMockSpeechProvider("fallback", { autoSelectOrder: 2 }), - ]); - - const result = await synthesizeSpeech({ - text: "Use the first persona-bound provider.", - cfg: { - tts: { - enabled: true, - provider: "mock", - persona: "alfred", - personas: { - alfred: { - fallbackPolicy: "fail", - providers: { - fallback: { - voice: "fallback-voice", - }, - }, - }, - }, - }, - }, - }); - - expect(result.success).toBe(true); - expect(result.provider).toBe("fallback"); - expect(result.fallbackFrom).toBe("mock"); - const skippedAttempt = requireAttempt(result.attempts, 0); - expect(skippedAttempt.provider).toBe("mock"); - expect(skippedAttempt.outcome).toBe("skipped"); - expect(skippedAttempt.reasonCode).toBe("not_configured"); - expect(skippedAttempt.persona).toBe("alfred"); - expect(skippedAttempt.personaBinding).toBe("missing"); - expect(skippedAttempt.error).toBe("mock: persona alfred has no provider binding"); - const successAttempt = requireAttempt(result.attempts, 1); - expect(successAttempt.provider).toBe("fallback"); - expect(successAttempt.outcome).toBe("success"); - expect(successAttempt.persona).toBe("alfred"); - expect(successAttempt.personaBinding).toBe("applied"); - }); -}); - -describe("speech-core per-agent TTS config", () => { - it("deep-merges the active agent TTS override over tts", () => { - const cfg = { - tts: { - enabled: true, - provider: "openai", - providers: { - openai: { - apiKey: "${OPENAI_API_KEY}", - voice: "coral", - speed: 1, - }, - }, - }, - agents: { - list: [ - { - id: "reader", - tts: { - provider: "openai", - providers: { - openai: { - voice: "nova", - }, - }, - }, - }, - ], - }, - } satisfies OpenClawConfig; - - const resolved = resolveTtsConfig(cfg, "reader"); - - const rawConfig = requireRecord(resolved.rawConfig, "resolved raw TTS config"); - expect(rawConfig.enabled).toBe(true); - expect(rawConfig.provider).toBe("openai"); - const providers = requireRecord(rawConfig.providers, "resolved raw TTS providers"); - const openai = requireRecord(providers.openai, "resolved OpenAI TTS provider config"); - expect(openai.apiKey).toBe("${OPENAI_API_KEY}"); - expect(openai.voice).toBe("nova"); - expect(openai.speed).toBe(1); - }); - - it("composes per-agent TTS overrides with active persona bindings", async () => { - const cfg = { - tts: { - enabled: true, - provider: "mock", - providers: { - mock: { - model: "base-model", - voice: "base-voice", - }, - }, - persona: "alfred", - personas: { - alfred: { - provider: "mock", - providers: { - mock: { - voice: "alfred-voice", - }, - }, - }, - jarvis: { - provider: "mock", - providers: { - mock: { - style: "jarvis-style", - }, - }, - }, - }, - }, - agents: { - list: [ - { - id: "reader", - tts: { - persona: "jarvis", - providers: { - mock: { - voice: "agent-voice", - }, - }, - }, - }, - ], - }, - } satisfies OpenClawConfig; - - let mediaDir: string | undefined; - try { - const result = await maybeApplyTtsToPayload({ - payload: { text: "This agent reply should use the composed persona config." }, - cfg, - channel: "slack", - kind: "final", - agentId: "reader", - }); - - expect(synthesizeMock).toHaveBeenCalled(); - const request = requireFirstSynthesisRequest("agent persona synthesis request"); - const providerConfig = requireRecord(request.providerConfig, "agent persona provider config"); - expect(providerConfig.model).toBe("base-model"); - expect(providerConfig.voice).toBe("agent-voice"); - expect(providerConfig.style).toBe("jarvis-style"); - expect(result.mediaUrl).toMatch(/voice---[a-f0-9-]+\.ogg$/); - mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; - } finally { - if (mediaDir) { - rmSync(mediaDir, { recursive: true, force: true }); - } - } - }); - - it("ignores prototype-pollution keys in agent TTS overrides", () => { - const cfg = { - tts: { - provider: "openai", - providers: { - openai: { - voice: "coral", - }, - }, - }, - agents: { - list: [ - { - id: "reader", - tts: JSON.parse( - '{"providers":{"openai":{"voice":"nova","__proto__":{"polluted":true}}}}', - ), - }, - ], - }, - } as OpenClawConfig; - - const resolved = resolveTtsConfig(cfg, "reader"); - - expect(resolved.rawConfig?.providers?.openai).toEqual({ voice: "nova" }); - expect(({} as Record).polluted).toBeUndefined(); - }); -}); -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/packages/speech-core/tsconfig.json b/packages/speech-core/tsconfig.json deleted file mode 100644 index 329cf33d90d7..000000000000 --- a/packages/speech-core/tsconfig.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "rootDir": "../.." - }, - "include": ["./*.ts", "./src/**/*.ts"], - "exclude": [ - "./**/*.test.ts", - "./dist/**", - "./node_modules/**", - "./src/test-support/**", - "./src/**/*test-helpers.ts", - "./src/**/*test-harness.ts", - "./src/**/*test-support.ts" - ] -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 43b0ed6ff41c..5e273594d4d1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2323,12 +2323,6 @@ importers: packages/session-url-contract: {} - packages/speech-core: - dependencies: - openclaw: - specifier: workspace:* - version: link:../.. - packages/terminal-core: dependencies: '@clack/prompts': diff --git a/qa/scenarios/media/webchat-auto-tts.yaml b/qa/scenarios/media/webchat-auto-tts.yaml index c6326cc3bfad..e9cbb21c4f33 100644 --- a/qa/scenarios/media/webchat-auto-tts.yaml +++ b/qa/scenarios/media/webchat-auto-tts.yaml @@ -19,7 +19,7 @@ scenario: - docs/tools/media-overview.md - docs/concepts/qa-e2e-automation.md codeRefs: - - packages/speech-core/src/tts.ts + - src/tts/runtime-api.ts - src/gateway/server-methods/chat-webchat-media.ts - src/gateway/managed-image-attachments.ts - src/gateway/server-methods/artifacts.ts diff --git a/scripts/lib/extension-package-boundary.ts b/scripts/lib/extension-package-boundary.ts index 5c677d3b6608..0ef871cb64e0 100644 --- a/scripts/lib/extension-package-boundary.ts +++ b/scripts/lib/extension-package-boundary.ts @@ -287,7 +287,6 @@ export const EXTENSION_PACKAGE_BOUNDARY_XAI_PATHS = { "@openclaw/anthropic-vertex/api.js": ["./.boundary-stubs/anthropic-vertex-api.d.ts"], "@openclaw/ollama/api.js": ["./.boundary-stubs/ollama-api.d.ts"], "@openclaw/ollama/runtime-api.js": ["./.boundary-stubs/ollama-runtime-api.d.ts"], - "@openclaw/speech-core/runtime-api.js": ["./.boundary-stubs/speech-core-runtime-api.d.ts"], } as const; type ExtensionPackageBoundaryTsConfigJson = { diff --git a/scripts/lib/tsdown-output-roots.mjs b/scripts/lib/tsdown-output-roots.mjs index b1ed9a7981d4..d7dcaa14f420 100644 --- a/scripts/lib/tsdown-output-roots.mjs +++ b/scripts/lib/tsdown-output-roots.mjs @@ -13,7 +13,6 @@ const TSDOWN_PACKAGE_NAMES = [ "net-policy", "normalization-core", "retry", - "speech-core", "terminal-core", "acp-core", ]; diff --git a/src/agents/cli-runner.reliability.test.ts b/src/agents/cli-runner.reliability.test.ts index ec78b66eed8d..7d16594be296 100644 --- a/src/agents/cli-runner.reliability.test.ts +++ b/src/agents/cli-runner.reliability.test.ts @@ -97,6 +97,8 @@ vi.mock("../skills/research/autocapture.js", () => ({ vi.mock("../tts/tts-settings.js", () => ({ buildTtsSystemPromptHint: vi.fn(() => undefined), + resolveModelOverridePolicy: vi.fn(), + setTtsMachinePrefsPathResolver: vi.fn(), })); const mockGetGlobalHookRunner = vi.mocked(getGlobalHookRunner); diff --git a/src/agents/cli-runner/helpers.system-prompt.test.ts b/src/agents/cli-runner/helpers.system-prompt.test.ts index d07c3fd216be..d4a78be19358 100644 --- a/src/agents/cli-runner/helpers.system-prompt.test.ts +++ b/src/agents/cli-runner/helpers.system-prompt.test.ts @@ -5,6 +5,8 @@ import { buildCliAgentSystemPrompt } from "./helpers.js"; vi.mock("../../tts/tts-settings.js", () => ({ buildTtsSystemPromptHint: vi.fn(() => undefined), + resolveModelOverridePolicy: vi.fn(), + setTtsMachinePrefsPathResolver: vi.fn(), })); describe("buildCliAgentSystemPrompt", () => { diff --git a/src/agents/cli-runner/prepare.test.ts b/src/agents/cli-runner/prepare.test.ts index f41442825574..a26cf2cc41d3 100644 --- a/src/agents/cli-runner/prepare.test.ts +++ b/src/agents/cli-runner/prepare.test.ts @@ -100,6 +100,8 @@ vi.mock("../../plugins/hook-runner-global.js", () => ({ vi.mock("../../tts/tts-settings.js", () => ({ buildTtsSystemPromptHint: vi.fn(() => undefined), + resolveModelOverridePolicy: vi.fn(), + setTtsMachinePrefsPathResolver: vi.fn(), })); vi.mock("../video-generation-task-status.js", () => ({ diff --git a/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts b/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts index 06f4e1a7c4cc..de4777a45bff 100644 --- a/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts +++ b/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts @@ -466,6 +466,8 @@ vi.mock("../../../infra/net/undici-global-dispatcher.js", () => ({ vi.mock("../../../tts/tts-settings.js", () => ({ buildTtsSystemPromptHint: () => undefined, + resolveModelOverridePolicy: () => undefined, + setTtsMachinePrefsPathResolver: () => undefined, })); vi.mock("../../bootstrap-files.js", async () => { diff --git a/src/agents/embedded-agent-runner/system-prompt.test.ts b/src/agents/embedded-agent-runner/system-prompt.test.ts index 4ae50bbe9c4b..c0a7e2f81a0a 100644 --- a/src/agents/embedded-agent-runner/system-prompt.test.ts +++ b/src/agents/embedded-agent-runner/system-prompt.test.ts @@ -10,6 +10,8 @@ import { applySystemPromptToSession, buildEmbeddedSystemPrompt } from "./system- vi.mock("../../tts/tts-settings.js", () => ({ buildTtsSystemPromptHint: vi.fn(() => undefined), + resolveModelOverridePolicy: vi.fn(), + setTtsMachinePrefsPathResolver: vi.fn(), })); describe("applySystemPromptToSession", () => { diff --git a/src/agents/system-prompt-config.test.ts b/src/agents/system-prompt-config.test.ts index f23f1f9389e2..d31bf1388fa3 100644 --- a/src/agents/system-prompt-config.test.ts +++ b/src/agents/system-prompt-config.test.ts @@ -6,6 +6,8 @@ import { buildConfiguredAgentSystemPrompt } from "./system-prompt-config.js"; vi.mock("../tts/tts-settings.js", () => ({ buildTtsSystemPromptHint: vi.fn(() => undefined), + resolveModelOverridePolicy: vi.fn(), + setTtsMachinePrefsPathResolver: vi.fn(), })); function buildPrompt(config: OpenClawConfig, agentId = "main"): string { diff --git a/src/auto-reply/reply/commands-system-prompt.test.ts b/src/auto-reply/reply/commands-system-prompt.test.ts index 817bdc7b961d..8ad29a3e7fa0 100644 --- a/src/auto-reply/reply/commands-system-prompt.test.ts +++ b/src/auto-reply/reply/commands-system-prompt.test.ts @@ -71,6 +71,8 @@ vi.mock("../../agents/agent-tools.js", () => ({ vi.mock("../../tts/tts-settings.js", () => ({ buildTtsSystemPromptHint: vi.fn(() => undefined), + resolveModelOverridePolicy: vi.fn(), + setTtsMachinePrefsPathResolver: vi.fn(), })); function makeParams(): HandleCommandsParams { diff --git a/src/channels/plugins/types.core.ts b/src/channels/plugins/types.core.ts index 281e2f99e974..cf0b4a096640 100644 --- a/src/channels/plugins/types.core.ts +++ b/src/channels/plugins/types.core.ts @@ -254,7 +254,7 @@ export type ChannelGroupContext = { /** TTS voice delivery behavior advertised by a channel plugin. */ /** * Container tokens (file-extension shape, no leading dot) that the host - * speech-core pipeline knows how to pre-transcode synthesized audio into. + * TTS pipeline knows how to pre-transcode synthesized audio into. * Channels that benefit from a specific container — currently only * iMessage, which needs Apple's native voice-memo CAF descriptor — name * one here. Adding a new entry requires extending the host transcoder diff --git a/src/gateway/server-methods/talk-shared.ts b/src/gateway/server-methods/talk-shared.ts index 455a8b736f8b..8f18b000779f 100644 --- a/src/gateway/server-methods/talk-shared.ts +++ b/src/gateway/server-methods/talk-shared.ts @@ -7,12 +7,6 @@ import { normalizeOptionalString, } from "@openclaw/normalization-core/string-coerce"; import { ErrorCodes } from "../../../packages/gateway-protocol/src/index.js"; -import { - getVoiceProviderConfig, - providerMatchesId, - resolveSupportedVoiceModelRefs, - type VoiceModelProvider, -} from "../../../packages/speech-core/voice-models.js"; import { resolveRealtimeBootstrapContextInstructions } from "../../agents/realtime-bootstrap-context.js"; import type { TalkRealtimeConfig } from "../../config/types.gateway.js"; import type { OpenClawConfig } from "../../config/types.js"; @@ -32,6 +26,12 @@ import type { RealtimeVoiceProviderConfig, } from "../../talk/provider-types.js"; import type { TalkBrain, TalkEvent, TalkMode, TalkTransport } from "../../talk/talk-events.js"; +import { + getVoiceProviderConfig, + providerMatchesId, + resolveSupportedVoiceModelRefs, + type VoiceModelProvider, +} from "../../tts/voice-models.js"; import { ADMIN_SCOPE } from "../operator-scopes.js"; import type { TalkHandoffTurnResult } from "../talk-handoff.js"; diff --git a/src/gateway/server-methods/talk.ts b/src/gateway/server-methods/talk.ts index a8fab5fee74f..7bd8ac06fb83 100644 --- a/src/gateway/server-methods/talk.ts +++ b/src/gateway/server-methods/talk.ts @@ -15,15 +15,6 @@ import { validateTalkModeParams, validateTalkSpeakParams, } from "../../../packages/gateway-protocol/src/index.js"; -import { - withSpeakerSelectionCompat, - withSpeakerSelectionFallbackCompat, -} from "../../../packages/speech-core/speaker.js"; -import { - CODE_HEAVY_SPOKEN_FALLBACK, - isCodeHeavySpeechText, -} from "../../../packages/speech-core/src/speech-text.js"; -import { getVoiceProviderConfig } from "../../../packages/speech-core/voice-models.js"; import { readConfigFileSnapshot } from "../../config/config.js"; import { redactConfigObject } from "../../config/redact-snapshot.js"; import { @@ -54,12 +45,18 @@ import { getSpeechProvider, listSpeechProviders, } from "../../tts/provider-registry.js"; +import { + withSpeakerSelectionCompat, + withSpeakerSelectionFallbackCompat, +} from "../../tts/speaker.js"; +import { CODE_HEAVY_SPOKEN_FALLBACK, isCodeHeavySpeechText } from "../../tts/speech-text.js"; import { getResolvedSpeechProviderConfig, resolveTtsConfig, synthesizeSpeech, type TtsDirectiveOverrides, } from "../../tts/tts.js"; +import { getVoiceProviderConfig } from "../../tts/voice-models.js"; import { ADMIN_SCOPE, READ_SCOPE, TALK_SECRETS_SCOPE } from "../operator-scopes.js"; import { resolveConfiguredSecretInputString } from "../resolve-configured-secret-input-string.js"; import { formatForLog } from "../ws-log.js"; diff --git a/src/gateway/server.talk-runtime.test.ts b/src/gateway/server.talk-runtime.test.ts index 671a9a1d952c..4336ecf37eb6 100644 --- a/src/gateway/server.talk-runtime.test.ts +++ b/src/gateway/server.talk-runtime.test.ts @@ -2,7 +2,7 @@ * Tests gateway talk runtime wiring for speech provider execution. */ import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { CODE_HEAVY_SPOKEN_FALLBACK } from "../../packages/speech-core/src/speech-text.js"; +import { CODE_HEAVY_SPOKEN_FALLBACK } from "../tts/speech-text.js"; import { invokeTalkSpeakDirect, type TalkSpeakTestPayload, diff --git a/src/infra/outbound/reply-payload-parts.ts b/src/infra/outbound/reply-payload-parts.ts new file mode 100644 index 000000000000..239f11dbefb0 --- /dev/null +++ b/src/infra/outbound/reply-payload-parts.ts @@ -0,0 +1,71 @@ +import { normalizeStringEntries } from "../../../packages/normalization-core/src/string-normalization.js"; + +/** Derived sendability facts for text/media outbound payload delivery. */ +export type SendableOutboundReplyParts = { + /** Raw text selected for delivery before trimming. */ + text: string; + /** Text after trimming whitespace for sendability checks. */ + trimmedText: string; + /** Normalized non-empty media URLs. */ + mediaUrls: string[]; + /** Number of normalized media URLs. */ + mediaCount: number; + /** Whether trimmed text is sendable. */ + hasText: boolean; + /** Whether at least one media URL is sendable. */ + hasMedia: boolean; + /** Whether the payload has any sendable text or media. */ + hasContent: boolean; +}; + +/** Prefer multi-attachment payloads, then fall back to the legacy single-media field. */ +export function resolveOutboundMediaUrls(payload: { + mediaUrls?: string[]; + mediaUrl?: string; +}): string[] { + if (payload.mediaUrls?.length) { + return payload.mediaUrls; + } + if (payload.mediaUrl) { + return [payload.mediaUrl]; + } + return []; +} + +/** Count outbound media items after legacy single-media fallback normalization. */ +export function countOutboundMedia(payload: { mediaUrls?: string[]; mediaUrl?: string }): number { + return resolveOutboundMediaUrls(payload).length; +} + +/** Check whether an outbound payload includes any media after normalization. */ +export function hasOutboundMedia(payload: { mediaUrls?: string[]; mediaUrl?: string }): boolean { + return countOutboundMedia(payload) > 0; +} + +/** Check whether an outbound payload includes text, optionally trimming whitespace first. */ +export function hasOutboundText(payload: { text?: string }, options?: { trim?: boolean }): boolean { + const text = options?.trim ? payload.text?.trim() : payload.text; + return Boolean(text); +} + +/** Normalize reply payload text/media into a trimmed, sendable shape for delivery paths. */ +export function resolveSendableOutboundReplyParts( + payload: { text?: string; mediaUrls?: string[]; mediaUrl?: string }, + options?: { text?: string }, +): SendableOutboundReplyParts { + const text = options?.text ?? payload.text ?? ""; + const trimmedText = text.trim(); + const mediaUrls = normalizeStringEntries(resolveOutboundMediaUrls(payload)); + const mediaCount = mediaUrls.length; + const hasText = Boolean(trimmedText); + const hasMedia = mediaCount > 0; + return { + text, + trimmedText, + mediaUrls, + mediaCount, + hasText, + hasMedia, + hasContent: hasText || hasMedia, + }; +} diff --git a/src/plugin-sdk/facade-runtime.test.ts b/src/plugin-sdk/facade-runtime.test.ts index 75c253aab303..452676556f45 100644 --- a/src/plugin-sdk/facade-runtime.test.ts +++ b/src/plugin-sdk/facade-runtime.test.ts @@ -700,7 +700,7 @@ describe("plugin-sdk facade runtime", () => { } }); - it("does not treat package-backed speech-core as a bundled extension facade", () => { + it("does not treat the core-owned speech runtime as a bundled extension facade", () => { setRuntimeConfigSnapshot({}); expect( diff --git a/src/plugin-sdk/reply-payload.ts b/src/plugin-sdk/reply-payload.ts index 50059bcfdb58..c6e5df1673bf 100644 --- a/src/plugin-sdk/reply-payload.ts +++ b/src/plugin-sdk/reply-payload.ts @@ -1,9 +1,15 @@ // Reply payload helpers normalize plugin reply targets, text, media, and approval metadata. import { normalizeLowercaseStringOrEmpty } from "../../packages/normalization-core/src/string-coerce.js"; -import { normalizeStringEntries } from "../../packages/normalization-core/src/string-normalization.js"; import type { ReplyPayload as InternalReplyPayload } from "../auto-reply/reply-payload.js"; import type { ChannelOutboundAdapter } from "../channels/plugins/outbound.types.js"; import { normalizeOutboundReplyPayload as normalizeCoreOutboundReplyPayload } from "../infra/outbound/reply-payload-normalize.js"; +import { + countOutboundMedia, + hasOutboundMedia, + hasOutboundText, + resolveOutboundMediaUrls, + resolveSendableOutboundReplyParts, +} from "../infra/outbound/reply-payload-parts.js"; import { createReplyToFanout } from "../infra/outbound/reply-policy.js"; import { hasReplyPayloadContent } from "../interactive/payload.js"; @@ -68,21 +74,13 @@ export type ReasoningReplyPayload = { }; /** Derived sendability facts for text/media outbound payload delivery. */ -export type SendableOutboundReplyParts = { - /** Raw text selected for delivery before trimming. */ - text: string; - /** Text after trimming whitespace for sendability checks. */ - trimmedText: string; - /** Normalized non-empty media URLs. */ - mediaUrls: string[]; - /** Number of normalized media URLs. */ - mediaCount: number; - /** Whether trimmed text is sendable. */ - hasText: boolean; - /** Whether at least one media URL is sendable. */ - hasMedia: boolean; - /** Whether the payload has any sendable text or media. */ - hasContent: boolean; +export type { SendableOutboundReplyParts } from "../infra/outbound/reply-payload-parts.js"; +export { + countOutboundMedia, + hasOutboundMedia, + hasOutboundText, + resolveOutboundMediaUrls, + resolveSendableOutboundReplyParts, }; type SendPayloadContext = Parameters>[0]; @@ -139,41 +137,11 @@ export function createNormalizedOutboundDeliverer( }; } -/** Prefer multi-attachment payloads, then fall back to the legacy single-media field. */ -export function resolveOutboundMediaUrls(payload: { - mediaUrls?: string[]; - mediaUrl?: string; -}): string[] { - if (payload.mediaUrls?.length) { - return payload.mediaUrls; - } - if (payload.mediaUrl) { - return [payload.mediaUrl]; - } - return []; -} - /** Resolve media URLs from a channel sendPayload context after legacy fallback normalization. */ export function resolvePayloadMediaUrls(payload: SendPayloadContext["payload"]): string[] { return resolveOutboundMediaUrls(payload); } -/** Count outbound media items after legacy single-media fallback normalization. */ -export function countOutboundMedia(payload: { mediaUrls?: string[]; mediaUrl?: string }): number { - return resolveOutboundMediaUrls(payload).length; -} - -/** Check whether an outbound payload includes any media after normalization. */ -export function hasOutboundMedia(payload: { mediaUrls?: string[]; mediaUrl?: string }): boolean { - return countOutboundMedia(payload) > 0; -} - -/** Check whether an outbound payload includes text, optionally trimming whitespace first. */ -export function hasOutboundText(payload: { text?: string }, options?: { trim?: boolean }): boolean { - const text = options?.trim ? payload.text?.trim() : payload.text; - return Boolean(text); -} - /** Check whether an outbound payload includes any sendable text, media, or rich reply content. */ export function hasOutboundReplyContent( payload: { @@ -189,28 +157,6 @@ export function hasOutboundReplyContent( return hasReplyPayloadContent(payload, { trimText: options?.trimText }); } -/** Normalize reply payload text/media into a trimmed, sendable shape for delivery paths. */ -export function resolveSendableOutboundReplyParts( - payload: { text?: string; mediaUrls?: string[]; mediaUrl?: string }, - options?: { text?: string }, -): SendableOutboundReplyParts { - const text = options?.text ?? payload.text ?? ""; - const trimmedText = text.trim(); - const mediaUrls = normalizeStringEntries(resolveOutboundMediaUrls(payload)); - const mediaCount = mediaUrls.length; - const hasText = Boolean(trimmedText); - const hasMedia = mediaCount > 0; - return { - text, - trimmedText, - mediaUrls, - mediaCount, - hasText, - hasMedia, - hasContent: hasText || hasMedia, - }; -} - /** Preserve caller-provided chunking, but fall back to the full text when chunkers return nothing. */ export function resolveTextChunksWithFallback(text: string, chunks: readonly string[]): string[] { if (chunks.length > 0) { diff --git a/src/plugin-sdk/tts-runtime.ts b/src/plugin-sdk/tts-runtime.ts index 97f0cb1aed36..896bc89f8d85 100644 --- a/src/plugin-sdk/tts-runtime.ts +++ b/src/plugin-sdk/tts-runtime.ts @@ -1,17 +1,6 @@ -// TTS runtime exports expose text-to-speech runtime helpers through the plugin SDK. -import { maybeApplyTtsToPayload as maybeApplyTtsToPayloadCore } from "../../packages/speech-core/src/tts-payload.js"; -import { textToSpeech as textToSpeechCore } from "../../packages/speech-core/src/tts-synthesis.js"; -import { persistTtsAudioToMediaStore } from "../tts/tts-audio-store.js"; - -export type { TtsResult } from "../../packages/speech-core/src/tts-types.js"; - -export function textToSpeech(params: Parameters[0]) { - return textToSpeechCore(params, persistTtsAudioToMediaStore); -} - -export function maybeApplyTtsToPayload(params: Parameters[0]) { - return maybeApplyTtsToPayloadCore(params, persistTtsAudioToMediaStore); -} +// TTS runtime exports expose host-owned text-to-speech helpers through the plugin SDK. +export { maybeApplyTtsToPayload, textToSpeech } from "../tts/tts.js"; +export type { TtsResult } from "../tts/tts-runtime-types.js"; export { TtsAutoSchema, @@ -23,8 +12,6 @@ export { /** Compatibility no-op retained for callers that prewarm facade runtimes generically. */ export function prewarmTtsRuntimeFacade(): void {} -// Pure synthesis stays in speech-core. File-backed helpers above inject the -// core media-store owner so package code never imports from src. export { buildTtsSystemPromptHint, getLastTtsAttempt, @@ -63,4 +50,4 @@ export { type TtsSynthesisStreamResult, type TtsStreamResult, type TtsTelephonyResult, -} from "../../packages/speech-core/runtime-api.js"; +} from "../tts/runtime-api.js"; diff --git a/src/plugins/capability-provider-runtime.ts b/src/plugins/capability-provider-runtime.ts index 2d7085ce82f2..75a28dc732b0 100644 --- a/src/plugins/capability-provider-runtime.ts +++ b/src/plugins/capability-provider-runtime.ts @@ -1,6 +1,6 @@ import { sortUniqueStrings } from "@openclaw/normalization-core/string-normalization"; -import { resolveVoiceModelRefs } from "../../packages/speech-core/voice-models.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { resolveVoiceModelRefs } from "../tts/voice-models.js"; import { getLoadedRuntimePluginRegistry, registryContainsRuntimePluginIds, diff --git a/src/plugins/model-catalog-registration.ts b/src/plugins/model-catalog-registration.ts index 2c2404e2fd86..1db05fec4d65 100644 --- a/src/plugins/model-catalog-registration.ts +++ b/src/plugins/model-catalog-registration.ts @@ -14,7 +14,7 @@ import { synthesizeVoiceModelCatalogEntries, type VoiceModelCapabilities, type VoiceModelProvider, -} from "../../packages/speech-core/voice-models.js"; +} from "../tts/voice-models.js"; import type { PluginDiagnostic } from "./manifest-types.js"; import { projectProviderCatalogResultToUnifiedTextRows } from "./provider-catalog-unified-text.js"; import type { PluginRecord, PluginRegistry } from "./registry-types.js"; diff --git a/src/plugins/runtime/runtime-tts-request.ts b/src/plugins/runtime/runtime-tts-request.ts index e735cd4a6a25..1376af3261e8 100644 --- a/src/plugins/runtime/runtime-tts-request.ts +++ b/src/plugins/runtime/runtime-tts-request.ts @@ -1,2 +1,2 @@ -// Lazy runtime bridge for speech-core request pre-resolution. -export { prepareTtsRequest } from "../../../packages/speech-core/runtime-api.js"; +// Lazy runtime bridge for TTS request pre-resolution. +export { prepareTtsRequest } from "../../tts/runtime-api.js"; diff --git a/packages/speech-core/src/tts.ts b/src/tts/runtime-api.ts similarity index 57% rename from packages/speech-core/src/tts.ts rename to src/tts/runtime-api.ts index 0c736b24b236..eaf9b7d8bccf 100644 --- a/packages/speech-core/src/tts.ts +++ b/src/tts/runtime-api.ts @@ -1,44 +1,62 @@ -import type { TtsProvider } from "openclaw/plugin-sdk/config-contracts"; -import { parseTtsDirectives, summarizeText } from "openclaw/plugin-sdk/speech-core"; +// Runtime speech API barrel for TTS preferences, synthesis, streaming, and test +// helpers used by speech-capable plugins. +import type { TtsProvider } from "../config/types.js"; +import { parseTtsDirectives } from "./directives.js"; +import { summarizeText } from "./tts-core.js"; import { getResolvedSpeechProviderConfig, resolveTtsProvider } from "./tts-provider-resolution.js"; import { resolveModelOverridePolicy, type ResolvedTtsConfig } from "./tts-settings.js"; import { formatTtsProviderError, sanitizeTtsErrorForLog } from "./tts-synthesis-support.js"; import { + resolveTtsSynthesisTarget, shouldDeliverTtsAsVoice, supportsNativeVoiceNoteTts, supportsTranscodedVoiceNoteTts, - resolveTtsSynthesisTarget, } from "./tts-synthesis.js"; -export type { - TtsDirectiveOverrides, - TtsDirectiveParseResult, -} from "openclaw/plugin-sdk/speech-core"; - -export function getTtsProvider(config: ResolvedTtsConfig, prefsPath: string): TtsProvider { - return resolveTtsProvider(config, prefsPath); -} - +export { setSpeechRuntimeAvailabilityGuard } from "./runtime-availability.js"; +export { + buildTtsSystemPromptHint, + getTtsMaxLength, + getTtsPersona, + isSummarizationEnabled, + isTtsEnabled, + listTtsPersonas, + resolveTtsAutoMode, + resolveTtsConfig, + resolveTtsPrefsPath, + setTtsMachinePrefsPathResolver, + type ResolvedTtsConfig, + type ResolvedTtsModelOverrides, +} from "./tts-settings.js"; +export { + setSummarizationEnabled, + setTtsAutoMode, + setTtsEnabled, + setTtsMaxLength, + setTtsPersona, + setTtsProvider, +} from "./tts-settings-writes.js"; export { getLastTtsAttempt, listSpeechVoices, setLastTtsAttempt } from "./tts-payload.js"; export { getResolvedSpeechProviderConfig, isTtsProviderConfigured, resolveTtsProviderOrder, } from "./tts-provider-resolution.js"; -export { - prepareTtsRequest, - resolveExplicitTtsOverrides, - type PreparedTtsRequest, -} from "./tts-request.js"; +export { prepareTtsRequest, resolveExplicitTtsOverrides } from "./tts-request.js"; export { streamSpeech, textToSpeechStream } from "./tts-streaming.js"; export { synthesizeSpeech } from "./tts-synthesis.js"; export { textToSpeechTelephony } from "./tts-telephony.js"; +export type { TtsDirectiveOverrides, TtsDirectiveParseResult } from "./provider-types.js"; export type { TtsStreamResult, TtsSynthesisResult, TtsSynthesisStreamResult, TtsTelephonyResult, -} from "./tts-types.js"; +} from "./tts-runtime-types.js"; + +export function getTtsProvider(config: ResolvedTtsConfig, prefsPath: string): TtsProvider { + return resolveTtsProvider(config, prefsPath); +} export const testApi = { parseTtsDirectives, diff --git a/packages/speech-core/src/runtime-availability.ts b/src/tts/runtime-availability.ts similarity index 89% rename from packages/speech-core/src/runtime-availability.ts rename to src/tts/runtime-availability.ts index 56aeaf1a7e67..ff565ce92a15 100644 --- a/packages/speech-core/src/runtime-availability.ts +++ b/src/tts/runtime-availability.ts @@ -1,4 +1,4 @@ -/** Host-owned availability guard shared by every speech-core entrypoint. */ +/** Host-owned availability guard shared by every speech runtime entrypoint. */ let assertRuntimeAvailable: (() => void) | undefined; diff --git a/packages/speech-core/speaker.ts b/src/tts/speaker.ts similarity index 96% rename from packages/speech-core/speaker.ts rename to src/tts/speaker.ts index cb730fbd4a15..f1705d2f0953 100644 --- a/packages/speech-core/speaker.ts +++ b/src/tts/speaker.ts @@ -1,6 +1,6 @@ // Speaker-selection compatibility helpers for plugins that renamed voice fields // over time but still need one normalized config object. -export type SpeakerSelectionConfig = Record; +type SpeakerSelectionConfig = Record; function readString(value: unknown): string | undefined { return typeof value === "string" && value.trim() ? value.trim() : undefined; diff --git a/packages/speech-core/src/speech-text.test.ts b/src/tts/speech-text.test.ts similarity index 98% rename from packages/speech-core/src/speech-text.test.ts rename to src/tts/speech-text.test.ts index da1dbbb820c6..8e871cdab91d 100644 --- a/packages/speech-core/src/speech-text.test.ts +++ b/src/tts/speech-text.test.ts @@ -1,5 +1,5 @@ -import { stripMarkdown } from "openclaw/plugin-sdk/text-chunking"; import { describe, expect, it } from "vitest"; +import { stripMarkdown } from "../shared/text/strip-markdown.js"; import { CODE_HEAVY_SPOKEN_FALLBACK, isCodeHeavySpeechText, diff --git a/packages/speech-core/src/speech-text.ts b/src/tts/speech-text.ts similarity index 98% rename from packages/speech-core/src/speech-text.ts rename to src/tts/speech-text.ts index e53359241402..71ca409b63f3 100644 --- a/packages/speech-core/src/speech-text.ts +++ b/src/tts/speech-text.ts @@ -1,4 +1,4 @@ -import { stripMarkdown } from "openclaw/plugin-sdk/text-chunking"; +import { stripMarkdown } from "../shared/text/strip-markdown.js"; export const CODE_HEAVY_SPOKEN_FALLBACK = "I've put the detailed response on screen."; diff --git a/src/tts/tts-audio-store.ts b/src/tts/tts-audio-store.ts index 48ea8afbf954..9a733cec9475 100644 --- a/src/tts/tts-audio-store.ts +++ b/src/tts/tts-audio-store.ts @@ -1,8 +1,8 @@ -// File-backed TTS output is owned by the canonical media store, not speech-core. +// File-backed TTS output is owned by the canonical media store. import { mimeTypeFromFilePath } from "@openclaw/media-core/mime"; -import type { TtsAudioPersistence } from "../../packages/speech-core/src/tts-synthesis.js"; import { resolveGeneratedMediaMaxBytes } from "../media/configured-max-bytes.js"; import { saveMediaBuffer } from "../media/store.js"; +import type { TtsAudioPersistence } from "./tts-synthesis.js"; const TTS_MEDIA_SUBDIR = "tool-speech-synthesis"; diff --git a/packages/speech-core/src/tts-payload.ts b/src/tts/tts-payload.ts similarity index 92% rename from packages/speech-core/src/tts-payload.ts rename to src/tts/tts-payload.ts index 63baa369d04b..33524db196d9 100644 --- a/packages/speech-core/src/tts-payload.ts +++ b/src/tts/tts-payload.ts @@ -1,25 +1,20 @@ -import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import { - markReplyPayloadAsTtsSupplement, - resolveSendableOutboundReplyParts, - type ReplyPayload, -} from "openclaw/plugin-sdk/reply-payload"; -import { isVerbose, logVerbose } from "openclaw/plugin-sdk/runtime-env"; -import { - canonicalizeSpeechProviderId, - getSpeechProvider, - parseTtsDirectives, - summarizeText, - type SpeechVoiceOption, -} from "openclaw/plugin-sdk/speech-core"; -import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; +import { markReplyPayloadAsTtsSupplement, type ReplyPayload } from "../auto-reply/reply-payload.js"; +import type { OpenClawConfig } from "../config/types.js"; +import { isVerbose, logVerbose } from "../globals.js"; +import { resolveSendableOutboundReplyParts } from "../infra/outbound/reply-payload-parts.js"; +import { truncateUtf16Safe } from "../utils.js"; +import { parseTtsDirectives } from "./directives.js"; +import { canonicalizeSpeechProviderId, getSpeechProvider } from "./provider-registry.js"; +import type { SpeechVoiceOption } from "./provider-types.js"; import { assertSpeechRuntimeAvailable, isSpeechRuntimeAvailable } from "./runtime-availability.js"; import { isCodeHeavySpeechText, normalizeSpeechText } from "./speech-text.js"; +import { summarizeText } from "./tts-core.js"; import { getResolvedSpeechProviderConfig, resolveSpeechProviderTimeoutMs, resolveTtsProvider, } from "./tts-provider-resolution.js"; +import type { TtsStatusEntry } from "./tts-runtime-types.js"; import { getTtsMaxLength, isSummarizationEnabled, @@ -29,7 +24,6 @@ import { type ResolvedTtsConfig, } from "./tts-settings.js"; import { textToSpeech, type TtsAudioPersistence } from "./tts-synthesis.js"; -import type { TtsStatusEntry } from "./tts-types.js"; let lastTtsAttempt: TtsStatusEntry | undefined; diff --git a/packages/speech-core/src/tts-provider-resolution.ts b/src/tts/tts-provider-resolution.ts similarity index 96% rename from packages/speech-core/src/tts-provider-resolution.ts rename to src/tts/tts-provider-resolution.ts index d5c811e78bb4..06317b038f63 100644 --- a/packages/speech-core/src/tts-provider-resolution.ts +++ b/src/tts/tts-provider-resolution.ts @@ -1,33 +1,23 @@ +import { clampTimerTimeoutMs } from "../../packages/normalization-core/src/number-coercion.js"; +import { + normalizeLowercaseStringOrEmpty, + normalizeOptionalString, +} from "../../packages/normalization-core/src/string-coerce.js"; import type { OpenClawConfig, ResolvedTtsPersona, TtsConfig, TtsProvider, -} from "openclaw/plugin-sdk/config-contracts"; -import { clampTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime"; +} from "../config/types.js"; +import type { SpeechProviderPlugin } from "../plugins/types.js"; import { canonicalizeSpeechProviderId, getSpeechProvider, listSpeechProviders, normalizeSpeechProviderId, - type SpeechProviderConfig, - type SpeechProviderPlugin, -} from "openclaw/plugin-sdk/speech-core"; -import { - normalizeLowercaseStringOrEmpty, - normalizeOptionalString, -} from "openclaw/plugin-sdk/string-coerce-runtime"; -import { withSpeakerSelectionCompat } from "../speaker.js"; -import { - resolvePrimaryVoiceProviderCandidate, - resolveSupportedVoiceModelRefs, - resolveVoiceModelRefs, - resolveVoiceProviderCandidates, - voiceProviderSupportsModel, - type VoiceModelProvider, - type VoiceModelRef, - type VoiceProviderCandidate, -} from "../voice-models.js"; +} from "./provider-registry.js"; +import type { SpeechProviderConfig } from "./provider-types.js"; +import { withSpeakerSelectionCompat } from "./speaker.js"; import { DEFAULT_TTS_TIMEOUT_MS, asProviderConfig, @@ -39,6 +29,16 @@ import { resolveTtsRuntimeConfig, type ResolvedTtsConfig, } from "./tts-settings.js"; +import { + resolvePrimaryVoiceProviderCandidate, + resolveSupportedVoiceModelRefs, + resolveVoiceModelRefs, + resolveVoiceProviderCandidates, + voiceProviderSupportsModel, + type VoiceModelProvider, + type VoiceModelRef, + type VoiceProviderCandidate, +} from "./voice-models.js"; function resolvePositiveTimeoutMs(timeoutMs: number | undefined): number | undefined { return typeof timeoutMs === "number" && Number.isFinite(timeoutMs) && timeoutMs > 0 diff --git a/packages/speech-core/src/tts-request.ts b/src/tts/tts-request.ts similarity index 87% rename from packages/speech-core/src/tts-request.ts rename to src/tts/tts-request.ts index 8466a0ad7506..762e51fbd81c 100644 --- a/packages/speech-core/src/tts-request.ts +++ b/src/tts/tts-request.ts @@ -1,17 +1,16 @@ -import type { OpenClawConfig, TtsConfig } from "openclaw/plugin-sdk/config-contracts"; -import { mergeDeep } from "openclaw/plugin-sdk/plugin-config-runtime"; -import { - canonicalizeSpeechProviderId, - getSpeechProvider, - parseTtsDirectives, - type SpeechProviderOverrides, - type TtsDirectiveOverrides, - type TtsDirectiveParseResult, -} from "openclaw/plugin-sdk/speech-core"; +import type { OpenClawConfig, TtsConfig } from "../config/types.js"; +import { mergeDeep } from "../infra/deep-merge.js"; +import { parseTtsDirectives } from "./directives.js"; +import { canonicalizeSpeechProviderId, getSpeechProvider } from "./provider-registry.js"; +import type { + SpeechProviderOverrides, + TtsDirectiveOverrides, + TtsDirectiveParseResult, +} from "./provider-types.js"; import { resolveTtsProvider } from "./tts-provider-resolution.js"; import { resolveTtsConfig, resolveTtsPrefsPath, resolveTtsRuntimeConfig } from "./tts-settings.js"; -export type PreparedTtsRequest = { +type PreparedTtsRequest = { cfg: OpenClawConfig; directives: TtsDirectiveParseResult; }; diff --git a/src/tts/tts-runtime-fallbacks.test.ts b/src/tts/tts-runtime-fallbacks.test.ts new file mode 100644 index 000000000000..e6364038f95c --- /dev/null +++ b/src/tts/tts-runtime-fallbacks.test.ts @@ -0,0 +1,462 @@ +import { rmSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + clearRuntimeConfigSnapshot, + createMockSpeechProvider, + createTtsConfig, + expectTtsPayloadResult, + installSpeechProviders, + maybeApplyTtsToPayload, + maybeApplyTtsToPayloadCore, + prefsPathFor, + prepareSynthesisMock, + requireFirstSynthesisRequest, + requireRecord, + setSummarizationEnabled, + setTtsMachinePrefsPathResolver, + setTtsMaxLength, + synthesizeMock, + synthesizeSpeech, + testApi, + transcodeAudioBufferMock, + type OpenClawConfig, +} from "./tts-runtime.test-support.js"; + +describe("TTS runtime provider fallback and delivery behavior", () => { + afterEach(() => { + setTtsMachinePrefsPathResolver(); + clearRuntimeConfigSnapshot(); + delete (Object.prototype as Record).polluted; + synthesizeMock.mockClear(); + prepareSynthesisMock.mockClear(); + transcodeAudioBufferMock.mockClear(); + installSpeechProviders([createMockSpeechProvider()]); + }); + + it("ignores voiceModel refs that are not speech models", async () => { + installSpeechProviders([ + createMockSpeechProvider("openai", { + autoSelectOrder: 10, + defaultModel: "gpt-4o-mini-tts", + models: ["gpt-4o-mini-tts"], + resolveConfig: ({ rawConfig }) => { + const providers = requireRecord(rawConfig.providers, "raw provider configs"); + return { + model: "gpt-4o-mini-tts", + modelId: "gpt-4o-mini-tts", + ...requireRecord(providers.openai, "raw openai provider config"), + }; + }, + }), + ]); + + const result = await synthesizeSpeech({ + text: "Use speech provider default for unsupported realtime model.", + cfg: { + agents: { + defaults: { + voiceModel: { primary: "openai/gpt-realtime-2" }, + }, + }, + tts: { + enabled: true, + provider: "openai", + prefsPath: "/tmp/openclaw-speech-core-realtime-voice-model-ignored-test.json", + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("openai"); + expect(result.providerModel).toBe("gpt-4o-mini-tts"); + const request = requireFirstSynthesisRequest("speech model fallback request"); + expect(request.providerConfig).toMatchObject({ + model: "gpt-4o-mini-tts", + modelId: "gpt-4o-mini-tts", + }); + }); + + it("uses the first speech-supported voiceModel fallback as the default provider", async () => { + installSpeechProviders([ + createMockSpeechProvider("openai", { + autoSelectOrder: 1, + models: ["gpt-4o-mini-tts"], + }), + createMockSpeechProvider("elevenlabs", { + autoSelectOrder: 99, + models: ["eleven_multilingual_v2"], + }), + ]); + + const result = await synthesizeSpeech({ + text: "Use first speech-supported voice model.", + cfg: { + agents: { + defaults: { + voiceModel: { + primary: "openai/gpt-realtime-2", + fallbacks: ["elevenlabs/eleven_multilingual_v2"], + }, + }, + }, + tts: { + enabled: true, + prefsPath: "/tmp/openclaw-speech-core-supported-voice-model-provider-test.json", + }, + } as OpenClawConfig, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("elevenlabs"); + expect(result.providerModel).toBe("eleven_multilingual_v2"); + expect(result.attemptedProviders).toEqual(["elevenlabs"]); + }); + + it("maps speakerVoice provider config to provider-compatible voice fields", async () => { + const result = await synthesizeSpeech({ + text: "Use the configured speaker.", + cfg: { + tts: { + enabled: true, + provider: "mock", + providers: { + mock: { + speakerVoice: "cedar", + speakerVoiceId: "voice-123", + voice: "legacy-voice", + voiceName: "legacy-name", + voiceId: "legacy-id", + }, + }, + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + expect(result.providerVoice).toBe("voice-123"); + const request = requireFirstSynthesisRequest("speaker voice synthesis request"); + expect(request.providerConfig).toMatchObject({ + speakerVoice: "cedar", + voice: "cedar", + voiceName: "cedar", + speakerVoiceId: "voice-123", + voiceId: "voice-123", + }); + }); + + it("preserves alias-keyed provider config when resolving canonical providers", async () => { + installSpeechProviders([ + createMockSpeechProvider("xiaomi", { + aliases: ["mimo"], + resolveConfig: ({ rawConfig }) => { + const providers = requireRecord(rawConfig.providers, "raw provider configs"); + return requireRecord(providers.xiaomi ?? providers.mimo, "raw xiaomi provider config"); + }, + }), + ]); + + const result = await synthesizeSpeech({ + text: "Use alias provider config.", + cfg: { + tts: { + enabled: true, + provider: "xiaomi", + providers: { + mimo: { apiKey: "fake" }, + }, + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("alias provider synthesis request"); + expect(request.providerConfig).toMatchObject({ apiKey: "fake" }); + }); + + it("maps speakerVoice persona provider config to provider-compatible voice fields", async () => { + const result = await synthesizeSpeech({ + text: "Use the persona speaker.", + cfg: { + tts: { + enabled: true, + provider: "mock", + persona: "narrator", + personas: { + narrator: { + providers: { + mock: { + speakerVoice: "marin", + }, + }, + }, + }, + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + expect(result.providerVoice).toBe("marin"); + const request = requireFirstSynthesisRequest("persona speaker voice synthesis request"); + expect(request.providerConfig).toMatchObject({ + speakerVoice: "marin", + voice: "marin", + voiceName: "marin", + }); + }); + + it.each(["feishu", "whatsapp"] as const)( + "marks %s voice-note TTS for channel-side transcoding when provider returns mp3", + async (channel) => { + expect(testApi.supportsTranscodedVoiceNoteTts(channel)).toBe(true); + await expectTtsPayloadResult({ + channel, + prefsName: `openclaw-speech-core-tts-${channel}-mp3-test`, + text: `This ${channel} reply should be transcoded by the channel.`, + target: "voice-note", + audioAsVoice: true, + mediaExtension: "mp3", + providerResult: { + audioBuffer: Buffer.from("mp3"), + outputFormat: "mp3", + fileExtension: ".mp3", + voiceCompatible: false, + }, + }); + }, + ); + + it("keeps non-native voice-note channels as regular audio files", async () => { + await expectTtsPayloadResult({ + channel: "slack", + prefsName: "openclaw-speech-core-tts-slack-test", + text: "Slack replies should be delivered as regular audio attachments.", + target: "audio-file", + audioAsVoice: undefined, + }); + }); + + it("preserves the text reply when auto-TTS audio persistence fails", async () => { + const payload = { text: "This text must still be delivered when media storage rejects audio." }; + const result = await maybeApplyTtsToPayloadCore( + { + payload, + cfg: createTtsConfig("openclaw-speech-core-auto-persistence-failure-test"), + channel: "slack", + kind: "final", + }, + async () => { + throw new Error("Media exceeds configured limit"); + }, + ); + + expect(result).toBe(payload); + }); + + it("normalizes voice-note Markdown once before synthesis", async () => { + const text = + 'This short explanation keeps the fenced literal below from becoming code-heavy.\n\n```md\nconst literal = "[x](y)";\n```'; + let mediaDir: string | undefined; + try { + const result = await maybeApplyTtsToPayload({ + payload: { text }, + cfg: createTtsConfig("openclaw-speech-core-once-normalized-markdown-test"), + channel: "telegram", + kind: "final", + }); + + const request = requireFirstSynthesisRequest("once-normalized voice-note synthesis request"); + expect(request.text).toBe( + 'This short explanation keeps the fenced literal below from becoming code-heavy.\n\nconst literal = "[x](y)";', + ); + expect(result.text).toBe(text); + mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; + } finally { + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } + }); + + it("skips channel auto-TTS audio for code-heavy replies", async () => { + const text = "```ts\nexport function answer() {\n return 42;\n}\n```"; + const result = await maybeApplyTtsToPayload({ + payload: { text }, + cfg: createTtsConfig("openclaw-speech-core-code-heavy-voice-note-test"), + channel: "telegram", + kind: "final", + }); + + expect(synthesizeMock).not.toHaveBeenCalled(); + expect(result).toEqual({ text }); + }); + + it("synthesizes code-heavy explicitly tagged hidden TTS text", async () => { + const cfg = createTtsConfig("openclaw-speech-core-code-heavy-hidden-tts-test"); + let mediaDir: string | undefined; + try { + const result = await maybeApplyTtsToPayload({ + payload: { + text: '[[tts:text]]```ts\nconst detailedAnswer = "this code should still be spoken";\n```[[/tts:text]]', + audioAsVoice: true, + }, + cfg, + channel: "telegram", + kind: "final", + }); + + expect(synthesizeMock).toHaveBeenCalled(); + const request = requireFirstSynthesisRequest("code-heavy hidden TTS request"); + expect(request.text).toBe('const detailedAnswer = "this code should still be spoken";'); + expect(result.text).toBeUndefined(); + mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; + } finally { + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } + }); + + it("synthesizes explicitly tagged short hidden TTS text", async () => { + const cfg = createTtsConfig("openclaw-speech-core-short-hidden-tts-test"); + let mediaDir: string | undefined; + try { + const result = await maybeApplyTtsToPayload({ + payload: { + text: "[[tts:text]]hello[[/tts:text]]", + audioAsVoice: true, + }, + cfg, + channel: "telegram", + kind: "final", + }); + + expect(synthesizeMock).toHaveBeenCalled(); + const request = requireFirstSynthesisRequest("hidden TTS request"); + expect(request.text).toBe("hello"); + expect(result.mediaUrl).toMatch(/voice---[a-f0-9-]+\.ogg$/); + expect(result.audioAsVoice).toBe(true); + expect(result.text).toBeUndefined(); + expect(result.ttsSupplement).toBeUndefined(); + mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; + } finally { + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } + }); + + it("truncates long TTS text on a UTF-16 boundary", async () => { + const prefsName = "openclaw-speech-core-utf16-truncate-test"; + const prefsPath = prefsPathFor(prefsName); + const cfg = createTtsConfig(prefsName); + setTtsMaxLength(prefsPath, 11); + setSummarizationEnabled(prefsPath, false); + let mediaDir: string | undefined; + try { + const result = await maybeApplyTtsToPayload({ + payload: { text: `${"a".repeat(7)}😀tail long enough for TTS` }, + cfg, + channel: "telegram", + kind: "final", + }); + + expect(synthesizeMock).toHaveBeenCalled(); + const request = requireFirstSynthesisRequest("utf16 truncated TTS request"); + const spokenText = String(request.text); + expect(spokenText).toBe(`${"a".repeat(7)}...`); + expect(result.spokenText).toBe(spokenText); + mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; + } finally { + rmSync(prefsPath, { force: true }); + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } + }); + + it("skips block delivery kind in final mode (accumulated final tail synthesizes instead)", async () => { + synthesizeMock.mockClear(); + const cfg = createTtsConfig("openclaw-speech-core-block-kind-tts-test"); + const result = await maybeApplyTtsToPayload({ + payload: { text: "WebChat block stream chunks defer TTS to the final tail." }, + cfg, + channel: "webchat", + kind: "block", + }); + + expect(synthesizeMock).not.toHaveBeenCalled(); + expect((result as { trustedLocalMedia?: boolean }).trustedLocalMedia).toBeUndefined(); + expect(result.text).toBe("WebChat block stream chunks defer TTS to the final tail."); + }); + + it("skips tool delivery kind in final mode", async () => { + synthesizeMock.mockClear(); + const cfg = createTtsConfig("openclaw-speech-core-tool-kind-tts-test"); + const result = await maybeApplyTtsToPayload({ + payload: { text: "Intermediate tool output should not be spoken." }, + cfg, + channel: "webchat", + kind: "tool", + }); + + expect(synthesizeMock).not.toHaveBeenCalled(); + expect((result as { trustedLocalMedia?: boolean }).trustedLocalMedia).toBeUndefined(); + expect(result.text).toBe("Intermediate tool output should not be spoken."); + }); + + it("keeps skipping untagged short TTS text", async () => { + const cfg = createTtsConfig("openclaw-speech-core-short-plain-tts-test"); + const result = await maybeApplyTtsToPayload({ + payload: { + text: "hello", + audioAsVoice: true, + }, + cfg, + channel: "telegram", + kind: "final", + }); + + expect(synthesizeMock).not.toHaveBeenCalled(); + expect(result).toEqual({ + text: "hello", + audioAsVoice: true, + }); + }); + + it("skips auto TTS for legacy final media directives", async () => { + synthesizeMock.mockClear(); + const cfg = createTtsConfig("openclaw-speech-core-media-directive-tts-test"); + const result = await maybeApplyTtsToPayload({ + payload: { text: "Here is the render.\nMEDIA:/tmp/render.png" }, + cfg, + channel: "telegram", + kind: "final", + }); + + expect(synthesizeMock).not.toHaveBeenCalled(); + expect(result).toEqual({ text: "Here is the render.\nMEDIA:/tmp/render.png" }); + }); + + it("keeps skipping explicit tagged TTS text that strips to empty markdown", async () => { + const cfg = createTtsConfig("openclaw-speech-core-empty-hidden-tts-test"); + const result = await maybeApplyTtsToPayload({ + payload: { + text: "[[tts:text]]***[[/tts:text]]", + audioAsVoice: true, + }, + cfg, + channel: "telegram", + kind: "final", + }); + + expect(synthesizeMock).not.toHaveBeenCalled(); + expect(result).toEqual({ + audioAsVoice: true, + }); + }); +}); diff --git a/src/tts/tts-runtime-models.test.ts b/src/tts/tts-runtime-models.test.ts new file mode 100644 index 000000000000..c90d13ddaab7 --- /dev/null +++ b/src/tts/tts-runtime-models.test.ts @@ -0,0 +1,341 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + MAX_TIMER_TIMEOUT_MS, + clearRuntimeConfigSnapshot, + createMockSpeechProvider, + installSpeechProviders, + prepareSynthesisMock, + requireAttempt, + requireFirstSynthesisRequest, + requireRecord, + setTtsMachinePrefsPathResolver, + synthesizeMock, + synthesizeSpeech, + textToSpeechStream, + transcodeAudioBufferMock, + type OpenClawConfig, + type SpeechSynthesisRequest, +} from "./tts-runtime.test-support.js"; + +describe("TTS runtime voice model and streaming behavior", () => { + afterEach(() => { + setTtsMachinePrefsPathResolver(); + clearRuntimeConfigSnapshot(); + delete (Object.prototype as Record).polluted; + synthesizeMock.mockClear(); + prepareSynthesisMock.mockClear(); + transcodeAudioBufferMock.mockClear(); + installSpeechProviders([createMockSpeechProvider()]); + }); + + it("caps oversized voice model TTS timeouts before synthesis", async () => { + installSpeechProviders([ + createMockSpeechProvider("mock", { autoSelectOrder: 1, models: ["mock-tts"] }), + ]); + + const result = await synthesizeSpeech({ + text: "Use capped explicit timeout.", + cfg: { + agents: { + defaults: { + voiceModel: { primary: "mock/mock-tts", timeoutMs: Number.MAX_SAFE_INTEGER }, + }, + }, + tts: { + enabled: true, + provider: "mock", + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("voice model capped timeout request"); + expect(request.timeoutMs).toBe(MAX_TIMER_TIMEOUT_MS); + }); + + it("uses agents.defaults.voiceModel as the default speech provider and model", async () => { + installSpeechProviders([ + createMockSpeechProvider("mock", { autoSelectOrder: 1 }), + createMockSpeechProvider("openai", { + autoSelectOrder: 10, + models: ["gpt-4o-mini-tts"], + resolveConfig: ({ rawConfig }) => { + const providers = requireRecord(rawConfig.providers, "raw provider configs"); + return { + model: "provider-default-model", + modelId: "provider-default-model", + ...requireRecord(providers.openai, "raw openai provider config"), + }; + }, + }), + ]); + + const result = await synthesizeSpeech({ + text: "Use configured voice model.", + cfg: { + agents: { + defaults: { + voiceModel: { primary: "openai/gpt-4o-mini-tts", timeoutMs: 12_345 }, + }, + }, + tts: { + enabled: true, + prefsPath: "/tmp/openclaw-speech-core-voice-model-default-test.json", + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("openai"); + expect(result.providerModel).toBe("gpt-4o-mini-tts"); + const request = requireFirstSynthesisRequest("voice model synthesis request"); + expect(request.providerConfig).toMatchObject({ + model: "gpt-4o-mini-tts", + modelId: "gpt-4o-mini-tts", + }); + expect(request.timeoutMs).toBe(12_345); + }); + + it("keeps explicit provider model aliases ahead of voiceModel defaults", async () => { + installSpeechProviders([ + createMockSpeechProvider("openrouter", { + models: ["explicit-model", "default-model"], + resolveConfig: ({ rawConfig }) => { + const providers = requireRecord(rawConfig.providers, "raw provider configs"); + return requireRecord(providers.openrouter, "raw openrouter provider config"); + }, + }), + ]); + + const result = await synthesizeSpeech({ + text: "Prefer explicit model alias.", + cfg: { + agents: { + defaults: { + voiceModel: { primary: "openrouter/default-model" }, + }, + }, + tts: { + enabled: true, + provider: "openrouter", + prefsPath: "/tmp/openclaw-speech-core-explicit-model-alias-test.json", + providers: { + openrouter: { + modelId: "explicit-model", + }, + }, + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("explicit model alias synthesis request"); + const providerConfig = requireRecord(request.providerConfig, "provider config"); + expect(providerConfig).toMatchObject({ + modelId: "explicit-model", + }); + expect(providerConfig.model).toBeUndefined(); + }); + + it("tries voiceModel fallbacks before auto-selected speech providers", async () => { + installSpeechProviders([ + createMockSpeechProvider("mock", { autoSelectOrder: 1 }), + createMockSpeechProvider("openai", { + autoSelectOrder: 10, + models: ["gpt-4o-mini-tts"], + isConfigured: () => false, + }), + createMockSpeechProvider("elevenlabs", { + autoSelectOrder: 99, + models: ["eleven_multilingual_v2"], + }), + ]); + + const result = await synthesizeSpeech({ + text: "Use configured voice model fallback.", + cfg: { + agents: { + defaults: { + voiceModel: { + primary: "openai/gpt-4o-mini-tts", + fallbacks: ["elevenlabs/eleven_multilingual_v2"], + }, + }, + }, + tts: { + enabled: true, + prefsPath: "/tmp/openclaw-speech-core-voice-model-fallback-test.json", + }, + } as OpenClawConfig, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("elevenlabs"); + expect(result.fallbackFrom).toBe("openai"); + expect(result.providerModel).toBe("eleven_multilingual_v2"); + }); + + it("tries same-provider voiceModel fallbacks as separate model attempts", async () => { + const synthesize = vi.fn(async (request: SpeechSynthesisRequest) => { + if (request.providerConfig.model === "bad-tts") { + throw new Error("unavailable model"); + } + return { + audioBuffer: Buffer.from("voice"), + fileExtension: ".ogg", + outputFormat: "ogg", + voiceCompatible: request.target === "voice-note", + }; + }); + installSpeechProviders([ + createMockSpeechProvider("openai", { + autoSelectOrder: 10, + models: ["bad-tts", "good-tts"], + synthesize, + }), + ]); + + const result = await synthesizeSpeech({ + text: "Use same-provider fallback model.", + cfg: { + agents: { + defaults: { + voiceModel: { + primary: "openai/bad-tts", + fallbacks: ["openai/good-tts"], + }, + }, + }, + tts: { + enabled: true, + prefsPath: "/tmp/openclaw-speech-core-same-provider-voice-model-fallback-test.json", + }, + } as OpenClawConfig, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("openai"); + expect(result.providerModel).toBe("good-tts"); + expect(result.attemptedProviders).toEqual(["openai", "openai"]); + expect(synthesize.mock.calls.map(([request]) => request.providerConfig.model)).toEqual([ + "bad-tts", + "good-tts", + ]); + }); + + it("skips non-streaming providers before using a streaming fallback", async () => { + const release = vi.fn(async () => {}); + const streamSynthesize = vi.fn(async () => ({ + audioStream: new ReadableStream({ + start(controller) { + controller.close(); + }, + }), + fileExtension: ".pcm", + outputFormat: "pcm", + voiceCompatible: false, + release, + })); + installSpeechProviders([ + createMockSpeechProvider("buffered", { autoSelectOrder: 1 }), + createMockSpeechProvider("streaming", { + autoSelectOrder: 2, + streamSynthesize, + }), + ]); + + const result = await textToSpeechStream({ + text: "Use streaming fallback.", + cfg: { + tts: { + enabled: true, + provider: "buffered", + prefsPath: "/tmp/openclaw-speech-core-streaming-fallback-test.json", + }, + } as OpenClawConfig, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("streaming"); + expect(result.fallbackFrom).toBe("buffered"); + expect(result.attemptedProviders).toEqual(["buffered", "streaming"]); + expect(result.outputFormat).toBe("pcm"); + expect(result.fileExtension).toBe(".pcm"); + expect(result.target).toBe("audio-file"); + expect(result.release).toBe(release); + const skippedAttempt = requireAttempt(result.attempts, 0); + expect(skippedAttempt).toMatchObject({ + provider: "buffered", + outcome: "skipped", + reasonCode: "unsupported_for_streaming", + personaBinding: "none", + error: "buffered does not support streaming TTS", + }); + expect(skippedAttempt).not.toHaveProperty("latencyMs"); + expect(requireAttempt(result.attempts, 1)).toMatchObject({ + provider: "streaming", + outcome: "success", + reasonCode: "success", + }); + expect(streamSynthesize).toHaveBeenCalledOnce(); + }); + + it("classifies streaming timeouts before falling back with raw text", async () => { + const timeoutStreamSynthesize = vi.fn(async () => { + const error = new Error("stalled"); + error.name = "AbortError"; + throw error; + }); + const fallbackStreamSynthesize = vi.fn(async () => ({ + audioStream: new ReadableStream({ + start(controller) { + controller.close(); + }, + }), + fileExtension: ".pcm", + outputFormat: "pcm", + voiceCompatible: false, + })); + installSpeechProviders([ + createMockSpeechProvider("primary", { + autoSelectOrder: 1, + streamSynthesize: timeoutStreamSynthesize, + }), + createMockSpeechProvider("fallback", { + autoSelectOrder: 2, + streamSynthesize: fallbackStreamSynthesize, + }), + ]); + const text = "## Keep [streaming Markdown](https://example.com) raw!!!!!"; + + const result = await textToSpeechStream({ + text, + cfg: { + tts: { + enabled: true, + provider: "primary", + prefsPath: "/tmp/openclaw-speech-core-streaming-timeout-test.json", + }, + } as OpenClawConfig, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("fallback"); + expect(result.fallbackFrom).toBe("primary"); + expect(requireAttempt(result.attempts, 0)).toMatchObject({ + provider: "primary", + outcome: "failed", + reasonCode: "timeout", + error: "primary: request timed out", + }); + expect(requireAttempt(result.attempts, 1)).toMatchObject({ + provider: "fallback", + outcome: "success", + reasonCode: "success", + }); + expect(fallbackStreamSynthesize).toHaveBeenCalledWith(expect.objectContaining({ text })); + }); +}); diff --git a/src/tts/tts-runtime-personas.test.ts b/src/tts/tts-runtime-personas.test.ts new file mode 100644 index 000000000000..f66aff76b9b9 --- /dev/null +++ b/src/tts/tts-runtime-personas.test.ts @@ -0,0 +1,496 @@ +import { rmSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + clearRuntimeConfigSnapshot, + createMockSpeechProvider, + getTtsPersona, + getTtsProvider, + installSpeechProviders, + isTtsProviderConfigured, + maybeApplyTtsToPayload, + prepareSynthesisMock, + requireAttempt, + requireFirstCallParam, + requireFirstSynthesisRequest, + requireRecord, + resolveTtsConfig, + setTtsMachinePrefsPathResolver, + synthesizeMock, + synthesizeSpeech, + textToSpeechTelephony, + transcodeAudioBufferMock, + type OpenClawConfig, + type ReplyPayload, + type SpeechTelephonySynthesisRequest, +} from "./tts-runtime.test-support.js"; + +describe("TTS runtime persona behavior", () => { + afterEach(() => { + setTtsMachinePrefsPathResolver(); + clearRuntimeConfigSnapshot(); + delete (Object.prototype as Record).polluted; + synthesizeMock.mockClear(); + prepareSynthesisMock.mockClear(); + transcodeAudioBufferMock.mockClear(); + installSpeechProviders([createMockSpeechProvider()]); + }); + + it("selects persona preferred provider before config fallback", () => { + const cfg: OpenClawConfig = { + tts: { + enabled: true, + provider: "other", + persona: "alfred", + personas: { + alfred: { + label: "Alfred", + provider: "mock", + providers: { + mock: { + voice: "Algieba", + }, + }, + }, + }, + }, + }; + const config = resolveTtsConfig(cfg); + const prefsPath = "/tmp/openclaw-speech-core-persona-provider.json"; + + expect(getTtsPersona(config, prefsPath)?.id).toBe("alfred"); + expect(getTtsProvider(config, prefsPath)).toBe("mock"); + }); + + it("treats provider configuration errors as unconfigured", () => { + installSpeechProviders([ + createMockSpeechProvider("broken", { + resolveConfig: () => { + throw new Error("invalid provider URL"); + }, + }), + ]); + const prefsPath = "/tmp/openclaw-speech-core-invalid-provider.json"; + setTtsMachinePrefsPathResolver(() => prefsPath); + const cfg = { + tts: { + providers: { broken: {} }, + }, + } as OpenClawConfig; + const config = resolveTtsConfig(cfg); + + expect(isTtsProviderConfigured(config, "broken", cfg)).toBe(false); + expect(getTtsProvider(config, prefsPath)).toBe(""); + }); + + it("merges active persona provider binding into synthesis config", async () => { + setTtsMachinePrefsPathResolver(() => "/tmp/openclaw-speech-core-persona-merge.json"); + const cfg: OpenClawConfig = { + tts: { + enabled: true, + provider: "mock", + providers: { + mock: { + model: "base-model", + voice: "base-voice", + }, + }, + persona: "alfred", + personas: { + alfred: { + provider: "mock", + providers: { + mock: { + voice: "persona-voice", + style: "dry", + }, + }, + }, + }, + }, + }; + + const payload: ReplyPayload = { + text: "This reply should use persona-specific provider configuration.", + }; + + let mediaDir: string | undefined; + try { + const result = await maybeApplyTtsToPayload({ + payload, + cfg, + channel: "slack", + kind: "final", + }); + + expect(synthesizeMock).toHaveBeenCalled(); + const request = requireFirstSynthesisRequest("persona synthesis request"); + const providerConfig = requireRecord(request.providerConfig, "persona provider config"); + expect(providerConfig.model).toBe("base-model"); + expect(providerConfig.voice).toBe("persona-voice"); + expect(providerConfig.style).toBe("dry"); + expect(result.mediaUrl).toMatch(/voice---[a-f0-9-]+\.ogg$/); + + mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; + } finally { + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } + }); + + it("does not mark skipped unregistered providers as missing persona bindings", async () => { + const result = await synthesizeSpeech({ + text: "Use fallback provider.", + cfg: { + tts: { + enabled: true, + provider: "missing", + persona: "alfred", + personas: { + alfred: { + providers: { + missing: { + voice: "configured-but-unregistered", + }, + }, + }, + }, + }, + }, + }); + + expect(result.success).toBe(true); + const attempt = requireAttempt(result.attempts, 0); + expect(attempt.provider).toBe("missing"); + expect(attempt.outcome).toBe("skipped"); + expect(attempt.reasonCode).toBe("no_provider_registered"); + expect(attempt.persona).toBe("alfred"); + expect(attempt).not.toHaveProperty("personaBinding"); + }); + + it("does not mark skipped telephony providers as missing persona bindings", async () => { + const result = await textToSpeechTelephony({ + text: "Use telephony provider.", + cfg: { + tts: { + enabled: true, + provider: "mock", + persona: "alfred", + personas: { + alfred: { + providers: { + mock: { + voice: "persona-voice", + }, + }, + }, + }, + }, + }, + }); + + expect(result.success).toBe(false); + const attempt = requireAttempt(result.attempts, 0); + expect(attempt.provider).toBe("mock"); + expect(attempt.outcome).toBe("skipped"); + expect(attempt.reasonCode).toBe("unsupported_for_telephony"); + expect(attempt.persona).toBe("alfred"); + expect(attempt).not.toHaveProperty("personaBinding"); + }); + + it("passes directive overrides to telephony synthesis providers", async () => { + const synthesizeTelephonyMock = vi.fn(async (_request: SpeechTelephonySynthesisRequest) => ({ + audioBuffer: Buffer.from("voice"), + outputFormat: "pcm", + sampleRate: 24_000, + })); + installSpeechProviders([ + createMockSpeechProvider("mock", { + synthesizeTelephony: synthesizeTelephonyMock, + }), + ]); + + const text = "## Keep [telephony Markdown](https://example.com) raw!!!!!"; + const result = await textToSpeechTelephony({ + text, + cfg: { + tts: { + enabled: true, + provider: "mock", + providers: { + mock: { + modelId: "telephony-model", + voiceId: "default-voice", + }, + }, + }, + }, + overrides: { + providerOverrides: { + mock: { + speakerVoice: "directed-voice", + speed: 1.5, + }, + }, + }, + }); + + expect(result.success).toBe(true); + expect(result.providerModel).toBe("telephony-model"); + expect(result.providerVoice).toBe("directed-voice"); + expect(synthesizeTelephonyMock).toHaveBeenCalledOnce(); + const telephonyRequest = requireRecord( + requireFirstCallParam(synthesizeTelephonyMock.mock.calls, "telephony synthesis"), + "telephony synthesis request", + ); + expect(telephonyRequest.providerOverrides).toEqual({ + speakerVoice: "directed-voice", + speed: 1.5, + }); + expect(telephonyRequest.text).toBe(text); + expect(telephonyRequest).not.toHaveProperty("target"); + }); + + it("uses provider defaults when fallback policy allows missing persona bindings", async () => { + await synthesizeSpeech({ + text: "Use neutral provider defaults.", + cfg: { + tts: { + enabled: true, + provider: "mock", + persona: "alfred", + personas: { + alfred: { + fallbackPolicy: "provider-defaults", + }, + }, + }, + }, + }); + + expect(prepareSynthesisMock).toHaveBeenCalledOnce(); + const prepareContext = requireRecord( + requireFirstCallParam(prepareSynthesisMock.mock.calls, "prepare synthesis"), + "prepare synthesis context", + ); + expect(prepareContext.persona).toBeUndefined(); + expect(prepareContext.personaProviderConfig).toBeUndefined(); + }); + + it("preserves persona metadata by default when provider bindings are missing", async () => { + await synthesizeSpeech({ + text: "Use persona prompt.", + cfg: { + tts: { + enabled: true, + provider: "mock", + persona: "alfred", + personas: { + alfred: { + label: "Alfred", + }, + }, + }, + }, + }); + + expect(prepareSynthesisMock).toHaveBeenCalledOnce(); + const prepareContext = requireRecord( + requireFirstCallParam(prepareSynthesisMock.mock.calls, "prepare synthesis"), + "prepare synthesis context", + ); + const persona = requireRecord(prepareContext.persona, "prepare synthesis persona"); + expect(persona.id).toBe("alfred"); + expect(prepareContext.personaProviderConfig).toBeUndefined(); + }); + + it("skips unbound providers under fail policy while allowing bound fallbacks", async () => { + installSpeechProviders([ + createMockSpeechProvider("mock", { autoSelectOrder: 1 }), + createMockSpeechProvider("fallback", { autoSelectOrder: 2 }), + ]); + + const result = await synthesizeSpeech({ + text: "Use the first persona-bound provider.", + cfg: { + tts: { + enabled: true, + provider: "mock", + persona: "alfred", + personas: { + alfred: { + fallbackPolicy: "fail", + providers: { + fallback: { + voice: "fallback-voice", + }, + }, + }, + }, + }, + }, + }); + + expect(result.success).toBe(true); + expect(result.provider).toBe("fallback"); + expect(result.fallbackFrom).toBe("mock"); + const skippedAttempt = requireAttempt(result.attempts, 0); + expect(skippedAttempt.provider).toBe("mock"); + expect(skippedAttempt.outcome).toBe("skipped"); + expect(skippedAttempt.reasonCode).toBe("not_configured"); + expect(skippedAttempt.persona).toBe("alfred"); + expect(skippedAttempt.personaBinding).toBe("missing"); + expect(skippedAttempt.error).toBe("mock: persona alfred has no provider binding"); + const successAttempt = requireAttempt(result.attempts, 1); + expect(successAttempt.provider).toBe("fallback"); + expect(successAttempt.outcome).toBe("success"); + expect(successAttempt.persona).toBe("alfred"); + expect(successAttempt.personaBinding).toBe("applied"); + }); +}); + +describe("TTS runtime per-agent config", () => { + it("deep-merges the active agent TTS override over tts", () => { + const cfg = { + tts: { + enabled: true, + provider: "openai", + providers: { + openai: { + apiKey: "example", + voice: "coral", + speed: 1, + }, + }, + }, + agents: { + list: [ + { + id: "reader", + tts: { + provider: "openai", + providers: { + openai: { + voice: "nova", + }, + }, + }, + }, + ], + }, + } satisfies OpenClawConfig; + + const resolved = resolveTtsConfig(cfg, "reader"); + + const rawConfig = requireRecord(resolved.rawConfig, "resolved raw TTS config"); + expect(rawConfig.enabled).toBe(true); + expect(rawConfig.provider).toBe("openai"); + const providers = requireRecord(rawConfig.providers, "resolved raw TTS providers"); + const openai = requireRecord(providers.openai, "resolved OpenAI TTS provider config"); + expect(openai.apiKey).toBe("example"); + expect(openai.voice).toBe("nova"); + expect(openai.speed).toBe(1); + }); + + it("composes per-agent TTS overrides with active persona bindings", async () => { + const cfg = { + tts: { + enabled: true, + provider: "mock", + providers: { + mock: { + model: "base-model", + voice: "base-voice", + }, + }, + persona: "alfred", + personas: { + alfred: { + provider: "mock", + providers: { + mock: { + voice: "alfred-voice", + }, + }, + }, + jarvis: { + provider: "mock", + providers: { + mock: { + style: "jarvis-style", + }, + }, + }, + }, + }, + agents: { + list: [ + { + id: "reader", + tts: { + persona: "jarvis", + providers: { + mock: { + voice: "agent-voice", + }, + }, + }, + }, + ], + }, + } satisfies OpenClawConfig; + + let mediaDir: string | undefined; + try { + const result = await maybeApplyTtsToPayload({ + payload: { text: "This agent reply should use the composed persona config." }, + cfg, + channel: "slack", + kind: "final", + agentId: "reader", + }); + + expect(synthesizeMock).toHaveBeenCalled(); + const request = requireFirstSynthesisRequest("agent persona synthesis request"); + const providerConfig = requireRecord(request.providerConfig, "agent persona provider config"); + expect(providerConfig.model).toBe("base-model"); + expect(providerConfig.voice).toBe("agent-voice"); + expect(providerConfig.style).toBe("jarvis-style"); + expect(result.mediaUrl).toMatch(/voice---[a-f0-9-]+\.ogg$/); + mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; + } finally { + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } + }); + + it("ignores prototype-pollution keys in agent TTS overrides", () => { + const cfg = { + tts: { + provider: "openai", + providers: { + openai: { + voice: "coral", + }, + }, + }, + agents: { + list: [ + { + id: "reader", + tts: JSON.parse( + '{"providers":{"openai":{"voice":"nova","__proto__":{"polluted":true}}}}', + ), + }, + ], + }, + } as OpenClawConfig; + + const resolved = resolveTtsConfig(cfg, "reader"); + + expect(resolved.rawConfig?.providers?.openai).toEqual({ voice: "nova" }); + expect(({} as Record).polluted).toBeUndefined(); + }); +}); diff --git a/src/tts/tts-runtime-routing.test.ts b/src/tts/tts-runtime-routing.test.ts new file mode 100644 index 000000000000..18b74d854a59 --- /dev/null +++ b/src/tts/tts-runtime-routing.test.ts @@ -0,0 +1,447 @@ +import { rmSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + CODE_HEAVY_SPOKEN_FALLBACK, + MAX_TIMER_TIMEOUT_MS, + buildTtsSystemPromptHint, + clearRuntimeConfigSnapshot, + createMockSpeechProvider, + createTtsConfig, + expectTtsPayloadResult, + installSpeechProviders, + listSpeechVoices, + nativeVoiceNoteChannels, + prefsPathFor, + prepareSynthesisMock, + prepareTtsRequest, + requireFirstCallParam, + requireFirstSynthesisRequest, + requireRecord, + resolveTtsConfig, + resolveTtsPrefsPath, + setRuntimeConfigSnapshot, + setTtsMachinePrefsPathResolver, + synthesizeMock, + synthesizeSpeech, + testApi, + textToSpeech, + textToSpeechCore, + transcodeAudioBufferMock, + type OpenClawConfig, + type SpeechListVoicesRequest, + type TtsConfig, +} from "./tts-runtime.test-support.js"; + +describe("TTS runtime native voice-note routing", () => { + afterEach(() => { + setTtsMachinePrefsPathResolver(); + clearRuntimeConfigSnapshot(); + delete (Object.prototype as Record).polluted; + synthesizeMock.mockClear(); + prepareSynthesisMock.mockClear(); + transcodeAudioBufferMock.mockClear(); + installSpeechProviders([createMockSpeechProvider()]); + }); + + it("prefers the environment preference path over migrated machine state", () => { + const previousEnvPath = process.env.OPENCLAW_TTS_PREFS; + const envPath = prefsPathFor("env-override"); + setTtsMachinePrefsPathResolver(() => prefsPathFor("machine-state")); + process.env.OPENCLAW_TTS_PREFS = envPath; + try { + expect(resolveTtsPrefsPath(resolveTtsConfig({}))).toBe(envPath); + } finally { + if (previousEnvPath === undefined) { + delete process.env.OPENCLAW_TTS_PREFS; + } else { + process.env.OPENCLAW_TTS_PREFS = previousEnvPath; + } + } + }); + + it("resolves voice delivery support from channel capabilities", () => { + for (const channel of nativeVoiceNoteChannels) { + expect(testApi.supportsNativeVoiceNoteTts(channel)).toBe(true); + expect(testApi.supportsNativeVoiceNoteTts(channel.toUpperCase())).toBe(true); + } + expect(testApi.supportsNativeVoiceNoteTts("slack")).toBe(false); + expect(testApi.supportsNativeVoiceNoteTts(undefined)).toBe(false); + }); + + it("tells generic TTS guidance to defer to MEMORY voice-delivery instructions", () => { + const hint = buildTtsSystemPromptHint(createTtsConfig("openclaw-speech-core-tts-hint-test")); + + expect(hint).toContain("Voice (TTS) is enabled."); + expect(hint).toContain( + "If workspace context (especially MEMORY.md) tells you not to use [[tts:...]] or to use a local/non-tagged voice workflow, follow that workspace instruction instead.", + ); + expect(hint).toContain( + "Use [[tts:...]] and optional [[tts:text]]...[[/tts:text]] to control voice/expressiveness.", + ); + }); + + it("prepares deep-merged surface config and directive inputs", () => { + const cfg: OpenClawConfig = { + tts: { + provider: "mock", + modelOverrides: { allowProvider: false }, + providers: { + mock: { + model: "base-model", + voiceSettings: { stability: 0.4 }, + }, + }, + }, + }; + + const prepared = prepareTtsRequest({ + cfg, + override: { + modelOverrides: { allowProvider: true }, + providers: { + mock: { + voice: "surface-voice", + voiceSettings: { speed: 1.1 }, + }, + }, + }, + text: "Hello [[tts:text]]Speak this instead[[/tts:text]] caller", + }); + + expect(prepared.cfg).not.toBe(cfg); + expect(prepared.cfg.tts?.providers?.mock).toEqual({ + model: "base-model", + voice: "surface-voice", + voiceSettings: { stability: 0.4, speed: 1.1 }, + }); + expect(prepared.cfg.tts?.modelOverrides?.allowProvider).toBe(true); + expect(prepared.directives).toEqual({ + cleanedText: "Hello caller", + hasDirective: true, + overrides: { + ttsText: "Speak this instead", + }, + ttsText: "Speak this instead", + warnings: [], + }); + expect(cfg.tts?.providers?.mock).toEqual({ + model: "base-model", + voiceSettings: { stability: 0.4 }, + }); + }); + + it("sanitizes blocked override keys while preparing TTS config", () => { + const prepared = prepareTtsRequest({ + cfg: { + tts: { + provider: "mock", + providers: { mock: { model: "base-model" } }, + }, + }, + override: JSON.parse( + '{"__proto__":{"polluted":"top"},"providers":{"mock":{"voice":"safe","__proto__":{"polluted":"nested"}}}}', + ) as TtsConfig, + text: "[[tts:text]]Speak this instead[[/tts:text]]", + }); + + expect((Object.prototype as Record).polluted).toBeUndefined(); + expect(prepared.cfg.tts).not.toHaveProperty("polluted"); + expect(prepared.cfg.tts?.providers?.mock).toEqual({ + model: "base-model", + voice: "safe", + }); + expect(prepared.directives.cleanedText).toBe(""); + expect(prepared.directives.ttsText).toBe("Speak this instead"); + }); + + it("marks Discord auto TTS replies as native voice messages", async () => { + await expectTtsPayloadResult({ + channel: "discord", + prefsName: "openclaw-speech-core-tts-test", + text: "This Discord reply should be delivered as a native voice note.", + target: "voice-note", + audioAsVoice: true, + }); + }); + + it("keeps compatible audio-file synthesis deliverable as a voice memo", async () => { + await expectTtsPayloadResult({ + channel: "voice-memo-chat", + prefsName: "openclaw-speech-core-tts-voice-memo-mp3-test", + text: "This reply should be delivered as a native voice memo.", + target: "audio-file", + audioAsVoice: true, + mediaExtension: "mp3", + providerResult: { + audioBuffer: Buffer.from("mp3"), + outputFormat: "mp3", + fileExtension: ".mp3", + voiceCompatible: false, + }, + }); + }); + + it("does not mark unsupported audio-file output as a voice memo", async () => { + await expectTtsPayloadResult({ + channel: "voice-memo-chat", + prefsName: "openclaw-speech-core-tts-voice-memo-ogg-test", + text: "This reply should stay a regular audio attachment.", + target: "audio-file", + audioAsVoice: undefined, + }); + }); + + it("pre-transcodes synthesized mp3 to opus-in-CAF when the host can satisfy preferAudioFileFormat", async () => { + transcodeAudioBufferMock.mockResolvedValueOnce({ + ok: true, + buffer: Buffer.from("transcoded-caf"), + }); + await expectTtsPayloadResult({ + channel: "voice-memo-chat", + prefsName: "openclaw-speech-core-tts-voice-memo-caf-transcode-test", + text: "This reply should be pre-transcoded to a native voice-memo CAF.", + target: "audio-file", + audioAsVoice: true, + mediaExtension: "caf", + providerResult: { + audioBuffer: Buffer.from("mp3"), + outputFormat: "mp3", + fileExtension: ".mp3", + voiceCompatible: false, + }, + }); + expect(transcodeAudioBufferMock).toHaveBeenCalledOnce(); + const transcodeRequest = requireRecord( + requireFirstCallParam(transcodeAudioBufferMock.mock.calls as unknown[][], "transcode"), + "transcode request", + ); + expect(transcodeRequest.sourceExtension).toBe("mp3"); + expect(transcodeRequest.targetExtension).toBe("caf"); + }); + + it("falls back to the original mp3 buffer when the host transcoder fails", async () => { + transcodeAudioBufferMock.mockResolvedValueOnce({ + ok: false, + reason: "transcoder-failed", + detail: "exit-1", + }); + // Even though the transcode failed, the original mp3 still satisfies the + // channel audioFileFormats list, so the channel still flips audioAsVoice. + // The user gets a voice memo bubble, possibly with bad duration, instead + // of a regression. The failure is logged via the call site in tts.ts. + await expectTtsPayloadResult({ + channel: "voice-memo-chat", + prefsName: "openclaw-speech-core-tts-voice-memo-caf-fallback-test", + text: "This reply should fall back to the original mp3.", + target: "audio-file", + audioAsVoice: true, + mediaExtension: "mp3", + providerResult: { + audioBuffer: Buffer.from("mp3"), + outputFormat: "mp3", + fileExtension: ".mp3", + voiceCompatible: false, + }, + }); + }); + + it("uses the active runtime snapshot when source config still contains TTS SecretRefs", async () => { + const sourceConfig = { + tts: { + enabled: true, + provider: "mock", + providers: { + mock: { + apiKey: { source: "exec", provider: "mockexec", id: "minimax/tts/apiKey" }, + }, + }, + }, + } as unknown as OpenClawConfig; + const runtimeConfig = { + tts: { + enabled: true, + provider: "mock", + providers: { + mock: { + apiKey: "test-key", + }, + }, + }, + } as unknown as OpenClawConfig; + installSpeechProviders([ + createMockSpeechProvider("mock", { + isConfigured: ({ providerConfig }) => providerConfig.apiKey === "test-key", + resolveConfig: ({ rawConfig }) => { + const providers = rawConfig.providers as Record | undefined; + return providers?.mock ?? {}; + }, + }), + ]); + setRuntimeConfigSnapshot(runtimeConfig, sourceConfig); + + const result = await synthesizeSpeech({ + text: "Runtime snapshot TTS SecretRef", + cfg: sourceConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + expect(synthesizeMock).toHaveBeenCalled(); + const request = requireFirstSynthesisRequest("runtime snapshot synthesis request"); + expect(request.cfg).toBe(runtimeConfig); + const providerConfig = requireRecord(request.providerConfig, "provider config"); + expect(providerConfig.apiKey).toBe("test-key"); + }); + + it("uses provider default TTS timeout when the call and config omit timeoutMs", async () => { + installSpeechProviders([createMockSpeechProvider("mock", { defaultTimeoutMs: 600_000 })]); + + const result = await synthesizeSpeech({ + text: "Use provider timeout.", + cfg: { + tts: { + enabled: true, + provider: "mock", + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("provider default timeout synthesis request"); + expect(request.timeoutMs).toBe(600_000); + }); + + it("normalizes non-streaming synthesis text before calling the provider", async () => { + const result = await synthesizeSpeech({ + text: "## Update\n\nRead the [guide](https://example.com/guide)!!!!!", + cfg: createTtsConfig("openclaw-speech-core-talk-markdown-test"), + disableFallback: true, + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("normalized talk synthesis request"); + expect(request.text).toBe("Update\n\nRead the guide!"); + }); + + it("speaks stripped code through the explicit textToSpeech conversion path", async () => { + let mediaDir: string | undefined; + try { + const result = await textToSpeech({ + text: "```ts\nconst answer = 42;\n```", + cfg: createTtsConfig("openclaw-speech-core-code-convert-test"), + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("explicit code conversion request"); + expect(request.text).toBe("const answer = 42;"); + expect(request.text).not.toBe(CODE_HEAVY_SPOKEN_FALLBACK); + mediaDir = result.audioPath ? path.dirname(result.audioPath) : undefined; + } finally { + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } + }); + + it("returns a normal TTS failure when audio persistence rejects", async () => { + const result = await textToSpeechCore( + { + text: "Store this synthesized reply.", + cfg: createTtsConfig("openclaw-speech-core-persistence-failure-test"), + }, + async () => { + throw new Error("Media exceeds configured limit"); + }, + ); + + expect(result).toMatchObject({ + success: false, + error: "TTS audio persistence failed", + provider: "mock", + }); + }); + + it("resolves the configured timeout for voice listing", async () => { + const listVoicesMock = vi.fn(async (_request: SpeechListVoicesRequest) => []); + installSpeechProviders([ + createMockSpeechProvider("mock", { + defaultTimeoutMs: 60_000, + listVoices: listVoicesMock, + }), + ]); + + await listSpeechVoices({ + provider: "mock", + cfg: { + tts: { + enabled: true, + provider: "mock", + timeoutMs: 45_000, + }, + } as OpenClawConfig, + }); + + expect(listVoicesMock).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 45_000 })); + }); + + it("caps oversized provider default TTS timeouts before synthesis", async () => { + installSpeechProviders([ + createMockSpeechProvider("mock", { defaultTimeoutMs: Number.MAX_SAFE_INTEGER }), + ]); + + const result = await synthesizeSpeech({ + text: "Use capped provider timeout.", + cfg: { + tts: { + enabled: true, + provider: "mock", + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("provider default capped timeout request"); + expect(request.timeoutMs).toBe(MAX_TIMER_TIMEOUT_MS); + }); + + it("ignores nonpositive provider default TTS timeouts", async () => { + installSpeechProviders([createMockSpeechProvider("mock", { defaultTimeoutMs: 0 })]); + + const result = await synthesizeSpeech({ + text: "Use fallback timeout.", + cfg: { + tts: { + enabled: true, + provider: "mock", + }, + } as OpenClawConfig, + disableFallback: true, + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("provider default fallback timeout request"); + expect(request.timeoutMs).toBe(30_000); + }); + + it("keeps explicit TTS config timeout ahead of provider default timeout", async () => { + installSpeechProviders([createMockSpeechProvider("mock", { defaultTimeoutMs: 600_000 })]); + + await synthesizeSpeech({ + text: "Use configured timeout.", + cfg: { + tts: { + enabled: true, + provider: "mock", + timeoutMs: 45_000, + }, + } as OpenClawConfig, + disableFallback: true, + }); + + const request = requireFirstSynthesisRequest("configured timeout synthesis request"); + expect(request.timeoutMs).toBe(45_000); + }); +}); diff --git a/packages/speech-core/src/tts-types.ts b/src/tts/tts-runtime-types.ts similarity index 100% rename from packages/speech-core/src/tts-types.ts rename to src/tts/tts-runtime-types.ts diff --git a/src/tts/tts-runtime.test-support.ts b/src/tts/tts-runtime.test-support.ts new file mode 100644 index 000000000000..577e04d895c3 --- /dev/null +++ b/src/tts/tts-runtime.test-support.ts @@ -0,0 +1,297 @@ +// TTS runtime tests cover speech synthesis behavior. +import crypto from "node:crypto"; +import { mkdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { OpenClawConfig, TtsConfig } from "openclaw/plugin-sdk/config-contracts"; +import { MAX_TIMER_TIMEOUT_MS as MAX_TIMER_TIMEOUT_MS_CORE } from "openclaw/plugin-sdk/number-runtime"; +import type { ReplyPayload } from "openclaw/plugin-sdk/reply-payload"; +import { + clearRuntimeConfigSnapshot as clearRuntimeConfigSnapshotCore, + setRuntimeConfigSnapshot as setRuntimeConfigSnapshotCore, +} from "openclaw/plugin-sdk/runtime-config-snapshot"; +import type { + SpeechListVoicesRequest, + SpeechProviderPlugin, + SpeechProviderPrepareSynthesisContext, + SpeechSynthesisRequest, + SpeechTelephonySynthesisRequest, +} from "openclaw/plugin-sdk/speech-core"; +import { expect, vi } from "vitest"; +import { CODE_HEAVY_SPOKEN_FALLBACK as CODE_HEAVY_SPOKEN_FALLBACK_CORE } from "./speech-text.js"; +import type { TtsAudioPersistence } from "./tts-synthesis.js"; + +type MockSpeechSynthesisResult = Awaited>; + +const synthesizeMock = vi.hoisted(() => + vi.fn( + async (request: SpeechSynthesisRequest): Promise => ({ + audioBuffer: Buffer.from("voice"), + fileExtension: ".ogg", + outputFormat: "ogg", + voiceCompatible: request.target === "voice-note", + }), + ), +); +const prepareSynthesisMock = vi.hoisted(() => + vi.fn(async (_ctx: SpeechProviderPrepareSynthesisContext) => undefined), +); + +const listSpeechProvidersMock = vi.hoisted(() => vi.fn()); +const getSpeechProviderMock = vi.hoisted(() => vi.fn()); +const transcodeAudioBufferMock = vi.hoisted(() => + // Default off: most tests rely on the synthesized buffer reaching the + // channel unchanged. Tests that exercise the pre-transcode branch override + // per-call via `transcodeAudioBufferMock.mockResolvedValueOnce(...)`. + // Typed as the helper's full return shape so per-call overrides aren't + // narrowed to the default's literal. + vi.fn< + () => Promise< + | { ok: true; buffer: Buffer } + | { + ok: false; + reason: + | "platform-unsupported" + | "invalid-extension" + | "noop-same-container" + | "no-recipe" + | "transcoder-failed"; + detail?: string; + } + > + >(async () => ({ ok: false, reason: "platform-unsupported" })), +); + +vi.mock("../media/media-services.js", () => ({ + transcodeAudioBuffer: transcodeAudioBufferMock, +})); + +vi.mock("../channels/plugins/tts-capabilities.js", () => ({ + normalizeChannelId: (channel: string | undefined) => channel?.trim().toLowerCase() ?? null, + resolveChannelTtsVoiceDelivery: (channel: string | undefined) => { + const normalized = channel?.trim().toLowerCase(); + if (normalized === "voice-memo-chat") { + return { + synthesisTarget: "audio-file", + audioFileFormats: ["mp3", "caf", "audio/mpeg", "audio/x-caf"], + preferAudioFileFormat: "caf", + }; + } + if (normalized === "feishu" || normalized === "whatsapp") { + return { synthesisTarget: "voice-note", transcodesAudio: true }; + } + if (normalized === "discord" || normalized === "matrix" || normalized === "telegram") { + return { synthesisTarget: "voice-note" }; + } + return undefined; + }, +})); + +vi.mock("./provider-registry.js", async () => { + const actual = + await vi.importActual("./provider-registry.js"); + const mockProvider: SpeechProviderPlugin = { + id: "mock", + label: "Mock", + autoSelectOrder: 1, + isConfigured: () => true, + prepareSynthesis: prepareSynthesisMock, + synthesize: synthesizeMock, + }; + listSpeechProvidersMock.mockImplementation(() => [mockProvider]); + getSpeechProviderMock.mockImplementation((providerId: string) => + providerId === "mock" ? mockProvider : null, + ); + return { + ...actual, + canonicalizeSpeechProviderId: (providerId: string | undefined) => + providerId?.trim().toLowerCase() || undefined, + normalizeSpeechProviderId: (providerId: string | undefined) => + providerId?.trim().toLowerCase() || undefined, + getSpeechProvider: getSpeechProviderMock, + listSpeechProviders: listSpeechProvidersMock, + }; +}); + +vi.mock("./tts-core.js", async () => { + const actual = await vi.importActual("./tts-core.js"); + return { ...actual, scheduleCleanup: vi.fn() }; +}); + +export const { + testApi, + buildTtsSystemPromptHint, + getTtsPersona, + getTtsProvider, + isTtsProviderConfigured, + listSpeechVoices, + prepareTtsRequest, + resolveTtsConfig, + resolveTtsPrefsPath, + setTtsMachinePrefsPathResolver, + setSummarizationEnabled, + setTtsMaxLength, + synthesizeSpeech, + textToSpeechStream, + textToSpeechTelephony, +} = await import("./runtime-api.js"); +export const { maybeApplyTtsToPayload: maybeApplyTtsToPayloadCore } = + await import("./tts-payload.js"); +export const { textToSpeech: textToSpeechCore } = await import("./tts-synthesis.js"); + +export const CODE_HEAVY_SPOKEN_FALLBACK = CODE_HEAVY_SPOKEN_FALLBACK_CORE; +export const MAX_TIMER_TIMEOUT_MS = MAX_TIMER_TIMEOUT_MS_CORE; +export function clearRuntimeConfigSnapshot(): void { + clearRuntimeConfigSnapshotCore(); +} +export const setRuntimeConfigSnapshot = ( + ...args: Parameters +) => setRuntimeConfigSnapshotCore(...args); + +export const nativeVoiceNoteChannels = [ + "discord", + "feishu", + "matrix", + "telegram", + "whatsapp", +] as const; + +export function createMockSpeechProvider( + id = "mock", + options: Partial = {}, +): SpeechProviderPlugin { + return { + id, + label: id, + autoSelectOrder: id === "mock" ? 1 : 2, + isConfigured: () => true, + prepareSynthesis: prepareSynthesisMock, + synthesize: synthesizeMock, + ...options, + }; +} + +export function installSpeechProviders(providers: SpeechProviderPlugin[]): void { + listSpeechProvidersMock.mockImplementation(() => providers); + getSpeechProviderMock.mockImplementation( + (providerId: string) => providers.find((provider) => provider.id === providerId) ?? null, + ); +} + +// macOS os.tmpdir() is a /var -> /private/var symlink and fs-safe rejects +// symlinked store roots; resolve the canonical dir before writing prefs. +const PREFS_TMP_DIR = realpathSync(os.tmpdir()); + +async function persistTestTtsAudio({ + audioBuffer, + fileExtension, +}: Parameters[0]): Promise { + const dir = path.join(PREFS_TMP_DIR, `openclaw-speech-core-media-${crypto.randomUUID()}`); + mkdirSync(dir, { recursive: true }); + const audioPath = path.join(dir, `voice---${crypto.randomUUID()}${fileExtension}`); + writeFileSync(audioPath, audioBuffer); + return audioPath; +} + +export function textToSpeech(params: Parameters[0]) { + return textToSpeechCore(params, persistTestTtsAudio); +} + +export function maybeApplyTtsToPayload(params: Parameters[0]) { + return maybeApplyTtsToPayloadCore(params, persistTestTtsAudio); +} + +export function prefsPathFor(prefsName: string): string { + return path.join(PREFS_TMP_DIR, `${prefsName}.json`); +} + +export function createTtsConfig(prefsName: string): OpenClawConfig { + setTtsMachinePrefsPathResolver(() => prefsPathFor(prefsName)); + return { + tts: { + enabled: true, + provider: "mock", + }, + }; +} + +export function requireRecord(value: unknown, label: string): Record { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`expected ${label} to be a record`); + } + return value as Record; +} + +export function requireFirstCallParam(calls: ReadonlyArray, label: string) { + const call = calls[0]; + if (!call) { + throw new Error(`expected ${label} call`); + } + return call[0]; +} + +export function requireFirstSynthesisRequest(label: string): Record { + return requireRecord(requireFirstCallParam(synthesizeMock.mock.calls, label), label); +} + +export function requireAttempt(attempts: unknown[] | undefined, index: number) { + if (!attempts) { + throw new Error("expected synthesis attempts"); + } + return requireRecord(attempts[index], `synthesis attempt ${index}`); +} + +export async function expectTtsPayloadResult(params: { + channel: string; + prefsName: string; + text: string; + target: "voice-note" | "audio-file"; + audioAsVoice: true | undefined; + providerResult?: MockSpeechSynthesisResult; + mediaExtension?: string; + kind?: "tool" | "block" | "final"; +}) { + if (params.providerResult) { + synthesizeMock.mockResolvedValueOnce(params.providerResult); + } + const cfg = createTtsConfig(params.prefsName); + let mediaDir: string | undefined; + try { + const result = await maybeApplyTtsToPayload({ + payload: { text: params.text }, + cfg, + channel: params.channel, + kind: params.kind ?? "final", + }); + + expect(synthesizeMock).toHaveBeenCalled(); + const request = requireRecord( + synthesizeMock.mock.calls.at(-1)?.[0], + "latest synthesis request", + ); + expect(request.target).toBe(params.target); + expect(result.audioAsVoice).toBe(params.audioAsVoice); + expect(result.mediaUrl).toMatch( + new RegExp(`voice---[a-f0-9-]+\\.${params.mediaExtension ?? "ogg"}$`), + ); + expect(result.spokenText).toBe(params.text); + expect(result.ttsSupplement).toEqual({ spokenText: params.text }); + expect((result as { trustedLocalMedia?: boolean }).trustedLocalMedia).toBe(true); + + mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; + } finally { + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } +} + +export { prepareSynthesisMock, synthesizeMock, transcodeAudioBufferMock }; +export type { + OpenClawConfig, + ReplyPayload, + SpeechListVoicesRequest, + SpeechSynthesisRequest, + SpeechTelephonySynthesisRequest, + TtsConfig, +}; diff --git a/packages/speech-core/src/tts-settings-writes.ts b/src/tts/tts-settings-writes.ts similarity index 87% rename from packages/speech-core/src/tts-settings-writes.ts rename to src/tts/tts-settings-writes.ts index 01188251a771..41d93562b395 100644 --- a/packages/speech-core/src/tts-settings-writes.ts +++ b/src/tts/tts-settings-writes.ts @@ -1,8 +1,8 @@ // TTS preference mutations stay off the agent prompt's read-only import path. import path from "node:path"; -import type { TtsAutoMode, TtsProvider } from "openclaw/plugin-sdk/config-contracts"; -import { privateFileStoreSync } from "openclaw/plugin-sdk/security-runtime"; -import { canonicalizeSpeechProviderId } from "openclaw/plugin-sdk/speech-core"; +import type { TtsAutoMode, TtsProvider } from "../config/types.js"; +import { privateFileStoreSync } from "../infra/private-file-store.js"; +import { canonicalizeSpeechProviderId } from "./provider-registry.js"; import { normalizeTtsPersonaId, readTtsPrefs, type TtsUserPrefs } from "./tts-settings.js"; function updateTtsPrefs(prefsPath: string, update: (prefs: TtsUserPrefs) => void): void { diff --git a/src/tts/tts-settings.ts b/src/tts/tts-settings.ts index a30c9c550262..b2b93f03bfb5 100644 --- a/src/tts/tts-settings.ts +++ b/src/tts/tts-settings.ts @@ -1,5 +1,397 @@ -// Lightweight core facade for TTS settings used by agent and status hot paths. -export { - buildTtsSystemPromptHint, - resolveTtsSettingsSnapshot, -} from "../../packages/speech-core/src/tts-settings.js"; +// Lightweight TTS settings resolution shared by agent prompts, status, and speech runtime. +import { existsSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { + normalizeOptionalLowercaseString, + normalizeOptionalString, +} from "../../packages/normalization-core/src/string-coerce.js"; +import { + getRuntimeConfigSnapshot, + getRuntimeConfigSourceSnapshot, + selectApplicableRuntimeConfig, +} from "../config/runtime-snapshot.js"; +import type { + OpenClawConfig, + ResolvedTtsPersona, + TtsAutoMode, + TtsConfig, + TtsModelOverrideConfig, + TtsProvider, +} from "../config/types.js"; +import { resolveConfigDir, resolveUserPath } from "../utils.js"; +import { normalizeSpeechProviderId } from "./provider-registry-core.js"; +import type { SpeechProviderConfig } from "./provider-types.js"; +import { withSpeakerSelectionCompat } from "./speaker.js"; +import { normalizeTtsAutoMode } from "./tts-auto-mode.js"; +import { resolveEffectiveTtsConfig, type TtsConfigResolutionContext } from "./tts-config.js"; +import type { ResolvedTtsConfig, ResolvedTtsModelOverrides } from "./tts-types.js"; + +export type { ResolvedTtsConfig, ResolvedTtsModelOverrides }; + +export const DEFAULT_TTS_TIMEOUT_MS = 30_000; +const DEFAULT_TTS_MAX_LENGTH = 1500; +const DEFAULT_TTS_SUMMARIZE = true; +const DEFAULT_MAX_TEXT_LENGTH = 4096; +let machinePrefsPathResolver: () => string | undefined = () => undefined; + +export function setTtsMachinePrefsPathResolver(resolver?: () => string | undefined): void { + machinePrefsPathResolver = resolver ?? (() => undefined); +} + +export type TtsUserPrefs = { + tts?: { + auto?: TtsAutoMode; + enabled?: boolean; + provider?: TtsProvider; + persona?: string | null; + maxLength?: number; + summarize?: boolean; + }; +}; + +function resolveConfiguredTtsAutoMode(raw: TtsConfig): TtsAutoMode { + return normalizeTtsAutoMode(raw.auto) ?? (raw.enabled ? "always" : "off"); +} + +export function normalizeConfiguredSpeechProviderId( + providerId: string | undefined, +): TtsProvider | undefined { + const normalized = normalizeSpeechProviderId(providerId); + if (!normalized) { + return undefined; + } + return normalized === "edge" ? "microsoft" : normalized; +} + +export function normalizeTtsPersonaId(personaId: string | null | undefined): string | undefined { + return normalizeOptionalLowercaseString(personaId ?? undefined); +} + +function resolveTtsPrefsPathValue(prefsPath: string | undefined): string { + // Scoped agent paths must win over the migrated machine-wide default. + if (prefsPath?.trim()) { + return resolveUserPath(prefsPath.trim()); + } + const envPath = process.env.OPENCLAW_TTS_PREFS?.trim(); + if (envPath) { + return resolveUserPath(envPath); + } + const machinePath = machinePrefsPathResolver()?.trim(); + if (machinePath) { + return resolveUserPath(machinePath); + } + return path.join(resolveConfigDir(process.env), "settings", "tts.json"); +} + +export function resolveModelOverridePolicy( + overrides: TtsModelOverrideConfig | undefined, +): ResolvedTtsModelOverrides { + const enabled = overrides?.enabled ?? true; + if (!enabled) { + return { + enabled: false, + allowText: false, + allowProvider: false, + allowVoice: false, + allowModelId: false, + allowVoiceSettings: false, + allowNormalization: false, + allowSeed: false, + }; + } + const allow = (value: boolean | undefined, defaultValue = true) => value ?? defaultValue; + return { + enabled: true, + allowText: allow(overrides?.allowText), + allowProvider: allow(overrides?.allowProvider, false), + allowVoice: allow(overrides?.allowVoice), + allowModelId: allow(overrides?.allowModelId), + allowVoiceSettings: allow(overrides?.allowVoiceSettings), + allowNormalization: allow(overrides?.allowNormalization), + allowSeed: allow(overrides?.allowSeed), + }; +} + +export function resolveTtsRuntimeConfig(cfg: OpenClawConfig): OpenClawConfig { + return ( + selectApplicableRuntimeConfig({ + inputConfig: cfg, + runtimeConfig: getRuntimeConfigSnapshot(), + runtimeSourceConfig: getRuntimeConfigSourceSnapshot(), + }) ?? cfg + ); +} + +export function asProviderConfig(value: unknown): SpeechProviderConfig { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? withSpeakerSelectionCompat(value as SpeechProviderConfig) + : {}; +} + +export function asProviderConfigMap(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : {}; +} + +export function hasOwnProperty(value: object, key: string): boolean { + return Object.hasOwn(value, key); +} + +function normalizeProviderConfigMap( + value: unknown, +): Record | undefined { + const rawMap = asProviderConfigMap(value); + if (Object.keys(rawMap).length === 0) { + return undefined; + } + const next: Record = {}; + for (const [providerId, providerConfig] of Object.entries(rawMap)) { + const normalized = normalizeConfiguredSpeechProviderId(providerId) ?? providerId; + next[normalized] = asProviderConfig(providerConfig); + } + return next; +} + +function collectTtsPersonas(raw: TtsConfig): Record { + const rawPersonas = asProviderConfigMap(raw.personas); + const personas: Record = {}; + for (const [id, value] of Object.entries(rawPersonas)) { + const normalizedId = normalizeTtsPersonaId(id); + if (!normalizedId || typeof value !== "object" || value === null || Array.isArray(value)) { + continue; + } + const persona = value as Omit; + personas[normalizedId] = { + ...persona, + id: normalizedId, + provider: normalizeConfiguredSpeechProviderId(persona.provider) ?? persona.provider, + providers: normalizeProviderConfigMap(persona.providers), + }; + } + return personas; +} + +function collectDirectProviderConfigEntries(raw: TtsConfig): Record { + const entries: Record = {}; + const rawProviders = asProviderConfigMap(raw.providers); + for (const [providerId, value] of Object.entries(rawProviders)) { + const normalized = normalizeConfiguredSpeechProviderId(providerId) ?? providerId; + entries[normalized] = asProviderConfig(value); + } + const reservedKeys = new Set([ + "auto", + "enabled", + "maxTextLength", + "mode", + "modelOverrides", + "persona", + "personas", + "prefsPath", + "provider", + "providers", + "summaryModel", + "timeoutMs", + ]); + for (const [key, value] of Object.entries(raw as Record)) { + if (reservedKeys.has(key)) { + continue; + } + if (typeof value !== "object" || value === null || Array.isArray(value)) { + continue; + } + const normalized = normalizeConfiguredSpeechProviderId(key) ?? key; + entries[normalized] ??= asProviderConfig(value); + } + return entries; +} + +export function resolveTtsConfig( + cfgInput: OpenClawConfig, + contextOrAgentId?: string | TtsConfigResolutionContext, +): ResolvedTtsConfig { + const cfg = resolveTtsRuntimeConfig(cfgInput); + const raw: TtsConfig = resolveEffectiveTtsConfig(cfg, contextOrAgentId); + const providerSource = raw.provider ? "config" : "default"; + const timeoutMs = raw.timeoutMs ?? DEFAULT_TTS_TIMEOUT_MS; + const timeoutMsSource = raw.timeoutMs === undefined ? "default" : "config"; + return { + auto: resolveConfiguredTtsAutoMode(raw), + mode: raw.mode ?? "final", + provider: + normalizeConfiguredSpeechProviderId(raw.provider) ?? + (providerSource === "config" ? (normalizeOptionalLowercaseString(raw.provider) ?? "") : ""), + providerSource, + persona: normalizeTtsPersonaId(raw.persona), + personas: collectTtsPersonas(raw), + summaryModel: normalizeOptionalString(raw.summaryModel), + modelOverrides: resolveModelOverridePolicy(raw.modelOverrides), + providerConfigs: collectDirectProviderConfigEntries(raw), + prefsPath: (raw as TtsConfig & { prefsPath?: string }).prefsPath, + maxTextLength: raw.maxTextLength ?? DEFAULT_MAX_TEXT_LENGTH, + timeoutMs, + timeoutMsSource, + rawConfig: raw, + sourceConfig: cfg, + }; +} + +export function resolveTtsPrefsPath(config: ResolvedTtsConfig): string { + return resolveTtsPrefsPathValue(config.prefsPath); +} + +export function readTtsPrefs(prefsPath: string): TtsUserPrefs { + try { + if (!existsSync(prefsPath)) { + return {}; + } + const parsed: unknown = JSON.parse(readFileSync(prefsPath, "utf8")); + return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as TtsUserPrefs) + : {}; + } catch { + return {}; + } +} + +function resolveTtsAutoModeFromPrefs(prefs: TtsUserPrefs): TtsAutoMode | undefined { + const auto = normalizeTtsAutoMode(prefs.tts?.auto); + if (auto) { + return auto; + } + if (typeof prefs.tts?.enabled === "boolean") { + return prefs.tts.enabled ? "always" : "off"; + } + return undefined; +} + +export function resolveTtsAutoMode(params: { + config: ResolvedTtsConfig; + prefsPath: string; + sessionAuto?: string; +}): TtsAutoMode { + const sessionAuto = normalizeTtsAutoMode(params.sessionAuto); + if (sessionAuto) { + return sessionAuto; + } + return resolveTtsAutoModeFromPrefs(readTtsPrefs(params.prefsPath)) ?? params.config.auto; +} + +function resolveTtsPersonaIdFromPrefs( + config: ResolvedTtsConfig, + prefs: TtsUserPrefs, +): string | undefined { + if (prefs.tts && hasOwnProperty(prefs.tts, "persona")) { + return normalizeTtsPersonaId(prefs.tts.persona); + } + return normalizeTtsPersonaId(config.persona); +} + +export function resolveTtsPersonaFromPrefs( + config: ResolvedTtsConfig, + prefs: TtsUserPrefs, +): ResolvedTtsPersona | undefined { + const personaId = resolveTtsPersonaIdFromPrefs(config, prefs); + return personaId ? config.personas[personaId] : undefined; +} + +type ResolvedTtsSettingsSnapshot = { + autoMode: TtsAutoMode; + config: ResolvedTtsConfig; + maxLength: number; + persona?: ResolvedTtsPersona; + personaId?: string; + preferredProvider?: TtsProvider; + prefsPath: string; + summarize: boolean; +}; + +export function resolveTtsSettingsSnapshot(params: { + cfg: OpenClawConfig; + sessionAuto?: string; + agentId?: string; + channelId?: string; + accountId?: string; +}): ResolvedTtsSettingsSnapshot { + const config = resolveTtsConfig(params.cfg, { + agentId: params.agentId, + channelId: params.channelId, + accountId: params.accountId, + }); + const prefsPath = resolveTtsPrefsPath(config); + const prefs = readTtsPrefs(prefsPath); + const personaId = resolveTtsPersonaIdFromPrefs(config, prefs); + const persona = personaId ? config.personas[personaId] : undefined; + const preferredProvider = + normalizeConfiguredSpeechProviderId(prefs.tts?.provider) ?? + normalizeConfiguredSpeechProviderId(persona?.provider) ?? + (config.providerSource === "config" + ? (normalizeConfiguredSpeechProviderId(config.provider) ?? config.provider) + : undefined); + return { + autoMode: + normalizeTtsAutoMode(params.sessionAuto) ?? resolveTtsAutoModeFromPrefs(prefs) ?? config.auto, + config, + maxLength: prefs.tts?.maxLength ?? DEFAULT_TTS_MAX_LENGTH, + ...(persona ? { persona } : {}), + ...(personaId ? { personaId } : {}), + ...(preferredProvider ? { preferredProvider } : {}), + prefsPath, + summarize: prefs.tts?.summarize ?? DEFAULT_TTS_SUMMARIZE, + }; +} + +export function buildTtsSystemPromptHint( + cfg: OpenClawConfig, + agentId?: string, +): string | undefined { + const settings = resolveTtsSettingsSnapshot({ cfg, agentId }); + if (settings.autoMode === "off") { + return undefined; + } + const autoHint = + settings.autoMode === "inbound" + ? "Only use TTS when the user's last message includes audio/voice." + : settings.autoMode === "tagged" + ? "Only use TTS when you include [[tts:key=value]] directives or a [[tts:text]]...[[/tts:text]] block." + : undefined; + return [ + "Voice (TTS) is enabled.", + autoHint, + settings.persona + ? `Active TTS persona: ${settings.persona.label ?? settings.persona.id}${settings.persona.description ? ` - ${settings.persona.description}` : ""}.` + : undefined, + `Keep spoken text ≤${settings.maxLength} chars to avoid auto-summary (summary ${settings.summarize ? "on" : "off"}).`, + "If workspace context (especially MEMORY.md) tells you not to use [[tts:...]] or to use a local/non-tagged voice workflow, follow that workspace instruction instead.", + "Use [[tts:...]] and optional [[tts:text]]...[[/tts:text]] to control voice/expressiveness.", + ] + .filter(Boolean) + .join("\n"); +} + +export function isTtsEnabled( + config: ResolvedTtsConfig, + prefsPath: string, + sessionAuto?: string, +): boolean { + return resolveTtsAutoMode({ config, prefsPath, sessionAuto }) !== "off"; +} + +export function getTtsPersona( + config: ResolvedTtsConfig, + prefsPath: string, +): ResolvedTtsPersona | undefined { + return resolveTtsPersonaFromPrefs(config, readTtsPrefs(prefsPath)); +} + +export function listTtsPersonas(config: ResolvedTtsConfig): ResolvedTtsPersona[] { + return Object.values(config.personas).toSorted((left, right) => left.id.localeCompare(right.id)); +} + +export function getTtsMaxLength(prefsPath: string): number { + return readTtsPrefs(prefsPath).tts?.maxLength ?? DEFAULT_TTS_MAX_LENGTH; +} + +export function isSummarizationEnabled(prefsPath: string): boolean { + return readTtsPrefs(prefsPath).tts?.summarize ?? DEFAULT_TTS_SUMMARIZE; +} diff --git a/packages/speech-core/src/tts-streaming.ts b/src/tts/tts-streaming.ts similarity index 94% rename from packages/speech-core/src/tts-streaming.ts rename to src/tts/tts-streaming.ts index ad8092680f61..2b3613f3d5ec 100644 --- a/packages/speech-core/src/tts-streaming.ts +++ b/src/tts/tts-streaming.ts @@ -1,9 +1,9 @@ -import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import type { TtsDirectiveOverrides } from "openclaw/plugin-sdk/speech-core"; +import type { OpenClawConfig } from "../config/types.js"; +import type { TtsDirectiveOverrides } from "./provider-types.js"; import { assertSpeechRuntimeAvailable } from "./runtime-availability.js"; +import type { TtsStreamResult, TtsSynthesisStreamResult } from "./tts-runtime-types.js"; import { executeTtsProviderAttempts, resolveTtsRequestSetup } from "./tts-synthesis-support.js"; import { resolveTtsSynthesisTarget } from "./tts-synthesis.js"; -import type { TtsStreamResult, TtsSynthesisStreamResult } from "./tts-types.js"; export async function streamSpeech(params: { text: string; diff --git a/packages/speech-core/src/tts-synthesis-support.ts b/src/tts/tts-synthesis-support.ts similarity index 95% rename from packages/speech-core/src/tts-synthesis-support.ts rename to src/tts/tts-synthesis-support.ts index 5095db7b7c3d..079856db39b0 100644 --- a/packages/speech-core/src/tts-synthesis-support.ts +++ b/src/tts/tts-synthesis-support.ts @@ -1,18 +1,9 @@ -import type { - OpenClawConfig, - ResolvedTtsPersona, - TtsProvider, -} from "openclaw/plugin-sdk/config-contracts"; -import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; -import { redactSensitiveText } from "openclaw/plugin-sdk/logging-core"; -import { logVerbose } from "openclaw/plugin-sdk/runtime-env"; -import { - canonicalizeSpeechProviderId, - getSpeechProvider, - type SpeechProviderConfig, - type SpeechProviderOverrides, -} from "openclaw/plugin-sdk/speech-core"; -import type { VoiceModelRef, VoiceProviderCandidate } from "../voice-models.js"; +import type { OpenClawConfig, ResolvedTtsPersona, TtsProvider } from "../config/types.js"; +import { logVerbose } from "../globals.js"; +import { formatErrorMessage } from "../infra/errors.js"; +import { redactSensitiveText } from "../logging/redact.js"; +import { canonicalizeSpeechProviderId, getSpeechProvider } from "./provider-registry.js"; +import type { SpeechProviderConfig, SpeechProviderOverrides } from "./provider-types.js"; import { getResolvedSpeechProviderConfigForVoiceModel, mergeProviderConfigWithPersona, @@ -22,6 +13,7 @@ import { resolveTtsProvider, resolveTtsProviderCandidates, } from "./tts-provider-resolution.js"; +import type { TtsProviderAttempt } from "./tts-runtime-types.js"; import { getTtsPersona, resolveTtsConfig, @@ -29,7 +21,7 @@ import { resolveTtsRuntimeConfig, type ResolvedTtsConfig, } from "./tts-settings.js"; -import type { TtsProviderAttempt } from "./tts-types.js"; +import type { VoiceModelRef, VoiceProviderCandidate } from "./voice-models.js"; export function formatTtsProviderError(provider: TtsProvider, err: unknown): string { const error = err instanceof Error ? err : new Error(String(err)); diff --git a/packages/speech-core/src/tts-synthesis.ts b/src/tts/tts-synthesis.ts similarity index 94% rename from packages/speech-core/src/tts-synthesis.ts rename to src/tts/tts-synthesis.ts index 659e8cfa131c..57d72f41bfd8 100644 --- a/packages/speech-core/src/tts-synthesis.ts +++ b/src/tts/tts-synthesis.ts @@ -1,16 +1,16 @@ -import { resolveChannelTtsVoiceDelivery } from "openclaw/plugin-sdk/channel-targets"; -import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import { transcodeAudioBuffer } from "openclaw/plugin-sdk/media-runtime"; -import { logVerbose } from "openclaw/plugin-sdk/runtime-env"; -import type { TtsDirectiveOverrides } from "openclaw/plugin-sdk/speech-core"; +import { resolveChannelTtsVoiceDelivery } from "../channels/plugins/tts-capabilities.js"; +import type { OpenClawConfig } from "../config/types.js"; +import { logVerbose } from "../globals.js"; +import { transcodeAudioBuffer } from "../media/media-services.js"; +import type { TtsDirectiveOverrides } from "./provider-types.js"; import { assertSpeechRuntimeAvailable } from "./runtime-availability.js"; import { normalizeSpeechText } from "./speech-text.js"; +import type { TtsResult, TtsSynthesisResult } from "./tts-runtime-types.js"; import { executeTtsProviderAttempts, resolveTtsRequestSetup, sanitizeTtsErrorForLog, } from "./tts-synthesis-support.js"; -import type { TtsResult, TtsSynthesisResult } from "./tts-types.js"; export type TtsAudioPersistence = (params: { audioBuffer: Buffer; diff --git a/packages/speech-core/src/tts-telephony.ts b/src/tts/tts-telephony.ts similarity index 89% rename from packages/speech-core/src/tts-telephony.ts rename to src/tts/tts-telephony.ts index 76e00fe937fc..3441b2ea147e 100644 --- a/packages/speech-core/src/tts-telephony.ts +++ b/src/tts/tts-telephony.ts @@ -1,8 +1,8 @@ -import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import type { TtsDirectiveOverrides } from "openclaw/plugin-sdk/speech-core"; +import type { OpenClawConfig } from "../config/types.js"; +import type { TtsDirectiveOverrides } from "./provider-types.js"; import { assertSpeechRuntimeAvailable } from "./runtime-availability.js"; +import type { TtsTelephonyResult } from "./tts-runtime-types.js"; import { executeTtsProviderAttempts, resolveTtsRequestSetup } from "./tts-synthesis-support.js"; -import type { TtsTelephonyResult } from "./tts-types.js"; export async function textToSpeechTelephony(params: { text: string; diff --git a/src/tts/tts.test.ts b/src/tts/tts.test.ts index b9e27841d946..b1bf6f510a46 100644 --- a/src/tts/tts.test.ts +++ b/src/tts/tts.test.ts @@ -1,41 +1,12 @@ -// TTS integration tests cover text-to-speech command behavior. -import { readFileSync } from "node:fs"; +// TTS integration tests cover host runtime availability behavior. import { afterEach, describe, expect, it } from "vitest"; import { setActiveDegradedSecretOwners } from "../secrets/runtime-degraded-state.js"; -function readSource(relativePath: string): string { - return readFileSync(new URL(relativePath, import.meta.url), "utf8"); -} - describe("tts runtime facade", () => { afterEach(() => { setActiveDegradedSecretOwners([]); }); - it("routes public TTS helpers through the core speech package", () => { - const publicFacadeSource = readSource("./tts.ts"); - const runtimeFacadeSource = readSource("../plugin-sdk/tts-runtime.ts"); - - expect(publicFacadeSource).toContain('} from "../plugin-sdk/tts-runtime.js";'); - expect(publicFacadeSource).toContain("setSpeechRuntimeAvailabilityGuard"); - expect(runtimeFacadeSource).toContain('from "../../packages/speech-core/runtime-api.js";'); - expect(runtimeFacadeSource).not.toContain('dirName: "speech-core"'); - }); - - it("keeps agent prompt TTS settings off the synthesis runtime chain", () => { - const agentConfigSource = readSource("../agents/system-prompt-config.ts"); - const settingsFacadeSource = readSource("./tts-settings.ts"); - const packageSettingsSource = readSource("../../packages/speech-core/src/tts-settings.ts"); - - expect(agentConfigSource).toContain('from "../tts/tts-settings.js";'); - expect(settingsFacadeSource).toContain( - 'from "../../packages/speech-core/src/tts-settings.js";', - ); - expect(settingsFacadeSource).not.toContain("tts-runtime"); - expect(packageSettingsSource).toContain('from "openclaw/plugin-sdk/speech-settings";'); - expect(packageSettingsSource).not.toContain("plugin-sdk/media-runtime"); - }); - it("blocks explicit synthesis but preserves text delivery when TTS is cold", async () => { setActiveDegradedSecretOwners([ { diff --git a/src/tts/tts.ts b/src/tts/tts.ts index face3ad7dd58..930ed326435c 100644 --- a/src/tts/tts.ts +++ b/src/tts/tts.ts @@ -1,10 +1,13 @@ /** Public TTS runtime barrel exposed to core callers and plugin SDK facades. */ +import { assertSecretOwnerAvailable } from "../secrets/runtime-degraded-state.js"; +import { readConfigMachineState } from "../state/config-machine-state.js"; import { setSpeechRuntimeAvailabilityGuard, setTtsMachinePrefsPathResolver, -} from "../../packages/speech-core/runtime-api.js"; -import { assertSecretOwnerAvailable } from "../secrets/runtime-degraded-state.js"; -import { readConfigMachineState } from "../state/config-machine-state.js"; +} from "./runtime-api.js"; +import { persistTtsAudioToMediaStore } from "./tts-audio-store.js"; +import { maybeApplyTtsToPayload as maybeApplyTtsToPayloadCore } from "./tts-payload.js"; +import { textToSpeech as textToSpeechCore } from "./tts-synthesis.js"; setSpeechRuntimeAvailabilityGuard(() => { assertSecretOwnerAvailable("capability", "tts"); @@ -12,6 +15,14 @@ setSpeechRuntimeAvailabilityGuard(() => { setTtsMachinePrefsPathResolver(() => readConfigMachineState("tts.prefsPath")); +export function textToSpeech(params: Parameters[0]) { + return textToSpeechCore(params, persistTtsAudioToMediaStore); +} + +export function maybeApplyTtsToPayload(params: Parameters[0]) { + return maybeApplyTtsToPayloadCore(params, persistTtsAudioToMediaStore); +} + export { getLastTtsAttempt, getResolvedSpeechProviderConfig, @@ -23,7 +34,6 @@ export { isTtsProviderConfigured, listSpeechVoices, listTtsPersonas, - maybeApplyTtsToPayload, resolveExplicitTtsOverrides, resolveTtsAutoMode, resolveTtsConfig, @@ -36,7 +46,6 @@ export { setTtsPersona, setTtsProvider, synthesizeSpeech, - textToSpeech, type ResolvedTtsConfig, type TtsDirectiveOverrides, -} from "../plugin-sdk/tts-runtime.js"; +} from "./runtime-api.js"; diff --git a/packages/speech-core/voice-models.ts b/src/tts/voice-models.ts similarity index 98% rename from packages/speech-core/voice-models.ts rename to src/tts/voice-models.ts index d998c5100607..f01dd1f76e9d 100644 --- a/packages/speech-core/voice-models.ts +++ b/src/tts/voice-models.ts @@ -1,7 +1,7 @@ // Voice model catalog helpers shared by TTS and realtime voice plugins. import { parseModelCatalogRef } from "@openclaw/model-catalog-core/model-catalog-refs"; -export type VoiceModelCapability = "tts" | "realtime_transcription" | "realtime_voice"; +type VoiceModelCapability = "tts" | "realtime_transcription" | "realtime_voice"; /** Capability flags advertised by a voice model catalog entry. */ export type VoiceModelCapabilities = Partial>; @@ -23,7 +23,7 @@ export type VoiceModelProvider = { }; /** Synthesized voice model catalog row exposed to provider/model selection. */ -export type VoiceModelCatalogEntry = { +type VoiceModelCatalogEntry = { kind: "voice"; provider: string; model: string; diff --git a/test/e2e/qa-lab/runtime/media-talk-gateway.ts b/test/e2e/qa-lab/runtime/media-talk-gateway.ts index 3085256c3636..b27c2ef94fbc 100644 --- a/test/e2e/qa-lab/runtime/media-talk-gateway.ts +++ b/test/e2e/qa-lab/runtime/media-talk-gateway.ts @@ -50,7 +50,7 @@ const SCENARIOS = { docsRefs: ["docs/tools/tts.md", "docs/tools/media-overview.md"], codeRefs: [ SOURCE_PATH, - "packages/speech-core/src/tts.ts", + "src/tts/runtime-api.ts", "src/gateway/managed-image-attachments.ts", "src/gateway/server-methods/artifacts.ts", ], diff --git a/test/vitest-scoped-config.test.ts b/test/vitest-scoped-config.test.ts index 687a121b703b..a674dc84e047 100644 --- a/test/vitest-scoped-config.test.ts +++ b/test/vitest-scoped-config.test.ts @@ -247,13 +247,13 @@ describe("createScopedVitestConfig", () => { it("keeps broad package scoped cli directory filters aligned with repo-root include patterns", () => { const config = createScopedVitestConfig(["packages/**/*.test.ts"], { - argv: ["vitest", "run", "packages/speech-core"], + argv: ["vitest", "run", "packages/normalization-core"], dir: "packages", env: {}, passWithNoTests: true, }); - expect(requireTestConfig(config).include).toEqual(["speech-core/**/*.test.*"]); + expect(requireTestConfig(config).include).toEqual(["normalization-core/**/*.test.*"]); }); it("relativizes scoped include and exclude patterns to the configured dir", () => { diff --git a/tsconfig.json b/tsconfig.json index c59dae092ee1..a42c3a7ee9a6 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -250,11 +250,6 @@ "@openclaw/net-policy/url-protocol": ["./packages/net-policy/src/url-protocol.ts"], "@openclaw/net-policy/url-userinfo": ["./packages/net-policy/src/url-userinfo.ts"], "@openclaw/net-policy/*": ["./packages/net-policy/src/*"], - "@openclaw/speech-core": ["./packages/speech-core/runtime-api.ts"], - "@openclaw/speech-core/runtime-api": ["./packages/speech-core/runtime-api.ts"], - "@openclaw/speech-core/speaker": ["./packages/speech-core/speaker.ts"], - "@openclaw/speech-core/voice-models": ["./packages/speech-core/voice-models.ts"], - "@openclaw/speech-core/*": ["./packages/speech-core/*"], "@openclaw/sdk": ["./packages/sdk/src/index.ts"], "@openclaw/plugin-sdk/*": ["./src/plugin-sdk/*.ts"], "openclaw/plugin-sdk/account-id": ["./src/plugin-sdk/account-id.ts"], diff --git a/tsdown.config.ts b/tsdown.config.ts index 2dcafbe5956d..303e18e0acb6 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -410,14 +410,6 @@ function buildPackageDistEntriesFromExports(packageDir: string): Record a.localeCompare(b))); } -function buildSpeechCoreDistEntries(): Record { - return { - "runtime-api": "packages/speech-core/runtime-api.ts", - speaker: "packages/speech-core/speaker.ts", - "voice-models": "packages/speech-core/voice-models.ts", - }; -} - function buildLlmCoreDistEntries(): Record { return { index: "packages/llm-core/src/index.ts", @@ -458,10 +450,6 @@ function shouldExternalizeNetPolicyDependency(id: string): boolean { return id === "ipaddr.js" || id.startsWith("ipaddr.js/"); } -function shouldExternalizeSpeechCoreDependency(id: string): boolean { - return id === "openclaw" || id.startsWith("openclaw/"); -} - function shouldExternalizeLlmCoreDependency(id: string): boolean { return id === "typebox" || id.startsWith("typebox/"); } @@ -665,12 +653,6 @@ const configs = [ neverBundle: shouldExternalizeTerminalCoreDependency, }, }), - nodeWorkspacePackageBuildConfig("speech-core", { - entry: buildSpeechCoreDistEntries(), - deps: { - neverBundle: shouldExternalizeSpeechCoreDependency, - }, - }), nodeWorkspacePackageBuildConfig("llm-core", { entry: buildLlmCoreDistEntries(), deps: {