From 5219c9353dc8a256e33224f01ddbb06e078edcb8 Mon Sep 17 00:00:00 2001 From: Alix-007 Date: Sat, 11 Jul 2026 15:55:06 +0800 Subject: [PATCH] fix(tts): bound voice-list requests (#102865) * fix(microsoft): add timeout to voices list request * fix(tts): bound voice-list requests Co-authored-by: llagy009 <0668001470@xydigit.com> --------- Co-authored-by: llagy009 <0668001470@xydigit.com> Co-authored-by: Peter Steinberger --- docs/plugins/architecture-internals.md | 1 + .../azure-speech/speech-provider.test.ts | 5 +-- extensions/azure-speech/speech-provider.ts | 2 +- extensions/elevenlabs/speech-provider.test.ts | 34 ++++++++++++++----- extensions/elevenlabs/speech-provider.ts | 3 ++ extensions/inworld/speech-provider.test.ts | 13 +++++++ extensions/inworld/speech-provider.ts | 1 + extensions/microsoft/speech-provider.test.ts | 32 +++++++++++++++++ extensions/microsoft/speech-provider.ts | 11 ++++-- packages/speech-core/src/tts.test.ts | 27 +++++++++++++++ packages/speech-core/src/tts.ts | 5 +++ src/tts/provider-types.ts | 2 ++ 12 files changed, 123 insertions(+), 13 deletions(-) diff --git a/docs/plugins/architecture-internals.md b/docs/plugins/architecture-internals.md index 071f6566573a..bb7bb68a33a8 100644 --- a/docs/plugins/architecture-internals.md +++ b/docs/plugins/architecture-internals.md @@ -455,6 +455,7 @@ Notes: - Uses core `messages.tts` configuration and provider selection. - Returns PCM audio buffer + sample rate. Plugins must resample/encode for providers. - `listVoices` is optional per provider. Use it for vendor-owned voice pickers or setup flows. +- Core passes a resolved request deadline to provider `listVoices` hooks; provider-specific timeout settings may override it. - Voice listings can include richer metadata such as locale, gender, and personality tags for provider-aware pickers. - OpenAI and ElevenLabs support telephony today. Microsoft does not. diff --git a/extensions/azure-speech/speech-provider.test.ts b/extensions/azure-speech/speech-provider.test.ts index cd750319252e..fc847a171ec3 100644 --- a/extensions/azure-speech/speech-provider.test.ts +++ b/extensions/azure-speech/speech-provider.test.ts @@ -252,7 +252,8 @@ describe("buildAzureSpeechProvider", () => { it("lists voices through config or explicit request auth", async () => { const provider = buildAzureSpeechProvider(); const voices = await provider.listVoices?.({ - providerConfig: { apiKey: "key", region: "eastus" }, + providerConfig: { apiKey: "key", region: "eastus", timeoutMs: 45_000 }, + timeoutMs: 30_000, }); expect(voices).toEqual([{ id: "en-US-JennyNeural", name: "Jenny" }]); @@ -261,7 +262,7 @@ describe("buildAzureSpeechProvider", () => { baseUrl: "https://eastus.tts.speech.microsoft.com", endpoint: undefined, region: "eastus", - timeoutMs: undefined, + timeoutMs: 45_000, }); }); }); diff --git a/extensions/azure-speech/speech-provider.ts b/extensions/azure-speech/speech-provider.ts index d996f2e452fa..6810da073dd9 100644 --- a/extensions/azure-speech/speech-provider.ts +++ b/extensions/azure-speech/speech-provider.ts @@ -259,7 +259,7 @@ export function buildAzureSpeechProvider(): SpeechProviderPlugin { baseUrl: req.baseUrl ?? config?.baseUrl, endpoint: config?.endpoint, region: config?.region ?? readAzureSpeechEnvRegion(), - timeoutMs: config?.timeoutMs, + timeoutMs: config?.timeoutMs ?? req.timeoutMs, }); }, isConfigured: ({ providerConfig }) => { diff --git a/extensions/elevenlabs/speech-provider.test.ts b/extensions/elevenlabs/speech-provider.test.ts index c01b3375dd60..aee17f9d2652 100644 --- a/extensions/elevenlabs/speech-provider.test.ts +++ b/extensions/elevenlabs/speech-provider.test.ts @@ -3,17 +3,20 @@ import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; import { isValidElevenLabsVoiceId } from "./shared.js"; import { buildElevenLabsSpeechProvider } from "./speech-provider.js"; +const fetchWithSsrFGuardMock = vi.hoisted(() => vi.fn()); + vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({ - fetchWithSsrFGuard: async ({ - url, - init, - }: { + fetchWithSsrFGuard: async (params: { url: string; init?: RequestInit; - }): Promise<{ response: Response; release: () => Promise }> => ({ - response: await globalThis.fetch(url, init), - release: vi.fn(async () => {}), - }), + timeoutMs?: number; + }): Promise<{ response: Response; release: () => Promise }> => { + fetchWithSsrFGuardMock(params); + return { + response: await globalThis.fetch(params.url, params.init), + release: vi.fn(async () => {}), + }; + }, ssrfPolicyFromHttpBaseUrlAllowedHostname: () => undefined, })); @@ -38,6 +41,7 @@ describe("elevenlabs speech provider", () => { afterEach(() => { globalThis.fetch = originalFetch; + fetchWithSsrFGuardMock.mockClear(); vi.restoreAllMocks(); }); @@ -54,6 +58,20 @@ describe("elevenlabs speech provider", () => { ]); }); + it("forwards the core-resolved voice-list timeout", async () => { + globalThis.fetch = vi.fn(async () => Response.json({ voices: [] })) as unknown as typeof fetch; + const provider = buildElevenLabsSpeechProvider(); + + await provider.listVoices?.({ + providerConfig: { apiKey: "xi-test" }, + timeoutMs: 30_000, + }); + + expect(fetchWithSsrFGuardMock).toHaveBeenCalledWith( + expect.objectContaining({ timeoutMs: 30_000 }), + ); + }); + it("keeps non-equivalent deprecated ElevenLabs TTS model IDs", async () => { const provider = buildElevenLabsSpeechProvider(); const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => { diff --git a/extensions/elevenlabs/speech-provider.ts b/extensions/elevenlabs/speech-provider.ts index b3e0f4f9640b..76c1d8e6bdf9 100644 --- a/extensions/elevenlabs/speech-provider.ts +++ b/extensions/elevenlabs/speech-provider.ts @@ -356,6 +356,7 @@ function parseDirectiveToken(ctx: SpeechDirectiveTokenParseContext) { async function listElevenLabsVoices(params: { apiKey: string; baseUrl?: string; + timeoutMs?: number; }): Promise { const normalizedBaseUrl = normalizeElevenLabsBaseUrl(params.baseUrl); const { response, release } = await fetchWithSsrFGuard({ @@ -365,6 +366,7 @@ async function listElevenLabsVoices(params: { "xi-api-key": params.apiKey, }, }, + timeoutMs: params.timeoutMs, policy: ssrfPolicyFromHttpBaseUrlAllowedHostname(normalizedBaseUrl), auditContext: "elevenlabs.voices", }); @@ -538,6 +540,7 @@ export function buildElevenLabsSpeechProvider(): SpeechProviderPlugin { return listElevenLabsVoices({ apiKey, baseUrl: req.baseUrl ?? config?.baseUrl, + timeoutMs: req.timeoutMs, }); }, isConfigured: ({ providerConfig }) => diff --git a/extensions/inworld/speech-provider.test.ts b/extensions/inworld/speech-provider.test.ts index f66e443cd3d0..484a65d43c27 100644 --- a/extensions/inworld/speech-provider.test.ts +++ b/extensions/inworld/speech-provider.test.ts @@ -72,6 +72,19 @@ describe("buildInworldSpeechProvider", () => { expect(provider.models).toContain("inworld-tts-1.5-mini"); }); + it("forwards the core-resolved voice-list timeout", async () => { + const provider = buildInworldSpeechProvider(); + + await provider.listVoices?.({ + providerConfig: { apiKey: "test-key" }, + timeoutMs: 30_000, + }); + + expect(listInworldVoicesMock).toHaveBeenCalledWith( + expect.objectContaining({ apiKey: "test-key", timeoutMs: 30_000 }), + ); + }); + it("normalizes provider-owned speech config from raw provider config", () => { const provider = buildInworldSpeechProvider(); const resolved = provider.resolveConfig?.({ diff --git a/extensions/inworld/speech-provider.ts b/extensions/inworld/speech-provider.ts index f1c77da66e11..0aa81c489f5c 100644 --- a/extensions/inworld/speech-provider.ts +++ b/extensions/inworld/speech-provider.ts @@ -171,6 +171,7 @@ export function buildInworldSpeechProvider(): SpeechProviderPlugin { return listInworldVoices({ apiKey, baseUrl: req.baseUrl ?? config?.baseUrl, + timeoutMs: req.timeoutMs, }); }, isConfigured: ({ providerConfig }) => diff --git a/extensions/microsoft/speech-provider.test.ts b/extensions/microsoft/speech-provider.test.ts index 822facbca755..1d66159d4f95 100644 --- a/extensions/microsoft/speech-provider.test.ts +++ b/extensions/microsoft/speech-provider.test.ts @@ -11,6 +11,19 @@ import { import { afterEach, describe, expect, it, vi } from "vitest"; import { installDebugProxyTestResetHooks } from "../test-support/debug-proxy-env-test-helpers.js"; +const fetchWithSsrFGuardMock = vi.hoisted(() => vi.fn()); + +vi.mock("openclaw/plugin-sdk/ssrf-runtime", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + fetchWithSsrFGuard: (...args: Parameters) => { + fetchWithSsrFGuardMock(...args); + return actual.fetchWithSsrFGuard(...args); + }, + }; +}); + vi.mock("node-edge-tts", () => ({ EdgeTTS: class { async ttsPromise(): Promise {} @@ -83,6 +96,9 @@ describe("listMicrosoftVoices", () => { personalities: ["Friendly", "Positive"], }, ]); + expect(fetchWithSsrFGuardMock).toHaveBeenLastCalledWith( + expect.objectContaining({ timeoutMs: 30_000 }), + ); }); it("throws on Microsoft voice list failures", async () => { @@ -95,6 +111,22 @@ describe("listMicrosoftVoices", () => { await expect(listMicrosoftVoices()).rejects.toThrow("Microsoft voices API error (503)"); }); + it("prefers the configured provider request timeout", async () => { + globalThis.fetch = vi + .fn() + .mockResolvedValue(new Response("[]", { status: 200 })) as unknown as typeof globalThis.fetch; + const listVoices = buildMicrosoftSpeechProvider().listVoices; + if (!listVoices) { + throw new Error("expected Microsoft voice listing support"); + } + + await listVoices({ providerConfig: { timeoutMs: 2_345 }, timeoutMs: 1_234 }); + + expect(fetchWithSsrFGuardMock).toHaveBeenLastCalledWith( + expect.objectContaining({ timeoutMs: 2_345 }), + ); + }); + it("records voice discovery exchanges in debug proxy capture mode", async () => { const tempDir = mkdtempSync(path.join(os.tmpdir(), "microsoft-voices-capture-")); proxyReset.captureProxyEnv(); diff --git a/extensions/microsoft/speech-provider.ts b/extensions/microsoft/speech-provider.ts index a8ec98163e92..582ecbacfabc 100644 --- a/extensions/microsoft/speech-provider.ts +++ b/extensions/microsoft/speech-provider.ts @@ -31,6 +31,7 @@ import { edgeTTS, inferEdgeExtension } from "./tts.js"; const DEFAULT_EDGE_VOICE = "en-US-MichelleNeural"; const DEFAULT_EDGE_LANG = "en-US"; const DEFAULT_EDGE_OUTPUT_FORMAT = "audio-24khz-48kbitrate-mono-mp3"; +const DEFAULT_MICROSOFT_VOICE_LIST_TIMEOUT_MS = 30_000; type MicrosoftProviderConfig = { enabled: boolean; @@ -141,7 +142,9 @@ export function isCjkDominant(text: string): boolean { const DEFAULT_CHINESE_EDGE_VOICE = "zh-CN-XiaoxiaoNeural"; const DEFAULT_CHINESE_EDGE_LANG = "zh-CN"; -export async function listMicrosoftVoices(): Promise { +export async function listMicrosoftVoices( + timeoutMs = DEFAULT_MICROSOFT_VOICE_LIST_TIMEOUT_MS, +): Promise { const url = "https://speech.platform.bing.com/consumer/speech/synthesize/readaloud/voices/list" + `?trustedclienttoken=${TRUSTED_CLIENT_TOKEN}`; @@ -153,6 +156,7 @@ export async function listMicrosoftVoices(): Promise { }, policy: ssrfPolicyFromHttpBaseUrlAllowedHostname("https://speech.platform.bing.com"), auditContext: "microsoft.speech.voices", + timeoutMs, }); try { if (!isDebugProxyGlobalFetchPatchInstalled()) { @@ -239,7 +243,10 @@ export function buildMicrosoftSpeechProvider(): SpeechProviderPlugin { ? {} : { outputFormat: trimToUndefined(params.outputFormat) }), }), - listVoices: async () => await listMicrosoftVoices(), + listVoices: async (req) => { + const config = readMicrosoftProviderConfig(req.providerConfig ?? {}); + return await listMicrosoftVoices(config.timeoutMs ?? req.timeoutMs); + }, isConfigured: ({ providerConfig }) => readMicrosoftProviderConfig(providerConfig).enabled, synthesize: async (req) => { const config = readMicrosoftProviderConfig(req.providerConfig); diff --git a/packages/speech-core/src/tts.test.ts b/packages/speech-core/src/tts.test.ts index 56797987a060..6f134a02b689 100644 --- a/packages/speech-core/src/tts.test.ts +++ b/packages/speech-core/src/tts.test.ts @@ -9,6 +9,7 @@ import { setRuntimeConfigSnapshot, } from "openclaw/plugin-sdk/runtime-config-snapshot"; import type { + SpeechListVoicesRequest, SpeechProviderPlugin, SpeechProviderPrepareSynthesisContext, SpeechSynthesisRequest, @@ -113,6 +114,7 @@ const { buildTtsSystemPromptHint, getTtsPersona, getTtsProvider, + listSpeechVoices, maybeApplyTtsToPayload, resolveTtsConfig, setSummarizationEnabled, @@ -422,6 +424,31 @@ describe("speech-core native voice-note routing", () => { expect(request.timeoutMs).toBe(600_000); }); + it("resolves the configured timeout for voice listing", async () => { + const listVoicesMock = vi.fn(async (_request: SpeechListVoicesRequest) => []); + installSpeechProviders([ + createMockSpeechProvider("mock", { + defaultTimeoutMs: 60_000, + listVoices: listVoicesMock, + }), + ]); + + await listSpeechVoices({ + provider: "mock", + cfg: { + messages: { + tts: { + enabled: true, + provider: "mock", + timeoutMs: 45_000, + }, + }, + } as OpenClawConfig, + }); + + expect(listVoicesMock).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 45_000 })); + }); + it("caps oversized provider default TTS timeouts before synthesis", async () => { installSpeechProviders([ createMockSpeechProvider("mock", { defaultTimeoutMs: Number.MAX_SAFE_INTEGER }), diff --git a/packages/speech-core/src/tts.ts b/packages/speech-core/src/tts.ts index a37c672767af..afd8ad2aaa43 100644 --- a/packages/speech-core/src/tts.ts +++ b/packages/speech-core/src/tts.ts @@ -1924,11 +1924,16 @@ export async function listSpeechVoices(params: { if (!resolvedProvider.listVoices) { throw new Error(`speech provider ${provider} does not support voice listing`); } + const timeoutMs = resolveSpeechProviderTimeoutMs({ + config, + provider: resolvedProvider, + }); return await resolvedProvider.listVoices({ cfg, providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, cfg), apiKey: params.apiKey, baseUrl: params.baseUrl, + timeoutMs, }); } diff --git a/src/tts/provider-types.ts b/src/tts/provider-types.ts index 42cb04f061e7..c9cb1189021e 100644 --- a/src/tts/provider-types.ts +++ b/src/tts/provider-types.ts @@ -131,6 +131,8 @@ export type SpeechListVoicesRequest = { providerConfig?: SpeechProviderConfig; apiKey?: string; baseUrl?: string; + /** Core-resolved request timeout after config and provider defaults. */ + timeoutMs?: number; }; /** Provider hook input for resolving normalized config from raw OpenClaw config. */