mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(tts): bound voice-list requests (#102865)
* fix(microsoft): add timeout to voices list request * fix(tts): bound voice-list requests Co-authored-by: llagy009 <0668001470@xydigit.com> --------- Co-authored-by: llagy009 <0668001470@xydigit.com> Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
@@ -455,6 +455,7 @@ Notes:
|
||||
- Uses core `messages.tts` configuration and provider selection.
|
||||
- Returns PCM audio buffer + sample rate. Plugins must resample/encode for providers.
|
||||
- `listVoices` is optional per provider. Use it for vendor-owned voice pickers or setup flows.
|
||||
- Core passes a resolved request deadline to provider `listVoices` hooks; provider-specific timeout settings may override it.
|
||||
- Voice listings can include richer metadata such as locale, gender, and personality tags for provider-aware pickers.
|
||||
- OpenAI and ElevenLabs support telephony today. Microsoft does not.
|
||||
|
||||
|
||||
@@ -252,7 +252,8 @@ describe("buildAzureSpeechProvider", () => {
|
||||
it("lists voices through config or explicit request auth", async () => {
|
||||
const provider = buildAzureSpeechProvider();
|
||||
const voices = await provider.listVoices?.({
|
||||
providerConfig: { apiKey: "key", region: "eastus" },
|
||||
providerConfig: { apiKey: "key", region: "eastus", timeoutMs: 45_000 },
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
|
||||
expect(voices).toEqual([{ id: "en-US-JennyNeural", name: "Jenny" }]);
|
||||
@@ -261,7 +262,7 @@ describe("buildAzureSpeechProvider", () => {
|
||||
baseUrl: "https://eastus.tts.speech.microsoft.com",
|
||||
endpoint: undefined,
|
||||
region: "eastus",
|
||||
timeoutMs: undefined,
|
||||
timeoutMs: 45_000,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -259,7 +259,7 @@ export function buildAzureSpeechProvider(): SpeechProviderPlugin {
|
||||
baseUrl: req.baseUrl ?? config?.baseUrl,
|
||||
endpoint: config?.endpoint,
|
||||
region: config?.region ?? readAzureSpeechEnvRegion(),
|
||||
timeoutMs: config?.timeoutMs,
|
||||
timeoutMs: config?.timeoutMs ?? req.timeoutMs,
|
||||
});
|
||||
},
|
||||
isConfigured: ({ providerConfig }) => {
|
||||
|
||||
@@ -3,17 +3,20 @@ import { afterAll, afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { isValidElevenLabsVoiceId } from "./shared.js";
|
||||
import { buildElevenLabsSpeechProvider } from "./speech-provider.js";
|
||||
|
||||
const fetchWithSsrFGuardMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
|
||||
fetchWithSsrFGuard: async ({
|
||||
url,
|
||||
init,
|
||||
}: {
|
||||
fetchWithSsrFGuard: async (params: {
|
||||
url: string;
|
||||
init?: RequestInit;
|
||||
}): Promise<{ response: Response; release: () => Promise<void> }> => ({
|
||||
response: await globalThis.fetch(url, init),
|
||||
release: vi.fn(async () => {}),
|
||||
}),
|
||||
timeoutMs?: number;
|
||||
}): Promise<{ response: Response; release: () => Promise<void> }> => {
|
||||
fetchWithSsrFGuardMock(params);
|
||||
return {
|
||||
response: await globalThis.fetch(params.url, params.init),
|
||||
release: vi.fn(async () => {}),
|
||||
};
|
||||
},
|
||||
ssrfPolicyFromHttpBaseUrlAllowedHostname: () => undefined,
|
||||
}));
|
||||
|
||||
@@ -38,6 +41,7 @@ describe("elevenlabs speech provider", () => {
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
fetchWithSsrFGuardMock.mockClear();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
@@ -54,6 +58,20 @@ describe("elevenlabs speech provider", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("forwards the core-resolved voice-list timeout", async () => {
|
||||
globalThis.fetch = vi.fn(async () => Response.json({ voices: [] })) as unknown as typeof fetch;
|
||||
const provider = buildElevenLabsSpeechProvider();
|
||||
|
||||
await provider.listVoices?.({
|
||||
providerConfig: { apiKey: "xi-test" },
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ timeoutMs: 30_000 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps non-equivalent deprecated ElevenLabs TTS model IDs", async () => {
|
||||
const provider = buildElevenLabsSpeechProvider();
|
||||
const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => {
|
||||
|
||||
@@ -356,6 +356,7 @@ function parseDirectiveToken(ctx: SpeechDirectiveTokenParseContext) {
|
||||
async function listElevenLabsVoices(params: {
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
timeoutMs?: number;
|
||||
}): Promise<SpeechVoiceOption[]> {
|
||||
const normalizedBaseUrl = normalizeElevenLabsBaseUrl(params.baseUrl);
|
||||
const { response, release } = await fetchWithSsrFGuard({
|
||||
@@ -365,6 +366,7 @@ async function listElevenLabsVoices(params: {
|
||||
"xi-api-key": params.apiKey,
|
||||
},
|
||||
},
|
||||
timeoutMs: params.timeoutMs,
|
||||
policy: ssrfPolicyFromHttpBaseUrlAllowedHostname(normalizedBaseUrl),
|
||||
auditContext: "elevenlabs.voices",
|
||||
});
|
||||
@@ -538,6 +540,7 @@ export function buildElevenLabsSpeechProvider(): SpeechProviderPlugin {
|
||||
return listElevenLabsVoices({
|
||||
apiKey,
|
||||
baseUrl: req.baseUrl ?? config?.baseUrl,
|
||||
timeoutMs: req.timeoutMs,
|
||||
});
|
||||
},
|
||||
isConfigured: ({ providerConfig }) =>
|
||||
|
||||
@@ -72,6 +72,19 @@ describe("buildInworldSpeechProvider", () => {
|
||||
expect(provider.models).toContain("inworld-tts-1.5-mini");
|
||||
});
|
||||
|
||||
it("forwards the core-resolved voice-list timeout", async () => {
|
||||
const provider = buildInworldSpeechProvider();
|
||||
|
||||
await provider.listVoices?.({
|
||||
providerConfig: { apiKey: "test-key" },
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
|
||||
expect(listInworldVoicesMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ apiKey: "test-key", timeoutMs: 30_000 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("normalizes provider-owned speech config from raw provider config", () => {
|
||||
const provider = buildInworldSpeechProvider();
|
||||
const resolved = provider.resolveConfig?.({
|
||||
|
||||
@@ -171,6 +171,7 @@ export function buildInworldSpeechProvider(): SpeechProviderPlugin {
|
||||
return listInworldVoices({
|
||||
apiKey,
|
||||
baseUrl: req.baseUrl ?? config?.baseUrl,
|
||||
timeoutMs: req.timeoutMs,
|
||||
});
|
||||
},
|
||||
isConfigured: ({ providerConfig }) =>
|
||||
|
||||
@@ -11,6 +11,19 @@ import {
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { installDebugProxyTestResetHooks } from "../test-support/debug-proxy-env-test-helpers.js";
|
||||
|
||||
const fetchWithSsrFGuardMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/ssrf-runtime", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("openclaw/plugin-sdk/ssrf-runtime")>();
|
||||
return {
|
||||
...actual,
|
||||
fetchWithSsrFGuard: (...args: Parameters<typeof actual.fetchWithSsrFGuard>) => {
|
||||
fetchWithSsrFGuardMock(...args);
|
||||
return actual.fetchWithSsrFGuard(...args);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("node-edge-tts", () => ({
|
||||
EdgeTTS: class {
|
||||
async ttsPromise(): Promise<void> {}
|
||||
@@ -83,6 +96,9 @@ describe("listMicrosoftVoices", () => {
|
||||
personalities: ["Friendly", "Positive"],
|
||||
},
|
||||
]);
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({ timeoutMs: 30_000 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("throws on Microsoft voice list failures", async () => {
|
||||
@@ -95,6 +111,22 @@ describe("listMicrosoftVoices", () => {
|
||||
await expect(listMicrosoftVoices()).rejects.toThrow("Microsoft voices API error (503)");
|
||||
});
|
||||
|
||||
it("prefers the configured provider request timeout", async () => {
|
||||
globalThis.fetch = vi
|
||||
.fn()
|
||||
.mockResolvedValue(new Response("[]", { status: 200 })) as unknown as typeof globalThis.fetch;
|
||||
const listVoices = buildMicrosoftSpeechProvider().listVoices;
|
||||
if (!listVoices) {
|
||||
throw new Error("expected Microsoft voice listing support");
|
||||
}
|
||||
|
||||
await listVoices({ providerConfig: { timeoutMs: 2_345 }, timeoutMs: 1_234 });
|
||||
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({ timeoutMs: 2_345 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("records voice discovery exchanges in debug proxy capture mode", async () => {
|
||||
const tempDir = mkdtempSync(path.join(os.tmpdir(), "microsoft-voices-capture-"));
|
||||
proxyReset.captureProxyEnv();
|
||||
|
||||
@@ -31,6 +31,7 @@ import { edgeTTS, inferEdgeExtension } from "./tts.js";
|
||||
const DEFAULT_EDGE_VOICE = "en-US-MichelleNeural";
|
||||
const DEFAULT_EDGE_LANG = "en-US";
|
||||
const DEFAULT_EDGE_OUTPUT_FORMAT = "audio-24khz-48kbitrate-mono-mp3";
|
||||
const DEFAULT_MICROSOFT_VOICE_LIST_TIMEOUT_MS = 30_000;
|
||||
|
||||
type MicrosoftProviderConfig = {
|
||||
enabled: boolean;
|
||||
@@ -141,7 +142,9 @@ export function isCjkDominant(text: string): boolean {
|
||||
const DEFAULT_CHINESE_EDGE_VOICE = "zh-CN-XiaoxiaoNeural";
|
||||
const DEFAULT_CHINESE_EDGE_LANG = "zh-CN";
|
||||
|
||||
export async function listMicrosoftVoices(): Promise<SpeechVoiceOption[]> {
|
||||
export async function listMicrosoftVoices(
|
||||
timeoutMs = DEFAULT_MICROSOFT_VOICE_LIST_TIMEOUT_MS,
|
||||
): Promise<SpeechVoiceOption[]> {
|
||||
const url =
|
||||
"https://speech.platform.bing.com/consumer/speech/synthesize/readaloud/voices/list" +
|
||||
`?trustedclienttoken=${TRUSTED_CLIENT_TOKEN}`;
|
||||
@@ -153,6 +156,7 @@ export async function listMicrosoftVoices(): Promise<SpeechVoiceOption[]> {
|
||||
},
|
||||
policy: ssrfPolicyFromHttpBaseUrlAllowedHostname("https://speech.platform.bing.com"),
|
||||
auditContext: "microsoft.speech.voices",
|
||||
timeoutMs,
|
||||
});
|
||||
try {
|
||||
if (!isDebugProxyGlobalFetchPatchInstalled()) {
|
||||
@@ -239,7 +243,10 @@ export function buildMicrosoftSpeechProvider(): SpeechProviderPlugin {
|
||||
? {}
|
||||
: { outputFormat: trimToUndefined(params.outputFormat) }),
|
||||
}),
|
||||
listVoices: async () => await listMicrosoftVoices(),
|
||||
listVoices: async (req) => {
|
||||
const config = readMicrosoftProviderConfig(req.providerConfig ?? {});
|
||||
return await listMicrosoftVoices(config.timeoutMs ?? req.timeoutMs);
|
||||
},
|
||||
isConfigured: ({ providerConfig }) => readMicrosoftProviderConfig(providerConfig).enabled,
|
||||
synthesize: async (req) => {
|
||||
const config = readMicrosoftProviderConfig(req.providerConfig);
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
setRuntimeConfigSnapshot,
|
||||
} from "openclaw/plugin-sdk/runtime-config-snapshot";
|
||||
import type {
|
||||
SpeechListVoicesRequest,
|
||||
SpeechProviderPlugin,
|
||||
SpeechProviderPrepareSynthesisContext,
|
||||
SpeechSynthesisRequest,
|
||||
@@ -113,6 +114,7 @@ const {
|
||||
buildTtsSystemPromptHint,
|
||||
getTtsPersona,
|
||||
getTtsProvider,
|
||||
listSpeechVoices,
|
||||
maybeApplyTtsToPayload,
|
||||
resolveTtsConfig,
|
||||
setSummarizationEnabled,
|
||||
@@ -422,6 +424,31 @@ describe("speech-core native voice-note routing", () => {
|
||||
expect(request.timeoutMs).toBe(600_000);
|
||||
});
|
||||
|
||||
it("resolves the configured timeout for voice listing", async () => {
|
||||
const listVoicesMock = vi.fn(async (_request: SpeechListVoicesRequest) => []);
|
||||
installSpeechProviders([
|
||||
createMockSpeechProvider("mock", {
|
||||
defaultTimeoutMs: 60_000,
|
||||
listVoices: listVoicesMock,
|
||||
}),
|
||||
]);
|
||||
|
||||
await listSpeechVoices({
|
||||
provider: "mock",
|
||||
cfg: {
|
||||
messages: {
|
||||
tts: {
|
||||
enabled: true,
|
||||
provider: "mock",
|
||||
timeoutMs: 45_000,
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig,
|
||||
});
|
||||
|
||||
expect(listVoicesMock).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 45_000 }));
|
||||
});
|
||||
|
||||
it("caps oversized provider default TTS timeouts before synthesis", async () => {
|
||||
installSpeechProviders([
|
||||
createMockSpeechProvider("mock", { defaultTimeoutMs: Number.MAX_SAFE_INTEGER }),
|
||||
|
||||
@@ -1924,11 +1924,16 @@ export async function listSpeechVoices(params: {
|
||||
if (!resolvedProvider.listVoices) {
|
||||
throw new Error(`speech provider ${provider} does not support voice listing`);
|
||||
}
|
||||
const timeoutMs = resolveSpeechProviderTimeoutMs({
|
||||
config,
|
||||
provider: resolvedProvider,
|
||||
});
|
||||
return await resolvedProvider.listVoices({
|
||||
cfg,
|
||||
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, cfg),
|
||||
apiKey: params.apiKey,
|
||||
baseUrl: params.baseUrl,
|
||||
timeoutMs,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -131,6 +131,8 @@ export type SpeechListVoicesRequest = {
|
||||
providerConfig?: SpeechProviderConfig;
|
||||
apiKey?: string;
|
||||
baseUrl?: string;
|
||||
/** Core-resolved request timeout after config and provider defaults. */
|
||||
timeoutMs?: number;
|
||||
};
|
||||
|
||||
/** Provider hook input for resolving normalized config from raw OpenClaw config. */
|
||||
|
||||
Reference in New Issue
Block a user