From 5de84bfc4d54852de148c5773be710997b1c046e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 2 Aug 2026 23:00:46 -0700 Subject: [PATCH] fix(openai): support the current speech model snapshot (#118475) * fix(openai): support the current speech model snapshot * docs(openai): clarify speech instructions apply to the model family --- docs/providers/openai.md | 9 ++- extensions/openai/speech-provider.test.ts | 97 +++++++++++++++++++++++ extensions/openai/tts.test.ts | 4 +- extensions/openai/tts.ts | 7 +- 4 files changed, 111 insertions(+), 6 deletions(-) diff --git a/docs/providers/openai.md b/docs/providers/openai.md index 8d9c58c5331a..8db61b40a79a 100644 --- a/docs/providers/openai.md +++ b/docs/providers/openai.md @@ -806,15 +806,16 @@ compatibility fallback when the shared | Model | `tts.providers.openai.model` | `gpt-4o-mini-tts` | | Voice | `tts.providers.openai.speakerVoice` | `coral` | | Speed | `tts.providers.openai.speed` | (unset) | - | Instructions | `tts.providers.openai.instructions` | (unset, `gpt-4o-mini-tts` only) | + | Instructions | `tts.providers.openai.instructions` | (unset, `gpt-4o-mini-tts` family only) | | Format | `tts.providers.openai.responseFormat` | `opus` for voice notes, `mp3` for files | | API key | `tts.providers.openai.apiKey` | Falls back to `OPENAI_API_KEY` | | Base URL | `tts.providers.openai.baseUrl` | `https://api.openai.com/v1` | | Extra body | `tts.providers.openai.extraBody` / `extra_body` | (unset) | - Available models: `gpt-4o-mini-tts`, `tts-1`, `tts-1-hd`. Available voices: - `alloy`, `ash`, `ballad`, `cedar`, `coral`, `echo`, `fable`, `juniper`, - `marin`, `onyx`, `nova`, `sage`, `shimmer`, `verse`. + Available models: `gpt-4o-mini-tts`, `gpt-4o-mini-tts-2025-12-15`, `tts-1`, + `tts-1-hd`. Available voices: `alloy`, `ash`, `ballad`, `cedar`, `coral`, + `echo`, `fable`, `juniper`, `marin`, `onyx`, `nova`, `sage`, `shimmer`, + `verse`. `extraBody` is merged into `/audio/speech` request JSON after OpenClaw's generated fields, so use it for OpenAI-compatible endpoints that require diff --git a/extensions/openai/speech-provider.test.ts b/extensions/openai/speech-provider.test.ts index 9a3b8e6f4a09..189340bb4fdb 100644 --- a/extensions/openai/speech-provider.test.ts +++ b/extensions/openai/speech-provider.test.ts @@ -1,7 +1,10 @@ // Openai tests cover speech provider plugin behavior. +import { createServer } from "node:http"; import { afterEach, describe, expect, it, vi } from "vitest"; import { buildOpenAISpeechProvider } from "./speech-provider.js"; +const OPENAI_TTS_SNAPSHOT = "gpt-4o-mini-tts-2025-12-15"; + vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({ fetchWithSsrFGuard: async ({ url, @@ -62,6 +65,13 @@ describe("buildOpenAISpeechProvider", () => { vi.restoreAllMocks(); }); + it("advertises official speech snapshots without changing the default model", () => { + const provider = buildOpenAISpeechProvider(); + + expect(provider.defaultModel).toBe("gpt-4o-mini-tts"); + expect(provider.models).toContain(OPENAI_TTS_SNAPSHOT); + }); + it("normalizes provider-owned speech config from raw provider config", () => { const provider = buildOpenAISpeechProvider(); const resolved = provider.resolveConfig?.({ @@ -177,6 +187,23 @@ describe("buildOpenAISpeechProvider", () => { overrides: { voice: "alloy" }, }); + expect( + provider.parseDirectiveToken?.({ + key: "openai_model", + value: OPENAI_TTS_SNAPSHOT, + policy: { + allowVoice: true, + allowModelId: true, + }, + providerConfig: { + baseUrl: "https://api.openai.com/v1/", + }, + } as never), + ).toEqual({ + handled: true, + overrides: { model: OPENAI_TTS_SNAPSHOT }, + }); + expect( provider.parseDirectiveToken?.({ key: "model", @@ -194,6 +221,76 @@ describe("buildOpenAISpeechProvider", () => { }); }); + it("sends dated speech snapshots through a real loopback HTTP request", async () => { + const provider = buildOpenAISpeechProvider(); + let receivedRequest: + | { + method: string | undefined; + url: string | undefined; + body: unknown; + } + | undefined; + const server = createServer((request, response) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + receivedRequest = { + method: request.method, + url: request.url, + body: JSON.parse(Buffer.concat(chunks).toString("utf8")) as unknown, + }; + response.writeHead(200, { "content-type": "audio/mpeg" }); + response.end(Buffer.from("snapshot-audio")); + }); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.removeListener("error", reject); + resolve(); + }); + }); + + try { + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("expected a loopback server address"); + } + + const result = await provider.synthesize({ + text: "snapshot request", + cfg: {} as never, + providerConfig: { + apiKey: "sk-test", + baseUrl: `http://127.0.0.1:${address.port}/v1`, + model: OPENAI_TTS_SNAPSHOT, + voice: "alloy", + instructions: " Speak warmly ", + }, + target: "audio-file", + timeoutMs: 1_000, + }); + + expect(receivedRequest).toEqual({ + method: "POST", + url: "/v1/audio/speech", + body: { + model: OPENAI_TTS_SNAPSHOT, + input: "snapshot request", + voice: "alloy", + response_format: "mp3", + instructions: "Speak warmly", + }, + }); + expect(result.audioBuffer).toEqual(Buffer.from("snapshot-audio")); + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } + }); + it("parses preferred-OpenAI speed directive within the supported range", () => { const provider = buildOpenAISpeechProvider(); diff --git a/extensions/openai/tts.test.ts b/extensions/openai/tts.test.ts index 92c4e217e1f1..60903e2d6243 100644 --- a/extensions/openai/tts.test.ts +++ b/extensions/openai/tts.test.ts @@ -101,13 +101,15 @@ describe("openai tts", () => { describe("isValidOpenAIModel", () => { it("matches the supported model set and rejects unsupported values", () => { expect(OPENAI_TTS_MODELS).toContain("gpt-4o-mini-tts"); + expect(OPENAI_TTS_MODELS).toContain("gpt-4o-mini-tts-2025-12-15"); expect(OPENAI_TTS_MODELS).toContain("tts-1"); expect(OPENAI_TTS_MODELS).toContain("tts-1-hd"); - expect(OPENAI_TTS_MODELS).toHaveLength(3); + expect(OPENAI_TTS_MODELS).toHaveLength(4); expect(Array.isArray(OPENAI_TTS_MODELS)).toBe(true); expect(OPENAI_TTS_MODELS.length).toBeGreaterThan(0); const cases = [ { model: "gpt-4o-mini-tts", expected: true }, + { model: "gpt-4o-mini-tts-2025-12-15", expected: true }, { model: "tts-1", expected: true }, { model: "tts-1-hd", expected: true }, { model: "invalid", expected: false }, diff --git a/extensions/openai/tts.ts b/extensions/openai/tts.ts index bebc252c9c43..d2e0248462f5 100644 --- a/extensions/openai/tts.ts +++ b/extensions/openai/tts.ts @@ -17,7 +17,12 @@ import { export const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1"; const DEFAULT_TTS_MAX_BYTES = 16 * 1024 * 1024; -export const OPENAI_TTS_MODELS = ["gpt-4o-mini-tts", "tts-1", "tts-1-hd"] as const; +export const OPENAI_TTS_MODELS = [ + "gpt-4o-mini-tts", + "gpt-4o-mini-tts-2025-12-15", + "tts-1", + "tts-1-hd", +] as const; export const OPENAI_TTS_VOICES = [ "alloy",