openclaw/extensions/openai/speech-provider.test.ts

// Openai tests cover speech provider plugin behavior.
import { createServer } from "node:http";
import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
import { afterEach, describe, expect, it, vi } from "vitest";
import { buildOpenAISpeechProvider } from "./speech-provider.js";

const OPENAI_TTS_SNAPSHOT = "gpt-4o-mini-tts-2025-12-15";

vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
  fetchWithSsrFGuard: async ({
    url,
    init,
  }: {
    url: string;
    init?: RequestInit;
  }): Promise<{ response: Response; release: () => Promise<void> }> => ({
    response: await globalThis.fetch(url, init),
    release: vi.fn(async () => {}),
  }),
  ssrfPolicyFromHttpBaseUrlAllowedHostname: () => undefined,
}));

function isSpeechRequestBody(value: unknown): value is {
  [key: string]: unknown;
  model?: string;
  voice?: string;
  speed?: number;
  response_format?: string;
} {
  return isRecord(value);
}

function parseRequestBody(init: RequestInit | undefined): {
  [key: string]: unknown;
  model?: string;
  voice?: string;
  speed?: number;
  response_format?: string;
} {
  if (typeof init?.body !== "string") {
    throw new Error("expected string request body");
  }
  const body: unknown = JSON.parse(init.body);
  if (!isSpeechRequestBody(body)) {
    throw new Error("expected OpenAI speech request body");
  }
  return body;
}

function mockSpeechFetchExpectingFormat(responseFormat: string) {
  const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => {
    const body = parseRequestBody(init);
    expect(body.response_format).toBe(responseFormat);
    return new Response(new Uint8Array([1, 2, 3]), { status: 200 });
  });
  globalThis.fetch = fetchMock as unknown as typeof fetch;
  return fetchMock;
}

describe("buildOpenAISpeechProvider", () => {
  const originalFetch = globalThis.fetch;

  afterEach(() => {
    globalThis.fetch = originalFetch;
    vi.unstubAllEnvs();
    vi.restoreAllMocks();
  });

  it("advertises official speech snapshots without changing the default model", () => {
    const provider = buildOpenAISpeechProvider();

    expect(provider.defaultModel).toBe("gpt-4o-mini-tts");
    expect(provider.models).toContain(OPENAI_TTS_SNAPSHOT);
  });

  it("normalizes provider-owned speech config from raw provider config", () => {
    const provider = buildOpenAISpeechProvider();
    const resolved = provider.resolveConfig?.({
      cfg: {} as never,
      timeoutMs: 30_000,
      rawConfig: {
        providers: {
          openai: {
            apiKey: "sk-test",
            baseUrl: "https://example.com/v1/",
            model: "tts-1",
            voice: "alloy",
            speed: 1.25,
            instructions: " Speak warmly ",
            responseFormat: " WAV ",
            extraBody: {
              lang: "en-US",
            },
          },
        },
      },
    });

    expect(resolved).toEqual({
      apiKey: "sk-test",
      baseUrl: "https://example.com/v1",
      model: "tts-1",
      voice: "alloy",
      speed: 1.25,
      instructions: "Speak warmly",
      responseFormat: "wav",
      extraBody: {
        lang: "en-US",
      },
    });
  });

  it("drops malformed speech speed values", () => {
    const provider = buildOpenAISpeechProvider();
    const resolved = provider.resolveConfig?.({
      cfg: {} as never,
      timeoutMs: 30_000,
      rawConfig: {
        providers: {
          openai: {
            speed: 4.5,
          },
        },
      },
    });

    expect(resolved?.speed).toBeUndefined();
  });

  it("passes custom endpoint speech speeds through", () => {
    const provider = buildOpenAISpeechProvider();
    const resolved = provider.resolveConfig?.({
      cfg: {} as never,
      timeoutMs: 30_000,
      rawConfig: {
        providers: {
          openai: {
            baseUrl: "https://tts.example.com/v1",
            speed: 4.5,
          },
        },
      },
    });

    expect(resolved?.speed).toBe(4.5);
  });

  it("uses talk base url overrides when validating speech speed", () => {
    const provider = buildOpenAISpeechProvider();

    const resolvedConfig = provider.resolveTalkConfig?.({
      cfg: {} as never,
      timeoutMs: 30_000,
      baseTtsConfig: {
        providers: {
          openai: {
            apiKey: "sk-base",
          },
        },
      },
      talkProviderConfig: {
        baseUrl: "https://tts.example.com/v1",
        speed: 4.5,
      },
    });

    expect(resolvedConfig?.baseUrl).toBe("https://tts.example.com/v1");
    expect(resolvedConfig?.speed).toBe(4.5);
  });

  it("parses OpenAI directive tokens against the resolved base url", () => {
    const provider = buildOpenAISpeechProvider();

    expect(
      provider.parseDirectiveToken?.({
        key: "voice",
        value: "alloy",
        policy: {
          allowVoice: true,
          allowModelId: true,
        },
        providerConfig: {
          baseUrl: "https://api.openai.com/v1/",
        },
      } as never),
    ).toEqual({
      handled: true,
      overrides: { voice: "alloy" },
    });

    expect(
      provider.parseDirectiveToken?.({
        key: "openai_model",
        value: OPENAI_TTS_SNAPSHOT,
        policy: {
          allowVoice: true,
          allowModelId: true,
        },
        providerConfig: {
          baseUrl: "https://api.openai.com/v1/",
        },
      } as never),
    ).toEqual({
      handled: true,
      overrides: { model: OPENAI_TTS_SNAPSHOT },
    });

    expect(
      provider.parseDirectiveToken?.({
        key: "model",
        value: "kokoro-custom-model",
        policy: {
          allowVoice: true,
          allowModelId: true,
        },
        providerConfig: {
          baseUrl: "https://api.openai.com/v1/",
        },
      } as never),
    ).toEqual({
      handled: false,
    });
  });

  it("sends dated speech snapshots through a real loopback HTTP request", async () => {
    const provider = buildOpenAISpeechProvider();
    let receivedRequest:
      | {
          method: string | undefined;
          url: string | undefined;
          body: unknown;
        }
      | undefined;
    const server = createServer((request, response) => {
      const chunks: Buffer[] = [];
      request.on("data", (chunk: Buffer) => chunks.push(chunk));
      request.on("end", () => {
        receivedRequest = {
          method: request.method,
          url: request.url,
          body: JSON.parse(Buffer.concat(chunks).toString("utf8")) as unknown,
        };
        response.writeHead(200, { "content-type": "audio/mpeg" });
        response.end(Buffer.from("snapshot-audio"));
      });
    });
    await new Promise<void>((resolve, reject) => {
      server.once("error", reject);
      server.listen(0, "127.0.0.1", () => {
        server.removeListener("error", reject);
        resolve();
      });
    });

    try {
      const address = server.address();
      if (!address || typeof address === "string") {
        throw new Error("expected a loopback server address");
      }

      const result = await provider.synthesize({
        text: "snapshot request",
        cfg: {} as never,
        providerConfig: {
          apiKey: "sk-test",
          baseUrl: `http://127.0.0.1:${address.port}/v1`,
          model: OPENAI_TTS_SNAPSHOT,
          voice: "alloy",
          instructions: " Speak warmly ",
        },
        target: "audio-file",
        timeoutMs: 1_000,
      });

      expect(receivedRequest).toEqual({
        method: "POST",
        url: "/v1/audio/speech",
        body: {
          model: OPENAI_TTS_SNAPSHOT,
          input: "snapshot request",
          voice: "alloy",
          response_format: "mp3",
          instructions: "Speak warmly",
        },
      });
      expect(result.audioBuffer).toEqual(Buffer.from("snapshot-audio"));
    } finally {
      server.closeAllConnections();
      await new Promise<void>((resolve, reject) => {
        server.close((error) => (error ? reject(error) : resolve()));
      });
    }
  });

  it("parses preferred-OpenAI speed directive within the supported range", () => {
    const provider = buildOpenAISpeechProvider();

    expect(
      provider.parseDirectiveToken?.({
        key: "speed",
        value: "1.5",
        policy: {
          allowVoice: true,
          allowModelId: true,
          allowVoiceSettings: true,
        },
        providerConfig: {
          baseUrl: "https://api.openai.com/v1/",
        },
      } as never),
    ).toEqual({
      handled: true,
      overrides: { speed: 1.5 },
    });
  });

  it("parses explicit openai_speed alias", () => {
    const provider = buildOpenAISpeechProvider();

    expect(
      provider.parseDirectiveToken?.({
        key: "openai_speed",
        value: "0.75",
        policy: {
          allowVoice: true,
          allowModelId: true,
          allowVoiceSettings: true,
        },
        providerConfig: {
          baseUrl: "https://api.openai.com/v1/",
        },
      } as never),
    ).toEqual({
      handled: true,
      overrides: { speed: 0.75 },
    });
  });

  it("ignores OpenAI speed directives when allowVoiceSettings is disabled", () => {
    const provider = buildOpenAISpeechProvider();

    expect(
      provider.parseDirectiveToken?.({
        key: "speed",
        value: "1.5",
        policy: {
          allowVoice: true,
          allowModelId: true,
          allowVoiceSettings: false,
        },
        providerConfig: {
          baseUrl: "https://api.openai.com/v1/",
        },
      } as never),
    ).toEqual({
      handled: true,
    });
  });

  it("warns on non-numeric OpenAI speed values", () => {
    const provider = buildOpenAISpeechProvider();

    expect(
      provider.parseDirectiveToken?.({
        key: "speed",
        value: "fast",
        policy: {
          allowVoice: true,
          allowModelId: true,
          allowVoiceSettings: true,
        },
        providerConfig: {
          baseUrl: "https://api.openai.com/v1/",
        },
      } as never),
    ).toEqual({
      handled: true,
      warnings: ['invalid OpenAI speed "fast" (0.25-4.0)'],
    });
  });

  it("warns on partial OpenAI speed values", () => {
    const provider = buildOpenAISpeechProvider();

    expect(
      provider.parseDirectiveToken?.({
        key: "speed",
        value: "1.5abc",
        policy: {
          allowVoice: true,
          allowModelId: true,
          allowVoiceSettings: true,
        },
        providerConfig: {
          baseUrl: "https://api.openai.com/v1/",
        },
      } as never),
    ).toEqual({
      handled: true,
      warnings: ['invalid OpenAI speed "1.5abc" (0.25-4.0)'],
    });
  });

  it("warns on OpenAI speed values outside the supported 0.25..4 range", () => {
    const provider = buildOpenAISpeechProvider();

    expect(
      provider.parseDirectiveToken?.({
        key: "speed",
        value: "5",
        policy: {
          allowVoice: true,
          allowModelId: true,
          allowVoiceSettings: true,
        },
        providerConfig: {
          baseUrl: "https://api.openai.com/v1/",
        },
      } as never),
    ).toEqual({
      handled: true,
      warnings: ['invalid OpenAI speed "5" (0.25-4.0)'],
    });
  });

  it("passes custom endpoint OpenAI-compatible speed directives through", () => {
    const provider = buildOpenAISpeechProvider();

    expect(
      provider.parseDirectiveToken?.({
        key: "speed",
        value: "4.5",
        policy: {
          allowVoice: true,
          allowModelId: true,
          allowVoiceSettings: true,
        },
        providerConfig: {
          baseUrl: "https://tts.example.com/v1",
        },
      } as never),
    ).toEqual({
      handled: true,
      overrides: { speed: 4.5 },
    });
  });

  it("uses OPENAI_TTS_BASE_URL when parsing OpenAI-compatible speed directives", () => {
    const previousBaseUrl = process.env.OPENAI_TTS_BASE_URL;
    process.env.OPENAI_TTS_BASE_URL = "https://tts.example.com/v1";
    try {
      const provider = buildOpenAISpeechProvider();

      expect(
        provider.parseDirectiveToken?.({
          key: "speed",
          value: "4.5",
          policy: {
            allowVoice: true,
            allowModelId: true,
            allowVoiceSettings: true,
          },
          providerConfig: {},
        } as never),
      ).toEqual({
        handled: true,
        overrides: { speed: 4.5 },
      });
    } finally {
      if (previousBaseUrl === undefined) {
        delete process.env.OPENAI_TTS_BASE_URL;
      } else {
        process.env.OPENAI_TTS_BASE_URL = previousBaseUrl;
      }
    }
  });

  it("treats a blank environment API key as missing across speech entry points", async () => {
    vi.stubEnv("OPENAI_API_KEY", "   ");
    const provider = buildOpenAISpeechProvider();
    const fetchMock = vi.fn(async () => new Response(new Uint8Array([1, 2, 3]), { status: 200 }));
    globalThis.fetch = fetchMock as unknown as typeof fetch;
    const providerConfig = {
      model: "gpt-4o-mini-tts",
      voice: "alloy",
    };

    expect(provider.isConfigured({ providerConfig, timeoutMs: 30_000 })).toBe(false);
    await expect(
      provider.synthesize({
        text: "hello",
        cfg: {} as never,
        providerConfig,
        target: "audio-file",
        timeoutMs: 1_000,
      }),
    ).rejects.toThrow("OpenAI API key missing");
    await expect(
      provider.synthesizeTelephony?.({
        text: "hello",
        cfg: {} as never,
        providerConfig,
        timeoutMs: 1_000,
      }),
    ).rejects.toThrow("OpenAI API key missing");
    expect(fetchMock).not.toHaveBeenCalled();
  });

  it("trims a valid environment API key for normal and telephony synthesis", async () => {
    vi.stubEnv("OPENAI_API_KEY", "  sk-env  ");
    const provider = buildOpenAISpeechProvider();
    const authorizationHeaders: Array<string | null> = [];
    globalThis.fetch = vi.fn(async (_url: string, init?: RequestInit) => {
      authorizationHeaders.push(new Headers(init?.headers).get("authorization"));
      return new Response(new Uint8Array([1, 2, 3]), { status: 200 });
    }) as unknown as typeof fetch;
    const providerConfig = {
      model: "gpt-4o-mini-tts",
      voice: "alloy",
    };

    expect(provider.isConfigured({ providerConfig, timeoutMs: 30_000 })).toBe(true);
    await provider.synthesize({
      text: "hello",
      cfg: {} as never,
      providerConfig,
      target: "audio-file",
      timeoutMs: 1_000,
    });
    await provider.synthesizeTelephony?.({
      text: "hello",
      cfg: {} as never,
      providerConfig,
      timeoutMs: 1_000,
    });

    expect(authorizationHeaders).toEqual(["Bearer sk-env", "Bearer sk-env"]);
  });

  it("preserves talk responseFormat overrides", () => {
    const provider = buildOpenAISpeechProvider();

    const resolvedConfig = provider.resolveTalkConfig?.({
      cfg: {} as never,
      timeoutMs: 30_000,
      baseTtsConfig: {
        providers: {
          openai: {
            apiKey: "sk-base",
            responseFormat: "mp3",
          },
        },
      },
      talkProviderConfig: {
        apiKey: "sk-talk",
        responseFormat: " WAV ",
      },
    });
    expect(resolvedConfig?.apiKey).toBe("sk-talk");
    expect(resolvedConfig?.responseFormat).toBe("wav");
  });

  it("maps Talk speak params onto OpenAI speech overrides", () => {
    const provider = buildOpenAISpeechProvider();

    expect(
      provider.resolveTalkOverrides?.({
        talkProviderConfig: {},
        params: {
          text: "Hello from talk mode.",
          voiceId: "nova",
          modelId: "tts-1",
          speed: 218 / 175,
        },
      }),
    ).toEqual({
      voice: "nova",
      model: "tts-1",
      speed: 218 / 175,
    });
  });

  it("uses wav for Groq-compatible OpenAI TTS endpoints", async () => {
    const provider = buildOpenAISpeechProvider();
    mockSpeechFetchExpectingFormat("wav");

    const result = await provider.synthesize({
      text: "hello",
      cfg: {} as never,
      providerConfig: {
        apiKey: "sk-test",
        baseUrl: "https://api.groq.com/openai/v1",
        model: "canopylabs/orpheus-v1-english",
        voice: "daniel",
      },
      target: "audio-file",
      timeoutMs: 1_000,
    });

    expect(result.outputFormat).toBe("wav");
    expect(result.fileExtension).toBe(".wav");
    expect(result.voiceCompatible).toBe(false);
  });

  it("applies the configured media byte cap to synthesized audio", async () => {
    const provider = buildOpenAISpeechProvider();
    globalThis.fetch = vi.fn(
      async () => new Response(new Uint8Array(2048), { status: 200 }),
    ) as unknown as typeof fetch;

    await expect(
      provider.synthesize({
        text: "hello",
        cfg: {
          agents: {
            defaults: {
              mediaMaxMb: 0.001,
            },
          },
        } as never,
        providerConfig: {
          apiKey: "sk-test",
          model: "gpt-4o-mini-tts",
          voice: "alloy",
        },
        target: "audio-file",
        timeoutMs: 1_000,
      }),
    ).rejects.toThrow("OpenAI TTS audio response exceeds");
  });

  it("applies provider overrides to telephony synthesis", async () => {
    const provider = buildOpenAISpeechProvider();
    const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => {
      const body = parseRequestBody(init);
      expect(body.model).toBe("tts-1");
      expect(body.voice).toBe("nova");
      expect(body.speed).toBe(1.25);
      expect(body.response_format).toBe("pcm");
      return new Response(new Uint8Array([1, 2, 3]), { status: 200 });
    });
    globalThis.fetch = fetchMock as unknown as typeof fetch;

    const result = await provider.synthesizeTelephony?.({
      text: "hello",
      cfg: {} as never,
      providerConfig: {
        apiKey: "sk-test",
        model: "gpt-4o-mini-tts",
        voice: "alloy",
        speed: 1,
      },
      providerOverrides: {
        model: "tts-1",
        voice: "nova",
        speed: 1.25,
      },
      timeoutMs: 1_000,
    });

    expect(result?.outputFormat).toBe("pcm");
    expect(fetchMock).toHaveBeenCalledTimes(1);
  });

  it.each([
    { responseFormat: "wav" as const, voiceCompatible: false },
    { responseFormat: "mp3" as const, voiceCompatible: true },
  ])(
    "marks configured $responseFormat voice-note compatibility as $voiceCompatible",
    async ({ responseFormat, voiceCompatible }) => {
      const provider = buildOpenAISpeechProvider();
      mockSpeechFetchExpectingFormat(responseFormat);

      const result = await provider.synthesize({
        text: "hello",
        cfg: {} as never,
        providerConfig: {
          apiKey: "sk-test",
          model: "gpt-4o-mini-tts",
          voice: "alloy",
          responseFormat,
        },
        target: "voice-note",
        timeoutMs: 1_000,
      });

      expect(result.outputFormat).toBe(responseFormat);
      expect(result.fileExtension).toBe(`.${responseFormat}`);
      expect(result.voiceCompatible).toBe(voiceCompatible);
    },
  );

  it("passes extra_body config through to OpenAI-compatible speech requests", async () => {
    const provider = buildOpenAISpeechProvider();
    const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => {
      const body = parseRequestBody(init);
      expect(body.model).toBe("custom-tts");
      expect(body.voice).toBe("custom-voice");
      expect(body.lang).toBe("en-US");
      expect(body.response_format).toBe("mp3");
      return new Response(new Uint8Array([1, 2, 3]), { status: 200 });
    });
    globalThis.fetch = fetchMock as unknown as typeof fetch;

    const result = await provider.synthesize({
      text: "hello",
      cfg: {} as never,
      providerConfig: {
        apiKey: "sk-test",
        baseUrl: "https://proxy.example.com/openai/v1",
        model: "custom-tts",
        voice: "custom-voice",
        responseFormat: "mp3",
        extra_body: {
          lang: "en-US",
        },
      },
      target: "audio-file",
      timeoutMs: 1_000,
    });

    expect(result.outputFormat).toBe("mp3");
    expect(fetchMock).toHaveBeenCalledTimes(1);
  });
});