diff --git a/extensions/openai/realtime-transcription-provider.test.ts b/extensions/openai/realtime-transcription-provider.test.ts index b4af80d0c8ea..0abc30faee3b 100644 --- a/extensions/openai/realtime-transcription-provider.test.ts +++ b/extensions/openai/realtime-transcription-provider.test.ts @@ -1060,41 +1060,4 @@ describe("buildOpenAIRealtimeTranscriptionProvider", () => { ]); session.close(); }); - - it("fails before retaining an oversized completed transcript", async () => { - const onError = vi.fn(); - const onTranscript = vi.fn(); - const provider = buildOpenAIRealtimeTranscriptionProvider(); - const session = provider.createSession({ - providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret - onError, - onTranscript, - }); - const socket = await connectFakeSession(session); - - emitJson(socket, { - type: "input_audio_buffer.committed", - item_id: "item-1", - previous_item_id: null, - }); - emitJson(socket, { - type: "conversation.item.input_audio_transcription.completed", - item_id: "item-1", - transcript: "x".repeat(256 * 1024 + 1), - }); - emitJson(socket, { - type: "conversation.item.input_audio_transcription.completed", - item_id: "item-1", - transcript: "late transcript", - }); - - expect(onError).toHaveBeenCalledExactlyOnceWith( - expect.objectContaining({ - message: "OpenAI realtime transcription exceeded the 256 KiB retained transcript limit", - }), - ); - expect(onTranscript).not.toHaveBeenCalled(); - expect(session.isConnected()).toBe(false); - session.close(); - }); }); diff --git a/extensions/openai/realtime-voice-bridge-connection.test.ts b/extensions/openai/realtime-voice-bridge-connection.test.ts index 0d0ab8002813..e7036b1742b9 100644 --- a/extensions/openai/realtime-voice-bridge-connection.test.ts +++ b/extensions/openai/realtime-voice-bridge-connection.test.ts @@ -4,146 +4,31 @@ import type { RealtimeVoiceBridge } from "openclaw/plugin-sdk/realtime-voice"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js"; -const INTERNAL_REALTIME_VOICE_PROVIDER = Symbol.for("openclaw.internal.realtime-voice-provider.v1"); - -function readInternalRealtimeVoiceProviderApi(provider: object) { - return Reflect.get(provider, INTERNAL_REALTIME_VOICE_PROVIDER) as { - isBrowserSessionConfigured: (ctx: { - cfg?: object; - providerConfig: Record; - agentId?: string; - }) => boolean; - isGatewayRelayConfigured: (ctx: { - cfg?: object; - providerConfig: Record; - agentId?: string; - }) => boolean | undefined; - resolveBrowserSessionCapabilities: (ctx: { - cfg?: object; - providerConfig: Record; - model?: string; - }) => { - handlesAgentConsult?: boolean; - supportsToolCalls?: boolean; - supportsVideoFrames?: boolean; - supportsGatewayControl?: boolean; - transports?: string[]; - }; - resolveGatewayRelayCapabilities: (ctx: { - cfg?: object; - providerConfig: Record; - model?: string; - }) => { - handlesAgentConsult?: boolean; - supportsToolCalls?: boolean; - transports?: string[]; - }; - validateGatewayRelayLaunch: (ctx: { - cfg?: object; - providerConfig: Record; - model?: string; - autoRespondToAudio?: boolean; - }) => string | undefined; - cancelBrowserSession: (request: Record, session: object) => Promise; - }; -} - -const { - FakeWebSocket, - execFileSyncMock, - fetchWithSsrFGuardMock, - isProviderAuthProfileConfiguredMock, - resolveProviderAuthProfileApiKeyMock, -} = vi.hoisted(() => { - type Listener = (...args: unknown[]) => void; - - class MockWebSocket { - static readonly OPEN = 1; - static readonly CLOSED = 3; - static instances: MockWebSocket[] = []; - - readonly listeners = new Map(); - readyState = 0; - sent: string[] = []; - closed = false; - terminated = false; - deferClose = false; - deferredClose: (() => void) | undefined; - args: unknown[]; - - constructor(...args: unknown[]) { - this.args = args; - MockWebSocket.instances.push(this); - } - - on(event: string, listener: Listener): this { - const listeners = this.listeners.get(event) ?? []; - listeners.push(listener); - this.listeners.set(event, listeners); - return this; - } - - emit(event: string, ...args: unknown[]): void { - for (const listener of this.listeners.get(event) ?? []) { - listener(...args); - } - } - - send(payload: string): void { - this.sent.push(payload); - } - - close(code?: number, reason?: string): void { - this.closed = true; - this.readyState = MockWebSocket.CLOSED; - const emitClose = () => this.emit("close", code ?? 1000, Buffer.from(reason ?? "")); - if (this.deferClose) { - this.deferredClose = emitClose; - return; - } - emitClose(); - } - - terminate(): void { - this.terminated = true; - this.close(1006, "terminated"); - } - - emitDeferredClose(): void { - const emitClose = this.deferredClose; - this.deferredClose = undefined; - emitClose?.(); - } - } - - return { - FakeWebSocket: MockWebSocket, - execFileSyncMock: vi.fn(), - fetchWithSsrFGuardMock: vi.fn(), - isProviderAuthProfileConfiguredMock: vi.fn(), - resolveProviderAuthProfileApiKeyMock: vi.fn(), - }; +const mocks = await vi.hoisted(async () => { + const { createOpenAIRealtimeMockState } = await import("./realtime-voice-test-support.js"); + return createOpenAIRealtimeMockState(); }); +const { FakeWebSocket, fetchWithSsrFGuardMock } = mocks; vi.mock("node:child_process", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, - execFileSync: execFileSyncMock, + execFileSync: mocks.execFileSyncMock, }; }); vi.mock("ws", () => ({ - default: FakeWebSocket, + default: mocks.FakeWebSocket, })); vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({ - fetchWithSsrFGuard: fetchWithSsrFGuardMock, + fetchWithSsrFGuard: mocks.fetchWithSsrFGuardMock, })); vi.mock("openclaw/plugin-sdk/provider-auth", () => ({ - isProviderAuthProfileConfigured: isProviderAuthProfileConfiguredMock, - resolveProviderAuthProfileApiKey: resolveProviderAuthProfileApiKeyMock, + isProviderAuthProfileConfigured: mocks.isProviderAuthProfileConfiguredMock, + resolveProviderAuthProfileApiKey: mocks.resolveProviderAuthProfileApiKeyMock, })); import { createOpenAIRealtimeTestSupport } from "./realtime-voice-test-support.js"; @@ -164,23 +49,19 @@ const { createRealtimeTool, createUnreadableToolName, createMalformedToolName, -} = createOpenAIRealtimeTestSupport({ FakeWebSocket, fetchWithSsrFGuardMock }); + resetTestState, + restoreTestEnvironment, + readInternalRealtimeVoiceProviderApi, + createQuicksilverBrowserBrokerFixture, +} = createOpenAIRealtimeTestSupport({ ...mocks, buildOpenAIRealtimeVoiceProvider }); describe("OpenAI realtime voice bridge connection", () => { beforeEach(() => { - FakeWebSocket.instances = []; - vi.stubEnv("OPENAI_API_KEY", ""); - execFileSyncMock.mockReset(); - fetchWithSsrFGuardMock.mockReset(); - isProviderAuthProfileConfiguredMock.mockReset(); - isProviderAuthProfileConfiguredMock.mockReturnValue(false); - resolveProviderAuthProfileApiKeyMock.mockReset(); - resolveProviderAuthProfileApiKeyMock.mockResolvedValue(undefined); + resetTestState(); }); afterEach(() => { - vi.useRealTimers(); - vi.unstubAllEnvs(); + restoreTestEnvironment(); }); it("adds OpenClaw attribution headers to native realtime websocket requests", () => { @@ -209,21 +90,11 @@ describe("OpenAI realtime voice bridge connection", () => { }); it("sends one shared GA policy and waits for session.updated on an attached sideband", async () => { - const createBrowserSession = vi.fn( - async (_request: unknown, _auth: unknown) => - ({ - provider: "openai", - transport: "webrtc" as const, - clientSecret: "gateway-token", - offerUrl: "/plugins/openai/realtime/calls", - }) as const, - ); + const { broker, createBrowserSession } = createQuicksilverBrowserBrokerFixture({ + session: { clientSecret: "gateway-token" }, + }); const provider = buildOpenAIRealtimeVoiceProvider({ - quicksilverBrowserSessionBroker: { - capabilities: { handlesAgentConsult: true as const }, - createBrowserSession, - cancelBrowserSession: vi.fn(async () => undefined), - }, + quicksilverBrowserSessionBroker: broker, }); const bindBridge = vi.fn(); const onEvent = vi.fn(); @@ -620,15 +491,10 @@ describe("OpenAI realtime voice bridge connection", () => { const { connecting, socket } = beginBridgeConnection(bridge); openSocket(socket); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "error", - error: { message: "invalid realtime session" }, - }), - ), - ); + emitServerEvent(socket, { + type: "error", + error: { message: "invalid realtime session" }, + }); await expect(connecting).rejects.toThrow("invalid realtime session"); expect(bridge.isConnected()).toBe(false); @@ -677,42 +543,45 @@ describe("OpenAI realtime voice bridge connection", () => { expect(bridge.isConnected()).toBe(false); }); - it("can disable automatic audio turn responses for agent-routed voice loops", async () => { - const bridge = createNativeBridge({ + it.each([ + { + $name: "automatic audio turn responses disabled", autoRespondToAudio: false, - }); - const { connecting, socket } = beginBridgeConnection(bridge); - - openSocket(socket); - emitSessionUpdated(socket); - await connecting; - - expectRecordFields( - requireNestedRecord(requireSession(socket), ["audio", "input", "turn_detection"]), - "turn detection", - { - create_response: false, - interrupt_response: false, - }, - ); - }); - - it("can disable realtime response interruption while keeping audio responses enabled", async () => { - const bridge = createNativeBridge({ + interruptResponseOnInputAudio: false, + expectedCreateResponse: false, + expectedInterruptResponse: false, + }, + { + $name: "realtime response interruption disabled", autoRespondToAudio: true, interruptResponseOnInputAudio: false, - }); - const socket = await connectReadyBridge(bridge); + expectedCreateResponse: true, + expectedInterruptResponse: false, + }, + ])( + "$name", + async ({ + autoRespondToAudio, + interruptResponseOnInputAudio, + expectedCreateResponse, + expectedInterruptResponse, + }) => { + const bridge = createNativeBridge({ + autoRespondToAudio, + interruptResponseOnInputAudio, + }); + const socket = await connectReadyBridge(bridge); - expectRecordFields( - requireNestedRecord(requireSession(socket), ["audio", "input", "turn_detection"]), - "turn detection", - { - create_response: true, - interrupt_response: false, - }, - ); - }); + expectRecordFields( + requireNestedRecord(requireSession(socket), ["audio", "input", "turn_detection"]), + "turn detection", + { + create_response: expectedCreateResponse, + interrupt_response: expectedInterruptResponse, + }, + ); + }, + ); it("can request PCM16 24 kHz realtime audio for Chrome command-pair bridges", async () => { const bridge = createNativeBridge({ diff --git a/extensions/openai/realtime-voice-bridge-events.test.ts b/extensions/openai/realtime-voice-bridge-events.test.ts index c8f37527210c..66f3382f2278 100644 --- a/extensions/openai/realtime-voice-bridge-events.test.ts +++ b/extensions/openai/realtime-voice-bridge-events.test.ts @@ -1,103 +1,31 @@ // Openai tests cover realtime voice provider plugin behavior. import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js"; -const { - FakeWebSocket, - execFileSyncMock, - fetchWithSsrFGuardMock, - isProviderAuthProfileConfiguredMock, - resolveProviderAuthProfileApiKeyMock, -} = vi.hoisted(() => { - type Listener = (...args: unknown[]) => void; - - class MockWebSocket { - static readonly OPEN = 1; - static readonly CLOSED = 3; - static instances: MockWebSocket[] = []; - - readonly listeners = new Map(); - readyState = 0; - sent: string[] = []; - closed = false; - terminated = false; - deferClose = false; - deferredClose: (() => void) | undefined; - args: unknown[]; - - constructor(...args: unknown[]) { - this.args = args; - MockWebSocket.instances.push(this); - } - - on(event: string, listener: Listener): this { - const listeners = this.listeners.get(event) ?? []; - listeners.push(listener); - this.listeners.set(event, listeners); - return this; - } - - emit(event: string, ...args: unknown[]): void { - for (const listener of this.listeners.get(event) ?? []) { - listener(...args); - } - } - - send(payload: string): void { - this.sent.push(payload); - } - - close(code?: number, reason?: string): void { - this.closed = true; - this.readyState = MockWebSocket.CLOSED; - const emitClose = () => this.emit("close", code ?? 1000, Buffer.from(reason ?? "")); - if (this.deferClose) { - this.deferredClose = emitClose; - return; - } - emitClose(); - } - - terminate(): void { - this.terminated = true; - this.close(1006, "terminated"); - } - - emitDeferredClose(): void { - const emitClose = this.deferredClose; - this.deferredClose = undefined; - emitClose?.(); - } - } - - return { - FakeWebSocket: MockWebSocket, - execFileSyncMock: vi.fn(), - fetchWithSsrFGuardMock: vi.fn(), - isProviderAuthProfileConfiguredMock: vi.fn(), - resolveProviderAuthProfileApiKeyMock: vi.fn(), - }; +const mocks = await vi.hoisted(async () => { + const { createOpenAIRealtimeMockState } = await import("./realtime-voice-test-support.js"); + return createOpenAIRealtimeMockState(); }); - vi.mock("node:child_process", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, - execFileSync: execFileSyncMock, + execFileSync: mocks.execFileSyncMock, }; }); vi.mock("ws", () => ({ - default: FakeWebSocket, + default: mocks.FakeWebSocket, })); vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({ - fetchWithSsrFGuard: fetchWithSsrFGuardMock, + fetchWithSsrFGuard: mocks.fetchWithSsrFGuardMock, })); vi.mock("openclaw/plugin-sdk/provider-auth", () => ({ - isProviderAuthProfileConfigured: isProviderAuthProfileConfiguredMock, - resolveProviderAuthProfileApiKey: resolveProviderAuthProfileApiKeyMock, + isProviderAuthProfileConfigured: mocks.isProviderAuthProfileConfiguredMock, + resolveProviderAuthProfileApiKey: mocks.resolveProviderAuthProfileApiKeyMock, })); import { createOpenAIRealtimeTestSupport } from "./realtime-voice-test-support.js"; @@ -105,91 +33,40 @@ const { parseSent, createNativeBridge, connectReadyBridge, + emitServerEvent, + emitAssistantPlayback, expectedResponseCancelEvent, hasSentEventType, -} = createOpenAIRealtimeTestSupport({ FakeWebSocket, fetchWithSsrFGuardMock }); + resetTestState, + restoreTestEnvironment, +} = createOpenAIRealtimeTestSupport({ ...mocks, buildOpenAIRealtimeVoiceProvider }); describe("OpenAI realtime voice bridge events", () => { beforeEach(() => { - FakeWebSocket.instances = []; - vi.stubEnv("OPENAI_API_KEY", ""); - execFileSyncMock.mockReset(); - fetchWithSsrFGuardMock.mockReset(); - isProviderAuthProfileConfiguredMock.mockReset(); - isProviderAuthProfileConfiguredMock.mockReturnValue(false); - resolveProviderAuthProfileApiKeyMock.mockReset(); - resolveProviderAuthProfileApiKeyMock.mockResolvedValue(undefined); + resetTestState(); }); afterEach(() => { - vi.useRealTimers(); - vi.unstubAllEnvs(); + restoreTestEnvironment(); }); - it("does not locally clear playback on speech-start events when input interruption is disabled", async () => { + it.each([ + { + $name: "input interruption disabled", + bridgeOptions: { autoRespondToAudio: true, interruptResponseOnInputAudio: false }, + }, + { + $name: "automatic audio responses disabled", + bridgeOptions: { autoRespondToAudio: false }, + }, + ])("$name", async ({ bridgeOptions }) => { const onAudio = vi.fn(); const onClearAudio = vi.fn(); - const bridge = createNativeBridge({ - autoRespondToAudio: true, - interruptResponseOnInputAudio: false, - onAudio, - onClearAudio, - }); + const bridge = createNativeBridge({ ...bridgeOptions, onAudio, onClearAudio }); const socket = await connectReadyBridge(bridge); - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })), - ); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "response.audio.delta", - item_id: "item_1", - delta: Buffer.from("assistant audio").toString("base64"), - }), - ), - ); - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "input_audio_buffer.speech_started" })), - ); - - expect(onAudio).toHaveBeenCalledTimes(1); - expect(onClearAudio).not.toHaveBeenCalled(); - expect(hasSentEventType(socket, "response.cancel")).toBe(false); - expect(hasSentEventType(socket, "conversation.item.truncate")).toBe(false); - }); - - it("keeps assistant playback active on server VAD when automatic audio responses are disabled", async () => { - const onAudio = vi.fn(); - const onClearAudio = vi.fn(); - const bridge = createNativeBridge({ - autoRespondToAudio: false, - onAudio, - onClearAudio, - }); - const socket = await connectReadyBridge(bridge); - - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })), - ); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "response.audio.delta", - item_id: "item_1", - delta: Buffer.from("assistant audio").toString("base64"), - }), - ), - ); - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "input_audio_buffer.speech_started" })), - ); + emitAssistantPlayback(socket); + emitServerEvent(socket, { type: "input_audio_buffer.speech_started" }); expect(onAudio).toHaveBeenCalledTimes(1); expect(onClearAudio).not.toHaveBeenCalled(); @@ -208,20 +85,7 @@ describe("OpenAI realtime voice bridge events", () => { const socket = await connectReadyBridge(bridge); bridge.setMediaTimestamp(1000); - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })), - ); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "response.audio.delta", - item_id: "item_1", - delta: Buffer.from("assistant audio").toString("base64"), - }), - ), - ); + emitAssistantPlayback(socket); bridge.setMediaTimestamp(1300); bridge.handleBargeIn?.({ audioPlaybackActive: true }); @@ -250,16 +114,11 @@ describe("OpenAI realtime voice bridge events", () => { bridge.setMediaTimestamp(1000); for (let index = 0; index < 300; index += 1) { - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "response.audio.delta", - item_id: "item_1", - delta: Buffer.from("assistant audio").toString("base64"), - }), - ), - ); + emitServerEvent(socket, { + type: "response.audio.delta", + item_id: "item_1", + delta: Buffer.from("assistant audio").toString("base64"), + }); } const marks = onMark.mock.calls.map(([markName]) => String(markName)); @@ -281,16 +140,11 @@ describe("OpenAI realtime voice bridge events", () => { expect(onClearAudio).toHaveBeenCalledWith("barge-in"); for (let index = 0; index < 300; index += 1) { - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "response.audio.delta", - item_id: "item_1", - delta: Buffer.from("assistant audio").toString("base64"), - }), - ), - ); + emitServerEvent(socket, { + type: "response.audio.delta", + item_id: "item_1", + delta: Buffer.from("assistant audio").toString("base64"), + }); } const latestMark = onMark.mock.calls.at(-1)?.[0]; if (typeof latestMark !== "string") { @@ -313,16 +167,11 @@ describe("OpenAI realtime voice bridge events", () => { bridge.setMediaTimestamp(1000); for (let index = 0; index < 3; index += 1) { - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "response.audio.delta", - item_id: "item_1", - delta: Buffer.from("assistant audio").toString("base64"), - }), - ), - ); + emitServerEvent(socket, { + type: "response.audio.delta", + item_id: "item_1", + delta: Buffer.from("assistant audio").toString("base64"), + }); } const marks = onMark.mock.calls.map(([markName]) => String(markName)); expect(marks).toHaveLength(3); @@ -349,25 +198,15 @@ describe("OpenAI realtime voice bridge events", () => { const socket = await connectReadyBridge(bridge); const audio = Buffer.from("assistant audio"); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "response.output_audio.delta", - item_id: "item_1", - delta: audio.toString("base64"), - }), - ), - ); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "response.output_audio_transcript.done", - transcript: "hello from current realtime events", - }), - ), - ); + emitServerEvent(socket, { + type: "response.output_audio.delta", + item_id: "item_1", + delta: audio.toString("base64"), + }); + emitServerEvent(socket, { + type: "response.output_audio_transcript.done", + transcript: "hello from current realtime events", + }); expect(onAudio).toHaveBeenCalledWith(audio); expect(onTranscript).toHaveBeenCalledWith( @@ -383,16 +222,11 @@ describe("OpenAI realtime voice bridge events", () => { const bridge = createNativeBridge({ onError, onEvent }); const socket = await connectReadyBridge(bridge); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "conversation.item.input_audio_transcription.failed", - item_id: "item_speech", - error: { code: "decoder_failure", message: "speech decoder exploded" }, - }), - ), - ); + emitServerEvent(socket, { + type: "conversation.item.input_audio_transcription.failed", + item_id: "item_speech", + error: { code: "decoder_failure", message: "speech decoder exploded" }, + }); expect(onError).toHaveBeenCalledWith( expect.objectContaining({ message: "speech decoder exploded" }), @@ -410,14 +244,8 @@ describe("OpenAI realtime voice bridge events", () => { const bridge = createNativeBridge({ onTranscript }); const socket = await connectReadyBridge(bridge); - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "response.text.delta", delta: "draft assistant" })), - ); - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "response.text.done", text: "corrected assistant" })), - ); + emitServerEvent(socket, { type: "response.text.delta", delta: "draft assistant" }); + emitServerEvent(socket, { type: "response.text.done", text: "corrected assistant" }); expect(onTranscript.mock.calls).toEqual([ ["assistant", "draft assistant", false], @@ -439,16 +267,7 @@ describe("OpenAI realtime voice bridge events", () => { }); const socket = await connectReadyBridge(bridge); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "response.output_audio.delta", - item_id: "item_1", - delta, - }), - ), - ); + emitServerEvent(socket, { type: "response.output_audio.delta", item_id: "item_1", delta }); expect(onAudio).not.toHaveBeenCalled(); expect(onError).toHaveBeenCalledWith( @@ -473,44 +292,24 @@ describe("OpenAI realtime voice bridge events", () => { const socket = await connectReadyBridge(bridge); const audio = Buffer.from("legacy assistant audio"); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "conversation.output_audio.delta", - data: audio.toString("base64"), - sample_rate: 24000, - channels: 1, - }), - ), - ); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "conversation.input_transcript.delta", - delta: "partial user", - }), - ), - ); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "conversation.output_transcript.delta", - delta: "partial assistant", - }), - ), - ); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "response.output_text.done", - text: "final assistant text", - }), - ), - ); + emitServerEvent(socket, { + type: "conversation.output_audio.delta", + data: audio.toString("base64"), + sample_rate: 24000, + channels: 1, + }); + emitServerEvent(socket, { + type: "conversation.input_transcript.delta", + delta: "partial user", + }); + emitServerEvent(socket, { + type: "conversation.output_transcript.delta", + delta: "partial assistant", + }); + emitServerEvent(socket, { + type: "response.output_text.done", + text: "final assistant text", + }); expect(onAudio).toHaveBeenCalledWith(audio); expect(onTranscript).toHaveBeenCalledWith("user", "partial user", false); @@ -522,21 +321,13 @@ describe("OpenAI realtime voice bridge events", () => { const onEvent = vi.fn(); const bridge = createNativeBridge({ onEvent }); const socket = await connectReadyBridge(bridge); - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })), - ); + emitServerEvent(socket, { type: "response.created", response: { id: "resp_1" } }); bridge.setMediaTimestamp(1000); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "response.audio.delta", - item_id: "item_1", - delta: Buffer.from("assistant audio").toString("base64"), - }), - ), - ); + emitServerEvent(socket, { + type: "response.audio.delta", + item_id: "item_1", + delta: Buffer.from("assistant audio").toString("base64"), + }); bridge.setMediaTimestamp(1300); bridge.handleBargeIn?.({ audioPlaybackActive: true }); @@ -564,20 +355,7 @@ describe("OpenAI realtime voice bridge events", () => { }); const socket = await connectReadyBridge(bridge); bridge.setMediaTimestamp(1000); - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })), - ); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "response.audio.delta", - item_id: "item_1", - delta: Buffer.from("assistant audio").toString("base64"), - }), - ), - ); + emitAssistantPlayback(socket); bridge.handleBargeIn?.({ audioPlaybackActive: true }); @@ -602,20 +380,7 @@ describe("OpenAI realtime voice bridge events", () => { }); const socket = await connectReadyBridge(bridge); bridge.setMediaTimestamp(1000); - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })), - ); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "response.audio.delta", - item_id: "item_1", - delta: Buffer.from("assistant audio").toString("base64"), - }), - ), - ); + emitAssistantPlayback(socket); bridge.handleBargeIn?.({ audioPlaybackActive: true, force: true }); @@ -647,20 +412,7 @@ describe("OpenAI realtime voice bridge events", () => { }); const socket = await connectReadyBridge(bridge); bridge.setMediaTimestamp(1000); - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })), - ); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "response.audio.delta", - item_id: "item_1", - delta: Buffer.from("assistant audio").toString("base64"), - }), - ), - ); + emitAssistantPlayback(socket); bridge.handleBargeIn?.({ audioPlaybackActive: true }); diff --git a/extensions/openai/realtime-voice-bridge-reconnect.test.ts b/extensions/openai/realtime-voice-bridge-reconnect.test.ts index 4ac6a8c0264b..a6740d7afabc 100644 --- a/extensions/openai/realtime-voice-bridge-reconnect.test.ts +++ b/extensions/openai/realtime-voice-bridge-reconnect.test.ts @@ -1,106 +1,33 @@ // Openai tests cover realtime voice provider plugin behavior. import type { RealtimeVoiceBridgeEvent } from "openclaw/plugin-sdk/realtime-voice"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js"; -const OPENAI_REALTIME_REJECTED_KEY_MESSAGE = - "OpenAI Realtime rejected the selected API key. Update or remove the active OpenAI API-key source"; - -const { - FakeWebSocket, - execFileSyncMock, - fetchWithSsrFGuardMock, - isProviderAuthProfileConfiguredMock, - resolveProviderAuthProfileApiKeyMock, -} = vi.hoisted(() => { - type Listener = (...args: unknown[]) => void; - - class MockWebSocket { - static readonly OPEN = 1; - static readonly CLOSED = 3; - static instances: MockWebSocket[] = []; - - readonly listeners = new Map(); - readyState = 0; - sent: string[] = []; - closed = false; - terminated = false; - deferClose = false; - deferredClose: (() => void) | undefined; - args: unknown[]; - - constructor(...args: unknown[]) { - this.args = args; - MockWebSocket.instances.push(this); - } - - on(event: string, listener: Listener): this { - const listeners = this.listeners.get(event) ?? []; - listeners.push(listener); - this.listeners.set(event, listeners); - return this; - } - - emit(event: string, ...args: unknown[]): void { - for (const listener of this.listeners.get(event) ?? []) { - listener(...args); - } - } - - send(payload: string): void { - this.sent.push(payload); - } - - close(code?: number, reason?: string): void { - this.closed = true; - this.readyState = MockWebSocket.CLOSED; - const emitClose = () => this.emit("close", code ?? 1000, Buffer.from(reason ?? "")); - if (this.deferClose) { - this.deferredClose = emitClose; - return; - } - emitClose(); - } - - terminate(): void { - this.terminated = true; - this.close(1006, "terminated"); - } - - emitDeferredClose(): void { - const emitClose = this.deferredClose; - this.deferredClose = undefined; - emitClose?.(); - } - } - - return { - FakeWebSocket: MockWebSocket, - execFileSyncMock: vi.fn(), - fetchWithSsrFGuardMock: vi.fn(), - isProviderAuthProfileConfiguredMock: vi.fn(), - resolveProviderAuthProfileApiKeyMock: vi.fn(), - }; +const mocks = await vi.hoisted(async () => { + const { createOpenAIRealtimeMockState } = await import("./realtime-voice-test-support.js"); + return createOpenAIRealtimeMockState(); }); +const { FakeWebSocket } = mocks; vi.mock("node:child_process", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, - execFileSync: execFileSyncMock, + execFileSync: mocks.execFileSyncMock, }; }); vi.mock("ws", () => ({ - default: FakeWebSocket, + default: mocks.FakeWebSocket, })); vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({ - fetchWithSsrFGuard: fetchWithSsrFGuardMock, + fetchWithSsrFGuard: mocks.fetchWithSsrFGuardMock, })); vi.mock("openclaw/plugin-sdk/provider-auth", () => ({ - isProviderAuthProfileConfigured: isProviderAuthProfileConfiguredMock, - resolveProviderAuthProfileApiKey: resolveProviderAuthProfileApiKeyMock, + isProviderAuthProfileConfigured: mocks.isProviderAuthProfileConfiguredMock, + resolveProviderAuthProfileApiKey: mocks.resolveProviderAuthProfileApiKeyMock, })); import { createOpenAIRealtimeTestSupport } from "./realtime-voice-test-support.js"; @@ -115,23 +42,18 @@ const { emitCompletedToolCalls, emitFunctionOutputAdded, connectReadyBridge, -} = createOpenAIRealtimeTestSupport({ FakeWebSocket, fetchWithSsrFGuardMock }); + resetTestState, + restoreTestEnvironment, + rejectedKeyMessage: OPENAI_REALTIME_REJECTED_KEY_MESSAGE, +} = createOpenAIRealtimeTestSupport({ ...mocks, buildOpenAIRealtimeVoiceProvider }); describe("OpenAI realtime voice bridge reconnect", () => { beforeEach(() => { - FakeWebSocket.instances = []; - vi.stubEnv("OPENAI_API_KEY", ""); - execFileSyncMock.mockReset(); - fetchWithSsrFGuardMock.mockReset(); - isProviderAuthProfileConfiguredMock.mockReset(); - isProviderAuthProfileConfiguredMock.mockReturnValue(false); - resolveProviderAuthProfileApiKeyMock.mockReset(); - resolveProviderAuthProfileApiKeyMock.mockResolvedValue(undefined); + resetTestState(); }); afterEach(() => { - vi.useRealTimers(); - vi.unstubAllEnvs(); + restoreTestEnvironment(); }); it("rotates realtime bridges on provider max-duration events without reporting an error", async () => { @@ -147,15 +69,10 @@ describe("OpenAI realtime voice bridge reconnect", () => { await connecting; expect(onReady).toHaveBeenCalledOnce(); - firstSocket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "error", - error: { message: "Your session hit the maximum duration of 60 minutes." }, - }), - ), - ); + emitServerEvent(firstSocket, { + type: "error", + error: { message: "Your session hit the maximum duration of 60 minutes." }, + }); expect(onError).not.toHaveBeenCalled(); expect(firstSocket.closed).toBe(true); @@ -394,15 +311,10 @@ describe("OpenAI realtime voice bridge reconnect", () => { firstSocket.readyState = FakeWebSocket.CLOSED; firstSocket.emit("close", 1006, Buffer.from("transient drop")); - firstSocket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "response.audio.delta", - delta: Buffer.from("late audio").toString("base64"), - }), - ), - ); + emitServerEvent(firstSocket, { + type: "response.audio.delta", + delta: Buffer.from("late audio").toString("base64"), + }); firstSocket.emit("error", new Error("late retry-wait failure")); expect(onAudio).not.toHaveBeenCalled(); expect(onError).not.toHaveBeenCalled(); diff --git a/extensions/openai/realtime-voice-bridge-tools.test.ts b/extensions/openai/realtime-voice-bridge-tools.test.ts index 12cc220588bc..d485afff5e23 100644 --- a/extensions/openai/realtime-voice-bridge-tools.test.ts +++ b/extensions/openai/realtime-voice-bridge-tools.test.ts @@ -1,102 +1,30 @@ // Openai tests cover realtime voice provider plugin behavior. import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js"; -const { - FakeWebSocket, - execFileSyncMock, - fetchWithSsrFGuardMock, - isProviderAuthProfileConfiguredMock, - resolveProviderAuthProfileApiKeyMock, -} = vi.hoisted(() => { - type Listener = (...args: unknown[]) => void; - - class MockWebSocket { - static readonly OPEN = 1; - static readonly CLOSED = 3; - static instances: MockWebSocket[] = []; - - readonly listeners = new Map(); - readyState = 0; - sent: string[] = []; - closed = false; - terminated = false; - deferClose = false; - deferredClose: (() => void) | undefined; - args: unknown[]; - - constructor(...args: unknown[]) { - this.args = args; - MockWebSocket.instances.push(this); - } - - on(event: string, listener: Listener): this { - const listeners = this.listeners.get(event) ?? []; - listeners.push(listener); - this.listeners.set(event, listeners); - return this; - } - - emit(event: string, ...args: unknown[]): void { - for (const listener of this.listeners.get(event) ?? []) { - listener(...args); - } - } - - send(payload: string): void { - this.sent.push(payload); - } - - close(code?: number, reason?: string): void { - this.closed = true; - this.readyState = MockWebSocket.CLOSED; - const emitClose = () => this.emit("close", code ?? 1000, Buffer.from(reason ?? "")); - if (this.deferClose) { - this.deferredClose = emitClose; - return; - } - emitClose(); - } - - terminate(): void { - this.terminated = true; - this.close(1006, "terminated"); - } - - emitDeferredClose(): void { - const emitClose = this.deferredClose; - this.deferredClose = undefined; - emitClose?.(); - } - } - - return { - FakeWebSocket: MockWebSocket, - execFileSyncMock: vi.fn(), - fetchWithSsrFGuardMock: vi.fn(), - isProviderAuthProfileConfiguredMock: vi.fn(), - resolveProviderAuthProfileApiKeyMock: vi.fn(), - }; +const mocks = await vi.hoisted(async () => { + const { createOpenAIRealtimeMockState } = await import("./realtime-voice-test-support.js"); + return createOpenAIRealtimeMockState(); }); - vi.mock("node:child_process", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, - execFileSync: execFileSyncMock, + execFileSync: mocks.execFileSyncMock, }; }); vi.mock("ws", () => ({ - default: FakeWebSocket, + default: mocks.FakeWebSocket, })); vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({ - fetchWithSsrFGuard: fetchWithSsrFGuardMock, + fetchWithSsrFGuard: mocks.fetchWithSsrFGuardMock, })); vi.mock("openclaw/plugin-sdk/provider-auth", () => ({ - isProviderAuthProfileConfigured: isProviderAuthProfileConfiguredMock, - resolveProviderAuthProfileApiKey: resolveProviderAuthProfileApiKeyMock, + isProviderAuthProfileConfigured: mocks.isProviderAuthProfileConfiguredMock, + resolveProviderAuthProfileApiKey: mocks.resolveProviderAuthProfileApiKeyMock, })); import { createOpenAIRealtimeTestSupport } from "./realtime-voice-test-support.js"; @@ -110,23 +38,17 @@ const { connectReadyBridge, expectedResponseCreateEvent, hasSentEventType, -} = createOpenAIRealtimeTestSupport({ FakeWebSocket, fetchWithSsrFGuardMock }); + resetTestState, + restoreTestEnvironment, +} = createOpenAIRealtimeTestSupport({ ...mocks, buildOpenAIRealtimeVoiceProvider }); describe("OpenAI realtime voice bridge tools", () => { beforeEach(() => { - FakeWebSocket.instances = []; - vi.stubEnv("OPENAI_API_KEY", ""); - execFileSyncMock.mockReset(); - fetchWithSsrFGuardMock.mockReset(); - isProviderAuthProfileConfiguredMock.mockReset(); - isProviderAuthProfileConfiguredMock.mockReturnValue(false); - resolveProviderAuthProfileApiKeyMock.mockReset(); - resolveProviderAuthProfileApiKeyMock.mockResolvedValue(undefined); + resetTestState(); }); afterEach(() => { - vi.useRealTimers(); - vi.unstubAllEnvs(); + restoreTestEnvironment(); }); it("executes tool calls only from successful response output", async () => { @@ -189,35 +111,6 @@ describe("OpenAI realtime voice bridge tools", () => { }); }); - it.each(["cancelled", "failed", "incomplete"])( - "ignores function calls from a %s response", - async (status) => { - const onToolCall = vi.fn(); - const bridge = createNativeBridge({ onToolCall }); - const socket = await connectReadyBridge(bridge); - - emitServerEvent(socket, { - type: "response.done", - response: { - id: "response_1", - status, - output: [ - { - id: "item_tool_1", - type: "function_call", - status: "completed", - name: "openclaw_agent_consult", - call_id: "call_1", - arguments: '{"question":"must stay inert"}', - }, - ], - }, - }); - - expect(onToolCall).not.toHaveBeenCalled(); - }, - ); - it("ignores malformed and unfinished response output items", async () => { const onToolCall = vi.fn(); const bridge = createNativeBridge({ onToolCall }); @@ -591,10 +484,7 @@ describe("OpenAI realtime voice bridge tools", () => { type: "conversation.item.done", detail: "itemType=function_call_output", }); - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_2" } })), - ); + emitServerEvent(socket, { type: "response.created", response: { id: "resp_2" } }); emitServerEvent(socket, { type: "response.done" }); expect(parseSent(socket).filter((event) => event.type === "response.create")).toHaveLength(1); @@ -675,10 +565,10 @@ describe("OpenAI realtime voice bridge tools", () => { expect(onError).not.toHaveBeenCalled(); expect(parseSent(socket).filter((event) => event.type === "response.create")).toHaveLength(1); - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_status" } })), - ); + emitServerEvent(socket, { + type: "response.created", + response: { id: "resp_status" }, + }); emitServerEvent(socket, { type: "response.done", response: { id: "resp_status", status: "completed", output: [] }, diff --git a/extensions/openai/realtime-voice-browser-auth.test.ts b/extensions/openai/realtime-voice-browser-auth.test.ts index 7f37fb805a69..e471350f0a57 100644 --- a/extensions/openai/realtime-voice-browser-auth.test.ts +++ b/extensions/openai/realtime-voice-browser-auth.test.ts @@ -2,105 +2,37 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js"; -const OPENAI_REALTIME_REJECTED_KEY_MESSAGE = - "OpenAI Realtime rejected the selected API key. Update or remove the active OpenAI API-key source"; - +const mocks = await vi.hoisted(async () => { + const { createOpenAIRealtimeMockState } = await import("./realtime-voice-test-support.js"); + return createOpenAIRealtimeMockState(); +}); const { FakeWebSocket, execFileSyncMock, fetchWithSsrFGuardMock, isProviderAuthProfileConfiguredMock, resolveProviderAuthProfileApiKeyMock, -} = vi.hoisted(() => { - type Listener = (...args: unknown[]) => void; - - class MockWebSocket { - static readonly OPEN = 1; - static readonly CLOSED = 3; - static instances: MockWebSocket[] = []; - - readonly listeners = new Map(); - readyState = 0; - sent: string[] = []; - closed = false; - terminated = false; - deferClose = false; - deferredClose: (() => void) | undefined; - args: unknown[]; - - constructor(...args: unknown[]) { - this.args = args; - MockWebSocket.instances.push(this); - } - - on(event: string, listener: Listener): this { - const listeners = this.listeners.get(event) ?? []; - listeners.push(listener); - this.listeners.set(event, listeners); - return this; - } - - emit(event: string, ...args: unknown[]): void { - for (const listener of this.listeners.get(event) ?? []) { - listener(...args); - } - } - - send(payload: string): void { - this.sent.push(payload); - } - - close(code?: number, reason?: string): void { - this.closed = true; - this.readyState = MockWebSocket.CLOSED; - const emitClose = () => this.emit("close", code ?? 1000, Buffer.from(reason ?? "")); - if (this.deferClose) { - this.deferredClose = emitClose; - return; - } - emitClose(); - } - - terminate(): void { - this.terminated = true; - this.close(1006, "terminated"); - } - - emitDeferredClose(): void { - const emitClose = this.deferredClose; - this.deferredClose = undefined; - emitClose?.(); - } - } - - return { - FakeWebSocket: MockWebSocket, - execFileSyncMock: vi.fn(), - fetchWithSsrFGuardMock: vi.fn(), - isProviderAuthProfileConfiguredMock: vi.fn(), - resolveProviderAuthProfileApiKeyMock: vi.fn(), - }; -}); +} = mocks; vi.mock("node:child_process", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, - execFileSync: execFileSyncMock, + execFileSync: mocks.execFileSyncMock, }; }); vi.mock("ws", () => ({ - default: FakeWebSocket, + default: mocks.FakeWebSocket, })); vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({ - fetchWithSsrFGuard: fetchWithSsrFGuardMock, + fetchWithSsrFGuard: mocks.fetchWithSsrFGuardMock, })); vi.mock("openclaw/plugin-sdk/provider-auth", () => ({ - isProviderAuthProfileConfigured: isProviderAuthProfileConfiguredMock, - resolveProviderAuthProfileApiKey: resolveProviderAuthProfileApiKeyMock, + isProviderAuthProfileConfigured: mocks.isProviderAuthProfileConfiguredMock, + resolveProviderAuthProfileApiKey: mocks.resolveProviderAuthProfileApiKeyMock, })); import { createOpenAIRealtimeTestSupport } from "./realtime-voice-test-support.js"; @@ -108,6 +40,7 @@ const { createNativeBridge, beginBridgeConnection, openSocket, + emitServerEvent, createJsonResponse, requireRecord, requireNestedRecord, @@ -118,23 +51,20 @@ const { requireFetchHeaders, requireFetchJsonBody, createTestJwt, -} = createOpenAIRealtimeTestSupport({ FakeWebSocket, fetchWithSsrFGuardMock }); + resetTestState, + restoreTestEnvironment, + mockRealtimeClientSecretResponse, + rejectedKeyMessage: OPENAI_REALTIME_REJECTED_KEY_MESSAGE, + createQuicksilverBrowserBrokerFixture, +} = createOpenAIRealtimeTestSupport({ ...mocks, buildOpenAIRealtimeVoiceProvider }); describe("OpenAI realtime voice browser authentication", () => { beforeEach(() => { - FakeWebSocket.instances = []; - vi.stubEnv("OPENAI_API_KEY", ""); - execFileSyncMock.mockReset(); - fetchWithSsrFGuardMock.mockReset(); - isProviderAuthProfileConfiguredMock.mockReset(); - isProviderAuthProfileConfiguredMock.mockReturnValue(false); - resolveProviderAuthProfileApiKeyMock.mockReset(); - resolveProviderAuthProfileApiKeyMock.mockResolvedValue(undefined); + resetTestState(); }); afterEach(() => { - vi.useRealTimers(); - vi.unstubAllEnvs(); + restoreTestEnvironment(); }); it("requires Platform auth for native realtime websocket bridges", async () => { @@ -154,35 +84,75 @@ describe("OpenAI realtime voice browser authentication", () => { expect(FakeWebSocket.instances).toHaveLength(0); }); - it("uses OPENAI_API_KEY for default GPT realtime bridges", async () => { - vi.stubEnv("OPENAI_API_KEY", "test-api-key-env"); - const provider = buildOpenAIRealtimeVoiceProvider(); - const bridge = provider.createBridge({ - cfg: {} as never, - providerConfig: { model: "gpt-realtime-2" }, - onAudio: vi.fn(), - onClearAudio: vi.fn(), - }); + it.each([ + { + $name: "environment API key", + environmentKey: "test-api-key-env", + profileKey: undefined, + configuredProfile: false, + expectedAuthorization: "Bearer test-api-key-env", + assertion: "environment" as const, + }, + { + $name: "API-key profile", + environmentKey: undefined, + profileKey: "test-api-key-profile", + configuredProfile: false, + expectedAuthorization: "Bearer test-api-key-profile", + assertion: "profile" as const, + }, + { + $name: "environment fallback after an unresolved configured profile", + environmentKey: "test-api-key-env", + profileKey: undefined, + configuredProfile: true, + expectedAuthorization: "Bearer test-api-key-env", + assertion: "fallback" as const, + }, + ])( + "$name", + async ({ environmentKey, profileKey, configuredProfile, expectedAuthorization, assertion }) => { + if (environmentKey) { + vi.stubEnv("OPENAI_API_KEY", environmentKey); + } + resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce(profileKey); + if (configuredProfile) { + isProviderAuthProfileConfiguredMock.mockReturnValueOnce(true); + } + const provider = buildOpenAIRealtimeVoiceProvider(); + const bridge = provider.createBridge({ + cfg: {} as never, + providerConfig: { model: "gpt-realtime-2" }, + onAudio: vi.fn(), + onClearAudio: vi.fn(), + }); - void bridge.connect(); - await vi.waitFor(() => expect(FakeWebSocket.instances.length).toBe(1)); - bridge.close(); + void bridge.connect(); + await vi.waitFor(() => expect(FakeWebSocket.instances.length).toBe(1)); + bridge.close(); - expect(resolveProviderAuthProfileApiKeyMock.mock.calls).toEqual([ - [ - { - provider: "openai", - cfg: {}, - profileTypes: ["api_key"], - includeExternalCliAuth: false, - }, - ], - ]); - expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled(); - const socket = FakeWebSocket.instances[0]; - const options = socket?.args[1] as { headers?: Record } | undefined; - expect(options?.headers?.Authorization).toBe("Bearer test-api-key-env"); - }); + if (assertion === "fallback") { + expect(resolveProviderAuthProfileApiKeyMock).toHaveBeenCalledTimes(1); + } else { + expect(resolveProviderAuthProfileApiKeyMock.mock.calls).toEqual([ + [ + { + provider: "openai", + cfg: {}, + profileTypes: ["api_key"], + includeExternalCliAuth: false, + }, + ], + ]); + } + if (assertion === "environment") { + expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled(); + } + const socket = FakeWebSocket.instances[0]; + const options = socket?.args[1] as { headers?: Record } | undefined; + expect(options?.headers?.Authorization).toBe(expectedAuthorization); + }, + ); it("does not use Codex OAuth profiles for default GPT realtime bridges", async () => { const provider = buildOpenAIRealtimeVoiceProvider(); @@ -211,57 +181,6 @@ describe("OpenAI realtime voice browser authentication", () => { expect(FakeWebSocket.instances).toHaveLength(0); }); - it("uses OPENAI_API_KEY when a configured API-key profile cannot be resolved", async () => { - vi.stubEnv("OPENAI_API_KEY", "test-api-key-env"); - resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce(undefined); - isProviderAuthProfileConfiguredMock.mockReturnValueOnce(true); - const provider = buildOpenAIRealtimeVoiceProvider(); - const bridge = provider.createBridge({ - cfg: {} as never, - providerConfig: { model: "gpt-realtime-2" }, - onAudio: vi.fn(), - onClearAudio: vi.fn(), - }); - - void bridge.connect(); - await vi.waitFor(() => expect(FakeWebSocket.instances.length).toBe(1)); - bridge.close(); - - expect(resolveProviderAuthProfileApiKeyMock).toHaveBeenCalledTimes(1); - const socket = FakeWebSocket.instances[0]; - const options = socket?.args[1] as { headers?: Record } | undefined; - expect(options?.headers?.Authorization).toBe("Bearer test-api-key-env"); - }); - - it("uses OpenAI API-key auth profiles", async () => { - resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce("test-api-key-profile"); - const provider = buildOpenAIRealtimeVoiceProvider(); - const bridge = provider.createBridge({ - cfg: {} as never, - providerConfig: { model: "gpt-realtime-2" }, - onAudio: vi.fn(), - onClearAudio: vi.fn(), - }); - - void bridge.connect(); - await vi.waitFor(() => expect(FakeWebSocket.instances.length).toBe(1)); - bridge.close(); - - expect(resolveProviderAuthProfileApiKeyMock.mock.calls).toEqual([ - [ - { - provider: "openai", - cfg: {}, - profileTypes: ["api_key"], - includeExternalCliAuth: false, - }, - ], - ]); - const socket = FakeWebSocket.instances[0]; - const options = socket?.args[1] as { headers?: Record } | undefined; - expect(options?.headers?.Authorization).toBe("Bearer test-api-key-profile"); - }); - it("keeps explicit OpenAI realtime API keys as the advanced override", () => { vi.stubEnv("OPENAI_API_KEY", "test-api-key-env"); resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce("test-api-key-profile"); @@ -305,13 +224,7 @@ describe("OpenAI realtime voice browser authentication", () => { it("returns browser-safe OpenClaw attribution headers for native WebRTC offers", async () => { vi.stubEnv("OPENCLAW_VERSION", "2026.3.22"); - fetchWithSsrFGuardMock.mockResolvedValueOnce({ - response: createJsonResponse({ - client_secret: { value: "client-secret-123" }, - expires_at: 1_765_000_000, - }), - release: vi.fn(async () => undefined), - }); + mockRealtimeClientSecretResponse({ expiresAt: 1_765_000_000 }); const provider = buildOpenAIRealtimeVoiceProvider(); if (!provider.createBrowserSession) { throw new Error("expected OpenAI realtime provider to support browser sessions"); @@ -401,12 +314,7 @@ describe("OpenAI realtime voice browser authentication", () => { it("resolves keychain OPENAI_API_KEY refs before creating browser sessions", async () => { vi.stubEnv("OPENAI_API_KEY", "keychain:openclaw:OPENAI_REALTIME_BROWSER_TEST"); execFileSyncMock.mockReturnValueOnce("test-api-key-browser-env\n"); - fetchWithSsrFGuardMock.mockResolvedValueOnce({ - response: createJsonResponse({ - client_secret: { value: "client-secret-123" }, - }), - release: vi.fn(async () => undefined), - }); + mockRealtimeClientSecretResponse(); const provider = buildOpenAIRealtimeVoiceProvider(); if (!provider.createBrowserSession) { throw new Error("expected OpenAI realtime provider to support browser sessions"); @@ -477,10 +385,7 @@ describe("OpenAI realtime voice browser authentication", () => { async ({ profileTypes }: { profileTypes?: readonly string[] }) => profileTypes?.includes("oauth") ? oauthToken : undefined, ); - fetchWithSsrFGuardMock.mockResolvedValueOnce({ - response: createJsonResponse({ client_secret: { value: "client-secret-123" } }), - release: vi.fn(async () => undefined), - }); + mockRealtimeClientSecretResponse(); const provider = buildOpenAIRealtimeVoiceProvider(); await provider.createBrowserSession?.({ @@ -508,13 +413,9 @@ describe("OpenAI realtime voice browser authentication", () => { ({ profileTypes }: { profileTypes?: readonly string[] }) => profileTypes?.includes("api_key") === true, ); - const createBrowserSession = vi.fn(); + const { broker, createBrowserSession } = createQuicksilverBrowserBrokerFixture(); const provider = buildOpenAIRealtimeVoiceProvider({ - quicksilverBrowserSessionBroker: { - capabilities: { handlesAgentConsult: true as const }, - createBrowserSession, - cancelBrowserSession: vi.fn(), - }, + quicksilverBrowserSessionBroker: broker, }); await expect( @@ -533,16 +434,6 @@ describe("OpenAI realtime voice browser authentication", () => { ); }); - it("requires Platform auth for browser sessions", async () => { - const provider = buildOpenAIRealtimeVoiceProvider(); - await expect( - provider.createBrowserSession?.({ - providerConfig: {}, - }), - ).rejects.toThrow("OpenAI Realtime voice requires an OpenAI Platform API key"); - expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled(); - }); - it("reports an unresolved Platform credential without trying another auth route", async () => { vi.stubEnv("OPENAI_API_KEY", "keychain:openclaw:OPENAI_REALTIME_MISSING_TEST"); execFileSyncMock.mockImplementationOnce(() => { @@ -605,12 +496,7 @@ describe("OpenAI realtime voice browser authentication", () => { it("uses OPENAI_API_KEY for default GPT browser sessions", async () => { vi.stubEnv("OPENAI_API_KEY", "test-api-key-env"); - fetchWithSsrFGuardMock.mockResolvedValueOnce({ - response: createJsonResponse({ - client_secret: { value: "client-secret-123" }, - }), - release: vi.fn(async () => undefined), - }); + mockRealtimeClientSecretResponse(); const provider = buildOpenAIRealtimeVoiceProvider(); if (!provider.createBrowserSession) { throw new Error("expected OpenAI realtime provider to support browser sessions"); @@ -673,24 +559,14 @@ describe("OpenAI realtime voice browser authentication", () => { const { connecting, socket } = beginBridgeConnection(bridge); openSocket(socket); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "error", - error: { message: "Incorrect API key provided: test-api-key-proj-***" }, - }), - ), - ); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "error", - error: { message: "Incorrect API key provided: test-api-key-proj-***" }, - }), - ), - ); + emitServerEvent(socket, { + type: "error", + error: { message: "Incorrect API key provided: test-api-key-proj-***" }, + }); + emitServerEvent(socket, { + type: "error", + error: { message: "Incorrect API key provided: test-api-key-proj-***" }, + }); await expect(connecting).rejects.toThrow(OPENAI_REALTIME_REJECTED_KEY_MESSAGE); expect(onError).not.toHaveBeenCalled(); @@ -699,64 +575,57 @@ describe("OpenAI realtime voice browser authentication", () => { expect(bridge.isConnected()).toBe(false); }); - it("normalizes structured direct OpenAI startup auth errors", async () => { - const bridge = createNativeBridge(); - const { connecting, socket } = beginBridgeConnection(bridge); - - openSocket(socket); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "error", - error: { - type: "invalid_request_error", - code: "invalid_api_key", - message: "Invalid API key", - }, - }), - ), - ); - - await expect(connecting).rejects.toThrow(OPENAI_REALTIME_REJECTED_KEY_MESSAGE); - expect(bridge.isConnected()).toBe(false); - }); - - it("normalizes direct OpenAI socket handshake auth errors", async () => { - const bridge = createNativeBridge(); - const { connecting, socket } = beginBridgeConnection(bridge); - - socket.emit("error", new Error("Unexpected server response: 401")); - - await expect(connecting).rejects.toThrow(OPENAI_REALTIME_REJECTED_KEY_MESSAGE); - expect(bridge.isConnected()).toBe(false); - }); - it.each([ - [ - "Azure deployment", - { + { + $name: "structured direct error expects normalization", + event: "structured" as const, + providerConfig: undefined, + expectedMessage: OPENAI_REALTIME_REJECTED_KEY_MESSAGE, + }, + { + $name: "direct handshake error expects normalization", + event: "handshake" as const, + providerConfig: undefined, + expectedMessage: OPENAI_REALTIME_REJECTED_KEY_MESSAGE, + }, + { + $name: "Azure handshake error expects raw preservation", + event: "handshake" as const, + providerConfig: { apiKey: "test-api-key-test", azureEndpoint: "https://example.openai.azure.com", azureDeployment: "realtime-prod", }, - ], - [ - "custom endpoint", - { + expectedMessage: "Unexpected server response: 401", + }, + { + $name: "custom-endpoint handshake error expects raw preservation", + event: "handshake" as const, + providerConfig: { apiKey: "test-api-key-test", azureEndpoint: "https://realtime-proxy.example.com", }, - ], - ])("preserves %s startup auth errors", async (_label, providerConfig) => { - const bridge = createNativeBridge({ - providerConfig, - }); + expectedMessage: "Unexpected server response: 401", + }, + ])("$name", async ({ event, providerConfig, expectedMessage }) => { + const bridge = createNativeBridge(providerConfig ? { providerConfig } : {}); const { connecting, socket } = beginBridgeConnection(bridge); - socket.emit("error", new Error("Unexpected server response: 401")); + if (event === "structured") { + openSocket(socket); + emitServerEvent(socket, { + type: "error", + error: { + type: "invalid_request_error", + code: "invalid_api_key", + message: "Invalid API key", + }, + }); + } else { + socket.emit("error", new Error("Unexpected server response: 401")); + } - await expect(connecting).rejects.toThrow("Unexpected server response: 401"); + await expect(connecting).rejects.toThrow(expectedMessage); expect(bridge.isConnected()).toBe(false); }); }); diff --git a/extensions/openai/realtime-voice-provider-routing.test.ts b/extensions/openai/realtime-voice-provider-routing.test.ts index 22946a22111a..2aeca07555db 100644 --- a/extensions/openai/realtime-voice-provider-routing.test.ts +++ b/extensions/openai/realtime-voice-provider-routing.test.ts @@ -2,174 +2,60 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js"; -const INTERNAL_REALTIME_VOICE_PROVIDER = Symbol.for("openclaw.internal.realtime-voice-provider.v1"); - -function readInternalRealtimeVoiceProviderApi(provider: object) { - return Reflect.get(provider, INTERNAL_REALTIME_VOICE_PROVIDER) as { - isBrowserSessionConfigured: (ctx: { - cfg?: object; - providerConfig: Record; - agentId?: string; - }) => boolean; - isGatewayRelayConfigured: (ctx: { - cfg?: object; - providerConfig: Record; - agentId?: string; - }) => boolean | undefined; - resolveBrowserSessionCapabilities: (ctx: { - cfg?: object; - providerConfig: Record; - model?: string; - }) => { - handlesAgentConsult?: boolean; - supportsToolCalls?: boolean; - supportsVideoFrames?: boolean; - supportsGatewayControl?: boolean; - transports?: string[]; - }; - resolveGatewayRelayCapabilities: (ctx: { - cfg?: object; - providerConfig: Record; - model?: string; - }) => { - handlesAgentConsult?: boolean; - supportsToolCalls?: boolean; - transports?: string[]; - }; - validateGatewayRelayLaunch: (ctx: { - cfg?: object; - providerConfig: Record; - model?: string; - autoRespondToAudio?: boolean; - }) => string | undefined; - cancelBrowserSession: (request: Record, session: object) => Promise; - }; -} - +const mocks = await vi.hoisted(async () => { + const { createOpenAIRealtimeMockState } = await import("./realtime-voice-test-support.js"); + return createOpenAIRealtimeMockState(); +}); const { - FakeWebSocket, execFileSyncMock, fetchWithSsrFGuardMock, isProviderAuthProfileConfiguredMock, resolveProviderAuthProfileApiKeyMock, -} = vi.hoisted(() => { - type Listener = (...args: unknown[]) => void; - - class MockWebSocket { - static readonly OPEN = 1; - static readonly CLOSED = 3; - static instances: MockWebSocket[] = []; - - readonly listeners = new Map(); - readyState = 0; - sent: string[] = []; - closed = false; - terminated = false; - deferClose = false; - deferredClose: (() => void) | undefined; - args: unknown[]; - - constructor(...args: unknown[]) { - this.args = args; - MockWebSocket.instances.push(this); - } - - on(event: string, listener: Listener): this { - const listeners = this.listeners.get(event) ?? []; - listeners.push(listener); - this.listeners.set(event, listeners); - return this; - } - - emit(event: string, ...args: unknown[]): void { - for (const listener of this.listeners.get(event) ?? []) { - listener(...args); - } - } - - send(payload: string): void { - this.sent.push(payload); - } - - close(code?: number, reason?: string): void { - this.closed = true; - this.readyState = MockWebSocket.CLOSED; - const emitClose = () => this.emit("close", code ?? 1000, Buffer.from(reason ?? "")); - if (this.deferClose) { - this.deferredClose = emitClose; - return; - } - emitClose(); - } - - terminate(): void { - this.terminated = true; - this.close(1006, "terminated"); - } - - emitDeferredClose(): void { - const emitClose = this.deferredClose; - this.deferredClose = undefined; - emitClose?.(); - } - } - - return { - FakeWebSocket: MockWebSocket, - execFileSyncMock: vi.fn(), - fetchWithSsrFGuardMock: vi.fn(), - isProviderAuthProfileConfiguredMock: vi.fn(), - resolveProviderAuthProfileApiKeyMock: vi.fn(), - }; -}); +} = mocks; vi.mock("node:child_process", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, - execFileSync: execFileSyncMock, + execFileSync: mocks.execFileSyncMock, }; }); vi.mock("ws", () => ({ - default: FakeWebSocket, + default: mocks.FakeWebSocket, })); vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({ - fetchWithSsrFGuard: fetchWithSsrFGuardMock, + fetchWithSsrFGuard: mocks.fetchWithSsrFGuardMock, })); vi.mock("openclaw/plugin-sdk/provider-auth", () => ({ - isProviderAuthProfileConfigured: isProviderAuthProfileConfiguredMock, - resolveProviderAuthProfileApiKey: resolveProviderAuthProfileApiKeyMock, + isProviderAuthProfileConfigured: mocks.isProviderAuthProfileConfiguredMock, + resolveProviderAuthProfileApiKey: mocks.resolveProviderAuthProfileApiKeyMock, })); import { createOpenAIRealtimeTestSupport } from "./realtime-voice-test-support.js"; const { - createJsonResponse, requireRecord, requireFetchJsonBody, createRealtimeTool, createUnreadableToolName, createMalformedToolName, createTestJwt, -} = createOpenAIRealtimeTestSupport({ FakeWebSocket, fetchWithSsrFGuardMock }); + resetTestState, + restoreTestEnvironment, + readInternalRealtimeVoiceProviderApi, + mockRealtimeClientSecretResponse, + createQuicksilverBrowserBrokerFixture, +} = createOpenAIRealtimeTestSupport({ ...mocks, buildOpenAIRealtimeVoiceProvider }); describe("OpenAI realtime voice provider routing", () => { beforeEach(() => { - FakeWebSocket.instances = []; - vi.stubEnv("OPENAI_API_KEY", ""); - execFileSyncMock.mockReset(); - fetchWithSsrFGuardMock.mockReset(); - isProviderAuthProfileConfiguredMock.mockReset(); - isProviderAuthProfileConfiguredMock.mockReturnValue(false); - resolveProviderAuthProfileApiKeyMock.mockReset(); - resolveProviderAuthProfileApiKeyMock.mockResolvedValue(undefined); + resetTestState(); }); afterEach(() => { - vi.useRealTimers(); - vi.unstubAllEnvs(); + restoreTestEnvironment(); }); it("declares realtime Talk capabilities for catalog selection", () => { @@ -206,51 +92,45 @@ describe("OpenAI realtime voice provider routing", () => { expect(bridge.supportsToolResultSuppression).toBe(true); }); - it("advertises quicksilver capabilities only for curated /v1/live models", () => { - const quicksilverBroker = { - capabilities: { - transports: ["webrtc" as const], - handlesAgentConsult: true as const, + it.each([ + { + $name: "browser capability projection", + surface: "browser" as const, + expected: { + transports: ["webrtc", "gateway-relay"], + handlesAgentConsult: true, supportsToolCalls: false, supportsVideoFrames: false, }, - createBrowserSession: vi.fn(), - cancelBrowserSession: vi.fn(), - }; + }, + { + $name: "gateway-relay capability projection", + surface: "gateway-relay" as const, + expected: { + transports: ["webrtc", "gateway-relay"], + handlesAgentConsult: true, + supportsToolCalls: false, + }, + }, + ])("$name", ({ surface, expected }) => { + const { broker } = createQuicksilverBrowserBrokerFixture(); const provider = buildOpenAIRealtimeVoiceProvider({ - quicksilverBrowserSessionBroker: quicksilverBroker, + quicksilverBrowserSessionBroker: broker, }); const internalApi = readInternalRealtimeVoiceProviderApi(provider); + const resolveCapabilities = + surface === "browser" + ? internalApi.resolveBrowserSessionCapabilities + : internalApi.resolveGatewayRelayCapabilities; expect( - internalApi.resolveBrowserSessionCapabilities({ + resolveCapabilities({ providerConfig: { model: "gpt-realtime-2.1" }, model: "gpt-live-1-codex", }), - ).toMatchObject({ - transports: ["webrtc", "gateway-relay"], - handlesAgentConsult: true, - supportsToolCalls: false, - supportsVideoFrames: false, - }); + ).toMatchObject(expected); expect( - internalApi.resolveGatewayRelayCapabilities({ - providerConfig: { model: "gpt-realtime-2.1" }, - model: "gpt-live-1-codex", - }), - ).toMatchObject({ - transports: ["webrtc", "gateway-relay"], - handlesAgentConsult: true, - supportsToolCalls: false, - }); - expect( - internalApi.resolveBrowserSessionCapabilities({ - providerConfig: { model: "gpt-realtime-2.1" }, - model: "gpt-live-1-mini", - }), - ).not.toHaveProperty("handlesAgentConsult"); - expect( - internalApi.resolveGatewayRelayCapabilities({ + resolveCapabilities({ providerConfig: { model: "gpt-realtime-2.1" }, model: "gpt-live-1-mini", }), @@ -258,10 +138,7 @@ describe("OpenAI realtime voice provider routing", () => { }); it("omits unsupported OpenAI tool names from browser sessions", async () => { - fetchWithSsrFGuardMock.mockResolvedValueOnce({ - response: createJsonResponse({ client_secret: { value: "client-secret-123" } }), - release: vi.fn(async () => undefined), - }); + mockRealtimeClientSecretResponse(); const provider = buildOpenAIRealtimeVoiceProvider(); if (!provider.createBrowserSession) { throw new Error("expected OpenAI realtime provider to support browser sessions"); @@ -304,23 +181,9 @@ describe("OpenAI realtime voice provider routing", () => { }); it("routes gpt-live Platform sessions through the native quicksilver broker", async () => { - const createBrowserSession = vi.fn(async (_request: unknown, _auth: unknown) => ({ - provider: "openai", - transport: "webrtc" as const, - clientSecret: "quicksilver-token", - offerUrl: "/plugins/openai/realtime/calls", - })); + const { broker, createBrowserSession } = createQuicksilverBrowserBrokerFixture(); const provider = buildOpenAIRealtimeVoiceProvider({ - quicksilverBrowserSessionBroker: { - capabilities: { - transports: ["webrtc" as const], - handlesAgentConsult: true as const, - supportsToolCalls: false, - supportsVideoFrames: false, - }, - createBrowserSession, - cancelBrowserSession: vi.fn(), - }, + quicksilverBrowserSessionBroker: broker, }); const request = { providerConfig: { apiKey: "test-api-key-platform" }, @@ -341,7 +204,150 @@ describe("OpenAI realtime voice provider routing", () => { expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled(); }); - it("routes an explicit unlisted gpt-live alias without advertising it as ready", async () => { + it.each([ + { + $name: "provider | gpt-live-1-mini | ChatGPT OAuth | standard endpoint | not ready", + surface: "provider" as const, + providerConfig: { model: "gpt-live-1-mini" }, + agentId: "main", + expected: false, + expectAgentDir: false, + }, + { + $name: "gateway-relay | gpt-live-1-mini | ChatGPT OAuth | standard endpoint | not ready", + surface: "gateway-relay" as const, + providerConfig: { model: "gpt-live-1-mini" }, + agentId: "main", + expected: false, + expectAgentDir: false, + }, + { + $name: "gateway-relay | gpt-live-1-mini | ChatGPT OAuth | Azure endpoint | not ready", + surface: "gateway-relay" as const, + providerConfig: { + model: "gpt-live-1-mini", + azureEndpoint: "https://example.openai.azure.com", + azureDeployment: "gpt-live", + }, + agentId: "main", + expected: false, + expectAgentDir: false, + }, + { + $name: "browser | gpt-live-1-mini | ChatGPT OAuth | standard endpoint | not ready", + surface: "browser" as const, + providerConfig: { model: "gpt-live-1-mini" }, + agentId: "main", + expected: false, + expectAgentDir: false, + }, + { + $name: + "gateway-relay | gpt-realtime-2.1 | Platform API key | standard endpoint | not applicable", + surface: "gateway-relay" as const, + providerConfig: { model: "gpt-realtime-2.1", apiKey: "test-api-key-platform" }, + agentId: "main", + expected: undefined, + expectAgentDir: false, + }, + { + $name: + "gateway-relay | gpt-realtime-2.1 | Platform API key | Azure endpoint | not applicable", + surface: "gateway-relay" as const, + providerConfig: { + model: "gpt-realtime-2.1", + apiKey: "test-api-key-platform", + azureEndpoint: "https://example.openai.azure.com", + }, + agentId: "main", + expected: undefined, + expectAgentDir: false, + }, + { + $name: + "gateway-relay | gpt-live-1-codex | Platform API key + OAuth | Azure endpoint | not ready", + surface: "gateway-relay" as const, + providerConfig: { + model: "gpt-live-1-codex", + apiKey: "test-api-key-platform", + azureEndpoint: "https://example.openai.azure.com", + }, + agentId: "main", + expected: false, + expectAgentDir: false, + }, + { + $name: + "gateway-relay | gpt-live-1-mini | Platform API key + OAuth | standard endpoint | not ready", + surface: "gateway-relay" as const, + providerConfig: { model: "gpt-live-1-mini", apiKey: "test-api-key-platform" }, + agentId: "main", + expected: false, + expectAgentDir: false, + }, + { + $name: "browser | gpt-live-1-mini | Platform API key + OAuth | standard endpoint | not ready", + surface: "browser" as const, + providerConfig: { model: "gpt-live-1-mini", apiKey: "test-api-key-platform" }, + agentId: "main", + expected: false, + expectAgentDir: false, + }, + { + $name: "gateway-relay | gpt-live-1-codex | ChatGPT OAuth | standard endpoint | ready", + surface: "gateway-relay" as const, + providerConfig: { model: "gpt-live-1-codex" }, + agentId: "main", + expected: true, + expectAgentDir: false, + }, + { + $name: + "gateway-relay | gpt-live-1-codex | voice-agent ChatGPT OAuth | standard endpoint | ready", + surface: "gateway-relay" as const, + providerConfig: { model: "gpt-live-1-codex" }, + agentId: "voice-agent", + expected: true, + expectAgentDir: true, + }, + { + $name: "browser | gpt-live-1-codex | ChatGPT OAuth | standard endpoint | ready", + surface: "browser" as const, + providerConfig: { model: "gpt-live-1-codex" }, + agentId: "main", + expected: true, + expectAgentDir: false, + }, + ])("$name", ({ surface, providerConfig, agentId, expected, expectAgentDir }) => { + isProviderAuthProfileConfiguredMock.mockImplementation( + ({ profileTypes }: { profileTypes?: readonly string[] }) => + profileTypes?.includes("oauth") === true, + ); + const { broker } = createQuicksilverBrowserBrokerFixture(); + const provider = buildOpenAIRealtimeVoiceProvider({ + quicksilverBrowserSessionBroker: broker, + }); + const cfg = { agents: { defaults: {} } } as never; + const internalApi = readInternalRealtimeVoiceProviderApi(provider); + const readiness = + surface === "provider" + ? provider.isConfigured({ cfg, providerConfig }) + : surface === "browser" + ? internalApi.isBrowserSessionConfigured({ cfg, providerConfig, agentId }) + : internalApi.isGatewayRelayConfigured({ cfg, providerConfig, agentId }); + + expect(readiness).toBe(expected); + if (expectAgentDir) { + expect(isProviderAuthProfileConfiguredMock).toHaveBeenCalledWith( + expect.objectContaining({ + agentDir: expect.stringContaining("voice-agent"), + profileTypes: ["oauth"], + }), + ); + } + }); + + it("routes an explicit unlisted gpt-live alias through the broker", async () => { const oauthToken = createTestJwt({ "https://api.openai.com/auth": { chatgpt_account_id: "account-123" }, }); @@ -349,27 +355,9 @@ describe("OpenAI realtime voice provider routing", () => { async ({ profileTypes }: { profileTypes?: readonly string[] }) => profileTypes?.includes("oauth") ? oauthToken : undefined, ); - isProviderAuthProfileConfiguredMock.mockImplementation( - ({ profileTypes }: { profileTypes?: readonly string[] }) => - profileTypes?.includes("oauth") === true, - ); - const createBrowserSession = vi.fn(async () => ({ - provider: "openai", - transport: "webrtc" as const, - clientSecret: "quicksilver-token", - offerUrl: "/plugins/openai/realtime/calls", - })); + const { broker, createBrowserSession } = createQuicksilverBrowserBrokerFixture(); const provider = buildOpenAIRealtimeVoiceProvider({ - quicksilverBrowserSessionBroker: { - capabilities: { - transports: ["webrtc" as const], - handlesAgentConsult: true as const, - supportsToolCalls: false, - supportsVideoFrames: false, - }, - createBrowserSession, - cancelBrowserSession: vi.fn(), - }, + quicksilverBrowserSessionBroker: broker, }); const cfg = { agents: { defaults: {} } } as never; const request = { @@ -382,104 +370,6 @@ describe("OpenAI realtime voice provider routing", () => { runAgentConsult: vi.fn(async () => ({ text: "Done" })), }; - expect(provider.isConfigured({ cfg, providerConfig: { model: "gpt-live-1-mini" } })).toBe( - false, - ); - expect( - readInternalRealtimeVoiceProviderApi(provider).isGatewayRelayConfigured({ - cfg, - providerConfig: { model: "gpt-live-1-mini" }, - agentId: "main", - }), - ).toBe(false); - expect( - readInternalRealtimeVoiceProviderApi(provider).isGatewayRelayConfigured({ - cfg, - providerConfig: { - model: "gpt-live-1-mini", - azureEndpoint: "https://example.openai.azure.com", - azureDeployment: "gpt-live", - }, - agentId: "main", - }), - ).toBe(false); - expect( - readInternalRealtimeVoiceProviderApi(provider).isBrowserSessionConfigured({ - cfg, - providerConfig: { model: "gpt-live-1-mini" }, - agentId: "main", - }), - ).toBe(false); - expect( - readInternalRealtimeVoiceProviderApi(provider).isGatewayRelayConfigured({ - cfg, - providerConfig: { model: "gpt-realtime-2.1", apiKey: "test-api-key-platform" }, - agentId: "main", - }), - ).toBeUndefined(); - expect( - readInternalRealtimeVoiceProviderApi(provider).isGatewayRelayConfigured({ - cfg, - providerConfig: { - model: "gpt-realtime-2.1", - apiKey: "test-api-key-platform", - azureEndpoint: "https://example.openai.azure.com", - }, - agentId: "main", - }), - ).toBeUndefined(); - expect( - readInternalRealtimeVoiceProviderApi(provider).isGatewayRelayConfigured({ - cfg, - providerConfig: { - model: "gpt-live-1-codex", - apiKey: "test-api-key-platform", - azureEndpoint: "https://example.openai.azure.com", - }, - agentId: "main", - }), - ).toBe(false); - expect( - readInternalRealtimeVoiceProviderApi(provider).isGatewayRelayConfigured({ - cfg, - providerConfig: { model: "gpt-live-1-mini", apiKey: "test-api-key-platform" }, - agentId: "main", - }), - ).toBe(false); - expect( - readInternalRealtimeVoiceProviderApi(provider).isBrowserSessionConfigured({ - cfg, - providerConfig: { model: "gpt-live-1-mini", apiKey: "test-api-key-platform" }, - agentId: "main", - }), - ).toBe(false); - expect( - readInternalRealtimeVoiceProviderApi(provider).isGatewayRelayConfigured({ - cfg, - providerConfig: { model: "gpt-live-1-codex" }, - agentId: "main", - }), - ).toBe(true); - expect( - readInternalRealtimeVoiceProviderApi(provider).isGatewayRelayConfigured({ - cfg, - providerConfig: { model: "gpt-live-1-codex" }, - agentId: "voice-agent", - }), - ).toBe(true); - expect(isProviderAuthProfileConfiguredMock).toHaveBeenCalledWith( - expect.objectContaining({ - agentDir: expect.stringContaining("voice-agent"), - profileTypes: ["oauth"], - }), - ); - expect( - readInternalRealtimeVoiceProviderApi(provider).isBrowserSessionConfigured({ - cfg, - providerConfig: { model: "gpt-live-1-codex" }, - agentId: "main", - }), - ).toBe(true); await provider.createBrowserSession?.(request); expect(createBrowserSession).toHaveBeenCalledWith(expect.objectContaining(request), { type: "oauth", @@ -521,18 +411,9 @@ describe("OpenAI realtime voice provider routing", () => { async ({ profileTypes }: { profileTypes?: readonly string[] }) => profileTypes?.includes("oauth") ? oauthToken : undefined, ); - const createBrowserSession = vi.fn(async () => ({ - provider: "openai", - transport: "webrtc" as const, - clientSecret: "quicksilver-token", - offerUrl: "/plugins/openai/realtime/calls", - })); + const { broker, createBrowserSession } = createQuicksilverBrowserBrokerFixture(); const provider = buildOpenAIRealtimeVoiceProvider({ - quicksilverBrowserSessionBroker: { - capabilities: { handlesAgentConsult: true as const }, - createBrowserSession, - cancelBrowserSession: vi.fn(), - }, + quicksilverBrowserSessionBroker: broker, }); await provider.createBrowserSession?.({ @@ -556,12 +437,9 @@ describe("OpenAI realtime voice provider routing", () => { ({ profileTypes }: { profileTypes?: readonly string[] }) => profileTypes?.includes("oauth") === true, ); + const { broker } = createQuicksilverBrowserBrokerFixture(); const provider = buildOpenAIRealtimeVoiceProvider({ - quicksilverBrowserSessionBroker: { - capabilities: { handlesAgentConsult: true as const }, - createBrowserSession: vi.fn(), - cancelBrowserSession: vi.fn(async () => undefined), - }, + quicksilverBrowserSessionBroker: broker, }); expect( readInternalRealtimeVoiceProviderApi(provider).resolveBrowserSessionCapabilities({ @@ -584,18 +462,11 @@ describe("OpenAI realtime voice provider routing", () => { ({ profileTypes }: { profileTypes?: readonly string[] }) => profileTypes?.includes("oauth") === true, ); - const createBrowserSession = vi.fn(async () => ({ - provider: "openai", - transport: "webrtc" as const, - clientSecret: "broker-token", - offerUrl: "/plugins/openai/realtime/calls", - })); + const { broker, createBrowserSession } = createQuicksilverBrowserBrokerFixture({ + session: { clientSecret: "broker-token" }, + }); const provider = buildOpenAIRealtimeVoiceProvider({ - quicksilverBrowserSessionBroker: { - capabilities: { handlesAgentConsult: true as const }, - createBrowserSession, - cancelBrowserSession: vi.fn(), - }, + quicksilverBrowserSessionBroker: broker, }); const cfg = { agents: { defaults: {} } } as never; const request = { @@ -628,23 +499,9 @@ describe("OpenAI realtime voice provider routing", () => { }); it("passes configured gpt-live model and voice to the native broker", async () => { - const createBrowserSession = vi.fn(async (_request: unknown, _auth: unknown) => ({ - provider: "openai", - transport: "webrtc" as const, - clientSecret: "quicksilver-token", - offerUrl: "/plugins/openai/realtime/calls", - })); + const { broker, createBrowserSession } = createQuicksilverBrowserBrokerFixture(); const provider = buildOpenAIRealtimeVoiceProvider({ - quicksilverBrowserSessionBroker: { - capabilities: { - transports: ["webrtc" as const], - handlesAgentConsult: true as const, - supportsToolCalls: false, - supportsVideoFrames: false, - }, - createBrowserSession, - cancelBrowserSession: vi.fn(), - }, + quicksilverBrowserSessionBroker: broker, }); await provider.createBrowserSession?.({ @@ -679,18 +536,9 @@ describe("OpenAI realtime voice provider routing", () => { }); it("explains both gpt-live authentication options when neither is available", async () => { - const createBrowserSession = vi.fn(); + const { broker, createBrowserSession } = createQuicksilverBrowserBrokerFixture(); const provider = buildOpenAIRealtimeVoiceProvider({ - quicksilverBrowserSessionBroker: { - capabilities: { - transports: ["webrtc" as const], - handlesAgentConsult: true as const, - supportsToolCalls: false, - supportsVideoFrames: false, - }, - createBrowserSession, - cancelBrowserSession: vi.fn(), - }, + quicksilverBrowserSessionBroker: broker, }); await expect( diff --git a/extensions/openai/realtime-voice-response-control.test.ts b/extensions/openai/realtime-voice-response-control.test.ts index 3fa31c456aa9..e37f6f309b10 100644 --- a/extensions/openai/realtime-voice-response-control.test.ts +++ b/extensions/openai/realtime-voice-response-control.test.ts @@ -1,103 +1,31 @@ // Openai tests cover realtime voice provider plugin behavior. import type { RealtimeVoiceBridge } from "openclaw/plugin-sdk/realtime-voice"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js"; -const { - FakeWebSocket, - execFileSyncMock, - fetchWithSsrFGuardMock, - isProviderAuthProfileConfiguredMock, - resolveProviderAuthProfileApiKeyMock, -} = vi.hoisted(() => { - type Listener = (...args: unknown[]) => void; - - class MockWebSocket { - static readonly OPEN = 1; - static readonly CLOSED = 3; - static instances: MockWebSocket[] = []; - - readonly listeners = new Map(); - readyState = 0; - sent: string[] = []; - closed = false; - terminated = false; - deferClose = false; - deferredClose: (() => void) | undefined; - args: unknown[]; - - constructor(...args: unknown[]) { - this.args = args; - MockWebSocket.instances.push(this); - } - - on(event: string, listener: Listener): this { - const listeners = this.listeners.get(event) ?? []; - listeners.push(listener); - this.listeners.set(event, listeners); - return this; - } - - emit(event: string, ...args: unknown[]): void { - for (const listener of this.listeners.get(event) ?? []) { - listener(...args); - } - } - - send(payload: string): void { - this.sent.push(payload); - } - - close(code?: number, reason?: string): void { - this.closed = true; - this.readyState = MockWebSocket.CLOSED; - const emitClose = () => this.emit("close", code ?? 1000, Buffer.from(reason ?? "")); - if (this.deferClose) { - this.deferredClose = emitClose; - return; - } - emitClose(); - } - - terminate(): void { - this.terminated = true; - this.close(1006, "terminated"); - } - - emitDeferredClose(): void { - const emitClose = this.deferredClose; - this.deferredClose = undefined; - emitClose?.(); - } - } - - return { - FakeWebSocket: MockWebSocket, - execFileSyncMock: vi.fn(), - fetchWithSsrFGuardMock: vi.fn(), - isProviderAuthProfileConfiguredMock: vi.fn(), - resolveProviderAuthProfileApiKeyMock: vi.fn(), - }; +const mocks = await vi.hoisted(async () => { + const { createOpenAIRealtimeMockState } = await import("./realtime-voice-test-support.js"); + return createOpenAIRealtimeMockState(); }); - vi.mock("node:child_process", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, - execFileSync: execFileSyncMock, + execFileSync: mocks.execFileSyncMock, }; }); vi.mock("ws", () => ({ - default: FakeWebSocket, + default: mocks.FakeWebSocket, })); vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({ - fetchWithSsrFGuard: fetchWithSsrFGuardMock, + fetchWithSsrFGuard: mocks.fetchWithSsrFGuardMock, })); vi.mock("openclaw/plugin-sdk/provider-auth", () => ({ - isProviderAuthProfileConfigured: isProviderAuthProfileConfiguredMock, - resolveProviderAuthProfileApiKey: resolveProviderAuthProfileApiKeyMock, + isProviderAuthProfileConfigured: mocks.isProviderAuthProfileConfiguredMock, + resolveProviderAuthProfileApiKey: mocks.resolveProviderAuthProfileApiKeyMock, })); import { createOpenAIRealtimeTestSupport } from "./realtime-voice-test-support.js"; @@ -108,29 +36,24 @@ const { beginBridgeConnection, openSocket, emitServerEvent, + emitAssistantPlayback, emitSessionUpdated, emitCompletedToolCalls, connectReadyBridge, expectedResponseCreateEvent, requireNestedRecord, expectRecordFields, -} = createOpenAIRealtimeTestSupport({ FakeWebSocket, fetchWithSsrFGuardMock }); + resetTestState, + restoreTestEnvironment, +} = createOpenAIRealtimeTestSupport({ ...mocks, buildOpenAIRealtimeVoiceProvider }); describe("OpenAI realtime voice response control", () => { beforeEach(() => { - FakeWebSocket.instances = []; - vi.stubEnv("OPENAI_API_KEY", ""); - execFileSyncMock.mockReset(); - fetchWithSsrFGuardMock.mockReset(); - isProviderAuthProfileConfiguredMock.mockReset(); - isProviderAuthProfileConfiguredMock.mockReturnValue(false); - resolveProviderAuthProfileApiKeyMock.mockReset(); - resolveProviderAuthProfileApiKeyMock.mockResolvedValue(undefined); + resetTestState(); }); afterEach(() => { - vi.useRealTimers(); - vi.unstubAllEnvs(); + restoreTestEnvironment(); }); it("suppresses auto responses before draining queued initial greeting audio", async () => { @@ -264,10 +187,7 @@ describe("OpenAI realtime voice response control", () => { it("defers manual response.create while a realtime response is active", async () => { const bridge = createNativeBridge(); const socket = await connectReadyBridge(bridge); - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })), - ); + emitServerEvent(socket, { type: "response.created", response: { id: "resp_1" } }); bridge.sendUserMessage?.("queued manual response"); @@ -310,18 +230,13 @@ describe("OpenAI realtime voice response control", () => { }, ); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "error", - error: { - event_id: responseCreateEvent.event_id, - message: "bad response request", - }, - }), - ), - ); + emitServerEvent(socket, { + type: "error", + error: { + event_id: responseCreateEvent.event_id, + message: "bad response request", + }, + }); expect(onError).toHaveBeenCalledWith(new Error("bad response request")); expectRecordFields( @@ -344,15 +259,10 @@ describe("OpenAI realtime voice response control", () => { (event) => event.type === "session.update", ); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "error", - error: { event_id: "unrelated-audio-event", message: "bad audio append" }, - }), - ), - ); + emitServerEvent(socket, { + type: "error", + error: { event_id: "unrelated-audio-event", message: "bad audio append" }, + }); expect(onError).toHaveBeenCalledWith(new Error("bad audio append")); expect(parseSent(socket).filter((event) => event.type === "session.update")).toHaveLength( @@ -388,18 +298,13 @@ describe("OpenAI realtime voice response control", () => { ).length; bridge.sendUserMessage?.("Say exactly: queued follow-up."); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "error", - error: { - event_id: firstResponseCreate.event_id, - message: "bad response request", - }, - }), - ), - ); + emitServerEvent(socket, { + type: "error", + error: { + event_id: firstResponseCreate.event_id, + message: "bad response request", + }, + }); const responseCreates = parseSent(socket).filter((event) => event.type === "response.create"); expect(responseCreates).toHaveLength(2); @@ -433,12 +338,10 @@ describe("OpenAI realtime voice response control", () => { expect(parseSent(socket).filter((event) => event.type === "response.create")).toHaveLength(1); for (let index = 0; index < 3; index += 1) { - socket.emit( - "message", - Buffer.from( - JSON.stringify({ type: "response.created", response: { id: `resp_control_${index}` } }), - ), - ); + emitServerEvent(socket, { + type: "response.created", + response: { id: `resp_control_${index}` }, + }); emitServerEvent(socket, { type: "response.done", response: { id: `resp_control_${index}`, status: "completed", output: [] }, @@ -452,13 +355,10 @@ describe("OpenAI realtime voice response control", () => { it("drains deferred response.create after response.cancelled", async () => { const bridge = createNativeBridge(); const socket = await connectReadyBridge(bridge); - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })), - ); + emitServerEvent(socket, { type: "response.created", response: { id: "resp_1" } }); bridge.sendUserMessage?.("queued after cancellation"); - socket.emit("message", Buffer.from(JSON.stringify({ type: "response.cancelled" }))); + emitServerEvent(socket, { type: "response.cancelled" }); expect(parseSent(socket).slice(-1)).toEqual([expectedResponseCreateEvent()]); }); @@ -467,10 +367,7 @@ describe("OpenAI realtime voice response control", () => { const onError = vi.fn(); const bridge = createNativeBridge({ onError }); const socket = await connectReadyBridge(bridge); - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })), - ); + emitServerEvent(socket, { type: "response.created", response: { id: "resp_1" } }); bridge.sendUserMessage?.("queued after cancellation error"); bridge.handleBargeIn?.({ audioPlaybackActive: true }); @@ -480,18 +377,13 @@ describe("OpenAI realtime voice response control", () => { if (!responseCancelEvent?.event_id) { throw new Error("expected response.cancel event id"); } - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "error", - error: { - event_id: responseCancelEvent.event_id, - message: "Cancellation failed: no active response found", - }, - }), - ), - ); + emitServerEvent(socket, { + type: "error", + error: { + event_id: responseCancelEvent.event_id, + message: "Cancellation failed: no active response found", + }, + }); expect(onError).not.toHaveBeenCalled(); expect(parseSent(socket).slice(-1)).toEqual([expectedResponseCreateEvent()]); @@ -502,20 +394,7 @@ describe("OpenAI realtime voice response control", () => { const bridge = createNativeBridge({ onError }); const socket = await connectReadyBridge(bridge); bridge.setMediaTimestamp(1000); - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })), - ); - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "response.audio.delta", - item_id: "item_1", - delta: Buffer.from("assistant audio").toString("base64"), - }), - ), - ); + emitAssistantPlayback(socket); bridge.setMediaTimestamp(1300); bridge.handleBargeIn?.({ audioPlaybackActive: true }); @@ -531,18 +410,13 @@ describe("OpenAI realtime voice response control", () => { (event) => event.type === "session.update", ).length; - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "error", - error: { - event_id: responseCancelEvent.event_id, - message: "Cancellation failed: no active response found", - }, - }), - ), - ); + emitServerEvent(socket, { + type: "error", + error: { + event_id: responseCancelEvent.event_id, + message: "Cancellation failed: no active response found", + }, + }); expect(onError).not.toHaveBeenCalled(); expect(parseSent(socket).filter((event) => event.type === "session.update")).toHaveLength( @@ -565,10 +439,7 @@ describe("OpenAI realtime voice response control", () => { vi.useFakeTimers(); const bridge = createNativeBridge(); const socket = await connectReadyBridge(bridge); - socket.emit( - "message", - Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })), - ); + emitServerEvent(socket, { type: "response.created", response: { id: "resp_1" } }); bridge.sendUserMessage?.("queued before reconnect"); expect(parseSent(socket).slice(-1)[0]?.type).toBe("conversation.item.create"); @@ -606,18 +477,13 @@ describe("OpenAI realtime voice response control", () => { if (!responseCreateEvent?.event_id) { throw new Error("expected response.create event id"); } - socket.emit( - "message", - Buffer.from( - JSON.stringify({ - type: "error", - error: { - event_id: responseCreateEvent.event_id, - message: "Conversation already has an active response in progress: resp_1", - }, - }), - ), - ); + emitServerEvent(socket, { + type: "error", + error: { + event_id: responseCreateEvent.event_id, + message: "Conversation already has an active response in progress: resp_1", + }, + }); const afterError = parseSent(socket); expect(afterError.filter((event) => event.type === "session.update")).toHaveLength(2); expectRecordFields( diff --git a/extensions/openai/realtime-voice-test-support.ts b/extensions/openai/realtime-voice-test-support.ts index 326445f9ddd4..4c70541a0d7e 100644 --- a/extensions/openai/realtime-voice-test-support.ts +++ b/extensions/openai/realtime-voice-test-support.ts @@ -1,11 +1,83 @@ import type { RealtimeVoiceBridge, RealtimeVoiceBridgeCreateRequest, + RealtimeVoiceBrowserSession, + RealtimeVoiceProviderPlugin, RealtimeVoiceTool, } from "openclaw/plugin-sdk/realtime-voice"; import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; import { expect, vi } from "vitest"; -import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js"; + +type Listener = (...args: unknown[]) => void; + +export function createOpenAIRealtimeMockState() { + class MockWebSocket { + static readonly OPEN = 1; + static readonly CLOSED = 3; + static instances: MockWebSocket[] = []; + + readonly listeners = new Map(); + readyState = 0; + sent: string[] = []; + closed = false; + terminated = false; + deferClose = false; + deferredClose: (() => void) | undefined; + args: unknown[]; + + constructor(...args: unknown[]) { + this.args = args; + MockWebSocket.instances.push(this); + } + + on(event: string, listener: Listener): this { + const listeners = this.listeners.get(event) ?? []; + listeners.push(listener); + this.listeners.set(event, listeners); + return this; + } + + emit(event: string, ...args: unknown[]): void { + for (const listener of this.listeners.get(event) ?? []) { + listener(...args); + } + } + + send(payload: string): void { + this.sent.push(payload); + } + + close(code?: number, reason?: string): void { + this.closed = true; + this.readyState = MockWebSocket.CLOSED; + const emitClose = () => this.emit("close", code ?? 1000, Buffer.from(reason ?? "")); + if (this.deferClose) { + this.deferredClose = emitClose; + return; + } + emitClose(); + } + + terminate(): void { + this.terminated = true; + this.close(1006, "terminated"); + } + + emitDeferredClose(): void { + const emitClose = this.deferredClose; + this.deferredClose = undefined; + emitClose?.(); + } + } + + return { + FakeWebSocket: MockWebSocket, + execFileSyncMock: vi.fn(), + fetchWithSsrFGuardMock: vi.fn(), + isProviderAuthProfileConfiguredMock: vi.fn(), + resolveProviderAuthProfileApiKeyMock: vi.fn(), + }; +} type FakeWebSocketLike = { sent: string[]; @@ -19,11 +91,65 @@ type FakeWebSocketConstructor = { instances: T[]; }; +type InternalRealtimeVoiceProviderApi = { + isBrowserSessionConfigured: (ctx: { + cfg?: object; + providerConfig: Record; + agentId?: string; + }) => boolean; + isGatewayRelayConfigured: (ctx: { + cfg?: object; + providerConfig: Record; + agentId?: string; + }) => boolean | undefined; + resolveBrowserSessionCapabilities: (ctx: { + cfg?: object; + providerConfig: Record; + model?: string; + }) => { + handlesAgentConsult?: boolean; + supportsToolCalls?: boolean; + supportsVideoFrames?: boolean; + supportsGatewayControl?: boolean; + transports?: string[]; + }; + resolveGatewayRelayCapabilities: (ctx: { + cfg?: object; + providerConfig: Record; + model?: string; + }) => { + handlesAgentConsult?: boolean; + supportsToolCalls?: boolean; + transports?: string[]; + }; + validateGatewayRelayLaunch: (ctx: { + cfg?: object; + providerConfig: Record; + model?: string; + autoRespondToAudio?: boolean; + }) => string | undefined; +}; + +const INTERNAL_REALTIME_VOICE_PROVIDER = Symbol.for("openclaw.internal.realtime-voice-provider.v1"); +const OPENAI_REALTIME_REJECTED_KEY_MESSAGE = + "OpenAI Realtime rejected the selected API key. Update or remove the active OpenAI API-key source"; + export function createOpenAIRealtimeTestSupport(deps: { FakeWebSocket: FakeWebSocketConstructor; + execFileSyncMock: ReturnType; fetchWithSsrFGuardMock: ReturnType; + isProviderAuthProfileConfiguredMock: ReturnType; + resolveProviderAuthProfileApiKeyMock: ReturnType; + buildOpenAIRealtimeVoiceProvider: () => RealtimeVoiceProviderPlugin; }) { - const { FakeWebSocket, fetchWithSsrFGuardMock } = deps; + const { + FakeWebSocket, + execFileSyncMock, + fetchWithSsrFGuardMock, + isProviderAuthProfileConfiguredMock, + resolveProviderAuthProfileApiKeyMock, + buildOpenAIRealtimeVoiceProvider, + } = deps; type FakeWebSocketInstance = T; type SentRealtimeEvent = { type: string; @@ -69,6 +195,31 @@ export function createOpenAIRealtimeTestSupport(dep return socket.sent.map((payload: string) => JSON.parse(payload) as SentRealtimeEvent); } + function resetTestState(): void { + FakeWebSocket.instances = []; + vi.stubEnv("OPENAI_API_KEY", ""); + execFileSyncMock.mockReset(); + fetchWithSsrFGuardMock.mockReset(); + isProviderAuthProfileConfiguredMock.mockReset(); + isProviderAuthProfileConfiguredMock.mockReturnValue(false); + resolveProviderAuthProfileApiKeyMock.mockReset(); + resolveProviderAuthProfileApiKeyMock.mockResolvedValue(undefined); + } + + function restoreTestEnvironment(): void { + vi.useRealTimers(); + vi.unstubAllEnvs(); + } + + function readInternalRealtimeVoiceProviderApi( + provider: object, + ): InternalRealtimeVoiceProviderApi { + return Reflect.get( + provider, + INTERNAL_REALTIME_VOICE_PROVIDER, + ) as InternalRealtimeVoiceProviderApi; + } + function createNativeBridge( overrides: Partial = {}, ): RealtimeVoiceBridge { @@ -109,6 +260,21 @@ export function createOpenAIRealtimeTestSupport(dep emitServerEvent(socket, { type: "session.updated" }); } + function emitAssistantPlayback( + socket: FakeWebSocketInstance, + overrides: { responseId?: string; itemId?: string; audio?: Buffer } = {}, + ): void { + emitServerEvent(socket, { + type: "response.created", + response: { id: overrides.responseId ?? "resp_1" }, + }); + emitServerEvent(socket, { + type: "response.audio.delta", + item_id: overrides.itemId ?? "item_1", + delta: (overrides.audio ?? Buffer.from("assistant audio")).toString("base64"), + }); + } + function emitCompletedToolCalls( socket: FakeWebSocketInstance, callIds: string[] = ["call_1"], @@ -182,6 +348,61 @@ export function createOpenAIRealtimeTestSupport(dep }); } + function mockRealtimeClientSecretResponse( + overrides: { clientSecret?: string; expiresAt?: number } = {}, + ): ReturnType { + const release = vi.fn(async () => undefined); + fetchWithSsrFGuardMock.mockResolvedValueOnce({ + response: createJsonResponse({ + client_secret: { value: overrides.clientSecret ?? "client-secret-123" }, + ...(overrides.expiresAt === undefined ? {} : { expires_at: overrides.expiresAt }), + }), + release, + }); + return release; + } + + function createQuicksilverBrowserBrokerFixture( + overrides: { + session?: { + provider?: "openai"; + transport?: "webrtc"; + clientSecret?: string; + offerUrl?: string; + }; + capabilities?: { + handlesAgentConsult?: true; + supportsToolCalls?: boolean; + supportsVideoFrames?: boolean; + transports?: Array<"webrtc">; + }; + } = {}, + ) { + const session: RealtimeVoiceBrowserSession = { + provider: "openai" as const, + transport: "webrtc" as const, + clientSecret: "quicksilver-token", + offerUrl: "/plugins/openai/realtime/calls", + ...overrides.session, + }; + const createBrowserSession = vi.fn( + async (_request: unknown, _auth: unknown): Promise => session, + ); + const cancelBrowserSession = vi.fn(async (_session: RealtimeVoiceBrowserSession) => undefined); + const broker = { + capabilities: { + transports: ["webrtc" as const], + handlesAgentConsult: true as const, + supportsToolCalls: false, + supportsVideoFrames: false, + ...overrides.capabilities, + }, + createBrowserSession, + cancelBrowserSession, + }; + return { broker, createBrowserSession, cancelBrowserSession }; + } + function requireRecord(value: unknown, label: string): Record { expect(isRecord(value), `${label} must be an object`).toBe(true); return value as Record; @@ -286,6 +507,9 @@ export function createOpenAIRealtimeTestSupport(dep } return { + resetTestState, + restoreTestEnvironment, + readInternalRealtimeVoiceProviderApi, parseSent, createNativeBridge, requireSocket, @@ -293,6 +517,7 @@ export function createOpenAIRealtimeTestSupport(dep openSocket, emitServerEvent, emitSessionUpdated, + emitAssistantPlayback, emitCompletedToolCalls, emitFunctionOutputAdded, expectedFunctionOutput, @@ -300,6 +525,9 @@ export function createOpenAIRealtimeTestSupport(dep expectedResponseCreateEvent, expectedResponseCancelEvent, createJsonResponse, + createQuicksilverBrowserBrokerFixture, + mockRealtimeClientSecretResponse, + rejectedKeyMessage: OPENAI_REALTIME_REJECTED_KEY_MESSAGE, requireRecord, requireNestedRecord, expectRecordFields,