ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-26 20:35:39 -06:00
Code Issues Packages Projects Releases Wiki Activity

refactor(openai): consolidate realtime test fixtures (#122670)

Browse Source 
* test(openai): consolidate realtime voice fixtures

* test(openai): table-drive realtime voice cases

* test(openai): remove redundant realtime coverage

Browser auth remains covered by 'requires Platform auth before minting browser realtime client secrets' in realtime-voice-browser-auth.test.ts.\n\nCancelled, failed, and incomplete tool output remains covered by 'normalizes .status without closing the reusable socket' in realtime-voice-terminal-outcomes.test.ts.\n\nCommitted oversized finals remain covered by the 'after item commit' row in realtime-transcription-provider.bounds.test.ts.
This commit is contained in:
Peter Steinberger
2026-08-12 08:39:26 -07:00
committed by GitHub
parent 9e8995384b
commit 0f4640a887
9 changed files with 831 additions and 1634 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/openai/realtime-transcription-provider.test.ts
-37
View File
@@ -1060,41 +1060,4 @@ describe("buildOpenAIRealtimeTranscriptionProvider", () => {
]);
session.close();
});
it("fails before retaining an oversized completed transcript", async () => {
const onError = vi.fn();
const onTranscript = vi.fn();
const provider = buildOpenAIRealtimeTranscriptionProvider();
const session = provider.createSession({
providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret
onError,
onTranscript,
});
const socket = await connectFakeSession(session);
emitJson(socket, {
type: "input_audio_buffer.committed",
item_id: "item-1",
previous_item_id: null,
});
emitJson(socket, {
type: "conversation.item.input_audio_transcription.completed",
item_id: "item-1",
transcript: "x".repeat(256 * 1024 + 1),
});
emitJson(socket, {
type: "conversation.item.input_audio_transcription.completed",
item_id: "item-1",
transcript: "late transcript",
});
expect(onError).toHaveBeenCalledExactlyOnceWith(
expect.objectContaining({
message: "OpenAI realtime transcription exceeded the 256 KiB retained transcript limit",
}),
);
expect(onTranscript).not.toHaveBeenCalled();
expect(session.isConnected()).toBe(false);
session.close();
});
});
extensions/openai/realtime-voice-bridge-connection.test.ts
+59 -190
View File
@@ -4,146 +4,31 @@ import type { RealtimeVoiceBridge } from "openclaw/plugin-sdk/realtime-voice";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js";
const INTERNAL_REALTIME_VOICE_PROVIDER = Symbol.for("openclaw.internal.realtime-voice-provider.v1");
function readInternalRealtimeVoiceProviderApi(provider: object) {
return Reflect.get(provider, INTERNAL_REALTIME_VOICE_PROVIDER) as {
isBrowserSessionConfigured: (ctx: {
cfg?: object;
providerConfig: Record<string, unknown>;
agentId?: string;
}) => boolean;
isGatewayRelayConfigured: (ctx: {
cfg?: object;
providerConfig: Record<string, unknown>;
agentId?: string;
}) => boolean | undefined;
resolveBrowserSessionCapabilities: (ctx: {
cfg?: object;
providerConfig: Record<string, unknown>;
model?: string;
}) => {
handlesAgentConsult?: boolean;
supportsToolCalls?: boolean;
supportsVideoFrames?: boolean;
supportsGatewayControl?: boolean;
transports?: string[];
};
resolveGatewayRelayCapabilities: (ctx: {
cfg?: object;
providerConfig: Record<string, unknown>;
model?: string;
}) => {
handlesAgentConsult?: boolean;
supportsToolCalls?: boolean;
transports?: string[];
};
validateGatewayRelayLaunch: (ctx: {
cfg?: object;
providerConfig: Record<string, unknown>;
model?: string;
autoRespondToAudio?: boolean;
}) => string | undefined;
cancelBrowserSession: (request: Record<string, unknown>, session: object) => Promise<void>;
};
}
const {
FakeWebSocket,
execFileSyncMock,
fetchWithSsrFGuardMock,
isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKeyMock,
} = vi.hoisted(() => {
type Listener = (...args: unknown[]) => void;
class MockWebSocket {
static readonly OPEN = 1;
static readonly CLOSED = 3;
static instances: MockWebSocket[] = [];
readonly listeners = new Map<string, Listener[]>();
readyState = 0;
sent: string[] = [];
closed = false;
terminated = false;
deferClose = false;
deferredClose: (() => void) | undefined;
args: unknown[];
constructor(...args: unknown[]) {
this.args = args;
MockWebSocket.instances.push(this);
}
on(event: string, listener: Listener): this {
const listeners = this.listeners.get(event) ?? [];
listeners.push(listener);
this.listeners.set(event, listeners);
return this;
}
emit(event: string, ...args: unknown[]): void {
for (const listener of this.listeners.get(event) ?? []) {
listener(...args);
}
}
send(payload: string): void {
this.sent.push(payload);
}
close(code?: number, reason?: string): void {
this.closed = true;
this.readyState = MockWebSocket.CLOSED;
const emitClose = () => this.emit("close", code ?? 1000, Buffer.from(reason ?? ""));
if (this.deferClose) {
this.deferredClose = emitClose;
return;
}
emitClose();
}
terminate(): void {
this.terminated = true;
this.close(1006, "terminated");
}
emitDeferredClose(): void {
const emitClose = this.deferredClose;
this.deferredClose = undefined;
emitClose?.();
}
}
return {
FakeWebSocket: MockWebSocket,
execFileSyncMock: vi.fn(),
fetchWithSsrFGuardMock: vi.fn(),
isProviderAuthProfileConfiguredMock: vi.fn(),
resolveProviderAuthProfileApiKeyMock: vi.fn(),
};
const mocks = await vi.hoisted(async () => {
const { createOpenAIRealtimeMockState } = await import("./realtime-voice-test-support.js");
return createOpenAIRealtimeMockState();
});
const { FakeWebSocket, fetchWithSsrFGuardMock } = mocks;
vi.mock("node:child_process", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:child_process")>();
return {
...actual,
execFileSync: execFileSyncMock,
execFileSync: mocks.execFileSyncMock,
};
});
vi.mock("ws", () => ({
default: FakeWebSocket,
default: mocks.FakeWebSocket,
}));
vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
fetchWithSsrFGuard: fetchWithSsrFGuardMock,
fetchWithSsrFGuard: mocks.fetchWithSsrFGuardMock,
}));
vi.mock("openclaw/plugin-sdk/provider-auth", () => ({
isProviderAuthProfileConfigured: isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKey: resolveProviderAuthProfileApiKeyMock,
isProviderAuthProfileConfigured: mocks.isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKey: mocks.resolveProviderAuthProfileApiKeyMock,
}));
import { createOpenAIRealtimeTestSupport } from "./realtime-voice-test-support.js";
@@ -164,23 +49,19 @@ const {
createRealtimeTool,
createUnreadableToolName,
createMalformedToolName,
} = createOpenAIRealtimeTestSupport({ FakeWebSocket, fetchWithSsrFGuardMock });
resetTestState,
restoreTestEnvironment,
readInternalRealtimeVoiceProviderApi,
createQuicksilverBrowserBrokerFixture,
} = createOpenAIRealtimeTestSupport({ ...mocks, buildOpenAIRealtimeVoiceProvider });
describe("OpenAI realtime voice bridge connection", () => {
beforeEach(() => {
FakeWebSocket.instances = [];
vi.stubEnv("OPENAI_API_KEY", "");
execFileSyncMock.mockReset();
fetchWithSsrFGuardMock.mockReset();
isProviderAuthProfileConfiguredMock.mockReset();
isProviderAuthProfileConfiguredMock.mockReturnValue(false);
resolveProviderAuthProfileApiKeyMock.mockReset();
resolveProviderAuthProfileApiKeyMock.mockResolvedValue(undefined);
resetTestState();
});
afterEach(() => {
vi.useRealTimers();
vi.unstubAllEnvs();
restoreTestEnvironment();
});
it("adds OpenClaw attribution headers to native realtime websocket requests", () => {
@@ -209,21 +90,11 @@ describe("OpenAI realtime voice bridge connection", () => {
});
it("sends one shared GA policy and waits for session.updated on an attached sideband", async () => {
const createBrowserSession = vi.fn(
async (_request: unknown, _auth: unknown) =>
({
provider: "openai",
transport: "webrtc" as const,
clientSecret: "gateway-token",
offerUrl: "/plugins/openai/realtime/calls",
}) as const,
);
const { broker, createBrowserSession } = createQuicksilverBrowserBrokerFixture({
session: { clientSecret: "gateway-token" },
});
const provider = buildOpenAIRealtimeVoiceProvider({
quicksilverBrowserSessionBroker: {
capabilities: { handlesAgentConsult: true as const },
createBrowserSession,
cancelBrowserSession: vi.fn(async () => undefined),
},
quicksilverBrowserSessionBroker: broker,
});
const bindBridge = vi.fn();
const onEvent = vi.fn();
@@ -620,15 +491,10 @@ describe("OpenAI realtime voice bridge connection", () => {
const { connecting, socket } = beginBridgeConnection(bridge);
openSocket(socket);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "error",
error: { message: "invalid realtime session" },
}),
),
);
emitServerEvent(socket, {
type: "error",
error: { message: "invalid realtime session" },
});
await expect(connecting).rejects.toThrow("invalid realtime session");
expect(bridge.isConnected()).toBe(false);
@@ -677,42 +543,45 @@ describe("OpenAI realtime voice bridge connection", () => {
expect(bridge.isConnected()).toBe(false);
});
it("can disable automatic audio turn responses for agent-routed voice loops", async () => {
const bridge = createNativeBridge({
it.each([
{
$name: "automatic audio turn responses disabled",
autoRespondToAudio: false,
});
const { connecting, socket } = beginBridgeConnection(bridge);
openSocket(socket);
emitSessionUpdated(socket);
await connecting;
expectRecordFields(
requireNestedRecord(requireSession(socket), ["audio", "input", "turn_detection"]),
"turn detection",
{
create_response: false,
interrupt_response: false,
},
);
});
it("can disable realtime response interruption while keeping audio responses enabled", async () => {
const bridge = createNativeBridge({
interruptResponseOnInputAudio: false,
expectedCreateResponse: false,
expectedInterruptResponse: false,
},
{
$name: "realtime response interruption disabled",
autoRespondToAudio: true,
interruptResponseOnInputAudio: false,
});
const socket = await connectReadyBridge(bridge);
expectedCreateResponse: true,
expectedInterruptResponse: false,
},
])(
"$name",
async ({
autoRespondToAudio,
interruptResponseOnInputAudio,
expectedCreateResponse,
expectedInterruptResponse,
}) => {
const bridge = createNativeBridge({
autoRespondToAudio,
interruptResponseOnInputAudio,
});
const socket = await connectReadyBridge(bridge);
expectRecordFields(
requireNestedRecord(requireSession(socket), ["audio", "input", "turn_detection"]),
"turn detection",
{
create_response: true,
interrupt_response: false,
},
);
});
expectRecordFields(
requireNestedRecord(requireSession(socket), ["audio", "input", "turn_detection"]),
"turn detection",
{
create_response: expectedCreateResponse,
interrupt_response: expectedInterruptResponse,
},
);
},
);
it("can request PCM16 24 kHz realtime audio for Chrome command-pair bridges", async () => {
const bridge = createNativeBridge({
extensions/openai/realtime-voice-bridge-events.test.ts
+89 -337
View File
@@ -1,103 +1,31 @@
// Openai tests cover realtime voice provider plugin behavior.
import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js";
const {
FakeWebSocket,
execFileSyncMock,
fetchWithSsrFGuardMock,
isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKeyMock,
} = vi.hoisted(() => {
type Listener = (...args: unknown[]) => void;
class MockWebSocket {
static readonly OPEN = 1;
static readonly CLOSED = 3;
static instances: MockWebSocket[] = [];
readonly listeners = new Map<string, Listener[]>();
readyState = 0;
sent: string[] = [];
closed = false;
terminated = false;
deferClose = false;
deferredClose: (() => void) | undefined;
args: unknown[];
constructor(...args: unknown[]) {
this.args = args;
MockWebSocket.instances.push(this);
}
on(event: string, listener: Listener): this {
const listeners = this.listeners.get(event) ?? [];
listeners.push(listener);
this.listeners.set(event, listeners);
return this;
}
emit(event: string, ...args: unknown[]): void {
for (const listener of this.listeners.get(event) ?? []) {
listener(...args);
}
}
send(payload: string): void {
this.sent.push(payload);
}
close(code?: number, reason?: string): void {
this.closed = true;
this.readyState = MockWebSocket.CLOSED;
const emitClose = () => this.emit("close", code ?? 1000, Buffer.from(reason ?? ""));
if (this.deferClose) {
this.deferredClose = emitClose;
return;
}
emitClose();
}
terminate(): void {
this.terminated = true;
this.close(1006, "terminated");
}
emitDeferredClose(): void {
const emitClose = this.deferredClose;
this.deferredClose = undefined;
emitClose?.();
}
}
return {
FakeWebSocket: MockWebSocket,
execFileSyncMock: vi.fn(),
fetchWithSsrFGuardMock: vi.fn(),
isProviderAuthProfileConfiguredMock: vi.fn(),
resolveProviderAuthProfileApiKeyMock: vi.fn(),
};
const mocks = await vi.hoisted(async () => {
const { createOpenAIRealtimeMockState } = await import("./realtime-voice-test-support.js");
return createOpenAIRealtimeMockState();
});
vi.mock("node:child_process", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:child_process")>();
return {
...actual,
execFileSync: execFileSyncMock,
execFileSync: mocks.execFileSyncMock,
};
});
vi.mock("ws", () => ({
default: FakeWebSocket,
default: mocks.FakeWebSocket,
}));
vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
fetchWithSsrFGuard: fetchWithSsrFGuardMock,
fetchWithSsrFGuard: mocks.fetchWithSsrFGuardMock,
}));
vi.mock("openclaw/plugin-sdk/provider-auth", () => ({
isProviderAuthProfileConfigured: isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKey: resolveProviderAuthProfileApiKeyMock,
isProviderAuthProfileConfigured: mocks.isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKey: mocks.resolveProviderAuthProfileApiKeyMock,
}));
import { createOpenAIRealtimeTestSupport } from "./realtime-voice-test-support.js";
@@ -105,91 +33,40 @@ const {
parseSent,
createNativeBridge,
connectReadyBridge,
emitServerEvent,
emitAssistantPlayback,
expectedResponseCancelEvent,
hasSentEventType,
} = createOpenAIRealtimeTestSupport({ FakeWebSocket, fetchWithSsrFGuardMock });
resetTestState,
restoreTestEnvironment,
} = createOpenAIRealtimeTestSupport({ ...mocks, buildOpenAIRealtimeVoiceProvider });
describe("OpenAI realtime voice bridge events", () => {
beforeEach(() => {
FakeWebSocket.instances = [];
vi.stubEnv("OPENAI_API_KEY", "");
execFileSyncMock.mockReset();
fetchWithSsrFGuardMock.mockReset();
isProviderAuthProfileConfiguredMock.mockReset();
isProviderAuthProfileConfiguredMock.mockReturnValue(false);
resolveProviderAuthProfileApiKeyMock.mockReset();
resolveProviderAuthProfileApiKeyMock.mockResolvedValue(undefined);
resetTestState();
});
afterEach(() => {
vi.useRealTimers();
vi.unstubAllEnvs();
restoreTestEnvironment();
});
it("does not locally clear playback on speech-start events when input interruption is disabled", async () => {
it.each([
{
$name: "input interruption disabled",
bridgeOptions: { autoRespondToAudio: true, interruptResponseOnInputAudio: false },
},
{
$name: "automatic audio responses disabled",
bridgeOptions: { autoRespondToAudio: false },
},
])("$name", async ({ bridgeOptions }) => {
const onAudio = vi.fn();
const onClearAudio = vi.fn();
const bridge = createNativeBridge({
autoRespondToAudio: true,
interruptResponseOnInputAudio: false,
onAudio,
onClearAudio,
});
const bridge = createNativeBridge({ ...bridgeOptions, onAudio, onClearAudio });
const socket = await connectReadyBridge(bridge);
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })),
);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "response.audio.delta",
item_id: "item_1",
delta: Buffer.from("assistant audio").toString("base64"),
}),
),
);
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "input_audio_buffer.speech_started" })),
);
expect(onAudio).toHaveBeenCalledTimes(1);
expect(onClearAudio).not.toHaveBeenCalled();
expect(hasSentEventType(socket, "response.cancel")).toBe(false);
expect(hasSentEventType(socket, "conversation.item.truncate")).toBe(false);
});
it("keeps assistant playback active on server VAD when automatic audio responses are disabled", async () => {
const onAudio = vi.fn();
const onClearAudio = vi.fn();
const bridge = createNativeBridge({
autoRespondToAudio: false,
onAudio,
onClearAudio,
});
const socket = await connectReadyBridge(bridge);
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })),
);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "response.audio.delta",
item_id: "item_1",
delta: Buffer.from("assistant audio").toString("base64"),
}),
),
);
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "input_audio_buffer.speech_started" })),
);
emitAssistantPlayback(socket);
emitServerEvent(socket, { type: "input_audio_buffer.speech_started" });
expect(onAudio).toHaveBeenCalledTimes(1);
expect(onClearAudio).not.toHaveBeenCalled();
@@ -208,20 +85,7 @@ describe("OpenAI realtime voice bridge events", () => {
const socket = await connectReadyBridge(bridge);
bridge.setMediaTimestamp(1000);
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })),
);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "response.audio.delta",
item_id: "item_1",
delta: Buffer.from("assistant audio").toString("base64"),
}),
),
);
emitAssistantPlayback(socket);
bridge.setMediaTimestamp(1300);
bridge.handleBargeIn?.({ audioPlaybackActive: true });
@@ -250,16 +114,11 @@ describe("OpenAI realtime voice bridge events", () => {
bridge.setMediaTimestamp(1000);
for (let index = 0; index < 300; index += 1) {
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "response.audio.delta",
item_id: "item_1",
delta: Buffer.from("assistant audio").toString("base64"),
}),
),
);
emitServerEvent(socket, {
type: "response.audio.delta",
item_id: "item_1",
delta: Buffer.from("assistant audio").toString("base64"),
});
}
const marks = onMark.mock.calls.map(([markName]) => String(markName));
@@ -281,16 +140,11 @@ describe("OpenAI realtime voice bridge events", () => {
expect(onClearAudio).toHaveBeenCalledWith("barge-in");
for (let index = 0; index < 300; index += 1) {
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "response.audio.delta",
item_id: "item_1",
delta: Buffer.from("assistant audio").toString("base64"),
}),
),
);
emitServerEvent(socket, {
type: "response.audio.delta",
item_id: "item_1",
delta: Buffer.from("assistant audio").toString("base64"),
});
}
const latestMark = onMark.mock.calls.at(-1)?.[0];
if (typeof latestMark !== "string") {
@@ -313,16 +167,11 @@ describe("OpenAI realtime voice bridge events", () => {
bridge.setMediaTimestamp(1000);
for (let index = 0; index < 3; index += 1) {
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "response.audio.delta",
item_id: "item_1",
delta: Buffer.from("assistant audio").toString("base64"),
}),
),
);
emitServerEvent(socket, {
type: "response.audio.delta",
item_id: "item_1",
delta: Buffer.from("assistant audio").toString("base64"),
});
}
const marks = onMark.mock.calls.map(([markName]) => String(markName));
expect(marks).toHaveLength(3);
@@ -349,25 +198,15 @@ describe("OpenAI realtime voice bridge events", () => {
const socket = await connectReadyBridge(bridge);
const audio = Buffer.from("assistant audio");
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "response.output_audio.delta",
item_id: "item_1",
delta: audio.toString("base64"),
}),
),
);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "response.output_audio_transcript.done",
transcript: "hello from current realtime events",
}),
),
);
emitServerEvent(socket, {
type: "response.output_audio.delta",
item_id: "item_1",
delta: audio.toString("base64"),
});
emitServerEvent(socket, {
type: "response.output_audio_transcript.done",
transcript: "hello from current realtime events",
});
expect(onAudio).toHaveBeenCalledWith(audio);
expect(onTranscript).toHaveBeenCalledWith(
@@ -383,16 +222,11 @@ describe("OpenAI realtime voice bridge events", () => {
const bridge = createNativeBridge({ onError, onEvent });
const socket = await connectReadyBridge(bridge);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "conversation.item.input_audio_transcription.failed",
item_id: "item_speech",
error: { code: "decoder_failure", message: "speech decoder exploded" },
}),
),
);
emitServerEvent(socket, {
type: "conversation.item.input_audio_transcription.failed",
item_id: "item_speech",
error: { code: "decoder_failure", message: "speech decoder exploded" },
});
expect(onError).toHaveBeenCalledWith(
expect.objectContaining({ message: "speech decoder exploded" }),
@@ -410,14 +244,8 @@ describe("OpenAI realtime voice bridge events", () => {
const bridge = createNativeBridge({ onTranscript });
const socket = await connectReadyBridge(bridge);
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "response.text.delta", delta: "draft assistant" })),
);
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "response.text.done", text: "corrected assistant" })),
);
emitServerEvent(socket, { type: "response.text.delta", delta: "draft assistant" });
emitServerEvent(socket, { type: "response.text.done", text: "corrected assistant" });
expect(onTranscript.mock.calls).toEqual([
["assistant", "draft assistant", false],
@@ -439,16 +267,7 @@ describe("OpenAI realtime voice bridge events", () => {
});
const socket = await connectReadyBridge(bridge);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "response.output_audio.delta",
item_id: "item_1",
delta,
}),
),
);
emitServerEvent(socket, { type: "response.output_audio.delta", item_id: "item_1", delta });
expect(onAudio).not.toHaveBeenCalled();
expect(onError).toHaveBeenCalledWith(
@@ -473,44 +292,24 @@ describe("OpenAI realtime voice bridge events", () => {
const socket = await connectReadyBridge(bridge);
const audio = Buffer.from("legacy assistant audio");
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "conversation.output_audio.delta",
data: audio.toString("base64"),
sample_rate: 24000,
channels: 1,
}),
),
);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "conversation.input_transcript.delta",
delta: "partial user",
}),
),
);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "conversation.output_transcript.delta",
delta: "partial assistant",
}),
),
);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "response.output_text.done",
text: "final assistant text",
}),
),
);
emitServerEvent(socket, {
type: "conversation.output_audio.delta",
data: audio.toString("base64"),
sample_rate: 24000,
channels: 1,
});
emitServerEvent(socket, {
type: "conversation.input_transcript.delta",
delta: "partial user",
});
emitServerEvent(socket, {
type: "conversation.output_transcript.delta",
delta: "partial assistant",
});
emitServerEvent(socket, {
type: "response.output_text.done",
text: "final assistant text",
});
expect(onAudio).toHaveBeenCalledWith(audio);
expect(onTranscript).toHaveBeenCalledWith("user", "partial user", false);
@@ -522,21 +321,13 @@ describe("OpenAI realtime voice bridge events", () => {
const onEvent = vi.fn();
const bridge = createNativeBridge({ onEvent });
const socket = await connectReadyBridge(bridge);
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })),
);
emitServerEvent(socket, { type: "response.created", response: { id: "resp_1" } });
bridge.setMediaTimestamp(1000);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "response.audio.delta",
item_id: "item_1",
delta: Buffer.from("assistant audio").toString("base64"),
}),
),
);
emitServerEvent(socket, {
type: "response.audio.delta",
item_id: "item_1",
delta: Buffer.from("assistant audio").toString("base64"),
});
bridge.setMediaTimestamp(1300);
bridge.handleBargeIn?.({ audioPlaybackActive: true });
@@ -564,20 +355,7 @@ describe("OpenAI realtime voice bridge events", () => {
});
const socket = await connectReadyBridge(bridge);
bridge.setMediaTimestamp(1000);
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })),
);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "response.audio.delta",
item_id: "item_1",
delta: Buffer.from("assistant audio").toString("base64"),
}),
),
);
emitAssistantPlayback(socket);
bridge.handleBargeIn?.({ audioPlaybackActive: true });
@@ -602,20 +380,7 @@ describe("OpenAI realtime voice bridge events", () => {
});
const socket = await connectReadyBridge(bridge);
bridge.setMediaTimestamp(1000);
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })),
);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "response.audio.delta",
item_id: "item_1",
delta: Buffer.from("assistant audio").toString("base64"),
}),
),
);
emitAssistantPlayback(socket);
bridge.handleBargeIn?.({ audioPlaybackActive: true, force: true });
@@ -647,20 +412,7 @@ describe("OpenAI realtime voice bridge events", () => {
});
const socket = await connectReadyBridge(bridge);
bridge.setMediaTimestamp(1000);
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })),
);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "response.audio.delta",
item_id: "item_1",
delta: Buffer.from("assistant audio").toString("base64"),
}),
),
);
emitAssistantPlayback(socket);
bridge.handleBargeIn?.({ audioPlaybackActive: true });
extensions/openai/realtime-voice-bridge-reconnect.test.ts
+24 -112
View File
@@ -1,106 +1,33 @@
// Openai tests cover realtime voice provider plugin behavior.
import type { RealtimeVoiceBridgeEvent } from "openclaw/plugin-sdk/realtime-voice";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js";
const OPENAI_REALTIME_REJECTED_KEY_MESSAGE =
"OpenAI Realtime rejected the selected API key. Update or remove the active OpenAI API-key source";
const {
FakeWebSocket,
execFileSyncMock,
fetchWithSsrFGuardMock,
isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKeyMock,
} = vi.hoisted(() => {
type Listener = (...args: unknown[]) => void;
class MockWebSocket {
static readonly OPEN = 1;
static readonly CLOSED = 3;
static instances: MockWebSocket[] = [];
readonly listeners = new Map<string, Listener[]>();
readyState = 0;
sent: string[] = [];
closed = false;
terminated = false;
deferClose = false;
deferredClose: (() => void) | undefined;
args: unknown[];
constructor(...args: unknown[]) {
this.args = args;
MockWebSocket.instances.push(this);
}
on(event: string, listener: Listener): this {
const listeners = this.listeners.get(event) ?? [];
listeners.push(listener);
this.listeners.set(event, listeners);
return this;
}
emit(event: string, ...args: unknown[]): void {
for (const listener of this.listeners.get(event) ?? []) {
listener(...args);
}
}
send(payload: string): void {
this.sent.push(payload);
}
close(code?: number, reason?: string): void {
this.closed = true;
this.readyState = MockWebSocket.CLOSED;
const emitClose = () => this.emit("close", code ?? 1000, Buffer.from(reason ?? ""));
if (this.deferClose) {
this.deferredClose = emitClose;
return;
}
emitClose();
}
terminate(): void {
this.terminated = true;
this.close(1006, "terminated");
}
emitDeferredClose(): void {
const emitClose = this.deferredClose;
this.deferredClose = undefined;
emitClose?.();
}
}
return {
FakeWebSocket: MockWebSocket,
execFileSyncMock: vi.fn(),
fetchWithSsrFGuardMock: vi.fn(),
isProviderAuthProfileConfiguredMock: vi.fn(),
resolveProviderAuthProfileApiKeyMock: vi.fn(),
};
const mocks = await vi.hoisted(async () => {
const { createOpenAIRealtimeMockState } = await import("./realtime-voice-test-support.js");
return createOpenAIRealtimeMockState();
});
const { FakeWebSocket } = mocks;
vi.mock("node:child_process", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:child_process")>();
return {
...actual,
execFileSync: execFileSyncMock,
execFileSync: mocks.execFileSyncMock,
};
});
vi.mock("ws", () => ({
default: FakeWebSocket,
default: mocks.FakeWebSocket,
}));
vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
fetchWithSsrFGuard: fetchWithSsrFGuardMock,
fetchWithSsrFGuard: mocks.fetchWithSsrFGuardMock,
}));
vi.mock("openclaw/plugin-sdk/provider-auth", () => ({
isProviderAuthProfileConfigured: isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKey: resolveProviderAuthProfileApiKeyMock,
isProviderAuthProfileConfigured: mocks.isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKey: mocks.resolveProviderAuthProfileApiKeyMock,
}));
import { createOpenAIRealtimeTestSupport } from "./realtime-voice-test-support.js";
@@ -115,23 +42,18 @@ const {
emitCompletedToolCalls,
emitFunctionOutputAdded,
connectReadyBridge,
} = createOpenAIRealtimeTestSupport({ FakeWebSocket, fetchWithSsrFGuardMock });
resetTestState,
restoreTestEnvironment,
rejectedKeyMessage: OPENAI_REALTIME_REJECTED_KEY_MESSAGE,
} = createOpenAIRealtimeTestSupport({ ...mocks, buildOpenAIRealtimeVoiceProvider });
describe("OpenAI realtime voice bridge reconnect", () => {
beforeEach(() => {
FakeWebSocket.instances = [];
vi.stubEnv("OPENAI_API_KEY", "");
execFileSyncMock.mockReset();
fetchWithSsrFGuardMock.mockReset();
isProviderAuthProfileConfiguredMock.mockReset();
isProviderAuthProfileConfiguredMock.mockReturnValue(false);
resolveProviderAuthProfileApiKeyMock.mockReset();
resolveProviderAuthProfileApiKeyMock.mockResolvedValue(undefined);
resetTestState();
});
afterEach(() => {
vi.useRealTimers();
vi.unstubAllEnvs();
restoreTestEnvironment();
});
it("rotates realtime bridges on provider max-duration events without reporting an error", async () => {
@@ -147,15 +69,10 @@ describe("OpenAI realtime voice bridge reconnect", () => {
await connecting;
expect(onReady).toHaveBeenCalledOnce();
firstSocket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "error",
error: { message: "Your session hit the maximum duration of 60 minutes." },
}),
),
);
emitServerEvent(firstSocket, {
type: "error",
error: { message: "Your session hit the maximum duration of 60 minutes." },
});
expect(onError).not.toHaveBeenCalled();
expect(firstSocket.closed).toBe(true);
@@ -394,15 +311,10 @@ describe("OpenAI realtime voice bridge reconnect", () => {
firstSocket.readyState = FakeWebSocket.CLOSED;
firstSocket.emit("close", 1006, Buffer.from("transient drop"));
firstSocket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "response.audio.delta",
delta: Buffer.from("late audio").toString("base64"),
}),
),
);
emitServerEvent(firstSocket, {
type: "response.audio.delta",
delta: Buffer.from("late audio").toString("base64"),
});
firstSocket.emit("error", new Error("late retry-wait failure"));
expect(onAudio).not.toHaveBeenCalled();
expect(onError).not.toHaveBeenCalled();
extensions/openai/realtime-voice-bridge-tools.test.ts
+19 -129
View File
@@ -1,102 +1,30 @@
// Openai tests cover realtime voice provider plugin behavior.
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js";
const {
FakeWebSocket,
execFileSyncMock,
fetchWithSsrFGuardMock,
isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKeyMock,
} = vi.hoisted(() => {
type Listener = (...args: unknown[]) => void;
class MockWebSocket {
static readonly OPEN = 1;
static readonly CLOSED = 3;
static instances: MockWebSocket[] = [];
readonly listeners = new Map<string, Listener[]>();
readyState = 0;
sent: string[] = [];
closed = false;
terminated = false;
deferClose = false;
deferredClose: (() => void) | undefined;
args: unknown[];
constructor(...args: unknown[]) {
this.args = args;
MockWebSocket.instances.push(this);
}
on(event: string, listener: Listener): this {
const listeners = this.listeners.get(event) ?? [];
listeners.push(listener);
this.listeners.set(event, listeners);
return this;
}
emit(event: string, ...args: unknown[]): void {
for (const listener of this.listeners.get(event) ?? []) {
listener(...args);
}
}
send(payload: string): void {
this.sent.push(payload);
}
close(code?: number, reason?: string): void {
this.closed = true;
this.readyState = MockWebSocket.CLOSED;
const emitClose = () => this.emit("close", code ?? 1000, Buffer.from(reason ?? ""));
if (this.deferClose) {
this.deferredClose = emitClose;
return;
}
emitClose();
}
terminate(): void {
this.terminated = true;
this.close(1006, "terminated");
}
emitDeferredClose(): void {
const emitClose = this.deferredClose;
this.deferredClose = undefined;
emitClose?.();
}
}
return {
FakeWebSocket: MockWebSocket,
execFileSyncMock: vi.fn(),
fetchWithSsrFGuardMock: vi.fn(),
isProviderAuthProfileConfiguredMock: vi.fn(),
resolveProviderAuthProfileApiKeyMock: vi.fn(),
};
const mocks = await vi.hoisted(async () => {
const { createOpenAIRealtimeMockState } = await import("./realtime-voice-test-support.js");
return createOpenAIRealtimeMockState();
});
vi.mock("node:child_process", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:child_process")>();
return {
...actual,
execFileSync: execFileSyncMock,
execFileSync: mocks.execFileSyncMock,
};
});
vi.mock("ws", () => ({
default: FakeWebSocket,
default: mocks.FakeWebSocket,
}));
vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
fetchWithSsrFGuard: fetchWithSsrFGuardMock,
fetchWithSsrFGuard: mocks.fetchWithSsrFGuardMock,
}));
vi.mock("openclaw/plugin-sdk/provider-auth", () => ({
isProviderAuthProfileConfigured: isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKey: resolveProviderAuthProfileApiKeyMock,
isProviderAuthProfileConfigured: mocks.isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKey: mocks.resolveProviderAuthProfileApiKeyMock,
}));
import { createOpenAIRealtimeTestSupport } from "./realtime-voice-test-support.js";
@@ -110,23 +38,17 @@ const {
connectReadyBridge,
expectedResponseCreateEvent,
hasSentEventType,
} = createOpenAIRealtimeTestSupport({ FakeWebSocket, fetchWithSsrFGuardMock });
resetTestState,
restoreTestEnvironment,
} = createOpenAIRealtimeTestSupport({ ...mocks, buildOpenAIRealtimeVoiceProvider });
describe("OpenAI realtime voice bridge tools", () => {
beforeEach(() => {
FakeWebSocket.instances = [];
vi.stubEnv("OPENAI_API_KEY", "");
execFileSyncMock.mockReset();
fetchWithSsrFGuardMock.mockReset();
isProviderAuthProfileConfiguredMock.mockReset();
isProviderAuthProfileConfiguredMock.mockReturnValue(false);
resolveProviderAuthProfileApiKeyMock.mockReset();
resolveProviderAuthProfileApiKeyMock.mockResolvedValue(undefined);
resetTestState();
});
afterEach(() => {
vi.useRealTimers();
vi.unstubAllEnvs();
restoreTestEnvironment();
});
it("executes tool calls only from successful response output", async () => {
@@ -189,35 +111,6 @@ describe("OpenAI realtime voice bridge tools", () => {
});
});
it.each(["cancelled", "failed", "incomplete"])(
"ignores function calls from a %s response",
async (status) => {
const onToolCall = vi.fn();
const bridge = createNativeBridge({ onToolCall });
const socket = await connectReadyBridge(bridge);
emitServerEvent(socket, {
type: "response.done",
response: {
id: "response_1",
status,
output: [
{
id: "item_tool_1",
type: "function_call",
status: "completed",
name: "openclaw_agent_consult",
call_id: "call_1",
arguments: '{"question":"must stay inert"}',
},
],
},
});
expect(onToolCall).not.toHaveBeenCalled();
},
);
it("ignores malformed and unfinished response output items", async () => {
const onToolCall = vi.fn();
const bridge = createNativeBridge({ onToolCall });
@@ -591,10 +484,7 @@ describe("OpenAI realtime voice bridge tools", () => {
type: "conversation.item.done",
detail: "itemType=function_call_output",
});
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_2" } })),
);
emitServerEvent(socket, { type: "response.created", response: { id: "resp_2" } });
emitServerEvent(socket, { type: "response.done" });
expect(parseSent(socket).filter((event) => event.type === "response.create")).toHaveLength(1);
@@ -675,10 +565,10 @@ describe("OpenAI realtime voice bridge tools", () => {
expect(onError).not.toHaveBeenCalled();
expect(parseSent(socket).filter((event) => event.type === "response.create")).toHaveLength(1);
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_status" } })),
);
emitServerEvent(socket, {
type: "response.created",
response: { id: "resp_status" },
});
emitServerEvent(socket, {
type: "response.done",
response: { id: "resp_status", status: "completed", output: [] },
extensions/openai/realtime-voice-browser-auth.test.ts
+140 -271
View File
@@ -2,105 +2,37 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js";
const OPENAI_REALTIME_REJECTED_KEY_MESSAGE =
"OpenAI Realtime rejected the selected API key. Update or remove the active OpenAI API-key source";
const mocks = await vi.hoisted(async () => {
const { createOpenAIRealtimeMockState } = await import("./realtime-voice-test-support.js");
return createOpenAIRealtimeMockState();
});
const {
FakeWebSocket,
execFileSyncMock,
fetchWithSsrFGuardMock,
isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKeyMock,
} = vi.hoisted(() => {
type Listener = (...args: unknown[]) => void;
class MockWebSocket {
static readonly OPEN = 1;
static readonly CLOSED = 3;
static instances: MockWebSocket[] = [];
readonly listeners = new Map<string, Listener[]>();
readyState = 0;
sent: string[] = [];
closed = false;
terminated = false;
deferClose = false;
deferredClose: (() => void) | undefined;
args: unknown[];
constructor(...args: unknown[]) {
this.args = args;
MockWebSocket.instances.push(this);
}
on(event: string, listener: Listener): this {
const listeners = this.listeners.get(event) ?? [];
listeners.push(listener);
this.listeners.set(event, listeners);
return this;
}
emit(event: string, ...args: unknown[]): void {
for (const listener of this.listeners.get(event) ?? []) {
listener(...args);
}
}
send(payload: string): void {
this.sent.push(payload);
}
close(code?: number, reason?: string): void {
this.closed = true;
this.readyState = MockWebSocket.CLOSED;
const emitClose = () => this.emit("close", code ?? 1000, Buffer.from(reason ?? ""));
if (this.deferClose) {
this.deferredClose = emitClose;
return;
}
emitClose();
}
terminate(): void {
this.terminated = true;
this.close(1006, "terminated");
}
emitDeferredClose(): void {
const emitClose = this.deferredClose;
this.deferredClose = undefined;
emitClose?.();
}
}
return {
FakeWebSocket: MockWebSocket,
execFileSyncMock: vi.fn(),
fetchWithSsrFGuardMock: vi.fn(),
isProviderAuthProfileConfiguredMock: vi.fn(),
resolveProviderAuthProfileApiKeyMock: vi.fn(),
};
});
} = mocks;
vi.mock("node:child_process", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:child_process")>();
return {
...actual,
execFileSync: execFileSyncMock,
execFileSync: mocks.execFileSyncMock,
};
});
vi.mock("ws", () => ({
default: FakeWebSocket,
default: mocks.FakeWebSocket,
}));
vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
fetchWithSsrFGuard: fetchWithSsrFGuardMock,
fetchWithSsrFGuard: mocks.fetchWithSsrFGuardMock,
}));
vi.mock("openclaw/plugin-sdk/provider-auth", () => ({
isProviderAuthProfileConfigured: isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKey: resolveProviderAuthProfileApiKeyMock,
isProviderAuthProfileConfigured: mocks.isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKey: mocks.resolveProviderAuthProfileApiKeyMock,
}));
import { createOpenAIRealtimeTestSupport } from "./realtime-voice-test-support.js";
@@ -108,6 +40,7 @@ const {
createNativeBridge,
beginBridgeConnection,
openSocket,
emitServerEvent,
createJsonResponse,
requireRecord,
requireNestedRecord,
@@ -118,23 +51,20 @@ const {
requireFetchHeaders,
requireFetchJsonBody,
createTestJwt,
} = createOpenAIRealtimeTestSupport({ FakeWebSocket, fetchWithSsrFGuardMock });
resetTestState,
restoreTestEnvironment,
mockRealtimeClientSecretResponse,
rejectedKeyMessage: OPENAI_REALTIME_REJECTED_KEY_MESSAGE,
createQuicksilverBrowserBrokerFixture,
} = createOpenAIRealtimeTestSupport({ ...mocks, buildOpenAIRealtimeVoiceProvider });
describe("OpenAI realtime voice browser authentication", () => {
beforeEach(() => {
FakeWebSocket.instances = [];
vi.stubEnv("OPENAI_API_KEY", "");
execFileSyncMock.mockReset();
fetchWithSsrFGuardMock.mockReset();
isProviderAuthProfileConfiguredMock.mockReset();
isProviderAuthProfileConfiguredMock.mockReturnValue(false);
resolveProviderAuthProfileApiKeyMock.mockReset();
resolveProviderAuthProfileApiKeyMock.mockResolvedValue(undefined);
resetTestState();
});
afterEach(() => {
vi.useRealTimers();
vi.unstubAllEnvs();
restoreTestEnvironment();
});
it("requires Platform auth for native realtime websocket bridges", async () => {
@@ -154,35 +84,75 @@ describe("OpenAI realtime voice browser authentication", () => {
expect(FakeWebSocket.instances).toHaveLength(0);
});
it("uses OPENAI_API_KEY for default GPT realtime bridges", async () => {
vi.stubEnv("OPENAI_API_KEY", "test-api-key-env");
const provider = buildOpenAIRealtimeVoiceProvider();
const bridge = provider.createBridge({
cfg: {} as never,
providerConfig: { model: "gpt-realtime-2" },
onAudio: vi.fn(),
onClearAudio: vi.fn(),
});
it.each([
{
$name: "environment API key",
environmentKey: "test-api-key-env",
profileKey: undefined,
configuredProfile: false,
expectedAuthorization: "Bearer test-api-key-env",
assertion: "environment" as const,
},
{
$name: "API-key profile",
environmentKey: undefined,
profileKey: "test-api-key-profile",
configuredProfile: false,
expectedAuthorization: "Bearer test-api-key-profile",
assertion: "profile" as const,
},
{
$name: "environment fallback after an unresolved configured profile",
environmentKey: "test-api-key-env",
profileKey: undefined,
configuredProfile: true,
expectedAuthorization: "Bearer test-api-key-env",
assertion: "fallback" as const,
},
])(
"$name",
async ({ environmentKey, profileKey, configuredProfile, expectedAuthorization, assertion }) => {
if (environmentKey) {
vi.stubEnv("OPENAI_API_KEY", environmentKey);
}
resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce(profileKey);
if (configuredProfile) {
isProviderAuthProfileConfiguredMock.mockReturnValueOnce(true);
}
const provider = buildOpenAIRealtimeVoiceProvider();
const bridge = provider.createBridge({
cfg: {} as never,
providerConfig: { model: "gpt-realtime-2" },
onAudio: vi.fn(),
onClearAudio: vi.fn(),
});
void bridge.connect();
await vi.waitFor(() => expect(FakeWebSocket.instances.length).toBe(1));
bridge.close();
void bridge.connect();
await vi.waitFor(() => expect(FakeWebSocket.instances.length).toBe(1));
bridge.close();
expect(resolveProviderAuthProfileApiKeyMock.mock.calls).toEqual([
[
{
provider: "openai",
cfg: {},
profileTypes: ["api_key"],
includeExternalCliAuth: false,
},
],
]);
expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled();
const socket = FakeWebSocket.instances[0];
const options = socket?.args[1] as { headers?: Record<string, string> } | undefined;
expect(options?.headers?.Authorization).toBe("Bearer test-api-key-env");
});
if (assertion === "fallback") {
expect(resolveProviderAuthProfileApiKeyMock).toHaveBeenCalledTimes(1);
} else {
expect(resolveProviderAuthProfileApiKeyMock.mock.calls).toEqual([
[
{
provider: "openai",
cfg: {},
profileTypes: ["api_key"],
includeExternalCliAuth: false,
},
],
]);
}
if (assertion === "environment") {
expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled();
}
const socket = FakeWebSocket.instances[0];
const options = socket?.args[1] as { headers?: Record<string, string> } | undefined;
expect(options?.headers?.Authorization).toBe(expectedAuthorization);
},
);
it("does not use Codex OAuth profiles for default GPT realtime bridges", async () => {
const provider = buildOpenAIRealtimeVoiceProvider();
@@ -211,57 +181,6 @@ describe("OpenAI realtime voice browser authentication", () => {
expect(FakeWebSocket.instances).toHaveLength(0);
});
it("uses OPENAI_API_KEY when a configured API-key profile cannot be resolved", async () => {
vi.stubEnv("OPENAI_API_KEY", "test-api-key-env");
resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce(undefined);
isProviderAuthProfileConfiguredMock.mockReturnValueOnce(true);
const provider = buildOpenAIRealtimeVoiceProvider();
const bridge = provider.createBridge({
cfg: {} as never,
providerConfig: { model: "gpt-realtime-2" },
onAudio: vi.fn(),
onClearAudio: vi.fn(),
});
void bridge.connect();
await vi.waitFor(() => expect(FakeWebSocket.instances.length).toBe(1));
bridge.close();
expect(resolveProviderAuthProfileApiKeyMock).toHaveBeenCalledTimes(1);
const socket = FakeWebSocket.instances[0];
const options = socket?.args[1] as { headers?: Record<string, string> } | undefined;
expect(options?.headers?.Authorization).toBe("Bearer test-api-key-env");
});
it("uses OpenAI API-key auth profiles", async () => {
resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce("test-api-key-profile");
const provider = buildOpenAIRealtimeVoiceProvider();
const bridge = provider.createBridge({
cfg: {} as never,
providerConfig: { model: "gpt-realtime-2" },
onAudio: vi.fn(),
onClearAudio: vi.fn(),
});
void bridge.connect();
await vi.waitFor(() => expect(FakeWebSocket.instances.length).toBe(1));
bridge.close();
expect(resolveProviderAuthProfileApiKeyMock.mock.calls).toEqual([
[
{
provider: "openai",
cfg: {},
profileTypes: ["api_key"],
includeExternalCliAuth: false,
},
],
]);
const socket = FakeWebSocket.instances[0];
const options = socket?.args[1] as { headers?: Record<string, string> } | undefined;
expect(options?.headers?.Authorization).toBe("Bearer test-api-key-profile");
});
it("keeps explicit OpenAI realtime API keys as the advanced override", () => {
vi.stubEnv("OPENAI_API_KEY", "test-api-key-env");
resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce("test-api-key-profile");
@@ -305,13 +224,7 @@ describe("OpenAI realtime voice browser authentication", () => {
it("returns browser-safe OpenClaw attribution headers for native WebRTC offers", async () => {
vi.stubEnv("OPENCLAW_VERSION", "2026.3.22");
fetchWithSsrFGuardMock.mockResolvedValueOnce({
response: createJsonResponse({
client_secret: { value: "client-secret-123" },
expires_at: 1_765_000_000,
}),
release: vi.fn(async () => undefined),
});
mockRealtimeClientSecretResponse({ expiresAt: 1_765_000_000 });
const provider = buildOpenAIRealtimeVoiceProvider();
if (!provider.createBrowserSession) {
throw new Error("expected OpenAI realtime provider to support browser sessions");
@@ -401,12 +314,7 @@ describe("OpenAI realtime voice browser authentication", () => {
it("resolves keychain OPENAI_API_KEY refs before creating browser sessions", async () => {
vi.stubEnv("OPENAI_API_KEY", "keychain:openclaw:OPENAI_REALTIME_BROWSER_TEST");
execFileSyncMock.mockReturnValueOnce("test-api-key-browser-env\n");
fetchWithSsrFGuardMock.mockResolvedValueOnce({
response: createJsonResponse({
client_secret: { value: "client-secret-123" },
}),
release: vi.fn(async () => undefined),
});
mockRealtimeClientSecretResponse();
const provider = buildOpenAIRealtimeVoiceProvider();
if (!provider.createBrowserSession) {
throw new Error("expected OpenAI realtime provider to support browser sessions");
@@ -477,10 +385,7 @@ describe("OpenAI realtime voice browser authentication", () => {
async ({ profileTypes }: { profileTypes?: readonly string[] }) =>
profileTypes?.includes("oauth") ? oauthToken : undefined,
);
fetchWithSsrFGuardMock.mockResolvedValueOnce({
response: createJsonResponse({ client_secret: { value: "client-secret-123" } }),
release: vi.fn(async () => undefined),
});
mockRealtimeClientSecretResponse();
const provider = buildOpenAIRealtimeVoiceProvider();
await provider.createBrowserSession?.({
@@ -508,13 +413,9 @@ describe("OpenAI realtime voice browser authentication", () => {
({ profileTypes }: { profileTypes?: readonly string[] }) =>
profileTypes?.includes("api_key") === true,
);
const createBrowserSession = vi.fn();
const { broker, createBrowserSession } = createQuicksilverBrowserBrokerFixture();
const provider = buildOpenAIRealtimeVoiceProvider({
quicksilverBrowserSessionBroker: {
capabilities: { handlesAgentConsult: true as const },
createBrowserSession,
cancelBrowserSession: vi.fn(),
},
quicksilverBrowserSessionBroker: broker,
});
await expect(
@@ -533,16 +434,6 @@ describe("OpenAI realtime voice browser authentication", () => {
);
});
it("requires Platform auth for browser sessions", async () => {
const provider = buildOpenAIRealtimeVoiceProvider();
await expect(
provider.createBrowserSession?.({
providerConfig: {},
}),
).rejects.toThrow("OpenAI Realtime voice requires an OpenAI Platform API key");
expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled();
});
it("reports an unresolved Platform credential without trying another auth route", async () => {
vi.stubEnv("OPENAI_API_KEY", "keychain:openclaw:OPENAI_REALTIME_MISSING_TEST");
execFileSyncMock.mockImplementationOnce(() => {
@@ -605,12 +496,7 @@ describe("OpenAI realtime voice browser authentication", () => {
it("uses OPENAI_API_KEY for default GPT browser sessions", async () => {
vi.stubEnv("OPENAI_API_KEY", "test-api-key-env");
fetchWithSsrFGuardMock.mockResolvedValueOnce({
response: createJsonResponse({
client_secret: { value: "client-secret-123" },
}),
release: vi.fn(async () => undefined),
});
mockRealtimeClientSecretResponse();
const provider = buildOpenAIRealtimeVoiceProvider();
if (!provider.createBrowserSession) {
throw new Error("expected OpenAI realtime provider to support browser sessions");
@@ -673,24 +559,14 @@ describe("OpenAI realtime voice browser authentication", () => {
const { connecting, socket } = beginBridgeConnection(bridge);
openSocket(socket);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "error",
error: { message: "Incorrect API key provided: test-api-key-proj-***" },
}),
),
);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "error",
error: { message: "Incorrect API key provided: test-api-key-proj-***" },
}),
),
);
emitServerEvent(socket, {
type: "error",
error: { message: "Incorrect API key provided: test-api-key-proj-***" },
});
emitServerEvent(socket, {
type: "error",
error: { message: "Incorrect API key provided: test-api-key-proj-***" },
});
await expect(connecting).rejects.toThrow(OPENAI_REALTIME_REJECTED_KEY_MESSAGE);
expect(onError).not.toHaveBeenCalled();
@@ -699,64 +575,57 @@ describe("OpenAI realtime voice browser authentication", () => {
expect(bridge.isConnected()).toBe(false);
});
it("normalizes structured direct OpenAI startup auth errors", async () => {
const bridge = createNativeBridge();
const { connecting, socket } = beginBridgeConnection(bridge);
openSocket(socket);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "error",
error: {
type: "invalid_request_error",
code: "invalid_api_key",
message: "Invalid API key",
},
}),
),
);
await expect(connecting).rejects.toThrow(OPENAI_REALTIME_REJECTED_KEY_MESSAGE);
expect(bridge.isConnected()).toBe(false);
});
it("normalizes direct OpenAI socket handshake auth errors", async () => {
const bridge = createNativeBridge();
const { connecting, socket } = beginBridgeConnection(bridge);
socket.emit("error", new Error("Unexpected server response: 401"));
await expect(connecting).rejects.toThrow(OPENAI_REALTIME_REJECTED_KEY_MESSAGE);
expect(bridge.isConnected()).toBe(false);
});
it.each([
[
"Azure deployment",
{
{
$name: "structured direct error expects normalization",
event: "structured" as const,
providerConfig: undefined,
expectedMessage: OPENAI_REALTIME_REJECTED_KEY_MESSAGE,
},
{
$name: "direct handshake error expects normalization",
event: "handshake" as const,
providerConfig: undefined,
expectedMessage: OPENAI_REALTIME_REJECTED_KEY_MESSAGE,
},
{
$name: "Azure handshake error expects raw preservation",
event: "handshake" as const,
providerConfig: {
apiKey: "test-api-key-test",
azureEndpoint: "https://example.openai.azure.com",
azureDeployment: "realtime-prod",
},
],
[
"custom endpoint",
{
expectedMessage: "Unexpected server response: 401",
},
{
$name: "custom-endpoint handshake error expects raw preservation",
event: "handshake" as const,
providerConfig: {
apiKey: "test-api-key-test",
azureEndpoint: "https://realtime-proxy.example.com",
},
],
])("preserves %s startup auth errors", async (_label, providerConfig) => {
const bridge = createNativeBridge({
providerConfig,
});
expectedMessage: "Unexpected server response: 401",
},
])("$name", async ({ event, providerConfig, expectedMessage }) => {
const bridge = createNativeBridge(providerConfig ? { providerConfig } : {});
const { connecting, socket } = beginBridgeConnection(bridge);
socket.emit("error", new Error("Unexpected server response: 401"));
if (event === "structured") {
openSocket(socket);
emitServerEvent(socket, {
type: "error",
error: {
type: "invalid_request_error",
code: "invalid_api_key",
message: "Invalid API key",
},
});
} else {
socket.emit("error", new Error("Unexpected server response: 401"));
}
await expect(connecting).rejects.toThrow("Unexpected server response: 401");
await expect(connecting).rejects.toThrow(expectedMessage);
expect(bridge.isConnected()).toBe(false);
});
});
extensions/openai/realtime-voice-provider-routing.test.ts
+206 -358
View File
@@ -2,174 +2,60 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js";
const INTERNAL_REALTIME_VOICE_PROVIDER = Symbol.for("openclaw.internal.realtime-voice-provider.v1");
function readInternalRealtimeVoiceProviderApi(provider: object) {
return Reflect.get(provider, INTERNAL_REALTIME_VOICE_PROVIDER) as {
isBrowserSessionConfigured: (ctx: {
cfg?: object;
providerConfig: Record<string, unknown>;
agentId?: string;
}) => boolean;
isGatewayRelayConfigured: (ctx: {
cfg?: object;
providerConfig: Record<string, unknown>;
agentId?: string;
}) => boolean | undefined;
resolveBrowserSessionCapabilities: (ctx: {
cfg?: object;
providerConfig: Record<string, unknown>;
model?: string;
}) => {
handlesAgentConsult?: boolean;
supportsToolCalls?: boolean;
supportsVideoFrames?: boolean;
supportsGatewayControl?: boolean;
transports?: string[];
};
resolveGatewayRelayCapabilities: (ctx: {
cfg?: object;
providerConfig: Record<string, unknown>;
model?: string;
}) => {
handlesAgentConsult?: boolean;
supportsToolCalls?: boolean;
transports?: string[];
};
validateGatewayRelayLaunch: (ctx: {
cfg?: object;
providerConfig: Record<string, unknown>;
model?: string;
autoRespondToAudio?: boolean;
}) => string | undefined;
cancelBrowserSession: (request: Record<string, unknown>, session: object) => Promise<void>;
};
}
const mocks = await vi.hoisted(async () => {
const { createOpenAIRealtimeMockState } = await import("./realtime-voice-test-support.js");
return createOpenAIRealtimeMockState();
});
const {
FakeWebSocket,
execFileSyncMock,
fetchWithSsrFGuardMock,
isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKeyMock,
} = vi.hoisted(() => {
type Listener = (...args: unknown[]) => void;
class MockWebSocket {
static readonly OPEN = 1;
static readonly CLOSED = 3;
static instances: MockWebSocket[] = [];
readonly listeners = new Map<string, Listener[]>();
readyState = 0;
sent: string[] = [];
closed = false;
terminated = false;
deferClose = false;
deferredClose: (() => void) | undefined;
args: unknown[];
constructor(...args: unknown[]) {
this.args = args;
MockWebSocket.instances.push(this);
}
on(event: string, listener: Listener): this {
const listeners = this.listeners.get(event) ?? [];
listeners.push(listener);
this.listeners.set(event, listeners);
return this;
}
emit(event: string, ...args: unknown[]): void {
for (const listener of this.listeners.get(event) ?? []) {
listener(...args);
}
}
send(payload: string): void {
this.sent.push(payload);
}
close(code?: number, reason?: string): void {
this.closed = true;
this.readyState = MockWebSocket.CLOSED;
const emitClose = () => this.emit("close", code ?? 1000, Buffer.from(reason ?? ""));
if (this.deferClose) {
this.deferredClose = emitClose;
return;
}
emitClose();
}
terminate(): void {
this.terminated = true;
this.close(1006, "terminated");
}
emitDeferredClose(): void {
const emitClose = this.deferredClose;
this.deferredClose = undefined;
emitClose?.();
}
}
return {
FakeWebSocket: MockWebSocket,
execFileSyncMock: vi.fn(),
fetchWithSsrFGuardMock: vi.fn(),
isProviderAuthProfileConfiguredMock: vi.fn(),
resolveProviderAuthProfileApiKeyMock: vi.fn(),
};
});
} = mocks;
vi.mock("node:child_process", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:child_process")>();
return {
...actual,
execFileSync: execFileSyncMock,
execFileSync: mocks.execFileSyncMock,
};
});
vi.mock("ws", () => ({
default: FakeWebSocket,
default: mocks.FakeWebSocket,
}));
vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
fetchWithSsrFGuard: fetchWithSsrFGuardMock,
fetchWithSsrFGuard: mocks.fetchWithSsrFGuardMock,
}));
vi.mock("openclaw/plugin-sdk/provider-auth", () => ({
isProviderAuthProfileConfigured: isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKey: resolveProviderAuthProfileApiKeyMock,
isProviderAuthProfileConfigured: mocks.isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKey: mocks.resolveProviderAuthProfileApiKeyMock,
}));
import { createOpenAIRealtimeTestSupport } from "./realtime-voice-test-support.js";
const {
createJsonResponse,
requireRecord,
requireFetchJsonBody,
createRealtimeTool,
createUnreadableToolName,
createMalformedToolName,
createTestJwt,
} = createOpenAIRealtimeTestSupport({ FakeWebSocket, fetchWithSsrFGuardMock });
resetTestState,
restoreTestEnvironment,
readInternalRealtimeVoiceProviderApi,
mockRealtimeClientSecretResponse,
createQuicksilverBrowserBrokerFixture,
} = createOpenAIRealtimeTestSupport({ ...mocks, buildOpenAIRealtimeVoiceProvider });
describe("OpenAI realtime voice provider routing", () => {
beforeEach(() => {
FakeWebSocket.instances = [];
vi.stubEnv("OPENAI_API_KEY", "");
execFileSyncMock.mockReset();
fetchWithSsrFGuardMock.mockReset();
isProviderAuthProfileConfiguredMock.mockReset();
isProviderAuthProfileConfiguredMock.mockReturnValue(false);
resolveProviderAuthProfileApiKeyMock.mockReset();
resolveProviderAuthProfileApiKeyMock.mockResolvedValue(undefined);
resetTestState();
});
afterEach(() => {
vi.useRealTimers();
vi.unstubAllEnvs();
restoreTestEnvironment();
});
it("declares realtime Talk capabilities for catalog selection", () => {
@@ -206,51 +92,45 @@ describe("OpenAI realtime voice provider routing", () => {
expect(bridge.supportsToolResultSuppression).toBe(true);
});
it("advertises quicksilver capabilities only for curated /v1/live models", () => {
const quicksilverBroker = {
capabilities: {
transports: ["webrtc" as const],
handlesAgentConsult: true as const,
it.each([
{
$name: "browser capability projection",
surface: "browser" as const,
expected: {
transports: ["webrtc", "gateway-relay"],
handlesAgentConsult: true,
supportsToolCalls: false,
supportsVideoFrames: false,
},
createBrowserSession: vi.fn(),
cancelBrowserSession: vi.fn(),
};
},
{
$name: "gateway-relay capability projection",
surface: "gateway-relay" as const,
expected: {
transports: ["webrtc", "gateway-relay"],
handlesAgentConsult: true,
supportsToolCalls: false,
},
},
])("$name", ({ surface, expected }) => {
const { broker } = createQuicksilverBrowserBrokerFixture();
const provider = buildOpenAIRealtimeVoiceProvider({
quicksilverBrowserSessionBroker: quicksilverBroker,
quicksilverBrowserSessionBroker: broker,
});
const internalApi = readInternalRealtimeVoiceProviderApi(provider);
const resolveCapabilities =
surface === "browser"
? internalApi.resolveBrowserSessionCapabilities
: internalApi.resolveGatewayRelayCapabilities;
expect(
internalApi.resolveBrowserSessionCapabilities({
resolveCapabilities({
providerConfig: { model: "gpt-realtime-2.1" },
model: "gpt-live-1-codex",
}),
).toMatchObject({
transports: ["webrtc", "gateway-relay"],
handlesAgentConsult: true,
supportsToolCalls: false,
supportsVideoFrames: false,
});
).toMatchObject(expected);
expect(
internalApi.resolveGatewayRelayCapabilities({
providerConfig: { model: "gpt-realtime-2.1" },
model: "gpt-live-1-codex",
}),
).toMatchObject({
transports: ["webrtc", "gateway-relay"],
handlesAgentConsult: true,
supportsToolCalls: false,
});
expect(
internalApi.resolveBrowserSessionCapabilities({
providerConfig: { model: "gpt-realtime-2.1" },
model: "gpt-live-1-mini",
}),
).not.toHaveProperty("handlesAgentConsult");
expect(
internalApi.resolveGatewayRelayCapabilities({
resolveCapabilities({
providerConfig: { model: "gpt-realtime-2.1" },
model: "gpt-live-1-mini",
}),
@@ -258,10 +138,7 @@ describe("OpenAI realtime voice provider routing", () => {
});
it("omits unsupported OpenAI tool names from browser sessions", async () => {
fetchWithSsrFGuardMock.mockResolvedValueOnce({
response: createJsonResponse({ client_secret: { value: "client-secret-123" } }),
release: vi.fn(async () => undefined),
});
mockRealtimeClientSecretResponse();
const provider = buildOpenAIRealtimeVoiceProvider();
if (!provider.createBrowserSession) {
throw new Error("expected OpenAI realtime provider to support browser sessions");
@@ -304,23 +181,9 @@ describe("OpenAI realtime voice provider routing", () => {
});
it("routes gpt-live Platform sessions through the native quicksilver broker", async () => {
const createBrowserSession = vi.fn(async (_request: unknown, _auth: unknown) => ({
provider: "openai",
transport: "webrtc" as const,
clientSecret: "quicksilver-token",
offerUrl: "/plugins/openai/realtime/calls",
}));
const { broker, createBrowserSession } = createQuicksilverBrowserBrokerFixture();
const provider = buildOpenAIRealtimeVoiceProvider({
quicksilverBrowserSessionBroker: {
capabilities: {
transports: ["webrtc" as const],
handlesAgentConsult: true as const,
supportsToolCalls: false,
supportsVideoFrames: false,
},
createBrowserSession,
cancelBrowserSession: vi.fn(),
},
quicksilverBrowserSessionBroker: broker,
});
const request = {
providerConfig: { apiKey: "test-api-key-platform" },
@@ -341,7 +204,150 @@ describe("OpenAI realtime voice provider routing", () => {
expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled();
});
it("routes an explicit unlisted gpt-live alias without advertising it as ready", async () => {
it.each([
{
$name: "provider | gpt-live-1-mini | ChatGPT OAuth | standard endpoint | not ready",
surface: "provider" as const,
providerConfig: { model: "gpt-live-1-mini" },
agentId: "main",
expected: false,
expectAgentDir: false,
},
{
$name: "gateway-relay | gpt-live-1-mini | ChatGPT OAuth | standard endpoint | not ready",
surface: "gateway-relay" as const,
providerConfig: { model: "gpt-live-1-mini" },
agentId: "main",
expected: false,
expectAgentDir: false,
},
{
$name: "gateway-relay | gpt-live-1-mini | ChatGPT OAuth | Azure endpoint | not ready",
surface: "gateway-relay" as const,
providerConfig: {
model: "gpt-live-1-mini",
azureEndpoint: "https://example.openai.azure.com",
azureDeployment: "gpt-live",
},
agentId: "main",
expected: false,
expectAgentDir: false,
},
{
$name: "browser | gpt-live-1-mini | ChatGPT OAuth | standard endpoint | not ready",
surface: "browser" as const,
providerConfig: { model: "gpt-live-1-mini" },
agentId: "main",
expected: false,
expectAgentDir: false,
},
{
$name:
"gateway-relay | gpt-realtime-2.1 | Platform API key | standard endpoint | not applicable",
surface: "gateway-relay" as const,
providerConfig: { model: "gpt-realtime-2.1", apiKey: "test-api-key-platform" },
agentId: "main",
expected: undefined,
expectAgentDir: false,
},
{
$name:
"gateway-relay | gpt-realtime-2.1 | Platform API key | Azure endpoint | not applicable",
surface: "gateway-relay" as const,
providerConfig: {
model: "gpt-realtime-2.1",
apiKey: "test-api-key-platform",
azureEndpoint: "https://example.openai.azure.com",
},
agentId: "main",
expected: undefined,
expectAgentDir: false,
},
{
$name:
"gateway-relay | gpt-live-1-codex | Platform API key + OAuth | Azure endpoint | not ready",
surface: "gateway-relay" as const,
providerConfig: {
model: "gpt-live-1-codex",
apiKey: "test-api-key-platform",
azureEndpoint: "https://example.openai.azure.com",
},
agentId: "main",
expected: false,
expectAgentDir: false,
},
{
$name:
"gateway-relay | gpt-live-1-mini | Platform API key + OAuth | standard endpoint | not ready",
surface: "gateway-relay" as const,
providerConfig: { model: "gpt-live-1-mini", apiKey: "test-api-key-platform" },
agentId: "main",
expected: false,
expectAgentDir: false,
},
{
$name: "browser | gpt-live-1-mini | Platform API key + OAuth | standard endpoint | not ready",
surface: "browser" as const,
providerConfig: { model: "gpt-live-1-mini", apiKey: "test-api-key-platform" },
agentId: "main",
expected: false,
expectAgentDir: false,
},
{
$name: "gateway-relay | gpt-live-1-codex | ChatGPT OAuth | standard endpoint | ready",
surface: "gateway-relay" as const,
providerConfig: { model: "gpt-live-1-codex" },
agentId: "main",
expected: true,
expectAgentDir: false,
},
{
$name:
"gateway-relay | gpt-live-1-codex | voice-agent ChatGPT OAuth | standard endpoint | ready",
surface: "gateway-relay" as const,
providerConfig: { model: "gpt-live-1-codex" },
agentId: "voice-agent",
expected: true,
expectAgentDir: true,
},
{
$name: "browser | gpt-live-1-codex | ChatGPT OAuth | standard endpoint | ready",
surface: "browser" as const,
providerConfig: { model: "gpt-live-1-codex" },
agentId: "main",
expected: true,
expectAgentDir: false,
},
])("$name", ({ surface, providerConfig, agentId, expected, expectAgentDir }) => {
isProviderAuthProfileConfiguredMock.mockImplementation(
({ profileTypes }: { profileTypes?: readonly string[] }) =>
profileTypes?.includes("oauth") === true,
);
const { broker } = createQuicksilverBrowserBrokerFixture();
const provider = buildOpenAIRealtimeVoiceProvider({
quicksilverBrowserSessionBroker: broker,
});
const cfg = { agents: { defaults: {} } } as never;
const internalApi = readInternalRealtimeVoiceProviderApi(provider);
const readiness =
surface === "provider"
? provider.isConfigured({ cfg, providerConfig })
: surface === "browser"
? internalApi.isBrowserSessionConfigured({ cfg, providerConfig, agentId })
: internalApi.isGatewayRelayConfigured({ cfg, providerConfig, agentId });
expect(readiness).toBe(expected);
if (expectAgentDir) {
expect(isProviderAuthProfileConfiguredMock).toHaveBeenCalledWith(
expect.objectContaining({
agentDir: expect.stringContaining("voice-agent"),
profileTypes: ["oauth"],
}),
);
}
});
it("routes an explicit unlisted gpt-live alias through the broker", async () => {
const oauthToken = createTestJwt({
"https://api.openai.com/auth": { chatgpt_account_id: "account-123" },
});
@@ -349,27 +355,9 @@ describe("OpenAI realtime voice provider routing", () => {
async ({ profileTypes }: { profileTypes?: readonly string[] }) =>
profileTypes?.includes("oauth") ? oauthToken : undefined,
);
isProviderAuthProfileConfiguredMock.mockImplementation(
({ profileTypes }: { profileTypes?: readonly string[] }) =>
profileTypes?.includes("oauth") === true,
);
const createBrowserSession = vi.fn(async () => ({
provider: "openai",
transport: "webrtc" as const,
clientSecret: "quicksilver-token",
offerUrl: "/plugins/openai/realtime/calls",
}));
const { broker, createBrowserSession } = createQuicksilverBrowserBrokerFixture();
const provider = buildOpenAIRealtimeVoiceProvider({
quicksilverBrowserSessionBroker: {
capabilities: {
transports: ["webrtc" as const],
handlesAgentConsult: true as const,
supportsToolCalls: false,
supportsVideoFrames: false,
},
createBrowserSession,
cancelBrowserSession: vi.fn(),
},
quicksilverBrowserSessionBroker: broker,
});
const cfg = { agents: { defaults: {} } } as never;
const request = {
@@ -382,104 +370,6 @@ describe("OpenAI realtime voice provider routing", () => {
runAgentConsult: vi.fn(async () => ({ text: "Done" })),
};
expect(provider.isConfigured({ cfg, providerConfig: { model: "gpt-live-1-mini" } })).toBe(
false,
);
expect(
readInternalRealtimeVoiceProviderApi(provider).isGatewayRelayConfigured({
cfg,
providerConfig: { model: "gpt-live-1-mini" },
agentId: "main",
}),
).toBe(false);
expect(
readInternalRealtimeVoiceProviderApi(provider).isGatewayRelayConfigured({
cfg,
providerConfig: {
model: "gpt-live-1-mini",
azureEndpoint: "https://example.openai.azure.com",
azureDeployment: "gpt-live",
},
agentId: "main",
}),
).toBe(false);
expect(
readInternalRealtimeVoiceProviderApi(provider).isBrowserSessionConfigured({
cfg,
providerConfig: { model: "gpt-live-1-mini" },
agentId: "main",
}),
).toBe(false);
expect(
readInternalRealtimeVoiceProviderApi(provider).isGatewayRelayConfigured({
cfg,
providerConfig: { model: "gpt-realtime-2.1", apiKey: "test-api-key-platform" },
agentId: "main",
}),
).toBeUndefined();
expect(
readInternalRealtimeVoiceProviderApi(provider).isGatewayRelayConfigured({
cfg,
providerConfig: {
model: "gpt-realtime-2.1",
apiKey: "test-api-key-platform",
azureEndpoint: "https://example.openai.azure.com",
},
agentId: "main",
}),
).toBeUndefined();
expect(
readInternalRealtimeVoiceProviderApi(provider).isGatewayRelayConfigured({
cfg,
providerConfig: {
model: "gpt-live-1-codex",
apiKey: "test-api-key-platform",
azureEndpoint: "https://example.openai.azure.com",
},
agentId: "main",
}),
).toBe(false);
expect(
readInternalRealtimeVoiceProviderApi(provider).isGatewayRelayConfigured({
cfg,
providerConfig: { model: "gpt-live-1-mini", apiKey: "test-api-key-platform" },
agentId: "main",
}),
).toBe(false);
expect(
readInternalRealtimeVoiceProviderApi(provider).isBrowserSessionConfigured({
cfg,
providerConfig: { model: "gpt-live-1-mini", apiKey: "test-api-key-platform" },
agentId: "main",
}),
).toBe(false);
expect(
readInternalRealtimeVoiceProviderApi(provider).isGatewayRelayConfigured({
cfg,
providerConfig: { model: "gpt-live-1-codex" },
agentId: "main",
}),
).toBe(true);
expect(
readInternalRealtimeVoiceProviderApi(provider).isGatewayRelayConfigured({
cfg,
providerConfig: { model: "gpt-live-1-codex" },
agentId: "voice-agent",
}),
).toBe(true);
expect(isProviderAuthProfileConfiguredMock).toHaveBeenCalledWith(
expect.objectContaining({
agentDir: expect.stringContaining("voice-agent"),
profileTypes: ["oauth"],
}),
);
expect(
readInternalRealtimeVoiceProviderApi(provider).isBrowserSessionConfigured({
cfg,
providerConfig: { model: "gpt-live-1-codex" },
agentId: "main",
}),
).toBe(true);
await provider.createBrowserSession?.(request);
expect(createBrowserSession).toHaveBeenCalledWith(expect.objectContaining(request), {
type: "oauth",
@@ -521,18 +411,9 @@ describe("OpenAI realtime voice provider routing", () => {
async ({ profileTypes }: { profileTypes?: readonly string[] }) =>
profileTypes?.includes("oauth") ? oauthToken : undefined,
);
const createBrowserSession = vi.fn(async () => ({
provider: "openai",
transport: "webrtc" as const,
clientSecret: "quicksilver-token",
offerUrl: "/plugins/openai/realtime/calls",
}));
const { broker, createBrowserSession } = createQuicksilverBrowserBrokerFixture();
const provider = buildOpenAIRealtimeVoiceProvider({
quicksilverBrowserSessionBroker: {
capabilities: { handlesAgentConsult: true as const },
createBrowserSession,
cancelBrowserSession: vi.fn(),
},
quicksilverBrowserSessionBroker: broker,
});
await provider.createBrowserSession?.({
@@ -556,12 +437,9 @@ describe("OpenAI realtime voice provider routing", () => {
({ profileTypes }: { profileTypes?: readonly string[] }) =>
profileTypes?.includes("oauth") === true,
);
const { broker } = createQuicksilverBrowserBrokerFixture();
const provider = buildOpenAIRealtimeVoiceProvider({
quicksilverBrowserSessionBroker: {
capabilities: { handlesAgentConsult: true as const },
createBrowserSession: vi.fn(),
cancelBrowserSession: vi.fn(async () => undefined),
},
quicksilverBrowserSessionBroker: broker,
});
expect(
readInternalRealtimeVoiceProviderApi(provider).resolveBrowserSessionCapabilities({
@@ -584,18 +462,11 @@ describe("OpenAI realtime voice provider routing", () => {
({ profileTypes }: { profileTypes?: readonly string[] }) =>
profileTypes?.includes("oauth") === true,
);
const createBrowserSession = vi.fn(async () => ({
provider: "openai",
transport: "webrtc" as const,
clientSecret: "broker-token",
offerUrl: "/plugins/openai/realtime/calls",
}));
const { broker, createBrowserSession } = createQuicksilverBrowserBrokerFixture({
session: { clientSecret: "broker-token" },
});
const provider = buildOpenAIRealtimeVoiceProvider({
quicksilverBrowserSessionBroker: {
capabilities: { handlesAgentConsult: true as const },
createBrowserSession,
cancelBrowserSession: vi.fn(),
},
quicksilverBrowserSessionBroker: broker,
});
const cfg = { agents: { defaults: {} } } as never;
const request = {
@@ -628,23 +499,9 @@ describe("OpenAI realtime voice provider routing", () => {
});
it("passes configured gpt-live model and voice to the native broker", async () => {
const createBrowserSession = vi.fn(async (_request: unknown, _auth: unknown) => ({
provider: "openai",
transport: "webrtc" as const,
clientSecret: "quicksilver-token",
offerUrl: "/plugins/openai/realtime/calls",
}));
const { broker, createBrowserSession } = createQuicksilverBrowserBrokerFixture();
const provider = buildOpenAIRealtimeVoiceProvider({
quicksilverBrowserSessionBroker: {
capabilities: {
transports: ["webrtc" as const],
handlesAgentConsult: true as const,
supportsToolCalls: false,
supportsVideoFrames: false,
},
createBrowserSession,
cancelBrowserSession: vi.fn(),
},
quicksilverBrowserSessionBroker: broker,
});
await provider.createBrowserSession?.({
@@ -679,18 +536,9 @@ describe("OpenAI realtime voice provider routing", () => {
});
it("explains both gpt-live authentication options when neither is available", async () => {
const createBrowserSession = vi.fn();
const { broker, createBrowserSession } = createQuicksilverBrowserBrokerFixture();
const provider = buildOpenAIRealtimeVoiceProvider({
quicksilverBrowserSessionBroker: {
capabilities: {
transports: ["webrtc" as const],
handlesAgentConsult: true as const,
supportsToolCalls: false,
supportsVideoFrames: false,
},
createBrowserSession,
cancelBrowserSession: vi.fn(),
},
quicksilverBrowserSessionBroker: broker,
});
await expect(
extensions/openai/realtime-voice-response-control.test.ts
+64 -198
View File
@@ -1,103 +1,31 @@
// Openai tests cover realtime voice provider plugin behavior.
import type { RealtimeVoiceBridge } from "openclaw/plugin-sdk/realtime-voice";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js";
const {
FakeWebSocket,
execFileSyncMock,
fetchWithSsrFGuardMock,
isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKeyMock,
} = vi.hoisted(() => {
type Listener = (...args: unknown[]) => void;
class MockWebSocket {
static readonly OPEN = 1;
static readonly CLOSED = 3;
static instances: MockWebSocket[] = [];
readonly listeners = new Map<string, Listener[]>();
readyState = 0;
sent: string[] = [];
closed = false;
terminated = false;
deferClose = false;
deferredClose: (() => void) | undefined;
args: unknown[];
constructor(...args: unknown[]) {
this.args = args;
MockWebSocket.instances.push(this);
}
on(event: string, listener: Listener): this {
const listeners = this.listeners.get(event) ?? [];
listeners.push(listener);
this.listeners.set(event, listeners);
return this;
}
emit(event: string, ...args: unknown[]): void {
for (const listener of this.listeners.get(event) ?? []) {
listener(...args);
}
}
send(payload: string): void {
this.sent.push(payload);
}
close(code?: number, reason?: string): void {
this.closed = true;
this.readyState = MockWebSocket.CLOSED;
const emitClose = () => this.emit("close", code ?? 1000, Buffer.from(reason ?? ""));
if (this.deferClose) {
this.deferredClose = emitClose;
return;
}
emitClose();
}
terminate(): void {
this.terminated = true;
this.close(1006, "terminated");
}
emitDeferredClose(): void {
const emitClose = this.deferredClose;
this.deferredClose = undefined;
emitClose?.();
}
}
return {
FakeWebSocket: MockWebSocket,
execFileSyncMock: vi.fn(),
fetchWithSsrFGuardMock: vi.fn(),
isProviderAuthProfileConfiguredMock: vi.fn(),
resolveProviderAuthProfileApiKeyMock: vi.fn(),
};
const mocks = await vi.hoisted(async () => {
const { createOpenAIRealtimeMockState } = await import("./realtime-voice-test-support.js");
return createOpenAIRealtimeMockState();
});
vi.mock("node:child_process", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:child_process")>();
return {
...actual,
execFileSync: execFileSyncMock,
execFileSync: mocks.execFileSyncMock,
};
});
vi.mock("ws", () => ({
default: FakeWebSocket,
default: mocks.FakeWebSocket,
}));
vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
fetchWithSsrFGuard: fetchWithSsrFGuardMock,
fetchWithSsrFGuard: mocks.fetchWithSsrFGuardMock,
}));
vi.mock("openclaw/plugin-sdk/provider-auth", () => ({
isProviderAuthProfileConfigured: isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKey: resolveProviderAuthProfileApiKeyMock,
isProviderAuthProfileConfigured: mocks.isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKey: mocks.resolveProviderAuthProfileApiKeyMock,
}));
import { createOpenAIRealtimeTestSupport } from "./realtime-voice-test-support.js";
@@ -108,29 +36,24 @@ const {
beginBridgeConnection,
openSocket,
emitServerEvent,
emitAssistantPlayback,
emitSessionUpdated,
emitCompletedToolCalls,
connectReadyBridge,
expectedResponseCreateEvent,
requireNestedRecord,
expectRecordFields,
} = createOpenAIRealtimeTestSupport({ FakeWebSocket, fetchWithSsrFGuardMock });
resetTestState,
restoreTestEnvironment,
} = createOpenAIRealtimeTestSupport({ ...mocks, buildOpenAIRealtimeVoiceProvider });
describe("OpenAI realtime voice response control", () => {
beforeEach(() => {
FakeWebSocket.instances = [];
vi.stubEnv("OPENAI_API_KEY", "");
execFileSyncMock.mockReset();
fetchWithSsrFGuardMock.mockReset();
isProviderAuthProfileConfiguredMock.mockReset();
isProviderAuthProfileConfiguredMock.mockReturnValue(false);
resolveProviderAuthProfileApiKeyMock.mockReset();
resolveProviderAuthProfileApiKeyMock.mockResolvedValue(undefined);
resetTestState();
});
afterEach(() => {
vi.useRealTimers();
vi.unstubAllEnvs();
restoreTestEnvironment();
});
it("suppresses auto responses before draining queued initial greeting audio", async () => {
@@ -264,10 +187,7 @@ describe("OpenAI realtime voice response control", () => {
it("defers manual response.create while a realtime response is active", async () => {
const bridge = createNativeBridge();
const socket = await connectReadyBridge(bridge);
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })),
);
emitServerEvent(socket, { type: "response.created", response: { id: "resp_1" } });
bridge.sendUserMessage?.("queued manual response");
@@ -310,18 +230,13 @@ describe("OpenAI realtime voice response control", () => {
},
);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "error",
error: {
event_id: responseCreateEvent.event_id,
message: "bad response request",
},
}),
),
);
emitServerEvent(socket, {
type: "error",
error: {
event_id: responseCreateEvent.event_id,
message: "bad response request",
},
});
expect(onError).toHaveBeenCalledWith(new Error("bad response request"));
expectRecordFields(
@@ -344,15 +259,10 @@ describe("OpenAI realtime voice response control", () => {
(event) => event.type === "session.update",
);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "error",
error: { event_id: "unrelated-audio-event", message: "bad audio append" },
}),
),
);
emitServerEvent(socket, {
type: "error",
error: { event_id: "unrelated-audio-event", message: "bad audio append" },
});
expect(onError).toHaveBeenCalledWith(new Error("bad audio append"));
expect(parseSent(socket).filter((event) => event.type === "session.update")).toHaveLength(
@@ -388,18 +298,13 @@ describe("OpenAI realtime voice response control", () => {
).length;
bridge.sendUserMessage?.("Say exactly: queued follow-up.");
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "error",
error: {
event_id: firstResponseCreate.event_id,
message: "bad response request",
},
}),
),
);
emitServerEvent(socket, {
type: "error",
error: {
event_id: firstResponseCreate.event_id,
message: "bad response request",
},
});
const responseCreates = parseSent(socket).filter((event) => event.type === "response.create");
expect(responseCreates).toHaveLength(2);
@@ -433,12 +338,10 @@ describe("OpenAI realtime voice response control", () => {
expect(parseSent(socket).filter((event) => event.type === "response.create")).toHaveLength(1);
for (let index = 0; index < 3; index += 1) {
socket.emit(
"message",
Buffer.from(
JSON.stringify({ type: "response.created", response: { id: `resp_control_${index}` } }),
),
);
emitServerEvent(socket, {
type: "response.created",
response: { id: `resp_control_${index}` },
});
emitServerEvent(socket, {
type: "response.done",
response: { id: `resp_control_${index}`, status: "completed", output: [] },
@@ -452,13 +355,10 @@ describe("OpenAI realtime voice response control", () => {
it("drains deferred response.create after response.cancelled", async () => {
const bridge = createNativeBridge();
const socket = await connectReadyBridge(bridge);
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })),
);
emitServerEvent(socket, { type: "response.created", response: { id: "resp_1" } });
bridge.sendUserMessage?.("queued after cancellation");
socket.emit("message", Buffer.from(JSON.stringify({ type: "response.cancelled" })));
emitServerEvent(socket, { type: "response.cancelled" });
expect(parseSent(socket).slice(-1)).toEqual([expectedResponseCreateEvent()]);
});
@@ -467,10 +367,7 @@ describe("OpenAI realtime voice response control", () => {
const onError = vi.fn();
const bridge = createNativeBridge({ onError });
const socket = await connectReadyBridge(bridge);
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })),
);
emitServerEvent(socket, { type: "response.created", response: { id: "resp_1" } });
bridge.sendUserMessage?.("queued after cancellation error");
bridge.handleBargeIn?.({ audioPlaybackActive: true });
@@ -480,18 +377,13 @@ describe("OpenAI realtime voice response control", () => {
if (!responseCancelEvent?.event_id) {
throw new Error("expected response.cancel event id");
}
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "error",
error: {
event_id: responseCancelEvent.event_id,
message: "Cancellation failed: no active response found",
},
}),
),
);
emitServerEvent(socket, {
type: "error",
error: {
event_id: responseCancelEvent.event_id,
message: "Cancellation failed: no active response found",
},
});
expect(onError).not.toHaveBeenCalled();
expect(parseSent(socket).slice(-1)).toEqual([expectedResponseCreateEvent()]);
@@ -502,20 +394,7 @@ describe("OpenAI realtime voice response control", () => {
const bridge = createNativeBridge({ onError });
const socket = await connectReadyBridge(bridge);
bridge.setMediaTimestamp(1000);
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })),
);
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "response.audio.delta",
item_id: "item_1",
delta: Buffer.from("assistant audio").toString("base64"),
}),
),
);
emitAssistantPlayback(socket);
bridge.setMediaTimestamp(1300);
bridge.handleBargeIn?.({ audioPlaybackActive: true });
@@ -531,18 +410,13 @@ describe("OpenAI realtime voice response control", () => {
(event) => event.type === "session.update",
).length;
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "error",
error: {
event_id: responseCancelEvent.event_id,
message: "Cancellation failed: no active response found",
},
}),
),
);
emitServerEvent(socket, {
type: "error",
error: {
event_id: responseCancelEvent.event_id,
message: "Cancellation failed: no active response found",
},
});
expect(onError).not.toHaveBeenCalled();
expect(parseSent(socket).filter((event) => event.type === "session.update")).toHaveLength(
@@ -565,10 +439,7 @@ describe("OpenAI realtime voice response control", () => {
vi.useFakeTimers();
const bridge = createNativeBridge();
const socket = await connectReadyBridge(bridge);
socket.emit(
"message",
Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })),
);
emitServerEvent(socket, { type: "response.created", response: { id: "resp_1" } });
bridge.sendUserMessage?.("queued before reconnect");
expect(parseSent(socket).slice(-1)[0]?.type).toBe("conversation.item.create");
@@ -606,18 +477,13 @@ describe("OpenAI realtime voice response control", () => {
if (!responseCreateEvent?.event_id) {
throw new Error("expected response.create event id");
}
socket.emit(
"message",
Buffer.from(
JSON.stringify({
type: "error",
error: {
event_id: responseCreateEvent.event_id,
message: "Conversation already has an active response in progress: resp_1",
},
}),
),
);
emitServerEvent(socket, {
type: "error",
error: {
event_id: responseCreateEvent.event_id,
message: "Conversation already has an active response in progress: resp_1",
},
});
const afterError = parseSent(socket);
expect(afterError.filter((event) => event.type === "session.update")).toHaveLength(2);
expectRecordFields(
extensions/openai/realtime-voice-test-support.ts
+230 -2
View File
@@ -1,11 +1,83 @@
import type {
RealtimeVoiceBridge,
RealtimeVoiceBridgeCreateRequest,
RealtimeVoiceBrowserSession,
RealtimeVoiceProviderPlugin,
RealtimeVoiceTool,
} from "openclaw/plugin-sdk/realtime-voice";
import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
import { expect, vi } from "vitest";
import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js";
type Listener = (...args: unknown[]) => void;
export function createOpenAIRealtimeMockState() {
class MockWebSocket {
static readonly OPEN = 1;
static readonly CLOSED = 3;
static instances: MockWebSocket[] = [];
readonly listeners = new Map<string, Listener[]>();
readyState = 0;
sent: string[] = [];
closed = false;
terminated = false;
deferClose = false;
deferredClose: (() => void) | undefined;
args: unknown[];
constructor(...args: unknown[]) {
this.args = args;
MockWebSocket.instances.push(this);
}
on(event: string, listener: Listener): this {
const listeners = this.listeners.get(event) ?? [];
listeners.push(listener);
this.listeners.set(event, listeners);
return this;
}
emit(event: string, ...args: unknown[]): void {
for (const listener of this.listeners.get(event) ?? []) {
listener(...args);
}
}
send(payload: string): void {
this.sent.push(payload);
}
close(code?: number, reason?: string): void {
this.closed = true;
this.readyState = MockWebSocket.CLOSED;
const emitClose = () => this.emit("close", code ?? 1000, Buffer.from(reason ?? ""));
if (this.deferClose) {
this.deferredClose = emitClose;
return;
}
emitClose();
}
terminate(): void {
this.terminated = true;
this.close(1006, "terminated");
}
emitDeferredClose(): void {
const emitClose = this.deferredClose;
this.deferredClose = undefined;
emitClose?.();
}
}
return {
FakeWebSocket: MockWebSocket,
execFileSyncMock: vi.fn(),
fetchWithSsrFGuardMock: vi.fn(),
isProviderAuthProfileConfiguredMock: vi.fn(),
resolveProviderAuthProfileApiKeyMock: vi.fn(),
};
}
type FakeWebSocketLike = {
sent: string[];
@@ -19,11 +91,65 @@ type FakeWebSocketConstructor<T extends FakeWebSocketLike> = {
instances: T[];
};
type InternalRealtimeVoiceProviderApi = {
isBrowserSessionConfigured: (ctx: {
cfg?: object;
providerConfig: Record<string, unknown>;
agentId?: string;
}) => boolean;
isGatewayRelayConfigured: (ctx: {
cfg?: object;
providerConfig: Record<string, unknown>;
agentId?: string;
}) => boolean | undefined;
resolveBrowserSessionCapabilities: (ctx: {
cfg?: object;
providerConfig: Record<string, unknown>;
model?: string;
}) => {
handlesAgentConsult?: boolean;
supportsToolCalls?: boolean;
supportsVideoFrames?: boolean;
supportsGatewayControl?: boolean;
transports?: string[];
};
resolveGatewayRelayCapabilities: (ctx: {
cfg?: object;
providerConfig: Record<string, unknown>;
model?: string;
}) => {
handlesAgentConsult?: boolean;
supportsToolCalls?: boolean;
transports?: string[];
};
validateGatewayRelayLaunch: (ctx: {
cfg?: object;
providerConfig: Record<string, unknown>;
model?: string;
autoRespondToAudio?: boolean;
}) => string | undefined;
};
const INTERNAL_REALTIME_VOICE_PROVIDER = Symbol.for("openclaw.internal.realtime-voice-provider.v1");
const OPENAI_REALTIME_REJECTED_KEY_MESSAGE =
"OpenAI Realtime rejected the selected API key. Update or remove the active OpenAI API-key source";
export function createOpenAIRealtimeTestSupport<T extends FakeWebSocketLike>(deps: {
FakeWebSocket: FakeWebSocketConstructor<T>;
execFileSyncMock: ReturnType<typeof vi.fn>;
fetchWithSsrFGuardMock: ReturnType<typeof vi.fn>;
isProviderAuthProfileConfiguredMock: ReturnType<typeof vi.fn>;
resolveProviderAuthProfileApiKeyMock: ReturnType<typeof vi.fn>;
buildOpenAIRealtimeVoiceProvider: () => RealtimeVoiceProviderPlugin;
}) {
const { FakeWebSocket, fetchWithSsrFGuardMock } = deps;
const {
FakeWebSocket,
execFileSyncMock,
fetchWithSsrFGuardMock,
isProviderAuthProfileConfiguredMock,
resolveProviderAuthProfileApiKeyMock,
buildOpenAIRealtimeVoiceProvider,
} = deps;
type FakeWebSocketInstance = T;
type SentRealtimeEvent = {
type: string;
@@ -69,6 +195,31 @@ export function createOpenAIRealtimeTestSupport<T extends FakeWebSocketLike>(dep
return socket.sent.map((payload: string) => JSON.parse(payload) as SentRealtimeEvent);
}
function resetTestState(): void {
FakeWebSocket.instances = [];
vi.stubEnv("OPENAI_API_KEY", "");
execFileSyncMock.mockReset();
fetchWithSsrFGuardMock.mockReset();
isProviderAuthProfileConfiguredMock.mockReset();
isProviderAuthProfileConfiguredMock.mockReturnValue(false);
resolveProviderAuthProfileApiKeyMock.mockReset();
resolveProviderAuthProfileApiKeyMock.mockResolvedValue(undefined);
}
function restoreTestEnvironment(): void {
vi.useRealTimers();
vi.unstubAllEnvs();
}
function readInternalRealtimeVoiceProviderApi(
provider: object,
): InternalRealtimeVoiceProviderApi {
return Reflect.get(
provider,
INTERNAL_REALTIME_VOICE_PROVIDER,
) as InternalRealtimeVoiceProviderApi;
}
function createNativeBridge(
overrides: Partial<RealtimeVoiceBridgeCreateRequest> = {},
): RealtimeVoiceBridge {
@@ -109,6 +260,21 @@ export function createOpenAIRealtimeTestSupport<T extends FakeWebSocketLike>(dep
emitServerEvent(socket, { type: "session.updated" });
}
function emitAssistantPlayback(
socket: FakeWebSocketInstance,
overrides: { responseId?: string; itemId?: string; audio?: Buffer } = {},
): void {
emitServerEvent(socket, {
type: "response.created",
response: { id: overrides.responseId ?? "resp_1" },
});
emitServerEvent(socket, {
type: "response.audio.delta",
item_id: overrides.itemId ?? "item_1",
delta: (overrides.audio ?? Buffer.from("assistant audio")).toString("base64"),
});
}
function emitCompletedToolCalls(
socket: FakeWebSocketInstance,
callIds: string[] = ["call_1"],
@@ -182,6 +348,61 @@ export function createOpenAIRealtimeTestSupport<T extends FakeWebSocketLike>(dep
});
}
function mockRealtimeClientSecretResponse(
overrides: { clientSecret?: string; expiresAt?: number } = {},
): ReturnType<typeof vi.fn> {
const release = vi.fn(async () => undefined);
fetchWithSsrFGuardMock.mockResolvedValueOnce({
response: createJsonResponse({
client_secret: { value: overrides.clientSecret ?? "client-secret-123" },
...(overrides.expiresAt === undefined ? {} : { expires_at: overrides.expiresAt }),
}),
release,
});
return release;
}
function createQuicksilverBrowserBrokerFixture(
overrides: {
session?: {
provider?: "openai";
transport?: "webrtc";
clientSecret?: string;
offerUrl?: string;
};
capabilities?: {
handlesAgentConsult?: true;
supportsToolCalls?: boolean;
supportsVideoFrames?: boolean;
transports?: Array<"webrtc">;
};
} = {},
) {
const session: RealtimeVoiceBrowserSession = {
provider: "openai" as const,
transport: "webrtc" as const,
clientSecret: "quicksilver-token",
offerUrl: "/plugins/openai/realtime/calls",
...overrides.session,
};
const createBrowserSession = vi.fn(
async (_request: unknown, _auth: unknown): Promise<RealtimeVoiceBrowserSession> => session,
);
const cancelBrowserSession = vi.fn(async (_session: RealtimeVoiceBrowserSession) => undefined);
const broker = {
capabilities: {
transports: ["webrtc" as const],
handlesAgentConsult: true as const,
supportsToolCalls: false,
supportsVideoFrames: false,
...overrides.capabilities,
},
createBrowserSession,
cancelBrowserSession,
};
return { broker, createBrowserSession, cancelBrowserSession };
}
function requireRecord(value: unknown, label: string): Record<string, unknown> {
expect(isRecord(value), `${label} must be an object`).toBe(true);
return value as Record<string, unknown>;
@@ -286,6 +507,9 @@ export function createOpenAIRealtimeTestSupport<T extends FakeWebSocketLike>(dep
}
return {
resetTestState,
restoreTestEnvironment,
readInternalRealtimeVoiceProviderApi,
parseSent,
createNativeBridge,
requireSocket,
@@ -293,6 +517,7 @@ export function createOpenAIRealtimeTestSupport<T extends FakeWebSocketLike>(dep
openSocket,
emitServerEvent,
emitSessionUpdated,
emitAssistantPlayback,
emitCompletedToolCalls,
emitFunctionOutputAdded,
expectedFunctionOutput,
@@ -300,6 +525,9 @@ export function createOpenAIRealtimeTestSupport<T extends FakeWebSocketLike>(dep
expectedResponseCreateEvent,
expectedResponseCancelEvent,
createJsonResponse,
createQuicksilverBrowserBrokerFixture,
mockRealtimeClientSecretResponse,
rejectedKeyMessage: OPENAI_REALTIME_REJECTED_KEY_MESSAGE,
requireRecord,
requireNestedRecord,
expectRecordFields,