From da94915d064989bf67455c1bd5938e58c42ba8d0 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 2 Aug 2026 03:07:01 +0800 Subject: [PATCH] test(openai): prove gateway startup audio delivery --- .../realtime-quicksilver-audio-buffer.test.ts | 14 +- ...ealtime-quicksilver-audio-pipeline.test.ts | 27 +-- ...me-quicksilver-gateway-bridge.live.test.ts | 161 ++++++++++++++++++ ...ealtime-quicksilver-gateway-bridge.test.ts | 36 +++- 4 files changed, 221 insertions(+), 17 deletions(-) diff --git a/extensions/openai/realtime-quicksilver-audio-buffer.test.ts b/extensions/openai/realtime-quicksilver-audio-buffer.test.ts index 85dfa89637e8..a43667f38198 100644 --- a/extensions/openai/realtime-quicksilver-audio-buffer.test.ts +++ b/extensions/openai/realtime-quicksilver-audio-buffer.test.ts @@ -25,15 +25,17 @@ describe("GPT-Live pending microphone audio", () => { it("retains the newest bounded tail across existing and oversized input", () => { const existing = Buffer.alloc(MAX_PENDING_AUDIO_BYTES, 0x01); const appended = appendOpenAIQuicksilverPendingAudio(existing, Buffer.from([0x02, 0x02])); - const oversized = Buffer.alloc(MAX_PENDING_AUDIO_BYTES + 2, 0x03); + const oversized = Buffer.alloc(MAX_PENDING_AUDIO_BYTES + 4, 0x03); + oversized.writeUInt16LE(0x1111, 0); + oversized.writeUInt16LE(0x2222, oversized.length - 2); + const expectedOversizedTail = Buffer.from(oversized.subarray(4)); + const oversizedResult = appendOpenAIQuicksilverPendingAudio(appended, oversized); + oversized.fill(0xff); expect(appended).toHaveLength(MAX_PENDING_AUDIO_BYTES); expect(appended.subarray(0, -2).every((byte) => byte === 0x01)).toBe(true); expect(appended.subarray(-2)).toEqual(Buffer.from([0x02, 0x02])); - expect( - appendOpenAIQuicksilverPendingAudio(appended, oversized).equals( - Buffer.alloc(MAX_PENDING_AUDIO_BYTES, 0x03), - ), - ).toBe(true); + expect(oversizedResult).toEqual(expectedOversizedTail); + expect(oversizedResult.readUInt16LE(-2 + oversizedResult.length)).toBe(0x2222); }); }); diff --git a/extensions/openai/realtime-quicksilver-audio-pipeline.test.ts b/extensions/openai/realtime-quicksilver-audio-pipeline.test.ts index 69500a8cd915..2061070280ad 100644 --- a/extensions/openai/realtime-quicksilver-audio-pipeline.test.ts +++ b/extensions/openai/realtime-quicksilver-audio-pipeline.test.ts @@ -29,6 +29,10 @@ type TestableAudioPeer = { }; }; +type TestableGatewayBridge = { + pendingAudio: Buffer; +}; + describe("GPT-Live gateway microphone audio pipeline", () => { it("retains the newest five seconds through delayed peer adoption and the RTP pump", async () => { vi.useFakeTimers(); @@ -65,12 +69,13 @@ describe("GPT-Live gateway microphone audio pipeline", () => { const connection = bridge.connect(); const source = Buffer.alloc(256 * OPENAI_QUICKSILVER_RELAY_FRAME_BYTES); for (let frame = 0; frame < 256; frame += 1) { - source - .subarray( - frame * OPENAI_QUICKSILVER_RELAY_FRAME_BYTES, - (frame + 1) * OPENAI_QUICKSILVER_RELAY_FRAME_BYTES, - ) - .fill(frame + 1); + const frameBuffer = source.subarray( + frame * OPENAI_QUICKSILVER_RELAY_FRAME_BYTES, + (frame + 1) * OPENAI_QUICKSILVER_RELAY_FRAME_BYTES, + ); + for (let sample = 0; sample < frameBuffer.length; sample += 2) { + frameBuffer.writeUInt16LE(frame + 1, sample); + } } for (let offset = 0; offset < source.length; offset += 8_192) { const callerBuffer = Buffer.from(source.subarray(offset, offset + 8_192)); @@ -78,6 +83,9 @@ describe("GPT-Live gateway microphone audio pipeline", () => { callerBuffer.fill(0xff); await vi.advanceTimersByTimeAsync(171); } + expect((bridge as unknown as TestableGatewayBridge).pendingAudio).toEqual( + source.subarray(source.length - MAX_PENDING_AUDIO_BYTES), + ); if (!peerCallbacks) { throw new Error("expected the bridge to start peer creation"); } @@ -113,10 +121,9 @@ describe("GPT-Live gateway microphone audio pipeline", () => { expect(testPeer.pendingAudio).toHaveLength(0); expect(emittedFrames).toHaveLength(MAX_PENDING_RELAY_FRAMES); - expect(emittedFrames.map((frame) => frame[0])).toEqual([ - ...Array.from({ length: MAX_PENDING_RELAY_FRAMES - 1 }, (_, index) => index + 7), - 0, - ]); + expect(emittedFrames.map((frame) => frame.readUInt16LE(0))).toEqual( + Array.from({ length: MAX_PENDING_RELAY_FRAMES }, (_, index) => index + 7), + ); expect(sendRtp).toHaveBeenCalledTimes(MAX_PENDING_RELAY_FRAMES); const packets = sendRtp.mock.calls.map( ([packet]) => diff --git a/extensions/openai/realtime-quicksilver-gateway-bridge.live.test.ts b/extensions/openai/realtime-quicksilver-gateway-bridge.live.test.ts index 41b542444519..0174dde16b67 100644 --- a/extensions/openai/realtime-quicksilver-gateway-bridge.live.test.ts +++ b/extensions/openai/realtime-quicksilver-gateway-bridge.live.test.ts @@ -2,13 +2,38 @@ import { readCodexCliCredentialsCached } from "openclaw/plugin-sdk/provider-auth import { describe, expect, it } from "vitest"; import { resolveCodexAuthIdentity } from "./openai-chatgpt-auth-identity.js"; import { OpenAIQuicksilverGatewayBridge } from "./realtime-quicksilver-gateway-bridge.js"; +import { + OpenAIQuicksilverAudioPeer, + type OpenAIQuicksilverAudioPeerContract, +} from "./realtime-quicksilver-peer.runtime.js"; import { resolveOpenAIChatGptSubscriptionAuth } from "./realtime-quicksilver-session.js"; import type { OpenAIQuicksilverAuth } from "./realtime-quicksilver-wire.js"; +import { buildOpenAISpeechProvider } from "./speech-provider.js"; const LIVE_ENABLED = process.env.OPENCLAW_LIVE_TEST === "1" && process.env.OPENCLAW_LIVE_GPT_LIVE === "1"; const describeLive = LIVE_ENABLED ? describe : describe.skip; const LIVE_TIMEOUT_MS = 60_000; +const MAX_PENDING_AUDIO_BYTES = 240_000; + +type TestableGatewayBridge = { + pendingAudio: Buffer; +}; + +async function waitForLiveCondition( + predicate: () => boolean, + describeFailure: () => string, + timeoutMs = 45_000, +): Promise { + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + if (predicate()) { + return; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error(describeFailure()); +} async function resolveLiveOAuthProfile(): Promise< Extract | undefined @@ -86,4 +111,140 @@ describeLive("OpenAI GPT-Live gateway WebRTC peer", () => { }, LIVE_TIMEOUT_MS, ); + + it( + "delivers microphone speech queued before the real media peer is adopted", + async ({ skip }) => { + const apiKey = process.env.OPENAI_API_KEY?.trim(); + if (!apiKey) { + skip("No OpenAI Platform API key is available for the speech fixture"); + return; + } + + const speechProvider = buildOpenAISpeechProvider(); + const synthesized = await speechProvider.synthesizeTelephony?.({ + text: "Please delegate the word glacier.", + cfg: { plugins: { enabled: true } } as never, + providerConfig: { + apiKey, + baseUrl: "https://api.openai.com/v1", + model: "gpt-4o-mini-tts", + voice: "alloy", + speed: 1.4, + }, + timeoutMs: 45_000, + }); + if (!synthesized) { + throw new Error("OpenAI speech provider did not return a telephony fixture"); + } + expect(synthesized.outputFormat).toBe("pcm"); + expect(synthesized.sampleRate).toBe(24_000); + const inputAudio = Buffer.concat([synthesized.audioBuffer, Buffer.alloc(24_000 * 2)]); + expect(inputAudio.byteLength).toBeLessThanOrEqual(MAX_PENDING_AUDIO_BYTES); + + let releasePeerAdoption!: () => void; + let peerCreated!: () => void; + const peerAdoption = new Promise((resolve) => { + releasePeerAdoption = resolve; + }); + const peerCreation = new Promise((resolve) => { + peerCreated = resolve; + }); + const eventTypes: string[] = []; + const finalUserTranscripts: string[] = []; + const errors: Error[] = []; + let closeNotifications = 0; + let closed = false; + let lateAudioBytes = 0; + const bridge = new OpenAIQuicksilverGatewayBridge({ + providerConfig: {}, + model: "gpt-live-1-boulder-alpha", + voice: "marin", + instructions: "Listen to the user. Do not speak or delegate.", + audioFormat: { encoding: "pcm16", sampleRateHz: 24_000, channels: 1 }, + onAudio: (audio) => { + if (closed) { + lateAudioBytes += audio.length; + } + }, + onClearAudio: () => undefined, + onEvent: (event) => eventTypes.push(event.type), + onReady: () => undefined, + onTranscript: (role, text, final) => { + if (role === "user" && final) { + finalUserTranscripts.push(text); + } + }, + onClose: () => { + closeNotifications += 1; + }, + onError: (error) => errors.push(error), + runAgentConsult: async () => ({ text: "Unexpected delegation." }), + logger: { debug: () => undefined, warn: () => undefined }, + resolveAuth: async () => ({ type: "api-key", token: apiKey }), + createPeer: async (callbacks, signal): Promise => { + const peer = await OpenAIQuicksilverAudioPeer.create({ callbacks, signal }); + peerCreated(); + await peerAdoption; + return peer; + }, + }); + const testBridge = bridge as unknown as TestableGatewayBridge; + + try { + const connection = bridge.connect(); + await Promise.race([ + peerCreation, + connection.then(() => { + throw new Error("Gateway bridge connected before the media peer adoption gate"); + }), + ]); + for (let offset = 0; offset < inputAudio.length; offset += 8_192) { + bridge.sendAudio(Buffer.from(inputAudio.subarray(offset, offset + 8_192))); + } + const prePeerPendingBytes = testBridge.pendingAudio.length; + expect(prePeerPendingBytes).toBe(inputAudio.length); + + releasePeerAdoption(); + await connection; + await waitForLiveCondition( + () => finalUserTranscripts.some((text) => text.toLowerCase().includes("glacier")), + () => + `GPT-Live did not transcribe startup audio: transcripts=${finalUserTranscripts.length} errors=${errors.map((error) => error.message).join(";")}`, + 30_000, + ); + + const postAdoptionPendingBytes = testBridge.pendingAudio.length; + expect(postAdoptionPendingBytes).toBe(0); + expect(eventTypes).toContain("turn.done"); + expect(bridge.isConnected()).toBe(true); + expect(errors).toStrictEqual([]); + + closed = true; + bridge.close(); + bridge.close(); + await new Promise((resolve) => setTimeout(resolve, 250)); + + expect(closeNotifications).toBe(1); + expect(lateAudioBytes).toBe(0); + expect(errors).toStrictEqual([]); + console.log( + JSON.stringify({ + proof: "gpt-live-gateway-pre-peer-transcription", + prePeerPendingBytes, + postAdoptionPendingBytes, + userTranscriptMarker: true, + closeNotifications, + lateAudioBytes, + errors: errors.length, + result: "pass", + }), + ); + } finally { + releasePeerAdoption(); + bridge.close(); + } + }, + LIVE_TIMEOUT_MS, + ); }); diff --git a/extensions/openai/realtime-quicksilver-gateway-bridge.test.ts b/extensions/openai/realtime-quicksilver-gateway-bridge.test.ts index 3bc00a66c2d2..839363f3a345 100644 --- a/extensions/openai/realtime-quicksilver-gateway-bridge.test.ts +++ b/extensions/openai/realtime-quicksilver-gateway-bridge.test.ts @@ -1,5 +1,6 @@ import { spawnSync } from "node:child_process"; import { describe, expect, it, vi } from "vitest"; +import { OPENAI_QUICKSILVER_RELAY_FRAME_BYTES } from "./realtime-quicksilver-audio-buffer.js"; import { OpenAIQuicksilverGatewayBridge } from "./realtime-quicksilver-gateway-bridge.js"; import { OpenAIQuicksilverAudioPeer, @@ -58,6 +59,7 @@ type TestableAudioPeer = { }; type TestableGatewayBridge = { + pendingAudio: Buffer; sideband?: { socket: FakeSocket; requestIds: { realtimeSessionId: string; sessionId: string; threadId: string }; @@ -357,6 +359,28 @@ describe("GPT-Live werift audio peer", () => { } }); + it("retains only the newest five seconds and releases it on close", async () => { + const peer = await OpenAIQuicksilverAudioPeer.create({ + callbacks: { onAudio: vi.fn(), onError: vi.fn() }, + iceServers: [], + }); + const testPeer = peer as unknown as TestableAudioPeer; + const maxPendingAudioBytes = OPENAI_QUICKSILVER_RELAY_FRAME_BYTES * 250; + const source = Buffer.alloc(maxPendingAudioBytes + OPENAI_QUICKSILVER_RELAY_FRAME_BYTES); + source.fill(0x11, 0, OPENAI_QUICKSILVER_RELAY_FRAME_BYTES); + source.fill(0x22, OPENAI_QUICKSILVER_RELAY_FRAME_BYTES); + const expectedTail = Buffer.from(source.subarray(OPENAI_QUICKSILVER_RELAY_FRAME_BYTES)); + + peer.sendAudio(source); + source.fill(0xff); + expect(testPeer.pendingAudio).toEqual(expectedTail); + + peer.close(); + expect(testPeer.pendingAudio).toHaveLength(0); + peer.sendAudio(Buffer.from([0x01, 0x02])); + expect(testPeer.pendingAudio).toHaveLength(0); + }); + it("consumes and zero-pads a sub-frame audio tail on the next tick", async () => { const peer = await OpenAIQuicksilverAudioPeer.create({ callbacks: { onAudio: vi.fn(), onError: vi.fn() }, @@ -535,6 +559,7 @@ describe("GPT-Live gateway relay bridge", () => { sendAudio: vi.fn(), close: vi.fn(), } satisfies OpenAIQuicksilverAudioPeerContract; + const onClose = vi.fn(); const bridge = new OpenAIQuicksilverGatewayBridge({ providerConfig: {}, model: "gpt-live-1-codex", @@ -542,6 +567,7 @@ describe("GPT-Live gateway relay bridge", () => { audioFormat: { encoding: "pcm16", sampleRateHz: 24_000, channels: 1 }, onAudio: vi.fn(), onClearAudio: vi.fn(), + onClose, runAgentConsult: vi.fn(async () => ({ text: "done" })), logger: { debug: vi.fn(), warn: vi.fn() }, resolveAuth: vi.fn(async () => ({ @@ -557,6 +583,7 @@ describe("GPT-Live gateway relay bridge", () => { return { bridge, connection, + onClose, peer, rejectPeer: (error: Error) => rejectPeer?.(error), resolvePeer: () => resolvePeer?.(peer), @@ -566,6 +593,7 @@ describe("GPT-Live gateway relay bridge", () => { it("preserves caller-owned microphone frames while the media peer is starting", async () => { const { bridge, connection, peer, resolvePeer } = createPendingPeerBridge(); try { + expect(bridge.connect()).toBe(connection); const source = Buffer.from([0x7f, 0x41]); bridge.sendAudio(source); source.fill(0); @@ -583,9 +611,15 @@ describe("GPT-Live gateway relay bridge", () => { }); it("discards queued microphone audio when closed before the media peer resolves", async () => { - const { bridge, connection, peer, resolvePeer } = createPendingPeerBridge(); + const { bridge, connection, onClose, peer, resolvePeer } = createPendingPeerBridge(); + const testBridge = bridge as unknown as TestableGatewayBridge; bridge.sendAudio(Buffer.from([0x41, 0x42])); bridge.close(); + bridge.close(); + + expect(testBridge.pendingAudio).toHaveLength(0); + expect(onClose).toHaveBeenCalledOnce(); + expect(onClose).toHaveBeenCalledWith("completed"); resolvePeer(); await expect(connection).rejects.toThrow("GPT-Live gateway relay bridge closed");