ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

test(openai): prove gateway startup audio delivery

Browse Source
This commit is contained in:
Vincent Koc
2026-08-02 03:07:01 +08:00
parent 7eeac6b004
commit da94915d06
4 changed files with 221 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/openai/realtime-quicksilver-audio-buffer.test.ts
+8 -6
View File
@@ -25,15 +25,17 @@ describe("GPT-Live pending microphone audio", () => {
it("retains the newest bounded tail across existing and oversized input", () => {
const existing = Buffer.alloc(MAX_PENDING_AUDIO_BYTES, 0x01);
const appended = appendOpenAIQuicksilverPendingAudio(existing, Buffer.from([0x02, 0x02]));
const oversized = Buffer.alloc(MAX_PENDING_AUDIO_BYTES + 2, 0x03);
const oversized = Buffer.alloc(MAX_PENDING_AUDIO_BYTES + 4, 0x03);
oversized.writeUInt16LE(0x1111, 0);
oversized.writeUInt16LE(0x2222, oversized.length - 2);
const expectedOversizedTail = Buffer.from(oversized.subarray(4));
const oversizedResult = appendOpenAIQuicksilverPendingAudio(appended, oversized);
oversized.fill(0xff);
expect(appended).toHaveLength(MAX_PENDING_AUDIO_BYTES);
expect(appended.subarray(0, -2).every((byte) => byte === 0x01)).toBe(true);
expect(appended.subarray(-2)).toEqual(Buffer.from([0x02, 0x02]));
expect(
appendOpenAIQuicksilverPendingAudio(appended, oversized).equals(
Buffer.alloc(MAX_PENDING_AUDIO_BYTES, 0x03),
),
).toBe(true);
expect(oversizedResult).toEqual(expectedOversizedTail);
expect(oversizedResult.readUInt16LE(-2 + oversizedResult.length)).toBe(0x2222);
});
});
extensions/openai/realtime-quicksilver-audio-pipeline.test.ts
+17 -10
View File
@@ -29,6 +29,10 @@ type TestableAudioPeer = {
};
};
type TestableGatewayBridge = {
pendingAudio: Buffer;
};
describe("GPT-Live gateway microphone audio pipeline", () => {
it("retains the newest five seconds through delayed peer adoption and the RTP pump", async () => {
vi.useFakeTimers();
@@ -65,12 +69,13 @@ describe("GPT-Live gateway microphone audio pipeline", () => {
const connection = bridge.connect();
const source = Buffer.alloc(256 * OPENAI_QUICKSILVER_RELAY_FRAME_BYTES);
for (let frame = 0; frame < 256; frame += 1) {
source
.subarray(
frame * OPENAI_QUICKSILVER_RELAY_FRAME_BYTES,
(frame + 1) * OPENAI_QUICKSILVER_RELAY_FRAME_BYTES,
)
.fill(frame + 1);
const frameBuffer = source.subarray(
frame * OPENAI_QUICKSILVER_RELAY_FRAME_BYTES,
(frame + 1) * OPENAI_QUICKSILVER_RELAY_FRAME_BYTES,
);
for (let sample = 0; sample < frameBuffer.length; sample += 2) {
frameBuffer.writeUInt16LE(frame + 1, sample);
}
}
for (let offset = 0; offset < source.length; offset += 8_192) {
const callerBuffer = Buffer.from(source.subarray(offset, offset + 8_192));
@@ -78,6 +83,9 @@ describe("GPT-Live gateway microphone audio pipeline", () => {
callerBuffer.fill(0xff);
await vi.advanceTimersByTimeAsync(171);
}
expect((bridge as unknown as TestableGatewayBridge).pendingAudio).toEqual(
source.subarray(source.length - MAX_PENDING_AUDIO_BYTES),
);
if (!peerCallbacks) {
throw new Error("expected the bridge to start peer creation");
}
@@ -113,10 +121,9 @@ describe("GPT-Live gateway microphone audio pipeline", () => {
expect(testPeer.pendingAudio).toHaveLength(0);
expect(emittedFrames).toHaveLength(MAX_PENDING_RELAY_FRAMES);
expect(emittedFrames.map((frame) => frame[0])).toEqual([
...Array.from({ length: MAX_PENDING_RELAY_FRAMES - 1 }, (_, index) => index + 7),
0,
]);
expect(emittedFrames.map((frame) => frame.readUInt16LE(0))).toEqual(
Array.from({ length: MAX_PENDING_RELAY_FRAMES }, (_, index) => index + 7),
);
expect(sendRtp).toHaveBeenCalledTimes(MAX_PENDING_RELAY_FRAMES);
const packets = sendRtp.mock.calls.map(
([packet]) =>
extensions/openai/realtime-quicksilver-gateway-bridge.live.test.ts
+161
View File
@@ -2,13 +2,38 @@ import { readCodexCliCredentialsCached } from "openclaw/plugin-sdk/provider-auth
import { describe, expect, it } from "vitest";
import { resolveCodexAuthIdentity } from "./openai-chatgpt-auth-identity.js";
import { OpenAIQuicksilverGatewayBridge } from "./realtime-quicksilver-gateway-bridge.js";
import {
OpenAIQuicksilverAudioPeer,
type OpenAIQuicksilverAudioPeerContract,
} from "./realtime-quicksilver-peer.runtime.js";
import { resolveOpenAIChatGptSubscriptionAuth } from "./realtime-quicksilver-session.js";
import type { OpenAIQuicksilverAuth } from "./realtime-quicksilver-wire.js";
import { buildOpenAISpeechProvider } from "./speech-provider.js";
const LIVE_ENABLED =
process.env.OPENCLAW_LIVE_TEST === "1" && process.env.OPENCLAW_LIVE_GPT_LIVE === "1";
const describeLive = LIVE_ENABLED ? describe : describe.skip;
const LIVE_TIMEOUT_MS = 60_000;
const MAX_PENDING_AUDIO_BYTES = 240_000;
type TestableGatewayBridge = {
pendingAudio: Buffer;
};
async function waitForLiveCondition(
predicate: () => boolean,
describeFailure: () => string,
timeoutMs = 45_000,
): Promise<void> {
const startedAt = Date.now();
while (Date.now() - startedAt < timeoutMs) {
if (predicate()) {
return;
}
await new Promise((resolve) => setTimeout(resolve, 100));
}
throw new Error(describeFailure());
}
async function resolveLiveOAuthProfile(): Promise<
Extract<OpenAIQuicksilverAuth, { type: "oauth" }> | undefined
@@ -86,4 +111,140 @@ describeLive("OpenAI GPT-Live gateway WebRTC peer", () => {
},
LIVE_TIMEOUT_MS,
);
it(
"delivers microphone speech queued before the real media peer is adopted",
async ({ skip }) => {
const apiKey = process.env.OPENAI_API_KEY?.trim();
if (!apiKey) {
skip("No OpenAI Platform API key is available for the speech fixture");
return;
}
const speechProvider = buildOpenAISpeechProvider();
const synthesized = await speechProvider.synthesizeTelephony?.({
text: "Please delegate the word glacier.",
cfg: { plugins: { enabled: true } } as never,
providerConfig: {
apiKey,
baseUrl: "https://api.openai.com/v1",
model: "gpt-4o-mini-tts",
voice: "alloy",
speed: 1.4,
},
timeoutMs: 45_000,
});
if (!synthesized) {
throw new Error("OpenAI speech provider did not return a telephony fixture");
}
expect(synthesized.outputFormat).toBe("pcm");
expect(synthesized.sampleRate).toBe(24_000);
const inputAudio = Buffer.concat([synthesized.audioBuffer, Buffer.alloc(24_000 * 2)]);
expect(inputAudio.byteLength).toBeLessThanOrEqual(MAX_PENDING_AUDIO_BYTES);
let releasePeerAdoption!: () => void;
let peerCreated!: () => void;
const peerAdoption = new Promise<void>((resolve) => {
releasePeerAdoption = resolve;
});
const peerCreation = new Promise<void>((resolve) => {
peerCreated = resolve;
});
const eventTypes: string[] = [];
const finalUserTranscripts: string[] = [];
const errors: Error[] = [];
let closeNotifications = 0;
let closed = false;
let lateAudioBytes = 0;
const bridge = new OpenAIQuicksilverGatewayBridge({
providerConfig: {},
model: "gpt-live-1-boulder-alpha",
voice: "marin",
instructions: "Listen to the user. Do not speak or delegate.",
audioFormat: { encoding: "pcm16", sampleRateHz: 24_000, channels: 1 },
onAudio: (audio) => {
if (closed) {
lateAudioBytes += audio.length;
}
},
onClearAudio: () => undefined,
onEvent: (event) => eventTypes.push(event.type),
onReady: () => undefined,
onTranscript: (role, text, final) => {
if (role === "user" && final) {
finalUserTranscripts.push(text);
}
},
onClose: () => {
closeNotifications += 1;
},
onError: (error) => errors.push(error),
runAgentConsult: async () => ({ text: "Unexpected delegation." }),
logger: { debug: () => undefined, warn: () => undefined },
resolveAuth: async () => ({ type: "api-key", token: apiKey }),
createPeer: async (callbacks, signal): Promise<OpenAIQuicksilverAudioPeerContract> => {
const peer = await OpenAIQuicksilverAudioPeer.create({ callbacks, signal });
peerCreated();
await peerAdoption;
return peer;
},
});
const testBridge = bridge as unknown as TestableGatewayBridge;
try {
const connection = bridge.connect();
await Promise.race([
peerCreation,
connection.then(() => {
throw new Error("Gateway bridge connected before the media peer adoption gate");
}),
]);
for (let offset = 0; offset < inputAudio.length; offset += 8_192) {
bridge.sendAudio(Buffer.from(inputAudio.subarray(offset, offset + 8_192)));
}
const prePeerPendingBytes = testBridge.pendingAudio.length;
expect(prePeerPendingBytes).toBe(inputAudio.length);
releasePeerAdoption();
await connection;
await waitForLiveCondition(
() => finalUserTranscripts.some((text) => text.toLowerCase().includes("glacier")),
() =>
`GPT-Live did not transcribe startup audio: transcripts=${finalUserTranscripts.length} errors=${errors.map((error) => error.message).join(";")}`,
30_000,
);
const postAdoptionPendingBytes = testBridge.pendingAudio.length;
expect(postAdoptionPendingBytes).toBe(0);
expect(eventTypes).toContain("turn.done");
expect(bridge.isConnected()).toBe(true);
expect(errors).toStrictEqual([]);
closed = true;
bridge.close();
bridge.close();
await new Promise((resolve) => setTimeout(resolve, 250));
expect(closeNotifications).toBe(1);
expect(lateAudioBytes).toBe(0);
expect(errors).toStrictEqual([]);
console.log(
JSON.stringify({
proof: "gpt-live-gateway-pre-peer-transcription",
prePeerPendingBytes,
postAdoptionPendingBytes,
userTranscriptMarker: true,
closeNotifications,
lateAudioBytes,
errors: errors.length,
result: "pass",
}),
);
} finally {
releasePeerAdoption();
bridge.close();
}
},
LIVE_TIMEOUT_MS,
);
});
extensions/openai/realtime-quicksilver-gateway-bridge.test.ts
+35 -1
View File
@@ -1,5 +1,6 @@
import { spawnSync } from "node:child_process";
import { describe, expect, it, vi } from "vitest";
import { OPENAI_QUICKSILVER_RELAY_FRAME_BYTES } from "./realtime-quicksilver-audio-buffer.js";
import { OpenAIQuicksilverGatewayBridge } from "./realtime-quicksilver-gateway-bridge.js";
import {
OpenAIQuicksilverAudioPeer,
@@ -58,6 +59,7 @@ type TestableAudioPeer = {
};
type TestableGatewayBridge = {
pendingAudio: Buffer;
sideband?: {
socket: FakeSocket;
requestIds: { realtimeSessionId: string; sessionId: string; threadId: string };
@@ -357,6 +359,28 @@ describe("GPT-Live werift audio peer", () => {
}
});
it("retains only the newest five seconds and releases it on close", async () => {
const peer = await OpenAIQuicksilverAudioPeer.create({
callbacks: { onAudio: vi.fn(), onError: vi.fn() },
iceServers: [],
});
const testPeer = peer as unknown as TestableAudioPeer;
const maxPendingAudioBytes = OPENAI_QUICKSILVER_RELAY_FRAME_BYTES * 250;
const source = Buffer.alloc(maxPendingAudioBytes + OPENAI_QUICKSILVER_RELAY_FRAME_BYTES);
source.fill(0x11, 0, OPENAI_QUICKSILVER_RELAY_FRAME_BYTES);
source.fill(0x22, OPENAI_QUICKSILVER_RELAY_FRAME_BYTES);
const expectedTail = Buffer.from(source.subarray(OPENAI_QUICKSILVER_RELAY_FRAME_BYTES));
peer.sendAudio(source);
source.fill(0xff);
expect(testPeer.pendingAudio).toEqual(expectedTail);
peer.close();
expect(testPeer.pendingAudio).toHaveLength(0);
peer.sendAudio(Buffer.from([0x01, 0x02]));
expect(testPeer.pendingAudio).toHaveLength(0);
});
it("consumes and zero-pads a sub-frame audio tail on the next tick", async () => {
const peer = await OpenAIQuicksilverAudioPeer.create({
callbacks: { onAudio: vi.fn(), onError: vi.fn() },
@@ -535,6 +559,7 @@ describe("GPT-Live gateway relay bridge", () => {
sendAudio: vi.fn(),
close: vi.fn(),
} satisfies OpenAIQuicksilverAudioPeerContract;
const onClose = vi.fn();
const bridge = new OpenAIQuicksilverGatewayBridge({
providerConfig: {},
model: "gpt-live-1-codex",
@@ -542,6 +567,7 @@ describe("GPT-Live gateway relay bridge", () => {
audioFormat: { encoding: "pcm16", sampleRateHz: 24_000, channels: 1 },
onAudio: vi.fn(),
onClearAudio: vi.fn(),
onClose,
runAgentConsult: vi.fn(async () => ({ text: "done" })),
logger: { debug: vi.fn(), warn: vi.fn() },
resolveAuth: vi.fn(async () => ({
@@ -557,6 +583,7 @@ describe("GPT-Live gateway relay bridge", () => {
return {
bridge,
connection,
onClose,
peer,
rejectPeer: (error: Error) => rejectPeer?.(error),
resolvePeer: () => resolvePeer?.(peer),
@@ -566,6 +593,7 @@ describe("GPT-Live gateway relay bridge", () => {
it("preserves caller-owned microphone frames while the media peer is starting", async () => {
const { bridge, connection, peer, resolvePeer } = createPendingPeerBridge();
try {
expect(bridge.connect()).toBe(connection);
const source = Buffer.from([0x7f, 0x41]);
bridge.sendAudio(source);
source.fill(0);
@@ -583,9 +611,15 @@ describe("GPT-Live gateway relay bridge", () => {
});
it("discards queued microphone audio when closed before the media peer resolves", async () => {
const { bridge, connection, peer, resolvePeer } = createPendingPeerBridge();
const { bridge, connection, onClose, peer, resolvePeer } = createPendingPeerBridge();
const testBridge = bridge as unknown as TestableGatewayBridge;
bridge.sendAudio(Buffer.from([0x41, 0x42]));
bridge.close();
bridge.close();
expect(testBridge.pendingAudio).toHaveLength(0);
expect(onClose).toHaveBeenCalledOnce();
expect(onClose).toHaveBeenCalledWith("completed");
resolvePeer();
await expect(connection).rejects.toThrow("GPT-Live gateway relay bridge closed");