diff --git a/extensions/openai/realtime-quicksilver-bridge.ts b/extensions/openai/realtime-quicksilver-bridge.ts index a6658d640302..95b012dbdb16 100644 --- a/extensions/openai/realtime-quicksilver-bridge.ts +++ b/extensions/openai/realtime-quicksilver-bridge.ts @@ -38,6 +38,7 @@ import { const OPENAI_QUICKSILVER_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024; const OPENAI_QUICKSILVER_READY_TIMEOUT_MS = 15_000; const OPENAI_QUICKSILVER_PENDING_AUDIO_CHUNKS = 320; +const OPENAI_QUICKSILVER_PENDING_AUDIO_BYTES = 1024 * 1024; const OPENAI_QUICKSILVER_SAMPLE_RATE = 24_000; const WEBSOCKET_OPEN = 1; @@ -88,6 +89,7 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge { private connectPromise: Promise | undefined; private readonly lifecycle = new OpenAIRealtimeVoiceLifecycle(); private pendingAudio: Buffer[] = []; + private pendingAudioBytes = 0; private activeDelegations = new Set(); private readonly flowId = randomUUID(); private readonly requestIds: OpenAIQuicksilverRequestIds = { @@ -141,7 +143,7 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge { ) { return; } - this.lifecycle.failure(connection); + this.failLifecycle(connection); throw error; } if (!this.lifecycle.isCurrent(connection) || connection.signal.aborted) { @@ -198,7 +200,7 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge { if (!this.lifecycle.acceptsEvents(connection) || reachedReady) { return; } - this.lifecycle.failure(connection); + this.failLifecycle(connection); failReady(error); this.closeSocket(reason, connected.socket); }; @@ -269,7 +271,7 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge { return; } const error = new Error("GPT-Live WebSocket closed before session.started"); - this.lifecycle.failure(connection); + this.failLifecycle(connection); failReady(error); this.lifecycle.close(connection, "error"); return; @@ -309,10 +311,11 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge { } sendAudio(audio: Buffer): void { + if (this.lifecycle.phase() === "terminal") { + return; + } if (!this.lifecycle.isReady() || this.socket?.readyState !== WEBSOCKET_OPEN) { - if (this.pendingAudio.length < OPENAI_QUICKSILVER_PENDING_AUDIO_CHUNKS) { - this.pendingAudio.push(audio); - } + this.enqueuePendingAudio(audio); return; } this.sendAudioNow(audio); @@ -360,6 +363,7 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge { if (!connection || !this.lifecycle.cancel()) { return; } + this.resetTerminalState(); if (this.socket?.readyState === WEBSOCKET_OPEN) { this.sendEvent({ type: "session.close" }); } @@ -425,7 +429,9 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge { } if (event.kind === "session-started") { if (this.lifecycle.ready(connection)) { - for (const audio of this.pendingAudio.splice(0)) { + const pendingAudio = this.pendingAudio.splice(0); + this.pendingAudioBytes = 0; + for (const audio of pendingAudio) { this.sendAudioNow(audio); } this.config.onReady?.(); @@ -535,7 +541,7 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge { error: Error, reason = "bridge error", ): boolean { - if (!this.lifecycle.failure(connection)) { + if (!this.failLifecycle(connection)) { return false; } this.config.onError?.(error); @@ -543,6 +549,31 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge { return true; } + private failLifecycle(connection: OpenAIRealtimeVoiceConnection): boolean { + if (!this.lifecycle.failure(connection)) { + return false; + } + this.resetTerminalState(); + return true; + } + + private enqueuePendingAudio(audio: Buffer): void { + if ( + this.pendingAudio.length >= OPENAI_QUICKSILVER_PENDING_AUDIO_CHUNKS || + this.pendingAudioBytes + audio.byteLength > OPENAI_QUICKSILVER_PENDING_AUDIO_BYTES + ) { + return; + } + this.pendingAudio.push(audio); + this.pendingAudioBytes += audio.byteLength; + } + + private resetTerminalState(): void { + this.pendingAudio = []; + this.pendingAudioBytes = 0; + this.activeDelegations.clear(); + } + private closeSocket(reason: string, socket = this.socket): void { try { socket?.close(1000, reason); @@ -559,6 +590,7 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge { if (!outcome) { return; } + this.resetTerminalState(); this.config.onClose?.(outcome); } } diff --git a/extensions/openai/realtime-voice-provider.ts b/extensions/openai/realtime-voice-provider.ts index e2b46a5cc891..d7b85ff489fe 100644 --- a/extensions/openai/realtime-voice-provider.ts +++ b/extensions/openai/realtime-voice-provider.ts @@ -575,6 +575,8 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { private static readonly MAX_RECONNECT_ATTEMPTS = 5; private static readonly BASE_RECONNECT_DELAY_MS = 1000; private static readonly CONNECT_TIMEOUT_MS = 10_000; + private static readonly MAX_PENDING_AUDIO_CHUNKS = 320; + private static readonly MAX_PENDING_AUDIO_BYTES = 1024 * 1024; readonly supportsToolResultContinuation = true; readonly supportsToolResultSuppression = true; @@ -583,6 +585,7 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { private connectPromise: Promise | undefined; private readonly lifecycle = new OpenAIRealtimeVoiceLifecycle(); private pendingAudio: Buffer[] = []; + private pendingAudioBytes = 0; private markQueue: string[] = []; private responseStartTimestamp: number | null = null; private responseActive = false; @@ -633,10 +636,11 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { } sendAudio(audio: Buffer): void { + if (this.lifecycle.phase() === "terminal") { + return; + } if (!this.lifecycle.isReady() || this.ws?.readyState !== WebSocket.OPEN) { - if (this.pendingAudio.length < 320) { - this.pendingAudio.push(audio); - } + this.enqueuePendingAudio(audio); return; } this.sendEvent({ @@ -704,6 +708,7 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { if (!connection || !this.lifecycle.cancel()) { return; } + this.resetTerminalState(); const ws = this.ws; this.ws = null; ws?.close(1000, "Bridge closed"); @@ -1061,7 +1066,9 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { type: "session.reconnect.exhausted", detail: `reason=${reason} attempts=${OpenAIRealtimeVoiceBridge.MAX_RECONNECT_ATTEMPTS}`, }); - this.lifecycle.failure(connection); + if (this.lifecycle.failure(connection)) { + this.resetTerminalState(); + } this.notifyClose(connection, "error"); return; } @@ -1272,7 +1279,9 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { this.sessionReadyFired = true; this.config.onReady?.(); } - for (const chunk of this.pendingAudio.splice(0)) { + const pendingAudio = this.pendingAudio.splice(0); + this.pendingAudioBytes = 0; + for (const chunk of pendingAudio) { this.sendAudio(chunk); } return; @@ -1591,6 +1600,27 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { this.deliveredToolCallKeys.clear(); } + private enqueuePendingAudio(audio: Buffer): void { + if ( + this.pendingAudio.length >= OpenAIRealtimeVoiceBridge.MAX_PENDING_AUDIO_CHUNKS || + this.pendingAudioBytes + audio.byteLength > OpenAIRealtimeVoiceBridge.MAX_PENDING_AUDIO_BYTES + ) { + return; + } + this.pendingAudio.push(audio); + this.pendingAudioBytes += audio.byteLength; + } + + private clearPendingAudio(): void { + this.pendingAudio = []; + this.pendingAudioBytes = 0; + } + + private resetTerminalState(): void { + this.clearPendingAudio(); + this.resetRealtimeSessionState(); + } + private failConnection( error: OpenAIRealtimeMalformedAudioError, ws: WebSocket, @@ -1601,7 +1631,7 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { } this.terminalError = error; this.lifecycle.failure(connection); - this.resetRealtimeSessionState(); + this.resetTerminalState(); try { this.config.onError?.(error); } finally { @@ -1621,6 +1651,7 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { if (!terminalOutcome) { return; } + this.resetTerminalState(); this.config.onClose?.(terminalOutcome); }