ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-27 12:56:01 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(openai): bound pre-ready realtime audio

Browse Source
This commit is contained in:
Vincent Koc
2026-07-30 12:35:24 +08:00
parent 81d165c612
commit 3133abed32
2 changed files with 77 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/openai/realtime-quicksilver-bridge.ts
+40 -8
View File
@@ -38,6 +38,7 @@ import {
const OPENAI_QUICKSILVER_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024;
const OPENAI_QUICKSILVER_READY_TIMEOUT_MS = 15_000;
const OPENAI_QUICKSILVER_PENDING_AUDIO_CHUNKS = 320;
const OPENAI_QUICKSILVER_PENDING_AUDIO_BYTES = 1024 * 1024;
const OPENAI_QUICKSILVER_SAMPLE_RATE = 24_000;
const WEBSOCKET_OPEN = 1;
@@ -88,6 +89,7 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge {
private connectPromise: Promise<void> | undefined;
private readonly lifecycle = new OpenAIRealtimeVoiceLifecycle();
private pendingAudio: Buffer[] = [];
private pendingAudioBytes = 0;
private activeDelegations = new Set<string>();
private readonly flowId = randomUUID();
private readonly requestIds: OpenAIQuicksilverRequestIds = {
@@ -141,7 +143,7 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge {
) {
return;
}
this.lifecycle.failure(connection);
this.failLifecycle(connection);
throw error;
}
if (!this.lifecycle.isCurrent(connection) || connection.signal.aborted) {
@@ -198,7 +200,7 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge {
if (!this.lifecycle.acceptsEvents(connection) || reachedReady) {
return;
}
this.lifecycle.failure(connection);
this.failLifecycle(connection);
failReady(error);
this.closeSocket(reason, connected.socket);
};
@@ -269,7 +271,7 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge {
return;
}
const error = new Error("GPT-Live WebSocket closed before session.started");
this.lifecycle.failure(connection);
this.failLifecycle(connection);
failReady(error);
this.lifecycle.close(connection, "error");
return;
@@ -309,10 +311,11 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge {
}
sendAudio(audio: Buffer): void {
if (this.lifecycle.phase() === "terminal") {
return;
}
if (!this.lifecycle.isReady() || this.socket?.readyState !== WEBSOCKET_OPEN) {
if (this.pendingAudio.length < OPENAI_QUICKSILVER_PENDING_AUDIO_CHUNKS) {
this.pendingAudio.push(audio);
}
this.enqueuePendingAudio(audio);
return;
}
this.sendAudioNow(audio);
@@ -360,6 +363,7 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge {
if (!connection || !this.lifecycle.cancel()) {
return;
}
this.resetTerminalState();
if (this.socket?.readyState === WEBSOCKET_OPEN) {
this.sendEvent({ type: "session.close" });
}
@@ -425,7 +429,9 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge {
}
if (event.kind === "session-started") {
if (this.lifecycle.ready(connection)) {
for (const audio of this.pendingAudio.splice(0)) {
const pendingAudio = this.pendingAudio.splice(0);
this.pendingAudioBytes = 0;
for (const audio of pendingAudio) {
this.sendAudioNow(audio);
}
this.config.onReady?.();
@@ -535,7 +541,7 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge {
error: Error,
reason = "bridge error",
): boolean {
if (!this.lifecycle.failure(connection)) {
if (!this.failLifecycle(connection)) {
return false;
}
this.config.onError?.(error);
@@ -543,6 +549,31 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge {
return true;
}
private failLifecycle(connection: OpenAIRealtimeVoiceConnection): boolean {
if (!this.lifecycle.failure(connection)) {
return false;
}
this.resetTerminalState();
return true;
}
private enqueuePendingAudio(audio: Buffer): void {
if (
this.pendingAudio.length >= OPENAI_QUICKSILVER_PENDING_AUDIO_CHUNKS ||
this.pendingAudioBytes + audio.byteLength > OPENAI_QUICKSILVER_PENDING_AUDIO_BYTES
) {
return;
}
this.pendingAudio.push(audio);
this.pendingAudioBytes += audio.byteLength;
}
private resetTerminalState(): void {
this.pendingAudio = [];
this.pendingAudioBytes = 0;
this.activeDelegations.clear();
}
private closeSocket(reason: string, socket = this.socket): void {
try {
socket?.close(1000, reason);
@@ -559,6 +590,7 @@ export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge {
if (!outcome) {
return;
}
this.resetTerminalState();
this.config.onClose?.(outcome);
}
}
extensions/openai/realtime-voice-provider.ts
+37 -6
View File
@@ -575,6 +575,8 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge {
private static readonly MAX_RECONNECT_ATTEMPTS = 5;
private static readonly BASE_RECONNECT_DELAY_MS = 1000;
private static readonly CONNECT_TIMEOUT_MS = 10_000;
private static readonly MAX_PENDING_AUDIO_CHUNKS = 320;
private static readonly MAX_PENDING_AUDIO_BYTES = 1024 * 1024;
readonly supportsToolResultContinuation = true;
readonly supportsToolResultSuppression = true;
@@ -583,6 +585,7 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge {
private connectPromise: Promise<void> | undefined;
private readonly lifecycle = new OpenAIRealtimeVoiceLifecycle();
private pendingAudio: Buffer[] = [];
private pendingAudioBytes = 0;
private markQueue: string[] = [];
private responseStartTimestamp: number | null = null;
private responseActive = false;
@@ -633,10 +636,11 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge {
}
sendAudio(audio: Buffer): void {
if (this.lifecycle.phase() === "terminal") {
return;
}
if (!this.lifecycle.isReady() || this.ws?.readyState !== WebSocket.OPEN) {
if (this.pendingAudio.length < 320) {
this.pendingAudio.push(audio);
}
this.enqueuePendingAudio(audio);
return;
}
this.sendEvent({
@@ -704,6 +708,7 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge {
if (!connection || !this.lifecycle.cancel()) {
return;
}
this.resetTerminalState();
const ws = this.ws;
this.ws = null;
ws?.close(1000, "Bridge closed");
@@ -1061,7 +1066,9 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge {
type: "session.reconnect.exhausted",
detail: `reason=${reason} attempts=${OpenAIRealtimeVoiceBridge.MAX_RECONNECT_ATTEMPTS}`,
});
this.lifecycle.failure(connection);
if (this.lifecycle.failure(connection)) {
this.resetTerminalState();
}
this.notifyClose(connection, "error");
return;
}
@@ -1272,7 +1279,9 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge {
this.sessionReadyFired = true;
this.config.onReady?.();
}
for (const chunk of this.pendingAudio.splice(0)) {
const pendingAudio = this.pendingAudio.splice(0);
this.pendingAudioBytes = 0;
for (const chunk of pendingAudio) {
this.sendAudio(chunk);
}
return;
@@ -1591,6 +1600,27 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge {
this.deliveredToolCallKeys.clear();
}
private enqueuePendingAudio(audio: Buffer): void {
if (
this.pendingAudio.length >= OpenAIRealtimeVoiceBridge.MAX_PENDING_AUDIO_CHUNKS ||
this.pendingAudioBytes + audio.byteLength > OpenAIRealtimeVoiceBridge.MAX_PENDING_AUDIO_BYTES
) {
return;
}
this.pendingAudio.push(audio);
this.pendingAudioBytes += audio.byteLength;
}
private clearPendingAudio(): void {
this.pendingAudio = [];
this.pendingAudioBytes = 0;
}
private resetTerminalState(): void {
this.clearPendingAudio();
this.resetRealtimeSessionState();
}
private failConnection(
error: OpenAIRealtimeMalformedAudioError,
ws: WebSocket,
@@ -1601,7 +1631,7 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge {
}
this.terminalError = error;
this.lifecycle.failure(connection);
this.resetRealtimeSessionState();
this.resetTerminalState();
try {
this.config.onError?.(error);
} finally {
@@ -1621,6 +1651,7 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge {
if (!terminalOutcome) {
return;
}
this.resetTerminalState();
this.config.onClose?.(terminalOutcome);
}