diff --git a/extensions/openai/realtime-transcription-provider.bounds.test.ts b/extensions/openai/realtime-transcription-provider.bounds.test.ts new file mode 100644 index 000000000000..3bb710870803 --- /dev/null +++ b/extensions/openai/realtime-transcription-provider.bounds.test.ts @@ -0,0 +1,227 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { buildOpenAIRealtimeTranscriptionProvider } from "./realtime-transcription-provider.js"; + +const { FakeWebSocket } = vi.hoisted(() => { + type Listener = (...args: unknown[]) => void; + + class MockWebSocket { + static readonly OPEN = 1; + static readonly CLOSED = 3; + static instances: MockWebSocket[] = []; + + readonly listeners = new Map(); + readyState = 0; + closed = false; + + constructor() { + MockWebSocket.instances.push(this); + } + + on(event: string, listener: Listener): this { + const listeners = this.listeners.get(event) ?? []; + listeners.push(listener); + this.listeners.set(event, listeners); + return this; + } + + emit(event: string, ...args: unknown[]): void { + for (const listener of this.listeners.get(event) ?? []) { + listener(...args); + } + } + + send(): void {} + + close(code?: number, reason?: string): void { + this.closed = true; + this.readyState = MockWebSocket.CLOSED; + this.emit("close", code ?? 1000, Buffer.from(reason ?? "")); + } + + terminate(): void { + this.closed = true; + this.readyState = MockWebSocket.CLOSED; + } + } + + return { FakeWebSocket: MockWebSocket }; +}); + +vi.mock("ws", () => ({ + default: FakeWebSocket, +})); + +type FakeWebSocketInstance = InstanceType; + +async function waitForFakeSocket(index = 0): Promise { + let socket: FakeWebSocketInstance | undefined; + await vi.waitFor(() => { + socket = FakeWebSocket.instances[index]; + if (!socket) { + throw new Error("expected session to create a websocket"); + } + }); + if (!socket) { + throw new Error("expected session to create a websocket"); + } + return socket; +} + +function emitJson(socket: FakeWebSocketInstance, event: Record): void { + socket.emit("message", Buffer.from(JSON.stringify(event))); +} + +async function connectFakeSession( + session: { connect(): Promise }, + socketIndex = 0, +): Promise { + const connecting = session.connect(); + const socket = await waitForFakeSocket(socketIndex); + socket.readyState = FakeWebSocket.OPEN; + socket.emit("open"); + emitJson(socket, { type: "session.updated" }); + await connecting; + return socket; +} + +describe("OpenAI realtime transcription terminal history bounds", () => { + beforeEach(() => { + FakeWebSocket.instances = []; + vi.stubEnv("OPENAI_API_KEY", ""); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("does not re-admit a terminal item after the settled frontier fills", async () => { + const onError = vi.fn(); + const onPartial = vi.fn(); + const transcripts: string[] = []; + const provider = buildOpenAIRealtimeTranscriptionProvider(); + const session = provider.createSession({ + providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret + onError, + onPartial, + onTranscript: (transcript) => transcripts.push(transcript), + }); + const socket = await connectFakeSession(session); + + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "oldest", + transcript: "first", + }); + for (let index = 0; index < 4095; index += 1) { + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: `settled-${index}`, + transcript: "", + }); + } + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "oldest", + transcript: "duplicate", + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.delta", + item_id: "oldest", + delta: "late partial", + }); + + expect(transcripts).toEqual(["first"]); + expect(onPartial).not.toHaveBeenCalled(); + expect(onError).not.toHaveBeenCalled(); + expect(session.isConnected()).toBe(true); + session.close(); + }); + + it("fails visibly instead of evicting terminal item history", async () => { + const onError = vi.fn(); + const onTranscript = vi.fn(); + const provider = buildOpenAIRealtimeTranscriptionProvider(); + const session = provider.createSession({ + providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret + onError, + onTranscript, + }); + const socket = await connectFakeSession(session); + + for (let index = 0; index < 4096; index += 1) { + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: `settled-${index}`, + transcript: "", + }); + } + emitJson(socket, { + type: "input_audio_buffer.committed", + item_id: "overflow-item", + previous_item_id: null, + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.failed", + item_id: "overflow-item", + error: { message: "provider failure" }, + }); + + expect(onError).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + message: "OpenAI realtime transcription exceeded the terminal item history limit", + }), + ); + expect(onTranscript).not.toHaveBeenCalled(); + expect(session.isConnected()).toBe(false); + + const reconnecting = session.connect(); + const replacementSocket = await waitForFakeSocket(1); + replacementSocket.readyState = FakeWebSocket.OPEN; + replacementSocket.emit("open"); + emitJson(replacementSocket, { type: "session.updated" }); + await reconnecting; + emitJson(replacementSocket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "replacement-item", + transcript: "replacement transcript", + }); + + expect(onTranscript).toHaveBeenCalledExactlyOnceWith("replacement transcript"); + expect(onError).toHaveBeenCalledTimes(1); + session.close(); + }); + + it("fails visibly when terminal item identities exceed 256 KiB", async () => { + const onError = vi.fn(); + const onTranscript = vi.fn(); + const provider = buildOpenAIRealtimeTranscriptionProvider(); + const session = provider.createSession({ + providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret + onError, + onTranscript, + }); + const socket = await connectFakeSession(session); + + for (let index = 0; index < 256; index += 1) { + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: `${index.toString().padStart(4, "0")}${"i".repeat(1020)}`, + transcript: "", + }); + } + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "overflow-item", + transcript: "must not emit", + }); + + expect(onError).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + message: "OpenAI realtime transcription exceeded the terminal item history limit", + }), + ); + expect(onTranscript).not.toHaveBeenCalled(); + expect(session.isConnected()).toBe(false); + session.close(); + }); +}); diff --git a/extensions/openai/realtime-transcription-provider.test.ts b/extensions/openai/realtime-transcription-provider.test.ts index 71cce622175f..d851d160e19f 100644 --- a/extensions/openai/realtime-transcription-provider.test.ts +++ b/extensions/openai/realtime-transcription-provider.test.ts @@ -45,6 +45,11 @@ const { FakeWebSocket, providerAuthMocks, ssrfMocks } = vi.hoisted(() => { this.readyState = MockWebSocket.CLOSED; this.emit("close", code ?? 1000, Buffer.from(reason ?? "")); } + + terminate(): void { + this.closed = true; + this.readyState = MockWebSocket.CLOSED; + } } return { @@ -83,10 +88,10 @@ function parseSent(socket: FakeWebSocketInstance): SentRealtimeEvent[] { return socket.sent.map((payload) => JSON.parse(payload) as SentRealtimeEvent); } -async function waitForFakeSocket(): Promise { +async function waitForFakeSocket(index = 0): Promise { let socket: FakeWebSocketInstance | undefined; await vi.waitFor(() => { - socket = FakeWebSocket.instances[0]; + socket = FakeWebSocket.instances[index]; if (!socket) { throw new Error("expected session to create a websocket"); } @@ -97,6 +102,23 @@ async function waitForFakeSocket(): Promise { return socket; } +function emitJson(socket: FakeWebSocketInstance, event: Record): void { + socket.emit("message", Buffer.from(JSON.stringify(event))); +} + +async function connectFakeSession( + session: { connect(): Promise }, + socketIndex = 0, +): Promise { + const connecting = session.connect(); + const socket = await waitForFakeSocket(socketIndex); + socket.readyState = FakeWebSocket.OPEN; + socket.emit("open"); + emitJson(socket, { type: "session.updated" }); + await connecting; + return socket; +} + function mockCallArg(mock: { mock: { calls: unknown[][] } }, index = 0): Record { const call = mock.mock.calls[index]; if (!call) { @@ -565,4 +587,493 @@ describe("buildOpenAIRealtimeTranscriptionProvider", () => { expect(transcripts).toEqual(["second final"]); session.close(); }); + + it("releases settled turns from the unresolved item budget", async () => { + const transcripts: string[] = []; + const onError = vi.fn(); + const provider = buildOpenAIRealtimeTranscriptionProvider(); + const session = provider.createSession({ + providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret + onError, + onTranscript: (transcript) => transcripts.push(transcript), + }); + const socket = await connectFakeSession(session); + + for (let index = 0; index < 128; index += 1) { + const itemId = `item-${index}`; + emitJson(socket, { + type: "input_audio_buffer.committed", + item_id: itemId, + previous_item_id: index === 0 ? null : `item-${index - 1}`, + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: itemId, + transcript: `turn-${index}`, + }); + } + emitJson(socket, { + type: "input_audio_buffer.committed", + item_id: "item-129", + previous_item_id: "item-128", + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "item-129", + transcript: "turn-129", + }); + emitJson(socket, { + type: "input_audio_buffer.committed", + item_id: "item-128", + previous_item_id: "item-127", + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "item-128", + transcript: "turn-128", + }); + + expect(transcripts).toHaveLength(130); + expect(transcripts.slice(-2)).toEqual(["turn-128", "turn-129"]); + expect(onError).not.toHaveBeenCalled(); + expect(session.isConnected()).toBe(true); + session.close(); + }); + + it("fails once when unresolved item correlation exceeds its session bound", async () => { + const onError = vi.fn(); + const onTranscript = vi.fn(); + const provider = buildOpenAIRealtimeTranscriptionProvider(); + const session = provider.createSession({ + providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret + onError, + onTranscript, + }); + const socket = await connectFakeSession(session); + + for (let index = 0; index < 64; index += 1) { + emitJson(socket, { + type: "input_audio_buffer.committed", + item_id: `item-${index}`, + previous_item_id: "missing-predecessor", + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: `item-${index}`, + transcript: `turn-${index}`, + }); + } + emitJson(socket, { + type: "conversation.item.input_audio_transcription.failed", + item_id: "overflow-item", + error: { message: "provider failure" }, + }); + + expect(onError).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + message: "OpenAI realtime transcription exceeded the 64 unresolved item limit", + }), + ); + expect(onTranscript).not.toHaveBeenCalled(); + expect(session.isConnected()).toBe(false); + + const reconnecting = session.connect(); + const replacementSocket = await waitForFakeSocket(1); + replacementSocket.readyState = FakeWebSocket.OPEN; + replacementSocket.emit("open"); + emitJson(replacementSocket, { type: "session.updated" }); + await reconnecting; + emitJson(replacementSocket, { + type: "input_audio_buffer.committed", + item_id: "replacement-item", + previous_item_id: null, + }); + emitJson(replacementSocket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "replacement-item", + transcript: "replacement transcript", + }); + + expect(onTranscript).toHaveBeenCalledExactlyOnceWith("replacement transcript"); + expect(onError).toHaveBeenCalledTimes(1); + session.close(); + session.close(); + }); + + it("fails once when aggregate in-progress transcript text exceeds 256 KiB", async () => { + const onError = vi.fn(); + const onPartial = vi.fn(); + const onTranscript = vi.fn(); + const provider = buildOpenAIRealtimeTranscriptionProvider(); + const session = provider.createSession({ + providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret + onError, + onPartial, + onTranscript, + }); + const socket = await connectFakeSession(session); + const exactLimit = "🙂".repeat((256 * 1024) / 4); + + emitJson(socket, { + type: "conversation.item.input_audio_transcription.delta", + item_id: "item-1", + delta: exactLimit, + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.delta", + item_id: "item-1", + delta: "x", + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "item-1", + transcript: "late transcript", + }); + + expect(onPartial).toHaveBeenCalledExactlyOnceWith(exactLimit); + expect(onError).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + message: "OpenAI realtime transcription exceeded the 256 KiB retained transcript limit", + }), + ); + expect(onTranscript).not.toHaveBeenCalled(); + expect(session.isConnected()).toBe(false); + session.close(); + }); + + it("accounts for UTF-8 surrogate pairs split across delta frames", async () => { + const onError = vi.fn(); + const onPartial = vi.fn(); + const provider = buildOpenAIRealtimeTranscriptionProvider(); + const session = provider.createSession({ + providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret + onError, + onPartial, + }); + const socket = await connectFakeSession(session); + const prefix = "x".repeat(256 * 1024 - 4); + + emitJson(socket, { + type: "conversation.item.input_audio_transcription.delta", + item_id: "item-1", + delta: prefix, + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.delta", + item_id: "item-1", + delta: "\ud83d", + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.delta", + item_id: "item-1", + delta: "\ude42", + }); + + expect(onPartial).toHaveBeenLastCalledWith(`${prefix}🙂`); + expect(onError).not.toHaveBeenCalled(); + expect(session.isConnected()).toBe(true); + session.close(); + }); + + it("ignores duplicate completion events without double-charging retained text", async () => { + const onError = vi.fn(); + const onPartial = vi.fn(); + const transcripts: string[] = []; + const provider = buildOpenAIRealtimeTranscriptionProvider(); + const session = provider.createSession({ + providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret + onError, + onPartial, + onTranscript: (transcript) => transcripts.push(transcript), + }); + const socket = await connectFakeSession(session); + const secondTranscript = "x".repeat(192 * 1024); + + emitJson(socket, { + type: "input_audio_buffer.committed", + item_id: "item-2", + previous_item_id: "item-1", + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "item-2", + transcript: secondTranscript, + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.delta", + item_id: "item-2", + delta: "late partial", + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.failed", + item_id: "item-2", + error: { message: "late failure" }, + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "item-2", + transcript: secondTranscript, + }); + emitJson(socket, { + type: "input_audio_buffer.committed", + item_id: "item-1", + previous_item_id: null, + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "item-1", + transcript: "first", + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "item-2", + transcript: "late duplicate", + }); + + expect(transcripts).toEqual(["first", secondTranscript]); + expect(onPartial).not.toHaveBeenCalled(); + expect(onError).not.toHaveBeenCalled(); + expect(session.isConnected()).toBe(true); + + const reconnecting = session.connect(); + const replacementSocket = await waitForFakeSocket(1); + replacementSocket.readyState = FakeWebSocket.OPEN; + replacementSocket.emit("open"); + emitJson(replacementSocket, { type: "session.updated" }); + await reconnecting; + emitJson(replacementSocket, { + type: "input_audio_buffer.committed", + item_id: "item-2", + previous_item_id: null, + }); + emitJson(replacementSocket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "item-2", + transcript: "new session", + }); + + expect(transcripts).toEqual(["first", secondTranscript, "new session"]); + session.close(); + }); + + it("tombstones terminal outcomes received before their commit", async () => { + const errors: string[] = []; + const partials: string[] = []; + const transcripts: string[] = []; + const provider = buildOpenAIRealtimeTranscriptionProvider(); + const session = provider.createSession({ + providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret + onError: (error) => errors.push(error.message), + onPartial: (partial) => partials.push(partial), + onTranscript: (transcript) => transcripts.push(transcript), + }); + const socket = await connectFakeSession(session); + + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "item-completed", + transcript: "first", + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "item-completed", + transcript: "duplicate", + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.delta", + item_id: "item-completed", + delta: "late partial", + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.failed", + item_id: "item-completed", + error: { message: "late failure" }, + }); + emitJson(socket, { + type: "input_audio_buffer.committed", + item_id: "item-completed", + previous_item_id: null, + }); + + emitJson(socket, { + type: "conversation.item.input_audio_transcription.failed", + item_id: "item-failed", + error: { message: "first failure" }, + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "item-failed", + transcript: "late completion", + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.delta", + item_id: "item-failed", + delta: "late partial", + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.failed", + item_id: "item-failed", + error: { message: "duplicate failure" }, + }); + emitJson(socket, { + type: "input_audio_buffer.committed", + item_id: "item-failed", + previous_item_id: null, + }); + + expect(transcripts).toEqual(["first"]); + expect(errors).toEqual(["first failure"]); + expect(partials).toEqual([]); + expect(session.isConnected()).toBe(true); + session.close(); + }); + + it("keeps active predecessor satisfaction as terminal history grows", async () => { + const transcripts: string[] = []; + const provider = buildOpenAIRealtimeTranscriptionProvider(); + const session = provider.createSession({ + providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret + onTranscript: (transcript) => transcripts.push(transcript), + }); + const socket = await connectFakeSession(session); + + emitJson(socket, { + type: "input_audio_buffer.committed", + item_id: "root", + previous_item_id: null, + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "root", + transcript: "root transcript", + }); + emitJson(socket, { + type: "input_audio_buffer.committed", + item_id: "waiting", + previous_item_id: "root", + }); + for (let index = 0; index < 64; index += 1) { + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: `uncommitted-${index}`, + transcript: "", + }); + } + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "waiting", + transcript: "waiting transcript", + }); + + expect(transcripts).toEqual(["root transcript", "waiting transcript"]); + expect(session.isConnected()).toBe(true); + session.close(); + }); + + it("keeps the first failed terminal outcome when completion arrives late", async () => { + const errors: string[] = []; + const transcripts: string[] = []; + const provider = buildOpenAIRealtimeTranscriptionProvider(); + const session = provider.createSession({ + providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret + onError: (error) => errors.push(error.message), + onTranscript: (transcript) => transcripts.push(transcript), + }); + const socket = await connectFakeSession(session); + + emitJson(socket, { + type: "input_audio_buffer.committed", + item_id: "item-2", + previous_item_id: "item-1", + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.failed", + item_id: "item-2", + error: { message: "second failed" }, + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "item-2", + transcript: "late second", + }); + emitJson(socket, { + type: "input_audio_buffer.committed", + item_id: "item-1", + previous_item_id: null, + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "item-1", + transcript: "first", + }); + + expect(errors).toEqual(["second failed"]); + expect(transcripts).toEqual(["first"]); + expect(session.isConnected()).toBe(true); + session.close(); + }); + + it("fails before retaining oversized correlation identities", async () => { + const onError = vi.fn(); + const onTranscript = vi.fn(); + const provider = buildOpenAIRealtimeTranscriptionProvider(); + const session = provider.createSession({ + providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret + onError, + onTranscript, + }); + const socket = await connectFakeSession(session); + + emitJson(socket, { + type: "conversation.item.input_audio_transcription.failed", + item_id: "i".repeat(1025), + error: { message: "provider failure" }, + }); + + expect(onError).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + message: "OpenAI realtime transcription exceeded the 1024-byte item identity limit", + }), + ); + expect(onTranscript).not.toHaveBeenCalled(); + expect(session.isConnected()).toBe(false); + session.close(); + }); + + it("fails before retaining an oversized completed transcript", async () => { + const onError = vi.fn(); + const onTranscript = vi.fn(); + const provider = buildOpenAIRealtimeTranscriptionProvider(); + const session = provider.createSession({ + providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret + onError, + onTranscript, + }); + const socket = await connectFakeSession(session); + + emitJson(socket, { + type: "input_audio_buffer.committed", + item_id: "item-1", + previous_item_id: null, + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "item-1", + transcript: "x".repeat(256 * 1024 + 1), + }); + emitJson(socket, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "item-1", + transcript: "late transcript", + }); + + expect(onError).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + message: "OpenAI realtime transcription exceeded the 256 KiB retained transcript limit", + }), + ); + expect(onTranscript).not.toHaveBeenCalled(); + expect(session.isConnected()).toBe(false); + session.close(); + }); }); diff --git a/extensions/openai/realtime-transcription-provider.ts b/extensions/openai/realtime-transcription-provider.ts index 1c99bad367e8..2ea9ab6bd924 100644 --- a/extensions/openai/realtime-transcription-provider.ts +++ b/extensions/openai/realtime-transcription-provider.ts @@ -75,11 +75,39 @@ const OPENAI_REALTIME_TRANSCRIPTION_CONNECT_TIMEOUT_MS = 10_000; const OPENAI_REALTIME_TRANSCRIPTION_MAX_RECONNECT_ATTEMPTS = 5; const OPENAI_REALTIME_TRANSCRIPTION_RECONNECT_DELAY_MS = 1000; const OPENAI_REALTIME_TRANSCRIPTION_DEFAULT_MODEL = "gpt-4o-transcribe"; +const OPENAI_REALTIME_TRANSCRIPTION_MAX_UNRESOLVED_ITEMS = 64; +const OPENAI_REALTIME_TRANSCRIPTION_MAX_ITEM_ID_BYTES = 1024; +const OPENAI_REALTIME_TRANSCRIPTION_MAX_RETAINED_TRANSCRIPT_BYTES = 256 * 1024; +const OPENAI_REALTIME_TRANSCRIPTION_MAX_SETTLED_ITEMS = 4096; +const OPENAI_REALTIME_TRANSCRIPTION_MAX_SETTLED_ID_BYTES = 256 * 1024; +const OPENAI_REALTIME_TRANSCRIPTION_ITEM_OVERFLOW_MESSAGE = + "OpenAI realtime transcription exceeded the 64 unresolved item limit"; +const OPENAI_REALTIME_TRANSCRIPTION_IDENTITY_OVERFLOW_MESSAGE = + "OpenAI realtime transcription exceeded the 1024-byte item identity limit"; +const OPENAI_REALTIME_TRANSCRIPTION_TEXT_OVERFLOW_MESSAGE = + "OpenAI realtime transcription exceeded the 256 KiB retained transcript limit"; +const OPENAI_REALTIME_TRANSCRIPTION_SETTLED_OVERFLOW_MESSAGE = + "OpenAI realtime transcription exceeded the terminal item history limit"; const OPENAI_REALTIME_TRANSCRIPTION_API_KEY_REQUIRED = "OpenAI Realtime transcription requires an OpenAI Platform API key"; const OPENAI_REALTIME_TRANSCRIPTION_API_KEY_REJECTED = "OpenAI Realtime transcription rejected the selected API key. Update or remove the active OpenAI API-key source"; +function appendedUtf8ByteLength(previous: string, appended: string): number { + const appendedBytes = Buffer.byteLength(appended, "utf8"); + if (!previous || !appended) { + return appendedBytes; + } + const previousCodeUnit = previous.charCodeAt(previous.length - 1); + const appendedCodeUnit = appended.charCodeAt(0); + const joinsSurrogatePair = + previousCodeUnit >= 0xd800 && + previousCodeUnit <= 0xdbff && + appendedCodeUnit >= 0xdc00 && + appendedCodeUnit <= 0xdfff; + return joinsSurrogatePair ? appendedBytes - 2 : appendedBytes; +} + function normalizeProviderConfig( config: RealtimeTranscriptionProviderConfig, ): OpenAIRealtimeTranscriptionProviderConfig { @@ -172,35 +200,112 @@ async function resolveOpenAIRealtimeTranscriptionAuthorization( function createOpenAIRealtimeTranscriptionSession( config: OpenAIRealtimeTranscriptionSessionConfig, ): RealtimeTranscriptionSession { - const pendingTranscripts = new Map(); + const pendingTranscripts = new Map(); const committedItemIds: string[] = []; - const committedItems = new Set(); - const previousItemIds = new Map(); - const settledItemIds = new Set(); + const committedItems = new Map(); const completedTranscripts = new Map(); + const trackedItemIds = new Set(); + const settledItemIds = new Set(); const unkeyedTranscript = "__openclaw_unkeyed_transcript__"; + let retainedTranscriptBytes = 0; + let settledItemIdBytes = 0; const resetTranscriptionState = () => { pendingTranscripts.clear(); committedItemIds.length = 0; committedItems.clear(); - previousItemIds.clear(); - settledItemIds.clear(); completedTranscripts.clear(); + trackedItemIds.clear(); + settledItemIds.clear(); + retainedTranscriptBytes = 0; + settledItemIdBytes = 0; }; - const commitItem = (itemId: string, previousItemId: string | null | undefined) => { - if (committedItems.has(itemId)) { + const failTerminal = (error: Error, transport: RealtimeTranscriptionWebSocketTransport) => { + resetTranscriptionState(); + transport.closeNow(); + try { + config.onError?.(error); + } catch { + // The provider terminal owns this outcome; observer failures must not + // re-enter shared error dispatch or duplicate the terminal callback. + } + }; + + const trackItem = ( + itemId: string, + transport: RealtimeTranscriptionWebSocketTransport, + ): boolean => { + if (settledItemIds.has(itemId) || completedTranscripts.has(itemId)) { + return false; + } + if (trackedItemIds.has(itemId)) { + return true; + } + if (trackedItemIds.size >= OPENAI_REALTIME_TRANSCRIPTION_MAX_UNRESOLVED_ITEMS) { + failTerminal(new Error(OPENAI_REALTIME_TRANSCRIPTION_ITEM_OVERFLOW_MESSAGE), transport); + return false; + } + if (Buffer.byteLength(itemId, "utf8") > OPENAI_REALTIME_TRANSCRIPTION_MAX_ITEM_ID_BYTES) { + failTerminal(new Error(OPENAI_REALTIME_TRANSCRIPTION_IDENTITY_OVERFLOW_MESSAGE), transport); + return false; + } + trackedItemIds.add(itemId); + return true; + }; + + const settleItem = ( + itemId: string, + transport: RealtimeTranscriptionWebSocketTransport, + ): boolean => { + const itemIdBytes = Buffer.byteLength(itemId, "utf8"); + if ( + settledItemIds.size >= OPENAI_REALTIME_TRANSCRIPTION_MAX_SETTLED_ITEMS || + itemIdBytes > OPENAI_REALTIME_TRANSCRIPTION_MAX_SETTLED_ID_BYTES - settledItemIdBytes + ) { + failTerminal(new Error(OPENAI_REALTIME_TRANSCRIPTION_SETTLED_OVERFLOW_MESSAGE), transport); + return false; + } + trackedItemIds.delete(itemId); + // Predecessor satisfaction belongs to each active item. Recording it here + // prevents terminal-history saturation from invalidating admitted state. + for (const [candidateId, previousItemId] of committedItems) { + if (previousItemId === itemId) { + committedItems.set(candidateId, null); + } + } + // Never evict terminal identities within a connection generation. Closing + // at the bound preserves first-terminal precedence without unbounded state. + settledItemIds.add(itemId); + settledItemIdBytes += itemIdBytes; + return true; + }; + + const commitItem = ( + itemId: string, + previousItemId: string | null | undefined, + transport: RealtimeTranscriptionWebSocketTransport, + ) => { + if (settledItemIds.has(itemId) || committedItems.has(itemId) || !trackItem(itemId, transport)) { return; } - committedItems.add(itemId); - previousItemIds.set(itemId, previousItemId); + if ( + previousItemId && + Buffer.byteLength(previousItemId, "utf8") > OPENAI_REALTIME_TRANSCRIPTION_MAX_ITEM_ID_BYTES + ) { + failTerminal(new Error(OPENAI_REALTIME_TRANSCRIPTION_IDENTITY_OVERFLOW_MESSAGE), transport); + return; + } + committedItems.set( + itemId, + previousItemId && settledItemIds.has(previousItemId) ? null : previousItemId, + ); committedItemIds.push(itemId); const arrivalOrder = committedItemIds.splice(0); const successors = new Map(); for (const candidateId of arrivalOrder) { - const previousId = previousItemIds.get(candidateId); + const previousId = committedItems.get(candidateId); if (previousId) { successors.set(previousId, candidateId); } @@ -215,7 +320,7 @@ function createOpenAIRealtimeTranscriptionSession( } }; for (const candidateId of arrivalOrder) { - const previousId = previousItemIds.get(candidateId); + const previousId = committedItems.get(candidateId); if (previousId == null || settledItemIds.has(previousId)) { appendChain(candidateId); } @@ -225,44 +330,75 @@ function createOpenAIRealtimeTranscriptionSession( } }; - const flushCompletedTranscripts = () => { + const flushCompletedTranscripts = ( + transport: RealtimeTranscriptionWebSocketTransport, + ): boolean => { while (committedItemIds.length > 0) { const itemId = committedItemIds[0]; if (!itemId || !completedTranscripts.has(itemId)) { - return; + return true; } - const previousItemId = previousItemIds.get(itemId); + const previousItemId = committedItems.get(itemId); if ( previousItemId && !settledItemIds.has(previousItemId) && !committedItems.has(previousItemId) ) { - return; + return true; } committedItemIds.shift(); committedItems.delete(itemId); - previousItemIds.delete(itemId); - settledItemIds.add(itemId); + if (!settleItem(itemId, transport)) { + return false; + } const transcript = completedTranscripts.get(itemId); completedTranscripts.delete(itemId); - pendingTranscripts.delete(itemId); + if (transcript) { + retainedTranscriptBytes -= Buffer.byteLength(transcript, "utf8"); + } if (transcript) { config.onTranscript?.(transcript); } } + return true; }; - const completeItem = (itemId: string | undefined, transcript: string | undefined) => { + const completeItem = ( + itemId: string | undefined, + transcript: string | undefined, + transport: RealtimeTranscriptionWebSocketTransport, + ): boolean => { const key = itemId ?? unkeyedTranscript; + if (itemId && !trackItem(itemId, transport)) { + return false; + } + const partialBytes = pendingTranscripts.get(key)?.bytes ?? 0; pendingTranscripts.delete(key); + retainedTranscriptBytes -= partialBytes; if (!itemId || !committedItems.has(itemId)) { + if (itemId) { + if (!settleItem(itemId, transport)) { + return false; + } + } else { + trackedItemIds.delete(key); + } if (transcript) { config.onTranscript?.(transcript); } - return; + return true; + } + const transcriptBytes = transcript ? Buffer.byteLength(transcript, "utf8") : 0; + if ( + transcriptBytes > + OPENAI_REALTIME_TRANSCRIPTION_MAX_RETAINED_TRANSCRIPT_BYTES - retainedTranscriptBytes + ) { + failTerminal(new Error(OPENAI_REALTIME_TRANSCRIPTION_TEXT_OVERFLOW_MESSAGE), transport); + return false; } completedTranscripts.set(itemId, transcript); - flushCompletedTranscripts(); + retainedTranscriptBytes += transcriptBytes; + return flushCompletedTranscripts(transport); }; const handleEvent = ( @@ -277,32 +413,63 @@ function createOpenAIRealtimeTranscriptionSession( case "input_audio_buffer.committed": if (event.item_id) { - commitItem(event.item_id, event.previous_item_id); + commitItem(event.item_id, event.previous_item_id, transport); } return; case "conversation.item.input_audio_transcription.delta": if (event.delta) { const key = event.item_id ?? unkeyedTranscript; - const pendingTranscript = `${pendingTranscripts.get(key) ?? ""}${event.delta}`; - pendingTranscripts.set(key, pendingTranscript); - config.onPartial?.(pendingTranscript); + if (!trackItem(key, transport)) { + return; + } + const pendingTranscript = pendingTranscripts.get(key); + const previousPartial = pendingTranscript?.text ?? ""; + const deltaBytes = appendedUtf8ByteLength(previousPartial, event.delta); + if ( + deltaBytes > + OPENAI_REALTIME_TRANSCRIPTION_MAX_RETAINED_TRANSCRIPT_BYTES - retainedTranscriptBytes + ) { + failTerminal(new Error(OPENAI_REALTIME_TRANSCRIPTION_TEXT_OVERFLOW_MESSAGE), transport); + return; + } + const partial = `${previousPartial}${event.delta}`; + pendingTranscripts.set(key, { + bytes: (pendingTranscript?.bytes ?? 0) + deltaBytes, + text: partial, + }); + retainedTranscriptBytes += deltaBytes; + config.onPartial?.(partial); } return; case "conversation.item.input_audio_transcription.completed": - completeItem(event.item_id, event.transcript); + completeItem(event.item_id, event.transcript, transport); return; case "conversation.item.input_audio_transcription.failed": - completeItem(event.item_id, undefined); - config.onError?.(new Error(readRealtimeErrorDetail(event.error))); + if ( + event.item_id && + (settledItemIds.has(event.item_id) || completedTranscripts.has(event.item_id)) + ) { + return; + } + if (completeItem(event.item_id, undefined, transport)) { + config.onError?.(new Error(readRealtimeErrorDetail(event.error))); + } return; - case "input_audio_buffer.speech_started": - pendingTranscripts.delete(event.item_id ?? unkeyedTranscript); + case "input_audio_buffer.speech_started": { + const key = event.item_id ?? unkeyedTranscript; + const partialBytes = pendingTranscripts.get(key)?.bytes ?? 0; + pendingTranscripts.delete(key); + retainedTranscriptBytes -= partialBytes; + if (!committedItems.has(key)) { + trackedItemIds.delete(key); + } config.onSpeechStart?.(); return; + } case "error": { const detail = readRealtimeErrorDetail(event.error);