From 970a82624aa4b03318bd039ead74b810d5e73bd4 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 8 Aug 2026 17:38:29 -0700 Subject: [PATCH] fix(elevenlabs): preserve repeated committed speech segments (#120792) --- .../realtime-transcription-provider.test.ts | 144 +++++++++++++++++- .../realtime-transcription-provider.ts | 19 ++- 2 files changed, 152 insertions(+), 11 deletions(-) diff --git a/extensions/elevenlabs/realtime-transcription-provider.test.ts b/extensions/elevenlabs/realtime-transcription-provider.test.ts index b09469dde059..c76b17c316f1 100644 --- a/extensions/elevenlabs/realtime-transcription-provider.test.ts +++ b/extensions/elevenlabs/realtime-transcription-provider.test.ts @@ -23,12 +23,14 @@ async function createRealtimeServer( options?: { initialEvent?: Record; events?: readonly Record[]; + eventsByConnection?: readonly (readonly Record[])[]; closeAfterEvents?: boolean; }, ) { const server = createServer(); const wss = new WebSocketServer({ noServer: true, maxPayload: 1024 * 1024 }); const clients = new Set(); + let connectionCount = 0; server.on("upgrade", (request, socket, head) => { onRequest(new URL(request.url ?? "/", "http://127.0.0.1")); wss.handleUpgrade(request, socket, head, (ws) => { @@ -37,7 +39,9 @@ async function createRealtimeServer( clients.delete(ws); }); ws.send(JSON.stringify(options?.initialEvent ?? { message_type: "session_started" })); - for (const event of options?.events ?? []) { + const events = options?.eventsByConnection?.[connectionCount] ?? options?.events ?? []; + connectionCount += 1; + for (const event of events) { ws.send(JSON.stringify(event)); } if (options?.closeAfterEvents) { @@ -267,6 +271,144 @@ describe("buildElevenLabsRealtimeTranscriptionProvider", () => { session.close(); }); + it.each([ + { + name: "delivers identical committed words from separate speech turns", + events: [ + { message_type: "partial_transcript", text: "yes" }, + { message_type: "committed_transcript", text: "yes" }, + { message_type: "partial_transcript", text: "yes" }, + { message_type: "committed_transcript", text: "yes" }, + ], + transcripts: ["yes", "yes"], + }, + { + name: "treats adjacent identical committed transcripts as separate segments", + events: [ + { message_type: "committed_transcript", text: "yes" }, + { message_type: "committed_transcript", text: "yes" }, + ], + transcripts: ["yes", "yes"], + }, + { + name: "suppresses a matching timestamp companion for the same committed segment", + events: [ + { message_type: "committed_transcript", text: "yes" }, + { message_type: "committed_transcript_with_timestamps", text: "yes" }, + ], + transcripts: ["yes"], + }, + { + name: "consumes the timestamp companion at most once", + events: [ + { message_type: "committed_transcript", text: "yes" }, + { message_type: "committed_transcript_with_timestamps", text: "yes" }, + { message_type: "committed_transcript_with_timestamps", text: "yes" }, + ], + transcripts: ["yes", "yes"], + }, + { + name: "preserves identical consecutive timestamp-only segments", + events: [ + { message_type: "committed_transcript_with_timestamps", text: "yes" }, + { message_type: "committed_transcript_with_timestamps", text: "yes" }, + ], + transcripts: ["yes", "yes"], + }, + { + name: "emits a timestamp-only segment without an earlier plain commit", + events: [{ message_type: "committed_transcript_with_timestamps", text: "yes" }], + transcripts: ["yes"], + }, + { + name: "does not suppress a timestamp transcript that differs from its commit", + events: [ + { message_type: "committed_transcript", text: "yes" }, + { message_type: "committed_transcript_with_timestamps", text: "no" }, + ], + transcripts: ["yes", "no"], + }, + { + name: "preserves a delayed timestamp companion across an interleaved partial", + events: [ + { message_type: "committed_transcript", text: "yes" }, + { message_type: "partial_transcript", text: "next turn" }, + { message_type: "committed_transcript_with_timestamps", text: "yes" }, + { message_type: "committed_transcript", text: "yes" }, + { message_type: "committed_transcript_with_timestamps", text: "yes" }, + ], + transcripts: ["yes", "yes"], + }, + { + name: "keeps timestamp companions attached to alternating committed segments", + events: [ + { message_type: "committed_transcript", text: "yes" }, + { message_type: "committed_transcript_with_timestamps", text: "yes" }, + { message_type: "committed_transcript", text: "no" }, + { message_type: "committed_transcript_with_timestamps", text: "no" }, + { message_type: "committed_transcript", text: "yes" }, + { message_type: "committed_transcript_with_timestamps", text: "yes" }, + ], + transcripts: ["yes", "no", "yes"], + }, + ])("$name", async ({ events, transcripts }) => { + const deliveryMarker = "transcript frames delivered"; + const baseUrl = await createRealtimeServer(() => undefined, { + events: [...events, { message_type: "partial_transcript", text: deliveryMarker }], + }); + const onError = vi.fn(); + const onPartial = vi.fn(); + const onSpeechStart = vi.fn(); + const onTranscript = vi.fn(); + const session = buildElevenLabsRealtimeTranscriptionProvider().createSession({ + providerConfig: { apiKey: "fixture-value", baseUrl }, + onError, + onPartial, + onSpeechStart, + onTranscript, + }); + + await session.connect(); + await vi.waitFor(() => expect(onPartial).toHaveBeenCalledWith(deliveryMarker)); + expect(onTranscript.mock.calls.map(([text]) => text)).toEqual(transcripts); + expect(onError).not.toHaveBeenCalled(); + expect(onSpeechStart).not.toHaveBeenCalled(); + session.close(); + }); + + it("does not suppress a timestamp-only transcript from a replacement session", async () => { + const firstMarker = "first session delivered"; + const secondMarker = "replacement session delivered"; + const baseUrl = await createRealtimeServer(() => undefined, { + eventsByConnection: [ + [ + { message_type: "committed_transcript", text: "yes" }, + { message_type: "partial_transcript", text: firstMarker }, + ], + [ + { message_type: "committed_transcript_with_timestamps", text: "yes" }, + { message_type: "partial_transcript", text: secondMarker }, + ], + ], + }); + const onPartial = vi.fn(); + const onTranscript = vi.fn(); + const session = buildElevenLabsRealtimeTranscriptionProvider().createSession({ + providerConfig: { apiKey: "fixture-value", baseUrl }, + onPartial, + onTranscript, + }); + + await session.connect(); + await vi.waitFor(() => expect(onPartial).toHaveBeenCalledWith(firstMarker)); + expect(onTranscript).toHaveBeenCalledExactlyOnceWith("yes"); + + await session.connect(); + await vi.waitFor(() => expect(onPartial).toHaveBeenCalledWith(secondMarker)); + expect(onTranscript.mock.calls).toEqual([["yes"], ["yes"]]); + session.close(); + }); + it("rejects whitespace-only environment keys before session creation", () => { resolveElevenLabsApiKeyWithProfileFallbackMock.mockReturnValue(null); vi.stubEnv("ELEVENLABS_API_KEY", ""); diff --git a/extensions/elevenlabs/realtime-transcription-provider.ts b/extensions/elevenlabs/realtime-transcription-provider.ts index 49b9ff1b5e66..ef14ac05ecce 100644 --- a/extensions/elevenlabs/realtime-transcription-provider.ts +++ b/extensions/elevenlabs/realtime-transcription-provider.ts @@ -169,15 +169,7 @@ function readErrorDetail(event: ElevenLabsRealtimeTranscriptionEvent): string { function createElevenLabsRealtimeTranscriptionSession( config: ElevenLabsRealtimeTranscriptionSessionConfig, ): RealtimeTranscriptionSession { - let lastTranscript: string | undefined; - - const emitTranscript = (text: string) => { - if (text === lastTranscript) { - return; - } - lastTranscript = text; - config.onTranscript?.(text); - }; + let pendingTimestampEcho: string | undefined; const sendAudioChunk = ( audio: Buffer, @@ -196,6 +188,7 @@ function createElevenLabsRealtimeTranscriptionSession( transport: RealtimeTranscriptionWebSocketTransport, ) => { if (event.message_type === "session_started") { + pendingTimestampEcho = undefined; transport.markReady(); return; } @@ -213,7 +206,13 @@ function createElevenLabsRealtimeTranscriptionSession( case "committed_transcript": case "committed_transcript_with_timestamps": if (event.text) { - emitTranscript(event.text); + // A committed segment can have one matching timestamp companion, never another turn. + const hasTimestamps = event.message_type !== "committed_transcript"; + const isEcho = hasTimestamps && pendingTimestampEcho === event.text; + pendingTimestampEcho = hasTimestamps ? undefined : event.text; + if (!isEcho) { + config.onTranscript?.(event.text); + } } return; default: