From 021b345340503f635fa5b05dfcb6d671e609781f Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 11 Aug 2026 23:49:16 +0800 Subject: [PATCH] fix(ui): gate realtime output generation by Gateway capability Punchcard-Session: golden-meadow-cedar-dv --- ui/src/pages/chat/chat-realtime.test.ts | 19 +++++++- ui/src/pages/chat/chat-realtime.ts | 8 +++- .../chat/realtime-talk-gateway-relay.test.ts | 46 ++++++++++++++++++- .../pages/chat/realtime-talk-gateway-relay.ts | 4 +- ui/src/pages/chat/realtime-talk-shared.ts | 1 + ui/src/pages/chat/realtime-talk.ts | 2 + 6 files changed, 74 insertions(+), 6 deletions(-) diff --git a/ui/src/pages/chat/chat-realtime.test.ts b/ui/src/pages/chat/chat-realtime.test.ts index 69abc479a185..64af86bc63b1 100644 --- a/ui/src/pages/chat/chat-realtime.test.ts +++ b/ui/src/pages/chat/chat-realtime.test.ts @@ -13,7 +13,11 @@ import { RealtimeTalkSession } from "./realtime-talk.ts"; type InspectableRealtimeTalkSession = { callbacks: RealtimeTalkCallbacks; options: { provider?: string; transport?: string }; - localOptions: { inputDeviceId?: string; videoDeviceId?: string }; + localOptions: { + inputDeviceId?: string; + videoDeviceId?: string; + supportsOutputGeneration?: boolean; + }; }; function inspectSession(state: ChatRealtimeState): InspectableRealtimeTalkSession { @@ -28,6 +32,7 @@ function createState(): ChatRealtimeState { const state = { client: {}, connected: true, + hello: null, settings: loadSettings(), sessionKey: "main", lastError: null, @@ -73,6 +78,18 @@ describe("chat realtime actions", () => { expect(startSpy).toHaveBeenCalledOnce(); }); + it("snapshots output-generation support when Talk starts", async () => { + const state = createState(); + state.hello = { + features: { capabilities: ["talk-output-generation"] }, + } as ChatRealtimeState["hello"]; + + await state.toggleRealtimeTalk(); + state.hello = null; + + expect(inspectSession(state).localOptions.supportsOutputGeneration).toBe(true); + }); + it("enables camera only after a video-capable voice session starts", async () => { const state = createState(); const setVideoEnabled = vi diff --git a/ui/src/pages/chat/chat-realtime.ts b/ui/src/pages/chat/chat-realtime.ts index 88bc9cebcdf2..514a1eb0b17b 100644 --- a/ui/src/pages/chat/chat-realtime.ts +++ b/ui/src/pages/chat/chat-realtime.ts @@ -1,5 +1,6 @@ -import type { GatewayBrowserClient } from "../../api/gateway.ts"; +import type { GatewayBrowserClient, GatewayHelloOk } from "../../api/gateway.ts"; import { loadSettings, patchSettings, type UiSettings } from "../../app/settings.ts"; +import { isGatewayCapabilityAdvertised } from "../../lib/gateway-methods.ts"; import { createRealtimeTalkConversationState, updateRealtimeTalkConversation, @@ -16,6 +17,7 @@ import { RealtimeTalkSession, type RealtimeTalkStatus } from "./realtime-talk.ts export type ChatRealtimeState = { client: GatewayBrowserClient | null; connected: boolean; + hello: GatewayHelloOk | null; settings: UiSettings; sessionKey: string; lastError?: string | null; @@ -166,6 +168,8 @@ export function attachChatRealtimeActions(state: ChatRealtimeState) { const inputDeviceId = talkSettings.realtimeTalkInputDeviceId?.trim() || undefined; const videoDeviceId = talkSettings.realtimeTalkVideoDeviceId?.trim() || undefined; const autoEnableCamera = talkSettings.talkCameraAutoEnable === true; + const supportsOutputGeneration = + isGatewayCapabilityAdvertised(state, "talk-output-generation") === true; let autoEnableCameraAttempted = false; state.realtimeTalkActive = true; state.realtimeTalkStatus = "connecting"; @@ -252,7 +256,7 @@ export function attachChatRealtimeActions(state: ChatRealtimeState) { }, }, {}, - { inputDeviceId, videoDeviceId }, + { inputDeviceId, videoDeviceId, supportsOutputGeneration }, ); state.realtimeTalkSession = session; try { diff --git a/ui/src/pages/chat/realtime-talk-gateway-relay.test.ts b/ui/src/pages/chat/realtime-talk-gateway-relay.test.ts index 481475dba23a..d37fcfd9e422 100644 --- a/ui/src/pages/chat/realtime-talk-gateway-relay.test.ts +++ b/ui/src/pages/chat/realtime-talk-gateway-relay.test.ts @@ -126,6 +126,7 @@ function createTransport(overrides: Partial = {}) callbacks: {}, client: createClient(), sessionKey: "main", + supportsOutputGeneration: true, ...overrides, }); } @@ -550,9 +551,9 @@ describe("GatewayRelayRealtimeTalkTransport", () => { transport.stop(); }); - it("cancels provider output when the first audio chunk exceeds the time budget", async () => { + it("includes output generation in cancellation when the Gateway advertises support", async () => { const client = createClient(); - const transport = createTransport({ client }); + const transport = createTransport({ client, supportsOutputGeneration: true }); await startTransport(transport); emitTalkEvent({ @@ -592,6 +593,47 @@ describe("GatewayRelayRealtimeTalkTransport", () => { transport.stop(); }); + it("preserves turn identity but omits output generation for a legacy Gateway", async () => { + const client = createClient(); + const transport = createTransport({ client, supportsOutputGeneration: false }); + + await startTransport(transport); + emitTalkEvent({ + relaySessionId: "relay-1", + type: "audio", + audioBase64: zeroPcmBase64(24000 * 11), + outputGeneration: 7, + talkEvent: { + id: "relay-1:2", + type: "output.audio.delta", + sessionId: "relay-1", + turnId: "turn-authoritative", + seq: 2, + timestamp: "2026-08-05T00:00:00.000Z", + mode: "realtime", + transport: "gateway-relay", + brain: "agent-consult", + payload: { byteLength: 48_000 }, + } satisfies RealtimeTalkEvent, + }); + + await waitForFast(() => + expect(requestCallsFor(client, "talk.session.cancelOutput")).toStrictEqual([ + [ + "talk.session.cancelOutput", + { + sessionId: "relay-1", + turnId: "turn-authoritative", + reason: "playback-overflow", + }, + ], + ]), + ); + expect(createdSources).toHaveLength(0); + + transport.stop(); + }); + it("reopens an overflowed generation after its matching clear", async () => { const transport = createTransport({ client: createClient() }); diff --git a/ui/src/pages/chat/realtime-talk-gateway-relay.ts b/ui/src/pages/chat/realtime-talk-gateway-relay.ts index efdfecf71259..c41a48aef8f6 100644 --- a/ui/src/pages/chat/realtime-talk-gateway-relay.ts +++ b/ui/src/pages/chat/realtime-talk-gateway-relay.ts @@ -709,7 +709,9 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport .request("talk.session.cancelOutput", { sessionId: this.session.relaySessionId, ...(turnId ? { turnId } : {}), - ...(outputGeneration ? { outputGeneration } : {}), + ...(this.ctx.supportsOutputGeneration === true && outputGeneration !== undefined + ? { outputGeneration } + : {}), reason, }) .then( diff --git a/ui/src/pages/chat/realtime-talk-shared.ts b/ui/src/pages/chat/realtime-talk-shared.ts index a0b543a03873..1afbe179d299 100644 --- a/ui/src/pages/chat/realtime-talk-shared.ts +++ b/ui/src/pages/chat/realtime-talk-shared.ts @@ -122,6 +122,7 @@ export type RealtimeTalkTransportContext = { callbacks: RealtimeTalkCallbacks; inputDeviceId?: string; videoDeviceId?: string; + supportsOutputGeneration?: boolean; consultThinkingLevel?: string; consultFastMode?: boolean; }; diff --git a/ui/src/pages/chat/realtime-talk.ts b/ui/src/pages/chat/realtime-talk.ts index 1e2bcc66bb62..4de305f9f354 100644 --- a/ui/src/pages/chat/realtime-talk.ts +++ b/ui/src/pages/chat/realtime-talk.ts @@ -45,6 +45,7 @@ type RealtimeTalkLaunchOptions = { type RealtimeTalkLocalOptions = { inputDeviceId?: string; videoDeviceId?: string; + supportsOutputGeneration?: boolean; }; const activeRealtimeTalkSessions = new Set(); @@ -243,6 +244,7 @@ export class RealtimeTalkSession { callbacks, inputDeviceId: this.localOptions.inputDeviceId, videoDeviceId: this.localOptions.videoDeviceId, + supportsOutputGeneration: this.localOptions.supportsOutputGeneration === true, consultThinkingLevel: session.consultThinkingLevel, consultFastMode: session.consultFastMode, });