fix(ui): gate realtime output generation by Gateway capability

Punchcard-Session: golden-meadow-cedar-dv
This commit is contained in:
Vincent Koc
2026-08-11 23:49:16 +08:00
parent 2b9b840513
commit 021b345340
6 changed files with 74 additions and 6 deletions
+18 -1
View File
@@ -13,7 +13,11 @@ import { RealtimeTalkSession } from "./realtime-talk.ts";
type InspectableRealtimeTalkSession = {
callbacks: RealtimeTalkCallbacks;
options: { provider?: string; transport?: string };
localOptions: { inputDeviceId?: string; videoDeviceId?: string };
localOptions: {
inputDeviceId?: string;
videoDeviceId?: string;
supportsOutputGeneration?: boolean;
};
};
function inspectSession(state: ChatRealtimeState): InspectableRealtimeTalkSession {
@@ -28,6 +32,7 @@ function createState(): ChatRealtimeState {
const state = {
client: {},
connected: true,
hello: null,
settings: loadSettings(),
sessionKey: "main",
lastError: null,
@@ -73,6 +78,18 @@ describe("chat realtime actions", () => {
expect(startSpy).toHaveBeenCalledOnce();
});
it("snapshots output-generation support when Talk starts", async () => {
const state = createState();
state.hello = {
features: { capabilities: ["talk-output-generation"] },
} as ChatRealtimeState["hello"];
await state.toggleRealtimeTalk();
state.hello = null;
expect(inspectSession(state).localOptions.supportsOutputGeneration).toBe(true);
});
it("enables camera only after a video-capable voice session starts", async () => {
const state = createState();
const setVideoEnabled = vi
+6 -2
View File
@@ -1,5 +1,6 @@
import type { GatewayBrowserClient } from "../../api/gateway.ts";
import type { GatewayBrowserClient, GatewayHelloOk } from "../../api/gateway.ts";
import { loadSettings, patchSettings, type UiSettings } from "../../app/settings.ts";
import { isGatewayCapabilityAdvertised } from "../../lib/gateway-methods.ts";
import {
createRealtimeTalkConversationState,
updateRealtimeTalkConversation,
@@ -16,6 +17,7 @@ import { RealtimeTalkSession, type RealtimeTalkStatus } from "./realtime-talk.ts
export type ChatRealtimeState = {
client: GatewayBrowserClient | null;
connected: boolean;
hello: GatewayHelloOk | null;
settings: UiSettings;
sessionKey: string;
lastError?: string | null;
@@ -166,6 +168,8 @@ export function attachChatRealtimeActions(state: ChatRealtimeState) {
const inputDeviceId = talkSettings.realtimeTalkInputDeviceId?.trim() || undefined;
const videoDeviceId = talkSettings.realtimeTalkVideoDeviceId?.trim() || undefined;
const autoEnableCamera = talkSettings.talkCameraAutoEnable === true;
const supportsOutputGeneration =
isGatewayCapabilityAdvertised(state, "talk-output-generation") === true;
let autoEnableCameraAttempted = false;
state.realtimeTalkActive = true;
state.realtimeTalkStatus = "connecting";
@@ -252,7 +256,7 @@ export function attachChatRealtimeActions(state: ChatRealtimeState) {
},
},
{},
{ inputDeviceId, videoDeviceId },
{ inputDeviceId, videoDeviceId, supportsOutputGeneration },
);
state.realtimeTalkSession = session;
try {
@@ -126,6 +126,7 @@ function createTransport(overrides: Partial<RealtimeTalkTransportContext> = {})
callbacks: {},
client: createClient(),
sessionKey: "main",
supportsOutputGeneration: true,
...overrides,
});
}
@@ -550,9 +551,9 @@ describe("GatewayRelayRealtimeTalkTransport", () => {
transport.stop();
});
it("cancels provider output when the first audio chunk exceeds the time budget", async () => {
it("includes output generation in cancellation when the Gateway advertises support", async () => {
const client = createClient();
const transport = createTransport({ client });
const transport = createTransport({ client, supportsOutputGeneration: true });
await startTransport(transport);
emitTalkEvent({
@@ -592,6 +593,47 @@ describe("GatewayRelayRealtimeTalkTransport", () => {
transport.stop();
});
it("preserves turn identity but omits output generation for a legacy Gateway", async () => {
const client = createClient();
const transport = createTransport({ client, supportsOutputGeneration: false });
await startTransport(transport);
emitTalkEvent({
relaySessionId: "relay-1",
type: "audio",
audioBase64: zeroPcmBase64(24000 * 11),
outputGeneration: 7,
talkEvent: {
id: "relay-1:2",
type: "output.audio.delta",
sessionId: "relay-1",
turnId: "turn-authoritative",
seq: 2,
timestamp: "2026-08-05T00:00:00.000Z",
mode: "realtime",
transport: "gateway-relay",
brain: "agent-consult",
payload: { byteLength: 48_000 },
} satisfies RealtimeTalkEvent,
});
await waitForFast(() =>
expect(requestCallsFor(client, "talk.session.cancelOutput")).toStrictEqual([
[
"talk.session.cancelOutput",
{
sessionId: "relay-1",
turnId: "turn-authoritative",
reason: "playback-overflow",
},
],
]),
);
expect(createdSources).toHaveLength(0);
transport.stop();
});
it("reopens an overflowed generation after its matching clear", async () => {
const transport = createTransport({ client: createClient() });
@@ -709,7 +709,9 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport
.request("talk.session.cancelOutput", {
sessionId: this.session.relaySessionId,
...(turnId ? { turnId } : {}),
...(outputGeneration ? { outputGeneration } : {}),
...(this.ctx.supportsOutputGeneration === true && outputGeneration !== undefined
? { outputGeneration }
: {}),
reason,
})
.then(
@@ -122,6 +122,7 @@ export type RealtimeTalkTransportContext = {
callbacks: RealtimeTalkCallbacks;
inputDeviceId?: string;
videoDeviceId?: string;
supportsOutputGeneration?: boolean;
consultThinkingLevel?: string;
consultFastMode?: boolean;
};
+2
View File
@@ -45,6 +45,7 @@ type RealtimeTalkLaunchOptions = {
type RealtimeTalkLocalOptions = {
inputDeviceId?: string;
videoDeviceId?: string;
supportsOutputGeneration?: boolean;
};
const activeRealtimeTalkSessions = new Set<RealtimeTalkSession>();
@@ -243,6 +244,7 @@ export class RealtimeTalkSession {
callbacks,
inputDeviceId: this.localOptions.inputDeviceId,
videoDeviceId: this.localOptions.videoDeviceId,
supportsOutputGeneration: this.localOptions.supportsOutputGeneration === true,
consultThinkingLevel: session.consultThinkingLevel,
consultFastMode: session.consultFastMode,
});