mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-26 04:15:48 -06:00
fix(ui): gate realtime output generation by Gateway capability
Punchcard-Session: golden-meadow-cedar-dv
This commit is contained in:
@@ -13,7 +13,11 @@ import { RealtimeTalkSession } from "./realtime-talk.ts";
|
||||
type InspectableRealtimeTalkSession = {
|
||||
callbacks: RealtimeTalkCallbacks;
|
||||
options: { provider?: string; transport?: string };
|
||||
localOptions: { inputDeviceId?: string; videoDeviceId?: string };
|
||||
localOptions: {
|
||||
inputDeviceId?: string;
|
||||
videoDeviceId?: string;
|
||||
supportsOutputGeneration?: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
function inspectSession(state: ChatRealtimeState): InspectableRealtimeTalkSession {
|
||||
@@ -28,6 +32,7 @@ function createState(): ChatRealtimeState {
|
||||
const state = {
|
||||
client: {},
|
||||
connected: true,
|
||||
hello: null,
|
||||
settings: loadSettings(),
|
||||
sessionKey: "main",
|
||||
lastError: null,
|
||||
@@ -73,6 +78,18 @@ describe("chat realtime actions", () => {
|
||||
expect(startSpy).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("snapshots output-generation support when Talk starts", async () => {
|
||||
const state = createState();
|
||||
state.hello = {
|
||||
features: { capabilities: ["talk-output-generation"] },
|
||||
} as ChatRealtimeState["hello"];
|
||||
|
||||
await state.toggleRealtimeTalk();
|
||||
state.hello = null;
|
||||
|
||||
expect(inspectSession(state).localOptions.supportsOutputGeneration).toBe(true);
|
||||
});
|
||||
|
||||
it("enables camera only after a video-capable voice session starts", async () => {
|
||||
const state = createState();
|
||||
const setVideoEnabled = vi
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { GatewayBrowserClient } from "../../api/gateway.ts";
|
||||
import type { GatewayBrowserClient, GatewayHelloOk } from "../../api/gateway.ts";
|
||||
import { loadSettings, patchSettings, type UiSettings } from "../../app/settings.ts";
|
||||
import { isGatewayCapabilityAdvertised } from "../../lib/gateway-methods.ts";
|
||||
import {
|
||||
createRealtimeTalkConversationState,
|
||||
updateRealtimeTalkConversation,
|
||||
@@ -16,6 +17,7 @@ import { RealtimeTalkSession, type RealtimeTalkStatus } from "./realtime-talk.ts
|
||||
export type ChatRealtimeState = {
|
||||
client: GatewayBrowserClient | null;
|
||||
connected: boolean;
|
||||
hello: GatewayHelloOk | null;
|
||||
settings: UiSettings;
|
||||
sessionKey: string;
|
||||
lastError?: string | null;
|
||||
@@ -166,6 +168,8 @@ export function attachChatRealtimeActions(state: ChatRealtimeState) {
|
||||
const inputDeviceId = talkSettings.realtimeTalkInputDeviceId?.trim() || undefined;
|
||||
const videoDeviceId = talkSettings.realtimeTalkVideoDeviceId?.trim() || undefined;
|
||||
const autoEnableCamera = talkSettings.talkCameraAutoEnable === true;
|
||||
const supportsOutputGeneration =
|
||||
isGatewayCapabilityAdvertised(state, "talk-output-generation") === true;
|
||||
let autoEnableCameraAttempted = false;
|
||||
state.realtimeTalkActive = true;
|
||||
state.realtimeTalkStatus = "connecting";
|
||||
@@ -252,7 +256,7 @@ export function attachChatRealtimeActions(state: ChatRealtimeState) {
|
||||
},
|
||||
},
|
||||
{},
|
||||
{ inputDeviceId, videoDeviceId },
|
||||
{ inputDeviceId, videoDeviceId, supportsOutputGeneration },
|
||||
);
|
||||
state.realtimeTalkSession = session;
|
||||
try {
|
||||
|
||||
@@ -126,6 +126,7 @@ function createTransport(overrides: Partial<RealtimeTalkTransportContext> = {})
|
||||
callbacks: {},
|
||||
client: createClient(),
|
||||
sessionKey: "main",
|
||||
supportsOutputGeneration: true,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
@@ -550,9 +551,9 @@ describe("GatewayRelayRealtimeTalkTransport", () => {
|
||||
transport.stop();
|
||||
});
|
||||
|
||||
it("cancels provider output when the first audio chunk exceeds the time budget", async () => {
|
||||
it("includes output generation in cancellation when the Gateway advertises support", async () => {
|
||||
const client = createClient();
|
||||
const transport = createTransport({ client });
|
||||
const transport = createTransport({ client, supportsOutputGeneration: true });
|
||||
|
||||
await startTransport(transport);
|
||||
emitTalkEvent({
|
||||
@@ -592,6 +593,47 @@ describe("GatewayRelayRealtimeTalkTransport", () => {
|
||||
transport.stop();
|
||||
});
|
||||
|
||||
it("preserves turn identity but omits output generation for a legacy Gateway", async () => {
|
||||
const client = createClient();
|
||||
const transport = createTransport({ client, supportsOutputGeneration: false });
|
||||
|
||||
await startTransport(transport);
|
||||
emitTalkEvent({
|
||||
relaySessionId: "relay-1",
|
||||
type: "audio",
|
||||
audioBase64: zeroPcmBase64(24000 * 11),
|
||||
outputGeneration: 7,
|
||||
talkEvent: {
|
||||
id: "relay-1:2",
|
||||
type: "output.audio.delta",
|
||||
sessionId: "relay-1",
|
||||
turnId: "turn-authoritative",
|
||||
seq: 2,
|
||||
timestamp: "2026-08-05T00:00:00.000Z",
|
||||
mode: "realtime",
|
||||
transport: "gateway-relay",
|
||||
brain: "agent-consult",
|
||||
payload: { byteLength: 48_000 },
|
||||
} satisfies RealtimeTalkEvent,
|
||||
});
|
||||
|
||||
await waitForFast(() =>
|
||||
expect(requestCallsFor(client, "talk.session.cancelOutput")).toStrictEqual([
|
||||
[
|
||||
"talk.session.cancelOutput",
|
||||
{
|
||||
sessionId: "relay-1",
|
||||
turnId: "turn-authoritative",
|
||||
reason: "playback-overflow",
|
||||
},
|
||||
],
|
||||
]),
|
||||
);
|
||||
expect(createdSources).toHaveLength(0);
|
||||
|
||||
transport.stop();
|
||||
});
|
||||
|
||||
it("reopens an overflowed generation after its matching clear", async () => {
|
||||
const transport = createTransport({ client: createClient() });
|
||||
|
||||
|
||||
@@ -709,7 +709,9 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport
|
||||
.request("talk.session.cancelOutput", {
|
||||
sessionId: this.session.relaySessionId,
|
||||
...(turnId ? { turnId } : {}),
|
||||
...(outputGeneration ? { outputGeneration } : {}),
|
||||
...(this.ctx.supportsOutputGeneration === true && outputGeneration !== undefined
|
||||
? { outputGeneration }
|
||||
: {}),
|
||||
reason,
|
||||
})
|
||||
.then(
|
||||
|
||||
@@ -122,6 +122,7 @@ export type RealtimeTalkTransportContext = {
|
||||
callbacks: RealtimeTalkCallbacks;
|
||||
inputDeviceId?: string;
|
||||
videoDeviceId?: string;
|
||||
supportsOutputGeneration?: boolean;
|
||||
consultThinkingLevel?: string;
|
||||
consultFastMode?: boolean;
|
||||
};
|
||||
|
||||
@@ -45,6 +45,7 @@ type RealtimeTalkLaunchOptions = {
|
||||
type RealtimeTalkLocalOptions = {
|
||||
inputDeviceId?: string;
|
||||
videoDeviceId?: string;
|
||||
supportsOutputGeneration?: boolean;
|
||||
};
|
||||
|
||||
const activeRealtimeTalkSessions = new Set<RealtimeTalkSession>();
|
||||
@@ -243,6 +244,7 @@ export class RealtimeTalkSession {
|
||||
callbacks,
|
||||
inputDeviceId: this.localOptions.inputDeviceId,
|
||||
videoDeviceId: this.localOptions.videoDeviceId,
|
||||
supportsOutputGeneration: this.localOptions.supportsOutputGeneration === true,
|
||||
consultThinkingLevel: session.consultThinkingLevel,
|
||||
consultFastMode: session.consultFastMode,
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user