From 3030100e92d175499bec3b149b659c2fd039460c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 2 Aug 2026 08:24:39 +0800 Subject: [PATCH 1/8] fix(talk): bound WebRTC tool-call lifecycle --- ui/src/pages/chat/realtime-talk-webrtc.ts | 115 ++++++++++++++++------ 1 file changed, 83 insertions(+), 32 deletions(-) diff --git a/ui/src/pages/chat/realtime-talk-webrtc.ts b/ui/src/pages/chat/realtime-talk-webrtc.ts index e1f6753ff173..248d5b24fd22 100644 --- a/ui/src/pages/chat/realtime-talk-webrtc.ts +++ b/ui/src/pages/chat/realtime-talk-webrtc.ts @@ -1,4 +1,5 @@ // Control UI chat module implements realtime talk webrtc behavior. +import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME } from "../../../../src/talk/describe-view-tool.js"; import { RealtimeTalkMediaStreamMeter } from "./realtime-talk-audio.ts"; import { RealtimeTalkCameraController } from "./realtime-talk-camera-controller.ts"; @@ -24,12 +25,18 @@ import { type RealtimeServerEvent, } from "./realtime-talk-webrtc-support.ts"; -type ToolBuffer = { +type CompletedToolCall = { + itemId: string; name: string; callId: string; args: string; }; +const MAX_REALTIME_TOOL_ARGUMENT_BYTES = 256_000; +// Realtime defines no replay window, so evicting terminal IDs could execute a +// very late duplicate. End an extreme session instead of weakening dedupe. +const MAX_COMPLETED_TOOL_CALL_IDS = 1_024; +const utf8Encoder = new TextEncoder(); const cancelledSetup = Symbol("cancelledSetup"); export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { @@ -42,7 +49,7 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { private responseActive = false; private responseCreateInFlight = false; private responseCreatePending = false; - private toolBuffers = new Map(); + private readonly completedToolCallIds = new Set(); private readonly offerExchange = new RealtimeTalkWebRtcOfferExchange(); private mediaSetupController: AbortController | null = null; private readonly camera: RealtimeTalkCameraController; @@ -267,7 +274,7 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { controller.abort(); } this.consultAbortControllers.clear(); - this.toolBuffers.clear(); + this.completedToolCallIds.clear(); this.responseActive = false; this.responseCreateInFlight = false; this.responseCreatePending = false; @@ -370,12 +377,9 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { this.emitAssistantTranscript(event, true); return; case "response.function_call_arguments.delta": - this.bufferToolDelta(event); - return; case "response.function_call_arguments.done": - void this.handleToolCall(event).catch((error: unknown) => { - this.reportToolResultSubmissionError(error); - }); + // Tool argument events are provisional and can also arrive for interrupted + // responses. Only the completed response owns executable calls. return; case "input_audio_buffer.speech_started": this.ctx.callbacks.onStatus?.("listening", "Speech detected"); @@ -392,6 +396,12 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { return; case "response.cancelled": case "response.done": + if (event.type === "response.done") { + this.handleCompletedResponse(event); + if (this.closed) { + return; + } + } this.responseActive = false; this.responseCreateInFlight = false; this.ctx.callbacks.onStatus?.("listening", this.extractResponseStatus(event)); @@ -485,41 +495,79 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { return message || code || type || "Realtime provider error"; } - private bufferToolDelta(event: RealtimeServerEvent): void { - const key = event.item_id ?? "unknown"; - const existing = this.toolBuffers.get(key); - if (existing) { - existing.args += event.delta ?? ""; + private handleCompletedResponse(event: RealtimeServerEvent): void { + const response = event.response; + if ( + !isRecord(response) || + response.status !== "completed" || + typeof response.id !== "string" || + !response.id.trim() || + !Array.isArray(response.output) + ) { return; } - this.toolBuffers.set(key, { - name: event.name ?? "", - callId: event.call_id ?? "", - args: event.delta ?? "", - }); + for (const output of response.output) { + if ( + !isRecord(output) || + output.type !== "function_call" || + (output.status !== undefined && output.status !== "completed") + ) { + continue; + } + const itemId = typeof output.id === "string" ? output.id.trim() : ""; + const callId = typeof output.call_id === "string" ? output.call_id.trim() : ""; + const name = typeof output.name === "string" ? output.name.trim() : ""; + const args = typeof output.arguments === "string" ? output.arguments : ""; + if (!itemId || !callId || !name || !args.trim()) { + continue; + } + if ( + name !== REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME && + name !== REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME && + name !== REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME + ) { + continue; + } + if (this.completedToolCallIds.has(callId)) { + continue; + } + if (this.completedToolCallIds.size >= MAX_COMPLETED_TOOL_CALL_IDS) { + this.failConnection("Realtime tool-call session limit exceeded"); + return; + } + this.completedToolCallIds.add(callId); + if (utf8Encoder.encode(args).byteLength > MAX_REALTIME_TOOL_ARGUMENT_BYTES) { + const message = "Realtime tool arguments exceed the 256000-byte UTF-8 limit"; + this.submitToolResult(callId, { error: message }); + this.emitTalkEvent({ + type: "tool.error", + callId, + itemId, + final: true, + payload: { name, message }, + }); + continue; + } + void this.handleToolCall({ itemId, callId, name, args }).catch((error: unknown) => { + this.reportToolResultSubmissionError(error); + }); + } } - private async handleToolCall(event: RealtimeServerEvent): Promise { - const key = event.item_id ?? "unknown"; - const buffered = this.toolBuffers.get(key); - this.toolBuffers.delete(key); - const name = buffered?.name || event.name || ""; - const callId = buffered?.callId || event.call_id || ""; - if (!callId) { - return; - } + private async handleToolCall(call: CompletedToolCall): Promise { + const { itemId, callId, name, args } = call; if (name === REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME) { await submitRealtimeTalkAgentControl({ ctx: this.ctx, callId, - args: buffered?.args || event.arguments || "{}", + args, emitTalkEvent: this.emitTalkEvent, submit: (toolCallId, result) => this.submitToolResult(toolCallId, result), }); return; } if (name === REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME) { - await this.handleDescribeViewToolCall(callId, key); + await this.handleDescribeViewToolCall(callId, itemId); return; } if (name !== REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) { @@ -528,8 +576,8 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { this.emitTalkEvent({ type: "tool.call", callId, - itemId: key, - payload: { name, args: buffered?.args || event.arguments || "{}" }, + itemId, + payload: { name, args }, }); const abortController = new AbortController(); this.consultAbortControllers.add(abortController); @@ -537,7 +585,7 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { await submitRealtimeTalkConsult({ ctx: this.ctx, callId, - args: buffered?.args || event.arguments || "{}", + args, signal: abortController.signal, emitTalkEvent: this.emitTalkEvent, submit: (toolCallId, result) => this.submitToolResult(toolCallId, result), @@ -594,6 +642,9 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { } private submitToolResult(callId: string, result: unknown): void { + if (this.closed) { + return; + } this.send({ type: "conversation.item.create", item: { From ddb2a198db9a1c0e4de0c026f66bb04856ed6314 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 2 Aug 2026 08:24:41 +0800 Subject: [PATCH 2/8] test(talk): cover WebRTC terminal tool calls --- .../pages/chat/realtime-talk-webrtc.test.ts | 281 ++++++++++++++++-- 1 file changed, 251 insertions(+), 30 deletions(-) diff --git a/ui/src/pages/chat/realtime-talk-webrtc.test.ts b/ui/src/pages/chat/realtime-talk-webrtc.test.ts index 8a662e211f2d..d45762142a74 100644 --- a/ui/src/pages/chat/realtime-talk-webrtc.test.ts +++ b/ui/src/pages/chat/realtime-talk-webrtc.test.ts @@ -1,7 +1,11 @@ // @vitest-environment jsdom import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME } from "../../../../src/talk/describe-view-tool.js"; import { waitForFast } from "../../test-helpers/wait-for.ts"; -import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME } from "./realtime-talk-shared.ts"; +import { + REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, +} from "./realtime-talk-shared.ts"; import { WebRtcSdpRealtimeTalkTransport } from "./realtime-talk-webrtc.ts"; let getUserMedia: ReturnType; @@ -107,15 +111,45 @@ function dispatchRealtimeEvent(peer: FakePeerConnection | undefined, event: unkn } function dispatchConsultToolCall(peer: FakePeerConnection | undefined): void { - dispatchRealtimeEvent(peer, { - type: "response.function_call_arguments.done", - item_id: "item-1", - call_id: "call-1", + dispatchCompletedToolCall(peer, { name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, arguments: JSON.stringify({ question: "status?" }), }); } +function dispatchCompletedToolCall( + peer: FakePeerConnection | undefined, + overrides: { + responseId?: string | null; + responseStatus?: string | null; + itemId?: string | null; + itemStatus?: string | null; + callId?: string | null; + name?: string | null; + arguments?: string | null; + } = {}, +): void { + const field = (value: string | null | undefined, fallback: string): string | undefined => + value === undefined ? fallback : (value ?? undefined); + dispatchRealtimeEvent(peer, { + type: "response.done", + response: { + id: field(overrides.responseId, "response-1"), + status: field(overrides.responseStatus, "completed"), + output: [ + { + type: "function_call", + id: field(overrides.itemId, "item-1"), + status: field(overrides.itemStatus, "completed"), + call_id: field(overrides.callId, "call-1"), + name: field(overrides.name, REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME), + arguments: field(overrides.arguments, JSON.stringify({ question: "status?" })), + }, + ], + }, + }); +} + function dispatchTranscription(peer: FakePeerConnection | undefined, transcript: string): void { dispatchRealtimeEvent(peer, { type: "conversation.item.input_audio_transcription.completed", @@ -135,13 +169,13 @@ async function startActiveConsult( await transport.start(); const peer = FakePeerConnection.instances[0]; - if (options.responseAlreadyActive) { - dispatchRealtimeEvent(peer, { type: "response.created" }); - } dispatchConsultToolCall(peer); await waitForFast(() => expect(request).toHaveBeenCalledWith("talk.client.toolCall", expect.any(Object)), ); + if (options.responseAlreadyActive) { + dispatchRealtimeEvent(peer, { type: "response.created" }); + } return { transport, peer }; } @@ -791,6 +825,213 @@ describe("WebRtcSdpRealtimeTalkTransport", () => { }, ); + it("executes authoritative completed response tool calls and ignores provisional or duplicate events", async () => { + stubAnswerSdpFetch(); + const request = vi.fn(async (method: string) => { + if (method === "talk.client.toolCall") { + return { runId: "run-1" }; + } + throw new Error(`unexpected request: ${method}`); + }); + const transport = createOpenAiTransport({ + addEventListener: vi.fn(() => () => undefined), + request, + }); + + await transport.start(); + const peer = FakePeerConnection.instances[0]; + dispatchRealtimeEvent(peer, { + type: "response.function_call_arguments.delta", + item_id: "item-1", + call_id: "call-1", + delta: JSON.stringify({ question: "provisional" }), + }); + dispatchRealtimeEvent(peer, { + type: "response.function_call_arguments.done", + item_id: "item-1", + call_id: "call-1", + name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + arguments: JSON.stringify({ question: "provisional" }), + }); + expect(request).not.toHaveBeenCalled(); + + dispatchCompletedToolCall(peer); + await waitForFast(() => + expect(request).toHaveBeenCalledWith("talk.client.toolCall", { + sessionKey: "main", + callId: "call-1", + name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + args: { question: "status?" }, + }), + ); + + dispatchCompletedToolCall(peer); + dispatchRealtimeEvent(peer, { + type: "response.function_call_arguments.delta", + item_id: "item-1", + call_id: "call-1", + delta: JSON.stringify({ question: "late" }), + }); + dispatchRealtimeEvent(peer, { + type: "response.function_call_arguments.done", + item_id: "item-1", + call_id: "call-1", + name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + arguments: JSON.stringify({ question: "late" }), + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(request).toHaveBeenCalledTimes(1); + transport.stop(); + }); + + it.each([ + { label: "cancelled response", responseStatus: "cancelled", itemStatus: "completed" }, + { label: "failed response", responseStatus: "failed", itemStatus: "completed" }, + { label: "incomplete response", responseStatus: "incomplete", itemStatus: "completed" }, + { label: "incomplete item", responseStatus: "completed", itemStatus: "incomplete" }, + ])("ignores function calls from a $label", async ({ responseStatus, itemStatus }) => { + stubAnswerSdpFetch(); + const request = vi.fn(); + const transport = createOpenAiTransport({ request }); + + await transport.start(); + dispatchCompletedToolCall(FakePeerConnection.instances[0], { + responseStatus, + itemStatus, + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(request).not.toHaveBeenCalled(); + transport.stop(); + }); + + it("requires response, item, call, name, and argument identities before executing tools", async () => { + stubAnswerSdpFetch(); + const request = vi.fn(); + const transport = createOpenAiTransport({ request }); + + await transport.start(); + const peer = FakePeerConnection.instances[0]; + for (const overrides of [ + { responseId: null, callId: "missing-response" }, + { itemId: null, callId: "missing-item" }, + { callId: null, itemId: "missing-call" }, + { name: null, callId: "missing-name", itemId: "missing-name" }, + { arguments: null, callId: "missing-args", itemId: "missing-args" }, + ]) { + dispatchCompletedToolCall(peer, overrides); + } + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(request).not.toHaveBeenCalled(); + transport.stop(); + }); + + it("accepts authoritative tool arguments at the 256000-byte UTF-8 limit", async () => { + stubAnswerSdpFetch(); + const request = vi.fn(async (method: string) => { + if (method === "talk.client.steer") { + return { ok: true, mode: "status" }; + } + throw new Error(`unexpected request: ${method}`); + }); + const transport = createOpenAiTransport({ request }); + const baseArgs = JSON.stringify({ text: "status" }); + const argumentsAtLimit = baseArgs + " ".repeat(256_000 - baseArgs.length); + expect(new TextEncoder().encode(argumentsAtLimit)).toHaveLength(256_000); + + await transport.start(); + dispatchCompletedToolCall(FakePeerConnection.instances[0], { + name: REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, + arguments: argumentsAtLimit, + }); + await waitForFast(() => + expect(request).toHaveBeenCalledWith("talk.client.steer", { + sessionKey: "main", + text: "status", + mode: "status", + }), + ); + + transport.stop(); + }); + + it("rejects oversized UTF-8 tool arguments once and returns a provider-visible error", async () => { + stubAnswerSdpFetch(); + const request = vi.fn(); + const onTalkEvent = vi.fn(); + const transport = createOpenAiTransport({ request }, { onTalkEvent }); + const oversizedArguments = JSON.stringify({ text: "é".repeat(128_000) }); + expect(new TextEncoder().encode(oversizedArguments).byteLength).toBeGreaterThan(256_000); + + await transport.start(); + const peer = FakePeerConnection.instances[0]; + dispatchCompletedToolCall(peer, { + name: REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, + arguments: oversizedArguments, + }); + dispatchCompletedToolCall(peer, { + name: REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, + arguments: oversizedArguments, + }); + + const outputs = sentRealtimeEvents(peer).filter( + (event) => + event.type === "conversation.item.create" && event.item?.type === "function_call_output", + ); + expect(outputs).toHaveLength(1); + expect(JSON.parse(String(outputs[0]?.item?.output))).toEqual({ + error: "Realtime tool arguments exceed the 256000-byte UTF-8 limit", + }); + expect(request).not.toHaveBeenCalled(); + expect(onTalkEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: "tool.error", + callId: "call-1", + itemId: "item-1", + final: true, + }), + ); + transport.stop(); + }); + + it("ends the session instead of evicting completed tool-call identities", async () => { + stubAnswerSdpFetch(); + const onStatus = vi.fn(); + const transport = createOpenAiTransport({}, { onStatus }); + + await transport.start(); + const peer = FakePeerConnection.instances[0]; + for (let index = 0; index < 1_024; index += 1) { + dispatchCompletedToolCall(peer, { + responseId: `response-${index}`, + itemId: `item-${index}`, + callId: `call-${index}`, + name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME, + arguments: "{}", + }); + } + dispatchCompletedToolCall(peer, { + responseId: "response-overflow", + itemId: "item-overflow", + callId: "call-overflow", + name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME, + arguments: "{}", + }); + + expect(onStatus).toHaveBeenCalledWith("error", "Realtime tool-call session limit exceeded"); + expect(peer?.channel.close).toHaveBeenCalledOnce(); + dispatchCompletedToolCall(peer, { + responseId: "response-late", + itemId: "item-late", + callId: "call-late", + name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME, + arguments: "{}", + }); + expect(peer?.channel.close).toHaveBeenCalledOnce(); + }); + it("aborts an in-flight OpenAI tool consult when the transport stops", async () => { vi.stubGlobal( "fetch", @@ -830,17 +1071,7 @@ describe("WebRtcSdpRealtimeTalkTransport", () => { await transport.start(); const peer = FakePeerConnection.instances[0]; - peer?.channel.dispatchEvent( - new MessageEvent("message", { - data: JSON.stringify({ - type: "response.function_call_arguments.done", - item_id: "item-1", - call_id: "call-1", - name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, - arguments: JSON.stringify({ question: "status?" }), - }), - }), - ); + dispatchConsultToolCall(peer); await waitForFast(() => expect(request).toHaveBeenCalledTimes(1)); expect(request).toHaveBeenCalledWith("talk.client.toolCall", { sessionKey: "main", @@ -1037,17 +1268,7 @@ describe("WebRtcSdpRealtimeTalkTransport", () => { await transport.start(); const peer = FakePeerConnection.instances[0]; - peer?.channel.dispatchEvent( - new MessageEvent("message", { - data: JSON.stringify({ - type: "response.function_call_arguments.done", - item_id: "item-1", - call_id: "call-1", - name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, - arguments: JSON.stringify({ question: "status?" }), - }), - }), - ); + dispatchConsultToolCall(peer); await waitForFast(() => expect(request).toHaveBeenCalledWith("talk.client.toolCall", expect.any(Object)), ); From b2d8f0d7bafbd0b4cbcb30da936e7efc3a0d26a7 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 2 Aug 2026 09:08:41 +0800 Subject: [PATCH 3/8] fix(talk): narrow completed response payload --- ui/src/pages/chat/realtime-talk-webrtc.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/pages/chat/realtime-talk-webrtc.ts b/ui/src/pages/chat/realtime-talk-webrtc.ts index 248d5b24fd22..11aae84ecc9e 100644 --- a/ui/src/pages/chat/realtime-talk-webrtc.ts +++ b/ui/src/pages/chat/realtime-talk-webrtc.ts @@ -496,7 +496,7 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { } private handleCompletedResponse(event: RealtimeServerEvent): void { - const response = event.response; + const response: unknown = event.response; if ( !isRecord(response) || response.status !== "completed" || From a648d58d76940ecb7167159f9d4f37e6cc7cdefe Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 2 Aug 2026 09:14:34 +0800 Subject: [PATCH 4/8] fix(talk): accept optional realtime item ids --- .../pages/chat/realtime-talk-webrtc.test.ts | 32 +++++++++++++++++-- ui/src/pages/chat/realtime-talk-webrtc.ts | 16 +++------- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/ui/src/pages/chat/realtime-talk-webrtc.test.ts b/ui/src/pages/chat/realtime-talk-webrtc.test.ts index d45762142a74..1c701016d589 100644 --- a/ui/src/pages/chat/realtime-talk-webrtc.test.ts +++ b/ui/src/pages/chat/realtime-talk-webrtc.test.ts @@ -906,7 +906,35 @@ describe("WebRtcSdpRealtimeTalkTransport", () => { transport.stop(); }); - it("requires response, item, call, name, and argument identities before executing tools", async () => { + it("accepts completed tool calls without optional response and item ids", async () => { + stubAnswerSdpFetch(); + const request = vi.fn(async (method: string) => { + if (method === "talk.client.steer") { + return { ok: true, mode: "status" }; + } + throw new Error(`unexpected request: ${method}`); + }); + const transport = createOpenAiTransport({ request }); + + await transport.start(); + dispatchCompletedToolCall(FakePeerConnection.instances[0], { + responseId: null, + itemId: null, + name: REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, + arguments: JSON.stringify({ text: "status" }), + }); + await waitForFast(() => + expect(request).toHaveBeenCalledWith("talk.client.steer", { + sessionKey: "main", + text: "status", + mode: "status", + }), + ); + + transport.stop(); + }); + + it("requires call, name, and arguments before executing tools", async () => { stubAnswerSdpFetch(); const request = vi.fn(); const transport = createOpenAiTransport({ request }); @@ -914,8 +942,6 @@ describe("WebRtcSdpRealtimeTalkTransport", () => { await transport.start(); const peer = FakePeerConnection.instances[0]; for (const overrides of [ - { responseId: null, callId: "missing-response" }, - { itemId: null, callId: "missing-item" }, { callId: null, itemId: "missing-call" }, { name: null, callId: "missing-name", itemId: "missing-name" }, { arguments: null, callId: "missing-args", itemId: "missing-args" }, diff --git a/ui/src/pages/chat/realtime-talk-webrtc.ts b/ui/src/pages/chat/realtime-talk-webrtc.ts index 11aae84ecc9e..c3ff65d3c750 100644 --- a/ui/src/pages/chat/realtime-talk-webrtc.ts +++ b/ui/src/pages/chat/realtime-talk-webrtc.ts @@ -26,7 +26,7 @@ import { } from "./realtime-talk-webrtc-support.ts"; type CompletedToolCall = { - itemId: string; + itemId?: string; name: string; callId: string; args: string; @@ -497,13 +497,7 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { private handleCompletedResponse(event: RealtimeServerEvent): void { const response: unknown = event.response; - if ( - !isRecord(response) || - response.status !== "completed" || - typeof response.id !== "string" || - !response.id.trim() || - !Array.isArray(response.output) - ) { + if (!isRecord(response) || response.status !== "completed" || !Array.isArray(response.output)) { return; } for (const output of response.output) { @@ -514,11 +508,11 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { ) { continue; } - const itemId = typeof output.id === "string" ? output.id.trim() : ""; + const itemId = typeof output.id === "string" ? output.id.trim() || undefined : undefined; const callId = typeof output.call_id === "string" ? output.call_id.trim() : ""; const name = typeof output.name === "string" ? output.name.trim() : ""; const args = typeof output.arguments === "string" ? output.arguments : ""; - if (!itemId || !callId || !name || !args.trim()) { + if (!callId || !name || !args.trim()) { continue; } if ( @@ -595,7 +589,7 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { } } - private async handleDescribeViewToolCall(callId: string, itemId: string): Promise { + private async handleDescribeViewToolCall(callId: string, itemId?: string): Promise { this.emitTalkEvent({ type: "tool.call", callId, From f82d2dde144ff257fb177e6b639169387ded23fe Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 2 Aug 2026 09:29:18 +0800 Subject: [PATCH 5/8] test(talk): align WebRTC tool-call fixtures --- .../chat/realtime-talk-webrtc-control.test.ts | 19 +++++-- .../chat/realtime-talk-webrtc-video.test.ts | 53 +++++++++++-------- 2 files changed, 45 insertions(+), 27 deletions(-) diff --git a/ui/src/pages/chat/realtime-talk-webrtc-control.test.ts b/ui/src/pages/chat/realtime-talk-webrtc-control.test.ts index b5b8df2b2f7b..f280b35fd6b2 100644 --- a/ui/src/pages/chat/realtime-talk-webrtc-control.test.ts +++ b/ui/src/pages/chat/realtime-talk-webrtc-control.test.ts @@ -72,11 +72,20 @@ function dispatchControlToolCall( peer?.channel.dispatchEvent( new MessageEvent("message", { data: JSON.stringify({ - type: "response.function_call_arguments.done", - item_id: "item-control", - call_id: "call-control", - name: REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, - arguments: JSON.stringify(args), + type: "response.done", + response: { + status: "completed", + output: [ + { + type: "function_call", + status: "completed", + id: "item-control", + call_id: "call-control", + name: REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, + arguments: JSON.stringify(args), + }, + ], + }, }), }), ); diff --git a/ui/src/pages/chat/realtime-talk-webrtc-video.test.ts b/ui/src/pages/chat/realtime-talk-webrtc-video.test.ts index a02a76dbafd7..091e810afe99 100644 --- a/ui/src/pages/chat/realtime-talk-webrtc-video.test.ts +++ b/ui/src/pages/chat/realtime-talk-webrtc-video.test.ts @@ -52,6 +52,32 @@ function sentRealtimeEvents(): Array> { ); } +function dispatchDescribeViewToolCall( + peer: FakePeerConnection | undefined, + ids: { itemId: string; callId: string }, +): void { + peer?.channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ + type: "response.done", + response: { + status: "completed", + output: [ + { + type: "function_call", + status: "completed", + id: ids.itemId, + call_id: ids.callId, + name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME, + arguments: "{}", + }, + ], + }, + }), + }), + ); +} + describe("OpenAI Realtime Video Talk", () => { beforeEach(() => { FakePeerConnection.instance = undefined; @@ -131,17 +157,7 @@ describe("OpenAI Realtime Video Talk", () => { await transport.setVideoEnabled(true); expect(onVideoStream).toHaveBeenCalledWith(camera); - peer?.channel.dispatchEvent( - new MessageEvent("message", { - data: JSON.stringify({ - type: "response.function_call_arguments.done", - item_id: "item-camera", - call_id: "call-camera", - name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME, - arguments: "{}", - }), - }), - ); + dispatchDescribeViewToolCall(peer, { itemId: "item-camera", callId: "call-camera" }); await Promise.resolve(); expect(sentRealtimeEvents()).not.toContainEqual( expect.objectContaining({ @@ -190,17 +206,10 @@ describe("OpenAI Realtime Video Talk", () => { expect(videoStop).toHaveBeenCalledOnce(); expect(audioStop).not.toHaveBeenCalled(); - peer?.channel.dispatchEvent( - new MessageEvent("message", { - data: JSON.stringify({ - type: "response.function_call_arguments.done", - item_id: "item-camera-off", - call_id: "call-camera-off", - name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME, - arguments: "{}", - }), - }), - ); + dispatchDescribeViewToolCall(peer, { + itemId: "item-camera-off", + callId: "call-camera-off", + }); await vi.waitFor(() => expect(sentRealtimeEvents()).toContainEqual({ type: "conversation.item.create", From 9d414464828c9e1152f4a5120d0f5a3d0b72b329 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 2 Aug 2026 09:43:58 +0800 Subject: [PATCH 6/8] test(talk): split WebRTC tool lifecycle coverage --- .../chat/realtime-talk-webrtc-control.test.ts | 267 ++++++++++++++++- .../pages/chat/realtime-talk-webrtc.test.ts | 273 +----------------- ui/src/pages/chat/realtime-talk-webrtc.ts | 4 +- 3 files changed, 270 insertions(+), 274 deletions(-) diff --git a/ui/src/pages/chat/realtime-talk-webrtc-control.test.ts b/ui/src/pages/chat/realtime-talk-webrtc-control.test.ts index f280b35fd6b2..a7cc69af1c60 100644 --- a/ui/src/pages/chat/realtime-talk-webrtc-control.test.ts +++ b/ui/src/pages/chat/realtime-talk-webrtc-control.test.ts @@ -1,7 +1,11 @@ // @vitest-environment jsdom import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME } from "../../../../src/talk/describe-view-tool.js"; import { waitForFast } from "../../test-helpers/wait-for.ts"; -import { REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME } from "./realtime-talk-shared.ts"; +import { + REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, +} from "./realtime-talk-shared.ts"; import { WebRtcSdpRealtimeTalkTransport } from "./realtime-talk-webrtc.ts"; class FakeDataChannel extends EventTarget { @@ -69,20 +73,41 @@ function dispatchControlToolCall( peer: FakePeerConnection | undefined, args: { text: string; mode: "status" | "steer" }, ): void { + dispatchCompletedToolCall(peer, { + name: REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, + arguments: JSON.stringify(args), + }); +} + +function dispatchCompletedToolCall( + peer: FakePeerConnection | undefined, + overrides: { + responseId?: string | null; + responseStatus?: string | null; + itemId?: string | null; + itemStatus?: string | null; + callId?: string | null; + name?: string | null; + arguments?: string | null; + } = {}, +): void { + const field = (value: string | null | undefined, fallback: string): string | undefined => + value === undefined ? fallback : (value ?? undefined); peer?.channel.dispatchEvent( new MessageEvent("message", { data: JSON.stringify({ type: "response.done", response: { - status: "completed", + id: field(overrides.responseId, "response-1"), + status: field(overrides.responseStatus, "completed"), output: [ { type: "function_call", - status: "completed", - id: "item-control", - call_id: "call-control", - name: REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, - arguments: JSON.stringify(args), + id: field(overrides.itemId, "item-control"), + status: field(overrides.itemStatus, "completed"), + call_id: field(overrides.callId, "call-control"), + name: field(overrides.name, REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME), + arguments: field(overrides.arguments, JSON.stringify({ text: "status" })), }, ], }, @@ -91,6 +116,14 @@ function dispatchControlToolCall( ); } +function sentRealtimeEvents(peer: FakePeerConnection | undefined): Array> { + return ( + peer?.channel.send.mock.calls.map( + ([payload]) => JSON.parse(String(payload)) as Record, + ) ?? [] + ); +} + describe("WebRtcSdpRealtimeTalkTransport control tool", () => { beforeEach(() => { FakePeerConnection.instances = []; @@ -162,6 +195,226 @@ describe("WebRtcSdpRealtimeTalkTransport control tool", () => { transport.stop(); }); + it("executes completed calls once and ignores provisional events", async () => { + const request = vi.fn(async (method: string) => { + if (method === "talk.client.toolCall") { + return { runId: "run-1" }; + } + throw new Error(`unexpected request: ${method}`); + }); + const transport = createOpenAiTransport({ + addEventListener: vi.fn(() => () => undefined), + request, + }); + + await transport.start(); + const peer = FakePeerConnection.instances[0]; + for (const type of [ + "response.function_call_arguments.delta", + "response.function_call_arguments.done", + ]) { + peer?.channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ + type, + item_id: "item-1", + call_id: "call-1", + name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + arguments: JSON.stringify({ question: "provisional" }), + delta: JSON.stringify({ question: "provisional" }), + }), + }), + ); + } + expect(request).not.toHaveBeenCalled(); + + dispatchCompletedToolCall(peer, { + itemId: "item-1", + callId: "call-1", + name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + arguments: JSON.stringify({ question: "status?" }), + }); + await waitForFast(() => + expect(request).toHaveBeenCalledWith("talk.client.toolCall", { + sessionKey: "main", + callId: "call-1", + name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + args: { question: "status?" }, + }), + ); + + dispatchCompletedToolCall(peer, { + responseId: "response-2", + itemId: "item-2", + callId: "call-1", + name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + arguments: JSON.stringify({ question: "late" }), + }); + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); + + expect(request).toHaveBeenCalledTimes(1); + transport.stop(); + }); + + it.each([ + { label: "cancelled response", responseStatus: "cancelled", itemStatus: "completed" }, + { label: "failed response", responseStatus: "failed", itemStatus: "completed" }, + { label: "incomplete response", responseStatus: "incomplete", itemStatus: "completed" }, + { label: "incomplete item", responseStatus: "completed", itemStatus: "incomplete" }, + ])("ignores function calls from a $label", async ({ responseStatus, itemStatus }) => { + const request = vi.fn(); + const transport = createOpenAiTransport({ request }); + + await transport.start(); + dispatchCompletedToolCall(FakePeerConnection.instances[0], { + responseStatus, + itemStatus, + }); + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); + + expect(request).not.toHaveBeenCalled(); + transport.stop(); + }); + + it("accepts completed calls without optional response and item ids", async () => { + const request = vi.fn(async (method: string) => { + if (method === "talk.client.steer") { + return { ok: true, mode: "status" }; + } + throw new Error(`unexpected request: ${method}`); + }); + const transport = createOpenAiTransport({ request }); + + await transport.start(); + dispatchCompletedToolCall(FakePeerConnection.instances[0], { + responseId: null, + itemId: null, + }); + await waitForFast(() => + expect(request).toHaveBeenCalledWith("talk.client.steer", { + sessionKey: "main", + text: "status", + mode: "status", + }), + ); + + transport.stop(); + }); + + it("requires call, name, and arguments before executing tools", async () => { + const request = vi.fn(); + const transport = createOpenAiTransport({ request }); + + await transport.start(); + const peer = FakePeerConnection.instances[0]; + for (const overrides of [ + { callId: null, itemId: "missing-call" }, + { name: null, callId: "missing-name", itemId: "missing-name" }, + { arguments: null, callId: "missing-args", itemId: "missing-args" }, + ]) { + dispatchCompletedToolCall(peer, overrides); + } + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); + + expect(request).not.toHaveBeenCalled(); + transport.stop(); + }); + + it("enforces the authoritative 256000-byte UTF-8 argument limit", async () => { + const request = vi.fn(async (method: string) => { + if (method === "talk.client.steer") { + return { ok: true, mode: "status" }; + } + throw new Error(`unexpected request: ${method}`); + }); + const onTalkEvent = vi.fn(); + const transport = createOpenAiTransport({ request }, { onTalkEvent }); + const baseArgs = JSON.stringify({ text: "status" }); + const argumentsAtLimit = baseArgs + " ".repeat(256_000 - baseArgs.length); + const oversizedArguments = JSON.stringify({ text: "é".repeat(128_000) }); + + await transport.start(); + const peer = FakePeerConnection.instances[0]; + dispatchCompletedToolCall(peer, { arguments: argumentsAtLimit }); + await waitForFast(() => expect(request).toHaveBeenCalledOnce()); + dispatchCompletedToolCall(peer, { + responseId: "response-2", + itemId: "item-2", + callId: "call-2", + arguments: oversizedArguments, + }); + dispatchCompletedToolCall(peer, { + responseId: "response-3", + itemId: "item-3", + callId: "call-2", + arguments: oversizedArguments, + }); + + expect(new TextEncoder().encode(argumentsAtLimit)).toHaveLength(256_000); + expect(new TextEncoder().encode(oversizedArguments).byteLength).toBeGreaterThan(256_000); + const outputs = sentRealtimeEvents(peer).filter( + (event) => + event.type === "conversation.item.create" && + (event.item as { type?: string } | undefined)?.type === "function_call_output", + ); + expect(outputs).toHaveLength(2); + expect( + JSON.parse(String((outputs[1]?.item as { output?: string } | undefined)?.output)), + ).toEqual({ + error: "Realtime tool arguments exceed the 256000-byte UTF-8 limit", + }); + expect(onTalkEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: "tool.error", + callId: "call-2", + itemId: "item-2", + final: true, + }), + ); + transport.stop(); + }); + + it("ends the session instead of evicting completed call identities", async () => { + const onStatus = vi.fn(); + const transport = createOpenAiTransport({}, { onStatus }); + + await transport.start(); + const peer = FakePeerConnection.instances[0]; + for (let index = 0; index < 1_024; index += 1) { + dispatchCompletedToolCall(peer, { + responseId: `response-${index}`, + itemId: `item-${index}`, + callId: `call-${index}`, + name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME, + arguments: "{}", + }); + } + dispatchCompletedToolCall(peer, { + responseId: "response-overflow", + itemId: "item-overflow", + callId: "call-overflow", + name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME, + arguments: "{}", + }); + + expect(onStatus).toHaveBeenCalledWith("error", "Realtime tool-call session limit exceeded"); + expect(peer?.channel.close).toHaveBeenCalledOnce(); + dispatchCompletedToolCall(peer, { + responseId: "response-late", + itemId: "item-late", + callId: "call-late", + name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME, + arguments: "{}", + }); + expect(peer?.channel.close).toHaveBeenCalledOnce(); + }); + it("surfaces OpenAI tool-result send failures without an unhandled rejection", async () => { const onStatus = vi.fn(); const onTalkEvent = vi.fn(); diff --git a/ui/src/pages/chat/realtime-talk-webrtc.test.ts b/ui/src/pages/chat/realtime-talk-webrtc.test.ts index 1c701016d589..8119cc05ba90 100644 --- a/ui/src/pages/chat/realtime-talk-webrtc.test.ts +++ b/ui/src/pages/chat/realtime-talk-webrtc.test.ts @@ -1,11 +1,7 @@ // @vitest-environment jsdom import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME } from "../../../../src/talk/describe-view-tool.js"; import { waitForFast } from "../../test-helpers/wait-for.ts"; -import { - REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, - REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, -} from "./realtime-talk-shared.ts"; +import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME } from "./realtime-talk-shared.ts"; import { WebRtcSdpRealtimeTalkTransport } from "./realtime-talk-webrtc.ts"; let getUserMedia: ReturnType; @@ -111,39 +107,19 @@ function dispatchRealtimeEvent(peer: FakePeerConnection | undefined, event: unkn } function dispatchConsultToolCall(peer: FakePeerConnection | undefined): void { - dispatchCompletedToolCall(peer, { - name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, - arguments: JSON.stringify({ question: "status?" }), - }); -} - -function dispatchCompletedToolCall( - peer: FakePeerConnection | undefined, - overrides: { - responseId?: string | null; - responseStatus?: string | null; - itemId?: string | null; - itemStatus?: string | null; - callId?: string | null; - name?: string | null; - arguments?: string | null; - } = {}, -): void { - const field = (value: string | null | undefined, fallback: string): string | undefined => - value === undefined ? fallback : (value ?? undefined); dispatchRealtimeEvent(peer, { type: "response.done", response: { - id: field(overrides.responseId, "response-1"), - status: field(overrides.responseStatus, "completed"), + id: "response-1", + status: "completed", output: [ { type: "function_call", - id: field(overrides.itemId, "item-1"), - status: field(overrides.itemStatus, "completed"), - call_id: field(overrides.callId, "call-1"), - name: field(overrides.name, REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME), - arguments: field(overrides.arguments, JSON.stringify({ question: "status?" })), + id: "item-1", + status: "completed", + call_id: "call-1", + name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + arguments: JSON.stringify({ question: "status?" }), }, ], }, @@ -825,239 +801,6 @@ describe("WebRtcSdpRealtimeTalkTransport", () => { }, ); - it("executes authoritative completed response tool calls and ignores provisional or duplicate events", async () => { - stubAnswerSdpFetch(); - const request = vi.fn(async (method: string) => { - if (method === "talk.client.toolCall") { - return { runId: "run-1" }; - } - throw new Error(`unexpected request: ${method}`); - }); - const transport = createOpenAiTransport({ - addEventListener: vi.fn(() => () => undefined), - request, - }); - - await transport.start(); - const peer = FakePeerConnection.instances[0]; - dispatchRealtimeEvent(peer, { - type: "response.function_call_arguments.delta", - item_id: "item-1", - call_id: "call-1", - delta: JSON.stringify({ question: "provisional" }), - }); - dispatchRealtimeEvent(peer, { - type: "response.function_call_arguments.done", - item_id: "item-1", - call_id: "call-1", - name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, - arguments: JSON.stringify({ question: "provisional" }), - }); - expect(request).not.toHaveBeenCalled(); - - dispatchCompletedToolCall(peer); - await waitForFast(() => - expect(request).toHaveBeenCalledWith("talk.client.toolCall", { - sessionKey: "main", - callId: "call-1", - name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, - args: { question: "status?" }, - }), - ); - - dispatchCompletedToolCall(peer); - dispatchRealtimeEvent(peer, { - type: "response.function_call_arguments.delta", - item_id: "item-1", - call_id: "call-1", - delta: JSON.stringify({ question: "late" }), - }); - dispatchRealtimeEvent(peer, { - type: "response.function_call_arguments.done", - item_id: "item-1", - call_id: "call-1", - name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, - arguments: JSON.stringify({ question: "late" }), - }); - await new Promise((resolve) => setTimeout(resolve, 0)); - - expect(request).toHaveBeenCalledTimes(1); - transport.stop(); - }); - - it.each([ - { label: "cancelled response", responseStatus: "cancelled", itemStatus: "completed" }, - { label: "failed response", responseStatus: "failed", itemStatus: "completed" }, - { label: "incomplete response", responseStatus: "incomplete", itemStatus: "completed" }, - { label: "incomplete item", responseStatus: "completed", itemStatus: "incomplete" }, - ])("ignores function calls from a $label", async ({ responseStatus, itemStatus }) => { - stubAnswerSdpFetch(); - const request = vi.fn(); - const transport = createOpenAiTransport({ request }); - - await transport.start(); - dispatchCompletedToolCall(FakePeerConnection.instances[0], { - responseStatus, - itemStatus, - }); - await new Promise((resolve) => setTimeout(resolve, 0)); - - expect(request).not.toHaveBeenCalled(); - transport.stop(); - }); - - it("accepts completed tool calls without optional response and item ids", async () => { - stubAnswerSdpFetch(); - const request = vi.fn(async (method: string) => { - if (method === "talk.client.steer") { - return { ok: true, mode: "status" }; - } - throw new Error(`unexpected request: ${method}`); - }); - const transport = createOpenAiTransport({ request }); - - await transport.start(); - dispatchCompletedToolCall(FakePeerConnection.instances[0], { - responseId: null, - itemId: null, - name: REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, - arguments: JSON.stringify({ text: "status" }), - }); - await waitForFast(() => - expect(request).toHaveBeenCalledWith("talk.client.steer", { - sessionKey: "main", - text: "status", - mode: "status", - }), - ); - - transport.stop(); - }); - - it("requires call, name, and arguments before executing tools", async () => { - stubAnswerSdpFetch(); - const request = vi.fn(); - const transport = createOpenAiTransport({ request }); - - await transport.start(); - const peer = FakePeerConnection.instances[0]; - for (const overrides of [ - { callId: null, itemId: "missing-call" }, - { name: null, callId: "missing-name", itemId: "missing-name" }, - { arguments: null, callId: "missing-args", itemId: "missing-args" }, - ]) { - dispatchCompletedToolCall(peer, overrides); - } - await new Promise((resolve) => setTimeout(resolve, 0)); - - expect(request).not.toHaveBeenCalled(); - transport.stop(); - }); - - it("accepts authoritative tool arguments at the 256000-byte UTF-8 limit", async () => { - stubAnswerSdpFetch(); - const request = vi.fn(async (method: string) => { - if (method === "talk.client.steer") { - return { ok: true, mode: "status" }; - } - throw new Error(`unexpected request: ${method}`); - }); - const transport = createOpenAiTransport({ request }); - const baseArgs = JSON.stringify({ text: "status" }); - const argumentsAtLimit = baseArgs + " ".repeat(256_000 - baseArgs.length); - expect(new TextEncoder().encode(argumentsAtLimit)).toHaveLength(256_000); - - await transport.start(); - dispatchCompletedToolCall(FakePeerConnection.instances[0], { - name: REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, - arguments: argumentsAtLimit, - }); - await waitForFast(() => - expect(request).toHaveBeenCalledWith("talk.client.steer", { - sessionKey: "main", - text: "status", - mode: "status", - }), - ); - - transport.stop(); - }); - - it("rejects oversized UTF-8 tool arguments once and returns a provider-visible error", async () => { - stubAnswerSdpFetch(); - const request = vi.fn(); - const onTalkEvent = vi.fn(); - const transport = createOpenAiTransport({ request }, { onTalkEvent }); - const oversizedArguments = JSON.stringify({ text: "é".repeat(128_000) }); - expect(new TextEncoder().encode(oversizedArguments).byteLength).toBeGreaterThan(256_000); - - await transport.start(); - const peer = FakePeerConnection.instances[0]; - dispatchCompletedToolCall(peer, { - name: REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, - arguments: oversizedArguments, - }); - dispatchCompletedToolCall(peer, { - name: REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, - arguments: oversizedArguments, - }); - - const outputs = sentRealtimeEvents(peer).filter( - (event) => - event.type === "conversation.item.create" && event.item?.type === "function_call_output", - ); - expect(outputs).toHaveLength(1); - expect(JSON.parse(String(outputs[0]?.item?.output))).toEqual({ - error: "Realtime tool arguments exceed the 256000-byte UTF-8 limit", - }); - expect(request).not.toHaveBeenCalled(); - expect(onTalkEvent).toHaveBeenCalledWith( - expect.objectContaining({ - type: "tool.error", - callId: "call-1", - itemId: "item-1", - final: true, - }), - ); - transport.stop(); - }); - - it("ends the session instead of evicting completed tool-call identities", async () => { - stubAnswerSdpFetch(); - const onStatus = vi.fn(); - const transport = createOpenAiTransport({}, { onStatus }); - - await transport.start(); - const peer = FakePeerConnection.instances[0]; - for (let index = 0; index < 1_024; index += 1) { - dispatchCompletedToolCall(peer, { - responseId: `response-${index}`, - itemId: `item-${index}`, - callId: `call-${index}`, - name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME, - arguments: "{}", - }); - } - dispatchCompletedToolCall(peer, { - responseId: "response-overflow", - itemId: "item-overflow", - callId: "call-overflow", - name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME, - arguments: "{}", - }); - - expect(onStatus).toHaveBeenCalledWith("error", "Realtime tool-call session limit exceeded"); - expect(peer?.channel.close).toHaveBeenCalledOnce(); - dispatchCompletedToolCall(peer, { - responseId: "response-late", - itemId: "item-late", - callId: "call-late", - name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME, - arguments: "{}", - }); - expect(peer?.channel.close).toHaveBeenCalledOnce(); - }); - it("aborts an in-flight OpenAI tool consult when the transport stops", async () => { vi.stubGlobal( "fetch", diff --git a/ui/src/pages/chat/realtime-talk-webrtc.ts b/ui/src/pages/chat/realtime-talk-webrtc.ts index c3ff65d3c750..66b948005bea 100644 --- a/ui/src/pages/chat/realtime-talk-webrtc.ts +++ b/ui/src/pages/chat/realtime-talk-webrtc.ts @@ -375,12 +375,12 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { case "response.audio_transcript.done": case "response.output_audio_transcript.done": this.emitAssistantTranscript(event, true); - return; + break; case "response.function_call_arguments.delta": case "response.function_call_arguments.done": // Tool argument events are provisional and can also arrive for interrupted // responses. Only the completed response owns executable calls. - return; + break; case "input_audio_buffer.speech_started": this.ctx.callbacks.onStatus?.("listening", "Speech detected"); this.emitTalkEvent({ type: "turn.started", payload: { source: event.type } }); From 9fc061124d3b97943fe4b71ef16f3f814daeea35 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 2 Aug 2026 09:46:25 +0800 Subject: [PATCH 7/8] docs(changelog): note bounded WebRTC tool calls --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d50996ecde7..8f04c0ae3edc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,7 @@ Docs: https://docs.openclaw.ai - **Control UI update reconciliation:** preserve an unresolved managed-update request across disconnects, accept the replacement Gateway version when it proves success, and otherwise show explicit recovery guidance instead of trusting an unrelated cached update result or failing silently. Fixes #116075. Thanks @shakkernerd. - **Control UI model readiness:** put AI setup first when no model is selectable, distinguish signed-in credentials from ready providers, and route accounts with no exposed models directly to provider recovery instead of leading with disabled default controls. - **Control UI Talk session isolation:** stop active realtime Talk media and retire its callbacks before chat session changes, Gateway disconnects, or pane disposal so previous-session audio, transcript, camera, and status updates cannot leak into the next view. Thanks @shakkernerd. +- **Control UI Realtime tool calls:** execute OpenAI WebRTC tools only from completed responses, bound retained call identities and UTF-8 arguments, and ignore provisional or late duplicate events so long Talk sessions cannot grow tool state without limit. - **Gateway reconnect event ordering:** reset the shared TypeScript client's outer event-sequence baseline for each replacement WebSocket, preventing gap recovery from comparing unrelated connection generations across Control UI, TUI, SDK, and browser extension clients. Thanks @shakkernerd. - **Skill Workshop offline apply:** preserve configless local proposal apply after upgrades under exclusive Gateway startup ownership, while keeping running Gateway snapshot invalidation fail-closed when CLI credentials are unavailable. - **macOS and Control UI keyboard navigation:** let Tab traverse links and controls inside embedded Dashboard, browser, and Canvas web views, and keep shortcuts working on non-Latin keyboard layouts without firing during IME composition. From 1ffcfdd168c05d33e764ac111cf9f49be1f7fa31 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 2 Aug 2026 09:56:39 +0800 Subject: [PATCH 8/8] test(talk): align browser E2E tool completion --- .../e2e/browser-talk-start-stop.e2e.test.ts | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/ui/src/e2e/browser-talk-start-stop.e2e.test.ts b/ui/src/e2e/browser-talk-start-stop.e2e.test.ts index 19ea6da0fe08..de086c64f74c 100644 --- a/ui/src/e2e/browser-talk-start-stop.e2e.test.ts +++ b/ui/src/e2e/browser-talk-start-stop.e2e.test.ts @@ -509,11 +509,21 @@ describeControlUiE2e("Control UI browser Talk", () => { channel?.dispatchEvent( new MessageEvent("message", { data: JSON.stringify({ - type: "response.function_call_arguments.done", - item_id: "item-camera", - call_id: "call-camera", - name: "describe_view", - arguments: "{}", + type: "response.done", + response: { + id: "response-camera", + status: "completed", + output: [ + { + type: "function_call", + id: "item-camera", + status: "completed", + call_id: "call-camera", + name: "describe_view", + arguments: "{}", + }, + ], + }, }), }), );