merge: bound WebRTC Talk tool lifecycle (#117732)

* commit '1ffcfdd168c05d33e764ac111cf9f49be1f7fa31':
  test(talk): align browser E2E tool completion
  docs(changelog): note bounded WebRTC tool calls
  test(talk): split WebRTC tool lifecycle coverage
  test(talk): align WebRTC tool-call fixtures
  fix(talk): accept optional realtime item ids
  fix(talk): narrow completed response payload
  test(talk): cover WebRTC terminal tool calls
  fix(talk): bound WebRTC tool-call lifecycle
This commit is contained in:
Vincent Koc
2026-08-02 10:16:26 +08:00
6 changed files with 415 additions and 98 deletions
+1
View File
@@ -74,6 +74,7 @@ Docs: https://docs.openclaw.ai
- **Control UI update reconciliation:** preserve an unresolved managed-update request across disconnects, accept the replacement Gateway version when it proves success, and otherwise show explicit recovery guidance instead of trusting an unrelated cached update result or failing silently. Fixes #116075. Thanks @shakkernerd.
- **Control UI model readiness:** put AI setup first when no model is selectable, distinguish signed-in credentials from ready providers, and route accounts with no exposed models directly to provider recovery instead of leading with disabled default controls.
- **Control UI Talk session isolation:** stop active realtime Talk media and retire its callbacks before chat session changes, Gateway disconnects, or pane disposal so previous-session audio, transcript, camera, and status updates cannot leak into the next view. Thanks @shakkernerd.
- **Control UI Realtime tool calls:** execute OpenAI WebRTC tools only from completed responses, bound retained call identities and UTF-8 arguments, and ignore provisional or late duplicate events so long Talk sessions cannot grow tool state without limit.
- **Gateway reconnect event ordering:** reset the shared TypeScript client's outer event-sequence baseline for each replacement WebSocket, preventing gap recovery from comparing unrelated connection generations across Control UI, TUI, SDK, and browser extension clients. Thanks @shakkernerd.
- **Skill Workshop offline apply:** preserve configless local proposal apply after upgrades under exclusive Gateway startup ownership, while keeping running Gateway snapshot invalidation fail-closed when CLI credentials are unavailable.
- **macOS and Control UI keyboard navigation:** let Tab traverse links and controls inside embedded Dashboard, browser, and Canvas web views, and keep shortcuts working on non-Latin keyboard layouts without firing during IME composition.
+15 -5
View File
@@ -509,11 +509,21 @@ describeControlUiE2e("Control UI browser Talk", () => {
channel?.dispatchEvent(
new MessageEvent("message", {
data: JSON.stringify({
type: "response.function_call_arguments.done",
item_id: "item-camera",
call_id: "call-camera",
name: "describe_view",
arguments: "{}",
type: "response.done",
response: {
id: "response-camera",
status: "completed",
output: [
{
type: "function_call",
id: "item-camera",
status: "completed",
call_id: "call-camera",
name: "describe_view",
arguments: "{}",
},
],
},
}),
}),
);
@@ -1,7 +1,11 @@
// @vitest-environment jsdom
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME } from "../../../../src/talk/describe-view-tool.js";
import { waitForFast } from "../../test-helpers/wait-for.ts";
import { REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME } from "./realtime-talk-shared.ts";
import {
REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME,
} from "./realtime-talk-shared.ts";
import { WebRtcSdpRealtimeTalkTransport } from "./realtime-talk-webrtc.ts";
class FakeDataChannel extends EventTarget {
@@ -69,19 +73,57 @@ function dispatchControlToolCall(
peer: FakePeerConnection | undefined,
args: { text: string; mode: "status" | "steer" },
): void {
dispatchCompletedToolCall(peer, {
name: REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME,
arguments: JSON.stringify(args),
});
}
function dispatchCompletedToolCall(
peer: FakePeerConnection | undefined,
overrides: {
responseId?: string | null;
responseStatus?: string | null;
itemId?: string | null;
itemStatus?: string | null;
callId?: string | null;
name?: string | null;
arguments?: string | null;
} = {},
): void {
const field = (value: string | null | undefined, fallback: string): string | undefined =>
value === undefined ? fallback : (value ?? undefined);
peer?.channel.dispatchEvent(
new MessageEvent("message", {
data: JSON.stringify({
type: "response.function_call_arguments.done",
item_id: "item-control",
call_id: "call-control",
name: REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME,
arguments: JSON.stringify(args),
type: "response.done",
response: {
id: field(overrides.responseId, "response-1"),
status: field(overrides.responseStatus, "completed"),
output: [
{
type: "function_call",
id: field(overrides.itemId, "item-control"),
status: field(overrides.itemStatus, "completed"),
call_id: field(overrides.callId, "call-control"),
name: field(overrides.name, REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME),
arguments: field(overrides.arguments, JSON.stringify({ text: "status" })),
},
],
},
}),
}),
);
}
function sentRealtimeEvents(peer: FakePeerConnection | undefined): Array<Record<string, unknown>> {
return (
peer?.channel.send.mock.calls.map(
([payload]) => JSON.parse(String(payload)) as Record<string, unknown>,
) ?? []
);
}
describe("WebRtcSdpRealtimeTalkTransport control tool", () => {
beforeEach(() => {
FakePeerConnection.instances = [];
@@ -153,6 +195,226 @@ describe("WebRtcSdpRealtimeTalkTransport control tool", () => {
transport.stop();
});
it("executes completed calls once and ignores provisional events", async () => {
const request = vi.fn(async (method: string) => {
if (method === "talk.client.toolCall") {
return { runId: "run-1" };
}
throw new Error(`unexpected request: ${method}`);
});
const transport = createOpenAiTransport({
addEventListener: vi.fn(() => () => undefined),
request,
});
await transport.start();
const peer = FakePeerConnection.instances[0];
for (const type of [
"response.function_call_arguments.delta",
"response.function_call_arguments.done",
]) {
peer?.channel.dispatchEvent(
new MessageEvent("message", {
data: JSON.stringify({
type,
item_id: "item-1",
call_id: "call-1",
name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
arguments: JSON.stringify({ question: "provisional" }),
delta: JSON.stringify({ question: "provisional" }),
}),
}),
);
}
expect(request).not.toHaveBeenCalled();
dispatchCompletedToolCall(peer, {
itemId: "item-1",
callId: "call-1",
name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
arguments: JSON.stringify({ question: "status?" }),
});
await waitForFast(() =>
expect(request).toHaveBeenCalledWith("talk.client.toolCall", {
sessionKey: "main",
callId: "call-1",
name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
args: { question: "status?" },
}),
);
dispatchCompletedToolCall(peer, {
responseId: "response-2",
itemId: "item-2",
callId: "call-1",
name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
arguments: JSON.stringify({ question: "late" }),
});
await new Promise<void>((resolve) => {
setTimeout(resolve, 0);
});
expect(request).toHaveBeenCalledTimes(1);
transport.stop();
});
it.each([
{ label: "cancelled response", responseStatus: "cancelled", itemStatus: "completed" },
{ label: "failed response", responseStatus: "failed", itemStatus: "completed" },
{ label: "incomplete response", responseStatus: "incomplete", itemStatus: "completed" },
{ label: "incomplete item", responseStatus: "completed", itemStatus: "incomplete" },
])("ignores function calls from a $label", async ({ responseStatus, itemStatus }) => {
const request = vi.fn();
const transport = createOpenAiTransport({ request });
await transport.start();
dispatchCompletedToolCall(FakePeerConnection.instances[0], {
responseStatus,
itemStatus,
});
await new Promise<void>((resolve) => {
setTimeout(resolve, 0);
});
expect(request).not.toHaveBeenCalled();
transport.stop();
});
it("accepts completed calls without optional response and item ids", async () => {
const request = vi.fn(async (method: string) => {
if (method === "talk.client.steer") {
return { ok: true, mode: "status" };
}
throw new Error(`unexpected request: ${method}`);
});
const transport = createOpenAiTransport({ request });
await transport.start();
dispatchCompletedToolCall(FakePeerConnection.instances[0], {
responseId: null,
itemId: null,
});
await waitForFast(() =>
expect(request).toHaveBeenCalledWith("talk.client.steer", {
sessionKey: "main",
text: "status",
mode: "status",
}),
);
transport.stop();
});
it("requires call, name, and arguments before executing tools", async () => {
const request = vi.fn();
const transport = createOpenAiTransport({ request });
await transport.start();
const peer = FakePeerConnection.instances[0];
for (const overrides of [
{ callId: null, itemId: "missing-call" },
{ name: null, callId: "missing-name", itemId: "missing-name" },
{ arguments: null, callId: "missing-args", itemId: "missing-args" },
]) {
dispatchCompletedToolCall(peer, overrides);
}
await new Promise<void>((resolve) => {
setTimeout(resolve, 0);
});
expect(request).not.toHaveBeenCalled();
transport.stop();
});
it("enforces the authoritative 256000-byte UTF-8 argument limit", async () => {
const request = vi.fn(async (method: string) => {
if (method === "talk.client.steer") {
return { ok: true, mode: "status" };
}
throw new Error(`unexpected request: ${method}`);
});
const onTalkEvent = vi.fn();
const transport = createOpenAiTransport({ request }, { onTalkEvent });
const baseArgs = JSON.stringify({ text: "status" });
const argumentsAtLimit = baseArgs + " ".repeat(256_000 - baseArgs.length);
const oversizedArguments = JSON.stringify({ text: "é".repeat(128_000) });
await transport.start();
const peer = FakePeerConnection.instances[0];
dispatchCompletedToolCall(peer, { arguments: argumentsAtLimit });
await waitForFast(() => expect(request).toHaveBeenCalledOnce());
dispatchCompletedToolCall(peer, {
responseId: "response-2",
itemId: "item-2",
callId: "call-2",
arguments: oversizedArguments,
});
dispatchCompletedToolCall(peer, {
responseId: "response-3",
itemId: "item-3",
callId: "call-2",
arguments: oversizedArguments,
});
expect(new TextEncoder().encode(argumentsAtLimit)).toHaveLength(256_000);
expect(new TextEncoder().encode(oversizedArguments).byteLength).toBeGreaterThan(256_000);
const outputs = sentRealtimeEvents(peer).filter(
(event) =>
event.type === "conversation.item.create" &&
(event.item as { type?: string } | undefined)?.type === "function_call_output",
);
expect(outputs).toHaveLength(2);
expect(
JSON.parse(String((outputs[1]?.item as { output?: string } | undefined)?.output)),
).toEqual({
error: "Realtime tool arguments exceed the 256000-byte UTF-8 limit",
});
expect(onTalkEvent).toHaveBeenCalledWith(
expect.objectContaining({
type: "tool.error",
callId: "call-2",
itemId: "item-2",
final: true,
}),
);
transport.stop();
});
it("ends the session instead of evicting completed call identities", async () => {
const onStatus = vi.fn();
const transport = createOpenAiTransport({}, { onStatus });
await transport.start();
const peer = FakePeerConnection.instances[0];
for (let index = 0; index < 1_024; index += 1) {
dispatchCompletedToolCall(peer, {
responseId: `response-${index}`,
itemId: `item-${index}`,
callId: `call-${index}`,
name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME,
arguments: "{}",
});
}
dispatchCompletedToolCall(peer, {
responseId: "response-overflow",
itemId: "item-overflow",
callId: "call-overflow",
name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME,
arguments: "{}",
});
expect(onStatus).toHaveBeenCalledWith("error", "Realtime tool-call session limit exceeded");
expect(peer?.channel.close).toHaveBeenCalledOnce();
dispatchCompletedToolCall(peer, {
responseId: "response-late",
itemId: "item-late",
callId: "call-late",
name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME,
arguments: "{}",
});
expect(peer?.channel.close).toHaveBeenCalledOnce();
});
it("surfaces OpenAI tool-result send failures without an unhandled rejection", async () => {
const onStatus = vi.fn();
const onTalkEvent = vi.fn();
@@ -52,6 +52,32 @@ function sentRealtimeEvents(): Array<Record<string, unknown>> {
);
}
function dispatchDescribeViewToolCall(
peer: FakePeerConnection | undefined,
ids: { itemId: string; callId: string },
): void {
peer?.channel.dispatchEvent(
new MessageEvent("message", {
data: JSON.stringify({
type: "response.done",
response: {
status: "completed",
output: [
{
type: "function_call",
status: "completed",
id: ids.itemId,
call_id: ids.callId,
name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME,
arguments: "{}",
},
],
},
}),
}),
);
}
describe("OpenAI Realtime Video Talk", () => {
beforeEach(() => {
FakePeerConnection.instance = undefined;
@@ -131,17 +157,7 @@ describe("OpenAI Realtime Video Talk", () => {
await transport.setVideoEnabled(true);
expect(onVideoStream).toHaveBeenCalledWith(camera);
peer?.channel.dispatchEvent(
new MessageEvent("message", {
data: JSON.stringify({
type: "response.function_call_arguments.done",
item_id: "item-camera",
call_id: "call-camera",
name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME,
arguments: "{}",
}),
}),
);
dispatchDescribeViewToolCall(peer, { itemId: "item-camera", callId: "call-camera" });
await Promise.resolve();
expect(sentRealtimeEvents()).not.toContainEqual(
expect.objectContaining({
@@ -190,17 +206,10 @@ describe("OpenAI Realtime Video Talk", () => {
expect(videoStop).toHaveBeenCalledOnce();
expect(audioStop).not.toHaveBeenCalled();
peer?.channel.dispatchEvent(
new MessageEvent("message", {
data: JSON.stringify({
type: "response.function_call_arguments.done",
item_id: "item-camera-off",
call_id: "call-camera-off",
name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME,
arguments: "{}",
}),
}),
);
dispatchDescribeViewToolCall(peer, {
itemId: "item-camera-off",
callId: "call-camera-off",
});
await vi.waitFor(() =>
expect(sentRealtimeEvents()).toContainEqual({
type: "conversation.item.create",
+20 -30
View File
@@ -108,11 +108,21 @@ function dispatchRealtimeEvent(peer: FakePeerConnection | undefined, event: unkn
function dispatchConsultToolCall(peer: FakePeerConnection | undefined): void {
dispatchRealtimeEvent(peer, {
type: "response.function_call_arguments.done",
item_id: "item-1",
call_id: "call-1",
name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
arguments: JSON.stringify({ question: "status?" }),
type: "response.done",
response: {
id: "response-1",
status: "completed",
output: [
{
type: "function_call",
id: "item-1",
status: "completed",
call_id: "call-1",
name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
arguments: JSON.stringify({ question: "status?" }),
},
],
},
});
}
@@ -135,13 +145,13 @@ async function startActiveConsult(
await transport.start();
const peer = FakePeerConnection.instances[0];
if (options.responseAlreadyActive) {
dispatchRealtimeEvent(peer, { type: "response.created" });
}
dispatchConsultToolCall(peer);
await waitForFast(() =>
expect(request).toHaveBeenCalledWith("talk.client.toolCall", expect.any(Object)),
);
if (options.responseAlreadyActive) {
dispatchRealtimeEvent(peer, { type: "response.created" });
}
return { transport, peer };
}
@@ -830,17 +840,7 @@ describe("WebRtcSdpRealtimeTalkTransport", () => {
await transport.start();
const peer = FakePeerConnection.instances[0];
peer?.channel.dispatchEvent(
new MessageEvent("message", {
data: JSON.stringify({
type: "response.function_call_arguments.done",
item_id: "item-1",
call_id: "call-1",
name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
arguments: JSON.stringify({ question: "status?" }),
}),
}),
);
dispatchConsultToolCall(peer);
await waitForFast(() => expect(request).toHaveBeenCalledTimes(1));
expect(request).toHaveBeenCalledWith("talk.client.toolCall", {
sessionKey: "main",
@@ -1037,17 +1037,7 @@ describe("WebRtcSdpRealtimeTalkTransport", () => {
await transport.start();
const peer = FakePeerConnection.instances[0];
peer?.channel.dispatchEvent(
new MessageEvent("message", {
data: JSON.stringify({
type: "response.function_call_arguments.done",
item_id: "item-1",
call_id: "call-1",
name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
arguments: JSON.stringify({ question: "status?" }),
}),
}),
);
dispatchConsultToolCall(peer);
await waitForFast(() =>
expect(request).toHaveBeenCalledWith("talk.client.toolCall", expect.any(Object)),
);
+80 -35
View File
@@ -1,4 +1,5 @@
// Control UI chat module implements realtime talk webrtc behavior.
import { isRecord } from "@openclaw/normalization-core/record-coerce";
import { REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME } from "../../../../src/talk/describe-view-tool.js";
import { RealtimeTalkMediaStreamMeter } from "./realtime-talk-audio.ts";
import { RealtimeTalkCameraController } from "./realtime-talk-camera-controller.ts";
@@ -24,12 +25,18 @@ import {
type RealtimeServerEvent,
} from "./realtime-talk-webrtc-support.ts";
type ToolBuffer = {
type CompletedToolCall = {
itemId?: string;
name: string;
callId: string;
args: string;
};
const MAX_REALTIME_TOOL_ARGUMENT_BYTES = 256_000;
// Realtime defines no replay window, so evicting terminal IDs could execute a
// very late duplicate. End an extreme session instead of weakening dedupe.
const MAX_COMPLETED_TOOL_CALL_IDS = 1_024;
const utf8Encoder = new TextEncoder();
const cancelledSetup = Symbol("cancelledSetup");
export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport {
@@ -42,7 +49,7 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport {
private responseActive = false;
private responseCreateInFlight = false;
private responseCreatePending = false;
private toolBuffers = new Map<string, ToolBuffer>();
private readonly completedToolCallIds = new Set<string>();
private readonly offerExchange = new RealtimeTalkWebRtcOfferExchange();
private mediaSetupController: AbortController | null = null;
private readonly camera: RealtimeTalkCameraController;
@@ -267,7 +274,7 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport {
controller.abort();
}
this.consultAbortControllers.clear();
this.toolBuffers.clear();
this.completedToolCallIds.clear();
this.responseActive = false;
this.responseCreateInFlight = false;
this.responseCreatePending = false;
@@ -368,15 +375,12 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport {
case "response.audio_transcript.done":
case "response.output_audio_transcript.done":
this.emitAssistantTranscript(event, true);
return;
break;
case "response.function_call_arguments.delta":
this.bufferToolDelta(event);
return;
case "response.function_call_arguments.done":
void this.handleToolCall(event).catch((error: unknown) => {
this.reportToolResultSubmissionError(error);
});
return;
// Tool argument events are provisional and can also arrive for interrupted
// responses. Only the completed response owns executable calls.
break;
case "input_audio_buffer.speech_started":
this.ctx.callbacks.onStatus?.("listening", "Speech detected");
this.emitTalkEvent({ type: "turn.started", payload: { source: event.type } });
@@ -392,6 +396,12 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport {
return;
case "response.cancelled":
case "response.done":
if (event.type === "response.done") {
this.handleCompletedResponse(event);
if (this.closed) {
return;
}
}
this.responseActive = false;
this.responseCreateInFlight = false;
this.ctx.callbacks.onStatus?.("listening", this.extractResponseStatus(event));
@@ -485,41 +495,73 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport {
return message || code || type || "Realtime provider error";
}
private bufferToolDelta(event: RealtimeServerEvent): void {
const key = event.item_id ?? "unknown";
const existing = this.toolBuffers.get(key);
if (existing) {
existing.args += event.delta ?? "";
private handleCompletedResponse(event: RealtimeServerEvent): void {
const response: unknown = event.response;
if (!isRecord(response) || response.status !== "completed" || !Array.isArray(response.output)) {
return;
}
this.toolBuffers.set(key, {
name: event.name ?? "",
callId: event.call_id ?? "",
args: event.delta ?? "",
});
for (const output of response.output) {
if (
!isRecord(output) ||
output.type !== "function_call" ||
(output.status !== undefined && output.status !== "completed")
) {
continue;
}
const itemId = typeof output.id === "string" ? output.id.trim() || undefined : undefined;
const callId = typeof output.call_id === "string" ? output.call_id.trim() : "";
const name = typeof output.name === "string" ? output.name.trim() : "";
const args = typeof output.arguments === "string" ? output.arguments : "";
if (!callId || !name || !args.trim()) {
continue;
}
if (
name !== REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME &&
name !== REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME &&
name !== REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME
) {
continue;
}
if (this.completedToolCallIds.has(callId)) {
continue;
}
if (this.completedToolCallIds.size >= MAX_COMPLETED_TOOL_CALL_IDS) {
this.failConnection("Realtime tool-call session limit exceeded");
return;
}
this.completedToolCallIds.add(callId);
if (utf8Encoder.encode(args).byteLength > MAX_REALTIME_TOOL_ARGUMENT_BYTES) {
const message = "Realtime tool arguments exceed the 256000-byte UTF-8 limit";
this.submitToolResult(callId, { error: message });
this.emitTalkEvent({
type: "tool.error",
callId,
itemId,
final: true,
payload: { name, message },
});
continue;
}
void this.handleToolCall({ itemId, callId, name, args }).catch((error: unknown) => {
this.reportToolResultSubmissionError(error);
});
}
}
private async handleToolCall(event: RealtimeServerEvent): Promise<void> {
const key = event.item_id ?? "unknown";
const buffered = this.toolBuffers.get(key);
this.toolBuffers.delete(key);
const name = buffered?.name || event.name || "";
const callId = buffered?.callId || event.call_id || "";
if (!callId) {
return;
}
private async handleToolCall(call: CompletedToolCall): Promise<void> {
const { itemId, callId, name, args } = call;
if (name === REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME) {
await submitRealtimeTalkAgentControl({
ctx: this.ctx,
callId,
args: buffered?.args || event.arguments || "{}",
args,
emitTalkEvent: this.emitTalkEvent,
submit: (toolCallId, result) => this.submitToolResult(toolCallId, result),
});
return;
}
if (name === REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME) {
await this.handleDescribeViewToolCall(callId, key);
await this.handleDescribeViewToolCall(callId, itemId);
return;
}
if (name !== REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) {
@@ -528,8 +570,8 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport {
this.emitTalkEvent({
type: "tool.call",
callId,
itemId: key,
payload: { name, args: buffered?.args || event.arguments || "{}" },
itemId,
payload: { name, args },
});
const abortController = new AbortController();
this.consultAbortControllers.add(abortController);
@@ -537,7 +579,7 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport {
await submitRealtimeTalkConsult({
ctx: this.ctx,
callId,
args: buffered?.args || event.arguments || "{}",
args,
signal: abortController.signal,
emitTalkEvent: this.emitTalkEvent,
submit: (toolCallId, result) => this.submitToolResult(toolCallId, result),
@@ -547,7 +589,7 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport {
}
}
private async handleDescribeViewToolCall(callId: string, itemId: string): Promise<void> {
private async handleDescribeViewToolCall(callId: string, itemId?: string): Promise<void> {
this.emitTalkEvent({
type: "tool.call",
callId,
@@ -594,6 +636,9 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport {
}
private submitToolResult(callId: string, result: unknown): void {
if (this.closed) {
return;
}
this.send({
type: "conversation.item.create",
item: {