diff --git a/docs/plugins/sdk-runtime.md b/docs/plugins/sdk-runtime.md index f8764f72e45a..907752c3799f 100644 --- a/docs/plugins/sdk-runtime.md +++ b/docs/plugins/sdk-runtime.md @@ -429,13 +429,20 @@ snapshots; OpenClaw owns all persistence and lifecycle coordination. session.close(); ``` - Both `sendAudio()` input and `audio` event output use PCM16LE, 24 kHz, mono audio. The callback - receives: + `sessionKey` selects the agent conversation and workspace. `provider`, `model`, `voice`, and + `language` optionally override its configured Talk defaults. Input and output use signed + PCM16 little-endian audio at 24 kHz, mono (`session.audio.encoding` is `"pcm16"`). - - `state` when the session starts listening, thinking, speaking, or enters an error state - - ordered `audio` chunks with generation, sequence, and presentation timestamps - - `clear` when buffered output from an earlier generation must be discarded - - one terminal `closed` event + `sendAudio()` accepts an optional `timestamp`: the current output playback position in + milliseconds, used to trim interrupted speech. The callback receives: + + - `state` (`idle`, `listening`, `thinking`, `speaking`, or `error`); `ptsMs` is the current + output position in milliseconds + - ordered `audio` chunks; `sequence` starts at zero and `ptsMs` is the chunk's presentation + time in milliseconds + - `clear`, with `barge-in` when the user interrupts or `cancel` for another cancellation; + discard buffered audio from older generations + - one terminal `closed`, with `completed` for a normal close or `error` after a failure After `closed`, `sendAudio()` throws while `cancelOutput()` and `close()` do nothing. A thrown or rejected callback closes the relay; if it fails during setup, `openSession()` rejects. Reopen a diff --git a/src/gateway/talk-plugin-session.test.ts b/src/gateway/talk-plugin-session.test.ts index 2e589d250bde..d2adcce8e68a 100644 --- a/src/gateway/talk-plugin-session.test.ts +++ b/src/gateway/talk-plugin-session.test.ts @@ -29,7 +29,10 @@ describe("plugin Talk session", () => { mocks.scope.mockReturnValue({ pluginId: "avatar", gatewayMethodDispatchAllowed: true, - client: { connect: { scopes: ["operator.talk"] } }, + client: { + connId: "plugin-http:127.0.0.1", + connect: { scopes: ["operator.talk"] }, + }, context: { logGateway: { warn: mocks.warn } }, }); mocks.createSession.mockResolvedValue({ relaySessionId: "relay-1" }); @@ -48,7 +51,7 @@ describe("plugin Talk session", () => { context: { logGateway: { warn: mocks.warn } }, request: { sessionKey: "agent:main:avatar", voice: "alloy" }, }); - expect(createParams.ownerId).toMatch(/^plugin:avatar:/); + expect(createParams.ownerId).toBe("plugin:avatar:plugin-http:127.0.0.1"); createParams.eventSink({ relaySessionId: "relay-1", type: "ready" }); createParams.eventSink({ relaySessionId: "relay-1", type: "audioStarted" }); @@ -94,6 +97,16 @@ describe("plugin Talk session", () => { }); }); + it("shares the route owner across opens so relay session limits apply", async () => { + await openPluginTalkSession({ sessionKey: "agent:main:first", onEvent: vi.fn() }); + await openPluginTalkSession({ sessionKey: "agent:main:second", onEvent: vi.fn() }); + + expect(mocks.createSession.mock.calls.map(([params]) => params.ownerId)).toEqual([ + "plugin:avatar:plugin-http:127.0.0.1", + "plugin:avatar:plugin-http:127.0.0.1", + ]); + }); + it("stops accepting media after the Gateway closes the session", async () => { const onEvent = vi.fn(); const session = await openPluginTalkSession({ @@ -154,7 +167,7 @@ describe("plugin Talk session", () => { expect(mocks.stopSession).toHaveBeenCalledWith({ relaySessionId: "relay-1", - connId: expect.stringMatching(/^plugin:avatar:/), + connId: "plugin:avatar:plugin-http:127.0.0.1", }); }); @@ -162,7 +175,10 @@ describe("plugin Talk session", () => { mocks.scope.mockReturnValue({ pluginId: "avatar", gatewayMethodDispatchAllowed: true, - client: { connect: { scopes: ["operator.read"] } }, + client: { + connId: "plugin-http:127.0.0.1", + connect: { scopes: ["operator.read"] }, + }, context: { logGateway: { warn: mocks.warn } }, }); diff --git a/src/gateway/talk-plugin-session.ts b/src/gateway/talk-plugin-session.ts index d4286d7779ac..7e21e4f1be73 100644 --- a/src/gateway/talk-plugin-session.ts +++ b/src/gateway/talk-plugin-session.ts @@ -1,4 +1,3 @@ -import { randomUUID } from "node:crypto"; import { formatErrorMessage } from "../infra/errors.js"; import { getPluginRuntimeGatewayRequestScope } from "../plugins/runtime/gateway-request-scope.js"; import { @@ -20,7 +19,12 @@ const PCM16_24KHZ_MONO_BYTES_PER_MS = 48; function requirePluginTalkScope() { const scope = getPluginRuntimeGatewayRequestScope(); - if (!scope?.context || !scope.pluginId || scope.gatewayMethodDispatchAllowed !== true) { + if ( + !scope?.context || + !scope.pluginId || + !scope.client?.connId || + scope.gatewayMethodDispatchAllowed !== true + ) { throw new Error( "Interactive Talk sessions require a plugin request route that declares the gatewayMethodDispatch contract.", ); @@ -31,7 +35,10 @@ function requirePluginTalkScope() { "Interactive Talk sessions require an authenticated plugin request with Talk access.", ); } - return { context: scope.context, pluginId: scope.pluginId }; + return { + context: scope.context, + ownerId: `plugin:${scope.pluginId}:${scope.client.connId}`, + }; } function createPluginTalkEventSink( @@ -129,8 +136,7 @@ export async function openPluginTalkSession( "Choose an OpenClaw session before starting voice so the conversation uses the intended agent and workspace.", ); } - const { context, pluginId } = requirePluginTalkScope(); - const ownerId = `plugin:${pluginId}:${randomUUID()}`; + const { context, ownerId } = requirePluginTalkScope(); const lifecycle: { relaySessionId?: string } = {}; let deliveryError: unknown; const events = createPluginTalkEventSink(params, (error) => { diff --git a/src/talk/plugin-session.ts b/src/talk/plugin-session.ts index 72668e33aa5f..6ad5bb114285 100644 --- a/src/talk/plugin-session.ts +++ b/src/talk/plugin-session.ts @@ -1,8 +1,11 @@ -export const PLUGIN_TALK_AUDIO_FORMAT = { - encoding: "pcm16le", - sampleRateHz: 24_000, - channels: 1, -} as const; +import { + REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ, + type RealtimeVoiceAudioFormat, +} from "./provider-types.js"; + +export const PLUGIN_TALK_AUDIO_FORMAT: Readonly< + Extract +> = REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ; export type PluginTalkSessionEvent = | { @@ -21,12 +24,12 @@ export type PluginTalkSessionEvent = | { type: "clear"; generation: number; - reason: "barge-in" | "cancel" | "replace" | "hangup" | "error"; + reason: "barge-in" | "cancel"; } | { type: "closed"; generation: number; - reason: "completed" | "error" | "replaced"; + reason: "completed" | "error"; }; export type OpenPluginTalkSessionParams = {