fix(plugin-sdk): tighten Talk session ownership

This commit is contained in:
Dallin Romney
2026-08-09 14:21:12 +08:00
parent 455a07f524
commit 8917aec98d
4 changed files with 54 additions and 22 deletions
+13 -6
View File
@@ -429,13 +429,20 @@ snapshots; OpenClaw owns all persistence and lifecycle coordination.
session.close();
```
Both `sendAudio()` input and `audio` event output use PCM16LE, 24 kHz, mono audio. The callback
receives:
`sessionKey` selects the agent conversation and workspace. `provider`, `model`, `voice`, and
`language` optionally override its configured Talk defaults. Input and output use signed
PCM16 little-endian audio at 24 kHz, mono (`session.audio.encoding` is `"pcm16"`).
- `state` when the session starts listening, thinking, speaking, or enters an error state
- ordered `audio` chunks with generation, sequence, and presentation timestamps
- `clear` when buffered output from an earlier generation must be discarded
- one terminal `closed` event
`sendAudio()` accepts an optional `timestamp`: the current output playback position in
milliseconds, used to trim interrupted speech. The callback receives:
- `state` (`idle`, `listening`, `thinking`, `speaking`, or `error`); `ptsMs` is the current
output position in milliseconds
- ordered `audio` chunks; `sequence` starts at zero and `ptsMs` is the chunk's presentation
time in milliseconds
- `clear`, with `barge-in` when the user interrupts or `cancel` for another cancellation;
discard buffered audio from older generations
- one terminal `closed`, with `completed` for a normal close or `error` after a failure
After `closed`, `sendAudio()` throws while `cancelOutput()` and `close()` do nothing. A thrown or
rejected callback closes the relay; if it fails during setup, `openSession()` rejects. Reopen a
+20 -4
View File
@@ -29,7 +29,10 @@ describe("plugin Talk session", () => {
mocks.scope.mockReturnValue({
pluginId: "avatar",
gatewayMethodDispatchAllowed: true,
client: { connect: { scopes: ["operator.talk"] } },
client: {
connId: "plugin-http:127.0.0.1",
connect: { scopes: ["operator.talk"] },
},
context: { logGateway: { warn: mocks.warn } },
});
mocks.createSession.mockResolvedValue({ relaySessionId: "relay-1" });
@@ -48,7 +51,7 @@ describe("plugin Talk session", () => {
context: { logGateway: { warn: mocks.warn } },
request: { sessionKey: "agent:main:avatar", voice: "alloy" },
});
expect(createParams.ownerId).toMatch(/^plugin:avatar:/);
expect(createParams.ownerId).toBe("plugin:avatar:plugin-http:127.0.0.1");
createParams.eventSink({ relaySessionId: "relay-1", type: "ready" });
createParams.eventSink({ relaySessionId: "relay-1", type: "audioStarted" });
@@ -94,6 +97,16 @@ describe("plugin Talk session", () => {
});
});
it("shares the route owner across opens so relay session limits apply", async () => {
await openPluginTalkSession({ sessionKey: "agent:main:first", onEvent: vi.fn() });
await openPluginTalkSession({ sessionKey: "agent:main:second", onEvent: vi.fn() });
expect(mocks.createSession.mock.calls.map(([params]) => params.ownerId)).toEqual([
"plugin:avatar:plugin-http:127.0.0.1",
"plugin:avatar:plugin-http:127.0.0.1",
]);
});
it("stops accepting media after the Gateway closes the session", async () => {
const onEvent = vi.fn();
const session = await openPluginTalkSession({
@@ -154,7 +167,7 @@ describe("plugin Talk session", () => {
expect(mocks.stopSession).toHaveBeenCalledWith({
relaySessionId: "relay-1",
connId: expect.stringMatching(/^plugin:avatar:/),
connId: "plugin:avatar:plugin-http:127.0.0.1",
});
});
@@ -162,7 +175,10 @@ describe("plugin Talk session", () => {
mocks.scope.mockReturnValue({
pluginId: "avatar",
gatewayMethodDispatchAllowed: true,
client: { connect: { scopes: ["operator.read"] } },
client: {
connId: "plugin-http:127.0.0.1",
connect: { scopes: ["operator.read"] },
},
context: { logGateway: { warn: mocks.warn } },
});
+11 -5
View File
@@ -1,4 +1,3 @@
import { randomUUID } from "node:crypto";
import { formatErrorMessage } from "../infra/errors.js";
import { getPluginRuntimeGatewayRequestScope } from "../plugins/runtime/gateway-request-scope.js";
import {
@@ -20,7 +19,12 @@ const PCM16_24KHZ_MONO_BYTES_PER_MS = 48;
function requirePluginTalkScope() {
const scope = getPluginRuntimeGatewayRequestScope();
if (!scope?.context || !scope.pluginId || scope.gatewayMethodDispatchAllowed !== true) {
if (
!scope?.context ||
!scope.pluginId ||
!scope.client?.connId ||
scope.gatewayMethodDispatchAllowed !== true
) {
throw new Error(
"Interactive Talk sessions require a plugin request route that declares the gatewayMethodDispatch contract.",
);
@@ -31,7 +35,10 @@ function requirePluginTalkScope() {
"Interactive Talk sessions require an authenticated plugin request with Talk access.",
);
}
return { context: scope.context, pluginId: scope.pluginId };
return {
context: scope.context,
ownerId: `plugin:${scope.pluginId}:${scope.client.connId}`,
};
}
function createPluginTalkEventSink(
@@ -129,8 +136,7 @@ export async function openPluginTalkSession(
"Choose an OpenClaw session before starting voice so the conversation uses the intended agent and workspace.",
);
}
const { context, pluginId } = requirePluginTalkScope();
const ownerId = `plugin:${pluginId}:${randomUUID()}`;
const { context, ownerId } = requirePluginTalkScope();
const lifecycle: { relaySessionId?: string } = {};
let deliveryError: unknown;
const events = createPluginTalkEventSink(params, (error) => {
+10 -7
View File
@@ -1,8 +1,11 @@
export const PLUGIN_TALK_AUDIO_FORMAT = {
encoding: "pcm16le",
sampleRateHz: 24_000,
channels: 1,
} as const;
import {
REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ,
type RealtimeVoiceAudioFormat,
} from "./provider-types.js";
export const PLUGIN_TALK_AUDIO_FORMAT: Readonly<
Extract<RealtimeVoiceAudioFormat, { encoding: "pcm16" }>
> = REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ;
export type PluginTalkSessionEvent =
| {
@@ -21,12 +24,12 @@ export type PluginTalkSessionEvent =
| {
type: "clear";
generation: number;
reason: "barge-in" | "cancel" | "replace" | "hangup" | "error";
reason: "barge-in" | "cancel";
}
| {
type: "closed";
generation: number;
reason: "completed" | "error" | "replaced";
reason: "completed" | "error";
};
export type OpenPluginTalkSessionParams = {