mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-27 12:56:01 -06:00
fix(plugin-sdk): tighten Talk session ownership
This commit is contained in:
@@ -429,13 +429,20 @@ snapshots; OpenClaw owns all persistence and lifecycle coordination.
|
||||
session.close();
|
||||
```
|
||||
|
||||
Both `sendAudio()` input and `audio` event output use PCM16LE, 24 kHz, mono audio. The callback
|
||||
receives:
|
||||
`sessionKey` selects the agent conversation and workspace. `provider`, `model`, `voice`, and
|
||||
`language` optionally override its configured Talk defaults. Input and output use signed
|
||||
PCM16 little-endian audio at 24 kHz, mono (`session.audio.encoding` is `"pcm16"`).
|
||||
|
||||
- `state` when the session starts listening, thinking, speaking, or enters an error state
|
||||
- ordered `audio` chunks with generation, sequence, and presentation timestamps
|
||||
- `clear` when buffered output from an earlier generation must be discarded
|
||||
- one terminal `closed` event
|
||||
`sendAudio()` accepts an optional `timestamp`: the current output playback position in
|
||||
milliseconds, used to trim interrupted speech. The callback receives:
|
||||
|
||||
- `state` (`idle`, `listening`, `thinking`, `speaking`, or `error`); `ptsMs` is the current
|
||||
output position in milliseconds
|
||||
- ordered `audio` chunks; `sequence` starts at zero and `ptsMs` is the chunk's presentation
|
||||
time in milliseconds
|
||||
- `clear`, with `barge-in` when the user interrupts or `cancel` for another cancellation;
|
||||
discard buffered audio from older generations
|
||||
- one terminal `closed`, with `completed` for a normal close or `error` after a failure
|
||||
|
||||
After `closed`, `sendAudio()` throws while `cancelOutput()` and `close()` do nothing. A thrown or
|
||||
rejected callback closes the relay; if it fails during setup, `openSession()` rejects. Reopen a
|
||||
|
||||
@@ -29,7 +29,10 @@ describe("plugin Talk session", () => {
|
||||
mocks.scope.mockReturnValue({
|
||||
pluginId: "avatar",
|
||||
gatewayMethodDispatchAllowed: true,
|
||||
client: { connect: { scopes: ["operator.talk"] } },
|
||||
client: {
|
||||
connId: "plugin-http:127.0.0.1",
|
||||
connect: { scopes: ["operator.talk"] },
|
||||
},
|
||||
context: { logGateway: { warn: mocks.warn } },
|
||||
});
|
||||
mocks.createSession.mockResolvedValue({ relaySessionId: "relay-1" });
|
||||
@@ -48,7 +51,7 @@ describe("plugin Talk session", () => {
|
||||
context: { logGateway: { warn: mocks.warn } },
|
||||
request: { sessionKey: "agent:main:avatar", voice: "alloy" },
|
||||
});
|
||||
expect(createParams.ownerId).toMatch(/^plugin:avatar:/);
|
||||
expect(createParams.ownerId).toBe("plugin:avatar:plugin-http:127.0.0.1");
|
||||
|
||||
createParams.eventSink({ relaySessionId: "relay-1", type: "ready" });
|
||||
createParams.eventSink({ relaySessionId: "relay-1", type: "audioStarted" });
|
||||
@@ -94,6 +97,16 @@ describe("plugin Talk session", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("shares the route owner across opens so relay session limits apply", async () => {
|
||||
await openPluginTalkSession({ sessionKey: "agent:main:first", onEvent: vi.fn() });
|
||||
await openPluginTalkSession({ sessionKey: "agent:main:second", onEvent: vi.fn() });
|
||||
|
||||
expect(mocks.createSession.mock.calls.map(([params]) => params.ownerId)).toEqual([
|
||||
"plugin:avatar:plugin-http:127.0.0.1",
|
||||
"plugin:avatar:plugin-http:127.0.0.1",
|
||||
]);
|
||||
});
|
||||
|
||||
it("stops accepting media after the Gateway closes the session", async () => {
|
||||
const onEvent = vi.fn();
|
||||
const session = await openPluginTalkSession({
|
||||
@@ -154,7 +167,7 @@ describe("plugin Talk session", () => {
|
||||
|
||||
expect(mocks.stopSession).toHaveBeenCalledWith({
|
||||
relaySessionId: "relay-1",
|
||||
connId: expect.stringMatching(/^plugin:avatar:/),
|
||||
connId: "plugin:avatar:plugin-http:127.0.0.1",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -162,7 +175,10 @@ describe("plugin Talk session", () => {
|
||||
mocks.scope.mockReturnValue({
|
||||
pluginId: "avatar",
|
||||
gatewayMethodDispatchAllowed: true,
|
||||
client: { connect: { scopes: ["operator.read"] } },
|
||||
client: {
|
||||
connId: "plugin-http:127.0.0.1",
|
||||
connect: { scopes: ["operator.read"] },
|
||||
},
|
||||
context: { logGateway: { warn: mocks.warn } },
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import { getPluginRuntimeGatewayRequestScope } from "../plugins/runtime/gateway-request-scope.js";
|
||||
import {
|
||||
@@ -20,7 +19,12 @@ const PCM16_24KHZ_MONO_BYTES_PER_MS = 48;
|
||||
|
||||
function requirePluginTalkScope() {
|
||||
const scope = getPluginRuntimeGatewayRequestScope();
|
||||
if (!scope?.context || !scope.pluginId || scope.gatewayMethodDispatchAllowed !== true) {
|
||||
if (
|
||||
!scope?.context ||
|
||||
!scope.pluginId ||
|
||||
!scope.client?.connId ||
|
||||
scope.gatewayMethodDispatchAllowed !== true
|
||||
) {
|
||||
throw new Error(
|
||||
"Interactive Talk sessions require a plugin request route that declares the gatewayMethodDispatch contract.",
|
||||
);
|
||||
@@ -31,7 +35,10 @@ function requirePluginTalkScope() {
|
||||
"Interactive Talk sessions require an authenticated plugin request with Talk access.",
|
||||
);
|
||||
}
|
||||
return { context: scope.context, pluginId: scope.pluginId };
|
||||
return {
|
||||
context: scope.context,
|
||||
ownerId: `plugin:${scope.pluginId}:${scope.client.connId}`,
|
||||
};
|
||||
}
|
||||
|
||||
function createPluginTalkEventSink(
|
||||
@@ -129,8 +136,7 @@ export async function openPluginTalkSession(
|
||||
"Choose an OpenClaw session before starting voice so the conversation uses the intended agent and workspace.",
|
||||
);
|
||||
}
|
||||
const { context, pluginId } = requirePluginTalkScope();
|
||||
const ownerId = `plugin:${pluginId}:${randomUUID()}`;
|
||||
const { context, ownerId } = requirePluginTalkScope();
|
||||
const lifecycle: { relaySessionId?: string } = {};
|
||||
let deliveryError: unknown;
|
||||
const events = createPluginTalkEventSink(params, (error) => {
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
export const PLUGIN_TALK_AUDIO_FORMAT = {
|
||||
encoding: "pcm16le",
|
||||
sampleRateHz: 24_000,
|
||||
channels: 1,
|
||||
} as const;
|
||||
import {
|
||||
REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ,
|
||||
type RealtimeVoiceAudioFormat,
|
||||
} from "./provider-types.js";
|
||||
|
||||
export const PLUGIN_TALK_AUDIO_FORMAT: Readonly<
|
||||
Extract<RealtimeVoiceAudioFormat, { encoding: "pcm16" }>
|
||||
> = REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ;
|
||||
|
||||
export type PluginTalkSessionEvent =
|
||||
| {
|
||||
@@ -21,12 +24,12 @@ export type PluginTalkSessionEvent =
|
||||
| {
|
||||
type: "clear";
|
||||
generation: number;
|
||||
reason: "barge-in" | "cancel" | "replace" | "hangup" | "error";
|
||||
reason: "barge-in" | "cancel";
|
||||
}
|
||||
| {
|
||||
type: "closed";
|
||||
generation: number;
|
||||
reason: "completed" | "error" | "replaced";
|
||||
reason: "completed" | "error";
|
||||
};
|
||||
|
||||
export type OpenPluginTalkSessionParams = {
|
||||
|
||||
Reference in New Issue
Block a user