From 92146f9f803b441b3f0cfe421ebcdc7d43b9f136 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 18:10:26 -0700 Subject: [PATCH] refactor(talk): share audio-energy stats and speech-threshold gate across voice surfaces (#109466) --- .../.generated/plugin-sdk-api-baseline.sha256 | 4 +- docs/plugins/sdk-migration.md | 2 +- docs/plugins/sdk-subpaths.md | 2 +- .../src/realtime-local-audio-transport.ts | 35 ++------ .../src/webhook/realtime-audio-pacer.test.ts | 22 +---- .../src/webhook/realtime-audio-pacer.ts | 74 +--------------- .../src/webhook/realtime-handler.ts | 12 ++- scripts/plugin-sdk-surface-report.mjs | 6 +- src/plugin-sdk/realtime-voice.ts | 6 ++ src/talk/audio-energy.test.ts | 65 ++++++++++++++ src/talk/audio-energy.ts | 87 +++++++++++++++++++ 11 files changed, 185 insertions(+), 130 deletions(-) create mode 100644 src/talk/audio-energy.test.ts create mode 100644 src/talk/audio-energy.ts diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 1d98ee8072ab..8d6c6e2b91a7 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -3fc61e844ced788c276503228bb6248a6fdaff0f9276e6e248ecdac66ad63c7f plugin-sdk-api-baseline.json -cfe57713951912169ef66cd8f40e9f329f57fdf268070e0994f0f8aa0ca2b8c1 plugin-sdk-api-baseline.jsonl +e53aad267b77bc3ae033e271e3501ece9e8e4113d243ea9b0740e627bb2acf26 plugin-sdk-api-baseline.json +e73d9ab6836d36fdfe47452e3f0337d5b15577e395cedb152fcb843928c3cd72 plugin-sdk-api-baseline.jsonl diff --git a/docs/plugins/sdk-migration.md b/docs/plugins/sdk-migration.md index 090b698716e3..aa04d81d38ef 100644 --- a/docs/plugins/sdk-migration.md +++ b/docs/plugins/sdk-migration.md @@ -504,7 +504,7 @@ SDK. | `plugin-sdk/speech-core` | Shared speech core | Speech provider types, registry, directives, normalization | | `plugin-sdk/speech-settings` | Speech settings | Lightweight TTS config resolution and normalization primitives without provider registries or synthesis runtime | | `plugin-sdk/realtime-transcription` | Realtime transcription helpers | Provider types, registry helpers, and shared WebSocket session helper | - | `plugin-sdk/realtime-voice` | Realtime voice helpers | Provider types, registry/resolution helpers, bridge session helpers, shared agent talk-back queues, active-run voice control, transcript/event health, echo suppression, consult question matching, forced-consult coordination, turn-context tracking, output activity tracking, and fast context consult helpers | + | `plugin-sdk/realtime-voice` | Realtime voice helpers | Provider types, registry/resolution helpers, bridge session helpers, audio-energy/speech-onset gates, shared agent talk-back queues, active-run voice control, transcript/event health, echo suppression, consult question matching, forced-consult coordination, turn-context tracking, output activity tracking, and fast context consult helpers | | `plugin-sdk/image-generation` | Image-generation helpers | Image generation provider types plus image asset/data URL helpers and the OpenAI-compatible image provider builder | | `plugin-sdk/image-generation-core` | Shared image-generation core | Image-generation types, failover, auth, and registry helpers | | `plugin-sdk/music-generation` | Music-generation helpers | Music-generation provider/request/result types | diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index a2c80b6d39ff..a9885b5a50f4 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -360,7 +360,7 @@ usage endpoint failed or returned no usable usage data. | `plugin-sdk/speech-settings` | Lightweight TTS config resolution and normalization primitives without provider registries or synthesis runtime | | `plugin-sdk/realtime-transcription` | Realtime transcription provider types, registry helpers, and shared WebSocket session helper | | `plugin-sdk/realtime-bootstrap-context` | Realtime profile bootstrap helper for bounded `IDENTITY.md`, `USER.md`, and `SOUL.md` context injection | - | `plugin-sdk/realtime-voice` | Realtime voice provider types, registry helpers, and shared realtime voice behavior helpers, including output activity tracking | + | `plugin-sdk/realtime-voice` | Realtime voice provider types, registry helpers, shared audio-energy/speech-onset gates, and realtime voice behavior helpers, including output activity tracking | | `plugin-sdk/image-generation` | Image generation provider types plus image asset/data URL helpers and the OpenAI-compatible image provider builder | | `plugin-sdk/image-generation-core` | Shared image-generation types, failover, auth, and registry helpers | | `plugin-sdk/music-generation` | Music generation provider/request/result types | diff --git a/extensions/google-meet/src/realtime-local-audio-transport.ts b/extensions/google-meet/src/realtime-local-audio-transport.ts index 6bf2db8c9816..876369c725da 100644 --- a/extensions/google-meet/src/realtime-local-audio-transport.ts +++ b/extensions/google-meet/src/realtime-local-audio-transport.ts @@ -2,6 +2,7 @@ import { spawn } from "node:child_process"; import type { Writable } from "node:stream"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import type { RuntimeLogger } from "openclaw/plugin-sdk/plugin-runtime"; +import { createSpeechThresholdGate, readPcm16AudioStats } from "openclaw/plugin-sdk/realtime-voice"; import type { MeetRealtimeAudioTransport } from "./realtime-audio-transport.js"; type BridgeProcess = { @@ -66,23 +67,6 @@ function terminateBridgeProcess(proc: BridgeProcess, signal: NodeJS.Signals = "S timer.unref?.(); } -function readPcm16Stats(audio: Buffer): { rms: number; peak: number } { - let sumSquares = 0; - let peak = 0; - let samples = 0; - for (let offset = 0; offset + 1 < audio.byteLength; offset += 2) { - const sample = audio.readInt16LE(offset); - const abs = Math.abs(sample); - peak = Math.max(peak, abs); - sumSquares += sample * sample; - samples += 1; - } - return { - rms: samples > 0 ? Math.sqrt(sumSquares / samples) : 0, - peak, - }; -} - export function createLocalMeetRealtimeAudioTransport(params: { inputCommand: string[]; outputCommand: string[]; @@ -229,24 +213,23 @@ export function createLocalMeetRealtimeAudioTransport(params: { return; } const command = splitCommand(params.bargeInInputCommand ?? []); - let lastBargeInAt = 0; + const bargeInGate = createSpeechThresholdGate({ + rmsThreshold: params.bargeInRmsThreshold, + peakThreshold: params.bargeInPeakThreshold, + cooldownMs: params.bargeInCooldownMs, + }); bargeInInputProcess = spawnFn(command.command, command.args, { stdio: ["ignore", "pipe", "pipe"], }); bargeInInputProcess.stdout?.on("data", (chunk) => { const audio = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); - const now = Date.now(); - if (stopped || now - lastBargeInAt < params.bargeInCooldownMs) { + if (stopped) { return; } - const stats = readPcm16Stats(audio); - if (stats.rms < params.bargeInRmsThreshold && stats.peak < params.bargeInPeakThreshold) { + const stats = readPcm16AudioStats(audio); + if (!bargeInGate.accept(stats, { nowMs: Date.now(), onTrigger: () => onBargeIn(audio) })) { return; } - if (!onBargeIn(audio)) { - return; - } - lastBargeInAt = now; params.logger.debug?.( `[google-meet] human barge-in detected by local input (rms=${Math.round( stats.rms, diff --git a/extensions/voice-call/src/webhook/realtime-audio-pacer.test.ts b/extensions/voice-call/src/webhook/realtime-audio-pacer.test.ts index 71d7c60c57e2..2d5df569aa31 100644 --- a/extensions/voice-call/src/webhook/realtime-audio-pacer.test.ts +++ b/extensions/voice-call/src/webhook/realtime-audio-pacer.test.ts @@ -1,6 +1,6 @@ // Voice Call tests cover realtime audio pacer plugin behavior. import { afterEach, describe, expect, it, vi } from "vitest"; -import { RealtimeAudioPacer, RealtimeMulawSpeechStartDetector } from "./realtime-audio-pacer.js"; +import { RealtimeAudioPacer } from "./realtime-audio-pacer.js"; type RealtimeAudioSerializer = ConstructorParameters[0]["serializer"]; @@ -120,23 +120,3 @@ describe("RealtimeAudioPacer", () => { ]); }); }); - -describe("RealtimeMulawSpeechStartDetector", () => { - it("detects a speech start after consecutive loud chunks and resets after quiet", () => { - const detector = new RealtimeMulawSpeechStartDetector({ - requiredLoudChunks: 2, - requiredQuietChunks: 2, - rmsThreshold: 0.02, - }); - const silence = Buffer.alloc(160, 0xff); - const speech = Buffer.alloc(160, 0x00); - - expect(detector.accept(speech)).toBe(false); - expect(detector.accept(speech)).toBe(true); - expect(detector.accept(speech)).toBe(false); - expect(detector.accept(silence)).toBe(false); - expect(detector.accept(silence)).toBe(false); - expect(detector.accept(speech)).toBe(false); - expect(detector.accept(speech)).toBe(true); - }); -}); diff --git a/extensions/voice-call/src/webhook/realtime-audio-pacer.ts b/extensions/voice-call/src/webhook/realtime-audio-pacer.ts index 4e0a0250db50..e214a4019e1a 100644 --- a/extensions/voice-call/src/webhook/realtime-audio-pacer.ts +++ b/extensions/voice-call/src/webhook/realtime-audio-pacer.ts @@ -1,18 +1,9 @@ -// Realtime telephony audio pacing and speech-start detection for mulaw streams. +// Realtime telephony audio pacing for mulaw streams. const TELEPHONY_SAMPLE_RATE = 8_000; const TELEPHONY_CHUNK_BYTES = 160; const TELEPHONY_CHUNK_MS = 20; -const DEFAULT_SPEECH_RMS_THRESHOLD = 0.035; -const DEFAULT_REQUIRED_LOUD_CHUNKS = 4; -const DEFAULT_REQUIRED_QUIET_CHUNKS = 12; const DEFAULT_MAX_QUEUED_AUDIO_BYTES = TELEPHONY_SAMPLE_RATE * 120; -const PCM16_MAX_AMPLITUDE = 32768; -const MULAW_LINEAR_SAMPLES = new Int16Array(256); - -for (let i = 0; i < MULAW_LINEAR_SAMPLES.length; i += 1) { - MULAW_LINEAR_SAMPLES[i] = decodeMulawSample(i); -} /** Queue item sent over the realtime provider media stream. */ type RealtimeAudioQueueItem = @@ -162,66 +153,3 @@ export class RealtimeAudioPacer { } } } - -/** Calculate normalized RMS from mulaw bytes. */ -function calculateMulawRms(muLaw: Buffer): number { - if (muLaw.length === 0) { - return 0; - } - let sum = 0; - for (const sample of muLaw) { - const normalized = (MULAW_LINEAR_SAMPLES[sample] ?? 0) / PCM16_MAX_AMPLITUDE; - sum += normalized * normalized; - } - return Math.sqrt(sum / muLaw.length); -} - -/** Detect likely speech start from consecutive loud mulaw chunks. */ -export class RealtimeMulawSpeechStartDetector { - private loudChunks = 0; - private quietChunks = DEFAULT_REQUIRED_QUIET_CHUNKS; - private speaking = false; - - constructor( - private readonly params: { - requiredLoudChunks?: number; - requiredQuietChunks?: number; - rmsThreshold?: number; - } = {}, - ) {} - - /** Accept one mulaw chunk and return true only on transition into speaking. */ - accept(muLaw: Buffer): boolean { - const rms = calculateMulawRms(muLaw); - const threshold = this.params.rmsThreshold ?? DEFAULT_SPEECH_RMS_THRESHOLD; - if (rms >= threshold) { - this.quietChunks = 0; - this.loudChunks += 1; - const requiredLoudChunks = this.params.requiredLoudChunks ?? DEFAULT_REQUIRED_LOUD_CHUNKS; - if (!this.speaking && this.loudChunks >= requiredLoudChunks) { - this.speaking = true; - return true; - } - return false; - } - - this.loudChunks = 0; - this.quietChunks += 1; - const requiredQuietChunks = this.params.requiredQuietChunks ?? DEFAULT_REQUIRED_QUIET_CHUNKS; - if (this.quietChunks >= requiredQuietChunks) { - this.speaking = false; - } - return false; - } -} - -/** Decode one G.711 mulaw byte to a linear PCM sample. */ -function decodeMulawSample(value: number): number { - const muLaw = ~value & 0xff; - const sign = muLaw & 0x80; - const exponent = (muLaw >> 4) & 0x07; - const mantissa = muLaw & 0x0f; - let sample = ((mantissa << 3) + 132) << exponent; - sample -= 132; - return sign ? -sample : sample; -} diff --git a/extensions/voice-call/src/webhook/realtime-handler.ts b/extensions/voice-call/src/webhook/realtime-handler.ts index c7e40e4c8893..7b0cf7ec5bea 100644 --- a/extensions/voice-call/src/webhook/realtime-handler.ts +++ b/extensions/voice-call/src/webhook/realtime-handler.ts @@ -10,7 +10,9 @@ import { } from "openclaw/plugin-sdk/number-runtime"; import { buildRealtimeVoiceAgentConsultWorkingResponse, + calculateMulawRms, createRealtimeVoiceForcedConsultCoordinator, + createSpeechThresholdGate, createTalkSessionController, createRealtimeVoiceBridgeSession, REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, @@ -35,7 +37,7 @@ import type { CallManager } from "../manager.js"; import type { VoiceCallProvider } from "../providers/base.js"; import type { CallRecord, NormalizedEvent } from "../types.js"; import type { WebhookResponsePayload } from "../webhook.types.js"; -import { RealtimeAudioPacer, RealtimeMulawSpeechStartDetector } from "./realtime-audio-pacer.js"; +import { RealtimeAudioPacer } from "./realtime-audio-pacer.js"; import { type StreamFrameAdapter, TelnyxStreamFrameAdapter, @@ -711,8 +713,10 @@ export class RealtimeCallHandler { } }, }); - const speechDetector = new RealtimeMulawSpeechStartDetector({ - requiredLoudChunks: BARGE_IN_REQUIRED_LOUD_CHUNKS, + const speechDetector = createSpeechThresholdGate({ + rmsThreshold: 0.035, + speechFrames: BARGE_IN_REQUIRED_LOUD_CHUNKS, + silenceFrames: 12, }); const interruptResponseOnInputAudio = typeof this.providerConfig.interruptResponseOnInputAudio === "boolean" @@ -947,7 +951,7 @@ export class RealtimeCallHandler { this.activeTelephonyClosersByCallId.set(callSid, closeTelephony); const sendAudioToSession = session.sendAudio.bind(session); session.sendAudio = (audio) => { - if (speechDetector.accept(audio)) { + if (speechDetector.accept({ rms: calculateMulawRms(audio), peak: 0 })) { console.log( `[voice-call] realtime local speech detected callId=${callId} providerCallId=${callSid}`, ); diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index 1a048887f895..58edfb129f21 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -250,7 +250,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +4: bounded plugin blob store options, entry, entry info, and store types. // +6: shared progress receipt tracker + compositor snapshot across channel barrels. // +1: selectPreferredLocalModelId shares app-guided local model ranking across providers. - 8006, + // +4: shared audio-energy stats and speech-threshold gate through realtime-voice. + 8010, env, ), publicFunctionExports: readPluginSdkSurfaceBudgetEnv( @@ -275,7 +276,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +3: lightweight speech settings normalizers and config resolver. // +1: unified implicit-mention policy resolver. // +1: selectPreferredLocalModelId shares app-guided local model ranking across providers. - 4473, + // +3: PCM16/mu-law energy readers and speech-threshold gate factory. + 4476, env, ), publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv( diff --git a/src/plugin-sdk/realtime-voice.ts b/src/plugin-sdk/realtime-voice.ts index 432bacc4df3a..3ac2ef9f5b3a 100644 --- a/src/plugin-sdk/realtime-voice.ts +++ b/src/plugin-sdk/realtime-voice.ts @@ -183,6 +183,12 @@ export { type RealtimeVoiceTranscriptEntry, type RealtimeVoiceTranscriptHealth, } from "../talk/session-log-runtime.js"; +export { + calculateMulawRms, + createSpeechThresholdGate, + readPcm16AudioStats, + type AudioEnergyStats, +} from "../talk/audio-energy.js"; export { convertPcmToMulaw8k, mulawToPcm, diff --git a/src/talk/audio-energy.test.ts b/src/talk/audio-energy.test.ts new file mode 100644 index 000000000000..2b9a510a6a30 --- /dev/null +++ b/src/talk/audio-energy.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it, vi } from "vitest"; +import { + calculateMulawRms, + createSpeechThresholdGate, + readPcm16AudioStats, +} from "./audio-energy.js"; + +function pcm16(...samples: number[]): Buffer { + const audio = Buffer.alloc(samples.length * Int16Array.BYTES_PER_ELEMENT); + samples.forEach((sample, index) => { + audio.writeInt16LE(sample, index * Int16Array.BYTES_PER_ELEMENT); + }); + return audio; +} + +describe("audio energy", () => { + it("reads silence and PCM16 tone-burst RMS/peak", () => { + expect(readPcm16AudioStats(Buffer.alloc(8))).toEqual({ rms: 0, peak: 0 }); + expect(readPcm16AudioStats(pcm16(0, 1_000, -1_000, 0))).toEqual({ + rms: Math.sqrt(500_000), + peak: 1_000, + }); + }); + + it("matches the existing normalized G.711 mu-law levels", () => { + expect(calculateMulawRms(Buffer.alloc(160, 0xff))).toBe(0); + expect(calculateMulawRms(Buffer.alloc(160, 0x00))).toBe(32_124 / 32_768); + }); +}); + +describe("speech threshold gate", () => { + it("fires on a sustained threshold-edge onset and rearms after quiet hold", () => { + const gate = createSpeechThresholdGate({ + rmsThreshold: 10, + speechFrames: 2, + silenceFrames: 2, + }); + const loud = { rms: 10, peak: 10 }; + const quiet = { rms: 9, peak: 9 }; + + expect(gate.accept(loud)).toBe(false); + expect(gate.accept(loud)).toBe(true); + expect(gate.accept(loud)).toBe(false); + expect(gate.accept(quiet)).toBe(false); + expect(gate.accept(quiet)).toBe(false); + expect(gate.accept(loud)).toBe(false); + expect(gate.accept(loud)).toBe(true); + }); + + it("uses RMS-or-peak thresholds, caller veto, and cooldown re-trigger", () => { + const gate = createSpeechThresholdGate({ + rmsThreshold: 10, + peakThreshold: 100, + cooldownMs: 1_000, + }); + const onTrigger = vi.fn(() => true); + const peakOnly = { rms: 9, peak: 100 }; + + expect(gate.accept(peakOnly, { nowMs: 1_000, onTrigger: () => false })).toBe(false); + expect(gate.accept(peakOnly, { nowMs: 1_000, onTrigger })).toBe(true); + expect(gate.accept(peakOnly, { nowMs: 1_999, onTrigger })).toBe(false); + expect(gate.accept(peakOnly, { nowMs: 2_000, onTrigger })).toBe(true); + expect(onTrigger).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/talk/audio-energy.ts b/src/talk/audio-energy.ts new file mode 100644 index 000000000000..8cc4f281ae01 --- /dev/null +++ b/src/talk/audio-energy.ts @@ -0,0 +1,87 @@ +import { mulawToPcm } from "./audio-codec.js"; + +const PCM16_MAX_AMPLITUDE = 32_768; +const MULAW_LINEAR_SAMPLES = (() => { + const encoded = Buffer.from([...Array(256).keys()]); + const decoded = mulawToPcm(encoded); + return Int16Array.from(encoded, (_, index) => decoded.readInt16LE(index * 2)); +})(); + +export type AudioEnergyStats = { peak: number; rms: number }; + +/** Read RMS and absolute peak from complete little-endian signed PCM16 samples. */ +export function readPcm16AudioStats(audio: Buffer): AudioEnergyStats { + let sumSquares = 0; + let peak = 0; + const samples = Math.floor(audio.byteLength / 2); + for (let index = 0; index < samples; index += 1) { + const sample = audio.readInt16LE(index * 2); + peak = Math.max(peak, Math.abs(sample)); + sumSquares += sample * sample; + } + return { rms: samples > 0 ? Math.sqrt(sumSquares / samples) : 0, peak }; +} + +/** Calculate normalized RMS from G.711 mu-law bytes. */ +export function calculateMulawRms(muLaw: Buffer): number { + if (muLaw.length === 0) { + return 0; + } + let sumSquares = 0; + for (const encoded of muLaw) { + const normalized = (MULAW_LINEAR_SAMPLES[encoded] ?? 0) / PCM16_MAX_AMPLITUDE; + sumSquares += normalized * normalized; + } + return Math.sqrt(sumSquares / muLaw.length); +} + +/** Build an OR-threshold gate with optional sustained onset, silence hold, and cooldown. */ +export function createSpeechThresholdGate(options: { + cooldownMs?: number; + peakThreshold?: number; + rmsThreshold?: number; + silenceFrames?: number; + speechFrames?: number; +}) { + const speechFrames = Math.max(1, Math.floor(options.speechFrames ?? 1)); + const silenceFrames = Math.max(0, Math.floor(options.silenceFrames ?? 0)); + const cooldownMs = Math.max(0, options.cooldownMs ?? 0); + let loudFrames = 0; + let quietFrames = 0; + let speaking = false; + let lastTriggerAt = Number.NEGATIVE_INFINITY; + + return { + accept( + stats: AudioEnergyStats, + acceptOptions: { nowMs?: number; onTrigger?: () => boolean } = {}, + ): boolean { + const loud = + (options.rmsThreshold !== undefined && stats.rms >= options.rmsThreshold) || + (options.peakThreshold !== undefined && stats.peak >= options.peakThreshold); + if (!loud) { + loudFrames = 0; + if (speaking && ++quietFrames >= silenceFrames) { + speaking = false; + } + return false; + } + + quietFrames = 0; + loudFrames += 1; + if (speaking || loudFrames < speechFrames) { + return false; + } + const nowMs = acceptOptions.nowMs ?? Date.now(); + if (nowMs - lastTriggerAt < cooldownMs || acceptOptions.onTrigger?.() === false) { + return false; + } + lastTriggerAt = nowMs; + speaking = silenceFrames > 0; + if (!speaking) { + loudFrames = 0; + } + return true; + }, + }; +}