ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

refactor(talk): share audio-energy stats and speech-threshold gate across voice surfaces (#109466)

Browse Source
This commit is contained in:
Peter Steinberger
2026-07-16 18:10:26 -07:00
committed by GitHub
parent 0f1b0e6679
commit 92146f9f80
11 changed files with 185 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/.generated/plugin-sdk-api-baseline.sha256
+2 -2
View File
@@ -1,2 +1,2 @@
3fc61e844ced788c276503228bb6248a6fdaff0f9276e6e248ecdac66ad63c7f plugin-sdk-api-baseline.json
cfe57713951912169ef66cd8f40e9f329f57fdf268070e0994f0f8aa0ca2b8c1 plugin-sdk-api-baseline.jsonl
e53aad267b77bc3ae033e271e3501ece9e8e4113d243ea9b0740e627bb2acf26 plugin-sdk-api-baseline.json
e73d9ab6836d36fdfe47452e3f0337d5b15577e395cedb152fcb843928c3cd72 plugin-sdk-api-baseline.jsonl
docs/plugins/sdk-migration.md
+1 -1
View File
@@ -504,7 +504,7 @@ SDK.
| `plugin-sdk/speech-core` | Shared speech core | Speech provider types, registry, directives, normalization |
| `plugin-sdk/speech-settings` | Speech settings | Lightweight TTS config resolution and normalization primitives without provider registries or synthesis runtime |
| `plugin-sdk/realtime-transcription` | Realtime transcription helpers | Provider types, registry helpers, and shared WebSocket session helper |
| `plugin-sdk/realtime-voice` | Realtime voice helpers | Provider types, registry/resolution helpers, bridge session helpers, shared agent talk-back queues, active-run voice control, transcript/event health, echo suppression, consult question matching, forced-consult coordination, turn-context tracking, output activity tracking, and fast context consult helpers |
| `plugin-sdk/realtime-voice` | Realtime voice helpers | Provider types, registry/resolution helpers, bridge session helpers, audio-energy/speech-onset gates, shared agent talk-back queues, active-run voice control, transcript/event health, echo suppression, consult question matching, forced-consult coordination, turn-context tracking, output activity tracking, and fast context consult helpers |
| `plugin-sdk/image-generation` | Image-generation helpers | Image generation provider types plus image asset/data URL helpers and the OpenAI-compatible image provider builder |
| `plugin-sdk/image-generation-core` | Shared image-generation core | Image-generation types, failover, auth, and registry helpers |
| `plugin-sdk/music-generation` | Music-generation helpers | Music-generation provider/request/result types |
docs/plugins/sdk-subpaths.md
+1 -1
View File
@@ -360,7 +360,7 @@ usage endpoint failed or returned no usable usage data.
| `plugin-sdk/speech-settings` | Lightweight TTS config resolution and normalization primitives without provider registries or synthesis runtime |
| `plugin-sdk/realtime-transcription` | Realtime transcription provider types, registry helpers, and shared WebSocket session helper |
| `plugin-sdk/realtime-bootstrap-context` | Realtime profile bootstrap helper for bounded `IDENTITY.md`, `USER.md`, and `SOUL.md` context injection |
| `plugin-sdk/realtime-voice` | Realtime voice provider types, registry helpers, and shared realtime voice behavior helpers, including output activity tracking |
| `plugin-sdk/realtime-voice` | Realtime voice provider types, registry helpers, shared audio-energy/speech-onset gates, and realtime voice behavior helpers, including output activity tracking |
| `plugin-sdk/image-generation` | Image generation provider types plus image asset/data URL helpers and the OpenAI-compatible image provider builder |
| `plugin-sdk/image-generation-core` | Shared image-generation types, failover, auth, and registry helpers |
| `plugin-sdk/music-generation` | Music generation provider/request/result types |
extensions/google-meet/src/realtime-local-audio-transport.ts
+9 -26
View File
@@ -2,6 +2,7 @@ import { spawn } from "node:child_process";
import type { Writable } from "node:stream";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import type { RuntimeLogger } from "openclaw/plugin-sdk/plugin-runtime";
import { createSpeechThresholdGate, readPcm16AudioStats } from "openclaw/plugin-sdk/realtime-voice";
import type { MeetRealtimeAudioTransport } from "./realtime-audio-transport.js";
type BridgeProcess = {
@@ -66,23 +67,6 @@ function terminateBridgeProcess(proc: BridgeProcess, signal: NodeJS.Signals = "S
timer.unref?.();
}
function readPcm16Stats(audio: Buffer): { rms: number; peak: number } {
let sumSquares = 0;
let peak = 0;
let samples = 0;
for (let offset = 0; offset + 1 < audio.byteLength; offset += 2) {
const sample = audio.readInt16LE(offset);
const abs = Math.abs(sample);
peak = Math.max(peak, abs);
sumSquares += sample * sample;
samples += 1;
}
return {
rms: samples > 0 ? Math.sqrt(sumSquares / samples) : 0,
peak,
};
}
export function createLocalMeetRealtimeAudioTransport(params: {
inputCommand: string[];
outputCommand: string[];
@@ -229,24 +213,23 @@ export function createLocalMeetRealtimeAudioTransport(params: {
return;
}
const command = splitCommand(params.bargeInInputCommand ?? []);
let lastBargeInAt = 0;
const bargeInGate = createSpeechThresholdGate({
rmsThreshold: params.bargeInRmsThreshold,
peakThreshold: params.bargeInPeakThreshold,
cooldownMs: params.bargeInCooldownMs,
});
bargeInInputProcess = spawnFn(command.command, command.args, {
stdio: ["ignore", "pipe", "pipe"],
});
bargeInInputProcess.stdout?.on("data", (chunk) => {
const audio = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
const now = Date.now();
if (stopped || now - lastBargeInAt < params.bargeInCooldownMs) {
if (stopped) {
return;
}
const stats = readPcm16Stats(audio);
if (stats.rms < params.bargeInRmsThreshold && stats.peak < params.bargeInPeakThreshold) {
const stats = readPcm16AudioStats(audio);
if (!bargeInGate.accept(stats, { nowMs: Date.now(), onTrigger: () => onBargeIn(audio) })) {
return;
}
if (!onBargeIn(audio)) {
return;
}
lastBargeInAt = now;
params.logger.debug?.(
`[google-meet] human barge-in detected by local input (rms=${Math.round(
stats.rms,
extensions/voice-call/src/webhook/realtime-audio-pacer.test.ts
+1 -21
View File
@@ -1,6 +1,6 @@
// Voice Call tests cover realtime audio pacer plugin behavior.
import { afterEach, describe, expect, it, vi } from "vitest";
import { RealtimeAudioPacer, RealtimeMulawSpeechStartDetector } from "./realtime-audio-pacer.js";
import { RealtimeAudioPacer } from "./realtime-audio-pacer.js";
type RealtimeAudioSerializer = ConstructorParameters<typeof RealtimeAudioPacer>[0]["serializer"];
@@ -120,23 +120,3 @@ describe("RealtimeAudioPacer", () => {
]);
});
});
describe("RealtimeMulawSpeechStartDetector", () => {
it("detects a speech start after consecutive loud chunks and resets after quiet", () => {
const detector = new RealtimeMulawSpeechStartDetector({
requiredLoudChunks: 2,
requiredQuietChunks: 2,
rmsThreshold: 0.02,
});
const silence = Buffer.alloc(160, 0xff);
const speech = Buffer.alloc(160, 0x00);
expect(detector.accept(speech)).toBe(false);
expect(detector.accept(speech)).toBe(true);
expect(detector.accept(speech)).toBe(false);
expect(detector.accept(silence)).toBe(false);
expect(detector.accept(silence)).toBe(false);
expect(detector.accept(speech)).toBe(false);
expect(detector.accept(speech)).toBe(true);
});
});
extensions/voice-call/src/webhook/realtime-audio-pacer.ts
+1 -73
View File
@@ -1,18 +1,9 @@
// Realtime telephony audio pacing and speech-start detection for mulaw streams.
// Realtime telephony audio pacing for mulaw streams.
const TELEPHONY_SAMPLE_RATE = 8_000;
const TELEPHONY_CHUNK_BYTES = 160;
const TELEPHONY_CHUNK_MS = 20;
const DEFAULT_SPEECH_RMS_THRESHOLD = 0.035;
const DEFAULT_REQUIRED_LOUD_CHUNKS = 4;
const DEFAULT_REQUIRED_QUIET_CHUNKS = 12;
const DEFAULT_MAX_QUEUED_AUDIO_BYTES = TELEPHONY_SAMPLE_RATE * 120;
const PCM16_MAX_AMPLITUDE = 32768;
const MULAW_LINEAR_SAMPLES = new Int16Array(256);
for (let i = 0; i < MULAW_LINEAR_SAMPLES.length; i += 1) {
MULAW_LINEAR_SAMPLES[i] = decodeMulawSample(i);
}
/** Queue item sent over the realtime provider media stream. */
type RealtimeAudioQueueItem =
@@ -162,66 +153,3 @@ export class RealtimeAudioPacer {
}
}
}
/** Calculate normalized RMS from mulaw bytes. */
function calculateMulawRms(muLaw: Buffer): number {
if (muLaw.length === 0) {
return 0;
}
let sum = 0;
for (const sample of muLaw) {
const normalized = (MULAW_LINEAR_SAMPLES[sample] ?? 0) / PCM16_MAX_AMPLITUDE;
sum += normalized * normalized;
}
return Math.sqrt(sum / muLaw.length);
}
/** Detect likely speech start from consecutive loud mulaw chunks. */
export class RealtimeMulawSpeechStartDetector {
private loudChunks = 0;
private quietChunks = DEFAULT_REQUIRED_QUIET_CHUNKS;
private speaking = false;
constructor(
private readonly params: {
requiredLoudChunks?: number;
requiredQuietChunks?: number;
rmsThreshold?: number;
} = {},
) {}
/** Accept one mulaw chunk and return true only on transition into speaking. */
accept(muLaw: Buffer): boolean {
const rms = calculateMulawRms(muLaw);
const threshold = this.params.rmsThreshold ?? DEFAULT_SPEECH_RMS_THRESHOLD;
if (rms >= threshold) {
this.quietChunks = 0;
this.loudChunks += 1;
const requiredLoudChunks = this.params.requiredLoudChunks ?? DEFAULT_REQUIRED_LOUD_CHUNKS;
if (!this.speaking && this.loudChunks >= requiredLoudChunks) {
this.speaking = true;
return true;
}
return false;
}
this.loudChunks = 0;
this.quietChunks += 1;
const requiredQuietChunks = this.params.requiredQuietChunks ?? DEFAULT_REQUIRED_QUIET_CHUNKS;
if (this.quietChunks >= requiredQuietChunks) {
this.speaking = false;
}
return false;
}
}
/** Decode one G.711 mulaw byte to a linear PCM sample. */
function decodeMulawSample(value: number): number {
const muLaw = ~value & 0xff;
const sign = muLaw & 0x80;
const exponent = (muLaw >> 4) & 0x07;
const mantissa = muLaw & 0x0f;
let sample = ((mantissa << 3) + 132) << exponent;
sample -= 132;
return sign ? -sample : sample;
}
extensions/voice-call/src/webhook/realtime-handler.ts
+8 -4
View File
@@ -10,7 +10,9 @@ import {
} from "openclaw/plugin-sdk/number-runtime";
import {
buildRealtimeVoiceAgentConsultWorkingResponse,
calculateMulawRms,
createRealtimeVoiceForcedConsultCoordinator,
createSpeechThresholdGate,
createTalkSessionController,
createRealtimeVoiceBridgeSession,
REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
@@ -35,7 +37,7 @@ import type { CallManager } from "../manager.js";
import type { VoiceCallProvider } from "../providers/base.js";
import type { CallRecord, NormalizedEvent } from "../types.js";
import type { WebhookResponsePayload } from "../webhook.types.js";
import { RealtimeAudioPacer, RealtimeMulawSpeechStartDetector } from "./realtime-audio-pacer.js";
import { RealtimeAudioPacer } from "./realtime-audio-pacer.js";
import {
type StreamFrameAdapter,
TelnyxStreamFrameAdapter,
@@ -711,8 +713,10 @@ export class RealtimeCallHandler {
}
},
});
const speechDetector = new RealtimeMulawSpeechStartDetector({
requiredLoudChunks: BARGE_IN_REQUIRED_LOUD_CHUNKS,
const speechDetector = createSpeechThresholdGate({
rmsThreshold: 0.035,
speechFrames: BARGE_IN_REQUIRED_LOUD_CHUNKS,
silenceFrames: 12,
});
const interruptResponseOnInputAudio =
typeof this.providerConfig.interruptResponseOnInputAudio === "boolean"
@@ -947,7 +951,7 @@ export class RealtimeCallHandler {
this.activeTelephonyClosersByCallId.set(callSid, closeTelephony);
const sendAudioToSession = session.sendAudio.bind(session);
session.sendAudio = (audio) => {
if (speechDetector.accept(audio)) {
if (speechDetector.accept({ rms: calculateMulawRms(audio), peak: 0 })) {
console.log(
`[voice-call] realtime local speech detected callId=${callId} providerCallId=${callSid}`,
);
scripts/plugin-sdk-surface-report.mjs
+4 -2
View File
@@ -250,7 +250,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) {
// +4: bounded plugin blob store options, entry, entry info, and store types.
// +6: shared progress receipt tracker + compositor snapshot across channel barrels.
// +1: selectPreferredLocalModelId shares app-guided local model ranking across providers.
8006,
// +4: shared audio-energy stats and speech-threshold gate through realtime-voice.
8010,
env,
),
publicFunctionExports: readPluginSdkSurfaceBudgetEnv(
@@ -275,7 +276,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) {
// +3: lightweight speech settings normalizers and config resolver.
// +1: unified implicit-mention policy resolver.
// +1: selectPreferredLocalModelId shares app-guided local model ranking across providers.
4473,
// +3: PCM16/mu-law energy readers and speech-threshold gate factory.
4476,
env,
),
publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv(
src/plugin-sdk/realtime-voice.ts
+6
View File
@@ -183,6 +183,12 @@ export {
type RealtimeVoiceTranscriptEntry,
type RealtimeVoiceTranscriptHealth,
} from "../talk/session-log-runtime.js";
export {
calculateMulawRms,
createSpeechThresholdGate,
readPcm16AudioStats,
type AudioEnergyStats,
} from "../talk/audio-energy.js";
export {
convertPcmToMulaw8k,
mulawToPcm,
src/talk/audio-energy.test.ts
+65
View File
@@ -0,0 +1,65 @@
import { describe, expect, it, vi } from "vitest";
import {
calculateMulawRms,
createSpeechThresholdGate,
readPcm16AudioStats,
} from "./audio-energy.js";
function pcm16(...samples: number[]): Buffer {
const audio = Buffer.alloc(samples.length * Int16Array.BYTES_PER_ELEMENT);
samples.forEach((sample, index) => {
audio.writeInt16LE(sample, index * Int16Array.BYTES_PER_ELEMENT);
});
return audio;
}
describe("audio energy", () => {
it("reads silence and PCM16 tone-burst RMS/peak", () => {
expect(readPcm16AudioStats(Buffer.alloc(8))).toEqual({ rms: 0, peak: 0 });
expect(readPcm16AudioStats(pcm16(0, 1_000, -1_000, 0))).toEqual({
rms: Math.sqrt(500_000),
peak: 1_000,
});
});
it("matches the existing normalized G.711 mu-law levels", () => {
expect(calculateMulawRms(Buffer.alloc(160, 0xff))).toBe(0);
expect(calculateMulawRms(Buffer.alloc(160, 0x00))).toBe(32_124 / 32_768);
});
});
describe("speech threshold gate", () => {
it("fires on a sustained threshold-edge onset and rearms after quiet hold", () => {
const gate = createSpeechThresholdGate({
rmsThreshold: 10,
speechFrames: 2,
silenceFrames: 2,
});
const loud = { rms: 10, peak: 10 };
const quiet = { rms: 9, peak: 9 };
expect(gate.accept(loud)).toBe(false);
expect(gate.accept(loud)).toBe(true);
expect(gate.accept(loud)).toBe(false);
expect(gate.accept(quiet)).toBe(false);
expect(gate.accept(quiet)).toBe(false);
expect(gate.accept(loud)).toBe(false);
expect(gate.accept(loud)).toBe(true);
});
it("uses RMS-or-peak thresholds, caller veto, and cooldown re-trigger", () => {
const gate = createSpeechThresholdGate({
rmsThreshold: 10,
peakThreshold: 100,
cooldownMs: 1_000,
});
const onTrigger = vi.fn(() => true);
const peakOnly = { rms: 9, peak: 100 };
expect(gate.accept(peakOnly, { nowMs: 1_000, onTrigger: () => false })).toBe(false);
expect(gate.accept(peakOnly, { nowMs: 1_000, onTrigger })).toBe(true);
expect(gate.accept(peakOnly, { nowMs: 1_999, onTrigger })).toBe(false);
expect(gate.accept(peakOnly, { nowMs: 2_000, onTrigger })).toBe(true);
expect(onTrigger).toHaveBeenCalledTimes(2);
});
});
src/talk/audio-energy.ts
+87
View File
@@ -0,0 +1,87 @@
import { mulawToPcm } from "./audio-codec.js";
const PCM16_MAX_AMPLITUDE = 32_768;
const MULAW_LINEAR_SAMPLES = (() => {
const encoded = Buffer.from([...Array(256).keys()]);
const decoded = mulawToPcm(encoded);
return Int16Array.from(encoded, (_, index) => decoded.readInt16LE(index * 2));
})();
export type AudioEnergyStats = { peak: number; rms: number };
/** Read RMS and absolute peak from complete little-endian signed PCM16 samples. */
export function readPcm16AudioStats(audio: Buffer): AudioEnergyStats {
let sumSquares = 0;
let peak = 0;
const samples = Math.floor(audio.byteLength / 2);
for (let index = 0; index < samples; index += 1) {
const sample = audio.readInt16LE(index * 2);
peak = Math.max(peak, Math.abs(sample));
sumSquares += sample * sample;
}
return { rms: samples > 0 ? Math.sqrt(sumSquares / samples) : 0, peak };
}
/** Calculate normalized RMS from G.711 mu-law bytes. */
export function calculateMulawRms(muLaw: Buffer): number {
if (muLaw.length === 0) {
return 0;
}
let sumSquares = 0;
for (const encoded of muLaw) {
const normalized = (MULAW_LINEAR_SAMPLES[encoded] ?? 0) / PCM16_MAX_AMPLITUDE;
sumSquares += normalized * normalized;
}
return Math.sqrt(sumSquares / muLaw.length);
}
/** Build an OR-threshold gate with optional sustained onset, silence hold, and cooldown. */
export function createSpeechThresholdGate(options: {
cooldownMs?: number;
peakThreshold?: number;
rmsThreshold?: number;
silenceFrames?: number;
speechFrames?: number;
}) {
const speechFrames = Math.max(1, Math.floor(options.speechFrames ?? 1));
const silenceFrames = Math.max(0, Math.floor(options.silenceFrames ?? 0));
const cooldownMs = Math.max(0, options.cooldownMs ?? 0);
let loudFrames = 0;
let quietFrames = 0;
let speaking = false;
let lastTriggerAt = Number.NEGATIVE_INFINITY;
return {
accept(
stats: AudioEnergyStats,
acceptOptions: { nowMs?: number; onTrigger?: () => boolean } = {},
): boolean {
const loud =
(options.rmsThreshold !== undefined && stats.rms >= options.rmsThreshold) ||
(options.peakThreshold !== undefined && stats.peak >= options.peakThreshold);
if (!loud) {
loudFrames = 0;
if (speaking && ++quietFrames >= silenceFrames) {
speaking = false;
}
return false;
}
quietFrames = 0;
loudFrames += 1;
if (speaking || loudFrames < speechFrames) {
return false;
}
const nowMs = acceptOptions.nowMs ?? Date.now();
if (nowMs - lastTriggerAt < cooldownMs || acceptOptions.onTrigger?.() === false) {
return false;
}
lastTriggerAt = nowMs;
speaking = silenceFrames > 0;
if (!speaking) {
loudFrames = 0;
}
return true;
},
};
}