mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
perf(xai): lazy-load optional capability runtimes (#119374)
Punchcard-Session: coral-workshop-workshop-3f
This commit is contained in:
Vincent Koc
@@ -338,6 +338,7 @@ Use `isLoopbackHost(host)` when a plugin must accept only the local machine. It
|
||||
| `plugin-sdk/speech-settings` | Lightweight TTS config resolution and normalization primitives without provider registries or synthesis runtime |
|
||||
| `plugin-sdk/realtime-transcription` | Private-local after July 2026; Realtime transcription provider types, registry helpers, and shared WebSocket session helper |
|
||||
| `plugin-sdk/realtime-bootstrap-context` | Private-local after July 2026; Realtime profile bootstrap helper for bounded `IDENTITY.md`, `USER.md`, and `SOUL.md` context injection |
|
||||
| `plugin-sdk/realtime-voice-audio-queue` | Private-local JavaScript-only host runtime for bundled or separately published official plugins; narrow bounded audio queue seam for lazy realtime voice provider facades without importing the broader realtime voice runtime; not for third-party plugins |
|
||||
| `plugin-sdk/realtime-voice` | Private-local after July 2026; Realtime voice provider types, registry helpers, shared audio-energy/speech-onset gates, and realtime voice behavior helpers, including the transport-independent session harness and output activity tracking. For official runtime consumers, sender-auth contract revision 1 forwards ingress-authenticated `senderId` and `senderIsOwner` unchanged; ingress owns authentication, and consumers requiring the handoff must fail closed on other revisions. |
|
||||
| `plugin-sdk/meeting-runtime` | Browser-meeting session runtime, realtime audio engines/transports, `MeetingPlatformAdapter`, browser/node control, agent-consult, voice-call delegation, setup checks, and SoX command helpers |
|
||||
| `plugin-sdk/image-generation` | Private-local after July 2026; Image generation provider types plus image asset/data URL helpers and the OpenAI-compatible image provider builder |
|
||||
|
||||
@@ -9,7 +9,7 @@ import type {
|
||||
RealtimeVoiceProviderConfig,
|
||||
RealtimeVoiceProviderPlugin,
|
||||
} from "openclaw/plugin-sdk/realtime-voice";
|
||||
import { createRealtimeVoiceAudioQueue } from "openclaw/plugin-sdk/realtime-voice";
|
||||
import { createRealtimeVoiceAudioQueue } from "openclaw/plugin-sdk/realtime-voice-audio-queue";
|
||||
import { normalizeResolvedSecretInputString } from "openclaw/plugin-sdk/secret-input";
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import type { VideoGenerationProvider } from "openclaw/plugin-sdk/video-generation";
|
||||
|
||||
@@ -356,6 +356,9 @@
|
||||
"openclaw/plugin-sdk/realtime-bootstrap-context": [
|
||||
"../packages/plugin-sdk/dist/src/plugin-sdk/realtime-bootstrap-context.d.ts"
|
||||
],
|
||||
"openclaw/plugin-sdk/realtime-voice-audio-queue": [
|
||||
"../packages/plugin-sdk/dist/src/plugin-sdk/realtime-voice-audio-queue.d.ts"
|
||||
],
|
||||
"openclaw/plugin-sdk/realtime-voice": [
|
||||
"../packages/plugin-sdk/dist/src/plugin-sdk/realtime-voice.d.ts"
|
||||
],
|
||||
|
||||
+2
-51
@@ -1,16 +1,11 @@
|
||||
// Xai API module exposes the plugin public contract.
|
||||
import {
|
||||
normalizeOptionalLowercaseString,
|
||||
normalizeOptionalString,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { normalizeOptionalLowercaseString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import {
|
||||
applyXaiModelCompat,
|
||||
HTML_ENTITY_TOOL_CALL_ARGUMENTS_ENCODING,
|
||||
normalizeNativeXaiModelId,
|
||||
XAI_TOOL_SCHEMA_PROFILE,
|
||||
} from "./model-compat.js";
|
||||
import { XAI_BASE_URL } from "./model-definitions.js";
|
||||
import { isXaiProviderId } from "./provider-id.js";
|
||||
|
||||
export { buildXaiProvider } from "./provider-catalog.js";
|
||||
export { applyXaiConfig, applyXaiProviderConfig, XAI_DEFAULT_MODEL_REF } from "./onboard.js";
|
||||
@@ -29,22 +24,7 @@ export {
|
||||
export { isModernXaiModel, resolveXaiForwardCompatModel } from "./provider-models.js";
|
||||
export { applyXaiRuntimeModelCompat } from "./runtime-model-compat.js";
|
||||
export { applyXaiModelCompat, HTML_ENTITY_TOOL_CALL_ARGUMENTS_ENCODING, XAI_TOOL_SCHEMA_PROFILE };
|
||||
|
||||
const XAI_NATIVE_ENDPOINT_HOSTS = new Set(["api.x.ai"]);
|
||||
|
||||
function resolveHostname(value: string): string | undefined {
|
||||
try {
|
||||
return new URL(value).hostname.toLowerCase();
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function isXaiNativeEndpoint(baseUrl: unknown): boolean {
|
||||
return (
|
||||
typeof baseUrl === "string" && XAI_NATIVE_ENDPOINT_HOSTS.has(resolveHostname(baseUrl) ?? "")
|
||||
);
|
||||
}
|
||||
export { resolveXaiTransport } from "./provider-routing.js";
|
||||
|
||||
export function isXaiModelHint(modelId: string): boolean {
|
||||
return getModelProviderHint(modelId) === "x-ai";
|
||||
@@ -63,32 +43,3 @@ function getModelProviderHint(modelId: string): string | null {
|
||||
}
|
||||
return trimmed.slice(0, slashIndex) || null;
|
||||
}
|
||||
|
||||
function shouldUseXaiResponsesTransport(params: {
|
||||
provider: string;
|
||||
api?: unknown;
|
||||
baseUrl?: unknown;
|
||||
}): boolean {
|
||||
const hasDefaultXaiRoute =
|
||||
isXaiProviderId(params.provider) && !normalizeOptionalString(params.baseUrl);
|
||||
return params.api === "openai-responses"
|
||||
? hasDefaultXaiRoute
|
||||
: params.api === "openai-completions" &&
|
||||
(isXaiNativeEndpoint(params.baseUrl) || hasDefaultXaiRoute);
|
||||
}
|
||||
|
||||
export function resolveXaiTransport(params: {
|
||||
provider: string;
|
||||
api?: unknown;
|
||||
baseUrl?: unknown;
|
||||
}): { api: "openai-responses"; baseUrl?: string } | undefined {
|
||||
if (!shouldUseXaiResponsesTransport(params)) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
api: "openai-responses",
|
||||
baseUrl:
|
||||
normalizeOptionalString(params.baseUrl) ??
|
||||
(isXaiProviderId(params.provider) ? XAI_BASE_URL : undefined),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
import type { ImageGenerationProvider } from "openclaw/plugin-sdk/image-generation";
|
||||
import type { MediaUnderstandingProvider } from "openclaw/plugin-sdk/media-understanding";
|
||||
import {
|
||||
isProviderApiKeyConfigured,
|
||||
isProviderAuthProfileConfigured,
|
||||
} from "openclaw/plugin-sdk/provider-auth";
|
||||
import type {
|
||||
RealtimeTranscriptionProviderConfig,
|
||||
RealtimeTranscriptionProviderPlugin,
|
||||
} from "openclaw/plugin-sdk/realtime-transcription";
|
||||
import type {
|
||||
RealtimeVoiceAudioFormat,
|
||||
RealtimeVoiceBridgeCreateRequest,
|
||||
RealtimeVoiceProviderPlugin,
|
||||
} from "openclaw/plugin-sdk/realtime-voice";
|
||||
import { normalizeResolvedSecretInputString } from "openclaw/plugin-sdk/secret-input";
|
||||
import {
|
||||
isRecord,
|
||||
normalizeOptionalString,
|
||||
parseBooleanValue as readBoolean,
|
||||
parseFiniteNumber as readFiniteNumber,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import type {
|
||||
VideoGenerationProvider,
|
||||
VideoGenerationProviderCapabilities,
|
||||
} from "openclaw/plugin-sdk/video-generation";
|
||||
import { XAI_DEFAULT_IMAGE_MODEL, XAI_IMAGE_MODELS } from "./model-definitions.js";
|
||||
import {
|
||||
XAI_REALTIME_DEFAULT_MODEL,
|
||||
XAI_REALTIME_VOICES,
|
||||
hasXaiRealtimeApiKeyInput,
|
||||
normalizeXaiRealtimeProviderConfig,
|
||||
} from "./realtime-voice-config.js";
|
||||
|
||||
export const XAI_IMAGE_DEFAULT_TIMEOUT_MS = 600_000;
|
||||
export const XAI_SUPPORTED_IMAGE_ASPECT_RATIOS = [
|
||||
"1:1",
|
||||
"16:9",
|
||||
"9:16",
|
||||
"4:3",
|
||||
"3:4",
|
||||
"3:2",
|
||||
"2:3",
|
||||
"2:1",
|
||||
"1:2",
|
||||
"19.5:9",
|
||||
"9:19.5",
|
||||
"20:9",
|
||||
"9:20",
|
||||
] as const;
|
||||
|
||||
export function createXaiImageGenerationProviderMetadata() {
|
||||
return {
|
||||
id: "xai",
|
||||
label: "xAI",
|
||||
defaultModel: XAI_DEFAULT_IMAGE_MODEL,
|
||||
defaultTimeoutMs: XAI_IMAGE_DEFAULT_TIMEOUT_MS,
|
||||
models: [...XAI_IMAGE_MODELS],
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxCount: 4,
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
supportsSize: false,
|
||||
},
|
||||
edit: {
|
||||
enabled: true,
|
||||
maxCount: 4,
|
||||
maxInputImages: 3,
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
supportsSize: false,
|
||||
},
|
||||
geometry: {
|
||||
aspectRatios: [...XAI_SUPPORTED_IMAGE_ASPECT_RATIOS],
|
||||
resolutions: ["1K", "2K"],
|
||||
},
|
||||
},
|
||||
} satisfies Omit<ImageGenerationProvider, "generateImage">;
|
||||
}
|
||||
|
||||
export function createXaiMediaUnderstandingProviderMetadata() {
|
||||
return {
|
||||
id: "xai",
|
||||
capabilities: ["audio"],
|
||||
autoPriority: { audio: 25 },
|
||||
} satisfies Omit<MediaUnderstandingProvider, "transcribeAudio">;
|
||||
}
|
||||
|
||||
export const DEFAULT_XAI_VIDEO_BASE_URL = "https://api.x.ai/v1";
|
||||
export const DEFAULT_XAI_VIDEO_MODEL = "grok-imagine-video";
|
||||
const XAI_VIDEO_15_MODEL = "grok-imagine-video-1.5";
|
||||
export const XAI_VIDEO_DEFAULT_TIMEOUT_MS = 600_000;
|
||||
export const XAI_VIDEO_ASPECT_RATIOS = new Set(["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3"]);
|
||||
const XAI_VIDEO_15_CAPABILITIES = {
|
||||
imageToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputImages: 1,
|
||||
maxDurationSeconds: 15,
|
||||
aspectRatios: [...XAI_VIDEO_ASPECT_RATIOS],
|
||||
resolutions: ["480P", "720P", "1080P"],
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
},
|
||||
videoToVideo: {
|
||||
enabled: false,
|
||||
},
|
||||
} satisfies VideoGenerationProviderCapabilities;
|
||||
|
||||
const XAI_VIDEO_15_MODEL_IDS = new Set([
|
||||
XAI_VIDEO_15_MODEL,
|
||||
"grok-imagine-video-1.5-preview",
|
||||
"grok-imagine-video-1.5-2026-05-30",
|
||||
]);
|
||||
|
||||
export function isXaiVideo15Model(model: string | undefined): boolean {
|
||||
const normalized = normalizeOptionalString(model);
|
||||
return normalized ? XAI_VIDEO_15_MODEL_IDS.has(normalized) : false;
|
||||
}
|
||||
|
||||
export function createXaiVideoGenerationProviderMetadata() {
|
||||
return {
|
||||
id: "xai",
|
||||
label: "xAI",
|
||||
defaultModel: DEFAULT_XAI_VIDEO_MODEL,
|
||||
defaultTimeoutMs: XAI_VIDEO_DEFAULT_TIMEOUT_MS,
|
||||
models: [DEFAULT_XAI_VIDEO_MODEL, XAI_VIDEO_15_MODEL],
|
||||
catalogByModel: {
|
||||
[XAI_VIDEO_15_MODEL]: {
|
||||
capabilities: XAI_VIDEO_15_CAPABILITIES,
|
||||
modes: ["imageToVideo"],
|
||||
},
|
||||
},
|
||||
isConfigured: (ctx) => isProviderApiKeyConfigured({ provider: "xai", ...ctx }),
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxVideos: 1,
|
||||
maxDurationSeconds: 15,
|
||||
aspectRatios: [...XAI_VIDEO_ASPECT_RATIOS],
|
||||
resolutions: ["480P", "720P"],
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
},
|
||||
imageToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputImages: 7,
|
||||
maxDurationSeconds: 15,
|
||||
aspectRatios: [...XAI_VIDEO_ASPECT_RATIOS],
|
||||
resolutions: ["480P", "720P"],
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
},
|
||||
videoToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputVideos: 1,
|
||||
maxDurationSeconds: 10,
|
||||
supportsAspectRatio: false,
|
||||
supportsResolution: false,
|
||||
},
|
||||
},
|
||||
resolveModelCapabilities: ({ model }) =>
|
||||
isXaiVideo15Model(model) ? XAI_VIDEO_15_CAPABILITIES : undefined,
|
||||
} satisfies Omit<VideoGenerationProvider, "generateVideo">;
|
||||
}
|
||||
|
||||
export type XaiRealtimeTranscriptionEncoding = "pcm" | "mulaw" | "alaw";
|
||||
|
||||
type XaiRealtimeTranscriptionProviderConfig = {
|
||||
apiKey?: string;
|
||||
baseUrl?: string;
|
||||
sampleRate?: number;
|
||||
encoding?: XaiRealtimeTranscriptionEncoding;
|
||||
interimResults?: boolean;
|
||||
endpointingMs?: number;
|
||||
language?: string;
|
||||
};
|
||||
|
||||
function normalizeRealtimeTranscriptionEncoding(
|
||||
value: unknown,
|
||||
): XaiRealtimeTranscriptionEncoding | undefined {
|
||||
const normalized = normalizeOptionalString(value)?.toLowerCase();
|
||||
if (!normalized) {
|
||||
return undefined;
|
||||
}
|
||||
if (normalized === "ulaw" || normalized === "g711_ulaw" || normalized === "g711-mulaw") {
|
||||
return "mulaw";
|
||||
}
|
||||
if (normalized === "g711_alaw" || normalized === "g711-alaw") {
|
||||
return "alaw";
|
||||
}
|
||||
if (normalized === "pcm" || normalized === "mulaw" || normalized === "alaw") {
|
||||
return normalized;
|
||||
}
|
||||
throw new Error(`Invalid xAI realtime transcription encoding: ${normalized}`);
|
||||
}
|
||||
|
||||
export function normalizeXaiRealtimeTranscriptionProviderConfig(
|
||||
config: RealtimeTranscriptionProviderConfig,
|
||||
): XaiRealtimeTranscriptionProviderConfig {
|
||||
const raw = isRecord(config) ? config : undefined;
|
||||
const providers = isRecord(raw?.providers) ? raw.providers : undefined;
|
||||
const nested = providers?.xai ?? raw?.xai ?? raw;
|
||||
const xai = isRecord(nested) ? nested : {};
|
||||
return {
|
||||
apiKey: normalizeResolvedSecretInputString({
|
||||
value: xai.apiKey,
|
||||
path: "plugins.entries.voice-call.config.streaming.providers.xai.apiKey",
|
||||
}),
|
||||
baseUrl: normalizeOptionalString(xai.baseUrl),
|
||||
sampleRate: readFiniteNumber(xai.sampleRate ?? xai.sample_rate),
|
||||
encoding: normalizeRealtimeTranscriptionEncoding(xai.encoding),
|
||||
interimResults: readBoolean(xai.interimResults ?? xai.interim_results),
|
||||
endpointingMs: readFiniteNumber(xai.endpointingMs ?? xai.endpointing ?? xai.silenceDurationMs),
|
||||
language: normalizeOptionalString(xai.language),
|
||||
};
|
||||
}
|
||||
|
||||
export function createXaiRealtimeTranscriptionProviderMetadata() {
|
||||
return {
|
||||
id: "xai",
|
||||
label: "xAI Realtime Transcription",
|
||||
aliases: ["xai-realtime", "grok-stt-streaming"],
|
||||
autoSelectOrder: 25,
|
||||
resolveConfig: ({ rawConfig }) => normalizeXaiRealtimeTranscriptionProviderConfig(rawConfig),
|
||||
isConfigured: ({ providerConfig, cfg }) =>
|
||||
Boolean(
|
||||
normalizeXaiRealtimeTranscriptionProviderConfig(providerConfig).apiKey ??
|
||||
normalizeOptionalString(process.env.XAI_API_KEY),
|
||||
) || isProviderAuthProfileConfigured({ provider: "xai", cfg }),
|
||||
} satisfies Omit<RealtimeTranscriptionProviderPlugin, "createSession">;
|
||||
}
|
||||
|
||||
const XAI_REALTIME_AUDIO_FORMAT_G711_ULAW_8KHZ = {
|
||||
encoding: "g711_ulaw",
|
||||
sampleRateHz: 8000,
|
||||
channels: 1,
|
||||
} satisfies RealtimeVoiceAudioFormat;
|
||||
const XAI_REALTIME_AUDIO_FORMAT_PCM16_24KHZ = {
|
||||
encoding: "pcm16",
|
||||
sampleRateHz: 24000,
|
||||
channels: 1,
|
||||
} satisfies RealtimeVoiceAudioFormat;
|
||||
|
||||
export function createXaiRealtimeVoiceProviderMetadata() {
|
||||
return {
|
||||
id: "xai",
|
||||
label: "xAI Grok Voice",
|
||||
aliases: ["xai-realtime-voice", "grok-voice"],
|
||||
defaultModel: XAI_REALTIME_DEFAULT_MODEL,
|
||||
voices: XAI_REALTIME_VOICES,
|
||||
autoSelectOrder: 25,
|
||||
capabilities: {
|
||||
transports: ["gateway-relay"],
|
||||
inputAudioFormats: [
|
||||
XAI_REALTIME_AUDIO_FORMAT_G711_ULAW_8KHZ,
|
||||
XAI_REALTIME_AUDIO_FORMAT_PCM16_24KHZ,
|
||||
],
|
||||
outputAudioFormats: [
|
||||
XAI_REALTIME_AUDIO_FORMAT_G711_ULAW_8KHZ,
|
||||
XAI_REALTIME_AUDIO_FORMAT_PCM16_24KHZ,
|
||||
],
|
||||
supportsBargeIn: true,
|
||||
handlesInputAudioBargeIn: true,
|
||||
supportsToolCalls: true,
|
||||
supportsSessionResumption: true,
|
||||
},
|
||||
resolveConfig: ({ rawConfig }) => normalizeXaiRealtimeProviderConfig(rawConfig),
|
||||
isConfigured: ({ providerConfig, cfg }) =>
|
||||
hasXaiRealtimeApiKeyInput(normalizeXaiRealtimeProviderConfig(providerConfig).apiKey, cfg),
|
||||
} satisfies Omit<RealtimeVoiceProviderPlugin, "createBridge" | "createBrowserSession">;
|
||||
}
|
||||
|
||||
export function assertXaiRealtimeVoiceRequestSupported(
|
||||
req: RealtimeVoiceBridgeCreateRequest,
|
||||
): void {
|
||||
const config = normalizeXaiRealtimeProviderConfig(req.providerConfig);
|
||||
if (req.autoRespondToAudio === false) {
|
||||
throw new Error(
|
||||
'xAI realtime voice requires automatic server-VAD responses; use consultRouting: "provider-direct"',
|
||||
);
|
||||
}
|
||||
if ((req.interruptResponseOnInputAudio ?? config.interruptResponseOnInputAudio) === false) {
|
||||
throw new Error("xAI realtime voice requires automatic server-VAD interruption handling");
|
||||
}
|
||||
}
|
||||
@@ -12,25 +12,12 @@ import {
|
||||
normalizeOptionalLowercaseString,
|
||||
normalizeOptionalString,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { XAI_BASE_URL, XAI_DEFAULT_IMAGE_MODEL, XAI_IMAGE_MODELS } from "./model-definitions.js";
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 600_000;
|
||||
|
||||
const XAI_SUPPORTED_ASPECT_RATIOS = [
|
||||
"1:1",
|
||||
"16:9",
|
||||
"9:16",
|
||||
"4:3",
|
||||
"3:4",
|
||||
"3:2",
|
||||
"2:3",
|
||||
"2:1",
|
||||
"1:2",
|
||||
"19.5:9",
|
||||
"9:19.5",
|
||||
"20:9",
|
||||
"9:20",
|
||||
] as const;
|
||||
import {
|
||||
XAI_IMAGE_DEFAULT_TIMEOUT_MS,
|
||||
XAI_SUPPORTED_IMAGE_ASPECT_RATIOS,
|
||||
createXaiImageGenerationProviderMetadata,
|
||||
} from "./capability-provider-metadata.js";
|
||||
import { XAI_BASE_URL } from "./model-definitions.js";
|
||||
|
||||
function resolveImageForEdit(
|
||||
input: (ImageGenerationSourceImage & { url?: string }) | undefined,
|
||||
@@ -66,7 +53,7 @@ function buildBody(params: {
|
||||
};
|
||||
|
||||
const aspect = normalizeOptionalString(params.req.aspectRatio);
|
||||
if (aspect && (XAI_SUPPORTED_ASPECT_RATIOS as readonly string[]).includes(aspect)) {
|
||||
if (aspect && (XAI_SUPPORTED_IMAGE_ASPECT_RATIOS as readonly string[]).includes(aspect)) {
|
||||
body.aspect_ratio = aspect;
|
||||
}
|
||||
|
||||
@@ -93,35 +80,13 @@ function buildBody(params: {
|
||||
}
|
||||
|
||||
export function buildXaiImageGenerationProvider(): ImageGenerationProvider {
|
||||
const metadata = createXaiImageGenerationProviderMetadata();
|
||||
return createOpenAiCompatibleImageGenerationProvider({
|
||||
id: "xai",
|
||||
label: "xAI",
|
||||
defaultModel: XAI_DEFAULT_IMAGE_MODEL,
|
||||
models: [...XAI_IMAGE_MODELS],
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxCount: 4,
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
supportsSize: false,
|
||||
},
|
||||
edit: {
|
||||
enabled: true,
|
||||
maxCount: 4,
|
||||
maxInputImages: 3,
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
supportsSize: false,
|
||||
},
|
||||
geometry: {
|
||||
aspectRatios: [...XAI_SUPPORTED_ASPECT_RATIOS],
|
||||
resolutions: ["1K", "2K"],
|
||||
},
|
||||
},
|
||||
...metadata,
|
||||
defaultBaseUrl: XAI_BASE_URL,
|
||||
resolveBaseUrl: ({ req }) => resolveXaiImageBaseUrl(req),
|
||||
resolveAllowPrivateNetwork: () => false,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
defaultTimeoutMs: XAI_IMAGE_DEFAULT_TIMEOUT_MS,
|
||||
buildGenerateRequest: ({ req, inputImages, model, count }) => ({
|
||||
kind: "json",
|
||||
body: buildBody({ req, inputImages, model, count }),
|
||||
|
||||
+18
-18
@@ -5,15 +5,19 @@ import { defineSingleProviderPluginEntry } from "openclaw/plugin-sdk/provider-en
|
||||
import { buildProviderReplayFamilyHooks } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { defaultToolStreamExtraParams } from "openclaw/plugin-sdk/provider-stream-shared";
|
||||
import { jsonResult } from "openclaw/plugin-sdk/provider-web-search";
|
||||
import {
|
||||
buildXaiImageGenerationProvider,
|
||||
normalizeXaiModelId,
|
||||
resolveXaiTransport,
|
||||
} from "./api.js";
|
||||
import {
|
||||
buildMissingCodeExecutionApiKeyPayload,
|
||||
createCodeExecutionToolDefinition,
|
||||
} from "./code-execution-tool-shared.js";
|
||||
import {
|
||||
createLazyXaiImageGenerationProvider,
|
||||
createLazyXaiMediaUnderstandingProvider,
|
||||
createLazyXaiRealtimeTranscriptionProvider,
|
||||
createLazyXaiRealtimeVoiceProvider,
|
||||
createLazyXaiSpeechProvider,
|
||||
createLazyXaiVideoGenerationProvider,
|
||||
} from "./lazy-capability-providers.js";
|
||||
import { normalizeNativeXaiModelId } from "./model-compat.js";
|
||||
import { applyXaiConfig, XAI_DEFAULT_MODEL_REF } from "./onboard.js";
|
||||
import {
|
||||
buildLiveXaiOAuthProvider,
|
||||
@@ -27,9 +31,7 @@ import {
|
||||
resolveXaiForwardCompatModel,
|
||||
} from "./provider-models.js";
|
||||
import { resolveThinkingProfile } from "./provider-policy-api.js";
|
||||
import { buildXaiRealtimeTranscriptionProvider } from "./realtime-transcription-provider.js";
|
||||
import { buildXaiRealtimeVoiceProvider } from "./realtime-voice-provider.js";
|
||||
import { buildXaiSpeechProvider } from "./speech-provider.js";
|
||||
import { resolveXaiTransport } from "./provider-routing.js";
|
||||
import {
|
||||
readPluginCodeExecutionConfig,
|
||||
resolveCodeExecutionEnabled,
|
||||
@@ -41,8 +43,6 @@ import {
|
||||
} from "./src/tool-auth-shared.js";
|
||||
import { resolveEffectiveXSearchConfig } from "./src/x-search-config.js";
|
||||
import { wrapXaiProviderStream } from "./stream.js";
|
||||
import { buildXaiMediaUnderstandingProvider } from "./stt.js";
|
||||
import { buildXaiVideoGenerationProvider } from "./video-generation-provider.js";
|
||||
import { createXaiWebSearchProvider } from "./web-search.js";
|
||||
import {
|
||||
buildMissingXSearchApiKeyPayload,
|
||||
@@ -52,7 +52,7 @@ import {
|
||||
createXaiDeviceCodeAuthMethod,
|
||||
createXaiOAuthAuthMethod,
|
||||
refreshXaiOAuthCredential,
|
||||
} from "./xai-oauth.js";
|
||||
} from "./xai-oauth-entry.js";
|
||||
|
||||
const PROVIDER_ID = "xai";
|
||||
|
||||
@@ -278,7 +278,7 @@ export default defineSingleProviderPluginEntry({
|
||||
normalizeResolvedModel: ({ model }) => normalizeXaiResolvedModel(model),
|
||||
normalizeTransport: ({ provider, api, baseUrl }) =>
|
||||
resolveXaiTransport({ provider, api, baseUrl }),
|
||||
normalizeModelId: ({ modelId }) => normalizeXaiModelId(modelId),
|
||||
normalizeModelId: ({ modelId }) => normalizeNativeXaiModelId(modelId),
|
||||
resolveDynamicModel: (ctx) => resolveXaiForwardCompatModel({ providerId: PROVIDER_ID, ctx }),
|
||||
refreshOAuth: refreshXaiOAuthCredential,
|
||||
resolveThinkingProfile,
|
||||
@@ -287,12 +287,12 @@ export default defineSingleProviderPluginEntry({
|
||||
}),
|
||||
register(api) {
|
||||
api.registerWebSearchProvider(createXaiWebSearchProvider());
|
||||
api.registerMediaUnderstandingProvider(buildXaiMediaUnderstandingProvider());
|
||||
api.registerVideoGenerationProvider(buildXaiVideoGenerationProvider());
|
||||
api.registerImageGenerationProvider(buildXaiImageGenerationProvider());
|
||||
api.registerSpeechProvider(buildXaiSpeechProvider());
|
||||
api.registerRealtimeTranscriptionProvider(buildXaiRealtimeTranscriptionProvider());
|
||||
api.registerRealtimeVoiceProvider(buildXaiRealtimeVoiceProvider());
|
||||
api.registerMediaUnderstandingProvider(createLazyXaiMediaUnderstandingProvider());
|
||||
api.registerVideoGenerationProvider(createLazyXaiVideoGenerationProvider());
|
||||
api.registerImageGenerationProvider(createLazyXaiImageGenerationProvider());
|
||||
api.registerSpeechProvider(createLazyXaiSpeechProvider());
|
||||
api.registerRealtimeTranscriptionProvider(createLazyXaiRealtimeTranscriptionProvider());
|
||||
api.registerRealtimeVoiceProvider(createLazyXaiRealtimeVoiceProvider());
|
||||
api.registerTool((ctx) => createLazyCodeExecutionTool(ctx), { name: "code_execution" });
|
||||
api.registerTool((ctx) => createLazyXSearchTool(ctx), { name: "x_search" });
|
||||
},
|
||||
|
||||
@@ -0,0 +1,838 @@
|
||||
import type {
|
||||
RealtimeVoiceBridge,
|
||||
RealtimeVoiceBridgeCreateRequest,
|
||||
} from "openclaw/plugin-sdk/realtime-voice";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const runtimeMocks = vi.hoisted(() => {
|
||||
const generateImage = vi.fn();
|
||||
const transcribeAudio = vi.fn();
|
||||
const generateVideo = vi.fn();
|
||||
const listVoices = vi.fn();
|
||||
const synthesize = vi.fn();
|
||||
const streamSynthesize = vi.fn();
|
||||
const synthesizeTelephony = vi.fn();
|
||||
const transcriptionConnect = vi.fn();
|
||||
const transcriptionSendAudio = vi.fn();
|
||||
const transcriptionClose = vi.fn();
|
||||
const transcriptionIsConnected = vi.fn();
|
||||
const createTranscriptionSession = vi.fn();
|
||||
const voiceConnect = vi.fn();
|
||||
const voiceSendAudio = vi.fn();
|
||||
const voiceSetMediaTimestamp = vi.fn();
|
||||
const voiceSendUserMessage = vi.fn();
|
||||
const voiceTriggerGreeting = vi.fn();
|
||||
const voiceHandleBargeIn = vi.fn();
|
||||
const voiceSubmitToolResult = vi.fn();
|
||||
const voiceAcknowledgeMark = vi.fn();
|
||||
const voiceClose = vi.fn();
|
||||
const voiceIsConnected = vi.fn();
|
||||
const createVoiceBridge = vi.fn();
|
||||
const buildImageProvider = vi.fn();
|
||||
const buildMediaProvider = vi.fn();
|
||||
const buildVideoProvider = vi.fn();
|
||||
const buildSpeechProvider = vi.fn();
|
||||
const buildTranscriptionProvider = vi.fn();
|
||||
const buildVoiceProvider = vi.fn();
|
||||
|
||||
return {
|
||||
generateImage,
|
||||
transcribeAudio,
|
||||
generateVideo,
|
||||
listVoices,
|
||||
synthesize,
|
||||
streamSynthesize,
|
||||
synthesizeTelephony,
|
||||
transcriptionConnect,
|
||||
transcriptionSendAudio,
|
||||
transcriptionClose,
|
||||
transcriptionIsConnected,
|
||||
createTranscriptionSession,
|
||||
voiceConnect,
|
||||
voiceSendAudio,
|
||||
voiceSetMediaTimestamp,
|
||||
voiceSendUserMessage,
|
||||
voiceTriggerGreeting,
|
||||
voiceHandleBargeIn,
|
||||
voiceSubmitToolResult,
|
||||
voiceAcknowledgeMark,
|
||||
voiceClose,
|
||||
voiceIsConnected,
|
||||
createVoiceBridge,
|
||||
buildImageProvider,
|
||||
buildMediaProvider,
|
||||
buildVideoProvider,
|
||||
buildSpeechProvider,
|
||||
buildTranscriptionProvider,
|
||||
buildVoiceProvider,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("./image-generation-provider.js", () => ({
|
||||
buildXaiImageGenerationProvider: runtimeMocks.buildImageProvider,
|
||||
}));
|
||||
vi.mock("./stt.js", () => ({
|
||||
buildXaiMediaUnderstandingProvider: runtimeMocks.buildMediaProvider,
|
||||
}));
|
||||
vi.mock("./video-generation-provider.js", () => ({
|
||||
buildXaiVideoGenerationProvider: runtimeMocks.buildVideoProvider,
|
||||
}));
|
||||
vi.mock("./speech-provider.js", () => ({
|
||||
buildXaiSpeechProvider: runtimeMocks.buildSpeechProvider,
|
||||
}));
|
||||
vi.mock("./realtime-transcription-provider.js", () => ({
|
||||
buildXaiRealtimeTranscriptionProvider: runtimeMocks.buildTranscriptionProvider,
|
||||
}));
|
||||
vi.mock("./realtime-voice-provider.js", () => ({
|
||||
buildXaiRealtimeVoiceProvider: runtimeMocks.buildVoiceProvider,
|
||||
}));
|
||||
|
||||
function createDeferred<T>() {
|
||||
let resolve!: (value: T | PromiseLike<T>) => void;
|
||||
let reject!: (reason?: unknown) => void;
|
||||
const promise = new Promise<T>((resolvePromise, rejectPromise) => {
|
||||
resolve = resolvePromise;
|
||||
reject = rejectPromise;
|
||||
});
|
||||
return { promise, reject, resolve };
|
||||
}
|
||||
|
||||
async function loadLazyProviders() {
|
||||
return await import("./lazy-capability-providers.js");
|
||||
}
|
||||
|
||||
function createVoiceRequest(
|
||||
overrides: Partial<RealtimeVoiceBridgeCreateRequest> = {},
|
||||
): RealtimeVoiceBridgeCreateRequest {
|
||||
return {
|
||||
providerConfig: {},
|
||||
onAudio() {},
|
||||
onClearAudio() {},
|
||||
onError() {},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
for (const value of Object.values(runtimeMocks)) {
|
||||
value.mockReset();
|
||||
}
|
||||
|
||||
runtimeMocks.generateImage.mockResolvedValue({ images: [] });
|
||||
runtimeMocks.transcribeAudio.mockResolvedValue({ text: "transcript" });
|
||||
runtimeMocks.generateVideo.mockResolvedValue({ videos: [] });
|
||||
runtimeMocks.listVoices.mockResolvedValue([]);
|
||||
runtimeMocks.synthesize.mockResolvedValue({ audioBuffer: Buffer.alloc(0) });
|
||||
runtimeMocks.streamSynthesize.mockResolvedValue({ audioStream: {} });
|
||||
runtimeMocks.synthesizeTelephony.mockResolvedValue({ audioBuffer: Buffer.alloc(0) });
|
||||
runtimeMocks.transcriptionConnect.mockResolvedValue(undefined);
|
||||
runtimeMocks.transcriptionIsConnected.mockReturnValue(false);
|
||||
runtimeMocks.voiceConnect.mockResolvedValue(undefined);
|
||||
runtimeMocks.voiceIsConnected.mockReturnValue(false);
|
||||
|
||||
runtimeMocks.createTranscriptionSession.mockReturnValue({
|
||||
connect: runtimeMocks.transcriptionConnect,
|
||||
sendAudio: runtimeMocks.transcriptionSendAudio,
|
||||
close: runtimeMocks.transcriptionClose,
|
||||
isConnected: runtimeMocks.transcriptionIsConnected,
|
||||
});
|
||||
runtimeMocks.createVoiceBridge.mockImplementation(
|
||||
() =>
|
||||
({
|
||||
supportsToolResultContinuation: false,
|
||||
connect: runtimeMocks.voiceConnect,
|
||||
sendAudio: runtimeMocks.voiceSendAudio,
|
||||
setMediaTimestamp: runtimeMocks.voiceSetMediaTimestamp,
|
||||
sendUserMessage: runtimeMocks.voiceSendUserMessage,
|
||||
triggerGreeting: runtimeMocks.voiceTriggerGreeting,
|
||||
handleBargeIn: runtimeMocks.voiceHandleBargeIn,
|
||||
submitToolResult: runtimeMocks.voiceSubmitToolResult,
|
||||
acknowledgeMark: runtimeMocks.voiceAcknowledgeMark,
|
||||
close: runtimeMocks.voiceClose,
|
||||
isConnected: runtimeMocks.voiceIsConnected,
|
||||
}) satisfies RealtimeVoiceBridge,
|
||||
);
|
||||
|
||||
runtimeMocks.buildImageProvider.mockReturnValue({
|
||||
generateImage: runtimeMocks.generateImage,
|
||||
});
|
||||
runtimeMocks.buildMediaProvider.mockReturnValue({
|
||||
transcribeAudio: runtimeMocks.transcribeAudio,
|
||||
});
|
||||
runtimeMocks.buildVideoProvider.mockReturnValue({
|
||||
generateVideo: runtimeMocks.generateVideo,
|
||||
});
|
||||
runtimeMocks.buildSpeechProvider.mockReturnValue({
|
||||
listVoices: runtimeMocks.listVoices,
|
||||
synthesize: runtimeMocks.synthesize,
|
||||
streamSynthesize: runtimeMocks.streamSynthesize,
|
||||
synthesizeTelephony: runtimeMocks.synthesizeTelephony,
|
||||
});
|
||||
runtimeMocks.buildTranscriptionProvider.mockReturnValue({
|
||||
createSession: runtimeMocks.createTranscriptionSession,
|
||||
});
|
||||
runtimeMocks.buildVoiceProvider.mockReturnValue({
|
||||
createBridge: runtimeMocks.createVoiceBridge,
|
||||
});
|
||||
});
|
||||
|
||||
describe("xAI lazy capability providers", () => {
|
||||
it("keeps heavy builders unloaded until their capability methods run", async () => {
|
||||
const lazy = await loadLazyProviders();
|
||||
const image = lazy.createLazyXaiImageGenerationProvider();
|
||||
const media = lazy.createLazyXaiMediaUnderstandingProvider();
|
||||
const video = lazy.createLazyXaiVideoGenerationProvider();
|
||||
const speech = lazy.createLazyXaiSpeechProvider();
|
||||
const transcription = lazy.createLazyXaiRealtimeTranscriptionProvider();
|
||||
const voice = lazy.createLazyXaiRealtimeVoiceProvider();
|
||||
|
||||
expect(
|
||||
[
|
||||
runtimeMocks.buildImageProvider,
|
||||
runtimeMocks.buildMediaProvider,
|
||||
runtimeMocks.buildVideoProvider,
|
||||
runtimeMocks.buildSpeechProvider,
|
||||
runtimeMocks.buildTranscriptionProvider,
|
||||
runtimeMocks.buildVoiceProvider,
|
||||
].map((mock) => mock.mock.calls.length),
|
||||
).toEqual([0, 0, 0, 0, 0, 0]);
|
||||
expect(transcription.label).toBe("xAI Realtime Transcription");
|
||||
expect(voice.label).toBe("xAI Grok Voice");
|
||||
|
||||
await image.generateImage({} as never);
|
||||
await media.transcribeAudio?.({} as never);
|
||||
await video.generateVideo({} as never);
|
||||
await speech.synthesize({} as never);
|
||||
await speech.listVoices?.({} as never);
|
||||
|
||||
expect(runtimeMocks.buildImageProvider).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.buildMediaProvider).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.buildVideoProvider).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.buildSpeechProvider).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.generateImage).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.transcribeAudio).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.generateVideo).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.synthesize).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.listVoices).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("keeps the newest transcription audio ordered while the runtime loads", async () => {
|
||||
const lazy = await loadLazyProviders();
|
||||
const session = lazy.createLazyXaiRealtimeTranscriptionProvider().createSession({
|
||||
providerConfig: {},
|
||||
});
|
||||
const first = Buffer.alloc(1024 * 1024, 0x01);
|
||||
const second = Buffer.alloc(1024 * 1024, 0x02);
|
||||
const third = Buffer.alloc(1024 * 1024, 0x03);
|
||||
|
||||
session.sendAudio(first);
|
||||
session.sendAudio(second);
|
||||
session.sendAudio(third);
|
||||
await session.connect();
|
||||
|
||||
expect(runtimeMocks.buildTranscriptionProvider).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.transcriptionSendAudio.mock.calls.map(([audio]) => audio)).toEqual([
|
||||
second,
|
||||
third,
|
||||
]);
|
||||
expect(runtimeMocks.transcriptionConnect).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.transcriptionSendAudio.mock.invocationCallOrder.at(-1)).toBeLessThan(
|
||||
runtimeMocks.transcriptionConnect.mock.invocationCallOrder[0]!,
|
||||
);
|
||||
});
|
||||
|
||||
it("closes a transcription session that finishes loading after the wrapper closes", async () => {
|
||||
const lazy = await loadLazyProviders();
|
||||
const session = lazy.createLazyXaiRealtimeTranscriptionProvider().createSession({
|
||||
providerConfig: {},
|
||||
});
|
||||
|
||||
const connectPromise = session.connect();
|
||||
session.close();
|
||||
session.close();
|
||||
await connectPromise;
|
||||
|
||||
expect(runtimeMocks.createTranscriptionSession).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.transcriptionConnect).not.toHaveBeenCalled();
|
||||
expect(runtimeMocks.transcriptionClose).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("reopens transcription after close without replaying discarded audio", async () => {
|
||||
const lazy = await loadLazyProviders();
|
||||
const session = lazy.createLazyXaiRealtimeTranscriptionProvider().createSession({
|
||||
providerConfig: {},
|
||||
});
|
||||
const first = Buffer.from([0x01]);
|
||||
const discarded = Buffer.from([0x02]);
|
||||
const second = Buffer.from([0x03]);
|
||||
|
||||
session.sendAudio(first);
|
||||
await session.connect();
|
||||
session.close();
|
||||
session.close();
|
||||
session.sendAudio(discarded);
|
||||
|
||||
const reconnectPromise = session.connect();
|
||||
session.sendAudio(second);
|
||||
await reconnectPromise;
|
||||
|
||||
expect(runtimeMocks.transcriptionConnect).toHaveBeenCalledTimes(2);
|
||||
expect(runtimeMocks.transcriptionClose).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.transcriptionSendAudio.mock.calls.map(([audio]) => audio)).toEqual([
|
||||
first,
|
||||
second,
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves voice startup ordering and waits to trigger the greeting", async () => {
|
||||
const connecting = createDeferred<void>();
|
||||
const forwarded: string[] = [];
|
||||
runtimeMocks.voiceConnect.mockReturnValue(connecting.promise);
|
||||
runtimeMocks.voiceSendAudio.mockImplementation((audio: Buffer) => {
|
||||
forwarded.push(`audio:${audio[0]}`);
|
||||
});
|
||||
runtimeMocks.voiceSetMediaTimestamp.mockImplementation((timestamp: number) => {
|
||||
forwarded.push(`timestamp:${timestamp}`);
|
||||
});
|
||||
runtimeMocks.voiceSendUserMessage.mockImplementation((text: string) => {
|
||||
forwarded.push(`user:${text}`);
|
||||
});
|
||||
runtimeMocks.voiceSubmitToolResult.mockImplementation((callId: string) => {
|
||||
forwarded.push(`tool:${callId}`);
|
||||
});
|
||||
runtimeMocks.voiceTriggerGreeting.mockImplementation((instructions?: string) => {
|
||||
forwarded.push(`greeting:${instructions ?? ""}`);
|
||||
});
|
||||
const lazy = await loadLazyProviders();
|
||||
const bridge = lazy.createLazyXaiRealtimeVoiceProvider().createBridge(createVoiceRequest());
|
||||
const first = Buffer.from([0x01]);
|
||||
const second = Buffer.from([0x02]);
|
||||
|
||||
bridge.sendAudio(first);
|
||||
bridge.setMediaTimestamp(42);
|
||||
bridge.sendUserMessage?.("hello");
|
||||
await bridge.submitToolResult("call-1", { ok: true });
|
||||
bridge.triggerGreeting?.("welcome");
|
||||
const connectPromise = bridge.connect();
|
||||
await vi.waitFor(() => expect(runtimeMocks.voiceConnect).toHaveBeenCalledOnce());
|
||||
bridge.sendAudio(second);
|
||||
|
||||
expect(runtimeMocks.voiceSetMediaTimestamp).not.toHaveBeenCalled();
|
||||
expect(runtimeMocks.voiceSendAudio).not.toHaveBeenCalled();
|
||||
expect(runtimeMocks.voiceSendUserMessage).not.toHaveBeenCalled();
|
||||
expect(runtimeMocks.voiceSubmitToolResult).not.toHaveBeenCalled();
|
||||
expect(runtimeMocks.voiceTriggerGreeting).not.toHaveBeenCalled();
|
||||
|
||||
connecting.resolve();
|
||||
await connectPromise;
|
||||
expect(runtimeMocks.voiceSetMediaTimestamp).toHaveBeenCalledWith(42);
|
||||
expect(runtimeMocks.voiceSendAudio.mock.calls.map(([audio]) => audio)).toEqual([first, second]);
|
||||
expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledWith("hello");
|
||||
expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledWith(
|
||||
"call-1",
|
||||
{ ok: true },
|
||||
undefined,
|
||||
);
|
||||
expect(runtimeMocks.voiceTriggerGreeting).toHaveBeenCalledWith("welcome");
|
||||
expect(forwarded).toEqual([
|
||||
"audio:1",
|
||||
"timestamp:42",
|
||||
"user:hello",
|
||||
"tool:call-1",
|
||||
"greeting:welcome",
|
||||
"audio:2",
|
||||
]);
|
||||
});
|
||||
|
||||
it("moves the latest pending timestamp and greeting to the operation tail", async () => {
|
||||
const forwarded: string[] = [];
|
||||
runtimeMocks.voiceSetMediaTimestamp.mockImplementation((timestamp: number) => {
|
||||
forwarded.push(`timestamp:${timestamp}`);
|
||||
});
|
||||
runtimeMocks.voiceSendUserMessage.mockImplementation((text: string) => {
|
||||
forwarded.push(`user:${text}`);
|
||||
});
|
||||
runtimeMocks.voiceSendAudio.mockImplementation((audio: Buffer) => {
|
||||
forwarded.push(`audio:${audio[0]}`);
|
||||
});
|
||||
runtimeMocks.voiceTriggerGreeting.mockImplementation((instructions?: string) => {
|
||||
forwarded.push(`greeting:${String(instructions)}`);
|
||||
});
|
||||
const lazy = await loadLazyProviders();
|
||||
const bridge = lazy.createLazyXaiRealtimeVoiceProvider().createBridge(createVoiceRequest());
|
||||
|
||||
bridge.setMediaTimestamp(1);
|
||||
bridge.sendUserMessage?.("middle");
|
||||
bridge.setMediaTimestamp(2);
|
||||
bridge.triggerGreeting?.("superseded");
|
||||
bridge.sendAudio(Buffer.from([0x03]));
|
||||
bridge.triggerGreeting?.();
|
||||
await bridge.connect();
|
||||
|
||||
expect(forwarded).toEqual(["user:middle", "timestamp:2", "audio:3", "greeting:undefined"]);
|
||||
|
||||
runtimeMocks.voiceTriggerGreeting.mockClear();
|
||||
runtimeMocks.voiceIsConnected.mockReturnValue(false);
|
||||
bridge.triggerGreeting?.("provider-owned-reconnect");
|
||||
expect(runtimeMocks.voiceTriggerGreeting).toHaveBeenCalledWith("provider-owned-reconnect");
|
||||
});
|
||||
|
||||
it("bounds pending voice user messages by aggregate bytes", async () => {
|
||||
const lazy = await loadLazyProviders();
|
||||
const onError = vi.fn();
|
||||
const bridge = lazy
|
||||
.createLazyXaiRealtimeVoiceProvider()
|
||||
.createBridge(createVoiceRequest({ onError }));
|
||||
const accepted = "a".repeat(200 * 1024);
|
||||
|
||||
bridge.sendUserMessage?.(accepted);
|
||||
bridge.sendUserMessage?.("b".repeat(64 * 1024));
|
||||
await bridge.connect();
|
||||
|
||||
expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledWith(accepted);
|
||||
expect(onError).toHaveBeenCalledOnce();
|
||||
expect(onError.mock.calls[0]?.[0]).toEqual(
|
||||
new Error("xAI realtime voice pending user message overflow during lazy startup"),
|
||||
);
|
||||
});
|
||||
|
||||
it("bounds pending voice tool results by aggregate serialized bytes", async () => {
|
||||
const lazy = await loadLazyProviders();
|
||||
const onError = vi.fn();
|
||||
const bridge = lazy
|
||||
.createLazyXaiRealtimeVoiceProvider()
|
||||
.createBridge(createVoiceRequest({ onError }));
|
||||
const accepted = { text: "a".repeat(200 * 1024) };
|
||||
|
||||
await bridge.submitToolResult("call-1", accepted);
|
||||
await bridge.submitToolResult("call-2", { text: "b".repeat(64 * 1024) });
|
||||
await bridge.connect();
|
||||
|
||||
expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledWith("call-1", accepted, undefined);
|
||||
expect(onError).toHaveBeenCalledOnce();
|
||||
expect(onError.mock.calls[0]?.[0]).toEqual(
|
||||
new Error("xAI realtime voice pending tool result overflow during lazy startup"),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps voice payloads byte-bounded until the underlying connect resolves", async () => {
|
||||
const connecting = createDeferred<void>();
|
||||
runtimeMocks.voiceConnect.mockReturnValue(connecting.promise);
|
||||
const lazy = await loadLazyProviders();
|
||||
const onError = vi.fn();
|
||||
const bridge = lazy
|
||||
.createLazyXaiRealtimeVoiceProvider()
|
||||
.createBridge(createVoiceRequest({ onError }));
|
||||
const acceptedMessage = "a".repeat(200 * 1024);
|
||||
const acceptedResult = { text: "b".repeat(200 * 1024) };
|
||||
|
||||
const connectPromise = bridge.connect();
|
||||
await vi.waitFor(() => expect(runtimeMocks.voiceConnect).toHaveBeenCalledOnce());
|
||||
bridge.sendUserMessage?.(acceptedMessage);
|
||||
bridge.sendUserMessage?.("c".repeat(64 * 1024));
|
||||
await bridge.submitToolResult("call-1", acceptedResult);
|
||||
await bridge.submitToolResult("call-2", { text: "d".repeat(64 * 1024) });
|
||||
|
||||
expect(runtimeMocks.voiceSendUserMessage).not.toHaveBeenCalled();
|
||||
expect(runtimeMocks.voiceSubmitToolResult).not.toHaveBeenCalled();
|
||||
expect(onError.mock.calls.map(([error]) => (error as Error).message)).toEqual([
|
||||
"xAI realtime voice pending user message overflow during lazy startup",
|
||||
"xAI realtime voice pending tool result overflow during lazy startup",
|
||||
]);
|
||||
|
||||
connecting.resolve();
|
||||
await connectPromise;
|
||||
|
||||
expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledWith(acceptedMessage);
|
||||
expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledWith(
|
||||
"call-1",
|
||||
acceptedResult,
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it("drains voice input queued while an earlier tool result is submitting", async () => {
|
||||
const submitting = createDeferred<void>();
|
||||
const forwarded: string[] = [];
|
||||
runtimeMocks.voiceSubmitToolResult
|
||||
.mockImplementationOnce((callId: string) => {
|
||||
forwarded.push(`tool:${callId}`);
|
||||
return submitting.promise;
|
||||
})
|
||||
.mockImplementation((callId: string) => {
|
||||
forwarded.push(`tool:${callId}`);
|
||||
});
|
||||
runtimeMocks.voiceSendUserMessage.mockImplementation((text: string) => {
|
||||
forwarded.push(`user:${text}`);
|
||||
});
|
||||
runtimeMocks.voiceSetMediaTimestamp.mockImplementation((timestamp: number) => {
|
||||
forwarded.push(`timestamp:${timestamp}`);
|
||||
});
|
||||
const lazy = await loadLazyProviders();
|
||||
const bridge = lazy.createLazyXaiRealtimeVoiceProvider().createBridge(createVoiceRequest());
|
||||
|
||||
await bridge.submitToolResult("call-1", { text: "first" });
|
||||
const connectPromise = bridge.connect();
|
||||
await vi.waitFor(() => expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledOnce());
|
||||
bridge.sendUserMessage?.("arrived-during-flush");
|
||||
bridge.setMediaTimestamp(84);
|
||||
await bridge.submitToolResult("call-2", { text: "second" });
|
||||
submitting.resolve();
|
||||
await connectPromise;
|
||||
|
||||
expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledWith("arrived-during-flush");
|
||||
expect(runtimeMocks.voiceSetMediaTimestamp).toHaveBeenLastCalledWith(84);
|
||||
expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledTimes(2);
|
||||
expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenLastCalledWith(
|
||||
"call-2",
|
||||
{ text: "second" },
|
||||
undefined,
|
||||
);
|
||||
expect(forwarded).toEqual([
|
||||
"tool:call-1",
|
||||
"user:arrived-during-flush",
|
||||
"timestamp:84",
|
||||
"tool:call-2",
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps an in-flight voice tool result charged against the startup byte cap", async () => {
|
||||
const submitting = createDeferred<void>();
|
||||
runtimeMocks.voiceSubmitToolResult.mockReturnValueOnce(submitting.promise);
|
||||
const lazy = await loadLazyProviders();
|
||||
const onError = vi.fn();
|
||||
const bridge = lazy
|
||||
.createLazyXaiRealtimeVoiceProvider()
|
||||
.createBridge(createVoiceRequest({ onError }));
|
||||
|
||||
await bridge.submitToolResult("call-1", { text: "a".repeat(200 * 1024) });
|
||||
const connectPromise = bridge.connect();
|
||||
await vi.waitFor(() => expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledOnce());
|
||||
await bridge.submitToolResult("call-2", { text: "b".repeat(64 * 1024) });
|
||||
|
||||
expect(onError.mock.calls[0]?.[0]).toEqual(
|
||||
new Error("xAI realtime voice pending tool result overflow during lazy startup"),
|
||||
);
|
||||
expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledOnce();
|
||||
|
||||
submitting.resolve();
|
||||
await connectPromise;
|
||||
expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("forwards all voice input admitted during the final connect handoff exactly once", async () => {
|
||||
const connecting = createDeferred<void>();
|
||||
runtimeMocks.voiceConnect.mockReturnValue(connecting.promise);
|
||||
runtimeMocks.voiceIsConnected.mockReturnValue(true);
|
||||
const lazy = await loadLazyProviders();
|
||||
const bridge = lazy.createLazyXaiRealtimeVoiceProvider().createBridge(createVoiceRequest());
|
||||
const audio = Buffer.from([0x01]);
|
||||
|
||||
const firstConnect = bridge.connect();
|
||||
const secondConnect = bridge.connect();
|
||||
await vi.waitFor(() => expect(runtimeMocks.voiceConnect).toHaveBeenCalledOnce());
|
||||
connecting.resolve();
|
||||
queueMicrotask(() => {
|
||||
bridge.sendAudio(audio);
|
||||
bridge.setMediaTimestamp(84);
|
||||
bridge.sendUserMessage?.("arrived-during-handoff");
|
||||
void bridge.submitToolResult("call-1", { text: "tool-result" });
|
||||
bridge.triggerGreeting?.("welcome");
|
||||
});
|
||||
await Promise.all([firstConnect, secondConnect]);
|
||||
|
||||
expect(runtimeMocks.voiceSendAudio).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.voiceSendAudio).toHaveBeenCalledWith(audio);
|
||||
expect(runtimeMocks.voiceSetMediaTimestamp).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.voiceSetMediaTimestamp).toHaveBeenCalledWith(84);
|
||||
expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledWith("arrived-during-handoff");
|
||||
expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledWith(
|
||||
"call-1",
|
||||
{ text: "tool-result" },
|
||||
undefined,
|
||||
);
|
||||
expect(runtimeMocks.voiceTriggerGreeting).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.voiceTriggerGreeting).toHaveBeenCalledWith("welcome");
|
||||
});
|
||||
|
||||
it("clears pending voice byte budgets when closed before connect", async () => {
|
||||
const lazy = await loadLazyProviders();
|
||||
const onError = vi.fn();
|
||||
const bridge = lazy
|
||||
.createLazyXaiRealtimeVoiceProvider()
|
||||
.createBridge(createVoiceRequest({ onError }));
|
||||
|
||||
bridge.sendUserMessage?.("stale".repeat(40 * 1024));
|
||||
bridge.setMediaTimestamp(42);
|
||||
await bridge.submitToolResult("stale-call", { text: "x".repeat(200 * 1024) });
|
||||
bridge.close();
|
||||
|
||||
const connectPromise = bridge.connect();
|
||||
bridge.sendUserMessage?.("fresh".repeat(40 * 1024));
|
||||
await bridge.submitToolResult("fresh-call", { text: "y".repeat(200 * 1024) });
|
||||
await connectPromise;
|
||||
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
expect(runtimeMocks.voiceSetMediaTimestamp).not.toHaveBeenCalled();
|
||||
expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledWith("fresh".repeat(40 * 1024));
|
||||
expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledWith(
|
||||
"fresh-call",
|
||||
{ text: "y".repeat(200 * 1024) },
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it("closes a voice bridge that finishes loading after the wrapper closes", async () => {
|
||||
const lazy = await loadLazyProviders();
|
||||
const onClose = vi.fn();
|
||||
const bridge = lazy
|
||||
.createLazyXaiRealtimeVoiceProvider()
|
||||
.createBridge(createVoiceRequest({ onClose }));
|
||||
|
||||
const connectPromise = bridge.connect();
|
||||
bridge.close();
|
||||
bridge.close();
|
||||
await connectPromise;
|
||||
|
||||
expect(runtimeMocks.createVoiceBridge).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.voiceConnect).not.toHaveBeenCalled();
|
||||
expect(runtimeMocks.voiceClose).toHaveBeenCalledOnce();
|
||||
expect(onClose).toHaveBeenCalledOnce();
|
||||
expect(onClose).toHaveBeenCalledWith("completed");
|
||||
});
|
||||
|
||||
it("reopens voice after close without replaying discarded input", async () => {
|
||||
const lazy = await loadLazyProviders();
|
||||
const onClose = vi.fn();
|
||||
const bridge = lazy
|
||||
.createLazyXaiRealtimeVoiceProvider()
|
||||
.createBridge(createVoiceRequest({ onClose }));
|
||||
const first = Buffer.from([0x01]);
|
||||
const discarded = Buffer.from([0x02]);
|
||||
const second = Buffer.from([0x03]);
|
||||
|
||||
bridge.sendAudio(first);
|
||||
await bridge.connect();
|
||||
bridge.close();
|
||||
bridge.close();
|
||||
bridge.sendAudio(discarded);
|
||||
|
||||
const reconnectPromise = bridge.connect();
|
||||
bridge.sendAudio(second);
|
||||
await reconnectPromise;
|
||||
|
||||
expect(runtimeMocks.voiceConnect).toHaveBeenCalledTimes(2);
|
||||
expect(runtimeMocks.voiceClose).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.voiceSendAudio.mock.calls.map(([audio]) => audio)).toEqual([first, second]);
|
||||
expect(onClose).toHaveBeenCalledOnce();
|
||||
expect(onClose).toHaveBeenCalledWith("completed");
|
||||
});
|
||||
|
||||
it("keeps a replacement voice generation open after closing a pending connect", async () => {
|
||||
const firstConnect = createDeferred<void>();
|
||||
runtimeMocks.voiceConnect
|
||||
.mockReturnValueOnce(firstConnect.promise)
|
||||
.mockResolvedValueOnce(undefined);
|
||||
const lazy = await loadLazyProviders();
|
||||
const onClose = vi.fn();
|
||||
const bridge = lazy
|
||||
.createLazyXaiRealtimeVoiceProvider()
|
||||
.createBridge(createVoiceRequest({ onClose }));
|
||||
|
||||
const staleConnect = bridge.connect();
|
||||
await vi.waitFor(() => expect(runtimeMocks.voiceConnect).toHaveBeenCalledOnce());
|
||||
const staleRequest = runtimeMocks.createVoiceBridge.mock.calls[0]?.[0] as
|
||||
| RealtimeVoiceBridgeCreateRequest
|
||||
| undefined;
|
||||
bridge.close();
|
||||
const replacementConnect = bridge.connect();
|
||||
await replacementConnect;
|
||||
staleRequest?.onClose?.("error");
|
||||
bridge.sendUserMessage?.("replacement-still-open");
|
||||
firstConnect.resolve();
|
||||
await staleConnect;
|
||||
|
||||
expect(runtimeMocks.voiceConnect).toHaveBeenCalledTimes(2);
|
||||
expect(runtimeMocks.createVoiceBridge).toHaveBeenCalledTimes(2);
|
||||
expect(runtimeMocks.voiceClose).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledWith("replacement-still-open");
|
||||
expect(onClose).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("ignores nonterminal callbacks from a superseded voice generation", async () => {
|
||||
const onAudio = vi.fn();
|
||||
const onClearAudio = vi.fn();
|
||||
const onMark = vi.fn();
|
||||
const onTranscript = vi.fn();
|
||||
const onEvent = vi.fn();
|
||||
const onToolCall = vi.fn();
|
||||
const onReady = vi.fn();
|
||||
const onError = vi.fn();
|
||||
const lazy = await loadLazyProviders();
|
||||
const bridge = lazy.createLazyXaiRealtimeVoiceProvider().createBridge(
|
||||
createVoiceRequest({
|
||||
onAudio,
|
||||
onClearAudio,
|
||||
onMark,
|
||||
onTranscript,
|
||||
onEvent,
|
||||
onToolCall,
|
||||
onReady,
|
||||
onError,
|
||||
}),
|
||||
);
|
||||
|
||||
await bridge.connect();
|
||||
const staleRequest = runtimeMocks.createVoiceBridge.mock.calls[0]?.[0] as
|
||||
| RealtimeVoiceBridgeCreateRequest
|
||||
| undefined;
|
||||
bridge.close();
|
||||
await bridge.connect();
|
||||
const currentRequest = runtimeMocks.createVoiceBridge.mock.calls[1]?.[0] as
|
||||
| RealtimeVoiceBridgeCreateRequest
|
||||
| undefined;
|
||||
const staleAudio = Buffer.from([0x01]);
|
||||
const staleError = new Error("stale");
|
||||
const staleEvent = { direction: "server" as const, type: "stale" };
|
||||
const staleToolCall = {
|
||||
itemId: "stale-item",
|
||||
callId: "stale-call",
|
||||
name: "stale-tool",
|
||||
args: {},
|
||||
};
|
||||
|
||||
staleRequest?.onAudio(staleAudio);
|
||||
staleRequest?.onClearAudio("barge-in");
|
||||
staleRequest?.onMark?.("stale-mark");
|
||||
staleRequest?.onTranscript?.("assistant", "stale", true);
|
||||
staleRequest?.onEvent?.(staleEvent);
|
||||
staleRequest?.onToolCall?.(staleToolCall);
|
||||
staleRequest?.onReady?.();
|
||||
staleRequest?.onError?.(staleError);
|
||||
|
||||
expect(onAudio).not.toHaveBeenCalled();
|
||||
expect(onClearAudio).not.toHaveBeenCalled();
|
||||
expect(onMark).not.toHaveBeenCalled();
|
||||
expect(onTranscript).not.toHaveBeenCalled();
|
||||
expect(onEvent).not.toHaveBeenCalled();
|
||||
expect(onToolCall).not.toHaveBeenCalled();
|
||||
expect(onReady).not.toHaveBeenCalled();
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
|
||||
const currentAudio = Buffer.from([0x02]);
|
||||
const currentError = new Error("current");
|
||||
const currentEvent = { direction: "server" as const, type: "current" };
|
||||
const currentToolCall = {
|
||||
itemId: "current-item",
|
||||
callId: "current-call",
|
||||
name: "current-tool",
|
||||
args: {},
|
||||
};
|
||||
currentRequest?.onAudio(currentAudio);
|
||||
currentRequest?.onClearAudio("barge-in");
|
||||
currentRequest?.onMark?.("current-mark");
|
||||
currentRequest?.onTranscript?.("assistant", "current", true);
|
||||
currentRequest?.onEvent?.(currentEvent);
|
||||
currentRequest?.onToolCall?.(currentToolCall);
|
||||
currentRequest?.onReady?.();
|
||||
currentRequest?.onError?.(currentError);
|
||||
|
||||
expect(onAudio).toHaveBeenCalledWith(currentAudio);
|
||||
expect(onClearAudio).toHaveBeenCalledWith("barge-in");
|
||||
expect(onMark).toHaveBeenCalledWith("current-mark");
|
||||
expect(onTranscript).toHaveBeenCalledWith("assistant", "current", true);
|
||||
expect(onEvent).toHaveBeenCalledWith(currentEvent);
|
||||
expect(onToolCall).toHaveBeenCalledWith(currentToolCall);
|
||||
expect(onReady).toHaveBeenCalledOnce();
|
||||
expect(onError).toHaveBeenCalledWith(currentError);
|
||||
});
|
||||
|
||||
it("reports queued voice flush failure as a terminal error", async () => {
|
||||
const failure = new Error("tool result rejected");
|
||||
runtimeMocks.voiceSubmitToolResult.mockRejectedValueOnce(failure);
|
||||
const lazy = await loadLazyProviders();
|
||||
const onClose = vi.fn();
|
||||
const bridge = lazy
|
||||
.createLazyXaiRealtimeVoiceProvider()
|
||||
.createBridge(createVoiceRequest({ onClose }));
|
||||
|
||||
await bridge.submitToolResult("call-1", { text: "queued" });
|
||||
await expect(bridge.connect()).rejects.toThrow(failure);
|
||||
const loadedRequest = runtimeMocks.createVoiceBridge.mock.calls[0]?.[0] as
|
||||
| RealtimeVoiceBridgeCreateRequest
|
||||
| undefined;
|
||||
loadedRequest?.onClose?.("completed");
|
||||
bridge.sendAudio(Buffer.from([0x01]));
|
||||
|
||||
expect(runtimeMocks.voiceClose).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.voiceSendAudio).not.toHaveBeenCalled();
|
||||
expect(onClose).toHaveBeenCalledOnce();
|
||||
expect(onClose).toHaveBeenCalledWith("error");
|
||||
});
|
||||
|
||||
it("reopens voice only after an explicit connect following provider termination", async () => {
|
||||
const lazy = await loadLazyProviders();
|
||||
const onClose = vi.fn();
|
||||
const bridge = lazy
|
||||
.createLazyXaiRealtimeVoiceProvider()
|
||||
.createBridge(createVoiceRequest({ onClose }));
|
||||
const discarded = Buffer.from([0x01]);
|
||||
const accepted = Buffer.from([0x02]);
|
||||
|
||||
await bridge.connect();
|
||||
const loadedRequest = runtimeMocks.createVoiceBridge.mock.calls[0]?.[0] as
|
||||
| RealtimeVoiceBridgeCreateRequest
|
||||
| undefined;
|
||||
loadedRequest?.onClose?.("error");
|
||||
bridge.sendAudio(discarded);
|
||||
|
||||
const reconnectPromise = bridge.connect();
|
||||
bridge.sendAudio(accepted);
|
||||
await reconnectPromise;
|
||||
|
||||
expect(runtimeMocks.voiceConnect).toHaveBeenCalledTimes(2);
|
||||
expect(runtimeMocks.voiceSendAudio).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.voiceSendAudio).toHaveBeenCalledWith(accepted);
|
||||
expect(onClose).toHaveBeenCalledOnce();
|
||||
expect(onClose).toHaveBeenCalledWith("error");
|
||||
});
|
||||
|
||||
it("reports explicit voice close once when the provider also reports completion", async () => {
|
||||
const lazy = await loadLazyProviders();
|
||||
const onClose = vi.fn();
|
||||
const bridge = lazy
|
||||
.createLazyXaiRealtimeVoiceProvider()
|
||||
.createBridge(createVoiceRequest({ onClose }));
|
||||
|
||||
await bridge.connect();
|
||||
const loadedRequest = runtimeMocks.createVoiceBridge.mock.calls[0]?.[0] as
|
||||
| RealtimeVoiceBridgeCreateRequest
|
||||
| undefined;
|
||||
runtimeMocks.voiceClose.mockImplementation(() => loadedRequest?.onClose?.("completed"));
|
||||
bridge.close();
|
||||
bridge.close();
|
||||
|
||||
expect(runtimeMocks.voiceClose).toHaveBeenCalledOnce();
|
||||
expect(onClose).toHaveBeenCalledOnce();
|
||||
expect(onClose).toHaveBeenCalledWith("completed");
|
||||
});
|
||||
|
||||
it("keeps realtime voice request validation synchronous", async () => {
|
||||
const lazy = await loadLazyProviders();
|
||||
const provider = lazy.createLazyXaiRealtimeVoiceProvider();
|
||||
|
||||
expect(() => provider.createBridge(createVoiceRequest({ autoRespondToAudio: false }))).toThrow(
|
||||
"xAI realtime voice requires automatic server-VAD responses",
|
||||
);
|
||||
expect(runtimeMocks.buildVoiceProvider).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,667 @@
|
||||
import type { ImageGenerationProvider } from "openclaw/plugin-sdk/image-generation";
|
||||
import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
|
||||
import type { MediaUnderstandingProvider } from "openclaw/plugin-sdk/media-understanding";
|
||||
import type {
|
||||
RealtimeTranscriptionProviderPlugin,
|
||||
RealtimeTranscriptionSession,
|
||||
RealtimeTranscriptionSessionCreateRequest,
|
||||
} from "openclaw/plugin-sdk/realtime-transcription";
|
||||
import type {
|
||||
RealtimeVoiceBridge,
|
||||
RealtimeVoiceBridgeCreateRequest,
|
||||
RealtimeVoiceProviderPlugin,
|
||||
RealtimeVoiceToolResultOptions,
|
||||
} from "openclaw/plugin-sdk/realtime-voice";
|
||||
import { createRealtimeVoiceAudioQueue } from "openclaw/plugin-sdk/realtime-voice-audio-queue";
|
||||
import type {
|
||||
SpeechProviderPlugin,
|
||||
SpeechSynthesisStreamRequest,
|
||||
SpeechTelephonySynthesisRequest,
|
||||
} from "openclaw/plugin-sdk/speech";
|
||||
import type { VideoGenerationProvider } from "openclaw/plugin-sdk/video-generation";
|
||||
import {
|
||||
assertXaiRealtimeVoiceRequestSupported,
|
||||
createXaiImageGenerationProviderMetadata,
|
||||
createXaiMediaUnderstandingProviderMetadata,
|
||||
createXaiRealtimeTranscriptionProviderMetadata,
|
||||
createXaiRealtimeVoiceProviderMetadata,
|
||||
createXaiVideoGenerationProviderMetadata,
|
||||
normalizeXaiRealtimeTranscriptionProviderConfig,
|
||||
} from "./capability-provider-metadata.js";
|
||||
import { createXaiSpeechProviderMetadata } from "./speech-provider-metadata.js";
|
||||
|
||||
const MAX_LAZY_REALTIME_TRANSCRIPTION_AUDIO_BYTES = 2 * 1024 * 1024;
|
||||
const MAX_LAZY_REALTIME_VOICE_USER_MESSAGES = 128;
|
||||
const MAX_LAZY_REALTIME_VOICE_USER_MESSAGE_BYTES = 256 * 1024;
|
||||
const MAX_LAZY_REALTIME_VOICE_TOOL_RESULTS = 128;
|
||||
const MAX_LAZY_REALTIME_VOICE_TOOL_RESULT_BYTES = 256 * 1024;
|
||||
|
||||
function serializedJsonBytes(value: unknown): number | undefined {
|
||||
try {
|
||||
const serialized = JSON.stringify(value);
|
||||
return typeof serialized === "string" ? Buffer.byteLength(serialized, "utf8") : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
const loadXaiImageGenerationProvider = createLazyRuntimeModule(async () =>
|
||||
(await import("./image-generation-provider.js")).buildXaiImageGenerationProvider(),
|
||||
);
|
||||
const loadXaiMediaUnderstandingProvider = createLazyRuntimeModule(async () =>
|
||||
(await import("./stt.js")).buildXaiMediaUnderstandingProvider(),
|
||||
);
|
||||
const loadXaiRealtimeTranscriptionProvider = createLazyRuntimeModule(async () =>
|
||||
(await import("./realtime-transcription-provider.js")).buildXaiRealtimeTranscriptionProvider(),
|
||||
);
|
||||
const loadXaiRealtimeVoiceProvider = createLazyRuntimeModule(async () =>
|
||||
(await import("./realtime-voice-provider.js")).buildXaiRealtimeVoiceProvider(),
|
||||
);
|
||||
const loadXaiSpeechProvider = createLazyRuntimeModule(async () =>
|
||||
(await import("./speech-provider.js")).buildXaiSpeechProvider(),
|
||||
);
|
||||
const loadXaiVideoGenerationProvider = createLazyRuntimeModule(async () =>
|
||||
(await import("./video-generation-provider.js")).buildXaiVideoGenerationProvider(),
|
||||
);
|
||||
|
||||
function createPendingTranscriptionAudioQueue(): {
|
||||
clear: () => void;
|
||||
drain: () => Buffer[];
|
||||
enqueue: (audio: Buffer) => void;
|
||||
} {
|
||||
let chunks: Array<Buffer | undefined> = [];
|
||||
let head = 0;
|
||||
let bytes = 0;
|
||||
const clear = () => {
|
||||
chunks = [];
|
||||
head = 0;
|
||||
bytes = 0;
|
||||
};
|
||||
return {
|
||||
clear,
|
||||
drain: () => {
|
||||
const pending = chunks.slice(head).filter((chunk): chunk is Buffer => chunk !== undefined);
|
||||
clear();
|
||||
return pending;
|
||||
},
|
||||
enqueue: (audio) => {
|
||||
if (audio.byteLength > MAX_LAZY_REALTIME_TRANSCRIPTION_AUDIO_BYTES) {
|
||||
return;
|
||||
}
|
||||
const chunk = Buffer.from(audio);
|
||||
chunks.push(chunk);
|
||||
bytes += chunk.byteLength;
|
||||
while (bytes > MAX_LAZY_REALTIME_TRANSCRIPTION_AUDIO_BYTES && head < chunks.length) {
|
||||
const dropped = chunks[head];
|
||||
chunks[head] = undefined;
|
||||
head += 1;
|
||||
bytes -= dropped?.byteLength ?? 0;
|
||||
}
|
||||
if (head > 256 && head * 2 >= chunks.length) {
|
||||
chunks = chunks.slice(head);
|
||||
head = 0;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createLazyXaiRealtimeTranscriptionSession(
|
||||
req: RealtimeTranscriptionSessionCreateRequest,
|
||||
): RealtimeTranscriptionSession {
|
||||
let session: RealtimeTranscriptionSession | undefined;
|
||||
let sessionPromise: Promise<RealtimeTranscriptionSession> | undefined;
|
||||
let activeConnect:
|
||||
| {
|
||||
generation: number;
|
||||
promise: Promise<void>;
|
||||
}
|
||||
| undefined;
|
||||
let generation = 0;
|
||||
let closedSessionGeneration: number | undefined;
|
||||
let closed = false;
|
||||
let acceptsInput = false;
|
||||
const pendingAudio = createPendingTranscriptionAudioQueue();
|
||||
|
||||
const closeSession = (
|
||||
closeGeneration: number,
|
||||
loadedSession: RealtimeTranscriptionSession | undefined = session,
|
||||
) => {
|
||||
if (!loadedSession || closedSessionGeneration === closeGeneration) {
|
||||
return;
|
||||
}
|
||||
closedSessionGeneration = closeGeneration;
|
||||
loadedSession.close();
|
||||
};
|
||||
const loadSession = async () => {
|
||||
if (!sessionPromise) {
|
||||
sessionPromise = loadXaiRealtimeTranscriptionProvider().then((provider) =>
|
||||
provider.createSession(req),
|
||||
);
|
||||
}
|
||||
session = await sessionPromise;
|
||||
return session;
|
||||
};
|
||||
const beginConnectGeneration = () => {
|
||||
if (closed) {
|
||||
generation += 1;
|
||||
closed = false;
|
||||
}
|
||||
return generation;
|
||||
};
|
||||
|
||||
return {
|
||||
connect: async () => {
|
||||
const connectGeneration = beginConnectGeneration();
|
||||
if (activeConnect?.generation === connectGeneration) {
|
||||
await activeConnect.promise;
|
||||
return;
|
||||
}
|
||||
const promise = (async () => {
|
||||
const loadedSession = await loadSession();
|
||||
if (connectGeneration !== generation || closed) {
|
||||
if (connectGeneration === generation && closed) {
|
||||
closeSession(connectGeneration, loadedSession);
|
||||
}
|
||||
return;
|
||||
}
|
||||
for (const audio of pendingAudio.drain()) {
|
||||
loadedSession.sendAudio(audio);
|
||||
}
|
||||
acceptsInput = true;
|
||||
await loadedSession.connect();
|
||||
if (connectGeneration === generation && closed) {
|
||||
closeSession(connectGeneration, loadedSession);
|
||||
}
|
||||
})();
|
||||
const connectTask = { generation: connectGeneration, promise };
|
||||
activeConnect = connectTask;
|
||||
try {
|
||||
await promise;
|
||||
} finally {
|
||||
if (activeConnect === connectTask) {
|
||||
activeConnect = undefined;
|
||||
}
|
||||
}
|
||||
},
|
||||
sendAudio: (audio) => {
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
if (acceptsInput && session) {
|
||||
session.sendAudio(audio);
|
||||
return;
|
||||
}
|
||||
pendingAudio.enqueue(audio);
|
||||
},
|
||||
close: () => {
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
closed = true;
|
||||
acceptsInput = false;
|
||||
pendingAudio.clear();
|
||||
closeSession(generation);
|
||||
},
|
||||
isConnected: () => !closed && (session?.isConnected() ?? false),
|
||||
};
|
||||
}
|
||||
|
||||
function createLazyXaiRealtimeVoiceBridge(
|
||||
req: RealtimeVoiceBridgeCreateRequest,
|
||||
): RealtimeVoiceBridge {
|
||||
assertXaiRealtimeVoiceRequestSupported(req);
|
||||
type PendingVoiceOperation =
|
||||
| { type: "audio" }
|
||||
| { timestamp: number; type: "media-timestamp" }
|
||||
| { bytes: number; text: string; type: "user-message" }
|
||||
| { instructions?: string; type: "greeting" }
|
||||
| {
|
||||
bytes: number;
|
||||
callId: string;
|
||||
options?: RealtimeVoiceToolResultOptions;
|
||||
result: unknown;
|
||||
type: "tool-result";
|
||||
};
|
||||
type PendingMediaTimestamp = Extract<PendingVoiceOperation, { type: "media-timestamp" }>;
|
||||
type PendingVoiceGreeting = Extract<PendingVoiceOperation, { type: "greeting" }>;
|
||||
|
||||
let bridge: RealtimeVoiceBridge | undefined;
|
||||
let bridgeState:
|
||||
| {
|
||||
generation: number;
|
||||
promise: Promise<RealtimeVoiceBridge>;
|
||||
}
|
||||
| undefined;
|
||||
let activeConnect:
|
||||
| {
|
||||
generation: number;
|
||||
promise: Promise<void>;
|
||||
}
|
||||
| undefined;
|
||||
let generation = 0;
|
||||
let terminalGeneration: number | undefined;
|
||||
let closed = false;
|
||||
let acceptsInput = false;
|
||||
let pendingMediaTimestamp: PendingMediaTimestamp | undefined;
|
||||
let pendingGreeting: PendingVoiceGreeting | undefined;
|
||||
let pendingUserMessageCount = 0;
|
||||
let pendingUserMessageBytes = 0;
|
||||
let pendingToolResultCount = 0;
|
||||
let pendingToolResultBytes = 0;
|
||||
const closedBridges = new WeakSet<RealtimeVoiceBridge>();
|
||||
const pendingAudio = createRealtimeVoiceAudioQueue("reject-newest");
|
||||
const pendingOperations: PendingVoiceOperation[] = [];
|
||||
|
||||
const clearPendingInput = () => {
|
||||
pendingAudio.clear();
|
||||
pendingOperations.length = 0;
|
||||
pendingMediaTimestamp = undefined;
|
||||
pendingGreeting = undefined;
|
||||
pendingUserMessageCount = 0;
|
||||
pendingUserMessageBytes = 0;
|
||||
pendingToolResultCount = 0;
|
||||
pendingToolResultBytes = 0;
|
||||
};
|
||||
const emitTerminal = (
|
||||
terminalForGeneration: number,
|
||||
outcome: Parameters<NonNullable<RealtimeVoiceBridgeCreateRequest["onClose"]>>[0],
|
||||
) => {
|
||||
if (terminalForGeneration !== generation || terminalGeneration === terminalForGeneration) {
|
||||
return;
|
||||
}
|
||||
terminalGeneration = terminalForGeneration;
|
||||
acceptsInput = false;
|
||||
clearPendingInput();
|
||||
req.onClose?.(outcome);
|
||||
};
|
||||
const closeBridge = (loadedBridge: RealtimeVoiceBridge | undefined = bridge) => {
|
||||
if (!loadedBridge || closedBridges.has(loadedBridge)) {
|
||||
return;
|
||||
}
|
||||
closedBridges.add(loadedBridge);
|
||||
loadedBridge.close();
|
||||
};
|
||||
const acceptsProviderCallback = (callbackGeneration: number) =>
|
||||
callbackGeneration === generation && !closed && terminalGeneration !== callbackGeneration;
|
||||
const guardProviderCallback = <TArgs extends unknown[]>(
|
||||
callbackGeneration: number,
|
||||
callback: (...args: TArgs) => void,
|
||||
) => {
|
||||
return (...args: TArgs) => {
|
||||
if (acceptsProviderCallback(callbackGeneration)) {
|
||||
callback(...args);
|
||||
}
|
||||
};
|
||||
};
|
||||
const loadBridge = async (loadGeneration: number) => {
|
||||
const existingState = bridgeState;
|
||||
const state =
|
||||
existingState?.generation === loadGeneration
|
||||
? existingState
|
||||
: {
|
||||
generation: loadGeneration,
|
||||
promise: loadXaiRealtimeVoiceProvider().then((provider) =>
|
||||
provider.createBridge({
|
||||
...req,
|
||||
// An explicit wrapper reconnect owns a new provider bridge. Guard every
|
||||
// nonterminal callback so late events cannot reach its replacement.
|
||||
onAudio: guardProviderCallback(loadGeneration, req.onAudio),
|
||||
onClearAudio: guardProviderCallback(loadGeneration, req.onClearAudio),
|
||||
...(req.onMark
|
||||
? { onMark: guardProviderCallback(loadGeneration, req.onMark) }
|
||||
: {}),
|
||||
...(req.onTranscript
|
||||
? { onTranscript: guardProviderCallback(loadGeneration, req.onTranscript) }
|
||||
: {}),
|
||||
...(req.onEvent
|
||||
? { onEvent: guardProviderCallback(loadGeneration, req.onEvent) }
|
||||
: {}),
|
||||
...(req.onToolCall
|
||||
? { onToolCall: guardProviderCallback(loadGeneration, req.onToolCall) }
|
||||
: {}),
|
||||
...(req.onReady
|
||||
? { onReady: guardProviderCallback(loadGeneration, req.onReady) }
|
||||
: {}),
|
||||
...(req.onError
|
||||
? { onError: guardProviderCallback(loadGeneration, req.onError) }
|
||||
: {}),
|
||||
onClose: (outcome) => emitTerminal(loadGeneration, outcome),
|
||||
}),
|
||||
),
|
||||
};
|
||||
if (state !== existingState) {
|
||||
bridgeState = state;
|
||||
}
|
||||
const loadedBridge = await state.promise;
|
||||
if (bridgeState === state && loadGeneration === generation) {
|
||||
bridge = loadedBridge;
|
||||
}
|
||||
return loadedBridge;
|
||||
};
|
||||
const replacePendingOperation = <T extends PendingVoiceOperation>(
|
||||
previous: T | undefined,
|
||||
next: T,
|
||||
): T => {
|
||||
if (previous) {
|
||||
const previousIndex = pendingOperations.indexOf(previous);
|
||||
if (previousIndex >= 0) {
|
||||
pendingOperations.splice(previousIndex, 1);
|
||||
}
|
||||
}
|
||||
pendingOperations.push(next);
|
||||
return next;
|
||||
};
|
||||
const beginConnectGeneration = () => {
|
||||
if (closed || terminalGeneration === generation) {
|
||||
generation += 1;
|
||||
closed = false;
|
||||
acceptsInput = false;
|
||||
bridge = undefined;
|
||||
}
|
||||
return generation;
|
||||
};
|
||||
const acceptsCurrentInput = () => !closed && terminalGeneration !== generation;
|
||||
const flushPendingInput = async (
|
||||
loadedBridge: RealtimeVoiceBridge,
|
||||
connectGeneration: number,
|
||||
) => {
|
||||
if (connectGeneration !== generation || !acceptsCurrentInput()) {
|
||||
return;
|
||||
}
|
||||
while (true) {
|
||||
if (connectGeneration !== generation || !acceptsCurrentInput()) {
|
||||
return;
|
||||
}
|
||||
const operation = pendingOperations.shift();
|
||||
if (!operation) {
|
||||
// Queue exhaustion and direct admission must change in the same turn.
|
||||
// An await between them can strand input admitted by the next microtask.
|
||||
acceptsInput = true;
|
||||
return;
|
||||
}
|
||||
switch (operation.type) {
|
||||
case "audio": {
|
||||
const chunk = pendingAudio.dequeue();
|
||||
if (!chunk) {
|
||||
throw new Error("xAI realtime voice pending audio queue invariant violated");
|
||||
}
|
||||
loadedBridge.sendAudio(chunk);
|
||||
break;
|
||||
}
|
||||
case "media-timestamp":
|
||||
if (pendingMediaTimestamp === operation) {
|
||||
pendingMediaTimestamp = undefined;
|
||||
}
|
||||
loadedBridge.setMediaTimestamp(operation.timestamp);
|
||||
break;
|
||||
case "user-message":
|
||||
loadedBridge.sendUserMessage?.(operation.text);
|
||||
break;
|
||||
case "tool-result":
|
||||
await loadedBridge.submitToolResult(
|
||||
operation.callId,
|
||||
operation.result,
|
||||
operation.options,
|
||||
);
|
||||
break;
|
||||
case "greeting":
|
||||
if (pendingGreeting === operation) {
|
||||
pendingGreeting = undefined;
|
||||
}
|
||||
loadedBridge.triggerGreeting?.(operation.instructions);
|
||||
break;
|
||||
}
|
||||
if (connectGeneration !== generation || !acceptsCurrentInput()) {
|
||||
return;
|
||||
}
|
||||
if (operation.type === "user-message") {
|
||||
pendingUserMessageCount -= 1;
|
||||
pendingUserMessageBytes -= operation.bytes;
|
||||
} else if (operation.type === "tool-result") {
|
||||
pendingToolResultCount -= 1;
|
||||
pendingToolResultBytes -= operation.bytes;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
get supportsToolResultContinuation() {
|
||||
return bridge?.supportsToolResultContinuation ?? false;
|
||||
},
|
||||
connect: async () => {
|
||||
const connectGeneration = beginConnectGeneration();
|
||||
if (activeConnect?.generation === connectGeneration) {
|
||||
await activeConnect.promise;
|
||||
return;
|
||||
}
|
||||
const promise = (async () => {
|
||||
const loadedBridge = await loadBridge(connectGeneration);
|
||||
if (connectGeneration !== generation || !acceptsCurrentInput()) {
|
||||
closeBridge(loadedBridge);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await loadedBridge.connect();
|
||||
} catch (error) {
|
||||
if (connectGeneration === generation) {
|
||||
acceptsInput = false;
|
||||
terminalGeneration = connectGeneration;
|
||||
clearPendingInput();
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (connectGeneration !== generation || !acceptsCurrentInput()) {
|
||||
closeBridge(loadedBridge);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await flushPendingInput(loadedBridge, connectGeneration);
|
||||
} catch (error) {
|
||||
emitTerminal(connectGeneration, "error");
|
||||
closeBridge(loadedBridge);
|
||||
throw error;
|
||||
}
|
||||
if (connectGeneration !== generation || !acceptsCurrentInput()) {
|
||||
closeBridge(loadedBridge);
|
||||
}
|
||||
})();
|
||||
const connectTask = { generation: connectGeneration, promise };
|
||||
activeConnect = connectTask;
|
||||
try {
|
||||
await promise;
|
||||
} finally {
|
||||
if (activeConnect === connectTask) {
|
||||
activeConnect = undefined;
|
||||
}
|
||||
}
|
||||
},
|
||||
sendAudio: (audio) => {
|
||||
if (!acceptsCurrentInput()) {
|
||||
return;
|
||||
}
|
||||
if (acceptsInput && bridge) {
|
||||
bridge.sendAudio(audio);
|
||||
return;
|
||||
}
|
||||
if (pendingAudio.enqueue(audio)) {
|
||||
pendingOperations.push({ type: "audio" });
|
||||
}
|
||||
},
|
||||
setMediaTimestamp: (timestamp) => {
|
||||
if (!acceptsCurrentInput()) {
|
||||
return;
|
||||
}
|
||||
if (acceptsInput && bridge) {
|
||||
bridge.setMediaTimestamp(timestamp);
|
||||
return;
|
||||
}
|
||||
pendingMediaTimestamp = replacePendingOperation(pendingMediaTimestamp, {
|
||||
timestamp,
|
||||
type: "media-timestamp",
|
||||
});
|
||||
},
|
||||
sendUserMessage: (text) => {
|
||||
if (!acceptsCurrentInput()) {
|
||||
return;
|
||||
}
|
||||
if (acceptsInput && bridge) {
|
||||
bridge.sendUserMessage?.(text);
|
||||
return;
|
||||
}
|
||||
const messageBytes = Buffer.byteLength(text, "utf8");
|
||||
if (
|
||||
pendingUserMessageCount >= MAX_LAZY_REALTIME_VOICE_USER_MESSAGES ||
|
||||
pendingUserMessageBytes + messageBytes > MAX_LAZY_REALTIME_VOICE_USER_MESSAGE_BYTES
|
||||
) {
|
||||
req.onError?.(
|
||||
new Error("xAI realtime voice pending user message overflow during lazy startup"),
|
||||
);
|
||||
return;
|
||||
}
|
||||
pendingOperations.push({
|
||||
bytes: messageBytes,
|
||||
text,
|
||||
type: "user-message",
|
||||
});
|
||||
pendingUserMessageCount += 1;
|
||||
pendingUserMessageBytes += messageBytes;
|
||||
},
|
||||
triggerGreeting: (instructions) => {
|
||||
if (!acceptsCurrentInput()) {
|
||||
return;
|
||||
}
|
||||
if (acceptsInput && bridge) {
|
||||
bridge.triggerGreeting?.(instructions);
|
||||
return;
|
||||
}
|
||||
pendingGreeting = replacePendingOperation(pendingGreeting, {
|
||||
instructions,
|
||||
type: "greeting",
|
||||
});
|
||||
},
|
||||
handleBargeIn: (options) => {
|
||||
if (acceptsCurrentInput()) {
|
||||
bridge?.handleBargeIn?.(options);
|
||||
}
|
||||
},
|
||||
submitToolResult: (callId, result, options) => {
|
||||
if (!acceptsCurrentInput()) {
|
||||
return;
|
||||
}
|
||||
if (acceptsInput && bridge) {
|
||||
return bridge.submitToolResult(callId, result, options);
|
||||
}
|
||||
const pending = { callId, result, ...(options ? { options } : {}) };
|
||||
const resultBytes = serializedJsonBytes(pending);
|
||||
if (
|
||||
resultBytes === undefined ||
|
||||
pendingToolResultCount >= MAX_LAZY_REALTIME_VOICE_TOOL_RESULTS ||
|
||||
pendingToolResultBytes + resultBytes > MAX_LAZY_REALTIME_VOICE_TOOL_RESULT_BYTES
|
||||
) {
|
||||
req.onError?.(
|
||||
new Error("xAI realtime voice pending tool result overflow during lazy startup"),
|
||||
);
|
||||
return;
|
||||
}
|
||||
pendingOperations.push({
|
||||
...pending,
|
||||
bytes: resultBytes,
|
||||
type: "tool-result",
|
||||
});
|
||||
pendingToolResultCount += 1;
|
||||
pendingToolResultBytes += resultBytes;
|
||||
},
|
||||
acknowledgeMark: (markName) => {
|
||||
if (acceptsCurrentInput()) {
|
||||
bridge?.acknowledgeMark(markName);
|
||||
}
|
||||
},
|
||||
close: () => {
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
const closeGeneration = generation;
|
||||
closed = true;
|
||||
acceptsInput = false;
|
||||
clearPendingInput();
|
||||
closeBridge();
|
||||
// A bridge closed before its first connect has no provider-owned
|
||||
// connection to report the terminal outcome.
|
||||
emitTerminal(closeGeneration, "completed");
|
||||
},
|
||||
isConnected: () => acceptsCurrentInput() && (bridge?.isConnected() ?? false),
|
||||
};
|
||||
}
|
||||
|
||||
export function createLazyXaiImageGenerationProvider(): ImageGenerationProvider {
|
||||
return {
|
||||
...createXaiImageGenerationProviderMetadata(),
|
||||
generateImage: async (req) => (await loadXaiImageGenerationProvider()).generateImage(req),
|
||||
};
|
||||
}
|
||||
|
||||
export function createLazyXaiMediaUnderstandingProvider(): MediaUnderstandingProvider {
|
||||
return {
|
||||
...createXaiMediaUnderstandingProviderMetadata(),
|
||||
transcribeAudio: async (req) => {
|
||||
const provider = await loadXaiMediaUnderstandingProvider();
|
||||
if (!provider.transcribeAudio) {
|
||||
throw new Error("xAI media understanding provider missing transcribeAudio");
|
||||
}
|
||||
return await provider.transcribeAudio(req);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createLazyXaiVideoGenerationProvider(): VideoGenerationProvider {
|
||||
return {
|
||||
...createXaiVideoGenerationProviderMetadata(),
|
||||
generateVideo: async (req) => (await loadXaiVideoGenerationProvider()).generateVideo(req),
|
||||
};
|
||||
}
|
||||
|
||||
export function createLazyXaiSpeechProvider(): SpeechProviderPlugin {
|
||||
return {
|
||||
...createXaiSpeechProviderMetadata(),
|
||||
listVoices: async (req) => {
|
||||
const provider = await loadXaiSpeechProvider();
|
||||
if (!provider.listVoices) {
|
||||
throw new Error("xAI speech provider missing listVoices");
|
||||
}
|
||||
return await provider.listVoices(req);
|
||||
},
|
||||
synthesize: async (req) => await (await loadXaiSpeechProvider()).synthesize(req),
|
||||
streamSynthesize: async (req: SpeechSynthesisStreamRequest) => {
|
||||
const provider = await loadXaiSpeechProvider();
|
||||
if (!provider.streamSynthesize) {
|
||||
throw new Error("xAI speech provider missing streamSynthesize");
|
||||
}
|
||||
return await provider.streamSynthesize(req);
|
||||
},
|
||||
synthesizeTelephony: async (req: SpeechTelephonySynthesisRequest) => {
|
||||
const provider = await loadXaiSpeechProvider();
|
||||
if (!provider.synthesizeTelephony) {
|
||||
throw new Error("xAI speech provider missing synthesizeTelephony");
|
||||
}
|
||||
return await provider.synthesizeTelephony(req);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createLazyXaiRealtimeTranscriptionProvider(): RealtimeTranscriptionProviderPlugin {
|
||||
return {
|
||||
...createXaiRealtimeTranscriptionProviderMetadata(),
|
||||
createSession: (req) => {
|
||||
// Preserve synchronous config validation even though transport code loads on connect().
|
||||
normalizeXaiRealtimeTranscriptionProviderConfig(req.providerConfig);
|
||||
return createLazyXaiRealtimeTranscriptionSession(req);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createLazyXaiRealtimeVoiceProvider(): RealtimeVoiceProviderPlugin {
|
||||
return {
|
||||
...createXaiRealtimeVoiceProviderMetadata(),
|
||||
createBridge: createLazyXaiRealtimeVoiceBridge,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { XAI_BASE_URL } from "./model-definitions.js";
|
||||
import { isXaiProviderId } from "./provider-id.js";
|
||||
|
||||
const XAI_NATIVE_ENDPOINT_HOSTS = new Set(["api.x.ai"]);
|
||||
|
||||
function resolveHostname(value: string): string | undefined {
|
||||
try {
|
||||
return new URL(value).hostname.toLowerCase();
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function isXaiNativeEndpoint(baseUrl: unknown): boolean {
|
||||
return (
|
||||
typeof baseUrl === "string" && XAI_NATIVE_ENDPOINT_HOSTS.has(resolveHostname(baseUrl) ?? "")
|
||||
);
|
||||
}
|
||||
|
||||
function shouldUseXaiResponsesTransport(params: {
|
||||
provider: string;
|
||||
api?: unknown;
|
||||
baseUrl?: unknown;
|
||||
}): boolean {
|
||||
const hasDefaultXaiRoute =
|
||||
isXaiProviderId(params.provider) && !normalizeOptionalString(params.baseUrl);
|
||||
return params.api === "openai-responses"
|
||||
? hasDefaultXaiRoute
|
||||
: params.api === "openai-completions" &&
|
||||
(isXaiNativeEndpoint(params.baseUrl) || hasDefaultXaiRoute);
|
||||
}
|
||||
|
||||
export function resolveXaiTransport(params: {
|
||||
provider: string;
|
||||
api?: unknown;
|
||||
baseUrl?: unknown;
|
||||
}): { api: "openai-responses"; baseUrl?: string } | undefined {
|
||||
if (!shouldUseXaiResponsesTransport(params)) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
api: "openai-responses",
|
||||
baseUrl:
|
||||
normalizeOptionalString(params.baseUrl) ??
|
||||
(isXaiProviderId(params.provider) ? XAI_BASE_URL : undefined),
|
||||
};
|
||||
}
|
||||
@@ -1,38 +1,22 @@
|
||||
// Xai provider module implements model/runtime integration.
|
||||
import {
|
||||
isProviderAuthProfileConfigured,
|
||||
type OpenClawConfig,
|
||||
} from "openclaw/plugin-sdk/provider-auth";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth";
|
||||
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import {
|
||||
createRealtimeTranscriptionWebSocketSession,
|
||||
type RealtimeTranscriptionProviderConfig,
|
||||
type RealtimeTranscriptionProviderPlugin,
|
||||
type RealtimeTranscriptionSession,
|
||||
type RealtimeTranscriptionSessionCreateRequest,
|
||||
type RealtimeTranscriptionWebSocketTransport,
|
||||
} from "openclaw/plugin-sdk/realtime-transcription";
|
||||
import { normalizeResolvedSecretInputString } from "openclaw/plugin-sdk/secret-input";
|
||||
import { isRecord, normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import {
|
||||
normalizeOptionalString,
|
||||
parseBooleanValue as readBoolean,
|
||||
parseFiniteNumber as readFiniteNumber,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
createXaiRealtimeTranscriptionProviderMetadata,
|
||||
normalizeXaiRealtimeTranscriptionProviderConfig,
|
||||
type XaiRealtimeTranscriptionEncoding,
|
||||
} from "./capability-provider-metadata.js";
|
||||
import { XAI_BASE_URL } from "./model-definitions.js";
|
||||
import { xaiUserAgentHeaderFor } from "./src/xai-user-agent.js";
|
||||
|
||||
type XaiRealtimeTranscriptionEncoding = "pcm" | "mulaw" | "alaw";
|
||||
|
||||
type XaiRealtimeTranscriptionProviderConfig = {
|
||||
apiKey?: string;
|
||||
baseUrl?: string;
|
||||
sampleRate?: number;
|
||||
encoding?: XaiRealtimeTranscriptionEncoding;
|
||||
interimResults?: boolean;
|
||||
endpointingMs?: number;
|
||||
language?: string;
|
||||
};
|
||||
|
||||
type XaiRealtimeTranscriptionSessionConfig = RealtimeTranscriptionSessionCreateRequest & {
|
||||
apiKey: string;
|
||||
// Late-bound bearer; called per (re)connect.
|
||||
@@ -64,33 +48,6 @@ const XAI_REALTIME_STT_MAX_RECONNECT_ATTEMPTS = 5;
|
||||
const XAI_REALTIME_STT_RECONNECT_DELAY_MS = 1000;
|
||||
const XAI_REALTIME_STT_MAX_QUEUED_BYTES = 2 * 1024 * 1024;
|
||||
|
||||
function readRecord(value: unknown): Record<string, unknown> | undefined {
|
||||
return value && typeof value === "object" ? (value as Record<string, unknown>) : undefined;
|
||||
}
|
||||
|
||||
function readNestedXaiConfig(rawConfig: RealtimeTranscriptionProviderConfig) {
|
||||
const raw = readRecord(rawConfig);
|
||||
const providers = readRecord(raw?.providers);
|
||||
return readRecord(providers?.xai ?? raw?.xai ?? raw) ?? {};
|
||||
}
|
||||
|
||||
function normalizeEncoding(value: unknown): XaiRealtimeTranscriptionEncoding | undefined {
|
||||
const normalized = normalizeOptionalString(value)?.toLowerCase();
|
||||
if (!normalized) {
|
||||
return undefined;
|
||||
}
|
||||
if (normalized === "ulaw" || normalized === "g711_ulaw" || normalized === "g711-mulaw") {
|
||||
return "mulaw";
|
||||
}
|
||||
if (normalized === "g711_alaw" || normalized === "g711-alaw") {
|
||||
return "alaw";
|
||||
}
|
||||
if (normalized === "pcm" || normalized === "mulaw" || normalized === "alaw") {
|
||||
return normalized;
|
||||
}
|
||||
throw new Error(`Invalid xAI realtime transcription encoding: ${normalized}`);
|
||||
}
|
||||
|
||||
function normalizeXaiRealtimeBaseUrl(value?: string): string {
|
||||
return normalizeOptionalString(value ?? process.env.XAI_BASE_URL) ?? XAI_BASE_URL;
|
||||
}
|
||||
@@ -109,29 +66,11 @@ function toXaiRealtimeWsUrl(config: XaiRealtimeTranscriptionSessionConfig): stri
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
function normalizeProviderConfig(
|
||||
config: RealtimeTranscriptionProviderConfig,
|
||||
): XaiRealtimeTranscriptionProviderConfig {
|
||||
const raw = readNestedXaiConfig(config);
|
||||
return {
|
||||
apiKey: normalizeResolvedSecretInputString({
|
||||
value: raw.apiKey,
|
||||
path: "plugins.entries.voice-call.config.streaming.providers.xai.apiKey",
|
||||
}),
|
||||
baseUrl: normalizeOptionalString(raw.baseUrl),
|
||||
sampleRate: readFiniteNumber(raw.sampleRate ?? raw.sample_rate),
|
||||
encoding: normalizeEncoding(raw.encoding),
|
||||
interimResults: readBoolean(raw.interimResults ?? raw.interim_results),
|
||||
endpointingMs: readFiniteNumber(raw.endpointingMs ?? raw.endpointing ?? raw.silenceDurationMs),
|
||||
language: normalizeOptionalString(raw.language),
|
||||
};
|
||||
}
|
||||
|
||||
function readErrorDetail(value: unknown): string {
|
||||
if (typeof value === "string") {
|
||||
return value;
|
||||
}
|
||||
const record = readRecord(value);
|
||||
const record = isRecord(value) ? value : undefined;
|
||||
const message = normalizeOptionalString(record?.message);
|
||||
const code = normalizeOptionalString(record?.code);
|
||||
return message ?? code ?? "xAI realtime transcription error";
|
||||
@@ -233,18 +172,9 @@ function createXaiRealtimeTranscriptionSession(
|
||||
|
||||
export function buildXaiRealtimeTranscriptionProvider(): RealtimeTranscriptionProviderPlugin {
|
||||
return {
|
||||
id: "xai",
|
||||
label: "xAI Realtime Transcription",
|
||||
aliases: ["xai-realtime", "grok-stt-streaming"],
|
||||
autoSelectOrder: 25,
|
||||
resolveConfig: ({ rawConfig }) => normalizeProviderConfig(rawConfig),
|
||||
isConfigured: ({ providerConfig, cfg }) =>
|
||||
Boolean(
|
||||
normalizeProviderConfig(providerConfig).apiKey ??
|
||||
normalizeOptionalString(process.env.XAI_API_KEY),
|
||||
) || isProviderAuthProfileConfigured({ provider: "xai", cfg }),
|
||||
...createXaiRealtimeTranscriptionProviderMetadata(),
|
||||
createSession: (req) => {
|
||||
const config = normalizeProviderConfig(req.providerConfig);
|
||||
const config = normalizeXaiRealtimeTranscriptionProviderConfig(req.providerConfig);
|
||||
// createSession must stay sync per RealtimeTranscriptionProviderPlugin; bearer is resolved lazily in headers().
|
||||
const seedApiKey =
|
||||
normalizeOptionalString(config.apiKey) ?? normalizeOptionalString(process.env.XAI_API_KEY);
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth";
|
||||
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
|
||||
export async function resolveXaiRealtimeApiKey(
|
||||
configApiKey: string | undefined,
|
||||
cfg: OpenClawConfig | undefined,
|
||||
): Promise<string> {
|
||||
const direct =
|
||||
normalizeOptionalString(configApiKey) ?? normalizeOptionalString(process.env.XAI_API_KEY);
|
||||
if (direct) {
|
||||
return direct;
|
||||
}
|
||||
const auth = await resolveApiKeyForProvider({ provider: "xai", cfg });
|
||||
const oauthKey = normalizeOptionalString(auth?.apiKey);
|
||||
if (oauthKey) {
|
||||
return oauthKey;
|
||||
}
|
||||
throw new Error(
|
||||
"xAI credentials missing for realtime voice. Sign in with `openclaw onboard --auth-choice xai-oauth`, run `openclaw onboard --auth-choice xai-api-key`, or set XAI_API_KEY.",
|
||||
);
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import type {
|
||||
import { RealtimeVoiceSessionLifecycle } from "openclaw/plugin-sdk/realtime-voice";
|
||||
import { sleepWithAbort } from "openclaw/plugin-sdk/runtime-env";
|
||||
import WebSocket from "ws";
|
||||
import { resolveXaiRealtimeApiKey } from "./realtime-voice-auth.runtime.js";
|
||||
import {
|
||||
XAI_REALTIME_BASE_RECONNECT_DELAY_MS,
|
||||
XAI_REALTIME_CONNECT_TIMEOUT_MS,
|
||||
@@ -21,7 +22,6 @@ import {
|
||||
XAI_REALTIME_MAX_RECONNECT_ATTEMPTS,
|
||||
XAI_REALTIME_WS_MAX_PAYLOAD_BYTES,
|
||||
readXaiRealtimeErrorDetail,
|
||||
resolveXaiRealtimeApiKey,
|
||||
toXaiRealtimeWsUrl,
|
||||
type XaiRealtimeEvent,
|
||||
} from "./realtime-voice-config.js";
|
||||
|
||||
@@ -2,7 +2,6 @@ import {
|
||||
isProviderAuthProfileConfigured,
|
||||
type OpenClawConfig,
|
||||
} from "openclaw/plugin-sdk/provider-auth";
|
||||
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import type {
|
||||
RealtimeVoiceBridgeCreateRequest,
|
||||
RealtimeVoiceProviderConfig,
|
||||
@@ -228,25 +227,6 @@ export function toXaiRealtimeWsUrl(
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
export async function resolveXaiRealtimeApiKey(
|
||||
configApiKey: string | undefined,
|
||||
cfg: OpenClawConfig | undefined,
|
||||
): Promise<string> {
|
||||
const direct =
|
||||
normalizeOptionalString(configApiKey) ?? normalizeOptionalString(process.env.XAI_API_KEY);
|
||||
if (direct) {
|
||||
return direct;
|
||||
}
|
||||
const auth = await resolveApiKeyForProvider({ provider: "xai", cfg });
|
||||
const oauthKey = normalizeOptionalString(auth?.apiKey);
|
||||
if (oauthKey) {
|
||||
return oauthKey;
|
||||
}
|
||||
throw new Error(
|
||||
"xAI credentials missing for realtime voice. Sign in with `openclaw onboard --auth-choice xai-oauth`, run `openclaw onboard --auth-choice xai-api-key`, or set XAI_API_KEY.",
|
||||
);
|
||||
}
|
||||
|
||||
export function hasXaiRealtimeApiKeyInput(
|
||||
configApiKey: string | undefined,
|
||||
cfg: OpenClawConfig | undefined,
|
||||
|
||||
@@ -1,54 +1,21 @@
|
||||
import type { RealtimeVoiceProviderPlugin } from "openclaw/plugin-sdk/realtime-voice";
|
||||
import {
|
||||
REALTIME_VOICE_AUDIO_FORMAT_G711_ULAW_8KHZ,
|
||||
REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ,
|
||||
} from "openclaw/plugin-sdk/realtime-voice";
|
||||
assertXaiRealtimeVoiceRequestSupported,
|
||||
createXaiRealtimeVoiceProviderMetadata,
|
||||
} from "./capability-provider-metadata.js";
|
||||
import { resolveXaiRealtimeApiKey } from "./realtime-voice-auth.runtime.js";
|
||||
import { XaiRealtimeVoiceBridge } from "./realtime-voice-bridge.js";
|
||||
import {
|
||||
XAI_REALTIME_DEFAULT_MODEL,
|
||||
XAI_REALTIME_VOICES,
|
||||
hasXaiRealtimeApiKeyInput,
|
||||
normalizeXaiRealtimeBaseUrl,
|
||||
normalizeXaiRealtimeProviderConfig,
|
||||
resolveXaiRealtimeApiKey,
|
||||
} from "./realtime-voice-config.js";
|
||||
|
||||
export function buildXaiRealtimeVoiceProvider(): RealtimeVoiceProviderPlugin {
|
||||
return {
|
||||
id: "xai",
|
||||
label: "xAI Grok Voice",
|
||||
aliases: ["xai-realtime-voice", "grok-voice"],
|
||||
defaultModel: XAI_REALTIME_DEFAULT_MODEL,
|
||||
voices: XAI_REALTIME_VOICES,
|
||||
autoSelectOrder: 25,
|
||||
capabilities: {
|
||||
transports: ["gateway-relay"],
|
||||
inputAudioFormats: [
|
||||
REALTIME_VOICE_AUDIO_FORMAT_G711_ULAW_8KHZ,
|
||||
REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ,
|
||||
],
|
||||
outputAudioFormats: [
|
||||
REALTIME_VOICE_AUDIO_FORMAT_G711_ULAW_8KHZ,
|
||||
REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ,
|
||||
],
|
||||
supportsBargeIn: true,
|
||||
handlesInputAudioBargeIn: true,
|
||||
supportsToolCalls: true,
|
||||
supportsSessionResumption: true,
|
||||
},
|
||||
resolveConfig: ({ rawConfig }) => normalizeXaiRealtimeProviderConfig(rawConfig),
|
||||
isConfigured: ({ providerConfig, cfg }) =>
|
||||
hasXaiRealtimeApiKeyInput(normalizeXaiRealtimeProviderConfig(providerConfig).apiKey, cfg),
|
||||
...createXaiRealtimeVoiceProviderMetadata(),
|
||||
createBridge: (req) => {
|
||||
const config = normalizeXaiRealtimeProviderConfig(req.providerConfig);
|
||||
if (req.autoRespondToAudio === false) {
|
||||
throw new Error(
|
||||
'xAI realtime voice requires automatic server-VAD responses; use consultRouting: "provider-direct"',
|
||||
);
|
||||
}
|
||||
if ((req.interruptResponseOnInputAudio ?? config.interruptResponseOnInputAudio) === false) {
|
||||
throw new Error("xAI realtime voice requires automatic server-VAD interruption handling");
|
||||
}
|
||||
assertXaiRealtimeVoiceRequestSupported(req);
|
||||
return new XaiRealtimeVoiceBridge({
|
||||
...req,
|
||||
apiKey: config.apiKey,
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
import { isProviderAuthProfileConfigured } from "openclaw/plugin-sdk/provider-auth";
|
||||
import { normalizeResolvedSecretInputString } from "openclaw/plugin-sdk/secret-input";
|
||||
import type {
|
||||
SpeechDirectiveTokenParseContext,
|
||||
SpeechProviderConfig,
|
||||
SpeechProviderOverrides,
|
||||
SpeechProviderPlugin,
|
||||
SpeechSynthesisTarget,
|
||||
} from "openclaw/plugin-sdk/speech";
|
||||
import {
|
||||
asFiniteNumberInRange,
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
normalizeOptionalString,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { XAI_BASE_URL } from "./model-definitions.js";
|
||||
|
||||
const XAI_SPEECH_RESPONSE_FORMATS = ["mp3", "wav", "pcm", "mulaw", "alaw"] as const;
|
||||
|
||||
export type XaiSpeechResponseFormat = (typeof XAI_SPEECH_RESPONSE_FORMATS)[number];
|
||||
|
||||
type XaiTtsProviderConfig = {
|
||||
apiKey?: string;
|
||||
baseUrl: string;
|
||||
voiceId: string;
|
||||
language?: string;
|
||||
speed?: number;
|
||||
responseFormat?: XaiSpeechResponseFormat;
|
||||
};
|
||||
|
||||
type XaiTtsProviderOverrides = {
|
||||
voiceId?: string;
|
||||
language?: string;
|
||||
speed?: number;
|
||||
};
|
||||
|
||||
export const XAI_TTS_FALLBACK_VOICES = ["ara", "eve", "leo", "rex", "sal"] as const;
|
||||
|
||||
export function normalizeXaiTtsBaseUrl(baseUrl?: string): string {
|
||||
return normalizeOptionalString(baseUrl)?.replace(/\/+$/, "") ?? XAI_BASE_URL;
|
||||
}
|
||||
|
||||
export function isValidXaiTtsVoice(voice: string): boolean {
|
||||
return normalizeOptionalString(voice) !== undefined;
|
||||
}
|
||||
|
||||
export function normalizeXaiLanguageCode(value: unknown): string | undefined {
|
||||
const normalized = normalizeOptionalString(value)?.toLowerCase();
|
||||
if (!normalized) {
|
||||
return undefined;
|
||||
}
|
||||
if (normalized === "auto" || /^[a-z]{2,3}(?:-[a-z]{2,4})?$/.test(normalized)) {
|
||||
return normalized;
|
||||
}
|
||||
throw new Error(
|
||||
`xAI language must be "auto" or a BCP-47 tag (e.g. "en", "pt-br", "zh-cn"); got: ${normalized}`,
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeXaiSpeechSpeed(value: unknown): number | undefined {
|
||||
return asFiniteNumberInRange(value, { min: 0.7, max: 1.5 });
|
||||
}
|
||||
|
||||
function normalizeXaiSpeechResponseFormat(value: unknown): XaiSpeechResponseFormat | undefined {
|
||||
const next = normalizeLowercaseStringOrEmpty(value);
|
||||
if (!next) {
|
||||
return undefined;
|
||||
}
|
||||
if (XAI_SPEECH_RESPONSE_FORMATS.some((format) => format === next)) {
|
||||
return next as XaiSpeechResponseFormat;
|
||||
}
|
||||
throw new Error(`Invalid xAI speech responseFormat: ${next}`);
|
||||
}
|
||||
|
||||
export function resolveXaiSpeechResponseFormat(
|
||||
target: SpeechSynthesisTarget | undefined,
|
||||
configuredFormat?: XaiSpeechResponseFormat,
|
||||
): XaiSpeechResponseFormat {
|
||||
// Voice-note consumers may transcode without raw codec/rate metadata.
|
||||
// Keep streamed output and buffered fallback self-describing.
|
||||
return target === "voice-note" ? "mp3" : (configuredFormat ?? "mp3");
|
||||
}
|
||||
|
||||
export function xaiSpeechResponseFormatToFileExtension(
|
||||
format: XaiSpeechResponseFormat,
|
||||
): ".mp3" | ".pcm" | ".wav" | ".mulaw" | ".alaw" {
|
||||
switch (format) {
|
||||
case "wav":
|
||||
return ".wav";
|
||||
case "pcm":
|
||||
return ".pcm";
|
||||
case "mulaw":
|
||||
return ".mulaw";
|
||||
case "alaw":
|
||||
return ".alaw";
|
||||
default:
|
||||
return ".mp3";
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeXaiSpeechProviderConfig(
|
||||
rawConfig: Record<string, unknown>,
|
||||
): XaiTtsProviderConfig {
|
||||
const providers = rawConfig.providers as Record<string, unknown> | undefined;
|
||||
const xai = (providers?.xai ?? rawConfig.xai ?? rawConfig) as Record<string, unknown>;
|
||||
return {
|
||||
apiKey: normalizeResolvedSecretInputString({
|
||||
value: xai.apiKey,
|
||||
path: "tts.providers.xai.apiKey",
|
||||
}),
|
||||
baseUrl: normalizeXaiTtsBaseUrl(
|
||||
normalizeOptionalString(xai.baseUrl) ??
|
||||
normalizeOptionalString(process.env.XAI_BASE_URL) ??
|
||||
XAI_BASE_URL,
|
||||
),
|
||||
voiceId: normalizeOptionalString(xai.voiceId ?? xai.voice) ?? "eve",
|
||||
language: normalizeXaiLanguageCode(xai.language ?? xai.languageCode),
|
||||
speed: normalizeXaiSpeechSpeed(xai.speed),
|
||||
responseFormat: normalizeXaiSpeechResponseFormat(xai.responseFormat),
|
||||
};
|
||||
}
|
||||
|
||||
export function readXaiSpeechProviderConfig(config: SpeechProviderConfig): XaiTtsProviderConfig {
|
||||
const normalized = normalizeXaiSpeechProviderConfig({});
|
||||
return {
|
||||
apiKey: normalizeOptionalString(config.apiKey) ?? normalized.apiKey,
|
||||
baseUrl: normalizeOptionalString(config.baseUrl) ?? normalized.baseUrl,
|
||||
voiceId: normalizeOptionalString(config.voiceId ?? config.voice) ?? normalized.voiceId,
|
||||
language:
|
||||
normalizeXaiLanguageCode(config.language ?? config.languageCode) ?? normalized.language,
|
||||
speed: normalizeXaiSpeechSpeed(config.speed) ?? normalized.speed,
|
||||
responseFormat:
|
||||
normalizeXaiSpeechResponseFormat(config.responseFormat) ?? normalized.responseFormat,
|
||||
};
|
||||
}
|
||||
|
||||
export function readXaiSpeechOverrides(
|
||||
overrides: SpeechProviderOverrides | undefined,
|
||||
): XaiTtsProviderOverrides {
|
||||
if (!overrides) {
|
||||
return {};
|
||||
}
|
||||
return {
|
||||
voiceId: normalizeOptionalString(overrides.voiceId ?? overrides.voice),
|
||||
language: normalizeXaiLanguageCode(overrides.language),
|
||||
speed: normalizeXaiSpeechSpeed(overrides.speed),
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveDirectXaiAudioApiKey(configApiKey?: string): string | undefined {
|
||||
return normalizeOptionalString(configApiKey) ?? normalizeOptionalString(process.env.XAI_API_KEY);
|
||||
}
|
||||
|
||||
function parseXaiSpeechDirectiveToken(ctx: SpeechDirectiveTokenParseContext): {
|
||||
handled: boolean;
|
||||
overrides?: SpeechProviderOverrides;
|
||||
warnings?: string[];
|
||||
} {
|
||||
switch (ctx.key) {
|
||||
case "voice":
|
||||
case "voice_id":
|
||||
case "voiceid":
|
||||
case "xai_voice":
|
||||
case "xaivoice":
|
||||
if (!ctx.policy.allowVoice) {
|
||||
return { handled: true };
|
||||
}
|
||||
if (!isValidXaiTtsVoice(ctx.value)) {
|
||||
return { handled: true, warnings: [`invalid xAI voice "${ctx.value}"`] };
|
||||
}
|
||||
return { handled: true, overrides: { voiceId: ctx.value } };
|
||||
default:
|
||||
return { handled: false };
|
||||
}
|
||||
}
|
||||
|
||||
export function createXaiSpeechProviderMetadata(): Omit<
|
||||
SpeechProviderPlugin,
|
||||
"listVoices" | "synthesize" | "streamSynthesize" | "synthesizeTelephony"
|
||||
> {
|
||||
return {
|
||||
id: "xai",
|
||||
label: "xAI",
|
||||
autoSelectOrder: 25,
|
||||
models: [],
|
||||
voices: XAI_TTS_FALLBACK_VOICES,
|
||||
resolveConfig: ({ rawConfig }) => normalizeXaiSpeechProviderConfig(rawConfig),
|
||||
parseDirectiveToken: parseXaiSpeechDirectiveToken,
|
||||
resolveTalkConfig: ({ baseTtsConfig, talkProviderConfig }) => {
|
||||
const base = normalizeXaiSpeechProviderConfig(baseTtsConfig);
|
||||
const responseFormat = normalizeXaiSpeechResponseFormat(talkProviderConfig.responseFormat);
|
||||
return {
|
||||
...base,
|
||||
...(talkProviderConfig.apiKey === undefined
|
||||
? {}
|
||||
: {
|
||||
apiKey: normalizeResolvedSecretInputString({
|
||||
value: talkProviderConfig.apiKey,
|
||||
path: "talk.providers.xai.apiKey",
|
||||
}),
|
||||
}),
|
||||
...(normalizeOptionalString(talkProviderConfig.baseUrl) === undefined
|
||||
? {}
|
||||
: {
|
||||
baseUrl: normalizeXaiTtsBaseUrl(normalizeOptionalString(talkProviderConfig.baseUrl)),
|
||||
}),
|
||||
...(normalizeOptionalString(talkProviderConfig.voiceId) === undefined
|
||||
? {}
|
||||
: { voiceId: normalizeOptionalString(talkProviderConfig.voiceId) }),
|
||||
...(normalizeXaiLanguageCode(
|
||||
talkProviderConfig.language ?? talkProviderConfig.languageCode,
|
||||
) === undefined
|
||||
? {}
|
||||
: {
|
||||
language: normalizeXaiLanguageCode(
|
||||
talkProviderConfig.language ?? talkProviderConfig.languageCode,
|
||||
),
|
||||
}),
|
||||
...(normalizeXaiSpeechSpeed(talkProviderConfig.speed) === undefined
|
||||
? {}
|
||||
: { speed: normalizeXaiSpeechSpeed(talkProviderConfig.speed) }),
|
||||
...(responseFormat === undefined ? {} : { responseFormat }),
|
||||
};
|
||||
},
|
||||
resolveTalkOverrides: ({ params }) => ({
|
||||
...(normalizeOptionalString(params.voiceId ?? params.voice) === undefined
|
||||
? {}
|
||||
: { voiceId: normalizeOptionalString(params.voiceId ?? params.voice) }),
|
||||
...(normalizeXaiLanguageCode(params.language ?? params.languageCode) === undefined
|
||||
? {}
|
||||
: { language: normalizeXaiLanguageCode(params.language ?? params.languageCode) }),
|
||||
...(normalizeXaiSpeechSpeed(params.speed) === undefined
|
||||
? {}
|
||||
: { speed: normalizeXaiSpeechSpeed(params.speed) }),
|
||||
}),
|
||||
isConfigured: ({ providerConfig, cfg }) =>
|
||||
Boolean(resolveDirectXaiAudioApiKey(readXaiSpeechProviderConfig(providerConfig).apiKey)) ||
|
||||
isProviderAuthProfileConfigured({ provider: "xai", cfg }),
|
||||
};
|
||||
}
|
||||
@@ -1,146 +1,25 @@
|
||||
// Xai provider module implements model/runtime integration.
|
||||
import { resolveGeneratedMediaMaxBytes } from "openclaw/plugin-sdk/media-generation-runtime";
|
||||
import {
|
||||
isProviderAuthProfileConfigured,
|
||||
type OpenClawConfig,
|
||||
} from "openclaw/plugin-sdk/provider-auth";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth";
|
||||
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import { normalizeResolvedSecretInputString } from "openclaw/plugin-sdk/secret-input";
|
||||
import {
|
||||
trimToUndefined,
|
||||
type SpeechDirectiveTokenParseContext,
|
||||
type SpeechProviderConfig,
|
||||
type SpeechProviderOverrides,
|
||||
type SpeechProviderPlugin,
|
||||
type SpeechSynthesisRequest,
|
||||
type SpeechSynthesisTarget,
|
||||
import type {
|
||||
SpeechProviderPlugin,
|
||||
SpeechSynthesisRequest,
|
||||
SpeechSynthesisTarget,
|
||||
} from "openclaw/plugin-sdk/speech";
|
||||
import { resolveSpeechProviderApiKey } from "openclaw/plugin-sdk/speech-core";
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import {
|
||||
asFiniteNumberInRange,
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import {
|
||||
isValidXaiTtsVoice,
|
||||
listXaiTtsVoices,
|
||||
normalizeXaiLanguageCode,
|
||||
normalizeXaiTtsBaseUrl,
|
||||
XAI_BASE_URL,
|
||||
createXaiSpeechProviderMetadata,
|
||||
readXaiSpeechOverrides,
|
||||
readXaiSpeechProviderConfig,
|
||||
resolveDirectXaiAudioApiKey,
|
||||
resolveXaiSpeechResponseFormat,
|
||||
xaiSpeechResponseFormatToFileExtension,
|
||||
XAI_TTS_FALLBACK_VOICES,
|
||||
xaiTTS,
|
||||
xaiTTSStream,
|
||||
} from "./tts.js";
|
||||
|
||||
const XAI_SPEECH_RESPONSE_FORMATS = ["mp3", "wav", "pcm", "mulaw", "alaw"] as const;
|
||||
|
||||
type XaiSpeechResponseFormat = (typeof XAI_SPEECH_RESPONSE_FORMATS)[number];
|
||||
|
||||
type XaiTtsProviderConfig = {
|
||||
apiKey?: string;
|
||||
baseUrl: string;
|
||||
voiceId: string;
|
||||
language?: string;
|
||||
speed?: number;
|
||||
responseFormat?: XaiSpeechResponseFormat;
|
||||
};
|
||||
|
||||
type XaiTtsProviderOverrides = {
|
||||
voiceId?: string;
|
||||
language?: string;
|
||||
speed?: number;
|
||||
};
|
||||
|
||||
function normalizeXaiSpeechSpeed(value: unknown): number | undefined {
|
||||
return asFiniteNumberInRange(value, { min: 0.7, max: 1.5 });
|
||||
}
|
||||
|
||||
function normalizeXaiSpeechResponseFormat(value: unknown): XaiSpeechResponseFormat | undefined {
|
||||
const next = normalizeLowercaseStringOrEmpty(value);
|
||||
if (!next) {
|
||||
return undefined;
|
||||
}
|
||||
if (XAI_SPEECH_RESPONSE_FORMATS.some((format) => format === next)) {
|
||||
return next as XaiSpeechResponseFormat;
|
||||
}
|
||||
throw new Error(`Invalid xAI speech responseFormat: ${next}`);
|
||||
}
|
||||
|
||||
function resolveSpeechResponseFormat(
|
||||
target: SpeechSynthesisTarget | undefined,
|
||||
configuredFormat?: XaiSpeechResponseFormat,
|
||||
): XaiSpeechResponseFormat {
|
||||
// Voice-note consumers may transcode without raw codec/rate metadata.
|
||||
// Keep streamed output and buffered fallback self-describing.
|
||||
if (target === "voice-note") {
|
||||
return "mp3";
|
||||
}
|
||||
return configuredFormat ?? "mp3";
|
||||
}
|
||||
|
||||
function responseFormatToFileExtension(
|
||||
format: XaiSpeechResponseFormat,
|
||||
): ".mp3" | ".pcm" | ".wav" | ".mulaw" | ".alaw" {
|
||||
switch (format) {
|
||||
case "wav":
|
||||
return ".wav";
|
||||
case "pcm":
|
||||
return ".pcm";
|
||||
case "mulaw":
|
||||
return ".mulaw";
|
||||
case "alaw":
|
||||
return ".alaw";
|
||||
default:
|
||||
return ".mp3";
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeXaiProviderConfig(rawConfig: Record<string, unknown>): XaiTtsProviderConfig {
|
||||
const providers = rawConfig?.providers as Record<string, unknown> | undefined;
|
||||
const xai = (providers?.xai ?? rawConfig?.xai ?? rawConfig) as Record<string, unknown>;
|
||||
return {
|
||||
apiKey: normalizeResolvedSecretInputString({
|
||||
value: xai?.apiKey,
|
||||
path: "tts.providers.xai.apiKey",
|
||||
}),
|
||||
baseUrl: normalizeXaiTtsBaseUrl(
|
||||
trimToUndefined(xai?.baseUrl) ?? trimToUndefined(process.env.XAI_BASE_URL) ?? XAI_BASE_URL,
|
||||
),
|
||||
voiceId: trimToUndefined(xai?.voiceId ?? xai?.voice) ?? "eve",
|
||||
language: normalizeXaiLanguageCode(trimToUndefined(xai?.language ?? xai?.languageCode)),
|
||||
speed: normalizeXaiSpeechSpeed(xai?.speed),
|
||||
responseFormat: normalizeXaiSpeechResponseFormat(xai?.responseFormat),
|
||||
};
|
||||
}
|
||||
|
||||
function readXaiProviderConfig(config: SpeechProviderConfig): XaiTtsProviderConfig {
|
||||
const normalized = normalizeXaiProviderConfig({});
|
||||
return {
|
||||
apiKey: trimToUndefined(config.apiKey) ?? normalized.apiKey,
|
||||
baseUrl: trimToUndefined(config.baseUrl) ?? normalized.baseUrl,
|
||||
voiceId: trimToUndefined(config.voiceId ?? config.voice) ?? normalized.voiceId,
|
||||
language:
|
||||
normalizeXaiLanguageCode(trimToUndefined(config.language ?? config.languageCode)) ??
|
||||
normalized.language,
|
||||
speed: normalizeXaiSpeechSpeed(config.speed) ?? normalized.speed,
|
||||
responseFormat:
|
||||
normalizeXaiSpeechResponseFormat(config.responseFormat) ?? normalized.responseFormat,
|
||||
};
|
||||
}
|
||||
|
||||
function readXaiOverrides(overrides: SpeechProviderOverrides | undefined): XaiTtsProviderOverrides {
|
||||
if (!overrides) {
|
||||
return {};
|
||||
}
|
||||
return {
|
||||
voiceId: trimToUndefined(overrides.voiceId ?? overrides.voice),
|
||||
language: normalizeXaiLanguageCode(trimToUndefined(overrides.language)),
|
||||
speed: normalizeXaiSpeechSpeed(overrides.speed),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveDirectXaiAudioApiKey(configApiKey?: string): string | undefined {
|
||||
return resolveSpeechProviderApiKey(configApiKey, process.env.XAI_API_KEY);
|
||||
}
|
||||
normalizeXaiTtsBaseUrl,
|
||||
type XaiSpeechResponseFormat,
|
||||
} from "./speech-provider-metadata.js";
|
||||
import { listXaiTtsVoices, xaiTTS, xaiTTSStream } from "./tts.js";
|
||||
|
||||
async function resolveXaiSpeechSynthesisRequest(
|
||||
req: Pick<
|
||||
@@ -149,8 +28,8 @@ async function resolveXaiSpeechSynthesisRequest(
|
||||
> & { target?: SpeechSynthesisTarget },
|
||||
forcedResponseFormat?: XaiSpeechResponseFormat,
|
||||
) {
|
||||
const config = readXaiProviderConfig(req.providerConfig);
|
||||
const overrides = readXaiOverrides(req.providerOverrides);
|
||||
const config = readXaiSpeechProviderConfig(req.providerConfig);
|
||||
const overrides = readXaiSpeechOverrides(req.providerOverrides);
|
||||
return {
|
||||
text: req.text,
|
||||
apiKey: await resolveXaiAudioApiKey(config.apiKey, req.cfg),
|
||||
@@ -159,114 +38,33 @@ async function resolveXaiSpeechSynthesisRequest(
|
||||
language: overrides.language ?? config.language,
|
||||
speed: overrides.speed ?? config.speed,
|
||||
responseFormat:
|
||||
forcedResponseFormat ?? resolveSpeechResponseFormat(req.target, config.responseFormat),
|
||||
forcedResponseFormat ?? resolveXaiSpeechResponseFormat(req.target, config.responseFormat),
|
||||
timeoutMs: req.timeoutMs,
|
||||
maxBytes: resolveGeneratedMediaMaxBytes(req.cfg, "audio"),
|
||||
};
|
||||
}
|
||||
|
||||
function parseDirectiveToken(ctx: SpeechDirectiveTokenParseContext): {
|
||||
handled: boolean;
|
||||
overrides?: SpeechProviderOverrides;
|
||||
warnings?: string[];
|
||||
} {
|
||||
switch (ctx.key) {
|
||||
case "voice":
|
||||
case "voice_id":
|
||||
case "voiceid":
|
||||
case "xai_voice":
|
||||
case "xaivoice":
|
||||
if (!ctx.policy.allowVoice) {
|
||||
return { handled: true };
|
||||
}
|
||||
if (!isValidXaiTtsVoice(ctx.value)) {
|
||||
return { handled: true, warnings: [`invalid xAI voice "${ctx.value}"`] };
|
||||
}
|
||||
return { handled: true, overrides: { voiceId: ctx.value } };
|
||||
default:
|
||||
return { handled: false };
|
||||
}
|
||||
}
|
||||
|
||||
export function buildXaiSpeechProvider(): SpeechProviderPlugin {
|
||||
return {
|
||||
id: "xai",
|
||||
label: "xAI",
|
||||
autoSelectOrder: 25,
|
||||
models: [],
|
||||
voices: XAI_TTS_FALLBACK_VOICES,
|
||||
resolveConfig: ({ rawConfig }) => normalizeXaiProviderConfig(rawConfig),
|
||||
parseDirectiveToken,
|
||||
resolveTalkConfig: ({ baseTtsConfig, talkProviderConfig }) => {
|
||||
const base = normalizeXaiProviderConfig(baseTtsConfig);
|
||||
const responseFormat = normalizeXaiSpeechResponseFormat(talkProviderConfig.responseFormat);
|
||||
return {
|
||||
...base,
|
||||
...(talkProviderConfig.apiKey === undefined
|
||||
? {}
|
||||
: {
|
||||
apiKey: normalizeResolvedSecretInputString({
|
||||
value: talkProviderConfig.apiKey,
|
||||
path: "talk.providers.xai.apiKey",
|
||||
}),
|
||||
}),
|
||||
...(trimToUndefined(talkProviderConfig.baseUrl) == null
|
||||
? {}
|
||||
: { baseUrl: normalizeXaiTtsBaseUrl(trimToUndefined(talkProviderConfig.baseUrl)) }),
|
||||
...(trimToUndefined(talkProviderConfig.voiceId) == null
|
||||
? {}
|
||||
: { voiceId: trimToUndefined(talkProviderConfig.voiceId) }),
|
||||
...(normalizeXaiLanguageCode(
|
||||
trimToUndefined(talkProviderConfig.language ?? talkProviderConfig.languageCode),
|
||||
) == null
|
||||
? {}
|
||||
: {
|
||||
language: normalizeXaiLanguageCode(
|
||||
trimToUndefined(talkProviderConfig.language ?? talkProviderConfig.languageCode),
|
||||
),
|
||||
}),
|
||||
...(normalizeXaiSpeechSpeed(talkProviderConfig.speed) == null
|
||||
? {}
|
||||
: { speed: normalizeXaiSpeechSpeed(talkProviderConfig.speed) }),
|
||||
...(responseFormat == null ? {} : { responseFormat }),
|
||||
};
|
||||
},
|
||||
resolveTalkOverrides: ({ params }) => ({
|
||||
...(trimToUndefined(params.voiceId ?? params.voice) == null
|
||||
? {}
|
||||
: { voiceId: trimToUndefined(params.voiceId ?? params.voice) }),
|
||||
...(normalizeXaiLanguageCode(trimToUndefined(params.language ?? params.languageCode)) == null
|
||||
? {}
|
||||
: {
|
||||
language: normalizeXaiLanguageCode(
|
||||
trimToUndefined(params.language ?? params.languageCode),
|
||||
),
|
||||
}),
|
||||
...(normalizeXaiSpeechSpeed(params.speed) == null
|
||||
? {}
|
||||
: { speed: normalizeXaiSpeechSpeed(params.speed) }),
|
||||
}),
|
||||
...createXaiSpeechProviderMetadata(),
|
||||
listVoices: async (req) => {
|
||||
const config = readXaiProviderConfig(req.providerConfig ?? {});
|
||||
const directApiKey = trimToUndefined(req.apiKey) ?? config.apiKey;
|
||||
const config = readXaiSpeechProviderConfig(req.providerConfig ?? {});
|
||||
const directApiKey = normalizeOptionalString(req.apiKey) ?? config.apiKey;
|
||||
const apiKey = await resolveOptionalXaiAudioApiKey(directApiKey, req.cfg);
|
||||
if (!apiKey) {
|
||||
return XAI_TTS_FALLBACK_VOICES.map((voice) => ({ id: voice, name: voice }));
|
||||
}
|
||||
return await listXaiTtsVoices({
|
||||
apiKey,
|
||||
baseUrl: normalizeXaiTtsBaseUrl(trimToUndefined(req.baseUrl) ?? config.baseUrl),
|
||||
baseUrl: normalizeXaiTtsBaseUrl(normalizeOptionalString(req.baseUrl) ?? config.baseUrl),
|
||||
});
|
||||
},
|
||||
isConfigured: ({ providerConfig, cfg }) =>
|
||||
Boolean(resolveDirectXaiAudioApiKey(readXaiProviderConfig(providerConfig).apiKey)) ||
|
||||
isProviderAuthProfileConfigured({ provider: "xai", cfg }),
|
||||
synthesize: async (req) => {
|
||||
const params = await resolveXaiSpeechSynthesisRequest(req);
|
||||
return {
|
||||
audioBuffer: await xaiTTS(params),
|
||||
outputFormat: params.responseFormat,
|
||||
fileExtension: responseFormatToFileExtension(params.responseFormat),
|
||||
fileExtension: xaiSpeechResponseFormatToFileExtension(params.responseFormat),
|
||||
voiceCompatible: false,
|
||||
};
|
||||
},
|
||||
@@ -276,7 +74,7 @@ export function buildXaiSpeechProvider(): SpeechProviderPlugin {
|
||||
return {
|
||||
audioStream: stream.audioStream,
|
||||
outputFormat: params.responseFormat,
|
||||
fileExtension: responseFormatToFileExtension(params.responseFormat),
|
||||
fileExtension: xaiSpeechResponseFormatToFileExtension(params.responseFormat),
|
||||
voiceCompatible: false,
|
||||
release: stream.release,
|
||||
};
|
||||
@@ -304,7 +102,7 @@ async function resolveOptionalXaiAudioApiKey(
|
||||
return undefined;
|
||||
}
|
||||
const auth = await resolveApiKeyForProvider({ provider: "xai", cfg });
|
||||
return trimToUndefined(auth?.apiKey);
|
||||
return normalizeOptionalString(auth?.apiKey);
|
||||
}
|
||||
|
||||
async function resolveXaiAudioApiKey(
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
resolveProviderHttpRequestConfig,
|
||||
} from "openclaw/plugin-sdk/provider-http";
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { createXaiMediaUnderstandingProviderMetadata } from "./capability-provider-metadata.js";
|
||||
import { XAI_BASE_URL } from "./model-definitions.js";
|
||||
|
||||
type XaiSttResponse = {
|
||||
@@ -80,9 +81,7 @@ export function buildXaiMediaUnderstandingProvider(): MediaUnderstandingProvider
|
||||
// before transcribeAudio runs, so an OAuth profile (when configured) reaches
|
||||
// here as `params.apiKey` already. No plugin-side fallback required.
|
||||
return {
|
||||
id: "xai",
|
||||
capabilities: ["audio"],
|
||||
autoPriority: { audio: 25 },
|
||||
...createXaiMediaUnderstandingProviderMetadata(),
|
||||
transcribeAudio: transcribeXaiAudio,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -353,6 +353,9 @@
|
||||
"openclaw/plugin-sdk/realtime-bootstrap-context": [
|
||||
"../../packages/plugin-sdk/dist/src/plugin-sdk/realtime-bootstrap-context.d.ts"
|
||||
],
|
||||
"openclaw/plugin-sdk/realtime-voice-audio-queue": [
|
||||
"../../packages/plugin-sdk/dist/src/plugin-sdk/realtime-voice-audio-queue.d.ts"
|
||||
],
|
||||
"openclaw/plugin-sdk/realtime-voice": [
|
||||
"../../packages/plugin-sdk/dist/src/plugin-sdk/realtime-voice.d.ts"
|
||||
],
|
||||
|
||||
@@ -1,14 +1,9 @@
|
||||
// Xai tests cover tts plugin behavior.
|
||||
import { mockPinnedHostnameResolution } from "openclaw/plugin-sdk/test-env";
|
||||
import { beforeEach, afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
isValidXaiTtsVoice,
|
||||
listXaiTtsVoices,
|
||||
XAI_BASE_URL,
|
||||
XAI_TTS_FALLBACK_VOICES,
|
||||
xaiTTS,
|
||||
xaiTTSStream,
|
||||
} from "./tts.js";
|
||||
import { XAI_BASE_URL } from "./model-definitions.js";
|
||||
import { isValidXaiTtsVoice, XAI_TTS_FALLBACK_VOICES } from "./speech-provider-metadata.js";
|
||||
import { listXaiTtsVoices, xaiTTS, xaiTTSStream } from "./tts.js";
|
||||
|
||||
const { FakeWebSocket } = vi.hoisted(() => {
|
||||
type Listener = (...args: unknown[]) => void;
|
||||
|
||||
+6
-30
@@ -13,28 +13,18 @@ import {
|
||||
ssrfPolicyFromHttpBaseUrlAllowedHostname,
|
||||
} from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import WebSocket, { type RawData } from "ws";
|
||||
import { XAI_BASE_URL } from "./api.js";
|
||||
import { XAI_BASE_URL } from "./model-definitions.js";
|
||||
import {
|
||||
isValidXaiTtsVoice,
|
||||
normalizeXaiLanguageCode,
|
||||
normalizeXaiTtsBaseUrl,
|
||||
} from "./speech-provider-metadata.js";
|
||||
import { xaiUserAgentHeaderFor } from "./src/xai-user-agent.js";
|
||||
export { XAI_BASE_URL };
|
||||
|
||||
const DEFAULT_TTS_MAX_BYTES = 16 * 1024 * 1024;
|
||||
const XAI_TTS_VOICE_LIST_TIMEOUT_MS = 30_000;
|
||||
const XAI_TTS_VOICE_LIST_MAX_BYTES = 1024 * 1024;
|
||||
const XAI_TTS_STREAM_TEXT_DELTA_MAX_CHARS = 15_000;
|
||||
export const XAI_TTS_FALLBACK_VOICES = ["ara", "eve", "leo", "rex", "sal"] as const;
|
||||
|
||||
export function normalizeXaiTtsBaseUrl(baseUrl?: string): string {
|
||||
const trimmed = baseUrl?.trim();
|
||||
if (!trimmed) {
|
||||
return XAI_BASE_URL;
|
||||
}
|
||||
return trimmed.replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
export function isValidXaiTtsVoice(voice: string): boolean {
|
||||
return trimToUndefined(voice) !== undefined;
|
||||
}
|
||||
|
||||
export async function listXaiTtsVoices(params: {
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
@@ -82,20 +72,6 @@ export async function listXaiTtsVoices(params: {
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeXaiLanguageCode(value: unknown): string | undefined {
|
||||
const trimmed = trimToUndefined(value);
|
||||
if (!trimmed) {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = trimmed.toLowerCase();
|
||||
if (normalized === "auto" || /^[a-z]{2,3}(?:-[a-z]{2,4})?$/.test(normalized)) {
|
||||
return normalized;
|
||||
}
|
||||
throw new Error(
|
||||
`xAI language must be "auto" or a BCP-47 tag (e.g. "en", "pt-br", "zh-cn"); got: ${normalized}`,
|
||||
);
|
||||
}
|
||||
|
||||
type XaiTtsResponseFormat = "mp3" | "wav" | "pcm" | "mulaw" | "alaw";
|
||||
|
||||
const XAI_NATIVE_TTS_STREAM_HOST = "api.x.ai";
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// Xai provider module implements model/runtime integration.
|
||||
import { toImageDataUrl } from "openclaw/plugin-sdk/image-generation";
|
||||
import { resolveGeneratedMediaMaxBytes } from "openclaw/plugin-sdk/media-generation-runtime";
|
||||
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
|
||||
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import {
|
||||
assertOkOrThrowHttpError,
|
||||
@@ -17,42 +16,24 @@ import {
|
||||
import { isRecord, normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import type {
|
||||
VideoGenerationProvider,
|
||||
VideoGenerationProviderCapabilities,
|
||||
VideoGenerationRequest,
|
||||
} from "openclaw/plugin-sdk/video-generation";
|
||||
import {
|
||||
DEFAULT_XAI_VIDEO_BASE_URL,
|
||||
DEFAULT_XAI_VIDEO_MODEL,
|
||||
XAI_VIDEO_ASPECT_RATIOS,
|
||||
XAI_VIDEO_DEFAULT_TIMEOUT_MS,
|
||||
createXaiVideoGenerationProviderMetadata,
|
||||
isXaiVideo15Model,
|
||||
} from "./capability-provider-metadata.js";
|
||||
import {
|
||||
downloadXaiVideo,
|
||||
fetchXaiVideoResponse,
|
||||
type XaiVideoRequestPolicy,
|
||||
} from "./video-generation-transport.js";
|
||||
|
||||
const DEFAULT_XAI_VIDEO_BASE_URL = "https://api.x.ai/v1";
|
||||
const DEFAULT_XAI_VIDEO_MODEL = "grok-imagine-video";
|
||||
const XAI_VIDEO_15_MODEL = "grok-imagine-video-1.5";
|
||||
const XAI_VIDEO_15_MODEL_IDS = new Set([
|
||||
XAI_VIDEO_15_MODEL,
|
||||
"grok-imagine-video-1.5-preview",
|
||||
"grok-imagine-video-1.5-2026-05-30",
|
||||
]);
|
||||
const DEFAULT_TIMEOUT_MS = 600_000;
|
||||
const POLL_INTERVAL_MS = 5_000;
|
||||
const MAX_POLL_ATTEMPTS = 120;
|
||||
const XAI_VIDEO_ASPECT_RATIOS = new Set(["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3"]);
|
||||
const XAI_VIDEO_15_CAPABILITIES = {
|
||||
imageToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputImages: 1,
|
||||
maxDurationSeconds: 15,
|
||||
aspectRatios: [...XAI_VIDEO_ASPECT_RATIOS],
|
||||
resolutions: ["480P", "720P", "1080P"],
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
},
|
||||
videoToVideo: {
|
||||
enabled: false,
|
||||
},
|
||||
} satisfies VideoGenerationProviderCapabilities;
|
||||
const XAI_VIDEO_MALFORMED_RESPONSE = "xAI video generation response malformed";
|
||||
// xAI documents these as the only meaningful values; everything else (queued,
|
||||
// processing, submitted, pending, in_progress, ...) means "keep polling".
|
||||
@@ -169,11 +150,6 @@ function isReferenceImage(input: VideoGenerationSourceInput): boolean {
|
||||
return normalizeOptionalString(input.role)?.toLowerCase() === "reference_image";
|
||||
}
|
||||
|
||||
function isXaiVideo15Model(model: string | undefined): boolean {
|
||||
const normalized = normalizeOptionalString(model);
|
||||
return normalized ? XAI_VIDEO_15_MODEL_IDS.has(normalized) : false;
|
||||
}
|
||||
|
||||
function isFirstFrameImage(input: VideoGenerationSourceInput): boolean {
|
||||
const role = normalizeOptionalString(input.role)?.toLowerCase();
|
||||
return role === undefined || role === "first_frame";
|
||||
@@ -385,9 +361,9 @@ async function pollXaiVideo(
|
||||
},
|
||||
timeoutMs: createProviderOperationTimeoutResolver({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
defaultTimeoutMs: XAI_VIDEO_DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
defaultTimeoutMs: XAI_VIDEO_DEFAULT_TIMEOUT_MS,
|
||||
allowPrivateNetwork: params.allowPrivateNetwork,
|
||||
dispatcherPolicy: params.dispatcherPolicy,
|
||||
fetchFn: params.fetchFn,
|
||||
@@ -418,52 +394,7 @@ async function pollXaiVideo(
|
||||
|
||||
export function buildXaiVideoGenerationProvider(): VideoGenerationProvider {
|
||||
return {
|
||||
id: "xai",
|
||||
label: "xAI",
|
||||
defaultModel: DEFAULT_XAI_VIDEO_MODEL,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
models: [DEFAULT_XAI_VIDEO_MODEL, XAI_VIDEO_15_MODEL],
|
||||
catalogByModel: {
|
||||
[XAI_VIDEO_15_MODEL]: {
|
||||
capabilities: XAI_VIDEO_15_CAPABILITIES,
|
||||
modes: ["imageToVideo"],
|
||||
},
|
||||
},
|
||||
isConfigured: (ctx) => isProviderApiKeyConfigured({ provider: "xai", ...ctx }),
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxVideos: 1,
|
||||
maxDurationSeconds: 15,
|
||||
aspectRatios: [...XAI_VIDEO_ASPECT_RATIOS],
|
||||
resolutions: ["480P", "720P"],
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
},
|
||||
imageToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputImages: 7,
|
||||
maxDurationSeconds: 15,
|
||||
aspectRatios: [...XAI_VIDEO_ASPECT_RATIOS],
|
||||
resolutions: ["480P", "720P"],
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
},
|
||||
videoToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputVideos: 1,
|
||||
maxDurationSeconds: 10,
|
||||
supportsAspectRatio: false,
|
||||
supportsResolution: false,
|
||||
},
|
||||
},
|
||||
resolveModelCapabilities: ({ model }): VideoGenerationProviderCapabilities | undefined => {
|
||||
if (!isXaiVideo15Model(model)) {
|
||||
return undefined;
|
||||
}
|
||||
return XAI_VIDEO_15_CAPABILITIES;
|
||||
},
|
||||
...createXaiVideoGenerationProviderMetadata(),
|
||||
async generateVideo(req) {
|
||||
// Validate provider/model mode constraints before auth or HTTP setup so
|
||||
// unsupported 1.5 requests cannot be submitted and billed accidentally.
|
||||
@@ -507,7 +438,7 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider {
|
||||
body: createBody,
|
||||
timeoutMs: resolveProviderOperationTimeoutMs({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
defaultTimeoutMs: XAI_VIDEO_DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
fetchFn,
|
||||
allowPrivateNetwork,
|
||||
@@ -528,7 +459,7 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider {
|
||||
headers,
|
||||
timeoutMs: resolveProviderOperationTimeoutMs({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
defaultTimeoutMs: XAI_VIDEO_DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
baseUrl,
|
||||
allowPrivateNetwork,
|
||||
@@ -543,9 +474,9 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider {
|
||||
url: videoUrl,
|
||||
timeoutMs: createProviderOperationTimeoutResolver({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
defaultTimeoutMs: XAI_VIDEO_DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
defaultTimeoutMs: XAI_VIDEO_DEFAULT_TIMEOUT_MS,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
fetchFn,
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const oauthRuntimeMocks = vi.hoisted(() => ({
|
||||
loginXaiDeviceCode: vi.fn(),
|
||||
refreshXaiOAuthCredential: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./xai-oauth.js", () => oauthRuntimeMocks);
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
oauthRuntimeMocks.loginXaiDeviceCode.mockReset();
|
||||
oauthRuntimeMocks.refreshXaiOAuthCredential.mockReset();
|
||||
oauthRuntimeMocks.loginXaiDeviceCode.mockResolvedValue({ profiles: [] });
|
||||
oauthRuntimeMocks.refreshXaiOAuthCredential.mockResolvedValue({
|
||||
type: "oauth",
|
||||
provider: "xai",
|
||||
access: "next-access",
|
||||
refresh: "next-refresh",
|
||||
expires: 123,
|
||||
});
|
||||
});
|
||||
|
||||
describe("xAI OAuth lazy entry", () => {
|
||||
it("loads OAuth runtime only when an auth operation runs", async () => {
|
||||
const entry = await import("./xai-oauth-entry.js");
|
||||
const method = entry.createXaiOAuthAuthMethod();
|
||||
|
||||
expect(oauthRuntimeMocks.loginXaiDeviceCode).not.toHaveBeenCalled();
|
||||
expect(oauthRuntimeMocks.refreshXaiOAuthCredential).not.toHaveBeenCalled();
|
||||
|
||||
await method.run({} as never);
|
||||
expect(oauthRuntimeMocks.loginXaiDeviceCode).toHaveBeenCalledOnce();
|
||||
|
||||
await entry.refreshXaiOAuthCredential({
|
||||
type: "oauth",
|
||||
provider: "xai",
|
||||
access: "access",
|
||||
refresh: "refresh",
|
||||
expires: 1,
|
||||
});
|
||||
expect(oauthRuntimeMocks.refreshXaiOAuthCredential).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,56 @@
|
||||
import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
|
||||
import type { ProviderAuthMethod } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import type { OAuthCredential } from "openclaw/plugin-sdk/provider-auth";
|
||||
|
||||
const PROVIDER_ID = "xai";
|
||||
const XAI_OAUTH_METHOD_ID = "oauth";
|
||||
const XAI_OAUTH_CHOICE_ID = "xai-oauth";
|
||||
const XAI_DEVICE_CODE_METHOD_ID = "device-code";
|
||||
const XAI_DEVICE_CODE_CHOICE_ID = "xai-device-code";
|
||||
|
||||
const loadXaiOAuthRuntime = createLazyRuntimeModule(() => import("./xai-oauth.js"));
|
||||
|
||||
export function createXaiOAuthAuthMethod(): ProviderAuthMethod {
|
||||
return {
|
||||
id: XAI_OAUTH_METHOD_ID,
|
||||
label: "xAI OAuth",
|
||||
hint: "Remote-friendly browser sign-in without a localhost callback",
|
||||
kind: "oauth",
|
||||
wizard: {
|
||||
choiceId: XAI_OAUTH_CHOICE_ID,
|
||||
choiceLabel: "xAI OAuth",
|
||||
choiceHint: "Remote-friendly browser sign-in without a localhost callback",
|
||||
groupId: PROVIDER_ID,
|
||||
groupLabel: "xAI (Grok)",
|
||||
groupHint: "API key or OAuth",
|
||||
methodId: XAI_OAUTH_METHOD_ID,
|
||||
},
|
||||
run: async (ctx) => (await loadXaiOAuthRuntime()).loginXaiDeviceCode(ctx),
|
||||
};
|
||||
}
|
||||
|
||||
export function createXaiDeviceCodeAuthMethod(): ProviderAuthMethod {
|
||||
return {
|
||||
id: XAI_DEVICE_CODE_METHOD_ID,
|
||||
label: "xAI device code",
|
||||
hint: "Deprecated alias for xAI OAuth device-code login",
|
||||
kind: "device_code",
|
||||
wizard: {
|
||||
choiceId: XAI_DEVICE_CODE_CHOICE_ID,
|
||||
choiceLabel: "xAI device code",
|
||||
choiceHint: "Compatibility alias for xAI OAuth device-code sign-in",
|
||||
assistantVisibility: "manual-only",
|
||||
groupId: PROVIDER_ID,
|
||||
groupLabel: "xAI (Grok)",
|
||||
groupHint: "API key or OAuth",
|
||||
methodId: XAI_DEVICE_CODE_METHOD_ID,
|
||||
},
|
||||
run: async (ctx) => (await loadXaiOAuthRuntime()).loginXaiDeviceCode(ctx),
|
||||
};
|
||||
}
|
||||
|
||||
export async function refreshXaiOAuthCredential(
|
||||
credential: OAuthCredential,
|
||||
): Promise<OAuthCredential> {
|
||||
return await (await loadXaiOAuthRuntime()).refreshXaiOAuthCredential(credential);
|
||||
}
|
||||
@@ -6,11 +6,8 @@ import {
|
||||
} from "openclaw/plugin-sdk/plugin-test-runtime";
|
||||
import type { OAuthCredential } from "openclaw/plugin-sdk/provider-auth";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
createXaiDeviceCodeAuthMethod,
|
||||
createXaiOAuthAuthMethod,
|
||||
refreshXaiOAuthCredential,
|
||||
} from "./xai-oauth.js";
|
||||
import { createXaiDeviceCodeAuthMethod, createXaiOAuthAuthMethod } from "./xai-oauth-entry.js";
|
||||
import { refreshXaiOAuthCredential } from "./xai-oauth.js";
|
||||
|
||||
const XAI_OAUTH_CLIENT_ID = "b1a00492-073a-47ea-816f-4c329264a828";
|
||||
const XAI_OAUTH_SCOPE = "openid profile email offline_access grok-cli:access api:access";
|
||||
|
||||
@@ -5,7 +5,7 @@ import {
|
||||
resolveExpiresAtMsFromDurationSeconds,
|
||||
resolveExpiresAtMsFromEpochSeconds,
|
||||
} from "openclaw/plugin-sdk/number-runtime";
|
||||
import type { ProviderAuthContext, ProviderAuthMethod } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import type { ProviderAuthContext } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import {
|
||||
buildOauthProviderAuthResult,
|
||||
toFormUrlEncoded,
|
||||
@@ -19,10 +19,6 @@ import { applyXaiOAuthConfig, XAI_OAUTH_DEFAULT_MODEL_REF } from "./onboard.js";
|
||||
import { xaiUserAgent } from "./src/xai-user-agent.js";
|
||||
|
||||
const PROVIDER_ID = "xai";
|
||||
const XAI_OAUTH_METHOD_ID = "oauth";
|
||||
const XAI_OAUTH_CHOICE_ID = "xai-oauth";
|
||||
const XAI_DEVICE_CODE_METHOD_ID = "device-code";
|
||||
const XAI_DEVICE_CODE_CHOICE_ID = "xai-device-code";
|
||||
const XAI_OAUTH_CLIENT_ID = "b1a00492-073a-47ea-816f-4c329264a828";
|
||||
const XAI_OAUTH_SCOPE = "openid profile email offline_access grok-cli:access api:access";
|
||||
const XAI_OAUTH_ISSUER = "https://auth.x.ai";
|
||||
@@ -607,7 +603,7 @@ async function noteXaiDeviceCode(
|
||||
);
|
||||
}
|
||||
|
||||
async function loginXaiDeviceCode(ctx: ProviderAuthContext): Promise<ProviderAuthResult> {
|
||||
export async function loginXaiDeviceCode(ctx: ProviderAuthContext): Promise<ProviderAuthResult> {
|
||||
const progress = ctx.prompter.progress("Starting xAI OAuth...");
|
||||
try {
|
||||
const discovery = await fetchXaiDeviceCodeDiscovery(
|
||||
@@ -708,42 +704,3 @@ export async function refreshXaiOAuthCredential(
|
||||
issuer: XAI_OAUTH_ISSUER,
|
||||
} as OAuthCredential;
|
||||
}
|
||||
|
||||
export function createXaiOAuthAuthMethod(): ProviderAuthMethod {
|
||||
return {
|
||||
id: XAI_OAUTH_METHOD_ID,
|
||||
label: "xAI OAuth",
|
||||
hint: "Remote-friendly browser sign-in without a localhost callback",
|
||||
kind: "oauth",
|
||||
wizard: {
|
||||
choiceId: XAI_OAUTH_CHOICE_ID,
|
||||
choiceLabel: "xAI OAuth",
|
||||
choiceHint: "Remote-friendly browser sign-in without a localhost callback",
|
||||
groupId: PROVIDER_ID,
|
||||
groupLabel: "xAI (Grok)",
|
||||
groupHint: "API key or OAuth",
|
||||
methodId: XAI_OAUTH_METHOD_ID,
|
||||
},
|
||||
run: async (ctx) => loginXaiDeviceCode(ctx),
|
||||
};
|
||||
}
|
||||
|
||||
export function createXaiDeviceCodeAuthMethod(): ProviderAuthMethod {
|
||||
return {
|
||||
id: XAI_DEVICE_CODE_METHOD_ID,
|
||||
label: "xAI device code",
|
||||
hint: "Deprecated alias for xAI OAuth device-code login",
|
||||
kind: "device_code",
|
||||
wizard: {
|
||||
choiceId: XAI_DEVICE_CODE_CHOICE_ID,
|
||||
choiceLabel: "xAI device code",
|
||||
choiceHint: "Compatibility alias for xAI OAuth device-code sign-in",
|
||||
assistantVisibility: "manual-only",
|
||||
groupId: PROVIDER_ID,
|
||||
groupLabel: "xAI (Grok)",
|
||||
groupHint: "API key or OAuth",
|
||||
methodId: XAI_DEVICE_CODE_METHOD_ID,
|
||||
},
|
||||
run: async (ctx) => loginXaiDeviceCode(ctx),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -170,6 +170,7 @@
|
||||
"!dist/plugin-sdk/qa-runner-runtime.d.ts",
|
||||
"!dist/plugin-sdk/realtime-bootstrap-context.d.ts",
|
||||
"!dist/plugin-sdk/realtime-transcription.d.ts",
|
||||
"!dist/plugin-sdk/realtime-voice-audio-queue.d.ts",
|
||||
"!dist/plugin-sdk/realtime-voice.d.ts",
|
||||
"!dist/plugin-sdk/reply-payload-testing.js",
|
||||
"!dist/plugin-sdk/reply-payload-testing.d.ts",
|
||||
@@ -1126,6 +1127,9 @@
|
||||
"./plugin-sdk/realtime-bootstrap-context": {
|
||||
"default": "./dist/plugin-sdk/realtime-bootstrap-context.js"
|
||||
},
|
||||
"./plugin-sdk/realtime-voice-audio-queue": {
|
||||
"default": "./dist/plugin-sdk/realtime-voice-audio-queue.js"
|
||||
},
|
||||
"./plugin-sdk/realtime-voice": {
|
||||
"default": "./dist/plugin-sdk/realtime-voice.js"
|
||||
},
|
||||
|
||||
@@ -232,6 +232,7 @@
|
||||
"reply-history",
|
||||
"realtime-transcription",
|
||||
"realtime-bootstrap-context",
|
||||
"realtime-voice-audio-queue",
|
||||
"realtime-voice",
|
||||
"meeting-runtime",
|
||||
"transcripts",
|
||||
|
||||
@@ -124,6 +124,7 @@
|
||||
"qa-runtime",
|
||||
"realtime-bootstrap-context",
|
||||
"realtime-transcription",
|
||||
"realtime-voice-audio-queue",
|
||||
"realtime-voice",
|
||||
"reply-payload-testing",
|
||||
"reply-reference",
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
/** Production-private queue seam for lazy realtime voice provider facades. */
|
||||
export {
|
||||
createRealtimeVoiceAudioQueue,
|
||||
type RealtimeVoiceAudioQueue,
|
||||
} from "../talk/realtime-session-lifecycle.js";
|
||||
@@ -237,6 +237,19 @@ describe("RealtimeVoiceSessionLifecycle", () => {
|
||||
});
|
||||
|
||||
describe("createRealtimeVoiceAudioQueue", () => {
|
||||
it("releases byte budget as queued audio is consumed", () => {
|
||||
const queue = createRealtimeVoiceAudioQueue("reject-newest");
|
||||
const first = Buffer.alloc(512 * 1024, 0x01);
|
||||
const second = Buffer.alloc(512 * 1024, 0x02);
|
||||
|
||||
expect(queue.enqueue(first)).toBe(true);
|
||||
expect(queue.enqueue(second)).toBe(true);
|
||||
expect(queue.enqueue(Buffer.from([0x03]))).toBe(false);
|
||||
expect(queue.dequeue()).toEqual(first);
|
||||
expect(queue.enqueue(Buffer.from([0x03]))).toBe(true);
|
||||
expect(queue.drain()).toEqual([second, Buffer.from([0x03])]);
|
||||
});
|
||||
|
||||
it("drops the oldest audio and resets accounting on clear", () => {
|
||||
const queue = createRealtimeVoiceAudioQueue("drop-oldest");
|
||||
for (let index = 0; index < 322; index += 1) {
|
||||
|
||||
@@ -5,6 +5,7 @@ type RealtimeVoiceAudioOverflowPolicy = "drop-oldest" | "reject-newest";
|
||||
|
||||
export type RealtimeVoiceAudioQueue = {
|
||||
clear: () => void;
|
||||
dequeue: () => Buffer | undefined;
|
||||
drain: () => Buffer[];
|
||||
enqueue: (audio: Buffer) => boolean;
|
||||
};
|
||||
@@ -22,6 +23,13 @@ export function createRealtimeVoiceAudioQueue(
|
||||
|
||||
return {
|
||||
clear,
|
||||
dequeue: () => {
|
||||
const chunk = chunks.shift();
|
||||
if (chunk) {
|
||||
bytes -= chunk.byteLength;
|
||||
}
|
||||
return chunk;
|
||||
},
|
||||
drain: () => {
|
||||
const drained = chunks;
|
||||
clear();
|
||||
|
||||
Reference in New Issue
Block a user