diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index 4ab07f3d618a..293215aa1833 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -338,6 +338,7 @@ Use `isLoopbackHost(host)` when a plugin must accept only the local machine. It | `plugin-sdk/speech-settings` | Lightweight TTS config resolution and normalization primitives without provider registries or synthesis runtime | | `plugin-sdk/realtime-transcription` | Private-local after July 2026; Realtime transcription provider types, registry helpers, and shared WebSocket session helper | | `plugin-sdk/realtime-bootstrap-context` | Private-local after July 2026; Realtime profile bootstrap helper for bounded `IDENTITY.md`, `USER.md`, and `SOUL.md` context injection | + | `plugin-sdk/realtime-voice-audio-queue` | Private-local JavaScript-only host runtime for bundled or separately published official plugins; narrow bounded audio queue seam for lazy realtime voice provider facades without importing the broader realtime voice runtime; not for third-party plugins | | `plugin-sdk/realtime-voice` | Private-local after July 2026; Realtime voice provider types, registry helpers, shared audio-energy/speech-onset gates, and realtime voice behavior helpers, including the transport-independent session harness and output activity tracking. For official runtime consumers, sender-auth contract revision 1 forwards ingress-authenticated `senderId` and `senderIsOwner` unchanged; ingress owns authentication, and consumers requiring the handoff must fail closed on other revisions. | | `plugin-sdk/meeting-runtime` | Browser-meeting session runtime, realtime audio engines/transports, `MeetingPlatformAdapter`, browser/node control, agent-consult, voice-call delegation, setup checks, and SoX command helpers | | `plugin-sdk/image-generation` | Private-local after July 2026; Image generation provider types plus image asset/data URL helpers and the OpenAI-compatible image provider builder | diff --git a/extensions/google/index.ts b/extensions/google/index.ts index d9d913cbfa35..36a4dd3e0efb 100644 --- a/extensions/google/index.ts +++ b/extensions/google/index.ts @@ -9,7 +9,7 @@ import type { RealtimeVoiceProviderConfig, RealtimeVoiceProviderPlugin, } from "openclaw/plugin-sdk/realtime-voice"; -import { createRealtimeVoiceAudioQueue } from "openclaw/plugin-sdk/realtime-voice"; +import { createRealtimeVoiceAudioQueue } from "openclaw/plugin-sdk/realtime-voice-audio-queue"; import { normalizeResolvedSecretInputString } from "openclaw/plugin-sdk/secret-input"; import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; import type { VideoGenerationProvider } from "openclaw/plugin-sdk/video-generation"; diff --git a/extensions/tsconfig.package-boundary.paths.json b/extensions/tsconfig.package-boundary.paths.json index bc644fb725c3..52c2ab71731e 100644 --- a/extensions/tsconfig.package-boundary.paths.json +++ b/extensions/tsconfig.package-boundary.paths.json @@ -356,6 +356,9 @@ "openclaw/plugin-sdk/realtime-bootstrap-context": [ "../packages/plugin-sdk/dist/src/plugin-sdk/realtime-bootstrap-context.d.ts" ], + "openclaw/plugin-sdk/realtime-voice-audio-queue": [ + "../packages/plugin-sdk/dist/src/plugin-sdk/realtime-voice-audio-queue.d.ts" + ], "openclaw/plugin-sdk/realtime-voice": [ "../packages/plugin-sdk/dist/src/plugin-sdk/realtime-voice.d.ts" ], diff --git a/extensions/xai/api.ts b/extensions/xai/api.ts index 1dfd8a761cc8..702dfa60cecd 100644 --- a/extensions/xai/api.ts +++ b/extensions/xai/api.ts @@ -1,16 +1,11 @@ // Xai API module exposes the plugin public contract. -import { - normalizeOptionalLowercaseString, - normalizeOptionalString, -} from "openclaw/plugin-sdk/string-coerce-runtime"; +import { normalizeOptionalLowercaseString } from "openclaw/plugin-sdk/string-coerce-runtime"; import { applyXaiModelCompat, HTML_ENTITY_TOOL_CALL_ARGUMENTS_ENCODING, normalizeNativeXaiModelId, XAI_TOOL_SCHEMA_PROFILE, } from "./model-compat.js"; -import { XAI_BASE_URL } from "./model-definitions.js"; -import { isXaiProviderId } from "./provider-id.js"; export { buildXaiProvider } from "./provider-catalog.js"; export { applyXaiConfig, applyXaiProviderConfig, XAI_DEFAULT_MODEL_REF } from "./onboard.js"; @@ -29,22 +24,7 @@ export { export { isModernXaiModel, resolveXaiForwardCompatModel } from "./provider-models.js"; export { applyXaiRuntimeModelCompat } from "./runtime-model-compat.js"; export { applyXaiModelCompat, HTML_ENTITY_TOOL_CALL_ARGUMENTS_ENCODING, XAI_TOOL_SCHEMA_PROFILE }; - -const XAI_NATIVE_ENDPOINT_HOSTS = new Set(["api.x.ai"]); - -function resolveHostname(value: string): string | undefined { - try { - return new URL(value).hostname.toLowerCase(); - } catch { - return undefined; - } -} - -function isXaiNativeEndpoint(baseUrl: unknown): boolean { - return ( - typeof baseUrl === "string" && XAI_NATIVE_ENDPOINT_HOSTS.has(resolveHostname(baseUrl) ?? "") - ); -} +export { resolveXaiTransport } from "./provider-routing.js"; export function isXaiModelHint(modelId: string): boolean { return getModelProviderHint(modelId) === "x-ai"; @@ -63,32 +43,3 @@ function getModelProviderHint(modelId: string): string | null { } return trimmed.slice(0, slashIndex) || null; } - -function shouldUseXaiResponsesTransport(params: { - provider: string; - api?: unknown; - baseUrl?: unknown; -}): boolean { - const hasDefaultXaiRoute = - isXaiProviderId(params.provider) && !normalizeOptionalString(params.baseUrl); - return params.api === "openai-responses" - ? hasDefaultXaiRoute - : params.api === "openai-completions" && - (isXaiNativeEndpoint(params.baseUrl) || hasDefaultXaiRoute); -} - -export function resolveXaiTransport(params: { - provider: string; - api?: unknown; - baseUrl?: unknown; -}): { api: "openai-responses"; baseUrl?: string } | undefined { - if (!shouldUseXaiResponsesTransport(params)) { - return undefined; - } - return { - api: "openai-responses", - baseUrl: - normalizeOptionalString(params.baseUrl) ?? - (isXaiProviderId(params.provider) ? XAI_BASE_URL : undefined), - }; -} diff --git a/extensions/xai/capability-provider-metadata.ts b/extensions/xai/capability-provider-metadata.ts new file mode 100644 index 000000000000..0b8799cc1441 --- /dev/null +++ b/extensions/xai/capability-provider-metadata.ts @@ -0,0 +1,288 @@ +import type { ImageGenerationProvider } from "openclaw/plugin-sdk/image-generation"; +import type { MediaUnderstandingProvider } from "openclaw/plugin-sdk/media-understanding"; +import { + isProviderApiKeyConfigured, + isProviderAuthProfileConfigured, +} from "openclaw/plugin-sdk/provider-auth"; +import type { + RealtimeTranscriptionProviderConfig, + RealtimeTranscriptionProviderPlugin, +} from "openclaw/plugin-sdk/realtime-transcription"; +import type { + RealtimeVoiceAudioFormat, + RealtimeVoiceBridgeCreateRequest, + RealtimeVoiceProviderPlugin, +} from "openclaw/plugin-sdk/realtime-voice"; +import { normalizeResolvedSecretInputString } from "openclaw/plugin-sdk/secret-input"; +import { + isRecord, + normalizeOptionalString, + parseBooleanValue as readBoolean, + parseFiniteNumber as readFiniteNumber, +} from "openclaw/plugin-sdk/string-coerce-runtime"; +import type { + VideoGenerationProvider, + VideoGenerationProviderCapabilities, +} from "openclaw/plugin-sdk/video-generation"; +import { XAI_DEFAULT_IMAGE_MODEL, XAI_IMAGE_MODELS } from "./model-definitions.js"; +import { + XAI_REALTIME_DEFAULT_MODEL, + XAI_REALTIME_VOICES, + hasXaiRealtimeApiKeyInput, + normalizeXaiRealtimeProviderConfig, +} from "./realtime-voice-config.js"; + +export const XAI_IMAGE_DEFAULT_TIMEOUT_MS = 600_000; +export const XAI_SUPPORTED_IMAGE_ASPECT_RATIOS = [ + "1:1", + "16:9", + "9:16", + "4:3", + "3:4", + "3:2", + "2:3", + "2:1", + "1:2", + "19.5:9", + "9:19.5", + "20:9", + "9:20", +] as const; + +export function createXaiImageGenerationProviderMetadata() { + return { + id: "xai", + label: "xAI", + defaultModel: XAI_DEFAULT_IMAGE_MODEL, + defaultTimeoutMs: XAI_IMAGE_DEFAULT_TIMEOUT_MS, + models: [...XAI_IMAGE_MODELS], + capabilities: { + generate: { + maxCount: 4, + supportsAspectRatio: true, + supportsResolution: true, + supportsSize: false, + }, + edit: { + enabled: true, + maxCount: 4, + maxInputImages: 3, + supportsAspectRatio: true, + supportsResolution: true, + supportsSize: false, + }, + geometry: { + aspectRatios: [...XAI_SUPPORTED_IMAGE_ASPECT_RATIOS], + resolutions: ["1K", "2K"], + }, + }, + } satisfies Omit; +} + +export function createXaiMediaUnderstandingProviderMetadata() { + return { + id: "xai", + capabilities: ["audio"], + autoPriority: { audio: 25 }, + } satisfies Omit; +} + +export const DEFAULT_XAI_VIDEO_BASE_URL = "https://api.x.ai/v1"; +export const DEFAULT_XAI_VIDEO_MODEL = "grok-imagine-video"; +const XAI_VIDEO_15_MODEL = "grok-imagine-video-1.5"; +export const XAI_VIDEO_DEFAULT_TIMEOUT_MS = 600_000; +export const XAI_VIDEO_ASPECT_RATIOS = new Set(["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3"]); +const XAI_VIDEO_15_CAPABILITIES = { + imageToVideo: { + enabled: true, + maxVideos: 1, + maxInputImages: 1, + maxDurationSeconds: 15, + aspectRatios: [...XAI_VIDEO_ASPECT_RATIOS], + resolutions: ["480P", "720P", "1080P"], + supportsAspectRatio: true, + supportsResolution: true, + }, + videoToVideo: { + enabled: false, + }, +} satisfies VideoGenerationProviderCapabilities; + +const XAI_VIDEO_15_MODEL_IDS = new Set([ + XAI_VIDEO_15_MODEL, + "grok-imagine-video-1.5-preview", + "grok-imagine-video-1.5-2026-05-30", +]); + +export function isXaiVideo15Model(model: string | undefined): boolean { + const normalized = normalizeOptionalString(model); + return normalized ? XAI_VIDEO_15_MODEL_IDS.has(normalized) : false; +} + +export function createXaiVideoGenerationProviderMetadata() { + return { + id: "xai", + label: "xAI", + defaultModel: DEFAULT_XAI_VIDEO_MODEL, + defaultTimeoutMs: XAI_VIDEO_DEFAULT_TIMEOUT_MS, + models: [DEFAULT_XAI_VIDEO_MODEL, XAI_VIDEO_15_MODEL], + catalogByModel: { + [XAI_VIDEO_15_MODEL]: { + capabilities: XAI_VIDEO_15_CAPABILITIES, + modes: ["imageToVideo"], + }, + }, + isConfigured: (ctx) => isProviderApiKeyConfigured({ provider: "xai", ...ctx }), + capabilities: { + generate: { + maxVideos: 1, + maxDurationSeconds: 15, + aspectRatios: [...XAI_VIDEO_ASPECT_RATIOS], + resolutions: ["480P", "720P"], + supportsAspectRatio: true, + supportsResolution: true, + }, + imageToVideo: { + enabled: true, + maxVideos: 1, + maxInputImages: 7, + maxDurationSeconds: 15, + aspectRatios: [...XAI_VIDEO_ASPECT_RATIOS], + resolutions: ["480P", "720P"], + supportsAspectRatio: true, + supportsResolution: true, + }, + videoToVideo: { + enabled: true, + maxVideos: 1, + maxInputVideos: 1, + maxDurationSeconds: 10, + supportsAspectRatio: false, + supportsResolution: false, + }, + }, + resolveModelCapabilities: ({ model }) => + isXaiVideo15Model(model) ? XAI_VIDEO_15_CAPABILITIES : undefined, + } satisfies Omit; +} + +export type XaiRealtimeTranscriptionEncoding = "pcm" | "mulaw" | "alaw"; + +type XaiRealtimeTranscriptionProviderConfig = { + apiKey?: string; + baseUrl?: string; + sampleRate?: number; + encoding?: XaiRealtimeTranscriptionEncoding; + interimResults?: boolean; + endpointingMs?: number; + language?: string; +}; + +function normalizeRealtimeTranscriptionEncoding( + value: unknown, +): XaiRealtimeTranscriptionEncoding | undefined { + const normalized = normalizeOptionalString(value)?.toLowerCase(); + if (!normalized) { + return undefined; + } + if (normalized === "ulaw" || normalized === "g711_ulaw" || normalized === "g711-mulaw") { + return "mulaw"; + } + if (normalized === "g711_alaw" || normalized === "g711-alaw") { + return "alaw"; + } + if (normalized === "pcm" || normalized === "mulaw" || normalized === "alaw") { + return normalized; + } + throw new Error(`Invalid xAI realtime transcription encoding: ${normalized}`); +} + +export function normalizeXaiRealtimeTranscriptionProviderConfig( + config: RealtimeTranscriptionProviderConfig, +): XaiRealtimeTranscriptionProviderConfig { + const raw = isRecord(config) ? config : undefined; + const providers = isRecord(raw?.providers) ? raw.providers : undefined; + const nested = providers?.xai ?? raw?.xai ?? raw; + const xai = isRecord(nested) ? nested : {}; + return { + apiKey: normalizeResolvedSecretInputString({ + value: xai.apiKey, + path: "plugins.entries.voice-call.config.streaming.providers.xai.apiKey", + }), + baseUrl: normalizeOptionalString(xai.baseUrl), + sampleRate: readFiniteNumber(xai.sampleRate ?? xai.sample_rate), + encoding: normalizeRealtimeTranscriptionEncoding(xai.encoding), + interimResults: readBoolean(xai.interimResults ?? xai.interim_results), + endpointingMs: readFiniteNumber(xai.endpointingMs ?? xai.endpointing ?? xai.silenceDurationMs), + language: normalizeOptionalString(xai.language), + }; +} + +export function createXaiRealtimeTranscriptionProviderMetadata() { + return { + id: "xai", + label: "xAI Realtime Transcription", + aliases: ["xai-realtime", "grok-stt-streaming"], + autoSelectOrder: 25, + resolveConfig: ({ rawConfig }) => normalizeXaiRealtimeTranscriptionProviderConfig(rawConfig), + isConfigured: ({ providerConfig, cfg }) => + Boolean( + normalizeXaiRealtimeTranscriptionProviderConfig(providerConfig).apiKey ?? + normalizeOptionalString(process.env.XAI_API_KEY), + ) || isProviderAuthProfileConfigured({ provider: "xai", cfg }), + } satisfies Omit; +} + +const XAI_REALTIME_AUDIO_FORMAT_G711_ULAW_8KHZ = { + encoding: "g711_ulaw", + sampleRateHz: 8000, + channels: 1, +} satisfies RealtimeVoiceAudioFormat; +const XAI_REALTIME_AUDIO_FORMAT_PCM16_24KHZ = { + encoding: "pcm16", + sampleRateHz: 24000, + channels: 1, +} satisfies RealtimeVoiceAudioFormat; + +export function createXaiRealtimeVoiceProviderMetadata() { + return { + id: "xai", + label: "xAI Grok Voice", + aliases: ["xai-realtime-voice", "grok-voice"], + defaultModel: XAI_REALTIME_DEFAULT_MODEL, + voices: XAI_REALTIME_VOICES, + autoSelectOrder: 25, + capabilities: { + transports: ["gateway-relay"], + inputAudioFormats: [ + XAI_REALTIME_AUDIO_FORMAT_G711_ULAW_8KHZ, + XAI_REALTIME_AUDIO_FORMAT_PCM16_24KHZ, + ], + outputAudioFormats: [ + XAI_REALTIME_AUDIO_FORMAT_G711_ULAW_8KHZ, + XAI_REALTIME_AUDIO_FORMAT_PCM16_24KHZ, + ], + supportsBargeIn: true, + handlesInputAudioBargeIn: true, + supportsToolCalls: true, + supportsSessionResumption: true, + }, + resolveConfig: ({ rawConfig }) => normalizeXaiRealtimeProviderConfig(rawConfig), + isConfigured: ({ providerConfig, cfg }) => + hasXaiRealtimeApiKeyInput(normalizeXaiRealtimeProviderConfig(providerConfig).apiKey, cfg), + } satisfies Omit; +} + +export function assertXaiRealtimeVoiceRequestSupported( + req: RealtimeVoiceBridgeCreateRequest, +): void { + const config = normalizeXaiRealtimeProviderConfig(req.providerConfig); + if (req.autoRespondToAudio === false) { + throw new Error( + 'xAI realtime voice requires automatic server-VAD responses; use consultRouting: "provider-direct"', + ); + } + if ((req.interruptResponseOnInputAudio ?? config.interruptResponseOnInputAudio) === false) { + throw new Error("xAI realtime voice requires automatic server-VAD interruption handling"); + } +} diff --git a/extensions/xai/image-generation-provider.ts b/extensions/xai/image-generation-provider.ts index d5951174b465..27e6bee6643c 100644 --- a/extensions/xai/image-generation-provider.ts +++ b/extensions/xai/image-generation-provider.ts @@ -12,25 +12,12 @@ import { normalizeOptionalLowercaseString, normalizeOptionalString, } from "openclaw/plugin-sdk/string-coerce-runtime"; -import { XAI_BASE_URL, XAI_DEFAULT_IMAGE_MODEL, XAI_IMAGE_MODELS } from "./model-definitions.js"; - -const DEFAULT_TIMEOUT_MS = 600_000; - -const XAI_SUPPORTED_ASPECT_RATIOS = [ - "1:1", - "16:9", - "9:16", - "4:3", - "3:4", - "3:2", - "2:3", - "2:1", - "1:2", - "19.5:9", - "9:19.5", - "20:9", - "9:20", -] as const; +import { + XAI_IMAGE_DEFAULT_TIMEOUT_MS, + XAI_SUPPORTED_IMAGE_ASPECT_RATIOS, + createXaiImageGenerationProviderMetadata, +} from "./capability-provider-metadata.js"; +import { XAI_BASE_URL } from "./model-definitions.js"; function resolveImageForEdit( input: (ImageGenerationSourceImage & { url?: string }) | undefined, @@ -66,7 +53,7 @@ function buildBody(params: { }; const aspect = normalizeOptionalString(params.req.aspectRatio); - if (aspect && (XAI_SUPPORTED_ASPECT_RATIOS as readonly string[]).includes(aspect)) { + if (aspect && (XAI_SUPPORTED_IMAGE_ASPECT_RATIOS as readonly string[]).includes(aspect)) { body.aspect_ratio = aspect; } @@ -93,35 +80,13 @@ function buildBody(params: { } export function buildXaiImageGenerationProvider(): ImageGenerationProvider { + const metadata = createXaiImageGenerationProviderMetadata(); return createOpenAiCompatibleImageGenerationProvider({ - id: "xai", - label: "xAI", - defaultModel: XAI_DEFAULT_IMAGE_MODEL, - models: [...XAI_IMAGE_MODELS], - capabilities: { - generate: { - maxCount: 4, - supportsAspectRatio: true, - supportsResolution: true, - supportsSize: false, - }, - edit: { - enabled: true, - maxCount: 4, - maxInputImages: 3, - supportsAspectRatio: true, - supportsResolution: true, - supportsSize: false, - }, - geometry: { - aspectRatios: [...XAI_SUPPORTED_ASPECT_RATIOS], - resolutions: ["1K", "2K"], - }, - }, + ...metadata, defaultBaseUrl: XAI_BASE_URL, resolveBaseUrl: ({ req }) => resolveXaiImageBaseUrl(req), resolveAllowPrivateNetwork: () => false, - defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + defaultTimeoutMs: XAI_IMAGE_DEFAULT_TIMEOUT_MS, buildGenerateRequest: ({ req, inputImages, model, count }) => ({ kind: "json", body: buildBody({ req, inputImages, model, count }), diff --git a/extensions/xai/index.ts b/extensions/xai/index.ts index adeffc507493..7c9771e90ce9 100644 --- a/extensions/xai/index.ts +++ b/extensions/xai/index.ts @@ -5,15 +5,19 @@ import { defineSingleProviderPluginEntry } from "openclaw/plugin-sdk/provider-en import { buildProviderReplayFamilyHooks } from "openclaw/plugin-sdk/provider-model-shared"; import { defaultToolStreamExtraParams } from "openclaw/plugin-sdk/provider-stream-shared"; import { jsonResult } from "openclaw/plugin-sdk/provider-web-search"; -import { - buildXaiImageGenerationProvider, - normalizeXaiModelId, - resolveXaiTransport, -} from "./api.js"; import { buildMissingCodeExecutionApiKeyPayload, createCodeExecutionToolDefinition, } from "./code-execution-tool-shared.js"; +import { + createLazyXaiImageGenerationProvider, + createLazyXaiMediaUnderstandingProvider, + createLazyXaiRealtimeTranscriptionProvider, + createLazyXaiRealtimeVoiceProvider, + createLazyXaiSpeechProvider, + createLazyXaiVideoGenerationProvider, +} from "./lazy-capability-providers.js"; +import { normalizeNativeXaiModelId } from "./model-compat.js"; import { applyXaiConfig, XAI_DEFAULT_MODEL_REF } from "./onboard.js"; import { buildLiveXaiOAuthProvider, @@ -27,9 +31,7 @@ import { resolveXaiForwardCompatModel, } from "./provider-models.js"; import { resolveThinkingProfile } from "./provider-policy-api.js"; -import { buildXaiRealtimeTranscriptionProvider } from "./realtime-transcription-provider.js"; -import { buildXaiRealtimeVoiceProvider } from "./realtime-voice-provider.js"; -import { buildXaiSpeechProvider } from "./speech-provider.js"; +import { resolveXaiTransport } from "./provider-routing.js"; import { readPluginCodeExecutionConfig, resolveCodeExecutionEnabled, @@ -41,8 +43,6 @@ import { } from "./src/tool-auth-shared.js"; import { resolveEffectiveXSearchConfig } from "./src/x-search-config.js"; import { wrapXaiProviderStream } from "./stream.js"; -import { buildXaiMediaUnderstandingProvider } from "./stt.js"; -import { buildXaiVideoGenerationProvider } from "./video-generation-provider.js"; import { createXaiWebSearchProvider } from "./web-search.js"; import { buildMissingXSearchApiKeyPayload, @@ -52,7 +52,7 @@ import { createXaiDeviceCodeAuthMethod, createXaiOAuthAuthMethod, refreshXaiOAuthCredential, -} from "./xai-oauth.js"; +} from "./xai-oauth-entry.js"; const PROVIDER_ID = "xai"; @@ -278,7 +278,7 @@ export default defineSingleProviderPluginEntry({ normalizeResolvedModel: ({ model }) => normalizeXaiResolvedModel(model), normalizeTransport: ({ provider, api, baseUrl }) => resolveXaiTransport({ provider, api, baseUrl }), - normalizeModelId: ({ modelId }) => normalizeXaiModelId(modelId), + normalizeModelId: ({ modelId }) => normalizeNativeXaiModelId(modelId), resolveDynamicModel: (ctx) => resolveXaiForwardCompatModel({ providerId: PROVIDER_ID, ctx }), refreshOAuth: refreshXaiOAuthCredential, resolveThinkingProfile, @@ -287,12 +287,12 @@ export default defineSingleProviderPluginEntry({ }), register(api) { api.registerWebSearchProvider(createXaiWebSearchProvider()); - api.registerMediaUnderstandingProvider(buildXaiMediaUnderstandingProvider()); - api.registerVideoGenerationProvider(buildXaiVideoGenerationProvider()); - api.registerImageGenerationProvider(buildXaiImageGenerationProvider()); - api.registerSpeechProvider(buildXaiSpeechProvider()); - api.registerRealtimeTranscriptionProvider(buildXaiRealtimeTranscriptionProvider()); - api.registerRealtimeVoiceProvider(buildXaiRealtimeVoiceProvider()); + api.registerMediaUnderstandingProvider(createLazyXaiMediaUnderstandingProvider()); + api.registerVideoGenerationProvider(createLazyXaiVideoGenerationProvider()); + api.registerImageGenerationProvider(createLazyXaiImageGenerationProvider()); + api.registerSpeechProvider(createLazyXaiSpeechProvider()); + api.registerRealtimeTranscriptionProvider(createLazyXaiRealtimeTranscriptionProvider()); + api.registerRealtimeVoiceProvider(createLazyXaiRealtimeVoiceProvider()); api.registerTool((ctx) => createLazyCodeExecutionTool(ctx), { name: "code_execution" }); api.registerTool((ctx) => createLazyXSearchTool(ctx), { name: "x_search" }); }, diff --git a/extensions/xai/lazy-capability-providers.test.ts b/extensions/xai/lazy-capability-providers.test.ts new file mode 100644 index 000000000000..9d4089ffdb9a --- /dev/null +++ b/extensions/xai/lazy-capability-providers.test.ts @@ -0,0 +1,838 @@ +import type { + RealtimeVoiceBridge, + RealtimeVoiceBridgeCreateRequest, +} from "openclaw/plugin-sdk/realtime-voice"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const runtimeMocks = vi.hoisted(() => { + const generateImage = vi.fn(); + const transcribeAudio = vi.fn(); + const generateVideo = vi.fn(); + const listVoices = vi.fn(); + const synthesize = vi.fn(); + const streamSynthesize = vi.fn(); + const synthesizeTelephony = vi.fn(); + const transcriptionConnect = vi.fn(); + const transcriptionSendAudio = vi.fn(); + const transcriptionClose = vi.fn(); + const transcriptionIsConnected = vi.fn(); + const createTranscriptionSession = vi.fn(); + const voiceConnect = vi.fn(); + const voiceSendAudio = vi.fn(); + const voiceSetMediaTimestamp = vi.fn(); + const voiceSendUserMessage = vi.fn(); + const voiceTriggerGreeting = vi.fn(); + const voiceHandleBargeIn = vi.fn(); + const voiceSubmitToolResult = vi.fn(); + const voiceAcknowledgeMark = vi.fn(); + const voiceClose = vi.fn(); + const voiceIsConnected = vi.fn(); + const createVoiceBridge = vi.fn(); + const buildImageProvider = vi.fn(); + const buildMediaProvider = vi.fn(); + const buildVideoProvider = vi.fn(); + const buildSpeechProvider = vi.fn(); + const buildTranscriptionProvider = vi.fn(); + const buildVoiceProvider = vi.fn(); + + return { + generateImage, + transcribeAudio, + generateVideo, + listVoices, + synthesize, + streamSynthesize, + synthesizeTelephony, + transcriptionConnect, + transcriptionSendAudio, + transcriptionClose, + transcriptionIsConnected, + createTranscriptionSession, + voiceConnect, + voiceSendAudio, + voiceSetMediaTimestamp, + voiceSendUserMessage, + voiceTriggerGreeting, + voiceHandleBargeIn, + voiceSubmitToolResult, + voiceAcknowledgeMark, + voiceClose, + voiceIsConnected, + createVoiceBridge, + buildImageProvider, + buildMediaProvider, + buildVideoProvider, + buildSpeechProvider, + buildTranscriptionProvider, + buildVoiceProvider, + }; +}); + +vi.mock("./image-generation-provider.js", () => ({ + buildXaiImageGenerationProvider: runtimeMocks.buildImageProvider, +})); +vi.mock("./stt.js", () => ({ + buildXaiMediaUnderstandingProvider: runtimeMocks.buildMediaProvider, +})); +vi.mock("./video-generation-provider.js", () => ({ + buildXaiVideoGenerationProvider: runtimeMocks.buildVideoProvider, +})); +vi.mock("./speech-provider.js", () => ({ + buildXaiSpeechProvider: runtimeMocks.buildSpeechProvider, +})); +vi.mock("./realtime-transcription-provider.js", () => ({ + buildXaiRealtimeTranscriptionProvider: runtimeMocks.buildTranscriptionProvider, +})); +vi.mock("./realtime-voice-provider.js", () => ({ + buildXaiRealtimeVoiceProvider: runtimeMocks.buildVoiceProvider, +})); + +function createDeferred() { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, reject, resolve }; +} + +async function loadLazyProviders() { + return await import("./lazy-capability-providers.js"); +} + +function createVoiceRequest( + overrides: Partial = {}, +): RealtimeVoiceBridgeCreateRequest { + return { + providerConfig: {}, + onAudio() {}, + onClearAudio() {}, + onError() {}, + ...overrides, + }; +} + +beforeEach(() => { + vi.resetModules(); + for (const value of Object.values(runtimeMocks)) { + value.mockReset(); + } + + runtimeMocks.generateImage.mockResolvedValue({ images: [] }); + runtimeMocks.transcribeAudio.mockResolvedValue({ text: "transcript" }); + runtimeMocks.generateVideo.mockResolvedValue({ videos: [] }); + runtimeMocks.listVoices.mockResolvedValue([]); + runtimeMocks.synthesize.mockResolvedValue({ audioBuffer: Buffer.alloc(0) }); + runtimeMocks.streamSynthesize.mockResolvedValue({ audioStream: {} }); + runtimeMocks.synthesizeTelephony.mockResolvedValue({ audioBuffer: Buffer.alloc(0) }); + runtimeMocks.transcriptionConnect.mockResolvedValue(undefined); + runtimeMocks.transcriptionIsConnected.mockReturnValue(false); + runtimeMocks.voiceConnect.mockResolvedValue(undefined); + runtimeMocks.voiceIsConnected.mockReturnValue(false); + + runtimeMocks.createTranscriptionSession.mockReturnValue({ + connect: runtimeMocks.transcriptionConnect, + sendAudio: runtimeMocks.transcriptionSendAudio, + close: runtimeMocks.transcriptionClose, + isConnected: runtimeMocks.transcriptionIsConnected, + }); + runtimeMocks.createVoiceBridge.mockImplementation( + () => + ({ + supportsToolResultContinuation: false, + connect: runtimeMocks.voiceConnect, + sendAudio: runtimeMocks.voiceSendAudio, + setMediaTimestamp: runtimeMocks.voiceSetMediaTimestamp, + sendUserMessage: runtimeMocks.voiceSendUserMessage, + triggerGreeting: runtimeMocks.voiceTriggerGreeting, + handleBargeIn: runtimeMocks.voiceHandleBargeIn, + submitToolResult: runtimeMocks.voiceSubmitToolResult, + acknowledgeMark: runtimeMocks.voiceAcknowledgeMark, + close: runtimeMocks.voiceClose, + isConnected: runtimeMocks.voiceIsConnected, + }) satisfies RealtimeVoiceBridge, + ); + + runtimeMocks.buildImageProvider.mockReturnValue({ + generateImage: runtimeMocks.generateImage, + }); + runtimeMocks.buildMediaProvider.mockReturnValue({ + transcribeAudio: runtimeMocks.transcribeAudio, + }); + runtimeMocks.buildVideoProvider.mockReturnValue({ + generateVideo: runtimeMocks.generateVideo, + }); + runtimeMocks.buildSpeechProvider.mockReturnValue({ + listVoices: runtimeMocks.listVoices, + synthesize: runtimeMocks.synthesize, + streamSynthesize: runtimeMocks.streamSynthesize, + synthesizeTelephony: runtimeMocks.synthesizeTelephony, + }); + runtimeMocks.buildTranscriptionProvider.mockReturnValue({ + createSession: runtimeMocks.createTranscriptionSession, + }); + runtimeMocks.buildVoiceProvider.mockReturnValue({ + createBridge: runtimeMocks.createVoiceBridge, + }); +}); + +describe("xAI lazy capability providers", () => { + it("keeps heavy builders unloaded until their capability methods run", async () => { + const lazy = await loadLazyProviders(); + const image = lazy.createLazyXaiImageGenerationProvider(); + const media = lazy.createLazyXaiMediaUnderstandingProvider(); + const video = lazy.createLazyXaiVideoGenerationProvider(); + const speech = lazy.createLazyXaiSpeechProvider(); + const transcription = lazy.createLazyXaiRealtimeTranscriptionProvider(); + const voice = lazy.createLazyXaiRealtimeVoiceProvider(); + + expect( + [ + runtimeMocks.buildImageProvider, + runtimeMocks.buildMediaProvider, + runtimeMocks.buildVideoProvider, + runtimeMocks.buildSpeechProvider, + runtimeMocks.buildTranscriptionProvider, + runtimeMocks.buildVoiceProvider, + ].map((mock) => mock.mock.calls.length), + ).toEqual([0, 0, 0, 0, 0, 0]); + expect(transcription.label).toBe("xAI Realtime Transcription"); + expect(voice.label).toBe("xAI Grok Voice"); + + await image.generateImage({} as never); + await media.transcribeAudio?.({} as never); + await video.generateVideo({} as never); + await speech.synthesize({} as never); + await speech.listVoices?.({} as never); + + expect(runtimeMocks.buildImageProvider).toHaveBeenCalledOnce(); + expect(runtimeMocks.buildMediaProvider).toHaveBeenCalledOnce(); + expect(runtimeMocks.buildVideoProvider).toHaveBeenCalledOnce(); + expect(runtimeMocks.buildSpeechProvider).toHaveBeenCalledOnce(); + expect(runtimeMocks.generateImage).toHaveBeenCalledOnce(); + expect(runtimeMocks.transcribeAudio).toHaveBeenCalledOnce(); + expect(runtimeMocks.generateVideo).toHaveBeenCalledOnce(); + expect(runtimeMocks.synthesize).toHaveBeenCalledOnce(); + expect(runtimeMocks.listVoices).toHaveBeenCalledOnce(); + }); + + it("keeps the newest transcription audio ordered while the runtime loads", async () => { + const lazy = await loadLazyProviders(); + const session = lazy.createLazyXaiRealtimeTranscriptionProvider().createSession({ + providerConfig: {}, + }); + const first = Buffer.alloc(1024 * 1024, 0x01); + const second = Buffer.alloc(1024 * 1024, 0x02); + const third = Buffer.alloc(1024 * 1024, 0x03); + + session.sendAudio(first); + session.sendAudio(second); + session.sendAudio(third); + await session.connect(); + + expect(runtimeMocks.buildTranscriptionProvider).toHaveBeenCalledOnce(); + expect(runtimeMocks.transcriptionSendAudio.mock.calls.map(([audio]) => audio)).toEqual([ + second, + third, + ]); + expect(runtimeMocks.transcriptionConnect).toHaveBeenCalledOnce(); + expect(runtimeMocks.transcriptionSendAudio.mock.invocationCallOrder.at(-1)).toBeLessThan( + runtimeMocks.transcriptionConnect.mock.invocationCallOrder[0]!, + ); + }); + + it("closes a transcription session that finishes loading after the wrapper closes", async () => { + const lazy = await loadLazyProviders(); + const session = lazy.createLazyXaiRealtimeTranscriptionProvider().createSession({ + providerConfig: {}, + }); + + const connectPromise = session.connect(); + session.close(); + session.close(); + await connectPromise; + + expect(runtimeMocks.createTranscriptionSession).toHaveBeenCalledOnce(); + expect(runtimeMocks.transcriptionConnect).not.toHaveBeenCalled(); + expect(runtimeMocks.transcriptionClose).toHaveBeenCalledOnce(); + }); + + it("reopens transcription after close without replaying discarded audio", async () => { + const lazy = await loadLazyProviders(); + const session = lazy.createLazyXaiRealtimeTranscriptionProvider().createSession({ + providerConfig: {}, + }); + const first = Buffer.from([0x01]); + const discarded = Buffer.from([0x02]); + const second = Buffer.from([0x03]); + + session.sendAudio(first); + await session.connect(); + session.close(); + session.close(); + session.sendAudio(discarded); + + const reconnectPromise = session.connect(); + session.sendAudio(second); + await reconnectPromise; + + expect(runtimeMocks.transcriptionConnect).toHaveBeenCalledTimes(2); + expect(runtimeMocks.transcriptionClose).toHaveBeenCalledOnce(); + expect(runtimeMocks.transcriptionSendAudio.mock.calls.map(([audio]) => audio)).toEqual([ + first, + second, + ]); + }); + + it("preserves voice startup ordering and waits to trigger the greeting", async () => { + const connecting = createDeferred(); + const forwarded: string[] = []; + runtimeMocks.voiceConnect.mockReturnValue(connecting.promise); + runtimeMocks.voiceSendAudio.mockImplementation((audio: Buffer) => { + forwarded.push(`audio:${audio[0]}`); + }); + runtimeMocks.voiceSetMediaTimestamp.mockImplementation((timestamp: number) => { + forwarded.push(`timestamp:${timestamp}`); + }); + runtimeMocks.voiceSendUserMessage.mockImplementation((text: string) => { + forwarded.push(`user:${text}`); + }); + runtimeMocks.voiceSubmitToolResult.mockImplementation((callId: string) => { + forwarded.push(`tool:${callId}`); + }); + runtimeMocks.voiceTriggerGreeting.mockImplementation((instructions?: string) => { + forwarded.push(`greeting:${instructions ?? ""}`); + }); + const lazy = await loadLazyProviders(); + const bridge = lazy.createLazyXaiRealtimeVoiceProvider().createBridge(createVoiceRequest()); + const first = Buffer.from([0x01]); + const second = Buffer.from([0x02]); + + bridge.sendAudio(first); + bridge.setMediaTimestamp(42); + bridge.sendUserMessage?.("hello"); + await bridge.submitToolResult("call-1", { ok: true }); + bridge.triggerGreeting?.("welcome"); + const connectPromise = bridge.connect(); + await vi.waitFor(() => expect(runtimeMocks.voiceConnect).toHaveBeenCalledOnce()); + bridge.sendAudio(second); + + expect(runtimeMocks.voiceSetMediaTimestamp).not.toHaveBeenCalled(); + expect(runtimeMocks.voiceSendAudio).not.toHaveBeenCalled(); + expect(runtimeMocks.voiceSendUserMessage).not.toHaveBeenCalled(); + expect(runtimeMocks.voiceSubmitToolResult).not.toHaveBeenCalled(); + expect(runtimeMocks.voiceTriggerGreeting).not.toHaveBeenCalled(); + + connecting.resolve(); + await connectPromise; + expect(runtimeMocks.voiceSetMediaTimestamp).toHaveBeenCalledWith(42); + expect(runtimeMocks.voiceSendAudio.mock.calls.map(([audio]) => audio)).toEqual([first, second]); + expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledWith("hello"); + expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledWith( + "call-1", + { ok: true }, + undefined, + ); + expect(runtimeMocks.voiceTriggerGreeting).toHaveBeenCalledWith("welcome"); + expect(forwarded).toEqual([ + "audio:1", + "timestamp:42", + "user:hello", + "tool:call-1", + "greeting:welcome", + "audio:2", + ]); + }); + + it("moves the latest pending timestamp and greeting to the operation tail", async () => { + const forwarded: string[] = []; + runtimeMocks.voiceSetMediaTimestamp.mockImplementation((timestamp: number) => { + forwarded.push(`timestamp:${timestamp}`); + }); + runtimeMocks.voiceSendUserMessage.mockImplementation((text: string) => { + forwarded.push(`user:${text}`); + }); + runtimeMocks.voiceSendAudio.mockImplementation((audio: Buffer) => { + forwarded.push(`audio:${audio[0]}`); + }); + runtimeMocks.voiceTriggerGreeting.mockImplementation((instructions?: string) => { + forwarded.push(`greeting:${String(instructions)}`); + }); + const lazy = await loadLazyProviders(); + const bridge = lazy.createLazyXaiRealtimeVoiceProvider().createBridge(createVoiceRequest()); + + bridge.setMediaTimestamp(1); + bridge.sendUserMessage?.("middle"); + bridge.setMediaTimestamp(2); + bridge.triggerGreeting?.("superseded"); + bridge.sendAudio(Buffer.from([0x03])); + bridge.triggerGreeting?.(); + await bridge.connect(); + + expect(forwarded).toEqual(["user:middle", "timestamp:2", "audio:3", "greeting:undefined"]); + + runtimeMocks.voiceTriggerGreeting.mockClear(); + runtimeMocks.voiceIsConnected.mockReturnValue(false); + bridge.triggerGreeting?.("provider-owned-reconnect"); + expect(runtimeMocks.voiceTriggerGreeting).toHaveBeenCalledWith("provider-owned-reconnect"); + }); + + it("bounds pending voice user messages by aggregate bytes", async () => { + const lazy = await loadLazyProviders(); + const onError = vi.fn(); + const bridge = lazy + .createLazyXaiRealtimeVoiceProvider() + .createBridge(createVoiceRequest({ onError })); + const accepted = "a".repeat(200 * 1024); + + bridge.sendUserMessage?.(accepted); + bridge.sendUserMessage?.("b".repeat(64 * 1024)); + await bridge.connect(); + + expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledOnce(); + expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledWith(accepted); + expect(onError).toHaveBeenCalledOnce(); + expect(onError.mock.calls[0]?.[0]).toEqual( + new Error("xAI realtime voice pending user message overflow during lazy startup"), + ); + }); + + it("bounds pending voice tool results by aggregate serialized bytes", async () => { + const lazy = await loadLazyProviders(); + const onError = vi.fn(); + const bridge = lazy + .createLazyXaiRealtimeVoiceProvider() + .createBridge(createVoiceRequest({ onError })); + const accepted = { text: "a".repeat(200 * 1024) }; + + await bridge.submitToolResult("call-1", accepted); + await bridge.submitToolResult("call-2", { text: "b".repeat(64 * 1024) }); + await bridge.connect(); + + expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledOnce(); + expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledWith("call-1", accepted, undefined); + expect(onError).toHaveBeenCalledOnce(); + expect(onError.mock.calls[0]?.[0]).toEqual( + new Error("xAI realtime voice pending tool result overflow during lazy startup"), + ); + }); + + it("keeps voice payloads byte-bounded until the underlying connect resolves", async () => { + const connecting = createDeferred(); + runtimeMocks.voiceConnect.mockReturnValue(connecting.promise); + const lazy = await loadLazyProviders(); + const onError = vi.fn(); + const bridge = lazy + .createLazyXaiRealtimeVoiceProvider() + .createBridge(createVoiceRequest({ onError })); + const acceptedMessage = "a".repeat(200 * 1024); + const acceptedResult = { text: "b".repeat(200 * 1024) }; + + const connectPromise = bridge.connect(); + await vi.waitFor(() => expect(runtimeMocks.voiceConnect).toHaveBeenCalledOnce()); + bridge.sendUserMessage?.(acceptedMessage); + bridge.sendUserMessage?.("c".repeat(64 * 1024)); + await bridge.submitToolResult("call-1", acceptedResult); + await bridge.submitToolResult("call-2", { text: "d".repeat(64 * 1024) }); + + expect(runtimeMocks.voiceSendUserMessage).not.toHaveBeenCalled(); + expect(runtimeMocks.voiceSubmitToolResult).not.toHaveBeenCalled(); + expect(onError.mock.calls.map(([error]) => (error as Error).message)).toEqual([ + "xAI realtime voice pending user message overflow during lazy startup", + "xAI realtime voice pending tool result overflow during lazy startup", + ]); + + connecting.resolve(); + await connectPromise; + + expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledOnce(); + expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledWith(acceptedMessage); + expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledOnce(); + expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledWith( + "call-1", + acceptedResult, + undefined, + ); + }); + + it("drains voice input queued while an earlier tool result is submitting", async () => { + const submitting = createDeferred(); + const forwarded: string[] = []; + runtimeMocks.voiceSubmitToolResult + .mockImplementationOnce((callId: string) => { + forwarded.push(`tool:${callId}`); + return submitting.promise; + }) + .mockImplementation((callId: string) => { + forwarded.push(`tool:${callId}`); + }); + runtimeMocks.voiceSendUserMessage.mockImplementation((text: string) => { + forwarded.push(`user:${text}`); + }); + runtimeMocks.voiceSetMediaTimestamp.mockImplementation((timestamp: number) => { + forwarded.push(`timestamp:${timestamp}`); + }); + const lazy = await loadLazyProviders(); + const bridge = lazy.createLazyXaiRealtimeVoiceProvider().createBridge(createVoiceRequest()); + + await bridge.submitToolResult("call-1", { text: "first" }); + const connectPromise = bridge.connect(); + await vi.waitFor(() => expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledOnce()); + bridge.sendUserMessage?.("arrived-during-flush"); + bridge.setMediaTimestamp(84); + await bridge.submitToolResult("call-2", { text: "second" }); + submitting.resolve(); + await connectPromise; + + expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledWith("arrived-during-flush"); + expect(runtimeMocks.voiceSetMediaTimestamp).toHaveBeenLastCalledWith(84); + expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledTimes(2); + expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenLastCalledWith( + "call-2", + { text: "second" }, + undefined, + ); + expect(forwarded).toEqual([ + "tool:call-1", + "user:arrived-during-flush", + "timestamp:84", + "tool:call-2", + ]); + }); + + it("keeps an in-flight voice tool result charged against the startup byte cap", async () => { + const submitting = createDeferred(); + runtimeMocks.voiceSubmitToolResult.mockReturnValueOnce(submitting.promise); + const lazy = await loadLazyProviders(); + const onError = vi.fn(); + const bridge = lazy + .createLazyXaiRealtimeVoiceProvider() + .createBridge(createVoiceRequest({ onError })); + + await bridge.submitToolResult("call-1", { text: "a".repeat(200 * 1024) }); + const connectPromise = bridge.connect(); + await vi.waitFor(() => expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledOnce()); + await bridge.submitToolResult("call-2", { text: "b".repeat(64 * 1024) }); + + expect(onError.mock.calls[0]?.[0]).toEqual( + new Error("xAI realtime voice pending tool result overflow during lazy startup"), + ); + expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledOnce(); + + submitting.resolve(); + await connectPromise; + expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledOnce(); + }); + + it("forwards all voice input admitted during the final connect handoff exactly once", async () => { + const connecting = createDeferred(); + runtimeMocks.voiceConnect.mockReturnValue(connecting.promise); + runtimeMocks.voiceIsConnected.mockReturnValue(true); + const lazy = await loadLazyProviders(); + const bridge = lazy.createLazyXaiRealtimeVoiceProvider().createBridge(createVoiceRequest()); + const audio = Buffer.from([0x01]); + + const firstConnect = bridge.connect(); + const secondConnect = bridge.connect(); + await vi.waitFor(() => expect(runtimeMocks.voiceConnect).toHaveBeenCalledOnce()); + connecting.resolve(); + queueMicrotask(() => { + bridge.sendAudio(audio); + bridge.setMediaTimestamp(84); + bridge.sendUserMessage?.("arrived-during-handoff"); + void bridge.submitToolResult("call-1", { text: "tool-result" }); + bridge.triggerGreeting?.("welcome"); + }); + await Promise.all([firstConnect, secondConnect]); + + expect(runtimeMocks.voiceSendAudio).toHaveBeenCalledOnce(); + expect(runtimeMocks.voiceSendAudio).toHaveBeenCalledWith(audio); + expect(runtimeMocks.voiceSetMediaTimestamp).toHaveBeenCalledOnce(); + expect(runtimeMocks.voiceSetMediaTimestamp).toHaveBeenCalledWith(84); + expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledOnce(); + expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledWith("arrived-during-handoff"); + expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledOnce(); + expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledWith( + "call-1", + { text: "tool-result" }, + undefined, + ); + expect(runtimeMocks.voiceTriggerGreeting).toHaveBeenCalledOnce(); + expect(runtimeMocks.voiceTriggerGreeting).toHaveBeenCalledWith("welcome"); + }); + + it("clears pending voice byte budgets when closed before connect", async () => { + const lazy = await loadLazyProviders(); + const onError = vi.fn(); + const bridge = lazy + .createLazyXaiRealtimeVoiceProvider() + .createBridge(createVoiceRequest({ onError })); + + bridge.sendUserMessage?.("stale".repeat(40 * 1024)); + bridge.setMediaTimestamp(42); + await bridge.submitToolResult("stale-call", { text: "x".repeat(200 * 1024) }); + bridge.close(); + + const connectPromise = bridge.connect(); + bridge.sendUserMessage?.("fresh".repeat(40 * 1024)); + await bridge.submitToolResult("fresh-call", { text: "y".repeat(200 * 1024) }); + await connectPromise; + + expect(onError).not.toHaveBeenCalled(); + expect(runtimeMocks.voiceSetMediaTimestamp).not.toHaveBeenCalled(); + expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledOnce(); + expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledWith("fresh".repeat(40 * 1024)); + expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledOnce(); + expect(runtimeMocks.voiceSubmitToolResult).toHaveBeenCalledWith( + "fresh-call", + { text: "y".repeat(200 * 1024) }, + undefined, + ); + }); + + it("closes a voice bridge that finishes loading after the wrapper closes", async () => { + const lazy = await loadLazyProviders(); + const onClose = vi.fn(); + const bridge = lazy + .createLazyXaiRealtimeVoiceProvider() + .createBridge(createVoiceRequest({ onClose })); + + const connectPromise = bridge.connect(); + bridge.close(); + bridge.close(); + await connectPromise; + + expect(runtimeMocks.createVoiceBridge).toHaveBeenCalledOnce(); + expect(runtimeMocks.voiceConnect).not.toHaveBeenCalled(); + expect(runtimeMocks.voiceClose).toHaveBeenCalledOnce(); + expect(onClose).toHaveBeenCalledOnce(); + expect(onClose).toHaveBeenCalledWith("completed"); + }); + + it("reopens voice after close without replaying discarded input", async () => { + const lazy = await loadLazyProviders(); + const onClose = vi.fn(); + const bridge = lazy + .createLazyXaiRealtimeVoiceProvider() + .createBridge(createVoiceRequest({ onClose })); + const first = Buffer.from([0x01]); + const discarded = Buffer.from([0x02]); + const second = Buffer.from([0x03]); + + bridge.sendAudio(first); + await bridge.connect(); + bridge.close(); + bridge.close(); + bridge.sendAudio(discarded); + + const reconnectPromise = bridge.connect(); + bridge.sendAudio(second); + await reconnectPromise; + + expect(runtimeMocks.voiceConnect).toHaveBeenCalledTimes(2); + expect(runtimeMocks.voiceClose).toHaveBeenCalledOnce(); + expect(runtimeMocks.voiceSendAudio.mock.calls.map(([audio]) => audio)).toEqual([first, second]); + expect(onClose).toHaveBeenCalledOnce(); + expect(onClose).toHaveBeenCalledWith("completed"); + }); + + it("keeps a replacement voice generation open after closing a pending connect", async () => { + const firstConnect = createDeferred(); + runtimeMocks.voiceConnect + .mockReturnValueOnce(firstConnect.promise) + .mockResolvedValueOnce(undefined); + const lazy = await loadLazyProviders(); + const onClose = vi.fn(); + const bridge = lazy + .createLazyXaiRealtimeVoiceProvider() + .createBridge(createVoiceRequest({ onClose })); + + const staleConnect = bridge.connect(); + await vi.waitFor(() => expect(runtimeMocks.voiceConnect).toHaveBeenCalledOnce()); + const staleRequest = runtimeMocks.createVoiceBridge.mock.calls[0]?.[0] as + | RealtimeVoiceBridgeCreateRequest + | undefined; + bridge.close(); + const replacementConnect = bridge.connect(); + await replacementConnect; + staleRequest?.onClose?.("error"); + bridge.sendUserMessage?.("replacement-still-open"); + firstConnect.resolve(); + await staleConnect; + + expect(runtimeMocks.voiceConnect).toHaveBeenCalledTimes(2); + expect(runtimeMocks.createVoiceBridge).toHaveBeenCalledTimes(2); + expect(runtimeMocks.voiceClose).toHaveBeenCalledOnce(); + expect(runtimeMocks.voiceSendUserMessage).toHaveBeenCalledWith("replacement-still-open"); + expect(onClose).toHaveBeenCalledOnce(); + }); + + it("ignores nonterminal callbacks from a superseded voice generation", async () => { + const onAudio = vi.fn(); + const onClearAudio = vi.fn(); + const onMark = vi.fn(); + const onTranscript = vi.fn(); + const onEvent = vi.fn(); + const onToolCall = vi.fn(); + const onReady = vi.fn(); + const onError = vi.fn(); + const lazy = await loadLazyProviders(); + const bridge = lazy.createLazyXaiRealtimeVoiceProvider().createBridge( + createVoiceRequest({ + onAudio, + onClearAudio, + onMark, + onTranscript, + onEvent, + onToolCall, + onReady, + onError, + }), + ); + + await bridge.connect(); + const staleRequest = runtimeMocks.createVoiceBridge.mock.calls[0]?.[0] as + | RealtimeVoiceBridgeCreateRequest + | undefined; + bridge.close(); + await bridge.connect(); + const currentRequest = runtimeMocks.createVoiceBridge.mock.calls[1]?.[0] as + | RealtimeVoiceBridgeCreateRequest + | undefined; + const staleAudio = Buffer.from([0x01]); + const staleError = new Error("stale"); + const staleEvent = { direction: "server" as const, type: "stale" }; + const staleToolCall = { + itemId: "stale-item", + callId: "stale-call", + name: "stale-tool", + args: {}, + }; + + staleRequest?.onAudio(staleAudio); + staleRequest?.onClearAudio("barge-in"); + staleRequest?.onMark?.("stale-mark"); + staleRequest?.onTranscript?.("assistant", "stale", true); + staleRequest?.onEvent?.(staleEvent); + staleRequest?.onToolCall?.(staleToolCall); + staleRequest?.onReady?.(); + staleRequest?.onError?.(staleError); + + expect(onAudio).not.toHaveBeenCalled(); + expect(onClearAudio).not.toHaveBeenCalled(); + expect(onMark).not.toHaveBeenCalled(); + expect(onTranscript).not.toHaveBeenCalled(); + expect(onEvent).not.toHaveBeenCalled(); + expect(onToolCall).not.toHaveBeenCalled(); + expect(onReady).not.toHaveBeenCalled(); + expect(onError).not.toHaveBeenCalled(); + + const currentAudio = Buffer.from([0x02]); + const currentError = new Error("current"); + const currentEvent = { direction: "server" as const, type: "current" }; + const currentToolCall = { + itemId: "current-item", + callId: "current-call", + name: "current-tool", + args: {}, + }; + currentRequest?.onAudio(currentAudio); + currentRequest?.onClearAudio("barge-in"); + currentRequest?.onMark?.("current-mark"); + currentRequest?.onTranscript?.("assistant", "current", true); + currentRequest?.onEvent?.(currentEvent); + currentRequest?.onToolCall?.(currentToolCall); + currentRequest?.onReady?.(); + currentRequest?.onError?.(currentError); + + expect(onAudio).toHaveBeenCalledWith(currentAudio); + expect(onClearAudio).toHaveBeenCalledWith("barge-in"); + expect(onMark).toHaveBeenCalledWith("current-mark"); + expect(onTranscript).toHaveBeenCalledWith("assistant", "current", true); + expect(onEvent).toHaveBeenCalledWith(currentEvent); + expect(onToolCall).toHaveBeenCalledWith(currentToolCall); + expect(onReady).toHaveBeenCalledOnce(); + expect(onError).toHaveBeenCalledWith(currentError); + }); + + it("reports queued voice flush failure as a terminal error", async () => { + const failure = new Error("tool result rejected"); + runtimeMocks.voiceSubmitToolResult.mockRejectedValueOnce(failure); + const lazy = await loadLazyProviders(); + const onClose = vi.fn(); + const bridge = lazy + .createLazyXaiRealtimeVoiceProvider() + .createBridge(createVoiceRequest({ onClose })); + + await bridge.submitToolResult("call-1", { text: "queued" }); + await expect(bridge.connect()).rejects.toThrow(failure); + const loadedRequest = runtimeMocks.createVoiceBridge.mock.calls[0]?.[0] as + | RealtimeVoiceBridgeCreateRequest + | undefined; + loadedRequest?.onClose?.("completed"); + bridge.sendAudio(Buffer.from([0x01])); + + expect(runtimeMocks.voiceClose).toHaveBeenCalledOnce(); + expect(runtimeMocks.voiceSendAudio).not.toHaveBeenCalled(); + expect(onClose).toHaveBeenCalledOnce(); + expect(onClose).toHaveBeenCalledWith("error"); + }); + + it("reopens voice only after an explicit connect following provider termination", async () => { + const lazy = await loadLazyProviders(); + const onClose = vi.fn(); + const bridge = lazy + .createLazyXaiRealtimeVoiceProvider() + .createBridge(createVoiceRequest({ onClose })); + const discarded = Buffer.from([0x01]); + const accepted = Buffer.from([0x02]); + + await bridge.connect(); + const loadedRequest = runtimeMocks.createVoiceBridge.mock.calls[0]?.[0] as + | RealtimeVoiceBridgeCreateRequest + | undefined; + loadedRequest?.onClose?.("error"); + bridge.sendAudio(discarded); + + const reconnectPromise = bridge.connect(); + bridge.sendAudio(accepted); + await reconnectPromise; + + expect(runtimeMocks.voiceConnect).toHaveBeenCalledTimes(2); + expect(runtimeMocks.voiceSendAudio).toHaveBeenCalledOnce(); + expect(runtimeMocks.voiceSendAudio).toHaveBeenCalledWith(accepted); + expect(onClose).toHaveBeenCalledOnce(); + expect(onClose).toHaveBeenCalledWith("error"); + }); + + it("reports explicit voice close once when the provider also reports completion", async () => { + const lazy = await loadLazyProviders(); + const onClose = vi.fn(); + const bridge = lazy + .createLazyXaiRealtimeVoiceProvider() + .createBridge(createVoiceRequest({ onClose })); + + await bridge.connect(); + const loadedRequest = runtimeMocks.createVoiceBridge.mock.calls[0]?.[0] as + | RealtimeVoiceBridgeCreateRequest + | undefined; + runtimeMocks.voiceClose.mockImplementation(() => loadedRequest?.onClose?.("completed")); + bridge.close(); + bridge.close(); + + expect(runtimeMocks.voiceClose).toHaveBeenCalledOnce(); + expect(onClose).toHaveBeenCalledOnce(); + expect(onClose).toHaveBeenCalledWith("completed"); + }); + + it("keeps realtime voice request validation synchronous", async () => { + const lazy = await loadLazyProviders(); + const provider = lazy.createLazyXaiRealtimeVoiceProvider(); + + expect(() => provider.createBridge(createVoiceRequest({ autoRespondToAudio: false }))).toThrow( + "xAI realtime voice requires automatic server-VAD responses", + ); + expect(runtimeMocks.buildVoiceProvider).not.toHaveBeenCalled(); + }); +}); diff --git a/extensions/xai/lazy-capability-providers.ts b/extensions/xai/lazy-capability-providers.ts new file mode 100644 index 000000000000..29eb5ba00ffa --- /dev/null +++ b/extensions/xai/lazy-capability-providers.ts @@ -0,0 +1,667 @@ +import type { ImageGenerationProvider } from "openclaw/plugin-sdk/image-generation"; +import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime"; +import type { MediaUnderstandingProvider } from "openclaw/plugin-sdk/media-understanding"; +import type { + RealtimeTranscriptionProviderPlugin, + RealtimeTranscriptionSession, + RealtimeTranscriptionSessionCreateRequest, +} from "openclaw/plugin-sdk/realtime-transcription"; +import type { + RealtimeVoiceBridge, + RealtimeVoiceBridgeCreateRequest, + RealtimeVoiceProviderPlugin, + RealtimeVoiceToolResultOptions, +} from "openclaw/plugin-sdk/realtime-voice"; +import { createRealtimeVoiceAudioQueue } from "openclaw/plugin-sdk/realtime-voice-audio-queue"; +import type { + SpeechProviderPlugin, + SpeechSynthesisStreamRequest, + SpeechTelephonySynthesisRequest, +} from "openclaw/plugin-sdk/speech"; +import type { VideoGenerationProvider } from "openclaw/plugin-sdk/video-generation"; +import { + assertXaiRealtimeVoiceRequestSupported, + createXaiImageGenerationProviderMetadata, + createXaiMediaUnderstandingProviderMetadata, + createXaiRealtimeTranscriptionProviderMetadata, + createXaiRealtimeVoiceProviderMetadata, + createXaiVideoGenerationProviderMetadata, + normalizeXaiRealtimeTranscriptionProviderConfig, +} from "./capability-provider-metadata.js"; +import { createXaiSpeechProviderMetadata } from "./speech-provider-metadata.js"; + +const MAX_LAZY_REALTIME_TRANSCRIPTION_AUDIO_BYTES = 2 * 1024 * 1024; +const MAX_LAZY_REALTIME_VOICE_USER_MESSAGES = 128; +const MAX_LAZY_REALTIME_VOICE_USER_MESSAGE_BYTES = 256 * 1024; +const MAX_LAZY_REALTIME_VOICE_TOOL_RESULTS = 128; +const MAX_LAZY_REALTIME_VOICE_TOOL_RESULT_BYTES = 256 * 1024; + +function serializedJsonBytes(value: unknown): number | undefined { + try { + const serialized = JSON.stringify(value); + return typeof serialized === "string" ? Buffer.byteLength(serialized, "utf8") : undefined; + } catch { + return undefined; + } +} + +const loadXaiImageGenerationProvider = createLazyRuntimeModule(async () => + (await import("./image-generation-provider.js")).buildXaiImageGenerationProvider(), +); +const loadXaiMediaUnderstandingProvider = createLazyRuntimeModule(async () => + (await import("./stt.js")).buildXaiMediaUnderstandingProvider(), +); +const loadXaiRealtimeTranscriptionProvider = createLazyRuntimeModule(async () => + (await import("./realtime-transcription-provider.js")).buildXaiRealtimeTranscriptionProvider(), +); +const loadXaiRealtimeVoiceProvider = createLazyRuntimeModule(async () => + (await import("./realtime-voice-provider.js")).buildXaiRealtimeVoiceProvider(), +); +const loadXaiSpeechProvider = createLazyRuntimeModule(async () => + (await import("./speech-provider.js")).buildXaiSpeechProvider(), +); +const loadXaiVideoGenerationProvider = createLazyRuntimeModule(async () => + (await import("./video-generation-provider.js")).buildXaiVideoGenerationProvider(), +); + +function createPendingTranscriptionAudioQueue(): { + clear: () => void; + drain: () => Buffer[]; + enqueue: (audio: Buffer) => void; +} { + let chunks: Array = []; + let head = 0; + let bytes = 0; + const clear = () => { + chunks = []; + head = 0; + bytes = 0; + }; + return { + clear, + drain: () => { + const pending = chunks.slice(head).filter((chunk): chunk is Buffer => chunk !== undefined); + clear(); + return pending; + }, + enqueue: (audio) => { + if (audio.byteLength > MAX_LAZY_REALTIME_TRANSCRIPTION_AUDIO_BYTES) { + return; + } + const chunk = Buffer.from(audio); + chunks.push(chunk); + bytes += chunk.byteLength; + while (bytes > MAX_LAZY_REALTIME_TRANSCRIPTION_AUDIO_BYTES && head < chunks.length) { + const dropped = chunks[head]; + chunks[head] = undefined; + head += 1; + bytes -= dropped?.byteLength ?? 0; + } + if (head > 256 && head * 2 >= chunks.length) { + chunks = chunks.slice(head); + head = 0; + } + }, + }; +} + +function createLazyXaiRealtimeTranscriptionSession( + req: RealtimeTranscriptionSessionCreateRequest, +): RealtimeTranscriptionSession { + let session: RealtimeTranscriptionSession | undefined; + let sessionPromise: Promise | undefined; + let activeConnect: + | { + generation: number; + promise: Promise; + } + | undefined; + let generation = 0; + let closedSessionGeneration: number | undefined; + let closed = false; + let acceptsInput = false; + const pendingAudio = createPendingTranscriptionAudioQueue(); + + const closeSession = ( + closeGeneration: number, + loadedSession: RealtimeTranscriptionSession | undefined = session, + ) => { + if (!loadedSession || closedSessionGeneration === closeGeneration) { + return; + } + closedSessionGeneration = closeGeneration; + loadedSession.close(); + }; + const loadSession = async () => { + if (!sessionPromise) { + sessionPromise = loadXaiRealtimeTranscriptionProvider().then((provider) => + provider.createSession(req), + ); + } + session = await sessionPromise; + return session; + }; + const beginConnectGeneration = () => { + if (closed) { + generation += 1; + closed = false; + } + return generation; + }; + + return { + connect: async () => { + const connectGeneration = beginConnectGeneration(); + if (activeConnect?.generation === connectGeneration) { + await activeConnect.promise; + return; + } + const promise = (async () => { + const loadedSession = await loadSession(); + if (connectGeneration !== generation || closed) { + if (connectGeneration === generation && closed) { + closeSession(connectGeneration, loadedSession); + } + return; + } + for (const audio of pendingAudio.drain()) { + loadedSession.sendAudio(audio); + } + acceptsInput = true; + await loadedSession.connect(); + if (connectGeneration === generation && closed) { + closeSession(connectGeneration, loadedSession); + } + })(); + const connectTask = { generation: connectGeneration, promise }; + activeConnect = connectTask; + try { + await promise; + } finally { + if (activeConnect === connectTask) { + activeConnect = undefined; + } + } + }, + sendAudio: (audio) => { + if (closed) { + return; + } + if (acceptsInput && session) { + session.sendAudio(audio); + return; + } + pendingAudio.enqueue(audio); + }, + close: () => { + if (closed) { + return; + } + closed = true; + acceptsInput = false; + pendingAudio.clear(); + closeSession(generation); + }, + isConnected: () => !closed && (session?.isConnected() ?? false), + }; +} + +function createLazyXaiRealtimeVoiceBridge( + req: RealtimeVoiceBridgeCreateRequest, +): RealtimeVoiceBridge { + assertXaiRealtimeVoiceRequestSupported(req); + type PendingVoiceOperation = + | { type: "audio" } + | { timestamp: number; type: "media-timestamp" } + | { bytes: number; text: string; type: "user-message" } + | { instructions?: string; type: "greeting" } + | { + bytes: number; + callId: string; + options?: RealtimeVoiceToolResultOptions; + result: unknown; + type: "tool-result"; + }; + type PendingMediaTimestamp = Extract; + type PendingVoiceGreeting = Extract; + + let bridge: RealtimeVoiceBridge | undefined; + let bridgeState: + | { + generation: number; + promise: Promise; + } + | undefined; + let activeConnect: + | { + generation: number; + promise: Promise; + } + | undefined; + let generation = 0; + let terminalGeneration: number | undefined; + let closed = false; + let acceptsInput = false; + let pendingMediaTimestamp: PendingMediaTimestamp | undefined; + let pendingGreeting: PendingVoiceGreeting | undefined; + let pendingUserMessageCount = 0; + let pendingUserMessageBytes = 0; + let pendingToolResultCount = 0; + let pendingToolResultBytes = 0; + const closedBridges = new WeakSet(); + const pendingAudio = createRealtimeVoiceAudioQueue("reject-newest"); + const pendingOperations: PendingVoiceOperation[] = []; + + const clearPendingInput = () => { + pendingAudio.clear(); + pendingOperations.length = 0; + pendingMediaTimestamp = undefined; + pendingGreeting = undefined; + pendingUserMessageCount = 0; + pendingUserMessageBytes = 0; + pendingToolResultCount = 0; + pendingToolResultBytes = 0; + }; + const emitTerminal = ( + terminalForGeneration: number, + outcome: Parameters>[0], + ) => { + if (terminalForGeneration !== generation || terminalGeneration === terminalForGeneration) { + return; + } + terminalGeneration = terminalForGeneration; + acceptsInput = false; + clearPendingInput(); + req.onClose?.(outcome); + }; + const closeBridge = (loadedBridge: RealtimeVoiceBridge | undefined = bridge) => { + if (!loadedBridge || closedBridges.has(loadedBridge)) { + return; + } + closedBridges.add(loadedBridge); + loadedBridge.close(); + }; + const acceptsProviderCallback = (callbackGeneration: number) => + callbackGeneration === generation && !closed && terminalGeneration !== callbackGeneration; + const guardProviderCallback = ( + callbackGeneration: number, + callback: (...args: TArgs) => void, + ) => { + return (...args: TArgs) => { + if (acceptsProviderCallback(callbackGeneration)) { + callback(...args); + } + }; + }; + const loadBridge = async (loadGeneration: number) => { + const existingState = bridgeState; + const state = + existingState?.generation === loadGeneration + ? existingState + : { + generation: loadGeneration, + promise: loadXaiRealtimeVoiceProvider().then((provider) => + provider.createBridge({ + ...req, + // An explicit wrapper reconnect owns a new provider bridge. Guard every + // nonterminal callback so late events cannot reach its replacement. + onAudio: guardProviderCallback(loadGeneration, req.onAudio), + onClearAudio: guardProviderCallback(loadGeneration, req.onClearAudio), + ...(req.onMark + ? { onMark: guardProviderCallback(loadGeneration, req.onMark) } + : {}), + ...(req.onTranscript + ? { onTranscript: guardProviderCallback(loadGeneration, req.onTranscript) } + : {}), + ...(req.onEvent + ? { onEvent: guardProviderCallback(loadGeneration, req.onEvent) } + : {}), + ...(req.onToolCall + ? { onToolCall: guardProviderCallback(loadGeneration, req.onToolCall) } + : {}), + ...(req.onReady + ? { onReady: guardProviderCallback(loadGeneration, req.onReady) } + : {}), + ...(req.onError + ? { onError: guardProviderCallback(loadGeneration, req.onError) } + : {}), + onClose: (outcome) => emitTerminal(loadGeneration, outcome), + }), + ), + }; + if (state !== existingState) { + bridgeState = state; + } + const loadedBridge = await state.promise; + if (bridgeState === state && loadGeneration === generation) { + bridge = loadedBridge; + } + return loadedBridge; + }; + const replacePendingOperation = ( + previous: T | undefined, + next: T, + ): T => { + if (previous) { + const previousIndex = pendingOperations.indexOf(previous); + if (previousIndex >= 0) { + pendingOperations.splice(previousIndex, 1); + } + } + pendingOperations.push(next); + return next; + }; + const beginConnectGeneration = () => { + if (closed || terminalGeneration === generation) { + generation += 1; + closed = false; + acceptsInput = false; + bridge = undefined; + } + return generation; + }; + const acceptsCurrentInput = () => !closed && terminalGeneration !== generation; + const flushPendingInput = async ( + loadedBridge: RealtimeVoiceBridge, + connectGeneration: number, + ) => { + if (connectGeneration !== generation || !acceptsCurrentInput()) { + return; + } + while (true) { + if (connectGeneration !== generation || !acceptsCurrentInput()) { + return; + } + const operation = pendingOperations.shift(); + if (!operation) { + // Queue exhaustion and direct admission must change in the same turn. + // An await between them can strand input admitted by the next microtask. + acceptsInput = true; + return; + } + switch (operation.type) { + case "audio": { + const chunk = pendingAudio.dequeue(); + if (!chunk) { + throw new Error("xAI realtime voice pending audio queue invariant violated"); + } + loadedBridge.sendAudio(chunk); + break; + } + case "media-timestamp": + if (pendingMediaTimestamp === operation) { + pendingMediaTimestamp = undefined; + } + loadedBridge.setMediaTimestamp(operation.timestamp); + break; + case "user-message": + loadedBridge.sendUserMessage?.(operation.text); + break; + case "tool-result": + await loadedBridge.submitToolResult( + operation.callId, + operation.result, + operation.options, + ); + break; + case "greeting": + if (pendingGreeting === operation) { + pendingGreeting = undefined; + } + loadedBridge.triggerGreeting?.(operation.instructions); + break; + } + if (connectGeneration !== generation || !acceptsCurrentInput()) { + return; + } + if (operation.type === "user-message") { + pendingUserMessageCount -= 1; + pendingUserMessageBytes -= operation.bytes; + } else if (operation.type === "tool-result") { + pendingToolResultCount -= 1; + pendingToolResultBytes -= operation.bytes; + } + } + }; + + return { + get supportsToolResultContinuation() { + return bridge?.supportsToolResultContinuation ?? false; + }, + connect: async () => { + const connectGeneration = beginConnectGeneration(); + if (activeConnect?.generation === connectGeneration) { + await activeConnect.promise; + return; + } + const promise = (async () => { + const loadedBridge = await loadBridge(connectGeneration); + if (connectGeneration !== generation || !acceptsCurrentInput()) { + closeBridge(loadedBridge); + return; + } + try { + await loadedBridge.connect(); + } catch (error) { + if (connectGeneration === generation) { + acceptsInput = false; + terminalGeneration = connectGeneration; + clearPendingInput(); + } + throw error; + } + if (connectGeneration !== generation || !acceptsCurrentInput()) { + closeBridge(loadedBridge); + return; + } + try { + await flushPendingInput(loadedBridge, connectGeneration); + } catch (error) { + emitTerminal(connectGeneration, "error"); + closeBridge(loadedBridge); + throw error; + } + if (connectGeneration !== generation || !acceptsCurrentInput()) { + closeBridge(loadedBridge); + } + })(); + const connectTask = { generation: connectGeneration, promise }; + activeConnect = connectTask; + try { + await promise; + } finally { + if (activeConnect === connectTask) { + activeConnect = undefined; + } + } + }, + sendAudio: (audio) => { + if (!acceptsCurrentInput()) { + return; + } + if (acceptsInput && bridge) { + bridge.sendAudio(audio); + return; + } + if (pendingAudio.enqueue(audio)) { + pendingOperations.push({ type: "audio" }); + } + }, + setMediaTimestamp: (timestamp) => { + if (!acceptsCurrentInput()) { + return; + } + if (acceptsInput && bridge) { + bridge.setMediaTimestamp(timestamp); + return; + } + pendingMediaTimestamp = replacePendingOperation(pendingMediaTimestamp, { + timestamp, + type: "media-timestamp", + }); + }, + sendUserMessage: (text) => { + if (!acceptsCurrentInput()) { + return; + } + if (acceptsInput && bridge) { + bridge.sendUserMessage?.(text); + return; + } + const messageBytes = Buffer.byteLength(text, "utf8"); + if ( + pendingUserMessageCount >= MAX_LAZY_REALTIME_VOICE_USER_MESSAGES || + pendingUserMessageBytes + messageBytes > MAX_LAZY_REALTIME_VOICE_USER_MESSAGE_BYTES + ) { + req.onError?.( + new Error("xAI realtime voice pending user message overflow during lazy startup"), + ); + return; + } + pendingOperations.push({ + bytes: messageBytes, + text, + type: "user-message", + }); + pendingUserMessageCount += 1; + pendingUserMessageBytes += messageBytes; + }, + triggerGreeting: (instructions) => { + if (!acceptsCurrentInput()) { + return; + } + if (acceptsInput && bridge) { + bridge.triggerGreeting?.(instructions); + return; + } + pendingGreeting = replacePendingOperation(pendingGreeting, { + instructions, + type: "greeting", + }); + }, + handleBargeIn: (options) => { + if (acceptsCurrentInput()) { + bridge?.handleBargeIn?.(options); + } + }, + submitToolResult: (callId, result, options) => { + if (!acceptsCurrentInput()) { + return; + } + if (acceptsInput && bridge) { + return bridge.submitToolResult(callId, result, options); + } + const pending = { callId, result, ...(options ? { options } : {}) }; + const resultBytes = serializedJsonBytes(pending); + if ( + resultBytes === undefined || + pendingToolResultCount >= MAX_LAZY_REALTIME_VOICE_TOOL_RESULTS || + pendingToolResultBytes + resultBytes > MAX_LAZY_REALTIME_VOICE_TOOL_RESULT_BYTES + ) { + req.onError?.( + new Error("xAI realtime voice pending tool result overflow during lazy startup"), + ); + return; + } + pendingOperations.push({ + ...pending, + bytes: resultBytes, + type: "tool-result", + }); + pendingToolResultCount += 1; + pendingToolResultBytes += resultBytes; + }, + acknowledgeMark: (markName) => { + if (acceptsCurrentInput()) { + bridge?.acknowledgeMark(markName); + } + }, + close: () => { + if (closed) { + return; + } + const closeGeneration = generation; + closed = true; + acceptsInput = false; + clearPendingInput(); + closeBridge(); + // A bridge closed before its first connect has no provider-owned + // connection to report the terminal outcome. + emitTerminal(closeGeneration, "completed"); + }, + isConnected: () => acceptsCurrentInput() && (bridge?.isConnected() ?? false), + }; +} + +export function createLazyXaiImageGenerationProvider(): ImageGenerationProvider { + return { + ...createXaiImageGenerationProviderMetadata(), + generateImage: async (req) => (await loadXaiImageGenerationProvider()).generateImage(req), + }; +} + +export function createLazyXaiMediaUnderstandingProvider(): MediaUnderstandingProvider { + return { + ...createXaiMediaUnderstandingProviderMetadata(), + transcribeAudio: async (req) => { + const provider = await loadXaiMediaUnderstandingProvider(); + if (!provider.transcribeAudio) { + throw new Error("xAI media understanding provider missing transcribeAudio"); + } + return await provider.transcribeAudio(req); + }, + }; +} + +export function createLazyXaiVideoGenerationProvider(): VideoGenerationProvider { + return { + ...createXaiVideoGenerationProviderMetadata(), + generateVideo: async (req) => (await loadXaiVideoGenerationProvider()).generateVideo(req), + }; +} + +export function createLazyXaiSpeechProvider(): SpeechProviderPlugin { + return { + ...createXaiSpeechProviderMetadata(), + listVoices: async (req) => { + const provider = await loadXaiSpeechProvider(); + if (!provider.listVoices) { + throw new Error("xAI speech provider missing listVoices"); + } + return await provider.listVoices(req); + }, + synthesize: async (req) => await (await loadXaiSpeechProvider()).synthesize(req), + streamSynthesize: async (req: SpeechSynthesisStreamRequest) => { + const provider = await loadXaiSpeechProvider(); + if (!provider.streamSynthesize) { + throw new Error("xAI speech provider missing streamSynthesize"); + } + return await provider.streamSynthesize(req); + }, + synthesizeTelephony: async (req: SpeechTelephonySynthesisRequest) => { + const provider = await loadXaiSpeechProvider(); + if (!provider.synthesizeTelephony) { + throw new Error("xAI speech provider missing synthesizeTelephony"); + } + return await provider.synthesizeTelephony(req); + }, + }; +} + +export function createLazyXaiRealtimeTranscriptionProvider(): RealtimeTranscriptionProviderPlugin { + return { + ...createXaiRealtimeTranscriptionProviderMetadata(), + createSession: (req) => { + // Preserve synchronous config validation even though transport code loads on connect(). + normalizeXaiRealtimeTranscriptionProviderConfig(req.providerConfig); + return createLazyXaiRealtimeTranscriptionSession(req); + }, + }; +} + +export function createLazyXaiRealtimeVoiceProvider(): RealtimeVoiceProviderPlugin { + return { + ...createXaiRealtimeVoiceProviderMetadata(), + createBridge: createLazyXaiRealtimeVoiceBridge, + }; +} diff --git a/extensions/xai/provider-routing.ts b/extensions/xai/provider-routing.ts new file mode 100644 index 000000000000..ac4f18ea3603 --- /dev/null +++ b/extensions/xai/provider-routing.ts @@ -0,0 +1,48 @@ +import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { XAI_BASE_URL } from "./model-definitions.js"; +import { isXaiProviderId } from "./provider-id.js"; + +const XAI_NATIVE_ENDPOINT_HOSTS = new Set(["api.x.ai"]); + +function resolveHostname(value: string): string | undefined { + try { + return new URL(value).hostname.toLowerCase(); + } catch { + return undefined; + } +} + +function isXaiNativeEndpoint(baseUrl: unknown): boolean { + return ( + typeof baseUrl === "string" && XAI_NATIVE_ENDPOINT_HOSTS.has(resolveHostname(baseUrl) ?? "") + ); +} + +function shouldUseXaiResponsesTransport(params: { + provider: string; + api?: unknown; + baseUrl?: unknown; +}): boolean { + const hasDefaultXaiRoute = + isXaiProviderId(params.provider) && !normalizeOptionalString(params.baseUrl); + return params.api === "openai-responses" + ? hasDefaultXaiRoute + : params.api === "openai-completions" && + (isXaiNativeEndpoint(params.baseUrl) || hasDefaultXaiRoute); +} + +export function resolveXaiTransport(params: { + provider: string; + api?: unknown; + baseUrl?: unknown; +}): { api: "openai-responses"; baseUrl?: string } | undefined { + if (!shouldUseXaiResponsesTransport(params)) { + return undefined; + } + return { + api: "openai-responses", + baseUrl: + normalizeOptionalString(params.baseUrl) ?? + (isXaiProviderId(params.provider) ? XAI_BASE_URL : undefined), + }; +} diff --git a/extensions/xai/realtime-transcription-provider.ts b/extensions/xai/realtime-transcription-provider.ts index a2aa55c0482a..6c149908c9d4 100644 --- a/extensions/xai/realtime-transcription-provider.ts +++ b/extensions/xai/realtime-transcription-provider.ts @@ -1,38 +1,22 @@ // Xai provider module implements model/runtime integration. -import { - isProviderAuthProfileConfigured, - type OpenClawConfig, -} from "openclaw/plugin-sdk/provider-auth"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth"; import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import { createRealtimeTranscriptionWebSocketSession, - type RealtimeTranscriptionProviderConfig, type RealtimeTranscriptionProviderPlugin, type RealtimeTranscriptionSession, type RealtimeTranscriptionSessionCreateRequest, type RealtimeTranscriptionWebSocketTransport, } from "openclaw/plugin-sdk/realtime-transcription"; -import { normalizeResolvedSecretInputString } from "openclaw/plugin-sdk/secret-input"; +import { isRecord, normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; import { - normalizeOptionalString, - parseBooleanValue as readBoolean, - parseFiniteNumber as readFiniteNumber, -} from "openclaw/plugin-sdk/string-coerce-runtime"; + createXaiRealtimeTranscriptionProviderMetadata, + normalizeXaiRealtimeTranscriptionProviderConfig, + type XaiRealtimeTranscriptionEncoding, +} from "./capability-provider-metadata.js"; import { XAI_BASE_URL } from "./model-definitions.js"; import { xaiUserAgentHeaderFor } from "./src/xai-user-agent.js"; -type XaiRealtimeTranscriptionEncoding = "pcm" | "mulaw" | "alaw"; - -type XaiRealtimeTranscriptionProviderConfig = { - apiKey?: string; - baseUrl?: string; - sampleRate?: number; - encoding?: XaiRealtimeTranscriptionEncoding; - interimResults?: boolean; - endpointingMs?: number; - language?: string; -}; - type XaiRealtimeTranscriptionSessionConfig = RealtimeTranscriptionSessionCreateRequest & { apiKey: string; // Late-bound bearer; called per (re)connect. @@ -64,33 +48,6 @@ const XAI_REALTIME_STT_MAX_RECONNECT_ATTEMPTS = 5; const XAI_REALTIME_STT_RECONNECT_DELAY_MS = 1000; const XAI_REALTIME_STT_MAX_QUEUED_BYTES = 2 * 1024 * 1024; -function readRecord(value: unknown): Record | undefined { - return value && typeof value === "object" ? (value as Record) : undefined; -} - -function readNestedXaiConfig(rawConfig: RealtimeTranscriptionProviderConfig) { - const raw = readRecord(rawConfig); - const providers = readRecord(raw?.providers); - return readRecord(providers?.xai ?? raw?.xai ?? raw) ?? {}; -} - -function normalizeEncoding(value: unknown): XaiRealtimeTranscriptionEncoding | undefined { - const normalized = normalizeOptionalString(value)?.toLowerCase(); - if (!normalized) { - return undefined; - } - if (normalized === "ulaw" || normalized === "g711_ulaw" || normalized === "g711-mulaw") { - return "mulaw"; - } - if (normalized === "g711_alaw" || normalized === "g711-alaw") { - return "alaw"; - } - if (normalized === "pcm" || normalized === "mulaw" || normalized === "alaw") { - return normalized; - } - throw new Error(`Invalid xAI realtime transcription encoding: ${normalized}`); -} - function normalizeXaiRealtimeBaseUrl(value?: string): string { return normalizeOptionalString(value ?? process.env.XAI_BASE_URL) ?? XAI_BASE_URL; } @@ -109,29 +66,11 @@ function toXaiRealtimeWsUrl(config: XaiRealtimeTranscriptionSessionConfig): stri return url.toString(); } -function normalizeProviderConfig( - config: RealtimeTranscriptionProviderConfig, -): XaiRealtimeTranscriptionProviderConfig { - const raw = readNestedXaiConfig(config); - return { - apiKey: normalizeResolvedSecretInputString({ - value: raw.apiKey, - path: "plugins.entries.voice-call.config.streaming.providers.xai.apiKey", - }), - baseUrl: normalizeOptionalString(raw.baseUrl), - sampleRate: readFiniteNumber(raw.sampleRate ?? raw.sample_rate), - encoding: normalizeEncoding(raw.encoding), - interimResults: readBoolean(raw.interimResults ?? raw.interim_results), - endpointingMs: readFiniteNumber(raw.endpointingMs ?? raw.endpointing ?? raw.silenceDurationMs), - language: normalizeOptionalString(raw.language), - }; -} - function readErrorDetail(value: unknown): string { if (typeof value === "string") { return value; } - const record = readRecord(value); + const record = isRecord(value) ? value : undefined; const message = normalizeOptionalString(record?.message); const code = normalizeOptionalString(record?.code); return message ?? code ?? "xAI realtime transcription error"; @@ -233,18 +172,9 @@ function createXaiRealtimeTranscriptionSession( export function buildXaiRealtimeTranscriptionProvider(): RealtimeTranscriptionProviderPlugin { return { - id: "xai", - label: "xAI Realtime Transcription", - aliases: ["xai-realtime", "grok-stt-streaming"], - autoSelectOrder: 25, - resolveConfig: ({ rawConfig }) => normalizeProviderConfig(rawConfig), - isConfigured: ({ providerConfig, cfg }) => - Boolean( - normalizeProviderConfig(providerConfig).apiKey ?? - normalizeOptionalString(process.env.XAI_API_KEY), - ) || isProviderAuthProfileConfigured({ provider: "xai", cfg }), + ...createXaiRealtimeTranscriptionProviderMetadata(), createSession: (req) => { - const config = normalizeProviderConfig(req.providerConfig); + const config = normalizeXaiRealtimeTranscriptionProviderConfig(req.providerConfig); // createSession must stay sync per RealtimeTranscriptionProviderPlugin; bearer is resolved lazily in headers(). const seedApiKey = normalizeOptionalString(config.apiKey) ?? normalizeOptionalString(process.env.XAI_API_KEY); diff --git a/extensions/xai/realtime-voice-auth.runtime.ts b/extensions/xai/realtime-voice-auth.runtime.ts new file mode 100644 index 000000000000..6b735960658d --- /dev/null +++ b/extensions/xai/realtime-voice-auth.runtime.ts @@ -0,0 +1,22 @@ +import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth"; +import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; +import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; + +export async function resolveXaiRealtimeApiKey( + configApiKey: string | undefined, + cfg: OpenClawConfig | undefined, +): Promise { + const direct = + normalizeOptionalString(configApiKey) ?? normalizeOptionalString(process.env.XAI_API_KEY); + if (direct) { + return direct; + } + const auth = await resolveApiKeyForProvider({ provider: "xai", cfg }); + const oauthKey = normalizeOptionalString(auth?.apiKey); + if (oauthKey) { + return oauthKey; + } + throw new Error( + "xAI credentials missing for realtime voice. Sign in with `openclaw onboard --auth-choice xai-oauth`, run `openclaw onboard --auth-choice xai-api-key`, or set XAI_API_KEY.", + ); +} diff --git a/extensions/xai/realtime-voice-bridge.ts b/extensions/xai/realtime-voice-bridge.ts index 1dc505e69e76..38c62167872a 100644 --- a/extensions/xai/realtime-voice-bridge.ts +++ b/extensions/xai/realtime-voice-bridge.ts @@ -12,6 +12,7 @@ import type { import { RealtimeVoiceSessionLifecycle } from "openclaw/plugin-sdk/realtime-voice"; import { sleepWithAbort } from "openclaw/plugin-sdk/runtime-env"; import WebSocket from "ws"; +import { resolveXaiRealtimeApiKey } from "./realtime-voice-auth.runtime.js"; import { XAI_REALTIME_BASE_RECONNECT_DELAY_MS, XAI_REALTIME_CONNECT_TIMEOUT_MS, @@ -21,7 +22,6 @@ import { XAI_REALTIME_MAX_RECONNECT_ATTEMPTS, XAI_REALTIME_WS_MAX_PAYLOAD_BYTES, readXaiRealtimeErrorDetail, - resolveXaiRealtimeApiKey, toXaiRealtimeWsUrl, type XaiRealtimeEvent, } from "./realtime-voice-config.js"; diff --git a/extensions/xai/realtime-voice-config.ts b/extensions/xai/realtime-voice-config.ts index b5f9244c389c..88156b81d230 100644 --- a/extensions/xai/realtime-voice-config.ts +++ b/extensions/xai/realtime-voice-config.ts @@ -2,7 +2,6 @@ import { isProviderAuthProfileConfigured, type OpenClawConfig, } from "openclaw/plugin-sdk/provider-auth"; -import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import type { RealtimeVoiceBridgeCreateRequest, RealtimeVoiceProviderConfig, @@ -228,25 +227,6 @@ export function toXaiRealtimeWsUrl( return url.toString(); } -export async function resolveXaiRealtimeApiKey( - configApiKey: string | undefined, - cfg: OpenClawConfig | undefined, -): Promise { - const direct = - normalizeOptionalString(configApiKey) ?? normalizeOptionalString(process.env.XAI_API_KEY); - if (direct) { - return direct; - } - const auth = await resolveApiKeyForProvider({ provider: "xai", cfg }); - const oauthKey = normalizeOptionalString(auth?.apiKey); - if (oauthKey) { - return oauthKey; - } - throw new Error( - "xAI credentials missing for realtime voice. Sign in with `openclaw onboard --auth-choice xai-oauth`, run `openclaw onboard --auth-choice xai-api-key`, or set XAI_API_KEY.", - ); -} - export function hasXaiRealtimeApiKeyInput( configApiKey: string | undefined, cfg: OpenClawConfig | undefined, diff --git a/extensions/xai/realtime-voice-provider.ts b/extensions/xai/realtime-voice-provider.ts index 16ae7e8035e6..234d9befc5aa 100644 --- a/extensions/xai/realtime-voice-provider.ts +++ b/extensions/xai/realtime-voice-provider.ts @@ -1,54 +1,21 @@ import type { RealtimeVoiceProviderPlugin } from "openclaw/plugin-sdk/realtime-voice"; import { - REALTIME_VOICE_AUDIO_FORMAT_G711_ULAW_8KHZ, - REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ, -} from "openclaw/plugin-sdk/realtime-voice"; + assertXaiRealtimeVoiceRequestSupported, + createXaiRealtimeVoiceProviderMetadata, +} from "./capability-provider-metadata.js"; +import { resolveXaiRealtimeApiKey } from "./realtime-voice-auth.runtime.js"; import { XaiRealtimeVoiceBridge } from "./realtime-voice-bridge.js"; import { - XAI_REALTIME_DEFAULT_MODEL, - XAI_REALTIME_VOICES, - hasXaiRealtimeApiKeyInput, normalizeXaiRealtimeBaseUrl, normalizeXaiRealtimeProviderConfig, - resolveXaiRealtimeApiKey, } from "./realtime-voice-config.js"; export function buildXaiRealtimeVoiceProvider(): RealtimeVoiceProviderPlugin { return { - id: "xai", - label: "xAI Grok Voice", - aliases: ["xai-realtime-voice", "grok-voice"], - defaultModel: XAI_REALTIME_DEFAULT_MODEL, - voices: XAI_REALTIME_VOICES, - autoSelectOrder: 25, - capabilities: { - transports: ["gateway-relay"], - inputAudioFormats: [ - REALTIME_VOICE_AUDIO_FORMAT_G711_ULAW_8KHZ, - REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ, - ], - outputAudioFormats: [ - REALTIME_VOICE_AUDIO_FORMAT_G711_ULAW_8KHZ, - REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ, - ], - supportsBargeIn: true, - handlesInputAudioBargeIn: true, - supportsToolCalls: true, - supportsSessionResumption: true, - }, - resolveConfig: ({ rawConfig }) => normalizeXaiRealtimeProviderConfig(rawConfig), - isConfigured: ({ providerConfig, cfg }) => - hasXaiRealtimeApiKeyInput(normalizeXaiRealtimeProviderConfig(providerConfig).apiKey, cfg), + ...createXaiRealtimeVoiceProviderMetadata(), createBridge: (req) => { const config = normalizeXaiRealtimeProviderConfig(req.providerConfig); - if (req.autoRespondToAudio === false) { - throw new Error( - 'xAI realtime voice requires automatic server-VAD responses; use consultRouting: "provider-direct"', - ); - } - if ((req.interruptResponseOnInputAudio ?? config.interruptResponseOnInputAudio) === false) { - throw new Error("xAI realtime voice requires automatic server-VAD interruption handling"); - } + assertXaiRealtimeVoiceRequestSupported(req); return new XaiRealtimeVoiceBridge({ ...req, apiKey: config.apiKey, diff --git a/extensions/xai/speech-provider-metadata.ts b/extensions/xai/speech-provider-metadata.ts new file mode 100644 index 000000000000..a14261514f35 --- /dev/null +++ b/extensions/xai/speech-provider-metadata.ts @@ -0,0 +1,239 @@ +import { isProviderAuthProfileConfigured } from "openclaw/plugin-sdk/provider-auth"; +import { normalizeResolvedSecretInputString } from "openclaw/plugin-sdk/secret-input"; +import type { + SpeechDirectiveTokenParseContext, + SpeechProviderConfig, + SpeechProviderOverrides, + SpeechProviderPlugin, + SpeechSynthesisTarget, +} from "openclaw/plugin-sdk/speech"; +import { + asFiniteNumberInRange, + normalizeLowercaseStringOrEmpty, + normalizeOptionalString, +} from "openclaw/plugin-sdk/string-coerce-runtime"; +import { XAI_BASE_URL } from "./model-definitions.js"; + +const XAI_SPEECH_RESPONSE_FORMATS = ["mp3", "wav", "pcm", "mulaw", "alaw"] as const; + +export type XaiSpeechResponseFormat = (typeof XAI_SPEECH_RESPONSE_FORMATS)[number]; + +type XaiTtsProviderConfig = { + apiKey?: string; + baseUrl: string; + voiceId: string; + language?: string; + speed?: number; + responseFormat?: XaiSpeechResponseFormat; +}; + +type XaiTtsProviderOverrides = { + voiceId?: string; + language?: string; + speed?: number; +}; + +export const XAI_TTS_FALLBACK_VOICES = ["ara", "eve", "leo", "rex", "sal"] as const; + +export function normalizeXaiTtsBaseUrl(baseUrl?: string): string { + return normalizeOptionalString(baseUrl)?.replace(/\/+$/, "") ?? XAI_BASE_URL; +} + +export function isValidXaiTtsVoice(voice: string): boolean { + return normalizeOptionalString(voice) !== undefined; +} + +export function normalizeXaiLanguageCode(value: unknown): string | undefined { + const normalized = normalizeOptionalString(value)?.toLowerCase(); + if (!normalized) { + return undefined; + } + if (normalized === "auto" || /^[a-z]{2,3}(?:-[a-z]{2,4})?$/.test(normalized)) { + return normalized; + } + throw new Error( + `xAI language must be "auto" or a BCP-47 tag (e.g. "en", "pt-br", "zh-cn"); got: ${normalized}`, + ); +} + +function normalizeXaiSpeechSpeed(value: unknown): number | undefined { + return asFiniteNumberInRange(value, { min: 0.7, max: 1.5 }); +} + +function normalizeXaiSpeechResponseFormat(value: unknown): XaiSpeechResponseFormat | undefined { + const next = normalizeLowercaseStringOrEmpty(value); + if (!next) { + return undefined; + } + if (XAI_SPEECH_RESPONSE_FORMATS.some((format) => format === next)) { + return next as XaiSpeechResponseFormat; + } + throw new Error(`Invalid xAI speech responseFormat: ${next}`); +} + +export function resolveXaiSpeechResponseFormat( + target: SpeechSynthesisTarget | undefined, + configuredFormat?: XaiSpeechResponseFormat, +): XaiSpeechResponseFormat { + // Voice-note consumers may transcode without raw codec/rate metadata. + // Keep streamed output and buffered fallback self-describing. + return target === "voice-note" ? "mp3" : (configuredFormat ?? "mp3"); +} + +export function xaiSpeechResponseFormatToFileExtension( + format: XaiSpeechResponseFormat, +): ".mp3" | ".pcm" | ".wav" | ".mulaw" | ".alaw" { + switch (format) { + case "wav": + return ".wav"; + case "pcm": + return ".pcm"; + case "mulaw": + return ".mulaw"; + case "alaw": + return ".alaw"; + default: + return ".mp3"; + } +} + +function normalizeXaiSpeechProviderConfig( + rawConfig: Record, +): XaiTtsProviderConfig { + const providers = rawConfig.providers as Record | undefined; + const xai = (providers?.xai ?? rawConfig.xai ?? rawConfig) as Record; + return { + apiKey: normalizeResolvedSecretInputString({ + value: xai.apiKey, + path: "tts.providers.xai.apiKey", + }), + baseUrl: normalizeXaiTtsBaseUrl( + normalizeOptionalString(xai.baseUrl) ?? + normalizeOptionalString(process.env.XAI_BASE_URL) ?? + XAI_BASE_URL, + ), + voiceId: normalizeOptionalString(xai.voiceId ?? xai.voice) ?? "eve", + language: normalizeXaiLanguageCode(xai.language ?? xai.languageCode), + speed: normalizeXaiSpeechSpeed(xai.speed), + responseFormat: normalizeXaiSpeechResponseFormat(xai.responseFormat), + }; +} + +export function readXaiSpeechProviderConfig(config: SpeechProviderConfig): XaiTtsProviderConfig { + const normalized = normalizeXaiSpeechProviderConfig({}); + return { + apiKey: normalizeOptionalString(config.apiKey) ?? normalized.apiKey, + baseUrl: normalizeOptionalString(config.baseUrl) ?? normalized.baseUrl, + voiceId: normalizeOptionalString(config.voiceId ?? config.voice) ?? normalized.voiceId, + language: + normalizeXaiLanguageCode(config.language ?? config.languageCode) ?? normalized.language, + speed: normalizeXaiSpeechSpeed(config.speed) ?? normalized.speed, + responseFormat: + normalizeXaiSpeechResponseFormat(config.responseFormat) ?? normalized.responseFormat, + }; +} + +export function readXaiSpeechOverrides( + overrides: SpeechProviderOverrides | undefined, +): XaiTtsProviderOverrides { + if (!overrides) { + return {}; + } + return { + voiceId: normalizeOptionalString(overrides.voiceId ?? overrides.voice), + language: normalizeXaiLanguageCode(overrides.language), + speed: normalizeXaiSpeechSpeed(overrides.speed), + }; +} + +export function resolveDirectXaiAudioApiKey(configApiKey?: string): string | undefined { + return normalizeOptionalString(configApiKey) ?? normalizeOptionalString(process.env.XAI_API_KEY); +} + +function parseXaiSpeechDirectiveToken(ctx: SpeechDirectiveTokenParseContext): { + handled: boolean; + overrides?: SpeechProviderOverrides; + warnings?: string[]; +} { + switch (ctx.key) { + case "voice": + case "voice_id": + case "voiceid": + case "xai_voice": + case "xaivoice": + if (!ctx.policy.allowVoice) { + return { handled: true }; + } + if (!isValidXaiTtsVoice(ctx.value)) { + return { handled: true, warnings: [`invalid xAI voice "${ctx.value}"`] }; + } + return { handled: true, overrides: { voiceId: ctx.value } }; + default: + return { handled: false }; + } +} + +export function createXaiSpeechProviderMetadata(): Omit< + SpeechProviderPlugin, + "listVoices" | "synthesize" | "streamSynthesize" | "synthesizeTelephony" +> { + return { + id: "xai", + label: "xAI", + autoSelectOrder: 25, + models: [], + voices: XAI_TTS_FALLBACK_VOICES, + resolveConfig: ({ rawConfig }) => normalizeXaiSpeechProviderConfig(rawConfig), + parseDirectiveToken: parseXaiSpeechDirectiveToken, + resolveTalkConfig: ({ baseTtsConfig, talkProviderConfig }) => { + const base = normalizeXaiSpeechProviderConfig(baseTtsConfig); + const responseFormat = normalizeXaiSpeechResponseFormat(talkProviderConfig.responseFormat); + return { + ...base, + ...(talkProviderConfig.apiKey === undefined + ? {} + : { + apiKey: normalizeResolvedSecretInputString({ + value: talkProviderConfig.apiKey, + path: "talk.providers.xai.apiKey", + }), + }), + ...(normalizeOptionalString(talkProviderConfig.baseUrl) === undefined + ? {} + : { + baseUrl: normalizeXaiTtsBaseUrl(normalizeOptionalString(talkProviderConfig.baseUrl)), + }), + ...(normalizeOptionalString(talkProviderConfig.voiceId) === undefined + ? {} + : { voiceId: normalizeOptionalString(talkProviderConfig.voiceId) }), + ...(normalizeXaiLanguageCode( + talkProviderConfig.language ?? talkProviderConfig.languageCode, + ) === undefined + ? {} + : { + language: normalizeXaiLanguageCode( + talkProviderConfig.language ?? talkProviderConfig.languageCode, + ), + }), + ...(normalizeXaiSpeechSpeed(talkProviderConfig.speed) === undefined + ? {} + : { speed: normalizeXaiSpeechSpeed(talkProviderConfig.speed) }), + ...(responseFormat === undefined ? {} : { responseFormat }), + }; + }, + resolveTalkOverrides: ({ params }) => ({ + ...(normalizeOptionalString(params.voiceId ?? params.voice) === undefined + ? {} + : { voiceId: normalizeOptionalString(params.voiceId ?? params.voice) }), + ...(normalizeXaiLanguageCode(params.language ?? params.languageCode) === undefined + ? {} + : { language: normalizeXaiLanguageCode(params.language ?? params.languageCode) }), + ...(normalizeXaiSpeechSpeed(params.speed) === undefined + ? {} + : { speed: normalizeXaiSpeechSpeed(params.speed) }), + }), + isConfigured: ({ providerConfig, cfg }) => + Boolean(resolveDirectXaiAudioApiKey(readXaiSpeechProviderConfig(providerConfig).apiKey)) || + isProviderAuthProfileConfigured({ provider: "xai", cfg }), + }; +} diff --git a/extensions/xai/speech-provider.ts b/extensions/xai/speech-provider.ts index 89f518747b87..df5314a97879 100644 --- a/extensions/xai/speech-provider.ts +++ b/extensions/xai/speech-provider.ts @@ -1,146 +1,25 @@ // Xai provider module implements model/runtime integration. import { resolveGeneratedMediaMaxBytes } from "openclaw/plugin-sdk/media-generation-runtime"; -import { - isProviderAuthProfileConfigured, - type OpenClawConfig, -} from "openclaw/plugin-sdk/provider-auth"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth"; import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; -import { normalizeResolvedSecretInputString } from "openclaw/plugin-sdk/secret-input"; -import { - trimToUndefined, - type SpeechDirectiveTokenParseContext, - type SpeechProviderConfig, - type SpeechProviderOverrides, - type SpeechProviderPlugin, - type SpeechSynthesisRequest, - type SpeechSynthesisTarget, +import type { + SpeechProviderPlugin, + SpeechSynthesisRequest, + SpeechSynthesisTarget, } from "openclaw/plugin-sdk/speech"; -import { resolveSpeechProviderApiKey } from "openclaw/plugin-sdk/speech-core"; +import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; import { - asFiniteNumberInRange, - normalizeLowercaseStringOrEmpty, -} from "openclaw/plugin-sdk/string-coerce-runtime"; -import { - isValidXaiTtsVoice, - listXaiTtsVoices, - normalizeXaiLanguageCode, - normalizeXaiTtsBaseUrl, - XAI_BASE_URL, + createXaiSpeechProviderMetadata, + readXaiSpeechOverrides, + readXaiSpeechProviderConfig, + resolveDirectXaiAudioApiKey, + resolveXaiSpeechResponseFormat, + xaiSpeechResponseFormatToFileExtension, XAI_TTS_FALLBACK_VOICES, - xaiTTS, - xaiTTSStream, -} from "./tts.js"; - -const XAI_SPEECH_RESPONSE_FORMATS = ["mp3", "wav", "pcm", "mulaw", "alaw"] as const; - -type XaiSpeechResponseFormat = (typeof XAI_SPEECH_RESPONSE_FORMATS)[number]; - -type XaiTtsProviderConfig = { - apiKey?: string; - baseUrl: string; - voiceId: string; - language?: string; - speed?: number; - responseFormat?: XaiSpeechResponseFormat; -}; - -type XaiTtsProviderOverrides = { - voiceId?: string; - language?: string; - speed?: number; -}; - -function normalizeXaiSpeechSpeed(value: unknown): number | undefined { - return asFiniteNumberInRange(value, { min: 0.7, max: 1.5 }); -} - -function normalizeXaiSpeechResponseFormat(value: unknown): XaiSpeechResponseFormat | undefined { - const next = normalizeLowercaseStringOrEmpty(value); - if (!next) { - return undefined; - } - if (XAI_SPEECH_RESPONSE_FORMATS.some((format) => format === next)) { - return next as XaiSpeechResponseFormat; - } - throw new Error(`Invalid xAI speech responseFormat: ${next}`); -} - -function resolveSpeechResponseFormat( - target: SpeechSynthesisTarget | undefined, - configuredFormat?: XaiSpeechResponseFormat, -): XaiSpeechResponseFormat { - // Voice-note consumers may transcode without raw codec/rate metadata. - // Keep streamed output and buffered fallback self-describing. - if (target === "voice-note") { - return "mp3"; - } - return configuredFormat ?? "mp3"; -} - -function responseFormatToFileExtension( - format: XaiSpeechResponseFormat, -): ".mp3" | ".pcm" | ".wav" | ".mulaw" | ".alaw" { - switch (format) { - case "wav": - return ".wav"; - case "pcm": - return ".pcm"; - case "mulaw": - return ".mulaw"; - case "alaw": - return ".alaw"; - default: - return ".mp3"; - } -} - -function normalizeXaiProviderConfig(rawConfig: Record): XaiTtsProviderConfig { - const providers = rawConfig?.providers as Record | undefined; - const xai = (providers?.xai ?? rawConfig?.xai ?? rawConfig) as Record; - return { - apiKey: normalizeResolvedSecretInputString({ - value: xai?.apiKey, - path: "tts.providers.xai.apiKey", - }), - baseUrl: normalizeXaiTtsBaseUrl( - trimToUndefined(xai?.baseUrl) ?? trimToUndefined(process.env.XAI_BASE_URL) ?? XAI_BASE_URL, - ), - voiceId: trimToUndefined(xai?.voiceId ?? xai?.voice) ?? "eve", - language: normalizeXaiLanguageCode(trimToUndefined(xai?.language ?? xai?.languageCode)), - speed: normalizeXaiSpeechSpeed(xai?.speed), - responseFormat: normalizeXaiSpeechResponseFormat(xai?.responseFormat), - }; -} - -function readXaiProviderConfig(config: SpeechProviderConfig): XaiTtsProviderConfig { - const normalized = normalizeXaiProviderConfig({}); - return { - apiKey: trimToUndefined(config.apiKey) ?? normalized.apiKey, - baseUrl: trimToUndefined(config.baseUrl) ?? normalized.baseUrl, - voiceId: trimToUndefined(config.voiceId ?? config.voice) ?? normalized.voiceId, - language: - normalizeXaiLanguageCode(trimToUndefined(config.language ?? config.languageCode)) ?? - normalized.language, - speed: normalizeXaiSpeechSpeed(config.speed) ?? normalized.speed, - responseFormat: - normalizeXaiSpeechResponseFormat(config.responseFormat) ?? normalized.responseFormat, - }; -} - -function readXaiOverrides(overrides: SpeechProviderOverrides | undefined): XaiTtsProviderOverrides { - if (!overrides) { - return {}; - } - return { - voiceId: trimToUndefined(overrides.voiceId ?? overrides.voice), - language: normalizeXaiLanguageCode(trimToUndefined(overrides.language)), - speed: normalizeXaiSpeechSpeed(overrides.speed), - }; -} - -function resolveDirectXaiAudioApiKey(configApiKey?: string): string | undefined { - return resolveSpeechProviderApiKey(configApiKey, process.env.XAI_API_KEY); -} + normalizeXaiTtsBaseUrl, + type XaiSpeechResponseFormat, +} from "./speech-provider-metadata.js"; +import { listXaiTtsVoices, xaiTTS, xaiTTSStream } from "./tts.js"; async function resolveXaiSpeechSynthesisRequest( req: Pick< @@ -149,8 +28,8 @@ async function resolveXaiSpeechSynthesisRequest( > & { target?: SpeechSynthesisTarget }, forcedResponseFormat?: XaiSpeechResponseFormat, ) { - const config = readXaiProviderConfig(req.providerConfig); - const overrides = readXaiOverrides(req.providerOverrides); + const config = readXaiSpeechProviderConfig(req.providerConfig); + const overrides = readXaiSpeechOverrides(req.providerOverrides); return { text: req.text, apiKey: await resolveXaiAudioApiKey(config.apiKey, req.cfg), @@ -159,114 +38,33 @@ async function resolveXaiSpeechSynthesisRequest( language: overrides.language ?? config.language, speed: overrides.speed ?? config.speed, responseFormat: - forcedResponseFormat ?? resolveSpeechResponseFormat(req.target, config.responseFormat), + forcedResponseFormat ?? resolveXaiSpeechResponseFormat(req.target, config.responseFormat), timeoutMs: req.timeoutMs, maxBytes: resolveGeneratedMediaMaxBytes(req.cfg, "audio"), }; } -function parseDirectiveToken(ctx: SpeechDirectiveTokenParseContext): { - handled: boolean; - overrides?: SpeechProviderOverrides; - warnings?: string[]; -} { - switch (ctx.key) { - case "voice": - case "voice_id": - case "voiceid": - case "xai_voice": - case "xaivoice": - if (!ctx.policy.allowVoice) { - return { handled: true }; - } - if (!isValidXaiTtsVoice(ctx.value)) { - return { handled: true, warnings: [`invalid xAI voice "${ctx.value}"`] }; - } - return { handled: true, overrides: { voiceId: ctx.value } }; - default: - return { handled: false }; - } -} - export function buildXaiSpeechProvider(): SpeechProviderPlugin { return { - id: "xai", - label: "xAI", - autoSelectOrder: 25, - models: [], - voices: XAI_TTS_FALLBACK_VOICES, - resolveConfig: ({ rawConfig }) => normalizeXaiProviderConfig(rawConfig), - parseDirectiveToken, - resolveTalkConfig: ({ baseTtsConfig, talkProviderConfig }) => { - const base = normalizeXaiProviderConfig(baseTtsConfig); - const responseFormat = normalizeXaiSpeechResponseFormat(talkProviderConfig.responseFormat); - return { - ...base, - ...(talkProviderConfig.apiKey === undefined - ? {} - : { - apiKey: normalizeResolvedSecretInputString({ - value: talkProviderConfig.apiKey, - path: "talk.providers.xai.apiKey", - }), - }), - ...(trimToUndefined(talkProviderConfig.baseUrl) == null - ? {} - : { baseUrl: normalizeXaiTtsBaseUrl(trimToUndefined(talkProviderConfig.baseUrl)) }), - ...(trimToUndefined(talkProviderConfig.voiceId) == null - ? {} - : { voiceId: trimToUndefined(talkProviderConfig.voiceId) }), - ...(normalizeXaiLanguageCode( - trimToUndefined(talkProviderConfig.language ?? talkProviderConfig.languageCode), - ) == null - ? {} - : { - language: normalizeXaiLanguageCode( - trimToUndefined(talkProviderConfig.language ?? talkProviderConfig.languageCode), - ), - }), - ...(normalizeXaiSpeechSpeed(talkProviderConfig.speed) == null - ? {} - : { speed: normalizeXaiSpeechSpeed(talkProviderConfig.speed) }), - ...(responseFormat == null ? {} : { responseFormat }), - }; - }, - resolveTalkOverrides: ({ params }) => ({ - ...(trimToUndefined(params.voiceId ?? params.voice) == null - ? {} - : { voiceId: trimToUndefined(params.voiceId ?? params.voice) }), - ...(normalizeXaiLanguageCode(trimToUndefined(params.language ?? params.languageCode)) == null - ? {} - : { - language: normalizeXaiLanguageCode( - trimToUndefined(params.language ?? params.languageCode), - ), - }), - ...(normalizeXaiSpeechSpeed(params.speed) == null - ? {} - : { speed: normalizeXaiSpeechSpeed(params.speed) }), - }), + ...createXaiSpeechProviderMetadata(), listVoices: async (req) => { - const config = readXaiProviderConfig(req.providerConfig ?? {}); - const directApiKey = trimToUndefined(req.apiKey) ?? config.apiKey; + const config = readXaiSpeechProviderConfig(req.providerConfig ?? {}); + const directApiKey = normalizeOptionalString(req.apiKey) ?? config.apiKey; const apiKey = await resolveOptionalXaiAudioApiKey(directApiKey, req.cfg); if (!apiKey) { return XAI_TTS_FALLBACK_VOICES.map((voice) => ({ id: voice, name: voice })); } return await listXaiTtsVoices({ apiKey, - baseUrl: normalizeXaiTtsBaseUrl(trimToUndefined(req.baseUrl) ?? config.baseUrl), + baseUrl: normalizeXaiTtsBaseUrl(normalizeOptionalString(req.baseUrl) ?? config.baseUrl), }); }, - isConfigured: ({ providerConfig, cfg }) => - Boolean(resolveDirectXaiAudioApiKey(readXaiProviderConfig(providerConfig).apiKey)) || - isProviderAuthProfileConfigured({ provider: "xai", cfg }), synthesize: async (req) => { const params = await resolveXaiSpeechSynthesisRequest(req); return { audioBuffer: await xaiTTS(params), outputFormat: params.responseFormat, - fileExtension: responseFormatToFileExtension(params.responseFormat), + fileExtension: xaiSpeechResponseFormatToFileExtension(params.responseFormat), voiceCompatible: false, }; }, @@ -276,7 +74,7 @@ export function buildXaiSpeechProvider(): SpeechProviderPlugin { return { audioStream: stream.audioStream, outputFormat: params.responseFormat, - fileExtension: responseFormatToFileExtension(params.responseFormat), + fileExtension: xaiSpeechResponseFormatToFileExtension(params.responseFormat), voiceCompatible: false, release: stream.release, }; @@ -304,7 +102,7 @@ async function resolveOptionalXaiAudioApiKey( return undefined; } const auth = await resolveApiKeyForProvider({ provider: "xai", cfg }); - return trimToUndefined(auth?.apiKey); + return normalizeOptionalString(auth?.apiKey); } async function resolveXaiAudioApiKey( diff --git a/extensions/xai/stt.ts b/extensions/xai/stt.ts index 9051384f8fc6..2f61674c78b6 100644 --- a/extensions/xai/stt.ts +++ b/extensions/xai/stt.ts @@ -13,6 +13,7 @@ import { resolveProviderHttpRequestConfig, } from "openclaw/plugin-sdk/provider-http"; import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { createXaiMediaUnderstandingProviderMetadata } from "./capability-provider-metadata.js"; import { XAI_BASE_URL } from "./model-definitions.js"; type XaiSttResponse = { @@ -80,9 +81,7 @@ export function buildXaiMediaUnderstandingProvider(): MediaUnderstandingProvider // before transcribeAudio runs, so an OAuth profile (when configured) reaches // here as `params.apiKey` already. No plugin-side fallback required. return { - id: "xai", - capabilities: ["audio"], - autoPriority: { audio: 25 }, + ...createXaiMediaUnderstandingProviderMetadata(), transcribeAudio: transcribeXaiAudio, }; } diff --git a/extensions/xai/tsconfig.json b/extensions/xai/tsconfig.json index 43db97d95e7f..0fcdf320592b 100644 --- a/extensions/xai/tsconfig.json +++ b/extensions/xai/tsconfig.json @@ -353,6 +353,9 @@ "openclaw/plugin-sdk/realtime-bootstrap-context": [ "../../packages/plugin-sdk/dist/src/plugin-sdk/realtime-bootstrap-context.d.ts" ], + "openclaw/plugin-sdk/realtime-voice-audio-queue": [ + "../../packages/plugin-sdk/dist/src/plugin-sdk/realtime-voice-audio-queue.d.ts" + ], "openclaw/plugin-sdk/realtime-voice": [ "../../packages/plugin-sdk/dist/src/plugin-sdk/realtime-voice.d.ts" ], diff --git a/extensions/xai/tts.test.ts b/extensions/xai/tts.test.ts index 67247315c617..332381b02387 100644 --- a/extensions/xai/tts.test.ts +++ b/extensions/xai/tts.test.ts @@ -1,14 +1,9 @@ // Xai tests cover tts plugin behavior. import { mockPinnedHostnameResolution } from "openclaw/plugin-sdk/test-env"; import { beforeEach, afterEach, describe, expect, it, vi } from "vitest"; -import { - isValidXaiTtsVoice, - listXaiTtsVoices, - XAI_BASE_URL, - XAI_TTS_FALLBACK_VOICES, - xaiTTS, - xaiTTSStream, -} from "./tts.js"; +import { XAI_BASE_URL } from "./model-definitions.js"; +import { isValidXaiTtsVoice, XAI_TTS_FALLBACK_VOICES } from "./speech-provider-metadata.js"; +import { listXaiTtsVoices, xaiTTS, xaiTTSStream } from "./tts.js"; const { FakeWebSocket } = vi.hoisted(() => { type Listener = (...args: unknown[]) => void; diff --git a/extensions/xai/tts.ts b/extensions/xai/tts.ts index bcae115cfee5..8f984178bd39 100644 --- a/extensions/xai/tts.ts +++ b/extensions/xai/tts.ts @@ -13,28 +13,18 @@ import { ssrfPolicyFromHttpBaseUrlAllowedHostname, } from "openclaw/plugin-sdk/ssrf-runtime"; import WebSocket, { type RawData } from "ws"; -import { XAI_BASE_URL } from "./api.js"; +import { XAI_BASE_URL } from "./model-definitions.js"; +import { + isValidXaiTtsVoice, + normalizeXaiLanguageCode, + normalizeXaiTtsBaseUrl, +} from "./speech-provider-metadata.js"; import { xaiUserAgentHeaderFor } from "./src/xai-user-agent.js"; -export { XAI_BASE_URL }; const DEFAULT_TTS_MAX_BYTES = 16 * 1024 * 1024; const XAI_TTS_VOICE_LIST_TIMEOUT_MS = 30_000; const XAI_TTS_VOICE_LIST_MAX_BYTES = 1024 * 1024; const XAI_TTS_STREAM_TEXT_DELTA_MAX_CHARS = 15_000; -export const XAI_TTS_FALLBACK_VOICES = ["ara", "eve", "leo", "rex", "sal"] as const; - -export function normalizeXaiTtsBaseUrl(baseUrl?: string): string { - const trimmed = baseUrl?.trim(); - if (!trimmed) { - return XAI_BASE_URL; - } - return trimmed.replace(/\/+$/, ""); -} - -export function isValidXaiTtsVoice(voice: string): boolean { - return trimToUndefined(voice) !== undefined; -} - export async function listXaiTtsVoices(params: { apiKey: string; baseUrl?: string; @@ -82,20 +72,6 @@ export async function listXaiTtsVoices(params: { } } -export function normalizeXaiLanguageCode(value: unknown): string | undefined { - const trimmed = trimToUndefined(value); - if (!trimmed) { - return undefined; - } - const normalized = trimmed.toLowerCase(); - if (normalized === "auto" || /^[a-z]{2,3}(?:-[a-z]{2,4})?$/.test(normalized)) { - return normalized; - } - throw new Error( - `xAI language must be "auto" or a BCP-47 tag (e.g. "en", "pt-br", "zh-cn"); got: ${normalized}`, - ); -} - type XaiTtsResponseFormat = "mp3" | "wav" | "pcm" | "mulaw" | "alaw"; const XAI_NATIVE_TTS_STREAM_HOST = "api.x.ai"; diff --git a/extensions/xai/video-generation-provider.ts b/extensions/xai/video-generation-provider.ts index 279640d1a6e3..494d72a57683 100644 --- a/extensions/xai/video-generation-provider.ts +++ b/extensions/xai/video-generation-provider.ts @@ -1,7 +1,6 @@ // Xai provider module implements model/runtime integration. import { toImageDataUrl } from "openclaw/plugin-sdk/image-generation"; import { resolveGeneratedMediaMaxBytes } from "openclaw/plugin-sdk/media-generation-runtime"; -import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import { assertOkOrThrowHttpError, @@ -17,42 +16,24 @@ import { import { isRecord, normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; import type { VideoGenerationProvider, - VideoGenerationProviderCapabilities, VideoGenerationRequest, } from "openclaw/plugin-sdk/video-generation"; +import { + DEFAULT_XAI_VIDEO_BASE_URL, + DEFAULT_XAI_VIDEO_MODEL, + XAI_VIDEO_ASPECT_RATIOS, + XAI_VIDEO_DEFAULT_TIMEOUT_MS, + createXaiVideoGenerationProviderMetadata, + isXaiVideo15Model, +} from "./capability-provider-metadata.js"; import { downloadXaiVideo, fetchXaiVideoResponse, type XaiVideoRequestPolicy, } from "./video-generation-transport.js"; -const DEFAULT_XAI_VIDEO_BASE_URL = "https://api.x.ai/v1"; -const DEFAULT_XAI_VIDEO_MODEL = "grok-imagine-video"; -const XAI_VIDEO_15_MODEL = "grok-imagine-video-1.5"; -const XAI_VIDEO_15_MODEL_IDS = new Set([ - XAI_VIDEO_15_MODEL, - "grok-imagine-video-1.5-preview", - "grok-imagine-video-1.5-2026-05-30", -]); -const DEFAULT_TIMEOUT_MS = 600_000; const POLL_INTERVAL_MS = 5_000; const MAX_POLL_ATTEMPTS = 120; -const XAI_VIDEO_ASPECT_RATIOS = new Set(["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3"]); -const XAI_VIDEO_15_CAPABILITIES = { - imageToVideo: { - enabled: true, - maxVideos: 1, - maxInputImages: 1, - maxDurationSeconds: 15, - aspectRatios: [...XAI_VIDEO_ASPECT_RATIOS], - resolutions: ["480P", "720P", "1080P"], - supportsAspectRatio: true, - supportsResolution: true, - }, - videoToVideo: { - enabled: false, - }, -} satisfies VideoGenerationProviderCapabilities; const XAI_VIDEO_MALFORMED_RESPONSE = "xAI video generation response malformed"; // xAI documents these as the only meaningful values; everything else (queued, // processing, submitted, pending, in_progress, ...) means "keep polling". @@ -169,11 +150,6 @@ function isReferenceImage(input: VideoGenerationSourceInput): boolean { return normalizeOptionalString(input.role)?.toLowerCase() === "reference_image"; } -function isXaiVideo15Model(model: string | undefined): boolean { - const normalized = normalizeOptionalString(model); - return normalized ? XAI_VIDEO_15_MODEL_IDS.has(normalized) : false; -} - function isFirstFrameImage(input: VideoGenerationSourceInput): boolean { const role = normalizeOptionalString(input.role)?.toLowerCase(); return role === undefined || role === "first_frame"; @@ -385,9 +361,9 @@ async function pollXaiVideo( }, timeoutMs: createProviderOperationTimeoutResolver({ deadline, - defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + defaultTimeoutMs: XAI_VIDEO_DEFAULT_TIMEOUT_MS, }), - defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + defaultTimeoutMs: XAI_VIDEO_DEFAULT_TIMEOUT_MS, allowPrivateNetwork: params.allowPrivateNetwork, dispatcherPolicy: params.dispatcherPolicy, fetchFn: params.fetchFn, @@ -418,52 +394,7 @@ async function pollXaiVideo( export function buildXaiVideoGenerationProvider(): VideoGenerationProvider { return { - id: "xai", - label: "xAI", - defaultModel: DEFAULT_XAI_VIDEO_MODEL, - defaultTimeoutMs: DEFAULT_TIMEOUT_MS, - models: [DEFAULT_XAI_VIDEO_MODEL, XAI_VIDEO_15_MODEL], - catalogByModel: { - [XAI_VIDEO_15_MODEL]: { - capabilities: XAI_VIDEO_15_CAPABILITIES, - modes: ["imageToVideo"], - }, - }, - isConfigured: (ctx) => isProviderApiKeyConfigured({ provider: "xai", ...ctx }), - capabilities: { - generate: { - maxVideos: 1, - maxDurationSeconds: 15, - aspectRatios: [...XAI_VIDEO_ASPECT_RATIOS], - resolutions: ["480P", "720P"], - supportsAspectRatio: true, - supportsResolution: true, - }, - imageToVideo: { - enabled: true, - maxVideos: 1, - maxInputImages: 7, - maxDurationSeconds: 15, - aspectRatios: [...XAI_VIDEO_ASPECT_RATIOS], - resolutions: ["480P", "720P"], - supportsAspectRatio: true, - supportsResolution: true, - }, - videoToVideo: { - enabled: true, - maxVideos: 1, - maxInputVideos: 1, - maxDurationSeconds: 10, - supportsAspectRatio: false, - supportsResolution: false, - }, - }, - resolveModelCapabilities: ({ model }): VideoGenerationProviderCapabilities | undefined => { - if (!isXaiVideo15Model(model)) { - return undefined; - } - return XAI_VIDEO_15_CAPABILITIES; - }, + ...createXaiVideoGenerationProviderMetadata(), async generateVideo(req) { // Validate provider/model mode constraints before auth or HTTP setup so // unsupported 1.5 requests cannot be submitted and billed accidentally. @@ -507,7 +438,7 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider { body: createBody, timeoutMs: resolveProviderOperationTimeoutMs({ deadline, - defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + defaultTimeoutMs: XAI_VIDEO_DEFAULT_TIMEOUT_MS, }), fetchFn, allowPrivateNetwork, @@ -528,7 +459,7 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider { headers, timeoutMs: resolveProviderOperationTimeoutMs({ deadline, - defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + defaultTimeoutMs: XAI_VIDEO_DEFAULT_TIMEOUT_MS, }), baseUrl, allowPrivateNetwork, @@ -543,9 +474,9 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider { url: videoUrl, timeoutMs: createProviderOperationTimeoutResolver({ deadline, - defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + defaultTimeoutMs: XAI_VIDEO_DEFAULT_TIMEOUT_MS, }), - defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + defaultTimeoutMs: XAI_VIDEO_DEFAULT_TIMEOUT_MS, allowPrivateNetwork, dispatcherPolicy, fetchFn, diff --git a/extensions/xai/xai-oauth-entry.test.ts b/extensions/xai/xai-oauth-entry.test.ts new file mode 100644 index 000000000000..dce876435a8d --- /dev/null +++ b/extensions/xai/xai-oauth-entry.test.ts @@ -0,0 +1,44 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const oauthRuntimeMocks = vi.hoisted(() => ({ + loginXaiDeviceCode: vi.fn(), + refreshXaiOAuthCredential: vi.fn(), +})); + +vi.mock("./xai-oauth.js", () => oauthRuntimeMocks); + +beforeEach(() => { + vi.resetModules(); + oauthRuntimeMocks.loginXaiDeviceCode.mockReset(); + oauthRuntimeMocks.refreshXaiOAuthCredential.mockReset(); + oauthRuntimeMocks.loginXaiDeviceCode.mockResolvedValue({ profiles: [] }); + oauthRuntimeMocks.refreshXaiOAuthCredential.mockResolvedValue({ + type: "oauth", + provider: "xai", + access: "next-access", + refresh: "next-refresh", + expires: 123, + }); +}); + +describe("xAI OAuth lazy entry", () => { + it("loads OAuth runtime only when an auth operation runs", async () => { + const entry = await import("./xai-oauth-entry.js"); + const method = entry.createXaiOAuthAuthMethod(); + + expect(oauthRuntimeMocks.loginXaiDeviceCode).not.toHaveBeenCalled(); + expect(oauthRuntimeMocks.refreshXaiOAuthCredential).not.toHaveBeenCalled(); + + await method.run({} as never); + expect(oauthRuntimeMocks.loginXaiDeviceCode).toHaveBeenCalledOnce(); + + await entry.refreshXaiOAuthCredential({ + type: "oauth", + provider: "xai", + access: "access", + refresh: "refresh", + expires: 1, + }); + expect(oauthRuntimeMocks.refreshXaiOAuthCredential).toHaveBeenCalledOnce(); + }); +}); diff --git a/extensions/xai/xai-oauth-entry.ts b/extensions/xai/xai-oauth-entry.ts new file mode 100644 index 000000000000..7512a4df6d12 --- /dev/null +++ b/extensions/xai/xai-oauth-entry.ts @@ -0,0 +1,56 @@ +import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime"; +import type { ProviderAuthMethod } from "openclaw/plugin-sdk/plugin-entry"; +import type { OAuthCredential } from "openclaw/plugin-sdk/provider-auth"; + +const PROVIDER_ID = "xai"; +const XAI_OAUTH_METHOD_ID = "oauth"; +const XAI_OAUTH_CHOICE_ID = "xai-oauth"; +const XAI_DEVICE_CODE_METHOD_ID = "device-code"; +const XAI_DEVICE_CODE_CHOICE_ID = "xai-device-code"; + +const loadXaiOAuthRuntime = createLazyRuntimeModule(() => import("./xai-oauth.js")); + +export function createXaiOAuthAuthMethod(): ProviderAuthMethod { + return { + id: XAI_OAUTH_METHOD_ID, + label: "xAI OAuth", + hint: "Remote-friendly browser sign-in without a localhost callback", + kind: "oauth", + wizard: { + choiceId: XAI_OAUTH_CHOICE_ID, + choiceLabel: "xAI OAuth", + choiceHint: "Remote-friendly browser sign-in without a localhost callback", + groupId: PROVIDER_ID, + groupLabel: "xAI (Grok)", + groupHint: "API key or OAuth", + methodId: XAI_OAUTH_METHOD_ID, + }, + run: async (ctx) => (await loadXaiOAuthRuntime()).loginXaiDeviceCode(ctx), + }; +} + +export function createXaiDeviceCodeAuthMethod(): ProviderAuthMethod { + return { + id: XAI_DEVICE_CODE_METHOD_ID, + label: "xAI device code", + hint: "Deprecated alias for xAI OAuth device-code login", + kind: "device_code", + wizard: { + choiceId: XAI_DEVICE_CODE_CHOICE_ID, + choiceLabel: "xAI device code", + choiceHint: "Compatibility alias for xAI OAuth device-code sign-in", + assistantVisibility: "manual-only", + groupId: PROVIDER_ID, + groupLabel: "xAI (Grok)", + groupHint: "API key or OAuth", + methodId: XAI_DEVICE_CODE_METHOD_ID, + }, + run: async (ctx) => (await loadXaiOAuthRuntime()).loginXaiDeviceCode(ctx), + }; +} + +export async function refreshXaiOAuthCredential( + credential: OAuthCredential, +): Promise { + return await (await loadXaiOAuthRuntime()).refreshXaiOAuthCredential(credential); +} diff --git a/extensions/xai/xai-oauth.test.ts b/extensions/xai/xai-oauth.test.ts index aced32f0855a..21aab462aa92 100644 --- a/extensions/xai/xai-oauth.test.ts +++ b/extensions/xai/xai-oauth.test.ts @@ -6,11 +6,8 @@ import { } from "openclaw/plugin-sdk/plugin-test-runtime"; import type { OAuthCredential } from "openclaw/plugin-sdk/provider-auth"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { - createXaiDeviceCodeAuthMethod, - createXaiOAuthAuthMethod, - refreshXaiOAuthCredential, -} from "./xai-oauth.js"; +import { createXaiDeviceCodeAuthMethod, createXaiOAuthAuthMethod } from "./xai-oauth-entry.js"; +import { refreshXaiOAuthCredential } from "./xai-oauth.js"; const XAI_OAUTH_CLIENT_ID = "b1a00492-073a-47ea-816f-4c329264a828"; const XAI_OAUTH_SCOPE = "openid profile email offline_access grok-cli:access api:access"; diff --git a/extensions/xai/xai-oauth.ts b/extensions/xai/xai-oauth.ts index db193db67e75..6836cf84c9bb 100644 --- a/extensions/xai/xai-oauth.ts +++ b/extensions/xai/xai-oauth.ts @@ -5,7 +5,7 @@ import { resolveExpiresAtMsFromDurationSeconds, resolveExpiresAtMsFromEpochSeconds, } from "openclaw/plugin-sdk/number-runtime"; -import type { ProviderAuthContext, ProviderAuthMethod } from "openclaw/plugin-sdk/plugin-entry"; +import type { ProviderAuthContext } from "openclaw/plugin-sdk/plugin-entry"; import { buildOauthProviderAuthResult, toFormUrlEncoded, @@ -19,10 +19,6 @@ import { applyXaiOAuthConfig, XAI_OAUTH_DEFAULT_MODEL_REF } from "./onboard.js"; import { xaiUserAgent } from "./src/xai-user-agent.js"; const PROVIDER_ID = "xai"; -const XAI_OAUTH_METHOD_ID = "oauth"; -const XAI_OAUTH_CHOICE_ID = "xai-oauth"; -const XAI_DEVICE_CODE_METHOD_ID = "device-code"; -const XAI_DEVICE_CODE_CHOICE_ID = "xai-device-code"; const XAI_OAUTH_CLIENT_ID = "b1a00492-073a-47ea-816f-4c329264a828"; const XAI_OAUTH_SCOPE = "openid profile email offline_access grok-cli:access api:access"; const XAI_OAUTH_ISSUER = "https://auth.x.ai"; @@ -607,7 +603,7 @@ async function noteXaiDeviceCode( ); } -async function loginXaiDeviceCode(ctx: ProviderAuthContext): Promise { +export async function loginXaiDeviceCode(ctx: ProviderAuthContext): Promise { const progress = ctx.prompter.progress("Starting xAI OAuth..."); try { const discovery = await fetchXaiDeviceCodeDiscovery( @@ -708,42 +704,3 @@ export async function refreshXaiOAuthCredential( issuer: XAI_OAUTH_ISSUER, } as OAuthCredential; } - -export function createXaiOAuthAuthMethod(): ProviderAuthMethod { - return { - id: XAI_OAUTH_METHOD_ID, - label: "xAI OAuth", - hint: "Remote-friendly browser sign-in without a localhost callback", - kind: "oauth", - wizard: { - choiceId: XAI_OAUTH_CHOICE_ID, - choiceLabel: "xAI OAuth", - choiceHint: "Remote-friendly browser sign-in without a localhost callback", - groupId: PROVIDER_ID, - groupLabel: "xAI (Grok)", - groupHint: "API key or OAuth", - methodId: XAI_OAUTH_METHOD_ID, - }, - run: async (ctx) => loginXaiDeviceCode(ctx), - }; -} - -export function createXaiDeviceCodeAuthMethod(): ProviderAuthMethod { - return { - id: XAI_DEVICE_CODE_METHOD_ID, - label: "xAI device code", - hint: "Deprecated alias for xAI OAuth device-code login", - kind: "device_code", - wizard: { - choiceId: XAI_DEVICE_CODE_CHOICE_ID, - choiceLabel: "xAI device code", - choiceHint: "Compatibility alias for xAI OAuth device-code sign-in", - assistantVisibility: "manual-only", - groupId: PROVIDER_ID, - groupLabel: "xAI (Grok)", - groupHint: "API key or OAuth", - methodId: XAI_DEVICE_CODE_METHOD_ID, - }, - run: async (ctx) => loginXaiDeviceCode(ctx), - }; -} diff --git a/package.json b/package.json index 50899541549d..b4e7b78eabaa 100644 --- a/package.json +++ b/package.json @@ -170,6 +170,7 @@ "!dist/plugin-sdk/qa-runner-runtime.d.ts", "!dist/plugin-sdk/realtime-bootstrap-context.d.ts", "!dist/plugin-sdk/realtime-transcription.d.ts", + "!dist/plugin-sdk/realtime-voice-audio-queue.d.ts", "!dist/plugin-sdk/realtime-voice.d.ts", "!dist/plugin-sdk/reply-payload-testing.js", "!dist/plugin-sdk/reply-payload-testing.d.ts", @@ -1126,6 +1127,9 @@ "./plugin-sdk/realtime-bootstrap-context": { "default": "./dist/plugin-sdk/realtime-bootstrap-context.js" }, + "./plugin-sdk/realtime-voice-audio-queue": { + "default": "./dist/plugin-sdk/realtime-voice-audio-queue.js" + }, "./plugin-sdk/realtime-voice": { "default": "./dist/plugin-sdk/realtime-voice.js" }, diff --git a/scripts/lib/plugin-sdk-entrypoints.json b/scripts/lib/plugin-sdk-entrypoints.json index 89b23f26306f..9cf93e1ecaa5 100644 --- a/scripts/lib/plugin-sdk-entrypoints.json +++ b/scripts/lib/plugin-sdk-entrypoints.json @@ -232,6 +232,7 @@ "reply-history", "realtime-transcription", "realtime-bootstrap-context", + "realtime-voice-audio-queue", "realtime-voice", "meeting-runtime", "transcripts", diff --git a/scripts/lib/plugin-sdk-private-local-only-subpaths.json b/scripts/lib/plugin-sdk-private-local-only-subpaths.json index 9ec713bada00..3a552e30a7dc 100644 --- a/scripts/lib/plugin-sdk-private-local-only-subpaths.json +++ b/scripts/lib/plugin-sdk-private-local-only-subpaths.json @@ -124,6 +124,7 @@ "qa-runtime", "realtime-bootstrap-context", "realtime-transcription", + "realtime-voice-audio-queue", "realtime-voice", "reply-payload-testing", "reply-reference", diff --git a/src/plugin-sdk/realtime-voice-audio-queue.ts b/src/plugin-sdk/realtime-voice-audio-queue.ts new file mode 100644 index 000000000000..353ba552e9ae --- /dev/null +++ b/src/plugin-sdk/realtime-voice-audio-queue.ts @@ -0,0 +1,5 @@ +/** Production-private queue seam for lazy realtime voice provider facades. */ +export { + createRealtimeVoiceAudioQueue, + type RealtimeVoiceAudioQueue, +} from "../talk/realtime-session-lifecycle.js"; diff --git a/src/plugin-sdk/realtime-voice.test.ts b/src/plugin-sdk/realtime-voice.test.ts index 3a614dbf7c99..4fb73be7b16d 100644 --- a/src/plugin-sdk/realtime-voice.test.ts +++ b/src/plugin-sdk/realtime-voice.test.ts @@ -237,6 +237,19 @@ describe("RealtimeVoiceSessionLifecycle", () => { }); describe("createRealtimeVoiceAudioQueue", () => { + it("releases byte budget as queued audio is consumed", () => { + const queue = createRealtimeVoiceAudioQueue("reject-newest"); + const first = Buffer.alloc(512 * 1024, 0x01); + const second = Buffer.alloc(512 * 1024, 0x02); + + expect(queue.enqueue(first)).toBe(true); + expect(queue.enqueue(second)).toBe(true); + expect(queue.enqueue(Buffer.from([0x03]))).toBe(false); + expect(queue.dequeue()).toEqual(first); + expect(queue.enqueue(Buffer.from([0x03]))).toBe(true); + expect(queue.drain()).toEqual([second, Buffer.from([0x03])]); + }); + it("drops the oldest audio and resets accounting on clear", () => { const queue = createRealtimeVoiceAudioQueue("drop-oldest"); for (let index = 0; index < 322; index += 1) { diff --git a/src/talk/realtime-session-lifecycle.ts b/src/talk/realtime-session-lifecycle.ts index 3b3925c9d283..8a96ada13b11 100644 --- a/src/talk/realtime-session-lifecycle.ts +++ b/src/talk/realtime-session-lifecycle.ts @@ -5,6 +5,7 @@ type RealtimeVoiceAudioOverflowPolicy = "drop-oldest" | "reject-newest"; export type RealtimeVoiceAudioQueue = { clear: () => void; + dequeue: () => Buffer | undefined; drain: () => Buffer[]; enqueue: (audio: Buffer) => boolean; }; @@ -22,6 +23,13 @@ export function createRealtimeVoiceAudioQueue( return { clear, + dequeue: () => { + const chunk = chunks.shift(); + if (chunk) { + bytes -= chunk.byteLength; + } + return chunk; + }, drain: () => { const drained = chunks; clear();