diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index a77a4da1edbe..b53a2b488eba 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -141,6 +141,6 @@ a94c9ff59cc361b04f8731a47d722c4bc88cfabf942221369a8cb67bdcda7249 module/tool-pl 9d6ab352913a573b226e054e1dc8c6d088493aea9954950c65585923b5b6895a module/tool-send 541df9dea799f25e83ea483d481ecebc5b91c016effab593c54d3efe3ee6517b module/web-media c1fe90c8d833e82aa6a94885b4ec5cb54d0463ce5a3bee25ad24124ff8cc2d20 module/webhook-ingress -9f2739dfe035d9a053fcd678aa76be79980832495041b770c326ed973a979bee module/webhook-request-guards +af8c5e1c84ec9d365a7aa0d15eb63f0dd9e7b50526023e6212bba020c229f224 module/webhook-request-guards 1cc469eacda2818a116ab7b63847b9ddf86225e8d02e62fac48e618ba41a9852 module/widget-html 9161b36ec0ab062ea41b363c894fcd672a7727f21cb726739f99f9c184fce69d module/zod diff --git a/docs/nodes/talk.md b/docs/nodes/talk.md index e84198b13468..3b155150da30 100644 --- a/docs/nodes/talk.md +++ b/docs/nodes/talk.md @@ -95,34 +95,52 @@ Supported keys: `voice` / `voice_id` / `voiceId`, `model` / `model_id` / `modelI } ``` -OpenAI browser and iOS WebRTC Talk use Platform credentials in this order: -the configured realtime API key, an `openai` API-key profile, then -`OPENAI_API_KEY`. ChatGPT/Codex OAuth authenticates the subscription Codex -backend, not the public OpenAI Realtime API, and does not configure Talk, -Voice Call, or Discord realtime voice. Configure a Platform API key even when -agent turns use Codex OAuth. +OpenAI browser WebRTC Talk supports native GPT-Live through +`https://api.openai.com/v1/live`. Set `talk.realtime.model` to +`gpt-live-1-codex` (recommended) or `gpt-live-1-boulder-alpha`; `gpt-live-1` +and `gpt-live-1-mini` are not valid on this route. GPT-Live prefers a ChatGPT +OAuth subscription profile and falls back to Platform API-key auth, whose +`/v1/live` access is currently +[waitlist-gated](https://openai.com/form/gpt-live-1-in-the-api/). -| Key | Default | Notes | -| ---------------------------------------- | ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `agentId` | configured default agent | Owns Talk sessions created without an explicit agent-scoped session key. | -| `provider` | - | Active Talk TTS provider. Use `elevenlabs`, `mlx`, or `system` for macOS-local playback paths. | -| `providers..voiceId` | - | ElevenLabs falls back to `ELEVENLABS_VOICE_ID` / `SAG_VOICE_ID`, or the first available voice with an API key. | -| `speechLocale` | device default | BCP 47 locale for Android, iOS, and macOS native speech recognition, plus the iOS system-voice fallback. Apple Speech may use network services; Android also forwards the language component to realtime input transcription. | -| `providers.elevenlabs.modelId` | `eleven_v3` | | -| `providers.mlx.modelId` | `mlx-community/Soprano-80M-bf16` | | -| `providers.elevenlabs.apiKey` | - | Falls back to `ELEVENLABS_API_KEY` (or gateway shell profile if available). | -| `silenceTimeoutMs` | `700` ms macOS/Android, `900` ms iOS | Pause window before Talk sends the transcript. | -| `interruptOnSpeech` | `true` | | -| `outputFormat` | `pcm_44100` macOS/iOS, `pcm_24000` Android | Set `mp3_*` to force MP3 streaming. | -| `consultThinkingLevel` | unset | Thinking level override for the agent run behind realtime `openclaw_agent_consult` calls. | -| `consultFastMode` | unset | Fast-mode override for realtime `openclaw_agent_consult` calls. | -| `realtime.provider` | - | `openai` for WebRTC, `google` for provider WebSocket, or a bridge-only provider through Gateway relay. | -| `realtime.providers.` | - | Provider-owned realtime config. Browsers receive only ephemeral/constrained session credentials, never a standard API key. | -| `realtime.providers.openai.speakerVoice` | `alloy` | Built-in OpenAI Realtime voice id (the older `voice` key still works but is deprecated). Current `gpt-realtime-2.1` voices: `alloy`, `ash`, `ballad`, `cedar`, `coral`, `echo`, `marin`, `sage`, `shimmer`, `verse`; `marin` and `cedar` are recommended for best quality. | -| `realtime.transport` | - | `webrtc`: client-owned OpenAI WebRTC on iOS and in the browser. `provider-websocket`: browser-owned, stays on Gateway relay on iOS. `gateway-relay`: keeps provider audio on the Gateway; Android uses realtime only with this transport. | -| `realtime.brain` | - | `agent-consult` routes realtime tool calls through Gateway policy; `direct-tools` is legacy direct-tool compatibility; `none` is for transcription/external orchestration. | -| `realtime.consultRouting` | - | `provider-direct` preserves the provider's direct reply when it skips `openclaw_agent_consult`; `force-agent-consult` routes finalized user transcripts through OpenClaw instead. | -| `realtime.instructions` | - | Appends provider-facing system instructions to OpenClaw's built-in realtime prompt. | +GPT-Live accepts `alloy`, `ash`, `ballad`, `cedar`, `coral`, `echo`, `marin`, +`sage`, `shimmer`, and `verse`. A `403 Voice session access denied` response is +overloaded: an invalid voice returns the same response. The legacy +`chatgpt.com` backend route also returns `403`; OpenClaw uses the native +`api.openai.com/v1/live` route instead. + +GPT-Live is limited to browser Talk WebRTC sessions. Telephony, Voice Call, +Gateway relay, provider WebSocket transports, iOS, and Android are unsupported. +The Gateway owns the authenticated sideband and routes delegated work through +the configured OpenClaw agent; the browser receives neither the OAuth token nor +a Platform API key. + +For GA `gpt-realtime-*` browser and iOS WebRTC sessions, Platform credentials +remain required in this order: the configured realtime API key, an `openai` +API-key profile, then `OPENAI_API_KEY`. ChatGPT OAuth does not configure those +GA sessions, Voice Call, Gateway relay, or Discord realtime voice. + +| Key | Default | Notes | +| ---------------------------------------- | ------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `agentId` | configured default agent | Owns Talk sessions created without an explicit agent-scoped session key. | +| `provider` | - | Active Talk TTS provider. Use `elevenlabs`, `mlx`, or `system` for macOS-local playback paths. | +| `providers..voiceId` | - | ElevenLabs falls back to `ELEVENLABS_VOICE_ID` / `SAG_VOICE_ID`, or the first available voice with an API key. | +| `speechLocale` | device default | BCP 47 locale for Android, iOS, and macOS native speech recognition, plus the iOS system-voice fallback. Apple Speech may use network services; Android also forwards the language component to realtime input transcription. | +| `providers.elevenlabs.modelId` | `eleven_v3` | | +| `providers.mlx.modelId` | `mlx-community/Soprano-80M-bf16` | | +| `providers.elevenlabs.apiKey` | - | Falls back to `ELEVENLABS_API_KEY` (or gateway shell profile if available). | +| `silenceTimeoutMs` | `700` ms macOS/Android, `900` ms iOS | Pause window before Talk sends the transcript. | +| `interruptOnSpeech` | `true` | | +| `outputFormat` | `pcm_44100` macOS/iOS, `pcm_24000` Android | Set `mp3_*` to force MP3 streaming. | +| `consultThinkingLevel` | unset | Thinking level override for the agent run behind realtime `openclaw_agent_consult` calls. | +| `consultFastMode` | unset | Fast-mode override for realtime `openclaw_agent_consult` calls. | +| `realtime.provider` | - | `openai` for WebRTC, `google` for provider WebSocket, or a bridge-only provider through Gateway relay. | +| `realtime.providers.` | - | Provider-owned realtime config. Browsers receive only ephemeral/constrained session credentials, never a standard API key. | +| `realtime.providers.openai.speakerVoice` | `alloy` for GA; `marin` for GPT-Live | Built-in OpenAI Realtime voice id (the older `voice` key still works but is deprecated). Current `gpt-realtime-2.1` and GPT-Live voices: `alloy`, `ash`, `ballad`, `cedar`, `coral`, `echo`, `marin`, `sage`, `shimmer`, `verse`; `marin` and `cedar` are recommended for best quality. | +| `realtime.transport` | - | `webrtc`: client-owned OpenAI WebRTC on iOS and in the browser. `provider-websocket`: browser-owned, stays on Gateway relay on iOS. `gateway-relay`: keeps provider audio on the Gateway; Android uses realtime only with this transport. | +| `realtime.brain` | - | `agent-consult` routes realtime tool calls through Gateway policy; `direct-tools` is legacy direct-tool compatibility; `none` is for transcription/external orchestration. | +| `realtime.consultRouting` | - | `provider-direct` preserves the provider's direct reply when it skips `openclaw_agent_consult`; `force-agent-consult` routes finalized user transcripts through OpenClaw instead. | +| `realtime.instructions` | - | Appends provider-facing system instructions to OpenClaw's built-in realtime prompt. | `talk.catalog` exposes canonical provider ids and registry aliases, each provider's valid modes/transports/brain strategies/realtime audio formats/capability flags, and the runtime-selected readiness result. First-party Talk clients should read that catalog instead of maintaining provider aliases locally; treat an older Gateway that omits group readiness as unverified rather than definitively unconfigured. Streaming transcription providers are discovered through `talk.catalog.transcription`; the current Gateway relay uses the Voice Call streaming provider config until a dedicated Talk transcription config surface ships. diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index c5e5b0ee3305..b5a3c2aff8be 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -208,7 +208,7 @@ usage endpoint failed or returned no usable usage data. | `plugin-sdk/ssrf-runtime` | Pinned-dispatcher, SSRF-guarded fetch, SSRF error, SSRF policy helpers, and loopback/private host classification | | `plugin-sdk/secret-input` | Secret input parsing helpers | | `plugin-sdk/webhook-ingress` | Webhook request/target helpers and raw websocket/body coercion | - | `plugin-sdk/webhook-request-guards` | Request body size/timeout helpers and `runDetachedWebhookWork` for tracked post-ack processing | + | `plugin-sdk/webhook-request-guards` | Request body size/timeout helpers, canonical Gateway browser-origin acceptance via `resolveAcceptedBrowserOrigin`, and `runDetachedWebhookWork` for tracked post-ack processing | Use `isLoopbackHost(host)` when a plugin must accept only the local machine. It accepts `localhost`, IPv4 loopback literals across `127.0.0.0/8`, `::1`, bracketed IPv6, and IPv4-mapped IPv6 loopback literals. It parses IP literals rather than matching text prefixes, so a DNS name such as `127.0.0.1.evil.com` is not loopback. Use `isPrivateOrLoopbackHost(host)` only when private-network hosts such as RFC 1918 addresses are also valid. diff --git a/docs/providers/openai.md b/docs/providers/openai.md index b2fbd893ee85..87f9c6bbb28c 100644 --- a/docs/providers/openai.md +++ b/docs/providers/openai.md @@ -154,20 +154,18 @@ explicit runtime config. | Text-to-speech | `tts.provider: "openai"` / `tts` | Yes | | Batch speech-to-text | `tools.media.audio` / media understanding | Yes | | Streaming speech-to-text | Voice Call `streaming.provider: "openai"` | Yes | -| Realtime voice | Voice Call `realtime.provider: "openai"` / Control UI Talk `talk.realtime.provider: "openai"` | Yes (Platform API key) | +| Realtime voice | Voice Call `realtime.provider: "openai"` / Control UI Talk `talk.realtime.provider: "openai"` | Yes (Platform API key, or ChatGPT OAuth for browser GPT-Live) | | Embeddings | memory embedding provider | Yes | -OpenAI Realtime voice normally goes through the public **OpenAI Platform -Realtime API** and requires a Platform API key. Codex OAuth tokens authenticate -the ChatGPT Codex backend instead; they are not interchangeable with Platform -API keys for the public Realtime endpoints. +GA OpenAI Realtime voice goes through the public **OpenAI Platform Realtime +API** and requires a Platform API key. Browser GPT-Live is the exception: its +native `api.openai.com/v1/live` route prefers a ChatGPT OAuth profile and falls +back to Platform API-key auth when that account has waitlist-gated access. Platform auth is resolved in this order: configured realtime API key, `openai` -API-key profile, then `OPENAI_API_KEY`. ChatGPT/Codex OAuth can still -authenticate agent models and other explicitly supported subscription -surfaces, but it does not configure Talk, Voice Call, Discord realtime voice, -or realtime transcription. +API-key profile, then `OPENAI_API_KEY`. ChatGPT OAuth does not configure GA +Talk, Voice Call, Discord realtime voice, or realtime transcription. If API-key auth reports missing billing, top up Platform credits at [platform.openai.com/account/billing](https://platform.openai.com/account/billing) @@ -909,19 +907,63 @@ compatibility fallback when the shared Set the model explicitly to `gpt-realtime-2.1-mini` when you prefer the smaller, lower-cost Realtime 2.1 variant. - - **GPT-Live (upcoming).** OpenAI's full-duplex `gpt-live-1` and - `gpt-live-1-mini` models replaced ChatGPT voice mode in July 2026; the - developer API is rolling out to early-access organizations. OpenClaw - recognizes the model family but does not run it yet: GPT-Live sessions are - WebRTC-only, own their turn-taking (no VAD), and delegate agent work - through a handoff event protocol that OpenClaw's realtime transports do - not implement yet. Configuring a `gpt-live-*` model fails closed with - guidance on both the WebSocket bridge and Talk browser sessions instead of - silently connecting audio without agent access. API access is also gated - per OpenAI organization during early access. Keep `gpt-realtime-2.1` (the - default) until GPT-Live support lands. - + #### GPT-Live browser Talk + + GPT-Live is supported for browser Talk WebRTC sessions using a ChatGPT + OAuth subscription profile. The complete path was verified on 2026-07-28 + with a ChatGPT Pro account: call creation returned `201 Created`, and the + authenticated sideband emitted `session.started` for the same `rtc_*` call. + + Use `gpt-live-1-codex` (recommended) or + `gpt-live-1-boulder-alpha`. The values `gpt-live-1` and + `gpt-live-1-mini` are not valid on this route. Opt in explicitly with + `talk.realtime.model`; `gpt-realtime-2.1` remains the GA default. + + GPT-Live accepts these voices: `alloy`, `ash`, `ballad`, `cedar`, `coral`, + `echo`, `marin`, `sage`, `shimmer`, and `verse`. OpenClaw defaults to + `marin` and maps unknown or unsupported configured voices back to it. + + ```json5 + { + talk: { + realtime: { + provider: "openai", + model: "gpt-live-1-codex", + transport: "webrtc", + }, + }, + } + ``` + + + Platform API-key access to `/v1/live` is waitlist-gated and commonly returns + `400 model_not_found` without enrollment. Use a ChatGPT OAuth profile, or request Platform access with the + [GPT-Live API access form](https://openai.com/form/gpt-live-1-in-the-api/). + + + A `403 Voice session access denied` response is overloaded and does not by + itself prove an account entitlement problem: an invalid voice produces the + same response. First verify the model and voice against the accepted lists + above, then verify that the selected ChatGPT OAuth profile and + `chatgpt-account-id` belong to the same account. + + GPT-Live remains limited to browser Talk WebRTC sessions. Voice + Call/telephony, Gateway relay, provider WebSocket transports, and Android + are unsupported. The Gateway owns the authenticated sideband and routes + delegated work through the configured OpenClaw agent; the browser never + receives the OAuth token. + + The canonical OpenClaw path creates the call on `api.openai.com/v1/live` + and joins its sideband there. The legacy `chatgpt.com` backend route returns + `403` and is not used. + + Maintainers can exercise OpenClaw's complete OAuth path with the opt-in + live test. It skips when no ChatGPT OAuth credential is available and + never prints token material: + + ```bash + OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_GPT_LIVE=1 node scripts/run-vitest.mjs run --config test/vitest/vitest.live.config.ts extensions/openai/realtime-quicksilver.live.test.ts + ``` Backend OpenAI realtime bridges use the GA Realtime WebSocket session @@ -939,13 +981,17 @@ compatibility fallback when the shared - Control UI Talk uses OpenAI browser realtime sessions with a Gateway- - minted ephemeral client secret and a direct browser WebRTC SDP exchange - against the OpenAI Realtime API when using Platform credentials. The - Gateway mints that client secret with the selected `openai` credential. + Control UI Talk uses OpenAI browser WebRTC sessions. GA + `gpt-realtime-*` models use a Gateway-minted ephemeral client secret and a + direct browser SDP exchange when Platform credentials are available. Configured realtime keys, API-key profiles, and `OPENAI_API_KEY` use that - path in that order. ChatGPT/Codex OAuth is not a Platform Realtime - credential and is not used as a fallback. + path in that order. ChatGPT OAuth is not a Platform Realtime credential and + is not used for GA models. GPT-Live instead uses the native Gateway offer + broker described above, prefers ChatGPT OAuth when both auth modes are + configured, and falls back to Platform API-key access when the account has + waitlist-gated `/v1/live` access. + Gateway relay and Voice Call backend realtime WebSocket bridges continue to + require Platform credentials and a GA model. Maintainer live verification is available with `OPENAI_API_KEY=... GEMINI_API_KEY=... node --import tsx scripts/dev/realtime-talk-live-smoke.ts`; the OpenAI legs verify both the backend WebSocket bridge and the browser diff --git a/docs/web/control-ui.md b/docs/web/control-ui.md index 632b033b589b..312db97b0cd8 100644 --- a/docs/web/control-ui.md +++ b/docs/web/control-ui.md @@ -456,13 +456,13 @@ The macOS app keeps its native link-browser sidebar for links clicked in the das - Talk mode uses a registered realtime voice provider. Configure OpenAI with `talk.realtime.provider: "openai"`. Browser WebRTC uses Platform auth in this order: `talk.realtime.providers.openai.apiKey`, an `openai` API-key profile, then `OPENAI_API_KEY`. When none exists and the bundled Codex runtime is active, it automatically falls back to the Gateway's logged-in ChatGPT/Codex subscription; OpenAI OAuth/Codex agent sessions activate that runtime without another Talk auth setting. This experimental fallback does not support Gateway relay or backend voice bridges. Configure Google with `talk.realtime.provider: "google"` plus `talk.realtime.providers.google.apiKey`. The browser never receives a standard provider API key or the Codex OAuth token: Platform OpenAI receives an ephemeral Realtime client secret, Codex OAuth receives a one-use Gateway reservation for a Codex app-server WebRTC exchange, and Google Live receives a one-use constrained Live API auth token for a browser WebSocket session. Providers that only expose a backend realtime bridge run through the Gateway relay transport, so credentials and vendor sockets stay server-side while browser audio moves through authenticated Gateway RPCs. Platform sessions use the Gateway's direct-tool prompt. Codex OAuth preserves Codex's native prompt, model selection, and agent handoff while adding configured Talk instructions as developer context. `talk.client.create` does not accept caller-provided instruction overrides. + Talk mode uses a registered realtime voice provider. Configure OpenAI with `talk.realtime.provider: "openai"`. GA `gpt-realtime-*` browser WebRTC uses Platform auth in this order: `talk.realtime.providers.openai.apiKey`, an `openai` API-key profile, then `OPENAI_API_KEY`. Native GPT-Live uses `api.openai.com/v1/live` through the Gateway offer broker and prefers a ChatGPT OAuth subscription profile over Platform auth; Platform API-key access remains waitlist-gated. GPT-Live is browser-only and does not support Gateway relay or backend voice bridges. Configure Google with `talk.realtime.provider: "google"` plus `talk.realtime.providers.google.apiKey`. The browser never receives a standard provider API key or a ChatGPT OAuth token: Platform GA OpenAI receives an ephemeral Realtime client secret, native GPT-Live receives a one-use Gateway reservation, and Google Live receives a one-use constrained Live API auth token for a browser WebSocket session. Providers that only expose a backend realtime bridge run through the Gateway relay transport, so credentials and vendor sockets stay server-side while browser audio moves through authenticated Gateway RPCs. Platform GA sessions use the Gateway's direct-tool prompt, while GPT-Live delegates through its Gateway-owned sideband. `talk.client.create` does not accept caller-provided instruction overrides. - Persistent provider, model, voice, transport, reasoning effort, exact VAD threshold, silence duration, and prefix padding defaults live in **Settings → Communications → Talk**; changing them requires `operator.admin` access. Codex OAuth owns its model and does not apply the Platform model, reasoning, VAD, or camera controls. Configuring Gateway relay forces the backend relay path; configuring WebRTC keeps the session client-owned and fails instead of silently falling back to relay if the provider cannot create a browser session. + Persistent provider, model, voice, transport, reasoning effort, exact VAD threshold, silence duration, and prefix padding defaults live in **Settings → Communications → Talk**; changing them requires `operator.admin` access. Configuring Gateway relay forces the backend relay path; configuring WebRTC keeps the session client-owned and fails instead of silently falling back to relay if the provider cannot create a browser session. The Talk control itself is the microphone button in the composer toolbar. Its caret lists **System default** and every microphone exposed by the browser, including USB, Bluetooth, and virtual inputs. The selected device ID stays browser-local and is never sent to the Gateway; if that exact device disappears, Talk asks you to choose another input instead of silently recording from a different microphone. While Talk is live, the microphone button becomes a pill showing the live input-level meter; clicking it stops voice input, and hovering it reveals the stop glyph. Screen readers announce `Connecting voice input...`, `Listening...`, or `Asking OpenClaw...` while a realtime tool call is consulting the configured larger model through `talk.client.toolCall`. Stopping a running agent response stays a separate square **Stop** control next to the pill. - **Video Talk** is available for OpenAI Platform Realtime WebRTC and Google Live browser sessions; Codex OAuth Talk is audio-only. Click the camera button, allow camera and microphone access, and confirm the local preview. OpenAI sends one bounded JPEG frame over its browser data channel when `describe_view` requests visual context. Google Live sends bounded JPEG frames directly from the browser to the provider at the supported maximum of one frame per second and answers `describe_view` function calls with the camera-stream state. Camera frames never pass through the Gateway. Stopping Talk closes the preview and releases both media tracks. See Google's [Live API capabilities](https://ai.google.dev/gemini-api/docs/live-api/capabilities#video) and [function-calling guide](https://ai.google.dev/gemini-api/docs/live-api/tools) for the provider wire contracts. + **Video Talk** is available for OpenAI Platform Realtime WebRTC and Google Live browser sessions; GPT-Live is audio-only. Click the camera button, allow camera and microphone access, and confirm the local preview. OpenAI sends one bounded JPEG frame over its browser data channel when `describe_view` requests visual context. Google Live sends bounded JPEG frames directly from the browser to the provider at the supported maximum of one frame per second and answers `describe_view` function calls with the camera-stream state. Camera frames never pass through the Gateway. Stopping Talk closes the preview and releases both media tracks. See Google's [Live API capabilities](https://ai.google.dev/gemini-api/docs/live-api/capabilities#video) and [function-calling guide](https://ai.google.dev/gemini-api/docs/live-api/tools) for the provider wire contracts. Maintainer live smoke: `OPENAI_API_KEY=... GEMINI_API_KEY=... node --import tsx scripts/dev/realtime-talk-live-smoke.ts` verifies the OpenAI backend WebSocket bridge, OpenAI browser WebRTC SDP exchange, Google Live constrained-token browser setup with a JPEG frame and `describe_view` function roundtrip, and the Gateway relay browser adapter with fake microphone media. The command prints provider status only and does not log secrets. diff --git a/extensions/openai/index.test.ts b/extensions/openai/index.test.ts index 62b4f543c67f..6366a171bd0f 100644 --- a/extensions/openai/index.test.ts +++ b/extensions/openai/index.test.ts @@ -156,6 +156,35 @@ describe("openai plugin", () => { vi.restoreAllMocks(); }); + it("registers the native GPT-Live offer route and cleanup lifecycle", () => { + const registerHttpRoute = vi.fn(); + const registerRuntimeLifecycle = vi.fn(); + plugin.register( + createTestPluginApi({ + id: "openai", + name: "OpenAI Provider", + source: "test", + config: {}, + runtime: { config: { current: vi.fn(() => ({})) } } as never, + registerHttpRoute, + registerRuntimeLifecycle, + }), + ); + + expect(registerHttpRoute).toHaveBeenCalledWith({ + path: "/plugins/openai/realtime/calls", + auth: "plugin", + match: "exact", + handler: expect.any(Function), + }); + expect(registerRuntimeLifecycle).toHaveBeenCalledWith( + expect.objectContaining({ + id: "openai-quicksilver-realtime-browser-session", + cleanup: expect.any(Function), + }), + ); + }); + it("generates PNG buffers from the OpenAI Images API", async () => { const { resolveApiKeySpy, postJsonRequestSpy } = mockOpenAIImageApiResponse({ finalUrl: "https://api.openai.com/v1/images/generations", diff --git a/extensions/openai/index.ts b/extensions/openai/index.ts index b1a34b687b8b..34e454501043 100644 --- a/extensions/openai/index.ts +++ b/extensions/openai/index.ts @@ -1,4 +1,5 @@ // Openai plugin entrypoint registers its OpenClaw integration. +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { resolvePluginConfigObject } from "openclaw/plugin-sdk/plugin-config-runtime"; import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; import { buildProviderToolCompatFamilyHooks } from "openclaw/plugin-sdk/provider-tools"; @@ -10,6 +11,10 @@ import { resolveOpenAIPromptOverlayMode, resolveOpenAISystemPromptContribution, } from "./prompt-overlay.js"; +import { + createOpenAIQuicksilverBrowserSessionBroker, + OPENAI_QUICKSILVER_OFFER_PATH, +} from "./realtime-quicksilver-session.js"; import { buildOpenAIRealtimeTranscriptionProvider } from "./realtime-transcription-provider.js"; import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js"; import { buildOpenAISpeechProvider } from "./speech-provider.js"; @@ -20,6 +25,26 @@ export default definePluginEntry({ name: "OpenAI Provider", description: "Bundled OpenAI provider plugins", register(api) { + const quicksilverSession = + api.registrationMode === "full" + ? createOpenAIQuicksilverBrowserSessionBroker({ + getConfig: () => api.runtime.config.current() as OpenClawConfig, + logger: api.logger, + }) + : undefined; + if (quicksilverSession) { + api.registerHttpRoute({ + path: OPENAI_QUICKSILVER_OFFER_PATH, + auth: "plugin", + match: "exact", + handler: quicksilverSession.handler, + }); + api.lifecycle.registerRuntimeLifecycle({ + id: "openai-quicksilver-realtime-browser-session", + description: "Close GPT-Live browser sidebands when the OpenAI plugin stops", + cleanup: () => quicksilverSession.cleanup(), + }); + } const openAIToolCompatHooks = buildProviderToolCompatFamilyHooks("openai"); const buildProviderWithPromptContribution = >( provider: T, @@ -45,7 +70,11 @@ export default definePluginEntry({ api.registerMemoryEmbeddingProvider(openAiMemoryEmbeddingProviderAdapter); api.registerImageGenerationProvider(buildOpenAIImageGenerationProvider()); api.registerRealtimeTranscriptionProvider(buildOpenAIRealtimeTranscriptionProvider()); - api.registerRealtimeVoiceProvider(buildOpenAIRealtimeVoiceProvider()); + api.registerRealtimeVoiceProvider( + buildOpenAIRealtimeVoiceProvider({ + quicksilverBrowserSessionBroker: quicksilverSession?.broker, + }), + ); api.registerSpeechProvider(buildOpenAISpeechProvider()); api.registerMediaUnderstandingProvider(openaiMediaUnderstandingProvider); api.registerVideoGenerationProvider(buildOpenAIVideoGenerationProvider()); diff --git a/extensions/openai/package.json b/extensions/openai/package.json index 62f6c5244f39..ad523740e9e8 100644 --- a/extensions/openai/package.json +++ b/extensions/openai/package.json @@ -5,7 +5,8 @@ "description": "OpenClaw OpenAI provider plugins", "type": "module", "dependencies": { - "ws": "8.21.1" + "ws": "8.21.1", + "zod": "4.4.3" }, "devDependencies": { "@openclaw/plugin-sdk": "workspace:*" diff --git a/extensions/openai/realtime-quicksilver-delegation.test.ts b/extensions/openai/realtime-quicksilver-delegation.test.ts new file mode 100644 index 000000000000..078346e3fae7 --- /dev/null +++ b/extensions/openai/realtime-quicksilver-delegation.test.ts @@ -0,0 +1,435 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + chunkOpenAIQuicksilverAppendText, + parseOpenAIQuicksilverEvent, +} from "./realtime-quicksilver-wire.js"; +import { + createRequest, + createResponseHarness, + parseSent, + emitSideband, + createBroker, +} from "./realtime-quicksilver.test-helpers.js"; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); +}); + +describe("GPT-Live sideband protocol", () => { + it.each(["session.updated", "output_audio.delta"])("ignores %s server-side", (type) => { + expect(parseOpenAIQuicksilverEvent(JSON.stringify({ type }))).toEqual({ + kind: "ignored", + eventType: type, + }); + }); + + it("parses session expiry and transcript events", () => { + expect( + parseOpenAIQuicksilverEvent( + JSON.stringify({ type: "session.started", session: { expires_at: 123 } }), + ), + ).toEqual({ kind: "session-started", expiresAt: 123 }); + expect( + parseOpenAIQuicksilverEvent( + JSON.stringify({ type: "input_transcript.added", item: { text: "hel" } }), + ), + ).toEqual({ kind: "transcript-delta", role: "user", text: "hel" }); + expect( + parseOpenAIQuicksilverEvent( + JSON.stringify({ type: "output_transcript.added", item: { text: "wor" } }), + ), + ).toEqual({ kind: "transcript-delta", role: "assistant", text: "wor" }); + expect( + parseOpenAIQuicksilverEvent( + JSON.stringify({ type: "turn.done", turn: { role: "user", transcript: "hello" } }), + ), + ).toEqual({ kind: "transcript-done", role: "user", text: "hello" }); + }); + + it("parses client delegations and ignores non-client targets", () => { + expect( + parseOpenAIQuicksilverEvent( + JSON.stringify({ + type: "delegation.created", + item: { + type: "delegation", + target: "client", + id: "delegation-1", + content: [ + { type: "input_text", text: "curl https://exa" }, + { type: "output_text", text: "ignored" }, + { type: "input_text", text: "mple.com" }, + ], + }, + }), + ), + ).toEqual({ kind: "delegation", id: "delegation-1", prompt: "curl https://example.com" }); + expect( + parseOpenAIQuicksilverEvent( + JSON.stringify({ + type: "delegation.created", + item: { type: "delegation", target: "server", id: "delegation-2", content: [] }, + }), + ), + ).toEqual({ kind: "ignored", eventType: "delegation.created" }); + }); + + it("parses errors, reports unknown events, and rejects malformed JSON", () => { + expect( + parseOpenAIQuicksilverEvent( + JSON.stringify({ type: "error", error: { message: "call failed" } }), + ), + ).toEqual({ kind: "error", message: "call failed", fatalAuth: false }); + expect( + parseOpenAIQuicksilverEvent( + JSON.stringify({ + type: "error", + message: "top-level failure", + error: { message: "nested failure" }, + }), + ), + ).toEqual({ kind: "error", message: "top-level failure", fatalAuth: false }); + expect( + parseOpenAIQuicksilverEvent( + JSON.stringify({ type: "error", error: { code: "invalid_token" } }), + ), + ).toEqual({ + kind: "error", + message: '{"code":"invalid_token"}', + fatalAuth: true, + }); + expect(parseOpenAIQuicksilverEvent(JSON.stringify({ type: "future.event" }))).toEqual({ + kind: "unknown", + eventType: "future.event", + }); + expect(parseOpenAIQuicksilverEvent("not-json")).toBeNull(); + }); + + it("chunks appends by UTF-8 bytes without splitting characters", () => { + const text = `${"a".repeat(499)}🙂${"b".repeat(501)}`; + const chunks = chunkOpenAIQuicksilverAppendText(text); + expect(chunks.join("")).toBe(text); + expect(chunks.length).toBeGreaterThan(1); + for (const chunk of chunks) { + expect(Buffer.byteLength(chunk, "utf8")).toBeLessThanOrEqual(500); + } + }); + + it("wraps delegated input and appends the raw speakable result", async () => { + const runAgentConsult = vi.fn(async ({ prompt }: { prompt: string }) => ({ + text: `Result for ${prompt}`, + })); + const { realtime, sockets } = createBroker({ runAgentConsult }); + try { + const reservation = await realtime.broker.createBrowserSession( + { + providerConfig: {}, + model: "gpt-live-1", + runAgentConsult, + }, + { type: "api-key", token: "platform-key" }, + ); + if (reservation.transport !== "webrtc") { + throw new Error("Expected WebRTC reservation"); + } + const response = createResponseHarness(); + await realtime.handler(createRequest({ token: reservation.clientSecret }), response.res); + const socket = sockets[0]; + if (!socket) { + throw new Error("Expected sideband socket"); + } + expect(socket.sent).toEqual([]); + socket.emit( + "message", + Buffer.from( + JSON.stringify({ + type: "delegation.created", + item: { + type: "delegation", + target: "client", + id: "delegation-1", + content: [ + { type: "input_text", text: "first " }, + { type: "input_text", text: "task" }, + ], + }, + }), + ), + false, + ); + await vi.waitFor(() => + expect(runAgentConsult).toHaveBeenCalledWith({ + prompt: "\n first task\n", + signal: expect.any(AbortSignal), + }), + ); + await vi.waitFor(() => + expect(parseSent(socket)).toContainEqual({ + type: "delegation.context.append", + delegation_item_id: "delegation-1", + channel: "speakable", + content: [ + { + type: "input_text", + text: "Result for \n first task\n", + }, + ], + }), + ); + await realtime.cleanup(); + expect(parseSent(socket).at(-1)).toEqual({ type: "session.close" }); + expect(socket.closed).toBe(true); + } finally { + await realtime.cleanup(); + } + }); + + it("adds the in-call transcript delta to each delegation and resets it", async () => { + const runAgentConsult = vi.fn(async (_params: { prompt: string; signal?: AbortSignal }) => ({ + text: "Done", + })); + const { realtime, sockets } = createBroker({ runAgentConsult }); + try { + const reservation = await realtime.broker.createBrowserSession( + { providerConfig: {}, model: "gpt-live-1-codex", runAgentConsult }, + { type: "api-key", token: "platform-key" }, + ); + if (reservation.transport !== "webrtc") { + throw new Error("Expected WebRTC reservation"); + } + await realtime.handler( + createRequest({ token: reservation.clientSecret }), + createResponseHarness().res, + ); + const socket = sockets[0]; + if (!socket) { + throw new Error("Expected sideband socket"); + } + + emitSideband(socket, { type: "input_transcript.added", item: { text: "hel" } }); + emitSideband(socket, { + type: "turn.done", + turn: { role: "user", transcript: "hello" }, + }); + emitSideband(socket, { type: "output_transcript.added", item: { text: "ack" } }); + emitSideband(socket, { + type: "delegation.created", + item: { + type: "delegation", + target: "client", + id: "delegation-1", + content: [{ type: "input_text", text: "check weather" }], + }, + }); + await vi.waitFor(() => expect(runAgentConsult).toHaveBeenCalledTimes(1)); + expect(runAgentConsult.mock.calls[0]?.[0]?.prompt).toBe( + "\n check weather\n user: hello\nassistant: ack\n", + ); + + emitSideband(socket, { + type: "turn.done", + turn: { role: "user", transcript: "second context" }, + }); + emitSideband(socket, { + type: "delegation.created", + item: { + type: "delegation", + target: "client", + id: "delegation-2", + content: [{ type: "input_text", text: "next task" }], + }, + }); + await vi.waitFor(() => expect(runAgentConsult).toHaveBeenCalledTimes(2)); + expect(runAgentConsult.mock.calls[1]?.[0]?.prompt).toBe( + "\n next task\n user: second context\n", + ); + } finally { + await realtime.cleanup(); + } + }); + + it("aborts the in-flight consult when a newer delegation arrives", async () => { + const signals: AbortSignal[] = []; + const resolutions: Array<(value: { text: string }) => void> = []; + const runAgentConsult = vi.fn( + ({ signal }: { prompt: string; signal?: AbortSignal }) => + new Promise<{ text: string }>((resolve) => { + if (signal) { + signals.push(signal); + } + resolutions.push(resolve); + }), + ); + const { realtime, sockets } = createBroker({ runAgentConsult }); + try { + const reservation = await realtime.broker.createBrowserSession( + { providerConfig: {}, model: "gpt-live-1-codex", runAgentConsult }, + { type: "api-key", token: "platform-key" }, + ); + if (reservation.transport !== "webrtc") { + throw new Error("Expected WebRTC reservation"); + } + await realtime.handler( + createRequest({ token: reservation.clientSecret }), + createResponseHarness().res, + ); + const socket = sockets[0]; + if (!socket) { + throw new Error("Expected sideband socket"); + } + + for (const [id, text] of [ + ["delegation-1", "first"], + ["delegation-2", "second"], + ] as const) { + emitSideband(socket, { + type: "delegation.created", + item: { + type: "delegation", + target: "client", + id, + content: [{ type: "input_text", text }], + }, + }); + await vi.waitFor(() => + expect(runAgentConsult).toHaveBeenCalledTimes(id.endsWith("1") ? 1 : 2), + ); + } + + expect(signals[0]?.aborted).toBe(true); + expect(signals[1]?.aborted).toBe(false); + resolutions[0]?.({ text: "stale" }); + resolutions[1]?.({ text: "fresh" }); + await vi.waitFor(() => + expect(parseSent(socket)).toContainEqual( + expect.objectContaining({ + delegation_item_id: "delegation-2", + channel: "speakable", + }), + ), + ); + expect(parseSent(socket)).not.toContainEqual( + expect.objectContaining({ delegation_item_id: "delegation-1" }), + ); + } finally { + await realtime.cleanup(); + } + }); + + it("skips empty delegations and keeps their transcript for the next one", async () => { + const runAgentConsult = vi.fn(async (_params: { prompt: string; signal?: AbortSignal }) => ({ + text: "Done", + })); + const { realtime, sockets } = createBroker({ runAgentConsult }); + try { + const reservation = await realtime.broker.createBrowserSession( + { providerConfig: {}, model: "gpt-live-1-codex", runAgentConsult }, + { type: "api-key", token: "platform-key" }, + ); + if (reservation.transport !== "webrtc") { + throw new Error("Expected WebRTC reservation"); + } + await realtime.handler( + createRequest({ token: reservation.clientSecret }), + createResponseHarness().res, + ); + const socket = sockets[0]; + if (!socket) { + throw new Error("Expected sideband socket"); + } + emitSideband(socket, { + type: "turn.done", + turn: { role: "user", transcript: "hello" }, + }); + emitSideband(socket, { + type: "delegation.created", + item: { + type: "delegation", + target: "client", + id: "empty", + content: [{ type: "input_text", text: " " }], + }, + }); + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); + expect(runAgentConsult).not.toHaveBeenCalled(); + + emitSideband(socket, { + type: "delegation.created", + item: { + type: "delegation", + target: "client", + id: "delegation-1", + content: [{ type: "input_text", text: "check weather" }], + }, + }); + await vi.waitFor(() => expect(runAgentConsult).toHaveBeenCalledTimes(1)); + expect(runAgentConsult.mock.calls[0]?.[0]?.prompt).toBe( + "\n check weather\n user: hello\n", + ); + } finally { + await realtime.cleanup(); + } + }); + + it("returns a speakable failure when the delegated agent fails", async () => { + const runAgentConsult = vi.fn(async () => { + throw new Error("workspace unavailable"); + }); + const { realtime, sockets, logger } = createBroker({ runAgentConsult }); + try { + const reservation = await realtime.broker.createBrowserSession( + { providerConfig: {}, model: "gpt-live-1", runAgentConsult }, + { type: "api-key", token: "platform-key" }, + ); + if (reservation.transport !== "webrtc") { + throw new Error("Expected WebRTC reservation"); + } + await realtime.handler( + createRequest({ token: reservation.clientSecret }), + createResponseHarness().res, + ); + const socket = sockets[0]; + if (!socket) { + throw new Error("Expected sideband socket"); + } + socket.emit( + "message", + Buffer.from( + JSON.stringify({ + type: "delegation.created", + item: { + type: "delegation", + target: "client", + id: "delegation-failed", + content: [{ type: "input_text", text: "do work" }], + }, + }), + ), + false, + ); + await vi.waitFor(() => { + expect(parseSent(socket)).toContainEqual( + expect.objectContaining({ + type: "delegation.context.append", + delegation_item_id: "delegation-failed", + channel: "speakable", + content: [ + { + type: "input_text", + text: "The agent task failed. Tell the user it did not complete and offer to try again.", + }, + ], + }), + ); + }); + // The raw failure detail must stay in Gateway logs, never on the provider sideband. + expect(JSON.stringify(parseSent(socket))).not.toContain("workspace unavailable"); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining("workspace unavailable")); + } finally { + await realtime.cleanup(); + } + }); +}); diff --git a/extensions/openai/realtime-quicksilver-instructions.ts b/extensions/openai/realtime-quicksilver-instructions.ts new file mode 100644 index 000000000000..a0bd382da641 --- /dev/null +++ b/extensions/openai/realtime-quicksilver-instructions.ts @@ -0,0 +1,37 @@ +const OPENAI_QUICKSILVER_DELEGATION_INSTRUCTIONS = `You are OpenClaw's realtime voice layer. You have no tools of your own. +Delegate any request that requires real work, reasoning, current information, or actions to the client through a delegation. +Keep the conversation natural while delegated work runs. +Context on the commentary channel is silent background. You may use it, but never read it aloud. +Context on the speakable channel is your answer to deliver naturally in your own words. Never mention the channel or the delegation.`; + +export type OpenAIQuicksilverTranscriptEntry = { + role: "user" | "assistant"; + text: string; +}; + +export function buildOpenAIQuicksilverInstructions(operatorInstructions?: string): string { + const operator = operatorInstructions?.trim(); + return operator + ? `${OPENAI_QUICKSILVER_DELEGATION_INSTRUCTIONS}\n\n${operator}` + : OPENAI_QUICKSILVER_DELEGATION_INSTRUCTIONS; +} + +function escapeXmlText(value: string): string { + return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">"); +} + +export function buildOpenAIQuicksilverDelegationPrompt(params: { + input: string; + transcript: readonly OpenAIQuicksilverTranscriptEntry[]; +}): string { + const input = escapeXmlText(params.input); + const transcript = params.transcript + .map((entry) => ({ role: entry.role, text: entry.text.trim() })) + .filter((entry) => entry.text.length > 0) + .map((entry) => `${entry.role}: ${entry.text}`) + .join("\n"); + const transcriptElement = transcript + ? `\n ${escapeXmlText(transcript)}` + : ""; + return `\n ${input}${transcriptElement}\n`; +} diff --git a/extensions/openai/realtime-quicksilver-session.test.ts b/extensions/openai/realtime-quicksilver-session.test.ts new file mode 100644 index 000000000000..51b5d92d3be1 --- /dev/null +++ b/extensions/openai/realtime-quicksilver-session.test.ts @@ -0,0 +1,640 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { OPENAI_QUICKSILVER_OFFER_PATH } from "./realtime-quicksilver-session.js"; +import { buildOpenAIQuicksilverSession } from "./realtime-quicksilver-wire.js"; +import { + FakeSocket, + createRequest, + createPreflightRequest, + createResponseHarness, + createCallResponse, + emitSideband, + createBroker, +} from "./realtime-quicksilver.test-helpers.js"; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); +}); + +describe("GPT-Live session shaping", () => { + it("maps initial roles and normalizes voices without an id field", () => { + expect( + buildOpenAIQuicksilverSession({ + model: "gpt-live-1", + instructions: " Speak briefly. ", + voice: "CEDAR", + initialItems: [ + { role: "user", text: "Question" }, + { role: "assistant", text: "Answer" }, + ], + }), + ).toEqual({ + model: "gpt-live-1", + instructions: "Speak briefly.", + audio: { output: { voice: "cedar" } }, + delegation: { type: "client" }, + initial_items: [ + { + type: "message", + role: "user", + content: [{ type: "input_text", text: "Question" }], + }, + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "Answer" }], + }, + ], + }); + expect( + buildOpenAIQuicksilverSession({ + model: "gpt-live-1-mini", + voice: "not-a-live-voice", + initialItems: [], + }), + ).toEqual({ + model: "gpt-live-1-mini", + instructions: "", + audio: { output: { voice: "marin" } }, + delegation: { type: "client" }, + }); + }); + + it.each([ + "alloy", + "ash", + "ballad", + "cedar", + "coral", + "echo", + "marin", + "sage", + "shimmer", + "verse", + ])("accepts the live-proven %s voice", (voice) => { + expect(buildOpenAIQuicksilverSession({ model: "gpt-live-1-codex", voice }).audio).toEqual({ + output: { voice }, + }); + }); + + it.each(["arbor", "breeze", "cove", "ember", "juniper", "maple", "sol", "spruce", "vale"])( + "falls back from the rejected %s voice", + (voice) => { + expect(buildOpenAIQuicksilverSession({ model: "gpt-live-1-codex", voice }).audio).toEqual({ + output: { voice: "marin" }, + }); + }, + ); + + it("bounds initial items to the newest context", () => { + const session = buildOpenAIQuicksilverSession({ + model: "gpt-live-1-codex", + initialItems: Array.from({ length: 20 }, (_, index) => ({ + role: index % 2 === 0 ? ("user" as const) : ("assistant" as const), + text: `${index}:${"x".repeat(1_000)}`, + })), + }); + + expect(session.initial_items).toHaveLength(10); + expect(session.initial_items?.[0]?.content[0]?.text).toMatch(/^10:/); + expect(session.initial_items?.at(-1)?.content[0]?.text).toMatch(/^19:/); + expect(session.initial_items?.every((item) => item.content[0]?.text.length === 800)).toBe(true); + }); +}); + +describe("GPT-Live offer broker", () => { + it.each([ + { + name: "OAuth", + auth: { type: "oauth" as const, token: "oauth-token", accountId: "account-123" }, + authorization: "Bearer oauth-token", + accountId: "account-123", + }, + { + name: "API key", + auth: { type: "api-key" as const, token: "platform-key" }, + authorization: "Bearer platform-key", + accountId: undefined, + }, + ])("uses matching $name headers on signaling and the API sideband", async (authCase) => { + vi.stubEnv("OPENCLAW_VERSION", "2026.7.2-test"); + let signalingUrl: string | undefined; + let signalingHeaders: Record | undefined; + const fetchImpl = vi.fn(async (url: string | URL | Request, init?: RequestInit) => { + signalingUrl = typeof url === "string" ? url : url instanceof URL ? url.href : url.url; + signalingHeaders = init?.headers as Record | undefined; + return createCallResponse("v=answer\r\n", "rtc_header-parity"); + }) as unknown as typeof fetch; + const { realtime, socketRequests } = createBroker({ fetchImpl }); + try { + const reservation = await realtime.broker.createBrowserSession( + { + providerConfig: {}, + model: "gpt-live-1", + runAgentConsult: vi.fn(async () => ({ text: "Done" })), + }, + authCase.auth, + ); + if (reservation.transport !== "webrtc") { + throw new Error("Expected WebRTC reservation"); + } + await realtime.handler( + createRequest({ token: reservation.clientSecret }), + createResponseHarness().res, + ); + + const sideband = socketRequests[0]; + expect(signalingUrl).toBe("https://api.openai.com/v1/live"); + expect(signalingUrl).not.toContain("?"); + expect(sideband?.url).toBe("wss://api.openai.com/v1/live/rtc_header-parity"); + expect(signalingHeaders).toMatchObject({ + Authorization: authCase.authorization, + "OpenAI-Alpha": "quicksilver=v2", + "User-Agent": "openclaw/2026.7.2-test", + originator: "openclaw", + version: "2026.7.2-test", + "session-id": expect.any(String), + "thread-id": expect.any(String), + "x-session-id": expect.any(String), + "Content-Type": expect.stringMatching(/^multipart\/form-data; boundary=/), + }); + expect(sideband?.headers).toMatchObject({ + Authorization: authCase.authorization, + "OpenAI-Alpha": "quicksilver=v2", + "User-Agent": "openclaw/2026.7.2-test", + originator: "openclaw", + version: "2026.7.2-test", + "session-id": signalingHeaders?.["session-id"], + "thread-id": signalingHeaders?.["thread-id"], + "x-session-id": signalingHeaders?.["x-session-id"], + }); + expect(signalingHeaders?.["session-id"]).not.toBe(signalingHeaders?.["x-session-id"]); + expect(signalingHeaders?.["thread-id"]).not.toBe(signalingHeaders?.["x-session-id"]); + expect(signalingHeaders?.["thread-id"]).not.toBe(signalingHeaders?.["session-id"]); + if (authCase.accountId) { + expect(signalingHeaders?.["chatgpt-account-id"]).toBe(authCase.accountId); + expect(sideband?.headers?.["chatgpt-account-id"]).toBe(authCase.accountId); + } else { + expect(signalingHeaders).not.toHaveProperty("chatgpt-account-id"); + expect(sideband?.headers).not.toHaveProperty("chatgpt-account-id"); + } + } finally { + await realtime.cleanup(); + } + }); + + it("survives a connecting socket that errors during retry teardown", async () => { + // Regression: ws emits `error` asynchronously when a CONNECTING socket is closed. + // Without a retained listener that is an unhandled EventEmitter error and kills the + // Gateway process, so the retry must keep swallowing errors on discarded sockets. + class ErrorOnCloseSocket extends FakeSocket { + constructor() { + super("manual"); + } + override close(code?: number, reason?: string): void { + queueMicrotask(() => this.emit("error", new Error("socket hang up"))); + super.close(code, reason); + } + } + const { realtime, sockets } = createBroker({ + socketFactory: (attempt) => (attempt < 1 ? new ErrorOnCloseSocket() : new FakeSocket("open")), + }); + try { + const reservation = await realtime.broker.createBrowserSession( + { + providerConfig: {}, + model: "gpt-live-1-codex", + runAgentConsult: vi.fn(async () => ({ text: "Done" })), + }, + { type: "api-key", token: "platform-key" }, + ); + if (reservation.transport !== "webrtc") { + throw new Error("Expected WebRTC reservation"); + } + const response = createResponseHarness(); + await realtime.handler(createRequest({ token: reservation.clientSecret }), response.res); + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); + + expect(response.res.statusCode).toBe(200); + expect(sockets[0]?.listenerCount("error")).toBeGreaterThan(0); + } finally { + await realtime.cleanup(); + } + }); + + it("retries transient sideband startup failures", async () => { + const { realtime, sockets } = createBroker({ + socketFactory: (attempt) => new FakeSocket(attempt < 2 ? "error" : "open"), + }); + try { + const reservation = await realtime.broker.createBrowserSession( + { + providerConfig: {}, + model: "gpt-live-1-codex", + runAgentConsult: vi.fn(async () => ({ text: "Done" })), + }, + { type: "api-key", token: "platform-key" }, + ); + if (reservation.transport !== "webrtc") { + throw new Error("Expected WebRTC reservation"); + } + const response = createResponseHarness(); + await realtime.handler(createRequest({ token: reservation.clientSecret }), response.res); + + expect(response.res.statusCode).toBe(200); + expect(sockets).toHaveLength(3); + expect(sockets[0]?.closeCode).toBe(1000); + expect(sockets[1]?.closeCode).toBe(1000); + expect(sockets[2]?.readyState).toBe(1); + } finally { + await realtime.cleanup(); + } + }); + + it("buffers sideband messages that arrive with the open handshake", async () => { + const runAgentConsult = vi.fn(async () => ({ text: "Done" })); + const { realtime, sockets } = createBroker({ + runAgentConsult, + socketFactory: () => { + const socket = new FakeSocket("manual"); + queueMicrotask(() => { + emitSideband(socket, { + type: "delegation.created", + item: { + type: "delegation", + target: "client", + id: "early-delegation", + content: [{ type: "input_text", text: "early task" }], + }, + }); + socket.readyState = 1; + socket.emit("open"); + }); + return socket; + }, + }); + try { + const reservation = await realtime.broker.createBrowserSession( + { providerConfig: {}, model: "gpt-live-1-codex", runAgentConsult }, + { type: "api-key", token: "platform-key" }, + ); + if (reservation.transport !== "webrtc") { + throw new Error("Expected WebRTC reservation"); + } + await realtime.handler( + createRequest({ token: reservation.clientSecret }), + createResponseHarness().res, + ); + + await vi.waitFor(() => expect(runAgentConsult).toHaveBeenCalledOnce()); + expect(sockets).toHaveLength(1); + } finally { + await realtime.cleanup(); + } + }); + + it.each(["error", "close"] as const)( + "fails safely when the sideband emits %s immediately after opening", + async (terminalEvent) => { + const { realtime, sockets, logger } = createBroker({ + socketFactory: () => { + const socket = new FakeSocket("manual"); + queueMicrotask(() => { + socket.readyState = 1; + socket.emit("open"); + if (terminalEvent === "error") { + socket.emit("error", new Error("post-open failure")); + } else { + socket.readyState = 3; + socket.emit("close"); + } + }); + return socket; + }, + }); + try { + const reservation = await realtime.broker.createBrowserSession( + { + providerConfig: {}, + model: "gpt-live-1-codex", + runAgentConsult: vi.fn(async () => ({ text: "Done" })), + }, + { type: "api-key", token: "platform-key" }, + ); + if (reservation.transport !== "webrtc") { + throw new Error("Expected WebRTC reservation"); + } + const response = createResponseHarness(); + await realtime.handler(createRequest({ token: reservation.clientSecret }), response.res); + + expect(response.res.statusCode).toBe(502); + expect(response.readBody()).toContain("sideband failed during startup"); + expect(sockets).toHaveLength(1); + if (terminalEvent === "error") { + expect(logger.warn).toHaveBeenCalledWith( + "OpenAI GPT-Live sideband socket failed: post-open failure", + ); + } + } finally { + await realtime.cleanup(); + } + }, + ); + + it("keeps nonfatal error frames alive but closes on fatal auth errors", async () => { + const { realtime, sockets, logger } = createBroker(); + try { + const reservation = await realtime.broker.createBrowserSession( + { + providerConfig: {}, + model: "gpt-live-1-codex", + runAgentConsult: vi.fn(async () => ({ text: "Done" })), + }, + { type: "api-key", token: "platform-key" }, + ); + if (reservation.transport !== "webrtc") { + throw new Error("Expected WebRTC reservation"); + } + await realtime.handler( + createRequest({ token: reservation.clientSecret }), + createResponseHarness().res, + ); + const socket = sockets[0]; + if (!socket) { + throw new Error("Expected sideband socket"); + } + + emitSideband(socket, { type: "error", message: "recoverable turn failure" }); + expect(socket.closed).toBe(false); + emitSideband(socket, { type: "error", error: { code: "invalid_token" } }); + expect(socket.closed).toBe(true); + expect(socket.closeCode).toBe(1000); + expect(logger.warn).toHaveBeenCalledTimes(2); + } finally { + await realtime.cleanup(); + } + }); + + it("treats binary sideband frames as protocol failures", async () => { + const { realtime, sockets, logger } = createBroker(); + try { + const reservation = await realtime.broker.createBrowserSession( + { + providerConfig: {}, + model: "gpt-live-1-codex", + runAgentConsult: vi.fn(async () => ({ text: "Done" })), + }, + { type: "api-key", token: "platform-key" }, + ); + if (reservation.transport !== "webrtc") { + throw new Error("Expected WebRTC reservation"); + } + await realtime.handler( + createRequest({ token: reservation.clientSecret }), + createResponseHarness().res, + ); + const socket = sockets[0]; + if (!socket) { + throw new Error("Expected sideband socket"); + } + + emitSideband(socket, { binary: true }, true); + expect(socket.closed).toBe(true); + expect(logger.warn).toHaveBeenCalledWith( + "OpenAI GPT-Live sideband returned an unexpected binary frame", + ); + } finally { + await realtime.cleanup(); + } + }); + + it("uses a relative single-use offer route and enforces CORS", async () => { + const { realtime } = createBroker(); + try { + const accepted = createResponseHarness(); + await realtime.handler(createPreflightRequest("https://control.example"), accepted.res); + expect(accepted.res.statusCode).toBe(204); + expect(accepted.setHeader).toHaveBeenCalledWith( + "Access-Control-Allow-Origin", + "https://control.example", + ); + expect(accepted.setHeader).toHaveBeenCalledWith( + "Access-Control-Allow-Private-Network", + "true", + ); + + const privateOrigin = "http://192.168.1.24:18789"; + const privatePreflight = createResponseHarness(); + await realtime.handler( + createPreflightRequest(privateOrigin, "192.168.1.24:18789"), + privatePreflight.res, + ); + expect(privatePreflight.res.statusCode).toBe(204); + expect(privatePreflight.setHeader).toHaveBeenCalledWith( + "Access-Control-Allow-Origin", + privateOrigin, + ); + + const privatePost = createResponseHarness(); + await realtime.handler( + createRequest({ + token: "invalid", + origin: privateOrigin, + host: "192.168.1.24:18789", + }), + privatePost.res, + ); + expect(privatePost.res.statusCode).toBe(401); + expect(privatePost.setHeader).toHaveBeenCalledWith( + "Access-Control-Allow-Origin", + privateOrigin, + ); + + const rejected = createResponseHarness(); + await realtime.handler( + createPreflightRequest("https://untrusted.example", "192.168.1.24:18789"), + rejected.res, + ); + expect(rejected.res.statusCode).toBe(403); + + const rejectedPost = createResponseHarness(); + await realtime.handler( + createRequest({ + token: "invalid", + origin: "https://untrusted.example", + host: "192.168.1.24:18789", + }), + rejectedPost.res, + ); + expect(rejectedPost.res.statusCode).toBe(403); + + const reservation = await realtime.broker.createBrowserSession( + { + providerConfig: {}, + model: "gpt-live-1", + voice: "invalid", + runAgentConsult: vi.fn(async () => ({ text: "Done" })), + }, + { type: "api-key", token: "platform-key" }, + ); + expect(reservation).toMatchObject({ + offerUrl: OPENAI_QUICKSILVER_OFFER_PATH, + model: "gpt-live-1", + voice: "marin", + expiresAt: expect.any(Number), + }); + if (reservation.transport !== "webrtc") { + throw new Error("Expected WebRTC reservation"); + } + const first = createResponseHarness(); + await realtime.handler( + createRequest({ token: reservation.clientSecret, origin: "https://control.example" }), + first.res, + ); + expect(first.res.statusCode).toBe(200); + expect(first.readBody()).toBe("v=answer\r\n"); + + const replay = createResponseHarness(); + await realtime.handler(createRequest({ token: reservation.clientSecret }), replay.res); + expect(replay.res.statusCode).toBe(401); + } finally { + await realtime.cleanup(); + } + }); + + it("rejects expired tokens, unsupported methods, and content types", async () => { + const now = vi.spyOn(Date, "now").mockReturnValue(1_000); + const { realtime } = createBroker(); + try { + const method = createResponseHarness(); + await realtime.handler(createRequest({ method: "GET" }), method.res); + expect(method.res.statusCode).toBe(405); + + const contentType = createResponseHarness(); + await realtime.handler(createRequest({ contentType: "application/json" }), contentType.res); + expect(contentType.res.statusCode).toBe(415); + + const reservation = await realtime.broker.createBrowserSession( + { + providerConfig: {}, + model: "gpt-live-1", + runAgentConsult: vi.fn(async () => ({ text: "Done" })), + }, + { type: "api-key", token: "platform-key" }, + ); + if (reservation.transport !== "webrtc") { + throw new Error("Expected WebRTC reservation"); + } + now.mockReturnValue(61_001); + const expired = createResponseHarness(); + await realtime.handler(createRequest({ token: reservation.clientSecret }), expired.res); + expect(expired.res.statusCode).toBe(401); + } finally { + await realtime.cleanup(); + } + }); + + it("caps pending and active sessions", async () => { + const { realtime } = createBroker(); + const runAgentConsult = vi.fn(async () => ({ text: "Done" })); + try { + await Promise.all( + Array.from({ length: 8 }, () => + realtime.broker.createBrowserSession( + { providerConfig: {}, model: "gpt-live-1", runAgentConsult }, + { type: "api-key", token: "platform-key" }, + ), + ), + ); + await expect( + realtime.broker.createBrowserSession( + { providerConfig: {}, model: "gpt-live-1", runAgentConsult }, + { type: "api-key", token: "platform-key" }, + ), + ).rejects.toThrow("Too many concurrent OpenAI GPT-Live sessions"); + } finally { + await realtime.cleanup(); + } + }); + + it("releases a reservation after an empty SDP offer", async () => { + const { realtime } = createBroker(); + const runAgentConsult = vi.fn(async () => ({ text: "Done" })); + try { + const reservation = await realtime.broker.createBrowserSession( + { providerConfig: {}, model: "gpt-live-1", runAgentConsult }, + { type: "api-key", token: "platform-key" }, + ); + if (reservation.transport !== "webrtc") { + throw new Error("Expected WebRTC reservation"); + } + const response = createResponseHarness(); + await realtime.handler( + createRequest({ token: reservation.clientSecret, body: " " }), + response.res, + ); + expect(response.res.statusCode).toBe(400); + + await expect( + Promise.all( + Array.from({ length: 8 }, () => + realtime.broker.createBrowserSession( + { providerConfig: {}, model: "gpt-live-1", runAgentConsult }, + { type: "api-key", token: "platform-key" }, + ), + ), + ), + ).resolves.toHaveLength(8); + } finally { + await realtime.cleanup(); + } + }); + + it("aborts a redeemed offer when its browser session is canceled", async () => { + let upstreamSignal: AbortSignal | undefined; + const fetchImpl = vi.fn( + async (_url: string | URL | Request, init?: RequestInit): Promise => + await new Promise((_resolve, reject) => { + upstreamSignal = init?.signal ?? undefined; + const rejectAbort = () => { + const reason = upstreamSignal?.reason; + reject(reason instanceof Error ? reason : new Error("aborted")); + }; + upstreamSignal?.addEventListener("abort", rejectAbort, { once: true }); + if (upstreamSignal?.aborted) { + rejectAbort(); + } + }), + ) as unknown as typeof fetch; + const { realtime, sockets } = createBroker({ fetchImpl }); + const runAgentConsult = vi.fn(async () => ({ text: "Done" })); + try { + const reservation = await realtime.broker.createBrowserSession( + { providerConfig: {}, model: "gpt-live-1", runAgentConsult }, + { type: "api-key", token: "platform-key" }, + ); + if (reservation.transport !== "webrtc") { + throw new Error("Expected WebRTC reservation"); + } + const response = createResponseHarness(); + const handling = realtime.handler( + createRequest({ token: reservation.clientSecret }), + response.res, + ); + await vi.waitFor(() => expect(upstreamSignal).toBeDefined()); + + realtime.broker.cancelBrowserSession(reservation); + + await expect(handling).resolves.toBe(true); + expect(upstreamSignal?.aborted).toBe(true); + expect(response.res.statusCode).toBe(502); + expect(response.readBody()).toContain("GPT-Live session canceled"); + expect(response.end).toHaveBeenCalledOnce(); + expect(sockets).toEqual([]); + } finally { + await realtime.cleanup(); + } + }); +}); diff --git a/extensions/openai/realtime-quicksilver-session.ts b/extensions/openai/realtime-quicksilver-session.ts new file mode 100644 index 000000000000..a95fba2308e1 --- /dev/null +++ b/extensions/openai/realtime-quicksilver-session.ts @@ -0,0 +1,679 @@ +// Native GPT-Live browser sessions: WebRTC offer broker plus gateway-owned sideband control. +import { randomBytes, randomUUID } from "node:crypto"; +import type { IncomingMessage, ServerResponse } from "node:http"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import type { PluginLogger } from "openclaw/plugin-sdk/plugin-entry"; +import { resolveProviderAuthProfileApiKey } from "openclaw/plugin-sdk/provider-auth"; +import type { + RealtimeVoiceBrowserSession, + RealtimeVoiceBrowserSessionCreateRequest, + RealtimeVoiceProviderCapabilities, +} from "openclaw/plugin-sdk/realtime-voice"; +import { + readRequestBodyWithLimit, + resolveAcceptedBrowserOrigin, +} from "openclaw/plugin-sdk/webhook-request-guards"; +import WebSocket, { type RawData } from "ws"; +import { resolveCodexAuthIdentity } from "./openai-chatgpt-auth-identity.js"; +import { + buildOpenAIQuicksilverDelegationPrompt, + type OpenAIQuicksilverTranscriptEntry, +} from "./realtime-quicksilver-instructions.js"; +import { + connectOpenAIQuicksilverSideband, + type OpenAIQuicksilverSocket, + type OpenAIQuicksilverSocketFactory, +} from "./realtime-quicksilver-sideband.js"; +import { + boundOpenAIQuicksilverContextItems, + buildOpenAIQuicksilverSession, + chunkOpenAIQuicksilverAppendText, + createOpenAIQuicksilverCall, + parseOpenAIQuicksilverEvent, + resolveOpenAIQuicksilverVoice, + type OpenAIQuicksilverAuth, + type OpenAIQuicksilverInboundEvent, + type OpenAIQuicksilverInitialItem, + type OpenAIQuicksilverRequestIds, +} from "./realtime-quicksilver-wire.js"; +export const OPENAI_QUICKSILVER_OFFER_PATH = "/plugins/openai/realtime/calls"; +export const OPENAI_QUICKSILVER_CAPABILITIES = { + transports: ["webrtc" as const], + handlesAgentConsult: true as const, + supportsToolCalls: false, + supportsVideoFrames: false, +} satisfies Partial & { handlesAgentConsult: true }; + +const OPENAI_QUICKSILVER_PENDING_TTL_MS = 60_000; +const OPENAI_QUICKSILVER_SESSION_TTL_MS = 30 * 60_000; +const OPENAI_QUICKSILVER_MAX_SESSIONS = 8; +const OPENAI_QUICKSILVER_MAX_SDP_BYTES = 256 * 1024; +const OPENAI_QUICKSILVER_UPSTREAM_TIMEOUT_MS = 30_000; +const WEBSOCKET_OPEN = 1; + +type OpenAIQuicksilverSessionRequest = RealtimeVoiceBrowserSessionCreateRequest & { + initialItems?: OpenAIQuicksilverInitialItem[]; +}; + +type PreparedOpenAIQuicksilverSessionRequest = OpenAIQuicksilverSessionRequest & { + model: string; + voice: string; +}; + +type PendingOffer = { + auth: OpenAIQuicksilverAuth; + expiresAt: number; + requestIds: OpenAIQuicksilverRequestIds; + request: PreparedOpenAIQuicksilverSessionRequest; +}; + +type ActiveSession = { + abortController: AbortController; + consultController?: AbortController; + socket: OpenAIQuicksilverSocket; + timer: NodeJS.Timeout; + token: string; + transcript: OpenAIQuicksilverTranscriptEntry[]; + partialTranscriptRole: "user" | "assistant" | undefined; +}; + +type ResponseDeliveryWaiter = { + result: Promise; + cancel: () => void; +}; + +function sendOpenAIQuicksilverAppend(params: { + socket: OpenAIQuicksilverSocket; + delegationItemId: string; + text: string; + channel: "speakable" | "commentary"; +}): void { + for (const chunk of chunkOpenAIQuicksilverAppendText(params.text)) { + params.socket.send( + JSON.stringify({ + type: "delegation.context.append", + delegation_item_id: params.delegationItemId, + channel: params.channel, + content: [{ type: "input_text", text: chunk }], + }), + ); + } +} + +const OPENAI_QUICKSILVER_CONSULT_FAILURE_TEXT = + "The agent task failed. Tell the user it did not complete and offer to try again."; + +function shortConsultFailureReason(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + return message.replaceAll(/\s+/g, " ").trim().slice(0, 180) || "unknown error"; +} + +function decodeQuicksilverTextFrame(data: RawData): string { + if (Array.isArray(data)) { + return Buffer.concat(data).toString("utf8"); + } + if (data instanceof ArrayBuffer) { + return Buffer.from(data).toString("utf8"); + } + return data.toString("utf8"); +} + +function appendOpenAIQuicksilverTranscript( + session: ActiveSession, + event: Extract, +): void { + const last = session.transcript.at(-1); + if (event.kind === "transcript-delta") { + if (last?.role === event.role && session.partialTranscriptRole === event.role) { + last.text += event.text; + } else { + session.transcript.push({ role: event.role, text: event.text }); + } + session.partialTranscriptRole = event.role; + } else { + if (last?.role === event.role && session.partialTranscriptRole === event.role) { + last.text = event.text; + } else { + session.transcript.push({ role: event.role, text: event.text }); + } + session.partialTranscriptRole = undefined; + } + session.transcript = boundOpenAIQuicksilverContextItems(session.transcript); +} + +function createResponseDeliveryWaiter( + res: ServerResponse, + onDelivered: () => void, +): ResponseDeliveryWaiter { + let settle!: (delivered: boolean) => void; + const result = new Promise((resolve) => { + settle = (delivered) => { + res.removeListener("finish", onFinish); + res.removeListener("close", onClose); + resolve(delivered); + }; + }); + const onFinish = () => { + onDelivered(); + settle(true); + }; + const onClose = () => settle(false); + res.once("finish", onFinish); + res.once("close", onClose); + return { result, cancel: () => settle(false) }; +} + +function respondText(res: ServerResponse, statusCode: number, body: string): void { + res.statusCode = statusCode; + res.setHeader("cache-control", "no-store"); + res.setHeader("content-type", "text/plain; charset=utf-8"); + res.setHeader("x-content-type-options", "nosniff"); + res.end(body); +} + +function applyRealtimeOfferCorsHeaders( + req: IncomingMessage, + res: ServerResponse, + cfg: OpenClawConfig | undefined, +): boolean { + if (!req.headers.origin) { + return true; + } + const origin = resolveAcceptedBrowserOrigin({ req, cfg }); + if (!origin) { + return false; + } + res.setHeader("Access-Control-Allow-Origin", origin); + res.setHeader("Vary", "Origin"); + return true; +} + +function readBearerToken(req: IncomingMessage): string | undefined { + const authorization = req.headers.authorization?.trim(); + return authorization?.match(/^Bearer\s+([^\s]+)$/i)?.[1]; +} + +export async function resolveOpenAIChatGptSubscriptionAuth(params: { + cfg?: OpenClawConfig; + agentDir?: string; +}): Promise | undefined> { + const token = await resolveProviderAuthProfileApiKey({ + provider: "openai", + cfg: params.cfg, + agentDir: params.agentDir, + profileTypes: ["oauth"], + includeExternalCliAuth: false, + }); + if (!token) { + return undefined; + } + const accountId = resolveCodexAuthIdentity({ accessToken: token }).accountId; + if (!accountId) { + throw new Error("The selected ChatGPT OAuth profile is missing its account id"); + } + return { type: "oauth", token, accountId }; +} + +export function createOpenAIQuicksilverBrowserSessionBroker(params: { + getConfig: () => OpenClawConfig | undefined; + logger: Pick; + fetchImpl?: typeof fetch; + webSocketFactory?: OpenAIQuicksilverSocketFactory; +}): { + broker: { + capabilities: Partial & { handlesAgentConsult: true }; + createBrowserSession: ( + request: OpenAIQuicksilverSessionRequest, + auth: OpenAIQuicksilverAuth, + ) => Promise; + cancelBrowserSession: (session: RealtimeVoiceBrowserSession) => void; + }; + handler: (req: IncomingMessage, res: ServerResponse) => Promise; + cleanup: () => Promise; +} { + const pendingOffers = new Map(); + const inFlightOffers = new Map(); + const activeSessions = new Map(); + const reservations = new Set(); + const inFlightHandlers = new Set>(); + const shutdownController = new AbortController(); + const createSocket = params.webSocketFactory ?? ((url, options) => new WebSocket(url, options)); + let cleanedUp = false; + + const finalizeSession = (session: ActiveSession) => { + if (activeSessions.get(session.token) !== session) { + return false; + } + activeSessions.delete(session.token); + reservations.delete(session.token); + clearTimeout(session.timer); + session.consultController?.abort(new Error("GPT-Live delegation stopped")); + session.abortController.abort(new Error("GPT-Live session closed")); + return true; + }; + + const closeSession = (session: ActiveSession) => { + if (!finalizeSession(session)) { + return; + } + if (session.socket.readyState === WEBSOCKET_OPEN) { + try { + session.socket.send(JSON.stringify({ type: "session.close" })); + } catch { + // The peer may have closed between readyState and send. + } + } + try { + session.socket.close(1000, "session closed"); + } catch { + // Socket teardown is best effort after ownership has been released. + } + }; + + const handleDelegation = async ( + session: ActiveSession, + delegationItemId: string, + prompt: string, + signal: AbortSignal, + runAgentConsult: NonNullable, + ) => { + let finalText: string; + try { + const result = await runAgentConsult({ prompt, signal }); + if (signal.aborted) { + return; + } + finalText = result.text; + } catch (error) { + if (signal.aborted) { + return; + } + // Agent failures can embed local paths, commands, or request data. The sideband is + // an external provider surface the model may speak aloud, so only a fixed message + // crosses it; the detail stays in Gateway logs. + params.logger.warn( + `OpenAI GPT-Live delegation consult failed: ${shortConsultFailureReason(error)}`, + ); + finalText = OPENAI_QUICKSILVER_CONSULT_FAILURE_TEXT; + } + if (session.socket.readyState !== WEBSOCKET_OPEN) { + return; + } + sendOpenAIQuicksilverAppend({ + socket: session.socket, + delegationItemId, + text: finalText, + channel: "speakable", + }); + }; + + const scheduleSessionExpiry = (session: ActiveSession, ttlMs: number) => { + clearTimeout(session.timer); + session.timer = setTimeout(() => closeSession(session), Math.max(0, ttlMs)); + session.timer.unref?.(); + }; + + const handleSidebandEvent = ( + session: ActiveSession, + event: OpenAIQuicksilverInboundEvent, + runAgentConsult: NonNullable, + ) => { + if (event.kind === "ignored") { + return; + } + if (event.kind === "unknown") { + params.logger.debug?.(`OpenAI GPT-Live ignored unknown sideband event: ${event.eventType}`); + return; + } + if (event.kind === "session-started") { + if (event.expiresAt !== undefined) { + const upstreamTtlMs = event.expiresAt * 1000 - Date.now(); + scheduleSessionExpiry(session, Math.min(OPENAI_QUICKSILVER_SESSION_TTL_MS, upstreamTtlMs)); + } + return; + } + if (event.kind === "transcript-delta" || event.kind === "transcript-done") { + appendOpenAIQuicksilverTranscript(session, event); + return; + } + if (event.kind === "error") { + params.logger.warn(`OpenAI GPT-Live sideband error: ${event.message}`); + if (event.fatalAuth) { + closeSession(session); + } + return; + } + const delegationInput = event.prompt; + if (!delegationInput.trim()) { + params.logger.debug?.("OpenAI GPT-Live ignored an empty client delegation"); + return; + } + // Drain only for a delegation we actually consult: the transcript is a + // once-delivered delta, so clearing it for an ignored event would strand + // that context and consult the next real delegation without it. + const transcript = session.transcript; + session.transcript = []; + session.partialTranscriptRole = undefined; + const prompt = buildOpenAIQuicksilverDelegationPrompt({ + input: delegationInput, + transcript, + }); + // A newer spoken task supersedes the prior consult so steering is immediate. + session.consultController?.abort(new Error("GPT-Live delegation superseded")); + const consultController = new AbortController(); + session.consultController = consultController; + const signal = AbortSignal.any([session.abortController.signal, consultController.signal]); + void handleDelegation(session, event.id, prompt, signal, runAgentConsult) + .catch((error: unknown) => { + params.logger.warn( + `OpenAI GPT-Live delegation failed: ${shortConsultFailureReason(error)}`, + ); + closeSession(session); + }) + .finally(() => { + if (session.consultController === consultController) { + session.consultController = undefined; + } + }); + }; + + const handleSidebandFrame = ( + session: ActiveSession, + data: RawData, + isBinary: boolean, + runAgentConsult: NonNullable, + ) => { + if (isBinary) { + params.logger.warn("OpenAI GPT-Live sideband returned an unexpected binary frame"); + closeSession(session); + return; + } + const event = parseOpenAIQuicksilverEvent(decodeQuicksilverTextFrame(data)); + if (event) { + handleSidebandEvent(session, event, runAgentConsult); + } + }; + + const attachSidebandHandlers = ( + session: ActiveSession, + runAgentConsult: NonNullable, + ) => { + session.socket.on("message", (data: RawData, isBinary: boolean) => { + handleSidebandFrame(session, data, isBinary, runAgentConsult); + }); + session.socket.on("error", (error: Error) => { + params.logger.warn(`OpenAI GPT-Live sideband socket failed: ${error.message}`); + closeSession(session); + }); + session.socket.on("close", () => { + finalizeSession(session); + }); + }; + + const prunePendingOffers = () => { + const now = Date.now(); + for (const [token, offer] of pendingOffers) { + if (offer.expiresAt <= now) { + pendingOffers.delete(token); + reservations.delete(token); + } + } + }; + + const broker = { + capabilities: OPENAI_QUICKSILVER_CAPABILITIES, + createBrowserSession: async ( + request: OpenAIQuicksilverSessionRequest, + auth: OpenAIQuicksilverAuth, + ): Promise => { + if (cleanedUp || shutdownController.signal.aborted) { + throw new Error("OpenAI GPT-Live sessions are stopping; restart Gateway and try again"); + } + if (!request.runAgentConsult) { + throw new Error("OpenAI GPT-Live requires the Gateway agent-consult runtime"); + } + prunePendingOffers(); + if (reservations.size >= OPENAI_QUICKSILVER_MAX_SESSIONS) { + throw new Error("Too many concurrent OpenAI GPT-Live sessions; try again in a minute"); + } + const model = request.model?.trim(); + if (!model) { + throw new Error("OpenAI GPT-Live requires a model"); + } + const voice = resolveOpenAIQuicksilverVoice(request.voice); + const token = randomBytes(32).toString("base64url"); + const expiresAt = Date.now() + OPENAI_QUICKSILVER_PENDING_TTL_MS; + pendingOffers.set(token, { + auth, + expiresAt, + requestIds: { + realtimeSessionId: randomUUID(), + sessionId: randomUUID(), + threadId: randomUUID(), + }, + request: { ...request, model, voice }, + }); + reservations.add(token); + return { + provider: "openai", + transport: "webrtc", + clientSecret: token, + offerUrl: OPENAI_QUICKSILVER_OFFER_PATH, + model, + voice, + expiresAt, + }; + }, + cancelBrowserSession: (session: RealtimeVoiceBrowserSession) => { + if (session.transport !== "webrtc") { + return; + } + pendingOffers.delete(session.clientSecret); + inFlightOffers.get(session.clientSecret)?.abort(new Error("GPT-Live session canceled")); + const active = activeSessions.get(session.clientSecret); + if (active) { + closeSession(active); + } else { + reservations.delete(session.clientSecret); + } + }, + }; + + const handleOffer = async (req: IncomingMessage, res: ServerResponse): Promise => { + const corsAllowed = applyRealtimeOfferCorsHeaders(req, res, params.getConfig()); + if (req.method === "OPTIONS") { + if (!corsAllowed) { + respondText(res, 403, "Origin not allowed"); + return true; + } + res.statusCode = 204; + res.setHeader("cache-control", "no-store"); + res.setHeader("Access-Control-Allow-Methods", "POST, OPTIONS"); + res.setHeader("Access-Control-Allow-Headers", "Authorization, Content-Type"); + res.setHeader( + "Vary", + "Origin, Access-Control-Request-Method, Access-Control-Request-Headers", + ); + if (req.headers["access-control-request-private-network"] === "true") { + res.setHeader("Access-Control-Allow-Private-Network", "true"); + } + res.setHeader("Access-Control-Max-Age", "600"); + res.end(); + return true; + } + if (!corsAllowed) { + respondText(res, 403, "Origin not allowed"); + return true; + } + if (req.method !== "POST") { + respondText(res, 405, "Method not allowed"); + return true; + } + if (!req.headers["content-type"]?.toLowerCase().startsWith("application/sdp")) { + respondText(res, 415, "Expected application/sdp"); + return true; + } + prunePendingOffers(); + const token = readBearerToken(req); + const offer = token ? pendingOffers.get(token) : undefined; + if (!token || !offer || offer.expiresAt <= Date.now()) { + respondText(res, 401, "Invalid or expired realtime session token"); + return true; + } + // Offer credentials are single-use so a captured browser request cannot join twice. + pendingOffers.delete(token); + const requestController = new AbortController(); + let browserDisconnected = false; + inFlightOffers.set(token, requestController); + const abortFromBrowser = () => { + browserDisconnected = true; + requestController.abort(new Error("Browser GPT-Live offer request closed")); + }; + req.once("aborted", abortFromBrowser); + res.once("close", abortFromBrowser); + const detachBrowserAbort = () => { + req.removeListener("aborted", abortFromBrowser); + res.removeListener("close", abortFromBrowser); + }; + const lifecycleSignal = AbortSignal.any([shutdownController.signal, requestController.signal]); + let session: ActiveSession | undefined; + let reservationTransferred = false; + let responseDeliveryWaiter: ResponseDeliveryWaiter | undefined; + try { + const sdp = await readRequestBodyWithLimit(req, { + maxBytes: OPENAI_QUICKSILVER_MAX_SDP_BYTES, + timeoutMs: 15_000, + }); + if (!sdp.trim()) { + respondText(res, 400, "SDP offer is required"); + return true; + } + const runAgentConsult = offer.request.runAgentConsult; + if (!runAgentConsult) { + throw new Error("OpenAI GPT-Live requires the Gateway agent-consult runtime"); + } + const upstreamSignal = AbortSignal.any([ + lifecycleSignal, + AbortSignal.timeout(OPENAI_QUICKSILVER_UPSTREAM_TIMEOUT_MS), + ]); + const call = await createOpenAIQuicksilverCall({ + auth: offer.auth, + requestIds: offer.requestIds, + sdp, + session: buildOpenAIQuicksilverSession({ + model: offer.request.model, + instructions: offer.request.instructions, + voice: offer.request.voice, + initialItems: offer.request.initialItems, + }), + signal: upstreamSignal, + fetchImpl: params.fetchImpl, + }); + const connected = await connectOpenAIQuicksilverSideband({ + auth: offer.auth, + createSocket, + requestIds: offer.requestIds, + signal: lifecycleSignal, + url: call.sidebandUrl, + }); + if (lifecycleSignal.aborted) { + connected.socket.close(1000, "session stopped"); + throw lifecycleSignal.reason; + } + const abortController = new AbortController(); + const timer = setTimeout(() => { + const active = activeSessions.get(token); + if (active) { + closeSession(active); + } + }, OPENAI_QUICKSILVER_SESSION_TTL_MS); + timer.unref?.(); + session = { + abortController, + socket: connected.socket, + timer, + token, + transcript: [], + partialTranscriptRole: undefined, + }; + activeSessions.set(token, session); + reservationTransferred = true; + attachSidebandHandlers(session, runAgentConsult); + const terminalEvent = connected.detachBuffer(); + for (const frame of connected.bufferedFrames) { + handleSidebandFrame(session, frame.data, frame.isBinary, runAgentConsult); + } + if (terminalEvent && activeSessions.get(token) === session) { + if (terminalEvent.kind === "error") { + params.logger.warn( + `OpenAI GPT-Live sideband socket failed: ${terminalEvent.error.message}`, + ); + closeSession(session); + } else { + finalizeSession(session); + } + } + if (activeSessions.get(token) !== session) { + throw new Error("OpenAI GPT-Live sideband failed during startup"); + } + + responseDeliveryWaiter = createResponseDeliveryWaiter(res, detachBrowserAbort); + res.statusCode = 200; + res.setHeader("cache-control", "no-store"); + res.setHeader("content-type", "application/sdp"); + res.setHeader("x-content-type-options", "nosniff"); + res.end(call.answerSdp); + const delivered = await responseDeliveryWaiter.result; + responseDeliveryWaiter = undefined; + if (!delivered || lifecycleSignal.aborted) { + closeSession(session); + } + return true; + } catch (error) { + if (session) { + closeSession(session); + } + if (browserDisconnected) { + return true; + } + respondText( + res, + 502, + error instanceof Error ? error.message : "OpenAI GPT-Live session failed", + ); + return true; + } finally { + responseDeliveryWaiter?.cancel(); + detachBrowserAbort(); + inFlightOffers.delete(token); + if (!reservationTransferred) { + reservations.delete(token); + } + } + }; + + const handler = (req: IncomingMessage, res: ServerResponse): Promise => { + const handling = handleOffer(req, res); + inFlightHandlers.add(handling); + return handling.finally(() => inFlightHandlers.delete(handling)); + }; + + const cleanup = async () => { + if (cleanedUp) { + return; + } + cleanedUp = true; + shutdownController.abort(new Error("OpenAI GPT-Live broker stopped")); + pendingOffers.clear(); + for (const controller of inFlightOffers.values()) { + controller.abort(new Error("OpenAI GPT-Live broker stopped")); + } + for (const session of activeSessions.values()) { + closeSession(session); + } + await Promise.allSettled(inFlightHandlers); + reservations.clear(); + }; + + return { broker, handler, cleanup }; +} diff --git a/extensions/openai/realtime-quicksilver-sideband.ts b/extensions/openai/realtime-quicksilver-sideband.ts new file mode 100644 index 000000000000..91bafa8ac0da --- /dev/null +++ b/extensions/openai/realtime-quicksilver-sideband.ts @@ -0,0 +1,202 @@ +import type { ClientOptions, RawData } from "ws"; +import { + openAIQuicksilverAuthHeaders, + type OpenAIQuicksilverAuth, + type OpenAIQuicksilverRequestIds, +} from "./realtime-quicksilver-wire.js"; + +const SIDEBAND_CONNECT_TIMEOUT_MS = 15_000; +const SIDEBAND_CONNECT_ATTEMPTS = 5; +const SIDEBAND_RETRY_BASE_MS = 200; +const EARLY_FRAME_MAX = 32; + +export type OpenAIQuicksilverSocket = { + readonly readyState: number; + send(payload: string): void; + close(code?: number, reason?: string): void; + on( + event: "message", + listener: (data: RawData, isBinary: boolean) => void, + ): OpenAIQuicksilverSocket; + on(event: "error", listener: (error: Error) => void): OpenAIQuicksilverSocket; + on(event: "close", listener: () => void): OpenAIQuicksilverSocket; + once(event: "open", listener: () => void): OpenAIQuicksilverSocket; + once(event: "error", listener: (error: Error) => void): OpenAIQuicksilverSocket; + once(event: "close", listener: () => void): OpenAIQuicksilverSocket; + off(event: "open", listener: () => void): OpenAIQuicksilverSocket; + off( + event: "message", + listener: (data: RawData, isBinary: boolean) => void, + ): OpenAIQuicksilverSocket; + off(event: "error", listener: (error: Error) => void): OpenAIQuicksilverSocket; + off(event: "close", listener: () => void): OpenAIQuicksilverSocket; +}; + +export type OpenAIQuicksilverSocketFactory = ( + url: string, + options: ClientOptions, +) => OpenAIQuicksilverSocket; + +type OpenAIQuicksilverBufferedFrame = { data: RawData; isBinary: boolean }; +type OpenAIQuicksilverTerminalEvent = { kind: "error"; error: Error } | { kind: "close" }; + +type OpenAIQuicksilverConnectedSideband = { + socket: OpenAIQuicksilverSocket; + bufferedFrames: OpenAIQuicksilverBufferedFrame[]; + detachBuffer: () => OpenAIQuicksilverTerminalEvent | undefined; +}; + +function waitForSocketOpen(params: { + socket: OpenAIQuicksilverSocket; + signal: AbortSignal; +}): Promise<{ detachTerminalListeners: () => OpenAIQuicksilverTerminalEvent | undefined }> { + return new Promise((resolve, reject) => { + let settled = false; + let opened = false; + let terminalEvent: OpenAIQuicksilverTerminalEvent | undefined; + const detachTerminalListeners = () => { + params.socket.off("error", onError); + params.socket.off("close", onClose); + return terminalEvent; + }; + const finish = (error?: Error) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + params.signal.removeEventListener("abort", onAbort); + params.socket.off("open", onOpen); + if (error) { + detachTerminalListeners(); + reject(error); + } else { + resolve({ detachTerminalListeners }); + } + }; + const onOpen = () => { + opened = true; + finish(); + }; + const onError = (error: Error) => { + if (opened) { + terminalEvent ??= { kind: "error", error }; + return; + } + finish(error); + }; + const onClose = () => { + if (opened) { + terminalEvent ??= { kind: "close" }; + return; + } + finish(new Error("GPT-Live sideband closed during startup")); + }; + const onAbort = () => + finish( + params.signal.reason instanceof Error + ? params.signal.reason + : new Error("GPT-Live session stopped during startup"), + ); + const timeout = setTimeout( + () => finish(new Error("GPT-Live sideband connection timed out")), + SIDEBAND_CONNECT_TIMEOUT_MS, + ); + timeout.unref?.(); + params.socket.once("open", onOpen); + params.socket.on("error", onError); + params.socket.on("close", onClose); + params.signal.addEventListener("abort", onAbort, { once: true }); + if (params.signal.aborted) { + onAbort(); + } + }); +} + +function waitForRetryDelay(ms: number, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + // Typed as Error so the rejection reason is provably an Error at the call site; + // onAbort normalizes a non-Error AbortSignal reason before it reaches here. + const finish = (error?: Error) => { + clearTimeout(timer); + signal.removeEventListener("abort", onAbort); + if (error) { + reject(error); + } else { + resolve(); + } + }; + const onAbort = () => { + const reason = signal.reason; + finish( + reason instanceof Error + ? reason + : new Error(reason === undefined ? "GPT-Live session stopped" : String(reason), { + cause: reason, + }), + ); + }; + const timer = setTimeout(() => finish(), ms); + timer.unref?.(); + signal.addEventListener("abort", onAbort, { once: true }); + if (signal.aborted) { + onAbort(); + } + }); +} + +export async function connectOpenAIQuicksilverSideband(params: { + auth: OpenAIQuicksilverAuth; + createSocket: OpenAIQuicksilverSocketFactory; + requestIds: OpenAIQuicksilverRequestIds; + signal: AbortSignal; + url: string; +}): Promise { + let lastError: unknown = new Error("GPT-Live sideband connection failed"); + for (let attempt = 0; attempt < SIDEBAND_CONNECT_ATTEMPTS; attempt += 1) { + if (params.signal.aborted) { + throw params.signal.reason; + } + const socket = params.createSocket(params.url, { + headers: openAIQuicksilverAuthHeaders(params.auth, params.requestIds), + }); + const bufferedFrames: OpenAIQuicksilverBufferedFrame[] = []; + const bufferFrame = (data: RawData, isBinary: boolean) => { + if (bufferedFrames.length < EARLY_FRAME_MAX) { + bufferedFrames.push({ data, isBinary }); + } + }; + socket.on("message", bufferFrame); + try { + const openHandoff = await waitForSocketOpen({ socket, signal: params.signal }); + return { + socket, + bufferedFrames, + detachBuffer: () => { + socket.off("message", bufferFrame); + return openHandoff.detachTerminalListeners(); + }, + }; + } catch (error) { + lastError = error; + socket.off("message", bufferFrame); + // waitForSocketOpen already detached its error listener, and closing a still + // CONNECTING ws emits `error` asynchronously. Without a listener that is an + // unhandled EventEmitter error, which is fatal to the Gateway process, so keep + // this swallow attached for the discarded socket's lifetime. + socket.on("error", () => {}); + try { + socket.close(1000, "retrying sideband"); + } catch { + // Failed startup sockets are already unusable. + } + if (params.signal.aborted) { + throw params.signal.reason; + } + if (attempt + 1 < SIDEBAND_CONNECT_ATTEMPTS) { + await waitForRetryDelay(SIDEBAND_RETRY_BASE_MS * 2 ** attempt, params.signal); + } + } + } + throw lastError; +} diff --git a/extensions/openai/realtime-quicksilver-wire.test.ts b/extensions/openai/realtime-quicksilver-wire.test.ts new file mode 100644 index 000000000000..1245b5818466 --- /dev/null +++ b/extensions/openai/realtime-quicksilver-wire.test.ts @@ -0,0 +1,207 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + buildOpenAIQuicksilverSession, + createOpenAIQuicksilverCall, +} from "./realtime-quicksilver-wire.js"; + +function createCallResponse(answer = "v=answer\r\n", callId = "rtc_test"): Response { + return new Response(answer, { + status: 201, + headers: { Location: `/v1/live/${callId}?source=test` }, + }); +} + +function createRequestIds(label: string) { + return { + realtimeSessionId: `${label}-realtime`, + sessionId: `${label}-session`, + threadId: `${label}-thread`, + }; +} + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe("GPT-Live call creation", () => { + it("uses one multipart /v1/live wire shape for OAuth and API-key auth", async () => { + vi.stubEnv("OPENCLAW_VERSION", "2026.7.2-test"); + const requests: Array<{ url: string; init?: RequestInit }> = []; + const fetchImpl = vi.fn(async (url: string | URL | Request, init?: RequestInit) => { + const resolvedUrl = typeof url === "string" ? url : url instanceof URL ? url.href : url.url; + requests.push({ url: resolvedUrl, init }); + return createCallResponse("v=answer\r\n", `rtc_${requests.length}`); + }) as unknown as typeof fetch; + const session = buildOpenAIQuicksilverSession({ + model: "gpt-live-1-codex", + instructions: "Speak briefly.", + voice: "marin", + }); + + await expect( + createOpenAIQuicksilverCall({ + auth: { type: "oauth", token: "oauth-token", accountId: "acct-1" }, + requestIds: createRequestIds("oauth"), + sdp: "v=oauth-offer\r\n", + session, + fetchImpl, + }), + ).resolves.toEqual({ + status: 201, + answerSdp: "v=answer\r\n", + callId: "rtc_1", + sidebandUrl: "wss://api.openai.com/v1/live/rtc_1", + }); + expect(requests[0]?.url).toBe("https://api.openai.com/v1/live"); + expect(requests[0]?.init?.headers).toEqual({ + Authorization: "Bearer oauth-token", + "OpenAI-Alpha": "quicksilver=v2", + "User-Agent": "openclaw/2026.7.2-test", + "chatgpt-account-id": "acct-1", + originator: "openclaw", + "session-id": "oauth-session", + "thread-id": "oauth-thread", + version: "2026.7.2-test", + "x-session-id": "oauth-realtime", + "Content-Type": expect.stringMatching(/^multipart\/form-data; boundary=/), + }); + + await expect( + createOpenAIQuicksilverCall({ + auth: { type: "api-key", token: "platform-key" }, + requestIds: createRequestIds("api-key"), + sdp: "v=api-offer\r\n", + session, + fetchImpl, + }), + ).resolves.toMatchObject({ status: 201, callId: "rtc_2" }); + expect(requests[1]?.url).toBe("https://api.openai.com/v1/live"); + expect(requests[1]?.init?.headers).toMatchObject({ + Authorization: "Bearer platform-key", + "session-id": "api-key-session", + "thread-id": "api-key-thread", + "x-session-id": "api-key-realtime", + }); + expect(requests[1]?.init?.headers).not.toHaveProperty("chatgpt-account-id"); + + for (const request of requests) { + expect(request.url).not.toContain("?"); + const headers = request.init?.headers as Record | undefined; + const boundary = headers?.["Content-Type"]?.split("boundary=")[1]; + const body = request.init?.body; + expect(boundary).toBeTruthy(); + expect(typeof body).toBe("string"); + expect(body).toContain(`--${boundary}\r\n`); + expect(body).toContain('name="sdp"\r\nContent-Type: application/sdp'); + expect(body).toContain('name="session"\r\nContent-Type: application/json'); + expect(body).toContain(JSON.stringify(session)); + } + }); + + it.each([ + { + name: "overloaded rejection", + status: 403, + body: "Voice session access denied.", + message: + "GPT-Live rejected the session (403). This overloaded response most often means the voice or model is invalid for /v1/live. Accepted voices: alloy, ash, ballad, cedar, coral, echo, marin, sage, shimmer, verse. Accepted models: gpt-live-1-codex, gpt-live-1-boulder-alpha. Account access may also be unavailable; verify the selected ChatGPT OAuth profile and chatgpt-account-id.", + }, + { + name: "Platform waitlist denial", + status: 400, + body: '{"error":{"code":"model_not_found","message":"The model does not exist or you do not have access"}}', + message: + "OpenAI Platform API-key access to /v1/live is waitlist-gated. Use a ChatGPT OAuth profile or request access at https://openai.com/form/gpt-live-1-in-the-api/", + }, + { + name: "unsupported route model", + status: 400, + body: "Field `session.model` is not allowed for this Codex realtime session", + message: + "The GPT-Live model value is not permitted on /v1/live. Accepted values are gpt-live-1-codex and gpt-live-1-boulder-alpha.", + }, + ])("maps $name", async ({ status, body, message }) => { + const fetchImpl = vi.fn(async () => new Response(body, { status })); + const promise = createOpenAIQuicksilverCall({ + auth: { type: "api-key", token: "platform-key" }, + requestIds: createRequestIds("error"), + sdp: "v=offer\r\n", + session: buildOpenAIQuicksilverSession({ model: "gpt-live-1-codex" }), + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + await expect(promise).rejects.toMatchObject({ + name: "OpenAIQuicksilverCallError", + status, + message, + }); + }); + + it.each([ + { location: null, callId: "rtc_header_fallback" }, + { location: null, callId: "019eb97d-8e9a-7ff3-94b0-ea019babd5d7" }, + { location: "http://[invalid", callId: "rtc_malformed_location_fallback" }, + { location: "/v1/live/not-a-call", callId: "rtc_invalid_path_fallback" }, + ])("falls back to openai-session-id for Location $location", async ({ location, callId }) => { + const fetchImpl = vi.fn( + async () => + new Response("v=answer\r\n", { + status: 201, + headers: { + ...(location ? { Location: location } : {}), + "openai-session-id": callId, + }, + }), + ); + + await expect( + createOpenAIQuicksilverCall({ + auth: { type: "oauth", token: "oauth-token", accountId: "acct-1" }, + requestIds: createRequestIds("header-fallback"), + sdp: "v=offer\r\n", + session: buildOpenAIQuicksilverSession({ model: "gpt-live-1-codex" }), + fetchImpl: fetchImpl as unknown as typeof fetch, + }), + ).resolves.toMatchObject({ callId }); + }); + + it("accepts a UUID call id from Location", async () => { + const callId = "019eb97d-8e9a-7ff3-94b0-ea019babd5d7"; + const fetchImpl = vi.fn(async () => createCallResponse("v=answer\r\n", callId)); + + await expect( + createOpenAIQuicksilverCall({ + auth: { type: "oauth", token: "oauth-token", accountId: "acct-1" }, + requestIds: createRequestIds("uuid-location"), + sdp: "v=offer\r\n", + session: buildOpenAIQuicksilverSession({ model: "gpt-live-1-codex" }), + fetchImpl: fetchImpl as unknown as typeof fetch, + }), + ).resolves.toMatchObject({ + callId, + sidebandUrl: `wss://api.openai.com/v1/live/${callId}`, + }); + }); + + it("rejects an empty SDP answer", async () => { + const fetchImpl = vi.fn( + async () => + new Response("", { + status: 201, + headers: { Location: "/v1/live/rtc_empty_answer" }, + }), + ); + await expect( + createOpenAIQuicksilverCall({ + auth: { type: "oauth", token: "oauth-token", accountId: "acct-1" }, + requestIds: createRequestIds("empty-answer"), + sdp: "v=offer\r\n", + session: buildOpenAIQuicksilverSession({ model: "gpt-live-1-codex" }), + fetchImpl: fetchImpl as unknown as typeof fetch, + }), + ).rejects.toMatchObject({ + name: "OpenAIQuicksilverCallError", + status: 201, + message: "GPT-Live call creation returned an empty SDP answer", + }); + }); +}); diff --git a/extensions/openai/realtime-quicksilver-wire.ts b/extensions/openai/realtime-quicksilver-wire.ts new file mode 100644 index 000000000000..75f5161870d6 --- /dev/null +++ b/extensions/openai/realtime-quicksilver-wire.ts @@ -0,0 +1,509 @@ +// GPT-Live frameless session, call-creation, and sideband event wire contracts. +import { randomBytes } from "node:crypto"; +import { resolveProviderRequestHeaders } from "openclaw/plugin-sdk/provider-http"; +import { z } from "zod"; + +const OPENAI_QUICKSILVER_APPEND_MAX_BYTES = 500; +const OPENAI_QUICKSILVER_CONTEXT_MAX_ENTRIES = 16; +const OPENAI_QUICKSILVER_CONTEXT_MAX_ITEM_CHARS = 800; +const OPENAI_QUICKSILVER_CONTEXT_MAX_UTF8_BYTES = 8_000; +const OPENAI_QUICKSILVER_CALL_URL = "https://api.openai.com/v1/live"; +const OPENAI_GPT_LIVE_WAITLIST_URL = "https://openai.com/form/gpt-live-1-in-the-api/"; + +const OPENAI_QUICKSILVER_VOICES = [ + "alloy", + "ash", + "ballad", + "cedar", + "coral", + "echo", + "marin", + "sage", + "shimmer", + "verse", +] as const; + +type OpenAIQuicksilverVoice = (typeof OPENAI_QUICKSILVER_VOICES)[number]; + +export type OpenAIQuicksilverAuth = + | { type: "api-key"; token: string } + | { type: "oauth"; token: string; accountId: string }; + +export type OpenAIQuicksilverRequestIds = { + realtimeSessionId: string; + sessionId: string; + threadId: string; +}; + +export type OpenAIQuicksilverInitialItem = { + role: "user" | "assistant"; + text: string; +}; + +type OpenAIQuicksilverSession = { + model: string; + instructions: string; + audio: { output: { voice: OpenAIQuicksilverVoice } }; + delegation: { type: "client" }; + initial_items?: Array<{ + type: "message"; + role: "user" | "assistant"; + content: Array<{ type: "input_text" | "output_text"; text: string }>; + }>; +}; + +const eventEnvelopeSchema = z.object({ type: z.string() }).passthrough(); +const sessionStartedSchema = z + .object({ + type: z.literal("session.started"), + session: z.object({ expires_at: z.number().optional() }).passthrough(), + }) + .passthrough(); +const transcriptAddedSchema = z + .object({ + item: z.object({ text: z.string() }).passthrough(), + }) + .passthrough(); +const turnDoneSchema = z + .object({ + turn: z + .object({ + role: z.enum(["user", "assistant"]), + transcript: z.string(), + }) + .passthrough(), + }) + .passthrough(); +const delegationSchema = z + .object({ + type: z.literal("delegation.created"), + item: z + .object({ + type: z.string(), + target: z.string(), + id: z.string().optional(), + content: z + .array( + z + .object({ + type: z.string(), + text: z.string().optional(), + }) + .passthrough(), + ) + .optional(), + }) + .passthrough(), + }) + .passthrough(); + +export type OpenAIQuicksilverInboundEvent = + | { kind: "ignored"; eventType: string } + | { kind: "session-started"; expiresAt?: number } + | { kind: "transcript-delta"; role: "user" | "assistant"; text: string } + | { kind: "transcript-done"; role: "user" | "assistant"; text: string } + | { kind: "delegation"; id: string; prompt: string } + | { kind: "error"; message: string; fatalAuth: boolean } + | { kind: "unknown"; eventType: string }; + +class OpenAIQuicksilverCallError extends Error { + constructor( + message: string, + readonly status?: number, + ) { + super(message); + this.name = "OpenAIQuicksilverCallError"; + } +} + +export function resolveOpenAIQuicksilverVoice(value: unknown): OpenAIQuicksilverVoice { + if (typeof value === "string") { + const normalized = value.trim().toLowerCase(); + if (OPENAI_QUICKSILVER_VOICES.includes(normalized as OpenAIQuicksilverVoice)) { + return normalized as OpenAIQuicksilverVoice; + } + } + return "marin"; +} + +export function buildOpenAIQuicksilverSession(params: { + model: string; + instructions?: string; + voice?: string; + initialItems?: readonly OpenAIQuicksilverInitialItem[]; +}): OpenAIQuicksilverSession { + const initialItems = boundOpenAIQuicksilverContextItems(params.initialItems ?? []).map( + (item) => ({ + type: "message" as const, + role: item.role, + content: [ + { + type: item.role === "assistant" ? ("output_text" as const) : ("input_text" as const), + text: item.text, + }, + ], + }), + ); + return { + model: params.model, + instructions: params.instructions?.trim() ?? "", + audio: { output: { voice: resolveOpenAIQuicksilverVoice(params.voice) } }, + delegation: { type: "client" }, + ...(initialItems && initialItems.length > 0 ? { initial_items: initialItems } : {}), + }; +} + +function truncateOpenAIQuicksilverContextText(text: string, maxBytes: number): string { + let result = ""; + let bytes = 0; + let characters = 0; + for (const character of text) { + const characterBytes = Buffer.byteLength(character, "utf8"); + if ( + characters >= OPENAI_QUICKSILVER_CONTEXT_MAX_ITEM_CHARS || + bytes + characterBytes > maxBytes + ) { + break; + } + result += character; + bytes += characterBytes; + characters += 1; + } + return result; +} + +export function boundOpenAIQuicksilverContextItems( + items: readonly OpenAIQuicksilverInitialItem[], +): OpenAIQuicksilverInitialItem[] { + let remainingBytes = OPENAI_QUICKSILVER_CONTEXT_MAX_UTF8_BYTES; + const newestFirst: OpenAIQuicksilverInitialItem[] = []; + for ( + let index = items.length - 1; + index >= 0 && newestFirst.length < OPENAI_QUICKSILVER_CONTEXT_MAX_ENTRIES; + index -= 1 + ) { + const item = items[index]; + if (!item || remainingBytes <= 0) { + continue; + } + const text = truncateOpenAIQuicksilverContextText(item.text, remainingBytes); + if (!text) { + continue; + } + newestFirst.push({ role: item.role, text }); + remainingBytes -= Buffer.byteLength(text, "utf8"); + } + return newestFirst.toReversed(); +} + +export function openAIQuicksilverAuthHeaders( + auth: OpenAIQuicksilverAuth, + requestIds: OpenAIQuicksilverRequestIds, +): Record { + const attributionHeaders = + resolveProviderRequestHeaders({ + provider: "openai", + baseUrl: "https://api.openai.com/v1/live", + capability: "audio", + transport: "http", + defaultHeaders: {}, + }) ?? {}; + // x-oai-attestation is optional and intentionally omitted on unsupported clients. + return { + ...attributionHeaders, + Authorization: `Bearer ${auth.token}`, + "OpenAI-Alpha": "quicksilver=v2", + "session-id": requestIds.sessionId, + "thread-id": requestIds.threadId, + "x-session-id": requestIds.realtimeSessionId, + ...(auth.type === "oauth" + ? { + "chatgpt-account-id": auth.accountId, + } + : {}), + }; +} + +function buildOpenAIQuicksilverMultipartBody(params: { + sdp: string; + session: OpenAIQuicksilverSession; +}): { body: string; contentType: string } { + const sessionJson = JSON.stringify(params.session); + let boundary: string; + do { + boundary = `openclaw-quicksilver-${randomBytes(18).toString("hex")}`; + } while (params.sdp.includes(boundary) || sessionJson.includes(boundary)); + return { + body: [ + `--${boundary}\r\n`, + 'Content-Disposition: form-data; name="sdp"\r\n', + "Content-Type: application/sdp\r\n\r\n", + params.sdp, + "\r\n", + `--${boundary}\r\n`, + 'Content-Disposition: form-data; name="session"\r\n', + "Content-Type: application/json\r\n\r\n", + sessionJson, + "\r\n", + `--${boundary}--\r\n`, + ].join(""), + contentType: `multipart/form-data; boundary=${boundary}`, + }; +} + +function isOpenAIQuicksilverCallId(value: string): boolean { + return ( + /^rtc_[\w-]+$/.test(value) || + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value) + ); +} + +function decodeOpenAIQuicksilverCallId(params: { + location: string | null; + openAiSessionId: string | null; + callUrl: string; +}): string { + const sessionId = params.openAiSessionId?.trim() ?? ""; + if (!params.location) { + if (isOpenAIQuicksilverCallId(sessionId)) { + return sessionId; + } + throw new OpenAIQuicksilverCallError( + sessionId + ? "GPT-Live call response returned an invalid openai-session-id" + : "GPT-Live call response missing Location and openai-session-id headers", + ); + } + let pathname: string; + try { + pathname = new URL(params.location, params.callUrl).pathname; + } catch { + if (isOpenAIQuicksilverCallId(sessionId)) { + return sessionId; + } + throw new OpenAIQuicksilverCallError("GPT-Live call response returned an invalid Location"); + } + const callId = pathname.split("/").filter(Boolean).find(isOpenAIQuicksilverCallId); + if (!callId) { + if (isOpenAIQuicksilverCallId(sessionId)) { + return sessionId; + } + throw new OpenAIQuicksilverCallError("GPT-Live call response Location has no valid call id"); + } + return callId; +} + +function describeOpenAIQuicksilverCallError(status: number, detail: string): string { + const normalized = detail.toLowerCase(); + if (status === 403) { + return "GPT-Live rejected the session (403). This overloaded response most often means the voice or model is invalid for /v1/live. Accepted voices: alloy, ash, ballad, cedar, coral, echo, marin, sage, shimmer, verse. Accepted models: gpt-live-1-codex, gpt-live-1-boulder-alpha. Account access may also be unavailable; verify the selected ChatGPT OAuth profile and chatgpt-account-id."; + } + if ( + status === 400 && + (normalized.includes("model_not_found") || + normalized.includes("does not exist or you do not have access")) + ) { + return `OpenAI Platform API-key access to /v1/live is waitlist-gated. Use a ChatGPT OAuth profile or request access at ${OPENAI_GPT_LIVE_WAITLIST_URL}`; + } + if ( + status === 400 && + normalized.includes("session.model") && + normalized.includes("not allowed") + ) { + return "The GPT-Live model value is not permitted on /v1/live. Accepted values are gpt-live-1-codex and gpt-live-1-boulder-alpha."; + } + return `GPT-Live call creation failed (${status})${detail ? `: ${detail}` : ""}`; +} + +export async function createOpenAIQuicksilverCall(params: { + auth: OpenAIQuicksilverAuth; + sdp: string; + session: OpenAIQuicksilverSession; + requestIds: OpenAIQuicksilverRequestIds; + signal?: AbortSignal; + fetchImpl?: typeof fetch; +}): Promise<{ status: number; answerSdp: string; callId: string; sidebandUrl: string }> { + const authHeaders = openAIQuicksilverAuthHeaders(params.auth, params.requestIds); + const multipart = buildOpenAIQuicksilverMultipartBody({ + sdp: params.sdp, + session: params.session, + }); + + const response = await (params.fetchImpl ?? fetch)(OPENAI_QUICKSILVER_CALL_URL, { + method: "POST", + headers: { ...authHeaders, "Content-Type": multipart.contentType }, + body: multipart.body, + signal: params.signal, + }); + if (!response.ok) { + const detail = (await response.text().catch(() => "")).trim().slice(0, 500); + throw new OpenAIQuicksilverCallError( + describeOpenAIQuicksilverCallError(response.status, detail), + response.status, + ); + } + const answerSdp = await response.text(); + if (!answerSdp.trim()) { + throw new OpenAIQuicksilverCallError( + "GPT-Live call creation returned an empty SDP answer", + response.status, + ); + } + const callId = decodeOpenAIQuicksilverCallId({ + location: response.headers.get("Location"), + openAiSessionId: response.headers.get("openai-session-id"), + callUrl: OPENAI_QUICKSILVER_CALL_URL, + }); + return { + status: response.status, + answerSdp, + callId, + sidebandUrl: `wss://api.openai.com/v1/live/${callId}`, + }; +} + +function readQuicksilverErrorMessage(value: unknown): string { + if (typeof value === "string" && value.trim()) { + return value.trim(); + } + if (value && typeof value === "object") { + const record = value as Record; + if (typeof record.message === "string" && record.message.trim()) { + return record.message.trim(); + } + const error = record.error; + if (error && typeof error === "object") { + const nestedMessage = (error as Record).message; + if (typeof nestedMessage === "string" && nestedMessage.trim()) { + return nestedMessage.trim(); + } + } + if (typeof error === "string" && error.trim()) { + return error.trim(); + } + try { + const serialized = JSON.stringify(error ?? value); + if (serialized && serialized !== "{}") { + return serialized; + } + } catch { + // Fall through to the stable generic diagnostic. + } + } + return "GPT-Live sideband error"; +} + +function isFatalQuicksilverAuthError(value: unknown): boolean { + if (!value || typeof value !== "object") { + return false; + } + const record = value as Record; + const error = + record.error && typeof record.error === "object" + ? (record.error as Record) + : undefined; + const status = record.status ?? error?.status; + if (status === 401 || status === "401") { + return true; + } + const code = + typeof (record.code ?? error?.code) === "string" + ? String(record.code ?? error?.code).toLowerCase() + : ""; + return ["authentication_error", "invalid_api_key", "invalid_token", "token_expired"].includes( + code, + ); +} + +export function parseOpenAIQuicksilverEvent(payload: string): OpenAIQuicksilverInboundEvent | null { + let decoded: unknown; + try { + decoded = JSON.parse(payload); + } catch { + return null; + } + const envelope = eventEnvelopeSchema.safeParse(decoded); + if (!envelope.success) { + return null; + } + const eventType = envelope.data.type; + if (eventType === "session.started") { + const started = sessionStartedSchema.safeParse(decoded); + if (!started.success) { + return { kind: "ignored", eventType }; + } + const expiresAt = started.data.session.expires_at; + return { + kind: "session-started", + ...(expiresAt !== undefined ? { expiresAt } : {}), + }; + } + if (eventType === "input_transcript.added" || eventType === "output_transcript.added") { + const transcript = transcriptAddedSchema.safeParse(decoded); + return transcript.success + ? { + kind: "transcript-delta", + role: eventType === "input_transcript.added" ? "user" : "assistant", + text: transcript.data.item.text, + } + : { kind: "ignored", eventType }; + } + if (eventType === "turn.done") { + const turn = turnDoneSchema.safeParse(decoded); + return turn.success + ? { kind: "transcript-done", role: turn.data.turn.role, text: turn.data.turn.transcript } + : { kind: "ignored", eventType }; + } + if (eventType === "session.updated" || eventType === "output_audio.delta") { + return { kind: "ignored", eventType }; + } + if (eventType === "delegation.created") { + const delegation = delegationSchema.safeParse(decoded); + if (!delegation.success) { + return { kind: "ignored", eventType }; + } + const { item } = delegation.data; + if (item.type !== "delegation" || item.target !== "client" || !item.id) { + return { kind: "ignored", eventType }; + } + return { + kind: "delegation", + id: item.id, + prompt: (item.content ?? []) + .filter((part) => part.type === "input_text") + .map((part) => part.text ?? "") + .join(""), + }; + } + if (eventType === "error") { + return { + kind: "error", + message: readQuicksilverErrorMessage(decoded), + fatalAuth: isFatalQuicksilverAuthError(decoded), + }; + } + return { kind: "unknown", eventType }; +} + +export function chunkOpenAIQuicksilverAppendText(text: string): string[] { + if (Buffer.byteLength(text, "utf8") <= OPENAI_QUICKSILVER_APPEND_MAX_BYTES) { + return [text]; + } + const chunks: string[] = []; + let current = ""; + let currentBytes = 0; + for (const character of text) { + const characterBytes = Buffer.byteLength(character, "utf8"); + if (current && currentBytes + characterBytes > OPENAI_QUICKSILVER_APPEND_MAX_BYTES) { + chunks.push(current); + current = ""; + currentBytes = 0; + } + current += character; + currentBytes += characterBytes; + } + if (current) { + chunks.push(current); + } + return chunks; +} diff --git a/extensions/openai/realtime-quicksilver.live.test.ts b/extensions/openai/realtime-quicksilver.live.test.ts new file mode 100644 index 000000000000..7ba749d25a76 --- /dev/null +++ b/extensions/openai/realtime-quicksilver.live.test.ts @@ -0,0 +1,199 @@ +import { randomUUID } from "node:crypto"; +import { readCodexCliCredentialsCached } from "openclaw/plugin-sdk/provider-auth"; +import type { Page } from "playwright"; +import { describe, expect, it } from "vitest"; +import WebSocket, { type RawData } from "ws"; +import { resolveCodexAuthIdentity } from "./openai-chatgpt-auth-identity.js"; +import { + buildOpenAIQuicksilverSession, + createOpenAIQuicksilverCall, + openAIQuicksilverAuthHeaders, + type OpenAIQuicksilverAuth, +} from "./realtime-quicksilver-wire.js"; + +const LIVE_ENABLED = + process.env.OPENCLAW_LIVE_TEST === "1" && process.env.OPENCLAW_LIVE_GPT_LIVE === "1"; +const describeLive = LIVE_ENABLED ? describe : describe.skip; +const LIVE_TIMEOUT_MS = 60_000; + +type BrowserWithGptLivePeer = typeof globalThis & { + openclawGptLivePeer?: RTCPeerConnection; +}; + +function decodeTextFrame(data: RawData): string { + if (Array.isArray(data)) { + return Buffer.concat(data).toString("utf8"); + } + if (data instanceof ArrayBuffer) { + return Buffer.from(data).toString("utf8"); + } + return data.toString("utf8"); +} + +async function createBrowserOffer(page: Page): Promise { + return await page.evaluate(async () => { + const peer = new RTCPeerConnection(); + peer.addTransceiver("audio", { direction: "sendrecv" }); + peer.createDataChannel("oai-events"); + const offer = await peer.createOffer(); + await peer.setLocalDescription(offer); + const sdp = offer.sdp; + if (!sdp) { + peer.close(); + throw new Error("Chromium did not produce a GPT-Live SDP offer"); + } + (globalThis as BrowserWithGptLivePeer).openclawGptLivePeer = peer; + return sdp; + }); +} + +async function applyBrowserAnswer(page: Page, sdp: string): Promise { + await page.evaluate(async (answerSdp) => { + const peer = (globalThis as BrowserWithGptLivePeer).openclawGptLivePeer; + if (!peer) { + throw new Error("GPT-Live browser peer is unavailable"); + } + await peer.setRemoteDescription({ type: "answer", sdp: answerSdp }); + }, sdp); +} + +async function closeBrowserPeer(page: Page): Promise { + await page.evaluate(() => { + const target = globalThis as BrowserWithGptLivePeer; + target.openclawGptLivePeer?.close(); + delete target.openclawGptLivePeer; + }); +} + +async function waitForSidebandSessionStarted(params: { + url: string; + headers: Record; +}): Promise<{ socket: WebSocket; sessionId: string }> { + return await new Promise((resolve, reject) => { + const socket = new WebSocket(params.url, { headers: params.headers }); + const timeout = setTimeout(() => { + socket.close(1000, "session-start timeout"); + reject(new Error("GPT-Live sideband did not emit session.started")); + }, 15_000); + const finish = (result: { socket: WebSocket; sessionId: string } | Error) => { + clearTimeout(timeout); + socket.off("message", onMessage); + socket.off("error", onError); + socket.off("close", onClose); + if (result instanceof Error) { + reject(result); + } else { + resolve(result); + } + }; + const onMessage = (data: RawData, isBinary: boolean) => { + if (isBinary) { + return; + } + let decoded: unknown; + try { + decoded = JSON.parse(decodeTextFrame(data)); + } catch { + return; + } + if (!decoded || typeof decoded !== "object") { + return; + } + const event = decoded as { type?: unknown; session?: { id?: unknown } }; + if (event.type === "session.started" && typeof event.session?.id === "string") { + finish({ socket, sessionId: event.session.id }); + } + }; + const onError = (error: Error) => finish(error); + const onClose = () => finish(new Error("GPT-Live sideband closed before session.started")); + socket.on("message", onMessage); + socket.once("error", onError); + socket.once("close", onClose); + }); +} + +async function sendSessionClose(socket: WebSocket): Promise { + await new Promise((resolve, reject) => { + socket.send(JSON.stringify({ type: "session.close" }), (error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }); + socket.close(1000, "test complete"); +} + +async function resolveLiveOAuthProfile(): Promise< + Extract | undefined +> { + const credential = readCodexCliCredentialsCached({ + allowKeychainPrompt: false, + ttlMs: 0, + }); + if (!credential) { + return undefined; + } + const accountId = + credential.accountId ?? resolveCodexAuthIdentity({ accessToken: credential.access }).accountId; + return accountId ? { type: "oauth", token: credential.access, accountId } : undefined; +} + +describeLive("GPT-Live OAuth WebRTC", () => { + it( + "creates a call and joins the authenticated sideband", + async ({ skip }) => { + const auth = await resolveLiveOAuthProfile(); + if (!auth) { + skip("No ChatGPT OAuth profile is available"); + return; + } + + const { chromium } = await import("playwright"); + const browser = await chromium.launch({ + headless: true, + args: ["--no-sandbox", "--use-fake-device-for-media-stream"], + }); + const page = await browser.newPage(); + let sideband: WebSocket | undefined; + try { + const offerSdp = await createBrowserOffer(page); + const requestIds = { + realtimeSessionId: randomUUID(), + sessionId: randomUUID(), + threadId: randomUUID(), + }; + const call = await createOpenAIQuicksilverCall({ + auth, + requestIds, + sdp: offerSdp, + session: buildOpenAIQuicksilverSession({ + model: "gpt-live-1-codex", + instructions: "Keep this transport verification session silent.", + voice: "marin", + }), + }); + + expect(call.status).toBe(201); + expect(call.callId).toMatch(/^rtc_[\w-]+$/); + expect(call.answerSdp).toMatch(/^v=0/m); + await applyBrowserAnswer(page, call.answerSdp); + + const started = await waitForSidebandSessionStarted({ + url: call.sidebandUrl, + headers: openAIQuicksilverAuthHeaders(auth, requestIds), + }); + sideband = started.socket; + expect(started.sessionId).toBe(call.callId); + await sendSessionClose(sideband); + sideband = undefined; + } finally { + sideband?.close(1000, "test cleanup"); + await closeBrowserPeer(page).catch(() => undefined); + await browser.close(); + } + }, + LIVE_TIMEOUT_MS, + ); +}); diff --git a/extensions/openai/realtime-quicksilver.test-helpers.ts b/extensions/openai/realtime-quicksilver.test-helpers.ts new file mode 100644 index 000000000000..985fbbd802b3 --- /dev/null +++ b/extensions/openai/realtime-quicksilver.test-helpers.ts @@ -0,0 +1,141 @@ +import { EventEmitter } from "node:events"; +import type { IncomingMessage, ServerResponse } from "node:http"; +import { Readable } from "node:stream"; +import { vi } from "vitest"; +import { createOpenAIQuicksilverBrowserSessionBroker } from "./realtime-quicksilver-session.js"; + +export class FakeSocket extends EventEmitter { + readyState: 0 | 1 | 2 | 3 = 0; + sent: string[] = []; + closed = false; + closeCode?: number; + closeReason?: string; + + constructor(autoEvent: "open" | "error" | "close" | "manual" = "open") { + super(); + queueMicrotask(() => { + if (autoEvent === "open") { + this.readyState = 1; + this.emit("open"); + } else if (autoEvent === "error") { + this.emit("error", new Error("transient sideband failure")); + } else if (autoEvent === "close") { + this.emit("close"); + } + }); + } + + send(payload: string): void { + this.sent.push(payload); + } + + close(code?: number, reason?: string): void { + if (this.closed) { + return; + } + this.closed = true; + this.closeCode = code; + this.closeReason = reason; + this.readyState = 3; + this.emit("close"); + } +} + +export function createRequest(params: { + method?: string; + token?: string; + origin?: string; + host?: string; + contentType?: string; + body?: string; +}): IncomingMessage { + return Object.assign(Readable.from([params.body ?? "v=offer\r\n"]), { + method: params.method ?? "POST", + headers: { + ...(params.token ? { authorization: `Bearer ${params.token}` } : {}), + ...(params.contentType === undefined + ? { "content-type": "application/sdp" } + : params.contentType + ? { "content-type": params.contentType } + : {}), + ...(params.origin ? { origin: params.origin } : {}), + ...(params.host ? { host: params.host } : {}), + }, + }) as unknown as IncomingMessage; +} + +export function createPreflightRequest(origin: string, host?: string): IncomingMessage { + return Object.assign(Readable.from([]), { + method: "OPTIONS", + headers: { + origin, + ...(host ? { host } : {}), + "access-control-request-method": "POST", + "access-control-request-headers": "authorization,content-type", + "access-control-request-private-network": "true", + }, + }) as unknown as IncomingMessage; +} + +export function createResponseHarness(): { + res: ServerResponse; + end: ReturnType; + setHeader: ReturnType; + readBody: () => string; +} { + let body = ""; + const setHeader = vi.fn(); + const end = vi.fn((value?: string) => { + body = value ?? ""; + queueMicrotask(() => res.emit("finish")); + }); + const res = Object.assign(new EventEmitter(), { + statusCode: 200, + setHeader, + end, + }) as unknown as ServerResponse; + return { res, end, setHeader, readBody: () => body }; +} + +export function createCallResponse(answer = "v=answer\r\n", callId = "rtc_test"): Response { + return new Response(answer, { + status: 201, + headers: { Location: `/v1/live/${callId}?source=test` }, + }); +} + +export function parseSent(socket: FakeSocket): Array> { + return socket.sent.map((payload) => JSON.parse(payload) as Record); +} + +export function emitSideband(socket: FakeSocket, payload: unknown, isBinary = false): void { + socket.emit("message", Buffer.from(JSON.stringify(payload)), isBinary); +} + +export function createBroker(params?: { + fetchImpl?: typeof fetch; + runAgentConsult?: (params: { prompt: string; signal?: AbortSignal }) => Promise<{ text: string }>; + socketFactory?: (attempt: number) => FakeSocket; +}) { + const sockets: FakeSocket[] = []; + const socketRequests: Array<{ url: string; headers?: Record }> = []; + const logger = { debug: vi.fn(), warn: vi.fn() }; + const realtime = createOpenAIQuicksilverBrowserSessionBroker({ + getConfig: () => ({ + gateway: { controlUi: { allowedOrigins: ["https://control.example"] } }, + }), + logger, + fetchImpl: params?.fetchImpl ?? vi.fn(async () => createCallResponse()), + webSocketFactory: (url, options) => { + const socket = params?.socketFactory?.(sockets.length) ?? new FakeSocket(); + sockets.push(socket); + socketRequests.push({ + url, + headers: options.headers as Record | undefined, + }); + return socket; + }, + }); + const runAgentConsult = params?.runAgentConsult ?? vi.fn(async () => ({ text: "Done" })); + return { realtime, sockets, socketRequests, logger, runAgentConsult }; +} diff --git a/extensions/openai/realtime-quicksilver.test.ts b/extensions/openai/realtime-quicksilver.test.ts index 20d7e2101ced..577dceeab829 100644 --- a/extensions/openai/realtime-quicksilver.test.ts +++ b/extensions/openai/realtime-quicksilver.test.ts @@ -3,7 +3,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { isOpenAIGptLiveModel, OPENAI_GPT_LIVE_BRIDGE_UNSUPPORTED_MESSAGE, - OPENAI_GPT_LIVE_BROWSER_SESSION_UNSUPPORTED_MESSAGE, } from "./realtime-quicksilver.js"; import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js"; @@ -34,28 +33,18 @@ describe("openai gpt-live model detection", () => { }); }); -describe("openai realtime voice provider with gpt-live models", () => { +describe("openai realtime voice provider gpt-live transport guard", () => { beforeEach(() => { mintSecretMock.mockReset(); mintSecretMock.mockResolvedValue({ value: "ek_test", expiresAt: 1234 }); }); - it("fails closed for gpt-live browser sessions with guidance", async () => { - const provider = buildOpenAIRealtimeVoiceProvider(); - await expect( - provider.createBrowserSession?.({ - providerConfig: { apiKey: "test-key" }, - model: "gpt-live-1", - }), - ).rejects.toThrow(OPENAI_GPT_LIVE_BROWSER_SESSION_UNSUPPORTED_MESSAGE); - expect(mintSecretMock).not.toHaveBeenCalled(); - }); - it("keeps GA realtime browser sessions working", async () => { const provider = buildOpenAIRealtimeVoiceProvider(); const session = await provider.createBrowserSession?.({ providerConfig: { apiKey: "test-key" }, model: "gpt-realtime-2.1", + instructions: "Keep GA behavior unchanged.", }); expect(session).toMatchObject({ transport: "webrtc", @@ -64,6 +53,7 @@ describe("openai realtime voice provider with gpt-live models", () => { expect(mintSecretMock.mock.calls[0]?.[0]?.session).toMatchObject({ type: "realtime", model: "gpt-realtime-2.1", + instructions: "Keep GA behavior unchanged.", }); }); diff --git a/extensions/openai/realtime-quicksilver.ts b/extensions/openai/realtime-quicksilver.ts index 9d70c8529922..5bb525ae3afc 100644 --- a/extensions/openai/realtime-quicksilver.ts +++ b/extensions/openai/realtime-quicksilver.ts @@ -1,24 +1,12 @@ -// GPT-Live (OpenAI "quicksilver" session type) gating for realtime voice. -// -// GPT-Live models (gpt-live-1, gpt-live-1-mini) are full-duplex quicksilver -// sessions: WebRTC-only (the API rejects websocket session creation with -// "Quicksilver sessions require WebRTC", probed 2026-07-11), no client -// turn_detection or tools, and agent delegation happens via -// conversation.handoff.* events instead of GA function-call events. The Talk -// browser client only speaks the GA Realtime protocol today, so routing -// GPT-Live into it would connect audio while agent consult silently never -// fires. Both transports fail closed with guidance until the handoff protocol -// is implemented (wire shape reference: openai/codex codex-rs realtime v1); -// the API is also org-gated until OpenAI opens GPT-Live access. +// GPT-Live (OpenAI "quicksilver") browser Talk routing uses the live-proven +// ChatGPT OAuth WebRTC and sideband path. Relay and backend WebSocket bridges +// stay unsupported; Platform API-key access remains waitlist-gated. const OPENAI_GPT_LIVE_MODEL_PREFIX = "gpt-live"; export const OPENAI_GPT_LIVE_BRIDGE_UNSUPPORTED_MESSAGE = "GPT-Live models are not supported on the realtime WebSocket bridge: OpenAI requires WebRTC for quicksilver sessions. Set a gpt-realtime model for this transport."; -export const OPENAI_GPT_LIVE_BROWSER_SESSION_UNSUPPORTED_MESSAGE = - "GPT-Live models are not supported for Talk browser sessions yet: quicksilver sessions delegate through conversation.handoff events that the Talk client does not implement. Set a gpt-realtime model until GPT-Live support lands."; - export function isOpenAIGptLiveModel(model: string | undefined): boolean { if (!model) { return false; diff --git a/extensions/openai/realtime-voice-provider.test.ts b/extensions/openai/realtime-voice-provider.test.ts index 116c66f56b2e..13aca9c0c49b 100644 --- a/extensions/openai/realtime-voice-provider.test.ts +++ b/extensions/openai/realtime-voice-provider.test.ts @@ -4,6 +4,29 @@ import type { RealtimeVoiceBridge, RealtimeVoiceTool } from "openclaw/plugin-sdk import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js"; +const INTERNAL_REALTIME_VOICE_PROVIDER = Symbol.for("openclaw.internal.realtime-voice-provider.v1"); + +function readInternalRealtimeVoiceProviderApi(provider: object) { + return Reflect.get(provider, INTERNAL_REALTIME_VOICE_PROVIDER) as { + isBrowserSessionConfigured: (ctx: { + cfg?: object; + providerConfig: Record; + agentId?: string; + }) => boolean; + resolveBrowserSessionCapabilities: (ctx: { + cfg?: object; + providerConfig: Record; + model?: string; + }) => { + handlesAgentConsult?: boolean; + supportsToolCalls?: boolean; + supportsVideoFrames?: boolean; + transports?: string[]; + }; + cancelBrowserSession: (request: Record, session: object) => Promise; + }; +} + const { FakeWebSocket, execFileSyncMock, @@ -269,6 +292,14 @@ function createMalformedToolName(name: unknown): RealtimeVoiceTool { } as unknown as RealtimeVoiceTool; } +function createTestJwt(payload: Record): string { + return [ + Buffer.from(JSON.stringify({ alg: "none" })).toString("base64url"), + Buffer.from(JSON.stringify(payload)).toString("base64url"), + "test-signature", + ].join("."); +} + describe("buildOpenAIRealtimeVoiceProvider", () => { beforeEach(() => { FakeWebSocket.instances = []; @@ -320,6 +351,35 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { expect(bridge.supportsToolResultSuppression).toBe(true); }); + it("uses quicksilver capabilities for a request-model override", () => { + const quicksilverBroker = { + capabilities: { + transports: ["webrtc" as const], + handlesAgentConsult: true as const, + supportsToolCalls: false, + supportsVideoFrames: false, + }, + createBrowserSession: vi.fn(), + cancelBrowserSession: vi.fn(), + }; + const provider = buildOpenAIRealtimeVoiceProvider({ + quicksilverBrowserSessionBroker: quicksilverBroker, + }); + const internalApi = readInternalRealtimeVoiceProviderApi(provider); + + expect( + internalApi.resolveBrowserSessionCapabilities({ + providerConfig: { model: "gpt-realtime-2.1" }, + model: "gpt-live-1-mini", + }), + ).toMatchObject({ + transports: ["webrtc"], + handlesAgentConsult: true, + supportsToolCalls: false, + supportsVideoFrames: false, + }); + }); + it("adds OpenClaw attribution headers to native realtime websocket requests", () => { vi.stubEnv("OPENCLAW_VERSION", "2026.3.22"); const provider = buildOpenAIRealtimeVoiceProvider(); @@ -726,9 +786,254 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { }); }); - it("requires Platform auth for browser sessions", async () => { + it("routes gpt-live Platform sessions through the native quicksilver broker", async () => { + const createBrowserSession = vi.fn(async (_request: unknown, _auth: unknown) => ({ + provider: "openai", + transport: "webrtc" as const, + clientSecret: "quicksilver-token", + offerUrl: "/plugins/openai/realtime/calls", + })); + const provider = buildOpenAIRealtimeVoiceProvider({ + quicksilverBrowserSessionBroker: { + capabilities: { + transports: ["webrtc" as const], + handlesAgentConsult: true as const, + supportsToolCalls: false, + supportsVideoFrames: false, + }, + createBrowserSession, + cancelBrowserSession: vi.fn(), + }, + }); + const request = { + providerConfig: { apiKey: "sk-platform" }, // pragma: allowlist secret + model: "gpt-live-1", + agentId: "main", + workspaceDir: "/tmp/openclaw-agent-workspace", + initialItems: [], + runAgentConsult: vi.fn(async () => ({ text: "Done" })), + }; + + await expect(provider.createBrowserSession?.(request)).resolves.toMatchObject({ + offerUrl: "/plugins/openai/realtime/calls", + }); + expect(createBrowserSession).toHaveBeenCalledWith(expect.objectContaining(request), { + type: "api-key", + token: "sk-platform", // pragma: allowlist secret + }); + expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled(); + }); + + it("routes gpt-live ChatGPT OAuth sessions through the native quicksilver broker", async () => { + const oauthToken = createTestJwt({ + "https://api.openai.com/auth": { chatgpt_account_id: "account-123" }, + }); + resolveProviderAuthProfileApiKeyMock.mockImplementation( + async ({ profileTypes }: { profileTypes?: readonly string[] }) => + profileTypes?.includes("oauth") ? oauthToken : undefined, + ); + isProviderAuthProfileConfiguredMock.mockImplementation( + ({ profileTypes }: { profileTypes?: readonly string[] }) => + profileTypes?.includes("oauth") === true, + ); + const createBrowserSession = vi.fn(async () => ({ + provider: "openai", + transport: "webrtc" as const, + clientSecret: "quicksilver-token", + offerUrl: "/plugins/openai/realtime/calls", + })); + const provider = buildOpenAIRealtimeVoiceProvider({ + quicksilverBrowserSessionBroker: { + capabilities: { + transports: ["webrtc" as const], + handlesAgentConsult: true as const, + supportsToolCalls: false, + supportsVideoFrames: false, + }, + createBrowserSession, + cancelBrowserSession: vi.fn(), + }, + }); + const cfg = { agents: { defaults: {} } } as never; + const request = { + cfg, + providerConfig: {}, + model: "gpt-live-1-mini", + agentId: "main", + workspaceDir: "/tmp/openclaw-agent-workspace", + initialItems: [], + runAgentConsult: vi.fn(async () => ({ text: "Done" })), + }; + + expect(provider.isConfigured({ cfg, providerConfig: { model: "gpt-live-1-mini" } })).toBe( + false, + ); + expect( + readInternalRealtimeVoiceProviderApi(provider).isBrowserSessionConfigured({ + cfg, + providerConfig: { model: "gpt-live-1-mini" }, + agentId: "main", + }), + ).toBe(true); + await provider.createBrowserSession?.(request); + expect(createBrowserSession).toHaveBeenCalledWith(expect.objectContaining(request), { + type: "oauth", + token: oauthToken, + accountId: "account-123", + }); + expect(resolveProviderAuthProfileApiKeyMock).toHaveBeenCalledWith( + expect.objectContaining({ + provider: "openai", + profileTypes: ["oauth"], + includeExternalCliAuth: false, + }), + ); + }); + + it("prefers ChatGPT OAuth over Platform auth for gpt-live", async () => { + const oauthToken = createTestJwt({ + "https://api.openai.com/auth": { chatgpt_account_id: "account-123" }, + }); + resolveProviderAuthProfileApiKeyMock.mockImplementation( + async ({ profileTypes }: { profileTypes?: readonly string[] }) => + profileTypes?.includes("oauth") ? oauthToken : undefined, + ); + const createBrowserSession = vi.fn(async () => ({ + provider: "openai", + transport: "webrtc" as const, + clientSecret: "quicksilver-token", + offerUrl: "/plugins/openai/realtime/calls", + })); + const provider = buildOpenAIRealtimeVoiceProvider({ + quicksilverBrowserSessionBroker: { + capabilities: { handlesAgentConsult: true as const }, + createBrowserSession, + cancelBrowserSession: vi.fn(), + }, + }); + + await provider.createBrowserSession?.({ + providerConfig: { apiKey: "sk-platform" }, // pragma: allowlist secret + model: "gpt-live-1-codex", + agentId: "main", + workspaceDir: "/tmp/openclaw-agent-workspace", + initialItems: [], + runAgentConsult: vi.fn(async () => ({ text: "Done" })), + } as never); + + expect(createBrowserSession).toHaveBeenCalledWith(expect.any(Object), { + type: "oauth", + token: oauthToken, + accountId: "account-123", + }); + }); + + it("keeps Platform precedence for GA realtime when OAuth is also available", async () => { + const oauthToken = createTestJwt({ + "https://api.openai.com/auth": { chatgpt_account_id: "account-123" }, + }); + resolveProviderAuthProfileApiKeyMock.mockImplementation( + async ({ profileTypes }: { profileTypes?: readonly string[] }) => + profileTypes?.includes("oauth") ? oauthToken : undefined, + ); + fetchWithSsrFGuardMock.mockResolvedValueOnce({ + response: createJsonResponse({ client_secret: { value: "client-secret-123" } }), + release: vi.fn(async () => undefined), + }); const provider = buildOpenAIRealtimeVoiceProvider(); + await provider.createBrowserSession?.({ + providerConfig: { apiKey: "sk-platform" }, // pragma: allowlist secret + model: "gpt-realtime-2.1", + }); + + expect(resolveProviderAuthProfileApiKeyMock).not.toHaveBeenCalledWith( + expect.objectContaining({ profileTypes: ["oauth"] }), + ); + expectRecordFields(requireFetchHeaders(), "fetch headers", { + Authorization: "Bearer sk-platform", // pragma: allowlist secret + }); + }); + + it("passes configured gpt-live model and voice to the native broker", async () => { + const createBrowserSession = vi.fn(async (_request: unknown, _auth: unknown) => ({ + provider: "openai", + transport: "webrtc" as const, + clientSecret: "quicksilver-token", + offerUrl: "/plugins/openai/realtime/calls", + })); + const provider = buildOpenAIRealtimeVoiceProvider({ + quicksilverBrowserSessionBroker: { + capabilities: { + transports: ["webrtc" as const], + handlesAgentConsult: true as const, + supportsToolCalls: false, + supportsVideoFrames: false, + }, + createBrowserSession, + cancelBrowserSession: vi.fn(), + }, + }); + + await provider.createBrowserSession?.({ + providerConfig: { + apiKey: "sk-platform", // pragma: allowlist secret + model: "gpt-live-1", + speakerVoice: "cedar", + }, + instructions: "Always address the caller as Captain.", + agentId: "voice-agent", + workspaceDir: "/tmp/openclaw-agent-workspace", + initialItems: [], + runAgentConsult: vi.fn(async () => ({ text: "Done" })), + } as never); + + expect(createBrowserSession).toHaveBeenCalledWith( + expect.objectContaining({ model: "gpt-live-1", voice: "cedar" }), + { type: "api-key", token: "sk-platform" }, // pragma: allowlist secret + ); + const quicksilverRequest = requireRecord( + createBrowserSession.mock.calls[0]?.[0], + "quicksilver request", + ); + expect(quicksilverRequest.instructions).toMatch(/^You are OpenClaw's realtime voice layer\./); + expect(quicksilverRequest.instructions).toContain( + "Context on the commentary channel is silent background", + ); + expect(quicksilverRequest.instructions).toContain( + "Context on the speakable channel is your answer", + ); + expect(quicksilverRequest.instructions).toMatch(/Always address the caller as Captain\.$/); + }); + + it("explains both gpt-live authentication options when neither is available", async () => { + const createBrowserSession = vi.fn(); + const provider = buildOpenAIRealtimeVoiceProvider({ + quicksilverBrowserSessionBroker: { + capabilities: { + transports: ["webrtc" as const], + handlesAgentConsult: true as const, + supportsToolCalls: false, + supportsVideoFrames: false, + }, + createBrowserSession, + cancelBrowserSession: vi.fn(), + }, + }); + + await expect( + provider.createBrowserSession?.({ + providerConfig: {}, + model: "gpt-live-1", + }), + ).rejects.toThrow( + "GPT-Live Talk requires either an OpenAI Platform API key or a ChatGPT OAuth subscription profile", + ); + expect(createBrowserSession).not.toHaveBeenCalled(); + }); + + it("requires Platform auth for browser sessions", async () => { + const provider = buildOpenAIRealtimeVoiceProvider(); await expect( provider.createBrowserSession?.({ providerConfig: {}, diff --git a/extensions/openai/realtime-voice-provider.ts b/extensions/openai/realtime-voice-provider.ts index 42d73494a9de..59328f1e0c40 100644 --- a/extensions/openai/realtime-voice-provider.ts +++ b/extensions/openai/realtime-voice-provider.ts @@ -1,6 +1,7 @@ // Openai provider module implements model/runtime integration. import { execFileSync } from "node:child_process"; import { randomUUID } from "node:crypto"; +import { resolveAgentDir } from "openclaw/plugin-sdk/agent-runtime"; import { canonicalizeBase64 } from "openclaw/plugin-sdk/media-runtime"; import { isProviderAuthProfileConfigured, @@ -43,10 +44,15 @@ import { resolveOpenAIProviderConfigRecord, trimToUndefined, } from "./realtime-provider-shared.js"; +import { buildOpenAIQuicksilverInstructions } from "./realtime-quicksilver-instructions.js"; +import { + createOpenAIQuicksilverBrowserSessionBroker, + OPENAI_QUICKSILVER_CAPABILITIES, + resolveOpenAIChatGptSubscriptionAuth, +} from "./realtime-quicksilver-session.js"; import { isOpenAIGptLiveModel, OPENAI_GPT_LIVE_BRIDGE_UNSUPPORTED_MESSAGE, - OPENAI_GPT_LIVE_BROWSER_SESSION_UNSUPPORTED_MESSAGE, } from "./realtime-quicksilver.js"; type OpenAIRealtimeVoice = @@ -273,6 +279,10 @@ type OpenAIRealtimeApiKeyResolution = const OPENAI_REALTIME_PLATFORM_AUTH_REQUIRED = "OpenAI Realtime voice requires an OpenAI Platform API key"; +const OPENAI_GPT_LIVE_AUTH_REQUIRED = + "GPT-Live Talk requires either an OpenAI Platform API key or a ChatGPT OAuth subscription profile"; +const OPENAI_GPT_LIVE_AUTHORED_PLATFORM_AUTH_UNAVAILABLE = + "GPT-Live Talk requires a working OpenAI Platform API key or ChatGPT OAuth subscription profile. The selected Platform API-key source could not be resolved, so OAuth fallback was not used; fix or remove it."; const OPENAI_REALTIME_API_KEY_REQUIRED = "OpenAI Realtime voice requires an API key"; const OPENAI_REALTIME_CONFIGURED_API_KEY_REJECTED = "OpenAI Realtime rejected the selected API key. Update or remove the active OpenAI API-key source"; @@ -474,6 +484,20 @@ function hasOpenAIRealtimePlatformAuthInput(params: { return hasOpenAIRealtimeApiKeyInput(undefined); } +function hasOpenAIChatGptSubscriptionAuthInput(params: { + cfg: RealtimeVoiceBrowserSessionCreateRequest["cfg"] | undefined; + agentId?: string; +}): boolean { + return isProviderAuthProfileConfigured({ + provider: "openai", + cfg: params.cfg, + agentDir: + params.cfg && params.agentId ? resolveAgentDir(params.cfg, params.agentId) : undefined, + profileTypes: ["oauth"], + includeExternalCliAuth: false, + }); +} + function isOpenAIRealtimeMaxSessionDurationError(detail: string): boolean { const normalized = detail.toLowerCase(); return ( @@ -1549,14 +1573,104 @@ function resolveOpenAIRealtimeBrowserOfferHeaders(): Record | un return Object.keys(browserHeaders).length > 0 ? browserHeaders : undefined; } +type OpenAIQuicksilverBrowserSessionBroker = ReturnType< + typeof createOpenAIQuicksilverBrowserSessionBroker +>["broker"]; + +type OpenAIInternalRealtimeBrowserSessionCreateRequest = + RealtimeVoiceBrowserSessionCreateRequest & { + agentId: string; + workspaceDir: string; + initialItems: Array<{ + role: "user" | "assistant"; + text: string; + }>; + }; + +type OpenAIInternalRealtimeVoiceCapabilities = RealtimeVoiceProviderCapabilities & { + handlesAgentConsult?: boolean; +}; + +type OpenAIInternalRealtimeVoiceProviderApi = { + isBrowserSessionConfigured: (ctx: { + cfg?: RealtimeVoiceBrowserSessionCreateRequest["cfg"]; + providerConfig: RealtimeVoiceProviderConfig; + agentId?: string; + }) => boolean; + resolveBrowserSessionCapabilities?: (ctx: { + cfg?: RealtimeVoiceBrowserSessionCreateRequest["cfg"]; + providerConfig: RealtimeVoiceProviderConfig; + model?: string; + }) => OpenAIInternalRealtimeVoiceCapabilities; + cancelBrowserSession?: ( + request: OpenAIInternalRealtimeBrowserSessionCreateRequest, + session: RealtimeVoiceBrowserSession, + ) => Promise | void; +}; + +const INTERNAL_REALTIME_VOICE_PROVIDER = Symbol.for("openclaw.internal.realtime-voice-provider.v1"); + +const quicksilverBrokerBySession = new WeakMap< + RealtimeVoiceBrowserSession, + OpenAIQuicksilverBrowserSessionBroker +>(); + async function createOpenAIRealtimeBrowserSession( - req: RealtimeVoiceBrowserSessionCreateRequest, + req: OpenAIInternalRealtimeBrowserSessionCreateRequest, + quicksilverBroker: OpenAIQuicksilverBrowserSessionBroker | undefined, ): Promise { + const rawConfig = resolveOpenAIProviderConfigRecord(req.providerConfig); const config = normalizeProviderConfig(req.providerConfig); if (config.azureEndpoint || config.azureDeployment) { throw new Error("OpenAI Realtime browser sessions do not support Azure endpoints yet"); } + const model = req.model ?? config.model ?? OPENAI_REALTIME_DEFAULT_MODEL; + if (isOpenAIGptLiveModel(model)) { + if (!quicksilverBroker) { + throw new Error("OpenAI GPT-Live browser session broker is unavailable"); + } + const configuredVoice = trimToUndefined(rawConfig?.speakerVoice ?? rawConfig?.voice); + const quicksilverRequest = { + ...req, + model, + instructions: buildOpenAIQuicksilverInstructions(req.instructions), + ...(req.voice ? {} : configuredVoice ? { voice: configuredVoice } : {}), + }; + const subscriptionAuth = await resolveOpenAIChatGptSubscriptionAuth({ + cfg: req.cfg, + agentDir: req.cfg ? resolveAgentDir(req.cfg, req.agentId) : undefined, + }); + if (subscriptionAuth) { + const session = await quicksilverBroker.createBrowserSession( + quicksilverRequest, + subscriptionAuth, + ); + quicksilverBrokerBySession.set(session, quicksilverBroker); + return session; + } + const auth = await resolveOpenAIRealtimePlatformAuth({ + configuredApiKey: config.apiKey, + cfg: req.cfg, + }); + if (auth.status === "available") { + const session = await quicksilverBroker.createBrowserSession(quicksilverRequest, { + type: "api-key", + token: auth.value, + }); + quicksilverBrokerBySession.set(session, quicksilverBroker); + return session; + } + if ( + hasOpenAIRealtimePlatformAuthInput({ + configuredApiKey: config.apiKey, + cfg: req.cfg, + }) + ) { + throw new Error(OPENAI_GPT_LIVE_AUTHORED_PLATFORM_AUTH_UNAVAILABLE); + } + throw new Error(OPENAI_GPT_LIVE_AUTH_REQUIRED); + } const auth = await resolveOpenAIRealtimePlatformAuth({ configuredApiKey: config.apiKey, cfg: req.cfg, @@ -1565,10 +1679,6 @@ async function createOpenAIRealtimeBrowserSession( throw new Error(OPENAI_REALTIME_PLATFORM_AUTH_REQUIRED); } - const model = req.model ?? config.model ?? OPENAI_REALTIME_DEFAULT_MODEL; - if (isOpenAIGptLiveModel(model)) { - throw new Error(OPENAI_GPT_LIVE_BROWSER_SESSION_UNSUPPORTED_MESSAGE); - } const voice = normalizeOpenAIRealtimeVoice(req.voice) ?? config.voice ?? "alloy"; const tools = normalizeOpenAIRealtimeTools(req.tools); const session: Record = { @@ -1625,7 +1735,18 @@ async function createOpenAIRealtimeBrowserSession( }; } -export function buildOpenAIRealtimeVoiceProvider(): RealtimeVoiceProviderPlugin { +async function cancelOpenAIRealtimeBrowserSession( + _req: OpenAIInternalRealtimeBrowserSessionCreateRequest, + session: RealtimeVoiceBrowserSession, +): Promise { + const quicksilverBroker = quicksilverBrokerBySession.get(session); + quicksilverBrokerBySession.delete(session); + quicksilverBroker?.cancelBrowserSession(session); +} + +export function buildOpenAIRealtimeVoiceProvider(options?: { + quicksilverBrowserSessionBroker?: OpenAIQuicksilverBrowserSessionBroker; +}): RealtimeVoiceProviderPlugin { const provider: RealtimeVoiceProviderPlugin = { id: "openai", label: "OpenAI Realtime Voice", @@ -1638,10 +1759,15 @@ export function buildOpenAIRealtimeVoiceProvider(): RealtimeVoiceProviderPlugin if (config.azureEndpoint || config.azureDeployment) { return hasOpenAIRealtimeApiKeyInput(config.apiKey); } - return hasOpenAIRealtimePlatformAuthInput({ - configuredApiKey: config.apiKey, - cfg, - }); + if ( + hasOpenAIRealtimePlatformAuthInput({ + configuredApiKey: config.apiKey, + cfg, + }) + ) { + return true; + } + return false; }, createBridge: (req) => { const config = normalizeProviderConfig(req.providerConfig); @@ -1666,8 +1792,47 @@ export function buildOpenAIRealtimeVoiceProvider(): RealtimeVoiceProviderPlugin azureApiVersion: config.azureApiVersion, }); }, - createBrowserSession: createOpenAIRealtimeBrowserSession, + createBrowserSession: (req) => + createOpenAIRealtimeBrowserSession( + req as OpenAIInternalRealtimeBrowserSessionCreateRequest, + options?.quicksilverBrowserSessionBroker, + ), }; + const internalApi: OpenAIInternalRealtimeVoiceProviderApi = { + isBrowserSessionConfigured: ({ cfg, providerConfig, agentId }) => { + const config = normalizeProviderConfig(providerConfig); + if (config.azureEndpoint || config.azureDeployment) { + return false; + } + const model = config.model ?? OPENAI_REALTIME_DEFAULT_MODEL; + if (isOpenAIGptLiveModel(model)) { + return ( + options?.quicksilverBrowserSessionBroker !== undefined && + (hasOpenAIRealtimePlatformAuthInput({ + configuredApiKey: config.apiKey, + cfg, + }) || + hasOpenAIChatGptSubscriptionAuthInput({ cfg, agentId })) + ); + } + return false; + }, + resolveBrowserSessionCapabilities: ({ providerConfig, model }) => { + const config = normalizeProviderConfig(providerConfig); + if (isOpenAIGptLiveModel(model ?? config.model)) { + return { + ...OPENAI_REALTIME_CAPABILITIES, + ...OPENAI_QUICKSILVER_CAPABILITIES, + }; + } + return OPENAI_REALTIME_CAPABILITIES; + }, + cancelBrowserSession: cancelOpenAIRealtimeBrowserSession, + }; + Object.defineProperty(provider, INTERNAL_REALTIME_VOICE_PROVIDER, { + configurable: true, + value: internalApi, + }); return provider; } /* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 53e0d69cacfc..747505da0081 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1480,6 +1480,9 @@ importers: ws: specifier: 8.21.1 version: 8.21.1 + zod: + specifier: 4.4.3 + version: 4.4.3 devDependencies: '@openclaw/plugin-sdk': specifier: workspace:* diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index 0dae0aa43ea3..e75950721ef9 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -195,7 +195,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +7: bounded archive extraction, entry reads, errors, and policy types. // +3: root-bounded walk iterator, options, and entry contract. // +5: pinned secret create/read functions and their options contract. - 4755, + // +1: canonical Gateway browser-origin acceptance for browser-facing plugin routes. + 4756, env, ), publicFunctionExports: readPluginSdkSurfaceBudgetEnv( @@ -228,7 +229,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +2: bounded archive extraction and single-entry reads. // +1: root-bounded directory walk iterator. // +4: pinned secret create and synchronous/asynchronous reads. - 2876, + // +1: canonical Gateway browser-origin acceptance for browser-facing plugin routes. + 2877, env, ), publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv( diff --git a/src/gateway/auth.ts b/src/gateway/auth.ts index bc6abf9fcc61..134c49cd423d 100644 --- a/src/gateway/auth.ts +++ b/src/gateway/auth.ts @@ -15,6 +15,7 @@ import { } from "./auth-rate-limit.js"; import type { ResolvedGatewayAuth } from "./auth-resolve.js"; import { + isLocalDirectRequest, isLoopbackAddress, resolveLocalInterfaceAddressMatch, resolveRequestClientIp, @@ -28,6 +29,7 @@ export { resolveGatewayAuth, type ResolvedGatewayAuth, } from "./auth-resolve.js"; +export { hasForwardedRequestHeaders, isLocalDirectRequest } from "./net.js"; const LEGACY_OPENCLAW_ENV_NOTE = " Legacy CLAWDBOT_* and MOLTBOT_* environment variables are ignored; use OPENCLAW_* names."; @@ -149,38 +151,6 @@ function resolveTailscaleClientIp(req?: IncomingMessage): string | undefined { }); } -/** Detect forwarded/proxy headers that make loopback requests ineligible for direct-local auth. */ -/** Return true when forwarded headers make loopback direct-local auth unsafe. */ -export function hasForwardedRequestHeaders(req?: IncomingMessage): boolean { - if (!req) { - return false; - } - const headers = req.headers ?? {}; - - return Boolean( - headers.forwarded || - headers["x-real-ip"] || - Object.keys(headers).some((header) => - normalizeLowercaseStringOrEmpty(header).startsWith("x-forwarded-"), - ), - ); -} - -/** Return whether a request is a clean loopback request without forwarded identity headers. */ -export function isLocalDirectRequest( - req?: IncomingMessage, - _trustedProxies?: string[], - _allowRealIpFallback = false, -): boolean { - if (!req) { - return false; - } - if (!hasForwardedRequestHeaders(req)) { - return isLoopbackAddress(req.socket?.remoteAddress); - } - return false; -} - function getTailscaleUser(req?: IncomingMessage): TailscaleUser | null { if (!req) { return null; diff --git a/src/gateway/http-auth-utils.ts b/src/gateway/http-auth-utils.ts index 64faa1a2dcd6..af11a6294c82 100644 --- a/src/gateway/http-auth-utils.ts +++ b/src/gateway/http-auth-utils.ts @@ -24,6 +24,7 @@ import { import { sendGatewayAuthFailure, sendMissingScopeForbidden } from "./http-common.js"; import { ADMIN_SCOPE, CLI_DEFAULT_OPERATOR_SCOPES } from "./method-scopes.js"; import { authorizeOperatorScopesForMethod } from "./method-scopes.js"; +import { resolveBrowserOriginPolicy } from "./origin-check.js"; import { resolveSharedGatewaySessionGeneration } from "./server/ws-shared-generation.js"; export function getHeader(req: IncomingMessage, name: string): string | undefined { @@ -69,13 +70,7 @@ export function resolveHttpBrowserOriginPolicy( req: IncomingMessage, cfg = getRuntimeConfig(), ): NonNullable[0]["browserOriginPolicy"]> { - return { - requestHost: getHeader(req, "host"), - origin: getHeader(req, "origin"), - allowedOrigins: cfg.gateway?.controlUi?.allowedOrigins, - allowHostHeaderOriginFallback: - cfg.gateway?.controlUi?.dangerouslyAllowHostHeaderOriginFallback === true, - }; + return resolveBrowserOriginPolicy({ req, cfg }); } function usesSharedSecretHttpAuth(auth: SharedSecretGatewayAuth | undefined): boolean { diff --git a/src/gateway/net.ts b/src/gateway/net.ts index 7f217c793f2e..8af7d4a2c4df 100644 --- a/src/gateway/net.ts +++ b/src/gateway/net.ts @@ -57,6 +57,32 @@ export function isLoopbackAddress(ip: string | undefined): boolean { return isLoopbackIpAddress(ip); } +/** Detect forwarded/proxy headers that make loopback requests ineligible for direct-local auth. */ +export function hasForwardedRequestHeaders(req?: IncomingMessage): boolean { + if (!req) { + return false; + } + const headers = req.headers ?? {}; + return Boolean( + headers.forwarded || + headers["x-real-ip"] || + Object.keys(headers).some((header) => + normalizeLowercaseStringOrEmpty(header).startsWith("x-forwarded-"), + ), + ); +} + +/** Return whether a request is a clean loopback request without forwarded identity headers. */ +export function isLocalDirectRequest( + req?: IncomingMessage, + _trustedProxies?: string[], + _allowRealIpFallback = false, +): boolean { + return Boolean( + req && !hasForwardedRequestHeaders(req) && isLoopbackAddress(req.socket?.remoteAddress), + ); +} + export function resolveLocalInterfaceAddressMatch( ip: string | undefined, snapshot?: NetworkInterfacesSnapshot, diff --git a/src/gateway/origin-check.test.ts b/src/gateway/origin-check.test.ts index 7add60b6e46d..6e93db0c3cb6 100644 --- a/src/gateway/origin-check.test.ts +++ b/src/gateway/origin-check.test.ts @@ -1,7 +1,12 @@ // Browser origin tests document same-origin, private-network, loopback, forwarded // host, and explicit allowlist decisions for gateway browser surfaces. +import type { IncomingMessage } from "node:http"; import { describe, expect, it } from "vitest"; -import { checkBrowserOrigin, normalizeChromeExtensionOrigin } from "./origin-check.js"; +import { + checkBrowserOrigin, + normalizeChromeExtensionOrigin, + resolveAcceptedBrowserOrigin, +} from "./origin-check.js"; describe("checkBrowserOrigin", () => { it.each([ @@ -184,3 +189,25 @@ describe("checkBrowserOrigin", () => { ).toBeUndefined(); }); }); + +describe("resolveAcceptedBrowserOrigin", () => { + it("applies the configured Host-header fallback through the canonical request resolver", () => { + const origin = "https://gateway.example.com:18789"; + const req = { + headers: { host: "gateway.example.com:18789", origin }, + socket: { remoteAddress: "203.0.113.10" }, + } as IncomingMessage; + + expect( + resolveAcceptedBrowserOrigin({ + req, + cfg: { + gateway: { + controlUi: { dangerouslyAllowHostHeaderOriginFallback: true }, + }, + }, + }), + ).toBe(origin); + expect(resolveAcceptedBrowserOrigin({ req, cfg: {} })).toBeUndefined(); + }); +}); diff --git a/src/gateway/origin-check.ts b/src/gateway/origin-check.ts index 8cc37a447fd2..0b7a6e729059 100644 --- a/src/gateway/origin-check.ts +++ b/src/gateway/origin-check.ts @@ -1,11 +1,18 @@ // Browser Origin validator for gateway HTTP and websocket requests. +import type { IncomingMessage } from "node:http"; import net from "node:net"; import { isPrivateOrLoopbackIpAddress } from "@openclaw/net-policy/ip"; import { normalizeLowercaseStringOrEmpty, normalizeOptionalLowercaseString, } from "@openclaw/normalization-core/string-coerce"; -import { isLoopbackHost, normalizeHostHeader, resolveHostName } from "./net.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { + isLocalDirectRequest, + isLoopbackHost, + normalizeHostHeader, + resolveHostName, +} from "./net.js"; type OriginCheckResult = | { @@ -14,6 +21,31 @@ type OriginCheckResult = } | { ok: false; reason: string }; +type BrowserOriginPolicy = { + requestHost?: string; + origin?: string; + allowedOrigins?: string[]; + allowHostHeaderOriginFallback?: boolean; +}; + +function headerValue(value: string | string[] | undefined): string | undefined { + return Array.isArray(value) ? value[0] : value; +} + +/** Gather the canonical Gateway browser-origin policy inputs for one HTTP request. */ +export function resolveBrowserOriginPolicy(params: { + req: IncomingMessage; + cfg?: OpenClawConfig; +}): BrowserOriginPolicy { + return { + requestHost: headerValue(params.req.headers.host), + origin: headerValue(params.req.headers.origin), + allowedOrigins: params.cfg?.gateway?.controlUi?.allowedOrigins, + allowHostHeaderOriginFallback: + params.cfg?.gateway?.controlUi?.dangerouslyAllowHostHeaderOriginFallback === true, + }; +} + function parseOrigin( originRaw?: string, ): { origin: string; protocol: string; host: string; hostname: string } | null { @@ -98,6 +130,25 @@ export function checkBrowserOrigin(params: { return { ok: false, reason: "origin not allowed" }; } +/** Return the request Origin only when the Gateway's canonical browser policy accepts it. */ +export function resolveAcceptedBrowserOrigin(params: { + req: IncomingMessage; + cfg?: OpenClawConfig; +}): string | undefined { + const policy = resolveBrowserOriginPolicy(params); + const origin = policy.origin?.trim(); + if (!origin) { + return undefined; + } + return checkBrowserOrigin({ + ...policy, + origin, + isLocalClient: isLocalDirectRequest(params.req), + }).ok + ? origin + : undefined; +} + function isTrustedSameOriginHost(hostHeader: string, isLocalClient?: boolean): boolean { const hostname = resolveHostName(hostHeader); if (!hostname) { diff --git a/src/gateway/server-methods/talk-client.ts b/src/gateway/server-methods/talk-client.ts index d558c0c9b866..2b24667638f2 100644 --- a/src/gateway/server-methods/talk-client.ts +++ b/src/gateway/server-methods/talk-client.ts @@ -14,7 +14,10 @@ import { validateTalkClientTranscriptParams, } from "../../../packages/gateway-protocol/src/index.js"; import { resolveAgentWorkspaceDir } from "../../agents/agent-scope.js"; +import { normalizeTalkSection } from "../../config/talk.js"; +import { createPluginRuntime } from "../../plugins/runtime/index.js"; import { buildAgentMainSessionKey } from "../../routing/session-key.js"; +import { consultRealtimeVoiceAgent } from "../../talk/agent-consult-runtime.js"; import { REALTIME_VOICE_AGENT_CONSULT_TOOL, REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, @@ -203,11 +206,18 @@ export const talkClientHandlers: GatewayRequestHandlers = { ); return; } + const launchOptions = buildRealtimeVoiceLaunchOptions({ + requested: typedParams, + defaults: realtimeConfig, + }); + const requestedAgentId = resolveTalkSessionAgentId(runtimeConfig, typedParams.sessionKey); const resolution = resolveConfiguredRealtimeVoiceProvider({ configuredProviderId: realtimeConfig.provider, providerConfigs: realtimeConfig.providers, + ...(launchOptions.model ? { providerConfigOverrides: { model: launchOptions.model } } : {}), cfg: runtimeConfig, cfgForResolve: runtimeConfig, + agentId: requestedAgentId, defaultModel: realtimeConfig.model, surface: "browser-session", noRegisteredProviderMessage: "No realtime voice provider registered", @@ -216,6 +226,7 @@ export const talkClientHandlers: GatewayRequestHandlers = { provider: resolution.provider, providerConfig: resolution.providerConfig, cfg: runtimeConfig, + model: launchOptions.model, surface: "browser-session", }); if (wantsCameraFrames && providerCapabilities?.supportsVideoFrames !== true) { @@ -229,10 +240,6 @@ export const talkClientHandlers: GatewayRequestHandlers = { ); return; } - const launchOptions = buildRealtimeVoiceLaunchOptions({ - requested: typedParams, - defaults: realtimeConfig, - }); const realtimeContext = await resolveTalkRealtimeProviderInstructions({ config: runtimeConfig, configuredInstructions: realtimeConfig.instructions, @@ -277,6 +284,31 @@ export const talkClientHandlers: GatewayRequestHandlers = { providerCapabilities?.handlesAgentConsult === true ? normalizeOptionalString(realtimeContext.instructions) : buildRealtimeInstructions(realtimeContext.instructions); + let consultAgentRuntime: ReturnType["agent"] | undefined; + const runAgentConsult: NonNullable< + InternalRealtimeVoiceBrowserSessionCreateRequest["runAgentConsult"] + > = async ({ prompt, signal }) => { + consultAgentRuntime ??= createPluginRuntime().agent; + const talkConfig = normalizeTalkSection(runtimeConfig.talk); + return await consultRealtimeVoiceAgent({ + cfg: runtimeConfig, + agentRuntime: consultAgentRuntime, + logger: context.logGateway, + agentId, + sessionKey, + messageProvider: "webchat", + lane: "talk", + runIdPrefix: "talk-realtime-consult", + args: { question: prompt }, + transcript: initialItems, + surface: "a browser Talk session", + userLabel: "User", + questionSourceLabel: "user", + thinkLevel: talkConfig?.consultThinkingLevel, + fastMode: talkConfig?.consultFastMode, + abortSignal: signal, + }); + }; const browserSessionRequest: InternalRealtimeVoiceBrowserSessionCreateRequest = { cfg: runtimeConfig, agentId, @@ -284,6 +316,7 @@ export const talkClientHandlers: GatewayRequestHandlers = { providerConfig: resolution.providerConfig, instructions, initialItems, + runAgentConsult, ...(tools.length > 0 ? { tools } : {}), ...launchOptions, }; diff --git a/src/gateway/server-methods/talk.test.ts b/src/gateway/server-methods/talk.test.ts index 10c86c12e7b6..62f22689b615 100644 --- a/src/gateway/server-methods/talk.test.ts +++ b/src/gateway/server-methods/talk.test.ts @@ -80,6 +80,8 @@ const mocks = vi.hoisted(() => ({ assertClientVoiceSessionOpen: vi.fn(), registerClientVoiceConsultRun: vi.fn(), resolveOpenClientVoiceSessionId: vi.fn(), + consultRealtimeVoiceAgent: vi.fn(async () => ({ text: "agent answer" })), + agentRuntime: {}, })); vi.mock("../../config/config.js", () => ({ @@ -127,6 +129,18 @@ vi.mock("../../talk/agent-run-control.js", () => ({ controlRealtimeVoiceAgentRun: mocks.controlRealtimeVoiceAgentRun, })); +vi.mock("../../talk/agent-consult-runtime.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + consultRealtimeVoiceAgent: mocks.consultRealtimeVoiceAgent, + }; +}); + +vi.mock("../../plugins/runtime/index.js", () => ({ + createPluginRuntime: () => ({ agent: mocks.agentRuntime }), +})); + vi.mock("../../agents/realtime-bootstrap-context.js", () => ({ resolveRealtimeBootstrapContextInstructions: mocks.resolveRealtimeBootstrapContextInstructions, })); @@ -2778,6 +2792,7 @@ describe("talk.client.create handler", () => { configuredProviderId: "openai", providerConfigs: { openai: { apiKey: "openai-key" } }, defaultModel: "gpt-realtime", + agentId: "main", surface: "browser-session", }); const createInput = mockCallArg(createBrowserSession) as Record; @@ -2802,6 +2817,29 @@ describe("talk.client.create handler", () => { expect(createInput.tools).not.toContainEqual( expect.objectContaining({ name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME }), ); + expect(createInput.runAgentConsult).toEqual(expect.any(Function)); + const consultSignal = new AbortController().signal; + await ( + createInput.runAgentConsult as (params: { + prompt: string; + signal?: AbortSignal; + }) => Promise<{ text: string }> + )({ prompt: "Check the repository", signal: consultSignal }); + expect(mocks.consultRealtimeVoiceAgent).toHaveBeenCalledWith( + expect.objectContaining({ + cfg: expect.any(Object), + agentRuntime: mocks.agentRuntime, + agentId: "main", + sessionKey: "main", + args: { question: "Check the repository" }, + transcript: [ + { role: "user", text: `2:${"🙂".repeat(799)}` }, + { role: "assistant", text: `3:${"🙂".repeat(799)}` }, + ], + surface: "a browser Talk session", + abortSignal: consultSignal, + }), + ); expect(createInput).not.toHaveProperty("provider"); expect(createInput).not.toHaveProperty("providers"); expect(createInput).not.toHaveProperty("transport"); @@ -2828,6 +2866,62 @@ describe("talk.client.create handler", () => { }); }); + it("passes a requested model override into selection and capability resolution", async () => { + const createBrowserSession = vi.fn(async () => ({ + provider: "openai", + transport: "webrtc" as const, + clientSecret: "secret", + })); + const provider = { + id: "openai", + label: "OpenAI Realtime", + isConfigured: () => true, + createBrowserSession, + createBridge: vi.fn(), + }; + mocks.resolveConfiguredRealtimeVoiceProvider.mockReturnValue({ + provider, + providerConfig: { model: "gpt-live-1" }, + }); + mocks.resolveRealtimeVoiceProviderCapabilities.mockReturnValueOnce({ + transports: ["webrtc"], + handlesAgentConsult: true, + supportsToolCalls: false, + supportsVideoFrames: false, + }); + const respond = vi.fn(); + + await callTalkHandler("talk.client.create", { + params: { sessionKey: "main", model: "gpt-live-1" }, + respond, + context: { + getRuntimeConfig: () => + ({ + talk: { + realtime: { + provider: "openai", + model: "gpt-realtime-2.1", + }, + }, + }) as OpenClawConfig, + }, + }); + + expect(mocks.resolveConfiguredRealtimeVoiceProvider).toHaveBeenCalledWith( + expect.objectContaining({ providerConfigOverrides: { model: "gpt-live-1" } }), + ); + expect(mocks.resolveRealtimeVoiceProviderCapabilities).toHaveBeenCalledWith( + expect.objectContaining({ model: "gpt-live-1" }), + ); + const createInput = mockCallArg(createBrowserSession) as Record; + expectRecordFields(createInput, { + model: "gpt-live-1", + runAgentConsult: expect.any(Function), + }); + expect(createInput).not.toHaveProperty("tools"); + expectRespondOk(respond, { provider: "openai", transport: "webrtc" }); + }); + it("lets native agent handoff own the Codex OAuth prompt and omits direct tools", async () => { mocks.resolveClientVoiceAgentSessionId.mockReturnValue(undefined); mocks.resolveRealtimeBootstrapContextInstructions.mockResolvedValue("Bounded profile context."); diff --git a/src/plugin-sdk/webhook-request-guards.ts b/src/plugin-sdk/webhook-request-guards.ts index 4362717c49a5..3b8613b14648 100644 --- a/src/plugin-sdk/webhook-request-guards.ts +++ b/src/plugin-sdk/webhook-request-guards.ts @@ -13,6 +13,8 @@ import { runWithGatewayIndependentRootWorkContinuation } from "../process/gatewa import type { FixedWindowRateLimiter } from "./webhook-memory-guards.js"; import { resolveWebhookIntegerOption } from "./webhook-numeric-options.js"; +export { resolveAcceptedBrowserOrigin } from "../gateway/origin-check.js"; + /** Body-read profile for webhook payload limits before or after authentication. */ export type WebhookBodyReadProfile = "pre-auth" | "post-auth"; diff --git a/src/talk/agent-consult-runtime.test.ts b/src/talk/agent-consult-runtime.test.ts index 4b704fd0e789..7638ef01f5a2 100644 --- a/src/talk/agent-consult-runtime.test.ts +++ b/src/talk/agent-consult-runtime.test.ts @@ -205,6 +205,30 @@ describe("realtime voice agent consult runtime", () => { expect(resolveRealtimeVoiceAgentConsultToolsAllow("none")).toStrictEqual([]); }); + it("does not start a consult after its caller has closed", async () => { + const { runtime, runEmbeddedAgent } = createAgentRuntime(); + const controller = new AbortController(); + controller.abort(new Error("voice session closed")); + + await expect( + consultRealtimeVoiceAgent({ + cfg: {} as never, + agentRuntime: runtime as never, + logger: { warn: vi.fn() }, + sessionKey: "voice:closed", + messageProvider: "voice", + lane: "voice", + runIdPrefix: "voice-realtime-consult:closed", + args: { question: "Do work" }, + transcript: [], + surface: "a live voice session", + userLabel: "User", + abortSignal: controller.signal, + }), + ).rejects.toThrow("voice session closed"); + expect(runEmbeddedAgent).not.toHaveBeenCalled(); + }); + it("runs an embedded agent using the shared session and prompt contract", async () => { const { runtime, runEmbeddedAgent, sessionStore } = createAgentRuntime(); diff --git a/src/talk/agent-consult-runtime.ts b/src/talk/agent-consult-runtime.ts index beb20c303ec8..b972235d5ba7 100644 --- a/src/talk/agent-consult-runtime.ts +++ b/src/talk/agent-consult-runtime.ts @@ -262,7 +262,9 @@ export async function consultRealtimeVoiceAgent(params: { toolsAllow?: string[]; extraSystemPrompt?: string; fallbackText?: string; + abortSignal?: AbortSignal; }): Promise { + params.abortSignal?.throwIfAborted(); const agentId = params.agentId ?? resolveDefaultAgentId(params.cfg); const agentDir = params.agentRuntime.resolveAgentDir(params.cfg, agentId); const workspaceDir = params.agentRuntime.resolveAgentWorkspaceDir(params.cfg, agentId); @@ -310,6 +312,12 @@ export async function consultRealtimeVoiceAgent(params: { assertRealtimeVoiceAgentConsultModelSelectionUnlocked(modelLockParams); }, }); + const abortFromCaller = () => lifecycleAbortController.abort(params.abortSignal?.reason); + if (params.abortSignal?.aborted) { + abortFromCaller(); + } else { + params.abortSignal?.addEventListener("abort", abortFromCaller, { once: true }); + } try { return await sessionWorkAdmission.run(async () => { @@ -402,6 +410,7 @@ export async function consultRealtimeVoiceAgent(params: { return { text }; }); } finally { + params.abortSignal?.removeEventListener("abort", abortFromCaller); sessionWorkAdmission.release(); } } diff --git a/src/talk/provider-internal.ts b/src/talk/provider-internal.ts index 07753985422d..aac7bf4b554c 100644 --- a/src/talk/provider-internal.ts +++ b/src/talk/provider-internal.ts @@ -35,10 +35,13 @@ type InternalRealtimeVoiceProviderApi = { isBrowserSessionConfigured: (ctx: { cfg?: OpenClawConfig; providerConfig: RealtimeVoiceProviderConfig; + agentId?: string; }) => boolean; resolveBrowserSessionCapabilities?: (ctx: { cfg?: OpenClawConfig; providerConfig: RealtimeVoiceProviderConfig; + /** Effective per-session model after request overrides. */ + model?: string; }) => InternalRealtimeVoiceProviderCapabilities; cancelBrowserSession?: ( request: InternalRealtimeVoiceBrowserSessionCreateRequest, @@ -63,11 +66,13 @@ export function isInternalRealtimeVoiceBrowserSessionConfigured(params: { provider: RealtimeVoiceProviderPlugin; cfg?: OpenClawConfig; providerConfig: RealtimeVoiceProviderConfig; + agentId?: string; }): boolean { return ( readInternalRealtimeVoiceProviderApi(params.provider)?.isBrowserSessionConfigured({ cfg: params.cfg, providerConfig: params.providerConfig, + agentId: params.agentId, }) === true ); } @@ -76,11 +81,13 @@ export function resolveInternalRealtimeVoiceBrowserSessionCapabilities(params: { provider: RealtimeVoiceProviderPlugin; cfg?: OpenClawConfig; providerConfig: RealtimeVoiceProviderConfig; + model?: string; }): InternalRealtimeVoiceProviderCapabilities | undefined { return readInternalRealtimeVoiceProviderApi(params.provider)?.resolveBrowserSessionCapabilities?.( { cfg: params.cfg, providerConfig: params.providerConfig, + model: params.model, }, ); } diff --git a/src/talk/provider-resolver.test.ts b/src/talk/provider-resolver.test.ts index 17bc1b87b804..f861351e05fe 100644 --- a/src/talk/provider-resolver.test.ts +++ b/src/talk/provider-resolver.test.ts @@ -1,5 +1,5 @@ // Talk provider resolver tests cover provider selection from config. -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import type { RealtimeVoiceProviderPlugin } from "../plugins/types.js"; import { resolveConfiguredRealtimeVoiceProvider, @@ -11,9 +11,13 @@ const INTERNAL_REALTIME_VOICE_PROVIDER = Symbol.for("openclaw.internal.realtime- function attachInternalRealtimeVoiceProviderApi( provider: RealtimeVoiceProviderPlugin, api: { - isBrowserSessionConfigured: () => boolean; + isBrowserSessionConfigured: (ctx: { + providerConfig: Record; + agentId?: string; + }) => boolean; resolveBrowserSessionCapabilities?: (ctx: { providerConfig: Record; + model?: string; }) => object; }, ): void { @@ -65,6 +69,9 @@ describe("realtime voice provider resolver", () => { }); it("keeps browser-only providers out of bridge auto-selection", () => { + const isBrowserSessionConfigured = vi.fn( + ({ agentId }: { agentId?: string }) => agentId === "voice-agent", + ); const browserOnly: RealtimeVoiceProviderPlugin = { id: "browser-only", label: "Browser only", @@ -75,7 +82,7 @@ describe("realtime voice provider resolver", () => { }, }; attachInternalRealtimeVoiceProviderApi(browserOnly, { - isBrowserSessionConfigured: () => true, + isBrowserSessionConfigured, }); const bridge: RealtimeVoiceProviderPlugin = { id: "bridge", @@ -96,10 +103,14 @@ describe("realtime voice provider resolver", () => { expect( resolveConfiguredRealtimeVoiceProvider({ cfg: {}, + agentId: "voice-agent", providers: [browserOnly, bridge], surface: "browser-session", }).provider.id, ).toBe("browser-only"); + expect(isBrowserSessionConfigured).toHaveBeenCalledWith( + expect.objectContaining({ agentId: "voice-agent" }), + ); }); it("applies a default model before provider config resolution", () => { @@ -187,11 +198,11 @@ describe("realtime voice provider resolver", () => { }; attachInternalRealtimeVoiceProviderApi(provider, { isBrowserSessionConfigured: () => true, - resolveBrowserSessionCapabilities: ({ providerConfig }) => ({ + resolveBrowserSessionCapabilities: ({ providerConfig, model }) => ({ transports: ["webrtc"], inputAudioFormats: [], outputAudioFormats: [], - supportsVideoFrames: providerConfig.authMode !== "native", + supportsVideoFrames: providerConfig.authMode !== "native" && model === "gpt-live-1", }), }); @@ -199,8 +210,17 @@ describe("realtime voice provider resolver", () => { resolveRealtimeVoiceProviderCapabilities({ provider, providerConfig: { authMode: "native" }, + model: "gpt-live-1", surface: "browser-session", })?.supportsVideoFrames, ).toBe(false); + expect( + resolveRealtimeVoiceProviderCapabilities({ + provider, + providerConfig: { authMode: "oauth" }, + model: "gpt-live-1", + surface: "browser-session", + })?.supportsVideoFrames, + ).toBe(true); }); }); diff --git a/src/talk/provider-resolver.ts b/src/talk/provider-resolver.ts index 3d1002a50a23..989a6299b82f 100644 --- a/src/talk/provider-resolver.ts +++ b/src/talk/provider-resolver.ts @@ -30,6 +30,8 @@ export type ResolveConfiguredRealtimeVoiceProviderParams = { cfg?: OpenClawConfig; /** Alternate config object used by generic provider selection internals. */ cfgForResolve?: OpenClawConfig; + /** Agent whose browser-session auth store should be inspected. */ + agentId?: string; /** Test/runtime override for the provider list. */ providers?: RealtimeVoiceProviderPlugin[]; /** Model injected before provider-specific resolveConfig runs. */ @@ -43,6 +45,8 @@ export function resolveRealtimeVoiceProviderCapabilities(params: { provider: RealtimeVoiceProviderPlugin; providerConfig: RealtimeVoiceProviderConfig; cfg?: OpenClawConfig; + /** Effective per-session model after request overrides. */ + model?: string; surface?: "browser-session" | "bridge"; }): InternalRealtimeVoiceProviderCapabilities | undefined { if (params.surface === "browser-session") { @@ -58,6 +62,7 @@ export function isRealtimeVoiceProviderConfigured(params: { provider: RealtimeVoiceProviderPlugin; cfg?: OpenClawConfig; providerConfig: RealtimeVoiceProviderConfig; + agentId?: string; surface?: "browser-session" | "bridge"; }): boolean { if ( @@ -111,6 +116,7 @@ export function resolveConfiguredRealtimeVoiceProvider( provider, cfg, providerConfig, + agentId: params.agentId, surface: params.surface, }), }); diff --git a/src/talk/provider-types.ts b/src/talk/provider-types.ts index 89274b11a8ef..b2e7f1ec44cc 100644 --- a/src/talk/provider-types.ts +++ b/src/talk/provider-types.ts @@ -128,6 +128,8 @@ export type RealtimeVoiceBrowserSessionCreateRequest = { silenceDurationMs?: number; prefixPaddingMs?: number; reasoningEffort?: string; + /** Host-injected agent delegation runner for provider-owned browser control channels. */ + runAgentConsult?: (params: { prompt: string; signal?: AbortSignal }) => Promise<{ text: string }>; }; export type RealtimeVoiceBrowserAudioContract = { diff --git a/test/scripts/test-live-shard.test.ts b/test/scripts/test-live-shard.test.ts index 4cdf668d9c08..d393c1f9cc52 100644 --- a/test/scripts/test-live-shard.test.ts +++ b/test/scripts/test-live-shard.test.ts @@ -129,6 +129,7 @@ describe("scripts/test-live-shard", () => { expect(selectLiveShardFiles("native-live-extensions-openai", allFiles)).toEqual([ "extensions/openai/openai-provider.live.test.ts", "extensions/openai/openai.live.test.ts", + "extensions/openai/realtime-quicksilver.live.test.ts", ]); expect(selectLiveShardFiles("native-live-extensions-l-n", allFiles)).toEqual([ "extensions/memory-lancedb/memory-lancedb.live.test.ts",