mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
feat(openai): support GPT-Live over backend WebSocket (#115622)
* feat(openai): add GPT-Live websocket bridge * chore: defer GPT-Live changelog to release * fix(android): preserve GPT-Live relay readiness gate * fix(openai): normalize GPT-Live assistant completion
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
87a66ddf2d
commit
7441362014
+21
-18
@@ -16,7 +16,7 @@ Talk mode covers five runtime shapes:
|
||||
|
||||
Native Talk is a continuous loop: listen for speech, send the transcript to the model through the active session, wait for the response, then speak it via the configured Talk provider (`talk.speak`).
|
||||
|
||||
Client-owned realtime Talk normally forwards provider tool calls through `talk.client.toolCall` instead of calling `chat.send` directly. GPT-Live delegates on a Gateway-owned sideband, and the Gateway binds each delegation to the browser connection that owns the Talk session. While a realtime consult is active, clients can call `talk.client.steer` or `talk.session.steer` to classify spoken input as `status`, `steer`, `cancel`, or `followup`; this includes GPT-Live delegations. Accepted steering queues into the active embedded run; rejected steering returns a reason such as `no_active_run`, `not_streaming`, or `compacting`. A newer GPT-Live spoken task also supersedes the running delegation.
|
||||
Client-owned realtime Talk normally forwards provider tool calls through `talk.client.toolCall` instead of calling `chat.send` directly. GPT-Live delegates on a Gateway-owned sideband, and the Gateway binds each delegation to the browser connection that owns the Talk session. While a realtime consult is active, clients can call `talk.client.steer` or `talk.session.steer` to classify spoken input as `status`, `steer`, `cancel`, or `followup`; this includes GPT-Live delegations. Accepted steering queues into the active embedded run; rejected steering returns a reason such as `no_active_run`, `not_streaming`, or `compacting`. A newer GPT-Live spoken task also supersedes the running delegation. Gateway-relayed GPT-Live uses the normal relay consult path.
|
||||
|
||||
Finalized realtime user and assistant utterances are always appended live to the active agent session, so later chat and voice turns share one history. Client-owned transports report their finalized transcripts with stable entry ids; Gateway relay sessions append the same events server-side. Provider sessions also receive the bounded realtime profile context used by Discord voice.
|
||||
|
||||
@@ -95,22 +95,24 @@ Supported keys: `voice` / `voice_id` / `voiceId`, `model` / `model_id` / `modelI
|
||||
}
|
||||
```
|
||||
|
||||
OpenAI browser WebRTC Talk supports native GPT-Live through
|
||||
`https://api.openai.com/v1/live`. Set `talk.realtime.model` to
|
||||
OpenAI Talk supports native GPT-Live through `https://api.openai.com/v1/live`.
|
||||
Set `talk.realtime.model` to
|
||||
`gpt-live-1-codex` (recommended) or `gpt-live-1-boulder-alpha`; `gpt-live-1`
|
||||
and `gpt-live-1-mini` are not valid on this route. GPT-Live prefers a ChatGPT
|
||||
OAuth subscription profile and falls back to Platform API-key auth, whose
|
||||
`/v1/live` access is currently
|
||||
and `gpt-live-1-mini` are not valid on this route. Browser WebRTC prefers a
|
||||
ChatGPT OAuth subscription profile and falls back to Platform API-key auth.
|
||||
Gateway relay and other backend bridges connect directly over the Frameless
|
||||
Bidi WebSocket and require Platform API-key auth, whose `/v1/live` access is
|
||||
currently
|
||||
[waitlist-gated](https://openai.com/form/gpt-live-1-in-the-api/).
|
||||
|
||||
The quickest setup is the Control UI: **Settings → Talk**, pick **OpenAI** and
|
||||
a `gpt-live-*` model. The OAuth prerequisite is an OpenClaw auth profile
|
||||
created with `openclaw models auth login --provider openai` — an existing
|
||||
Codex CLI sign-in is not read. GPT-Live also requires the bundled `openai`
|
||||
plugin registered in full mode; a restrictive `plugins.allow` list fails
|
||||
session creation with "OpenAI GPT-Live browser session broker is unavailable".
|
||||
Runtime bounds: 8 concurrent sessions per Gateway, 30-minute session TTL,
|
||||
60-second single-use offer tokens.
|
||||
a `gpt-live-*` model. Choose **WebRTC** for the browser-owned route or
|
||||
**Gateway relay** for the server-owned WebSocket route. WebRTC OAuth requires
|
||||
an OpenClaw auth profile created with
|
||||
`openclaw models auth login --provider openai`; an existing Codex CLI sign-in
|
||||
is not read. The WebRTC route also requires the bundled `openai` plugin in full
|
||||
registration mode. Its broker allows 8 concurrent sessions per Gateway, a
|
||||
30-minute session TTL, and 60-second single-use offer tokens.
|
||||
|
||||
GPT-Live accepts `alloy`, `ash`, `ballad`, `cedar`, `coral`, `echo`, `marin`,
|
||||
`sage`, `shimmer`, and `verse`. A `403 Voice session access denied` response is
|
||||
@@ -118,11 +120,12 @@ overloaded: an invalid voice returns the same response. The legacy
|
||||
`chatgpt.com` backend route also returns `403`; OpenClaw uses the native
|
||||
`api.openai.com/v1/live` route instead.
|
||||
|
||||
GPT-Live is limited to browser Talk WebRTC sessions. Telephony, Voice Call,
|
||||
Gateway relay, provider WebSocket transports, iOS, and Android are unsupported.
|
||||
The Gateway owns the authenticated sideband and routes delegated work through
|
||||
the configured OpenClaw agent; the browser receives neither the OAuth token nor
|
||||
a Platform API key.
|
||||
GPT-Live also works through Gateway relay and backend voice bridges, including
|
||||
Discord voice and Voice Call/telephony. Those paths keep the
|
||||
Platform API key on the Gateway and use one direct provider WebSocket; OpenClaw
|
||||
converts telephony G.711 u-law audio to and from GPT-Live's 24 kHz PCM contract.
|
||||
The browser WebRTC path remains available for ChatGPT OAuth sessions and keeps
|
||||
the OAuth token off the browser.
|
||||
|
||||
For GA `gpt-realtime-2.1`, `gpt-realtime-2.1-mini`, and `gpt-realtime-2`
|
||||
browser sessions, Platform credentials remain preferred in this order: the
|
||||
|
||||
+59
-35
@@ -142,26 +142,28 @@ explicit runtime config.
|
||||
|
||||
## OpenClaw feature coverage
|
||||
|
||||
| OpenAI capability | OpenClaw surface | Status |
|
||||
| ------------------------- | --------------------------------------------------------------------------------------------- | --------------------------------------------------------------- |
|
||||
| Chat / Responses | `openai/<model>` model provider | Yes |
|
||||
| Codex subscription models | `openai/<model>` with OpenAI OAuth | Yes |
|
||||
| Legacy Codex model refs | old Codex model refs, `codex-cli/<model>` | Repaired by doctor to `openai/<model>` |
|
||||
| Codex app-server harness | Codex-compatible HTTPS route with runtime unset/`auto`, or explicit `agentRuntime.id: codex` | Yes |
|
||||
| Server-side web search | Native OpenAI Responses tool | Yes, when web search is enabled and no other provider is pinned |
|
||||
| Images | `image_generate` | Yes |
|
||||
| Videos | `video_generate` | Yes |
|
||||
| Text-to-speech | `tts.provider: "openai"` / `tts` | Yes |
|
||||
| Batch speech-to-text | `tools.media.audio` / media understanding | Yes |
|
||||
| Streaming speech-to-text | Voice Call `streaming.provider: "openai"` | Yes |
|
||||
| Realtime voice | Voice Call `realtime.provider: "openai"` / Control UI Talk `talk.realtime.provider: "openai"` | Yes (Platform API key, or ChatGPT OAuth for browser GPT-Live) |
|
||||
| Embeddings | memory embedding provider | Yes |
|
||||
| OpenAI capability | OpenClaw surface | Status |
|
||||
| ------------------------- | --------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- |
|
||||
| Chat / Responses | `openai/<model>` model provider | Yes |
|
||||
| Codex subscription models | `openai/<model>` with OpenAI OAuth | Yes |
|
||||
| Legacy Codex model refs | old Codex model refs, `codex-cli/<model>` | Repaired by doctor to `openai/<model>` |
|
||||
| Codex app-server harness | Codex-compatible HTTPS route with runtime unset/`auto`, or explicit `agentRuntime.id: codex` | Yes |
|
||||
| Server-side web search | Native OpenAI Responses tool | Yes, when web search is enabled and no other provider is pinned |
|
||||
| Images | `image_generate` | Yes |
|
||||
| Videos | `video_generate` | Yes |
|
||||
| Text-to-speech | `tts.provider: "openai"` / `tts` | Yes |
|
||||
| Batch speech-to-text | `tools.media.audio` / media understanding | Yes |
|
||||
| Streaming speech-to-text | Voice Call `streaming.provider: "openai"` | Yes |
|
||||
| Realtime voice | Voice Call `realtime.provider: "openai"` / Control UI Talk `talk.realtime.provider: "openai"` | Yes (Platform API key, or ChatGPT OAuth for browser GPT-Live WebRTC) |
|
||||
| Embeddings | memory embedding provider | Yes |
|
||||
|
||||
<Note>
|
||||
GA OpenAI Realtime voice goes through the public **OpenAI Platform Realtime
|
||||
API** and requires a Platform API key. Browser GPT-Live is the exception: its
|
||||
native `api.openai.com/v1/live` route prefers a ChatGPT OAuth profile and falls
|
||||
back to Platform API-key auth when that account has waitlist-gated access.
|
||||
API** and requires a Platform API key. GPT-Live browser WebRTC is the exception:
|
||||
its native `api.openai.com/v1/live` route prefers a ChatGPT OAuth profile and
|
||||
falls back to Platform API-key auth when that account has waitlist-gated access.
|
||||
GPT-Live Gateway relay and backend voice bridges use a direct Frameless Bidi
|
||||
WebSocket and require Platform API-key auth.
|
||||
|
||||
Platform auth is resolved in this order: configured realtime API key, `openai`
|
||||
API-key profile, then `OPENAI_API_KEY`. ChatGPT OAuth does not configure GA
|
||||
@@ -934,12 +936,14 @@ compatibility fallback when the shared
|
||||
Gateway relay, provider WebSocket transports, Discord realtime voice, and
|
||||
other backend Realtime bridges remain Platform-key-only.
|
||||
|
||||
#### GPT-Live browser Talk
|
||||
#### GPT-Live Talk
|
||||
|
||||
GPT-Live is supported for browser Talk WebRTC sessions using a ChatGPT
|
||||
OAuth subscription profile. The complete path was verified on 2026-07-28
|
||||
with a ChatGPT Pro account: call creation returned `201 Created`, and the
|
||||
authenticated sideband emitted `session.started` for the same `rtc_*` call.
|
||||
GPT-Live has two native transport paths. Browser Talk WebRTC uses a ChatGPT
|
||||
OAuth subscription profile (or an enrolled Platform API key) through the
|
||||
Gateway offer broker. Gateway relay and backend voice bridges use the
|
||||
Frameless Bidi `wss://api.openai.com/v1/live?model=...` endpoint directly,
|
||||
with no browser or WebRTC provider connection, and require a Platform API
|
||||
key. The direct wire contract matches public Codex realtime v3.
|
||||
|
||||
Use `gpt-live-1-codex` (recommended) or
|
||||
`gpt-live-1-boulder-alpha`. The values `gpt-live-1` and
|
||||
@@ -950,7 +954,7 @@ compatibility fallback when the shared
|
||||
`echo`, `marin`, `sage`, `shimmer`, and `verse`. OpenClaw defaults to
|
||||
`marin` and maps unknown or unsupported configured voices back to it.
|
||||
|
||||
Prerequisites, in order:
|
||||
Browser WebRTC prerequisites, in order:
|
||||
|
||||
1. A ChatGPT OAuth auth profile: `openclaw models auth login --provider openai`.
|
||||
An existing Codex CLI (`~/.codex`) sign-in is **not** read; the profile
|
||||
@@ -979,6 +983,22 @@ compatibility fallback when the shared
|
||||
}
|
||||
```
|
||||
|
||||
For the server-owned WebSocket path, select Gateway relay and configure a
|
||||
Platform API key through `talk.realtime.providers.openai.apiKey`, an
|
||||
`openai` API-key profile, or `OPENAI_API_KEY`:
|
||||
|
||||
```json5
|
||||
{
|
||||
talk: {
|
||||
realtime: {
|
||||
provider: "openai",
|
||||
model: "gpt-live-1-codex",
|
||||
transport: "gateway-relay",
|
||||
},
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
<Warning>
|
||||
Platform API-key access to `/v1/live` is waitlist-gated and commonly returns
|
||||
`400 model_not_found` without enrollment. Use a ChatGPT OAuth profile, or request Platform access with the
|
||||
@@ -991,15 +1011,18 @@ compatibility fallback when the shared
|
||||
above, then verify that the selected ChatGPT OAuth profile and
|
||||
`chatgpt-account-id` belong to the same account.
|
||||
|
||||
GPT-Live remains limited to browser Talk WebRTC sessions. Voice
|
||||
Call/telephony, Gateway relay, provider WebSocket transports, and Android
|
||||
are unsupported. The Gateway owns the authenticated sideband and routes
|
||||
delegated work through the configured OpenClaw agent; the browser never
|
||||
receives the OAuth token.
|
||||
The direct WebSocket bridge enables Gateway relay, Discord voice, and Voice
|
||||
Call/telephony. OpenClaw converts G.711 u-law telephony
|
||||
audio to and from GPT-Live's 24 kHz PCM stream. The Gateway routes provider
|
||||
delegations through the configured OpenClaw agent and keeps the Platform
|
||||
API key server-side. The browser WebRTC route remains available for
|
||||
ChatGPT OAuth and never receives the OAuth token.
|
||||
|
||||
The canonical OpenClaw path creates the call on `api.openai.com/v1/live`
|
||||
and joins its sideband there. The legacy `chatgpt.com` backend route returns
|
||||
`403` and is not used.
|
||||
The WebRTC path creates a call on `api.openai.com/v1/live` and joins its
|
||||
sideband there. The backend path opens `/v1/live?model=...`, sends a
|
||||
Frameless `session.update`, then carries PCM audio, transcripts,
|
||||
delegations, and delegation results over that one socket. The legacy
|
||||
`chatgpt.com` backend route returns `403` and is not used.
|
||||
|
||||
Maintainers can exercise OpenClaw's complete OAuth path with the opt-in
|
||||
live test. It skips when no ChatGPT OAuth credential is available and
|
||||
@@ -1010,8 +1033,9 @@ compatibility fallback when the shared
|
||||
```
|
||||
|
||||
<Note>
|
||||
Backend OpenAI realtime bridges use the GA Realtime WebSocket session
|
||||
shape, which does not accept `session.temperature`. Azure OpenAI
|
||||
GA backend OpenAI realtime bridges use the Realtime WebSocket session
|
||||
shape, which does not accept `session.temperature`; GPT-Live uses the
|
||||
separate Frameless Bidi shape. Azure OpenAI
|
||||
deployments remain available via `azureEndpoint` and `azureDeployment` and
|
||||
keep the deployment-compatible session shape (including `temperature`).
|
||||
Supports bidirectional tool calling and G.711 u-law audio.
|
||||
@@ -1034,8 +1058,8 @@ compatibility fallback when the shared
|
||||
GPT-Live prefers ChatGPT OAuth when both auth modes are configured and
|
||||
falls back to Platform API-key access when the account has waitlist-gated
|
||||
`/v1/live` access.
|
||||
Gateway relay and Voice Call backend realtime WebSocket bridges continue to
|
||||
require Platform credentials and a GA model.
|
||||
Gateway relay and Voice Call backend realtime WebSocket bridges require
|
||||
Platform credentials and support both GA and GPT-Live models.
|
||||
Maintainer live verification is available with
|
||||
`OPENAI_API_KEY=... GEMINI_API_KEY=... node --import tsx scripts/dev/realtime-talk-live-smoke.ts`;
|
||||
the OpenAI legs verify both the backend WebSocket bridge and the browser
|
||||
|
||||
@@ -253,7 +253,7 @@ select it to open the owning Approvals page.
|
||||
- Chat with the model via Gateway WS (`chat.history`, `chat.send`, `chat.abort`, `chat.inject`). Archived sessions keep the composer disabled and show a banner with an **Unarchive** action before the conversation can continue.
|
||||
- Chat history refreshes request a bounded recent window with per-message text caps, so large sessions do not force the browser to render a full transcript payload before chat becomes usable.
|
||||
- Hovering or keyboard-focusing a public GitHub issue or pull request link shows its state, title, author, recent activity, comments, and change statistics. The connected Gateway fetches and caches public metadata without changing the link target, including when the UI uses a remote Gateway. The Gateway uses `GH_TOKEN` or `GITHUB_TOKEN` when available, after confirming the repository is public; otherwise it uses GitHub's anonymous API with a longer cache.
|
||||
- Talk through browser realtime sessions. OpenAI uses direct WebRTC, Google Live uses a constrained one-use browser token over WebSocket, and backend-only realtime voice plugins use the Gateway relay transport. Video-capable browser sessions can choose a device-local camera in Settings or flip cameras from the live preview; the browser captures JPEG frames for the realtime provider without streaming camera video through the Gateway. Client-owned provider sessions start with `talk.client.create`; Gateway relay sessions start with `talk.session.create`. The relay keeps provider credentials on the Gateway while the browser streams microphone PCM through `talk.session.appendAudio`, forwards `openclaw_agent_consult` provider tool calls through `talk.client.toolCall` for Gateway policy and the larger configured OpenClaw model, and routes active-run voice steering through `talk.client.steer` or `talk.session.steer`. GPT-Live delegates on the Gateway-owned sideband, but each delegation has the same spoken-confirmation gate and browser-owned `talk.client.steer` lifecycle; a newer spoken task can also supersede the running delegation. Configure the realtime provider, model, and speaker voice on **Settings → Talk**, whose pickers come from `talk.catalog` and show whether the selection is ready to use.
|
||||
- Talk through browser realtime sessions. OpenAI supports browser WebRTC and Gateway-relayed provider WebSockets, Google Live uses a constrained one-use browser token over WebSocket, and backend-only realtime voice plugins use Gateway relay. Video-capable browser sessions can choose a device-local camera in Settings or flip cameras from the live preview; the browser captures JPEG frames for the realtime provider without streaming camera video through the Gateway. Client-owned provider sessions start with `talk.client.create`; Gateway relay sessions start with `talk.session.create`. The relay keeps provider credentials on the Gateway while the browser streams microphone PCM through `talk.session.appendAudio`, forwards provider delegations or `openclaw_agent_consult` tool calls through Gateway policy and the larger configured OpenClaw model, and routes active-run voice steering through `talk.client.steer` or `talk.session.steer`. Browser WebRTC GPT-Live delegates on the Gateway-owned sideband, but each delegation has the same spoken-confirmation gate and browser-owned `talk.client.steer` lifecycle; a newer spoken task can also supersede the running delegation. Gateway-relayed GPT-Live uses the normal relay consult and steering path. Configure the realtime provider, model, and speaker voice on **Settings → Talk**, whose pickers come from `talk.catalog` and show whether the selection is ready to use.
|
||||
- Stream tool calls and live tool output cards in Chat (agent events). Tool activity renders as kind-aware rows: shell commands show the syntax-highlighted command with terminal-style output; supported edit and write calls show bounded inline diffs, line numbers when available, and `+added -removed` stats; and consecutive calls collapse into a summary such as "Ran 13 commands, read 6 files, edited 9 files". While a run is live, the newest running call names the group header. Expand a row to inspect its remaining arguments and raw output.
|
||||
- Optional AI purpose titles for complex tool calls (long shell commands, argument-heavy plugin tools), enabled with `gateway.controlUi.toolTitles: true` (default off). Titles come from the batched `chat.toolTitles` method through standard utility-model routing — an explicit `utilityModel` (operator-chosen provider, like other utility tasks), else the session provider's declared small-model default — and cache gateway-side per agent. When the opt-in is off or no cheap model is usable, rows keep their deterministic labels and no model call happens.
|
||||
- Start or dismiss ephemeral model-suggested follow-up tasks; accepted suggestions open a fresh managed-worktree session with the proposed prompt.
|
||||
@@ -467,7 +467,7 @@ The macOS app keeps its native link-browser sidebar for links clicked in the das
|
||||
|
||||
</Accordion>
|
||||
<Accordion title="Talk mode (browser realtime)">
|
||||
Talk mode uses a registered realtime voice provider. Configure OpenAI with `talk.realtime.provider: "openai"`. GA `gpt-realtime-*` browser WebRTC uses Platform auth in this order: `talk.realtime.providers.openai.apiKey`, an `openai` API-key profile, then `OPENAI_API_KEY`. Native GPT-Live uses `api.openai.com/v1/live` through the Gateway offer broker and prefers a ChatGPT OAuth subscription profile over Platform auth; Platform API-key access remains waitlist-gated. GPT-Live is browser-only and does not support Gateway relay or backend voice bridges. Configure Google with `talk.realtime.provider: "google"` plus `talk.realtime.providers.google.apiKey`. The browser never receives a standard provider API key or a ChatGPT OAuth token: Platform GA OpenAI receives an ephemeral Realtime client secret, native GPT-Live receives a one-use Gateway reservation, and Google Live receives a one-use constrained Live API auth token for a browser WebSocket session. Providers that only expose a backend realtime bridge run through the Gateway relay transport, so credentials and vendor sockets stay server-side while browser audio moves through authenticated Gateway RPCs. Platform GA sessions use the Gateway's direct-tool prompt, while GPT-Live delegates through its Gateway-owned sideband. `talk.client.create` does not accept caller-provided instruction overrides.
|
||||
Talk mode uses a registered realtime voice provider. Configure OpenAI with `talk.realtime.provider: "openai"`. GA `gpt-realtime-*` browser WebRTC uses Platform auth in this order: `talk.realtime.providers.openai.apiKey`, an `openai` API-key profile, then `OPENAI_API_KEY`. Native GPT-Live browser WebRTC uses `api.openai.com/v1/live` through the Gateway offer broker and prefers a ChatGPT OAuth subscription profile over Platform auth. GPT-Live also supports Gateway relay through a direct Frameless Bidi WebSocket with Platform API-key auth and no browser or WebRTC provider connection; Platform `/v1/live` access remains waitlist-gated. Configure Google with `talk.realtime.provider: "google"` plus `talk.realtime.providers.google.apiKey`. The browser never receives a standard provider API key or a ChatGPT OAuth token: Platform GA OpenAI receives an ephemeral Realtime client secret, native GPT-Live WebRTC receives a one-use Gateway reservation, and Google Live receives a one-use constrained Live API auth token for a browser WebSocket session. Gateway relay keeps provider credentials and vendor sockets server-side while browser audio moves through authenticated Gateway RPCs. Platform GA sessions use the Gateway's direct-tool prompt, while GPT-Live uses provider delegations. `talk.client.create` does not accept caller-provided instruction overrides.
|
||||
|
||||
Persistent provider, model, voice, transport, reasoning effort, exact VAD threshold, silence duration, and prefix padding defaults live in **Settings → Communications → Talk**; changing them requires `operator.admin` access. Configuring Gateway relay forces the backend relay path; configuring WebRTC keeps the session client-owned and fails instead of silently falling back to relay if the provider cannot create a browser session.
|
||||
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { ClientOptions } from "ws";
|
||||
import { OpenAIQuicksilverVoiceBridge } from "./realtime-quicksilver-bridge.js";
|
||||
import type {
|
||||
OpenAIQuicksilverSocket,
|
||||
OpenAIQuicksilverSocketFactory,
|
||||
} from "./realtime-quicksilver-sideband.js";
|
||||
|
||||
class FakeSocket extends EventEmitter {
|
||||
readyState = 0;
|
||||
readonly sent: string[] = [];
|
||||
|
||||
open(): void {
|
||||
this.readyState = 1;
|
||||
this.emit("open");
|
||||
}
|
||||
|
||||
send(payload: string): void {
|
||||
this.sent.push(payload);
|
||||
const event = JSON.parse(payload) as { type?: string };
|
||||
if (event.type === "session.update") {
|
||||
queueMicrotask(() =>
|
||||
this.serverEvent({
|
||||
type: "session.started",
|
||||
session: { id: "live-1", expires_at: Math.floor(Date.now() / 1000) + 60 },
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
close(): void {
|
||||
if (this.readyState === 3) {
|
||||
return;
|
||||
}
|
||||
this.readyState = 3;
|
||||
queueMicrotask(() => this.emit("close"));
|
||||
}
|
||||
|
||||
serverEvent(event: unknown): void {
|
||||
this.emit("message", Buffer.from(JSON.stringify(event)), false);
|
||||
}
|
||||
}
|
||||
|
||||
function createHarness(params?: { audioFormat?: "pcm16" | "g711_ulaw" }) {
|
||||
const socket = new FakeSocket();
|
||||
const connections: Array<{ url: string; options: ClientOptions }> = [];
|
||||
const webSocketFactory: OpenAIQuicksilverSocketFactory = (url, options) => {
|
||||
connections.push({ url, options });
|
||||
queueMicrotask(() => socket.open());
|
||||
return socket as unknown as OpenAIQuicksilverSocket;
|
||||
};
|
||||
const onAudio = vi.fn();
|
||||
const onTranscript = vi.fn();
|
||||
const onToolCall = vi.fn();
|
||||
const onReady = vi.fn();
|
||||
const onError = vi.fn();
|
||||
const onClose = vi.fn();
|
||||
const onEvent = vi.fn();
|
||||
const bridge = new OpenAIQuicksilverVoiceBridge({
|
||||
providerConfig: {},
|
||||
model: "gpt-live-1-codex",
|
||||
voice: "marin",
|
||||
instructions: "Use delegation for real work.",
|
||||
audioFormat:
|
||||
params?.audioFormat === "g711_ulaw"
|
||||
? { encoding: "g711_ulaw", sampleRateHz: 8000, channels: 1 }
|
||||
: { encoding: "pcm16", sampleRateHz: 24000, channels: 1 },
|
||||
resolveAuth: async () => ({ type: "api-key", token: "test-key" }),
|
||||
webSocketFactory,
|
||||
onAudio,
|
||||
onClearAudio: vi.fn(),
|
||||
onTranscript,
|
||||
onToolCall,
|
||||
onReady,
|
||||
onError,
|
||||
onClose,
|
||||
onEvent,
|
||||
});
|
||||
return {
|
||||
bridge,
|
||||
connections,
|
||||
onAudio,
|
||||
onClose,
|
||||
onError,
|
||||
onEvent,
|
||||
onReady,
|
||||
onToolCall,
|
||||
onTranscript,
|
||||
socket,
|
||||
};
|
||||
}
|
||||
|
||||
function sentEvents(socket: FakeSocket): Array<Record<string, unknown>> {
|
||||
return socket.sent.map((payload) => JSON.parse(payload) as Record<string, unknown>);
|
||||
}
|
||||
|
||||
describe("OpenAIQuicksilverVoiceBridge", () => {
|
||||
it("connects directly to /v1/live and completes the Frameless Bidi handshake", async () => {
|
||||
const harness = createHarness();
|
||||
await harness.bridge.connect();
|
||||
|
||||
expect(harness.connections).toHaveLength(1);
|
||||
expect(harness.connections[0]?.url).toBe("wss://api.openai.com/v1/live?model=gpt-live-1-codex");
|
||||
expect(harness.connections[0]?.options.headers).toMatchObject({
|
||||
Authorization: "Bearer test-key",
|
||||
"OpenAI-Alpha": "quicksilver=v2",
|
||||
});
|
||||
expect(sentEvents(harness.socket)[0]).toEqual({
|
||||
type: "session.update",
|
||||
session: {
|
||||
instructions: "Use delegation for real work.",
|
||||
audio: { output: { voice: "marin" } },
|
||||
delegation: { type: "client" },
|
||||
},
|
||||
});
|
||||
expect(harness.bridge.isConnected()).toBe(true);
|
||||
expect(harness.bridge.handlesInputAudioBargeIn).toBe(false);
|
||||
expect(harness.onReady).toHaveBeenCalledOnce();
|
||||
|
||||
harness.bridge.close();
|
||||
await vi.waitFor(() => expect(harness.onClose).toHaveBeenCalledWith("completed"));
|
||||
});
|
||||
|
||||
it("maps audio, transcripts, and delegations onto the shared bridge contract", async () => {
|
||||
const harness = createHarness();
|
||||
await harness.bridge.connect();
|
||||
harness.socket.serverEvent({
|
||||
type: "output_audio.delta",
|
||||
audio: Buffer.from([1, 2, 3, 4]).toString("base64"),
|
||||
});
|
||||
harness.socket.serverEvent({
|
||||
type: "input_transcript.added",
|
||||
item: { text: "hello" },
|
||||
});
|
||||
harness.socket.serverEvent({
|
||||
type: "turn.done",
|
||||
turn: { role: "user", transcript: "hello there" },
|
||||
});
|
||||
harness.socket.serverEvent({
|
||||
type: "delegation.created",
|
||||
item: {
|
||||
type: "delegation",
|
||||
target: "client",
|
||||
id: "delegation-1",
|
||||
content: [{ type: "input_text", text: "check the repository" }],
|
||||
},
|
||||
});
|
||||
|
||||
expect(harness.onAudio).toHaveBeenCalledWith(Buffer.from([1, 2, 3, 4]));
|
||||
expect(harness.onTranscript).toHaveBeenNthCalledWith(1, "user", "hello", false);
|
||||
expect(harness.onTranscript).toHaveBeenNthCalledWith(2, "user", "hello there", true);
|
||||
expect(harness.onToolCall).toHaveBeenCalledWith({
|
||||
itemId: "delegation-1",
|
||||
callId: "delegation-1",
|
||||
name: "openclaw_agent_consult",
|
||||
args: { question: "check the repository" },
|
||||
});
|
||||
|
||||
harness.bridge.submitToolResult("delegation-1", { text: "The repository is clean." });
|
||||
expect(sentEvents(harness.socket).at(-1)).toEqual({
|
||||
type: "delegation.context.append",
|
||||
delegation_item_id: "delegation-1",
|
||||
channel: "speakable",
|
||||
content: [{ type: "input_text", text: "The repository is clean." }],
|
||||
});
|
||||
});
|
||||
|
||||
it("normalizes assistant completion to the shared response lifecycle", async () => {
|
||||
const harness = createHarness();
|
||||
await harness.bridge.connect();
|
||||
|
||||
harness.socket.serverEvent({
|
||||
type: "turn.done",
|
||||
turn: { role: "user", transcript: "hello" },
|
||||
});
|
||||
expect(harness.onEvent).toHaveBeenLastCalledWith({
|
||||
direction: "server",
|
||||
type: "turn.done",
|
||||
});
|
||||
|
||||
harness.socket.serverEvent({
|
||||
type: "turn.done",
|
||||
turn: { role: "assistant", transcript: "hi there" },
|
||||
});
|
||||
expect(harness.onEvent).toHaveBeenLastCalledWith({
|
||||
direction: "server",
|
||||
type: "response.done",
|
||||
});
|
||||
});
|
||||
|
||||
it("converts telephony mu-law audio to and from GPT-Live PCM16", async () => {
|
||||
const harness = createHarness({ audioFormat: "g711_ulaw" });
|
||||
await harness.bridge.connect();
|
||||
harness.bridge.sendAudio(Buffer.alloc(160, 0xff));
|
||||
|
||||
const inputEvent = sentEvents(harness.socket).at(-1);
|
||||
expect(inputEvent?.type).toBe("input_audio.append");
|
||||
expect(Buffer.from(String(inputEvent?.audio), "base64")).toHaveLength(960);
|
||||
|
||||
harness.socket.serverEvent({
|
||||
type: "output_audio.delta",
|
||||
audio: Buffer.alloc(960).toString("base64"),
|
||||
});
|
||||
expect(harness.onAudio).toHaveBeenCalledWith(Buffer.alloc(160, 0xff));
|
||||
});
|
||||
|
||||
it("uses session context for forced consult results without a provider delegation", async () => {
|
||||
const harness = createHarness();
|
||||
await harness.bridge.connect();
|
||||
harness.bridge.submitToolResult("forced-consult", { text: "Forced answer" });
|
||||
|
||||
expect(sentEvents(harness.socket).at(-1)).toEqual({
|
||||
type: "session.context.append",
|
||||
channel: "speakable",
|
||||
content: [{ type: "input_text", text: "Forced answer" }],
|
||||
});
|
||||
|
||||
harness.bridge.triggerGreeting();
|
||||
expect(sentEvents(harness.socket).at(-1)).toEqual({
|
||||
type: "session.context.append",
|
||||
channel: "speakable",
|
||||
content: [{ type: "input_text", text: "Greet the user briefly." }],
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,423 @@
|
||||
// GPT-Live backend bridge over the Frameless Bidi WebSocket protocol used by Codex realtime v3.
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { canonicalizeBase64 } from "openclaw/plugin-sdk/media-runtime";
|
||||
import {
|
||||
captureWsEvent,
|
||||
createDebugProxyWebSocketAgent,
|
||||
resolveDebugProxySettings,
|
||||
} from "openclaw/plugin-sdk/proxy-capture";
|
||||
import {
|
||||
convertPcmToMulaw8k,
|
||||
mulawToPcm,
|
||||
REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
|
||||
resamplePcm,
|
||||
type RealtimeVoiceBridge,
|
||||
type RealtimeVoiceBridgeCreateRequest,
|
||||
type RealtimeVoiceToolResultOptions,
|
||||
} from "openclaw/plugin-sdk/realtime-voice";
|
||||
import WebSocket, { type RawData } from "ws";
|
||||
import {
|
||||
connectOpenAIQuicksilverSideband,
|
||||
type OpenAIQuicksilverSocket,
|
||||
type OpenAIQuicksilverSocketFactory,
|
||||
} from "./realtime-quicksilver-sideband.js";
|
||||
import {
|
||||
buildOpenAIQuicksilverSessionUpdate,
|
||||
buildOpenAIQuicksilverWebSocketUrl,
|
||||
chunkOpenAIQuicksilverAppendText,
|
||||
parseOpenAIQuicksilverEvent,
|
||||
type OpenAIQuicksilverAuth,
|
||||
type OpenAIQuicksilverInboundEvent,
|
||||
type OpenAIQuicksilverRequestIds,
|
||||
} from "./realtime-quicksilver-wire.js";
|
||||
|
||||
const OPENAI_QUICKSILVER_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024;
|
||||
const OPENAI_QUICKSILVER_READY_TIMEOUT_MS = 15_000;
|
||||
const OPENAI_QUICKSILVER_PENDING_AUDIO_CHUNKS = 320;
|
||||
const OPENAI_QUICKSILVER_SAMPLE_RATE = 24_000;
|
||||
const WEBSOCKET_OPEN = 1;
|
||||
|
||||
type OpenAIQuicksilverVoiceBridgeConfig = RealtimeVoiceBridgeCreateRequest & {
|
||||
model: string;
|
||||
voice?: string;
|
||||
resolveAuth: () => Promise<OpenAIQuicksilverAuth>;
|
||||
webSocketFactory?: OpenAIQuicksilverSocketFactory;
|
||||
};
|
||||
|
||||
function decodeTextFrame(data: RawData): string {
|
||||
if (Array.isArray(data)) {
|
||||
return Buffer.concat(data).toString("utf8");
|
||||
}
|
||||
if (data instanceof ArrayBuffer) {
|
||||
return Buffer.from(data).toString("utf8");
|
||||
}
|
||||
return data.toString("utf8");
|
||||
}
|
||||
|
||||
function toolResultText(result: unknown): string {
|
||||
if (typeof result === "string") {
|
||||
return result;
|
||||
}
|
||||
if (result && typeof result === "object") {
|
||||
const record = result as Record<string, unknown>;
|
||||
for (const key of ["text", "result", "output", "error"]) {
|
||||
const value = record[key];
|
||||
if (typeof value === "string" && value.trim()) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
return JSON.stringify(result) ?? String(result);
|
||||
} catch {
|
||||
return String(result);
|
||||
}
|
||||
}
|
||||
|
||||
export class OpenAIQuicksilverVoiceBridge implements RealtimeVoiceBridge {
|
||||
readonly supportsToolResultContinuation = true;
|
||||
readonly supportsToolResultSuppression = true;
|
||||
readonly handlesInputAudioBargeIn = false;
|
||||
|
||||
private socket: OpenAIQuicksilverSocket | undefined;
|
||||
private stopController = new AbortController();
|
||||
private ready = false;
|
||||
private intentionallyClosed = false;
|
||||
private closeNotified = false;
|
||||
private pendingAudio: Buffer[] = [];
|
||||
private activeDelegations = new Set<string>();
|
||||
private readonly flowId = randomUUID();
|
||||
private readonly requestIds: OpenAIQuicksilverRequestIds = {
|
||||
realtimeSessionId: randomUUID(),
|
||||
sessionId: randomUUID(),
|
||||
threadId: randomUUID(),
|
||||
};
|
||||
|
||||
constructor(private readonly config: OpenAIQuicksilverVoiceBridgeConfig) {}
|
||||
|
||||
async connect(): Promise<void> {
|
||||
if (this.ready) {
|
||||
return;
|
||||
}
|
||||
this.intentionallyClosed = false;
|
||||
this.closeNotified = false;
|
||||
if (this.stopController.signal.aborted) {
|
||||
this.stopController = new AbortController();
|
||||
}
|
||||
const auth = await this.config.resolveAuth();
|
||||
const url = buildOpenAIQuicksilverWebSocketUrl(this.config.model);
|
||||
const createSocket = this.config.webSocketFactory ?? this.createSocketFactory();
|
||||
const connected = await connectOpenAIQuicksilverSideband({
|
||||
auth,
|
||||
createSocket,
|
||||
requestIds: this.requestIds,
|
||||
signal: this.stopController.signal,
|
||||
url,
|
||||
});
|
||||
this.socket = connected.socket;
|
||||
captureWsEvent({
|
||||
url,
|
||||
direction: "local",
|
||||
kind: "ws-open",
|
||||
flowId: this.flowId,
|
||||
meta: { provider: "openai", capability: "gpt-live-voice" },
|
||||
});
|
||||
|
||||
let resolveReady!: () => void;
|
||||
let rejectReady!: (error: Error) => void;
|
||||
const readyPromise = new Promise<void>((resolve, reject) => {
|
||||
resolveReady = resolve;
|
||||
rejectReady = reject;
|
||||
});
|
||||
const readyTimeout = setTimeout(() => {
|
||||
rejectReady(new Error("GPT-Live WebSocket did not emit session.started"));
|
||||
this.closeSocket("session-start timeout");
|
||||
}, OPENAI_QUICKSILVER_READY_TIMEOUT_MS);
|
||||
readyTimeout.unref?.();
|
||||
const settleReady = () => {
|
||||
clearTimeout(readyTimeout);
|
||||
resolveReady();
|
||||
};
|
||||
const failReady = (error: Error) => {
|
||||
clearTimeout(readyTimeout);
|
||||
rejectReady(error);
|
||||
};
|
||||
|
||||
connected.socket.on("message", (data: RawData, isBinary: boolean) => {
|
||||
if (isBinary) {
|
||||
const error = new Error("GPT-Live WebSocket returned an unexpected binary frame");
|
||||
if (!this.ready) {
|
||||
failReady(error);
|
||||
}
|
||||
this.fail(error);
|
||||
return;
|
||||
}
|
||||
const payload = decodeTextFrame(data);
|
||||
captureWsEvent({
|
||||
url,
|
||||
direction: "inbound",
|
||||
kind: "ws-frame",
|
||||
flowId: this.flowId,
|
||||
payload,
|
||||
meta: { provider: "openai", capability: "gpt-live-voice" },
|
||||
});
|
||||
const event = parseOpenAIQuicksilverEvent(payload);
|
||||
if (event) {
|
||||
this.handleEvent(event, settleReady, failReady);
|
||||
}
|
||||
});
|
||||
connected.socket.on("error", (error: Error) => {
|
||||
if (!this.ready) {
|
||||
failReady(error);
|
||||
}
|
||||
this.fail(error);
|
||||
});
|
||||
connected.socket.on("close", () => {
|
||||
const wasReady = this.ready;
|
||||
this.ready = false;
|
||||
this.socket = undefined;
|
||||
if (!wasReady) {
|
||||
failReady(new Error("GPT-Live WebSocket closed before session.started"));
|
||||
}
|
||||
this.notifyClose(this.intentionallyClosed ? "completed" : "error");
|
||||
});
|
||||
|
||||
const terminalEvent = connected.detachBuffer();
|
||||
this.sendEvent(
|
||||
buildOpenAIQuicksilverSessionUpdate({
|
||||
instructions: this.config.instructions,
|
||||
voice: this.config.voice,
|
||||
}),
|
||||
);
|
||||
for (const frame of connected.bufferedFrames) {
|
||||
if (!frame.isBinary) {
|
||||
const event = parseOpenAIQuicksilverEvent(decodeTextFrame(frame.data));
|
||||
if (event) {
|
||||
this.handleEvent(event, settleReady, failReady);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (terminalEvent) {
|
||||
const error =
|
||||
terminalEvent.kind === "error"
|
||||
? terminalEvent.error
|
||||
: new Error("GPT-Live WebSocket closed during startup");
|
||||
failReady(error);
|
||||
this.fail(error);
|
||||
}
|
||||
await readyPromise;
|
||||
}
|
||||
|
||||
sendAudio(audio: Buffer): void {
|
||||
if (!this.ready || this.socket?.readyState !== WEBSOCKET_OPEN) {
|
||||
if (this.pendingAudio.length < OPENAI_QUICKSILVER_PENDING_AUDIO_CHUNKS) {
|
||||
this.pendingAudio.push(audio);
|
||||
}
|
||||
return;
|
||||
}
|
||||
this.sendAudioNow(audio);
|
||||
}
|
||||
|
||||
setMediaTimestamp(_ts: number): void {}
|
||||
|
||||
sendUserMessage(text: string): void {
|
||||
this.sendContext("session.context.append", undefined, text);
|
||||
}
|
||||
|
||||
triggerGreeting(instructions?: string): void {
|
||||
this.sendContext(
|
||||
"session.context.append",
|
||||
undefined,
|
||||
instructions ?? "Greet the user briefly.",
|
||||
"speakable",
|
||||
);
|
||||
}
|
||||
|
||||
submitToolResult(
|
||||
callId: string,
|
||||
result: unknown,
|
||||
options?: RealtimeVoiceToolResultOptions,
|
||||
): void {
|
||||
const channel = options?.suppressResponse || options?.willContinue ? "commentary" : "speakable";
|
||||
const type = this.activeDelegations.has(callId)
|
||||
? "delegation.context.append"
|
||||
: "session.context.append";
|
||||
this.sendContext(
|
||||
type,
|
||||
type === "delegation.context.append" ? callId : undefined,
|
||||
toolResultText(result),
|
||||
channel,
|
||||
);
|
||||
if (!options?.willContinue) {
|
||||
this.activeDelegations.delete(callId);
|
||||
}
|
||||
}
|
||||
|
||||
acknowledgeMark(_markName?: string): void {}
|
||||
|
||||
close(): void {
|
||||
this.intentionallyClosed = true;
|
||||
this.stopController.abort(new Error("GPT-Live bridge closed"));
|
||||
if (this.socket?.readyState === WEBSOCKET_OPEN) {
|
||||
this.sendEvent({ type: "session.close" });
|
||||
}
|
||||
this.closeSocket("bridge closed");
|
||||
this.ready = false;
|
||||
}
|
||||
|
||||
isConnected(): boolean {
|
||||
return this.ready && this.socket?.readyState === WEBSOCKET_OPEN;
|
||||
}
|
||||
|
||||
handleBargeIn(): void {
|
||||
// Frameless Bidi owns interruption from incoming audio and exposes no client cancel event.
|
||||
this.config.onClearAudio("barge-in");
|
||||
}
|
||||
|
||||
private createSocketFactory(): OpenAIQuicksilverSocketFactory {
|
||||
return (url, options) => {
|
||||
const proxyAgent = createDebugProxyWebSocketAgent(resolveDebugProxySettings());
|
||||
return new WebSocket(url, {
|
||||
...options,
|
||||
maxPayload: OPENAI_QUICKSILVER_MAX_PAYLOAD_BYTES,
|
||||
...(proxyAgent ? { agent: proxyAgent } : {}),
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
private handleEvent(
|
||||
event: OpenAIQuicksilverInboundEvent,
|
||||
settleReady: () => void,
|
||||
failReady: (error: Error) => void,
|
||||
): void {
|
||||
if (event.kind === "ignored" || event.kind === "unknown") {
|
||||
return;
|
||||
}
|
||||
if (event.kind === "session-started") {
|
||||
if (!this.ready) {
|
||||
this.ready = true;
|
||||
for (const audio of this.pendingAudio.splice(0)) {
|
||||
this.sendAudioNow(audio);
|
||||
}
|
||||
this.config.onReady?.();
|
||||
}
|
||||
this.config.onEvent?.({ direction: "server", type: "session.started" });
|
||||
settleReady();
|
||||
return;
|
||||
}
|
||||
if (event.kind === "audio") {
|
||||
const canonical = canonicalizeBase64(event.data);
|
||||
if (!canonical) {
|
||||
this.fail(new Error("GPT-Live WebSocket returned malformed base64 audio"));
|
||||
return;
|
||||
}
|
||||
const pcm = Buffer.from(canonical, "base64");
|
||||
this.config.onAudio(
|
||||
this.config.audioFormat?.encoding === "g711_ulaw"
|
||||
? convertPcmToMulaw8k(pcm, OPENAI_QUICKSILVER_SAMPLE_RATE)
|
||||
: pcm,
|
||||
);
|
||||
this.config.onEvent?.({ direction: "server", type: "output_audio.delta" });
|
||||
return;
|
||||
}
|
||||
if (event.kind === "transcript-delta" || event.kind === "transcript-done") {
|
||||
this.config.onTranscript?.(event.role, event.text, event.kind === "transcript-done");
|
||||
this.config.onEvent?.({
|
||||
direction: "server",
|
||||
type:
|
||||
event.kind === "transcript-done"
|
||||
? event.role === "assistant"
|
||||
? "response.done"
|
||||
: "turn.done"
|
||||
: `${event.role === "user" ? "input" : "output"}_transcript.added`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (event.kind === "delegation") {
|
||||
this.activeDelegations.add(event.id);
|
||||
this.config.onEvent?.({
|
||||
direction: "server",
|
||||
type: "delegation.created",
|
||||
itemId: event.id,
|
||||
});
|
||||
this.config.onToolCall?.({
|
||||
itemId: event.id,
|
||||
callId: event.id,
|
||||
name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
|
||||
args: { question: event.prompt },
|
||||
});
|
||||
return;
|
||||
}
|
||||
const error = new Error(event.message);
|
||||
if (!this.ready) {
|
||||
failReady(error);
|
||||
this.closeSocket("session start failed");
|
||||
}
|
||||
this.config.onEvent?.({ direction: "server", type: "error", detail: event.message });
|
||||
this.config.onError?.(error);
|
||||
if (event.fatalAuth) {
|
||||
this.closeSocket("authentication failed");
|
||||
}
|
||||
}
|
||||
|
||||
private sendAudioNow(audio: Buffer): void {
|
||||
const pcm =
|
||||
this.config.audioFormat?.encoding === "g711_ulaw"
|
||||
? resamplePcm(mulawToPcm(audio), 8_000, OPENAI_QUICKSILVER_SAMPLE_RATE)
|
||||
: audio;
|
||||
this.sendEvent({ type: "input_audio.append", audio: pcm.toString("base64") });
|
||||
}
|
||||
|
||||
private sendContext(
|
||||
type: "delegation.context.append" | "session.context.append",
|
||||
delegationItemId: string | undefined,
|
||||
text: string,
|
||||
channel?: "speakable" | "commentary",
|
||||
): void {
|
||||
for (const chunk of chunkOpenAIQuicksilverAppendText(text)) {
|
||||
this.sendEvent({
|
||||
type,
|
||||
...(delegationItemId ? { delegation_item_id: delegationItemId } : {}),
|
||||
...(channel ? { channel } : {}),
|
||||
content: [{ type: "input_text", text: chunk }],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private sendEvent(event: object): void {
|
||||
if (!this.socket || this.socket.readyState !== WEBSOCKET_OPEN) {
|
||||
return;
|
||||
}
|
||||
const payload = JSON.stringify(event);
|
||||
captureWsEvent({
|
||||
url: buildOpenAIQuicksilverWebSocketUrl(this.config.model),
|
||||
direction: "outbound",
|
||||
kind: "ws-frame",
|
||||
flowId: this.flowId,
|
||||
payload,
|
||||
meta: { provider: "openai", capability: "gpt-live-voice" },
|
||||
});
|
||||
this.socket.send(payload);
|
||||
}
|
||||
|
||||
private fail(error: Error): void {
|
||||
this.config.onError?.(error);
|
||||
this.closeSocket("bridge error");
|
||||
}
|
||||
|
||||
private closeSocket(reason: string): void {
|
||||
try {
|
||||
this.socket?.close(1000, reason);
|
||||
} catch {
|
||||
// Closing is best effort once the bridge reaches a terminal state.
|
||||
}
|
||||
}
|
||||
|
||||
private notifyClose(reason: "completed" | "error"): void {
|
||||
if (this.closeNotified) {
|
||||
return;
|
||||
}
|
||||
this.closeNotified = true;
|
||||
this.config.onClose?.(reason);
|
||||
}
|
||||
}
|
||||
@@ -17,13 +17,22 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("GPT-Live sideband protocol", () => {
|
||||
it.each(["session.updated", "output_audio.delta"])("ignores %s server-side", (type) => {
|
||||
it("ignores session.updated server-side", () => {
|
||||
const type = "session.updated";
|
||||
expect(parseOpenAIQuicksilverEvent(JSON.stringify({ type }))).toEqual({
|
||||
kind: "ignored",
|
||||
eventType: type,
|
||||
});
|
||||
});
|
||||
|
||||
it("parses direct WebSocket audio", () => {
|
||||
expect(
|
||||
parseOpenAIQuicksilverEvent(
|
||||
JSON.stringify({ type: "output_audio.delta", audio: "AQIDBA==" }),
|
||||
),
|
||||
).toEqual({ kind: "audio", data: "AQIDBA==" });
|
||||
});
|
||||
|
||||
it("parses session expiry and transcript events", () => {
|
||||
expect(
|
||||
parseOpenAIQuicksilverEvent(
|
||||
|
||||
@@ -344,6 +344,11 @@ export function createOpenAIQuicksilverBrowserSessionBroker(params: {
|
||||
}
|
||||
return;
|
||||
}
|
||||
// WebRTC carries media separately; only the direct backend bridge consumes
|
||||
// Frameless output_audio.delta events received on the provider WebSocket.
|
||||
if (event.kind === "audio") {
|
||||
return;
|
||||
}
|
||||
const delegationInput = event.prompt;
|
||||
if (!delegationInput.trim()) {
|
||||
params.logger.debug?.("OpenAI GPT-Live ignored an empty client delegation");
|
||||
|
||||
@@ -54,6 +54,11 @@ type OpenAIQuicksilverSession = {
|
||||
}>;
|
||||
};
|
||||
|
||||
type OpenAIQuicksilverSessionUpdate = {
|
||||
type: "session.update";
|
||||
session: Omit<OpenAIQuicksilverSession, "model">;
|
||||
};
|
||||
|
||||
const eventEnvelopeSchema = z.object({ type: z.string() }).passthrough();
|
||||
const sessionStartedSchema = z
|
||||
.object({
|
||||
@@ -66,6 +71,12 @@ const transcriptAddedSchema = z
|
||||
item: z.object({ text: z.string() }).passthrough(),
|
||||
})
|
||||
.passthrough();
|
||||
const outputAudioDeltaSchema = z
|
||||
.object({
|
||||
type: z.literal("output_audio.delta"),
|
||||
audio: z.string(),
|
||||
})
|
||||
.passthrough();
|
||||
const turnDoneSchema = z
|
||||
.object({
|
||||
turn: z
|
||||
@@ -102,6 +113,7 @@ const delegationSchema = z
|
||||
export type OpenAIQuicksilverInboundEvent =
|
||||
| { kind: "ignored"; eventType: string }
|
||||
| { kind: "session-started"; expiresAt?: number }
|
||||
| { kind: "audio"; data: string }
|
||||
| { kind: "transcript-delta"; role: "user" | "assistant"; text: string }
|
||||
| { kind: "transcript-done"; role: "user" | "assistant"; text: string }
|
||||
| { kind: "delegation"; id: string; prompt: string }
|
||||
@@ -155,6 +167,26 @@ export function buildOpenAIQuicksilverSession(params: {
|
||||
};
|
||||
}
|
||||
|
||||
/** Builds the direct Frameless Bidi WebSocket handshake used by Codex realtime v3. */
|
||||
export function buildOpenAIQuicksilverSessionUpdate(params: {
|
||||
instructions?: string;
|
||||
voice?: string;
|
||||
initialItems?: readonly OpenAIQuicksilverInitialItem[];
|
||||
}): OpenAIQuicksilverSessionUpdate {
|
||||
const { model: _model, ...session } = buildOpenAIQuicksilverSession({
|
||||
model: "direct-websocket",
|
||||
...params,
|
||||
});
|
||||
return { type: "session.update", session };
|
||||
}
|
||||
|
||||
export function buildOpenAIQuicksilverWebSocketUrl(model: string): string {
|
||||
const url = new URL(OPENAI_QUICKSILVER_CALL_URL);
|
||||
url.protocol = "wss:";
|
||||
url.searchParams.set("model", model);
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
function truncateOpenAIQuicksilverContextText(text: string, maxBytes: number): string {
|
||||
let result = "";
|
||||
let bytes = 0;
|
||||
@@ -501,7 +533,13 @@ export function parseOpenAIQuicksilverEvent(payload: string): OpenAIQuicksilverI
|
||||
? { kind: "transcript-done", role: turn.data.turn.role, text: turn.data.turn.transcript }
|
||||
: { kind: "ignored", eventType };
|
||||
}
|
||||
if (eventType === "session.updated" || eventType === "output_audio.delta") {
|
||||
if (eventType === "output_audio.delta") {
|
||||
const audio = outputAudioDeltaSchema.safeParse(decoded);
|
||||
return audio.success
|
||||
? { kind: "audio", data: audio.data.audio }
|
||||
: { kind: "ignored", eventType };
|
||||
}
|
||||
if (eventType === "session.updated") {
|
||||
return { kind: "ignored", eventType };
|
||||
}
|
||||
if (eventType === "delegation.created") {
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { Page } from "playwright";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import WebSocket, { type RawData } from "ws";
|
||||
import { resolveCodexAuthIdentity } from "./openai-chatgpt-auth-identity.js";
|
||||
import { OpenAIQuicksilverVoiceBridge } from "./realtime-quicksilver-bridge.js";
|
||||
import {
|
||||
createOpenAIQuicksilverBrowserSessionBroker,
|
||||
OPENAI_QUICKSILVER_OFFER_PATH,
|
||||
@@ -176,6 +177,36 @@ async function closeServer(server: Server): Promise<void> {
|
||||
});
|
||||
}
|
||||
|
||||
describeLive("GPT-Live Platform WebSocket", () => {
|
||||
it(
|
||||
"opens a Frameless Bidi session without a browser or WebRTC",
|
||||
async ({ skip }) => {
|
||||
const apiKey = process.env.OPENAI_API_KEY?.trim();
|
||||
if (!apiKey) {
|
||||
skip("No OpenAI Platform API key is available");
|
||||
return;
|
||||
}
|
||||
const bridge = new OpenAIQuicksilverVoiceBridge({
|
||||
providerConfig: {},
|
||||
model: "gpt-live-1-codex",
|
||||
voice: "marin",
|
||||
instructions: "Keep this transport verification session silent.",
|
||||
audioFormat: { encoding: "pcm16", sampleRateHz: 24000, channels: 1 },
|
||||
resolveAuth: async () => ({ type: "api-key", token: apiKey }),
|
||||
onAudio: () => {},
|
||||
onClearAudio: () => {},
|
||||
});
|
||||
try {
|
||||
await bridge.connect();
|
||||
expect(bridge.isConnected()).toBe(true);
|
||||
} finally {
|
||||
bridge.close();
|
||||
}
|
||||
},
|
||||
LIVE_TIMEOUT_MS,
|
||||
);
|
||||
});
|
||||
|
||||
describeLive("OpenAI OAuth WebRTC", () => {
|
||||
it(
|
||||
"creates a call and joins the authenticated sideband",
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
// Openai tests cover GPT-Live (quicksilver) realtime voice gating.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
isOpenAIGptLiveModel,
|
||||
OPENAI_GPT_LIVE_BRIDGE_UNSUPPORTED_MESSAGE,
|
||||
} from "./realtime-quicksilver.js";
|
||||
import { isOpenAIGptLiveModel } from "./realtime-quicksilver.js";
|
||||
import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js";
|
||||
|
||||
const mintSecretMock = vi.hoisted(() => vi.fn());
|
||||
@@ -33,7 +30,7 @@ describe("openai gpt-live model detection", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("openai realtime voice provider gpt-live transport guard", () => {
|
||||
describe("openai realtime voice provider gpt-live transport routing", () => {
|
||||
beforeEach(() => {
|
||||
mintSecretMock.mockReset();
|
||||
mintSecretMock.mockResolvedValue({ value: "ek_test", expiresAt: 1234 });
|
||||
@@ -57,18 +54,18 @@ describe("openai realtime voice provider gpt-live transport guard", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects gpt-live models on the realtime WebSocket bridge", () => {
|
||||
it("routes gpt-live models to a backend WebSocket bridge", () => {
|
||||
const provider = buildOpenAIRealtimeVoiceProvider();
|
||||
const callbacks = {
|
||||
onAudio: vi.fn(),
|
||||
onClearAudio: vi.fn(),
|
||||
};
|
||||
expect(() =>
|
||||
expect(
|
||||
provider.createBridge({
|
||||
...callbacks,
|
||||
providerConfig: { apiKey: "test-key", model: "gpt-live-1" },
|
||||
providerConfig: { apiKey: "test-key", model: "gpt-live-1-codex" },
|
||||
}),
|
||||
).toThrow(OPENAI_GPT_LIVE_BRIDGE_UNSUPPORTED_MESSAGE);
|
||||
).toMatchObject({ supportsToolResultContinuation: true });
|
||||
expect(() =>
|
||||
provider.createBridge({
|
||||
...callbacks,
|
||||
@@ -76,4 +73,20 @@ describe("openai realtime voice provider gpt-live transport guard", () => {
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects Azure credentials before creating a Platform GPT-Live bridge", () => {
|
||||
const provider = buildOpenAIRealtimeVoiceProvider();
|
||||
expect(() =>
|
||||
provider.createBridge({
|
||||
providerConfig: {
|
||||
apiKey: "azure-test-key",
|
||||
model: "gpt-live-1-codex",
|
||||
azureEndpoint: "https://example.openai.azure.com",
|
||||
azureDeployment: "realtime",
|
||||
},
|
||||
onAudio: vi.fn(),
|
||||
onClearAudio: vi.fn(),
|
||||
}),
|
||||
).toThrow("GPT-Live backend WebSocket sessions do not support Azure");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,12 +1,8 @@
|
||||
// GPT-Live (OpenAI "quicksilver") browser Talk routing uses the live-proven
|
||||
// ChatGPT OAuth WebRTC and sideband path. Relay and backend WebSocket bridges
|
||||
// stay unsupported; Platform API-key access remains waitlist-gated.
|
||||
// GPT-Live (OpenAI "quicksilver") routing uses either the ChatGPT OAuth browser
|
||||
// WebRTC path or the Platform API-key Frameless Bidi backend WebSocket path.
|
||||
|
||||
const OPENAI_GPT_LIVE_MODEL_PREFIX = "gpt-live";
|
||||
|
||||
export const OPENAI_GPT_LIVE_BRIDGE_UNSUPPORTED_MESSAGE =
|
||||
"GPT-Live models are not supported on the realtime WebSocket bridge: OpenAI requires WebRTC for quicksilver sessions. Set a gpt-realtime model for this transport.";
|
||||
|
||||
export function isOpenAIGptLiveModel(model: string | undefined): boolean {
|
||||
if (!model) {
|
||||
return false;
|
||||
|
||||
@@ -44,16 +44,14 @@ import {
|
||||
resolveOpenAIProviderConfigRecord,
|
||||
trimToUndefined,
|
||||
} from "./realtime-provider-shared.js";
|
||||
import { OpenAIQuicksilverVoiceBridge } from "./realtime-quicksilver-bridge.js";
|
||||
import { buildOpenAIQuicksilverInstructions } from "./realtime-quicksilver-instructions.js";
|
||||
import {
|
||||
createOpenAIQuicksilverBrowserSessionBroker,
|
||||
OPENAI_QUICKSILVER_CAPABILITIES,
|
||||
resolveOpenAIChatGptSubscriptionAuth,
|
||||
} from "./realtime-quicksilver-session.js";
|
||||
import {
|
||||
isOpenAIGptLiveModel,
|
||||
OPENAI_GPT_LIVE_BRIDGE_UNSUPPORTED_MESSAGE,
|
||||
} from "./realtime-quicksilver.js";
|
||||
import { isOpenAIGptLiveModel } from "./realtime-quicksilver.js";
|
||||
|
||||
type OpenAIRealtimeVoice =
|
||||
| "alloy"
|
||||
@@ -1811,8 +1809,28 @@ export function buildOpenAIRealtimeVoiceProvider(options?: {
|
||||
},
|
||||
createBridge: (req) => {
|
||||
const config = normalizeProviderConfig(req.providerConfig);
|
||||
if (isOpenAIGptLiveModel(config.model)) {
|
||||
throw new Error(OPENAI_GPT_LIVE_BRIDGE_UNSUPPORTED_MESSAGE);
|
||||
const model = config.model;
|
||||
if (model && isOpenAIGptLiveModel(model)) {
|
||||
if (config.azureEndpoint || config.azureDeployment) {
|
||||
throw new Error(
|
||||
"GPT-Live backend WebSocket sessions do not support Azure endpoints or deployments",
|
||||
);
|
||||
}
|
||||
return new OpenAIQuicksilverVoiceBridge({
|
||||
...req,
|
||||
model,
|
||||
voice: config.voice,
|
||||
instructions: buildOpenAIQuicksilverInstructions(req.instructions),
|
||||
resolveAuth: async () => ({
|
||||
type: "api-key",
|
||||
token: (
|
||||
await requireOpenAIRealtimePlatformAuth({
|
||||
configuredApiKey: config.apiKey,
|
||||
cfg: req.cfg,
|
||||
})
|
||||
).value,
|
||||
}),
|
||||
});
|
||||
}
|
||||
return new OpenAIRealtimeVoiceBridge({
|
||||
...req,
|
||||
|
||||
@@ -231,6 +231,7 @@ function parseWebSocketMessage(data: RawData): Record<string, unknown> {
|
||||
|
||||
async function withBargeInHarness(
|
||||
params: {
|
||||
bridgeHandlesInputAudioBargeIn?: boolean;
|
||||
handlesProviderBargeIn?: boolean;
|
||||
interruptResponseOnInputAudio?: boolean;
|
||||
providerCallId: string;
|
||||
@@ -251,7 +252,13 @@ async function withBargeInHarness(
|
||||
const call = makeCallRecord(params.providerCallId);
|
||||
const createBridge = vi.fn((request: RealtimeBridgeRequest) => {
|
||||
callbacks = request;
|
||||
return makeBridge({ handleBargeIn, sendAudio });
|
||||
return makeBridge({
|
||||
handleBargeIn,
|
||||
sendAudio,
|
||||
...(params.bridgeHandlesInputAudioBargeIn === undefined
|
||||
? {}
|
||||
: { handlesInputAudioBargeIn: params.bridgeHandlesInputAudioBargeIn }),
|
||||
});
|
||||
});
|
||||
const capabilities = params.handlesProviderBargeIn
|
||||
? PROVIDER_BARGE_IN_CAPABILITIES
|
||||
@@ -1009,6 +1016,34 @@ describe("RealtimeCallHandler path routing", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("lets a session bridge override provider-level barge-in capabilities", async () => {
|
||||
await withBargeInHarness(
|
||||
{
|
||||
bridgeHandlesInputAudioBargeIn: false,
|
||||
handlesProviderBargeIn: true,
|
||||
providerCallId: "CA-bridge-local-barge-in",
|
||||
},
|
||||
async ({ callbacks, call, handleBargeIn, outboundMessages, sendAudio, ws }) => {
|
||||
callbacks?.onAudio?.(Buffer.from([1, 2, 3]));
|
||||
for (let i = 0; i < 4; i += 1) {
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
event: "media",
|
||||
media: { payload: Buffer.alloc(160, 0x00).toString("base64") },
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
await waitForRealtimeTest(() => {
|
||||
expect(sendAudio).toHaveBeenCalledTimes(4);
|
||||
expect(requireCancelledTurn(call).turnId).toMatch(/^turn-\d+$/);
|
||||
expect(outboundMessages.some((message) => message.event === "clear")).toBe(true);
|
||||
});
|
||||
expect(handleBargeIn).toHaveBeenCalledWith({ audioPlaybackActive: true });
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("clears remote playback after local pacing and output state have finished", async () => {
|
||||
await withBargeInHarness(
|
||||
{ providerCallId: "CA-late-local-barge-in" },
|
||||
|
||||
@@ -589,7 +589,7 @@ export class RealtimeCallHandler {
|
||||
},
|
||||
onTalkEvent: (event) => appendRecentTalkEventMetadata(callRecord, event),
|
||||
});
|
||||
const providerHandlesInputAudioBargeIn =
|
||||
let providerHandlesInputAudioBargeIn =
|
||||
this.realtimeProvider.capabilities?.handlesInputAudioBargeIn === true;
|
||||
const cancelOutputAudioForBargeIn = (
|
||||
source: "local" | "provider",
|
||||
@@ -895,6 +895,8 @@ export class RealtimeCallHandler {
|
||||
});
|
||||
},
|
||||
});
|
||||
providerHandlesInputAudioBargeIn =
|
||||
session.bridge.handlesInputAudioBargeIn ?? providerHandlesInputAudioBargeIn;
|
||||
const closeTelephony = (reason: TelephonyCloseReason) => {
|
||||
try {
|
||||
session.close();
|
||||
|
||||
@@ -193,6 +193,8 @@ export type RealtimeVoiceBridge = {
|
||||
supportsToolResultContinuation?: boolean;
|
||||
/** False when the provider cannot accept a tool result without starting a response. */
|
||||
supportsToolResultSuppression?: boolean;
|
||||
/** Per-session override for provider-confirmed input-audio barge-in handling. */
|
||||
handlesInputAudioBargeIn?: boolean;
|
||||
connect(): Promise<void>;
|
||||
sendAudio(audio: Buffer): void;
|
||||
setMediaTimestamp(ts: number): void;
|
||||
|
||||
Reference in New Issue
Block a user