ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-25 20:05:46 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(talk): admit opaque realtime routes

Browse Source
This commit is contained in:
Vincent Koc
2026-08-25 15:08:53 +08:00
parent 2bdbfebd2e
commit 95feb34d78
31 changed files with 475 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/android/app/src/test/java/ai/openclaw/app/voice/TalkModeConfigParsingTest.kt
+2 -2
View File
@@ -118,12 +118,12 @@ class TalkModeConfigParsingTest {
val providerLevelBrowserOnly =
json
.parseToJsonElement(
"""{"talk":{"realtime":{"provider":"openai","providers":{"openai":{"model":"gpt-live-1-codex"}}}}}""",
"""{"talk":{"realtime":{"provider":"openai","providers":{"openai":{"model":"gpt-live-test-canary"}}}}}""",
).jsonObject
val topLevelWins =
json
.parseToJsonElement(
"""{"talk":{"realtime":{"provider":"openai","model":"gpt-realtime-2.1","providers":{"openai":{"model":"gpt-live-1-codex"}}}}}""",
"""{"talk":{"realtime":{"provider":"openai","model":"gpt-realtime-2.1","providers":{"openai":{"model":"gpt-live-test-canary"}}}}}""",
).jsonObject
assertFalse(TalkModeGatewayConfigParser.parse(providerLevelBrowserOnly).realtimeRelayModelSupported)
apps/macos/Tests/OpenClawIPCTests/TalkModeGatewayConfigTests.swift
+4 -4
View File
@@ -64,7 +64,7 @@ struct TalkModeGatewayConfigTests {
"speakerVoice": "alloy",
],
],
"model": " gpt-live-1-codex ",
"model": " gpt-live-test-canary ",
"speakerVoice": " cedar ",
"mode": " Realtime ",
"transport": " Gateway-Relay ",
@@ -75,7 +75,7 @@ struct TalkModeGatewayConfigTests {
let parsed = Self.parse(snapshot)
#expect(parsed.realtimeProvider == "OpenAI")
#expect(parsed.realtimeModelId == "gpt-live-1-codex")
#expect(parsed.realtimeModelId == "gpt-live-test-canary")
#expect(parsed.realtimeSpeakerVoice == "cedar")
#expect(parsed.realtimeMode == "realtime")
#expect(parsed.realtimeTransport == "gateway-relay")
@@ -113,7 +113,7 @@ struct TalkModeGatewayConfigTests {
"provider": "OPENAI",
"providers": [
"openai": [
"model": "gpt-live-1-codex",
"model": "gpt-live-test-canary",
"speakerVoice": "cedar",
],
],
@@ -123,7 +123,7 @@ struct TalkModeGatewayConfigTests {
let parsed = Self.parse(snapshot)
#expect(parsed.realtimeProvider == "OPENAI")
#expect(parsed.realtimeModelId == "gpt-live-1-codex")
#expect(parsed.realtimeModelId == "gpt-live-test-canary")
#expect(parsed.realtimeSpeakerVoice == "cedar")
}
config/assertion-safety-baseline.txt
-1
View File
@@ -943,7 +943,6 @@ extensions/openai/realtime-quicksilver-bridge.ts 1
extensions/openai/realtime-quicksilver-delegation-controller.ts 1
extensions/openai/realtime-quicksilver-gateway-bridge.ts 1
extensions/openai/realtime-quicksilver-wire.ts 6
extensions/openai/realtime-quicksilver.ts 1
extensions/openai/realtime-voice-bridge.ts 3
extensions/openai/realtime-voice-provider.ts 1
extensions/openai/realtime-voice-session-policy.ts 4
docs/nodes/talk.md
+14 -12
View File
@@ -186,20 +186,22 @@ Supported keys: `voice` / `voice_id` / `voiceId`, `model` / `model_id` / `modelI
```
OpenAI browser WebRTC and Gateway-relay Talk support native GPT-Live through
`https://api.openai.com/v1/live`. Set `talk.realtime.model` to
`gpt-live-1-codex` (recommended) or `gpt-live-1-boulder-alpha`; `gpt-live-1`
and `gpt-live-1-mini` are not valid on this route. Browser and Gateway-relay
WebRTC prefer a ChatGPT OAuth subscription profile and fall back to Platform
API-key auth. Other backend bridges connect directly over the Frameless Bidi
WebSocket and require Platform API-key auth, whose `/v1/live` access is currently
`https://api.openai.com/v1/live`. Set `talk.realtime.model` to the opaque
GPT-Live model identifier issued for your account. OpenClaw intentionally does
not publish these account-scoped identifiers in its catalog or model picker.
Browser and Gateway-relay WebRTC prefer a ChatGPT OAuth subscription profile
and fall back to Platform API-key auth. Other backend bridges connect directly
over the Frameless Bidi WebSocket and require Platform API-key auth, whose
`/v1/live` access is currently
[waitlist-gated](https://openai.com/form/gpt-live-1-in-the-api/).
The quickest setup is the Control UI: **Settings → Talk**, pick **OpenAI** and
a `gpt-live-*` model. The OAuth prerequisite is an OpenClaw auth profile
created with `openclaw models auth login --provider openai` — an existing
Codex CLI sign-in is not read. GPT-Live also requires the bundled `openai`
plugin registered in full mode; a restrictive `plugins.allow` list fails
session creation with "OpenAI GPT-Live browser session broker is unavailable".
Select **OpenAI** under **Settings → Talk**, then set the account-issued model
identifier in `talk.realtime.model`. The OAuth prerequisite is an OpenClaw auth
profile created with `openclaw models auth login --provider openai` — an
existing Codex CLI sign-in is not read. GPT-Live also requires the bundled
`openai` plugin registered in full mode; a restrictive `plugins.allow` list
fails session creation with "OpenAI GPT-Live browser session broker is
unavailable".
Runtime bounds: 8 concurrent sessions per Gateway and a 30-minute session TTL.
Browser sessions also use 60-second single-use offer tokens.
docs/providers/openai.md
+7 -8
View File
@@ -1036,10 +1036,10 @@ value into `plugins.entries.openai.config.personality` when that key is unset.
and Voice Call use the Frameless Bidi
`wss://api.openai.com/v1/live?model=...` endpoint with Platform API-key auth.
Use `gpt-live-1-codex` (recommended) or
`gpt-live-1-boulder-alpha`. The values `gpt-live-1` and
`gpt-live-1-mini` are not valid on this route. Opt in explicitly with
`talk.realtime.model`; `gpt-realtime-2.1` remains the GA default.
Use the opaque GPT-Live model identifier issued for your account. OpenClaw
intentionally omits account-scoped identifiers from its public catalog and
model picker. Opt in explicitly with `talk.realtime.model`;
`gpt-realtime-2.1` remains the GA default.
GPT-Live accepts these voices: `alloy`, `ash`, `ballad`, `cedar`, `coral`,
`echo`, `marin`, `sage`, `shimmer`, and `verse`. OpenClaw defaults to
@@ -1051,8 +1051,7 @@ value into `plugins.entries.openai.config.personality` when that key is unset.
An existing Codex CLI (`~/.codex`) sign-in is **not** read; the profile
must exist in OpenClaw. A Platform API key with `/v1/live` access works
instead, but that access is waitlist-gated.
2. `talk.realtime.model` set to a `gpt-live-*` value — via **Settings →
Talk** in the Control UI or the config below.
2. `talk.realtime.model` set to the account-issued opaque value in config.
3. The bundled `openai` plugin registered in full mode. A restrictive
`plugins.allow` list fails with "OpenAI GPT-Live browser session broker
is unavailable".
@@ -1067,7 +1066,7 @@ value into `plugins.entries.openai.config.personality` when that key is unset.
talk: {
realtime: {
provider: "openai",
model: "gpt-live-1-codex",
model: "<account-issued-model>",
transport: "webrtc",
},
},
@@ -1084,7 +1083,7 @@ value into `plugins.entries.openai.config.personality` when that key is unset.
talk: {
realtime: {
provider: "openai",
model: "gpt-live-1-codex",
model: "<account-issued-model>",
transport: "gateway-relay",
},
},
extensions/openai/provider-policy-api.ts
+13
View File
@@ -26,6 +26,7 @@ import {
OPENAI_GPT_56_MODEL_ID,
OPENAI_GPT_56_SOL_MODEL_ID,
} from "./model-route-contract.js";
import { isOpenAIGptLiveModel } from "./realtime-quicksilver.js";
import { resolveUnifiedOpenAIThinkingProfile } from "./thinking-policy.js";
const OPENAI_RESPONSES_API = "openai-responses";
@@ -119,6 +120,18 @@ export function projectConfiguredModelRow(ctx: ProviderNormalizeResolvedModelCon
return null;
}
export function projectRealtimeVoicePublicConfig(ctx: {
providerConfig: Record<string, unknown>;
config: Record<string, unknown>;
}): Record<string, unknown> {
const model = normalizeOptionalString(ctx.config.model) ?? ctx.providerConfig.model;
if (!isOpenAIGptLiveModel(typeof model === "string" ? model : undefined)) {
return ctx.config;
}
const { model: _model, ...publicConfig } = ctx.config;
return publicConfig;
}
function firstRouteBaseUrl(...values: unknown[]): unknown {
for (const value of values) {
if (typeof value === "string") {
extensions/openai/realtime-audio-buffer-ownership.test.ts
+1 -1
View File
@@ -61,7 +61,7 @@ async function withRealtimeProvider(
})
: new OpenAIQuicksilverVoiceBridge({
providerConfig: {},
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
audioFormat: { encoding: "pcm16", sampleRateHz: 24000, channels: 1 },
resolveAuth: async () => ({ type: "api-key", token: "fixture-local" }),
webSocketFactory: (_url, options) => new WebSocket(endpoint, options),
extensions/openai/realtime-quicksilver-audio-pipeline.test.ts
+1 -1
View File
@@ -49,7 +49,7 @@ describe("GPT-Live gateway microphone audio pipeline", () => {
const onError = vi.fn();
const bridge = new OpenAIQuicksilverGatewayBridge({
providerConfig: {},
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
voice: "marin",
audioFormat: { encoding: "pcm16", sampleRateHz: 24_000, channels: 1 },
onAudio: vi.fn(),
extensions/openai/realtime-quicksilver-bridge.test.ts
+5 -3
View File
@@ -88,7 +88,7 @@ function createHarness(params?: {
const logger = { warn: vi.fn() };
const bridge = new OpenAIQuicksilverVoiceBridge({
providerConfig: {},
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
voice: "marin",
instructions: "Use delegation for real work.",
audioFormat:
@@ -132,7 +132,9 @@ describe("OpenAIQuicksilverVoiceBridge", () => {
await harness.bridge.connect();
expect(harness.connections).toHaveLength(1);
expect(harness.connections[0]?.url).toBe("wss://api.openai.com/v1/live?model=gpt-live-1-codex");
expect(harness.connections[0]?.url).toBe(
"wss://api.openai.com/v1/live?model=gpt-live-test-canary",
);
expect(harness.connections[0]?.options.headers).toMatchObject({
Authorization: "Bearer test-key",
"OpenAI-Alpha": "quicksilver=v2",
@@ -242,7 +244,7 @@ describe("OpenAIQuicksilverVoiceBridge", () => {
const sockets: FakeSocket[] = [];
const bridge = new OpenAIQuicksilverVoiceBridge({
providerConfig: {},
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
audioFormat: { encoding: "pcm16", sampleRateHz: 24000, channels: 1 },
resolveAuth: async () => ({ type: "api-key", token: "test-key" }),
webSocketFactory: (_url, _options) => {
extensions/openai/realtime-quicksilver-gateway-bridge-lifecycle.test.ts
+1 -1
View File
@@ -13,7 +13,7 @@ function createBridge(params: {
let socket: FakeSocket | undefined;
const bridge = new OpenAIQuicksilverGatewayBridge({
providerConfig: {},
model: "gpt-live-1-boulder-alpha",
model: "gpt-live-test-canary-alt",
voice: "marin",
audioFormat: { encoding: "pcm16", sampleRateHz: 24_000, channels: 1 },
onAudio: vi.fn(),
extensions/openai/realtime-quicksilver-gateway-bridge.live.test.ts
+2 -2
View File
@@ -87,7 +87,7 @@ describeLive("OpenAI GPT-Live gateway WebRTC peer", () => {
});
const bridge = new OpenAIQuicksilverGatewayBridge({
providerConfig: {},
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
voice: "marin",
instructions:
"This is a live transport check. Immediately say: OpenClaw gateway relay test OK.",
@@ -163,7 +163,7 @@ describeLive("OpenAI GPT-Live gateway WebRTC peer", () => {
let lateAudioBytes = 0;
const bridge = new OpenAIQuicksilverGatewayBridge({
providerConfig: {},
model: "gpt-live-1-boulder-alpha",
model: "gpt-live-test-canary-alt",
voice: "marin",
instructions: "Listen to the user. Do not speak or delegate.",
audioFormat: { encoding: "pcm16", sampleRateHz: 24_000, channels: 1 },
extensions/openai/realtime-quicksilver-gateway-bridge.test.ts
+4 -4
View File
@@ -609,7 +609,7 @@ describe("GPT-Live gateway relay bridge", () => {
const onClose = vi.fn();
const bridge = new OpenAIQuicksilverGatewayBridge({
providerConfig: {},
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
voice: "marin",
audioFormat: { encoding: "pcm16", sampleRateHz: 24_000, channels: 1 },
onAudio: vi.fn(),
@@ -810,7 +810,7 @@ describe("GPT-Live gateway relay bridge", () => {
const closePeer = vi.fn();
const bridge = new OpenAIQuicksilverGatewayBridge({
providerConfig: {},
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
voice: "marin",
audioFormat: { encoding: "pcm16", sampleRateHz: 24_000, channels: 1 },
onAudio: vi.fn(),
@@ -868,7 +868,7 @@ describe("GPT-Live gateway relay bridge", () => {
const onClose = vi.fn();
const bridge = new OpenAIQuicksilverGatewayBridge({
providerConfig: {},
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
voice: "marin",
instructions: "Speak briefly.",
audioFormat: { encoding: "pcm16", sampleRateHz: 24_000, channels: 1 },
@@ -944,7 +944,7 @@ describe("GPT-Live gateway relay bridge", () => {
const onError = vi.fn();
const bridge = new OpenAIQuicksilverGatewayBridge({
providerConfig: {},
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
voice: "marin",
audioFormat: { encoding: "pcm16", sampleRateHz: 24_000, channels: 1 },
onAudio: vi.fn(),
extensions/openai/realtime-quicksilver-session.test.ts
+13 -11
View File
@@ -86,7 +86,7 @@ describe("GPT-Live session shaping", () => {
"shimmer",
"verse",
])("accepts the live-proven %s voice", (voice) => {
expect(buildOpenAIQuicksilverSession({ model: "gpt-live-1-codex", voice }).audio).toEqual({
expect(buildOpenAIQuicksilverSession({ model: "gpt-live-test-canary", voice }).audio).toEqual({
output: { voice },
});
});
@@ -94,15 +94,17 @@ describe("GPT-Live session shaping", () => {
it.each(["arbor", "breeze", "cove", "ember", "juniper", "maple", "sol", "spruce", "vale"])(
"falls back from the rejected %s voice",
(voice) => {
expect(buildOpenAIQuicksilverSession({ model: "gpt-live-1-codex", voice }).audio).toEqual({
output: { voice: "marin" },
});
expect(buildOpenAIQuicksilverSession({ model: "gpt-live-test-canary", voice }).audio).toEqual(
{
output: { voice: "marin" },
},
);
},
);
it("bounds initial items to the newest context", () => {
const session = buildOpenAIQuicksilverSession({
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
initialItems: Array.from({ length: 20 }, (_, index) => ({
role: index % 2 === 0 ? ("user" as const) : ("assistant" as const),
text: `${index}:${"x".repeat(1_000)}`,
@@ -467,7 +469,7 @@ describe("GPT-Live offer broker", () => {
const reservation = await realtime.broker.createBrowserSession(
{
providerConfig: {},
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
runAgentConsult: vi.fn(async () => ({ text: "Done" })),
},
{ type: "api-key", token: "platform-key" },
@@ -496,7 +498,7 @@ describe("GPT-Live offer broker", () => {
const reservation = await realtime.broker.createBrowserSession(
{
providerConfig: {},
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
runAgentConsult: vi.fn(async () => ({ text: "Done" })),
},
{ type: "api-key", token: "platform-key" },
@@ -541,7 +543,7 @@ describe("GPT-Live offer broker", () => {
});
try {
const reservation = await realtime.broker.createBrowserSession(
{ providerConfig: {}, model: "gpt-live-1-codex", runAgentConsult },
{ providerConfig: {}, model: "gpt-live-test-canary", runAgentConsult },
{ type: "api-key", token: "platform-key" },
);
if (reservation.transport !== "webrtc") {
@@ -582,7 +584,7 @@ describe("GPT-Live offer broker", () => {
const reservation = await realtime.broker.createBrowserSession(
{
providerConfig: {},
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
runAgentConsult: vi.fn(async () => ({ text: "Done" })),
},
{ type: "api-key", token: "platform-key" },
@@ -613,7 +615,7 @@ describe("GPT-Live offer broker", () => {
const reservation = await realtime.broker.createBrowserSession(
{
providerConfig: {},
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
runAgentConsult: vi.fn(async () => ({ text: "Done" })),
},
{ type: "api-key", token: "platform-key" },
@@ -647,7 +649,7 @@ describe("GPT-Live offer broker", () => {
const reservation = await realtime.broker.createBrowserSession(
{
providerConfig: {},
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
runAgentConsult: vi.fn(async () => ({ text: "Done" })),
},
{ type: "api-key", token: "platform-key" },
extensions/openai/realtime-quicksilver-wire.test.ts
+9 -10
View File
@@ -34,7 +34,7 @@ describe("Realtime call creation", () => {
return createCallResponse("v=answer\r\n", `rtc_${requests.length}`);
}) as unknown as typeof fetch;
const session = buildOpenAIQuicksilverSession({
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
instructions: "Speak briefly.",
voice: "marin",
});
@@ -162,7 +162,7 @@ describe("Realtime call creation", () => {
status: 403,
body: "Voice session access denied.",
message:
"GPT-Live rejected the session (403). This overloaded response most often means the voice or model is invalid for /v1/live. Accepted voices: alloy, ash, ballad, cedar, coral, echo, marin, sage, shimmer, verse. Accepted models: gpt-live-1-codex, gpt-live-1-boulder-alpha. Account access may also be unavailable; verify the selected ChatGPT OAuth profile and chatgpt-account-id.",
"GPT-Live rejected the session (403). The voice, model, or account access may be unavailable; verify the configured voice and selected ChatGPT OAuth profile.",
},
{
name: "Platform waitlist denial",
@@ -175,8 +175,7 @@ describe("Realtime call creation", () => {
name: "unsupported route model",
status: 400,
body: "Field `session.model` is not allowed for this Codex realtime session",
message:
"The GPT-Live model value is not permitted on /v1/live. Accepted values are gpt-live-1-codex and gpt-live-1-boulder-alpha.",
message: "The configured GPT-Live model is not permitted for this account.",
},
])("maps $name", async ({ status, body, message }) => {
const fetchImpl = vi.fn(async () => new Response(body, { status }));
@@ -184,7 +183,7 @@ describe("Realtime call creation", () => {
auth: { type: "api-key", token: "platform-key" },
requestIds: createRequestIds("error"),
sdp: "v=offer\r\n",
session: buildOpenAIQuicksilverSession({ model: "gpt-live-1-codex" }),
session: buildOpenAIQuicksilverSession({ model: "gpt-live-test-canary" }),
fetchImpl: fetchImpl as unknown as typeof fetch,
});
await expect(promise).rejects.toMatchObject({
@@ -197,7 +196,7 @@ describe("Realtime call creation", () => {
it.each([
{
name: "GPT-Live",
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
expectedMessage: "GPT-Live call creation failed (429)",
},
{
@@ -280,7 +279,7 @@ describe("Realtime call creation", () => {
auth: { type: "oauth", token: "oauth-token", accountId: "acct-1" },
requestIds: createRequestIds("header-fallback"),
sdp: "v=offer\r\n",
session: buildOpenAIQuicksilverSession({ model: "gpt-live-1-codex" }),
session: buildOpenAIQuicksilverSession({ model: "gpt-live-test-canary" }),
fetchImpl: fetchImpl as unknown as typeof fetch,
}),
).resolves.toMatchObject({ callId });
@@ -295,7 +294,7 @@ describe("Realtime call creation", () => {
auth: { type: "oauth", token: "oauth-token", accountId: "acct-1" },
requestIds: createRequestIds("uuid-location"),
sdp: "v=offer\r\n",
session: buildOpenAIQuicksilverSession({ model: "gpt-live-1-codex" }),
session: buildOpenAIQuicksilverSession({ model: "gpt-live-test-canary" }),
fetchImpl: fetchImpl as unknown as typeof fetch,
}),
).resolves.toMatchObject({
@@ -317,7 +316,7 @@ describe("Realtime call creation", () => {
auth: { type: "oauth", token: "oauth-token", accountId: "acct-1" },
requestIds: createRequestIds("empty-answer"),
sdp: "v=offer\r\n",
session: buildOpenAIQuicksilverSession({ model: "gpt-live-1-codex" }),
session: buildOpenAIQuicksilverSession({ model: "gpt-live-test-canary" }),
fetchImpl: fetchImpl as unknown as typeof fetch,
}),
).rejects.toMatchObject({
@@ -331,7 +330,7 @@ describe("Realtime call creation", () => {
{
label: "GPT-Live",
auth: { type: "oauth" as const, token: "oauth-token", accountId: "acct-1" },
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
location: "/v1/live/rtc_oversized_answer",
},
{
extensions/openai/realtime-quicksilver-wire.ts
+2 -2
View File
@@ -399,7 +399,7 @@ function decodeOpenAIQuicksilverCallId(params: {
function describeOpenAIQuicksilverCallError(status: number, detail: string): string {
const normalized = detail.toLowerCase();
if (status === 403) {
return "GPT-Live rejected the session (403). This overloaded response most often means the voice or model is invalid for /v1/live. Accepted voices: alloy, ash, ballad, cedar, coral, echo, marin, sage, shimmer, verse. Accepted models: gpt-live-1-codex, gpt-live-1-boulder-alpha. Account access may also be unavailable; verify the selected ChatGPT OAuth profile and chatgpt-account-id.";
return "GPT-Live rejected the session (403). The voice, model, or account access may be unavailable; verify the configured voice and selected ChatGPT OAuth profile.";
}
if (
status === 400 &&
@@ -413,7 +413,7 @@ function describeOpenAIQuicksilverCallError(status: number, detail: string): str
normalized.includes("session.model") &&
normalized.includes("not allowed")
) {
return "The GPT-Live model value is not permitted on /v1/live. Accepted values are gpt-live-1-codex and gpt-live-1-boulder-alpha.";
return "The configured GPT-Live model is not permitted for this account.";
}
return `GPT-Live call creation failed (${status})${detail ? `: ${detail}` : ""}`;
}
extensions/openai/realtime-quicksilver.live.test.ts
+2 -2
View File
@@ -223,7 +223,7 @@ describeLive("GPT-Live Platform WebSocket", () => {
}
const bridge = new OpenAIQuicksilverVoiceBridge({
providerConfig: {},
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
voice: "marin",
instructions: "Keep this transport verification session silent.",
audioFormat: { encoding: "pcm16", sampleRateHz: 24000, channels: 1 },
@@ -457,7 +457,7 @@ describeLive("OpenAI OAuth WebRTC", () => {
requestIds,
sdp: offerSdp,
session: buildOpenAIQuicksilverSession({
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
instructions: "Keep this transport verification session silent.",
voice: "marin",
}),
extensions/openai/realtime-quicksilver.test.ts
+3 -10
View File
@@ -1,6 +1,6 @@
// Openai tests cover GPT-Live (quicksilver) realtime voice gating.
import { beforeEach, describe, expect, it, vi } from "vitest";
import { isOpenAIGptLiveModel, isSupportedOpenAIGptLiveModel } from "./realtime-quicksilver.js";
import { isOpenAIGptLiveModel } from "./realtime-quicksilver.js";
import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js";
const mintSecretMock = vi.hoisted(() => vi.fn());
@@ -28,13 +28,6 @@ describe("openai gpt-live model detection", () => {
expect(isOpenAIGptLiveModel("gpt-realtime-2.1")).toBe(false);
expect(isOpenAIGptLiveModel("gpt-liveish")).toBe(false);
});
it("advertises only curated /v1/live models", () => {
expect(isSupportedOpenAIGptLiveModel("gpt-live-1-codex")).toBe(true);
expect(isSupportedOpenAIGptLiveModel(" GPT-Live-1-Boulder-Alpha ")).toBe(true);
expect(isSupportedOpenAIGptLiveModel("gpt-live-1")).toBe(false);
expect(isSupportedOpenAIGptLiveModel("gpt-live-1-mini")).toBe(false);
});
});
describe("openai realtime voice provider gpt-live transport routing", () => {
@@ -70,7 +63,7 @@ describe("openai realtime voice provider gpt-live transport routing", () => {
expect(
provider.createBridge({
...callbacks,
providerConfig: { apiKey: "test-key", model: "gpt-live-1-codex" },
providerConfig: { apiKey: "test-key", model: "gpt-live-test-canary" },
}),
).toMatchObject({ supportsToolResultContinuation: true });
expect(
@@ -94,7 +87,7 @@ describe("openai realtime voice provider gpt-live transport routing", () => {
provider.createBridge({
providerConfig: {
apiKey: "azure-test-key",
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
azureEndpoint: "https://example.openai.azure.com",
azureDeployment: "realtime",
},
extensions/openai/realtime-quicksilver.ts
-10
View File
@@ -3,8 +3,6 @@
const OPENAI_GPT_LIVE_MODEL_PREFIX = "gpt-live";
export const OPENAI_GPT_LIVE_MODELS = ["gpt-live-1-codex", "gpt-live-1-boulder-alpha"] as const;
export function isOpenAIGptLiveModel(model: string | undefined): boolean {
if (!model) {
return false;
@@ -15,11 +13,3 @@ export function isOpenAIGptLiveModel(model: string | undefined): boolean {
normalized.startsWith(`${OPENAI_GPT_LIVE_MODEL_PREFIX}-`)
);
}
export function isSupportedOpenAIGptLiveModel(model: string | undefined): boolean {
if (!model) {
return false;
}
const normalized = model.trim().toLowerCase();
return OPENAI_GPT_LIVE_MODELS.includes(normalized as (typeof OPENAI_GPT_LIVE_MODELS)[number]);
}
extensions/openai/realtime-voice-provider-routing.test.ts
+75 -51
View File
@@ -39,6 +39,8 @@ vi.mock("openclaw/plugin-sdk/provider-auth", async (importOriginal) => {
});
import { createOpenAIRealtimeTestSupport } from "./realtime-voice-test-support.js";
const OPAQUE_REALTIME_MODEL = "gpt-live-test-canary";
const {
requireRecord,
requireFetchJsonBody,
@@ -84,6 +86,47 @@ describe("OpenAI realtime voice provider routing", () => {
});
});
it("admits opaque realtime models without publishing them", () => {
const { broker } = createQuicksilverBrowserBrokerFixture();
const provider = buildOpenAIRealtimeVoiceProvider({
quicksilverBrowserSessionBroker: broker,
});
const internalApi = readInternalRealtimeVoiceProviderApi(provider);
const providerConfig = {
apiKey: "test-api-key-platform",
model: OPAQUE_REALTIME_MODEL,
};
expect(provider.models?.some((model) => model.startsWith("gpt-live"))).toBe(false);
expect(
internalApi.isGatewayRelayConfigured({
providerConfig,
agentId: "main",
}),
).toBe(true);
expect(
internalApi.resolveGatewayRelayCapabilities({
providerConfig,
model: OPAQUE_REALTIME_MODEL,
}),
).toMatchObject({
handlesAgentConsult: true,
supportsToolCalls: false,
});
expect(
internalApi.projectPublicConfig({
providerConfig,
config: { model: OPAQUE_REALTIME_MODEL },
}),
).toEqual({});
expect(
internalApi.projectPublicConfig({
providerConfig: { model: "gpt-realtime-2.1" },
config: { model: "gpt-realtime-2.1" },
}),
).toEqual({ model: "gpt-realtime-2.1" });
});
it("advertises continuing realtime tool results", () => {
const provider = buildOpenAIRealtimeVoiceProvider();
const bridge = provider.createBridge({
@@ -130,15 +173,15 @@ describe("OpenAI realtime voice provider routing", () => {
expect(
resolveCapabilities({
providerConfig: { model: "gpt-realtime-2.1" },
model: "gpt-live-1-codex",
model: OPAQUE_REALTIME_MODEL,
}),
).toMatchObject(expected);
expect(
resolveCapabilities({
providerConfig: { model: "gpt-realtime-2.1" },
model: "gpt-live-1-mini",
model: "gpt-live-test-canary-alt",
}),
).not.toHaveProperty("handlesAgentConsult");
).toMatchObject(expected);
});
it("omits unsupported OpenAI tool names from browser sessions", async () => {
@@ -235,26 +278,18 @@ describe("OpenAI realtime voice provider routing", () => {
it.each([
{
$name: "provider | gpt-live-1-mini | ChatGPT OAuth | standard endpoint | not ready",
$name: "provider | opaque model | ChatGPT OAuth | standard endpoint | Platform-only",
surface: "provider" as const,
providerConfig: { model: "gpt-live-1-mini" },
providerConfig: { model: OPAQUE_REALTIME_MODEL },
agentId: "main",
expected: false,
expectAgentDir: false,
},
{
$name: "gateway-relay | gpt-live-1-mini | ChatGPT OAuth | standard endpoint | not ready",
surface: "gateway-relay" as const,
providerConfig: { model: "gpt-live-1-mini" },
agentId: "main",
expected: false,
expectAgentDir: false,
},
{
$name: "gateway-relay | gpt-live-1-mini | ChatGPT OAuth | Azure endpoint | not ready",
$name: "gateway-relay | opaque model | ChatGPT OAuth | Azure endpoint | not ready",
surface: "gateway-relay" as const,
providerConfig: {
model: "gpt-live-1-mini",
model: OPAQUE_REALTIME_MODEL,
azureEndpoint: "https://example.openai.azure.com",
azureDeployment: "gpt-live",
},
@@ -262,14 +297,6 @@ describe("OpenAI realtime voice provider routing", () => {
expected: false,
expectAgentDir: false,
},
{
$name: "browser | gpt-live-1-mini | ChatGPT OAuth | standard endpoint | not ready",
surface: "browser" as const,
providerConfig: { model: "gpt-live-1-mini" },
agentId: "main",
expected: false,
expectAgentDir: false,
},
{
$name:
"gateway-relay | gpt-realtime-2.1 | Platform API key | standard endpoint | not applicable",
@@ -293,11 +320,10 @@ describe("OpenAI realtime voice provider routing", () => {
expectAgentDir: false,
},
{
$name:
"gateway-relay | gpt-live-1-codex | Platform API key + OAuth | Azure endpoint | not ready",
$name: "gateway-relay | opaque model | Platform API key + OAuth | Azure endpoint | not ready",
surface: "gateway-relay" as const,
providerConfig: {
model: "gpt-live-1-codex",
model: OPAQUE_REALTIME_MODEL,
apiKey: "test-api-key-platform",
azureEndpoint: "https://example.openai.azure.com",
},
@@ -306,43 +332,41 @@ describe("OpenAI realtime voice provider routing", () => {
expectAgentDir: false,
},
{
$name:
"gateway-relay | gpt-live-1-mini | Platform API key + OAuth | standard endpoint | not ready",
$name: "gateway-relay | opaque model | Platform API key + OAuth | standard endpoint | ready",
surface: "gateway-relay" as const,
providerConfig: { model: "gpt-live-1-mini", apiKey: "test-api-key-platform" },
agentId: "main",
expected: false,
expectAgentDir: false,
},
{
$name: "browser | gpt-live-1-mini | Platform API key + OAuth | standard endpoint | not ready",
surface: "browser" as const,
providerConfig: { model: "gpt-live-1-mini", apiKey: "test-api-key-platform" },
agentId: "main",
expected: false,
expectAgentDir: false,
},
{
$name: "gateway-relay | gpt-live-1-codex | ChatGPT OAuth | standard endpoint | ready",
surface: "gateway-relay" as const,
providerConfig: { model: "gpt-live-1-codex" },
providerConfig: { model: OPAQUE_REALTIME_MODEL, apiKey: "test-api-key-platform" },
agentId: "main",
expected: true,
expectAgentDir: false,
},
{
$name:
"gateway-relay | gpt-live-1-codex | voice-agent ChatGPT OAuth | standard endpoint | ready",
$name: "browser | opaque model | Platform API key + OAuth | standard endpoint | ready",
surface: "browser" as const,
providerConfig: { model: OPAQUE_REALTIME_MODEL, apiKey: "test-api-key-platform" },
agentId: "main",
expected: true,
expectAgentDir: false,
},
{
$name: "gateway-relay | opaque model | ChatGPT OAuth | standard endpoint | ready",
surface: "gateway-relay" as const,
providerConfig: { model: "gpt-live-1-codex" },
providerConfig: { model: OPAQUE_REALTIME_MODEL },
agentId: "main",
expected: true,
expectAgentDir: false,
},
{
$name: "gateway-relay | opaque model | voice-agent ChatGPT OAuth | standard endpoint | ready",
surface: "gateway-relay" as const,
providerConfig: { model: OPAQUE_REALTIME_MODEL },
agentId: "voice-agent",
expected: true,
expectAgentDir: true,
},
{
$name: "browser | gpt-live-1-codex | ChatGPT OAuth | standard endpoint | ready",
$name: "browser | opaque model | ChatGPT OAuth | standard endpoint | ready",
surface: "browser" as const,
providerConfig: { model: "gpt-live-1-codex" },
providerConfig: { model: OPAQUE_REALTIME_MODEL },
agentId: "main",
expected: true,
expectAgentDir: false,
@@ -447,7 +471,7 @@ describe("OpenAI realtime voice provider routing", () => {
await provider.createBrowserSession?.({
providerConfig: { apiKey: "test-api-key-platform" },
model: "gpt-live-1-codex",
model: OPAQUE_REALTIME_MODEL,
agentId: "main",
workspaceDir: "/tmp/openclaw-agent-workspace",
initialItems: [],
extensions/openai/realtime-voice-provider.ts
+11 -10
View File
@@ -10,6 +10,7 @@ import type {
} from "openclaw/plugin-sdk/realtime-voice";
import { REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ } from "openclaw/plugin-sdk/realtime-voice";
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
import { projectRealtimeVoicePublicConfig } from "./provider-policy-api.js";
import {
createOpenAIRealtimeClientSecret,
resolveOpenAIProviderConfigRecord,
@@ -22,7 +23,7 @@ import {
OPENAI_QUICKSILVER_CAPABILITIES,
resolveOpenAIChatGptSubscriptionAuth,
} from "./realtime-quicksilver-session.js";
import { isOpenAIGptLiveModel, isSupportedOpenAIGptLiveModel } from "./realtime-quicksilver.js";
import { isOpenAIGptLiveModel } from "./realtime-quicksilver.js";
import { OpenAIRealtimeBridge } from "./realtime-voice-bridge.js";
import {
OPENAI_REALTIME_CAPABILITIES,
@@ -106,6 +107,10 @@ type OpenAIInternalRealtimeVoiceProviderApi = {
providerConfig: RealtimeVoiceProviderConfig;
model?: string;
}) => OpenAIInternalRealtimeVoiceCapabilities;
projectPublicConfig?: (ctx: {
providerConfig: RealtimeVoiceProviderConfig;
config: RealtimeVoiceProviderConfig;
}) => RealtimeVoiceProviderConfig;
validateGatewayRelayLaunch?: (ctx: {
cfg?: RealtimeVoiceBrowserSessionCreateRequest["cfg"];
providerConfig: RealtimeVoiceProviderConfig;
@@ -435,9 +440,6 @@ export function buildOpenAIRealtimeVoiceProvider(options?: {
}
const model = config.model ?? OPENAI_REALTIME_DEFAULT_MODEL;
if (isOpenAIGptLiveModel(model)) {
if (!isSupportedOpenAIGptLiveModel(model)) {
return false;
}
return (
options?.quicksilverBrowserSessionBroker !== undefined &&
(hasOpenAIRealtimePlatformAuthInput({
@@ -460,7 +462,7 @@ export function buildOpenAIRealtimeVoiceProvider(options?: {
},
resolveBrowserSessionCapabilities: ({ cfg, providerConfig, agentId, model }) => {
const config = normalizeProviderConfig(providerConfig);
if (isSupportedOpenAIGptLiveModel(model ?? config.model)) {
if (isOpenAIGptLiveModel(model ?? config.model)) {
return {
...OPENAI_REALTIME_CAPABILITIES,
...OPENAI_QUICKSILVER_CAPABILITIES,
@@ -483,18 +485,16 @@ export function buildOpenAIRealtimeVoiceProvider(options?: {
return false;
}
return (
isSupportedOpenAIGptLiveModel(config.model) &&
(hasOpenAIRealtimePlatformAuthInput({
hasOpenAIRealtimePlatformAuthInput({
configuredApiKey: config.apiKey,
cfg,
agentId,
}) ||
hasOpenAIChatGptSubscriptionAuthInput({ cfg, agentId }))
}) || hasOpenAIChatGptSubscriptionAuthInput({ cfg, agentId })
);
},
resolveGatewayRelayCapabilities: ({ providerConfig, model }) => {
const config = normalizeProviderConfig(providerConfig);
if (isSupportedOpenAIGptLiveModel(model ?? config.model)) {
if (isOpenAIGptLiveModel(model ?? config.model)) {
return {
...OPENAI_REALTIME_CAPABILITIES,
...OPENAI_QUICKSILVER_CAPABILITIES,
@@ -502,6 +502,7 @@ export function buildOpenAIRealtimeVoiceProvider(options?: {
}
return OPENAI_REALTIME_CAPABILITIES;
},
projectPublicConfig: projectRealtimeVoicePublicConfig,
validateGatewayRelayLaunch: ({ providerConfig, model, autoRespondToAudio }) => {
const config = normalizeProviderConfig(providerConfig);
if (autoRespondToAudio === false && isOpenAIGptLiveModel(model ?? config.model)) {
extensions/openai/realtime-voice-session-policy.ts
-2
View File
@@ -34,7 +34,6 @@ import {
resolveOpenAIProviderConfigRecord,
} from "./realtime-provider-shared.js";
import { resolveOpenAIChatGptSubscriptionAuth } from "./realtime-quicksilver-session.js";
import { OPENAI_GPT_LIVE_MODELS } from "./realtime-quicksilver.js";
export type OpenAIRealtimeVoice =
| "alloy"
@@ -94,7 +93,6 @@ export const OPENAI_REALTIME_MODELS = [
"gpt-realtime-2.1",
"gpt-realtime-2.1-mini",
"gpt-realtime-2",
...OPENAI_GPT_LIVE_MODELS,
] as const;
export const OPENAI_REALTIME_INPUT_TRANSCRIPTION_MODEL = "gpt-4o-mini-transcribe";
export const OPENAI_REALTIME_CAPABILITIES: RealtimeVoiceProviderCapabilities = {
extensions/openai/realtime-voice-test-support.ts
+4
View File
@@ -124,6 +124,10 @@ type InternalRealtimeVoiceProviderApi = {
supportsToolCalls?: boolean;
transports?: string[];
};
projectPublicConfig: (ctx: {
providerConfig: Record<string, unknown>;
config: Record<string, unknown>;
}) => Record<string, unknown>;
validateGatewayRelayLaunch: (ctx: {
cfg?: object;
providerConfig: Record<string, unknown>;
scripts/control-ui-mock-dev.ts
+1 -7
View File
@@ -2098,13 +2098,7 @@ async function createChatPickerScenario(
configured: true,
defaultModel: "gpt-realtime-2.1",
transports: ["webrtc", "gateway-relay"],
models: [
"gpt-realtime-2.1",
"gpt-realtime-2.1-mini",
"gpt-realtime-2",
"gpt-live-1-codex",
"gpt-live-1-boulder-alpha",
],
models: ["gpt-realtime-2.1", "gpt-realtime-2.1-mini", "gpt-realtime-2"],
voices: [
"alloy",
"ash",
src/gateway/server-methods/talk-client.ts
+6 -1
View File
@@ -46,6 +46,7 @@ import {
import { REALTIME_VOICE_DESCRIBE_VIEW_TOOL } from "../../talk/describe-view-tool.js";
import {
cancelInternalRealtimeVoiceBrowserSession,
projectInternalRealtimeVoicePublicConfig,
type InternalRealtimeVoiceBrowserSessionCreateRequest,
} from "../../talk/provider-internal.js";
import {
@@ -434,7 +435,11 @@ export const talkClientHandlers: GatewayRequestHandlers = {
respond(
true,
{
...session,
...projectInternalRealtimeVoicePublicConfig({
provider: resolution.provider,
providerConfig: resolution.providerConfig,
config: session,
}),
voiceSessionId,
...(wantsGatewayControl ? { clientControl: { owner: "gateway" as const } } : {}),
},
src/gateway/server-methods/talk.test.ts
+147 -22
View File
@@ -13,6 +13,9 @@ import { REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME } from "../../talk/describe-view
import { buildTalkRealtimeConfig } from "./talk-shared.js";
import { talkHandlers } from "./talk.js";
const INTERNAL_REALTIME_VOICE_PROVIDER = Symbol.for("openclaw.internal.realtime-voice-provider.v1");
const OPAQUE_REALTIME_MODEL = "gpt-live-test-canary";
const mocks = vi.hoisted(() => ({
getRuntimeConfig: vi.fn<() => OpenClawConfig>(),
readConfigFileSnapshot: vi.fn(),
@@ -25,6 +28,7 @@ const mocks = vi.hoisted(() => ({
canonicalizeRealtimeVoiceProviderId: vi.fn((providerId: string | undefined) =>
providerId === "gemini-live" ? "google" : providerId?.trim().toLowerCase(),
),
getRealtimeVoiceProvider: vi.fn(),
listRealtimeVoiceProviders: vi.fn(() => []),
canonicalizeRealtimeTranscriptionProviderId: vi.fn((providerId: string | undefined) =>
providerId === "openai-realtime" ? "openai" : providerId?.trim().toLowerCase(),
@@ -111,6 +115,7 @@ vi.mock("../../tts/tts-synthesis.js", () => ({ synthesizeTalkSpeech: mocks.synth
vi.mock("../../talk/provider-registry.js", () => ({
canonicalizeRealtimeVoiceProviderId: mocks.canonicalizeRealtimeVoiceProviderId,
getRealtimeVoiceProvider: mocks.getRealtimeVoiceProvider,
listRealtimeVoiceProviders: mocks.listRealtimeVoiceProviders,
}));
@@ -566,7 +571,7 @@ describe("talk.catalog handler", () => {
id: "openai",
label: "OpenAI Realtime",
defaultModel: "gpt-realtime-2.1",
models: ["gpt-realtime-2.1", "gpt-live-1-codex"],
models: ["gpt-realtime-2.1", "gpt-live-test-canary"],
voices: ["alloy", "marin"],
resolveConfig: vi.fn(({ rawConfig }: { rawConfig: Record<string, unknown> }) => rawConfig),
isConfigured: vi.fn(() => false),
@@ -591,7 +596,7 @@ describe("talk.catalog handler", () => {
realtime: {
provider: "openai",
providers: { openai: { model: "gpt-realtime-2.1" } },
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
},
},
}) as OpenClawConfig,
@@ -602,7 +607,7 @@ describe("talk.catalog handler", () => {
realtime: { providers: Array<Record<string, unknown>> };
};
expect(catalog.realtime.providers[0]).toMatchObject({
models: ["gpt-realtime-2.1", "gpt-live-1-codex"],
models: ["gpt-realtime-2.1", "gpt-live-test-canary"],
voices: ["alloy", "marin"],
});
// Catalog readiness must mirror talk.client.create: top-level
@@ -610,14 +615,14 @@ describe("talk.catalog handler", () => {
// agent scope is consulted, or GPT-Live over OAuth reads as unconfigured.
expect(mocks.resolveConfiguredRealtimeVoiceProvider).toHaveBeenCalledWith(
expect.objectContaining({
providerConfigOverrides: { model: "gpt-live-1-codex" },
providerConfigOverrides: { model: "gpt-live-test-canary" },
agentId: expect.any(String),
}),
);
expect(mocks.isRealtimeVoiceProviderConfigured).toHaveBeenCalledWith(
expect.objectContaining({
agentId: expect.any(String),
providerConfig: expect.objectContaining({ model: "gpt-live-1-codex" }),
providerConfig: expect.objectContaining({ model: "gpt-live-test-canary" }),
surface: "browser-session",
}),
);
@@ -1088,6 +1093,14 @@ describe("talk.speak handler", () => {
describe("talk.config handler", () => {
beforeEach(() => {
vi.clearAllMocks();
mocks.getRealtimeVoiceProvider.mockImplementation((providerId: string | undefined) =>
mocks
.listRealtimeVoiceProviders()
.find(
(provider: { id: string; aliases?: string[] }) =>
provider.id === providerId || provider.aliases?.includes(providerId ?? ""),
),
);
});
it.each([
@@ -1167,6 +1180,102 @@ describe("talk.config handler", () => {
});
});
it("omits provider-owned opaque realtime models from config responses", async () => {
const provider = {
id: "openai",
label: "OpenAI Realtime",
isConfigured: () => true,
createBridge: vi.fn(),
[INTERNAL_REALTIME_VOICE_PROVIDER]: {
isBrowserSessionConfigured: () => true,
projectPublicConfig: ({ config }: { config: Record<string, unknown> }) => {
const { model: _model, ...publicConfig } = config;
return publicConfig;
},
},
};
const config = {
talk: {
realtime: {
provider: "openai",
model: OPAQUE_REALTIME_MODEL,
providers: { openai: { model: OPAQUE_REALTIME_MODEL } },
},
},
} as OpenClawConfig;
mocks.listRealtimeVoiceProviders.mockReturnValue([provider] as never);
mocks.readConfigFileSnapshot.mockResolvedValue({ config });
const respond = vi.fn();
await callTalkHandler("talk.config", {
params: {},
client: { connect: { scopes: ["operator.read"] } },
respond,
context: { getRuntimeConfig: () => config },
});
const response = expectRespondOk(respond);
expect(JSON.stringify(response)).not.toContain(OPAQUE_REALTIME_MODEL);
expect(response).toMatchObject({
config: {
talk: {
realtime: {
provider: "openai",
providers: { openai: {} },
},
},
},
});
});
it.each([
{
label: "runtime provider unavailable",
runtimePolicy: undefined,
registeredProviders: [],
},
{
label: "runtime provider blocked by policy",
runtimePolicy: { plugins: { enabled: false } },
registeredProviders: [{ id: "openai" }],
},
])("omits opaque realtime models when $label", async (testCase) => {
const config = {
...testCase.runtimePolicy,
talk: {
realtime: {
provider: "openai",
model: OPAQUE_REALTIME_MODEL,
providers: { openai: { model: OPAQUE_REALTIME_MODEL } },
},
},
} as OpenClawConfig;
mocks.listRealtimeVoiceProviders.mockReturnValue(testCase.registeredProviders as never);
mocks.getRealtimeVoiceProvider.mockReturnValue(undefined);
mocks.readConfigFileSnapshot.mockResolvedValue({ config });
const respond = vi.fn();
await callTalkHandler("talk.config", {
params: {},
client: { connect: { scopes: ["operator.read"] } },
respond,
context: { getRuntimeConfig: () => config },
});
const response = expectRespondOk(respond);
expect(JSON.stringify(response)).not.toContain(OPAQUE_REALTIME_MODEL);
expect(response).toMatchObject({
config: {
talk: {
realtime: {
provider: "openai",
providers: { openai: {} },
},
},
},
});
});
it("projects effective legacy realtime provider config for native routing", async () => {
const resolveConfig = vi.fn(
({ rawConfig }: { rawConfig: Record<string, unknown> }): Record<string, unknown> => ({
@@ -2045,11 +2154,11 @@ describe("talk.session unified handlers", () => {
{
label: "request override from a configured GA model",
configuredModel: "gpt-realtime-2.1",
requestedModel: "gpt-live-1-codex",
requestedModel: "gpt-live-test-canary",
},
{
label: "configured supported model without an override",
configuredModel: "gpt-live-1-codex",
configuredModel: "gpt-live-test-canary",
requestedModel: undefined,
},
])("resolves relay readiness from the effective model: $label", async (testCase) => {
@@ -2058,19 +2167,26 @@ describe("talk.session unified handlers", () => {
label: "OpenAI Realtime",
isConfigured: () => false,
createBridge: vi.fn(),
[INTERNAL_REALTIME_VOICE_PROVIDER]: {
isBrowserSessionConfigured: () => true,
projectPublicConfig: ({ config }: { config: Record<string, unknown> }) => {
const { model: _model, ...publicConfig } = config;
return publicConfig;
},
},
};
mocks.resolveConfiguredRealtimeVoiceProvider.mockImplementationOnce((input) => {
expect(input).toEqual(
expect.objectContaining({
agentId: "voice-agent",
providerConfigOverrides: { model: "gpt-live-1-codex" },
providerConfigOverrides: { model: "gpt-live-test-canary" },
defaultModel: testCase.configuredModel,
surface: "gateway-relay",
}),
);
return {
provider,
providerConfig: { model: "gpt-live-1-codex" },
providerConfig: { model: "gpt-live-test-canary" },
} as never;
});
mocks.createTalkRealtimeRelaySession.mockReturnValueOnce({
@@ -2083,7 +2199,7 @@ describe("talk.session unified handlers", () => {
outputEncoding: "pcm16",
outputSampleRateHz: 24000,
},
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
voice: "marin",
expiresAt: 1_797_986_400,
});
@@ -2117,8 +2233,8 @@ describe("talk.session unified handlers", () => {
expect(mocks.createTalkRealtimeRelaySession).toHaveBeenCalledWith(
expect.objectContaining({
provider,
providerConfig: { model: "gpt-live-1-codex" },
model: "gpt-live-1-codex",
providerConfig: { model: "gpt-live-test-canary" },
model: "gpt-live-test-canary",
sessionKey: "agent:voice-agent:main",
}),
);
@@ -2138,7 +2254,7 @@ describe("talk.session unified handlers", () => {
};
mocks.resolveConfiguredRealtimeVoiceProvider.mockReturnValue({
provider,
providerConfig: { model: "gpt-live-1-codex" },
providerConfig: { model: "gpt-live-test-canary" },
});
mocks.resolveInternalRealtimeVoiceGatewayRelayLaunchError.mockReturnValueOnce(
"GPT-Live gateway-relay sessions cannot use forced agent consult routing; GPT-Live delegates to the agent natively",
@@ -2151,7 +2267,7 @@ describe("talk.session unified handlers", () => {
transport: "gateway-relay",
brain: "agent-consult",
provider: "openai",
model: "gpt-live-1-codex",
model: "gpt-live-test-canary",
},
respond,
context: {
@@ -2160,7 +2276,7 @@ describe("talk.session unified handlers", () => {
talk: {
realtime: {
provider: "openai",
providers: { openai: { model: "gpt-live-1-codex" } },
providers: { openai: { model: "gpt-live-test-canary" } },
consultRouting: "force-agent-consult",
},
},
@@ -2171,8 +2287,8 @@ describe("talk.session unified handlers", () => {
expect(mocks.resolveInternalRealtimeVoiceGatewayRelayLaunchError).toHaveBeenCalledWith({
provider,
cfg: expect.any(Object),
providerConfig: { model: "gpt-live-1-codex" },
model: "gpt-live-1-codex",
providerConfig: { model: "gpt-live-test-canary" },
model: "gpt-live-test-canary",
autoRespondToAudio: false,
});
expectRespondError(respond, {
@@ -3088,6 +3204,7 @@ describe("talk.client.create handler", () => {
provider: "openai",
transport: "webrtc" as const,
clientSecret: "secret",
model: OPAQUE_REALTIME_MODEL,
}));
const provider = {
id: "openai",
@@ -3095,10 +3212,17 @@ describe("talk.client.create handler", () => {
isConfigured: () => true,
createBrowserSession,
createBridge: vi.fn(),
[INTERNAL_REALTIME_VOICE_PROVIDER]: {
isBrowserSessionConfigured: () => true,
projectPublicConfig: ({ config }: { config: Record<string, unknown> }) => {
const { model: _model, ...publicConfig } = config;
return publicConfig;
},
},
};
mocks.resolveConfiguredRealtimeVoiceProvider.mockReturnValue({
provider,
providerConfig: { model: "gpt-live-1" },
providerConfig: { model: OPAQUE_REALTIME_MODEL },
});
mocks.resolveRealtimeVoiceProviderCapabilities.mockReturnValueOnce({
transports: ["webrtc"],
@@ -3109,7 +3233,7 @@ describe("talk.client.create handler", () => {
const respond = vi.fn();
await callTalkHandler("talk.client.create", {
params: { sessionKey: "main", model: "gpt-live-1" },
params: { sessionKey: "main", model: OPAQUE_REALTIME_MODEL },
respond,
client: { connId: "conn-1", connect: { scopes: ["operator.write"] } },
context: {
@@ -3126,14 +3250,14 @@ describe("talk.client.create handler", () => {
});
expect(mocks.resolveConfiguredRealtimeVoiceProvider).toHaveBeenCalledWith(
expect.objectContaining({ providerConfigOverrides: { model: "gpt-live-1" } }),
expect.objectContaining({ providerConfigOverrides: { model: OPAQUE_REALTIME_MODEL } }),
);
expect(mocks.resolveRealtimeVoiceProviderCapabilities).toHaveBeenCalledWith(
expect.objectContaining({ agentId: "main", model: "gpt-live-1" }),
expect.objectContaining({ agentId: "main", model: OPAQUE_REALTIME_MODEL }),
);
const createInput = mockCallArg(createBrowserSession) as Record<string, unknown>;
expectRecordFields(createInput, {
model: "gpt-live-1",
model: OPAQUE_REALTIME_MODEL,
runAgentConsult: expect.any(Function),
});
await (
@@ -3144,6 +3268,7 @@ describe("talk.client.create handler", () => {
expect(consultInput).not.toHaveProperty("toolsAllow");
expect(createInput).not.toHaveProperty("tools");
expectRespondOk(respond, { provider: "openai", transport: "webrtc" });
expect(JSON.stringify(mockCallArg(respond, 0, 1))).not.toContain(OPAQUE_REALTIME_MODEL);
});
it("returns a Gateway-owned descriptor only after a supported reservation succeeds", async () => {
src/gateway/server-methods/talk.ts
+48 -4
View File
@@ -36,8 +36,10 @@ import {
isSecretOwnerAvailable,
} from "../../secrets/runtime-degraded-state.js";
import { resolveTalkSessionAgentId } from "../../talk/agent-target.js";
import { projectInternalRealtimeVoicePublicConfig } from "../../talk/provider-internal.js";
import {
canonicalizeRealtimeVoiceProviderId,
getRealtimeVoiceProvider,
listRealtimeVoiceProviders,
} from "../../talk/provider-registry.js";
import {
@@ -568,10 +570,14 @@ function resolveTalkResponseFromConfig(params: {
...effectiveRealtime,
}
: configuredPayload?.realtime;
const sourcePayload: TalkConfigResponse = {
...configuredPayload,
...(realtime ? { realtime } : {}),
};
const sourcePayload = projectTalkRealtimePublicModels({
payload: {
...configuredPayload,
...(realtime ? { realtime } : {}),
},
runtimeConfig: params.runtimeConfig,
effectiveProvider,
});
const payload = params.includeSecrets
? projectTalkSourcePayloadForSecrets(sourcePayload)
: sourcePayload;
@@ -630,6 +636,44 @@ function resolveTalkResponseFromConfig(params: {
};
}
function projectTalkRealtimePublicModels(params: {
payload: TalkConfigResponse;
runtimeConfig: OpenClawConfig;
effectiveProvider?: string;
}): TalkConfigResponse {
const realtime = params.payload.realtime;
if (!realtime) {
return params.payload;
}
const project = <T extends TalkProviderConfig>(
providerId: string | undefined,
config: T,
providerConfig: TalkProviderConfig = config,
): T => {
const provider = getRealtimeVoiceProvider(providerId, params.runtimeConfig);
return projectInternalRealtimeVoicePublicConfig({
...(provider ? { provider } : {}),
providerId,
providerConfig,
config,
});
};
const providers = realtime.providers
? Object.fromEntries(
Object.entries(realtime.providers).map(([id, config]) => [id, project(id, config)]),
)
: undefined;
const providerConfig = realtime.providers?.[params.effectiveProvider ?? ""] ?? {};
return {
...params.payload,
realtime: project(
params.effectiveProvider,
{ ...realtime, ...(providers ? { providers } : {}) },
providerConfig,
),
};
}
function projectTalkResolvedProviderConfig(params: {
includeSecrets: boolean;
sourceProviderConfig: TalkProviderConfig;
src/gateway/talk-realtime-relay-session-create.ts
+11 -5
View File
@@ -4,6 +4,7 @@ import { formatErrorMessage } from "../infra/errors.js";
import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME } from "../talk/agent-consult-tool.js";
import { buildRealtimeVoiceAgentCancelProviderResult } from "../talk/agent-run-control-shared.js";
import { resolveTalkSessionAgentId } from "../talk/agent-target.js";
import { projectInternalRealtimeVoicePublicConfig } from "../talk/provider-internal.js";
import {
REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ,
type RealtimeVoiceCloseReason,
@@ -69,6 +70,11 @@ export function createTalkRealtimeRelaySession(
params: CreateTalkRealtimeRelaySessionParams,
): TalkRealtimeRelaySessionResult {
enforceRelaySessionLimits(params.connId);
const publicModel = projectInternalRealtimeVoicePublicConfig({
provider: params.provider,
providerConfig: params.providerConfig,
config: { model: params.model },
}).model;
const forceAgentConsultOnFinalTranscript = params.forceAgentConsultOnFinalTranscript === true;
const relaySessionId = randomUUID();
const expiresAtMs = resolveExpiresAtMsFromDurationMs(RELAY_SESSION_TTL_MS);
@@ -377,7 +383,7 @@ export function createTalkRealtimeRelaySession(
const issue = realtimeRelayIssue({
message: outcome.message,
provider: params.provider.id,
model: params.model,
model: publicModel,
phase: "response",
});
const errorTalkEvent = harness.talk.recentEvents.findLast(
@@ -520,7 +526,7 @@ export function createTalkRealtimeRelaySession(
const issue = realtimeRelayIssue({
message: formatErrorMessage(error),
provider: params.provider.id,
model: params.model,
model: publicModel,
phase: ready ? "stream" : "connect",
});
failureEmitted = true;
@@ -543,7 +549,7 @@ export function createTalkRealtimeRelaySession(
const issue = realtimeRelayIssue({
message: "Realtime provider closed before the session became ready.",
provider: params.provider.id,
model: params.model,
model: publicModel,
phase: "connect",
});
emit(relayIssuePayload(relaySessionId, issue), {
@@ -651,7 +657,7 @@ export function createTalkRealtimeRelaySession(
const issue = realtimeRelayIssue({
message: formatErrorMessage(error),
provider: params.provider.id,
model: params.model,
model: publicModel,
phase: "connect",
});
failureEmitted = true;
@@ -673,7 +679,7 @@ export function createTalkRealtimeRelaySession(
outputEncoding: "pcm16",
outputSampleRateHz: REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ.sampleRateHz,
},
...(params.model ? { model: params.model } : {}),
...(publicModel ? { model: publicModel } : {}),
...(params.voice ? { voice: params.voice } : {}),
expiresAt: Math.floor(expiresAtMs / 1000),
};
src/gateway/talk-realtime-relay.test.ts
+25 -1
View File
@@ -1524,6 +1524,20 @@ describe("talk realtime gateway relay", () => {
bridgeRequest = req;
return bridge;
},
[Symbol.for("openclaw.internal.realtime-voice-provider.v1")]: {
isBrowserSessionConfigured: () => true,
projectPublicConfig: ({
config,
}: {
config: Record<string, unknown>;
}): Record<string, unknown> => {
if (config.model !== "gpt-live-test-canary") {
return config;
}
const { model: _model, ...publicConfig } = config;
return publicConfig;
},
},
};
const events: Array<{
event: string;
@@ -1578,7 +1592,6 @@ describe("talk realtime gateway relay", () => {
autoRespondToAudio: true,
interruptResponseOnInputAudio: true,
});
const readyPayload = findEventPayload(events, (payload) => payload.type === "ready");
expectRecordFields(readyPayload, {
relaySessionId: session.relaySessionId,
@@ -1816,6 +1829,17 @@ describe("talk realtime gateway relay", () => {
});
expectRecordFields(closePayload.talkEvent, { type: "session.closed", final: true });
expectDelivery(closePayload, false);
const opaqueSession = createTalkRealtimeRelaySession({
context,
connId: "conn-1",
provider,
providerConfig: { model: "gpt-live-test-canary" },
instructions: "be brief",
tools: [],
model: "gpt-live-test-canary",
});
expect(opaqueSession).not.toHaveProperty("model");
});
it("emits generic issue details when relay connect fails", async () => {
src/plugins/provider-policy-surface.ts
+13
View File
@@ -35,6 +35,11 @@ type ProviderProjectConfiguredModelRowContext = {
model: ProviderRuntimeModel;
};
type ProviderProjectRealtimeVoicePublicConfigContext = {
providerConfig: Record<string, unknown>;
config: Record<string, unknown>;
};
type EmbeddingProviderSetupInspection = {
provider: string;
reason: string;
@@ -76,6 +81,9 @@ export type BundledProviderPolicySurface = ProviderPolicySurface & {
projectConfiguredModelRow?: (
ctx: ProviderProjectConfiguredModelRowContext,
) => ProviderRuntimeModel | null | undefined;
projectRealtimeVoicePublicConfig?: (
ctx: ProviderProjectRealtimeVoicePublicConfigContext,
) => Record<string, unknown> | null | undefined;
};
const bundledProviderPolicySurfaceByPluginId = new Map<
@@ -120,6 +128,11 @@ function extractBundledProviderPolicySurface(
surface.projectConfiguredModelRow =
mod.projectConfiguredModelRow as BundledProviderPolicySurface["projectConfiguredModelRow"];
}
if (typeof mod.projectRealtimeVoicePublicConfig === "function") {
Object.assign(surface, {
projectRealtimeVoicePublicConfig: mod.projectRealtimeVoicePublicConfig,
});
}
return Object.keys(surface).length > 0 ? surface : null;
}
src/talk/provider-internal.ts
+34
View File
@@ -6,6 +6,7 @@
* RealtimeVoiceProviderPlugin contract.
*/
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { resolveBundledProviderPolicySurface } from "../plugins/provider-public-artifacts.js";
import type { RealtimeVoiceProviderPlugin } from "../plugins/types.js";
import type {
RealtimeVoiceBrowserSession,
@@ -57,6 +58,10 @@ type InternalRealtimeVoiceProviderApi = {
providerConfig: RealtimeVoiceProviderConfig;
model?: string;
}) => InternalRealtimeVoiceProviderCapabilities;
projectPublicConfig?: (ctx: {
providerConfig: RealtimeVoiceProviderConfig;
config: RealtimeVoiceProviderConfig;
}) => RealtimeVoiceProviderConfig;
validateGatewayRelayLaunch?: (ctx: {
cfg?: OpenClawConfig;
providerConfig: RealtimeVoiceProviderConfig;
@@ -138,6 +143,35 @@ export function resolveInternalRealtimeVoiceGatewayRelayCapabilities(params: {
});
}
export function projectInternalRealtimeVoicePublicConfig<
T extends RealtimeVoiceProviderConfig,
>(params: {
provider?: RealtimeVoiceProviderPlugin;
providerId?: string;
providerConfig: RealtimeVoiceProviderConfig;
config: T;
}): T {
const project =
(params.provider
? readInternalRealtimeVoiceProviderApi(params.provider)?.projectPublicConfig
: undefined) ??
(params.providerId
? resolveBundledProviderPolicySurface(params.providerId)?.projectRealtimeVoicePublicConfig
: undefined);
const projected = project?.({ providerConfig: params.providerConfig, config: params.config });
if (projected) {
return projected as T; // SAFETY: projections only remove or replace `model`; other fields stay intact.
}
if (params.provider) {
return params.config;
}
// A cold or policy-blocked owner cannot attest that its model id is public.
// Preserve the rest of the control-plane config, but fail closed on that field.
const { model: _model, ...publicConfig } = params.config;
// SAFETY: the fail-closed projection only removes `model`; every retained field preserves T.
return publicConfig as T;
}
export function resolveInternalRealtimeVoiceGatewayRelayLaunchError(params: {
provider: RealtimeVoiceProviderPlugin;
cfg?: OpenClawConfig;
ui/src/pages/config/talk.test.ts
+17 -17
View File
@@ -46,10 +46,10 @@ function createTalkMutationHarness(options: TalkMutationHarnessOptions = {}) {
label: "OpenAI",
configured: true,
aliases: options.aliases ?? [],
models: ["gpt-live-1-boulder-alpha"],
models: ["gpt-live-test-canary-alt"],
voices: ["marin"],
transports: options.transports ?? ["gateway-relay"],
defaultModel: options.defaultModel ?? "gpt-live-1-boulder-alpha",
defaultModel: options.defaultModel ?? "gpt-live-test-canary-alt",
},
{
id: "xai",
@@ -143,7 +143,7 @@ afterEach(() => {
});
describe("isTalkGptLiveModel", () => {
it.each(["gpt-live", "gpt-live-1-codex", " GPT-Live-1-Boulder-Alpha "])(
it.each(["gpt-live", "gpt-live-test-canary", " GPT-Live-Test-Canary-Alt "])(
"accepts the GPT-Live family: %s",
(model) => {
expect(isTalkGptLiveModel(model)).toBe(true);
@@ -278,7 +278,7 @@ describe("renderTalk", () => {
it.each([
["gpt-liveish", false],
["gpt-lively", false],
["gpt-live-1-codex", true],
["gpt-live-test-canary", true],
] as const)("renders the GPT-Live hint only for the exact family: %s", (model, showsHint) => {
const container = document.createElement("div");
render(
@@ -323,7 +323,7 @@ describe("renderTalk", () => {
describe("TalkSettingsPage realtime transport mutation", () => {
it("removes forced consult routing when OpenAI GPT-Live keeps gateway relay", async () => {
const removeFormValue = await selectModel("gpt-live-1-boulder-alpha", {
const removeFormValue = await selectModel("gpt-live-test-canary-alt", {
consultRouting: " Force-Agent-Consult ",
transports: ["gateway-relay"],
});
@@ -336,14 +336,14 @@ describe("TalkSettingsPage realtime transport mutation", () => {
it.each([
[
"provider-direct routing",
"gpt-live-1-boulder-alpha",
"gpt-live-test-canary-alt",
"provider-direct",
"openai",
"gateway-relay",
],
["another model", "gpt-realtime", "force-agent-consult", "openai", "gateway-relay"],
["another provider", "gpt-live-1-boulder-alpha", "force-agent-consult", "xai", "gateway-relay"],
["another transport", "gpt-live-1-boulder-alpha", "force-agent-consult", "openai", "webrtc"],
["another provider", "gpt-live-test-canary-alt", "force-agent-consult", "xai", "gateway-relay"],
["another transport", "gpt-live-test-canary-alt", "force-agent-consult", "openai", "webrtc"],
] as const)(
"preserves consult routing for %s",
async (_label, model, consultRouting, provider, transport) => {
@@ -378,8 +378,8 @@ describe("TalkSettingsPage realtime transport mutation", () => {
});
it.each([
["catalog default", "gpt-live-1-boulder-alpha", undefined],
["provider fallback", "gpt-realtime-2.1", "gpt-live-1-boulder-alpha"],
["catalog default", "gpt-live-test-canary-alt", undefined],
["provider fallback", "gpt-realtime-2.1", "gpt-live-test-canary-alt"],
])(
"removes forced consult when a provider switch activates a GPT-Live %s",
async (_label, defaultModel, openAIProviderModel) => {
@@ -407,19 +407,19 @@ describe("TalkSettingsPage realtime transport mutation", () => {
it("preserves transport when the catalog is unavailable", async () => {
expect(
await selectModel("gpt-live-1-boulder-alpha", { unavailable: true }),
await selectModel("gpt-live-test-canary-alt", { unavailable: true }),
).not.toHaveBeenCalled();
});
it("preserves transport when the provider advertises no transport capabilities", async () => {
expect(
await selectModel("gpt-live-1-boulder-alpha", { transports: [] }),
await selectModel("gpt-live-test-canary-alt", { transports: [] }),
).not.toHaveBeenCalled();
});
it("removes provider websocket from a selected GPT-Live model", async () => {
expect(
await selectModel("gpt-live-1-boulder-alpha", {
await selectModel("gpt-live-test-canary-alt", {
transport: "provider-websocket",
transports: ["provider-websocket", "webrtc"],
}),
@@ -428,13 +428,13 @@ describe("TalkSettingsPage realtime transport mutation", () => {
it("preserves a transport advertised by the explicit provider", async () => {
expect(
await selectModel("gpt-live-1-boulder-alpha", { transports: ["gateway-relay"] }),
await selectModel("gpt-live-test-canary-alt", { transports: ["gateway-relay"] }),
).not.toHaveBeenCalled();
});
it("resolves an explicit provider alias before preserving transport", async () => {
expect(
await selectModel("gpt-live-1-boulder-alpha", {
await selectModel("gpt-live-test-canary-alt", {
aliases: ["openai-preview"],
provider: "openai-preview",
transports: ["gateway-relay"],
@@ -444,7 +444,7 @@ describe("TalkSettingsPage realtime transport mutation", () => {
it("uses the auto-selected provider before preserving transport", async () => {
expect(
await selectModel("gpt-live-1-boulder-alpha", {
await selectModel("gpt-live-test-canary-alt", {
activeProvider: "openai",
provider: null,
transports: ["gateway-relay"],
@@ -454,7 +454,7 @@ describe("TalkSettingsPage realtime transport mutation", () => {
it("removes transport only when the resolved provider positively excludes it", async () => {
expect(
await selectModel("gpt-live-1-boulder-alpha", { transports: ["webrtc"] }),
await selectModel("gpt-live-test-canary-alt", { transports: ["webrtc"] }),
).toHaveBeenCalledOnce();
});