diff --git a/docs/plugins/google-meet.md b/docs/plugins/google-meet.md index 2b9b91c1b990..1e9779df0788 100644 --- a/docs/plugins/google-meet.md +++ b/docs/plugins/google-meet.md @@ -736,7 +736,7 @@ An `openclaw doctor --fix` migration exists for the legacy `realtime.provider: " provider: "openai", transcriptionProvider: "openai", voiceProvider: "google", - model: "gemini-2.5-flash-native-audio-preview-12-2025", + model: "gemini-3.1-flash-live-preview", agentId: "jay", toolPolicy: "owner", introMessage: "Say exactly: I'm here.", diff --git a/docs/plugins/voice-call.md b/docs/plugins/voice-call.md index 85b8bcbab03a..277b66b28bd6 100644 --- a/docs/plugins/voice-call.md +++ b/docs/plugins/voice-call.md @@ -328,7 +328,7 @@ for tool work, current information, memory lookups, or workspace state. Defaults: API key from `realtime.providers.google.apiKey`, `GEMINI_API_KEY`, - or `GOOGLE_API_KEY`; model `gemini-2.5-flash-native-audio-preview-12-2025`; + or `GOOGLE_API_KEY`; model `gemini-3.1-flash-live-preview`; voice `Kore`. `sessionResumption` and `contextWindowCompression` default on for longer, reconnectable calls. Use `silenceDurationMs`, `startSensitivity`, and `endSensitivity` to tune faster turn-taking on @@ -355,7 +355,7 @@ for tool work, current information, memory lookups, or workspace state. providers: { google: { apiKey: "${GEMINI_API_KEY}", - model: "gemini-2.5-flash-native-audio-preview-12-2025", + model: "gemini-3.1-flash-live-preview", speakerVoice: "Kore", silenceDurationMs: 500, startSensitivity: "high", diff --git a/docs/providers/google.md b/docs/providers/google.md index bd2192f074a8..0198a5e74410 100644 --- a/docs/providers/google.md +++ b/docs/providers/google.md @@ -342,14 +342,14 @@ Gemini Live API for backend audio bridges such as Voice Call and Google Meet. | Setting | Config path | Default | | --------------------- | ------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | -| Model | `plugins.entries.voice-call.config.realtime.providers.google.model` | `gemini-2.5-flash-native-audio-preview-12-2025` | +| Model | `plugins.entries.voice-call.config.realtime.providers.google.model` | `gemini-3.1-flash-live-preview` | | Voice | `...google.voice` | `Kore` | | Temperature | `...google.temperature` | (unset) | | VAD start sensitivity | `...google.startSensitivity` | (unset) | | VAD end sensitivity | `...google.endSensitivity` | (unset) | | Silence duration | `...google.silenceDurationMs` | (unset) | | Activity handling | `...google.activityHandling` | Google default, `start-of-activity-interrupts` | -| Turn coverage | `...google.turnCoverage` | Google default, `only-activity` | +| Turn coverage | `...google.turnCoverage` | Google default, `audio-activity-and-all-video` | | Disable auto VAD | `...google.automaticActivityDetectionDisabled` | `false` | | Session resumption | `...google.sessionResumption` | `true` | | Context compression | `...google.contextWindowCompression` | `true` | @@ -369,10 +369,10 @@ Example Voice Call realtime config: provider: "google", providers: { google: { - model: "gemini-2.5-flash-native-audio-preview-12-2025", + model: "gemini-3.1-flash-live-preview", speakerVoice: "Kore", activityHandling: "start-of-activity-interrupts", - turnCoverage: "only-activity", + turnCoverage: "audio-activity-and-all-video", }, }, }, @@ -393,6 +393,15 @@ Gemini API transcription is enabled without `languageCodes`; the current Google SDK rejects language-code hints on this API path. + +Gemini 3.1 Live accepts conversational text through realtime input and uses +sequential function calling. OpenClaw omits the older `NON_BLOCKING`, function +response scheduling, and affective-dialog fields for this model. Prefer +`thinkingLevel`; configured positive `thinkingBudget` values are mapped to the +nearest supported level, while `-1` leaves Google's default in place. See the +[Gemini Live capability comparison](https://ai.google.dev/gemini-api/docs/live-api/capabilities). + + Control UI Talk supports Google Live browser sessions with constrained one-use tokens. Backend-only realtime voice providers can also run through the generic diff --git a/docs/providers/qwen.md b/docs/providers/qwen.md index 1f35874de92c..808d921e0fc4 100644 --- a/docs/providers/qwen.md +++ b/docs/providers/qwen.md @@ -17,7 +17,8 @@ Qwen Cloud is an official external OpenClaw provider plugin with canonical id `q | API style | OpenAI-compatible | -`qwen3.7-plus` and `qwen3.6-plus` work with Coding Plan and Standard endpoints. For `qwen3.7-max`, use a **Standard (pay-as-you-go)** endpoint. +`qwen3.7-plus` and `qwen3.6-plus` work with Coding Plan and Standard endpoints. +For `qwen3.7-max` or `qwen3.6-flash`, use a **Standard (pay-as-you-go)** endpoint. ## Install plugin @@ -84,7 +85,7 @@ Choose your plan type and follow the setup steps. - **Best for:** pay-as-you-go access through the Standard Model Studio endpoint, including `qwen3.7-max`, which is not available on the Coding Plan. + **Best for:** pay-as-you-go access through the Standard Model Studio endpoint, including `qwen3.7-max` and `qwen3.6-flash`, which are not available on the Coding Plan. @@ -198,6 +199,7 @@ Plan configs omit models that only work on the Standard endpoint. | Model ref | Input | Context | Notes | | --------------------------- | ----------- | --------- | ----------------------- | | `qwen/qwen3.5-plus` | text, image | 1,000,000 | Default model | +| `qwen/qwen3.6-flash` | text, image | 1,000,000 | Standard endpoints only | | `qwen/qwen3.6-plus` | text, image | 1,000,000 | Coding Plan + Standard | | `qwen/qwen3.7-max` | text | 1,000,000 | Standard endpoints only | | `qwen/qwen3.7-plus` | text, image | 1,000,000 | Coding Plan + Standard | @@ -217,7 +219,8 @@ present in the static catalog. ## Thinking controls -`qwen3.7-max`, `qwen3.7-plus`, `qwen3.6-plus`, and `qwen/MiniMax-M2.5` are +`qwen3.7-max`, `qwen3.7-plus`, `qwen3.6-flash`, `qwen3.6-plus`, and +`qwen/MiniMax-M2.5` are reasoning-enabled in the built-in catalog. For reasoning models on the `qwen` family, the provider maps OpenClaw thinking levels to DashScope's top-level `enable_thinking` request flag: disabled thinking sends `enable_thinking: false`, @@ -263,15 +266,15 @@ See [Video generation](/tools/video-generation) for shared tool parameters, prov ## Advanced configuration - - `qwen3.7-plus` is available on Coding Plan and Standard endpoints. `qwen3.7-max` is Standard-only. The Standard (pay-as-you-go) endpoints are: + + `qwen3.7-plus` and `qwen3.6-plus` are available on Coding Plan and Standard endpoints. `qwen3.7-max` and `qwen3.6-flash` are Standard-only. The Standard (pay-as-you-go) endpoints are: - China: `dashscope.aliyuncs.com/compatible-mode/v1` - Global: `dashscope-intl.aliyuncs.com/compatible-mode/v1` - OpenClaw omits `qwen3.7-max` from Coding Plan catalogs. If a Coding Plan - endpoint returns an "unsupported model" error for it, switch to the matching - Standard endpoint and key. + OpenClaw omits `qwen3.7-max` and `qwen3.6-flash` from Coding Plan catalogs. + If a Coding Plan endpoint returns an "unsupported model" error for either, + switch to the matching Standard endpoint and key. diff --git a/extensions/google/index.test.ts b/extensions/google/index.test.ts index 9f9e8aee9167..8d550c1c44a8 100644 --- a/extensions/google/index.test.ts +++ b/extensions/google/index.test.ts @@ -404,6 +404,7 @@ describe("google provider plugin hooks", () => { if (!bridge) { throw new Error("expected Google realtime bridge"); } + expect(bridge.supportsToolResultContinuation).toBe(false); expect(bridge.sendAudio(Buffer.alloc(160))).toBeUndefined(); expect(bridge.setMediaTimestamp(20)).toBeUndefined(); expect(bridge.sendUserMessage?.("hello")).toBeUndefined(); diff --git a/extensions/google/index.ts b/extensions/google/index.ts index 724efd872566..dbccdf83b31e 100644 --- a/extensions/google/index.ts +++ b/extensions/google/index.ts @@ -245,7 +245,9 @@ function createLazyGoogleRealtimeVoiceBridge( } }; return { - supportsToolResultContinuation: true, + get supportsToolResultContinuation() { + return bridge?.supportsToolResultContinuation ?? false; + }, connect: async () => { const loadedBridge = await loadBridge(); if (closed) { diff --git a/extensions/google/realtime-voice-provider.test.ts b/extensions/google/realtime-voice-provider.test.ts index 1d26e2c7ef84..af55a0e1eee8 100644 --- a/extensions/google/realtime-voice-provider.test.ts +++ b/extensions/google/realtime-voice-provider.test.ts @@ -154,6 +154,7 @@ describe("buildGoogleRealtimeVoiceProvider", () => { it("declares realtime Talk capabilities for catalog selection", () => { const provider = buildGoogleRealtimeVoiceProvider(); + expect(provider.defaultModel).toBe("gemini-3.1-flash-live-preview"); expect(provider.capabilities).toEqual({ transports: ["provider-websocket", "gateway-relay"], inputAudioFormats: [ @@ -173,6 +174,35 @@ describe("buildGoogleRealtimeVoiceProvider", () => { }); }); + it("uses Gemini 3.1 Live-compatible defaults", async () => { + const provider = buildGoogleRealtimeVoiceProvider(); + const bridge = provider.createBridge({ + providerConfig: { + apiKey: "gemini-key", + enableAffectiveDialog: true, + thinkingBudget: 8_193, + }, + tools: [createRealtimeTool("openclaw_agent_consult")], + onAudio: vi.fn(), + onClearAudio: vi.fn(), + }); + + expect(bridge.supportsToolResultContinuation).toBe(false); + await bridge.connect(); + + const params = lastConnectParams(); + expect(params.model).toBe("gemini-3.1-flash-live-preview"); + expect(params.config.thinkingConfig).toEqual({ thinkingLevel: "HIGH" }); + expect(params.config).not.toHaveProperty("enableAffectiveDialog"); + const config = params.config as { + tools?: Array<{ functionDeclarations?: Array<{ behavior?: string; name?: string }> }>; + }; + expect(config.tools?.[0]?.functionDeclarations?.[0]).toMatchObject({ + name: "openclaw_agent_consult", + }); + expect(config.tools?.[0]?.functionDeclarations?.[0]).not.toHaveProperty("behavior"); + }); + it("normalizes provider config and cfg model-provider key fallback", () => { const provider = buildGoogleRealtimeVoiceProvider(); const resolved = provider.resolveConfig?.({ @@ -416,6 +446,7 @@ describe("buildGoogleRealtimeVoiceProvider", () => { const bridge = provider.createBridge({ providerConfig: { apiKey: "gemini-key", + model: "gemini-live-2.5-flash-preview", thinkingBudget: -1, }, onAudio: vi.fn(), @@ -427,6 +458,22 @@ describe("buildGoogleRealtimeVoiceProvider", () => { expect(lastConnectParams().config.thinkingConfig).toEqual({ thinkingBudget: -1 }); }); + it("omits adaptive thinking budgets for Gemini 3.1 Live", async () => { + const provider = buildGoogleRealtimeVoiceProvider(); + const bridge = provider.createBridge({ + providerConfig: { + apiKey: "gemini-key", + thinkingBudget: -1, + }, + onAudio: vi.fn(), + onClearAudio: vi.fn(), + }); + + await bridge.connect(); + + expect(lastConnectParams().config).not.toHaveProperty("thinkingConfig"); + }); + it("creates constrained browser sessions for Google Live Talk", async () => { const provider = buildGoogleRealtimeVoiceProvider(); @@ -546,6 +593,46 @@ describe("buildGoogleRealtimeVoiceProvider", () => { ]); }); + it("constrains default browser sessions to Gemini 3.1 capabilities", async () => { + const provider = buildGoogleRealtimeVoiceProvider(); + + const sessionLocal = await provider.createBrowserSession?.({ + providerConfig: { + apiKey: "gemini-key", + enableAffectiveDialog: true, + thinkingLevel: "low", + thinkingBudget: 8_193, + }, + tools: [createRealtimeTool("openclaw_agent_consult")], + }); + + const tokenConfig = requireFirstMockArg(createTokenMock, "Google Live auth token config") as { + config?: { + liveConnectConstraints?: { + config?: { + enableAffectiveDialog?: boolean; + thinkingConfig?: unknown; + tools?: Array<{ + functionDeclarations?: Array<{ behavior?: string; name?: string }>; + }>; + }; + model?: string; + }; + }; + }; + const constraints = tokenConfig.config?.liveConnectConstraints; + expect(constraints?.model).toBe("gemini-3.1-flash-live-preview"); + expect(constraints?.config?.thinkingConfig).toEqual({ thinkingLevel: "LOW" }); + expect(constraints?.config).not.toHaveProperty("enableAffectiveDialog"); + expect(constraints?.config?.tools?.[0]?.functionDeclarations?.[0]).toMatchObject({ + name: "openclaw_agent_consult", + }); + expect(constraints?.config?.tools?.[0]?.functionDeclarations?.[0]).not.toHaveProperty( + "behavior", + ); + expect(sessionLocal?.model).toBe("gemini-3.1-flash-live-preview"); + }); + it("rejects browser session expiry outside Date range", async () => { vi.spyOn(Date, "now").mockReturnValue(8_640_000_000_000_001); const provider = buildGoogleRealtimeVoiceProvider(); @@ -907,7 +994,7 @@ describe("buildGoogleRealtimeVoiceProvider", () => { expect(config.realtimeInputConfig?.automaticActivityDetection?.disabled).toBe(true); }); - it("sends text prompts as ordered client turns", async () => { + it("sends Gemini 3.1 text prompts as realtime input", async () => { const provider = buildGoogleRealtimeVoiceProvider(); const bridge = provider.createBridge({ providerConfig: { apiKey: "gemini-key" }, @@ -921,6 +1008,27 @@ describe("buildGoogleRealtimeVoiceProvider", () => { bridge.sendUserMessage?.(" Say hello. "); + expect(session.sendRealtimeInput).toHaveBeenCalledWith({ text: "Say hello." }); + expect(session.sendClientContent).not.toHaveBeenCalled(); + }); + + it("keeps ordered client turns for explicit Gemini 2.5 sessions", async () => { + const provider = buildGoogleRealtimeVoiceProvider(); + const bridge = provider.createBridge({ + providerConfig: { + apiKey: "gemini-key", + model: "gemini-live-2.5-flash-preview", + }, + onAudio: vi.fn(), + onClearAudio: vi.fn(), + }); + + await bridge.connect(); + lastConnectParams().callbacks.onopen(); + lastConnectParams().callbacks.onmessage({ setupComplete: { sessionId: "session-1" } }); + + bridge.sendUserMessage?.(" Say hello. "); + expect(session.sendClientContent).toHaveBeenCalledWith({ turns: [{ role: "user", parts: [{ text: "Say hello." }] }], turnComplete: true, @@ -1203,7 +1311,10 @@ describe("buildGoogleRealtimeVoiceProvider", () => { it("keeps Google Live consult calls open after continuing tool responses", async () => { const provider = buildGoogleRealtimeVoiceProvider(); const bridge = provider.createBridge({ - providerConfig: { apiKey: "gemini-key" }, + providerConfig: { + apiKey: "gemini-key", + model: "gemini-live-2.5-flash-preview", + }, onAudio: vi.fn(), onClearAudio: vi.fn(), onToolCall: vi.fn(), @@ -1249,6 +1360,46 @@ describe("buildGoogleRealtimeVoiceProvider", () => { }); }); + it("keeps Gemini 3.1 consult calls pending after rejecting continuation", async () => { + const provider = buildGoogleRealtimeVoiceProvider(); + const onError = vi.fn(); + const bridge = provider.createBridge({ + providerConfig: { apiKey: "gemini-key" }, + onAudio: vi.fn(), + onClearAudio: vi.fn(), + onToolCall: vi.fn(), + onError, + }); + + await bridge.connect(); + lastConnectParams().callbacks.onmessage({ + setupComplete: { sessionId: "session-1" }, + toolCall: { + functionCalls: [ + { id: "consult-call", name: "openclaw_agent_consult", args: { prompt: "hi" } }, + ], + }, + }); + + bridge.submitToolResult("consult-call", { status: "working" }, { willContinue: true }); + expect(session.sendToolResponse).not.toHaveBeenCalled(); + expect(requireFirstError(onError).message).toContain( + "does not support continuing tool responses", + ); + + bridge.submitToolResult("consult-call", { text: "The meeting starts at 3." }); + + expect(session.sendToolResponse).toHaveBeenCalledWith({ + functionResponses: [ + { + id: "consult-call", + name: "openclaw_agent_consult", + response: { text: "The meeting starts at 3." }, + }, + ], + }); + }); + it("does not send malformed Live API tool responses without a matching call name", async () => { const provider = buildGoogleRealtimeVoiceProvider(); const onError = vi.fn(); diff --git a/extensions/google/realtime-voice-provider.ts b/extensions/google/realtime-voice-provider.ts index b040413dfb86..a0957653d00b 100644 --- a/extensions/google/realtime-voice-provider.ts +++ b/extensions/google/realtime-voice-provider.ts @@ -50,8 +50,9 @@ import { normalizeOptionalString, } from "openclaw/plugin-sdk/string-coerce-runtime"; import { createGoogleGenAI } from "./google-genai-runtime.js"; +import { resolveGoogleGemini3ThinkingLevel } from "./thinking.js"; -const GOOGLE_REALTIME_DEFAULT_MODEL = "gemini-2.5-flash-native-audio-preview-12-2025"; +const GOOGLE_REALTIME_DEFAULT_MODEL = "gemini-3.1-flash-live-preview"; const GOOGLE_REALTIME_DEFAULT_VOICE = "Kore"; const GOOGLE_REALTIME_DEFAULT_API_VERSION = "v1beta"; const GOOGLE_REALTIME_INPUT_SAMPLE_RATE = 16_000; @@ -131,6 +132,7 @@ type GoogleLiveSession = { sendRealtimeInput: (params: { audio?: { data: string; mimeType: string }; audioStreamEnd?: boolean; + text?: string; }) => void; sendToolResponse: (params: { functionResponses: FunctionResponse[] | FunctionResponse }) => void; close: () => void; @@ -305,9 +307,30 @@ function mapTurnCoverage(value: GoogleRealtimeTurnCoverage | undefined): TurnCov } } -function buildThinkingConfig(config: GoogleRealtimeLiveConfig): ThinkingConfig | undefined { - if (config.thinkingLevel) { - return { thinkingLevel: config.thinkingLevel.toUpperCase() as ThinkingConfig["thinkingLevel"] }; +// Gemini 3.1 Live replaces client-content text and async tools with realtime text +// and sequential function responses; explicit older models keep their prior contract. +function isGemini31LiveModel(model: string): boolean { + const modelId = model.startsWith("models/") ? model.slice("models/".length) : model; + return modelId.startsWith("gemini-3.1-") && modelId.includes("-live"); +} + +function supportsAsyncFunctionCalling(model: string): boolean { + return !isGemini31LiveModel(model); +} + +function buildThinkingConfig( + config: GoogleRealtimeLiveConfig, + model: string, +): ThinkingConfig | undefined { + if (isGemini31LiveModel(model)) { + const thinkingLevel = resolveGoogleGemini3ThinkingLevel({ + modelId: model, + thinkingLevel: config.thinkingLevel, + thinkingBudget: config.thinkingBudget, + }); + return thinkingLevel + ? { thinkingLevel: thinkingLevel as ThinkingConfig["thinkingLevel"] } + : undefined; } if (typeof config.thinkingBudget === "number") { return { thinkingBudget: config.thinkingBudget }; @@ -343,7 +366,10 @@ function buildRealtimeInputConfig( return Object.keys(realtimeInputConfig).length > 0 ? realtimeInputConfig : undefined; } -function buildFunctionDeclarations(tools: RealtimeVoiceTool[] | undefined): FunctionDeclaration[] { +function buildFunctionDeclarations( + tools: RealtimeVoiceTool[] | undefined, + allowNonBlocking: boolean, +): FunctionDeclaration[] { const declarations: FunctionDeclaration[] = []; let omitted = 0; for (const tool of tools ?? []) { @@ -360,7 +386,7 @@ function buildFunctionDeclarations(tools: RealtimeVoiceTool[] | undefined): Func description: tool.description, parameters: tool.parameters as unknown as FunctionDeclaration["parameters"], }; - if (name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) { + if (allowNonBlocking && name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) { declaration.behavior = "NON_BLOCKING" as Behavior; } declarations.push(declaration); @@ -374,10 +400,16 @@ function buildFunctionDeclarations(tools: RealtimeVoiceTool[] | undefined): Func return declarations; } -function buildGoogleLiveConnectConfig(config: GoogleRealtimeLiveConfig): LiveConnectConfig { - const functionDeclarations = buildFunctionDeclarations(config.tools); +function buildGoogleLiveConnectConfig( + config: GoogleRealtimeLiveConfig, + model: string, +): LiveConnectConfig { + const functionDeclarations = buildFunctionDeclarations( + config.tools, + supportsAsyncFunctionCalling(model), + ); const realtimeInputConfig = buildRealtimeInputConfig(config); - const thinkingConfig = buildThinkingConfig(config); + const thinkingConfig = buildThinkingConfig(config, model); return { responseModalities: ["AUDIO" as Modality], ...(typeof config.temperature === "number" && config.temperature > 0 @@ -395,7 +427,7 @@ function buildGoogleLiveConnectConfig(config: GoogleRealtimeLiveConfig): LiveCon ...(realtimeInputConfig ? { realtimeInputConfig } : {}), inputAudioTranscription: {}, outputAudioTranscription: {}, - ...(typeof config.enableAffectiveDialog === "boolean" + ...(!isGemini31LiveModel(model) && typeof config.enableAffectiveDialog === "boolean" ? { enableAffectiveDialog: config.enableAffectiveDialog } : {}), ...(thinkingConfig ? { thinkingConfig } : {}), @@ -461,7 +493,7 @@ function formatGoogleLiveCloseEvent( } class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge { - readonly supportsToolResultContinuation = true; + readonly supportsToolResultContinuation: boolean; private session: GoogleLiveSession | null = null; private connected = false; @@ -473,6 +505,7 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge { private audioStreamEnded = false; private pendingFunctionNames = new Map(); private readonly audioFormat: RealtimeVoiceAudioFormat; + private readonly model: string; private resumptionHandle: string | undefined; private reconnectAttempts = 0; private reconnectTimer: ReturnType | undefined; @@ -484,6 +517,8 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge { constructor(private readonly config: GoogleRealtimeVoiceBridgeConfig) { this.audioFormat = config.audioFormat ?? REALTIME_VOICE_AUDIO_FORMAT_G711_ULAW_8KHZ; + this.model = config.model ?? GOOGLE_REALTIME_DEFAULT_MODEL; + this.supportsToolResultContinuation = supportsAsyncFunctionCalling(this.model); } async connect(): Promise { @@ -508,9 +543,9 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge { }); this.session = (await ai.live.connect({ - model: this.config.model ?? GOOGLE_REALTIME_DEFAULT_MODEL, + model: this.model, config: { - ...buildGoogleLiveConnectConfig(this.config), + ...buildGoogleLiveConnectConfig(this.config, this.model), ...(this.config.sessionResumption === false ? {} : { @@ -611,6 +646,10 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge { if (!normalized || !this.session || !this.connected || !this.sessionConfigured) { return; } + if (isGemini31LiveModel(this.model)) { + this.session.sendRealtimeInput({ text: normalized }); + return; + } this.session.sendClientContent({ turns: [{ role: "user", parts: [{ text: normalized }] }], turnComplete: true, @@ -642,6 +681,12 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge { } try { const isConsultTool = name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME; + if (options?.willContinue === true && !this.supportsToolResultContinuation) { + this.config.onError?.( + new Error(`Google Live model ${this.model} does not support continuing tool responses`), + ); + return; + } const functionResponse: FunctionResponse = { id: callId, name, @@ -650,7 +695,7 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge { ? (result as Record) : { output: result }, }; - if (isConsultTool) { + if (isConsultTool && this.supportsToolResultContinuation) { functionResponse.scheduling = "WHEN_IDLE" as FunctionResponseScheduling; if (options?.willContinue === true) { functionResponse.willContinue = true; @@ -959,14 +1004,17 @@ async function createGoogleRealtimeBrowserSession( newSessionExpireTime, liveConnectConstraints: { model, - config: buildGoogleLiveConnectConfig({ - ...config, - apiKey, + config: buildGoogleLiveConnectConfig( + { + ...config, + apiKey, + model, + voice, + instructions: req.instructions, + tools: req.tools, + }, model, - voice, - instructions: req.instructions, - tools: req.tools, - }), + ), }, }, }); diff --git a/extensions/qwen/api.ts b/extensions/qwen/api.ts index 7fdda179c76c..b148cad55740 100644 --- a/extensions/qwen/api.ts +++ b/extensions/qwen/api.ts @@ -8,6 +8,7 @@ export { isQwen36PlusSupportedBaseUrl, isQwenCodingPlanBaseUrl, isQwenStandardOnlyModelId, + QWEN_36_FLASH_MODEL_ID, QWEN_36_PLUS_MODEL_ID, QWEN_37_MAX_MODEL_ID, QWEN_37_PLUS_MODEL_ID, diff --git a/extensions/qwen/index.test.ts b/extensions/qwen/index.test.ts index f46bfe84533f..36c8e16fb496 100644 --- a/extensions/qwen/index.test.ts +++ b/extensions/qwen/index.test.ts @@ -7,6 +7,7 @@ import type { ProviderCatalogResult } from "openclaw/plugin-sdk/provider-catalog import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared"; import { describe, expect, it } from "vitest"; import { + QWEN_36_FLASH_MODEL_ID, QWEN_36_PLUS_MODEL_ID, QWEN_37_MAX_MODEL_ID, QWEN_37_PLUS_MODEL_ID, @@ -41,6 +42,7 @@ describe("qwen provider plugin", () => { baseUrl: QWEN_BASE_URL, models: [ { id: "qwen3.5-plus" }, + { id: QWEN_36_FLASH_MODEL_ID }, { id: QWEN_36_PLUS_MODEL_ID }, { id: QWEN_37_MAX_MODEL_ID }, { id: QWEN_37_PLUS_MODEL_ID }, @@ -81,6 +83,7 @@ describe("qwen provider plugin", () => { const catalogProvider = requireCatalogProvider(result); expect(catalogProvider.baseUrl).toBe("https://portal.qwen.ai/v1"); expect(catalogProvider.models?.map((model) => model.id)).toContain("qwen3.5-plus"); + expect(catalogProvider.models?.map((model) => model.id)).not.toContain(QWEN_36_FLASH_MODEL_ID); expect(catalogProvider.models?.map((model) => model.id)).not.toContain(QWEN_37_MAX_MODEL_ID); expect(catalogProvider.models?.map((model) => model.id)).not.toContain(QWEN_37_PLUS_MODEL_ID); }); diff --git a/extensions/qwen/index.ts b/extensions/qwen/index.ts index c6d7c855bc2d..c8229020dc7e 100644 --- a/extensions/qwen/index.ts +++ b/extensions/qwen/index.ts @@ -73,7 +73,7 @@ export default defineSingleProviderPluginEntry({ "Manage API keys: https://home.qwencloud.com/api-keys", "Docs: https://docs.qwencloud.com/", "Endpoint: dashscope.aliyuncs.com/compatible-mode/v1", - "Models: qwen3.7-max, qwen3.7-plus, qwen3.6-plus, qwen3.5-plus, etc.", + "Models: qwen3.7-max, qwen3.7-plus, qwen3.6-plus, qwen3.6-flash, qwen3.5-plus, etc.", ].join("\n"), noteTitle: "Qwen Cloud Standard (China)", wizard: { @@ -96,7 +96,7 @@ export default defineSingleProviderPluginEntry({ "Manage API keys: https://home.qwencloud.com/api-keys", "Docs: https://docs.qwencloud.com/", "Endpoint: dashscope-intl.aliyuncs.com/compatible-mode/v1", - "Models: qwen3.7-max, qwen3.7-plus, qwen3.6-plus, qwen3.5-plus, etc.", + "Models: qwen3.7-max, qwen3.7-plus, qwen3.6-plus, qwen3.6-flash, qwen3.5-plus, etc.", ].join("\n"), noteTitle: "Qwen Cloud Standard (Global/Intl)", wizard: { diff --git a/extensions/qwen/models.ts b/extensions/qwen/models.ts index 3d7bcd3d6b1c..dcf487f06781 100644 --- a/extensions/qwen/models.ts +++ b/extensions/qwen/models.ts @@ -18,6 +18,7 @@ export const QWEN_OAUTH_PROVIDER_ID = "qwen-oauth"; export const QWEN_OAUTH_BASE_URL = "https://portal.qwen.ai/v1"; export const QWEN_DEFAULT_MODEL_ID = "qwen3.5-plus"; +export const QWEN_36_FLASH_MODEL_ID = "qwen3.6-flash"; export const QWEN_36_PLUS_MODEL_ID = "qwen3.6-plus"; export const QWEN_37_MAX_MODEL_ID = "qwen3.7-max"; export const QWEN_37_PLUS_MODEL_ID = "qwen3.7-plus"; @@ -40,6 +41,15 @@ export const QWEN_MODEL_CATALOG: ReadonlyArray = [ contextWindow: 1_000_000, maxTokens: 65_536, }, + { + id: QWEN_36_FLASH_MODEL_ID, + name: QWEN_36_FLASH_MODEL_ID, + reasoning: true, + input: ["text", "image"], + cost: QWEN_DEFAULT_COST, + contextWindow: 1_000_000, + maxTokens: 65_536, + }, { id: QWEN_36_PLUS_MODEL_ID, name: QWEN_36_PLUS_MODEL_ID, @@ -152,9 +162,13 @@ export function isQwen36PlusSupportedBaseUrl(_baseUrl: string | undefined): bool return true; } -const QWEN_STANDARD_ONLY_MODEL_IDS = new Set([QWEN_37_MAX_MODEL_ID]); +const QWEN_STANDARD_ONLY_MODEL_IDS = new Set([ + QWEN_36_FLASH_MODEL_ID, + QWEN_37_MAX_MODEL_ID, +]); const QWEN_OAUTH_UNSUPPORTED_MODEL_IDS = new Set([ + QWEN_36_FLASH_MODEL_ID, QWEN_37_MAX_MODEL_ID, QWEN_37_PLUS_MODEL_ID, ]); diff --git a/extensions/qwen/openclaw.plugin.json b/extensions/qwen/openclaw.plugin.json index 3a19f56a57a2..a898cad4ff2e 100644 --- a/extensions/qwen/openclaw.plugin.json +++ b/extensions/qwen/openclaw.plugin.json @@ -60,6 +60,15 @@ } }, "suppressions": [ + { + "provider": "qwen", + "model": "qwen3.6-flash", + "reason": "qwen3.6-flash is not supported on the Qwen Coding Plan endpoint; use a Standard pay-as-you-go Qwen endpoint.", + "when": { + "baseUrlHosts": ["coding.dashscope.aliyuncs.com", "coding-intl.dashscope.aliyuncs.com"], + "providerConfigApiIn": ["qwen", "modelstudio"] + } + }, { "provider": "qwen", "model": "qwen3.7-max", @@ -69,6 +78,15 @@ "providerConfigApiIn": ["qwen", "modelstudio"] } }, + { + "provider": "modelstudio", + "model": "qwen3.6-flash", + "reason": "qwen3.6-flash is not supported on the Qwen Coding Plan endpoint; use a Standard pay-as-you-go Qwen endpoint.", + "when": { + "baseUrlHosts": ["coding.dashscope.aliyuncs.com", "coding-intl.dashscope.aliyuncs.com"], + "providerConfigApiIn": ["qwen", "modelstudio"] + } + }, { "provider": "modelstudio", "model": "qwen3.7-max", diff --git a/extensions/qwen/provider-catalog.test.ts b/extensions/qwen/provider-catalog.test.ts index 826e19ab8cc4..17405c8f77b7 100644 --- a/extensions/qwen/provider-catalog.test.ts +++ b/extensions/qwen/provider-catalog.test.ts @@ -4,6 +4,7 @@ import { applyQwenNativeStreamingUsageCompat, buildQwenProvider, QWEN_BASE_URL, + QWEN_36_FLASH_MODEL_ID, QWEN_37_MAX_MODEL_ID, QWEN_37_PLUS_MODEL_ID, QWEN_STANDARD_GLOBAL_BASE_URL, @@ -27,6 +28,7 @@ describe("qwen provider catalog", () => { const modelIds = getQwenModelIds(provider); expect(modelIds.length).toBeGreaterThan(0); expect(modelIds).toContain(QWEN_DEFAULT_MODEL_ID); + expect(modelIds).not.toContain(QWEN_36_FLASH_MODEL_ID); expect(modelIds).toContain("qwen3.6-plus"); expect(modelIds).not.toContain(QWEN_37_MAX_MODEL_ID); expect(modelIds).toContain(QWEN_37_PLUS_MODEL_ID); @@ -42,10 +44,18 @@ describe("qwen provider catalog", () => { expect(getQwenModelIds(coding)).toContain("qwen3.6-plus"); expect(getQwenModelIds(codingTrailingDot)).toContain("qwen3.6-plus"); expect(getQwenModelIds(standard)).toContain("qwen3.6-plus"); + expect(getQwenModelIds(coding)).not.toContain(QWEN_36_FLASH_MODEL_ID); + expect(getQwenModelIds(codingTrailingDot)).not.toContain(QWEN_36_FLASH_MODEL_ID); expect(getQwenModelIds(coding)).not.toContain(QWEN_37_MAX_MODEL_ID); expect(getQwenModelIds(coding)).toContain(QWEN_37_PLUS_MODEL_ID); expect(coding.models.find((model) => model.id === "qwen3.6-plus")?.reasoning).toBe(true); + expect(standard.models.find((model) => model.id === QWEN_36_FLASH_MODEL_ID)).toMatchObject({ + reasoning: true, + input: ["text", "image"], + contextWindow: 1_000_000, + maxTokens: 65_536, + }); expect(standard.models.find((model) => model.id === QWEN_37_MAX_MODEL_ID)).toMatchObject({ reasoning: true, input: ["text"], @@ -60,13 +70,17 @@ describe("qwen provider catalog", () => { }); }); - it("keeps Qwen 3.7 out of the portal catalog", () => { + it("keeps unsupported Qwen models out of the portal catalog", () => { const portal = buildQwenOAuthProvider(); const portalQwen36 = portal.models.find((model) => model.id === "qwen3.6-plus"); const manifestQwen36 = manifest.modelCatalog.providers["qwen-oauth"].models.find( (model) => model.id === "qwen3.6-plus", ); + expect(getQwenModelIds(portal)).not.toContain(QWEN_36_FLASH_MODEL_ID); + expect( + manifest.modelCatalog.providers["qwen-oauth"].models.map((model) => model.id), + ).not.toContain(QWEN_36_FLASH_MODEL_ID); expect(getQwenModelIds(portal)).not.toContain(QWEN_37_MAX_MODEL_ID); expect(getQwenModelIds(portal)).not.toContain(QWEN_37_PLUS_MODEL_ID); expect(portalQwen36?.reasoning).toBe(true); diff --git a/scripts/dev/realtime-talk-live-smoke.ts b/scripts/dev/realtime-talk-live-smoke.ts index 61e3a2235af3..648a278a20ae 100644 --- a/scripts/dev/realtime-talk-live-smoke.ts +++ b/scripts/dev/realtime-talk-live-smoke.ts @@ -16,8 +16,7 @@ const OPENAI_REALTIME_VOICE = process.env.OPENCLAW_REALTIME_OPENAI_VOICE?.trim() const DEFAULT_OPENAI_HTTP_TIMEOUT_MS = 30_000; const OPENAI_HTTP_RESPONSE_MAX_BYTES = 256 * 1024; const GOOGLE_REALTIME_MODEL = - process.env.OPENCLAW_REALTIME_GOOGLE_MODEL?.trim() || - "gemini-2.5-flash-native-audio-preview-12-2025"; + process.env.OPENCLAW_REALTIME_GOOGLE_MODEL?.trim() || "gemini-3.1-flash-live-preview"; const GOOGLE_REALTIME_VOICE = process.env.OPENCLAW_REALTIME_GOOGLE_VOICE?.trim() || "Kore"; const GOOGLE_LIVE_WS_URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained"; diff --git a/ui/src/pages/chat/realtime-talk-google-live.test.ts b/ui/src/pages/chat/realtime-talk-google-live.test.ts index dc345cb4288e..3f393ba3f967 100644 --- a/ui/src/pages/chat/realtime-talk-google-live.test.ts +++ b/ui/src/pages/chat/realtime-talk-google-live.test.ts @@ -442,6 +442,59 @@ describe("GoogleLiveRealtimeTalkTransport", () => { expect(onStatus).not.toHaveBeenCalledWith("listening"); }); + it("submits completed consults without asynchronous scheduling", async () => { + const listeners = new Set<(event: { event: string; payload?: unknown }) => void>(); + const client = { + addEventListener: vi.fn((listener: (event: { event: string; payload?: unknown }) => void) => { + listeners.add(listener); + return () => listeners.delete(listener); + }), + request: vi.fn(async (method: string) => { + expect(method).toBe("talk.client.toolCall"); + return { runId: "run-1" }; + }), + } as unknown as RealtimeTalkTransportContext["client"]; + const transport = createTransport({}, client); + await transport.start(); + const ws = latestWebSocket(); + + ws.emitMessage( + encodeJsonFrame({ + toolCall: { + functionCalls: [ + { + id: "call-1", + name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + args: { question: "check the session" }, + }, + ], + }, + }), + ); + await vi.waitFor(() => expect(listeners.size).toBe(1)); + for (const listener of listeners) { + listener({ + event: "chat", + payload: { runId: "run-1", state: "final", message: { text: "done" } }, + }); + } + + await vi.waitFor(() => + expect(ws.sent.map((payload) => JSON.parse(payload))).toContainEqual({ + toolResponse: { + functionResponses: [ + { + id: "call-1", + name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + response: { result: "done" }, + }, + ], + }, + }), + ); + transport.stop(); + }); + it("sends spoken active-control acknowledgements through Google Live", async () => { const client = createClient(); vi.mocked(client["request"]).mockImplementation(async (method) => { @@ -507,18 +560,8 @@ describe("GoogleLiveRealtimeTalkTransport", () => { expect(createdSources[0]?.stop).toHaveBeenCalledTimes(1); const sent = ws.sent.map((payload) => JSON.parse(payload)); expect(sent).toContainEqual({ - clientContent: { - turns: [ - { - role: "user", - parts: [ - { - text: expect.stringContaining('Status: "OpenClaw is working in read (running)."'), - }, - ], - }, - ], - turnComplete: true, + realtimeInput: { + text: expect.stringContaining('Status: "OpenClaw is working in read (running)."'), }, }); transport.stop(); @@ -590,18 +633,8 @@ describe("GoogleLiveRealtimeTalkTransport", () => { expect(createdSources[0]?.stop).toHaveBeenCalledTimes(1); const sent = ws.sent.map((payload) => JSON.parse(payload)); expect(sent).toContainEqual({ - clientContent: { - turns: [ - { - role: "user", - parts: [ - { - text: expect.stringContaining('Status: "Got it. I steered the active run."'), - }, - ], - }, - ], - turnComplete: true, + realtimeInput: { + text: expect.stringContaining('Status: "Got it. I steered the active run."'), }, }); transport.stop(); diff --git a/ui/src/pages/chat/realtime-talk-google-live.ts b/ui/src/pages/chat/realtime-talk-google-live.ts index b1fe72737334..a66296d76fdf 100644 --- a/ui/src/pages/chat/realtime-talk-google-live.ts +++ b/ui/src/pages/chat/realtime-talk-google-live.ts @@ -53,6 +53,16 @@ const GOOGLE_LIVE_WEBSOCKET_HOST = "generativelanguage.googleapis.com"; const GOOGLE_LIVE_WEBSOCKET_PATH = /^\/ws\/google\.ai\.generativelanguage\.v[0-9a-z]+\.GenerativeService\.BidiGenerateContent(?:Constrained)?$/; +// Browser sessions can still pin a 2.5 model, whose text and tool-response wire +// contract differs from the 3.1 default carried in new session metadata. +function isGemini31LiveModel(model: string | undefined): boolean { + if (!model) { + return true; + } + const modelId = model.startsWith("models/") ? model.slice("models/".length) : model; + return modelId.startsWith("gemini-3.1-") && modelId.includes("-live"); +} + export function buildGoogleLiveUrl(session: RealtimeTalkJsonPcmWebSocketSessionResult): string { let url: URL; try { @@ -392,7 +402,7 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { { id: callId, name: pending.name, - scheduling: "WHEN_IDLE", + ...(!isGemini31LiveModel(this.session.model) ? { scheduling: "WHEN_IDLE" } : {}), response: result && typeof result === "object" && !Array.isArray(result) ? result @@ -405,15 +415,18 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { private sendControlSpeechMessage(message: string): void { this.stopOutput(); + if (!isGemini31LiveModel(this.session.model)) { + this.send({ + clientContent: { + turns: [{ role: "user", parts: [{ text: message }] }], + turnComplete: true, + }, + }); + return; + } this.send({ - clientContent: { - turns: [ - { - role: "user", - parts: [{ text: message }], - }, - ], - turnComplete: true, + realtimeInput: { + text: message, }, }); }