diff --git a/extensions/google/base64.ts b/extensions/google/base64.ts new file mode 100644 index 000000000000..b83dbfce7757 --- /dev/null +++ b/extensions/google/base64.ts @@ -0,0 +1,27 @@ +// Google provider inbound-media boundary implements ProtoJSON base64 decoding. +// Every Google response path that receives inline media (Live, TTS, video, +// music) normalizes the URL-safe alphabet here before the shared strict +// validator, so ProtoJSON bytes fields are accepted without weakening the +// malformed-base64 guard for any surface. +import { canonicalizeBase64 } from "openclaw/plugin-sdk/media-runtime"; + +/** + * Convert a ProtoJSON URL-safe Base64 payload to the standard alphabet without + * validating the payload. Returns undefined when the input mixes alphabets, so + * callers can reject it before the shared strict validator runs once. + */ +export function toStandardGoogleProviderBase64(value: string): string | undefined { + const usesStandardAlphabet = value.includes("+") || value.includes("/"); + const usesUrlSafeAlphabet = value.includes("-") || value.includes("_"); + if (usesStandardAlphabet && usesUrlSafeAlphabet) { + return undefined; + } + return usesUrlSafeAlphabet + ? value.replace(/[-_]/g, (symbol) => (symbol === "-" ? "+" : "/")) + : value; +} + +export function canonicalizeGoogleProviderBase64(value: string): string | undefined { + const standard = toStandardGoogleProviderBase64(value); + return standard === undefined ? undefined : canonicalizeBase64(standard); +} diff --git a/extensions/google/image-generation-provider.test.ts b/extensions/google/image-generation-provider.test.ts index e4825051c916..8eb7446ae9ee 100644 --- a/extensions/google/image-generation-provider.test.ts +++ b/extensions/google/image-generation-provider.test.ts @@ -290,6 +290,72 @@ describe("Google image-generation provider", () => { ).rejects.toThrow("Google image generation response malformed"); }); + it("accepts URL-safe base64 image bytes", async () => { + mockGoogleApiKeyAuth(); + const imageBytes = Buffer.from([0xfb, 0xff, 0x50, 0x4e, 0x47]); + const imageBase64url = imageBytes.toString("base64url"); + expect(imageBase64url).toMatch(/[-_]/); + expect(imageBase64url).not.toMatch(/[+/]/); + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + jsonResponse({ + candidates: [ + { + content: { + parts: [ + { + inlineData: { + mimeType: "image/png", + data: imageBase64url, + }, + }, + ], + }, + }, + ], + }), + ), + ); + + const result = await buildGoogleImageGenerationProvider().generateImage({ + provider: "google", + model: "gemini-3.1-flash-image", + prompt: "draw a cat", + cfg: {}, + }); + + expect(result.images[0]?.buffer).toEqual(imageBytes); + }); + + it("rejects mixed-alphabet inline image data", async () => { + mockGoogleApiKeyAuth(); + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + jsonResponse({ + candidates: [ + { + content: { + parts: [{ inlineData: { mimeType: "image/png", data: "aGVsbG8+_" } }], + }, + }, + ], + }), + ), + ); + + const provider = buildGoogleImageGenerationProvider(); + await expect( + provider.generateImage({ + provider: "google", + model: "gemini-3.1-flash-image", + prompt: "draw a cat", + cfg: {}, + }), + ).rejects.toThrow("Google image generation response malformed"); + }); + it("accepts OAuth JSON auth and inline_data responses", async () => { vi.spyOn(providerAuthRuntime, "resolveApiKeyForProvider").mockResolvedValue({ apiKey: JSON.stringify({ token: "oauth-token" }), diff --git a/extensions/google/image-generation-provider.ts b/extensions/google/image-generation-provider.ts index 962a8263f32c..5959de04cc70 100644 --- a/extensions/google/image-generation-provider.ts +++ b/extensions/google/image-generation-provider.ts @@ -21,6 +21,7 @@ import { normalizeOptionalString, } from "openclaw/plugin-sdk/string-coerce-runtime"; import { normalizeGoogleModelId, resolveGoogleGenerativeAiHttpRequestConfig } from "./api.js"; +import { toStandardGoogleProviderBase64 } from "./base64.js"; const DEFAULT_GOOGLE_IMAGE_MODEL = "gemini-3.1-flash-image"; const DEFAULT_IMAGE_TIMEOUT_MS = 180_000; @@ -248,8 +249,12 @@ export function buildGoogleImageGenerationProvider(): ImageGenerationProvider { if (!data) { throw new Error(GOOGLE_IMAGE_MALFORMED_RESPONSE); } + const standardData = toStandardGoogleProviderBase64(data); + if (!standardData) { + throw new Error(GOOGLE_IMAGE_MALFORMED_RESPONSE); + } const image = generatedImageAssetFromBase64({ - base64: data, + base64: standardData, index: imageIndex, mimeType: normalizeOptionalString(inline.mimeType) ?? diff --git a/extensions/google/music-generation-provider.test.ts b/extensions/google/music-generation-provider.test.ts index b94b9eb23824..976ac5a2d16a 100644 --- a/extensions/google/music-generation-provider.test.ts +++ b/extensions/google/music-generation-provider.test.ts @@ -170,6 +170,7 @@ describe("google music generation provider", () => { it.each([ ["invalid alphabet", "not-base64!"], ["non-canonical pad bits", "ZE=="], + ["mixed alphabet", "aGVsbG8+_"], ])("rejects %s in inline audio", async (_scenario, data) => { mockGoogleAuth(); generateContentMock.mockResolvedValue({ @@ -193,6 +194,34 @@ describe("google music generation provider", () => { expect(generateContentMock).toHaveBeenCalledTimes(1); }); + it("accepts inline audio encoded with URL-safe base64", async () => { + mockGoogleAuth(); + const audio = Buffer.from([0xfb, 0xff, 0x49, 0x44, 0x33, 0x04, 0x00, 0x00]); + const audioBase64url = audio.toString("base64url"); + expect(audioBase64url).toMatch(/[-_]/); + expect(audioBase64url).not.toMatch(/[+/]/); + generateContentMock.mockResolvedValue({ + candidates: [ + { + content: { + parts: [{ inlineData: { data: audioBase64url, mimeType: "audio/mpeg" } }], + }, + finishReason: "STOP", + }, + ], + }); + + const result = await buildGoogleMusicGenerationProvider().generateMusic({ + provider: "google", + model: "lyria-3-clip-preview", + prompt: "upbeat synthpop anthem", + cfg: {}, + }); + + expect(result.tracks).toHaveLength(1); + expect(result.tracks[0]?.buffer).toEqual(audio); + }); + it("retries once when Lyria returns an unblocked text-only response", async () => { mockGoogleAuth(); generateContentMock diff --git a/extensions/google/music-generation-provider.ts b/extensions/google/music-generation-provider.ts index 533d0bb1f5a0..b1f8e21c690b 100644 --- a/extensions/google/music-generation-provider.ts +++ b/extensions/google/music-generation-provider.ts @@ -13,6 +13,7 @@ import { } from "openclaw/plugin-sdk/provider-http"; import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; import { resolveGoogleGenerativeAiApiOrigin } from "./api.js"; +import { toStandardGoogleProviderBase64 } from "./base64.js"; import { createGoogleMusicGenerationProviderMetadata, DEFAULT_GOOGLE_MUSIC_MODEL, @@ -95,9 +96,13 @@ function extractTracks(params: { payload: GoogleGenerateMusicResponse; model: st normalizeOptionalString(inline?.mimeType) || normalizeOptionalString(inline?.mime_type) || "audio/mpeg"; + const standardAudio = toStandardGoogleProviderBase64(data); + if (!standardAudio) { + throw new Error("Generated music asset contains malformed base64 audio data"); + } tracks.push( generatedMusicAssetFromBase64({ - base64: data, + base64: standardAudio, mimeType, fileName: resolveTrackFileName({ index: tracks.length, diff --git a/extensions/google/realtime-voice-provider.test.ts b/extensions/google/realtime-voice-provider.test.ts index 0f0b6c2596eb..4e0dc61b1b3c 100644 --- a/extensions/google/realtime-voice-provider.test.ts +++ b/extensions/google/realtime-voice-provider.test.ts @@ -1728,6 +1728,7 @@ describe("buildGoogleRealtimeVoiceProvider", () => { onClearAudio: vi.fn(), }); const pcm24k = Buffer.alloc(480); + pcm24k.set([0xfb, 0xff]); await bridge.connect(); lastConnectParams().callbacks.onmessage({ @@ -1738,7 +1739,7 @@ describe("buildGoogleRealtimeVoiceProvider", () => { { inlineData: { mimeType: "audio/L16;codec=pcm;rate=24000", - data: pcm24k.toString("base64"), + data: pcm24k.toString("base64url"), }, }, ], @@ -1787,6 +1788,7 @@ describe("buildGoogleRealtimeVoiceProvider", () => { it.each([ ["invalid alphabet", "not-base64!"], ["non-canonical pad bits", "ZE=="], + ["mixed alphabet", "aGVsbG8+_"], ])("terminates the session for %s in output audio", async (_scenario, data) => { const provider = buildGoogleRealtimeVoiceProvider(); const onAudio = vi.fn(); diff --git a/extensions/google/realtime-voice-provider.ts b/extensions/google/realtime-voice-provider.ts index fb6db9f67a2f..80fa6a695832 100644 --- a/extensions/google/realtime-voice-provider.ts +++ b/extensions/google/realtime-voice-provider.ts @@ -18,7 +18,6 @@ import { type ThinkingConfig, TurnCoverage, } from "@google/genai"; -import { canonicalizeBase64 } from "openclaw/plugin-sdk/media-runtime"; import { resolveExpiresAtMsFromDurationMs, timestampMsToIsoString, @@ -52,6 +51,7 @@ import { asFiniteNumber, normalizeOptionalString, } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { canonicalizeGoogleProviderBase64 } from "./base64.js"; import { createGoogleGenAI } from "./google-genai-runtime.js"; import { resolveGoogleGemini3ThinkingLevel } from "./thinking.js"; @@ -950,7 +950,7 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge { for (const part of content.modelTurn?.parts ?? []) { if (part.inlineData?.data) { - const canonicalAudio = canonicalizeBase64(part.inlineData.data); + const canonicalAudio = canonicalizeGoogleProviderBase64(part.inlineData.data); if (!canonicalAudio) { this.failConnection(new Error("Google Live stream returned malformed base64 audio data")); return; diff --git a/extensions/google/speech-provider.test.ts b/extensions/google/speech-provider.test.ts index cebe4ece180a..3147956daf11 100644 --- a/extensions/google/speech-provider.test.ts +++ b/extensions/google/speech-provider.test.ts @@ -374,6 +374,56 @@ describe("Google speech provider", () => { expect(requestSequence).toHaveBeenCalledTimes(2); }); + it("accepts Gemini audio with URL-safe base64", async () => { + const pcm = Buffer.from([0xfb, 0xff, 8, 0, 9, 0, 10, 0]); + const pcmBase64url = pcm.toString("base64url"); + expect(pcmBase64url).toMatch(/[-_]/); + expect(pcmBase64url).not.toMatch(/[+/]/); + const response = { + response: googleTtsResponse(pcmBase64url), + release: vi.fn(async () => {}), + }; + const requestSequence = vi.fn().mockResolvedValue(response); + postJsonRequestMock.mockImplementation(requestSequence); + const provider = buildGoogleSpeechProvider(); + + const result = await provider.synthesize({ + text: "Accept URL-safe audio.", + cfg: {}, + providerConfig: { + apiKey: "google-test-key", + }, + target: "audio-file", + timeoutMs: 5_000, + }); + + expect(result.audioBuffer.subarray(44)).toEqual(pcm); + expect(requestSequence).toHaveBeenCalledTimes(1); + }); + + it("rejects Gemini audio with a mixed base64 alphabet", async () => { + const malformedResponse = { + response: googleTtsResponse("aGVsbG8+_"), + release: vi.fn(async () => {}), + }; + const requestSequence = vi.fn().mockResolvedValue(malformedResponse); + postJsonRequestMock.mockImplementation(requestSequence); + const provider = buildGoogleSpeechProvider(); + + await expect( + provider.synthesize({ + text: "Reject mixed audio.", + cfg: {}, + providerConfig: { + apiKey: "google-test-key", + }, + target: "audio-file", + timeoutMs: 5_000, + }), + ).rejects.toThrow("Google TTS response returned malformed base64 audio data"); + expect(requestSequence).toHaveBeenCalledTimes(2); + }); + it("retries once when Gemini TTS fetch aborts", async () => { const pcm = Buffer.from([7, 0, 8, 0]); const abortError = new Error("This operation was aborted"); diff --git a/extensions/google/speech-provider.ts b/extensions/google/speech-provider.ts index dcf42c08a958..d53e04d50e7c 100644 --- a/extensions/google/speech-provider.ts +++ b/extensions/google/speech-provider.ts @@ -1,5 +1,5 @@ // Google provider module implements model/runtime integration. -import { canonicalizeBase64, transcodeAudioBufferToOpus } from "openclaw/plugin-sdk/media-runtime"; +import { transcodeAudioBufferToOpus } from "openclaw/plugin-sdk/media-runtime"; import { assertOkOrThrowProviderError, postJsonRequest, @@ -18,6 +18,7 @@ import type { import { asObject, trimToUndefined } from "openclaw/plugin-sdk/speech-core"; import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; import { resolveGoogleGenerativeAiHttpRequestConfig } from "./api.js"; +import { canonicalizeGoogleProviderBase64 } from "./base64.js"; const DEFAULT_GOOGLE_TTS_MODEL = "gemini-3.1-flash-tts-preview"; const DEFAULT_GOOGLE_TTS_VOICE = "Kore"; @@ -297,7 +298,7 @@ function extractGoogleSpeechPcm(payload: GoogleGenerateSpeechResponse): Buffer { if (!data) { continue; } - const canonicalAudio = canonicalizeBase64(data); + const canonicalAudio = canonicalizeGoogleProviderBase64(data); if (!canonicalAudio) { throw new Error("Google TTS response returned malformed base64 audio data"); } diff --git a/extensions/google/video-generation-provider.test.ts b/extensions/google/video-generation-provider.test.ts index 49f9ae4d4aed..dc3757333131 100644 --- a/extensions/google/video-generation-provider.test.ts +++ b/extensions/google/video-generation-provider.test.ts @@ -197,9 +197,47 @@ describe("google video generation provider", () => { expect(httpOptions).not.toHaveProperty("apiVersion"); }); + it("returns inline video bytes encoded with URL-safe base64", async () => { + vi.spyOn(providerAuthRuntime, "resolveApiKeyForProvider").mockResolvedValue({ + apiKey: "google-key", + source: "env", + mode: "api-key", + }); + const videoBytes = Buffer.from([0xfb, 0xff, 0x6d, 0x70, 0x34]); + const videoBase64url = videoBytes.toString("base64url"); + expect(videoBase64url).toMatch(/[-_]/); + expect(videoBase64url).not.toMatch(/[+/]/); + generateVideosMock.mockResolvedValue({ + done: true, + name: "operations/123", + response: { + generatedVideos: [ + { + video: { + videoBytes: videoBase64url, + mimeType: "video/mp4", + }, + }, + ], + }, + }); + + const result = await buildGoogleVideoGenerationProvider().generateVideo({ + provider: "google", + model: "veo-3.1-fast-generate-preview", + prompt: "A tiny robot watering a windowsill garden", + cfg: {}, + durationSeconds: 3, + }); + + expect(result.videos).toHaveLength(1); + expect(result.videos[0]?.buffer).toEqual(videoBytes); + }); + it.each([ ["invalid alphabet", "not-base64!"], ["non-canonical pad bits", "ZE=="], + ["mixed alphabet", "aGVsbG8+_"], ])("rejects %s in inline video bytes", async (_scenario, videoBytes) => { vi.spyOn(providerAuthRuntime, "resolveApiKeyForProvider").mockResolvedValue({ apiKey: "google-key", diff --git a/extensions/google/video-generation-provider.ts b/extensions/google/video-generation-provider.ts index edbb6ef613f7..e02e3274462f 100644 --- a/extensions/google/video-generation-provider.ts +++ b/extensions/google/video-generation-provider.ts @@ -1,6 +1,5 @@ // Google provider module implements model/runtime integration. import { resolveGeneratedMediaMaxBytes } from "openclaw/plugin-sdk/media-generation-runtime"; -import { canonicalizeBase64 } from "openclaw/plugin-sdk/media-runtime"; import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import { createProviderOperationDeadline, @@ -17,6 +16,7 @@ import type { VideoGenerationRequest, } from "openclaw/plugin-sdk/video-generation"; import { parseGeminiAuth, resolveGoogleGenerativeAiApiOrigin } from "./api.js"; +import { canonicalizeGoogleProviderBase64 } from "./base64.js"; import { createGoogleVideoGenerationProviderMetadata, DEFAULT_GOOGLE_VIDEO_MODEL, @@ -568,7 +568,7 @@ export function buildGoogleVideoGenerationProvider(): VideoGenerationProvider { | { videoBytes?: string; uri?: string; mimeType?: string } | undefined; if (inline?.videoBytes) { - const canonicalVideo = canonicalizeBase64(inline.videoBytes); + const canonicalVideo = canonicalizeGoogleProviderBase64(inline.videoBytes); if (!canonicalVideo) { throw new Error("Google video generation returned malformed base64 video data"); }