From e2e5d1bca685bffbda31b742e51c9e23377d950b Mon Sep 17 00:00:00 2001 From: Alix-007 Date: Wed, 26 Aug 2026 19:20:29 +0800 Subject: [PATCH] fix(google): retain provider-returned response model (#129606) * fix(google): preserve provider-returned response models Co-authored-by: Peter Steinberger * test(google): use typed Vertex stream options Co-authored-by: Peter Steinberger * fix(google): normalize Vertex response model resources Co-authored-by: Peter Steinberger --------- Co-authored-by: Peter Steinberger --- extensions/google/transport-stream.test.ts | 260 ++++++++++++++++++ extensions/google/transport-stream.ts | 12 + .../ai/src/providers/google-shared.test.ts | 134 +++++++++ packages/ai/src/providers/google-shared.ts | 12 + 4 files changed, 418 insertions(+) diff --git a/extensions/google/transport-stream.test.ts b/extensions/google/transport-stream.test.ts index 06a9d4079b63..e4b6bc247e4a 100644 --- a/extensions/google/transport-stream.test.ts +++ b/extensions/google/transport-stream.test.ts @@ -801,6 +801,266 @@ describe("google transport stream", () => { expect(result.content[2]).toHaveProperty("thoughtSignature", "Y2FsbF9zaWdfMQ=="); }); + it.each([ + { + provider: "google", + requested: "gemini-2.5-pro", + returned: ["gemini-2.5-pro-002"], + expected: "gemini-2.5-pro-002", + }, + { + provider: "google-vertex", + requested: "gemini-2.5-pro", + returned: ["gemini-2.5-pro-002"], + expected: "gemini-2.5-pro-002", + }, + { + provider: "google", + requested: "gemini-2.5-pro", + returned: ["gemini-2.5-pro"], + }, + { + provider: "google", + requested: "google/gemini-2.5-pro", + returned: ["gemini-2.5-pro"], + }, + { + provider: "google", + requested: "models/gemini-2.5-pro", + returned: ["gemini-2.5-pro"], + }, + { + provider: "google", + requested: "gemini-2.5-pro", + returned: ["models/gemini-2.5-pro"], + }, + { + provider: "google", + requested: "gemini-2.5-pro", + returned: ["google/gemini-2.5-pro"], + }, + { + provider: "google", + requested: "tunedModels/fixture-gemini", + returned: ["tunedModels/fixture-gemini"], + }, + { + provider: "google-vertex", + requested: "google/gemini-2.5-pro", + returned: ["gemini-2.5-pro"], + }, + { + provider: "google-vertex", + requested: "gemini-2.5-pro", + returned: ["publishers/google/models/gemini-2.5-pro"], + }, + { + provider: "google-vertex", + requested: "gemini-2.5-pro", + returned: [ + "projects/fixture-project/locations/global/publishers/google/models/gemini-2.5-pro", + ], + }, + { + provider: "google-vertex", + requested: "publishers/google/models/gemini-2.5-pro", + returned: ["gemini-2.5-pro"], + }, + { + provider: "google-vertex", + requested: + "projects/fixture-project/locations/global/publishers/google/models/gemini-2.5-pro", + returned: ["gemini-2.5-pro"], + }, + { + provider: "google-vertex", + requested: "gemini-2.5-pro", + returned: ["publishers/meta/models/gemini-2.5-pro"], + expected: "publishers/meta/models/gemini-2.5-pro", + }, + { + provider: "google-vertex", + requested: "gemini-2.5-pro", + returned: ["tunedModels/gemini-2.5-pro"], + expected: "tunedModels/gemini-2.5-pro", + }, + { + provider: "google", + requested: "gemini-2.5-pro", + returned: [undefined, "", " "], + }, + { + provider: "google", + requested: "gemini-2.5-pro", + returned: ["", "gemini-2.5-pro-002", "gemini-2.5-pro-003"], + expected: "gemini-2.5-pro-002", + }, + ])( + "retains the concrete provider-returned model only when it actually differs ($provider, $requested)", + async ({ provider, requested, returned, expected }) => { + vi.stubEnv("GOOGLE_CLOUD_PROJECT", "fixture-google-project"); + vi.stubEnv("GOOGLE_CLOUD_LOCATION", "global"); + guardedFetchMock.mockResolvedValueOnce( + buildSseResponse( + returned.map((modelVersion, index) => ({ + ...(modelVersion === undefined ? {} : { modelVersion }), + ...(index === returned.length - 1 + ? { + responseId: "actual-google-response", + candidates: [ + { content: { parts: [{ text: "actual response" }] }, finishReason: "STOP" }, + ], + } + : {}), + })), + ), + ); + + const model = + provider === "google" + ? buildGeminiModel({ id: requested }) + : buildGoogleVertexModel({ id: requested }); + const { buildGoogleProvider } = await import("./provider-registration.js"); + const streamFn = buildGoogleProvider().createStreamFn?.({ + provider, + modelId: requested, + model, + }); + if (!streamFn) { + throw new Error(`Missing registered ${provider} stream`); + } + const stream = await Promise.resolve( + streamFn( + model, + { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, + { + apiKey: "fixture-google-api-key", + }, + ), + ); + const result = await stream.result(); + + expect(result.responseId).toBe("actual-google-response"); + expect(result.stopReason).toBe("stop"); + if (expected) { + expect(result.responseModel).toBe(expected); + } else { + expect(result).not.toHaveProperty("responseModel"); + } + }, + ); + + it.each([ + { + provider: "google", + requested: "google/gemini-2.5-pro", + returned: "gemini-2.5-pro-002", + expected: "gemini-2.5-pro-002", + }, + { + provider: "google-vertex", + requested: "google/gemini-2.5-pro", + returned: "gemini-2.5-pro-002", + expected: "gemini-2.5-pro-002", + }, + { + provider: "google", + requested: "models/gemini-2.5-pro", + returned: "gemini-2.5-pro", + }, + { + provider: "google-vertex", + requested: "gemini-2.5-pro", + returned: "publishers/google/models/gemini-2.5-pro", + }, + { + provider: "google-vertex", + requested: "gemini-2.5-pro", + returned: "projects/fixture-project/locations/global/publishers/google/models/gemini-2.5-pro", + }, + ])( + "keeps the registered $provider model identity across actual localhost HTTP SSE", + async ({ provider, requested, returned, expected }) => { + vi.stubEnv("GOOGLE_CLOUD_PROJECT", "fixture-google-project"); + vi.stubEnv("GOOGLE_CLOUD_LOCATION", "global"); + const observedRequests: Array<{ method?: string; url?: string }> = []; + const server = createServer((request, response) => { + observedRequests.push({ method: request.method, url: request.url }); + response.writeHead(200, { "content-type": "text/event-stream" }); + response.end( + `data: ${JSON.stringify({ + responseId: "loopback-google-response", + modelVersion: returned, + candidates: [ + { content: { parts: [{ text: "actual response" }] }, finishReason: "STOP" }, + ], + })}\n\ndata: [DONE]\n\n`, + ); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + resolve(); + }); + }); + + try { + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Missing Google loopback server address"); + } + buildGuardedModelFetchMock.mockReturnValue(fetch); + const model = + provider === "google" + ? buildGeminiModel({ + id: requested, + baseUrl: `http://127.0.0.1:${address.port}/v1beta`, + }) + : buildGoogleVertexModel({ + id: requested, + baseUrl: `http://127.0.0.1:${address.port}`, + }); + const { buildGoogleProvider } = await import("./provider-registration.js"); + const streamFn = buildGoogleProvider().createStreamFn?.({ + provider, + modelId: requested, + model, + }); + if (!streamFn) { + throw new Error(`Missing registered ${provider} stream`); + } + const stream = await Promise.resolve( + streamFn( + model, + { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, + { + apiKey: "fixture-google-api-key", + }, + ), + ); + const result = await stream.result(); + + expect(observedRequests).toEqual([ + { method: "POST", url: expect.stringContaining(":streamGenerateContent?alt=sse") }, + ]); + expect(result).toMatchObject({ + responseId: "loopback-google-response", + stopReason: "stop", + }); + if (expected) { + expect(result.responseModel).toBe(expected); + } else { + expect(result).not.toHaveProperty("responseModel"); + } + } finally { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } + }, + ); + it.each([ { name: "includes billed tool-result prompt tokens in input accounting", diff --git a/extensions/google/transport-stream.ts b/extensions/google/transport-stream.ts index 540451f3ea0e..531b5847812c 100644 --- a/extensions/google/transport-stream.ts +++ b/extensions/google/transport-stream.ts @@ -121,6 +121,9 @@ type GoogleVideoSlots = Map, VideoContent>; const GOOGLE_GEMINI3_FIRST_RESPONSE_RETRY_DEFAULT_MS = 45_000; const GOOGLE_GEMINI3_FIRST_RESPONSE_RETRY_ENV = "OPENCLAW_GOOGLE_GEMINI_FIRST_RESPONSE_RETRY_MS"; const GOOGLE_SSE_EVENT_BOUNDARY_RE = /(?:\r\n|\r(?!\n)|\n){2}/u; +// Compare Google-owned publisher resources without changing outbound request paths. +const GOOGLE_VERTEX_MODEL_RESOURCE_PREFIX = + /^(?:projects\/[^/]+\/locations\/[^/]+\/)?publishers\/google\/models\//u; type GoogleTransportContentBlock = | { type: "text"; text: string; textSignature?: string } @@ -142,6 +145,7 @@ const GOOGLE_VERTEX_DEFAULT_API_VERSION = "v1"; type GoogleSseChunk = { responseId?: string; + modelVersion?: string; promptFeedback?: { blockReason?: string; blockReasonMessage?: string; @@ -1549,6 +1553,14 @@ function createGoogleTransportStreamFn(kind: CanonicalGoogleTransportApi): Strea })(sse.firstChunk); for await (const chunk of chunks) { output.responseId ||= chunk.responseId; + const responseModel = normalizeOptionalString(chunk.modelVersion); + if ( + responseModel && + resolveGoogleModelPath(model.id.replace(GOOGLE_VERTEX_MODEL_RESOURCE_PREFIX, "")) !== + resolveGoogleModelPath(responseModel.replace(GOOGLE_VERTEX_MODEL_RESOURCE_PREFIX, "")) + ) { + output.responseModel ||= responseModel; + } updateUsage(output, model, chunk, knownUsage); const candidate = chunk.candidates?.[0]; const promptFeedback = chunk.promptFeedback; diff --git a/packages/ai/src/providers/google-shared.test.ts b/packages/ai/src/providers/google-shared.test.ts index 6abdd371a987..a2e0fd344390 100644 --- a/packages/ai/src/providers/google-shared.test.ts +++ b/packages/ai/src/providers/google-shared.test.ts @@ -1,4 +1,5 @@ // Google shared provider tests cover response conversion and finish reasons. +import { createServer } from "node:http"; import { ApiError, BlockedReason, @@ -146,6 +147,7 @@ type GoogleResponseFixture = { finishReason?: FinishReason; finishMessage?: string; responseId?: string; + modelVersion?: string; usageMetadata?: GenerateContentResponse["usageMetadata"]; promptFeedback?: GenerateContentResponse["promptFeedback"]; }; @@ -321,6 +323,85 @@ describe("consumeGoogleGenerateContentStream", () => { expect(output.usage.cost.total).toBeGreaterThan(0); }); + it.each([ + { + api: "google-generative-ai", + requested: "gemini-test", + returned: ["gemini-test-002"], + expected: "gemini-test-002", + }, + { + api: "google-vertex", + requested: "gemini-test", + returned: ["gemini-test-002"], + expected: "gemini-test-002", + }, + { api: "google-generative-ai", requested: "gemini-test", returned: ["gemini-test"] }, + { api: "google-generative-ai", requested: "google/gemini-test", returned: ["gemini-test"] }, + { api: "google-generative-ai", requested: "models/gemini-test", returned: ["gemini-test"] }, + { api: "google-generative-ai", requested: "gemini-test", returned: ["models/gemini-test"] }, + { + api: "google-vertex", + requested: "publishers/google/models/gemini-test", + returned: ["gemini-test"], + }, + { + api: "google-vertex", + requested: "projects/fixture-project/locations/global/publishers/google/models/gemini-test", + returned: ["gemini-test"], + }, + { + api: "google-vertex", + requested: "gemini-test", + returned: ["publishers/google/models/gemini-test"], + }, + { + api: "google-vertex", + requested: "publishers/meta/models/gemini-test", + returned: ["gemini-test"], + expected: "gemini-test", + }, + { + api: "google-generative-ai", + requested: "tunedModels/fixture-gemini", + returned: ["tunedModels/fixture-gemini"], + }, + { + api: "google-generative-ai", + requested: "gemini-test", + returned: ["", "gemini-test-002", "gemini-test-003"], + expected: "gemini-test-002", + }, + ] as const)( + "retains an actually different $api SDK response model for $requested", + async ({ api, requested, returned, expected }) => { + const targetModel = { + ...model, + id: requested, + api, + provider: api === "google-vertex" ? "google-vertex" : "google", + } satisfies Model<"google-generative-ai" | "google-vertex">; + const { result } = await runGoogleFixture( + returned.map((modelVersion, index) => + googleResponse({ + modelVersion, + ...(index === returned.length - 1 + ? { parts: [{ text: "actual response" }], finishReason: FinishReason.STOP } + : {}), + }), + ), + { targetModel }, + ); + + expect(result.stopReason).toBe("stop"); + if (expected) { + expect(result.responseModel).toBe(expected); + } else { + expect(result).not.toHaveProperty("responseModel"); + } + }, + ); + it.each([ { name: "includes billed tool-result prompt tokens in input accounting", @@ -591,6 +672,59 @@ describe("consumeGoogleGenerateContentStream", () => { }); describe("runGoogleGenerateContentLifecycle", () => { + it("retains the actual SDK response model across localhost HTTP SSE", async () => { + const observedRequests: Array<{ method?: string; url?: string }> = []; + const server = createServer((request, response) => { + observedRequests.push({ method: request.method, url: request.url }); + response.writeHead(200, { "content-type": "text/event-stream" }); + response.end( + `data: ${JSON.stringify({ + responseId: "sdk-google-response", + modelVersion: "gemini-test-002", + candidates: [{ content: { parts: [{ text: "actual response" }] }, finishReason: "STOP" }], + })}\n\n`, + ); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + resolve(); + }); + }); + + try { + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Missing Google SDK loopback server address"); + } + const { result } = await runGoogleFixture([], { + createClient: () => + new GoogleGenAI({ + apiKey: "fixture-google-api-key", + httpOptions: { baseUrl: `http://127.0.0.1:${address.port}` }, + }), + buildParams: () => ({ + model: model.id, + contents: [{ role: "user", parts: [{ text: "hello" }] }], + }), + }); + + expect(observedRequests).toEqual([ + { method: "POST", url: expect.stringContaining(":streamGenerateContent?alt=sse") }, + ]); + expect(result).toMatchObject({ + responseId: "sdk-google-response", + responseModel: "gemini-test-002", + stopReason: "stop", + }); + } finally { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } + }); + it("reports SDK stream acceptance without fabricated HTTP metadata", async () => { const acceptanceObserver = vi.fn(); const options = withProviderAcceptanceObserver({}, acceptanceObserver); diff --git a/packages/ai/src/providers/google-shared.ts b/packages/ai/src/providers/google-shared.ts index fe57039fb391..584dc2b93614 100644 --- a/packages/ai/src/providers/google-shared.ts +++ b/packages/ai/src/providers/google-shared.ts @@ -49,6 +49,10 @@ import { type GoogleApiType = "google-generative-ai" | "google-vertex"; +// Google-owned SDK resource spellings identify the same model; other publishers do not. +const GOOGLE_MODEL_RESOURCE_PREFIX = + /^(?:(?:projects\/[^/]+\/locations\/[^/]+\/)?publishers\/google\/models\/|google\/|models\/)/u; + type GoogleThinkingLevel = `${ThinkingLevel}`; type GoogleToolChoice = "auto" | "none" | "any"; @@ -801,6 +805,14 @@ export async function consumeGoogleGenerateContentStream) { const value = chunk.usageMetadata[field];