From 8eaf917efbae732ebfaa2e2807ef86afdd0ba56e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 3 Aug 2026 10:47:13 -0700 Subject: [PATCH] fix(google): honor Cloud SDK credential location and Vertex billing project (#118745) --- extensions/google/index.test.ts | 82 ++++++++- extensions/google/manifest.test.ts | 26 +++ extensions/google/openclaw.plugin.json | 1 + extensions/google/transport-stream.test.ts | 129 ++++++++++++++- extensions/google/vertex-adc.ts | 42 +++-- src/secrets/provider-auth-evidence.ts | 39 ++++- src/secrets/provider-env-vars.dynamic.test.ts | 155 ++++++++++++++++++ 7 files changed, 446 insertions(+), 28 deletions(-) diff --git a/extensions/google/index.test.ts b/extensions/google/index.test.ts index 024286e5ad27..2eb55cbc515f 100644 --- a/extensions/google/index.test.ts +++ b/extensions/google/index.test.ts @@ -1,5 +1,5 @@ // Google tests cover index plugin behavior. -import { mkdtemp, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import type { Context, Model } from "openclaw/plugin-sdk/llm"; @@ -327,6 +327,86 @@ describe("google provider plugin hooks", () => { ).toBe("gcp-vertex-credentials"); }); + it("prefers relocated Google Cloud SDK ADC over the home fallback", async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-google-vertex-cloud-sdk-")); + const cloudSdkDir = path.join(tempDir, "cloud-sdk"); + const homeCredentialsDir = path.join(tempDir, "home", ".config", "gcloud"); + await Promise.all([ + mkdir(cloudSdkDir, { recursive: true }), + mkdir(homeCredentialsDir, { recursive: true }), + ]); + const relocatedCredentialsPath = path.join(cloudSdkDir, "application_default_credentials.json"); + const homeCredentialsPath = path.join( + homeCredentialsDir, + "application_default_credentials.json", + ); + await Promise.all([ + writeFile( + relocatedCredentialsPath, + JSON.stringify({ + type: "authorized_user", + client_id: "fixture-client", + client_secret: "fixture-secret", + refresh_token: "fixture-refresh", + }), + "utf8", + ), + writeFile(homeCredentialsPath, JSON.stringify({ type: "unsupported" }), "utf8"), + ]); + const { providers } = await registerProviderPlugin({ + plugin: googleProviderPlugin, + id: "google", + name: "Google Provider", + }); + const provider = requireRegisteredProvider(providers, "google-vertex"); + const env = { + CLOUDSDK_CONFIG: cloudSdkDir, + HOME: path.join(tempDir, "home"), + GOOGLE_CLOUD_PROJECT: "fixture-project", + GOOGLE_CLOUD_LOCATION: "global", + }; + + expect(provider.resolveConfigApiKey?.({ provider: "google-vertex", env })).toBe( + "gcp-vertex-credentials", + ); + expect(googleProviderDiscovery.resolveConfigApiKey?.({ provider: "google-vertex", env })).toBe( + "gcp-vertex-credentials", + ); + expect( + provider.resolveConfigApiKey?.({ + provider: "google-vertex", + env: { ...env, GOOGLE_APPLICATION_CREDENTIALS: homeCredentialsPath }, + }), + ).toBeUndefined(); + + await writeFile( + homeCredentialsPath, + JSON.stringify({ + type: "authorized_user", + client_id: "stale-client", + client_secret: "stale-secret", + refresh_token: "stale-refresh", + }), + "utf8", + ); + const missingRelocatedCredentialsEnv = { + ...env, + CLOUDSDK_CONFIG: path.join(tempDir, "missing-cloud-sdk"), + }; + expect( + provider.resolveConfigApiKey?.({ + provider: "google-vertex", + env: missingRelocatedCredentialsEnv, + }), + ).toBeUndefined(); + expect( + googleProviderDiscovery.resolveConfigApiKey?.({ + provider: "google-vertex", + env: missingRelocatedCredentialsEnv, + }), + ).toBeUndefined(); + }); + it("owns Gemini tool schema normalization for direct and CLI providers", async () => { const { providers } = await registerProviderPlugin({ plugin: googleProviderPlugin, diff --git a/extensions/google/manifest.test.ts b/extensions/google/manifest.test.ts index f27244e95999..c661bfec3966 100644 --- a/extensions/google/manifest.test.ts +++ b/extensions/google/manifest.test.ts @@ -5,6 +5,15 @@ import type { JsonSchemaObject } from "openclaw/plugin-sdk/json-schema-runtime"; import { describe, expect, it } from "vitest"; type GoogleManifest = { + setup?: { + providers?: Array<{ + id?: string; + authEvidence?: Array<{ + fileEnvVar?: string; + fallbackPaths?: string[]; + }>; + }>; + }; providerAuthChoices?: Array<{ provider?: string; method?: string; @@ -86,6 +95,23 @@ function loadManifest(): GoogleManifest { } describe("google manifest model catalog", () => { + it("checks relocated Cloud SDK ADC before platform-specific fallback paths", () => { + const vertex = loadManifest().setup?.providers?.find( + (provider) => provider.id === "google-vertex", + ); + + expect(vertex?.authEvidence).toEqual([ + expect.objectContaining({ + fileEnvVar: "GOOGLE_APPLICATION_CREDENTIALS", + fallbackPaths: [ + "${CLOUDSDK_CONFIG}/application_default_credentials.json", + "${HOME}/.config/gcloud/application_default_credentials.json", + "${APPDATA}/gcloud/application_default_credentials.json", + ], + }), + ]); + }); + it("offers Google AI Studio API keys without consumer CLI OAuth", () => { const choices = loadManifest().providerAuthChoices ?? []; diff --git a/extensions/google/openclaw.plugin.json b/extensions/google/openclaw.plugin.json index 3a1629889c5d..a17c9ad7830e 100644 --- a/extensions/google/openclaw.plugin.json +++ b/extensions/google/openclaw.plugin.json @@ -669,6 +669,7 @@ "type": "local-file-with-env", "fileEnvVar": "GOOGLE_APPLICATION_CREDENTIALS", "fallbackPaths": [ + "${CLOUDSDK_CONFIG}/application_default_credentials.json", "${HOME}/.config/gcloud/application_default_credentials.json", "${APPDATA}/gcloud/application_default_credentials.json" ], diff --git a/extensions/google/transport-stream.test.ts b/extensions/google/transport-stream.test.ts index 695b86690832..ee7854f26f57 100644 --- a/extensions/google/transport-stream.test.ts +++ b/extensions/google/transport-stream.test.ts @@ -137,7 +137,11 @@ async function runGoogleVertexStreamResult(params: { return stream.result(); } -async function useGoogleAuthorizedUserCredentials(label: string, refreshToken: string) { +async function useGoogleAuthorizedUserCredentials( + label: string, + refreshToken: string, + quotaProjectId?: string, +) { const tempDir = await mkdtemp(path.join(os.tmpdir(), `openclaw-google-vertex-${label}-`)); const credentialsPath = path.join(tempDir, "application_default_credentials.json"); await writeFile( @@ -147,6 +151,7 @@ async function useGoogleAuthorizedUserCredentials(label: string, refreshToken: s client_id: "client-id", client_secret: "client-secret", refresh_token: refreshToken, + ...(quotaProjectId ? { quota_project_id: quotaProjectId } : {}), }), "utf8", ); @@ -1469,6 +1474,37 @@ describe("google transport stream", () => { expect(tokenFetchMock).not.toHaveBeenCalled(); }); + it("never refreshes stale home ADC when the selected Cloud SDK directory has no credentials", async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-google-vertex-stale-home-")); + const homeCredentialsDir = path.join(tempDir, "home", ".config", "gcloud"); + await mkdir(homeCredentialsDir, { recursive: true }); + await writeFile( + path.join(homeCredentialsDir, "application_default_credentials.json"), + JSON.stringify({ + type: "authorized_user", + client_id: "stale-client", + client_secret: "stale-secret", + refresh_token: "stale-refresh", + }), + "utf8", + ); + vi.stubEnv("GOOGLE_APPLICATION_CREDENTIALS", ""); + vi.stubEnv("CLOUDSDK_CONFIG", path.join(tempDir, "missing-cloud-sdk")); + vi.stubEnv("HOME", path.join(tempDir, "home")); + vi.stubEnv("APPDATA", ""); + googleAuthGetAccessTokenMock.mockResolvedValueOnce("fixture-google-auth-token"); + const tokenFetchMock = vi.fn(); + + await expect(resolveGoogleVertexAuthorizedUserHeaders(tokenFetchMock)).resolves.toEqual({ + Authorization: "Bearer fixture-google-auth-token", + }); + expect(googleAuthMock).toHaveBeenCalledWith({ + scopes: ["https://www.googleapis.com/auth/cloud-platform"], + clientOptions: { transporterOptions: { timeout: 30_000 } }, + }); + expect(tokenFetchMock).not.toHaveBeenCalled(); + }); + it("bounds Google Vertex ADC files before google-auth-library reads them", async () => { const tempDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-google-vertex-adc-file-")); const credentialsPath = path.join(tempDir, "application_default_credentials.json"); @@ -1593,6 +1629,97 @@ describe("google transport stream", () => { expect(new Headers(guardedInit.headers).has("x-goog-api-key")).toBe(false); }); + it.each([ + { + scenario: "authorized-user ADC quota project", + credentialType: "authorized_user", + credentialQuotaProject: "fixture-json-billing", + envQuotaProject: "", + expectedQuotaProject: "fixture-json-billing", + }, + { + scenario: "environment quota project overriding authorized-user ADC", + credentialType: "authorized_user", + credentialQuotaProject: "fixture-json-billing", + envQuotaProject: "fixture-env-billing", + expectedQuotaProject: "fixture-env-billing", + }, + { + scenario: "google-auth service-account ADC quota project", + credentialType: "service_account", + credentialQuotaProject: "fixture-service-billing", + envQuotaProject: "", + expectedQuotaProject: "fixture-service-billing", + }, + { + scenario: "google-auth metadata ADC environment quota project", + credentialType: "metadata", + credentialQuotaProject: "", + envQuotaProject: "fixture-metadata-billing", + expectedQuotaProject: "fixture-metadata-billing", + }, + ])( + "forwards the $scenario on the actual Vertex request", + async ({ credentialType, credentialQuotaProject, envQuotaProject, expectedQuotaProject }) => { + const tokenFetchMock = vi.fn(); + vi.stubEnv("GOOGLE_CLOUD_PROJECT", "fixture-project"); + vi.stubEnv("GOOGLE_CLOUD_LOCATION", "global"); + vi.stubEnv("GOOGLE_CLOUD_QUOTA_PROJECT", envQuotaProject); + if (credentialType === "authorized_user") { + await useGoogleAuthorizedUserCredentials( + "quota-authorized-user", + "fixture-refresh-token", + credentialQuotaProject, + ); + tokenFetchMock.mockResolvedValueOnce( + new Response(JSON.stringify({ access_token: "fixture-vertex-token", expires_in: 3600 }), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ); + } else if (credentialType === "service_account") { + const tempDir = await mkdtemp( + path.join(os.tmpdir(), "openclaw-google-vertex-quota-service-"), + ); + const credentialsPath = path.join(tempDir, "application_default_credentials.json"); + await writeFile( + credentialsPath, + JSON.stringify({ + type: "service_account", + client_email: "fixture@example.invalid", + quota_project_id: credentialQuotaProject, + }), + "utf8", + ); + vi.stubEnv("GOOGLE_APPLICATION_CREDENTIALS", credentialsPath); + googleAuthGetAccessTokenMock.mockResolvedValueOnce("fixture-vertex-token"); + } else { + const tempDir = await mkdtemp( + path.join(os.tmpdir(), "openclaw-google-vertex-quota-metadata-"), + ); + vi.stubEnv("GOOGLE_APPLICATION_CREDENTIALS", ""); + vi.stubEnv("HOME", path.join(tempDir, "home")); + vi.stubEnv("APPDATA", ""); + googleAuthGetAccessTokenMock.mockResolvedValueOnce("fixture-vertex-token"); + } + guardedFetchMock.mockResolvedValueOnce( + buildSseResponse([ + { + candidates: [{ content: { parts: [{ text: "ok" }] }, finishReason: "STOP" }], + }, + ]), + ); + + await runGoogleVertexStreamResult({ fetch: tokenFetchMock }); + + const guardedCall = requireMockCall(guardedFetchMock, 0, "guarded fetch"); + expectHeaders(requireRequestInit(guardedCall, "guarded fetch"), { + Authorization: "Bearer fixture-vertex-token", + "x-goog-user-project": expectedQuotaProject, + }); + }, + ); + it("strips redundant google provider prefixes from Google Vertex model paths", async () => { const tempDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-google-vertex-prefix-")); vi.stubEnv("HOME", path.join(tempDir, "home")); diff --git a/extensions/google/vertex-adc.ts b/extensions/google/vertex-adc.ts index 18bb9c667fcd..3fc97f3dde4f 100644 --- a/extensions/google/vertex-adc.ts +++ b/extensions/google/vertex-adc.ts @@ -132,6 +132,11 @@ function resolveGoogleApplicationCredentialsPath( if (explicit) { return existsSync(explicit) ? explicit : undefined; } + const cloudSdkDir = normalizeOptionalString(env.CLOUDSDK_CONFIG); + if (cloudSdkDir) { + const cloudSdkFallback = path.join(cloudSdkDir, "application_default_credentials.json"); + return existsSync(cloudSdkFallback) ? cloudSdkFallback : undefined; + } const homeDir = normalizeOptionalString(env.HOME) ?? os.homedir(); const homeFallback = path.join( homeDir, @@ -449,22 +454,23 @@ export async function resolveGoogleVertexAuthorizedUserHeaders( fetchImpl?: typeof fetch, ): Promise> { const adcPath = resolveGoogleApplicationCredentialsPath(); - let adcConfig: GoogleAdcConfig | undefined; - if (adcPath) { - adcConfig = readGoogleAdcCredentials(adcPath); - const userAdc = resolveGoogleAuthorizedUserCredentials(adcConfig); - if (userAdc) { - const token = await refreshGoogleVertexAuthorizedUserAccessToken({ - credentialsPath: adcPath, - credentials: userAdc, - fetchImpl, - }); - return { Authorization: `Bearer ${token}` }; - } - } - // No file-based authorized_user ADC. Fall back to google-auth-library which - // handles GKE Workload Identity (metadata server), Workload Identity - // Federation (external_account), and service-account keys. - const token = await resolveGoogleVertexAccessTokenViaGoogleAuth(adcConfig); - return { Authorization: `Bearer ${token}` }; + const adcConfig = adcPath ? readGoogleAdcCredentials(adcPath) : undefined; + const userAdc = adcConfig ? resolveGoogleAuthorizedUserCredentials(adcConfig) : undefined; + // Google auth owns metadata, federation, and service-account ADC variants. + const token = + userAdc && adcPath + ? await refreshGoogleVertexAuthorizedUserAccessToken({ + credentialsPath: adcPath, + credentials: userAdc, + fetchImpl, + }) + : await resolveGoogleVertexAccessTokenViaGoogleAuth(adcConfig); + // Google auth gives the explicit billing project precedence over ADC metadata. + const quotaProject = + normalizeOptionalString(process.env.GOOGLE_CLOUD_QUOTA_PROJECT) ?? + normalizeOptionalString((adcConfig as Record | undefined)?.quota_project_id); + return { + Authorization: `Bearer ${token}`, + ...(quotaProject ? { "x-goog-user-project": quotaProject } : {}), + }; } diff --git a/src/secrets/provider-auth-evidence.ts b/src/secrets/provider-auth-evidence.ts index 0913fdea9e17..3194d08331a8 100644 --- a/src/secrets/provider-auth-evidence.ts +++ b/src/secrets/provider-auth-evidence.ts @@ -19,17 +19,33 @@ type ResolvedLocalProviderAuthEvidence = { source: string; }; -function expandAuthEvidencePath(rawPath: string, env: NodeJS.ProcessEnv): string | undefined { +function expandAuthEvidencePath( + rawPath: string, + env: NodeJS.ProcessEnv, +): { path: string; explicitOverride: boolean } | undefined { const trimmed = rawPath.trim(); if (!trimmed) { return undefined; } - const homeDir = normalizeOptionalPathInput(env.HOME) ?? os.homedir(); - const appDataDir = normalizeOptionalPathInput(env.APPDATA); - if (trimmed.includes("${APPDATA}") && !appDataDir) { - return undefined; - } - return trimmed.replaceAll("${HOME}", homeDir).replaceAll("${APPDATA}", appDataDir ?? ""); + let unresolvedPlaceholder = false; + let explicitOverride = false; + const expanded = trimmed.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/gu, (_match, name: string) => { + const value = + name === "HOME" + ? (normalizeOptionalPathInput(env.HOME) ?? os.homedir()) + : normalizeOptionalPathInput(env[name]); + if (!value) { + unresolvedPlaceholder = true; + return ""; + } + if (name !== "HOME" && name !== "APPDATA") { + explicitOverride = true; + } + return value; + }); + return unresolvedPlaceholder || expanded.includes("${") + ? undefined + : { path: expanded, explicitOverride }; } function hasRequiredAuthEvidenceEnv( @@ -58,9 +74,16 @@ function hasLocalFileAuthEvidence( } for (const rawPath of evidence.fallbackPaths ?? []) { const expandedPath = expandAuthEvidencePath(rawPath, env); - if (expandedPath && fs.existsSync(expandedPath)) { + if (!expandedPath) { + continue; + } + if (fs.existsSync(expandedPath.path)) { return true; } + // An explicit provider directory owns identity; never select stale platform credentials. + if (expandedPath.explicitOverride) { + return false; + } } return false; } diff --git a/src/secrets/provider-env-vars.dynamic.test.ts b/src/secrets/provider-env-vars.dynamic.test.ts index 21a93e03cd98..0309eb9ce643 100644 --- a/src/secrets/provider-env-vars.dynamic.test.ts +++ b/src/secrets/provider-env-vars.dynamic.test.ts @@ -1,6 +1,8 @@ /** Tests dynamic provider env-var discovery from plugin metadata. */ +import fs from "node:fs"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { sanitizeEnvVars } from "../agents/sandbox/sanitize-env-vars.js"; +import { resolveLocalProviderAuthEvidence } from "./provider-auth-evidence.js"; import { getProviderEnvVars, listKnownProviderAuthEnvVarNames, @@ -287,6 +289,159 @@ describe("provider env vars dynamic manifest metadata", () => { expect(snapshotOptions.preferPersisted).toBe(false); }); + it("expands provider-owned directory variables in manifest credential evidence", () => { + useRegistrySetupPlugin("external-cloud", "global", { + id: "external-cloud", + authEvidence: [ + { + type: "local-file-with-env", + fallbackPaths: ["${EXTERNAL_CLOUD_CONFIG}/application_default_credentials.json"], + requiresAllEnv: ["EXTERNAL_CLOUD_PROJECT"], + credentialMarker: "external-cloud-local-credentials", + source: "external cloud credentials", + }, + ], + }); + const evidence = resolveProviderAuthLookupMaps().authEvidenceMap["external-cloud"]; + const expectedPath = "/fixture/cloud-sdk/application_default_credentials.json"; + const existsSync = vi.spyOn(fs, "existsSync").mockImplementation((candidate) => { + return candidate === expectedPath; + }); + + try { + expect( + resolveLocalProviderAuthEvidence(evidence, { + EXTERNAL_CLOUD_CONFIG: "/fixture/cloud-sdk", + EXTERNAL_CLOUD_PROJECT: "fixture-project", + }), + ).toEqual({ + credentialMarker: "external-cloud-local-credentials", + source: "external cloud credentials", + }); + expect(existsSync).toHaveBeenCalledWith(expectedPath); + } finally { + existsSync.mockRestore(); + } + }); + + it("rejects stale home evidence when an explicit provider directory has no credentials", () => { + useRegistrySetupPlugin("external-cloud", "global", { + id: "external-cloud", + authEvidence: [ + { + type: "local-file-with-env", + fallbackPaths: [ + "${EXTERNAL_CLOUD_CONFIG}/credentials.json", + "${HOME}/credentials.json", + "${APPDATA}/credentials.json", + ], + credentialMarker: "external-cloud-local-credentials", + }, + ], + }); + const evidence = resolveProviderAuthLookupMaps().authEvidenceMap["external-cloud"]; + const existsSync = vi.spyOn(fs, "existsSync").mockImplementation((candidate) => { + return candidate === "/fixture/home/credentials.json"; + }); + + try { + expect( + resolveLocalProviderAuthEvidence(evidence, { + EXTERNAL_CLOUD_CONFIG: "/fixture/missing-cloud-sdk", + HOME: "/fixture/home", + APPDATA: "/fixture/appdata", + }), + ).toBeNull(); + expect(existsSync).toHaveBeenCalledOnce(); + expect(existsSync).toHaveBeenCalledWith("/fixture/missing-cloud-sdk/credentials.json"); + } finally { + existsSync.mockRestore(); + } + }); + + it("preserves home and appdata fallback when no provider directory is selected", () => { + const fallbackPaths = [ + "${EXTERNAL_CLOUD_CONFIG}/credentials.json", + "${HOME}/credentials.json", + "${APPDATA}/credentials.json", + ]; + const evidence = [ + { + type: "local-file-with-env" as const, + fallbackPaths, + credentialMarker: "external-cloud-local-credentials", + }, + ]; + const existsSync = vi.spyOn(fs, "existsSync").mockImplementation((candidate) => { + return ( + candidate === "/fixture/home/credentials.json" || + candidate === "/fixture/appdata/credentials.json" + ); + }); + + try { + expect(resolveLocalProviderAuthEvidence(evidence, { HOME: "/fixture/home" })).toEqual({ + credentialMarker: "external-cloud-local-credentials", + source: "local auth evidence", + }); + expect( + resolveLocalProviderAuthEvidence(evidence, { + EXTERNAL_CLOUD_CONFIG: " ", + HOME: "/fixture/missing-home", + APPDATA: "/fixture/appdata", + }), + ).toEqual({ + credentialMarker: "external-cloud-local-credentials", + source: "local auth evidence", + }); + } finally { + existsSync.mockRestore(); + } + }); + + it.each([ + { + scenario: "missing variable", + fallbackPath: "${EXTERNAL_CLOUD_CONFIG}/credentials.json", + env: {}, + }, + { + scenario: "blank variable", + fallbackPath: "${EXTERNAL_CLOUD_CONFIG}/credentials.json", + env: { EXTERNAL_CLOUD_CONFIG: " " }, + }, + { + scenario: "invalid variable name", + fallbackPath: "${EXTERNAL-CLOUD-CONFIG}/credentials.json", + env: { "EXTERNAL-CLOUD-CONFIG": "/fixture/cloud-sdk" }, + }, + { + scenario: "unterminated placeholder", + fallbackPath: "${EXTERNAL_CLOUD_CONFIG/credentials.json", + env: { EXTERNAL_CLOUD_CONFIG: "/fixture/cloud-sdk" }, + }, + ])("rejects manifest credential evidence with a $scenario", ({ fallbackPath, env }) => { + const existsSync = vi.spyOn(fs, "existsSync").mockReturnValue(true); + + try { + expect( + resolveLocalProviderAuthEvidence( + [ + { + type: "local-file-with-env", + fallbackPaths: [fallbackPath], + credentialMarker: "external-cloud-local-credentials", + }, + ], + env, + ), + ).toBeNull(); + expect(existsSync).not.toHaveBeenCalled(); + } finally { + existsSync.mockRestore(); + } + }); + it("reuses the current compatible metadata snapshot for workspace auth evidence", () => { pluginRegistryMocks.getCurrentPluginMetadataSnapshot.mockReturnValue( metadataSnapshot(