From 196d81195bf438ef095ba392445bb164e4e07310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mert=20Ba=C5=9Far?= <100199793+MertBasar0@users.noreply.github.com> Date: Tue, 4 Aug 2026 04:19:06 +0300 Subject: [PATCH] fix(auth): cooldown inline api key billing failures (#88709) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(auth): cooldown inline api key billing failures * fix: mark inline api key prompt failures * fix(auth): propagate inline api key cooldown to provider visibility * fix(types): fix test type errors in agents and ui * fix(auth): correct null check for cooldown gate and fix test case * fix(ui): remove unrelated test drift and fix expired toast expectation * fix(auth): preserve inline cooldown usageStats in warm worker snapshots * test(auth): update warm snapshot expectation with usageStats * test(auth): format inline cooldown worker proof * fix(auth): narrow inline cooldown classifier * fix(auth): mark managed file/exec SecretRef inline keys on auth/billing failure The inline-api-key cooldown marker only fired for literal, env-marker, and env SecretRef sources, while the resolution gate already covered managed file/exec SecretRef provider keys via coerceSecretRef. As a result a file/exec SecretRef models.providers..apiKey could never record a 402 billing cooldown, so the gate that would honor it never triggered. Align the runner failure-marker trigger with the gate predicate (isConfigBackedInlineProviderApiKey) so managed non-env SecretRef inline keys are marked on auth/billing failure. Adds focused regression coverage. Co-Authored-By: Claude Opus 4.8 * refactor(auth): drop now-unused isInlineProviderApiKeyAuth export The runner failure-marker now gates on isConfigBackedInlineProviderApiKey, so isInlineProviderApiKeyAuth has no remaining callers. Removing the dead export also trims one public callable export back under the plugin-SDK surface budget. Co-Authored-By: Claude Opus 4.8 * fix(auth): enforce inline cooldown for managed file/exec SecretRef keys The inline-api-key cooldown was recorded for managed (file/exec) SecretRef provider keys but never enforced: these keys resolve through the synthetic runtime path (resolveSyntheticLocalProviderAuth) and the explicit api-key override path, neither of which consulted the inline-key cooldown gate. So an exhausted file/exec SecretRef provider kept resolving and reporting available during an active inline-api-key: cooldown. Gate both resolution exits (and the provider-availability check) on isConfigBackedInlineProviderApiKey + assertInlineProviderApiKeyUsable, matching the literal/env paths. Local no-auth markers are not config-backed inline keys, so they stay untouched. Adds regression coverage for both resolution paths. Co-Authored-By: Claude Opus 4.8 * fix(auth): gate runtime SecretRef cooldown checks * fix(auth): honor inline key cooldowns in model-list and tool auth checks Direct hasRuntimeAvailableProviderAuth callers omitted the auth store, so inline provider keys in billing cooldown were still advertised as available in model browsing (models.list) and model-backed tool auth checks. Pass the scoped auth store at each call site and cover the cooled-key paths with regression tests, including profile fallback and cooldown expiry. Co-Authored-By: Claude Fable 5 * fix(auth): resolve merge CI failures (format + browse cooldown mock independence) - failover-retry-controller.ts: apply oxfmt formatting missed after the port. - model-auth-availability.ts: the browse-side inline cooldown check imported resolveInlineProviderApiKeyUnusableUntil from the auth-profiles usage module, which several command/directive tests partially mock — the new export was absent from those mocks, so every mocking test threw at the browse path. Read the inline cooldown via usage-state primitives (resolveProfileUnusableUntil + isAuthCooldownBypassedForProvider) and build the usage id with the same normalizeProviderId the write side uses, so the hot browse path no longer depends on the mocked usage module while keeping key resolution identical. Co-Authored-By: Claude Fable 5 * fix(auth): make model-auth inline cooldown reads mock-independent resolveApiKeyForProvider/assertInlineProviderApiKeyUsable read the inline provider API-key cooldown, which this PR routes through the auth-profiles usage module. Many existing suites partially mock that barrel (e.g. run.overflow-compaction.test.ts) without the new export, so those tests threw once merged code reached the inline path. Read the cooldown via usage-state primitives (resolveProfileUnusableUntil + isAuthCooldownBypassedForProvider) and build the `inline-api-key:` usage id with the same normalizeProviderId the write side uses, so model-auth no longer depends on the mocked usage module while key resolution stays identical. Verified across all 30 suites that mock the auth-profiles module. Co-Authored-By: Claude Fable 5 * fix(test): use main's terminal failure shape for the inline-key billing case Main reworked EmbeddedRunAttemptResult so attempt failures are reported via `terminal: { kind: "failed", source, error }` instead of the old `promptError`/`promptErrorSource` fields. The inline-key billing regression test still used the removed fields, which merged cleanly but broke check-test-types. Switch it to the same shape the sibling helper in this file already uses. Verified with `pnpm check:test-types` (exit 0). Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Mert Basar Co-authored-by: Ceviz Agent Co-authored-by: Claude Opus 4.8 --- ...th-profiles.markauthprofilefailure.test.ts | 40 ++ src/agents/auth-profiles.ts | 3 + src/agents/auth-profiles/persisted.ts | 12 +- src/agents/auth-profiles/store.ts | 4 +- src/agents/auth-profiles/usage.ts | 78 ++++ ...ed-agent.auth-profile-rotation.e2e.test.ts | 47 ++- src/agents/embedded-agent-runner/run-loop.ts | 1 + .../run/assistant-failover.test.ts | 22 ++ .../run/assistant-failover.ts | 2 +- .../run/failover-retry-controller.ts | 41 ++- .../run/prompt-failure.ts | 4 +- src/agents/model-auth-availability.ts | 22 +- src/agents/model-auth-model.ts | 60 ++- src/agents/model-auth-provider-config.ts | 65 ++++ src/agents/model-auth-provider.ts | 44 +++ src/agents/model-auth-runtime.ts | 43 ++- src/agents/model-auth.profiles.test.ts | 341 +++++++++++++++++- src/agents/model-auth.test.ts | 60 +++ src/agents/model-auth.ts | 1 + src/agents/model-provider-auth.test.ts | 1 + src/agents/model-provider-auth.ts | 37 +- src/agents/model-provider-auth.worker.test.ts | 60 ++- src/agents/tools/model-config.helpers.test.ts | 77 ++++ src/agents/tools/model-config.helpers.ts | 16 + src/gateway/server-methods/models.test.ts | 77 ++++ 25 files changed, 1112 insertions(+), 46 deletions(-) diff --git a/src/agents/auth-profiles.markauthprofilefailure.test.ts b/src/agents/auth-profiles.markauthprofilefailure.test.ts index b6d81e2baa09..2a2b264527df 100644 --- a/src/agents/auth-profiles.markauthprofilefailure.test.ts +++ b/src/agents/auth-profiles.markauthprofilefailure.test.ts @@ -27,6 +27,8 @@ import { import { calculateAuthProfileCooldownMs, markAuthProfileFailure, + markInlineProviderApiKeyFailure, + resolveInlineProviderApiKeyUsageId, setAuthProfileFailureHook, } from "./auth-profiles/usage.js"; @@ -168,6 +170,26 @@ describe("markAuthProfileFailure", () => { expectCooldownInRange(remainingMs, 4.5 * 60 * 60 * 1000, 5.5 * 60 * 60 * 1000); }); }); + it("records billing backoff for inline provider api keys without creating an auth profile", async () => { + await withAuthProfileStore(async ({ agentDir, store }) => { + const startedAt = Date.now(); + await markInlineProviderApiKeyFailure({ + store, + provider: "anthropic", + reason: "billing", + agentDir, + }); + + const usageId = resolveInlineProviderApiKeyUsageId("anthropic"); + const stats = store.usageStats?.[usageId]; + expect(store.profiles[usageId]).toBeUndefined(); + expect(stats?.disabledReason).toBe("billing"); + expect(typeof stats?.disabledUntil).toBe("number"); + const remainingMs = (stats?.disabledUntil as number) - startedAt; + expectCooldownInRange(remainingMs, 4.5 * 60 * 60 * 1000, 5.5 * 60 * 60 * 1000); + }); + }); + it("keeps persisted cooldownUntil unchanged across mid-window retries", async () => { await withAuthProfileStore(async ({ agentDir, store }) => { await markAuthProfileFailure({ @@ -376,6 +398,24 @@ describe("markAuthProfileFailure", () => { }); }); + it("fires the auth profile failure hook for inline provider api key failures", async () => { + await withAuthProfileStore(async ({ agentDir, store }) => { + const hook = vi.fn(); + setAuthProfileFailureHook(hook); + try { + await markInlineProviderApiKeyFailure({ + store, + provider: "anthropic", + reason: "billing", + agentDir, + }); + expect(hook).toHaveBeenCalledTimes(1); + } finally { + setAuthProfileFailureHook(undefined); + } + }); + }); + it("does not break failure recording when the hook throws", async () => { await withAuthProfileStore(async ({ agentDir, store }) => { const throwingHook = vi.fn(() => { diff --git a/src/agents/auth-profiles.ts b/src/agents/auth-profiles.ts index 4c2ff501b914..e9240d42ef31 100644 --- a/src/agents/auth-profiles.ts +++ b/src/agents/auth-profiles.ts @@ -97,6 +97,9 @@ export { markAuthProfileCooldown, markAuthProfileBlockedUntil, markAuthProfileFailure, + markInlineProviderApiKeyFailure, + resolveInlineProviderApiKeyUnusableUntil, + resolveInlineProviderApiKeyUsageId, resolveProfilesUnavailableReason, resolveProfileUnusableUntilForDisplay, setAuthProfileFailureHook, diff --git a/src/agents/auth-profiles/persisted.ts b/src/agents/auth-profiles/persisted.ts index 80b38b0964ad..6054251aaba8 100644 --- a/src/agents/auth-profiles/persisted.ts +++ b/src/agents/auth-profiles/persisted.ts @@ -44,6 +44,14 @@ type CredentialRejectReason = "non_object" | "invalid_type" | "missing_provider" type RejectedCredentialEntry = { key: string; reason: CredentialRejectReason }; const AUTH_PROFILE_TYPES = new Set(["api_key", "oauth", "token"]); +const INLINE_API_KEY_USAGE_ID_PREFIX = "inline-api-key:"; + +function isRetainedUsageStatsId( + profileId: string, + profiles: AuthProfileStore["profiles"], +): boolean { + return Boolean(profiles[profileId]) || profileId.startsWith(INLINE_API_KEY_USAGE_ID_PREFIX); +} // Persisted credential normalization accepts old field names and SecretRef-ish // values, then emits the current credential discriminated union. @@ -652,7 +660,9 @@ export function mergeAuthProfileStores( const mergedUsageStats = mergeRecord(base.usageStats, override.usageStats); const usageStats = mergedUsageStats ? Object.fromEntries( - Object.entries(mergedUsageStats).filter(([profileId]) => profiles[profileId]), + Object.entries(mergedUsageStats).filter(([profileId]) => + isRetainedUsageStatsId(profileId, profiles), + ), ) : undefined; const merged = { diff --git a/src/agents/auth-profiles/store.ts b/src/agents/auth-profiles/store.ts index 03eb7338956d..550d2bd5378b 100644 --- a/src/agents/auth-profiles/store.ts +++ b/src/agents/auth-profiles/store.ts @@ -539,7 +539,9 @@ function pruneAuthProfileStoreReferences( : undefined; store.usageStats = store.usageStats ? Object.fromEntries( - Object.entries(store.usageStats).filter(([profileId]) => keptProfileIds.has(profileId)), + Object.entries(store.usageStats).filter( + ([profileId]) => keptProfileIds.has(profileId) || profileId.startsWith("inline-api-key:"), + ), ) : undefined; store.runtimePersistedProfileIds = store.runtimePersistedProfileIds diff --git a/src/agents/auth-profiles/usage.ts b/src/agents/auth-profiles/usage.ts index dbb9f06bcaf9..1155a0cb06ec 100644 --- a/src/agents/auth-profiles/usage.ts +++ b/src/agents/auth-profiles/usage.ts @@ -73,6 +73,12 @@ function logDroppedAuthProfileBookkeeping(kind: string, profileId: string): void }); } +const INLINE_API_KEY_USAGE_ID_PREFIX = "inline-api-key:"; + +export function resolveInlineProviderApiKeyUsageId(provider: string): string { + return `${INLINE_API_KEY_USAGE_ID_PREFIX}${normalizeProviderId(provider)}`; +} + const FAILURE_REASON_PRIORITY: AuthProfileFailureReason[] = [ "auth_permanent", "auth", @@ -761,6 +767,20 @@ export function resolveProfileUnusableUntilForDisplay( return resolveProfileUnusableUntil(stats); } +export function resolveInlineProviderApiKeyUnusableUntil( + store: AuthProfileStore, + provider: string, +): number | null { + if (isAuthCooldownBypassedForProvider(provider)) { + return null; + } + const stats = store.usageStats?.[resolveInlineProviderApiKeyUsageId(provider)]; + if (!stats) { + return null; + } + return resolveProfileUnusableUntil(stats); +} + function resetUsageStats( existing: ProfileUsageStats | undefined, overrides?: Partial, @@ -1120,6 +1140,64 @@ export async function markAuthProfileBlockedUntil(params: { } } +export async function markInlineProviderApiKeyFailure(params: { + store: AuthProfileStore; + provider: string; + reason: AuthProfileFailureReason; + cfg?: OpenClawConfig; + agentDir?: string; + runId?: string; + modelId?: string; +}): Promise { + const { store, provider, reason, agentDir, runId, modelId } = params; + if (isAuthCooldownBypassedForProvider(provider)) { + return; + } + + const usageId = resolveInlineProviderApiKeyUsageId(provider); + const cfgResolved = resolveAuthCooldownConfig(); + + let nextStats: ProfileUsageStats | undefined; + let previousStats: ProfileUsageStats | undefined; + let updateTime = 0; + const updated = await authProfileUsageDeps.updateAuthProfileStoreWithLock({ + agentDir, + updater: (freshStore) => { + const now = Date.now(); + previousStats = freshStore.usageStats?.[usageId]; + updateTime = now; + nextStats = computeNextProfileUsageStats({ + existing: previousStats ?? {}, + now, + reason, + cfgResolved, + modelId, + }); + updateUsageStatsEntry(freshStore, usageId, () => nextStats as ProfileUsageStats); + return true; + }, + }); + if (updated) { + store.usageStats = updated.usageStats; + if (nextStats) { + logAuthProfileFailureStateChange({ + runId, + profileId: usageId, + provider, + reason, + previous: previousStats, + next: nextStats, + now: updateTime, + }); + } + notifyAuthProfileFailureHook(); + return; + } + if (updated === null) { + logDroppedAuthProfileBookkeeping("inline_api_key_failure", usageId); + } +} + /** * Mark a profile as transiently failed. Applies stepped backoff cooldown. * Cooldown times: 30s, 1min, 5min (capped). diff --git a/src/agents/embedded-agent-runner.run-embedded-agent.auth-profile-rotation.e2e.test.ts b/src/agents/embedded-agent-runner.run-embedded-agent.auth-profile-rotation.e2e.test.ts index 15a8259d05a3..47117da9f6d7 100644 --- a/src/agents/embedded-agent-runner.run-embedded-agent.auth-profile-rotation.e2e.test.ts +++ b/src/agents/embedded-agent-runner.run-embedded-agent.auth-profile-rotation.e2e.test.ts @@ -6,7 +6,10 @@ import type { AssistantMessage } from "openclaw/plugin-sdk/llm"; import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; import { redactIdentifier } from "../logging/redact-identifier.js"; -import type { AuthProfileFailureReason } from "./auth-profiles.js"; +import { + resolveInlineProviderApiKeyUsageId, + type AuthProfileFailureReason, +} from "./auth-profiles.js"; import { ensureAuthProfileStore, saveAuthProfileStore } from "./auth-profiles/store.js"; import type { EmbeddedRunAttemptResult } from "./embedded-agent-runner/run/types.js"; import { @@ -934,6 +937,48 @@ describe("runEmbeddedAgent auth profile rotation", () => { expect(sleepWithAbortMock).not.toHaveBeenCalled(); }); + it("marks inline provider api key billing prompt failures without an auth profile", async () => { + await withAgentWorkspace(async ({ agentDir, workspaceDir }) => { + await fs.writeFile( + path.join(agentDir, "auth-profiles.json"), + JSON.stringify({ version: 1, profiles: {} }), + ); + await fs.writeFile( + path.join(agentDir, "auth-state.json"), + JSON.stringify({ version: 1, usageStats: {} }), + ); + runEmbeddedAttemptMock.mockResolvedValueOnce( + makeAttempt({ + terminal: { kind: "failed", source: "prompt", error: new Error("insufficient credits") }, + }), + ); + + await expect( + runEmbeddedAgentInline({ + sessionId: "session:test", + sessionKey: "agent:test:inline-api-key-prompt-billing", + sessionFile: path.join(workspaceDir, "session.jsonl"), + workspaceDir, + agentDir, + config: makeConfig(), + prompt: "hello", + provider: "openai", + model: "mock-1", + authProfileIdSource: "auto", + timeoutMs: 5_000, + runId: "run:inline-api-key-prompt-billing", + }), + ).rejects.toThrow(/insufficient credits/); + + expect(runEmbeddedAttemptMock).toHaveBeenCalledTimes(1); + const usageStats = await readUsageStats(agentDir); + const usageId = resolveInlineProviderApiKeyUsageId("openai"); + expect(usageStats[usageId]?.disabledReason).toBe("billing"); + expect(typeof usageStats[usageId]?.disabledUntil).toBe("number"); + expect(usageStats["openai:p1"]).toBeUndefined(); + }); + }); + it("rotates on timeout without cooling down the timed-out profile", async () => { const { usageStats } = await runAutoPinnedRotationCase({ errorMessage: "request ended without sending any chunks", diff --git a/src/agents/embedded-agent-runner/run-loop.ts b/src/agents/embedded-agent-runner/run-loop.ts index 9d161824b699..fa824ae4fc35 100644 --- a/src/agents/embedded-agent-runner/run-loop.ts +++ b/src/agents/embedded-agent-runner/run-loop.ts @@ -262,6 +262,7 @@ export async function runPreparedEmbeddedLoop( getLastProfileId: () => preparedRuntime.snapshot().lastProfileId, getSessionId: () => sessionPromptState.sessionId, harnessOwnsTransport: () => preparedRuntime.snapshot().pluginHarnessOwnsTransport, + getApiKeyInfo, }); // Resolve the context engine once and reuse across retries to avoid // repeated initialization/connection overhead per attempt. diff --git a/src/agents/embedded-agent-runner/run/assistant-failover.test.ts b/src/agents/embedded-agent-runner/run/assistant-failover.test.ts index 4c2a3b9831e2..6897db0df0be 100644 --- a/src/agents/embedded-agent-runner/run/assistant-failover.test.ts +++ b/src/agents/embedded-agent-runner/run/assistant-failover.test.ts @@ -497,6 +497,28 @@ describe("handleAssistantFailover", () => { expect(warn).not.toHaveBeenCalled(); }); + it("marks inline auth failures even when no profile id is active", async () => { + const maybeMarkAuthProfileFailure = vi.fn(async () => {}); + + const outcome = await handleAssistantFailover( + makeParams({ + initialDecision: { action: "rotate_profile", reason: "billing" }, + failoverReason: "billing", + assistantProfileFailureReason: "billing", + lastProfileId: undefined, + advanceAuthProfile: vi.fn(async () => false), + maybeMarkAuthProfileFailure, + }), + ); + + expect(outcome.action).toBe("throw"); + expect(maybeMarkAuthProfileFailure).toHaveBeenCalledWith({ + profileId: undefined, + reason: "billing", + modelId: "claude-haiku-4-5-20251001", + }); + }); + it("marks provider-started timeout rotations against the failed profile", async () => { const maybeMarkAuthProfileFailure = vi.fn(async () => {}); diff --git a/src/agents/embedded-agent-runner/run/assistant-failover.ts b/src/agents/embedded-agent-runner/run/assistant-failover.ts index d178a68a20cf..7741ed25bb04 100644 --- a/src/agents/embedded-agent-runner/run/assistant-failover.ts +++ b/src/agents/embedded-agent-runner/run/assistant-failover.ts @@ -146,7 +146,7 @@ export async function handleAssistantFailover(params: { const timeoutFailure = terminal.timedOut; const failureReason = params.assistantProfileFailureReason; const markFailedProfile = async () => { - if (!failedProfileId || !failureReason) { + if (!failureReason) { return; } try { diff --git a/src/agents/embedded-agent-runner/run/failover-retry-controller.ts b/src/agents/embedded-agent-runner/run/failover-retry-controller.ts index 1f54c277b268..768e5e806c71 100644 --- a/src/agents/embedded-agent-runner/run/failover-retry-controller.ts +++ b/src/agents/embedded-agent-runner/run/failover-retry-controller.ts @@ -1,8 +1,13 @@ import { sanitizeForLog } from "../../../../packages/terminal-core/src/ansi.js"; import { sleepWithAbort } from "../../../infra/backoff.js"; -import { type AuthProfileFailureReason, markAuthProfileFailure } from "../../auth-profiles.js"; +import { + type AuthProfileFailureReason, + markAuthProfileFailure, + markInlineProviderApiKeyFailure, +} from "../../auth-profiles.js"; import type { FailoverReason } from "../../embedded-agent-helpers.js"; import { FailoverError, resolveFailoverStatus } from "../../failover-error.js"; +import { isConfigBackedInlineProviderApiKey, type ResolvedProviderAuth } from "../../model-auth.js"; import { log } from "../logger.js"; import { resolveAuthProfileFailureReason } from "./auth-profile-failure-policy.js"; import type { PreparedEmbeddedRunInput } from "./execution-context.js"; @@ -29,6 +34,7 @@ export function createEmbeddedRunFailoverRetryController(input: { getLastProfileId: () => string | undefined; getSessionId: () => string; harnessOwnsTransport: () => boolean; + getApiKeyInfo: () => ResolvedProviderAuth | null; }) { const { runParams: params, @@ -109,15 +115,42 @@ export function createEmbeddedRunFailoverRetryController(input: { return; } const { profileId, reason } = failure; - if (!profileId || !reason) { + if (!reason) { return; } if (input.harnessOwnsTransport() && reason === "timeout") { return; } - await markAuthProfileFailure({ + if (profileId) { + await markAuthProfileFailure({ + store: profileFailureStore, + profileId, + reason, + cfg: params.config, + agentDir, + runId: params.runId, + modelId: failure.modelId, + }); + return; + } + // Inline provider API keys have no auth profile, so record their + // billing/auth failures under the provider-scoped inline cooldown so the + // resolver stops handing back the exhausted key on the next turn. + const apiKeyInfo = input.getApiKeyInfo(); + if ( + apiKeyInfo?.mode !== "api-key" || + !isConfigBackedInlineProviderApiKey({ + cfg: params.config, + provider, + source: apiKeyInfo.source, + store: profileFailureStore, + }) + ) { + return; + } + await markInlineProviderApiKeyFailure({ store: profileFailureStore, - profileId, + provider, reason, cfg: params.config, agentDir, diff --git a/src/agents/embedded-agent-runner/run/prompt-failure.ts b/src/agents/embedded-agent-runner/run/prompt-failure.ts index e969b9489ae4..961b1fbb5efa 100644 --- a/src/agents/embedded-agent-runner/run/prompt-failure.ts +++ b/src/agents/embedded-agent-runner/run/prompt-failure.ts @@ -177,7 +177,7 @@ export async function handleEmbeddedPromptFailure(input: { profileRotated: false, }); if (failoverDecision.action === "rotate_profile" && (await input.advanceAttemptAuthProfile())) { - if (failedProfileId && promptProfileFailureReason) { + if (promptProfileFailureReason) { void input .maybeMarkAuthProfileFailure({ profileId: failedProfileId, @@ -223,7 +223,7 @@ export async function handleEmbeddedPromptFailure(input: { profileRotated: true, }); } - if (failedProfileId && promptProfileFailureReason) { + if (promptProfileFailureReason) { try { await input.maybeMarkAuthProfileFailure({ profileId: failedProfileId, diff --git a/src/agents/model-auth-availability.ts b/src/agents/model-auth-availability.ts index e7df215cd3cb..d72b8cafc044 100644 --- a/src/agents/model-auth-availability.ts +++ b/src/agents/model-auth-availability.ts @@ -1,6 +1,7 @@ /** Read-only provider/model auth availability with provider-route selection. */ import { findNormalizedProviderValue, + normalizeProviderId, normalizeProviderIdForAuth, } from "@openclaw/model-catalog-core/provider-id"; import { resolveAgentModelPrimaryValue } from "../config/model-input.js"; @@ -30,7 +31,11 @@ import { } from "./auth-profiles/read-only-availability.js"; import { getRuntimeAuthProfileStoreSnapshot } from "./auth-profiles/runtime-snapshots.js"; import type { AuthProfileCredential, AuthProfileStore } from "./auth-profiles/types.js"; -import { isProfileInCooldown } from "./auth-profiles/usage-state.js"; +import { + isAuthCooldownBypassedForProvider, + isProfileInCooldown, + resolveProfileUnusableUntil, +} from "./auth-profiles/usage-state.js"; import { listProviderEnvAuthLookupKeys, resolveProviderEnvAuthLookupMaps, @@ -503,6 +508,21 @@ export function createModelAuthAvailabilityResolver( if (binding.kind === "profile-incompatible") { return { availability: false, evidence: "profile" }; } + // Config-backed inline provider keys have no auth profile, so a recorded + // billing/auth cooldown must hide them from browse availability the same way + // it blocks their resolution — otherwise a cooled key still looks usable. + // Mirrors resolveInlineProviderApiKeyUnusableUntil, but reads the cooldown + // via usage-state primitives so this hot browse path stays independent of + // the auth-profiles usage module that many callers mock in tests. + const inlineUsageStats = isAuthCooldownBypassedForProvider(provider) + ? undefined + : store.usageStats?.[`inline-api-key:${normalizeProviderId(provider)}`]; + const inlineKeyUnusableUntil = inlineUsageStats + ? resolveProfileUnusableUntil(inlineUsageStats) + : null; + if (inlineKeyUnusableUntil != null && inlineKeyUnusableUntil > now) { + return { availability: false, evidence: "provider-config" }; + } if (binding.kind === "literal") { return { availability: modeAllowed(provider, target, configuredBearerMode), diff --git a/src/agents/model-auth-model.ts b/src/agents/model-auth-model.ts index 0ec4b4737043..30003289291c 100644 --- a/src/agents/model-auth-model.ts +++ b/src/agents/model-auth-model.ts @@ -127,23 +127,6 @@ export async function hasAvailableAuthForProvider(params: { if (authOverride === "aws-sdk") { return true; } - const envAuth = authConfig.resolveConfigAwareEnvApiKey(cfg, provider, params.workspaceDir); - if ( - envAuth && - isAuthModeAllowedForModel({ - provider, - modelApi: params.modelApi, - mode: envAuth.source.includes("OAUTH_TOKEN") ? "oauth" : "api-key", - }) - ) { - return true; - } - if (authConfig.resolveUsableCustomProviderApiKey({ cfg, provider })) { - return true; - } - if (resolveSyntheticLocalProviderAuth({ cfg, provider })) { - return true; - } const store = params.store ?? resolveScopedAuthProfileStore({ @@ -152,6 +135,49 @@ export async function hasAvailableAuthForProvider(params: { provider, preferredProfile, }); + // An inline provider key inside its billing/auth cooldown is not available + // auth: the resolver refuses to hand it back, so reporting it as available + // would strand callers on a credential they cannot use. + const inlineUnusableUntil = authConfig.resolveInlineProviderApiKeyCooldownUntil(store, provider); + const inlineProviderApiKeyUsable = + typeof inlineUnusableUntil !== "number" || inlineUnusableUntil <= Date.now(); + const envAuth = authConfig.resolveConfigAwareEnvApiKey(cfg, provider, params.workspaceDir); + if ( + envAuth && + isAuthModeAllowedForModel({ + provider, + modelApi: params.modelApi, + mode: envAuth.source.includes("OAUTH_TOKEN") ? "oauth" : "api-key", + }) && + (!authConfig.isConfigBackedInlineProviderApiKey({ + cfg, + provider, + source: envAuth.source, + store, + }) || + inlineProviderApiKeyUsable) + ) { + return true; + } + if ( + authConfig.resolveUsableCustomProviderApiKey({ cfg, provider }) && + inlineProviderApiKeyUsable + ) { + return true; + } + const syntheticLocalAuth = resolveSyntheticLocalProviderAuth({ cfg, provider }); + if ( + syntheticLocalAuth && + (!authConfig.isConfigBackedInlineProviderApiKey({ + cfg, + provider, + source: syntheticLocalAuth.source, + store, + }) || + inlineProviderApiKeyUsable) + ) { + return true; + } const order = resolveAuthProfileOrder({ cfg, store, diff --git a/src/agents/model-auth-provider-config.ts b/src/agents/model-auth-provider-config.ts index bfc495999da8..97202c963423 100644 --- a/src/agents/model-auth-provider-config.ts +++ b/src/agents/model-auth-provider-config.ts @@ -21,6 +21,10 @@ import { isStoredCredentialCompatibleWithAuthProvider, } from "./auth-profiles/order.js"; import type { AuthProfileCredential, AuthProfileStore } from "./auth-profiles/types.js"; +import { + isAuthCooldownBypassedForProvider, + resolveProfileUnusableUntil, +} from "./auth-profiles/usage-state.js"; import { resolveEnvApiKey, type EnvApiKeyResult } from "./model-auth-env.js"; import { CUSTOM_LOCAL_AUTH_MARKER, @@ -522,6 +526,67 @@ function hasExplicitProviderApiKeyConfig(providerConfig: ModelProviderConfig): b ); } +function isInlineProviderApiKeySource(source: string): boolean { + return ( + source === "models.json" || + source.endsWith(" (models.json secretref)") || + source.endsWith(" (models.json marker)") + ); +} + +/** True when a resolved credential came from an inline `models.providers..apiKey`. */ +export function isConfigBackedInlineProviderApiKey(params: { + cfg: OpenClawConfig | undefined; + provider: string; + source: string; + store?: AuthProfileStore; +}): boolean { + if (isInlineProviderApiKeySource(params.source)) { + return true; + } + const providerConfig = resolveProviderConfig(params.cfg, params.provider); + if (!providerConfig || !hasExplicitProviderApiKeyConfig(providerConfig)) { + return false; + } + if (coerceSecretRef(providerConfig.apiKey)) { + return true; + } + const perEntryRawKey = normalizeOptionalSecretInput(providerConfig.apiKey); + return Boolean(perEntryRawKey && !params.store?.profiles[perEntryRawKey]); +} + +// Reads the inline provider API-key cooldown via usage-state primitives instead +// of the auth-profiles usage module, so model-auth keeps working in the many +// tests that partially mock that module. Mirrors the usage-module helper of the +// same intent, using the same provider normalization as the write side so the +// `inline-api-key:` usage id matches what the failure marker records. +export function resolveInlineProviderApiKeyCooldownUntil( + store: AuthProfileStore, + provider: string, +): number | null { + if (isAuthCooldownBypassedForProvider(provider)) { + return null; + } + const stats = store.usageStats?.[`inline-api-key:${normalizeProviderId(provider)}`]; + return stats ? resolveProfileUnusableUntil(stats) : null; +} + +/** Fails closed while an inline provider API key is inside its billing/auth cooldown. */ +export function assertInlineProviderApiKeyUsable(params: { + store: AuthProfileStore; + provider: string; +}): void { + const unusableUntil = resolveInlineProviderApiKeyCooldownUntil(params.store, params.provider); + if (typeof unusableUntil !== "number" || unusableUntil <= Date.now()) { + return; + } + const waitMs = Math.max(0, unusableUntil - Date.now()); + const waitMinutes = Math.max(1, Math.ceil(waitMs / 60_000)); + throw new Error( + `Inline API key for provider "${params.provider}" is temporarily disabled after a provider auth/billing failure. Retry after about ${waitMinutes} minute${waitMinutes === 1 ? "" : "s"}, or switch to a different auth profile/API key.`, + ); +} + function isCustomLocalProviderConfig(providerConfig: ModelProviderConfig): boolean { return ( typeof providerConfig.baseUrl === "string" && diff --git a/src/agents/model-auth-provider.ts b/src/agents/model-auth-provider.ts index bbb1519f8a44..7b500e5c5bb5 100644 --- a/src/agents/model-auth-provider.ts +++ b/src/agents/model-auth-provider.ts @@ -229,6 +229,19 @@ export async function resolveApiKeyForProvider(params: { provider, inferredMode: envResolved.source.includes("OAUTH_TOKEN") ? "oauth" : "api-key", }); + if (resolvedMode === "api-key") { + const inlineStore = getScopedStore(); + if ( + authConfig.isConfigBackedInlineProviderApiKey({ + cfg, + provider, + source: envResolved.source, + store: inlineStore, + }) + ) { + authConfig.assertInlineProviderApiKeyUsable({ store: inlineStore, provider }); + } + } if ( !isAuthModeAllowedForModel({ provider, @@ -301,6 +314,11 @@ export async function resolveApiKeyForProvider(params: { secretSentinels: params.secretSentinels, }); if (runtimeCustomKey) { + // Managed (file/exec) SecretRef provider keys are config-backed inline + // credentials too, so they must honor the inline-key cooldown gate just + // like the literal/env paths below — otherwise a 402 cooldown is recorded + // but never enforced for these keys. + authConfig.assertInlineProviderApiKeyUsable({ store: getScopedStore(), provider }); return runtimeCustomKey; } const customKey = authConfig.resolveUsableCustomProviderApiKey({ @@ -309,6 +327,7 @@ export async function resolveApiKeyForProvider(params: { secretSentinels: params.secretSentinels, }); if (customKey) { + authConfig.assertInlineProviderApiKeyUsable({ store: getScopedStore(), provider }); return { apiKey: customKey.apiKey, source: customKey.source, @@ -461,6 +480,19 @@ export async function resolveApiKeyForProvider(params: { provider, inferredMode: envResolved.source.includes("OAUTH_TOKEN") ? "oauth" : "api-key", }); + if (resolvedMode === "api-key") { + const inlineStore = getScopedStore(); + if ( + authConfig.isConfigBackedInlineProviderApiKey({ + cfg, + provider, + source: envResolved.source, + store: inlineStore, + }) + ) { + authConfig.assertInlineProviderApiKeyUsable({ store: inlineStore, provider }); + } + } if ( isAuthModeAllowedForModel({ provider, @@ -496,6 +528,17 @@ export async function resolveApiKeyForProvider(params: { mode: managedRuntimeAuth.mode, }) ) { + const inlineStore = getScopedStore(); + if ( + authConfig.isConfigBackedInlineProviderApiKey({ + cfg, + provider, + source: managedRuntimeAuth.source, + store: inlineStore, + }) + ) { + authConfig.assertInlineProviderApiKeyUsable({ store: inlineStore, provider }); + } return managedRuntimeAuth; } @@ -511,6 +554,7 @@ export async function resolveApiKeyForProvider(params: { inferredMode: "api-key", }); if (isAuthModeAllowedForModel({ provider, modelApi: params.modelApi, mode })) { + authConfig.assertInlineProviderApiKeyUsable({ store: getScopedStore(), provider }); return { apiKey: customKey.apiKey, source: customKey.source, mode }; } } diff --git a/src/agents/model-auth-runtime.ts b/src/agents/model-auth-runtime.ts index 775035af1636..3138043294ab 100644 --- a/src/agents/model-auth-runtime.ts +++ b/src/agents/model-auth-runtime.ts @@ -8,6 +8,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; import { resolveProviderSyntheticAuthWithPlugin } from "../plugins/provider-runtime.js"; import { resolveRuntimeSyntheticAuthProviderRefState } from "../plugins/synthetic-auth.runtime.js"; import { mintSecretSentinel } from "../secrets/sentinel.js"; +import type { AuthProfileStore } from "./auth-profiles.js"; import { resolveProviderEnvAuthLookupMaps } from "./model-auth-env-vars.js"; import { resolveEnvApiKey, type EnvApiKeyLookupOptions } from "./model-auth-env.js"; import { CUSTOM_LOCAL_AUTH_MARKER, isNonSecretApiKeyMarker } from "./model-auth-markers.js"; @@ -140,12 +141,27 @@ export function hasRuntimeAvailableProviderAuth(params: { allowPluginSyntheticAuth?: boolean; runtimeLookup?: RuntimeProviderAuthLookup; modelApi?: string; + store?: AuthProfileStore; }): boolean { const provider = normalizeProviderId(params.provider); const authOverride = authConfig.resolveProviderAuthOverride(params.cfg, provider); if (authOverride === "aws-sdk") { return true; } + + // Callers that supply the auth store get inline provider keys hidden while + // their billing/auth cooldown is active, so browse and tool selection stop + // advertising a credential the resolver would refuse to hand back. + const inlineProviderApiKeyUsable = params.store + ? (() => { + const unusableUntil = authConfig.resolveInlineProviderApiKeyCooldownUntil( + params.store, + provider, + ); + return unusableUntil === null || unusableUntil <= Date.now(); + })() + : true; + const envAuth = resolveEnvApiKey(provider, params.env, { config: params.cfg, workspaceDir: params.workspaceDir, @@ -160,7 +176,14 @@ export function hasRuntimeAvailableProviderAuth(params: { provider, modelApi: params.modelApi, mode: envAuth.source.includes("OAUTH_TOKEN") ? "oauth" : "api-key", - }) + }) && + (!authConfig.isConfigBackedInlineProviderApiKey({ + cfg: params.cfg, + provider, + source: envAuth.source, + store: params.store, + }) || + inlineProviderApiKeyUsable) ) { return true; } @@ -169,11 +192,25 @@ export function hasRuntimeAvailableProviderAuth(params: { cfg: params.cfg, provider, env: params.env, - }) + }) && + inlineProviderApiKeyUsable ) { return true; } - if (resolveManagedSecretRefRuntimeProviderAuth({ cfg: params.cfg, provider })) { + const managedRuntimeAuth = resolveManagedSecretRefRuntimeProviderAuth({ + cfg: params.cfg, + provider, + }); + if ( + managedRuntimeAuth && + (!authConfig.isConfigBackedInlineProviderApiKey({ + cfg: params.cfg, + provider, + source: managedRuntimeAuth.source, + store: params.store, + }) || + inlineProviderApiKeyUsable) + ) { return true; } if (authConfig.hasSyntheticLocalProviderAuthConfig({ cfg: params.cfg, provider })) { diff --git a/src/agents/model-auth.profiles.test.ts b/src/agents/model-auth.profiles.test.ts index 9f1b52002635..b01b9ba4d6d0 100644 --- a/src/agents/model-auth.profiles.test.ts +++ b/src/agents/model-auth.profiles.test.ts @@ -11,13 +11,19 @@ import { clearRuntimeAuthProfileStoreSnapshots, ensureAuthProfileStore, } from "./auth-profiles/store.js"; -import type { OAuthCredential } from "./auth-profiles/types.js"; +import type { + AuthProfileCredential, + AuthProfileStore, + OAuthCredential, +} from "./auth-profiles/types.js"; +import { resolveInlineProviderApiKeyUsageId } from "./auth-profiles/usage.js"; import type { ClaudeCliCredential } from "./cli-credentials.js"; import { createRuntimeProviderAuthLookup, getApiKeyForModel, hasAvailableAuthForProvider, hasRuntimeAvailableProviderAuth, + isConfigBackedInlineProviderApiKey, resolveApiKeyForProvider, resolveEnvApiKey, resolveModelAuthMode, @@ -594,6 +600,51 @@ describe("getApiKeyForModel", () => { ); }); + it("uses the config default agent dir for inline provider cooldown checks", async () => { + await withOpenClawTestState( + { + layout: "state-only", + prefix: "openclaw-inline-cooldown-agent-dir-", + agentEnv: "clear", + }, + async (state) => { + const usageId = resolveInlineProviderApiKeyUsageId("demo-local"); + await state.writeAuthProfiles( + { + version: 1, + profiles: {}, + usageStats: { + [usageId]: { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing", + }, + }, + }, + "configured", + ); + + const cfg: OpenClawConfig = { + ...buildDemoLocalProviderCfg("DEMO_LOCAL_API_KEY"), + agents: { + list: [ + { + id: "configured", + default: true, + agentDir: state.agentDir("configured"), + }, + ], + }, + }; + + await withEnvAsync({ DEMO_LOCAL_API_KEY: "env-demo-key" }, async () => { + await expect(resolveApiKeyForProvider({ provider: "demo-local", cfg })).rejects.toThrow( + /Inline API key for provider "demo-local" is temporarily disabled/, + ); + }); + }, + ); + }); + it("reports the config default agent dir when provider auth is missing", async () => { await withOpenClawTestState( { @@ -963,6 +1014,64 @@ describe("getApiKeyForModel", () => { ).resolves.toBe(false); }); + it("hasAuthForModelProvider respects inline api key cooldown (visibility check)", async () => { + const store = { + version: 1 as const, + profiles: {}, + usageStats: {}, + } as unknown as AuthProfileStore; + const cfg: OpenClawConfig = { + models: { + providers: { + "anthropic-local": { + apiKey: "demo-key", + baseUrl: "http://127.0.0.1:8000/v1", + models: [testModelDefinition("demo-model")], + }, + }, + }, + } as unknown as OpenClawConfig; + + // Initially available + await expect( + hasAuthForModelProvider({ + provider: "anthropic-local", + cfg, + store, + }), + ).resolves.toBe(true); + + // Mark failure to trigger cooldown + const usageId = resolveInlineProviderApiKeyUsageId("anthropic-local"); + store.usageStats![usageId] = { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing", + }; + + // Now unavailable for visibility listing (cooldown) + await expect( + hasAuthForModelProvider({ + provider: "anthropic-local", + cfg, + store, + }), + ).resolves.toBe(false); + + // But still available if there is a healthy stored profile + store.profiles["test-profile"] = { + type: "api_key", + provider: "anthropic-local", + key: "profile-key", + } as unknown as AuthProfileCredential; + await expect( + hasAuthForModelProvider({ + provider: "anthropic-local", + cfg, + store, + }), + ).resolves.toBe(true); + }); + it("hasAvailableAuthForProvider('google') accepts GOOGLE_API_KEY fallback", async () => { await withEnvAsync( { @@ -1125,6 +1234,189 @@ describe("getApiKeyForModel", () => { expect(resolved.profileId).toBeUndefined(); }); + it("blocks explicit configured apiKey while its inline provider cooldown is active", async () => { + const usageId = resolveInlineProviderApiKeyUsageId("demo-local"); + await expect( + resolveApiKeyForProvider({ + provider: "demo-local", + store: { + version: 1, + profiles: {}, + usageStats: { + [usageId]: { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing", + }, + }, + }, + cfg: { + models: { + providers: { + "demo-local": { + baseUrl: "http://localhost:11434", + api: "openai-completions", + apiKey: "config-demo-key", + models: [], + }, + }, + }, + }, + }), + ).rejects.toThrow(/Inline API key for provider "demo-local" is temporarily disabled/); + }); + + it("blocks configured env-marker apiKey while its inline provider cooldown is active", async () => { + const usageId = resolveInlineProviderApiKeyUsageId("inline-cloud"); + await withEnvAsync({ INLINE_CLOUD_API_KEY: "env-cloud-key" }, async () => { + const store = { + version: 1 as const, + profiles: {}, + usageStats: { + [usageId]: { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing" as const, + }, + }, + }; + const cfg: OpenClawConfig = { + models: { + providers: { + "inline-cloud": { + baseUrl: "https://inline-cloud.example", + api: "openai-completions", + apiKey: "INLINE_CLOUD_API_KEY", + models: [], + }, + }, + }, + }; + + await expect( + resolveApiKeyForProvider({ + provider: "inline-cloud", + store, + cfg, + }), + ).rejects.toThrow(/Inline API key for provider "inline-cloud" is temporarily disabled/); + await expect( + hasAvailableAuthForProvider({ provider: "inline-cloud", store, cfg }), + ).resolves.toBe(false); + }); + }); + + it("recognizes managed non-env SecretRef apiKeys as config-backed inline provider keys", () => { + // Regression for the marker/gate asymmetry: file/exec SecretRef provider + // keys resolve to a source label that is not one of the inline source + // markers, so the failure marker used to skip them and their 402 billing + // cooldown was never recorded even though the gate would honor it. + const emptyStore = { version: 1 as const, profiles: {} }; + for (const secretRef of [ + { source: "file", provider: "default", id: "/run/secrets/inline-cloud" }, + { source: "exec", provider: "default", id: "print-inline-cloud-key" }, + ] as const) { + const cfg: OpenClawConfig = { + models: { + providers: { + "inline-cloud": { + baseUrl: "https://inline-cloud.example", + api: "openai-completions", + apiKey: secretRef, + models: [], + }, + }, + }, + }; + expect( + isConfigBackedInlineProviderApiKey({ + cfg, + provider: "inline-cloud", + source: `${secretRef.source}:${secretRef.provider}:${secretRef.id}`, + store: emptyStore, + }), + ).toBe(true); + } + }); + + it("keeps healthy stored profiles available when configured env auth is cooling down", async () => { + const usageId = resolveInlineProviderApiKeyUsageId("inline-cloud"); + await withEnvAsync({ INLINE_CLOUD_API_KEY: "env-cloud-key" }, async () => { + const store = { + version: 1 as const, + profiles: { + "inline-cloud:default": { + type: "api_key" as const, + provider: "inline-cloud" as const, + key: "stored-cloud-key", + }, + }, + usageStats: { + [usageId]: { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing" as const, + }, + }, + }; + const cfg: OpenClawConfig = { + models: { + providers: { + "inline-cloud": { + baseUrl: "https://inline-cloud.example", + api: "openai-completions", + apiKey: "INLINE_CLOUD_API_KEY", + models: [], + }, + }, + }, + }; + + await expect( + hasAvailableAuthForProvider({ provider: "inline-cloud", store, cfg }), + ).resolves.toBe(true); + const resolved = await resolveApiKeyForProvider({ provider: "inline-cloud", store, cfg }); + expect(resolved.apiKey).toBe("stored-cloud-key"); + expect(resolved.source).toBe("profile:inline-cloud:default"); + }); + }); + + it("blocks configured env SecretRef apiKey while its inline provider cooldown is active", async () => { + const usageId = resolveInlineProviderApiKeyUsageId("inline-cloud"); + await withEnvAsync({ INLINE_CLOUD_API_KEY: "env-cloud-key" }, async () => { + const store = { + version: 1 as const, + profiles: {}, + usageStats: { + [usageId]: { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing" as const, + }, + }, + }; + const cfg: OpenClawConfig = { + models: { + providers: { + "inline-cloud": { + baseUrl: "https://inline-cloud.example", + api: "openai-completions", + apiKey: { source: "env", provider: "default", id: "INLINE_CLOUD_API_KEY" }, + models: [], + }, + }, + }, + }; + + await expect( + resolveApiKeyForProvider({ + provider: "inline-cloud", + store, + cfg, + }), + ).rejects.toThrow(/Inline API key for provider "inline-cloud" is temporarily disabled/); + await expect( + hasAvailableAuthForProvider({ provider: "inline-cloud", store, cfg }), + ).resolves.toBe(false); + }); + }); + it("falls back to the stored synthetic local profile when no real auth exists", async () => { const resolved = await resolveDemoLocalApiKey({ envApiKey: undefined, @@ -1752,6 +2044,53 @@ describe("resolveApiKeyForProvider — per-entry apiKey as profile ID reference" }); }); + it("keeps env-first precedence ahead of stale inline cooldown for per-entry profile references", async () => { + const usageId = resolveInlineProviderApiKeyUsageId("openai"); + await withEnvAsync({ OPENAI_API_KEY: "sk-env-first" }, async () => { + const store = { + version: 1 as const, + profiles: { + "openai:key-b": { + type: "api_key" as const, + provider: "openai" as const, + key: "sk-profile-key", + }, + }, + usageStats: { + [usageId]: { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing" as const, + }, + }, + }; + const cfg: OpenClawConfig = { + models: { + providers: { + openai: { + api: "openai-completions", + baseUrl: "https://api.openai.com/v1", + apiKey: "openai:key-b", + models: [], + }, + }, + }, + }; + + const resolved = await resolveApiKeyForProvider({ + provider: "openai", + credentialPrecedence: "env-first", + cfg, + store, + }); + + expect(resolved.apiKey).toBe("sk-env-first"); + expect(resolved.source).toContain("OPENAI_API_KEY"); + await expect(hasAvailableAuthForProvider({ provider: "openai", store, cfg })).resolves.toBe( + true, + ); + }); + }); + it("does not bleed auth.order canonical provider profiles into a per-entry provider", async () => { // auth.order.openrouter should not be selected when resolving openrouter-minimax // that has its own per-entry apiKey = "openrouter:key-b" profile reference. diff --git a/src/agents/model-auth.test.ts b/src/agents/model-auth.test.ts index 6aced3440ccb..61012806df6a 100644 --- a/src/agents/model-auth.test.ts +++ b/src/agents/model-auth.test.ts @@ -1338,6 +1338,66 @@ describe("resolveApiKeyForProvider", () => { }); }); + // Regression: a 402 cooldown recorded under inline-api-key: must be + // enforced for managed (file/exec) SecretRef provider keys too, not just + // literal/env keys — otherwise the cooldown is written but never honored and + // the exhausted provider keeps resolving and reporting available. Covers both + // resolution paths: the synthetic-runtime path and the explicit api-key + // override path. + it.each([ + { name: "no auth override (synthetic-runtime path)", auth: undefined }, + { name: "explicit api-key override path", auth: "api-key" as const }, + ])( + "blocks a managed file SecretRef apiKey while its inline provider cooldown is active — $name", + async ({ auth }) => { + const cliproxyConfig = { + api: "openai-responses" as const, + apiKey: { source: "file", provider: "vault", id: "/cliproxy/api-key" } as const, + baseUrl: "https://cliproxy.example/v1", + models: [], + ...(auth ? { auth } : {}), + }; + const sourceConfig = { models: { providers: { cliproxyapi: cliproxyConfig } } }; + const runtimeConfig = { + models: { + providers: { + cliproxyapi: { + ...cliproxyConfig, + apiKey: "sk-runtime-cliproxy", // pragma: allowlist secret + }, + }, + }, + }; + setRuntimeConfigSnapshot(runtimeConfig, sourceConfig); + + const store = { + version: 1 as const, + profiles: {}, + usageStats: { + "inline-api-key:cliproxyapi": { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing" as const, + }, + }, + }; + + await expect( + resolveApiKeyForProvider({ provider: "cliproxyapi", cfg: sourceConfig, store }), + ).rejects.toThrow(/Inline API key for provider "cliproxyapi" is temporarily disabled/); + await expect( + hasAvailableAuthForProvider({ provider: "cliproxyapi", cfg: sourceConfig, store }), + ).resolves.toBe(false); + expect( + hasRuntimeAvailableProviderAuth({ + provider: "cliproxyapi", + cfg: sourceConfig, + allowPluginSyntheticAuth: false, + store, + }), + ).toBe(false); + }, + ); + it("does not treat a custom provider managed SecretRef marker as auth without a runtime snapshot", async () => { const sourceConfig = { models: { diff --git a/src/agents/model-auth.ts b/src/agents/model-auth.ts index 7f5248c6b32d..cdb1cc4e3f1a 100644 --- a/src/agents/model-auth.ts +++ b/src/agents/model-auth.ts @@ -22,6 +22,7 @@ export { getCustomProviderApiKey, hasSyntheticLocalProviderAuthConfig, hasUsableCustomProviderApiKey, + isConfigBackedInlineProviderApiKey, resolveProviderEntryApiKeyBinding, resolveProviderEntryApiKeyProfileReference, resolveUsableCustomProviderApiKey, diff --git a/src/agents/model-provider-auth.test.ts b/src/agents/model-provider-auth.test.ts index c182373ca83e..e4710d5614ba 100644 --- a/src/agents/model-provider-auth.test.ts +++ b/src/agents/model-provider-auth.test.ts @@ -669,6 +669,7 @@ describe("prepared provider auth state", () => { provider: "openai", }, }, + usageStats: {}, }, }, ], diff --git a/src/agents/model-provider-auth.ts b/src/agents/model-provider-auth.ts index e22e4215a366..7952be41db79 100644 --- a/src/agents/model-provider-auth.ts +++ b/src/agents/model-provider-auth.ts @@ -181,19 +181,6 @@ export async function hasAuthForModelProvider(params: { await new Promise((resolve) => { setImmediate(resolve); }); - if ( - hasRuntimeAvailableProviderAuth({ - provider, - cfg: params.cfg, - workspaceDir: params.workspaceDir, - env: params.env, - allowPluginSyntheticAuth: params.allowPluginSyntheticAuth, - runtimeLookup: params.runtimeAuthLookup ?? params.resolveRuntimeAuthLookup?.(), - modelApi: params.modelApi, - }) - ) { - return true; - } const slowPathAgentDir = params.agentDir ?? (params.agentId && params.cfg @@ -208,6 +195,21 @@ export async function hasAuthForModelProvider(params: { : ensureAuthProfileStore(slowPathAgentDir, { externalCli: externalCliDiscoveryForProviderAuth({ cfg: params.cfg, provider }), })); + + if ( + hasRuntimeAvailableProviderAuth({ + provider, + cfg: params.cfg, + workspaceDir: params.workspaceDir, + env: params.env, + allowPluginSyntheticAuth: params.allowPluginSyntheticAuth, + runtimeLookup: params.runtimeAuthLookup ?? params.resolveRuntimeAuthLookup?.(), + modelApi: params.modelApi, + store, + }) + ) { + return true; + } if (listProfilesForProvider(store, provider).length > 0) { return params.modelApi === undefined ? true @@ -535,9 +537,18 @@ function createProviderAuthWarmPresenceStore(store: AuthProfileStore): AuthProfi provider: credential.provider, }; } + const usageStats: AuthProfileStore["usageStats"] = {}; + if (store.usageStats) { + for (const [id, stats] of Object.entries(store.usageStats)) { + if (id.startsWith("inline-api-key:")) { + usageStats[id] = stats; + } + } + } return { version: store.version, profiles, + usageStats, }; } diff --git a/src/agents/model-provider-auth.worker.test.ts b/src/agents/model-provider-auth.worker.test.ts index b4f8c16eb192..7eceaf0cd0c9 100644 --- a/src/agents/model-provider-auth.worker.test.ts +++ b/src/agents/model-provider-auth.worker.test.ts @@ -5,7 +5,10 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { withEnvAsync } from "../test-utils/env.js"; -import { clearRuntimeAuthProfileStoreSnapshots } from "./auth-profiles.js"; +import { + clearRuntimeAuthProfileStoreSnapshots, + resolveInlineProviderApiKeyUsageId, +} from "./auth-profiles.js"; import { clearCurrentProviderAuthState } from "./model-provider-auth.js"; import { runProviderAuthWarmWorkerInput } from "./model-provider-auth.worker.js"; @@ -91,4 +94,59 @@ describe("provider auth warm worker", () => { }, ); }, 30_000); + + it("respects cooled-down inline api keys in the worker warm input", async () => { + const root = mkdtempSync(path.join(tmpdir(), "openclaw-provider-auth-worker-cooldown-")); + tempDirs.push(root); + + await withEnvAsync( + { + OPENCLAW_DISABLE_PERSISTED_PLUGIN_REGISTRY: "1", + OPENCLAW_STATE_DIR: path.join(root, "state"), + }, + async () => { + const agentDir = path.join(root, "agent"); + const cfg = { + agents: { list: [{ id: "main", agentDir }] }, + models: { + providers: { + "cooled-down": { + apiKey: "some-key", + baseUrl: "https://example.com/v1", + api: "openai", + models: [{ id: "some-model", name: "Some Model" }], + }, + }, + }, + } as unknown as OpenClawConfig; + + const usageId = resolveInlineProviderApiKeyUsageId("cooled-down"); + const result = await runProviderAuthWarmWorkerInput({ + cfg, + runtimeAuthStores: [ + { + agentDir, + store: { + version: 1, + profiles: {}, + usageStats: { + [usageId]: { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing", + }, + }, + }, + }, + ], + }); + + expect(result.status).toBe("ok"); + if (result.status !== "ok") { + return; + } + // Should NOT contain the provider because the inline key is in cooldown + expect(result.snapshot.agents[0]?.providers).not.toContainEqual(["cooled-down", true]); + }, + ); + }, 30_000); }); diff --git a/src/agents/tools/model-config.helpers.test.ts b/src/agents/tools/model-config.helpers.test.ts index 0fa13b14aad7..a1b8e8374d1b 100644 --- a/src/agents/tools/model-config.helpers.test.ts +++ b/src/agents/tools/model-config.helpers.test.ts @@ -204,6 +204,83 @@ describe("hasProviderAuthForTool", () => { ).toBe(false); expect(authMocks.resolveEnvApiKey).toHaveBeenCalledTimes(1); }); + + it("hides inline provider keys during billing cooldown, keeping profile fallback", () => { + // Regression: hasProviderAuthForTool used to call the runtime availability + // check without the auth store, so inline provider keys in billing cooldown + // were still advertised as usable tool auth. + const cfg = { + models: { + providers: { + hatchery: { + baseUrl: "https://example.com/v1", + apiKey: "sk-configured", // pragma: allowlist secret + models: [], + }, + }, + }, + } as OpenClawConfig; + const cooldownStats = (disabledUntil: number) => ({ + "inline-api-key:hatchery": { disabledUntil, disabledReason: "billing" as const }, + }); + + expect( + hasProviderAuthForTool({ + provider: "hatchery", + cfg, + authStore: { version: 1, profiles: {}, usageStats: cooldownStats(Date.now() + 60_000) }, + }), + ).toBe(false); + expect( + hasProviderAuthForTool({ + provider: "hatchery", + cfg, + authStore: { version: 1, profiles: {}, usageStats: cooldownStats(Date.now() - 60_000) }, + }), + ).toBe(true); + expect( + hasProviderAuthForTool({ + provider: "hatchery", + cfg, + authStore: { + version: 1, + profiles: { "hatchery:default": apiKey("hatchery", "sk-profile") }, + usageStats: cooldownStats(Date.now() + 60_000), + }, + }), + ).toBe(true); + }); + + it("hides inline provider keys during billing cooldown from direct API-key tool auth", () => { + const cfg = { + models: { + providers: { + hatchery: { + baseUrl: "https://example.com/v1", + apiKey: "sk-configured", // pragma: allowlist secret + models: [], + }, + }, + }, + } as OpenClawConfig; + + expect( + hasDirectProviderApiKeyAuthForTool({ + provider: "hatchery", + cfg, + authStore: { + version: 1, + profiles: {}, + usageStats: { + "inline-api-key:hatchery": { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing" as const, + }, + }, + }, + }), + ).toBe(false); + }); }); describe("resolveOpenAiImageMediaCandidate", () => { diff --git a/src/agents/tools/model-config.helpers.ts b/src/agents/tools/model-config.helpers.ts index 413c7bdc0fe8..275ad1537495 100644 --- a/src/agents/tools/model-config.helpers.ts +++ b/src/agents/tools/model-config.helpers.ts @@ -143,6 +143,14 @@ export function hasProviderAuthForTool(params: { workspaceDir: params.workspaceDir, allowPluginSyntheticAuth: false, runtimeLookup: params.runtimeLookup, + // Without the store, inline provider keys in billing cooldown would + // still be advertised as available to model-backed tools. + store: loadAuthStoreForProvider({ + provider: params.provider, + cfg: params.cfg, + agentDir: params.agentDir, + authStore: params.authStore, + }), }) ) { return true; @@ -260,6 +268,14 @@ function hasDirectProviderApiKeyAuthForTool(params: { workspaceDir: params.workspaceDir, modelApi: params.modelApi, allowPluginSyntheticAuth: false, + // Without the store, inline provider keys in billing cooldown would + // still be advertised as direct API-key auth for tools. + store: loadAuthStoreForProvider({ + provider: params.provider, + cfg: params.cfg, + agentDir: params.agentDir, + authStore: params.authStore, + }), }) ) { return true; diff --git a/src/gateway/server-methods/models.test.ts b/src/gateway/server-methods/models.test.ts index e958c23f367d..920034d21692 100644 --- a/src/gateway/server-methods/models.test.ts +++ b/src/gateway/server-methods/models.test.ts @@ -1274,6 +1274,83 @@ describe("models.list", () => { ); }); + it("hides inline provider keys during billing cooldown from model browsing", async () => { + // Regression: the models.list availability checker loaded the auth store + // for profile checks but did not pass it to the runtime availability check, + // so inline provider keys in billing cooldown stayed browseable. + await withOpenClawTestState( + { + layout: "state-only", + prefix: "openclaw-models-list-inline-cooldown-", + agentEnv: "main", + }, + async (state) => { + const runtimeConfig = { + models: { + providers: { + cliproxyapi: { + api: "openai-responses", + baseUrl: "https://cliproxy.example/v1", + apiKey: "sk-inline-cooldown", // pragma: allowlist secret + models: [], + }, + }, + }, + } as unknown as OpenClawConfig; + const catalog = [{ id: "qwen-remote", name: "Qwen Remote", provider: "cliproxyapi" }]; + const writeCooldown = (disabledUntil: number) => + state.writeAuthProfiles({ + version: 1, + profiles: {}, + usageStats: { + "inline-api-key:cliproxyapi": { + disabledUntil, + disabledReason: "billing", + }, + }, + }); + + await writeCooldown(Date.now() + 60_000); + const cooled = requestModelsList({ + view: "all", + runtimeConfig, + loadGatewayModelCatalog: vi.fn(() => Promise.resolve(catalog)), + reqId: "req-models-list-inline-cooldown-active", + }); + await cooled.request; + expect(cooled.respond).toHaveBeenCalledWith( + true, + { + models: [ + { id: "qwen-remote", name: "Qwen Remote", provider: "cliproxyapi", available: false }, + ], + }, + undefined, + ); + + // Expired cooldown proves the store reaches the runtime check instead + // of the row being unavailable for an unrelated reason. + await writeCooldown(Date.now() - 60_000); + const recovered = requestModelsList({ + view: "all", + runtimeConfig, + loadGatewayModelCatalog: vi.fn(() => Promise.resolve(catalog)), + reqId: "req-models-list-inline-cooldown-expired", + }); + await recovered.request; + expect(recovered.respond).toHaveBeenCalledWith( + true, + { + models: [ + { id: "qwen-remote", name: "Qwen Remote", provider: "cliproxyapi", available: true }, + ], + }, + undefined, + ); + }, + ); + }); + it("uses an exact hydrated runtime profile SecretRef as read-only proof", async () => { await withOpenClawTestState( {