diff --git a/src/agents/auth-profiles.markauthprofilefailure.test.ts b/src/agents/auth-profiles.markauthprofilefailure.test.ts index b6d81e2baa09..2a2b264527df 100644 --- a/src/agents/auth-profiles.markauthprofilefailure.test.ts +++ b/src/agents/auth-profiles.markauthprofilefailure.test.ts @@ -27,6 +27,8 @@ import { import { calculateAuthProfileCooldownMs, markAuthProfileFailure, + markInlineProviderApiKeyFailure, + resolveInlineProviderApiKeyUsageId, setAuthProfileFailureHook, } from "./auth-profiles/usage.js"; @@ -168,6 +170,26 @@ describe("markAuthProfileFailure", () => { expectCooldownInRange(remainingMs, 4.5 * 60 * 60 * 1000, 5.5 * 60 * 60 * 1000); }); }); + it("records billing backoff for inline provider api keys without creating an auth profile", async () => { + await withAuthProfileStore(async ({ agentDir, store }) => { + const startedAt = Date.now(); + await markInlineProviderApiKeyFailure({ + store, + provider: "anthropic", + reason: "billing", + agentDir, + }); + + const usageId = resolveInlineProviderApiKeyUsageId("anthropic"); + const stats = store.usageStats?.[usageId]; + expect(store.profiles[usageId]).toBeUndefined(); + expect(stats?.disabledReason).toBe("billing"); + expect(typeof stats?.disabledUntil).toBe("number"); + const remainingMs = (stats?.disabledUntil as number) - startedAt; + expectCooldownInRange(remainingMs, 4.5 * 60 * 60 * 1000, 5.5 * 60 * 60 * 1000); + }); + }); + it("keeps persisted cooldownUntil unchanged across mid-window retries", async () => { await withAuthProfileStore(async ({ agentDir, store }) => { await markAuthProfileFailure({ @@ -376,6 +398,24 @@ describe("markAuthProfileFailure", () => { }); }); + it("fires the auth profile failure hook for inline provider api key failures", async () => { + await withAuthProfileStore(async ({ agentDir, store }) => { + const hook = vi.fn(); + setAuthProfileFailureHook(hook); + try { + await markInlineProviderApiKeyFailure({ + store, + provider: "anthropic", + reason: "billing", + agentDir, + }); + expect(hook).toHaveBeenCalledTimes(1); + } finally { + setAuthProfileFailureHook(undefined); + } + }); + }); + it("does not break failure recording when the hook throws", async () => { await withAuthProfileStore(async ({ agentDir, store }) => { const throwingHook = vi.fn(() => { diff --git a/src/agents/auth-profiles.ts b/src/agents/auth-profiles.ts index 4c2ff501b914..e9240d42ef31 100644 --- a/src/agents/auth-profiles.ts +++ b/src/agents/auth-profiles.ts @@ -97,6 +97,9 @@ export { markAuthProfileCooldown, markAuthProfileBlockedUntil, markAuthProfileFailure, + markInlineProviderApiKeyFailure, + resolveInlineProviderApiKeyUnusableUntil, + resolveInlineProviderApiKeyUsageId, resolveProfilesUnavailableReason, resolveProfileUnusableUntilForDisplay, setAuthProfileFailureHook, diff --git a/src/agents/auth-profiles/persisted.ts b/src/agents/auth-profiles/persisted.ts index 80b38b0964ad..6054251aaba8 100644 --- a/src/agents/auth-profiles/persisted.ts +++ b/src/agents/auth-profiles/persisted.ts @@ -44,6 +44,14 @@ type CredentialRejectReason = "non_object" | "invalid_type" | "missing_provider" type RejectedCredentialEntry = { key: string; reason: CredentialRejectReason }; const AUTH_PROFILE_TYPES = new Set(["api_key", "oauth", "token"]); +const INLINE_API_KEY_USAGE_ID_PREFIX = "inline-api-key:"; + +function isRetainedUsageStatsId( + profileId: string, + profiles: AuthProfileStore["profiles"], +): boolean { + return Boolean(profiles[profileId]) || profileId.startsWith(INLINE_API_KEY_USAGE_ID_PREFIX); +} // Persisted credential normalization accepts old field names and SecretRef-ish // values, then emits the current credential discriminated union. @@ -652,7 +660,9 @@ export function mergeAuthProfileStores( const mergedUsageStats = mergeRecord(base.usageStats, override.usageStats); const usageStats = mergedUsageStats ? Object.fromEntries( - Object.entries(mergedUsageStats).filter(([profileId]) => profiles[profileId]), + Object.entries(mergedUsageStats).filter(([profileId]) => + isRetainedUsageStatsId(profileId, profiles), + ), ) : undefined; const merged = { diff --git a/src/agents/auth-profiles/store.ts b/src/agents/auth-profiles/store.ts index 03eb7338956d..550d2bd5378b 100644 --- a/src/agents/auth-profiles/store.ts +++ b/src/agents/auth-profiles/store.ts @@ -539,7 +539,9 @@ function pruneAuthProfileStoreReferences( : undefined; store.usageStats = store.usageStats ? Object.fromEntries( - Object.entries(store.usageStats).filter(([profileId]) => keptProfileIds.has(profileId)), + Object.entries(store.usageStats).filter( + ([profileId]) => keptProfileIds.has(profileId) || profileId.startsWith("inline-api-key:"), + ), ) : undefined; store.runtimePersistedProfileIds = store.runtimePersistedProfileIds diff --git a/src/agents/auth-profiles/usage.ts b/src/agents/auth-profiles/usage.ts index dbb9f06bcaf9..1155a0cb06ec 100644 --- a/src/agents/auth-profiles/usage.ts +++ b/src/agents/auth-profiles/usage.ts @@ -73,6 +73,12 @@ function logDroppedAuthProfileBookkeeping(kind: string, profileId: string): void }); } +const INLINE_API_KEY_USAGE_ID_PREFIX = "inline-api-key:"; + +export function resolveInlineProviderApiKeyUsageId(provider: string): string { + return `${INLINE_API_KEY_USAGE_ID_PREFIX}${normalizeProviderId(provider)}`; +} + const FAILURE_REASON_PRIORITY: AuthProfileFailureReason[] = [ "auth_permanent", "auth", @@ -761,6 +767,20 @@ export function resolveProfileUnusableUntilForDisplay( return resolveProfileUnusableUntil(stats); } +export function resolveInlineProviderApiKeyUnusableUntil( + store: AuthProfileStore, + provider: string, +): number | null { + if (isAuthCooldownBypassedForProvider(provider)) { + return null; + } + const stats = store.usageStats?.[resolveInlineProviderApiKeyUsageId(provider)]; + if (!stats) { + return null; + } + return resolveProfileUnusableUntil(stats); +} + function resetUsageStats( existing: ProfileUsageStats | undefined, overrides?: Partial, @@ -1120,6 +1140,64 @@ export async function markAuthProfileBlockedUntil(params: { } } +export async function markInlineProviderApiKeyFailure(params: { + store: AuthProfileStore; + provider: string; + reason: AuthProfileFailureReason; + cfg?: OpenClawConfig; + agentDir?: string; + runId?: string; + modelId?: string; +}): Promise { + const { store, provider, reason, agentDir, runId, modelId } = params; + if (isAuthCooldownBypassedForProvider(provider)) { + return; + } + + const usageId = resolveInlineProviderApiKeyUsageId(provider); + const cfgResolved = resolveAuthCooldownConfig(); + + let nextStats: ProfileUsageStats | undefined; + let previousStats: ProfileUsageStats | undefined; + let updateTime = 0; + const updated = await authProfileUsageDeps.updateAuthProfileStoreWithLock({ + agentDir, + updater: (freshStore) => { + const now = Date.now(); + previousStats = freshStore.usageStats?.[usageId]; + updateTime = now; + nextStats = computeNextProfileUsageStats({ + existing: previousStats ?? {}, + now, + reason, + cfgResolved, + modelId, + }); + updateUsageStatsEntry(freshStore, usageId, () => nextStats as ProfileUsageStats); + return true; + }, + }); + if (updated) { + store.usageStats = updated.usageStats; + if (nextStats) { + logAuthProfileFailureStateChange({ + runId, + profileId: usageId, + provider, + reason, + previous: previousStats, + next: nextStats, + now: updateTime, + }); + } + notifyAuthProfileFailureHook(); + return; + } + if (updated === null) { + logDroppedAuthProfileBookkeeping("inline_api_key_failure", usageId); + } +} + /** * Mark a profile as transiently failed. Applies stepped backoff cooldown. * Cooldown times: 30s, 1min, 5min (capped). diff --git a/src/agents/embedded-agent-runner.run-embedded-agent.auth-profile-rotation.e2e.test.ts b/src/agents/embedded-agent-runner.run-embedded-agent.auth-profile-rotation.e2e.test.ts index 15a8259d05a3..47117da9f6d7 100644 --- a/src/agents/embedded-agent-runner.run-embedded-agent.auth-profile-rotation.e2e.test.ts +++ b/src/agents/embedded-agent-runner.run-embedded-agent.auth-profile-rotation.e2e.test.ts @@ -6,7 +6,10 @@ import type { AssistantMessage } from "openclaw/plugin-sdk/llm"; import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; import { redactIdentifier } from "../logging/redact-identifier.js"; -import type { AuthProfileFailureReason } from "./auth-profiles.js"; +import { + resolveInlineProviderApiKeyUsageId, + type AuthProfileFailureReason, +} from "./auth-profiles.js"; import { ensureAuthProfileStore, saveAuthProfileStore } from "./auth-profiles/store.js"; import type { EmbeddedRunAttemptResult } from "./embedded-agent-runner/run/types.js"; import { @@ -934,6 +937,48 @@ describe("runEmbeddedAgent auth profile rotation", () => { expect(sleepWithAbortMock).not.toHaveBeenCalled(); }); + it("marks inline provider api key billing prompt failures without an auth profile", async () => { + await withAgentWorkspace(async ({ agentDir, workspaceDir }) => { + await fs.writeFile( + path.join(agentDir, "auth-profiles.json"), + JSON.stringify({ version: 1, profiles: {} }), + ); + await fs.writeFile( + path.join(agentDir, "auth-state.json"), + JSON.stringify({ version: 1, usageStats: {} }), + ); + runEmbeddedAttemptMock.mockResolvedValueOnce( + makeAttempt({ + terminal: { kind: "failed", source: "prompt", error: new Error("insufficient credits") }, + }), + ); + + await expect( + runEmbeddedAgentInline({ + sessionId: "session:test", + sessionKey: "agent:test:inline-api-key-prompt-billing", + sessionFile: path.join(workspaceDir, "session.jsonl"), + workspaceDir, + agentDir, + config: makeConfig(), + prompt: "hello", + provider: "openai", + model: "mock-1", + authProfileIdSource: "auto", + timeoutMs: 5_000, + runId: "run:inline-api-key-prompt-billing", + }), + ).rejects.toThrow(/insufficient credits/); + + expect(runEmbeddedAttemptMock).toHaveBeenCalledTimes(1); + const usageStats = await readUsageStats(agentDir); + const usageId = resolveInlineProviderApiKeyUsageId("openai"); + expect(usageStats[usageId]?.disabledReason).toBe("billing"); + expect(typeof usageStats[usageId]?.disabledUntil).toBe("number"); + expect(usageStats["openai:p1"]).toBeUndefined(); + }); + }); + it("rotates on timeout without cooling down the timed-out profile", async () => { const { usageStats } = await runAutoPinnedRotationCase({ errorMessage: "request ended without sending any chunks", diff --git a/src/agents/embedded-agent-runner/run-loop.ts b/src/agents/embedded-agent-runner/run-loop.ts index 9d161824b699..fa824ae4fc35 100644 --- a/src/agents/embedded-agent-runner/run-loop.ts +++ b/src/agents/embedded-agent-runner/run-loop.ts @@ -262,6 +262,7 @@ export async function runPreparedEmbeddedLoop( getLastProfileId: () => preparedRuntime.snapshot().lastProfileId, getSessionId: () => sessionPromptState.sessionId, harnessOwnsTransport: () => preparedRuntime.snapshot().pluginHarnessOwnsTransport, + getApiKeyInfo, }); // Resolve the context engine once and reuse across retries to avoid // repeated initialization/connection overhead per attempt. diff --git a/src/agents/embedded-agent-runner/run/assistant-failover.test.ts b/src/agents/embedded-agent-runner/run/assistant-failover.test.ts index 4c2a3b9831e2..6897db0df0be 100644 --- a/src/agents/embedded-agent-runner/run/assistant-failover.test.ts +++ b/src/agents/embedded-agent-runner/run/assistant-failover.test.ts @@ -497,6 +497,28 @@ describe("handleAssistantFailover", () => { expect(warn).not.toHaveBeenCalled(); }); + it("marks inline auth failures even when no profile id is active", async () => { + const maybeMarkAuthProfileFailure = vi.fn(async () => {}); + + const outcome = await handleAssistantFailover( + makeParams({ + initialDecision: { action: "rotate_profile", reason: "billing" }, + failoverReason: "billing", + assistantProfileFailureReason: "billing", + lastProfileId: undefined, + advanceAuthProfile: vi.fn(async () => false), + maybeMarkAuthProfileFailure, + }), + ); + + expect(outcome.action).toBe("throw"); + expect(maybeMarkAuthProfileFailure).toHaveBeenCalledWith({ + profileId: undefined, + reason: "billing", + modelId: "claude-haiku-4-5-20251001", + }); + }); + it("marks provider-started timeout rotations against the failed profile", async () => { const maybeMarkAuthProfileFailure = vi.fn(async () => {}); diff --git a/src/agents/embedded-agent-runner/run/assistant-failover.ts b/src/agents/embedded-agent-runner/run/assistant-failover.ts index d178a68a20cf..7741ed25bb04 100644 --- a/src/agents/embedded-agent-runner/run/assistant-failover.ts +++ b/src/agents/embedded-agent-runner/run/assistant-failover.ts @@ -146,7 +146,7 @@ export async function handleAssistantFailover(params: { const timeoutFailure = terminal.timedOut; const failureReason = params.assistantProfileFailureReason; const markFailedProfile = async () => { - if (!failedProfileId || !failureReason) { + if (!failureReason) { return; } try { diff --git a/src/agents/embedded-agent-runner/run/failover-retry-controller.ts b/src/agents/embedded-agent-runner/run/failover-retry-controller.ts index 1f54c277b268..768e5e806c71 100644 --- a/src/agents/embedded-agent-runner/run/failover-retry-controller.ts +++ b/src/agents/embedded-agent-runner/run/failover-retry-controller.ts @@ -1,8 +1,13 @@ import { sanitizeForLog } from "../../../../packages/terminal-core/src/ansi.js"; import { sleepWithAbort } from "../../../infra/backoff.js"; -import { type AuthProfileFailureReason, markAuthProfileFailure } from "../../auth-profiles.js"; +import { + type AuthProfileFailureReason, + markAuthProfileFailure, + markInlineProviderApiKeyFailure, +} from "../../auth-profiles.js"; import type { FailoverReason } from "../../embedded-agent-helpers.js"; import { FailoverError, resolveFailoverStatus } from "../../failover-error.js"; +import { isConfigBackedInlineProviderApiKey, type ResolvedProviderAuth } from "../../model-auth.js"; import { log } from "../logger.js"; import { resolveAuthProfileFailureReason } from "./auth-profile-failure-policy.js"; import type { PreparedEmbeddedRunInput } from "./execution-context.js"; @@ -29,6 +34,7 @@ export function createEmbeddedRunFailoverRetryController(input: { getLastProfileId: () => string | undefined; getSessionId: () => string; harnessOwnsTransport: () => boolean; + getApiKeyInfo: () => ResolvedProviderAuth | null; }) { const { runParams: params, @@ -109,15 +115,42 @@ export function createEmbeddedRunFailoverRetryController(input: { return; } const { profileId, reason } = failure; - if (!profileId || !reason) { + if (!reason) { return; } if (input.harnessOwnsTransport() && reason === "timeout") { return; } - await markAuthProfileFailure({ + if (profileId) { + await markAuthProfileFailure({ + store: profileFailureStore, + profileId, + reason, + cfg: params.config, + agentDir, + runId: params.runId, + modelId: failure.modelId, + }); + return; + } + // Inline provider API keys have no auth profile, so record their + // billing/auth failures under the provider-scoped inline cooldown so the + // resolver stops handing back the exhausted key on the next turn. + const apiKeyInfo = input.getApiKeyInfo(); + if ( + apiKeyInfo?.mode !== "api-key" || + !isConfigBackedInlineProviderApiKey({ + cfg: params.config, + provider, + source: apiKeyInfo.source, + store: profileFailureStore, + }) + ) { + return; + } + await markInlineProviderApiKeyFailure({ store: profileFailureStore, - profileId, + provider, reason, cfg: params.config, agentDir, diff --git a/src/agents/embedded-agent-runner/run/prompt-failure.ts b/src/agents/embedded-agent-runner/run/prompt-failure.ts index e969b9489ae4..961b1fbb5efa 100644 --- a/src/agents/embedded-agent-runner/run/prompt-failure.ts +++ b/src/agents/embedded-agent-runner/run/prompt-failure.ts @@ -177,7 +177,7 @@ export async function handleEmbeddedPromptFailure(input: { profileRotated: false, }); if (failoverDecision.action === "rotate_profile" && (await input.advanceAttemptAuthProfile())) { - if (failedProfileId && promptProfileFailureReason) { + if (promptProfileFailureReason) { void input .maybeMarkAuthProfileFailure({ profileId: failedProfileId, @@ -223,7 +223,7 @@ export async function handleEmbeddedPromptFailure(input: { profileRotated: true, }); } - if (failedProfileId && promptProfileFailureReason) { + if (promptProfileFailureReason) { try { await input.maybeMarkAuthProfileFailure({ profileId: failedProfileId, diff --git a/src/agents/model-auth-availability.ts b/src/agents/model-auth-availability.ts index e7df215cd3cb..d72b8cafc044 100644 --- a/src/agents/model-auth-availability.ts +++ b/src/agents/model-auth-availability.ts @@ -1,6 +1,7 @@ /** Read-only provider/model auth availability with provider-route selection. */ import { findNormalizedProviderValue, + normalizeProviderId, normalizeProviderIdForAuth, } from "@openclaw/model-catalog-core/provider-id"; import { resolveAgentModelPrimaryValue } from "../config/model-input.js"; @@ -30,7 +31,11 @@ import { } from "./auth-profiles/read-only-availability.js"; import { getRuntimeAuthProfileStoreSnapshot } from "./auth-profiles/runtime-snapshots.js"; import type { AuthProfileCredential, AuthProfileStore } from "./auth-profiles/types.js"; -import { isProfileInCooldown } from "./auth-profiles/usage-state.js"; +import { + isAuthCooldownBypassedForProvider, + isProfileInCooldown, + resolveProfileUnusableUntil, +} from "./auth-profiles/usage-state.js"; import { listProviderEnvAuthLookupKeys, resolveProviderEnvAuthLookupMaps, @@ -503,6 +508,21 @@ export function createModelAuthAvailabilityResolver( if (binding.kind === "profile-incompatible") { return { availability: false, evidence: "profile" }; } + // Config-backed inline provider keys have no auth profile, so a recorded + // billing/auth cooldown must hide them from browse availability the same way + // it blocks their resolution — otherwise a cooled key still looks usable. + // Mirrors resolveInlineProviderApiKeyUnusableUntil, but reads the cooldown + // via usage-state primitives so this hot browse path stays independent of + // the auth-profiles usage module that many callers mock in tests. + const inlineUsageStats = isAuthCooldownBypassedForProvider(provider) + ? undefined + : store.usageStats?.[`inline-api-key:${normalizeProviderId(provider)}`]; + const inlineKeyUnusableUntil = inlineUsageStats + ? resolveProfileUnusableUntil(inlineUsageStats) + : null; + if (inlineKeyUnusableUntil != null && inlineKeyUnusableUntil > now) { + return { availability: false, evidence: "provider-config" }; + } if (binding.kind === "literal") { return { availability: modeAllowed(provider, target, configuredBearerMode), diff --git a/src/agents/model-auth-model.ts b/src/agents/model-auth-model.ts index 0ec4b4737043..30003289291c 100644 --- a/src/agents/model-auth-model.ts +++ b/src/agents/model-auth-model.ts @@ -127,23 +127,6 @@ export async function hasAvailableAuthForProvider(params: { if (authOverride === "aws-sdk") { return true; } - const envAuth = authConfig.resolveConfigAwareEnvApiKey(cfg, provider, params.workspaceDir); - if ( - envAuth && - isAuthModeAllowedForModel({ - provider, - modelApi: params.modelApi, - mode: envAuth.source.includes("OAUTH_TOKEN") ? "oauth" : "api-key", - }) - ) { - return true; - } - if (authConfig.resolveUsableCustomProviderApiKey({ cfg, provider })) { - return true; - } - if (resolveSyntheticLocalProviderAuth({ cfg, provider })) { - return true; - } const store = params.store ?? resolveScopedAuthProfileStore({ @@ -152,6 +135,49 @@ export async function hasAvailableAuthForProvider(params: { provider, preferredProfile, }); + // An inline provider key inside its billing/auth cooldown is not available + // auth: the resolver refuses to hand it back, so reporting it as available + // would strand callers on a credential they cannot use. + const inlineUnusableUntil = authConfig.resolveInlineProviderApiKeyCooldownUntil(store, provider); + const inlineProviderApiKeyUsable = + typeof inlineUnusableUntil !== "number" || inlineUnusableUntil <= Date.now(); + const envAuth = authConfig.resolveConfigAwareEnvApiKey(cfg, provider, params.workspaceDir); + if ( + envAuth && + isAuthModeAllowedForModel({ + provider, + modelApi: params.modelApi, + mode: envAuth.source.includes("OAUTH_TOKEN") ? "oauth" : "api-key", + }) && + (!authConfig.isConfigBackedInlineProviderApiKey({ + cfg, + provider, + source: envAuth.source, + store, + }) || + inlineProviderApiKeyUsable) + ) { + return true; + } + if ( + authConfig.resolveUsableCustomProviderApiKey({ cfg, provider }) && + inlineProviderApiKeyUsable + ) { + return true; + } + const syntheticLocalAuth = resolveSyntheticLocalProviderAuth({ cfg, provider }); + if ( + syntheticLocalAuth && + (!authConfig.isConfigBackedInlineProviderApiKey({ + cfg, + provider, + source: syntheticLocalAuth.source, + store, + }) || + inlineProviderApiKeyUsable) + ) { + return true; + } const order = resolveAuthProfileOrder({ cfg, store, diff --git a/src/agents/model-auth-provider-config.ts b/src/agents/model-auth-provider-config.ts index bfc495999da8..97202c963423 100644 --- a/src/agents/model-auth-provider-config.ts +++ b/src/agents/model-auth-provider-config.ts @@ -21,6 +21,10 @@ import { isStoredCredentialCompatibleWithAuthProvider, } from "./auth-profiles/order.js"; import type { AuthProfileCredential, AuthProfileStore } from "./auth-profiles/types.js"; +import { + isAuthCooldownBypassedForProvider, + resolveProfileUnusableUntil, +} from "./auth-profiles/usage-state.js"; import { resolveEnvApiKey, type EnvApiKeyResult } from "./model-auth-env.js"; import { CUSTOM_LOCAL_AUTH_MARKER, @@ -522,6 +526,67 @@ function hasExplicitProviderApiKeyConfig(providerConfig: ModelProviderConfig): b ); } +function isInlineProviderApiKeySource(source: string): boolean { + return ( + source === "models.json" || + source.endsWith(" (models.json secretref)") || + source.endsWith(" (models.json marker)") + ); +} + +/** True when a resolved credential came from an inline `models.providers..apiKey`. */ +export function isConfigBackedInlineProviderApiKey(params: { + cfg: OpenClawConfig | undefined; + provider: string; + source: string; + store?: AuthProfileStore; +}): boolean { + if (isInlineProviderApiKeySource(params.source)) { + return true; + } + const providerConfig = resolveProviderConfig(params.cfg, params.provider); + if (!providerConfig || !hasExplicitProviderApiKeyConfig(providerConfig)) { + return false; + } + if (coerceSecretRef(providerConfig.apiKey)) { + return true; + } + const perEntryRawKey = normalizeOptionalSecretInput(providerConfig.apiKey); + return Boolean(perEntryRawKey && !params.store?.profiles[perEntryRawKey]); +} + +// Reads the inline provider API-key cooldown via usage-state primitives instead +// of the auth-profiles usage module, so model-auth keeps working in the many +// tests that partially mock that module. Mirrors the usage-module helper of the +// same intent, using the same provider normalization as the write side so the +// `inline-api-key:` usage id matches what the failure marker records. +export function resolveInlineProviderApiKeyCooldownUntil( + store: AuthProfileStore, + provider: string, +): number | null { + if (isAuthCooldownBypassedForProvider(provider)) { + return null; + } + const stats = store.usageStats?.[`inline-api-key:${normalizeProviderId(provider)}`]; + return stats ? resolveProfileUnusableUntil(stats) : null; +} + +/** Fails closed while an inline provider API key is inside its billing/auth cooldown. */ +export function assertInlineProviderApiKeyUsable(params: { + store: AuthProfileStore; + provider: string; +}): void { + const unusableUntil = resolveInlineProviderApiKeyCooldownUntil(params.store, params.provider); + if (typeof unusableUntil !== "number" || unusableUntil <= Date.now()) { + return; + } + const waitMs = Math.max(0, unusableUntil - Date.now()); + const waitMinutes = Math.max(1, Math.ceil(waitMs / 60_000)); + throw new Error( + `Inline API key for provider "${params.provider}" is temporarily disabled after a provider auth/billing failure. Retry after about ${waitMinutes} minute${waitMinutes === 1 ? "" : "s"}, or switch to a different auth profile/API key.`, + ); +} + function isCustomLocalProviderConfig(providerConfig: ModelProviderConfig): boolean { return ( typeof providerConfig.baseUrl === "string" && diff --git a/src/agents/model-auth-provider.ts b/src/agents/model-auth-provider.ts index bbb1519f8a44..7b500e5c5bb5 100644 --- a/src/agents/model-auth-provider.ts +++ b/src/agents/model-auth-provider.ts @@ -229,6 +229,19 @@ export async function resolveApiKeyForProvider(params: { provider, inferredMode: envResolved.source.includes("OAUTH_TOKEN") ? "oauth" : "api-key", }); + if (resolvedMode === "api-key") { + const inlineStore = getScopedStore(); + if ( + authConfig.isConfigBackedInlineProviderApiKey({ + cfg, + provider, + source: envResolved.source, + store: inlineStore, + }) + ) { + authConfig.assertInlineProviderApiKeyUsable({ store: inlineStore, provider }); + } + } if ( !isAuthModeAllowedForModel({ provider, @@ -301,6 +314,11 @@ export async function resolveApiKeyForProvider(params: { secretSentinels: params.secretSentinels, }); if (runtimeCustomKey) { + // Managed (file/exec) SecretRef provider keys are config-backed inline + // credentials too, so they must honor the inline-key cooldown gate just + // like the literal/env paths below — otherwise a 402 cooldown is recorded + // but never enforced for these keys. + authConfig.assertInlineProviderApiKeyUsable({ store: getScopedStore(), provider }); return runtimeCustomKey; } const customKey = authConfig.resolveUsableCustomProviderApiKey({ @@ -309,6 +327,7 @@ export async function resolveApiKeyForProvider(params: { secretSentinels: params.secretSentinels, }); if (customKey) { + authConfig.assertInlineProviderApiKeyUsable({ store: getScopedStore(), provider }); return { apiKey: customKey.apiKey, source: customKey.source, @@ -461,6 +480,19 @@ export async function resolveApiKeyForProvider(params: { provider, inferredMode: envResolved.source.includes("OAUTH_TOKEN") ? "oauth" : "api-key", }); + if (resolvedMode === "api-key") { + const inlineStore = getScopedStore(); + if ( + authConfig.isConfigBackedInlineProviderApiKey({ + cfg, + provider, + source: envResolved.source, + store: inlineStore, + }) + ) { + authConfig.assertInlineProviderApiKeyUsable({ store: inlineStore, provider }); + } + } if ( isAuthModeAllowedForModel({ provider, @@ -496,6 +528,17 @@ export async function resolveApiKeyForProvider(params: { mode: managedRuntimeAuth.mode, }) ) { + const inlineStore = getScopedStore(); + if ( + authConfig.isConfigBackedInlineProviderApiKey({ + cfg, + provider, + source: managedRuntimeAuth.source, + store: inlineStore, + }) + ) { + authConfig.assertInlineProviderApiKeyUsable({ store: inlineStore, provider }); + } return managedRuntimeAuth; } @@ -511,6 +554,7 @@ export async function resolveApiKeyForProvider(params: { inferredMode: "api-key", }); if (isAuthModeAllowedForModel({ provider, modelApi: params.modelApi, mode })) { + authConfig.assertInlineProviderApiKeyUsable({ store: getScopedStore(), provider }); return { apiKey: customKey.apiKey, source: customKey.source, mode }; } } diff --git a/src/agents/model-auth-runtime.ts b/src/agents/model-auth-runtime.ts index 775035af1636..3138043294ab 100644 --- a/src/agents/model-auth-runtime.ts +++ b/src/agents/model-auth-runtime.ts @@ -8,6 +8,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; import { resolveProviderSyntheticAuthWithPlugin } from "../plugins/provider-runtime.js"; import { resolveRuntimeSyntheticAuthProviderRefState } from "../plugins/synthetic-auth.runtime.js"; import { mintSecretSentinel } from "../secrets/sentinel.js"; +import type { AuthProfileStore } from "./auth-profiles.js"; import { resolveProviderEnvAuthLookupMaps } from "./model-auth-env-vars.js"; import { resolveEnvApiKey, type EnvApiKeyLookupOptions } from "./model-auth-env.js"; import { CUSTOM_LOCAL_AUTH_MARKER, isNonSecretApiKeyMarker } from "./model-auth-markers.js"; @@ -140,12 +141,27 @@ export function hasRuntimeAvailableProviderAuth(params: { allowPluginSyntheticAuth?: boolean; runtimeLookup?: RuntimeProviderAuthLookup; modelApi?: string; + store?: AuthProfileStore; }): boolean { const provider = normalizeProviderId(params.provider); const authOverride = authConfig.resolveProviderAuthOverride(params.cfg, provider); if (authOverride === "aws-sdk") { return true; } + + // Callers that supply the auth store get inline provider keys hidden while + // their billing/auth cooldown is active, so browse and tool selection stop + // advertising a credential the resolver would refuse to hand back. + const inlineProviderApiKeyUsable = params.store + ? (() => { + const unusableUntil = authConfig.resolveInlineProviderApiKeyCooldownUntil( + params.store, + provider, + ); + return unusableUntil === null || unusableUntil <= Date.now(); + })() + : true; + const envAuth = resolveEnvApiKey(provider, params.env, { config: params.cfg, workspaceDir: params.workspaceDir, @@ -160,7 +176,14 @@ export function hasRuntimeAvailableProviderAuth(params: { provider, modelApi: params.modelApi, mode: envAuth.source.includes("OAUTH_TOKEN") ? "oauth" : "api-key", - }) + }) && + (!authConfig.isConfigBackedInlineProviderApiKey({ + cfg: params.cfg, + provider, + source: envAuth.source, + store: params.store, + }) || + inlineProviderApiKeyUsable) ) { return true; } @@ -169,11 +192,25 @@ export function hasRuntimeAvailableProviderAuth(params: { cfg: params.cfg, provider, env: params.env, - }) + }) && + inlineProviderApiKeyUsable ) { return true; } - if (resolveManagedSecretRefRuntimeProviderAuth({ cfg: params.cfg, provider })) { + const managedRuntimeAuth = resolveManagedSecretRefRuntimeProviderAuth({ + cfg: params.cfg, + provider, + }); + if ( + managedRuntimeAuth && + (!authConfig.isConfigBackedInlineProviderApiKey({ + cfg: params.cfg, + provider, + source: managedRuntimeAuth.source, + store: params.store, + }) || + inlineProviderApiKeyUsable) + ) { return true; } if (authConfig.hasSyntheticLocalProviderAuthConfig({ cfg: params.cfg, provider })) { diff --git a/src/agents/model-auth.profiles.test.ts b/src/agents/model-auth.profiles.test.ts index 9f1b52002635..b01b9ba4d6d0 100644 --- a/src/agents/model-auth.profiles.test.ts +++ b/src/agents/model-auth.profiles.test.ts @@ -11,13 +11,19 @@ import { clearRuntimeAuthProfileStoreSnapshots, ensureAuthProfileStore, } from "./auth-profiles/store.js"; -import type { OAuthCredential } from "./auth-profiles/types.js"; +import type { + AuthProfileCredential, + AuthProfileStore, + OAuthCredential, +} from "./auth-profiles/types.js"; +import { resolveInlineProviderApiKeyUsageId } from "./auth-profiles/usage.js"; import type { ClaudeCliCredential } from "./cli-credentials.js"; import { createRuntimeProviderAuthLookup, getApiKeyForModel, hasAvailableAuthForProvider, hasRuntimeAvailableProviderAuth, + isConfigBackedInlineProviderApiKey, resolveApiKeyForProvider, resolveEnvApiKey, resolveModelAuthMode, @@ -594,6 +600,51 @@ describe("getApiKeyForModel", () => { ); }); + it("uses the config default agent dir for inline provider cooldown checks", async () => { + await withOpenClawTestState( + { + layout: "state-only", + prefix: "openclaw-inline-cooldown-agent-dir-", + agentEnv: "clear", + }, + async (state) => { + const usageId = resolveInlineProviderApiKeyUsageId("demo-local"); + await state.writeAuthProfiles( + { + version: 1, + profiles: {}, + usageStats: { + [usageId]: { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing", + }, + }, + }, + "configured", + ); + + const cfg: OpenClawConfig = { + ...buildDemoLocalProviderCfg("DEMO_LOCAL_API_KEY"), + agents: { + list: [ + { + id: "configured", + default: true, + agentDir: state.agentDir("configured"), + }, + ], + }, + }; + + await withEnvAsync({ DEMO_LOCAL_API_KEY: "env-demo-key" }, async () => { + await expect(resolveApiKeyForProvider({ provider: "demo-local", cfg })).rejects.toThrow( + /Inline API key for provider "demo-local" is temporarily disabled/, + ); + }); + }, + ); + }); + it("reports the config default agent dir when provider auth is missing", async () => { await withOpenClawTestState( { @@ -963,6 +1014,64 @@ describe("getApiKeyForModel", () => { ).resolves.toBe(false); }); + it("hasAuthForModelProvider respects inline api key cooldown (visibility check)", async () => { + const store = { + version: 1 as const, + profiles: {}, + usageStats: {}, + } as unknown as AuthProfileStore; + const cfg: OpenClawConfig = { + models: { + providers: { + "anthropic-local": { + apiKey: "demo-key", + baseUrl: "http://127.0.0.1:8000/v1", + models: [testModelDefinition("demo-model")], + }, + }, + }, + } as unknown as OpenClawConfig; + + // Initially available + await expect( + hasAuthForModelProvider({ + provider: "anthropic-local", + cfg, + store, + }), + ).resolves.toBe(true); + + // Mark failure to trigger cooldown + const usageId = resolveInlineProviderApiKeyUsageId("anthropic-local"); + store.usageStats![usageId] = { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing", + }; + + // Now unavailable for visibility listing (cooldown) + await expect( + hasAuthForModelProvider({ + provider: "anthropic-local", + cfg, + store, + }), + ).resolves.toBe(false); + + // But still available if there is a healthy stored profile + store.profiles["test-profile"] = { + type: "api_key", + provider: "anthropic-local", + key: "profile-key", + } as unknown as AuthProfileCredential; + await expect( + hasAuthForModelProvider({ + provider: "anthropic-local", + cfg, + store, + }), + ).resolves.toBe(true); + }); + it("hasAvailableAuthForProvider('google') accepts GOOGLE_API_KEY fallback", async () => { await withEnvAsync( { @@ -1125,6 +1234,189 @@ describe("getApiKeyForModel", () => { expect(resolved.profileId).toBeUndefined(); }); + it("blocks explicit configured apiKey while its inline provider cooldown is active", async () => { + const usageId = resolveInlineProviderApiKeyUsageId("demo-local"); + await expect( + resolveApiKeyForProvider({ + provider: "demo-local", + store: { + version: 1, + profiles: {}, + usageStats: { + [usageId]: { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing", + }, + }, + }, + cfg: { + models: { + providers: { + "demo-local": { + baseUrl: "http://localhost:11434", + api: "openai-completions", + apiKey: "config-demo-key", + models: [], + }, + }, + }, + }, + }), + ).rejects.toThrow(/Inline API key for provider "demo-local" is temporarily disabled/); + }); + + it("blocks configured env-marker apiKey while its inline provider cooldown is active", async () => { + const usageId = resolveInlineProviderApiKeyUsageId("inline-cloud"); + await withEnvAsync({ INLINE_CLOUD_API_KEY: "env-cloud-key" }, async () => { + const store = { + version: 1 as const, + profiles: {}, + usageStats: { + [usageId]: { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing" as const, + }, + }, + }; + const cfg: OpenClawConfig = { + models: { + providers: { + "inline-cloud": { + baseUrl: "https://inline-cloud.example", + api: "openai-completions", + apiKey: "INLINE_CLOUD_API_KEY", + models: [], + }, + }, + }, + }; + + await expect( + resolveApiKeyForProvider({ + provider: "inline-cloud", + store, + cfg, + }), + ).rejects.toThrow(/Inline API key for provider "inline-cloud" is temporarily disabled/); + await expect( + hasAvailableAuthForProvider({ provider: "inline-cloud", store, cfg }), + ).resolves.toBe(false); + }); + }); + + it("recognizes managed non-env SecretRef apiKeys as config-backed inline provider keys", () => { + // Regression for the marker/gate asymmetry: file/exec SecretRef provider + // keys resolve to a source label that is not one of the inline source + // markers, so the failure marker used to skip them and their 402 billing + // cooldown was never recorded even though the gate would honor it. + const emptyStore = { version: 1 as const, profiles: {} }; + for (const secretRef of [ + { source: "file", provider: "default", id: "/run/secrets/inline-cloud" }, + { source: "exec", provider: "default", id: "print-inline-cloud-key" }, + ] as const) { + const cfg: OpenClawConfig = { + models: { + providers: { + "inline-cloud": { + baseUrl: "https://inline-cloud.example", + api: "openai-completions", + apiKey: secretRef, + models: [], + }, + }, + }, + }; + expect( + isConfigBackedInlineProviderApiKey({ + cfg, + provider: "inline-cloud", + source: `${secretRef.source}:${secretRef.provider}:${secretRef.id}`, + store: emptyStore, + }), + ).toBe(true); + } + }); + + it("keeps healthy stored profiles available when configured env auth is cooling down", async () => { + const usageId = resolveInlineProviderApiKeyUsageId("inline-cloud"); + await withEnvAsync({ INLINE_CLOUD_API_KEY: "env-cloud-key" }, async () => { + const store = { + version: 1 as const, + profiles: { + "inline-cloud:default": { + type: "api_key" as const, + provider: "inline-cloud" as const, + key: "stored-cloud-key", + }, + }, + usageStats: { + [usageId]: { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing" as const, + }, + }, + }; + const cfg: OpenClawConfig = { + models: { + providers: { + "inline-cloud": { + baseUrl: "https://inline-cloud.example", + api: "openai-completions", + apiKey: "INLINE_CLOUD_API_KEY", + models: [], + }, + }, + }, + }; + + await expect( + hasAvailableAuthForProvider({ provider: "inline-cloud", store, cfg }), + ).resolves.toBe(true); + const resolved = await resolveApiKeyForProvider({ provider: "inline-cloud", store, cfg }); + expect(resolved.apiKey).toBe("stored-cloud-key"); + expect(resolved.source).toBe("profile:inline-cloud:default"); + }); + }); + + it("blocks configured env SecretRef apiKey while its inline provider cooldown is active", async () => { + const usageId = resolveInlineProviderApiKeyUsageId("inline-cloud"); + await withEnvAsync({ INLINE_CLOUD_API_KEY: "env-cloud-key" }, async () => { + const store = { + version: 1 as const, + profiles: {}, + usageStats: { + [usageId]: { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing" as const, + }, + }, + }; + const cfg: OpenClawConfig = { + models: { + providers: { + "inline-cloud": { + baseUrl: "https://inline-cloud.example", + api: "openai-completions", + apiKey: { source: "env", provider: "default", id: "INLINE_CLOUD_API_KEY" }, + models: [], + }, + }, + }, + }; + + await expect( + resolveApiKeyForProvider({ + provider: "inline-cloud", + store, + cfg, + }), + ).rejects.toThrow(/Inline API key for provider "inline-cloud" is temporarily disabled/); + await expect( + hasAvailableAuthForProvider({ provider: "inline-cloud", store, cfg }), + ).resolves.toBe(false); + }); + }); + it("falls back to the stored synthetic local profile when no real auth exists", async () => { const resolved = await resolveDemoLocalApiKey({ envApiKey: undefined, @@ -1752,6 +2044,53 @@ describe("resolveApiKeyForProvider — per-entry apiKey as profile ID reference" }); }); + it("keeps env-first precedence ahead of stale inline cooldown for per-entry profile references", async () => { + const usageId = resolveInlineProviderApiKeyUsageId("openai"); + await withEnvAsync({ OPENAI_API_KEY: "sk-env-first" }, async () => { + const store = { + version: 1 as const, + profiles: { + "openai:key-b": { + type: "api_key" as const, + provider: "openai" as const, + key: "sk-profile-key", + }, + }, + usageStats: { + [usageId]: { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing" as const, + }, + }, + }; + const cfg: OpenClawConfig = { + models: { + providers: { + openai: { + api: "openai-completions", + baseUrl: "https://api.openai.com/v1", + apiKey: "openai:key-b", + models: [], + }, + }, + }, + }; + + const resolved = await resolveApiKeyForProvider({ + provider: "openai", + credentialPrecedence: "env-first", + cfg, + store, + }); + + expect(resolved.apiKey).toBe("sk-env-first"); + expect(resolved.source).toContain("OPENAI_API_KEY"); + await expect(hasAvailableAuthForProvider({ provider: "openai", store, cfg })).resolves.toBe( + true, + ); + }); + }); + it("does not bleed auth.order canonical provider profiles into a per-entry provider", async () => { // auth.order.openrouter should not be selected when resolving openrouter-minimax // that has its own per-entry apiKey = "openrouter:key-b" profile reference. diff --git a/src/agents/model-auth.test.ts b/src/agents/model-auth.test.ts index 6aced3440ccb..61012806df6a 100644 --- a/src/agents/model-auth.test.ts +++ b/src/agents/model-auth.test.ts @@ -1338,6 +1338,66 @@ describe("resolveApiKeyForProvider", () => { }); }); + // Regression: a 402 cooldown recorded under inline-api-key: must be + // enforced for managed (file/exec) SecretRef provider keys too, not just + // literal/env keys — otherwise the cooldown is written but never honored and + // the exhausted provider keeps resolving and reporting available. Covers both + // resolution paths: the synthetic-runtime path and the explicit api-key + // override path. + it.each([ + { name: "no auth override (synthetic-runtime path)", auth: undefined }, + { name: "explicit api-key override path", auth: "api-key" as const }, + ])( + "blocks a managed file SecretRef apiKey while its inline provider cooldown is active — $name", + async ({ auth }) => { + const cliproxyConfig = { + api: "openai-responses" as const, + apiKey: { source: "file", provider: "vault", id: "/cliproxy/api-key" } as const, + baseUrl: "https://cliproxy.example/v1", + models: [], + ...(auth ? { auth } : {}), + }; + const sourceConfig = { models: { providers: { cliproxyapi: cliproxyConfig } } }; + const runtimeConfig = { + models: { + providers: { + cliproxyapi: { + ...cliproxyConfig, + apiKey: "sk-runtime-cliproxy", // pragma: allowlist secret + }, + }, + }, + }; + setRuntimeConfigSnapshot(runtimeConfig, sourceConfig); + + const store = { + version: 1 as const, + profiles: {}, + usageStats: { + "inline-api-key:cliproxyapi": { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing" as const, + }, + }, + }; + + await expect( + resolveApiKeyForProvider({ provider: "cliproxyapi", cfg: sourceConfig, store }), + ).rejects.toThrow(/Inline API key for provider "cliproxyapi" is temporarily disabled/); + await expect( + hasAvailableAuthForProvider({ provider: "cliproxyapi", cfg: sourceConfig, store }), + ).resolves.toBe(false); + expect( + hasRuntimeAvailableProviderAuth({ + provider: "cliproxyapi", + cfg: sourceConfig, + allowPluginSyntheticAuth: false, + store, + }), + ).toBe(false); + }, + ); + it("does not treat a custom provider managed SecretRef marker as auth without a runtime snapshot", async () => { const sourceConfig = { models: { diff --git a/src/agents/model-auth.ts b/src/agents/model-auth.ts index 7f5248c6b32d..cdb1cc4e3f1a 100644 --- a/src/agents/model-auth.ts +++ b/src/agents/model-auth.ts @@ -22,6 +22,7 @@ export { getCustomProviderApiKey, hasSyntheticLocalProviderAuthConfig, hasUsableCustomProviderApiKey, + isConfigBackedInlineProviderApiKey, resolveProviderEntryApiKeyBinding, resolveProviderEntryApiKeyProfileReference, resolveUsableCustomProviderApiKey, diff --git a/src/agents/model-provider-auth.test.ts b/src/agents/model-provider-auth.test.ts index c182373ca83e..e4710d5614ba 100644 --- a/src/agents/model-provider-auth.test.ts +++ b/src/agents/model-provider-auth.test.ts @@ -669,6 +669,7 @@ describe("prepared provider auth state", () => { provider: "openai", }, }, + usageStats: {}, }, }, ], diff --git a/src/agents/model-provider-auth.ts b/src/agents/model-provider-auth.ts index e22e4215a366..7952be41db79 100644 --- a/src/agents/model-provider-auth.ts +++ b/src/agents/model-provider-auth.ts @@ -181,19 +181,6 @@ export async function hasAuthForModelProvider(params: { await new Promise((resolve) => { setImmediate(resolve); }); - if ( - hasRuntimeAvailableProviderAuth({ - provider, - cfg: params.cfg, - workspaceDir: params.workspaceDir, - env: params.env, - allowPluginSyntheticAuth: params.allowPluginSyntheticAuth, - runtimeLookup: params.runtimeAuthLookup ?? params.resolveRuntimeAuthLookup?.(), - modelApi: params.modelApi, - }) - ) { - return true; - } const slowPathAgentDir = params.agentDir ?? (params.agentId && params.cfg @@ -208,6 +195,21 @@ export async function hasAuthForModelProvider(params: { : ensureAuthProfileStore(slowPathAgentDir, { externalCli: externalCliDiscoveryForProviderAuth({ cfg: params.cfg, provider }), })); + + if ( + hasRuntimeAvailableProviderAuth({ + provider, + cfg: params.cfg, + workspaceDir: params.workspaceDir, + env: params.env, + allowPluginSyntheticAuth: params.allowPluginSyntheticAuth, + runtimeLookup: params.runtimeAuthLookup ?? params.resolveRuntimeAuthLookup?.(), + modelApi: params.modelApi, + store, + }) + ) { + return true; + } if (listProfilesForProvider(store, provider).length > 0) { return params.modelApi === undefined ? true @@ -535,9 +537,18 @@ function createProviderAuthWarmPresenceStore(store: AuthProfileStore): AuthProfi provider: credential.provider, }; } + const usageStats: AuthProfileStore["usageStats"] = {}; + if (store.usageStats) { + for (const [id, stats] of Object.entries(store.usageStats)) { + if (id.startsWith("inline-api-key:")) { + usageStats[id] = stats; + } + } + } return { version: store.version, profiles, + usageStats, }; } diff --git a/src/agents/model-provider-auth.worker.test.ts b/src/agents/model-provider-auth.worker.test.ts index b4f8c16eb192..7eceaf0cd0c9 100644 --- a/src/agents/model-provider-auth.worker.test.ts +++ b/src/agents/model-provider-auth.worker.test.ts @@ -5,7 +5,10 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { withEnvAsync } from "../test-utils/env.js"; -import { clearRuntimeAuthProfileStoreSnapshots } from "./auth-profiles.js"; +import { + clearRuntimeAuthProfileStoreSnapshots, + resolveInlineProviderApiKeyUsageId, +} from "./auth-profiles.js"; import { clearCurrentProviderAuthState } from "./model-provider-auth.js"; import { runProviderAuthWarmWorkerInput } from "./model-provider-auth.worker.js"; @@ -91,4 +94,59 @@ describe("provider auth warm worker", () => { }, ); }, 30_000); + + it("respects cooled-down inline api keys in the worker warm input", async () => { + const root = mkdtempSync(path.join(tmpdir(), "openclaw-provider-auth-worker-cooldown-")); + tempDirs.push(root); + + await withEnvAsync( + { + OPENCLAW_DISABLE_PERSISTED_PLUGIN_REGISTRY: "1", + OPENCLAW_STATE_DIR: path.join(root, "state"), + }, + async () => { + const agentDir = path.join(root, "agent"); + const cfg = { + agents: { list: [{ id: "main", agentDir }] }, + models: { + providers: { + "cooled-down": { + apiKey: "some-key", + baseUrl: "https://example.com/v1", + api: "openai", + models: [{ id: "some-model", name: "Some Model" }], + }, + }, + }, + } as unknown as OpenClawConfig; + + const usageId = resolveInlineProviderApiKeyUsageId("cooled-down"); + const result = await runProviderAuthWarmWorkerInput({ + cfg, + runtimeAuthStores: [ + { + agentDir, + store: { + version: 1, + profiles: {}, + usageStats: { + [usageId]: { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing", + }, + }, + }, + }, + ], + }); + + expect(result.status).toBe("ok"); + if (result.status !== "ok") { + return; + } + // Should NOT contain the provider because the inline key is in cooldown + expect(result.snapshot.agents[0]?.providers).not.toContainEqual(["cooled-down", true]); + }, + ); + }, 30_000); }); diff --git a/src/agents/tools/model-config.helpers.test.ts b/src/agents/tools/model-config.helpers.test.ts index 0fa13b14aad7..a1b8e8374d1b 100644 --- a/src/agents/tools/model-config.helpers.test.ts +++ b/src/agents/tools/model-config.helpers.test.ts @@ -204,6 +204,83 @@ describe("hasProviderAuthForTool", () => { ).toBe(false); expect(authMocks.resolveEnvApiKey).toHaveBeenCalledTimes(1); }); + + it("hides inline provider keys during billing cooldown, keeping profile fallback", () => { + // Regression: hasProviderAuthForTool used to call the runtime availability + // check without the auth store, so inline provider keys in billing cooldown + // were still advertised as usable tool auth. + const cfg = { + models: { + providers: { + hatchery: { + baseUrl: "https://example.com/v1", + apiKey: "sk-configured", // pragma: allowlist secret + models: [], + }, + }, + }, + } as OpenClawConfig; + const cooldownStats = (disabledUntil: number) => ({ + "inline-api-key:hatchery": { disabledUntil, disabledReason: "billing" as const }, + }); + + expect( + hasProviderAuthForTool({ + provider: "hatchery", + cfg, + authStore: { version: 1, profiles: {}, usageStats: cooldownStats(Date.now() + 60_000) }, + }), + ).toBe(false); + expect( + hasProviderAuthForTool({ + provider: "hatchery", + cfg, + authStore: { version: 1, profiles: {}, usageStats: cooldownStats(Date.now() - 60_000) }, + }), + ).toBe(true); + expect( + hasProviderAuthForTool({ + provider: "hatchery", + cfg, + authStore: { + version: 1, + profiles: { "hatchery:default": apiKey("hatchery", "sk-profile") }, + usageStats: cooldownStats(Date.now() + 60_000), + }, + }), + ).toBe(true); + }); + + it("hides inline provider keys during billing cooldown from direct API-key tool auth", () => { + const cfg = { + models: { + providers: { + hatchery: { + baseUrl: "https://example.com/v1", + apiKey: "sk-configured", // pragma: allowlist secret + models: [], + }, + }, + }, + } as OpenClawConfig; + + expect( + hasDirectProviderApiKeyAuthForTool({ + provider: "hatchery", + cfg, + authStore: { + version: 1, + profiles: {}, + usageStats: { + "inline-api-key:hatchery": { + disabledUntil: Date.now() + 60_000, + disabledReason: "billing" as const, + }, + }, + }, + }), + ).toBe(false); + }); }); describe("resolveOpenAiImageMediaCandidate", () => { diff --git a/src/agents/tools/model-config.helpers.ts b/src/agents/tools/model-config.helpers.ts index 413c7bdc0fe8..275ad1537495 100644 --- a/src/agents/tools/model-config.helpers.ts +++ b/src/agents/tools/model-config.helpers.ts @@ -143,6 +143,14 @@ export function hasProviderAuthForTool(params: { workspaceDir: params.workspaceDir, allowPluginSyntheticAuth: false, runtimeLookup: params.runtimeLookup, + // Without the store, inline provider keys in billing cooldown would + // still be advertised as available to model-backed tools. + store: loadAuthStoreForProvider({ + provider: params.provider, + cfg: params.cfg, + agentDir: params.agentDir, + authStore: params.authStore, + }), }) ) { return true; @@ -260,6 +268,14 @@ function hasDirectProviderApiKeyAuthForTool(params: { workspaceDir: params.workspaceDir, modelApi: params.modelApi, allowPluginSyntheticAuth: false, + // Without the store, inline provider keys in billing cooldown would + // still be advertised as direct API-key auth for tools. + store: loadAuthStoreForProvider({ + provider: params.provider, + cfg: params.cfg, + agentDir: params.agentDir, + authStore: params.authStore, + }), }) ) { return true; diff --git a/src/gateway/server-methods/models.test.ts b/src/gateway/server-methods/models.test.ts index e958c23f367d..920034d21692 100644 --- a/src/gateway/server-methods/models.test.ts +++ b/src/gateway/server-methods/models.test.ts @@ -1274,6 +1274,83 @@ describe("models.list", () => { ); }); + it("hides inline provider keys during billing cooldown from model browsing", async () => { + // Regression: the models.list availability checker loaded the auth store + // for profile checks but did not pass it to the runtime availability check, + // so inline provider keys in billing cooldown stayed browseable. + await withOpenClawTestState( + { + layout: "state-only", + prefix: "openclaw-models-list-inline-cooldown-", + agentEnv: "main", + }, + async (state) => { + const runtimeConfig = { + models: { + providers: { + cliproxyapi: { + api: "openai-responses", + baseUrl: "https://cliproxy.example/v1", + apiKey: "sk-inline-cooldown", // pragma: allowlist secret + models: [], + }, + }, + }, + } as unknown as OpenClawConfig; + const catalog = [{ id: "qwen-remote", name: "Qwen Remote", provider: "cliproxyapi" }]; + const writeCooldown = (disabledUntil: number) => + state.writeAuthProfiles({ + version: 1, + profiles: {}, + usageStats: { + "inline-api-key:cliproxyapi": { + disabledUntil, + disabledReason: "billing", + }, + }, + }); + + await writeCooldown(Date.now() + 60_000); + const cooled = requestModelsList({ + view: "all", + runtimeConfig, + loadGatewayModelCatalog: vi.fn(() => Promise.resolve(catalog)), + reqId: "req-models-list-inline-cooldown-active", + }); + await cooled.request; + expect(cooled.respond).toHaveBeenCalledWith( + true, + { + models: [ + { id: "qwen-remote", name: "Qwen Remote", provider: "cliproxyapi", available: false }, + ], + }, + undefined, + ); + + // Expired cooldown proves the store reaches the runtime check instead + // of the row being unavailable for an unrelated reason. + await writeCooldown(Date.now() - 60_000); + const recovered = requestModelsList({ + view: "all", + runtimeConfig, + loadGatewayModelCatalog: vi.fn(() => Promise.resolve(catalog)), + reqId: "req-models-list-inline-cooldown-expired", + }); + await recovered.request; + expect(recovered.respond).toHaveBeenCalledWith( + true, + { + models: [ + { id: "qwen-remote", name: "Qwen Remote", provider: "cliproxyapi", available: true }, + ], + }, + undefined, + ); + }, + ); + }); + it("uses an exact hydrated runtime profile SecretRef as read-only proof", async () => { await withOpenClawTestState( {