diff --git a/src/agents/embedded-agent-runner/run.ts b/src/agents/embedded-agent-runner/run.ts index 12e6611e61c8..e525a6a0670a 100644 --- a/src/agents/embedded-agent-runner/run.ts +++ b/src/agents/embedded-agent-runner/run.ts @@ -137,7 +137,7 @@ import { type PostCompactionGuardObservation, } from "./post-compaction-loop-guard.js"; import { createEmbeddedRunReplayState, observeReplayMetadata } from "./replay-state.js"; -import { handleAssistantFailover } from "./run/assistant-failover.js"; +import { handleAssistantFailover, isShortWindowRateLimitMessage } from "./run/assistant-failover.js"; import { createEmbeddedRunStageTracker, EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE, @@ -1418,11 +1418,12 @@ async function runEmbeddedAgentInternal( }; const resolveRunAuthProfileFailureReason = ( failoverReason: FailoverReason | null, - opts?: { providerStarted?: boolean }, + opts?: { providerStarted?: boolean; transientRateLimit?: boolean }, ) => resolveAuthProfileFailureReason({ failoverReason, providerStarted: opts?.providerStarted, + transientRateLimit: opts?.transientRateLimit, policy: params.authProfileFailurePolicy, }); const maybeBackoffBeforeOverloadFailover = async (reason: FailoverReason | null) => { @@ -2695,6 +2696,9 @@ async function runEmbeddedAgentInternal( promptFailoverReason, { providerStarted: promptErrorSource === "prompt", + transientRateLimit: + promptFailoverReason === "rate_limit" && + isShortWindowRateLimitMessage(errorText), }, ); const promptFailoverFailure = @@ -2865,6 +2869,9 @@ async function runEmbeddedAgentInternal( assistantProfileFailoverReason, { providerStarted: assistantProviderStarted, + transientRateLimit: + assistantProfileFailoverReason === "rate_limit" && + isShortWindowRateLimitMessage(assistantForFailover?.errorMessage), }, ); const cloudCodeAssistFormatError = attempt.cloudCodeAssistFormatError; diff --git a/src/agents/embedded-agent-runner/run/assistant-failover.ts b/src/agents/embedded-agent-runner/run/assistant-failover.ts index f3e581818c9b..aa3b7b43aac9 100644 --- a/src/agents/embedded-agent-runner/run/assistant-failover.ts +++ b/src/agents/embedded-agent-runner/run/assistant-failover.ts @@ -110,6 +110,10 @@ function resolveShortWindowRateLimitRetry( return retryAfterSeconds !== null ? { retryAfterSeconds } : {}; } +export function isShortWindowRateLimitMessage(message: string | undefined): boolean { + return resolveShortWindowRateLimitRetry(message) !== null; +} + /** * Applies an assistant-stage failover decision and returns the next run action. * It owns auth-profile rotation, overload/rate-limit escalation, same-model diff --git a/src/agents/embedded-agent-runner/run/auth-profile-failure-policy.test.ts b/src/agents/embedded-agent-runner/run/auth-profile-failure-policy.test.ts index c2a03cd78267..3dcf39a05ab9 100644 --- a/src/agents/embedded-agent-runner/run/auth-profile-failure-policy.test.ts +++ b/src/agents/embedded-agent-runner/run/auth-profile-failure-policy.test.ts @@ -33,6 +33,40 @@ describe("resolveAuthProfileFailureReason", () => { ).toBeNull(); }); + it("keeps only transient local failures out of shared auth state", () => { + expect( + resolveAuthProfileFailureReason({ + failoverReason: "rate_limit", + policy: "local_transient", + }), + ).toBe("rate_limit"); + expect( + resolveAuthProfileFailureReason({ + failoverReason: "rate_limit", + policy: "local_transient", + transientRateLimit: true, + }), + ).toBeNull(); + expect( + resolveAuthProfileFailureReason({ + failoverReason: "overloaded", + policy: "local_transient", + }), + ).toBeNull(); + expect( + resolveAuthProfileFailureReason({ + failoverReason: "auth", + policy: "local_transient", + }), + ).toBe("auth"); + expect( + resolveAuthProfileFailureReason({ + failoverReason: "billing", + policy: "local_transient", + }), + ).toBe("billing"); + }); + it("only persists timeouts when the provider request started", () => { // Pre-provider timeout says nothing about credential health; started // provider timeouts can cool down the active profile. diff --git a/src/agents/embedded-agent-runner/run/auth-profile-failure-policy.ts b/src/agents/embedded-agent-runner/run/auth-profile-failure-policy.ts index 6b9b9f1f33c4..56025fe721c2 100644 --- a/src/agents/embedded-agent-runner/run/auth-profile-failure-policy.ts +++ b/src/agents/embedded-agent-runner/run/auth-profile-failure-policy.ts @@ -14,6 +14,7 @@ import type { AuthProfileFailurePolicy } from "./auth-profile-failure-policy.typ export function resolveAuthProfileFailureReason(params: { failoverReason: FailoverReason | null; providerStarted?: boolean; + transientRateLimit?: boolean; policy?: AuthProfileFailurePolicy; }): AuthProfileFailureReason | null { // Helper-local runs, transport/server failures, empty responses, and request-shape ("format") rejections @@ -28,6 +29,9 @@ export function resolveAuthProfileFailureReason(params: { if ( params.policy === "local" || !params.failoverReason || + (params.policy === "local_transient" && + (params.failoverReason === "overloaded" || + (params.failoverReason === "rate_limit" && params.transientRateLimit === true))) || params.failoverReason === "server_error" || params.failoverReason === "empty_response" || params.failoverReason === "format" diff --git a/src/agents/embedded-agent-runner/run/auth-profile-failure-policy.types.ts b/src/agents/embedded-agent-runner/run/auth-profile-failure-policy.types.ts index 50caa6b11f9b..a99e9ea01f2a 100644 --- a/src/agents/embedded-agent-runner/run/auth-profile-failure-policy.types.ts +++ b/src/agents/embedded-agent-runner/run/auth-profile-failure-policy.types.ts @@ -1,4 +1,4 @@ /** * Scope used when classifying auth-profile failures for retry/fallback decisions. */ -export type AuthProfileFailurePolicy = "shared" | "local"; +export type AuthProfileFailurePolicy = "shared" | "local" | "local_transient"; diff --git a/src/cron/isolated-agent.auth-profile-propagation.test.ts b/src/cron/isolated-agent.auth-profile-propagation.test.ts index 6931dc62d4b4..ac50119e6a89 100644 --- a/src/cron/isolated-agent.auth-profile-propagation.test.ts +++ b/src/cron/isolated-agent.auth-profile-propagation.test.ts @@ -1,5 +1,6 @@ // Auth profile propagation tests cover isolated agent auth profile forwarding. import { describe, expect, it } from "vitest"; +import type { AuthProfileFailurePolicy } from "../agents/embedded-agent-runner/run/auth-profile-failure-policy.types.js"; import { makeIsolatedAgentTurnJob, makeIsolatedAgentTurnParams, @@ -15,7 +16,11 @@ import { const runCronIsolatedAgentTurn = await loadRunCronIsolatedAgentTurn(); -function getEmbeddedAgentParams(): { authProfileId?: string; authProfileIdSource?: string } { +function getEmbeddedAgentParams(): { + authProfileId?: string; + authProfileIdSource?: string; + authProfileFailurePolicy?: AuthProfileFailurePolicy; +} { const params = runEmbeddedAgentMock.mock.calls[0]?.[0]; if (!params || typeof params !== "object" || Array.isArray(params)) { throw new Error("Expected embedded OpenClaw agent params to be an object"); @@ -23,9 +28,30 @@ function getEmbeddedAgentParams(): { authProfileId?: string; authProfileIdSource return params; } -describe("runCronIsolatedAgentTurn auth profile propagation (#20624)", () => { +describe("runCronIsolatedAgentTurn auth profile propagation (#20624, #90991)", () => { setupRunCronIsolatedAgentTurnSuite(); + it("uses transient-local auth cooldown policy for cron throttling failures", async () => { + mockRunCronFallbackPassthrough(); + + await runCronIsolatedAgentTurn( + makeIsolatedAgentTurnParams({ + job: makeIsolatedAgentTurnJob({ + delivery: { mode: "none" }, + payload: { kind: "agentTurn", message: "check status" }, + }), + message: "check status", + sessionKey: "cron:job-1", + lane: "cron", + }), + ); + + expect(runEmbeddedAgentMock).toHaveBeenCalledOnce(); + expect(getEmbeddedAgentParams()).toMatchObject({ + authProfileFailurePolicy: "local_transient", + }); + }); + it("passes authProfileId to runEmbeddedAgent when auth profiles exist", async () => { resolveConfiguredModelRefMock.mockReturnValue({ provider: "openrouter", diff --git a/src/cron/isolated-agent/run-executor.ts b/src/cron/isolated-agent/run-executor.ts index 2775b266cb0b..df062a3c421e 100644 --- a/src/cron/isolated-agent/run-executor.ts +++ b/src/cron/isolated-agent/run-executor.ts @@ -304,6 +304,9 @@ export function createCronPromptExecutor(params: { authProfileIdSource: params.liveSelection.authProfileId ? params.liveSelection.authProfileIdSource : undefined, + // Scheduled run: keep bursty cron overloaded/rate_limit local, while + // still sharing real credential/account failures across auth profiles. + authProfileFailurePolicy: "local_transient", thinkLevel: params.thinkLevel, fastMode: resolveFastModeState({ cfg: params.cfgWithAgentDefaults,