fix(cron): isolate transient auth cooldowns

Keep cron-local transient auth failures from polluting shared cooldowns while preserving real auth/billing/rate-limit propagation. Verified with focused auth/cron tests, type proof, autoreview, and clean CI.
This commit is contained in:
Chunyue Wang
2026-06-14 07:45:20 +08:00
committed by GitHub
parent edd76238fe
commit 5b21384ab6
7 changed files with 83 additions and 5 deletions
+9 -2
View File
@@ -137,7 +137,7 @@ import {
type PostCompactionGuardObservation,
} from "./post-compaction-loop-guard.js";
import { createEmbeddedRunReplayState, observeReplayMetadata } from "./replay-state.js";
import { handleAssistantFailover } from "./run/assistant-failover.js";
import { handleAssistantFailover, isShortWindowRateLimitMessage } from "./run/assistant-failover.js";
import {
createEmbeddedRunStageTracker,
EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE,
@@ -1418,11 +1418,12 @@ async function runEmbeddedAgentInternal(
};
const resolveRunAuthProfileFailureReason = (
failoverReason: FailoverReason | null,
opts?: { providerStarted?: boolean },
opts?: { providerStarted?: boolean; transientRateLimit?: boolean },
) =>
resolveAuthProfileFailureReason({
failoverReason,
providerStarted: opts?.providerStarted,
transientRateLimit: opts?.transientRateLimit,
policy: params.authProfileFailurePolicy,
});
const maybeBackoffBeforeOverloadFailover = async (reason: FailoverReason | null) => {
@@ -2695,6 +2696,9 @@ async function runEmbeddedAgentInternal(
promptFailoverReason,
{
providerStarted: promptErrorSource === "prompt",
transientRateLimit:
promptFailoverReason === "rate_limit" &&
isShortWindowRateLimitMessage(errorText),
},
);
const promptFailoverFailure =
@@ -2865,6 +2869,9 @@ async function runEmbeddedAgentInternal(
assistantProfileFailoverReason,
{
providerStarted: assistantProviderStarted,
transientRateLimit:
assistantProfileFailoverReason === "rate_limit" &&
isShortWindowRateLimitMessage(assistantForFailover?.errorMessage),
},
);
const cloudCodeAssistFormatError = attempt.cloudCodeAssistFormatError;
@@ -110,6 +110,10 @@ function resolveShortWindowRateLimitRetry(
return retryAfterSeconds !== null ? { retryAfterSeconds } : {};
}
export function isShortWindowRateLimitMessage(message: string | undefined): boolean {
return resolveShortWindowRateLimitRetry(message) !== null;
}
/**
* Applies an assistant-stage failover decision and returns the next run action.
* It owns auth-profile rotation, overload/rate-limit escalation, same-model
@@ -33,6 +33,40 @@ describe("resolveAuthProfileFailureReason", () => {
).toBeNull();
});
it("keeps only transient local failures out of shared auth state", () => {
expect(
resolveAuthProfileFailureReason({
failoverReason: "rate_limit",
policy: "local_transient",
}),
).toBe("rate_limit");
expect(
resolveAuthProfileFailureReason({
failoverReason: "rate_limit",
policy: "local_transient",
transientRateLimit: true,
}),
).toBeNull();
expect(
resolveAuthProfileFailureReason({
failoverReason: "overloaded",
policy: "local_transient",
}),
).toBeNull();
expect(
resolveAuthProfileFailureReason({
failoverReason: "auth",
policy: "local_transient",
}),
).toBe("auth");
expect(
resolveAuthProfileFailureReason({
failoverReason: "billing",
policy: "local_transient",
}),
).toBe("billing");
});
it("only persists timeouts when the provider request started", () => {
// Pre-provider timeout says nothing about credential health; started
// provider timeouts can cool down the active profile.
@@ -14,6 +14,7 @@ import type { AuthProfileFailurePolicy } from "./auth-profile-failure-policy.typ
export function resolveAuthProfileFailureReason(params: {
failoverReason: FailoverReason | null;
providerStarted?: boolean;
transientRateLimit?: boolean;
policy?: AuthProfileFailurePolicy;
}): AuthProfileFailureReason | null {
// Helper-local runs, transport/server failures, empty responses, and request-shape ("format") rejections
@@ -28,6 +29,9 @@ export function resolveAuthProfileFailureReason(params: {
if (
params.policy === "local" ||
!params.failoverReason ||
(params.policy === "local_transient" &&
(params.failoverReason === "overloaded" ||
(params.failoverReason === "rate_limit" && params.transientRateLimit === true))) ||
params.failoverReason === "server_error" ||
params.failoverReason === "empty_response" ||
params.failoverReason === "format"
@@ -1,4 +1,4 @@
/**
* Scope used when classifying auth-profile failures for retry/fallback decisions.
*/
export type AuthProfileFailurePolicy = "shared" | "local";
export type AuthProfileFailurePolicy = "shared" | "local" | "local_transient";
@@ -1,5 +1,6 @@
// Auth profile propagation tests cover isolated agent auth profile forwarding.
import { describe, expect, it } from "vitest";
import type { AuthProfileFailurePolicy } from "../agents/embedded-agent-runner/run/auth-profile-failure-policy.types.js";
import {
makeIsolatedAgentTurnJob,
makeIsolatedAgentTurnParams,
@@ -15,7 +16,11 @@ import {
const runCronIsolatedAgentTurn = await loadRunCronIsolatedAgentTurn();
function getEmbeddedAgentParams(): { authProfileId?: string; authProfileIdSource?: string } {
function getEmbeddedAgentParams(): {
authProfileId?: string;
authProfileIdSource?: string;
authProfileFailurePolicy?: AuthProfileFailurePolicy;
} {
const params = runEmbeddedAgentMock.mock.calls[0]?.[0];
if (!params || typeof params !== "object" || Array.isArray(params)) {
throw new Error("Expected embedded OpenClaw agent params to be an object");
@@ -23,9 +28,30 @@ function getEmbeddedAgentParams(): { authProfileId?: string; authProfileIdSource
return params;
}
describe("runCronIsolatedAgentTurn auth profile propagation (#20624)", () => {
describe("runCronIsolatedAgentTurn auth profile propagation (#20624, #90991)", () => {
setupRunCronIsolatedAgentTurnSuite();
it("uses transient-local auth cooldown policy for cron throttling failures", async () => {
mockRunCronFallbackPassthrough();
await runCronIsolatedAgentTurn(
makeIsolatedAgentTurnParams({
job: makeIsolatedAgentTurnJob({
delivery: { mode: "none" },
payload: { kind: "agentTurn", message: "check status" },
}),
message: "check status",
sessionKey: "cron:job-1",
lane: "cron",
}),
);
expect(runEmbeddedAgentMock).toHaveBeenCalledOnce();
expect(getEmbeddedAgentParams()).toMatchObject({
authProfileFailurePolicy: "local_transient",
});
});
it("passes authProfileId to runEmbeddedAgent when auth profiles exist", async () => {
resolveConfiguredModelRefMock.mockReturnValue({
provider: "openrouter",
+3
View File
@@ -304,6 +304,9 @@ export function createCronPromptExecutor(params: {
authProfileIdSource: params.liveSelection.authProfileId
? params.liveSelection.authProfileIdSource
: undefined,
// Scheduled run: keep bursty cron overloaded/rate_limit local, while
// still sharing real credential/account failures across auth profiles.
authProfileFailurePolicy: "local_transient",
thinkLevel: params.thinkLevel,
fastMode: resolveFastModeState({
cfg: params.cfgWithAgentDefaults,