mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-23 19:08:22 -06:00
fix(cron): isolate transient auth cooldowns
Keep cron-local transient auth failures from polluting shared cooldowns while preserving real auth/billing/rate-limit propagation. Verified with focused auth/cron tests, type proof, autoreview, and clean CI.
This commit is contained in:
@@ -137,7 +137,7 @@ import {
|
||||
type PostCompactionGuardObservation,
|
||||
} from "./post-compaction-loop-guard.js";
|
||||
import { createEmbeddedRunReplayState, observeReplayMetadata } from "./replay-state.js";
|
||||
import { handleAssistantFailover } from "./run/assistant-failover.js";
|
||||
import { handleAssistantFailover, isShortWindowRateLimitMessage } from "./run/assistant-failover.js";
|
||||
import {
|
||||
createEmbeddedRunStageTracker,
|
||||
EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE,
|
||||
@@ -1418,11 +1418,12 @@ async function runEmbeddedAgentInternal(
|
||||
};
|
||||
const resolveRunAuthProfileFailureReason = (
|
||||
failoverReason: FailoverReason | null,
|
||||
opts?: { providerStarted?: boolean },
|
||||
opts?: { providerStarted?: boolean; transientRateLimit?: boolean },
|
||||
) =>
|
||||
resolveAuthProfileFailureReason({
|
||||
failoverReason,
|
||||
providerStarted: opts?.providerStarted,
|
||||
transientRateLimit: opts?.transientRateLimit,
|
||||
policy: params.authProfileFailurePolicy,
|
||||
});
|
||||
const maybeBackoffBeforeOverloadFailover = async (reason: FailoverReason | null) => {
|
||||
@@ -2695,6 +2696,9 @@ async function runEmbeddedAgentInternal(
|
||||
promptFailoverReason,
|
||||
{
|
||||
providerStarted: promptErrorSource === "prompt",
|
||||
transientRateLimit:
|
||||
promptFailoverReason === "rate_limit" &&
|
||||
isShortWindowRateLimitMessage(errorText),
|
||||
},
|
||||
);
|
||||
const promptFailoverFailure =
|
||||
@@ -2865,6 +2869,9 @@ async function runEmbeddedAgentInternal(
|
||||
assistantProfileFailoverReason,
|
||||
{
|
||||
providerStarted: assistantProviderStarted,
|
||||
transientRateLimit:
|
||||
assistantProfileFailoverReason === "rate_limit" &&
|
||||
isShortWindowRateLimitMessage(assistantForFailover?.errorMessage),
|
||||
},
|
||||
);
|
||||
const cloudCodeAssistFormatError = attempt.cloudCodeAssistFormatError;
|
||||
|
||||
@@ -110,6 +110,10 @@ function resolveShortWindowRateLimitRetry(
|
||||
return retryAfterSeconds !== null ? { retryAfterSeconds } : {};
|
||||
}
|
||||
|
||||
export function isShortWindowRateLimitMessage(message: string | undefined): boolean {
|
||||
return resolveShortWindowRateLimitRetry(message) !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies an assistant-stage failover decision and returns the next run action.
|
||||
* It owns auth-profile rotation, overload/rate-limit escalation, same-model
|
||||
|
||||
@@ -33,6 +33,40 @@ describe("resolveAuthProfileFailureReason", () => {
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps only transient local failures out of shared auth state", () => {
|
||||
expect(
|
||||
resolveAuthProfileFailureReason({
|
||||
failoverReason: "rate_limit",
|
||||
policy: "local_transient",
|
||||
}),
|
||||
).toBe("rate_limit");
|
||||
expect(
|
||||
resolveAuthProfileFailureReason({
|
||||
failoverReason: "rate_limit",
|
||||
policy: "local_transient",
|
||||
transientRateLimit: true,
|
||||
}),
|
||||
).toBeNull();
|
||||
expect(
|
||||
resolveAuthProfileFailureReason({
|
||||
failoverReason: "overloaded",
|
||||
policy: "local_transient",
|
||||
}),
|
||||
).toBeNull();
|
||||
expect(
|
||||
resolveAuthProfileFailureReason({
|
||||
failoverReason: "auth",
|
||||
policy: "local_transient",
|
||||
}),
|
||||
).toBe("auth");
|
||||
expect(
|
||||
resolveAuthProfileFailureReason({
|
||||
failoverReason: "billing",
|
||||
policy: "local_transient",
|
||||
}),
|
||||
).toBe("billing");
|
||||
});
|
||||
|
||||
it("only persists timeouts when the provider request started", () => {
|
||||
// Pre-provider timeout says nothing about credential health; started
|
||||
// provider timeouts can cool down the active profile.
|
||||
|
||||
@@ -14,6 +14,7 @@ import type { AuthProfileFailurePolicy } from "./auth-profile-failure-policy.typ
|
||||
export function resolveAuthProfileFailureReason(params: {
|
||||
failoverReason: FailoverReason | null;
|
||||
providerStarted?: boolean;
|
||||
transientRateLimit?: boolean;
|
||||
policy?: AuthProfileFailurePolicy;
|
||||
}): AuthProfileFailureReason | null {
|
||||
// Helper-local runs, transport/server failures, empty responses, and request-shape ("format") rejections
|
||||
@@ -28,6 +29,9 @@ export function resolveAuthProfileFailureReason(params: {
|
||||
if (
|
||||
params.policy === "local" ||
|
||||
!params.failoverReason ||
|
||||
(params.policy === "local_transient" &&
|
||||
(params.failoverReason === "overloaded" ||
|
||||
(params.failoverReason === "rate_limit" && params.transientRateLimit === true))) ||
|
||||
params.failoverReason === "server_error" ||
|
||||
params.failoverReason === "empty_response" ||
|
||||
params.failoverReason === "format"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/**
|
||||
* Scope used when classifying auth-profile failures for retry/fallback decisions.
|
||||
*/
|
||||
export type AuthProfileFailurePolicy = "shared" | "local";
|
||||
export type AuthProfileFailurePolicy = "shared" | "local" | "local_transient";
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// Auth profile propagation tests cover isolated agent auth profile forwarding.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { AuthProfileFailurePolicy } from "../agents/embedded-agent-runner/run/auth-profile-failure-policy.types.js";
|
||||
import {
|
||||
makeIsolatedAgentTurnJob,
|
||||
makeIsolatedAgentTurnParams,
|
||||
@@ -15,7 +16,11 @@ import {
|
||||
|
||||
const runCronIsolatedAgentTurn = await loadRunCronIsolatedAgentTurn();
|
||||
|
||||
function getEmbeddedAgentParams(): { authProfileId?: string; authProfileIdSource?: string } {
|
||||
function getEmbeddedAgentParams(): {
|
||||
authProfileId?: string;
|
||||
authProfileIdSource?: string;
|
||||
authProfileFailurePolicy?: AuthProfileFailurePolicy;
|
||||
} {
|
||||
const params = runEmbeddedAgentMock.mock.calls[0]?.[0];
|
||||
if (!params || typeof params !== "object" || Array.isArray(params)) {
|
||||
throw new Error("Expected embedded OpenClaw agent params to be an object");
|
||||
@@ -23,9 +28,30 @@ function getEmbeddedAgentParams(): { authProfileId?: string; authProfileIdSource
|
||||
return params;
|
||||
}
|
||||
|
||||
describe("runCronIsolatedAgentTurn auth profile propagation (#20624)", () => {
|
||||
describe("runCronIsolatedAgentTurn auth profile propagation (#20624, #90991)", () => {
|
||||
setupRunCronIsolatedAgentTurnSuite();
|
||||
|
||||
it("uses transient-local auth cooldown policy for cron throttling failures", async () => {
|
||||
mockRunCronFallbackPassthrough();
|
||||
|
||||
await runCronIsolatedAgentTurn(
|
||||
makeIsolatedAgentTurnParams({
|
||||
job: makeIsolatedAgentTurnJob({
|
||||
delivery: { mode: "none" },
|
||||
payload: { kind: "agentTurn", message: "check status" },
|
||||
}),
|
||||
message: "check status",
|
||||
sessionKey: "cron:job-1",
|
||||
lane: "cron",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(runEmbeddedAgentMock).toHaveBeenCalledOnce();
|
||||
expect(getEmbeddedAgentParams()).toMatchObject({
|
||||
authProfileFailurePolicy: "local_transient",
|
||||
});
|
||||
});
|
||||
|
||||
it("passes authProfileId to runEmbeddedAgent when auth profiles exist", async () => {
|
||||
resolveConfiguredModelRefMock.mockReturnValue({
|
||||
provider: "openrouter",
|
||||
|
||||
@@ -304,6 +304,9 @@ export function createCronPromptExecutor(params: {
|
||||
authProfileIdSource: params.liveSelection.authProfileId
|
||||
? params.liveSelection.authProfileIdSource
|
||||
: undefined,
|
||||
// Scheduled run: keep bursty cron overloaded/rate_limit local, while
|
||||
// still sharing real credential/account failures across auth profiles.
|
||||
authProfileFailurePolicy: "local_transient",
|
||||
thinkLevel: params.thinkLevel,
|
||||
fastMode: resolveFastModeState({
|
||||
cfg: params.cfgWithAgentDefaults,
|
||||
|
||||
Reference in New Issue
Block a user