fix(agents): distinguish terminal aborts from retryable failures (#60388) (#62682)

* fix(agents): terminal-abort coverage beyond #87085's isTerminalAbort (closes #60388)

PR #87085 landed the base isTerminalAbort(signal) check (TimeoutError /
ClientDisconnectError on signal.reason) plus abortSignal threading through
the model-fallback and chat-side callers. This change adds the coverage that
PR #87085 did not include:

- ClientDisconnectError class + wiring in http-common.ts so the
  reason.name === "ClientDisconnectError" branch PR #87085 added is actually
  reachable (upstream watchClientDisconnect still aborts bare).
- cron run-budget string reasons (prefix match) — cron timer aborts with a
  plain string, which the Error-only base check skips.
- .cause-chain walking + isTerminalAbortFromError gated on the
  OPENCLAW_ABORTABLE_WRAPPER marker, for the embedded run-budget timer that
  aborts a private controller (not the caller signal).
- compaction-path abortSignal forwarding (compactEmbeddedPiSessionDirect).
- timedOutByRunBudget plumbing through attempt/failover-policy/assistant-failover
  so run-budget timeouts skip the fallback chain and wasted compaction.

* fix(agents): preserve wrapped restart aborts

* refactor: dedupe terminal abort classification

---------

Co-authored-by: Altay <altay@hey.com>
This commit is contained in:
simonusa
2026-07-06 07:48:50 -07:00
committed by GitHub
parent b3db79929f
commit 7cdbfc9649
16 changed files with 810 additions and 19 deletions
@@ -0,0 +1,74 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
vi.mock("../model-fallback.js", () => ({
runWithModelFallback: vi.fn(async (params: Record<string, unknown>) => ({
result: { ok: true, compacted: false, reason: "no-op" },
provider: params.provider,
model: params.model,
attempts: [],
})),
isFallbackSummaryError: () => false,
}));
vi.mock("./compact.queued.js", () => ({ compactEmbeddedAgentSession: vi.fn() }));
import { runWithModelFallback } from "../model-fallback.js";
import { compactEmbeddedAgentSessionDirect } from "./compact.js";
const runMock = vi.mocked(runWithModelFallback);
const baseParams = {
sessionId: "test-session",
sessionKey: "agent:main:test-session",
sessionFile: "/tmp/test-session.jsonl",
workspaceDir: "/tmp",
};
function configWithFallbacks(fallbacks: string[]): OpenClawConfig {
return {
agents: {
defaults: {
model: {
primary: "anthropic/claude-sonnet-4-6",
fallbacks,
},
},
},
} as OpenClawConfig;
}
describe("compactEmbeddedAgentSessionDirect abortSignal threading", () => {
beforeEach(() => {
runMock.mockClear();
});
it("forwards params.abortSignal to runWithModelFallback so terminal aborts during compaction short-circuit", async () => {
const controller = new AbortController();
await compactEmbeddedAgentSessionDirect({
...baseParams,
config: configWithFallbacks(["anthropic/claude-haiku-4-5", "openai/gpt-4.1-mini"]),
provider: "anthropic",
model: "claude-sonnet-4-6",
abortSignal: controller.signal,
});
expect(runMock).toHaveBeenCalledTimes(1);
const passedParams = runMock.mock.calls[0]?.[0];
expect(passedParams?.abortSignal).toBe(controller.signal);
});
it("passes undefined when no abortSignal is set (back-compat)", async () => {
await compactEmbeddedAgentSessionDirect({
...baseParams,
config: configWithFallbacks(["anthropic/claude-haiku-4-5"]),
provider: "anthropic",
model: "claude-sonnet-4-6",
});
expect(runMock).toHaveBeenCalledTimes(1);
const passedParams = runMock.mock.calls[0]?.[0];
expect(passedParams?.abortSignal).toBeUndefined();
});
});
+11 -4
View File
@@ -2325,6 +2325,7 @@ async function runEmbeddedAgentInternal(
attempt.setTerminalLifecycleMeta?.({ ...meta, aborted });
};
const timedOutDuringToolExecution = attempt.timedOutDuringToolExecution ?? false;
const timedOutByRunBudget = attempt.timedOutByRunBudget ?? false;
adoptActiveSessionId(sessionIdUsed);
if (sessionFileUsed && sessionFileUsed !== activeSessionFile) {
activeSessionFile = sessionFileUsed;
@@ -2502,10 +2503,12 @@ async function runEmbeddedAgentInternal(
);
throw new LiveSessionModelSwitchError(requestedSelection);
}
// ── Timeout-triggered compaction ──────────────────────────────────
// When the LLM times out with high context usage, compact before
// retrying to break the death spiral of repeated timeouts.
if (timedOut && !timedOutDuringCompaction && !timedOutDuringToolExecution) {
if (
timedOut &&
!timedOutDuringCompaction &&
!timedOutDuringToolExecution &&
!timedOutByRunBudget
) {
// Only consider prompt-side tokens here. API totals include output
// tokens, which can make a long generation look like high context
// pressure even when the prompt itself was small.
@@ -3241,6 +3244,7 @@ async function runEmbeddedAgentInternal(
failoverReason: promptFailoverReason,
harnessOwnsTransport: pluginHarnessOwnsTransport,
promptTimeoutFallbackSafe,
timedOutByRunBudget,
profileRotated: false,
});
if (
@@ -3281,6 +3285,7 @@ async function runEmbeddedAgentInternal(
failoverReason: promptFailoverReason,
harnessOwnsTransport: pluginHarnessOwnsTransport,
promptTimeoutFallbackSafe,
timedOutByRunBudget,
profileRotated: true,
});
}
@@ -3483,6 +3488,7 @@ async function runEmbeddedAgentInternal(
timedOutDuringCompaction,
timedOutDuringToolExecution,
harnessOwnsTransport: pluginHarnessOwnsTransport,
timedOutByRunBudget,
profileRotated: false,
});
const assistantFailoverOutcome = await handleAssistantFailover({
@@ -3496,6 +3502,7 @@ async function runEmbeddedAgentInternal(
idleTimedOut,
timedOutDuringCompaction,
timedOutDuringToolExecution,
timedOutByRunBudget,
allowSameModelIdleTimeoutRetry:
timedOut &&
idleTimedOut &&
@@ -7,16 +7,28 @@ function getAbortReason(signal: AbortSignal): unknown {
return "reason" in signal ? (signal as { reason?: unknown }).reason : undefined;
}
/** Marks AbortErrors produced by abortable() so provider aborts stay retryable. */
export const OPENCLAW_ABORTABLE_WRAPPER = Symbol.for("openclaw.abortable.wrapper");
export function isOpenClawAbortableWrapper(err: unknown): boolean {
return err !== null && typeof err === "object" && OPENCLAW_ABORTABLE_WRAPPER in err;
}
function tagAsAbortableWrapper(err: Error): Error {
(err as Error & { [OPENCLAW_ABORTABLE_WRAPPER]?: true })[OPENCLAW_ABORTABLE_WRAPPER] = true;
return err;
}
function makeAbortError(signal: AbortSignal): Error {
const reason = getAbortReason(signal);
if (reason instanceof Error) {
const err = new Error(reason.message, { cause: reason });
err.name = "AbortError";
return err;
return tagAsAbortableWrapper(err);
}
const err = reason ? new Error("aborted", { cause: reason }) : new Error("aborted");
err.name = "AbortError";
return err;
return tagAsAbortableWrapper(err);
}
/**
@@ -23,6 +23,7 @@ function makeParams(overrides: Partial<Params> = {}): Params {
idleTimedOut: false,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
timedOutByRunBudget: false,
allowSameModelIdleTimeoutRetry: false,
allowSameModelRateLimitRetry: true,
assistantProfileFailureReason: null,
@@ -130,6 +130,7 @@ export async function handleAssistantFailover(params: {
idleTimedOut: boolean;
timedOutDuringCompaction: boolean;
timedOutDuringToolExecution: boolean;
timedOutByRunBudget: boolean;
allowSameModelIdleTimeoutRetry: boolean;
allowSameModelRateLimitRetry: boolean;
assistantProfileFailureReason: AuthProfileFailureReason | null;
@@ -309,6 +310,7 @@ export async function handleAssistantFailover(params: {
idleTimedOut: params.idleTimedOut,
timedOutDuringCompaction: params.timedOutDuringCompaction,
timedOutDuringToolExecution: params.timedOutDuringToolExecution,
timedOutByRunBudget: params.timedOutByRunBudget,
profileRotated: true,
});
}
@@ -918,6 +918,7 @@ export async function runEmbeddedAttempt(
let idleTimedOut = false;
let timedOutDuringCompaction = false;
let timedOutDuringToolExecution = false;
let timedOutByRunBudget = false;
let promptError: unknown = null;
let emitDiagnosticRunCompleted:
| ((
@@ -3819,6 +3820,7 @@ export async function runEmbeddedAttempt(
) {
timedOutDuringCompaction = true;
}
timedOutByRunBudget = true;
abortRun(true);
if (!abortWarnTimer) {
abortWarnTimer = setTimeout(() => {
@@ -5668,6 +5670,7 @@ export async function runEmbeddedAttempt(
idleTimedOut,
timedOutDuringCompaction,
timedOutDuringToolExecution,
timedOutByRunBudget,
promptError: promptError ? formatErrorMessage(promptError) : undefined,
promptErrorSource,
terminalError: attemptTrajectoryTerminal.terminalError,
@@ -5688,6 +5691,7 @@ export async function runEmbeddedAttempt(
idleTimedOut,
timedOutDuringCompaction,
timedOutDuringToolExecution,
timedOutByRunBudget,
promptError: promptError ? formatErrorMessage(promptError) : undefined,
promptErrorSource,
terminalError: attemptTrajectoryTerminal.terminalError,
@@ -5714,6 +5718,7 @@ export async function runEmbeddedAttempt(
idleTimedOut,
timedOutDuringCompaction,
timedOutDuringToolExecution,
timedOutByRunBudget,
promptError: promptError ? formatErrorMessage(promptError) : undefined,
terminalError: attemptTrajectoryTerminal.terminalError,
});
@@ -5729,6 +5734,7 @@ export async function runEmbeddedAttempt(
idleTimedOut,
timedOutDuringCompaction,
timedOutDuringToolExecution,
timedOutByRunBudget,
promptError,
promptErrorSource,
preflightRecovery,
@@ -5785,6 +5791,7 @@ export async function runEmbeddedAttempt(
idleTimedOut,
timedOutDuringCompaction,
timedOutDuringToolExecution,
timedOutByRunBudget,
promptError: promptError ? formatErrorMessage(promptError) : undefined,
});
}
@@ -95,6 +95,43 @@ describe("resolveRunFailoverDecision", () => {
});
});
it("surfaces prompt run-budget timeouts instead of model fallback (#60388)", () => {
expect(
resolveRunFailoverDecision({
stage: "prompt",
aborted: true,
externalAbort: false,
fallbackConfigured: true,
failoverFailure: true,
failoverReason: "timeout",
promptTimeoutFallbackSafe: true,
timedOutByRunBudget: true,
profileRotated: true,
}),
).toEqual({
action: "surface_error",
reason: "timeout",
});
});
it("does not rotate prompt failures after the run budget is exhausted (#60388)", () => {
expect(
resolveRunFailoverDecision({
stage: "prompt",
aborted: true,
externalAbort: false,
fallbackConfigured: true,
failoverFailure: true,
failoverReason: "rate_limit",
timedOutByRunBudget: true,
profileRotated: false,
}),
).toEqual({
action: "surface_error",
reason: "rate_limit",
});
});
it("surfaces deterministic prompt format failures instead of rotating or falling back", () => {
expect(
resolveRunFailoverDecision({
@@ -145,6 +182,7 @@ describe("resolveRunFailoverDecision", () => {
idleTimedOut: false,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
timedOutByRunBudget: false,
profileRotated: false,
}),
).toEqual({
@@ -165,6 +203,7 @@ describe("resolveRunFailoverDecision", () => {
idleTimedOut: false,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
timedOutByRunBudget: false,
profileRotated: false,
}),
).toEqual({
@@ -187,6 +226,7 @@ describe("resolveRunFailoverDecision", () => {
idleTimedOut: false,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
timedOutByRunBudget: false,
profileRotated: false,
}),
).toEqual({
@@ -208,6 +248,7 @@ describe("resolveRunFailoverDecision", () => {
idleTimedOut: false,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
timedOutByRunBudget: false,
profileRotated: true,
}),
).toEqual({
@@ -229,6 +270,7 @@ describe("resolveRunFailoverDecision", () => {
idleTimedOut: false,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
timedOutByRunBudget: false,
profileRotated: true,
}),
).toEqual({
@@ -249,6 +291,7 @@ describe("resolveRunFailoverDecision", () => {
idleTimedOut: false,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
timedOutByRunBudget: false,
profileRotated: false,
}),
).toEqual({
@@ -286,6 +329,7 @@ describe("resolveRunFailoverDecision", () => {
idleTimedOut: false,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: true,
timedOutByRunBudget: false,
profileRotated: false,
}),
).toEqual({
@@ -348,6 +392,7 @@ describe("resolveRunFailoverDecision", () => {
idleTimedOut: false,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: true,
timedOutByRunBudget: false,
profileRotated: true,
}),
).toEqual({
@@ -368,6 +413,7 @@ describe("resolveRunFailoverDecision", () => {
idleTimedOut: false,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
timedOutByRunBudget: false,
profileRotated: false,
}),
).toEqual({
@@ -477,6 +523,7 @@ describe("resolveRunFailoverDecision", () => {
idleTimedOut: true,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: true,
timedOutByRunBudget: false,
profileRotated: false,
}),
).toEqual({
@@ -498,6 +545,7 @@ describe("resolveRunFailoverDecision", () => {
idleTimedOut: true,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: true,
timedOutByRunBudget: false,
profileRotated: true,
}),
).toEqual({
@@ -519,6 +567,7 @@ describe("resolveRunFailoverDecision", () => {
idleTimedOut: false,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
timedOutByRunBudget: false,
profileRotated: false,
}),
).toEqual({
@@ -540,6 +589,7 @@ describe("resolveRunFailoverDecision", () => {
idleTimedOut: true,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
timedOutByRunBudget: false,
profileRotated: false,
}),
).toEqual({
@@ -561,6 +611,7 @@ describe("resolveRunFailoverDecision", () => {
idleTimedOut: true,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
timedOutByRunBudget: false,
profileRotated: true,
}),
).toEqual({
@@ -659,6 +710,7 @@ describe("resolveRunFailoverDecision", () => {
idleTimedOut: true,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
timedOutByRunBudget: false,
profileRotated: true,
}),
).toEqual({
@@ -680,6 +732,7 @@ describe("resolveRunFailoverDecision", () => {
idleTimedOut: true,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
timedOutByRunBudget: false,
profileRotated: false,
}),
).toEqual({
@@ -687,6 +740,48 @@ describe("resolveRunFailoverDecision", () => {
reason: null,
});
});
it("does not rotate or fallback assistant timeouts that exhausted the run budget (#60388)", () => {
expect(
resolveRunFailoverDecision({
stage: "assistant",
aborted: true,
externalAbort: false,
fallbackConfigured: true,
failoverFailure: false,
failoverReason: null,
timedOut: true,
idleTimedOut: false,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
timedOutByRunBudget: true,
profileRotated: false,
}),
).toEqual({
action: "continue_normal",
});
});
it("does not fallback assistant run-budget timeouts even after profile rotation exhausted (#60388)", () => {
expect(
resolveRunFailoverDecision({
stage: "assistant",
aborted: true,
externalAbort: false,
fallbackConfigured: true,
failoverFailure: false,
failoverReason: null,
timedOut: true,
idleTimedOut: false,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
timedOutByRunBudget: true,
profileRotated: true,
}),
).toEqual({
action: "continue_normal",
});
});
});
describe("mergeRetryFailoverReason", () => {
@@ -51,6 +51,7 @@ type PromptDecisionParams = {
failoverReason: FailoverReason | null;
harnessOwnsTransport?: boolean;
promptTimeoutFallbackSafe?: boolean;
timedOutByRunBudget?: boolean;
profileRotated: boolean;
};
@@ -67,6 +68,7 @@ type AssistantDecisionParams = {
timedOutDuringCompaction: boolean;
timedOutDuringToolExecution: boolean;
harnessOwnsTransport?: boolean;
timedOutByRunBudget?: boolean;
profileRotated: boolean;
};
@@ -92,6 +94,9 @@ function isTerminalFormatFailure(params: {
}
function shouldRotatePrompt(params: PromptDecisionParams): boolean {
if (params.timedOutByRunBudget) {
return false;
}
return (
params.failoverFailure &&
params.failoverReason !== "timeout" &&
@@ -116,6 +121,9 @@ function shouldRotateAssistant(params: AssistantDecisionParams): boolean {
if (isTerminalFormatFailure(params)) {
return false;
}
if (params.timedOutByRunBudget) {
return false;
}
const timeoutFailure = isAssistantTimeoutFailure(params);
const harnessOwnedTimeout =
params.harnessOwnsTransport && (timeoutFailure || params.failoverReason === "timeout");
@@ -175,6 +183,12 @@ export function resolveRunFailoverDecision(params: RunFailoverDecisionParams): R
reason: params.failoverReason,
};
}
if (params.timedOutByRunBudget) {
return {
action: "surface_error",
reason: params.failoverReason,
};
}
if (params.harnessOwnsTransport && params.failoverReason === "timeout") {
// Plugin harness lifecycle timeouts must stay inside the harness boundary;
// only prompt request timeouts proven replay-safe may enter model fallback.
@@ -125,6 +125,7 @@ export type EmbeddedRunAttemptResult = {
timedOutDuringCompaction: boolean;
/** Optional because this type is re-exported as `AgentHarnessAttemptResult`. */
timedOutDuringToolExecution?: boolean;
timedOutByRunBudget?: boolean;
promptError: unknown;
/**
* Identifies which phase produced the promptError.
+484
View File
@@ -18,6 +18,7 @@ import { CommandLaneTaskTimeoutError } from "../process/command-queue.js";
import { AUTH_STORE_VERSION } from "./auth-profiles/constants.js";
import type { AuthProfileStore } from "./auth-profiles/types.js";
import { classifyEmbeddedAgentRunResultForModelFallback } from "./embedded-agent-runner/result-fallback-classifier.js";
import { OPENCLAW_ABORTABLE_WRAPPER } from "./embedded-agent-runner/run/abortable.js";
import type { EmbeddedAgentRunResult } from "./embedded-agent-runner/types.js";
import { FailoverError } from "./failover-error.js";
import { resetFallbackSkipCacheForTest } from "./fallback-skip-cache.js";
@@ -3459,6 +3460,489 @@ describe("runWithModelFallback", () => {
});
});
});
describe("terminal abort propagation", () => {
function makeAbortError(message = "aborted"): Error {
const err = new Error(message);
err.name = "AbortError";
return err;
}
function makeAbortableWrapper(reason: Error): Error {
const err = new Error(reason.message, { cause: reason });
err.name = "AbortError";
(err as Error & { [OPENCLAW_ABORTABLE_WRAPPER]?: true })[OPENCLAW_ABORTABLE_WRAPPER] = true;
return err;
}
function makeAbortWrapper(reason: Error): Error {
const err = new Error("aborted", { cause: reason });
err.name = "AbortError";
return err;
}
function makeTaggedAbortController(reason: Error): AbortController {
const controller = new AbortController();
controller.abort(reason);
return controller;
}
it("rethrows immediately when signal.reason has name=TimeoutError (run-budget timeout)", async () => {
const cfg = makeCfg();
const runError = makeAbortError("aborted");
const run = vi.fn().mockRejectedValue(runError);
const timeoutReason = new Error("request timed out");
timeoutReason.name = "TimeoutError";
const controller = makeTaggedAbortController(timeoutReason);
await expect(
runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
abortSignal: controller.signal,
}),
).rejects.toBe(runError);
expect(run).toHaveBeenCalledTimes(1);
});
it("rethrows immediately when signal.reason has name=ClientDisconnectError", async () => {
const cfg = makeCfg();
const runError = makeAbortError("aborted");
const run = vi.fn().mockRejectedValue(runError);
const disconnectReason = new Error("HTTP client disconnected");
disconnectReason.name = "ClientDisconnectError";
const controller = makeTaggedAbortController(disconnectReason);
await expect(
runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
abortSignal: controller.signal,
}),
).rejects.toBe(runError);
expect(run).toHaveBeenCalledTimes(1);
});
it("detects TimeoutError nested as cause of an outer Error", async () => {
const cfg = makeCfg();
const runError = makeAbortError("aborted");
const run = vi.fn().mockRejectedValue(runError);
const innerTimeout = new Error("request timed out");
innerTimeout.name = "TimeoutError";
const outerWrap = makeAbortableWrapper(innerTimeout);
const controller = makeTaggedAbortController(outerWrap);
await expect(
runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
abortSignal: controller.signal,
}),
).rejects.toBe(runError);
expect(run).toHaveBeenCalledTimes(1);
});
it("rethrows when thrown error has TimeoutError in cause chain (embedded run-budget timer)", async () => {
const cfg = makeCfg();
const innerTimeout = new Error("request timed out");
innerTimeout.name = "TimeoutError";
const outerAbort = makeAbortableWrapper(innerTimeout);
const run = vi.fn().mockRejectedValue(outerAbort);
await expect(
runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
}),
).rejects.toBe(outerAbort);
expect(run).toHaveBeenCalledTimes(1);
});
it("rethrows when thrown error has ClientDisconnectError in cause chain", async () => {
const cfg = makeCfg();
const innerDisconnect = new Error("client disconnected");
innerDisconnect.name = "ClientDisconnectError";
const outerAbort = makeAbortableWrapper(innerDisconnect);
const run = vi.fn().mockRejectedValue(outerAbort);
await expect(
runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
}),
).rejects.toBe(outerAbort);
expect(run).toHaveBeenCalledTimes(1);
});
it("rethrows when thrown error has restart abort in cause chain", async () => {
const cfg = makeCfg();
const restartAbort = createAgentRunRestartAbortError();
const outerAbort = makeAbortableWrapper(restartAbort);
const run = vi.fn().mockRejectedValueOnce(outerAbort).mockResolvedValueOnce("ok");
await expect(
runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
}),
).rejects.toBe(outerAbort);
expect(run).toHaveBeenCalledTimes(1);
});
it("rethrows when an unmarked AbortError wraps a restart abort", async () => {
const cfg = makeCfg();
const restartAbort = createAgentRunRestartAbortError();
const outerAbort = makeAbortWrapper(restartAbort);
const run = vi.fn().mockRejectedValueOnce(outerAbort).mockResolvedValueOnce("ok");
await expect(
runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
}),
).rejects.toBe(outerAbort);
expect(run).toHaveBeenCalledTimes(1);
});
it("discards deferred session suspension for private terminal abort wrappers", () => {
const timeout = new Error("request timed out");
timeout.name = "TimeoutError";
expect(
testing.shouldDiscardDeferredSessionSuspension({
error: makeAbortableWrapper(timeout),
}),
).toBe(true);
expect(
testing.shouldDiscardDeferredSessionSuspension({
error: makeAbortWrapper(createAgentRunRestartAbortError()),
}),
).toBe(true);
const providerTimeout = new Error("provider request timed out after 60s");
providerTimeout.name = "TimeoutError";
expect(
testing.shouldDiscardDeferredSessionSuspension({
error: makeAbortWrapper(providerTimeout),
}),
).toBe(false);
});
it("falls back normally when a provider wraps its own timeout as AbortError(cause: TimeoutError) WITHOUT the abortable() marker", async () => {
const cfg = makeCfg();
const providerInnerTimeout = new Error("provider request timed out after 60s");
providerInnerTimeout.name = "TimeoutError";
const unmarkedAbortError = new Error("aborted", { cause: providerInnerTimeout });
unmarkedAbortError.name = "AbortError";
const run = vi.fn().mockRejectedValueOnce(unmarkedAbortError).mockResolvedValueOnce("ok");
const result = await runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
});
expect(result.result).toBe("ok");
expect(run).toHaveBeenCalledTimes(2);
});
it("falls back normally when a top-level provider TimeoutError is thrown (not an AbortError wrapper)", async () => {
const cfg = makeCfg();
const directProviderTimeout = new Error("provider request timed out after 60s");
directProviderTimeout.name = "TimeoutError";
const run = vi.fn().mockRejectedValueOnce(directProviderTimeout).mockResolvedValueOnce("ok");
const result = await runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
});
expect(result.result).toBe("ok");
expect(run).toHaveBeenCalledTimes(2);
});
it("falls back normally when thrown error is generic AbortError without terminal cause", async () => {
const cfg = makeCfg();
const run = vi
.fn()
.mockRejectedValueOnce(new Error("provider transient failure"))
.mockResolvedValueOnce("ok");
const result = await runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
});
expect(result.result).toBe("ok");
expect(run).toHaveBeenCalledTimes(2);
});
it("skips fallback when the caller signal is aborted, even with a non-terminal reason", async () => {
const cfg = makeCfg();
const run = vi
.fn()
.mockRejectedValueOnce(new Error("provider had a sad day"))
.mockResolvedValueOnce("ok");
const genericReason = new Error("some unrelated abort");
const controller = makeTaggedAbortController(genericReason);
await expect(
runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
abortSignal: controller.signal,
}),
).rejects.toBeInstanceOf(Error);
expect(run).toHaveBeenCalledTimes(1);
});
it("falls back normally when no abortSignal is passed (back-compat)", async () => {
const cfg = makeCfg();
const run = vi
.fn()
.mockRejectedValueOnce(new Error("first attempt failed"))
.mockResolvedValueOnce("ok");
const result = await runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
});
expect(result.result).toBe("ok");
expect(run).toHaveBeenCalledTimes(2);
});
it("falls back normally when signal is provided but not aborted", async () => {
const cfg = makeCfg();
const run = vi
.fn()
.mockRejectedValueOnce(new Error("first attempt failed"))
.mockResolvedValueOnce("ok");
const controller = new AbortController();
const result = await runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
abortSignal: controller.signal,
});
expect(result.result).toBe("ok");
expect(run).toHaveBeenCalledTimes(2);
});
it("rethrows terminal abort even when error resembles a failover-normalizable error", async () => {
const cfg = makeCfg();
const rateLimitLikeError = Object.assign(new Error("RESOURCE_EXHAUSTED: quota exceeded"), {
status: 429,
name: "AbortError",
});
const run = vi.fn().mockRejectedValue(rateLimitLikeError);
const timeoutReason = new Error("request timed out");
timeoutReason.name = "TimeoutError";
const controller = new AbortController();
controller.abort(timeoutReason);
await expect(
runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
abortSignal: controller.signal,
}),
).rejects.toBe(rateLimitLikeError);
expect(run).toHaveBeenCalledTimes(1);
});
it("treats cron timeout string reason as terminal (covers plain-string abort)", async () => {
const cfg = makeCfg();
const run = vi.fn().mockRejectedValue(makeAbortError("aborted"));
const controller = new AbortController();
controller.abort("cron: job execution timed out");
await expect(
runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
abortSignal: controller.signal,
}),
).rejects.toBeInstanceOf(Error);
expect(run).toHaveBeenCalledTimes(1);
});
it("treats phase-suffixed cron timeout reason as terminal (covers `(last phase: ...)` variant)", async () => {
const cfg = makeCfg();
const run = vi.fn().mockRejectedValue(makeAbortError("aborted"));
const controller = new AbortController();
controller.abort("cron: job execution timed out (last phase: model_call_started)");
await expect(
runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
abortSignal: controller.signal,
}),
).rejects.toBeInstanceOf(Error);
expect(run).toHaveBeenCalledTimes(1);
});
it("treats isolated-agent setup-timeout (and phase suffix) as terminal", async () => {
const cfg = makeCfg();
const run = vi.fn().mockRejectedValue(makeAbortError("aborted"));
const controller = new AbortController();
controller.abort(
"cron: isolated agent setup timed out before runner start (last phase: workspace_provision)",
);
await expect(
runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
abortSignal: controller.signal,
}),
).rejects.toBeInstanceOf(Error);
expect(run).toHaveBeenCalledTimes(1);
});
it("treats isolated-agent pre-execution stall (and phase suffix) as terminal", async () => {
const cfg = makeCfg();
const run = vi.fn().mockRejectedValue(makeAbortError("aborted"));
const controller = new AbortController();
controller.abort(
"cron: isolated agent run stalled before execution start (last phase: runner_ready)",
);
await expect(
runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
abortSignal: controller.signal,
}),
).rejects.toBeInstanceOf(Error);
expect(run).toHaveBeenCalledTimes(1);
});
it("treats bare isolated-agent pre-execution stall as terminal", async () => {
const cfg = makeCfg();
const run = vi.fn().mockRejectedValue(makeAbortError("aborted"));
const controller = new AbortController();
controller.abort("cron: isolated agent run stalled before execution start");
await expect(
runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
abortSignal: controller.signal,
}),
).rejects.toBeInstanceOf(Error);
expect(run).toHaveBeenCalledTimes(1);
});
it("treats an Error whose .message matches a known terminal string as terminal", async () => {
const cfg = makeCfg();
const run = vi.fn().mockRejectedValue(makeAbortError("aborted"));
const wrapped = new Error("cron: job execution timed out");
const controller = new AbortController();
controller.abort(wrapped);
await expect(
runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
abortSignal: controller.signal,
}),
).rejects.toBeInstanceOf(Error);
expect(run).toHaveBeenCalledTimes(1);
});
it("does not classify an unrelated string reason as terminal (caller-abort still skips fallback)", async () => {
const cfg = makeCfg();
const run = vi
.fn()
.mockRejectedValueOnce(new Error("first attempt failed"))
.mockResolvedValueOnce("ok");
const controller = new AbortController();
controller.abort("some unrelated cancel reason");
await expect(
runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-sonnet-4-6",
run,
abortSignal: controller.signal,
}),
).rejects.toBeInstanceOf(Error);
expect(run).toHaveBeenCalledTimes(1);
});
});
});
describe("runWithImageModelFallback", () => {
+65 -5
View File
@@ -8,6 +8,7 @@ import {
resolveAgentModelPrimaryValue,
} from "../config/model-input.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { isCronTerminalAbortReasonText } from "../cron/service/execution-errors.js";
import { emitFailoverEvent } from "../infra/diagnostic-events.js";
import { formatErrorMessage, toErrorObject } from "../infra/errors.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
@@ -31,6 +32,7 @@ import { isActiveUnusableWindow } from "./auth-profiles/usage-state.js";
import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "./defaults.js";
import { isLikelyContextOverflowError } from "./embedded-agent-helpers/errors.js";
import type { FailoverReason } from "./embedded-agent-helpers/types.js";
import { isOpenClawAbortableWrapper } from "./embedded-agent-runner/run/abortable.js";
import {
FailoverError,
buildFailoverRemediationHint,
@@ -188,21 +190,74 @@ type ModelFallbackRunFn<T> = (
options?: ModelFallbackRunOptions,
) => Promise<T>;
function isTerminalAbortReasonString(reason: string): boolean {
return isCronTerminalAbortReasonText(reason);
}
function getErrorCauseCandidates(err: Error): unknown[] {
const candidates: unknown[] = [];
if ("cause" in err && err.cause !== undefined) {
candidates.push(err.cause);
if (err.cause instanceof Error && "cause" in err.cause && err.cause.cause !== undefined) {
candidates.push(err.cause.cause);
}
}
return candidates;
}
function isTerminalAbortCandidate(candidate: unknown): boolean {
if (typeof candidate === "string") {
return isTerminalAbortReasonString(candidate);
}
if (!(candidate instanceof Error)) {
return false;
}
if (isAgentRunRestartAbortReason(candidate)) {
return true;
}
if (candidate.name === "TimeoutError") {
return true;
}
if (candidate.name === "ClientDisconnectError") {
return true;
}
return isTerminalAbortReasonString(candidate.message);
}
function isTerminalAbort(signal: AbortSignal | undefined): boolean {
if (!signal?.aborted) {
return false;
}
const reason = signal.reason;
if (!(reason instanceof Error)) {
if (reason instanceof Error) {
const candidates: unknown[] = [reason, ...getErrorCauseCandidates(reason)];
return candidates.some(isTerminalAbortCandidate);
}
return isTerminalAbortCandidate(reason);
}
function isTerminalAbortFromError(err: unknown): boolean {
if (!(err instanceof Error)) {
return false;
}
if (isAgentRunRestartAbortReason(reason)) {
if (isAgentRunRestartAbortReason(err)) {
return true;
}
if (reason.name === "TimeoutError") {
return true;
const causeCandidates = getErrorCauseCandidates(err);
if (err.name !== "AbortError") {
return false;
}
return reason.name === "ClientDisconnectError";
for (const candidate of causeCandidates) {
if (isAgentRunRestartAbortReason(candidate)) {
return true;
}
}
if (!isOpenClawAbortableWrapper(err)) {
return false;
}
return causeCandidates.some(isTerminalAbortCandidate);
}
function isCallerAbortSignal(signal: AbortSignal | undefined): boolean {
@@ -357,6 +412,9 @@ async function runFallbackCandidate<T>(params: {
if (isAgentRunDirectAbortReason(err) || isAgentRunRestartAbortReason(err)) {
throw err;
}
if (isTerminalAbortFromError(err)) {
throw err;
}
// Normalize abort-wrapped rate-limit errors (e.g. Google Vertex RESOURCE_EXHAUSTED)
// so they become FailoverErrors and continue the fallback loop instead of aborting.
const normalizedFailover = coerceToFailoverError(err, {
@@ -809,6 +867,7 @@ export const testing = {
resolveImageFallbackCandidates,
resolveCooldownDecision,
resolveSessionSuspensionReason,
shouldDiscardDeferredSessionSuspension,
} as const;
export function resolveModelCandidateChain(
@@ -1310,6 +1369,7 @@ function shouldDiscardDeferredSessionSuspension(params: {
isCallerAbortSignal(params.abortSignal) ||
isAgentRunDirectAbortReason(params.error) ||
isAgentRunRestartAbortReason(params.error) ||
isTerminalAbortFromError(params.error) ||
isCommandLaneTaskTimeoutError(params.error) ||
isNonProviderRuntimeCoordinationError(params.error) ||
isLikelyContextOverflowError(formatErrorMessage(params.error))
+29 -7
View File
@@ -6,36 +6,58 @@ function formatCronAgentExecutionPhase(execution?: CronAgentExecutionStarted): s
return formatEmbeddedAgentExecutionPhase(execution?.phase);
}
const CRON_JOB_EXECUTION_TIMEOUT_ERROR = "cron: job execution timed out";
const CRON_SETUP_TIMEOUT_ERROR = "cron: isolated agent setup timed out before runner start";
const CRON_PRE_EXECUTION_TIMEOUT_ERROR = "cron: isolated agent run stalled before execution start";
const CRON_TIMEOUT_ERROR_PREFIXES: readonly string[] = [
CRON_JOB_EXECUTION_TIMEOUT_ERROR,
CRON_SETUP_TIMEOUT_ERROR,
CRON_PRE_EXECUTION_TIMEOUT_ERROR,
];
function hasCronTimeoutPrefix(error: string, prefix: string): boolean {
return error === prefix || error.startsWith(prefix + " ");
}
export function isCronTerminalAbortReasonText(error: string): boolean {
for (const prefix of CRON_TIMEOUT_ERROR_PREFIXES) {
if (hasCronTimeoutPrefix(error, prefix)) {
return true;
}
}
return false;
}
/** Formats the generic cron execution timeout message with last-known phase context when available. */
export function timeoutErrorMessage(execution?: CronAgentExecutionStarted): string {
const phase = formatCronAgentExecutionPhase(execution);
if (!phase) {
return "cron: job execution timed out";
return CRON_JOB_EXECUTION_TIMEOUT_ERROR;
}
return `cron: job execution timed out (last phase: ${phase})`;
return `${CRON_JOB_EXECUTION_TIMEOUT_ERROR} (last phase: ${phase})`;
}
/** Formats timeout text for runs that stalled before the isolated runner started. */
export function setupTimeoutErrorMessage(execution?: CronAgentExecutionStarted): string {
const phase = formatCronAgentExecutionPhase(execution);
if (!phase) {
return "cron: isolated agent setup timed out before runner start";
return CRON_SETUP_TIMEOUT_ERROR;
}
return `cron: isolated agent setup timed out before runner start (last phase: ${phase})`;
return `${CRON_SETUP_TIMEOUT_ERROR} (last phase: ${phase})`;
}
/** Returns true for the setup-timeout class that fires before the isolated runner starts. */
export function isSetupTimeoutErrorText(error: string): boolean {
return error.startsWith("cron: isolated agent setup timed out before runner start");
return hasCronTimeoutPrefix(error, CRON_SETUP_TIMEOUT_ERROR);
}
/** Formats timeout text for runs that stalled after setup but before execution start. */
export function preExecutionTimeoutErrorMessage(execution?: CronAgentExecutionStarted): string {
const phase = formatCronAgentExecutionPhase(execution);
if (!phase) {
return "cron: isolated agent run stalled before execution start";
return CRON_PRE_EXECUTION_TIMEOUT_ERROR;
}
return `cron: isolated agent run stalled before execution start (last phase: ${phase})`;
return `${CRON_PRE_EXECUTION_TIMEOUT_ERROR} (last phase: ${phase})`;
}
/** Extracts a human timeout/abort reason, falling back to the canonical cron timeout text. */
+9 -1
View File
@@ -134,6 +134,14 @@ export function setSseHeaders(res: ServerResponse) {
res.flushHeaders?.();
}
/** Abort reason used when the HTTP client disconnects before delivery. */
export class ClientDisconnectError extends Error {
constructor(message = "HTTP client disconnected") {
super(message);
this.name = "ClientDisconnectError";
}
}
export function watchClientDisconnect(
req: IncomingMessage,
res: ServerResponse,
@@ -153,7 +161,7 @@ export function watchClientDisconnect(
const handleClose = () => {
onDisconnect?.();
if (!abortController.signal.aborted) {
abortController.abort();
abortController.abort(new ClientDisconnectError());
}
};
for (const socket of sockets) {
+1
View File
@@ -822,6 +822,7 @@ function buildArtifactsCapture(params: {
runtimeArtifacts?.timedOutDuringCompaction ?? runtimeEnd?.timedOutDuringCompaction,
timedOutDuringToolExecution:
runtimeArtifacts?.timedOutDuringToolExecution ?? runtimeEnd?.timedOutDuringToolExecution,
timedOutByRunBudget: runtimeArtifacts?.timedOutByRunBudget ?? runtimeEnd?.timedOutByRunBudget,
promptError:
runtimeArtifacts?.promptError ?? runtimeEnd?.promptError ?? runtimeCompletion?.promptError,
promptErrorSource: runtimeArtifacts?.promptErrorSource ?? runtimeCompletion?.promptErrorSource,
+1
View File
@@ -268,6 +268,7 @@ describe("trajectory metadata", () => {
idleTimedOut: false,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
timedOutByRunBudget: false,
compactionCount: 1,
assistantTexts: ["done"],
finalPromptText: "run tests",
+2
View File
@@ -50,6 +50,7 @@ type BuildTrajectoryArtifactsParams = {
idleTimedOut: boolean;
timedOutDuringCompaction: boolean;
timedOutDuringToolExecution: boolean;
timedOutByRunBudget: boolean;
promptError?: string;
promptErrorSource?: string | null;
terminalError?: string;
@@ -321,6 +322,7 @@ export function buildTrajectoryArtifacts(
idleTimedOut: params.idleTimedOut,
timedOutDuringCompaction: params.timedOutDuringCompaction,
timedOutDuringToolExecution: params.timedOutDuringToolExecution,
timedOutByRunBudget: params.timedOutByRunBudget,
promptError: params.promptError,
promptErrorSource: params.promptErrorSource,
terminalError: params.terminalError,