diff --git a/src/agents/embedded-agent-runner/compact.abort-signal.test.ts b/src/agents/embedded-agent-runner/compact.abort-signal.test.ts new file mode 100644 index 000000000000..addca0a80ebe --- /dev/null +++ b/src/agents/embedded-agent-runner/compact.abort-signal.test.ts @@ -0,0 +1,74 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { OpenClawConfig } from "../../config/types.openclaw.js"; + +vi.mock("../model-fallback.js", () => ({ + runWithModelFallback: vi.fn(async (params: Record) => ({ + result: { ok: true, compacted: false, reason: "no-op" }, + provider: params.provider, + model: params.model, + attempts: [], + })), + isFallbackSummaryError: () => false, +})); + +vi.mock("./compact.queued.js", () => ({ compactEmbeddedAgentSession: vi.fn() })); + +import { runWithModelFallback } from "../model-fallback.js"; +import { compactEmbeddedAgentSessionDirect } from "./compact.js"; + +const runMock = vi.mocked(runWithModelFallback); + +const baseParams = { + sessionId: "test-session", + sessionKey: "agent:main:test-session", + sessionFile: "/tmp/test-session.jsonl", + workspaceDir: "/tmp", +}; + +function configWithFallbacks(fallbacks: string[]): OpenClawConfig { + return { + agents: { + defaults: { + model: { + primary: "anthropic/claude-sonnet-4-6", + fallbacks, + }, + }, + }, + } as OpenClawConfig; +} + +describe("compactEmbeddedAgentSessionDirect abortSignal threading", () => { + beforeEach(() => { + runMock.mockClear(); + }); + + it("forwards params.abortSignal to runWithModelFallback so terminal aborts during compaction short-circuit", async () => { + const controller = new AbortController(); + + await compactEmbeddedAgentSessionDirect({ + ...baseParams, + config: configWithFallbacks(["anthropic/claude-haiku-4-5", "openai/gpt-4.1-mini"]), + provider: "anthropic", + model: "claude-sonnet-4-6", + abortSignal: controller.signal, + }); + + expect(runMock).toHaveBeenCalledTimes(1); + const passedParams = runMock.mock.calls[0]?.[0]; + expect(passedParams?.abortSignal).toBe(controller.signal); + }); + + it("passes undefined when no abortSignal is set (back-compat)", async () => { + await compactEmbeddedAgentSessionDirect({ + ...baseParams, + config: configWithFallbacks(["anthropic/claude-haiku-4-5"]), + provider: "anthropic", + model: "claude-sonnet-4-6", + }); + + expect(runMock).toHaveBeenCalledTimes(1); + const passedParams = runMock.mock.calls[0]?.[0]; + expect(passedParams?.abortSignal).toBeUndefined(); + }); +}); diff --git a/src/agents/embedded-agent-runner/run.ts b/src/agents/embedded-agent-runner/run.ts index 45d38b801890..00591d56139a 100644 --- a/src/agents/embedded-agent-runner/run.ts +++ b/src/agents/embedded-agent-runner/run.ts @@ -2325,6 +2325,7 @@ async function runEmbeddedAgentInternal( attempt.setTerminalLifecycleMeta?.({ ...meta, aborted }); }; const timedOutDuringToolExecution = attempt.timedOutDuringToolExecution ?? false; + const timedOutByRunBudget = attempt.timedOutByRunBudget ?? false; adoptActiveSessionId(sessionIdUsed); if (sessionFileUsed && sessionFileUsed !== activeSessionFile) { activeSessionFile = sessionFileUsed; @@ -2502,10 +2503,12 @@ async function runEmbeddedAgentInternal( ); throw new LiveSessionModelSwitchError(requestedSelection); } - // ── Timeout-triggered compaction ────────────────────────────────── - // When the LLM times out with high context usage, compact before - // retrying to break the death spiral of repeated timeouts. - if (timedOut && !timedOutDuringCompaction && !timedOutDuringToolExecution) { + if ( + timedOut && + !timedOutDuringCompaction && + !timedOutDuringToolExecution && + !timedOutByRunBudget + ) { // Only consider prompt-side tokens here. API totals include output // tokens, which can make a long generation look like high context // pressure even when the prompt itself was small. @@ -3241,6 +3244,7 @@ async function runEmbeddedAgentInternal( failoverReason: promptFailoverReason, harnessOwnsTransport: pluginHarnessOwnsTransport, promptTimeoutFallbackSafe, + timedOutByRunBudget, profileRotated: false, }); if ( @@ -3281,6 +3285,7 @@ async function runEmbeddedAgentInternal( failoverReason: promptFailoverReason, harnessOwnsTransport: pluginHarnessOwnsTransport, promptTimeoutFallbackSafe, + timedOutByRunBudget, profileRotated: true, }); } @@ -3483,6 +3488,7 @@ async function runEmbeddedAgentInternal( timedOutDuringCompaction, timedOutDuringToolExecution, harnessOwnsTransport: pluginHarnessOwnsTransport, + timedOutByRunBudget, profileRotated: false, }); const assistantFailoverOutcome = await handleAssistantFailover({ @@ -3496,6 +3502,7 @@ async function runEmbeddedAgentInternal( idleTimedOut, timedOutDuringCompaction, timedOutDuringToolExecution, + timedOutByRunBudget, allowSameModelIdleTimeoutRetry: timedOut && idleTimedOut && diff --git a/src/agents/embedded-agent-runner/run/abortable.ts b/src/agents/embedded-agent-runner/run/abortable.ts index 14e35959f908..cd014dca5bfe 100644 --- a/src/agents/embedded-agent-runner/run/abortable.ts +++ b/src/agents/embedded-agent-runner/run/abortable.ts @@ -7,16 +7,28 @@ function getAbortReason(signal: AbortSignal): unknown { return "reason" in signal ? (signal as { reason?: unknown }).reason : undefined; } +/** Marks AbortErrors produced by abortable() so provider aborts stay retryable. */ +export const OPENCLAW_ABORTABLE_WRAPPER = Symbol.for("openclaw.abortable.wrapper"); + +export function isOpenClawAbortableWrapper(err: unknown): boolean { + return err !== null && typeof err === "object" && OPENCLAW_ABORTABLE_WRAPPER in err; +} + +function tagAsAbortableWrapper(err: Error): Error { + (err as Error & { [OPENCLAW_ABORTABLE_WRAPPER]?: true })[OPENCLAW_ABORTABLE_WRAPPER] = true; + return err; +} + function makeAbortError(signal: AbortSignal): Error { const reason = getAbortReason(signal); if (reason instanceof Error) { const err = new Error(reason.message, { cause: reason }); err.name = "AbortError"; - return err; + return tagAsAbortableWrapper(err); } const err = reason ? new Error("aborted", { cause: reason }) : new Error("aborted"); err.name = "AbortError"; - return err; + return tagAsAbortableWrapper(err); } /** diff --git a/src/agents/embedded-agent-runner/run/assistant-failover.test.ts b/src/agents/embedded-agent-runner/run/assistant-failover.test.ts index 66926f4d9f44..3c8b8330fcef 100644 --- a/src/agents/embedded-agent-runner/run/assistant-failover.test.ts +++ b/src/agents/embedded-agent-runner/run/assistant-failover.test.ts @@ -23,6 +23,7 @@ function makeParams(overrides: Partial = {}): Params { idleTimedOut: false, timedOutDuringCompaction: false, timedOutDuringToolExecution: false, + timedOutByRunBudget: false, allowSameModelIdleTimeoutRetry: false, allowSameModelRateLimitRetry: true, assistantProfileFailureReason: null, diff --git a/src/agents/embedded-agent-runner/run/assistant-failover.ts b/src/agents/embedded-agent-runner/run/assistant-failover.ts index 5a90155f6e8c..efadcc32a8ac 100644 --- a/src/agents/embedded-agent-runner/run/assistant-failover.ts +++ b/src/agents/embedded-agent-runner/run/assistant-failover.ts @@ -130,6 +130,7 @@ export async function handleAssistantFailover(params: { idleTimedOut: boolean; timedOutDuringCompaction: boolean; timedOutDuringToolExecution: boolean; + timedOutByRunBudget: boolean; allowSameModelIdleTimeoutRetry: boolean; allowSameModelRateLimitRetry: boolean; assistantProfileFailureReason: AuthProfileFailureReason | null; @@ -309,6 +310,7 @@ export async function handleAssistantFailover(params: { idleTimedOut: params.idleTimedOut, timedOutDuringCompaction: params.timedOutDuringCompaction, timedOutDuringToolExecution: params.timedOutDuringToolExecution, + timedOutByRunBudget: params.timedOutByRunBudget, profileRotated: true, }); } diff --git a/src/agents/embedded-agent-runner/run/attempt.ts b/src/agents/embedded-agent-runner/run/attempt.ts index c07cd5fe5d04..bb9d8014decd 100644 --- a/src/agents/embedded-agent-runner/run/attempt.ts +++ b/src/agents/embedded-agent-runner/run/attempt.ts @@ -918,6 +918,7 @@ export async function runEmbeddedAttempt( let idleTimedOut = false; let timedOutDuringCompaction = false; let timedOutDuringToolExecution = false; + let timedOutByRunBudget = false; let promptError: unknown = null; let emitDiagnosticRunCompleted: | (( @@ -3819,6 +3820,7 @@ export async function runEmbeddedAttempt( ) { timedOutDuringCompaction = true; } + timedOutByRunBudget = true; abortRun(true); if (!abortWarnTimer) { abortWarnTimer = setTimeout(() => { @@ -5668,6 +5670,7 @@ export async function runEmbeddedAttempt( idleTimedOut, timedOutDuringCompaction, timedOutDuringToolExecution, + timedOutByRunBudget, promptError: promptError ? formatErrorMessage(promptError) : undefined, promptErrorSource, terminalError: attemptTrajectoryTerminal.terminalError, @@ -5688,6 +5691,7 @@ export async function runEmbeddedAttempt( idleTimedOut, timedOutDuringCompaction, timedOutDuringToolExecution, + timedOutByRunBudget, promptError: promptError ? formatErrorMessage(promptError) : undefined, promptErrorSource, terminalError: attemptTrajectoryTerminal.terminalError, @@ -5714,6 +5718,7 @@ export async function runEmbeddedAttempt( idleTimedOut, timedOutDuringCompaction, timedOutDuringToolExecution, + timedOutByRunBudget, promptError: promptError ? formatErrorMessage(promptError) : undefined, terminalError: attemptTrajectoryTerminal.terminalError, }); @@ -5729,6 +5734,7 @@ export async function runEmbeddedAttempt( idleTimedOut, timedOutDuringCompaction, timedOutDuringToolExecution, + timedOutByRunBudget, promptError, promptErrorSource, preflightRecovery, @@ -5785,6 +5791,7 @@ export async function runEmbeddedAttempt( idleTimedOut, timedOutDuringCompaction, timedOutDuringToolExecution, + timedOutByRunBudget, promptError: promptError ? formatErrorMessage(promptError) : undefined, }); } diff --git a/src/agents/embedded-agent-runner/run/failover-policy.test.ts b/src/agents/embedded-agent-runner/run/failover-policy.test.ts index b36cfa6cf972..1c90ceab0c80 100644 --- a/src/agents/embedded-agent-runner/run/failover-policy.test.ts +++ b/src/agents/embedded-agent-runner/run/failover-policy.test.ts @@ -95,6 +95,43 @@ describe("resolveRunFailoverDecision", () => { }); }); + it("surfaces prompt run-budget timeouts instead of model fallback (#60388)", () => { + expect( + resolveRunFailoverDecision({ + stage: "prompt", + aborted: true, + externalAbort: false, + fallbackConfigured: true, + failoverFailure: true, + failoverReason: "timeout", + promptTimeoutFallbackSafe: true, + timedOutByRunBudget: true, + profileRotated: true, + }), + ).toEqual({ + action: "surface_error", + reason: "timeout", + }); + }); + + it("does not rotate prompt failures after the run budget is exhausted (#60388)", () => { + expect( + resolveRunFailoverDecision({ + stage: "prompt", + aborted: true, + externalAbort: false, + fallbackConfigured: true, + failoverFailure: true, + failoverReason: "rate_limit", + timedOutByRunBudget: true, + profileRotated: false, + }), + ).toEqual({ + action: "surface_error", + reason: "rate_limit", + }); + }); + it("surfaces deterministic prompt format failures instead of rotating or falling back", () => { expect( resolveRunFailoverDecision({ @@ -145,6 +182,7 @@ describe("resolveRunFailoverDecision", () => { idleTimedOut: false, timedOutDuringCompaction: false, timedOutDuringToolExecution: false, + timedOutByRunBudget: false, profileRotated: false, }), ).toEqual({ @@ -165,6 +203,7 @@ describe("resolveRunFailoverDecision", () => { idleTimedOut: false, timedOutDuringCompaction: false, timedOutDuringToolExecution: false, + timedOutByRunBudget: false, profileRotated: false, }), ).toEqual({ @@ -187,6 +226,7 @@ describe("resolveRunFailoverDecision", () => { idleTimedOut: false, timedOutDuringCompaction: false, timedOutDuringToolExecution: false, + timedOutByRunBudget: false, profileRotated: false, }), ).toEqual({ @@ -208,6 +248,7 @@ describe("resolveRunFailoverDecision", () => { idleTimedOut: false, timedOutDuringCompaction: false, timedOutDuringToolExecution: false, + timedOutByRunBudget: false, profileRotated: true, }), ).toEqual({ @@ -229,6 +270,7 @@ describe("resolveRunFailoverDecision", () => { idleTimedOut: false, timedOutDuringCompaction: false, timedOutDuringToolExecution: false, + timedOutByRunBudget: false, profileRotated: true, }), ).toEqual({ @@ -249,6 +291,7 @@ describe("resolveRunFailoverDecision", () => { idleTimedOut: false, timedOutDuringCompaction: false, timedOutDuringToolExecution: false, + timedOutByRunBudget: false, profileRotated: false, }), ).toEqual({ @@ -286,6 +329,7 @@ describe("resolveRunFailoverDecision", () => { idleTimedOut: false, timedOutDuringCompaction: false, timedOutDuringToolExecution: true, + timedOutByRunBudget: false, profileRotated: false, }), ).toEqual({ @@ -348,6 +392,7 @@ describe("resolveRunFailoverDecision", () => { idleTimedOut: false, timedOutDuringCompaction: false, timedOutDuringToolExecution: true, + timedOutByRunBudget: false, profileRotated: true, }), ).toEqual({ @@ -368,6 +413,7 @@ describe("resolveRunFailoverDecision", () => { idleTimedOut: false, timedOutDuringCompaction: false, timedOutDuringToolExecution: false, + timedOutByRunBudget: false, profileRotated: false, }), ).toEqual({ @@ -477,6 +523,7 @@ describe("resolveRunFailoverDecision", () => { idleTimedOut: true, timedOutDuringCompaction: false, timedOutDuringToolExecution: true, + timedOutByRunBudget: false, profileRotated: false, }), ).toEqual({ @@ -498,6 +545,7 @@ describe("resolveRunFailoverDecision", () => { idleTimedOut: true, timedOutDuringCompaction: false, timedOutDuringToolExecution: true, + timedOutByRunBudget: false, profileRotated: true, }), ).toEqual({ @@ -519,6 +567,7 @@ describe("resolveRunFailoverDecision", () => { idleTimedOut: false, timedOutDuringCompaction: false, timedOutDuringToolExecution: false, + timedOutByRunBudget: false, profileRotated: false, }), ).toEqual({ @@ -540,6 +589,7 @@ describe("resolveRunFailoverDecision", () => { idleTimedOut: true, timedOutDuringCompaction: false, timedOutDuringToolExecution: false, + timedOutByRunBudget: false, profileRotated: false, }), ).toEqual({ @@ -561,6 +611,7 @@ describe("resolveRunFailoverDecision", () => { idleTimedOut: true, timedOutDuringCompaction: false, timedOutDuringToolExecution: false, + timedOutByRunBudget: false, profileRotated: true, }), ).toEqual({ @@ -659,6 +710,7 @@ describe("resolveRunFailoverDecision", () => { idleTimedOut: true, timedOutDuringCompaction: false, timedOutDuringToolExecution: false, + timedOutByRunBudget: false, profileRotated: true, }), ).toEqual({ @@ -680,6 +732,7 @@ describe("resolveRunFailoverDecision", () => { idleTimedOut: true, timedOutDuringCompaction: false, timedOutDuringToolExecution: false, + timedOutByRunBudget: false, profileRotated: false, }), ).toEqual({ @@ -687,6 +740,48 @@ describe("resolveRunFailoverDecision", () => { reason: null, }); }); + + it("does not rotate or fallback assistant timeouts that exhausted the run budget (#60388)", () => { + expect( + resolveRunFailoverDecision({ + stage: "assistant", + aborted: true, + externalAbort: false, + fallbackConfigured: true, + failoverFailure: false, + failoverReason: null, + timedOut: true, + idleTimedOut: false, + timedOutDuringCompaction: false, + timedOutDuringToolExecution: false, + timedOutByRunBudget: true, + profileRotated: false, + }), + ).toEqual({ + action: "continue_normal", + }); + }); + + it("does not fallback assistant run-budget timeouts even after profile rotation exhausted (#60388)", () => { + expect( + resolveRunFailoverDecision({ + stage: "assistant", + aborted: true, + externalAbort: false, + fallbackConfigured: true, + failoverFailure: false, + failoverReason: null, + timedOut: true, + idleTimedOut: false, + timedOutDuringCompaction: false, + timedOutDuringToolExecution: false, + timedOutByRunBudget: true, + profileRotated: true, + }), + ).toEqual({ + action: "continue_normal", + }); + }); }); describe("mergeRetryFailoverReason", () => { diff --git a/src/agents/embedded-agent-runner/run/failover-policy.ts b/src/agents/embedded-agent-runner/run/failover-policy.ts index 983fa0f21bc5..12af742dd6fb 100644 --- a/src/agents/embedded-agent-runner/run/failover-policy.ts +++ b/src/agents/embedded-agent-runner/run/failover-policy.ts @@ -51,6 +51,7 @@ type PromptDecisionParams = { failoverReason: FailoverReason | null; harnessOwnsTransport?: boolean; promptTimeoutFallbackSafe?: boolean; + timedOutByRunBudget?: boolean; profileRotated: boolean; }; @@ -67,6 +68,7 @@ type AssistantDecisionParams = { timedOutDuringCompaction: boolean; timedOutDuringToolExecution: boolean; harnessOwnsTransport?: boolean; + timedOutByRunBudget?: boolean; profileRotated: boolean; }; @@ -92,6 +94,9 @@ function isTerminalFormatFailure(params: { } function shouldRotatePrompt(params: PromptDecisionParams): boolean { + if (params.timedOutByRunBudget) { + return false; + } return ( params.failoverFailure && params.failoverReason !== "timeout" && @@ -116,6 +121,9 @@ function shouldRotateAssistant(params: AssistantDecisionParams): boolean { if (isTerminalFormatFailure(params)) { return false; } + if (params.timedOutByRunBudget) { + return false; + } const timeoutFailure = isAssistantTimeoutFailure(params); const harnessOwnedTimeout = params.harnessOwnsTransport && (timeoutFailure || params.failoverReason === "timeout"); @@ -175,6 +183,12 @@ export function resolveRunFailoverDecision(params: RunFailoverDecisionParams): R reason: params.failoverReason, }; } + if (params.timedOutByRunBudget) { + return { + action: "surface_error", + reason: params.failoverReason, + }; + } if (params.harnessOwnsTransport && params.failoverReason === "timeout") { // Plugin harness lifecycle timeouts must stay inside the harness boundary; // only prompt request timeouts proven replay-safe may enter model fallback. diff --git a/src/agents/embedded-agent-runner/run/types.ts b/src/agents/embedded-agent-runner/run/types.ts index 076705ab95d3..b7fa980dfafc 100644 --- a/src/agents/embedded-agent-runner/run/types.ts +++ b/src/agents/embedded-agent-runner/run/types.ts @@ -125,6 +125,7 @@ export type EmbeddedRunAttemptResult = { timedOutDuringCompaction: boolean; /** Optional because this type is re-exported as `AgentHarnessAttemptResult`. */ timedOutDuringToolExecution?: boolean; + timedOutByRunBudget?: boolean; promptError: unknown; /** * Identifies which phase produced the promptError. diff --git a/src/agents/model-fallback.test.ts b/src/agents/model-fallback.test.ts index ef52f7410d97..17574e6b1b72 100644 --- a/src/agents/model-fallback.test.ts +++ b/src/agents/model-fallback.test.ts @@ -18,6 +18,7 @@ import { CommandLaneTaskTimeoutError } from "../process/command-queue.js"; import { AUTH_STORE_VERSION } from "./auth-profiles/constants.js"; import type { AuthProfileStore } from "./auth-profiles/types.js"; import { classifyEmbeddedAgentRunResultForModelFallback } from "./embedded-agent-runner/result-fallback-classifier.js"; +import { OPENCLAW_ABORTABLE_WRAPPER } from "./embedded-agent-runner/run/abortable.js"; import type { EmbeddedAgentRunResult } from "./embedded-agent-runner/types.js"; import { FailoverError } from "./failover-error.js"; import { resetFallbackSkipCacheForTest } from "./fallback-skip-cache.js"; @@ -3459,6 +3460,489 @@ describe("runWithModelFallback", () => { }); }); }); + + describe("terminal abort propagation", () => { + function makeAbortError(message = "aborted"): Error { + const err = new Error(message); + err.name = "AbortError"; + return err; + } + + function makeAbortableWrapper(reason: Error): Error { + const err = new Error(reason.message, { cause: reason }); + err.name = "AbortError"; + (err as Error & { [OPENCLAW_ABORTABLE_WRAPPER]?: true })[OPENCLAW_ABORTABLE_WRAPPER] = true; + return err; + } + + function makeAbortWrapper(reason: Error): Error { + const err = new Error("aborted", { cause: reason }); + err.name = "AbortError"; + return err; + } + + function makeTaggedAbortController(reason: Error): AbortController { + const controller = new AbortController(); + controller.abort(reason); + return controller; + } + + it("rethrows immediately when signal.reason has name=TimeoutError (run-budget timeout)", async () => { + const cfg = makeCfg(); + const runError = makeAbortError("aborted"); + const run = vi.fn().mockRejectedValue(runError); + + const timeoutReason = new Error("request timed out"); + timeoutReason.name = "TimeoutError"; + const controller = makeTaggedAbortController(timeoutReason); + + await expect( + runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + abortSignal: controller.signal, + }), + ).rejects.toBe(runError); + + expect(run).toHaveBeenCalledTimes(1); + }); + + it("rethrows immediately when signal.reason has name=ClientDisconnectError", async () => { + const cfg = makeCfg(); + const runError = makeAbortError("aborted"); + const run = vi.fn().mockRejectedValue(runError); + + const disconnectReason = new Error("HTTP client disconnected"); + disconnectReason.name = "ClientDisconnectError"; + const controller = makeTaggedAbortController(disconnectReason); + + await expect( + runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + abortSignal: controller.signal, + }), + ).rejects.toBe(runError); + + expect(run).toHaveBeenCalledTimes(1); + }); + + it("detects TimeoutError nested as cause of an outer Error", async () => { + const cfg = makeCfg(); + const runError = makeAbortError("aborted"); + const run = vi.fn().mockRejectedValue(runError); + + const innerTimeout = new Error("request timed out"); + innerTimeout.name = "TimeoutError"; + const outerWrap = makeAbortableWrapper(innerTimeout); + const controller = makeTaggedAbortController(outerWrap); + + await expect( + runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + abortSignal: controller.signal, + }), + ).rejects.toBe(runError); + + expect(run).toHaveBeenCalledTimes(1); + }); + + it("rethrows when thrown error has TimeoutError in cause chain (embedded run-budget timer)", async () => { + const cfg = makeCfg(); + const innerTimeout = new Error("request timed out"); + innerTimeout.name = "TimeoutError"; + const outerAbort = makeAbortableWrapper(innerTimeout); + const run = vi.fn().mockRejectedValue(outerAbort); + + await expect( + runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + }), + ).rejects.toBe(outerAbort); + + expect(run).toHaveBeenCalledTimes(1); + }); + + it("rethrows when thrown error has ClientDisconnectError in cause chain", async () => { + const cfg = makeCfg(); + const innerDisconnect = new Error("client disconnected"); + innerDisconnect.name = "ClientDisconnectError"; + const outerAbort = makeAbortableWrapper(innerDisconnect); + const run = vi.fn().mockRejectedValue(outerAbort); + + await expect( + runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + }), + ).rejects.toBe(outerAbort); + + expect(run).toHaveBeenCalledTimes(1); + }); + + it("rethrows when thrown error has restart abort in cause chain", async () => { + const cfg = makeCfg(); + const restartAbort = createAgentRunRestartAbortError(); + const outerAbort = makeAbortableWrapper(restartAbort); + const run = vi.fn().mockRejectedValueOnce(outerAbort).mockResolvedValueOnce("ok"); + + await expect( + runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + }), + ).rejects.toBe(outerAbort); + + expect(run).toHaveBeenCalledTimes(1); + }); + + it("rethrows when an unmarked AbortError wraps a restart abort", async () => { + const cfg = makeCfg(); + const restartAbort = createAgentRunRestartAbortError(); + const outerAbort = makeAbortWrapper(restartAbort); + const run = vi.fn().mockRejectedValueOnce(outerAbort).mockResolvedValueOnce("ok"); + + await expect( + runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + }), + ).rejects.toBe(outerAbort); + + expect(run).toHaveBeenCalledTimes(1); + }); + + it("discards deferred session suspension for private terminal abort wrappers", () => { + const timeout = new Error("request timed out"); + timeout.name = "TimeoutError"; + expect( + testing.shouldDiscardDeferredSessionSuspension({ + error: makeAbortableWrapper(timeout), + }), + ).toBe(true); + + expect( + testing.shouldDiscardDeferredSessionSuspension({ + error: makeAbortWrapper(createAgentRunRestartAbortError()), + }), + ).toBe(true); + + const providerTimeout = new Error("provider request timed out after 60s"); + providerTimeout.name = "TimeoutError"; + expect( + testing.shouldDiscardDeferredSessionSuspension({ + error: makeAbortWrapper(providerTimeout), + }), + ).toBe(false); + }); + + it("falls back normally when a provider wraps its own timeout as AbortError(cause: TimeoutError) WITHOUT the abortable() marker", async () => { + const cfg = makeCfg(); + const providerInnerTimeout = new Error("provider request timed out after 60s"); + providerInnerTimeout.name = "TimeoutError"; + const unmarkedAbortError = new Error("aborted", { cause: providerInnerTimeout }); + unmarkedAbortError.name = "AbortError"; + const run = vi.fn().mockRejectedValueOnce(unmarkedAbortError).mockResolvedValueOnce("ok"); + + const result = await runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + }); + + expect(result.result).toBe("ok"); + expect(run).toHaveBeenCalledTimes(2); + }); + + it("falls back normally when a top-level provider TimeoutError is thrown (not an AbortError wrapper)", async () => { + const cfg = makeCfg(); + const directProviderTimeout = new Error("provider request timed out after 60s"); + directProviderTimeout.name = "TimeoutError"; + const run = vi.fn().mockRejectedValueOnce(directProviderTimeout).mockResolvedValueOnce("ok"); + + const result = await runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + }); + + expect(result.result).toBe("ok"); + expect(run).toHaveBeenCalledTimes(2); + }); + + it("falls back normally when thrown error is generic AbortError without terminal cause", async () => { + const cfg = makeCfg(); + const run = vi + .fn() + .mockRejectedValueOnce(new Error("provider transient failure")) + .mockResolvedValueOnce("ok"); + + const result = await runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + }); + + expect(result.result).toBe("ok"); + expect(run).toHaveBeenCalledTimes(2); + }); + + it("skips fallback when the caller signal is aborted, even with a non-terminal reason", async () => { + const cfg = makeCfg(); + const run = vi + .fn() + .mockRejectedValueOnce(new Error("provider had a sad day")) + .mockResolvedValueOnce("ok"); + + const genericReason = new Error("some unrelated abort"); + const controller = makeTaggedAbortController(genericReason); + + await expect( + runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + abortSignal: controller.signal, + }), + ).rejects.toBeInstanceOf(Error); + + expect(run).toHaveBeenCalledTimes(1); + }); + + it("falls back normally when no abortSignal is passed (back-compat)", async () => { + const cfg = makeCfg(); + const run = vi + .fn() + .mockRejectedValueOnce(new Error("first attempt failed")) + .mockResolvedValueOnce("ok"); + + const result = await runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + }); + + expect(result.result).toBe("ok"); + expect(run).toHaveBeenCalledTimes(2); + }); + + it("falls back normally when signal is provided but not aborted", async () => { + const cfg = makeCfg(); + const run = vi + .fn() + .mockRejectedValueOnce(new Error("first attempt failed")) + .mockResolvedValueOnce("ok"); + + const controller = new AbortController(); + const result = await runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + abortSignal: controller.signal, + }); + + expect(result.result).toBe("ok"); + expect(run).toHaveBeenCalledTimes(2); + }); + + it("rethrows terminal abort even when error resembles a failover-normalizable error", async () => { + const cfg = makeCfg(); + + const rateLimitLikeError = Object.assign(new Error("RESOURCE_EXHAUSTED: quota exceeded"), { + status: 429, + name: "AbortError", + }); + + const run = vi.fn().mockRejectedValue(rateLimitLikeError); + + const timeoutReason = new Error("request timed out"); + timeoutReason.name = "TimeoutError"; + const controller = new AbortController(); + controller.abort(timeoutReason); + + await expect( + runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + abortSignal: controller.signal, + }), + ).rejects.toBe(rateLimitLikeError); + + expect(run).toHaveBeenCalledTimes(1); + }); + + it("treats cron timeout string reason as terminal (covers plain-string abort)", async () => { + const cfg = makeCfg(); + const run = vi.fn().mockRejectedValue(makeAbortError("aborted")); + + const controller = new AbortController(); + controller.abort("cron: job execution timed out"); + + await expect( + runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + abortSignal: controller.signal, + }), + ).rejects.toBeInstanceOf(Error); + + expect(run).toHaveBeenCalledTimes(1); + }); + + it("treats phase-suffixed cron timeout reason as terminal (covers `(last phase: ...)` variant)", async () => { + const cfg = makeCfg(); + const run = vi.fn().mockRejectedValue(makeAbortError("aborted")); + + const controller = new AbortController(); + controller.abort("cron: job execution timed out (last phase: model_call_started)"); + + await expect( + runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + abortSignal: controller.signal, + }), + ).rejects.toBeInstanceOf(Error); + + expect(run).toHaveBeenCalledTimes(1); + }); + + it("treats isolated-agent setup-timeout (and phase suffix) as terminal", async () => { + const cfg = makeCfg(); + const run = vi.fn().mockRejectedValue(makeAbortError("aborted")); + + const controller = new AbortController(); + controller.abort( + "cron: isolated agent setup timed out before runner start (last phase: workspace_provision)", + ); + + await expect( + runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + abortSignal: controller.signal, + }), + ).rejects.toBeInstanceOf(Error); + + expect(run).toHaveBeenCalledTimes(1); + }); + + it("treats isolated-agent pre-execution stall (and phase suffix) as terminal", async () => { + const cfg = makeCfg(); + const run = vi.fn().mockRejectedValue(makeAbortError("aborted")); + + const controller = new AbortController(); + controller.abort( + "cron: isolated agent run stalled before execution start (last phase: runner_ready)", + ); + + await expect( + runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + abortSignal: controller.signal, + }), + ).rejects.toBeInstanceOf(Error); + + expect(run).toHaveBeenCalledTimes(1); + }); + + it("treats bare isolated-agent pre-execution stall as terminal", async () => { + const cfg = makeCfg(); + const run = vi.fn().mockRejectedValue(makeAbortError("aborted")); + + const controller = new AbortController(); + controller.abort("cron: isolated agent run stalled before execution start"); + + await expect( + runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + abortSignal: controller.signal, + }), + ).rejects.toBeInstanceOf(Error); + + expect(run).toHaveBeenCalledTimes(1); + }); + + it("treats an Error whose .message matches a known terminal string as terminal", async () => { + const cfg = makeCfg(); + const run = vi.fn().mockRejectedValue(makeAbortError("aborted")); + + const wrapped = new Error("cron: job execution timed out"); + const controller = new AbortController(); + controller.abort(wrapped); + + await expect( + runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + abortSignal: controller.signal, + }), + ).rejects.toBeInstanceOf(Error); + + expect(run).toHaveBeenCalledTimes(1); + }); + + it("does not classify an unrelated string reason as terminal (caller-abort still skips fallback)", async () => { + const cfg = makeCfg(); + const run = vi + .fn() + .mockRejectedValueOnce(new Error("first attempt failed")) + .mockResolvedValueOnce("ok"); + + const controller = new AbortController(); + controller.abort("some unrelated cancel reason"); + + await expect( + runWithModelFallback({ + cfg, + provider: "anthropic", + model: "claude-sonnet-4-6", + run, + abortSignal: controller.signal, + }), + ).rejects.toBeInstanceOf(Error); + + expect(run).toHaveBeenCalledTimes(1); + }); + }); }); describe("runWithImageModelFallback", () => { diff --git a/src/agents/model-fallback.ts b/src/agents/model-fallback.ts index 9514d6b81be5..f5beb7003d1b 100644 --- a/src/agents/model-fallback.ts +++ b/src/agents/model-fallback.ts @@ -8,6 +8,7 @@ import { resolveAgentModelPrimaryValue, } from "../config/model-input.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { isCronTerminalAbortReasonText } from "../cron/service/execution-errors.js"; import { emitFailoverEvent } from "../infra/diagnostic-events.js"; import { formatErrorMessage, toErrorObject } from "../infra/errors.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; @@ -31,6 +32,7 @@ import { isActiveUnusableWindow } from "./auth-profiles/usage-state.js"; import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "./defaults.js"; import { isLikelyContextOverflowError } from "./embedded-agent-helpers/errors.js"; import type { FailoverReason } from "./embedded-agent-helpers/types.js"; +import { isOpenClawAbortableWrapper } from "./embedded-agent-runner/run/abortable.js"; import { FailoverError, buildFailoverRemediationHint, @@ -188,21 +190,74 @@ type ModelFallbackRunFn = ( options?: ModelFallbackRunOptions, ) => Promise; +function isTerminalAbortReasonString(reason: string): boolean { + return isCronTerminalAbortReasonText(reason); +} + +function getErrorCauseCandidates(err: Error): unknown[] { + const candidates: unknown[] = []; + if ("cause" in err && err.cause !== undefined) { + candidates.push(err.cause); + if (err.cause instanceof Error && "cause" in err.cause && err.cause.cause !== undefined) { + candidates.push(err.cause.cause); + } + } + return candidates; +} + +function isTerminalAbortCandidate(candidate: unknown): boolean { + if (typeof candidate === "string") { + return isTerminalAbortReasonString(candidate); + } + if (!(candidate instanceof Error)) { + return false; + } + if (isAgentRunRestartAbortReason(candidate)) { + return true; + } + if (candidate.name === "TimeoutError") { + return true; + } + if (candidate.name === "ClientDisconnectError") { + return true; + } + return isTerminalAbortReasonString(candidate.message); +} + function isTerminalAbort(signal: AbortSignal | undefined): boolean { if (!signal?.aborted) { return false; } const reason = signal.reason; - if (!(reason instanceof Error)) { + + if (reason instanceof Error) { + const candidates: unknown[] = [reason, ...getErrorCauseCandidates(reason)]; + return candidates.some(isTerminalAbortCandidate); + } + + return isTerminalAbortCandidate(reason); +} + +function isTerminalAbortFromError(err: unknown): boolean { + if (!(err instanceof Error)) { return false; } - if (isAgentRunRestartAbortReason(reason)) { + if (isAgentRunRestartAbortReason(err)) { return true; } - if (reason.name === "TimeoutError") { - return true; + const causeCandidates = getErrorCauseCandidates(err); + if (err.name !== "AbortError") { + return false; } - return reason.name === "ClientDisconnectError"; + for (const candidate of causeCandidates) { + if (isAgentRunRestartAbortReason(candidate)) { + return true; + } + } + if (!isOpenClawAbortableWrapper(err)) { + return false; + } + return causeCandidates.some(isTerminalAbortCandidate); } function isCallerAbortSignal(signal: AbortSignal | undefined): boolean { @@ -357,6 +412,9 @@ async function runFallbackCandidate(params: { if (isAgentRunDirectAbortReason(err) || isAgentRunRestartAbortReason(err)) { throw err; } + if (isTerminalAbortFromError(err)) { + throw err; + } // Normalize abort-wrapped rate-limit errors (e.g. Google Vertex RESOURCE_EXHAUSTED) // so they become FailoverErrors and continue the fallback loop instead of aborting. const normalizedFailover = coerceToFailoverError(err, { @@ -809,6 +867,7 @@ export const testing = { resolveImageFallbackCandidates, resolveCooldownDecision, resolveSessionSuspensionReason, + shouldDiscardDeferredSessionSuspension, } as const; export function resolveModelCandidateChain( @@ -1310,6 +1369,7 @@ function shouldDiscardDeferredSessionSuspension(params: { isCallerAbortSignal(params.abortSignal) || isAgentRunDirectAbortReason(params.error) || isAgentRunRestartAbortReason(params.error) || + isTerminalAbortFromError(params.error) || isCommandLaneTaskTimeoutError(params.error) || isNonProviderRuntimeCoordinationError(params.error) || isLikelyContextOverflowError(formatErrorMessage(params.error)) diff --git a/src/cron/service/execution-errors.ts b/src/cron/service/execution-errors.ts index e27870ed9843..912146b32a34 100644 --- a/src/cron/service/execution-errors.ts +++ b/src/cron/service/execution-errors.ts @@ -6,36 +6,58 @@ function formatCronAgentExecutionPhase(execution?: CronAgentExecutionStarted): s return formatEmbeddedAgentExecutionPhase(execution?.phase); } +const CRON_JOB_EXECUTION_TIMEOUT_ERROR = "cron: job execution timed out"; +const CRON_SETUP_TIMEOUT_ERROR = "cron: isolated agent setup timed out before runner start"; +const CRON_PRE_EXECUTION_TIMEOUT_ERROR = "cron: isolated agent run stalled before execution start"; +const CRON_TIMEOUT_ERROR_PREFIXES: readonly string[] = [ + CRON_JOB_EXECUTION_TIMEOUT_ERROR, + CRON_SETUP_TIMEOUT_ERROR, + CRON_PRE_EXECUTION_TIMEOUT_ERROR, +]; + +function hasCronTimeoutPrefix(error: string, prefix: string): boolean { + return error === prefix || error.startsWith(prefix + " "); +} + +export function isCronTerminalAbortReasonText(error: string): boolean { + for (const prefix of CRON_TIMEOUT_ERROR_PREFIXES) { + if (hasCronTimeoutPrefix(error, prefix)) { + return true; + } + } + return false; +} + /** Formats the generic cron execution timeout message with last-known phase context when available. */ export function timeoutErrorMessage(execution?: CronAgentExecutionStarted): string { const phase = formatCronAgentExecutionPhase(execution); if (!phase) { - return "cron: job execution timed out"; + return CRON_JOB_EXECUTION_TIMEOUT_ERROR; } - return `cron: job execution timed out (last phase: ${phase})`; + return `${CRON_JOB_EXECUTION_TIMEOUT_ERROR} (last phase: ${phase})`; } /** Formats timeout text for runs that stalled before the isolated runner started. */ export function setupTimeoutErrorMessage(execution?: CronAgentExecutionStarted): string { const phase = formatCronAgentExecutionPhase(execution); if (!phase) { - return "cron: isolated agent setup timed out before runner start"; + return CRON_SETUP_TIMEOUT_ERROR; } - return `cron: isolated agent setup timed out before runner start (last phase: ${phase})`; + return `${CRON_SETUP_TIMEOUT_ERROR} (last phase: ${phase})`; } /** Returns true for the setup-timeout class that fires before the isolated runner starts. */ export function isSetupTimeoutErrorText(error: string): boolean { - return error.startsWith("cron: isolated agent setup timed out before runner start"); + return hasCronTimeoutPrefix(error, CRON_SETUP_TIMEOUT_ERROR); } /** Formats timeout text for runs that stalled after setup but before execution start. */ export function preExecutionTimeoutErrorMessage(execution?: CronAgentExecutionStarted): string { const phase = formatCronAgentExecutionPhase(execution); if (!phase) { - return "cron: isolated agent run stalled before execution start"; + return CRON_PRE_EXECUTION_TIMEOUT_ERROR; } - return `cron: isolated agent run stalled before execution start (last phase: ${phase})`; + return `${CRON_PRE_EXECUTION_TIMEOUT_ERROR} (last phase: ${phase})`; } /** Extracts a human timeout/abort reason, falling back to the canonical cron timeout text. */ diff --git a/src/gateway/http-common.ts b/src/gateway/http-common.ts index b6c1cf17bfd8..fa6b56dcd714 100644 --- a/src/gateway/http-common.ts +++ b/src/gateway/http-common.ts @@ -134,6 +134,14 @@ export function setSseHeaders(res: ServerResponse) { res.flushHeaders?.(); } +/** Abort reason used when the HTTP client disconnects before delivery. */ +export class ClientDisconnectError extends Error { + constructor(message = "HTTP client disconnected") { + super(message); + this.name = "ClientDisconnectError"; + } +} + export function watchClientDisconnect( req: IncomingMessage, res: ServerResponse, @@ -153,7 +161,7 @@ export function watchClientDisconnect( const handleClose = () => { onDisconnect?.(); if (!abortController.signal.aborted) { - abortController.abort(); + abortController.abort(new ClientDisconnectError()); } }; for (const socket of sockets) { diff --git a/src/trajectory/export.ts b/src/trajectory/export.ts index 063391c68332..f34ac614f81d 100644 --- a/src/trajectory/export.ts +++ b/src/trajectory/export.ts @@ -822,6 +822,7 @@ function buildArtifactsCapture(params: { runtimeArtifacts?.timedOutDuringCompaction ?? runtimeEnd?.timedOutDuringCompaction, timedOutDuringToolExecution: runtimeArtifacts?.timedOutDuringToolExecution ?? runtimeEnd?.timedOutDuringToolExecution, + timedOutByRunBudget: runtimeArtifacts?.timedOutByRunBudget ?? runtimeEnd?.timedOutByRunBudget, promptError: runtimeArtifacts?.promptError ?? runtimeEnd?.promptError ?? runtimeCompletion?.promptError, promptErrorSource: runtimeArtifacts?.promptErrorSource ?? runtimeCompletion?.promptErrorSource, diff --git a/src/trajectory/metadata.test.ts b/src/trajectory/metadata.test.ts index 5c9b4f492038..807e67aadc1e 100644 --- a/src/trajectory/metadata.test.ts +++ b/src/trajectory/metadata.test.ts @@ -268,6 +268,7 @@ describe("trajectory metadata", () => { idleTimedOut: false, timedOutDuringCompaction: false, timedOutDuringToolExecution: false, + timedOutByRunBudget: false, compactionCount: 1, assistantTexts: ["done"], finalPromptText: "run tests", diff --git a/src/trajectory/metadata.ts b/src/trajectory/metadata.ts index 5cac15795549..2015560d44e9 100644 --- a/src/trajectory/metadata.ts +++ b/src/trajectory/metadata.ts @@ -50,6 +50,7 @@ type BuildTrajectoryArtifactsParams = { idleTimedOut: boolean; timedOutDuringCompaction: boolean; timedOutDuringToolExecution: boolean; + timedOutByRunBudget: boolean; promptError?: string; promptErrorSource?: string | null; terminalError?: string; @@ -321,6 +322,7 @@ export function buildTrajectoryArtifacts( idleTimedOut: params.idleTimedOut, timedOutDuringCompaction: params.timedOutDuringCompaction, timedOutDuringToolExecution: params.timedOutDuringToolExecution, + timedOutByRunBudget: params.timedOutByRunBudget, promptError: params.promptError, promptErrorSource: params.promptErrorSource, terminalError: params.terminalError,