mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-26 12:26:38 -06:00
* fix(agents): terminal-abort coverage beyond #87085's isTerminalAbort (closes #60388) PR #87085 landed the base isTerminalAbort(signal) check (TimeoutError / ClientDisconnectError on signal.reason) plus abortSignal threading through the model-fallback and chat-side callers. This change adds the coverage that PR #87085 did not include: - ClientDisconnectError class + wiring in http-common.ts so the reason.name === "ClientDisconnectError" branch PR #87085 added is actually reachable (upstream watchClientDisconnect still aborts bare). - cron run-budget string reasons (prefix match) — cron timer aborts with a plain string, which the Error-only base check skips. - .cause-chain walking + isTerminalAbortFromError gated on the OPENCLAW_ABORTABLE_WRAPPER marker, for the embedded run-budget timer that aborts a private controller (not the caller signal). - compaction-path abortSignal forwarding (compactEmbeddedPiSessionDirect). - timedOutByRunBudget plumbing through attempt/failover-policy/assistant-failover so run-budget timeouts skip the fallback chain and wasted compaction. * fix(agents): preserve wrapped restart aborts * refactor: dedupe terminal abort classification --------- Co-authored-by: Altay <altay@hey.com>
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
|
||||
vi.mock("../model-fallback.js", () => ({
|
||||
runWithModelFallback: vi.fn(async (params: Record<string, unknown>) => ({
|
||||
result: { ok: true, compacted: false, reason: "no-op" },
|
||||
provider: params.provider,
|
||||
model: params.model,
|
||||
attempts: [],
|
||||
})),
|
||||
isFallbackSummaryError: () => false,
|
||||
}));
|
||||
|
||||
vi.mock("./compact.queued.js", () => ({ compactEmbeddedAgentSession: vi.fn() }));
|
||||
|
||||
import { runWithModelFallback } from "../model-fallback.js";
|
||||
import { compactEmbeddedAgentSessionDirect } from "./compact.js";
|
||||
|
||||
const runMock = vi.mocked(runWithModelFallback);
|
||||
|
||||
const baseParams = {
|
||||
sessionId: "test-session",
|
||||
sessionKey: "agent:main:test-session",
|
||||
sessionFile: "/tmp/test-session.jsonl",
|
||||
workspaceDir: "/tmp",
|
||||
};
|
||||
|
||||
function configWithFallbacks(fallbacks: string[]): OpenClawConfig {
|
||||
return {
|
||||
agents: {
|
||||
defaults: {
|
||||
model: {
|
||||
primary: "anthropic/claude-sonnet-4-6",
|
||||
fallbacks,
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
}
|
||||
|
||||
describe("compactEmbeddedAgentSessionDirect abortSignal threading", () => {
|
||||
beforeEach(() => {
|
||||
runMock.mockClear();
|
||||
});
|
||||
|
||||
it("forwards params.abortSignal to runWithModelFallback so terminal aborts during compaction short-circuit", async () => {
|
||||
const controller = new AbortController();
|
||||
|
||||
await compactEmbeddedAgentSessionDirect({
|
||||
...baseParams,
|
||||
config: configWithFallbacks(["anthropic/claude-haiku-4-5", "openai/gpt-4.1-mini"]),
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
abortSignal: controller.signal,
|
||||
});
|
||||
|
||||
expect(runMock).toHaveBeenCalledTimes(1);
|
||||
const passedParams = runMock.mock.calls[0]?.[0];
|
||||
expect(passedParams?.abortSignal).toBe(controller.signal);
|
||||
});
|
||||
|
||||
it("passes undefined when no abortSignal is set (back-compat)", async () => {
|
||||
await compactEmbeddedAgentSessionDirect({
|
||||
...baseParams,
|
||||
config: configWithFallbacks(["anthropic/claude-haiku-4-5"]),
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
});
|
||||
|
||||
expect(runMock).toHaveBeenCalledTimes(1);
|
||||
const passedParams = runMock.mock.calls[0]?.[0];
|
||||
expect(passedParams?.abortSignal).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -2325,6 +2325,7 @@ async function runEmbeddedAgentInternal(
|
||||
attempt.setTerminalLifecycleMeta?.({ ...meta, aborted });
|
||||
};
|
||||
const timedOutDuringToolExecution = attempt.timedOutDuringToolExecution ?? false;
|
||||
const timedOutByRunBudget = attempt.timedOutByRunBudget ?? false;
|
||||
adoptActiveSessionId(sessionIdUsed);
|
||||
if (sessionFileUsed && sessionFileUsed !== activeSessionFile) {
|
||||
activeSessionFile = sessionFileUsed;
|
||||
@@ -2502,10 +2503,12 @@ async function runEmbeddedAgentInternal(
|
||||
);
|
||||
throw new LiveSessionModelSwitchError(requestedSelection);
|
||||
}
|
||||
// ── Timeout-triggered compaction ──────────────────────────────────
|
||||
// When the LLM times out with high context usage, compact before
|
||||
// retrying to break the death spiral of repeated timeouts.
|
||||
if (timedOut && !timedOutDuringCompaction && !timedOutDuringToolExecution) {
|
||||
if (
|
||||
timedOut &&
|
||||
!timedOutDuringCompaction &&
|
||||
!timedOutDuringToolExecution &&
|
||||
!timedOutByRunBudget
|
||||
) {
|
||||
// Only consider prompt-side tokens here. API totals include output
|
||||
// tokens, which can make a long generation look like high context
|
||||
// pressure even when the prompt itself was small.
|
||||
@@ -3241,6 +3244,7 @@ async function runEmbeddedAgentInternal(
|
||||
failoverReason: promptFailoverReason,
|
||||
harnessOwnsTransport: pluginHarnessOwnsTransport,
|
||||
promptTimeoutFallbackSafe,
|
||||
timedOutByRunBudget,
|
||||
profileRotated: false,
|
||||
});
|
||||
if (
|
||||
@@ -3281,6 +3285,7 @@ async function runEmbeddedAgentInternal(
|
||||
failoverReason: promptFailoverReason,
|
||||
harnessOwnsTransport: pluginHarnessOwnsTransport,
|
||||
promptTimeoutFallbackSafe,
|
||||
timedOutByRunBudget,
|
||||
profileRotated: true,
|
||||
});
|
||||
}
|
||||
@@ -3483,6 +3488,7 @@ async function runEmbeddedAgentInternal(
|
||||
timedOutDuringCompaction,
|
||||
timedOutDuringToolExecution,
|
||||
harnessOwnsTransport: pluginHarnessOwnsTransport,
|
||||
timedOutByRunBudget,
|
||||
profileRotated: false,
|
||||
});
|
||||
const assistantFailoverOutcome = await handleAssistantFailover({
|
||||
@@ -3496,6 +3502,7 @@ async function runEmbeddedAgentInternal(
|
||||
idleTimedOut,
|
||||
timedOutDuringCompaction,
|
||||
timedOutDuringToolExecution,
|
||||
timedOutByRunBudget,
|
||||
allowSameModelIdleTimeoutRetry:
|
||||
timedOut &&
|
||||
idleTimedOut &&
|
||||
|
||||
@@ -7,16 +7,28 @@ function getAbortReason(signal: AbortSignal): unknown {
|
||||
return "reason" in signal ? (signal as { reason?: unknown }).reason : undefined;
|
||||
}
|
||||
|
||||
/** Marks AbortErrors produced by abortable() so provider aborts stay retryable. */
|
||||
export const OPENCLAW_ABORTABLE_WRAPPER = Symbol.for("openclaw.abortable.wrapper");
|
||||
|
||||
export function isOpenClawAbortableWrapper(err: unknown): boolean {
|
||||
return err !== null && typeof err === "object" && OPENCLAW_ABORTABLE_WRAPPER in err;
|
||||
}
|
||||
|
||||
function tagAsAbortableWrapper(err: Error): Error {
|
||||
(err as Error & { [OPENCLAW_ABORTABLE_WRAPPER]?: true })[OPENCLAW_ABORTABLE_WRAPPER] = true;
|
||||
return err;
|
||||
}
|
||||
|
||||
function makeAbortError(signal: AbortSignal): Error {
|
||||
const reason = getAbortReason(signal);
|
||||
if (reason instanceof Error) {
|
||||
const err = new Error(reason.message, { cause: reason });
|
||||
err.name = "AbortError";
|
||||
return err;
|
||||
return tagAsAbortableWrapper(err);
|
||||
}
|
||||
const err = reason ? new Error("aborted", { cause: reason }) : new Error("aborted");
|
||||
err.name = "AbortError";
|
||||
return err;
|
||||
return tagAsAbortableWrapper(err);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -23,6 +23,7 @@ function makeParams(overrides: Partial<Params> = {}): Params {
|
||||
idleTimedOut: false,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
timedOutByRunBudget: false,
|
||||
allowSameModelIdleTimeoutRetry: false,
|
||||
allowSameModelRateLimitRetry: true,
|
||||
assistantProfileFailureReason: null,
|
||||
|
||||
@@ -130,6 +130,7 @@ export async function handleAssistantFailover(params: {
|
||||
idleTimedOut: boolean;
|
||||
timedOutDuringCompaction: boolean;
|
||||
timedOutDuringToolExecution: boolean;
|
||||
timedOutByRunBudget: boolean;
|
||||
allowSameModelIdleTimeoutRetry: boolean;
|
||||
allowSameModelRateLimitRetry: boolean;
|
||||
assistantProfileFailureReason: AuthProfileFailureReason | null;
|
||||
@@ -309,6 +310,7 @@ export async function handleAssistantFailover(params: {
|
||||
idleTimedOut: params.idleTimedOut,
|
||||
timedOutDuringCompaction: params.timedOutDuringCompaction,
|
||||
timedOutDuringToolExecution: params.timedOutDuringToolExecution,
|
||||
timedOutByRunBudget: params.timedOutByRunBudget,
|
||||
profileRotated: true,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -918,6 +918,7 @@ export async function runEmbeddedAttempt(
|
||||
let idleTimedOut = false;
|
||||
let timedOutDuringCompaction = false;
|
||||
let timedOutDuringToolExecution = false;
|
||||
let timedOutByRunBudget = false;
|
||||
let promptError: unknown = null;
|
||||
let emitDiagnosticRunCompleted:
|
||||
| ((
|
||||
@@ -3819,6 +3820,7 @@ export async function runEmbeddedAttempt(
|
||||
) {
|
||||
timedOutDuringCompaction = true;
|
||||
}
|
||||
timedOutByRunBudget = true;
|
||||
abortRun(true);
|
||||
if (!abortWarnTimer) {
|
||||
abortWarnTimer = setTimeout(() => {
|
||||
@@ -5668,6 +5670,7 @@ export async function runEmbeddedAttempt(
|
||||
idleTimedOut,
|
||||
timedOutDuringCompaction,
|
||||
timedOutDuringToolExecution,
|
||||
timedOutByRunBudget,
|
||||
promptError: promptError ? formatErrorMessage(promptError) : undefined,
|
||||
promptErrorSource,
|
||||
terminalError: attemptTrajectoryTerminal.terminalError,
|
||||
@@ -5688,6 +5691,7 @@ export async function runEmbeddedAttempt(
|
||||
idleTimedOut,
|
||||
timedOutDuringCompaction,
|
||||
timedOutDuringToolExecution,
|
||||
timedOutByRunBudget,
|
||||
promptError: promptError ? formatErrorMessage(promptError) : undefined,
|
||||
promptErrorSource,
|
||||
terminalError: attemptTrajectoryTerminal.terminalError,
|
||||
@@ -5714,6 +5718,7 @@ export async function runEmbeddedAttempt(
|
||||
idleTimedOut,
|
||||
timedOutDuringCompaction,
|
||||
timedOutDuringToolExecution,
|
||||
timedOutByRunBudget,
|
||||
promptError: promptError ? formatErrorMessage(promptError) : undefined,
|
||||
terminalError: attemptTrajectoryTerminal.terminalError,
|
||||
});
|
||||
@@ -5729,6 +5734,7 @@ export async function runEmbeddedAttempt(
|
||||
idleTimedOut,
|
||||
timedOutDuringCompaction,
|
||||
timedOutDuringToolExecution,
|
||||
timedOutByRunBudget,
|
||||
promptError,
|
||||
promptErrorSource,
|
||||
preflightRecovery,
|
||||
@@ -5785,6 +5791,7 @@ export async function runEmbeddedAttempt(
|
||||
idleTimedOut,
|
||||
timedOutDuringCompaction,
|
||||
timedOutDuringToolExecution,
|
||||
timedOutByRunBudget,
|
||||
promptError: promptError ? formatErrorMessage(promptError) : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -95,6 +95,43 @@ describe("resolveRunFailoverDecision", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("surfaces prompt run-budget timeouts instead of model fallback (#60388)", () => {
|
||||
expect(
|
||||
resolveRunFailoverDecision({
|
||||
stage: "prompt",
|
||||
aborted: true,
|
||||
externalAbort: false,
|
||||
fallbackConfigured: true,
|
||||
failoverFailure: true,
|
||||
failoverReason: "timeout",
|
||||
promptTimeoutFallbackSafe: true,
|
||||
timedOutByRunBudget: true,
|
||||
profileRotated: true,
|
||||
}),
|
||||
).toEqual({
|
||||
action: "surface_error",
|
||||
reason: "timeout",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not rotate prompt failures after the run budget is exhausted (#60388)", () => {
|
||||
expect(
|
||||
resolveRunFailoverDecision({
|
||||
stage: "prompt",
|
||||
aborted: true,
|
||||
externalAbort: false,
|
||||
fallbackConfigured: true,
|
||||
failoverFailure: true,
|
||||
failoverReason: "rate_limit",
|
||||
timedOutByRunBudget: true,
|
||||
profileRotated: false,
|
||||
}),
|
||||
).toEqual({
|
||||
action: "surface_error",
|
||||
reason: "rate_limit",
|
||||
});
|
||||
});
|
||||
|
||||
it("surfaces deterministic prompt format failures instead of rotating or falling back", () => {
|
||||
expect(
|
||||
resolveRunFailoverDecision({
|
||||
@@ -145,6 +182,7 @@ describe("resolveRunFailoverDecision", () => {
|
||||
idleTimedOut: false,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
timedOutByRunBudget: false,
|
||||
profileRotated: false,
|
||||
}),
|
||||
).toEqual({
|
||||
@@ -165,6 +203,7 @@ describe("resolveRunFailoverDecision", () => {
|
||||
idleTimedOut: false,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
timedOutByRunBudget: false,
|
||||
profileRotated: false,
|
||||
}),
|
||||
).toEqual({
|
||||
@@ -187,6 +226,7 @@ describe("resolveRunFailoverDecision", () => {
|
||||
idleTimedOut: false,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
timedOutByRunBudget: false,
|
||||
profileRotated: false,
|
||||
}),
|
||||
).toEqual({
|
||||
@@ -208,6 +248,7 @@ describe("resolveRunFailoverDecision", () => {
|
||||
idleTimedOut: false,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
timedOutByRunBudget: false,
|
||||
profileRotated: true,
|
||||
}),
|
||||
).toEqual({
|
||||
@@ -229,6 +270,7 @@ describe("resolveRunFailoverDecision", () => {
|
||||
idleTimedOut: false,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
timedOutByRunBudget: false,
|
||||
profileRotated: true,
|
||||
}),
|
||||
).toEqual({
|
||||
@@ -249,6 +291,7 @@ describe("resolveRunFailoverDecision", () => {
|
||||
idleTimedOut: false,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
timedOutByRunBudget: false,
|
||||
profileRotated: false,
|
||||
}),
|
||||
).toEqual({
|
||||
@@ -286,6 +329,7 @@ describe("resolveRunFailoverDecision", () => {
|
||||
idleTimedOut: false,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: true,
|
||||
timedOutByRunBudget: false,
|
||||
profileRotated: false,
|
||||
}),
|
||||
).toEqual({
|
||||
@@ -348,6 +392,7 @@ describe("resolveRunFailoverDecision", () => {
|
||||
idleTimedOut: false,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: true,
|
||||
timedOutByRunBudget: false,
|
||||
profileRotated: true,
|
||||
}),
|
||||
).toEqual({
|
||||
@@ -368,6 +413,7 @@ describe("resolveRunFailoverDecision", () => {
|
||||
idleTimedOut: false,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
timedOutByRunBudget: false,
|
||||
profileRotated: false,
|
||||
}),
|
||||
).toEqual({
|
||||
@@ -477,6 +523,7 @@ describe("resolveRunFailoverDecision", () => {
|
||||
idleTimedOut: true,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: true,
|
||||
timedOutByRunBudget: false,
|
||||
profileRotated: false,
|
||||
}),
|
||||
).toEqual({
|
||||
@@ -498,6 +545,7 @@ describe("resolveRunFailoverDecision", () => {
|
||||
idleTimedOut: true,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: true,
|
||||
timedOutByRunBudget: false,
|
||||
profileRotated: true,
|
||||
}),
|
||||
).toEqual({
|
||||
@@ -519,6 +567,7 @@ describe("resolveRunFailoverDecision", () => {
|
||||
idleTimedOut: false,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
timedOutByRunBudget: false,
|
||||
profileRotated: false,
|
||||
}),
|
||||
).toEqual({
|
||||
@@ -540,6 +589,7 @@ describe("resolveRunFailoverDecision", () => {
|
||||
idleTimedOut: true,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
timedOutByRunBudget: false,
|
||||
profileRotated: false,
|
||||
}),
|
||||
).toEqual({
|
||||
@@ -561,6 +611,7 @@ describe("resolveRunFailoverDecision", () => {
|
||||
idleTimedOut: true,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
timedOutByRunBudget: false,
|
||||
profileRotated: true,
|
||||
}),
|
||||
).toEqual({
|
||||
@@ -659,6 +710,7 @@ describe("resolveRunFailoverDecision", () => {
|
||||
idleTimedOut: true,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
timedOutByRunBudget: false,
|
||||
profileRotated: true,
|
||||
}),
|
||||
).toEqual({
|
||||
@@ -680,6 +732,7 @@ describe("resolveRunFailoverDecision", () => {
|
||||
idleTimedOut: true,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
timedOutByRunBudget: false,
|
||||
profileRotated: false,
|
||||
}),
|
||||
).toEqual({
|
||||
@@ -687,6 +740,48 @@ describe("resolveRunFailoverDecision", () => {
|
||||
reason: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not rotate or fallback assistant timeouts that exhausted the run budget (#60388)", () => {
|
||||
expect(
|
||||
resolveRunFailoverDecision({
|
||||
stage: "assistant",
|
||||
aborted: true,
|
||||
externalAbort: false,
|
||||
fallbackConfigured: true,
|
||||
failoverFailure: false,
|
||||
failoverReason: null,
|
||||
timedOut: true,
|
||||
idleTimedOut: false,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
timedOutByRunBudget: true,
|
||||
profileRotated: false,
|
||||
}),
|
||||
).toEqual({
|
||||
action: "continue_normal",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not fallback assistant run-budget timeouts even after profile rotation exhausted (#60388)", () => {
|
||||
expect(
|
||||
resolveRunFailoverDecision({
|
||||
stage: "assistant",
|
||||
aborted: true,
|
||||
externalAbort: false,
|
||||
fallbackConfigured: true,
|
||||
failoverFailure: false,
|
||||
failoverReason: null,
|
||||
timedOut: true,
|
||||
idleTimedOut: false,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
timedOutByRunBudget: true,
|
||||
profileRotated: true,
|
||||
}),
|
||||
).toEqual({
|
||||
action: "continue_normal",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("mergeRetryFailoverReason", () => {
|
||||
|
||||
@@ -51,6 +51,7 @@ type PromptDecisionParams = {
|
||||
failoverReason: FailoverReason | null;
|
||||
harnessOwnsTransport?: boolean;
|
||||
promptTimeoutFallbackSafe?: boolean;
|
||||
timedOutByRunBudget?: boolean;
|
||||
profileRotated: boolean;
|
||||
};
|
||||
|
||||
@@ -67,6 +68,7 @@ type AssistantDecisionParams = {
|
||||
timedOutDuringCompaction: boolean;
|
||||
timedOutDuringToolExecution: boolean;
|
||||
harnessOwnsTransport?: boolean;
|
||||
timedOutByRunBudget?: boolean;
|
||||
profileRotated: boolean;
|
||||
};
|
||||
|
||||
@@ -92,6 +94,9 @@ function isTerminalFormatFailure(params: {
|
||||
}
|
||||
|
||||
function shouldRotatePrompt(params: PromptDecisionParams): boolean {
|
||||
if (params.timedOutByRunBudget) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
params.failoverFailure &&
|
||||
params.failoverReason !== "timeout" &&
|
||||
@@ -116,6 +121,9 @@ function shouldRotateAssistant(params: AssistantDecisionParams): boolean {
|
||||
if (isTerminalFormatFailure(params)) {
|
||||
return false;
|
||||
}
|
||||
if (params.timedOutByRunBudget) {
|
||||
return false;
|
||||
}
|
||||
const timeoutFailure = isAssistantTimeoutFailure(params);
|
||||
const harnessOwnedTimeout =
|
||||
params.harnessOwnsTransport && (timeoutFailure || params.failoverReason === "timeout");
|
||||
@@ -175,6 +183,12 @@ export function resolveRunFailoverDecision(params: RunFailoverDecisionParams): R
|
||||
reason: params.failoverReason,
|
||||
};
|
||||
}
|
||||
if (params.timedOutByRunBudget) {
|
||||
return {
|
||||
action: "surface_error",
|
||||
reason: params.failoverReason,
|
||||
};
|
||||
}
|
||||
if (params.harnessOwnsTransport && params.failoverReason === "timeout") {
|
||||
// Plugin harness lifecycle timeouts must stay inside the harness boundary;
|
||||
// only prompt request timeouts proven replay-safe may enter model fallback.
|
||||
|
||||
@@ -125,6 +125,7 @@ export type EmbeddedRunAttemptResult = {
|
||||
timedOutDuringCompaction: boolean;
|
||||
/** Optional because this type is re-exported as `AgentHarnessAttemptResult`. */
|
||||
timedOutDuringToolExecution?: boolean;
|
||||
timedOutByRunBudget?: boolean;
|
||||
promptError: unknown;
|
||||
/**
|
||||
* Identifies which phase produced the promptError.
|
||||
|
||||
@@ -18,6 +18,7 @@ import { CommandLaneTaskTimeoutError } from "../process/command-queue.js";
|
||||
import { AUTH_STORE_VERSION } from "./auth-profiles/constants.js";
|
||||
import type { AuthProfileStore } from "./auth-profiles/types.js";
|
||||
import { classifyEmbeddedAgentRunResultForModelFallback } from "./embedded-agent-runner/result-fallback-classifier.js";
|
||||
import { OPENCLAW_ABORTABLE_WRAPPER } from "./embedded-agent-runner/run/abortable.js";
|
||||
import type { EmbeddedAgentRunResult } from "./embedded-agent-runner/types.js";
|
||||
import { FailoverError } from "./failover-error.js";
|
||||
import { resetFallbackSkipCacheForTest } from "./fallback-skip-cache.js";
|
||||
@@ -3459,6 +3460,489 @@ describe("runWithModelFallback", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("terminal abort propagation", () => {
|
||||
function makeAbortError(message = "aborted"): Error {
|
||||
const err = new Error(message);
|
||||
err.name = "AbortError";
|
||||
return err;
|
||||
}
|
||||
|
||||
function makeAbortableWrapper(reason: Error): Error {
|
||||
const err = new Error(reason.message, { cause: reason });
|
||||
err.name = "AbortError";
|
||||
(err as Error & { [OPENCLAW_ABORTABLE_WRAPPER]?: true })[OPENCLAW_ABORTABLE_WRAPPER] = true;
|
||||
return err;
|
||||
}
|
||||
|
||||
function makeAbortWrapper(reason: Error): Error {
|
||||
const err = new Error("aborted", { cause: reason });
|
||||
err.name = "AbortError";
|
||||
return err;
|
||||
}
|
||||
|
||||
function makeTaggedAbortController(reason: Error): AbortController {
|
||||
const controller = new AbortController();
|
||||
controller.abort(reason);
|
||||
return controller;
|
||||
}
|
||||
|
||||
it("rethrows immediately when signal.reason has name=TimeoutError (run-budget timeout)", async () => {
|
||||
const cfg = makeCfg();
|
||||
const runError = makeAbortError("aborted");
|
||||
const run = vi.fn().mockRejectedValue(runError);
|
||||
|
||||
const timeoutReason = new Error("request timed out");
|
||||
timeoutReason.name = "TimeoutError";
|
||||
const controller = makeTaggedAbortController(timeoutReason);
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
abortSignal: controller.signal,
|
||||
}),
|
||||
).rejects.toBe(runError);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rethrows immediately when signal.reason has name=ClientDisconnectError", async () => {
|
||||
const cfg = makeCfg();
|
||||
const runError = makeAbortError("aborted");
|
||||
const run = vi.fn().mockRejectedValue(runError);
|
||||
|
||||
const disconnectReason = new Error("HTTP client disconnected");
|
||||
disconnectReason.name = "ClientDisconnectError";
|
||||
const controller = makeTaggedAbortController(disconnectReason);
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
abortSignal: controller.signal,
|
||||
}),
|
||||
).rejects.toBe(runError);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("detects TimeoutError nested as cause of an outer Error", async () => {
|
||||
const cfg = makeCfg();
|
||||
const runError = makeAbortError("aborted");
|
||||
const run = vi.fn().mockRejectedValue(runError);
|
||||
|
||||
const innerTimeout = new Error("request timed out");
|
||||
innerTimeout.name = "TimeoutError";
|
||||
const outerWrap = makeAbortableWrapper(innerTimeout);
|
||||
const controller = makeTaggedAbortController(outerWrap);
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
abortSignal: controller.signal,
|
||||
}),
|
||||
).rejects.toBe(runError);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rethrows when thrown error has TimeoutError in cause chain (embedded run-budget timer)", async () => {
|
||||
const cfg = makeCfg();
|
||||
const innerTimeout = new Error("request timed out");
|
||||
innerTimeout.name = "TimeoutError";
|
||||
const outerAbort = makeAbortableWrapper(innerTimeout);
|
||||
const run = vi.fn().mockRejectedValue(outerAbort);
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
}),
|
||||
).rejects.toBe(outerAbort);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rethrows when thrown error has ClientDisconnectError in cause chain", async () => {
|
||||
const cfg = makeCfg();
|
||||
const innerDisconnect = new Error("client disconnected");
|
||||
innerDisconnect.name = "ClientDisconnectError";
|
||||
const outerAbort = makeAbortableWrapper(innerDisconnect);
|
||||
const run = vi.fn().mockRejectedValue(outerAbort);
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
}),
|
||||
).rejects.toBe(outerAbort);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rethrows when thrown error has restart abort in cause chain", async () => {
|
||||
const cfg = makeCfg();
|
||||
const restartAbort = createAgentRunRestartAbortError();
|
||||
const outerAbort = makeAbortableWrapper(restartAbort);
|
||||
const run = vi.fn().mockRejectedValueOnce(outerAbort).mockResolvedValueOnce("ok");
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
}),
|
||||
).rejects.toBe(outerAbort);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rethrows when an unmarked AbortError wraps a restart abort", async () => {
|
||||
const cfg = makeCfg();
|
||||
const restartAbort = createAgentRunRestartAbortError();
|
||||
const outerAbort = makeAbortWrapper(restartAbort);
|
||||
const run = vi.fn().mockRejectedValueOnce(outerAbort).mockResolvedValueOnce("ok");
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
}),
|
||||
).rejects.toBe(outerAbort);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("discards deferred session suspension for private terminal abort wrappers", () => {
|
||||
const timeout = new Error("request timed out");
|
||||
timeout.name = "TimeoutError";
|
||||
expect(
|
||||
testing.shouldDiscardDeferredSessionSuspension({
|
||||
error: makeAbortableWrapper(timeout),
|
||||
}),
|
||||
).toBe(true);
|
||||
|
||||
expect(
|
||||
testing.shouldDiscardDeferredSessionSuspension({
|
||||
error: makeAbortWrapper(createAgentRunRestartAbortError()),
|
||||
}),
|
||||
).toBe(true);
|
||||
|
||||
const providerTimeout = new Error("provider request timed out after 60s");
|
||||
providerTimeout.name = "TimeoutError";
|
||||
expect(
|
||||
testing.shouldDiscardDeferredSessionSuspension({
|
||||
error: makeAbortWrapper(providerTimeout),
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("falls back normally when a provider wraps its own timeout as AbortError(cause: TimeoutError) WITHOUT the abortable() marker", async () => {
|
||||
const cfg = makeCfg();
|
||||
const providerInnerTimeout = new Error("provider request timed out after 60s");
|
||||
providerInnerTimeout.name = "TimeoutError";
|
||||
const unmarkedAbortError = new Error("aborted", { cause: providerInnerTimeout });
|
||||
unmarkedAbortError.name = "AbortError";
|
||||
const run = vi.fn().mockRejectedValueOnce(unmarkedAbortError).mockResolvedValueOnce("ok");
|
||||
|
||||
const result = await runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
});
|
||||
|
||||
expect(result.result).toBe("ok");
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("falls back normally when a top-level provider TimeoutError is thrown (not an AbortError wrapper)", async () => {
|
||||
const cfg = makeCfg();
|
||||
const directProviderTimeout = new Error("provider request timed out after 60s");
|
||||
directProviderTimeout.name = "TimeoutError";
|
||||
const run = vi.fn().mockRejectedValueOnce(directProviderTimeout).mockResolvedValueOnce("ok");
|
||||
|
||||
const result = await runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
});
|
||||
|
||||
expect(result.result).toBe("ok");
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("falls back normally when thrown error is generic AbortError without terminal cause", async () => {
|
||||
const cfg = makeCfg();
|
||||
const run = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error("provider transient failure"))
|
||||
.mockResolvedValueOnce("ok");
|
||||
|
||||
const result = await runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
});
|
||||
|
||||
expect(result.result).toBe("ok");
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("skips fallback when the caller signal is aborted, even with a non-terminal reason", async () => {
|
||||
const cfg = makeCfg();
|
||||
const run = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error("provider had a sad day"))
|
||||
.mockResolvedValueOnce("ok");
|
||||
|
||||
const genericReason = new Error("some unrelated abort");
|
||||
const controller = makeTaggedAbortController(genericReason);
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
abortSignal: controller.signal,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(Error);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("falls back normally when no abortSignal is passed (back-compat)", async () => {
|
||||
const cfg = makeCfg();
|
||||
const run = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error("first attempt failed"))
|
||||
.mockResolvedValueOnce("ok");
|
||||
|
||||
const result = await runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
});
|
||||
|
||||
expect(result.result).toBe("ok");
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("falls back normally when signal is provided but not aborted", async () => {
|
||||
const cfg = makeCfg();
|
||||
const run = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error("first attempt failed"))
|
||||
.mockResolvedValueOnce("ok");
|
||||
|
||||
const controller = new AbortController();
|
||||
const result = await runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
abortSignal: controller.signal,
|
||||
});
|
||||
|
||||
expect(result.result).toBe("ok");
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("rethrows terminal abort even when error resembles a failover-normalizable error", async () => {
|
||||
const cfg = makeCfg();
|
||||
|
||||
const rateLimitLikeError = Object.assign(new Error("RESOURCE_EXHAUSTED: quota exceeded"), {
|
||||
status: 429,
|
||||
name: "AbortError",
|
||||
});
|
||||
|
||||
const run = vi.fn().mockRejectedValue(rateLimitLikeError);
|
||||
|
||||
const timeoutReason = new Error("request timed out");
|
||||
timeoutReason.name = "TimeoutError";
|
||||
const controller = new AbortController();
|
||||
controller.abort(timeoutReason);
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
abortSignal: controller.signal,
|
||||
}),
|
||||
).rejects.toBe(rateLimitLikeError);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("treats cron timeout string reason as terminal (covers plain-string abort)", async () => {
|
||||
const cfg = makeCfg();
|
||||
const run = vi.fn().mockRejectedValue(makeAbortError("aborted"));
|
||||
|
||||
const controller = new AbortController();
|
||||
controller.abort("cron: job execution timed out");
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
abortSignal: controller.signal,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(Error);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("treats phase-suffixed cron timeout reason as terminal (covers `(last phase: ...)` variant)", async () => {
|
||||
const cfg = makeCfg();
|
||||
const run = vi.fn().mockRejectedValue(makeAbortError("aborted"));
|
||||
|
||||
const controller = new AbortController();
|
||||
controller.abort("cron: job execution timed out (last phase: model_call_started)");
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
abortSignal: controller.signal,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(Error);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("treats isolated-agent setup-timeout (and phase suffix) as terminal", async () => {
|
||||
const cfg = makeCfg();
|
||||
const run = vi.fn().mockRejectedValue(makeAbortError("aborted"));
|
||||
|
||||
const controller = new AbortController();
|
||||
controller.abort(
|
||||
"cron: isolated agent setup timed out before runner start (last phase: workspace_provision)",
|
||||
);
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
abortSignal: controller.signal,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(Error);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("treats isolated-agent pre-execution stall (and phase suffix) as terminal", async () => {
|
||||
const cfg = makeCfg();
|
||||
const run = vi.fn().mockRejectedValue(makeAbortError("aborted"));
|
||||
|
||||
const controller = new AbortController();
|
||||
controller.abort(
|
||||
"cron: isolated agent run stalled before execution start (last phase: runner_ready)",
|
||||
);
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
abortSignal: controller.signal,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(Error);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("treats bare isolated-agent pre-execution stall as terminal", async () => {
|
||||
const cfg = makeCfg();
|
||||
const run = vi.fn().mockRejectedValue(makeAbortError("aborted"));
|
||||
|
||||
const controller = new AbortController();
|
||||
controller.abort("cron: isolated agent run stalled before execution start");
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
abortSignal: controller.signal,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(Error);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("treats an Error whose .message matches a known terminal string as terminal", async () => {
|
||||
const cfg = makeCfg();
|
||||
const run = vi.fn().mockRejectedValue(makeAbortError("aborted"));
|
||||
|
||||
const wrapped = new Error("cron: job execution timed out");
|
||||
const controller = new AbortController();
|
||||
controller.abort(wrapped);
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
abortSignal: controller.signal,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(Error);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not classify an unrelated string reason as terminal (caller-abort still skips fallback)", async () => {
|
||||
const cfg = makeCfg();
|
||||
const run = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error("first attempt failed"))
|
||||
.mockResolvedValueOnce("ok");
|
||||
|
||||
const controller = new AbortController();
|
||||
controller.abort("some unrelated cancel reason");
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
run,
|
||||
abortSignal: controller.signal,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(Error);
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("runWithImageModelFallback", () => {
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
resolveAgentModelPrimaryValue,
|
||||
} from "../config/model-input.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { isCronTerminalAbortReasonText } from "../cron/service/execution-errors.js";
|
||||
import { emitFailoverEvent } from "../infra/diagnostic-events.js";
|
||||
import { formatErrorMessage, toErrorObject } from "../infra/errors.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
@@ -31,6 +32,7 @@ import { isActiveUnusableWindow } from "./auth-profiles/usage-state.js";
|
||||
import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "./defaults.js";
|
||||
import { isLikelyContextOverflowError } from "./embedded-agent-helpers/errors.js";
|
||||
import type { FailoverReason } from "./embedded-agent-helpers/types.js";
|
||||
import { isOpenClawAbortableWrapper } from "./embedded-agent-runner/run/abortable.js";
|
||||
import {
|
||||
FailoverError,
|
||||
buildFailoverRemediationHint,
|
||||
@@ -188,21 +190,74 @@ type ModelFallbackRunFn<T> = (
|
||||
options?: ModelFallbackRunOptions,
|
||||
) => Promise<T>;
|
||||
|
||||
function isTerminalAbortReasonString(reason: string): boolean {
|
||||
return isCronTerminalAbortReasonText(reason);
|
||||
}
|
||||
|
||||
function getErrorCauseCandidates(err: Error): unknown[] {
|
||||
const candidates: unknown[] = [];
|
||||
if ("cause" in err && err.cause !== undefined) {
|
||||
candidates.push(err.cause);
|
||||
if (err.cause instanceof Error && "cause" in err.cause && err.cause.cause !== undefined) {
|
||||
candidates.push(err.cause.cause);
|
||||
}
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
|
||||
function isTerminalAbortCandidate(candidate: unknown): boolean {
|
||||
if (typeof candidate === "string") {
|
||||
return isTerminalAbortReasonString(candidate);
|
||||
}
|
||||
if (!(candidate instanceof Error)) {
|
||||
return false;
|
||||
}
|
||||
if (isAgentRunRestartAbortReason(candidate)) {
|
||||
return true;
|
||||
}
|
||||
if (candidate.name === "TimeoutError") {
|
||||
return true;
|
||||
}
|
||||
if (candidate.name === "ClientDisconnectError") {
|
||||
return true;
|
||||
}
|
||||
return isTerminalAbortReasonString(candidate.message);
|
||||
}
|
||||
|
||||
function isTerminalAbort(signal: AbortSignal | undefined): boolean {
|
||||
if (!signal?.aborted) {
|
||||
return false;
|
||||
}
|
||||
const reason = signal.reason;
|
||||
if (!(reason instanceof Error)) {
|
||||
|
||||
if (reason instanceof Error) {
|
||||
const candidates: unknown[] = [reason, ...getErrorCauseCandidates(reason)];
|
||||
return candidates.some(isTerminalAbortCandidate);
|
||||
}
|
||||
|
||||
return isTerminalAbortCandidate(reason);
|
||||
}
|
||||
|
||||
function isTerminalAbortFromError(err: unknown): boolean {
|
||||
if (!(err instanceof Error)) {
|
||||
return false;
|
||||
}
|
||||
if (isAgentRunRestartAbortReason(reason)) {
|
||||
if (isAgentRunRestartAbortReason(err)) {
|
||||
return true;
|
||||
}
|
||||
if (reason.name === "TimeoutError") {
|
||||
return true;
|
||||
const causeCandidates = getErrorCauseCandidates(err);
|
||||
if (err.name !== "AbortError") {
|
||||
return false;
|
||||
}
|
||||
return reason.name === "ClientDisconnectError";
|
||||
for (const candidate of causeCandidates) {
|
||||
if (isAgentRunRestartAbortReason(candidate)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (!isOpenClawAbortableWrapper(err)) {
|
||||
return false;
|
||||
}
|
||||
return causeCandidates.some(isTerminalAbortCandidate);
|
||||
}
|
||||
|
||||
function isCallerAbortSignal(signal: AbortSignal | undefined): boolean {
|
||||
@@ -357,6 +412,9 @@ async function runFallbackCandidate<T>(params: {
|
||||
if (isAgentRunDirectAbortReason(err) || isAgentRunRestartAbortReason(err)) {
|
||||
throw err;
|
||||
}
|
||||
if (isTerminalAbortFromError(err)) {
|
||||
throw err;
|
||||
}
|
||||
// Normalize abort-wrapped rate-limit errors (e.g. Google Vertex RESOURCE_EXHAUSTED)
|
||||
// so they become FailoverErrors and continue the fallback loop instead of aborting.
|
||||
const normalizedFailover = coerceToFailoverError(err, {
|
||||
@@ -809,6 +867,7 @@ export const testing = {
|
||||
resolveImageFallbackCandidates,
|
||||
resolveCooldownDecision,
|
||||
resolveSessionSuspensionReason,
|
||||
shouldDiscardDeferredSessionSuspension,
|
||||
} as const;
|
||||
|
||||
export function resolveModelCandidateChain(
|
||||
@@ -1310,6 +1369,7 @@ function shouldDiscardDeferredSessionSuspension(params: {
|
||||
isCallerAbortSignal(params.abortSignal) ||
|
||||
isAgentRunDirectAbortReason(params.error) ||
|
||||
isAgentRunRestartAbortReason(params.error) ||
|
||||
isTerminalAbortFromError(params.error) ||
|
||||
isCommandLaneTaskTimeoutError(params.error) ||
|
||||
isNonProviderRuntimeCoordinationError(params.error) ||
|
||||
isLikelyContextOverflowError(formatErrorMessage(params.error))
|
||||
|
||||
@@ -6,36 +6,58 @@ function formatCronAgentExecutionPhase(execution?: CronAgentExecutionStarted): s
|
||||
return formatEmbeddedAgentExecutionPhase(execution?.phase);
|
||||
}
|
||||
|
||||
const CRON_JOB_EXECUTION_TIMEOUT_ERROR = "cron: job execution timed out";
|
||||
const CRON_SETUP_TIMEOUT_ERROR = "cron: isolated agent setup timed out before runner start";
|
||||
const CRON_PRE_EXECUTION_TIMEOUT_ERROR = "cron: isolated agent run stalled before execution start";
|
||||
const CRON_TIMEOUT_ERROR_PREFIXES: readonly string[] = [
|
||||
CRON_JOB_EXECUTION_TIMEOUT_ERROR,
|
||||
CRON_SETUP_TIMEOUT_ERROR,
|
||||
CRON_PRE_EXECUTION_TIMEOUT_ERROR,
|
||||
];
|
||||
|
||||
function hasCronTimeoutPrefix(error: string, prefix: string): boolean {
|
||||
return error === prefix || error.startsWith(prefix + " ");
|
||||
}
|
||||
|
||||
export function isCronTerminalAbortReasonText(error: string): boolean {
|
||||
for (const prefix of CRON_TIMEOUT_ERROR_PREFIXES) {
|
||||
if (hasCronTimeoutPrefix(error, prefix)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Formats the generic cron execution timeout message with last-known phase context when available. */
|
||||
export function timeoutErrorMessage(execution?: CronAgentExecutionStarted): string {
|
||||
const phase = formatCronAgentExecutionPhase(execution);
|
||||
if (!phase) {
|
||||
return "cron: job execution timed out";
|
||||
return CRON_JOB_EXECUTION_TIMEOUT_ERROR;
|
||||
}
|
||||
return `cron: job execution timed out (last phase: ${phase})`;
|
||||
return `${CRON_JOB_EXECUTION_TIMEOUT_ERROR} (last phase: ${phase})`;
|
||||
}
|
||||
|
||||
/** Formats timeout text for runs that stalled before the isolated runner started. */
|
||||
export function setupTimeoutErrorMessage(execution?: CronAgentExecutionStarted): string {
|
||||
const phase = formatCronAgentExecutionPhase(execution);
|
||||
if (!phase) {
|
||||
return "cron: isolated agent setup timed out before runner start";
|
||||
return CRON_SETUP_TIMEOUT_ERROR;
|
||||
}
|
||||
return `cron: isolated agent setup timed out before runner start (last phase: ${phase})`;
|
||||
return `${CRON_SETUP_TIMEOUT_ERROR} (last phase: ${phase})`;
|
||||
}
|
||||
|
||||
/** Returns true for the setup-timeout class that fires before the isolated runner starts. */
|
||||
export function isSetupTimeoutErrorText(error: string): boolean {
|
||||
return error.startsWith("cron: isolated agent setup timed out before runner start");
|
||||
return hasCronTimeoutPrefix(error, CRON_SETUP_TIMEOUT_ERROR);
|
||||
}
|
||||
|
||||
/** Formats timeout text for runs that stalled after setup but before execution start. */
|
||||
export function preExecutionTimeoutErrorMessage(execution?: CronAgentExecutionStarted): string {
|
||||
const phase = formatCronAgentExecutionPhase(execution);
|
||||
if (!phase) {
|
||||
return "cron: isolated agent run stalled before execution start";
|
||||
return CRON_PRE_EXECUTION_TIMEOUT_ERROR;
|
||||
}
|
||||
return `cron: isolated agent run stalled before execution start (last phase: ${phase})`;
|
||||
return `${CRON_PRE_EXECUTION_TIMEOUT_ERROR} (last phase: ${phase})`;
|
||||
}
|
||||
|
||||
/** Extracts a human timeout/abort reason, falling back to the canonical cron timeout text. */
|
||||
|
||||
@@ -134,6 +134,14 @@ export function setSseHeaders(res: ServerResponse) {
|
||||
res.flushHeaders?.();
|
||||
}
|
||||
|
||||
/** Abort reason used when the HTTP client disconnects before delivery. */
|
||||
export class ClientDisconnectError extends Error {
|
||||
constructor(message = "HTTP client disconnected") {
|
||||
super(message);
|
||||
this.name = "ClientDisconnectError";
|
||||
}
|
||||
}
|
||||
|
||||
export function watchClientDisconnect(
|
||||
req: IncomingMessage,
|
||||
res: ServerResponse,
|
||||
@@ -153,7 +161,7 @@ export function watchClientDisconnect(
|
||||
const handleClose = () => {
|
||||
onDisconnect?.();
|
||||
if (!abortController.signal.aborted) {
|
||||
abortController.abort();
|
||||
abortController.abort(new ClientDisconnectError());
|
||||
}
|
||||
};
|
||||
for (const socket of sockets) {
|
||||
|
||||
@@ -822,6 +822,7 @@ function buildArtifactsCapture(params: {
|
||||
runtimeArtifacts?.timedOutDuringCompaction ?? runtimeEnd?.timedOutDuringCompaction,
|
||||
timedOutDuringToolExecution:
|
||||
runtimeArtifacts?.timedOutDuringToolExecution ?? runtimeEnd?.timedOutDuringToolExecution,
|
||||
timedOutByRunBudget: runtimeArtifacts?.timedOutByRunBudget ?? runtimeEnd?.timedOutByRunBudget,
|
||||
promptError:
|
||||
runtimeArtifacts?.promptError ?? runtimeEnd?.promptError ?? runtimeCompletion?.promptError,
|
||||
promptErrorSource: runtimeArtifacts?.promptErrorSource ?? runtimeCompletion?.promptErrorSource,
|
||||
|
||||
@@ -268,6 +268,7 @@ describe("trajectory metadata", () => {
|
||||
idleTimedOut: false,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
timedOutByRunBudget: false,
|
||||
compactionCount: 1,
|
||||
assistantTexts: ["done"],
|
||||
finalPromptText: "run tests",
|
||||
|
||||
@@ -50,6 +50,7 @@ type BuildTrajectoryArtifactsParams = {
|
||||
idleTimedOut: boolean;
|
||||
timedOutDuringCompaction: boolean;
|
||||
timedOutDuringToolExecution: boolean;
|
||||
timedOutByRunBudget: boolean;
|
||||
promptError?: string;
|
||||
promptErrorSource?: string | null;
|
||||
terminalError?: string;
|
||||
@@ -321,6 +322,7 @@ export function buildTrajectoryArtifacts(
|
||||
idleTimedOut: params.idleTimedOut,
|
||||
timedOutDuringCompaction: params.timedOutDuringCompaction,
|
||||
timedOutDuringToolExecution: params.timedOutDuringToolExecution,
|
||||
timedOutByRunBudget: params.timedOutByRunBudget,
|
||||
promptError: params.promptError,
|
||||
promptErrorSource: params.promptErrorSource,
|
||||
terminalError: params.terminalError,
|
||||
|
||||
Reference in New Issue
Block a user