fix(agents): stop fallback on gateway draining (#118101)

This commit is contained in:
Jason (Json)
2026-08-02 13:26:37 -06:00
committed by GitHub
parent bbc8d3b82f
commit dd8e07c11f
3 changed files with 83 additions and 1 deletions
+15
View File
@@ -4,6 +4,7 @@
*/
import { describe, expect, it } from "vitest";
import { createAgentRunStaleLifecycleError } from "../infra/agent-lifecycle-error.js";
import { GatewayDrainingError } from "../process/gateway-work-admission.js";
import { classifyFailoverSignal } from "./embedded-agent-helpers/errors.js";
import {
buildFailoverRemediationHint,
@@ -1428,6 +1429,20 @@ describe("failover-error", () => {
expect(isNonProviderRuntimeCoordinationError(abortWrapper)).toBe(true);
});
it("returns true for direct and nested gateway drain admission failures", () => {
const draining = new GatewayDrainingError();
const causeWrapper = new Error("session send failed", { cause: draining });
const aggregateWrapper = new AggregateError(
[new Error("cleanup failed"), { error: draining }],
"agent run failed",
);
for (const error of [draining, causeWrapper, aggregateWrapper]) {
expect(isNonProviderRuntimeCoordinationError(error)).toBe(true);
expect(resolveModelFallbackError(error)).toEqual({ kind: "coordination", error });
}
});
it("returns true when the coordination error is nested via cause", () => {
const wrapped = new Error("wrapper", { cause: makeSessionLockError() });
expect(isNonProviderRuntimeCoordinationError(wrapped)).toBe(true);
+13 -1
View File
@@ -6,7 +6,7 @@
import { parseStrictNonNegativeInteger } from "@openclaw/normalization-core/number-coercion";
import { formatCliCommand } from "../cli/command-format.js";
import { isAgentRunStaleLifecycleError } from "../infra/agent-lifecycle-error.js";
import { readErrorName } from "../infra/errors.js";
import { collectErrorGraphCandidates, readErrorName } from "../infra/errors.js";
import {
classifyFailoverSignal,
extractFailoverSignalDetails,
@@ -505,6 +505,13 @@ function hasStaleAgentRunLifecycleFailure(err: unknown): boolean {
);
}
function hasGatewayDrainingFailure(err: unknown): boolean {
return collectErrorGraphCandidates(err, (candidate) => {
const errors = candidate.errors;
return [candidate.error, candidate.cause, ...(Array.isArray(errors) ? errors : [])];
}).some((candidate) => readErrorName(candidate) === "GatewayDrainingError");
}
function hasDirectProviderFailureIdentity(err: unknown): boolean {
if (isFailoverError(err)) {
return true;
@@ -919,6 +926,11 @@ export function resolveModelFallbackError(
if (err instanceof AgentHarnessSessionSupersededError) {
return { kind: "coordination", error: err };
}
// Gateway admission can fail before any provider turn starts. Preserve that
// identity through wrappers and aggregates so fallback cannot blame a model.
if (hasGatewayDrainingFailure(err)) {
return { kind: "coordination", error: err };
}
const staleLifecycleFailure = hasStaleAgentRunLifecycleFailure(err);
if (
staleLifecycleFailure &&
+55
View File
@@ -16,6 +16,7 @@ import { createWarnLogCapture } from "../logging/test-helpers/warn-log-capture.j
import { setCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-snapshot.js";
import { clearCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-state.js";
import { loadPluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js";
import { GatewayDrainingError } from "../process/gateway-work-admission.js";
import { AgentRunTerminalOutcomeError } from "./agent-run-terminal-outcome.js";
import { AUTH_STORE_VERSION } from "./auth-profiles/constants.js";
import type { AuthProfileStore } from "./auth-profiles/types.js";
@@ -2080,6 +2081,60 @@ describe("runWithModelFallback", () => {
);
});
it.each([
["direct", () => new GatewayDrainingError()],
["cause", () => new Error("session send failed", { cause: new GatewayDrainingError() })],
[
"aggregate",
() =>
new AggregateError(
[new Error("cleanup failed"), new GatewayDrainingError()],
"agent run failed",
),
],
])("aborts fallback on %s gateway drain failures", async (_label, makeError) => {
const error = makeError();
const run = vi.fn().mockRejectedValueOnce(error).mockResolvedValueOnce("too late");
const onError = vi.fn();
const onFallbackStep = vi.fn();
await expect(
runWithModelFallback({
cfg: undefined,
provider: "openai",
model: "gpt-5.6-sol",
fallbacksOverride: ["openai/gpt-5.4-mini"],
skipAuthProfileRuntime: true,
run,
onError,
onFallbackStep,
}),
).rejects.toBe(error);
expect(run).toHaveBeenCalledTimes(1);
expect(onError).not.toHaveBeenCalled();
expect(onFallbackStep).not.toHaveBeenCalled();
});
it("still advances after a genuine provider rate limit", async () => {
const rateLimit = Object.assign(new Error("rate limit exceeded"), { status: 429 });
const run = vi.fn().mockRejectedValueOnce(rateLimit).mockResolvedValueOnce("fallback ok");
const result = await runWithModelFallback({
cfg: undefined,
provider: "openai",
model: "gpt-5.6-sol",
fallbacksOverride: ["openai/gpt-5.4-mini"],
skipAuthProfileRuntime: true,
run,
});
expect(run).toHaveBeenCalledTimes(2);
expect(result.result).toBe("fallback ok");
expect(result.provider).toBe("openai");
expect(result.model).toBe("gpt-5.4-mini");
expect(result.attempts[0]).toMatchObject({ reason: "rate_limit", status: 429 });
});
it("aborts the fallback chain on transcript continuation failures without candidate_failed attribution", async () => {
const cfg = makeCfg({
agents: {