mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-25 20:05:46 -06:00
fix(agents): stop fallback on gateway draining (#118101)
This commit is contained in:
@@ -4,6 +4,7 @@
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createAgentRunStaleLifecycleError } from "../infra/agent-lifecycle-error.js";
|
||||
import { GatewayDrainingError } from "../process/gateway-work-admission.js";
|
||||
import { classifyFailoverSignal } from "./embedded-agent-helpers/errors.js";
|
||||
import {
|
||||
buildFailoverRemediationHint,
|
||||
@@ -1428,6 +1429,20 @@ describe("failover-error", () => {
|
||||
expect(isNonProviderRuntimeCoordinationError(abortWrapper)).toBe(true);
|
||||
});
|
||||
|
||||
it("returns true for direct and nested gateway drain admission failures", () => {
|
||||
const draining = new GatewayDrainingError();
|
||||
const causeWrapper = new Error("session send failed", { cause: draining });
|
||||
const aggregateWrapper = new AggregateError(
|
||||
[new Error("cleanup failed"), { error: draining }],
|
||||
"agent run failed",
|
||||
);
|
||||
|
||||
for (const error of [draining, causeWrapper, aggregateWrapper]) {
|
||||
expect(isNonProviderRuntimeCoordinationError(error)).toBe(true);
|
||||
expect(resolveModelFallbackError(error)).toEqual({ kind: "coordination", error });
|
||||
}
|
||||
});
|
||||
|
||||
it("returns true when the coordination error is nested via cause", () => {
|
||||
const wrapped = new Error("wrapper", { cause: makeSessionLockError() });
|
||||
expect(isNonProviderRuntimeCoordinationError(wrapped)).toBe(true);
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
import { parseStrictNonNegativeInteger } from "@openclaw/normalization-core/number-coercion";
|
||||
import { formatCliCommand } from "../cli/command-format.js";
|
||||
import { isAgentRunStaleLifecycleError } from "../infra/agent-lifecycle-error.js";
|
||||
import { readErrorName } from "../infra/errors.js";
|
||||
import { collectErrorGraphCandidates, readErrorName } from "../infra/errors.js";
|
||||
import {
|
||||
classifyFailoverSignal,
|
||||
extractFailoverSignalDetails,
|
||||
@@ -505,6 +505,13 @@ function hasStaleAgentRunLifecycleFailure(err: unknown): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
function hasGatewayDrainingFailure(err: unknown): boolean {
|
||||
return collectErrorGraphCandidates(err, (candidate) => {
|
||||
const errors = candidate.errors;
|
||||
return [candidate.error, candidate.cause, ...(Array.isArray(errors) ? errors : [])];
|
||||
}).some((candidate) => readErrorName(candidate) === "GatewayDrainingError");
|
||||
}
|
||||
|
||||
function hasDirectProviderFailureIdentity(err: unknown): boolean {
|
||||
if (isFailoverError(err)) {
|
||||
return true;
|
||||
@@ -919,6 +926,11 @@ export function resolveModelFallbackError(
|
||||
if (err instanceof AgentHarnessSessionSupersededError) {
|
||||
return { kind: "coordination", error: err };
|
||||
}
|
||||
// Gateway admission can fail before any provider turn starts. Preserve that
|
||||
// identity through wrappers and aggregates so fallback cannot blame a model.
|
||||
if (hasGatewayDrainingFailure(err)) {
|
||||
return { kind: "coordination", error: err };
|
||||
}
|
||||
const staleLifecycleFailure = hasStaleAgentRunLifecycleFailure(err);
|
||||
if (
|
||||
staleLifecycleFailure &&
|
||||
|
||||
@@ -16,6 +16,7 @@ import { createWarnLogCapture } from "../logging/test-helpers/warn-log-capture.j
|
||||
import { setCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-snapshot.js";
|
||||
import { clearCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-state.js";
|
||||
import { loadPluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js";
|
||||
import { GatewayDrainingError } from "../process/gateway-work-admission.js";
|
||||
import { AgentRunTerminalOutcomeError } from "./agent-run-terminal-outcome.js";
|
||||
import { AUTH_STORE_VERSION } from "./auth-profiles/constants.js";
|
||||
import type { AuthProfileStore } from "./auth-profiles/types.js";
|
||||
@@ -2080,6 +2081,60 @@ describe("runWithModelFallback", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["direct", () => new GatewayDrainingError()],
|
||||
["cause", () => new Error("session send failed", { cause: new GatewayDrainingError() })],
|
||||
[
|
||||
"aggregate",
|
||||
() =>
|
||||
new AggregateError(
|
||||
[new Error("cleanup failed"), new GatewayDrainingError()],
|
||||
"agent run failed",
|
||||
),
|
||||
],
|
||||
])("aborts fallback on %s gateway drain failures", async (_label, makeError) => {
|
||||
const error = makeError();
|
||||
const run = vi.fn().mockRejectedValueOnce(error).mockResolvedValueOnce("too late");
|
||||
const onError = vi.fn();
|
||||
const onFallbackStep = vi.fn();
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg: undefined,
|
||||
provider: "openai",
|
||||
model: "gpt-5.6-sol",
|
||||
fallbacksOverride: ["openai/gpt-5.4-mini"],
|
||||
skipAuthProfileRuntime: true,
|
||||
run,
|
||||
onError,
|
||||
onFallbackStep,
|
||||
}),
|
||||
).rejects.toBe(error);
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
expect(onFallbackStep).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("still advances after a genuine provider rate limit", async () => {
|
||||
const rateLimit = Object.assign(new Error("rate limit exceeded"), { status: 429 });
|
||||
const run = vi.fn().mockRejectedValueOnce(rateLimit).mockResolvedValueOnce("fallback ok");
|
||||
|
||||
const result = await runWithModelFallback({
|
||||
cfg: undefined,
|
||||
provider: "openai",
|
||||
model: "gpt-5.6-sol",
|
||||
fallbacksOverride: ["openai/gpt-5.4-mini"],
|
||||
skipAuthProfileRuntime: true,
|
||||
run,
|
||||
});
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
expect(result.result).toBe("fallback ok");
|
||||
expect(result.provider).toBe("openai");
|
||||
expect(result.model).toBe("gpt-5.4-mini");
|
||||
expect(result.attempts[0]).toMatchObject({ reason: "rate_limit", status: 429 });
|
||||
});
|
||||
|
||||
it("aborts the fallback chain on transcript continuation failures without candidate_failed attribution", async () => {
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
|
||||
Reference in New Issue
Block a user