mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-20 01:21:41 -06:00
fix(failover): classify transient Undici error codes (#123042)
* fix(failover): classify transient Undici error codes * fix(failover): share transient network code policy Signed-off-by: sallyom <somalley@redhat.com> --------- Signed-off-by: sallyom <somalley@redhat.com> Co-authored-by: sallyom <somalley@redhat.com>
This commit is contained in:
@@ -25,6 +25,25 @@ describe("classifyAssistantFailoverReason", () => {
|
||||
expect(classifyAssistantFailoverReason(opencodeGoStalledStreamError)).toBe("timeout");
|
||||
});
|
||||
|
||||
it.each([
|
||||
"ENOTFOUND",
|
||||
"UND_ERR_CONNECT_TIMEOUT",
|
||||
"UND_ERR_DNS_RESOLVE_FAILED",
|
||||
"UND_ERR_CONNECT",
|
||||
"UND_ERR_SOCKET",
|
||||
"UND_ERR_HEADERS_TIMEOUT",
|
||||
"UND_ERR_BODY_TIMEOUT",
|
||||
])("classifies structured %s assistant errors as timeouts", (errorCode) => {
|
||||
expect(
|
||||
classifyAssistantFailoverReason({
|
||||
...opencodeGoStalledStreamError,
|
||||
provider: "demo-provider",
|
||||
errorCode,
|
||||
errorMessage: "provider connection closed",
|
||||
}),
|
||||
).toBe("timeout");
|
||||
});
|
||||
|
||||
it("does not classify caller-aborted assistant messages as provider failover", () => {
|
||||
expect(
|
||||
classifyAssistantFailoverReason({
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { normalizeOptionalLowercaseString } from "@openclaw/normalization-core/string-coerce";
|
||||
import { isTransientNetworkError } from "../../infra/retryable-network-errors.js";
|
||||
import {
|
||||
extractLeadingHttpStatus,
|
||||
parseApiErrorInfo,
|
||||
@@ -9,18 +10,9 @@ import {
|
||||
isRateLimitErrorMessage,
|
||||
} from "./message-patterns.js";
|
||||
import type { FailoverClassification, FailoverReason, FailoverSignal } from "./signal.js";
|
||||
const TIMEOUT_ERROR_CODES = new Set([
|
||||
"ETIMEDOUT",
|
||||
"ESOCKETTIMEDOUT",
|
||||
"ECONNRESET",
|
||||
"ECONNABORTED",
|
||||
"ECONNREFUSED",
|
||||
"ENETUNREACH",
|
||||
"EHOSTUNREACH",
|
||||
const FAILOVER_TIMEOUT_ERROR_CODES = new Set([
|
||||
"EHOSTDOWN",
|
||||
"ENETRESET",
|
||||
"EPIPE",
|
||||
"EAI_AGAIN",
|
||||
"ERR_STREAM_PREMATURE_CLOSE",
|
||||
]);
|
||||
const NO_BODY_HTTP_WRAPPER_RE =
|
||||
@@ -342,7 +334,10 @@ export function classifyFailoverReasonFromCode(raw: string | undefined): Failove
|
||||
case "OVERLOADED_ERROR":
|
||||
return "overloaded";
|
||||
default:
|
||||
return TIMEOUT_ERROR_CODES.has(normalized) ? "timeout" : null;
|
||||
return FAILOVER_TIMEOUT_ERROR_CODES.has(normalized) ||
|
||||
isTransientNetworkError({ code: normalized })
|
||||
? "timeout"
|
||||
: null;
|
||||
}
|
||||
}
|
||||
export function classifyCoreFailoverReasonFromErrorType(
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { parseRetryAfterHttpDateMs } from "@openclaw/ai/internal/retry-after";
|
||||
import milliseconds from "ms";
|
||||
import { isTransientNetworkError } from "../../infra/retryable-network-errors.js";
|
||||
import {
|
||||
extractErrorHttpStatus,
|
||||
extractLeadingHttpStatus,
|
||||
@@ -48,12 +49,13 @@ function resolveRetrySignalStatus(signal: Pick<FailoverSignal, "message" | "stat
|
||||
|
||||
/** Narrow evidence that replaying the same assistant request may succeed within this session. */
|
||||
export function hasTransientRetryEvidence(
|
||||
signal: Pick<FailoverSignal, "message" | "status">,
|
||||
signal: Pick<FailoverSignal, "code" | "message" | "status">,
|
||||
): boolean {
|
||||
const status = resolveRetrySignalStatus(signal);
|
||||
return (
|
||||
(status !== undefined && RETRYABLE_HTTP_STATUS_CODES.has(status)) ||
|
||||
TRANSIENT_RETRY_EVIDENCE_RE.test(signal.message ?? "")
|
||||
TRANSIENT_RETRY_EVIDENCE_RE.test(signal.message ?? "") ||
|
||||
isTransientNetworkError({ code: signal.code })
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -55,6 +55,15 @@ describe("isRetryableAssistantError", () => {
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("retries a structured transient Undici error", () => {
|
||||
expect(
|
||||
isRetryableAssistantError({
|
||||
...errorMessage("provider connection closed"),
|
||||
errorCode: "UND_ERR_HEADERS_TIMEOUT",
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"An error occurred while processing your request. You can retry your request.",
|
||||
"The system encountered an unexpected error. Try your request again.",
|
||||
|
||||
@@ -36,6 +36,8 @@ export function isRetryableAssistantError(message: AssistantMessage): boolean {
|
||||
const signal = {
|
||||
message: errorMessage,
|
||||
provider: message.provider,
|
||||
code: message.errorCode,
|
||||
errorType: message.errorType,
|
||||
...(status === undefined ? {} : { status }),
|
||||
};
|
||||
const classification = classifyFailoverSignal(signal);
|
||||
|
||||
Reference in New Issue
Block a user