mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-24 11:25:50 -06:00
fix(failover): retry truncated provider streams
This commit is contained in:
@@ -146,7 +146,6 @@ export const structuredMiscCases = [
|
||||
{ id: "retry-reset-before-headers", message: "upstream reset before headers" },
|
||||
{ id: "retry-websocket-closed", message: "WebSocket closed unexpectedly" },
|
||||
{ id: "retry-websocket-error", message: "WebSocket error" },
|
||||
{ id: "retry-anthropic-message-stop", message: "stream ended before message_stop" },
|
||||
{ id: "retry-http2-no-response", message: "HTTP2 request did not get a response" },
|
||||
{ id: "retry-delay", message: "retry delay 1000ms" },
|
||||
{
|
||||
@@ -155,6 +154,15 @@ export const structuredMiscCases = [
|
||||
},
|
||||
{ id: "retry-rate-limit-hyphen", message: "rate-limit reached" },
|
||||
]),
|
||||
...messageRows(retrySource, reason("timeout"), [
|
||||
{ id: "retry-anthropic-message-stop", message: "stream ended before message_stop" },
|
||||
{ id: "retry-bedrock-message-stop", message: "Bedrock stream ended before messageStop" },
|
||||
{ id: "retry-proxy-terminal-event", message: "Proxy stream ended before terminal event" },
|
||||
{
|
||||
id: "retry-google-terminal-finish-reason",
|
||||
message: "Google stream ended before a terminal finish reason",
|
||||
},
|
||||
]),
|
||||
...messageRows(errorsSource, null, [
|
||||
{
|
||||
id: "errors-json-parse-position",
|
||||
|
||||
@@ -14,6 +14,9 @@ export const failoverRetryExpectations = {
|
||||
"retry-websocket-closed": true,
|
||||
"retry-websocket-error": true,
|
||||
"retry-anthropic-message-stop": true,
|
||||
"retry-bedrock-message-stop": true,
|
||||
"retry-proxy-terminal-event": true,
|
||||
"retry-google-terminal-finish-reason": true,
|
||||
"retry-http2-no-response": true,
|
||||
"retry-delay": true,
|
||||
"retry-ended-without-terminal-response": true,
|
||||
|
||||
@@ -59,6 +59,8 @@ const STATUS_INTERNAL_SERVER_ERROR_RE = /\bstatus:\s*internal server error\b/i;
|
||||
const STATUS_INTERNAL_SERVER_ERROR_WITH_500_RE =
|
||||
/^(?=[\s\S]*\bstatus:\s*internal server error\b)(?=[\s\S]*\bcode["']?\s*[:=]\s*500\b)/i;
|
||||
const HTTP_5XX_STATUS_RE = /\bHTTP\s+5\d\d\b/i;
|
||||
const INCOMPLETE_TERMINAL_STREAM_RE =
|
||||
/^(?:[a-z][\w.-]* )?stream ended before (?:message_?stop|terminal event|a terminal finish reason)$/i;
|
||||
const BILLING_ERROR_HARD_402_RE =
|
||||
/["']?(?:status|code)["']?\s*[:=]\s*402\b|\bhttp\s*402\b|\berror(?:\s+code)?\s*[:=]?\s*402\b|^\s*402\s+payment/i;
|
||||
|
||||
@@ -145,6 +147,7 @@ const ERROR_PATTERNS = {
|
||||
// Codex and node-fetch expose these exact terminal transport messages.
|
||||
// Keep them anchored so unrelated local stream failures do not trigger model failover.
|
||||
/^stream disconnected before completion(?::[\s\S]*)?$/i,
|
||||
INCOMPLETE_TERMINAL_STREAM_RE,
|
||||
/^premature close of server response while trying to fetch\b/i,
|
||||
// Chinese provider error messages (ZhipuAI/GLM, Bailian, Kimi/Moonshot, DeepSeek, etc.)
|
||||
"网络错误",
|
||||
@@ -296,6 +299,9 @@ export function isRateLimitErrorMessage(raw: string): boolean {
|
||||
export function isTimeoutErrorMessage(raw: string): boolean {
|
||||
return matchesErrorPatterns(raw, ERROR_PATTERNS.timeout);
|
||||
}
|
||||
export function isIncompleteTerminalStreamMessage(raw: string): boolean {
|
||||
return INCOMPLETE_TERMINAL_STREAM_RE.test(raw.trim());
|
||||
}
|
||||
|
||||
/**
|
||||
* Provider stream completed with an explicit error finish/stop reason.
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
extractLeadingHttpStatus,
|
||||
extractProviderWrappedHttpStatus,
|
||||
} from "../../shared/assistant-error-format.js";
|
||||
import { isIncompleteTerminalStreamMessage } from "./message-patterns.js";
|
||||
import type { FailoverClassification, FailoverSignal } from "./signal.js";
|
||||
|
||||
type RateLimitWindow =
|
||||
@@ -17,7 +18,7 @@ const RETRYABLE_HTTP_STATUS_CODES = new Set([429, 500, 502, 503, 504, 524]);
|
||||
const RATE_LIMIT_RETRY_CONTEXT_RE =
|
||||
/rate.?limit|too many requests|resource[_ -]?exhausted|daily (?:request|usage) limit|requests? per day|tokens? per day|quota[_ -]?exceeded/i;
|
||||
const TRANSIENT_RETRY_EVIDENCE_RE =
|
||||
/overloaded|rate.?limit|too many requests|service.?unavailable|server.?error|internal.?error|provider.?returned.?error|network.?error|connection.?error|connection.?refused|connection.?lost|other side closed|fetch failed|upstream.?connect|reset before headers|socket hang up|socket connection was closed|timed? out|timeout|terminated|websocket.?closed|websocket.?error|ended without|stream ended before message_stop|http2 request did not get a response|retry delay|you can retry your request|try your request again|please retry your request|resource[_ -]?exhausted/i;
|
||||
/overloaded|rate.?limit|too many requests|service.?unavailable|server.?error|internal.?error|provider.?returned.?error|network.?error|connection.?error|connection.?refused|connection.?lost|other side closed|fetch failed|upstream.?connect|reset before headers|socket hang up|socket connection was closed|timed? out|timeout|terminated|websocket.?closed|websocket.?error|ended without|http2 request did not get a response|retry delay|you can retry your request|try your request again|please retry your request|resource[_ -]?exhausted/i;
|
||||
const LONG_WINDOW_RATE_LIMIT_RE =
|
||||
/\b(?:daily|weekly|monthly|tokens per day|requests per day|usage limit|subscription|insufficient[_ -]?quota|current quota|quota[_ -]?exceeded|(?:go|free)usagelimiterror|available balance|out of budget)\b/i;
|
||||
const SHORT_RATE_LIMIT_UNIT_RE =
|
||||
@@ -54,6 +55,7 @@ export function hasTransientRetryEvidence(
|
||||
const status = resolveRetrySignalStatus(signal);
|
||||
return (
|
||||
(status !== undefined && RETRYABLE_HTTP_STATUS_CODES.has(status)) ||
|
||||
isIncompleteTerminalStreamMessage(signal.message ?? "") ||
|
||||
TRANSIENT_RETRY_EVIDENCE_RE.test(signal.message ?? "") ||
|
||||
isTransientNetworkError({ code: signal.code })
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user