From 44b3d452c9711e85f39debdd29a4e26cb6280ea5 Mon Sep 17 00:00:00 2001 From: vyctorbrzezowski Date: Sat, 22 Aug 2026 09:16:25 +0000 Subject: [PATCH] fix(failover): retry truncated provider streams --- .../failover-classification.structured-misc.cases.ts | 10 +++++++++- .../failover/failover-retry.expected.test-support.ts | 3 +++ src/agents/failover/message-patterns.ts | 6 ++++++ src/agents/failover/retry-evidence.ts | 4 +++- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/agents/failover/failover-classification.structured-misc.cases.ts b/src/agents/failover/failover-classification.structured-misc.cases.ts index 7c7e4945a494..4535e881b950 100644 --- a/src/agents/failover/failover-classification.structured-misc.cases.ts +++ b/src/agents/failover/failover-classification.structured-misc.cases.ts @@ -146,7 +146,6 @@ export const structuredMiscCases = [ { id: "retry-reset-before-headers", message: "upstream reset before headers" }, { id: "retry-websocket-closed", message: "WebSocket closed unexpectedly" }, { id: "retry-websocket-error", message: "WebSocket error" }, - { id: "retry-anthropic-message-stop", message: "stream ended before message_stop" }, { id: "retry-http2-no-response", message: "HTTP2 request did not get a response" }, { id: "retry-delay", message: "retry delay 1000ms" }, { @@ -155,6 +154,15 @@ export const structuredMiscCases = [ }, { id: "retry-rate-limit-hyphen", message: "rate-limit reached" }, ]), + ...messageRows(retrySource, reason("timeout"), [ + { id: "retry-anthropic-message-stop", message: "stream ended before message_stop" }, + { id: "retry-bedrock-message-stop", message: "Bedrock stream ended before messageStop" }, + { id: "retry-proxy-terminal-event", message: "Proxy stream ended before terminal event" }, + { + id: "retry-google-terminal-finish-reason", + message: "Google stream ended before a terminal finish reason", + }, + ]), ...messageRows(errorsSource, null, [ { id: "errors-json-parse-position", diff --git a/src/agents/failover/failover-retry.expected.test-support.ts b/src/agents/failover/failover-retry.expected.test-support.ts index 0923b956e011..5bed4850631b 100644 --- a/src/agents/failover/failover-retry.expected.test-support.ts +++ b/src/agents/failover/failover-retry.expected.test-support.ts @@ -14,6 +14,9 @@ export const failoverRetryExpectations = { "retry-websocket-closed": true, "retry-websocket-error": true, "retry-anthropic-message-stop": true, + "retry-bedrock-message-stop": true, + "retry-proxy-terminal-event": true, + "retry-google-terminal-finish-reason": true, "retry-http2-no-response": true, "retry-delay": true, "retry-ended-without-terminal-response": true, diff --git a/src/agents/failover/message-patterns.ts b/src/agents/failover/message-patterns.ts index 25e73df1a158..add7d3d8894a 100644 --- a/src/agents/failover/message-patterns.ts +++ b/src/agents/failover/message-patterns.ts @@ -59,6 +59,8 @@ const STATUS_INTERNAL_SERVER_ERROR_RE = /\bstatus:\s*internal server error\b/i; const STATUS_INTERNAL_SERVER_ERROR_WITH_500_RE = /^(?=[\s\S]*\bstatus:\s*internal server error\b)(?=[\s\S]*\bcode["']?\s*[:=]\s*500\b)/i; const HTTP_5XX_STATUS_RE = /\bHTTP\s+5\d\d\b/i; +const INCOMPLETE_TERMINAL_STREAM_RE = + /^(?:[a-z][\w.-]* )?stream ended before (?:message_?stop|terminal event|a terminal finish reason)$/i; const BILLING_ERROR_HARD_402_RE = /["']?(?:status|code)["']?\s*[:=]\s*402\b|\bhttp\s*402\b|\berror(?:\s+code)?\s*[:=]?\s*402\b|^\s*402\s+payment/i; @@ -145,6 +147,7 @@ const ERROR_PATTERNS = { // Codex and node-fetch expose these exact terminal transport messages. // Keep them anchored so unrelated local stream failures do not trigger model failover. /^stream disconnected before completion(?::[\s\S]*)?$/i, + INCOMPLETE_TERMINAL_STREAM_RE, /^premature close of server response while trying to fetch\b/i, // Chinese provider error messages (ZhipuAI/GLM, Bailian, Kimi/Moonshot, DeepSeek, etc.) "网络错误", @@ -296,6 +299,9 @@ export function isRateLimitErrorMessage(raw: string): boolean { export function isTimeoutErrorMessage(raw: string): boolean { return matchesErrorPatterns(raw, ERROR_PATTERNS.timeout); } +export function isIncompleteTerminalStreamMessage(raw: string): boolean { + return INCOMPLETE_TERMINAL_STREAM_RE.test(raw.trim()); +} /** * Provider stream completed with an explicit error finish/stop reason. diff --git a/src/agents/failover/retry-evidence.ts b/src/agents/failover/retry-evidence.ts index 7a1f7708e59c..8183863275cf 100644 --- a/src/agents/failover/retry-evidence.ts +++ b/src/agents/failover/retry-evidence.ts @@ -6,6 +6,7 @@ import { extractLeadingHttpStatus, extractProviderWrappedHttpStatus, } from "../../shared/assistant-error-format.js"; +import { isIncompleteTerminalStreamMessage } from "./message-patterns.js"; import type { FailoverClassification, FailoverSignal } from "./signal.js"; type RateLimitWindow = @@ -17,7 +18,7 @@ const RETRYABLE_HTTP_STATUS_CODES = new Set([429, 500, 502, 503, 504, 524]); const RATE_LIMIT_RETRY_CONTEXT_RE = /rate.?limit|too many requests|resource[_ -]?exhausted|daily (?:request|usage) limit|requests? per day|tokens? per day|quota[_ -]?exceeded/i; const TRANSIENT_RETRY_EVIDENCE_RE = - /overloaded|rate.?limit|too many requests|service.?unavailable|server.?error|internal.?error|provider.?returned.?error|network.?error|connection.?error|connection.?refused|connection.?lost|other side closed|fetch failed|upstream.?connect|reset before headers|socket hang up|socket connection was closed|timed? out|timeout|terminated|websocket.?closed|websocket.?error|ended without|stream ended before message_stop|http2 request did not get a response|retry delay|you can retry your request|try your request again|please retry your request|resource[_ -]?exhausted/i; + /overloaded|rate.?limit|too many requests|service.?unavailable|server.?error|internal.?error|provider.?returned.?error|network.?error|connection.?error|connection.?refused|connection.?lost|other side closed|fetch failed|upstream.?connect|reset before headers|socket hang up|socket connection was closed|timed? out|timeout|terminated|websocket.?closed|websocket.?error|ended without|http2 request did not get a response|retry delay|you can retry your request|try your request again|please retry your request|resource[_ -]?exhausted/i; const LONG_WINDOW_RATE_LIMIT_RE = /\b(?:daily|weekly|monthly|tokens per day|requests per day|usage limit|subscription|insufficient[_ -]?quota|current quota|quota[_ -]?exceeded|(?:go|free)usagelimiterror|available balance|out of budget)\b/i; const SHORT_RATE_LIMIT_UNIT_RE = @@ -54,6 +55,7 @@ export function hasTransientRetryEvidence( const status = resolveRetrySignalStatus(signal); return ( (status !== undefined && RETRYABLE_HTTP_STATUS_CODES.has(status)) || + isIncompleteTerminalStreamMessage(signal.message ?? "") || TRANSIENT_RETRY_EVIDENCE_RE.test(signal.message ?? "") || isTransientNetworkError({ code: signal.code }) );