From 1fd8de8495bf97a27610085bdaac1e41e649bbba Mon Sep 17 00:00:00 2001 From: Andy Ye <35905412+TurboTheTurtle@users.noreply.github.com> Date: Tue, 26 May 2026 10:40:31 -0700 Subject: [PATCH] fix(telegram): treat ENETDOWN as transient network failure (#86762) --- extensions/telegram/src/fetch.test.ts | 17 +++++++++++++++++ extensions/telegram/src/fetch.ts | 1 + extensions/telegram/src/network-errors.test.ts | 2 ++ extensions/telegram/src/network-errors.ts | 2 ++ src/cron/retry-hint.test.ts | 1 + src/cron/retry-hint.ts | 2 +- ...unhandled-rejections.fatal-detection.test.ts | 3 +++ src/infra/unhandled-rejections.test.ts | 9 +++++++++ src/infra/unhandled-rejections.ts | 6 ++++-- 9 files changed, 40 insertions(+), 3 deletions(-) diff --git a/extensions/telegram/src/fetch.test.ts b/extensions/telegram/src/fetch.test.ts index 1b0d2c1e1c0b..7514abb5ccb4 100644 --- a/extensions/telegram/src/fetch.test.ts +++ b/extensions/telegram/src/fetch.test.ts @@ -1101,6 +1101,23 @@ describe("resolveTelegramFetch", () => { expect(undiciFetch).toHaveBeenCalledTimes(1); }); + it("retries sticky fallback when the local network is down during connect", async () => { + undiciFetch + .mockRejectedValueOnce(buildFetchFallbackError("ENETDOWN")) + .mockResolvedValueOnce({ ok: true } as Response); + + const resolved = resolveTelegramFetchOrThrow(undefined, { + network: { + autoSelectFamily: true, + }, + }); + + await resolved("https://api.telegram.org/botx/getUpdates"); + + expect(undiciFetch).toHaveBeenCalledTimes(2); + expect(getDispatcherFromUndiciCall(1)).not.toBe(getDispatcherFromUndiciCall(2)); + }); + it("keeps per-resolver transport policy isolated across multiple accounts", async () => { undiciFetch.mockResolvedValue({ ok: true } as Response); diff --git a/extensions/telegram/src/fetch.ts b/extensions/telegram/src/fetch.ts index a4fb39b07872..460f8694a3f0 100644 --- a/extensions/telegram/src/fetch.ts +++ b/extensions/telegram/src/fetch.ts @@ -113,6 +113,7 @@ type LookupFunction = ( const FALLBACK_RETRY_ERROR_CODES = [ "ETIMEDOUT", + "ENETDOWN", "ENETUNREACH", "EHOSTUNREACH", "UND_ERR_CONNECT_TIMEOUT", diff --git a/extensions/telegram/src/network-errors.test.ts b/extensions/telegram/src/network-errors.test.ts index 0c4ab12e9ef8..31dd0febb2a8 100644 --- a/extensions/telegram/src/network-errors.test.ts +++ b/extensions/telegram/src/network-errors.test.ts @@ -83,6 +83,7 @@ describe("isRecoverableTelegramNetworkError", () => { it.each([ ["ETIMEDOUT", "timeout"], + ["ENETDOWN", "network down"], ["ECONNABORTED", "aborted"], ["ERR_NETWORK", "network"], ])("detects recoverable error code %s", (code, message) => { @@ -218,6 +219,7 @@ describe("isSafeToRetrySendError", () => { ["ECONNREFUSED", "connect ECONNREFUSED", true], ["ENOTFOUND", "getaddrinfo ENOTFOUND", true], ["EAI_AGAIN", "getaddrinfo EAI_AGAIN", true], + ["ENETDOWN", "connect ENETDOWN", true], ["ENETUNREACH", "connect ENETUNREACH", true], ["EHOSTUNREACH", "connect EHOSTUNREACH", true], ["ECONNRESET", "read ECONNRESET", false], diff --git a/extensions/telegram/src/network-errors.ts b/extensions/telegram/src/network-errors.ts index a9fcc967e770..4be080954e15 100644 --- a/extensions/telegram/src/network-errors.ts +++ b/extensions/telegram/src/network-errors.ts @@ -12,6 +12,7 @@ const RECOVERABLE_ERROR_CODES = new Set([ "ECONNRESET", "ECONNREFUSED", "EPIPE", + "ENETDOWN", "ETIMEDOUT", "ESOCKETTIMEDOUT", "ENETUNREACH", @@ -41,6 +42,7 @@ const PRE_CONNECT_ERROR_CODES = new Set([ "ECONNREFUSED", // Server actively refused the connection (never reached Telegram) "ENOTFOUND", // DNS resolution failed (never sent) "EAI_AGAIN", // Transient DNS failure (never sent) + "ENETDOWN", // Local network interface is down before connect completes (never sent) "ENETUNREACH", // No route to host (never sent) "EHOSTUNREACH", // Host unreachable (never sent) ]); diff --git a/src/cron/retry-hint.test.ts b/src/cron/retry-hint.test.ts index 54fb14c1eb6c..f6e7b508b35b 100644 --- a/src/cron/retry-hint.test.ts +++ b/src/cron/retry-hint.test.ts @@ -16,6 +16,7 @@ describe("resolveCronExecutionRetryHint", () => { it("treats common network error codes as network when retryOn only includes network", () => { for (const code of [ "EAI_AGAIN", + "ENETDOWN", "EHOSTUNREACH", "EHOSTDOWN", "ENETRESET", diff --git a/src/cron/retry-hint.ts b/src/cron/retry-hint.ts index e84efd229c5b..baafc40d8fee 100644 --- a/src/cron/retry-hint.ts +++ b/src/cron/retry-hint.ts @@ -11,7 +11,7 @@ const TRANSIENT_PATTERNS: Record = { overloaded: /\b529\b|\boverloaded(?:_error)?\b|high demand|temporar(?:ily|y) overloaded|capacity exceeded/i, network: - /(network|fetch failed|socket|econnreset|econnrefused|eai_again|ehostunreach|ehostdown|enetreset|enetunreach|epipe)/i, + /(network|fetch failed|socket|econnreset|econnrefused|eai_again|enetdown|ehostunreach|ehostdown|enetreset|enetunreach|epipe)/i, timeout: /(timeout|etimedout)/i, server_error: /\b5\d{2}\b/, }; diff --git a/src/infra/unhandled-rejections.fatal-detection.test.ts b/src/infra/unhandled-rejections.fatal-detection.test.ts index e694e2c71e44..8deae074e35a 100644 --- a/src/infra/unhandled-rejections.fatal-detection.test.ts +++ b/src/infra/unhandled-rejections.fatal-detection.test.ts @@ -150,6 +150,9 @@ describe("installUnhandledRejectionHandler - fatal detection", () => { cause: { code: "UND_ERR_CONNECT_TIMEOUT", syscall: "connect" }, }), Object.assign(new Error("DNS resolve failed"), { code: "UND_ERR_DNS_RESOLVE_FAILED" }), + Object.assign(new Error("connect ENETDOWN 149.154.167.220:443"), { + code: "ENETDOWN", + }), Object.assign(new Error("Connection reset"), { code: "ECONNRESET" }), Object.assign(new Error("Timeout"), { code: "ETIMEDOUT" }), Object.assign( diff --git a/src/infra/unhandled-rejections.test.ts b/src/infra/unhandled-rejections.test.ts index 00a1ef256cee..b6008fdcfa39 100644 --- a/src/infra/unhandled-rejections.test.ts +++ b/src/infra/unhandled-rejections.test.ts @@ -60,6 +60,7 @@ describe("isTransientNetworkError", () => { "ESOCKETTIMEDOUT", "ECONNABORTED", "EPIPE", + "ENETDOWN", "EHOSTUNREACH", "ENETUNREACH", "EADDRNOTAVAIL", @@ -375,6 +376,12 @@ describe("isTransientUnhandledRejectionError", () => { const epipe = Object.assign(new Error("write EPIPE"), { code: "EPIPE" }); const sqlite = Object.assign(new Error("database is locked"), { code: "SQLITE_BUSY" }); const network = Object.assign(new Error("connection reset"), { code: "ECONNRESET" }); + const networkDown = Object.assign(new Error("connect ENETDOWN"), { + code: "ENETDOWN", + }); + const rawNetworkDown = new Error( + "connect ENETDOWN 149.154.167.220:443 - Local (10.0.10.40:50017)", + ); const hostUnreachable = Object.assign(new Error("connect EHOSTUNREACH"), { code: "EHOSTUNREACH", }); @@ -398,6 +405,8 @@ describe("isTransientUnhandledRejectionError", () => { expect(isBenignUncaughtExceptionError(epipe)).toBe(true); expect(isBenignUncaughtExceptionError(sqlite)).toBe(false); expect(isBenignUncaughtExceptionError(network)).toBe(false); + expect(isBenignUncaughtExceptionError(networkDown)).toBe(true); + expect(isBenignUncaughtExceptionError(rawNetworkDown)).toBe(true); expect(isBenignUncaughtExceptionError(hostUnreachable)).toBe(true); expect(isBenignUncaughtExceptionError(rawHostUnreachable)).toBe(true); expect(isBenignUncaughtExceptionError(addressUnavailable)).toBe(true); diff --git a/src/infra/unhandled-rejections.ts b/src/infra/unhandled-rejections.ts index 04f799451dab..9ceeffa68c1e 100644 --- a/src/infra/unhandled-rejections.ts +++ b/src/infra/unhandled-rejections.ts @@ -57,6 +57,7 @@ const TRANSIENT_NETWORK_CODES = new Set([ "ESOCKETTIMEDOUT", "ECONNABORTED", "EPIPE", + "ENETDOWN", "EHOSTUNREACH", "ENETUNREACH", "EADDRNOTAVAIL", @@ -93,6 +94,7 @@ const TRANSIENT_SQLITE_ERRCODES = new Set([5, 6, 10, 14]); const BENIGN_UNCAUGHT_EXCEPTION_CODES = new Set(["EPIPE", "EIO"]); const BENIGN_UNCAUGHT_EXCEPTION_NETWORK_CODES = new Set([ "ECONNREFUSED", + "ENETDOWN", "EHOSTUNREACH", "ENETUNREACH", "EADDRNOTAVAIL", @@ -106,9 +108,9 @@ const BENIGN_UNCAUGHT_EXCEPTION_NETWORK_CODES = new Set([ ]); const TRANSIENT_NETWORK_MESSAGE_CODE_RE = - /\b(ECONNRESET|ECONNREFUSED|ENOTFOUND|ETIMEDOUT|ESOCKETTIMEDOUT|ECONNABORTED|EPIPE|EHOSTUNREACH|ENETUNREACH|EADDRNOTAVAIL|EAI_AGAIN|EPROTO|UND_ERR_CONNECT_TIMEOUT|UND_ERR_DNS_RESOLVE_FAILED|UND_ERR_CONNECT|UND_ERR_SOCKET|UND_ERR_HEADERS_TIMEOUT|UND_ERR_BODY_TIMEOUT|ERR_HTTP2_INVALID_SESSION)\b/i; + /\b(ECONNRESET|ECONNREFUSED|ENOTFOUND|ETIMEDOUT|ESOCKETTIMEDOUT|ECONNABORTED|EPIPE|ENETDOWN|EHOSTUNREACH|ENETUNREACH|EADDRNOTAVAIL|EAI_AGAIN|EPROTO|UND_ERR_CONNECT_TIMEOUT|UND_ERR_DNS_RESOLVE_FAILED|UND_ERR_CONNECT|UND_ERR_SOCKET|UND_ERR_HEADERS_TIMEOUT|UND_ERR_BODY_TIMEOUT|ERR_HTTP2_INVALID_SESSION)\b/i; const BENIGN_UNCAUGHT_EXCEPTION_NETWORK_MESSAGE_CODE_RE = - /\b(ECONNREFUSED|EHOSTUNREACH|ENETUNREACH|EADDRNOTAVAIL|EAI_AGAIN|ENOTFOUND|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|UND_ERR_DNS_RESOLVE_FAILED|UND_ERR_CONNECT|ERR_HTTP2_INVALID_SESSION)\b/i; + /\b(ECONNREFUSED|ENETDOWN|EHOSTUNREACH|ENETUNREACH|EADDRNOTAVAIL|EAI_AGAIN|ENOTFOUND|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|UND_ERR_DNS_RESOLVE_FAILED|UND_ERR_CONNECT|ERR_HTTP2_INVALID_SESSION)\b/i; const TRANSIENT_SQLITE_MESSAGE_CODE_RE = /\b(SQLITE_BUSY|SQLITE_CANTOPEN|SQLITE_IOERR|SQLITE_LOCKED)\b/i;