fix(telegram): preserve pre-dispatch send custody (#122741)

This commit is contained in:
Peter Steinberger
2026-08-12 10:47:08 -07:00
committed by GitHub
parent bae30ae888
commit 2e86f7cc95
11 changed files with 244 additions and 59 deletions
@@ -1,7 +1,7 @@
// Telegram tests cover bot.fetch abort plugin behavior.
import { toErrorObject as toLintErrorObject } from "openclaw/plugin-sdk/error-runtime";
import { describe, expect, it, vi } from "vitest";
import { isTelegramPollingNetworkError } from "./network-errors.js";
import { isTelegramPollingNetworkError, TelegramRequestNotStartedError } from "./network-errors.js";
const { botCtorSpy, telegramBotDepsForTest } =
await import("./bot.create-telegram-bot.test-harness.js");
@@ -357,6 +357,36 @@ describe("createTelegramBot fetch abort", () => {
expect(cancelMisdirectedBody).toHaveBeenCalledOnce();
});
it.each([
["without fallback", false],
["after one fallback", true],
])(
"rejects a terminal actual 421 %s and cancels every response body",
async (_name, fallback) => {
const cancelBodies = Array.from({ length: fallback ? 2 : 1 }, () => vi.fn());
const fetchSpy = vi.fn();
for (const cancel of cancelBodies) {
fetchSpy.mockResolvedValueOnce(
new Response(new ReadableStream<Uint8Array>({ cancel }), { status: 421 }),
);
}
const forceFallback = fallback ? vi.fn(() => true) : undefined;
const { clientFetch } = createWrappedTelegramClientFetchWithTransport({
fetch: fetchSpy as typeof fetch,
...(forceFallback ? { forceFallback } : {}),
});
await expect(
clientFetch("https://api.telegram.org/bot123456:ABC/sendMessage"),
).rejects.toBeInstanceOf(TelegramRequestNotStartedError);
expect(fetchSpy).toHaveBeenCalledTimes(cancelBodies.length);
for (const cancel of cancelBodies) {
expect(cancel).toHaveBeenCalledOnce();
}
},
);
it("retries Telegram 421 fetch errors after forcing transport fallback", async () => {
const forceFallback = vi.fn(() => true);
const fetchSpy = vi
@@ -376,6 +406,22 @@ describe("createTelegramBot fetch abort", () => {
expect(fetchSpy).toHaveBeenCalledTimes(2);
});
it("keeps a thrown 421-shaped edge error distinct from request-not-started custody", async () => {
const edgeError = Object.assign(new Error("421 Misdirected Request"), { status: 421 });
const forceFallback = vi.fn(() => false);
const fetchSpy = vi.fn().mockRejectedValue(edgeError);
const { clientFetch } = createWrappedTelegramClientFetchWithTransport({
fetch: fetchSpy as typeof fetch,
forceFallback,
});
await expect(clientFetch("https://api.telegram.org/bot123456:ABC/sendMessage")).rejects.toBe(
edgeError,
);
expect(edgeError).not.toBeInstanceOf(TelegramRequestNotStartedError);
expect(forceFallback).toHaveBeenCalledWith("misdirected-request");
});
it("preserves the original fetch error when tagging cannot attach metadata", async () => {
const frozenError = Object.freeze(
Object.assign(new TypeError("fetch failed"), {
+3 -3
View File
@@ -4,7 +4,7 @@ import type { MarkdownTableMode } from "openclaw/plugin-sdk/config-contracts";
import { createChannelApiRetryRunner } from "openclaw/plugin-sdk/retry-runtime";
import type { RuntimeEnv } from "openclaw/plugin-sdk/runtime-env";
import { withTelegramApiErrorLogging } from "../api-logging.js";
import { isSafeToRetrySendError, isTelegramRateLimitError } from "../network-errors.js";
import { rethrowTelegramSendError, shouldRetryTelegramSendError } from "../network-errors.js";
import {
buildTelegramSendParams,
getTelegramNativeQuoteReplyMessageId,
@@ -32,7 +32,7 @@ export { buildTelegramSendParams } from "../reply-parameters.js";
function createTelegramDeliverySendRetry() {
return createChannelApiRetryRunner({
shouldRetry: (err) => isSafeToRetrySendError(err) || isTelegramRateLimitError(err),
shouldRetry: shouldRetryTelegramSendError,
strictShouldRetry: true,
retryAfterMaxDelayMs: TELEGRAM_OUTBOUND_RETRY_AFTER_CAP_MS,
});
@@ -61,7 +61,7 @@ export async function sendTelegramWithThreadFallback<T>(params: {
getTelegramNativeQuoteReplyMessageId(requestParams) && isTelegramQuoteParamError(error)
),
fn: () => requestWithRetry(() => params.send(requestParams), operation),
}),
}).catch(rethrowTelegramSendError),
});
return result;
}
+35 -2
View File
@@ -67,6 +67,7 @@ vi.mock("../sent-message-cache.js", async (importOriginal) => {
vi.resetModules();
const { deliverReplies } = await import("./delivery.js");
const { sendTelegramText } = await import("./delivery.send.js");
const { PlatformMessageNotDispatchedError } = await import("openclaw/plugin-sdk/error-runtime");
vi.mock("grammy", () => ({
API_CONSTANTS: {
@@ -84,6 +85,8 @@ vi.mock("grammy", () => ({
},
}));
const { TelegramRequestNotStartedError } = await import("../network-errors.js");
function createRuntime(withLog = true): RuntimeStub {
return {
error: vi.fn(),
@@ -285,10 +288,13 @@ function createWrappedConnectTimeoutHttpError(operation = "sendMessage") {
});
}
function createPlainHttpError(operation = "sendMessage") {
function createPlainHttpError(
operation = "sendMessage",
error: unknown = new TypeError("fetch failed"),
) {
return Object.assign(new Error(`Network request for '${operation}' failed!`), {
name: "HttpError",
error: new TypeError("fetch failed"),
error,
});
}
@@ -1651,6 +1657,33 @@ describe("deliverReplies", () => {
expect(runtime.error).toHaveBeenCalledTimes(1);
});
it("maps an exhausted request-not-started marker to streaming no-dispatch custody", async () => {
const runtime = createRuntime();
const terminal = createPlainHttpError("sendMessage", new TelegramRequestNotStartedError());
const sendMessage = vi.fn().mockRejectedValue(terminal);
let observed: unknown;
try {
await sendTelegramText(createBot({ sendMessage }), "123", "hello", runtime);
} catch (error) {
observed = error;
}
expect(observed).toBeInstanceOf(PlatformMessageNotDispatchedError);
expect(observed).toHaveProperty("cause", terminal);
expect(sendMessage).toHaveBeenCalledTimes(3);
});
it("keeps broad 421-shaped streaming send errors ambiguous", async () => {
const edgeError = Object.assign(new Error("421 Misdirected Request"), { status: 421 });
const sendMessage = vi.fn().mockRejectedValue(edgeError);
await expect(
sendTelegramText(createBot({ sendMessage }), "123", "hello", createRuntime()),
).rejects.toBe(edgeError);
expect(sendMessage).toHaveBeenCalledOnce();
});
it("does not retry DM topic media sends without the topic id", async () => {
const runtime = createRuntime();
const sendPhoto = vi.fn().mockRejectedValueOnce(createThreadNotFoundError("sendPhoto"));
@@ -1,4 +1,5 @@
import { isChannelPartialDeliveryError } from "openclaw/plugin-sdk/channel-inbound";
import { PlatformMessageNotDispatchedError } from "openclaw/plugin-sdk/error-runtime";
import { describe, expect, it, vi } from "vitest";
import { createTelegramChunkDeliveryTracker } from "./chunk-delivery.js";
@@ -9,6 +10,19 @@ describe("Telegram chunk delivery", () => {
it.each([
[telegramError(400, "content rejected"), true],
[Object.assign(new Error("dns failed"), { code: "ENOTFOUND" }), true],
[
new PlatformMessageNotDispatchedError("request not started", {
cause: new Error("transport unavailable"),
}),
true,
],
[
new PlatformMessageNotDispatchedError("payload rejected", {
cause: new Error("invalid payload"),
retryable: false,
}),
false,
],
[telegramError(400, "message thread not found"), false],
[telegramError(401, "unauthorized"), false],
[telegramError(429, "rate limited"), false],
+19 -8
View File
@@ -3,7 +3,11 @@ import type { ApiClientOptions } from "grammy";
import { responseWithRelease } from "openclaw/plugin-sdk/fetch-runtime";
import { normalizeOptionalLowercaseString } from "openclaw/plugin-sdk/string-coerce-runtime";
import type { TelegramTransport } from "./fetch.js";
import { isTelegramMisdirectedRequestError, tagTelegramNetworkError } from "./network-errors.js";
import {
isTelegramMisdirectedRequestError,
tagTelegramNetworkError,
TelegramRequestNotStartedError,
} from "./network-errors.js";
import { resolveTelegramRequestTimeoutMs } from "./request-timeouts.js";
type TelegramFetchInput = Parameters<NonNullable<ApiClientOptions["fetch"]>>[0];
@@ -142,7 +146,7 @@ export function createTelegramClientFetch(params: {
!requestSignal?.aborted &&
params.transport?.forceFallback?.(reason) === true;
const runFetch = async () => {
const runFetch = async (allowMisdirectedFallback = false): Promise<Response> => {
const controller = new AbortController();
const abortWith = (signal: Pick<TelegramAbortSignalLike, "reason">) =>
controller.abort(signal.reason);
@@ -195,6 +199,18 @@ export function createTelegramClientFetch(params: {
...init,
signal: controller.signal,
});
if (response.status === 421) {
const retry =
allowMisdirectedFallback && canForceTransportFallback("misdirected-request");
// HTTP 421 permits retrying a non-idempotent request;
// arbitrary thrown 421 shapes do not own that fact.
await response.body?.cancel().catch(() => undefined);
if (retry) {
await releaseRequest();
return runFetch();
}
throw new TelegramRequestNotStartedError();
}
// grammY consumes JSON after fetch resolves; keep its deadline and
// cancellation linked until the response body settles.
return responseWithRelease(response, releaseRequest);
@@ -208,12 +224,7 @@ export function createTelegramClientFetch(params: {
};
try {
const response = await runFetch();
if (response.status === 421 && canForceTransportFallback("misdirected-request")) {
await response.body?.cancel().catch(() => undefined);
return await runFetch();
}
return response;
return await runFetch(true);
} catch (err) {
if (
requestTimeoutMs &&
+10 -9
View File
@@ -6,6 +6,7 @@ import { expectDefined } from "@openclaw/normalization-core";
import { resolveFetch } from "openclaw/plugin-sdk/fetch-runtime";
import { MAX_DATE_TIMESTAMP_MS } from "openclaw/plugin-sdk/number-runtime";
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { isSafeToRetrySendError, TelegramRequestNotStartedError } from "./network-errors.js";
const setDefaultResultOrder = vi.hoisted(() => vi.fn());
const getDefaultResultOrder = vi.hoisted(() => vi.fn(() => "ipv4first"));
@@ -1053,12 +1054,7 @@ describe("resolveTelegramFetch", () => {
});
it("cools down a repeatedly failing sticky fallback and probes earlier attempts", async () => {
for (let i = 0; i < 7; i += 1) {
undiciFetch.mockRejectedValueOnce(buildFetchFallbackError("ENETUNREACH"));
}
undiciFetch
.mockRejectedValueOnce(buildFetchFallbackError("ENETUNREACH"))
.mockRejectedValueOnce(buildFetchFallbackError("ENETUNREACH"));
undiciFetch.mockRejectedValue(buildFetchFallbackError("ENETUNREACH"));
const resolved = resolveTelegramFetchOrThrow(undefined, {
network: {
@@ -1075,10 +1071,15 @@ describe("resolveTelegramFetch", () => {
"fetch failed",
);
}
await expect(resolved("https://api.telegram.org/botx/getUpdates")).rejects.toThrow(
"temporarily unhealthy",
);
let terminalError: unknown;
try {
await resolved("https://api.telegram.org/botx/getUpdates");
} catch (error) {
terminalError = error;
}
expect(terminalError).toBeInstanceOf(TelegramRequestNotStartedError);
expect(isSafeToRetrySendError(terminalError)).toBe(true);
expect(undiciFetch).toHaveBeenCalledTimes(9);
expect(getDispatcherFromUndiciCall(7)).toBe(getDispatcherFromUndiciCall(3));
expect(getDispatcherFromUndiciCall(8)).toBe(getDispatcherFromUndiciCall(1));
+5 -12
View File
@@ -32,6 +32,7 @@ import {
resolveTelegramDnsResultOrderDecision,
TELEGRAM_DNS_RESULT_ORDER_ENV,
} from "./network-config.js";
import { TelegramRequestNotStartedError } from "./network-errors.js";
import { getProxyUrlFromFetch, makeProxyFetch } from "./proxy.js";
const log = createSubsystemLogger("telegram/network");
@@ -423,18 +424,7 @@ function formatErrorCodes(err: unknown): string {
return codes.length > 0 ? codes.join(",") : "none";
}
class TelegramTransportAttemptUnhealthyError extends Error {
constructor(unhealthyUntilMs: number) {
const remainingMs = Math.max(0, unhealthyUntilMs - Date.now());
super(`telegram transport attempt temporarily unhealthy; retry after ${remainingMs}ms`);
this.name = "TelegramTransportAttemptUnhealthyError";
}
}
function shouldUseTelegramTransportFallback(err: unknown): boolean {
if (err instanceof TelegramTransportAttemptUnhealthyError) {
return true;
}
const ctx: TelegramTransportFallbackContext = {
message:
err && typeof err === "object" && "message" in err
@@ -651,7 +641,10 @@ export function resolveTelegramTransport(
if (!isFutureDateTimestampMs(health.unhealthyUntilMs)) {
return null;
}
return new TelegramTransportAttemptUnhealthyError(health.unhealthyUntilMs);
const remainingMs = Math.max(0, health.unhealthyUntilMs - Date.now());
return new TelegramRequestNotStartedError(
`Telegram transport attempts are cooling down; retry after ${remainingMs}ms`,
);
};
const recordAttemptFailure = (attemptIndex: number, err: unknown): void => {
+25 -2
View File
@@ -10,6 +10,7 @@ import {
isTelegramPollingNetworkError,
isTelegramServerError,
tagTelegramNetworkError,
TelegramRequestNotStartedError,
} from "./network-errors.js";
const errorWithCode = (message: string, code: string) =>
@@ -171,6 +172,17 @@ describe("isRecoverableTelegramNetworkError", () => {
).toBe(true);
});
it("keeps request-not-started markers recoverable across Telegram contexts", () => {
const marker = new TelegramRequestNotStartedError();
const wrapped = Object.assign(new Error("Network request for 'getUpdates' failed!"), {
name: "HttpError",
error: marker,
});
expect(isRecoverableTelegramNetworkError(marker, { context: "send" })).toBe(true);
expect(isRecoverableTelegramNetworkError(wrapped, { context: "polling" })).toBe(true);
});
it("returns false for unrelated errors", () => {
expect(isRecoverableTelegramNetworkError(new Error("invalid token"))).toBe(false);
});
@@ -279,6 +291,17 @@ describe("isSafeToRetrySendError", () => {
expect(isSafeToRetrySendError(wrapped)).toBe(false);
});
it("accepts only direct and exact grammY-wrapped request-not-started markers", () => {
const marker = new TelegramRequestNotStartedError();
expect(isSafeToRetrySendError(marker)).toBe(true);
expect(
isSafeToRetrySendError(
new MockHttpError("Network request for 'sendMessage' failed!", marker),
),
).toBe(true);
});
it.each([
["status", Object.assign(new Error("Misdirected Request"), { status: 421 })],
["statusCode", Object.assign(new Error("Misdirected Request"), { statusCode: "421" })],
@@ -297,8 +320,8 @@ describe("isSafeToRetrySendError", () => {
Object.assign(new Error("Misdirected Request"), { status: 421 }),
),
],
])("treats Telegram 421 Misdirected Request as safe to retry via %s", (_name, err) => {
expect(isSafeToRetrySendError(err)).toBe(true);
])("does not infer safe retry from broad Telegram 421 shape %s", (_name, err) => {
expect(isSafeToRetrySendError(err)).toBe(false);
});
it("does not parse malformed status strings as Telegram 421", () => {
+37 -19
View File
@@ -3,6 +3,7 @@ import {
collectErrorGraphCandidates,
extractErrorCode,
formatErrorMessage,
PlatformMessageNotDispatchedError,
readErrorName,
} from "openclaw/plugin-sdk/error-runtime";
import { parseStrictNonNegativeInteger } from "openclaw/plugin-sdk/number-runtime";
@@ -11,6 +12,27 @@ import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coer
const TELEGRAM_NETWORK_ORIGIN = Symbol("openclaw.telegram.network-origin");
export class TelegramRequestNotStartedError extends Error {
constructor(message = "Telegram request did not start") {
super(message);
this.name = "TelegramRequestNotStartedError";
}
}
function isTelegramRequestNotStartedError(err: unknown): boolean {
return (
err instanceof TelegramRequestNotStartedError ||
(readErrorName(err) === "HttpError" &&
(err as { error?: unknown }).error instanceof TelegramRequestNotStartedError)
);
}
export function rethrowTelegramSendError(err: unknown): never {
throw isTelegramRequestNotStartedError(err)
? new PlatformMessageNotDispatchedError("Telegram request not started", { cause: err })
: err;
}
const TELEGRAM_ADDITIONAL_TRANSIENT_ERROR_CODES = new Set([
"ENETDOWN",
"ESOCKETTIMEDOUT",
@@ -20,16 +42,9 @@ const TELEGRAM_ADDITIONAL_TRANSIENT_ERROR_CODES = new Set([
"ERR_NETWORK",
]);
/**
* Error codes that are safe to retry for non-idempotent send operations (e.g. sendMessage).
*
* These represent failures that occur *before* the request reaches Telegram's servers,
* meaning the message was definitely not delivered and it is safe to retry.
*
* Contrast with the full transient set, which includes codes like ECONNRESET and ETIMEDOUT
* that can fire *after* Telegram has already received and delivered a message — retrying
* those would cause duplicate messages.
*/
// These exact local codes fail before request publication and are safe for
// non-idempotent sends. Resets and timeouts can occur after delivery, so
// retrying them could duplicate a visible message.
const TELEGRAM_ADDITIONAL_PRE_CONNECT_ERROR_CODES = new Set([
"ENETDOWN", // Local network interface is down before connect completes (never sent)
"EHOSTUNREACH", // Host unreachable (never sent)
@@ -194,19 +209,15 @@ export function isTelegramPollingNetworkError(err: unknown): boolean {
return getTelegramNetworkErrorOrigin(err)?.method === "getupdates";
}
/**
* Returns true if the error is safe to retry for a non-idempotent Telegram send operation
* (e.g. sendMessage). Only matches errors that are guaranteed to have occurred *before*
* the request reached Telegram's servers, preventing duplicate message delivery.
*
* Use this instead of isRecoverableTelegramNetworkError for sendMessage/sendPhoto/etc.
* calls where a retry would create a duplicate visible message.
*/
/** True only for channel-owned no-send proof or proven pre-connect failures. */
export function isSafeToRetrySendError(err: unknown): boolean {
if (!err) {
return false;
}
if (isTelegramMisdirectedRequestError(err)) {
if (err instanceof PlatformMessageNotDispatchedError) {
return err.retryable;
}
if (isTelegramRequestNotStartedError(err)) {
return true;
}
for (const candidate of collectTelegramErrorCandidates(err)) {
@@ -217,6 +228,10 @@ export function isSafeToRetrySendError(err: unknown): boolean {
return false;
}
export function shouldRetryTelegramSendError(err: unknown): boolean {
return isSafeToRetrySendError(err) || isTelegramRateLimitError(err);
}
function hasTelegramErrorCode(err: unknown, matches: (code: number) => boolean): boolean {
for (const candidate of collectTelegramErrorCandidates(err)) {
if (!candidate || typeof candidate !== "object" || !("error_code" in candidate)) {
@@ -316,6 +331,9 @@ export function isRecoverableTelegramNetworkError(
if (!err) {
return false;
}
if (isTelegramRequestNotStartedError(err)) {
return true;
}
const allowMessageMatch =
typeof options.allowMessageMatch === "boolean"
? options.allowMessageMatch
+4 -3
View File
@@ -13,7 +13,7 @@ import { withTelegramApiErrorLogging } from "./api-logging.js";
import { normalizeTelegramApiRoot } from "./api-root.js";
import { asTelegramClientFetch, createTelegramClientFetch } from "./client-fetch.js";
import { resolveTelegramTransport, type TelegramTransport } from "./fetch.js";
import { isSafeToRetrySendError, isTelegramRateLimitError } from "./network-errors.js";
import { rethrowTelegramSendError, shouldRetryTelegramSendError } from "./network-errors.js";
import type { TelegramOutboundPromptContextMessage as TelegramMessageLike } from "./outbound-message-context.js";
import { makeProxyFetch } from "./proxy.js";
import {
@@ -571,14 +571,15 @@ export function createTelegramNonIdempotentRequestWithDiag(params: {
verbose?: boolean;
useApiErrorLogging?: boolean;
}): TelegramRequestWithDiag {
return createTelegramRequestWithDiag({
const request = createTelegramRequestWithDiag({
cfg: params.cfg,
account: params.account,
retry: params.retry,
verbose: params.verbose,
useApiErrorLogging: params.useApiErrorLogging,
retryAfterMaxDelayMs: TELEGRAM_OUTBOUND_RETRY_AFTER_CAP_MS,
shouldRetry: (err) => isSafeToRetrySendError(err) || isTelegramRateLimitError(err),
shouldRetry: shouldRetryTelegramSendError,
strictShouldRetry: true,
});
return (fn, label, options) => request(fn, label, options).catch(rethrowTelegramSendError);
}
+45
View File
@@ -69,6 +69,8 @@ const {
probeVideoDimensions,
} = getTelegramSendTestMocks();
const telegramSendModule = await importTelegramSendModule();
const { PlatformMessageNotDispatchedError } = await import("openclaw/plugin-sdk/error-runtime");
const { TelegramRequestNotStartedError } = await import("./network-errors.js");
const { getChildLogger, resetLogger, setLoggerOverride } =
await import("openclaw/plugin-sdk/runtime-env");
const {
@@ -3419,6 +3421,49 @@ describe("sendMessageTelegram", () => {
vi.useRealTimers();
});
it("maps an exhausted request-not-started marker to durable no-dispatch custody", async () => {
const chatId = "123";
const terminal = Object.assign(new Error("Network request for 'sendMessage' failed!"), {
name: "HttpError",
error: new TelegramRequestNotStartedError(),
});
const sendMessage = vi.fn().mockRejectedValue(terminal);
const api = { sendMessage } as unknown as { sendMessage: typeof sendMessage };
let observed: unknown;
try {
await sendMessageTelegram(chatId, "hi", {
cfg: TELEGRAM_TEST_CFG,
token: "tok",
api,
retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 },
});
} catch (error) {
observed = error;
}
expect(observed).toBeInstanceOf(PlatformMessageNotDispatchedError);
expect(observed).toHaveProperty("cause", terminal);
expect(sendMessage).toHaveBeenCalledTimes(2);
});
it("keeps broad 421-shaped durable send errors ambiguous", async () => {
const chatId = "123";
const edgeError = Object.assign(new Error("421 Misdirected Request"), { status: 421 });
const sendMessage = vi.fn().mockRejectedValue(edgeError);
const api = { sendMessage } as unknown as { sendMessage: typeof sendMessage };
await expect(
sendMessageTelegram(chatId, "hi", {
cfg: TELEGRAM_TEST_CFG,
token: "tok",
api,
retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 },
}),
).rejects.toBe(edgeError);
expect(sendMessage).toHaveBeenCalledOnce();
});
it("does not retry on non-transient errors", async () => {
const chatId = "123";
const sendMessage = vi.fn().mockRejectedValue(new Error("400: Bad Request"));