mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-27 21:07:01 -06:00
fix: stop retry storms after definitive channel rejections (#127353)
* fix(outbound): terminalize definitive channel rejections * refactor(outbound): rethrow unclassified Slack rejections by identity The Slack send boundary replaced every non-Error rejection with a synthetic Error before classifying. That changed the propagated value at all four send call sites, contradicting the stated contract that unlisted rejections keep their previous path, and forced a compensating one-level `cause` walk in `isSlackInvalidBlocksError` so the downstream `invalid_blocks` fallback could still match. The guard's second operand was also dead: `isRecord` accepts Error instances, so `!(err instanceof Error) || !isRecord(err)` never reached its right side for a plain object. Classify off the raw value and rethrow unclassified rejections by identity; the `cause`-walking compensator and its test go away with it. Distill the Telegram migration classifier's three-state result object into a message-or-nothing, and document the description-first and 52-bit id contracts inline. Production surface for the PR drops from +71/-11 to +62/-8. * test(agents): remove clock-tick race from workspace bootstrap ctime coverage The in-place-edit case added in #127769 assumes restoring mtime leaves ctime as the only changed stat field, and that it therefore differs. Linux and macOS stamp ctime from a coarse per-tick clock, so an edit landing in the same tick as the cached stat leaves ctimeMs equal: the ctime-only scenario never occurs, the cache correctly serves its entry, and the assertion fails. Measured 199/200 identical ctimes in a tight loop; the test failed 1 in 8 local runs and broke checks-node-compact-large-14 on CI. Re-touch until the kernel advances ctimeMs, then assert mtimeMs and size are unchanged so the scenario is provably ctime-only. Stripping ctimeMs from the cache identity still fails the test. * fix(slack): keep post-dispatch upload completion rejections ambiguous PlatformMessageNotDispatchedError is a provider assertion that no recipient-visible send began, and its contract says never use it after an ambiguous send. files.completeUploadExternal runs after onPlatformSendDispatch and is the one-time share operation, so a rejection there cannot prove the file was never shared however definitive its code reads. Drop the permanent-rejection classifier from that call and keep it on the pre-dispatch calls only (chat.postMessage, files.getUploadURLExternal, resolveChannelId). The upload test that pinned the old behavior asserted onPlatformSendDispatch had already fired, which is exactly the condition that forbids the claim; it now pins ambiguity instead. Also widens the workspace bootstrap ctime wait to a 1s deadline and reshapes it as a while loop, so a coarser filesystem tick cannot exhaust the bound. * test(slack): prove permanent rejection recovery * fix(test): stabilize Slack channel action routing * fix(test): retain channel parity for precise targets * refactor(outbound): drop unrelated test-routing changes * fix(telegram): require Bot API error code for migration rejection * test: repair Telegram tuples and preserve Slack test routing --------- Co-authored-by: Ayaan Zaidi <hi@obviy.us> Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
@@ -13,6 +13,7 @@ import { loadOutboundMediaFromUrl } from "openclaw/plugin-sdk/outbound-media";
|
||||
import { retryAsync } from "openclaw/plugin-sdk/retry-runtime";
|
||||
import { logVerbose, sleepWithAbort } from "openclaw/plugin-sdk/runtime-env";
|
||||
import { fetchWithSsrFGuard, type SsrFPolicy } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { formatSlackError } from "./errors.js";
|
||||
import {
|
||||
postSlackMessageWithIdentityFallback,
|
||||
@@ -41,6 +42,26 @@ const SLACK_DNS_RETRY_CODES = new Set(["EAI_AGAIN", "ENOTFOUND", "UND_ERR_DNS_RE
|
||||
const SLACK_DNS_RETRY_ATTEMPTS = 2;
|
||||
const SLACK_DNS_RETRY_BASE_DELAY_MS = 250;
|
||||
|
||||
// Slack reports provider verdicts as `slack_webapi_platform_error` with the code in
|
||||
// `data.error`; these two mean no recipient can ever see the message, so durable
|
||||
// recovery must stop retrying. Everything else rethrows by identity — the
|
||||
// `invalid_blocks` and custom-identity fallbacks match on the original value.
|
||||
// Pre-dispatch calls only: PlatformMessageNotDispatchedError asserts no send began,
|
||||
// so a call made after onPlatformSendDispatch must stay ambiguous instead.
|
||||
export function rethrowSlackPermanentOutboundApiRejection(err: unknown): never {
|
||||
const rawData =
|
||||
isRecord(err) && err.code === "slack_webapi_platform_error" ? err.data : undefined;
|
||||
const data = isRecord(rawData) ? rawData : undefined;
|
||||
const code = data?.error;
|
||||
if (data?.ok === false && (code === "messages_tab_disabled" || code === "account_inactive")) {
|
||||
throw new PlatformMessageNotDispatchedError(`Slack outbound delivery rejected: ${code}`, {
|
||||
cause: err,
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
function readSlackRequestErrorCode(value: unknown): string | undefined {
|
||||
if (!value || typeof value !== "object") {
|
||||
return undefined;
|
||||
@@ -236,7 +257,9 @@ export async function postSlackMessageBestEffort(params: {
|
||||
const basePayload = buildSlackPostMessagePayload(params);
|
||||
const postChatMessage = params.client.chat.postMessage.bind(params.client.chat);
|
||||
const post = async (payload: SlackPostMessagePayload, identity?: SlackPostMessageIdentity) => ({
|
||||
response: await withSlackDnsRequestRetry("chat.postMessage", () => postChatMessage(payload)),
|
||||
response: await withSlackDnsRequestRetry("chat.postMessage", () =>
|
||||
postChatMessage(payload),
|
||||
).catch(rethrowSlackPermanentOutboundApiRejection),
|
||||
identity,
|
||||
});
|
||||
const posted = await postSlackMessageWithIdentityFallback({
|
||||
@@ -287,7 +310,7 @@ export async function uploadSlackFile(params: {
|
||||
filename: uploadFileName,
|
||||
length: buffer.length,
|
||||
}),
|
||||
);
|
||||
).catch(rethrowSlackPermanentOutboundApiRejection);
|
||||
if (!uploadUrlResp.ok || !uploadUrlResp.upload_url || !uploadUrlResp.file_id) {
|
||||
throw new Error(`Failed to get upload URL: ${uploadUrlResp.error ?? "unknown error"}`);
|
||||
}
|
||||
@@ -349,6 +372,8 @@ export async function uploadSlackFile(params: {
|
||||
await params.onPlatformSendDispatch?.();
|
||||
// Slack allows this finalize call only once. Keep only the pre-connect DNS
|
||||
// retry; a timeout or broader retry would create an unknown-send state.
|
||||
// Dispatch is already recorded above, so this call is the ambiguous send:
|
||||
// no rejection here may claim non-dispatch, however definitive its code reads.
|
||||
const completionClient = params.completionClient ?? params.client;
|
||||
const completeResp = await withSlackDnsRequestRetry("files.completeUploadExternal", () =>
|
||||
completionClient.files.completeUploadExternal({
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
import { PlatformMessageNotDispatchedError } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createSlackSendTestClient } from "./blocks.test-helpers.js";
|
||||
import { rethrowSlackPermanentOutboundApiRejection } from "./client-delivery.js";
|
||||
import { isSlackInvalidBlocksError } from "./native-data-blocks.js";
|
||||
|
||||
const { sendMessageSlack } = await import("./send.js");
|
||||
const SLACK_TEST_CFG = { channels: { slack: { botToken: "xoxb-test" } } };
|
||||
const SLACK_TEXT_LIMIT = 8000;
|
||||
|
||||
function slackPlatformError(code: string): Error {
|
||||
return Object.assign(new Error(`An API error occurred: ${code}`), {
|
||||
code: "slack_webapi_platform_error",
|
||||
data: { ok: false, error: code },
|
||||
});
|
||||
}
|
||||
|
||||
describe("sendMessageSlack permanent provider rejections", () => {
|
||||
it.each(["messages_tab_disabled", "account_inactive"])(
|
||||
"marks Slack %s as a permanent non-dispatch",
|
||||
async (code) => {
|
||||
const client = createSlackSendTestClient();
|
||||
const rejection = slackPlatformError(code);
|
||||
const onPlatformSendDispatch = vi.fn();
|
||||
client.chat.postMessage.mockRejectedValueOnce(rejection);
|
||||
|
||||
const caught = await sendMessageSlack("channel:C123", "hello", {
|
||||
token: "xoxb-test",
|
||||
cfg: SLACK_TEST_CFG,
|
||||
client,
|
||||
onPlatformSendDispatch,
|
||||
}).catch((error: unknown) => error);
|
||||
|
||||
expect(caught).toBeInstanceOf(PlatformMessageNotDispatchedError);
|
||||
expect(caught).toMatchObject({ retryable: false, cause: rejection });
|
||||
expect(caught).toMatchObject({ message: expect.stringContaining(code) });
|
||||
expect(onPlatformSendDispatch).toHaveBeenCalledOnce();
|
||||
expect(client.chat.postMessage).toHaveBeenCalledOnce();
|
||||
},
|
||||
);
|
||||
|
||||
it("marks account_inactive from durable DM resolution as a permanent non-dispatch", async () => {
|
||||
const client = createSlackSendTestClient();
|
||||
const rejection = slackPlatformError("account_inactive");
|
||||
client.conversations.open.mockRejectedValueOnce(rejection);
|
||||
|
||||
const caught = await sendMessageSlack("user:U123", "hello", {
|
||||
token: "xoxb-test",
|
||||
cfg: SLACK_TEST_CFG,
|
||||
client,
|
||||
deliveryQueueId: "queue-dm-account-inactive",
|
||||
}).catch((error: unknown) => error);
|
||||
|
||||
expect(caught).toBeInstanceOf(PlatformMessageNotDispatchedError);
|
||||
expect(caught).toMatchObject({ retryable: false, cause: rejection });
|
||||
expect(client.chat.postMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retains earlier Slack delivery evidence when a later chunk is permanently rejected", async () => {
|
||||
const client = createSlackSendTestClient();
|
||||
client.chat.postMessage
|
||||
.mockResolvedValueOnce({ ts: "171234.100", channel: "C123" })
|
||||
.mockRejectedValueOnce(slackPlatformError("messages_tab_disabled"));
|
||||
const delivered: string[] = [];
|
||||
|
||||
const caught = await sendMessageSlack("channel:C123", "a".repeat(SLACK_TEXT_LIMIT + 1), {
|
||||
token: "xoxb-test",
|
||||
cfg: SLACK_TEST_CFG,
|
||||
client,
|
||||
onDeliveryResult: (result) => {
|
||||
delivered.push(result.messageId);
|
||||
},
|
||||
}).catch((error: unknown) => error);
|
||||
|
||||
expect(caught).toBeInstanceOf(PlatformMessageNotDispatchedError);
|
||||
expect(delivered).toEqual(["171234.100"]);
|
||||
expect(client.chat.postMessage).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("does not classify a persistence callback failure as a Slack API rejection", async () => {
|
||||
const client = createSlackSendTestClient();
|
||||
const callbackError = slackPlatformError("messages_tab_disabled");
|
||||
client.chat.postMessage.mockResolvedValueOnce({ ts: "171234.100", channel: "C123" });
|
||||
|
||||
const caught = await sendMessageSlack("channel:C123", "hello", {
|
||||
token: "xoxb-test",
|
||||
cfg: SLACK_TEST_CFG,
|
||||
client,
|
||||
onDeliveryResult: () => {
|
||||
throw callbackError;
|
||||
},
|
||||
}).catch((error: unknown) => error);
|
||||
|
||||
expect(caught).toBe(callbackError);
|
||||
expect(caught).not.toBeInstanceOf(PlatformMessageNotDispatchedError);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
"rate limit",
|
||||
Object.assign(new Error("A rate limit was exceeded"), {
|
||||
code: "slack_webapi_rate_limited_error",
|
||||
retryAfter: 1,
|
||||
}),
|
||||
],
|
||||
[
|
||||
"HTTP 500",
|
||||
Object.assign(new Error("An HTTP protocol error occurred"), {
|
||||
code: "slack_webapi_http_error",
|
||||
statusCode: 500,
|
||||
}),
|
||||
],
|
||||
[
|
||||
"network reset",
|
||||
Object.assign(new Error("A request error occurred: read ECONNRESET"), {
|
||||
code: "slack_webapi_request_error",
|
||||
original: Object.assign(new Error("read ECONNRESET"), { code: "ECONNRESET" }),
|
||||
}),
|
||||
],
|
||||
])("does not terminalize a Slack %s", async (_name, error) => {
|
||||
const client = createSlackSendTestClient();
|
||||
client.chat.postMessage.mockRejectedValueOnce(error);
|
||||
|
||||
const caught = await sendMessageSlack("channel:C123", "hello", {
|
||||
token: "xoxb-test",
|
||||
cfg: SLACK_TEST_CFG,
|
||||
client,
|
||||
}).catch((caughtError: unknown) => caughtError);
|
||||
|
||||
expect(caught).toBe(error);
|
||||
expect(caught).not.toBeInstanceOf(PlatformMessageNotDispatchedError);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["null", null],
|
||||
["an unlisted platform error code", slackPlatformError("channel_not_found")],
|
||||
])("rethrows %s by identity", (_name, rejection) => {
|
||||
let caught: unknown = "not thrown";
|
||||
try {
|
||||
rethrowSlackPermanentOutboundApiRejection(rejection);
|
||||
} catch (error) {
|
||||
caught = error;
|
||||
}
|
||||
expect(caught).toBe(rejection);
|
||||
});
|
||||
|
||||
it("keeps a non-Error invalid_blocks rejection matchable after the send boundary", () => {
|
||||
let caught: unknown = "not thrown";
|
||||
try {
|
||||
rethrowSlackPermanentOutboundApiRejection({ data: { error: "invalid_blocks" } });
|
||||
} catch (error) {
|
||||
caught = error;
|
||||
}
|
||||
expect(isSlackInvalidBlocksError(caught)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -35,6 +35,7 @@ import { buildSlackCompleteBlocksFallbackText } from "./blocks-fallback.js";
|
||||
import { validateSlackBlocksArray } from "./blocks-input.js";
|
||||
import {
|
||||
postSlackMessageBestEffort,
|
||||
rethrowSlackPermanentOutboundApiRejection,
|
||||
uploadSlackFile,
|
||||
withSlackDnsRequestRetry,
|
||||
} from "./client-delivery.js";
|
||||
@@ -608,7 +609,7 @@ async function resolveChannelId(
|
||||
}
|
||||
const response = await withSlackDnsRequestRetry("conversations.open", () =>
|
||||
client.conversations.open({ users: recipient.id }),
|
||||
);
|
||||
).catch(rethrowSlackPermanentOutboundApiRejection);
|
||||
const channelId = response.channel?.id;
|
||||
if (!channelId) {
|
||||
throw new Error("Failed to open Slack DM channel");
|
||||
|
||||
@@ -214,6 +214,13 @@ function createUploadTestClient(slackApiUrl = "https://slack.com/api/"): UploadT
|
||||
} as unknown as UploadTestClient;
|
||||
}
|
||||
|
||||
function slackPlatformError(code: string): Error {
|
||||
return Object.assign(new Error(`An API error occurred: ${code}`), {
|
||||
code: "slack_webapi_platform_error",
|
||||
data: { ok: false, error: code },
|
||||
});
|
||||
}
|
||||
|
||||
type UploadOverrides = Omit<Partial<Parameters<typeof sendMessageSlack>[2]>, "cfg" | "client">;
|
||||
type UploadParams = UploadOverrides & { mediaUrl: string; target?: string; message?: string };
|
||||
|
||||
@@ -317,6 +324,70 @@ describe("sendMessageSlack file upload with user IDs", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("marks account_inactive from files.getUploadURLExternal as a permanent non-dispatch", async () => {
|
||||
const rejection = slackPlatformError("account_inactive");
|
||||
const onPlatformSendDispatch = vi.fn();
|
||||
client.files.getUploadURLExternal.mockRejectedValueOnce(rejection);
|
||||
|
||||
const caught = await sendUpload(client, {
|
||||
mediaUrl: "/tmp/account-inactive.png",
|
||||
onPlatformSendDispatch,
|
||||
}).catch((error: unknown) => error);
|
||||
|
||||
expect(caught).toBeInstanceOf(PlatformMessageNotDispatchedError);
|
||||
expect(caught).toMatchObject({ retryable: false, cause: rejection });
|
||||
expect(globalThis.fetch).not.toHaveBeenCalled();
|
||||
expect(onPlatformSendDispatch).not.toHaveBeenCalled();
|
||||
expect(client.files.completeUploadExternal).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps a definitive completeUploadExternal rejection ambiguous", async () => {
|
||||
// Dispatch is recorded before this call, so even a code that reads as a final
|
||||
// verdict cannot prove the file was never shared; it must not become a
|
||||
// non-dispatch assertion. Pairs with the pre-dispatch getUploadURLExternal case.
|
||||
const rejection = slackPlatformError("messages_tab_disabled");
|
||||
const onPlatformSendDispatch = vi.fn();
|
||||
client.files.completeUploadExternal.mockRejectedValueOnce(rejection);
|
||||
|
||||
const caught = await sendUpload(client, {
|
||||
mediaUrl: "/tmp/messages-tab-disabled.png",
|
||||
onPlatformSendDispatch,
|
||||
}).catch((error: unknown) => error);
|
||||
|
||||
expect(onPlatformSendDispatch).toHaveBeenCalledOnce();
|
||||
expect(caught).toBe(rejection);
|
||||
expect(caught).not.toBeInstanceOf(PlatformMessageNotDispatchedError);
|
||||
});
|
||||
|
||||
it("keeps getUploadURLExternal network failures ambiguous", async () => {
|
||||
const rejection = Object.assign(new Error("read ECONNRESET"), {
|
||||
code: "slack_webapi_request_error",
|
||||
});
|
||||
client.files.getUploadURLExternal.mockRejectedValueOnce(rejection);
|
||||
|
||||
const caught = await sendUpload(client, {
|
||||
mediaUrl: "/tmp/network-failure.png",
|
||||
}).catch((error: unknown) => error);
|
||||
|
||||
expect(caught).toBe(rejection);
|
||||
expect(caught).not.toBeInstanceOf(PlatformMessageNotDispatchedError);
|
||||
});
|
||||
|
||||
it("keeps completeUploadExternal HTTP failures ambiguous", async () => {
|
||||
const rejection = Object.assign(new Error("Slack HTTP 500"), {
|
||||
code: "slack_webapi_http_error",
|
||||
statusCode: 500,
|
||||
});
|
||||
client.files.completeUploadExternal.mockRejectedValueOnce(rejection);
|
||||
|
||||
const caught = await sendUpload(client, {
|
||||
mediaUrl: "/tmp/http-failure.png",
|
||||
}).catch((error: unknown) => error);
|
||||
|
||||
expect(caught).toBe(rejection);
|
||||
expect(caught).not.toBeInstanceOf(PlatformMessageNotDispatchedError);
|
||||
});
|
||||
|
||||
it("disables image optimization for forced-media uploads", async () => {
|
||||
await sendUpload(client, {
|
||||
mediaUrl: "/tmp/original.png",
|
||||
|
||||
@@ -1964,6 +1964,34 @@ describe("deliverReplies", () => {
|
||||
expect(sendMessage).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("maps a supergroup migration rejection without rewriting the streaming target", async () => {
|
||||
const chatId = "-123456789";
|
||||
const migratedChatId = -1_001_234_567_890;
|
||||
const terminal = Object.assign(
|
||||
new Error("400: Bad Request: group chat was upgraded to a supergroup chat"),
|
||||
{
|
||||
name: "GrammyError",
|
||||
error_code: 400,
|
||||
description: "Bad Request: group chat was upgraded to a supergroup chat",
|
||||
parameters: { migrate_to_chat_id: migratedChatId },
|
||||
},
|
||||
);
|
||||
const sendMessage = vi.fn().mockRejectedValue(terminal);
|
||||
|
||||
const observed = await sendTelegramText(
|
||||
createBot({ sendMessage }),
|
||||
chatId,
|
||||
"hello",
|
||||
createRuntime(),
|
||||
).catch((error: unknown) => error);
|
||||
|
||||
expect(observed).toBeInstanceOf(PlatformMessageNotDispatchedError);
|
||||
expect(observed).toMatchObject({ cause: terminal, retryable: false });
|
||||
expect(observed).toMatchObject({ message: expect.stringContaining(String(migratedChatId)) });
|
||||
expect(sendMessage).toHaveBeenCalledOnce();
|
||||
expect(firstMockCallArg(sendMessage, 0)).toBe(chatId);
|
||||
});
|
||||
|
||||
it("keeps broad 421-shaped streaming send errors ambiguous", async () => {
|
||||
const edgeError = Object.assign(new Error("421 Misdirected Request"), { status: 421 });
|
||||
const sendMessage = vi.fn().mockRejectedValue(edgeError);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Telegram tests cover network errors plugin behavior.
|
||||
import { PlatformMessageNotDispatchedError } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
isRecoverableTelegramNetworkError,
|
||||
@@ -9,6 +10,7 @@ import {
|
||||
isTelegramClientRejection,
|
||||
isTelegramPollingNetworkError,
|
||||
isTelegramServerError,
|
||||
rethrowTelegramSendError,
|
||||
tagTelegramNetworkError,
|
||||
TelegramRequestNotStartedError,
|
||||
} from "./network-errors.js";
|
||||
@@ -18,6 +20,15 @@ const errorWithCode = (message: string, code: string) =>
|
||||
const errorWithTelegramCode = (message: string, error_code: number) =>
|
||||
Object.assign(new Error(message), { error_code });
|
||||
|
||||
function captureTelegramSendError(error: unknown): unknown {
|
||||
try {
|
||||
rethrowTelegramSendError(error);
|
||||
} catch (caught) {
|
||||
return caught;
|
||||
}
|
||||
throw new Error("Expected Telegram send error to be rethrown");
|
||||
}
|
||||
|
||||
const plainErrorPredicateCases = [
|
||||
{
|
||||
name: "isTelegramServerError",
|
||||
@@ -333,6 +344,79 @@ describe("isSafeToRetrySendError", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("rethrowTelegramSendError", () => {
|
||||
const migratedChatId = -1_001_234_567_890;
|
||||
const migrationError = Object.assign(
|
||||
new Error("400: Bad Request: group chat was upgraded to a supergroup chat"),
|
||||
{
|
||||
name: "GrammyError",
|
||||
error_code: 400,
|
||||
description: "Bad Request: group chat was upgraded to a supergroup chat",
|
||||
parameters: { migrate_to_chat_id: migratedChatId },
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
["direct grammY rejection", migrationError],
|
||||
[
|
||||
"nested provider rejection",
|
||||
Object.assign(new Error("Telegram send failed"), { cause: migrationError }),
|
||||
],
|
||||
])("marks a migrated supergroup as a permanent non-dispatch for %s", (_name, error) => {
|
||||
const caught = captureTelegramSendError(error);
|
||||
|
||||
expect(caught).toBeInstanceOf(PlatformMessageNotDispatchedError);
|
||||
expect(caught).toMatchObject({
|
||||
retryable: false,
|
||||
cause: error,
|
||||
});
|
||||
expect(caught).toMatchObject({ message: expect.stringContaining(String(migratedChatId)) });
|
||||
});
|
||||
|
||||
it.each([
|
||||
["rate limit", errorWithTelegramCode("Too Many Requests", 429)],
|
||||
["server failure", errorWithTelegramCode("Bad Gateway", 502)],
|
||||
["unrelated client rejection", errorWithTelegramCode("Bad Request: message is empty", 400)],
|
||||
["ambiguous network failure", errorWithCode("read ECONNRESET", "ECONNRESET")],
|
||||
...(["status", "statusCode"] as const).map((statusField): [string, Error] => [
|
||||
`non-Telegram ${statusField} lookalike`,
|
||||
Object.assign(new Error("migration-shaped HTTP error"), {
|
||||
[statusField]: 400,
|
||||
description: "Bad Request: group chat was upgraded to a supergroup chat",
|
||||
parameters: { migrate_to_chat_id: migratedChatId },
|
||||
}),
|
||||
]),
|
||||
[
|
||||
"migration parameter without matching description",
|
||||
Object.assign(new Error("different bad request"), {
|
||||
error_code: 400,
|
||||
description: "Bad Request: chat not found",
|
||||
parameters: { migrate_to_chat_id: migratedChatId },
|
||||
}),
|
||||
],
|
||||
["plain migration text", new Error("400: group chat was upgraded to a supergroup chat")],
|
||||
])("does not terminalize a %s", (_name, error) => {
|
||||
expect(captureTelegramSendError(error)).toBe(error);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["without response parameters", undefined],
|
||||
["with an unsafe replacement id", Number.MAX_SAFE_INTEGER + 1],
|
||||
])("terminalizes a migration response %s without surfacing a target", (_name, target) => {
|
||||
const error = Object.assign(new Error("migration"), {
|
||||
error_code: 400,
|
||||
description: "Bad Request: group chat was upgraded to a supergroup chat",
|
||||
...(target === undefined ? {} : { parameters: { migrate_to_chat_id: target } }),
|
||||
});
|
||||
|
||||
const caught = captureTelegramSendError(error);
|
||||
|
||||
expect(caught).toBeInstanceOf(PlatformMessageNotDispatchedError);
|
||||
expect(caught).toMatchObject({ retryable: false, cause: error });
|
||||
expect(caught).not.toMatchObject({ message: expect.stringContaining(String(target)) });
|
||||
});
|
||||
});
|
||||
|
||||
describe("isTelegramServerError", () => {
|
||||
it.each([
|
||||
["Internal Server Error", 500, true],
|
||||
|
||||
@@ -8,9 +8,14 @@ import {
|
||||
} from "openclaw/plugin-sdk/error-runtime";
|
||||
import { parseStrictNonNegativeInteger } from "openclaw/plugin-sdk/number-runtime";
|
||||
import { classifyTransientNetworkErrorCode } from "openclaw/plugin-sdk/retry-runtime";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import {
|
||||
isRecord,
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
|
||||
const TELEGRAM_NETWORK_ORIGIN = Symbol("openclaw.telegram.network-origin");
|
||||
const TELEGRAM_SUPERGROUP_MIGRATION_DESCRIPTION =
|
||||
"Bad Request: group chat was upgraded to a supergroup chat";
|
||||
|
||||
export class TelegramRequestNotStartedError extends Error {
|
||||
constructor(message = "Telegram request did not start") {
|
||||
@@ -28,9 +33,14 @@ function isTelegramRequestNotStartedError(err: unknown): boolean {
|
||||
}
|
||||
|
||||
export function rethrowTelegramSendError(err: unknown): never {
|
||||
throw isTelegramRequestNotStartedError(err)
|
||||
? new PlatformMessageNotDispatchedError("Telegram request not started", { cause: err })
|
||||
: err;
|
||||
if (isTelegramRequestNotStartedError(err)) {
|
||||
throw new PlatformMessageNotDispatchedError("Telegram request not started", { cause: err });
|
||||
}
|
||||
const migrationRejection = describeTelegramSupergroupMigration(err);
|
||||
if (migrationRejection === undefined) {
|
||||
throw err;
|
||||
}
|
||||
throw new PlatformMessageNotDispatchedError(migrationRejection, { cause: err, retryable: false });
|
||||
}
|
||||
|
||||
const TELEGRAM_ADDITIONAL_TRANSIENT_ERROR_CODES = new Set([
|
||||
@@ -139,6 +149,28 @@ function getNumericHttpStatus(err: unknown): number | undefined {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Once a basic group is upgraded, its old id answers every send with this exact Bot API
|
||||
// 400 (https://core.telegram.org/bots/api#making-requests). Matching the description
|
||||
// literally keeps every other 400 retryable; `migrate_to_chat_id` is bounded at 52
|
||||
// significant bits, so a non-safe integer is not the documented id and stays unreported.
|
||||
function describeTelegramSupergroupMigration(err: unknown): string | undefined {
|
||||
for (const candidate of collectTelegramErrorCandidates(err)) {
|
||||
if (!isRecord(candidate) || candidate.error_code !== 400) {
|
||||
continue;
|
||||
}
|
||||
if (candidate.description !== TELEGRAM_SUPERGROUP_MIGRATION_DESCRIPTION) {
|
||||
continue;
|
||||
}
|
||||
const migratedChatId = isRecord(candidate.parameters)
|
||||
? candidate.parameters.migrate_to_chat_id
|
||||
: undefined;
|
||||
return typeof migratedChatId === "number" && Number.isSafeInteger(migratedChatId)
|
||||
? `Telegram rejected send: group migrated to supergroup ${migratedChatId}`
|
||||
: "Telegram rejected send: group migrated to a supergroup";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function isTelegramMisdirectedRequestError(err: unknown): boolean {
|
||||
for (const candidate of collectTelegramErrorCandidates(err)) {
|
||||
const code = normalizeCode(getErrorCode(candidate));
|
||||
|
||||
@@ -3489,6 +3489,34 @@ describe("sendMessageTelegram", () => {
|
||||
expect(sendMessage).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("maps a supergroup migration rejection without rewriting the durable target", async () => {
|
||||
const chatId = "-123456789";
|
||||
const migratedChatId = -1_001_234_567_890;
|
||||
const terminal = Object.assign(
|
||||
new Error("400: Bad Request: group chat was upgraded to a supergroup chat"),
|
||||
{
|
||||
name: "GrammyError",
|
||||
error_code: 400,
|
||||
description: "Bad Request: group chat was upgraded to a supergroup chat",
|
||||
parameters: { migrate_to_chat_id: migratedChatId },
|
||||
},
|
||||
);
|
||||
const sendMessage = vi.fn().mockRejectedValue(terminal);
|
||||
const api = makeTelegramApiTestMock({ sendMessage });
|
||||
|
||||
const observed = await sendMessageTelegram(chatId, "hi", {
|
||||
cfg: TELEGRAM_TEST_CFG,
|
||||
token: "tok",
|
||||
api,
|
||||
}).catch((error: unknown) => error);
|
||||
|
||||
expect(observed).toBeInstanceOf(PlatformMessageNotDispatchedError);
|
||||
expect(observed).toMatchObject({ cause: terminal, retryable: false });
|
||||
expect(observed).toMatchObject({ message: expect.stringContaining(String(migratedChatId)) });
|
||||
expect(sendMessage).toHaveBeenCalledOnce();
|
||||
expect(firstMockCall(sendMessage, "sendMessage call")[0]).toBe(chatId);
|
||||
});
|
||||
|
||||
it("keeps broad 421-shaped durable send errors ambiguous", async () => {
|
||||
const chatId = "123";
|
||||
const edgeError = Object.assign(new Error("421 Misdirected Request"), { status: 421 });
|
||||
|
||||
@@ -561,6 +561,14 @@ const PRECISE_SOURCE_TEST_TARGETS = new Map<string, string[]>([
|
||||
"extensions/slack/src/monitor/provider.auth-test-token.test.ts",
|
||||
],
|
||||
],
|
||||
[
|
||||
"extensions/slack/src/channel-actions.ts",
|
||||
[
|
||||
"extensions/slack/src/actions.reactions-limit.test.ts",
|
||||
"extensions/slack/src/channel-actions-setup-status.contract.test.ts",
|
||||
"extensions/slack/src/message-tools.test.ts",
|
||||
],
|
||||
],
|
||||
[
|
||||
"src/gateway/worker-environments/worker-turn-launcher.ts",
|
||||
[
|
||||
@@ -1945,6 +1953,9 @@ function resolveToolingChangedTestTargets(changedPaths: string[], cwd = process.
|
||||
return null;
|
||||
}
|
||||
targets.push(...testTargets);
|
||||
if (CHANNEL_PLUGIN_SHAPE_PARITY_WIRING_PATHS.has(changedPath)) {
|
||||
targets.push(CHANNEL_PLUGIN_SHAPE_PARITY_TEST_TARGET);
|
||||
}
|
||||
}
|
||||
return [...new Set(targets)];
|
||||
}
|
||||
|
||||
@@ -763,6 +763,36 @@ describe("delivery-queue recovery", () => {
|
||||
expect(await loadPendingDeliveries(tmpDir())).toHaveLength(0);
|
||||
expect(readOutboundQueueStatus(tmpDir(), id)).toBeUndefined();
|
||||
});
|
||||
it("keeps a partially sent batch unknown when a later send has a permanent rejection", async () => {
|
||||
const id = await enqueueDemoRecoveryDelivery(["first", "second"]);
|
||||
const rejection = new PlatformMessageNotDispatchedError(
|
||||
"Slack outbound delivery rejected: messages_tab_disabled",
|
||||
{ cause: new Error("messages_tab_disabled"), retryable: false },
|
||||
);
|
||||
const partialFailure = new OutboundDeliveryError("second send failed", {
|
||||
cause: rejection,
|
||||
results: [{ channel: "demo-channel-c", messageId: "m1" }],
|
||||
payloadOutcomes: [
|
||||
{ index: 0, status: "sent", results: [{ channel: "demo-channel-c", messageId: "m1" }] },
|
||||
{
|
||||
index: 1,
|
||||
status: "failed",
|
||||
error: rejection,
|
||||
sentBeforeError: false,
|
||||
stage: "platform_send",
|
||||
},
|
||||
],
|
||||
stage: "platform_send",
|
||||
});
|
||||
|
||||
const { result } = await runRecovery({
|
||||
deliver: vi.fn().mockRejectedValue(partialFailure),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ recovered: 0, failed: 1 });
|
||||
await expectPendingEntry({ id, recoveryState: "unknown_after_send", retryCount: 0 });
|
||||
expect(readOutboundQueueStatus(tmpDir(), id)).toBe("pending");
|
||||
});
|
||||
it("keeps a best-effort recovery failure retryable when no payload was sent", async () => {
|
||||
await enqueueDemoRecoveryDelivery(["first"], { bestEffort: true });
|
||||
const deliver = vi.fn(async (params: PayloadOutcomeSink) => {
|
||||
@@ -1269,6 +1299,57 @@ describe("delivery-queue recovery", () => {
|
||||
await runIf(typedRejection, () => expect(deliver).toHaveBeenCalledOnce());
|
||||
await runIf(!typedRejection, () => expectMockMessageContaining(log.warn, "permanent error"));
|
||||
});
|
||||
it("persists a nested channel rejection as the only terminal across recovery restart", async () => {
|
||||
const operationId = "operation-channel-permanent-rejection";
|
||||
const scope = await createConversationRecoveryFixture(operationId);
|
||||
const rejection = new PlatformMessageNotDispatchedError(
|
||||
"Slack chat.postMessage rejected: messages_tab_disabled",
|
||||
{
|
||||
cause: new Error("messages_tab_disabled"),
|
||||
retryable: false,
|
||||
},
|
||||
);
|
||||
const deliver = vi.fn().mockRejectedValue(
|
||||
new OutboundDeliveryError("Slack delivery failed", {
|
||||
cause: rejection,
|
||||
payloadOutcomes: [
|
||||
{
|
||||
index: 0,
|
||||
status: "failed",
|
||||
error: rejection,
|
||||
sentBeforeError: false,
|
||||
stage: "platform_send",
|
||||
},
|
||||
],
|
||||
stage: "platform_send",
|
||||
}),
|
||||
);
|
||||
|
||||
try {
|
||||
const first = await runRecovery({ deliver });
|
||||
expect(first.result).toEqual(RECOVERY_SUMMARY.failed);
|
||||
expect(deliver).toHaveBeenCalledOnce();
|
||||
expect(getConversationDeliveryOperation(scope, operationId)).toMatchObject({
|
||||
status: "rejected",
|
||||
rejectionError: "Slack chat.postMessage rejected: messages_tab_disabled",
|
||||
});
|
||||
expect(await loadPendingDeliveries(tmpDir())).toHaveLength(0);
|
||||
expect(readOutboundQueueStatus(tmpDir(), operationId)).toBe("failed");
|
||||
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
const replay = vi.fn();
|
||||
const second = await runRecovery({ deliver: replay });
|
||||
expect(second.result).toEqual(RECOVERY_SUMMARY.empty);
|
||||
expect(replay).not.toHaveBeenCalled();
|
||||
expect(getConversationDeliveryOperation(scope, operationId)).toMatchObject({
|
||||
status: "rejected",
|
||||
rejectionError: "Slack chat.postMessage rejected: messages_tab_disabled",
|
||||
});
|
||||
expect(readOutboundQueueStatus(tmpDir(), operationId)).toBe("failed");
|
||||
} finally {
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
}
|
||||
});
|
||||
it("passes skipQueue: true to prevent re-enqueueing during recovery", async () => {
|
||||
await enqueueRecoveryDelivery();
|
||||
const deliver = vi.fn().mockResolvedValue([]);
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
// Root-owned integration may combine the public Slack plugin with the durable queue runtime.
|
||||
import { createServer, type Server } from "node:http";
|
||||
import type { AddressInfo, Socket } from "node:net";
|
||||
import { sendDurableMessageBatch } from "openclaw/plugin-sdk/channel-outbound";
|
||||
import {
|
||||
createEmptyPluginRegistry,
|
||||
createTestRegistry,
|
||||
resetPluginRuntimeStateForTest,
|
||||
resetGlobalHookRunner,
|
||||
setActivePluginRegistry,
|
||||
} from "openclaw/plugin-sdk/channel-test-helpers";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { drainPendingDeliveries } from "openclaw/plugin-sdk/delivery-queue-runtime";
|
||||
import { PlatformMessageNotDispatchedError } from "openclaw/plugin-sdk/error-runtime";
|
||||
import {
|
||||
closeOpenClawAgentDatabasesForTest,
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime-testing";
|
||||
import { withStateDirEnv } from "openclaw/plugin-sdk/test-env";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { getDeliveryQueueEntryStatus } from "../src/infra/delivery-queue-sqlite.js";
|
||||
import { OUTBOUND_DELIVERY_QUEUE_NAME } from "../src/infra/outbound/delivery-queue-media-staging.js";
|
||||
|
||||
const CLASSIFIED_CODES = ["messages_tab_disabled", "account_inactive"] as const;
|
||||
const DELIVERY_INTENT_PREFIX = "slack-loopback-permanent-rejection";
|
||||
|
||||
type SlackPermanentRejectionCode = (typeof CLASSIFIED_CODES)[number];
|
||||
|
||||
type SlackLoopbackRequest = {
|
||||
body: string;
|
||||
code?: SlackPermanentRejectionCode;
|
||||
method: string | undefined;
|
||||
text: string;
|
||||
url: string;
|
||||
};
|
||||
|
||||
type SlackLoopback = {
|
||||
apiUrl: string;
|
||||
requests: SlackLoopbackRequest[];
|
||||
close: () => Promise<void>;
|
||||
};
|
||||
|
||||
function readQueueTerminal(
|
||||
stateDir: string,
|
||||
intentId: string,
|
||||
): { retryCount: number; status: string } | undefined {
|
||||
const { db } = openOpenClawStateDatabase({
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
|
||||
});
|
||||
const row = db
|
||||
// sqlite-allow-raw: The proof reads one exact queue owner after terminalization.
|
||||
.prepare(
|
||||
"SELECT status, retry_count FROM delivery_queue_entries WHERE queue_name = ? AND id = ?",
|
||||
)
|
||||
.get(OUTBOUND_DELIVERY_QUEUE_NAME, intentId) as
|
||||
| { retry_count: number; status: string }
|
||||
| undefined;
|
||||
return row ? { retryCount: row.retry_count, status: row.status } : undefined;
|
||||
}
|
||||
|
||||
async function startSlackPermanentRejectionLoopback(): Promise<SlackLoopback> {
|
||||
const requests: SlackLoopbackRequest[] = [];
|
||||
const sockets = new Set<Socket>();
|
||||
const server: Server = createServer((request, response) => {
|
||||
const chunks: Buffer[] = [];
|
||||
request.on("data", (chunk: Buffer) => chunks.push(chunk));
|
||||
request.on("end", () => {
|
||||
const body = Buffer.concat(chunks).toString("utf8");
|
||||
const text = new URLSearchParams(body).get("text") ?? "";
|
||||
const code = CLASSIFIED_CODES.find((candidate) => text.includes(candidate));
|
||||
requests.push({
|
||||
body,
|
||||
...(code ? { code } : {}),
|
||||
method: request.method,
|
||||
text,
|
||||
url: request.url ?? "",
|
||||
});
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(JSON.stringify({ ok: false, error: code ?? "unexpected_test_request" }));
|
||||
});
|
||||
});
|
||||
server.on("connection", (socket) => {
|
||||
sockets.add(socket);
|
||||
socket.on("close", () => sockets.delete(socket));
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", resolve);
|
||||
});
|
||||
const { port } = server.address() as AddressInfo;
|
||||
return {
|
||||
apiUrl: `http://127.0.0.1:${port}/api/`,
|
||||
requests,
|
||||
close: async () => {
|
||||
for (const socket of sockets) {
|
||||
socket.destroy();
|
||||
}
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("Slack permanent rejections over real Web API transport", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
resetGlobalHookRunner();
|
||||
resetPluginRuntimeStateForTest();
|
||||
setActivePluginRegistry(createEmptyPluginRegistry());
|
||||
});
|
||||
|
||||
it("dead-letters both real platform rejections and never replays them after restart", async () => {
|
||||
const loopback = await startSlackPermanentRejectionLoopback();
|
||||
vi.stubEnv("SLACK_API_URL", loopback.apiUrl);
|
||||
try {
|
||||
const { slackPlugin } = await import("../extensions/slack/api.js");
|
||||
const cfg = {
|
||||
channels: { slack: { botToken: "xoxb-loopback" } },
|
||||
} satisfies OpenClawConfig;
|
||||
setActivePluginRegistry(
|
||||
createTestRegistry([{ pluginId: "slack", plugin: slackPlugin, source: "test" }]),
|
||||
);
|
||||
|
||||
await withStateDirEnv("openclaw-slack-permanent-loopback-", async ({ stateDir }) => {
|
||||
try {
|
||||
const log = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
|
||||
for (const [index, code] of CLASSIFIED_CODES.entries()) {
|
||||
const intentId = `${DELIVERY_INTENT_PREFIX}-${code}`;
|
||||
const text = `real transport permanent rejection: ${code}`;
|
||||
const staged = await sendDurableMessageBatch({
|
||||
cfg,
|
||||
channel: "slack",
|
||||
to: "channel:C123",
|
||||
accountId: "default",
|
||||
durability: "required",
|
||||
deliveryIntentId: intentId,
|
||||
completionRetention: {
|
||||
idPrefix: "slack-loopback-",
|
||||
maxAgeMs: 60_000,
|
||||
maxEntries: 10,
|
||||
},
|
||||
maxRetries: 10,
|
||||
payloads: [{ text }],
|
||||
deps: {
|
||||
slack: async () => {
|
||||
throw new PlatformMessageNotDispatchedError(
|
||||
"staged before transport for recovery proof",
|
||||
{ cause: new Error("loopback transport not released yet") },
|
||||
);
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(staged.status).toBe("failed");
|
||||
expect(loopback.requests).toHaveLength(index);
|
||||
expect(
|
||||
getDeliveryQueueEntryStatus(OUTBOUND_DELIVERY_QUEUE_NAME, intentId, stateDir),
|
||||
).toBe("pending");
|
||||
|
||||
log.warn.mockClear();
|
||||
await drainPendingDeliveries({
|
||||
drainKey: `slack:default:${code}`,
|
||||
logLabel: `Slack loopback ${code} recovery`,
|
||||
cfg,
|
||||
stateDir,
|
||||
log,
|
||||
selectEntry: (entry) => ({
|
||||
match: entry.id === intentId,
|
||||
bypassBackoff: true,
|
||||
}),
|
||||
});
|
||||
|
||||
expect(loopback.requests).toHaveLength(index + 1);
|
||||
expect(loopback.requests[index]).toMatchObject({
|
||||
code,
|
||||
method: "POST",
|
||||
text,
|
||||
url: "/api/chat.postMessage",
|
||||
});
|
||||
expect(log.warn).toHaveBeenCalledWith(expect.stringContaining(code));
|
||||
expect(readQueueTerminal(stateDir, intentId)).toEqual({
|
||||
retryCount: 1,
|
||||
status: "failed",
|
||||
});
|
||||
}
|
||||
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
for (const code of CLASSIFIED_CODES) {
|
||||
expect(
|
||||
getDeliveryQueueEntryStatus(
|
||||
OUTBOUND_DELIVERY_QUEUE_NAME,
|
||||
`${DELIVERY_INTENT_PREFIX}-${code}`,
|
||||
stateDir,
|
||||
),
|
||||
).toBe("failed");
|
||||
}
|
||||
|
||||
await drainPendingDeliveries({
|
||||
drainKey: "slack:default:post-restart",
|
||||
logLabel: "Slack loopback post-restart recovery",
|
||||
cfg,
|
||||
stateDir,
|
||||
log,
|
||||
selectEntry: (entry) => ({
|
||||
match: entry.channel === "slack",
|
||||
bypassBackoff: true,
|
||||
}),
|
||||
});
|
||||
expect(loopback.requests).toHaveLength(CLASSIFIED_CODES.length);
|
||||
|
||||
console.log(
|
||||
`[slack permanent-rejection proof] ${JSON.stringify({
|
||||
queueTerminal: "failed",
|
||||
restartReplayCount: 0,
|
||||
providerStatus: 200,
|
||||
classifiedCodes: CLASSIFIED_CODES,
|
||||
classification: "typed non-retryable",
|
||||
transport: "@slack/web-api HTTP to 127.0.0.1:<redacted>",
|
||||
})}`,
|
||||
);
|
||||
} finally {
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
await loopback.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,215 @@
|
||||
// Root-owned integration may combine the public Telegram plugin with the durable queue runtime.
|
||||
import { createServer, type Server } from "node:http";
|
||||
import type { AddressInfo, Socket } from "node:net";
|
||||
import { sendDurableMessageBatch } from "openclaw/plugin-sdk/channel-outbound";
|
||||
import {
|
||||
createEmptyPluginRegistry,
|
||||
createTestRegistry,
|
||||
resetPluginRuntimeStateForTest,
|
||||
resetGlobalHookRunner,
|
||||
setActivePluginRegistry,
|
||||
} from "openclaw/plugin-sdk/channel-test-helpers";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { drainPendingDeliveries } from "openclaw/plugin-sdk/delivery-queue-runtime";
|
||||
import { PlatformMessageNotDispatchedError } from "openclaw/plugin-sdk/error-runtime";
|
||||
import {
|
||||
closeOpenClawAgentDatabasesForTest,
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime-testing";
|
||||
import { withStateDirEnv } from "openclaw/plugin-sdk/test-env";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { getDeliveryQueueEntryStatus } from "../src/infra/delivery-queue-sqlite.js";
|
||||
import { OUTBOUND_DELIVERY_QUEUE_NAME } from "../src/infra/outbound/delivery-queue-media-staging.js";
|
||||
|
||||
const MIGRATION_DESCRIPTION = "Bad Request: group chat was upgraded to a supergroup chat";
|
||||
const DELIVERY_INTENT_ID = "telegram-loopback-permanent-rejection";
|
||||
|
||||
type TelegramLoopback = {
|
||||
apiRoot: string;
|
||||
requests: Array<{ body: string; method: string | undefined; url: string }>;
|
||||
close: () => Promise<void>;
|
||||
};
|
||||
|
||||
function readQueueTerminal(stateDir: string): { retryCount: number; status: string } | undefined {
|
||||
const { db } = openOpenClawStateDatabase({
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
|
||||
});
|
||||
const row = db
|
||||
// sqlite-allow-raw: The proof reads one exact queue owner after terminalization.
|
||||
.prepare(
|
||||
"SELECT status, retry_count FROM delivery_queue_entries WHERE queue_name = ? AND id = ?",
|
||||
)
|
||||
.get(OUTBOUND_DELIVERY_QUEUE_NAME, DELIVERY_INTENT_ID) as
|
||||
| { retry_count: number; status: string }
|
||||
| undefined;
|
||||
return row ? { retryCount: row.retry_count, status: row.status } : undefined;
|
||||
}
|
||||
|
||||
async function startTelegramMigrationLoopback(): Promise<TelegramLoopback> {
|
||||
const requests: TelegramLoopback["requests"] = [];
|
||||
const sockets = new Set<Socket>();
|
||||
const server: Server = createServer((request, response) => {
|
||||
const chunks: Buffer[] = [];
|
||||
request.on("data", (chunk: Buffer) => chunks.push(chunk));
|
||||
request.on("end", () => {
|
||||
requests.push({
|
||||
body: Buffer.concat(chunks).toString("utf8"),
|
||||
method: request.method,
|
||||
url: request.url ?? "",
|
||||
});
|
||||
response.writeHead(400, { "content-type": "application/json" });
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
ok: false,
|
||||
error_code: 400,
|
||||
description: MIGRATION_DESCRIPTION,
|
||||
parameters: { migrate_to_chat_id: -1_001_234_567_890 },
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
server.on("connection", (socket) => {
|
||||
sockets.add(socket);
|
||||
socket.on("close", () => sockets.delete(socket));
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", resolve);
|
||||
});
|
||||
const { port } = server.address() as AddressInfo;
|
||||
return {
|
||||
apiRoot: `http://127.0.0.1:${port}`,
|
||||
requests,
|
||||
close: async () => {
|
||||
for (const socket of sockets) {
|
||||
socket.destroy();
|
||||
}
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("Telegram permanent rejection over real Bot API transport", () => {
|
||||
afterEach(() => {
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
resetGlobalHookRunner();
|
||||
resetPluginRuntimeStateForTest();
|
||||
setActivePluginRegistry(createEmptyPluginRegistry());
|
||||
});
|
||||
|
||||
it("dead-letters one real migration rejection and never replays it after restart", async () => {
|
||||
const loopback = await startTelegramMigrationLoopback();
|
||||
try {
|
||||
const { telegramPlugin } = await import("../extensions/telegram/api.js");
|
||||
const cfg = {
|
||||
channels: {
|
||||
telegram: {
|
||||
botToken: "123456:loopback",
|
||||
apiRoot: loopback.apiRoot,
|
||||
},
|
||||
},
|
||||
} satisfies OpenClawConfig;
|
||||
setActivePluginRegistry(
|
||||
createTestRegistry([{ pluginId: "telegram", plugin: telegramPlugin, source: "test" }]),
|
||||
);
|
||||
|
||||
await withStateDirEnv("openclaw-telegram-permanent-loopback-", async ({ stateDir }) => {
|
||||
try {
|
||||
const staged = await sendDurableMessageBatch({
|
||||
cfg,
|
||||
channel: "telegram",
|
||||
to: "123",
|
||||
accountId: "default",
|
||||
durability: "required",
|
||||
deliveryIntentId: DELIVERY_INTENT_ID,
|
||||
completionRetention: {
|
||||
idPrefix: "telegram-loopback-",
|
||||
maxAgeMs: 60_000,
|
||||
maxEntries: 10,
|
||||
},
|
||||
maxRetries: 10,
|
||||
payloads: [{ text: "real transport permanent rejection" }],
|
||||
deps: {
|
||||
telegram: async () => {
|
||||
throw new PlatformMessageNotDispatchedError(
|
||||
"staged before transport for recovery proof",
|
||||
{ cause: new Error("loopback transport not released yet") },
|
||||
);
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(staged.status).toBe("failed");
|
||||
expect(loopback.requests).toHaveLength(0);
|
||||
expect(
|
||||
getDeliveryQueueEntryStatus(OUTBOUND_DELIVERY_QUEUE_NAME, DELIVERY_INTENT_ID, stateDir),
|
||||
).toBe("pending");
|
||||
|
||||
const log = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
|
||||
await drainPendingDeliveries({
|
||||
drainKey: "telegram:default",
|
||||
logLabel: "Telegram loopback permanent rejection recovery",
|
||||
cfg,
|
||||
stateDir,
|
||||
log,
|
||||
selectEntry: (entry) => ({
|
||||
match: entry.channel === "telegram",
|
||||
bypassBackoff: true,
|
||||
}),
|
||||
});
|
||||
|
||||
expect(loopback.requests).toHaveLength(1);
|
||||
expect(log.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining(
|
||||
"Telegram rejected send: group migrated to supergroup -1001234567890",
|
||||
),
|
||||
);
|
||||
expect(loopback.requests[0]).toMatchObject({ method: "POST" });
|
||||
expect(loopback.requests[0]?.url).toMatch(/\/sendMessage$/u);
|
||||
expect(loopback.requests[0]?.body).toContain("real transport permanent rejection");
|
||||
expect(readQueueTerminal(stateDir)).toEqual({ retryCount: 1, status: "failed" });
|
||||
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
expect(
|
||||
getDeliveryQueueEntryStatus(OUTBOUND_DELIVERY_QUEUE_NAME, DELIVERY_INTENT_ID, stateDir),
|
||||
).toBe("failed");
|
||||
|
||||
await drainPendingDeliveries({
|
||||
drainKey: "telegram:default",
|
||||
logLabel: "Telegram loopback post-restart recovery",
|
||||
cfg,
|
||||
stateDir,
|
||||
log,
|
||||
selectEntry: (entry) => ({
|
||||
match: entry.channel === "telegram",
|
||||
bypassBackoff: true,
|
||||
}),
|
||||
});
|
||||
expect(loopback.requests).toHaveLength(1);
|
||||
expect(
|
||||
getDeliveryQueueEntryStatus(OUTBOUND_DELIVERY_QUEUE_NAME, DELIVERY_INTENT_ID, stateDir),
|
||||
).toBe("failed");
|
||||
|
||||
console.log(
|
||||
`[telegram permanent-rejection proof] ${JSON.stringify({
|
||||
queueTerminal: "failed",
|
||||
restartReplayCount: 0,
|
||||
providerStatus: 400,
|
||||
classification: "typed non-retryable",
|
||||
transport: "grammY Bot API HTTP to 127.0.0.1:<redacted>",
|
||||
})}`,
|
||||
);
|
||||
} finally {
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
await loopback.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user