diff --git a/docs/channels/feishu.md b/docs/channels/feishu.md index 720590b8c96c..76be27a0d810 100644 --- a/docs/channels/feishu.md +++ b/docs/channels/feishu.md @@ -38,7 +38,7 @@ The wizard also asks for the API domain (Feishu vs Lark) and the group policy. I ## Inbound durability -OpenClaw durably queues authenticated `im.message.receive_v1` and `drive.notice.comment_add_v1` envelopes before agent dispatch. Pending or retryable events survive a Gateway restart, remain serialized per chat or document, and use Feishu's event ID to suppress duplicate queue entries while the active or retained completion record exists. +OpenClaw durably queues authenticated `im.message.receive_v1` and `drive.notice.comment_add_v1` envelopes before agent dispatch. In webhook mode, the durable `200` carries `x-openclaw-delivery-accepted: durable`; verification challenges, non-durable event types, and error responses omit the marker, so reverse proxies can require it to distinguish durable acceptance from a generic `200`. Pending or retryable events survive a Gateway restart, remain serialized per chat or document, and use Feishu's event ID to suppress duplicate queue entries while the active or retained completion record exists. If a WebSocket event cannot be persisted after bounded retries, OpenClaw closes that socket and forces a fresh authenticated connection instead of continuing past an uncommitted turn. Other Feishu event types, including reactions and VC meeting invitations, use their normal event paths and do not receive this durable-queue guarantee. diff --git a/docs/channels/googlechat.md b/docs/channels/googlechat.md index 7f83f84c5356..b8abd50ce019 100644 --- a/docs/channels/googlechat.md +++ b/docs/channels/googlechat.md @@ -143,7 +143,7 @@ Configure the tunnel ingress rules to route only the webhook path: ### Inbound durability -After request authentication, OpenClaw removes the add-on authorization object from storage and durably queues Google Chat `MESSAGE` events before returning `200`. A persistence failure returns `503`, allowing Google Chat to retry instead of acknowledging an event that could be lost. +After request authentication, OpenClaw removes the add-on authorization object from storage and durably queues Google Chat `MESSAGE` events before returning `200`. A persistence failure returns `503`, allowing Google Chat to retry instead of acknowledging an event that could be lost. A durably queued `200` carries `x-openclaw-delivery-accepted: durable`; non-message action acks and error responses omit the marker, so reverse proxies can require it to distinguish durable acceptance from a generic `200`. Pending or retryable messages survive a Gateway restart, remain serialized per space, and use the Google Chat message resource name to suppress duplicate queue entries while the active or retained completion record exists. Non-message actions keep their existing detached webhook path and do not receive this durable-queue guarantee. Delivery remains at least once across the queue-to-agent boundary, so a crash during handoff can replay a turn. diff --git a/docs/channels/nextcloud-talk.md b/docs/channels/nextcloud-talk.md index e5254b32800a..842e70930977 100644 --- a/docs/channels/nextcloud-talk.md +++ b/docs/channels/nextcloud-talk.md @@ -84,6 +84,7 @@ Minimal config: - Bots cannot initiate DMs. The user must message the bot first. - The webhook URL must be reachable from the Nextcloud server; set `webhookPublicUrl` when the gateway sits behind a proxy. Webhook requests are HMAC-SHA256 signed with the bot secret; invalid signatures are rejected and rate limited. +- HTTP 200 is returned only after the raw event is durably stored; storage failures return HTTP 500. The durable `200` carries `x-openclaw-delivery-accepted: durable` (signature, validation, and storage-error responses omit it), so reverse proxies can require the marker to distinguish OpenClaw acceptance from a generic `200`. - Media uploads are not supported by the bot API; outbound media is appended as an `Attachment: ` line. - The webhook payload does not distinguish DMs from rooms; set `apiUser` + `apiPassword` to enable room-type lookups (cached about 5 minutes). Without them, every conversation is treated as a room. - Outbound requests go through the SSRF guard. For a Nextcloud host on a trusted private/internal network, opt in with `channels.nextcloud-talk.network.dangerouslyAllowPrivateNetwork: true`. diff --git a/docs/channels/synology-chat.md b/docs/channels/synology-chat.md index ba704d157457..d8447a59adba 100644 --- a/docs/channels/synology-chat.md +++ b/docs/channels/synology-chat.md @@ -52,7 +52,7 @@ Webhook auth details: ## Inbound durability -After token, sender-policy, and rate-limit checks pass, OpenClaw removes the webhook token from the stored envelope and durably queues the event before acknowledging it. The route returns `204` only after that append succeeds; a persistence failure returns `503` so Synology Chat can retry instead of silently losing the message. +After token, sender-policy, and rate-limit checks pass, OpenClaw removes the webhook token from the stored envelope and durably queues the event before acknowledging it. The route returns `204` only after that append succeeds; a persistence failure returns `503` so Synology Chat can retry instead of silently losing the message. The durable `204` carries `x-openclaw-delivery-accepted: durable`; authentication, validation, and storage-error responses omit the marker, so reverse proxies can require it to distinguish durable acceptance from a generic response. Pending or retryable events survive a Gateway restart. Synology's stable `post_id` suppresses duplicate queue entries while the corresponding active or retained completion record exists. Delivery remains at least once across the queue-to-agent handoff, so a crash at that boundary can still replay a turn. diff --git a/docs/channels/zalo.md b/docs/channels/zalo.md index 94c278f2b51a..b79cf209ffb2 100644 --- a/docs/channels/zalo.md +++ b/docs/channels/zalo.md @@ -97,7 +97,7 @@ Group chats are supported by the plugin (`chatTypes: ["direct", "group"]`) and g - Zalo sends events with an `X-Bot-Api-Secret-Token` header, checked with a constant-time comparison. - Gateway HTTP handles webhook requests at `channels.zalo.webhookPath` (defaults to the webhook URL's path). - Requests must use `Content-Type: application/json` (or a `+json` media type). - - HTTP 200 is returned only after the raw event is durably stored; storage failures return HTTP 500. + - HTTP 200 is returned only after the raw event is durably stored; storage failures return HTTP 500. The durable `200` carries `x-openclaw-delivery-accepted: durable`, so reverse proxies can require it to distinguish OpenClaw acceptance from a generic `200` (authentication, validation, and storage-error responses omit it). - getUpdates polling and webhook are mutually exclusive per Zalo API docs. ## Supported message types diff --git a/extensions/feishu/src/feishu-ingress.test.ts b/extensions/feishu/src/feishu-ingress.test.ts index 1d653c8d57a8..eefa427d29b4 100644 --- a/extensions/feishu/src/feishu-ingress.test.ts +++ b/extensions/feishu/src/feishu-ingress.test.ts @@ -139,7 +139,7 @@ function signWebhookBody(rawBody: string, encryptKey: string): Record, + ingress: Pick, "invoke" | "invokeWebhook">, run: (url: string) => Promise, ) { const port = await getFreePort(); @@ -154,7 +154,8 @@ async function withWebhook( const monitor = monitorWebhook({ account, accountId: account.accountId, - eventDispatcher: eventDispatcher as Lark.EventDispatcher, + eventDispatcher: { invoke: ingress.invoke } as Lark.EventDispatcher, + invokeWebhookEvent: ingress.invokeWebhook, abortSignal: abortController.signal, runtime: createNonExitingRuntimeEnv(), }); @@ -197,7 +198,7 @@ describe("Feishu durable ingress", () => { const gatedQueue = { ...queue, enqueue } as FeishuIngressQueue; const ingress = startIngress({ queue: gatedQueue, dispatcher: createDispatcher() }); - await withWebhook({ invoke: ingress.invoke }, async (url) => { + await withWebhook(ingress, async (url) => { let responseSettled = false; const responsePromise = postWebhook( url, @@ -210,7 +211,9 @@ describe("Feishu durable ingress", () => { expect(responseSettled).toBe(false); releaseAppend(); - await expect(responsePromise.then((response) => response.status)).resolves.toBe(200); + const response = await responsePromise; + expect(response.status).toBe(200); + expect(response.headers.get("x-openclaw-delivery-accepted")).toBe("durable"); }); await ingress.stop(); }); @@ -225,9 +228,10 @@ describe("Feishu durable ingress", () => { const dispatch = vi.fn(async () => undefined); const ingress = startIngress({ queue: failingQueue, dispatcher: createDispatcher(dispatch) }); - await withWebhook({ invoke: ingress.invoke }, async (url) => { + await withWebhook(ingress, async (url) => { const response = await postWebhook(url, messageEnvelope({ eventId: "evt-append-fail" })); expect(response.status).toBe(500); + expect(response.headers.get("x-openclaw-delivery-accepted")).toBeNull(); }); expect(enqueue).toHaveBeenCalledTimes(3); diff --git a/extensions/feishu/src/feishu-ingress.ts b/extensions/feishu/src/feishu-ingress.ts index 9bbfa2e69a27..f3754f1c5797 100644 --- a/extensions/feishu/src/feishu-ingress.ts +++ b/extensions/feishu/src/feishu-ingress.ts @@ -50,8 +50,14 @@ type FeishuIngressOptions = { adoptionStallTimeoutMs?: number; }; +export type FeishuWebhookInvoker = ( + data: unknown, + params?: { needCheck?: boolean }, +) => Promise<{ kind: "durable" | "non-durable"; value: unknown }>; + type FeishuDurableIngress = { invoke: Lark.EventDispatcher["invoke"]; + invokeWebhook: FeishuWebhookInvoker; resolveLifecycle: (data: unknown) => FeishuIngressLifecycle | undefined; setSocketTerminator: (terminate: (() => void) | undefined) => void; start: () => void; @@ -434,7 +440,7 @@ export function createFeishuDurableIngress(options: FeishuIngressOptions): Feish options.runtime.error?.(`feishu ingress drain failed: ${formatErrorMessage(error)}`), }); - const invoke: Lark.EventDispatcher["invoke"] = async (data, params) => { + const invokeWebhook: FeishuWebhookInvoker = async (data, params) => { let rawEnvelope: string; try { const serialized = JSON.stringify(data); @@ -455,7 +461,10 @@ export function createFeishuDurableIngress(options: FeishuIngressOptions): Feish // exists. Claim-side validation then dead-letters them without retry. const facts = inspectFeishuIngressEnvelope(rawEnvelope, options.encryptKey, true); if (!facts) { - return await options.dispatcher.invoke(data, params); + return { + kind: "non-durable", + value: await options.dispatcher.invoke(data, params), + }; } try { await monitor.admit(rawEnvelope, { @@ -465,11 +474,14 @@ export function createFeishuDurableIngress(options: FeishuIngressOptions): Feish socketTerminator?.(); throw error; } - return undefined; + return { kind: "durable", value: undefined }; }; + const invoke: Lark.EventDispatcher["invoke"] = async (data, params) => + (await invokeWebhook(data, params)).value; return { invoke, + invokeWebhook, resolveLifecycle: (data) => { const eventId = isRecord(data) ? readString(data.event_id) : null; return eventId ? activeLifecycles.get(eventId) : undefined; diff --git a/extensions/feishu/src/monitor.account.ts b/extensions/feishu/src/monitor.account.ts index e57b082ba30b..cec1d4a22d7c 100644 --- a/extensions/feishu/src/monitor.account.ts +++ b/extensions/feishu/src/monitor.account.ts @@ -570,6 +570,7 @@ export async function monitorSingleAccount(params: MonitorSingleAccountParams): runtime, abortSignal, eventDispatcher: durableEventDispatcher, + ...(durableIngress ? { invokeWebhookEvent: durableIngress.invokeWebhook } : {}), ...(params.statusSink ? { statusSink: params.statusSink } : {}), }); } diff --git a/extensions/feishu/src/monitor.transport.ts b/extensions/feishu/src/monitor.transport.ts index e43a0707f241..978fdb031534 100644 --- a/extensions/feishu/src/monitor.transport.ts +++ b/extensions/feishu/src/monitor.transport.ts @@ -5,6 +5,7 @@ import * as Lark from "@larksuiteoapi/node-sdk"; import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; import { waitForAbortableDelay } from "./async.js"; import { createFeishuWSClient } from "./client.js"; +import type { FeishuWebhookInvoker } from "./feishu-ingress.js"; import { buildFeishuWebhookRateLimitKey } from "./monitor-rate-limit-key.js"; import { applyBasicWebhookRequestGuards, @@ -33,6 +34,7 @@ type MonitorTransportParams = { runtime?: RuntimeEnv; abortSignal?: AbortSignal; eventDispatcher: Lark.EventDispatcher; + invokeWebhookEvent?: FeishuWebhookInvoker; setSocketTerminator?: (terminate: (() => void) | undefined) => void; /** * Optional status sink for Feishu health tracking. Lifecycle callbacks @@ -42,6 +44,8 @@ type MonitorTransportParams = { statusSink?: FeishuStatusSink; }; +const FEISHU_WEBHOOK_ACCEPTED_HEADER = "x-openclaw-delivery-accepted"; +const FEISHU_WEBHOOK_ACCEPTED_VALUE = "durable"; const FEISHU_WS_RECONNECT_INITIAL_DELAY_MS = 1_000; const FEISHU_WS_RECONNECT_MAX_DELAY_MS = 30_000; const FEISHU_WS_LOG_ERROR_MAX_LENGTH = 500; @@ -330,6 +334,7 @@ export async function monitorWebhook({ runtime, abortSignal, eventDispatcher, + invokeWebhookEvent, statusSink, }: MonitorTransportParams): Promise { const log = runtime?.log ?? console.log; @@ -435,13 +440,22 @@ export async function monitorWebhook({ return; } - const value = await eventDispatcher.invoke(buildFeishuWebhookEnvelope(req, payload), { - needCheck: false, - }); + const envelope = buildFeishuWebhookEnvelope(req, payload); + const invocation = invokeWebhookEvent + ? await invokeWebhookEvent(envelope, { needCheck: false }) + : { + kind: "non-durable" as const, + value: await eventDispatcher.invoke(envelope, { needCheck: false }), + }; if (!res.headersSent) { + if (invocation.kind === "durable") { + // The ingress owner records this fact at admission; challenges and + // non-durable event types ack without claiming durable acceptance. + res.setHeader(FEISHU_WEBHOOK_ACCEPTED_HEADER, FEISHU_WEBHOOK_ACCEPTED_VALUE); + } res.statusCode = 200; res.setHeader("Content-Type", "application/json; charset=utf-8"); - res.end(JSON.stringify(value)); + res.end(JSON.stringify(invocation.value)); } } catch (err) { error(`feishu[${accountId}]: webhook handler error: ${String(err)}`); diff --git a/extensions/feishu/src/monitor.webhook-e2e.test.ts b/extensions/feishu/src/monitor.webhook-e2e.test.ts index 94b04496740f..c04604f40d06 100644 --- a/extensions/feishu/src/monitor.webhook-e2e.test.ts +++ b/extensions/feishu/src/monitor.webhook-e2e.test.ts @@ -31,6 +31,8 @@ vi.mock("./runtime.js", () => createFeishuRuntimeMockModule()); import { cleanupFeishuMonitorStateForTests } from "./monitor.cleanup.test-helpers.js"; import { monitorFeishuProvider } from "./monitor.js"; import { httpServers } from "./monitor.state.js"; +import { monitorWebhook } from "./monitor.transport.js"; +import type { ResolvedFeishuAccount } from "./types.js"; beforeAll(async () => { await import("./monitor.account.js"); @@ -339,6 +341,7 @@ describe("Feishu webhook signed-request e2e", () => { const response = await postSignedPayload(url, payload); expect(response.status).toBe(200); + expect(response.headers.get("x-openclaw-delivery-accepted")).toBeNull(); await expect(response.json()).resolves.toEqual({ challenge: "challenge-token" }); }, ); @@ -364,11 +367,152 @@ describe("Feishu webhook signed-request e2e", () => { const response = await postSignedPayload(url, payload); expect(response.status).toBe(200); + expect(response.headers.get("x-openclaw-delivery-accepted")).toBeNull(); expect(await response.text()).toContain("no unknown.event event handle"); }, ); }); + it("marks durably admitted message acks with the delivery-accepted header", async () => { + probeFeishuMock.mockResolvedValue({ ok: true, botOpenId: "bot_open_id" }); + + await withRunningWebhookMonitor( + { + accountId: "signed-durable-ack", + path: "/hook-e2e-durable-ack", + verificationToken: "verify_token", + encryptKey: "encrypt_key", + }, + monitorFeishuProvider, + async (url) => { + const payload = { + schema: "2.0", + header: { event_type: "im.message.receive_v1", event_id: "evt-durable-ack-1" }, + event: { message: { chat_id: "oc_durable_ack" } }, + }; + const response = await postSignedPayload(url, payload); + + expect(response.status).toBe(200); + expect(response.headers.get("x-openclaw-delivery-accepted")).toBe("durable"); + }, + ); + }); + + it("acks durable envelopes only after ingress admission resolves", async () => { + const accountId = "durable-ack-ordering"; + const path = "/hook-e2e-durable-ack-ordering"; + const port = await getFreePort(); + const abortController = new AbortController(); + let releaseAdmission: (() => void) | undefined; + const invoke = vi.fn( + async () => + await new Promise((resolve) => { + releaseAdmission = resolve; + }), + ); + const monitorPromise = monitorWebhook({ + account: { + accountId, + encryptKey: "encrypt_key", + config: { + enabled: true, + connectionMode: "webhook", + webhookHost: "127.0.0.1", + webhookPort: port, + webhookPath: path, + }, + } as ResolvedFeishuAccount, + accountId, + runtime: { log: vi.fn(), error: vi.fn(), exit: vi.fn() }, + abortSignal: abortController.signal, + eventDispatcher: { invoke } as never, + invokeWebhookEvent: async () => { + await invoke(); + return { kind: "durable", value: undefined }; + }, + }); + + try { + const url = `http://127.0.0.1:${port}${path}`; + await waitUntilServerReady(url); + + const payload = { + schema: "2.0", + header: { event_type: "im.message.receive_v1", event_id: "evt-durable-ack-ordering-1" }, + event: { message: { chat_id: "oc_durable_ack_ordering" } }, + }; + let acceptedResponseReceived = false; + const acceptedRequest = postSignedPayload(url, payload).then((response) => { + acceptedResponseReceived = true; + return response; + }); + await vi.waitFor(() => { + expect(invoke).toHaveBeenCalledTimes(1); + }); + expect(acceptedResponseReceived).toBe(false); + if (!releaseAdmission) { + throw new Error("expected pending Feishu durable admission"); + } + releaseAdmission(); + + const accepted = await acceptedRequest; + expect(accepted.status).toBe(200); + expect(accepted.headers.get("x-openclaw-delivery-accepted")).toBe("durable"); + } finally { + releaseAdmission?.(); + abortController.abort(); + await monitorPromise; + } + }); + + it("does not mark acks when durable admission fails", async () => { + const accountId = "durable-ack-failure"; + const path = "/hook-e2e-durable-ack-failure"; + const port = await getFreePort(); + const abortController = new AbortController(); + const invoke = vi.fn(async () => { + throw new Error("admission failed"); + }); + const monitorPromise = monitorWebhook({ + account: { + accountId, + encryptKey: "encrypt_key", + config: { + enabled: true, + connectionMode: "webhook", + webhookHost: "127.0.0.1", + webhookPort: port, + webhookPath: path, + }, + } as ResolvedFeishuAccount, + accountId, + runtime: { log: vi.fn(), error: vi.fn(), exit: vi.fn() }, + abortSignal: abortController.signal, + eventDispatcher: { invoke } as never, + invokeWebhookEvent: async () => { + await invoke(); + return { kind: "durable", value: undefined }; + }, + }); + + try { + const url = `http://127.0.0.1:${port}${path}`; + await waitUntilServerReady(url); + + const response = await postSignedPayload(url, { + schema: "2.0", + header: { event_type: "im.message.receive_v1", event_id: "evt-durable-ack-failure-1" }, + event: { message: { chat_id: "oc_durable_ack_failure" } }, + }); + expect(response.status).toBe(500); + expect(response.headers.get("x-openclaw-delivery-accepted")).toBeNull(); + expect(invoke).toHaveBeenCalledTimes(1); + } finally { + abortController.abort(); + await monitorPromise; + } + }); + it("does not emit unhandled-event warning for bot_p2p_chat_entered_v1", async () => { probeFeishuMock.mockResolvedValue({ ok: true, botOpenId: "bot_open_id" }); diff --git a/extensions/googlechat/src/monitor-webhook.test.ts b/extensions/googlechat/src/monitor-webhook.test.ts index 95fbb8b87f28..bd029d83b8e1 100644 --- a/extensions/googlechat/src/monitor-webhook.test.ts +++ b/extensions/googlechat/src/monitor-webhook.test.ts @@ -305,6 +305,7 @@ describe("googlechat monitor webhook", () => { expect(processEvent).not.toHaveBeenCalled(); expect(runDetachedWebhookWork).not.toHaveBeenCalled(); expect(res.statusCode).toBe(200); + expect(res.headers["x-openclaw-delivery-accepted"]).toBe("durable"); expect(res.headers["Content-Type"]).toBe("application/json"); expect(res.body).toBe("{}"); }); @@ -383,6 +384,7 @@ describe("googlechat monitor webhook", () => { target, ); expect(res.statusCode).toBe(200); + expect(res.headers["x-openclaw-delivery-accepted"]).toBeUndefined(); expect(res.headers["Content-Type"]).toBe("application/json"); expect(res.body).toBe("{}"); }); @@ -426,9 +428,11 @@ describe("googlechat monitor webhook", () => { await vi.waitFor(() => expect(ingressReceive).toHaveBeenCalledWith(raw)); expect(res.statusCode).toBe(0); + expect(res.headers["x-openclaw-delivery-accepted"]).toBeUndefined(); releaseAdmission({ kind: "durable" }); await expect(handling).resolves.toBe(true); expect(res.statusCode).toBe(200); + expect(res.headers["x-openclaw-delivery-accepted"]).toBe("durable"); }); it("returns 503 instead of acknowledging when durable admission fails", async () => { @@ -455,6 +459,7 @@ describe("googlechat monitor webhook", () => { const { processEvent, res } = await runWebhookHandler({ authorization: "Bearer valid" }); expect(res.statusCode).toBe(503); + expect(res.headers["x-openclaw-delivery-accepted"]).toBeUndefined(); expect(res.body).toBe("failed to persist event"); expect(processEvent).not.toHaveBeenCalled(); }); diff --git a/extensions/googlechat/src/monitor-webhook.ts b/extensions/googlechat/src/monitor-webhook.ts index b1260780be58..daed628df5c5 100644 --- a/extensions/googlechat/src/monitor-webhook.ts +++ b/extensions/googlechat/src/monitor-webhook.ts @@ -36,6 +36,8 @@ function extractBearerToken(header: unknown): string { const ADD_ON_PREAUTH_MAX_BYTES = 16 * 1024; const ADD_ON_PREAUTH_TIMEOUT_MS = 3_000; +const GOOGLECHAT_WEBHOOK_ACCEPTED_HEADER = "x-openclaw-delivery-accepted"; +const GOOGLECHAT_WEBHOOK_ACCEPTED_VALUE = "durable"; type ParsedGoogleChatInboundSuccess = { raw: Record; @@ -270,6 +272,11 @@ export function createGoogleChatWebhookRequestHandler(params: { }, ); } + if (admission.kind === "durable") { + // Only durably persisted turns claim the marker; ignored non-turn + // actions ack without it (same contract as #104407). + res.setHeader(GOOGLECHAT_WEBHOOK_ACCEPTED_HEADER, GOOGLECHAT_WEBHOOK_ACCEPTED_VALUE); + } } catch (error) { dispatchTarget.runtime.error?.( `[${dispatchTarget.account.accountId}] Google Chat durable admission failed: ${String(error)}`, diff --git a/extensions/nextcloud-talk/src/monitor.durable-ack.test.ts b/extensions/nextcloud-talk/src/monitor.durable-ack.test.ts index 39fd7828f8c6..bf177a2e7d7c 100644 --- a/extensions/nextcloud-talk/src/monitor.durable-ack.test.ts +++ b/extensions/nextcloud-talk/src/monitor.durable-ack.test.ts @@ -24,7 +24,9 @@ describe("Nextcloud Talk durable webhook acknowledgement", () => { await vi.waitFor(() => expect(onWebhook).toHaveBeenCalledTimes(1)); expect(settled).toBe(false); releaseAdmission(); - expect((await request).status).toBe(200); + const response = await request; + expect(response.status).toBe(200); + expect(response.headers.get("x-openclaw-delivery-accepted")).toBe("durable"); }); it("does not acknowledge a failed durable append", async () => { @@ -37,6 +39,18 @@ describe("Nextcloud Talk durable webhook acknowledgement", () => { const { body, headers } = createSignedCreateMessageRequest(); const response = await fetch(harness.webhookUrl, { method: "POST", headers, body }); expect(response.status).toBe(500); + expect(response.headers.get("x-openclaw-delivery-accepted")).toBeNull(); + }); + + it("does not mark ignored webhook events as durable", async () => { + const harness = await startWebhookServer({ + path: "/nextcloud-ignored-event", + onWebhook: vi.fn(async () => "ignored" as const), + }); + const { body, headers } = createSignedCreateMessageRequest(); + const response = await fetch(harness.webhookUrl, { method: "POST", headers, body }); + expect(response.status).toBe(200); + expect(response.headers.get("x-openclaw-delivery-accepted")).toBeNull(); }); it("maps permanent pre-admission payload failures to 400", async () => { @@ -49,6 +63,7 @@ describe("Nextcloud Talk durable webhook acknowledgement", () => { const { body, headers } = createSignedCreateMessageRequest(); const response = await fetch(harness.webhookUrl, { method: "POST", headers, body }); expect(response.status).toBe(400); + expect(response.headers.get("x-openclaw-delivery-accepted")).toBeNull(); expect(await response.json()).toEqual({ error: "Invalid payload format" }); }); }); diff --git a/extensions/nextcloud-talk/src/monitor.ts b/extensions/nextcloud-talk/src/monitor.ts index 4e4d64f81b2a..0893a04938fa 100644 --- a/extensions/nextcloud-talk/src/monitor.ts +++ b/extensions/nextcloud-talk/src/monitor.ts @@ -13,6 +13,8 @@ import { NextcloudTalkWebhookPayloadError } from "./webhook-spool-state.js"; const DEFAULT_WEBHOOK_MAX_BODY_BYTES = 1024 * 1024; const PREAUTH_WEBHOOK_MAX_BODY_BYTES = 64 * 1024; +const NEXTCLOUD_TALK_WEBHOOK_ACCEPTED_HEADER = "x-openclaw-delivery-accepted"; +const NEXTCLOUD_TALK_WEBHOOK_ACCEPTED_VALUE = "durable"; const PREAUTH_WEBHOOK_BODY_TIMEOUT_MS = 5_000; const HEALTH_PATH = "/healthz"; const WEBHOOK_AUTH_RATE_LIMIT_SCOPE = "nextcloud-talk-webhook-auth"; @@ -181,7 +183,15 @@ export function createNextcloudTalkWebhookServer(opts: NextcloudTalkWebhookServe // Nextcloud retries only a few times. Acknowledge only after the raw // envelope is durably admitted; append failure must remain retryable. - await onWebhook(body); + const admission = await onWebhook(body); + if (admission === "accepted") { + // Ignored non-message events still receive 200 but must not claim + // durable adoption. + res.setHeader( + NEXTCLOUD_TALK_WEBHOOK_ACCEPTED_HEADER, + NEXTCLOUD_TALK_WEBHOOK_ACCEPTED_VALUE, + ); + } writeJsonResponse(res, 200); } catch (err) { if (isRequestBodyLimitError(err, "PAYLOAD_TOO_LARGE")) { diff --git a/extensions/sms/src/webhook.test.ts b/extensions/sms/src/webhook.test.ts index 43b9acd37e6c..180450226068 100644 --- a/extensions/sms/src/webhook.test.ts +++ b/extensions/sms/src/webhook.test.ts @@ -160,6 +160,7 @@ describe("createSmsWebhookHandler", () => { await handler(createRequest(body, signature), res); expect(res.statusCode).toBe(200); + expect(res.setHeaderMock).toHaveBeenCalledWith("x-openclaw-delivery-accepted", "durable"); expect(enqueueSmsIngress).toHaveBeenCalledWith(parseTestTwilioForm(body)); }); @@ -178,6 +179,53 @@ describe("createSmsWebhookHandler", () => { ); expect(res.endMock).not.toHaveBeenCalled(); + expect(res.setHeaderMock).not.toHaveBeenCalledWith("x-openclaw-delivery-accepted", "durable"); + }); + + it("acknowledges only after the durable enqueue resolves", async () => { + const { body, signature } = createSignedSmsPayload(createMessageSid(3)); + let releaseAdmission: (() => void) | undefined; + enqueueSmsIngress.mockImplementationOnce( + async () => + await new Promise<{ kind: "accepted"; duplicate: boolean }>((resolve) => { + releaseAdmission = () => resolve({ kind: "accepted", duplicate: false }); + }), + ); + const handler = createSmsWebhookHandler({ + cfg: {}, + account: createAccount(), + ingress: createIngress(), + }); + const res = createResponse(); + + const handling = handler(createRequest(body, signature), res); + await vi.waitFor(() => expect(enqueueSmsIngress).toHaveBeenCalledTimes(1)); + expect(res.endMock).not.toHaveBeenCalled(); + if (!releaseAdmission) { + throw new Error("expected pending SMS durable admission"); + } + releaseAdmission(); + await handling; + + expect(res.statusCode).toBe(200); + expect(res.setHeaderMock).toHaveBeenCalledWith("x-openclaw-delivery-accepted", "durable"); + expect(res.endMock).toHaveBeenCalledTimes(1); + }); + + it("still acks durable when the enqueue reports a replayed duplicate", async () => { + const { body, signature } = createSignedSmsPayload(createMessageSid(4)); + enqueueSmsIngress.mockResolvedValueOnce({ kind: "accepted", duplicate: true }); + const handler = createSmsWebhookHandler({ + cfg: {}, + account: createAccount(), + ingress: createIngress(), + }); + const res = createResponse(); + + await handler(createRequest(body, signature), res); + + expect(res.statusCode).toBe(200); + expect(res.setHeaderMock).toHaveBeenCalledWith("x-openclaw-delivery-accepted", "durable"); }); it("rejects a signed webhook without a stable MessageSid", async () => { diff --git a/extensions/sms/src/webhook.ts b/extensions/sms/src/webhook.ts index 2735fbac894b..064158982425 100644 --- a/extensions/sms/src/webhook.ts +++ b/extensions/sms/src/webhook.ts @@ -16,6 +16,8 @@ import type { ResolvedSmsAccount } from "./types.js"; const INVALID_REQUEST_MAX_REQUESTS = 300; const CALLBACK_DISPATCH_MAX_REQUESTS = 30; +const SMS_WEBHOOK_ACCEPTED_HEADER = "x-openclaw-delivery-accepted"; +const SMS_WEBHOOK_ACCEPTED_VALUE = "durable"; // Count failed-auth traffic separately from the stricter dispatchable callback quota. // The over-budget decision is applied only after validation fails, so a same-key @@ -143,6 +145,8 @@ export function createSmsWebhookHandler(params: SmsWebhookHandlerParams) { } // Durable admission also reserves the monitor pump under this HTTP request's // detached work root, so the response can acknowledge immediately after commit. + // Duplicates map to the committed row, so replays still ack durable (#104407). + res.setHeader(SMS_WEBHOOK_ACCEPTED_HEADER, SMS_WEBHOOK_ACCEPTED_VALUE); respondTwiml(res, 200); return true; }; diff --git a/extensions/synology-chat/src/test-http-utils.ts b/extensions/synology-chat/src/test-http-utils.ts index 2184212b14da..d295070f4baf 100644 --- a/extensions/synology-chat/src/test-http-utils.ts +++ b/extensions/synology-chat/src/test-http-utils.ts @@ -45,17 +45,29 @@ export function makeStalledReq( return makeBaseReq(method, opts); } -export function makeRes(): ServerResponse & { status: number; body: string } { +export function makeRes(): ServerResponse & { + status: number; + body: string; + headers: Record; +} { const res = { status: 0, body: "", - writeHead(statusCode: number, _headers: Record) { + headers: {} as Record, + setHeader(name: string, value: string) { + res.headers[name.toLowerCase()] = value; + }, + writeHead(statusCode: number, _headers?: Record) { res.status = statusCode; }, end(body?: string) { res.body = body ?? ""; }, - } as unknown as ServerResponse & { status: number; body: string }; + } as unknown as ServerResponse & { + status: number; + body: string; + headers: Record; + }; Object.defineProperty(res, "statusCode", { configurable: true, enumerable: true, diff --git a/extensions/synology-chat/src/webhook-handler.test.ts b/extensions/synology-chat/src/webhook-handler.test.ts index 00748646d023..222f779249b8 100644 --- a/extensions/synology-chat/src/webhook-handler.test.ts +++ b/extensions/synology-chat/src/webhook-handler.test.ts @@ -265,10 +265,12 @@ describe("createWebhookHandler", () => { const pending = handler(makeReq("POST", validBody), res); await vi.waitFor(() => expect(receive).toHaveBeenCalledTimes(1)); expect(res.status).toBe(0); + expect(res.headers["x-openclaw-delivery-accepted"]).toBeUndefined(); resolveAdmission?.({ kind: "durable" }); await pending; expect(res.status).toBe(204); + expect(res.headers["x-openclaw-delivery-accepted"]).toBe("durable"); }); it("returns 503 without acknowledging when durable admission fails", async () => { @@ -283,6 +285,7 @@ describe("createWebhookHandler", () => { await handler(makeReq("POST", validBody), res); expect(res.status).toBe(503); + expect(res.headers["x-openclaw-delivery-accepted"]).toBeUndefined(); expect(res.body).toContain("Webhook admission failed"); }); diff --git a/extensions/synology-chat/src/webhook-handler.ts b/extensions/synology-chat/src/webhook-handler.ts index b9ab17d6dc2e..20546833cfe0 100644 --- a/extensions/synology-chat/src/webhook-handler.ts +++ b/extensions/synology-chat/src/webhook-handler.ts @@ -354,6 +354,9 @@ function parsePayload( }; } +const SYNOLOGY_WEBHOOK_ACCEPTED_HEADER = "x-openclaw-delivery-accepted"; +const SYNOLOGY_WEBHOOK_ACCEPTED_VALUE = "durable"; + /** Send a JSON response. */ function respondJson(res: ServerResponse, statusCode: number, body: Record) { res.writeHead(statusCode, { "Content-Type": "application/json" }); @@ -684,6 +687,9 @@ export function createWebhookHandler(deps: WebhookHandlerDeps) { respondJson(res, 400, { error: admitted.message }); return; } + // Only a durably admitted event is acknowledged here; mark the ack so + // proxies can distinguish it from other responses (same marker as #104407). + res.setHeader(SYNOLOGY_WEBHOOK_ACCEPTED_HEADER, SYNOLOGY_WEBHOOK_ACCEPTED_VALUE); respondNoContent(res); }; } diff --git a/extensions/zalo/src/monitor.webhook.test.ts b/extensions/zalo/src/monitor.webhook.test.ts index 53d7420d84ae..76e09a58ede8 100644 --- a/extensions/zalo/src/monitor.webhook.test.ts +++ b/extensions/zalo/src/monitor.webhook.test.ts @@ -218,7 +218,9 @@ describe("handleZaloWebhookRequest", () => { await vi.waitFor(() => expect(acceptWebhook).toHaveBeenCalledTimes(1)); expect(settled).toBe(false); releaseAdmission(); - expect((await responsePromise).status).toBe(200); + const response = await responsePromise; + expect(response.status).toBe(200); + expect(response.headers.get("x-openclaw-delivery-accepted")).toBe("durable"); }); } finally { releaseAdmission(); @@ -256,6 +258,7 @@ describe("handleZaloWebhookRequest", () => { body: '{"event_name":"message.text.received"}', }); expect(response.status).toBe(500); + expect(response.headers.get("x-openclaw-delivery-accepted")).toBeNull(); }); } finally { unregister(); diff --git a/extensions/zalo/src/monitor.webhook.ts b/extensions/zalo/src/monitor.webhook.ts index e7ad4c937674..6b83c7f7fb29 100644 --- a/extensions/zalo/src/monitor.webhook.ts +++ b/extensions/zalo/src/monitor.webhook.ts @@ -30,6 +30,9 @@ type ZaloWebhookTarget = { acceptWebhook: (rawEvent: string) => Promise; }; +const ZALO_WEBHOOK_ACCEPTED_HEADER = "x-openclaw-delivery-accepted"; +const ZALO_WEBHOOK_ACCEPTED_VALUE = "durable"; + const webhookTargets = new Map(); const webhookRateLimiter = createFixedWindowRateLimiter({ windowMs: WEBHOOK_RATE_LIMIT_DEFAULTS.windowMs, @@ -177,6 +180,9 @@ async function handleZaloWebhookRequest( return true; } + // The spool persisted the envelope above; mark the ack as durable so + // proxies can distinguish it from other 200s (same marker as #104407). + res.setHeader(ZALO_WEBHOOK_ACCEPTED_HEADER, ZALO_WEBHOOK_ACCEPTED_VALUE); res.statusCode = 200; res.end("ok"); return true;