diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc50c9c027a6..98d4c0cefa0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2435,7 +2435,11 @@ jobs: fi pnpm tool-display:check pnpm check:host-env-policy:swift - pnpm dup:check + if [[ "$FROZEN_TARGET" == "true" ]]; then + pnpm dup:check:coverage + else + pnpm dup:check + fi if has_package_script "check:coercion-helpers"; then pnpm check:coercion-helpers elif [[ "$HISTORICAL_TARGET" == "true" ]]; then diff --git a/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json b/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json index ff57f49e1c7d..8a178ae0c44e 100644 --- a/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json @@ -1 +1 @@ -{"contentHash":"a35378b670434316ba33a3ba4e595925be685daee9fb0f3c5989983525b18c50","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"} +{"contentHash":"812e818c8d7c013b2287c4502426123222037a4235e130a69481272912062866","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/agent-harness.json b/docs/.generated/plugin-sdk-api-baseline/agent-harness.json index 5cf1ada7b53e..3b4e4add6542 100644 --- a/docs/.generated/plugin-sdk-api-baseline/agent-harness.json +++ b/docs/.generated/plugin-sdk-api-baseline/agent-harness.json @@ -1 +1 @@ -{"contentHash":"b9c074a11791688e63213f51099ea21745b0e554f63ed13e694ea34a3e4328ac","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"} +{"contentHash":"f639fea8b8ee53626452bdbce156724b09a3a29bb05a134eb8e8f8fb8062d4da","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-core.json b/docs/.generated/plugin-sdk-api-baseline/channel-core.json index 31b85bb7faaa..dbcb1039396a 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-core.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-core.json @@ -1 +1 @@ -{"contentHash":"d8b7c867b29b4651375455e938ae634e7213e3e8c17444dc2b0ef091e46fda6c","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"} +{"contentHash":"02db4abe2f1f4578d438f7afacd68d44114e46b48b9582711e7e10191353e6c5","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json b/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json index 94ceeb3194f9..45b7d50451ab 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json @@ -1 +1 @@ -{"contentHash":"883bbab77caeff122bf8b7505dc9299484fba601e285a42d2c457697b7590b8e","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"} +{"contentHash":"6826237a93cc52b6039fddbd4b4e0e00e82b5c83cbadd36a055a50ab0480879c","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-inbound.json b/docs/.generated/plugin-sdk-api-baseline/channel-inbound.json index 31bd2c3e0d0d..0c74fb7bdaa4 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-inbound.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-inbound.json @@ -1 +1 @@ -{"contentHash":"a10b2b86f7fec9a99f951656b882414b73d04b5e16313bf0747e883da50c37df","entrypoint":"channel-inbound","importSpecifier":"openclaw/plugin-sdk/channel-inbound"} +{"contentHash":"e0eefe9ad871dbfe2bb8dac626e44ecfcd55c8b35667e9fd041799f2f45ea6ba","entrypoint":"channel-inbound","importSpecifier":"openclaw/plugin-sdk/channel-inbound"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-ingress-runtime.json b/docs/.generated/plugin-sdk-api-baseline/channel-ingress-runtime.json index 059ed0270106..0116d09a1ec1 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-ingress-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-ingress-runtime.json @@ -1 +1 @@ -{"contentHash":"5ea7b454922c17117a2a34aae8f2bfd9a45666a7268dff223063bde72b379aff","entrypoint":"channel-ingress-runtime","importSpecifier":"openclaw/plugin-sdk/channel-ingress-runtime"} +{"contentHash":"6f32863c74ad4ea076404754618d38015ad48281964f1fc24692aae8cd0fcad6","entrypoint":"channel-ingress-runtime","importSpecifier":"openclaw/plugin-sdk/channel-ingress-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-message.json b/docs/.generated/plugin-sdk-api-baseline/channel-message.json index 205deefc917c..e74031a0e4a1 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-message.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-message.json @@ -1 +1 @@ -{"contentHash":"2f85610859f047db1d76efbe52d9e63d8126ce8e944d53d6fdafb4e7f503b5dc","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"} +{"contentHash":"bc8881a906f40f0a3ede29eb83efc1e4d3e59b62c00154b2480d85d69cbe4010","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json b/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json index f1ffe9571fdc..dd24f428bb5a 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json @@ -1 +1 @@ -{"contentHash":"6b1f575e77dc71e72c4b2575f366e7e9990d391e85364844e0ecec2146e7f902","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"} +{"contentHash":"f4b35d03ac9df9788462f3e50b64819ff85aba0245b1a51208a288bd994edb8e","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-pairing.json b/docs/.generated/plugin-sdk-api-baseline/channel-pairing.json index 89e60b0ff9fe..13dd9510f1a9 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-pairing.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-pairing.json @@ -1 +1 @@ -{"contentHash":"d92b60c0b7109446db7d24172281dfd6f79196d6428b8e6fd59e99a3d4d9f016","entrypoint":"channel-pairing","importSpecifier":"openclaw/plugin-sdk/channel-pairing"} +{"contentHash":"d1c4d7478b9dcb6c4567cb599839c84b843e005ff39f5d31bc317f26d6745853","entrypoint":"channel-pairing","importSpecifier":"openclaw/plugin-sdk/channel-pairing"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json b/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json index a6647ae1cf97..c777e8b2845b 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json @@ -1 +1 @@ -{"contentHash":"72d713de6cb87ad8e6d9c49456477367a2d30c8157695ddb66ac57919558272b","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"} +{"contentHash":"80b5ab5fedd16c952c83747f4093db46e49640b3cfa1fdaa5a9727ff31edf0a7","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"} diff --git a/docs/.generated/plugin-sdk-api-baseline/core.json b/docs/.generated/plugin-sdk-api-baseline/core.json index f0957d7f1ba9..ca65ae570cac 100644 --- a/docs/.generated/plugin-sdk-api-baseline/core.json +++ b/docs/.generated/plugin-sdk-api-baseline/core.json @@ -1 +1 @@ -{"contentHash":"2ccfe8eb85378f3be3f78f79f2638b00faa943e351ff69464bfed44ed63e09c1","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"} +{"contentHash":"edfa6b7a219aef521935ac17a20c5cd74e91bf4988eb3fad5ac75fc2e5015596","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"} diff --git a/docs/.generated/plugin-sdk-api-baseline/discord.json b/docs/.generated/plugin-sdk-api-baseline/discord.json index c9fa00676e85..edeb54f4fba1 100644 --- a/docs/.generated/plugin-sdk-api-baseline/discord.json +++ b/docs/.generated/plugin-sdk-api-baseline/discord.json @@ -1 +1 @@ -{"contentHash":"d23cd5df763235dbe35ede1c23d26735f4c183b03dc216ac212aefe75c0c1f96","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"} +{"contentHash":"6490014377554ea6b62c657ab22c53fcec385b4ad24cdfccd95ebd9a79717e59","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"} diff --git a/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json b/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json index ccf2daaf63e6..78798cf75996 100644 --- a/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json +++ b/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json @@ -1 +1 @@ -{"contentHash":"f57a244e027b326090ed91cdc105c0c8c83b24318d99be0ec205fffd33504050","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"} +{"contentHash":"bb5122c6ac5f4dfe381493b9d128a303108434edca9434ad5528781d005271b8","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"} diff --git a/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json b/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json index a4180100b928..2f8591069a4b 100644 --- a/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json @@ -1 +1 @@ -{"contentHash":"5008615c80d86383e18527d71442f1501e4f02bdb3faa1494a48edf40fd66f84","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"} +{"contentHash":"9060d4011e1249ca8aa0b01f2e2825d71b29eff14dc84f7f702440dc01e73460","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/plugin-command-runtime.json b/docs/.generated/plugin-sdk-api-baseline/plugin-command-runtime.json index 81a60130d47d..349b5da67fc6 100644 --- a/docs/.generated/plugin-sdk-api-baseline/plugin-command-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/plugin-command-runtime.json @@ -1 +1 @@ -{"contentHash":"be5d21b9760636dd2956ac64b786213532e74ebb4cd1cc844a923b7bd5949be7","entrypoint":"plugin-command-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-command-runtime"} +{"contentHash":"45c491f92e76d0523e2fafc47f362620e2598a5fdef95aed50ce4e8ff108423d","entrypoint":"plugin-command-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-command-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json b/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json index d7bd00e6e002..eb8a21e5eb31 100644 --- a/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json +++ b/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json @@ -1 +1 @@ -{"contentHash":"6b73741c889f68cc19a5d1f8703df2eca650fe57803795045cff122d746cbb6a","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"} +{"contentHash":"a36c13699a318b3fb1b6701ab3280c45e5a4f5ca982cef29778d10af8a5ae97a","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"} diff --git a/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json b/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json index b8e9d32eb33e..b78837d1e262 100644 --- a/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json @@ -1 +1 @@ -{"contentHash":"15704fc50b81cebceb677a5a69a7bc8c97afd962081c647de58a027bd816f19d","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"} +{"contentHash":"bdf0b57a425cac872d156021006cce6813893bee30e77c5ef4c055343697dba0","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json b/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json index 3153f34476f5..084ac522181e 100644 --- a/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json @@ -1 +1 @@ -{"contentHash":"df8043d0630d95bc4c316da28bf675a713e7f1f0c304959d1b4b9c8b30fcbd0d","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"} +{"contentHash":"62f6dc31a2b1789b3667fe16c2d465681aa4c01e2cd9b2a6da062fd995235963","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/runtime-store.json b/docs/.generated/plugin-sdk-api-baseline/runtime-store.json index 3756a620df98..1069301d29d8 100644 --- a/docs/.generated/plugin-sdk-api-baseline/runtime-store.json +++ b/docs/.generated/plugin-sdk-api-baseline/runtime-store.json @@ -1 +1 @@ -{"contentHash":"01918241de10b9b625379aa143ea2b3bee3ebf53369dde648c3402227d3cb5f7","entrypoint":"runtime-store","importSpecifier":"openclaw/plugin-sdk/runtime-store"} +{"contentHash":"b33bfa1835dbfedd3fef5d48720d57cf166fd60dd21727f1c18c49162eb2ab87","entrypoint":"runtime-store","importSpecifier":"openclaw/plugin-sdk/runtime-store"} diff --git a/docs/.generated/plugin-sdk-api-baseline/session-catalog.json b/docs/.generated/plugin-sdk-api-baseline/session-catalog.json index ce235d30f317..1adceaef9175 100644 --- a/docs/.generated/plugin-sdk-api-baseline/session-catalog.json +++ b/docs/.generated/plugin-sdk-api-baseline/session-catalog.json @@ -1 +1 @@ -{"contentHash":"7bae1f99dfbd5920f9fa2e82abfefdf11e28c3150f15b22ee3fcf115a4dcef19","entrypoint":"session-catalog","importSpecifier":"openclaw/plugin-sdk/session-catalog"} +{"contentHash":"b8218ab6c7789147d0ced29d4dfa1487af511a2a91f12808587ee97c702947c9","entrypoint":"session-catalog","importSpecifier":"openclaw/plugin-sdk/session-catalog"} diff --git a/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json b/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json index b48deb66cf47..261089d57bc7 100644 --- a/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json +++ b/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json @@ -1 +1 @@ -{"contentHash":"8b0c71c8abe13163eee28ca65e6afb490d0ae3c16288292143ad021975b96ca6","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"} +{"contentHash":"3662752cd7db434787d728355a4fb46e8f4be5b88dece9c29d96225bc02da6db","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"} diff --git a/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json b/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json index bb5a15cdc566..b6f15aef72f9 100644 --- a/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json +++ b/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json @@ -1 +1 @@ -{"contentHash":"eb0096fd25551a7f4a58dcf6c6b53120f3f1ddc1248b2b962cca60c75f6daa12","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"} +{"contentHash":"66ab292503af6befc63d5962f3312a8ebab4ab3bf3a56f47fe4f6a465d7c40b9","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"} diff --git a/docs/plan/runners.md b/docs/plan/runners.md index ddb4840137b5..84bdfa6649ad 100644 --- a/docs/plan/runners.md +++ b/docs/plan/runners.md @@ -13,22 +13,22 @@ Proposal, revision 2. Supersedes revision 1 in place (2026-08-11, operator decision). Implementation in progress; update this table in every PR that advances a milestone. -| # | Milestone | Status | PRs | -| --- | ---------------------------------------------------------- | ----------- | ------------------------------------------- | -| 0 | This plan (revision 2) | landed | #122454 | -| 1a | Naming: session copy revert | landed | #120667 | -| 1b | Naming: devices consolidation | landed | #120689 | -| 1c | Cleanup: node-pairing → device-pairing merge | landed | #120726 | -| 2 | `openclaw resume` + web Continue in terminal | in progress | #120664, #122870 | -| 3 | `openclaw connect` one-paste onboarding + `/j/` join route | in progress | #120768, #122499 | -| 4 | Picker: grouping, placement, liveness, enrichment | in progress | #120804, #122531, #122635, #122774, #122923 | -| F | Real-wire session boundary harness | landed | #121212 | -| 5 | Public worker ingress path | landed | #122578, #122643 | -| 6 | Node worker provider (device runners) | in progress | #122683, #122769, #122829, #122939, #123013 | -| 7 | Bundle push consent + runner updates | not started | — | -| 8 | Stop-and-continue moves | not started | — | -| 9 | Deletions (ssh sandbox, openshell, exec-host clones, …) | not started | — | -| 10 | Cloud convergence (provisioners run `openclaw connect`) | not started | — | +| # | Milestone | Status | PRs | +| --- | ---------------------------------------------------------- | ----------- | ---------------------------------------------------- | +| 0 | This plan (revision 2) | landed | #122454 | +| 1a | Naming: session copy revert | landed | #120667 | +| 1b | Naming: devices consolidation | landed | #120689 | +| 1c | Cleanup: node-pairing → device-pairing merge | landed | #120726 | +| 2 | `openclaw resume` + web Continue in terminal | in progress | #120664, #122870 | +| 3 | `openclaw connect` one-paste onboarding + `/j/` join route | in progress | #120768, #122499 | +| 4 | Picker: grouping, placement, liveness, enrichment | in progress | #120804, #122531, #122635, #122774, #122923 | +| F | Real-wire session boundary harness | landed | #121212 | +| 5 | Public worker ingress path | landed | #122578, #122643 | +| 6 | Node worker provider (device runners) | in progress | #122683, #122769, #122829, #122939, #123013, #123033 | +| 7 | Bundle push consent + runner updates | not started | — | +| 8 | Stop-and-continue moves | not started | — | +| 9 | Deletions (ssh sandbox, openshell, exec-host clones, …) | not started | — | +| 10 | Cloud convergence (provisioners run `openclaw connect`) | not started | — | Revision history: revision 1 (2026-08-08) established the session/runner vocabulary, the naming rulings, and the milestone skeleton after a diff --git a/docs/plugins/sdk-channel-outbound.md b/docs/plugins/sdk-channel-outbound.md index d701b7be096b..7ce7198e442e 100644 --- a/docs/plugins/sdk-channel-outbound.md +++ b/docs/plugins/sdk-channel-outbound.md @@ -47,13 +47,15 @@ the transport callback instead of dispatching an event that was not made durable. At claim time it decodes the versioned payload, re-runs `inspect`, and rejects an id or lane mismatch before delivery. -`deliver` receives `onAdopted`, `onDeferred`, `onAdoptionFinalizing`, -`onAbandoned`, and `abortSignal`. Returning without an explicit handoff marks a -terminal no-dispatch event adopted. `admission` is always `exclusive`. A -deferred handoff keeps the claim held, while shutdown or abort leaves unadopted -work retryable. The monitor tracks delivery independently from claim settlement -because adoption can tombstone a row before the channel's delivery promise -returns. +`deliver` receives `onAdopted`, `onDeferred`, `onAdoptionFinalizing`, `onFailed`, +`onCancelled`, `onAbandoned`, and `abortSignal`. Use `onFailed` for delivery +errors, `onCancelled` for explicit pre-adoption cancellation that must preserve +retry accounting, and `onAbandoned` when a non-adopted turn should consume a +retry attempt. Returning without an explicit handoff marks a terminal +no-dispatch event adopted. `admission` is always `exclusive`. A deferred handoff +keeps the claim held, while shutdown or abort leaves unadopted work retryable. +The monitor tracks delivery independently from claim settlement because +adoption can tombstone a row before the channel's delivery promise returns. Optional settings include custom append delays, a `drain` option block for advanced drain ordering/concurrency/retry policy, an external `abortSignal`, a diff --git a/extensions/discord/src/monitor/inbound-job.test.ts b/extensions/discord/src/monitor/inbound-job.test.ts index 1ac1e8a5ce1f..636969d18be3 100644 --- a/extensions/discord/src/monitor/inbound-job.test.ts +++ b/extensions/discord/src/monitor/inbound-job.test.ts @@ -123,6 +123,7 @@ describe("buildDiscordInboundJob", () => { const ingressSettlement = { settle: vi.fn(async () => {}), abandon: vi.fn(async () => {}), + cancel: vi.fn(async () => {}), }; const job = buildDiscordInboundJob(ctx, { ingressSettlement }); const overrideAbortController = new AbortController(); diff --git a/extensions/discord/src/monitor/inbound-job.ts b/extensions/discord/src/monitor/inbound-job.ts index f62a68410ffa..94bb395dee5e 100644 --- a/extensions/discord/src/monitor/inbound-job.ts +++ b/extensions/discord/src/monitor/inbound-job.ts @@ -26,6 +26,7 @@ export type DiscordInboundJob = { ingressSettlement?: { settle: () => Promise; abandon: (error?: unknown) => Promise; + cancel: () => Promise; }; }; diff --git a/extensions/discord/src/monitor/ingress.test.ts b/extensions/discord/src/monitor/ingress.test.ts index 18ab32ade29e..ccf7e6fee135 100644 --- a/extensions/discord/src/monitor/ingress.test.ts +++ b/extensions/discord/src/monitor/ingress.test.ts @@ -116,6 +116,31 @@ describe("Discord durable ingress", () => { }); }); + it("rejects unstable message identity before durable allocation", async () => { + await withQueue(async (queue) => { + const dispatch = vi.fn(); + const monitor = createDiscordIngressMonitor({ + accountId: "default", + client: {} as never, + runtime: runtime(), + queue, + dispatch, + }); + monitor.start(); + try { + const missingMessageId = { ...createRawMessage("missing"), id: undefined }; + const missingChannelId = { ...createRawMessage("missing"), channel_id: undefined }; + + await expect(monitor.accept(missingMessageId as never)).rejects.toThrow("snowflake"); + await expect(monitor.accept(missingChannelId as never)).rejects.toThrow("channel_id"); + expect(await queue.listPending({ limit: "all" })).toEqual([]); + expect(dispatch).not.toHaveBeenCalled(); + } finally { + await monitor.stop(); + } + }); + }); + it("recovers a claimed row with a fresh drain and dispatches it exactly once", async () => { await withQueue(async (queue) => { const monitors: DiscordIngressMonitor[] = []; diff --git a/extensions/discord/src/monitor/ingress.ts b/extensions/discord/src/monitor/ingress.ts index c5052bf539bc..a2c438a77774 100644 --- a/extensions/discord/src/monitor/ingress.ts +++ b/extensions/discord/src/monitor/ingress.ts @@ -3,6 +3,7 @@ import { GatewayDispatchEvents, type APIMessage } from "discord-api-types/v10"; import { createChannelIngressError, createChannelIngressMonitor, + DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS, type ChannelIngressQueue, type ChannelIngressMonitorDeliveryResult, type ChannelIngressMonitorLifecycle, @@ -147,6 +148,10 @@ export function createDiscordIngressMonitor(params: { }, appendRetryDelaysMs: [0], drain: { + retryPolicy: { + maxAttempts: DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS, + deadLetterMinAgeMs: 0, + }, resolveNonRetryableFailure: (error) => { if (error instanceof DiscordIngressPayloadError) { return { reason: "invalid-event", message: error.message }; diff --git a/extensions/discord/src/monitor/message-dispatcher.ts b/extensions/discord/src/monitor/message-dispatcher.ts index 32d284409c83..ed534ba09ef3 100644 --- a/extensions/discord/src/monitor/message-dispatcher.ts +++ b/extensions/discord/src/monitor/message-dispatcher.ts @@ -139,7 +139,7 @@ export function createDiscordMessageDispatcher( } const abortSignal = last.abortSignal; if (abortSignal?.aborted) { - await admissionLifecycle.onAbandoned(); + await ingress.cancel(); return; } try { @@ -157,7 +157,7 @@ export function createDiscordMessageDispatcher( turnAdoptionLifecycle: admissionLifecycle, }); if (abortSignal?.aborted) { - await ingress.abandon(abortSignal.reason); + await ingress.cancel(); return; } if (!ctx) { @@ -211,7 +211,7 @@ export function createDiscordMessageDispatcher( turnAdoptionLifecycle: admissionLifecycle, }); if (abortSignal?.aborted) { - await ingress.abandon(abortSignal.reason); + await ingress.cancel(); return; } if (!ctx) { @@ -232,7 +232,10 @@ export function createDiscordMessageDispatcher( } messageRunQueue.enqueue(buildDiscordInboundJob(ctx, { ingressSettlement: ingress })); } catch (error) { - await admissionLifecycle.onAbandoned(); + if (abortSignal?.aborted) { + await ingress.cancel(); + return; + } throw error; } }, @@ -244,7 +247,8 @@ export function createDiscordMessageDispatcher( onCancel: (entries) => { for (const entry of entries) { pendingDebounceEntries.delete(entry); - const settlement = Promise.resolve(entry.turnAdoptionLifecycle?.onAbandoned()) + const settlement = fanInChannelIngressLifecycles([entry.turnAdoptionLifecycle]) + .cancel() .catch((error: unknown) => { params.runtime.error( danger(`discord ingress cancellation settlement failed: ${String(error)}`), @@ -271,6 +275,10 @@ export function createDiscordMessageDispatcher( const reason = dispatcherShutdown.signal.aborted ? (dispatcherShutdown.signal.reason ?? new Error("discord dispatcher shut down")) : (options?.abortSignal?.reason ?? new Error("discord dispatch aborted")); + if (options?.turnAdoptionLifecycle) { + await fanInChannelIngressLifecycles([options.turnAdoptionLifecycle]).cancel(); + return { kind: "deferred" }; + } return { kind: "failed-retryable", error: reason }; } // Filter bot-own messages before they enter the debounce queue. diff --git a/extensions/discord/src/monitor/message-handler.ingress-recovery.test.ts b/extensions/discord/src/monitor/message-handler.ingress-recovery.test.ts new file mode 100644 index 000000000000..7119e8b62e49 --- /dev/null +++ b/extensions/discord/src/monitor/message-handler.ingress-recovery.test.ts @@ -0,0 +1,263 @@ +// Discord tests cover durable retry recovery through full handler replacement. +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { APIMessage } from "discord-api-types/v10"; +import { + type ChannelIngressQueue, + DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS, +} from "openclaw/plugin-sdk/channel-outbound"; +import { createDeferred } from "openclaw/plugin-sdk/extension-shared"; +import { + closeOpenClawStateDatabaseForTest, + createChannelIngressQueueForTests, +} from "openclaw/plugin-sdk/plugin-state-test-runtime"; +import { describe, expect, it, vi } from "vitest"; +import { createDiscordIngressMonitor } from "./ingress.js"; +import { createDiscordMessageHandler } from "./message-handler.js"; +import { createDiscordHandlerParams } from "./message-handler.test-helpers.js"; + +type DiscordIngressPayload = { + version: 1; + receivedAt: number; + rawMessage: APIMessage; +}; + +type DiscordQueue = ChannelIngressQueue; + +function rawMessage(id: string, channelId = "lane-a"): APIMessage { + return { + id, + channel_id: channelId, + content: "hello", + author: { + id: "user-1", + username: "alice", + discriminator: "0", + avatar: null, + }, + attachments: [], + embeds: [], + mentions: [], + mention_roles: [], + mention_everyone: false, + timestamp: new Date(0).toISOString(), + edited_timestamp: null, + components: [], + pinned: false, + type: 0, + tts: false, + } as unknown as APIMessage; +} + +async function withQueue(run: (queue: DiscordQueue) => Promise): Promise { + const created = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-discord-recovery-")); + const stateDir = await fs.realpath(created); + const queue = createChannelIngressQueueForTests({ + channelId: "discord", + accountId: "default", + stateDir, + }); + try { + await run(queue); + } finally { + closeOpenClawStateDatabaseForTest(); + await fs.rm(stateDir, { recursive: true, force: true }); + } +} + +async function seedPendingFailure(params: { + queue: DiscordQueue; + id: string; + attempts: number; + laneKey?: string; +}): Promise { + await params.queue.enqueue( + params.id, + { version: 1, receivedAt: 1, rawMessage: rawMessage(params.id) }, + { laneKey: params.laneKey ?? "channel:lane-a", receivedAt: 1 }, + ); + for (let attempt = 1; attempt <= params.attempts; attempt += 1) { + const claim = await params.queue.claim(params.id, { ownerId: `seed-${attempt}` }); + if (!claim) { + throw new Error(`Expected ${params.id} to be claimable for seed attempt ${attempt}`); + } + await params.queue.release(claim, { + lastError: `prior genuine failure ${attempt}`, + releasedAt: 10 + attempt, + }); + } +} + +async function retryFacts(queue: DiscordQueue, id: string) { + const record = (await queue.listPending({ limit: "all" })).find((entry) => entry.id === id); + if (!record) { + throw new Error(`Expected pending Discord ingress row ${id}`); + } + return { + attempts: record.attempts, + lastAttemptAt: record.lastAttemptAt, + lastError: record.lastError, + }; +} + +function createHandler(params: { + queue: DiscordQueue; + preflight: (input: { data: { message?: { id?: string } } }) => Promise; + debounceMs?: number; + beforeDispatch?: () => Promise; +}) { + const handlerParams = createDiscordHandlerParams(); + handlerParams.cfg.messages = { inbound: { debounceMs: params.debounceMs ?? 0 } }; + return createDiscordMessageHandler({ + ...handlerParams, + client: {} as never, + testing: { + preflightDiscordMessage: params.preflight as never, + createIngressMonitor: (monitorParams) => + createDiscordIngressMonitor({ + ...monitorParams, + queue: params.queue, + dispatch: params.beforeDispatch + ? async (event, lifecycle) => { + await params.beforeDispatch?.(); + return await monitorParams.dispatch(event, lifecycle); + } + : monitorParams.dispatch, + }), + }, + }); +} + +describe("Discord durable ingress replacement recovery", () => { + it("terminally settles a preexisting exhausted poison row before its follower", async () => { + await withQueue(async (queue) => { + await seedPendingFailure({ + queue, + id: "poison", + attempts: DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS, + }); + await queue.enqueue( + "follower", + { version: 1, receivedAt: 2, rawMessage: rawMessage("follower") }, + { laneKey: "channel:lane-a", receivedAt: 2 }, + ); + const dispatched: string[] = []; + const handler = createHandler({ + queue, + preflight: vi.fn(async ({ data }) => { + const id = data.message?.id ?? "unknown"; + dispatched.push(id); + if (id === "poison") { + throw new Error("recovered poison failure"); + } + return null; + }), + }); + try { + await vi.waitFor(async () => { + await expect(queue.enqueue("poison", {} as DiscordIngressPayload)).resolves.toMatchObject( + { kind: "failed", record: { reason: "retry-limit-exceeded" } }, + ); + await expect( + queue.enqueue("follower", {} as DiscordIngressPayload), + ).resolves.toMatchObject({ kind: "completed" }); + }); + expect(dispatched).toEqual(["poison", "follower"]); + expect(await queue.listPending({ limit: "all" })).toEqual([]); + expect(await queue.listClaims()).toEqual([]); + } finally { + await handler.deactivate(); + } + }); + }); + + it("preserves retry facts across every Discord cancellation route and replacement", async () => { + await withQueue(async (queue) => { + await seedPendingFailure({ + queue, + id: "poison", + attempts: DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS - 1, + }); + await queue.enqueue( + "follower", + { version: 1, receivedAt: 2, rawMessage: rawMessage("follower") }, + { laneKey: "channel:lane-a", receivedAt: 2 }, + ); + const expectedFacts = await retryFacts(queue, "poison"); + + const dispatchEntered = createDeferred(); + const releaseDispatch = createDeferred(); + const beforeDispatch = async () => { + dispatchEntered.resolve(); + await releaseDispatch.promise; + }; + const beforeDispatchPreflight = vi.fn(async () => null); + const beforeDispatchHandler = createHandler({ + queue, + preflight: beforeDispatchPreflight, + beforeDispatch, + }); + await dispatchEntered.promise; + const beforeDispatchStop = beforeDispatchHandler.deactivate(); + await Promise.resolve(); + releaseDispatch.resolve(); + await beforeDispatchStop; + expect(beforeDispatchPreflight).not.toHaveBeenCalled(); + expect(await retryFacts(queue, "poison")).toEqual(expectedFacts); + + const bufferedPreflight = vi.fn(async () => null); + const bufferedHandler = createHandler({ + queue, + preflight: bufferedPreflight, + debounceMs: 60_000, + }); + await vi.waitFor(async () => expect(await queue.listClaims()).toHaveLength(1)); + await bufferedHandler.deactivate(); + expect(bufferedPreflight).not.toHaveBeenCalled(); + expect(await retryFacts(queue, "poison")).toEqual(expectedFacts); + + const preflightEntered = createDeferred(); + const releasePreflight = createDeferred(); + const activePreflight = vi.fn(async () => { + preflightEntered.resolve(); + await releasePreflight.promise; + return null; + }); + const activeHandler = createHandler({ queue, preflight: activePreflight }); + await preflightEntered.promise; + const activeStop = activeHandler.deactivate(); + await Promise.resolve(); + releasePreflight.resolve(); + await activeStop; + expect(activePreflight).toHaveBeenCalledTimes(1); + expect(await retryFacts(queue, "poison")).toEqual(expectedFacts); + + const finalDispatches: string[] = []; + const replacement = createHandler({ + queue, + preflight: vi.fn(async ({ data }) => { + const id = data.message?.id ?? "unknown"; + finalDispatches.push(id); + if (id === "poison") { + throw new Error("final genuine failure"); + } + return null; + }), + }); + try { + await vi.waitFor(async () => { + await expect(queue.enqueue("poison", {} as DiscordIngressPayload)).resolves.toMatchObject( + { kind: "failed", record: { reason: "retry-limit-exceeded" } }, + ); + await expect( + queue.enqueue("follower", {} as DiscordIngressPayload), + ).resolves.toMatchObject({ kind: "completed" }); + }); + expect(finalDispatches).toEqual(["poison", "follower"]); + } finally { + await replacement.deactivate(); + } + }); + }); +}); diff --git a/extensions/discord/src/monitor/message-handler.queue.test.ts b/extensions/discord/src/monitor/message-handler.queue.test.ts index d6c66a8baef2..ff8688fc70ff 100644 --- a/extensions/discord/src/monitor/message-handler.queue.test.ts +++ b/extensions/discord/src/monitor/message-handler.queue.test.ts @@ -1,19 +1,35 @@ // Discord tests cover message handler.queue plugin behavior. import { getEventListeners } from "node:events"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { APIMessage } from "discord-api-types/v10"; +import { fanInChannelIngressLifecycles } from "openclaw/plugin-sdk/channel-ingress-runtime"; +import { + type ChannelIngressQueue, + DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS, +} from "openclaw/plugin-sdk/channel-outbound"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { createDeferred } from "openclaw/plugin-sdk/extension-shared"; +import { + closeOpenClawStateDatabaseForTest, + createChannelIngressQueueForTests, +} from "openclaw/plugin-sdk/plugin-state-test-runtime"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { DiscordIngressLifecycle } from "./ingress.js"; +import { buildDiscordInboundJob } from "./inbound-job.js"; +import { createDiscordIngressMonitor, type DiscordIngressLifecycle } from "./ingress.js"; import { createDiscordMessageHandler as createDurableDiscordMessageHandler } from "./message-handler.js"; import { createDiscordMessageHandler, preflightDiscordMessageMock, processDiscordMessageMock, } from "./message-handler.module-test-helpers.js"; +import { createBaseDiscordMessageContext } from "./message-handler.test-harness.js"; import { createDiscordHandlerParams, createDiscordPreflightContext, } from "./message-handler.test-helpers.js"; +import { createDiscordMessageRunQueue } from "./message-run-queue.js"; type SetStatusFn = (patch: Record) => void; type MockCallSource = { mock: { calls: Array> } }; @@ -35,6 +51,8 @@ function expectStatusPatch(setStatus: MockCallSource, expected: Record; + onFailed: ReturnType; + onCancelled: ReturnType; onAbandoned: ReturnType; } { return { @@ -42,10 +60,61 @@ function createIngressLifecycle(): DiscordIngressLifecycle & { onAdopted: vi.fn(async () => {}), onDeferred: vi.fn(), onAdoptionFinalizing: vi.fn(), + onFailed: vi.fn(async () => {}), + onCancelled: vi.fn(async () => {}), onAbandoned: vi.fn(async () => {}), }; } +type DiscordIngressPayload = { + version: 1; + receivedAt: number; + rawMessage: APIMessage; +}; + +async function withDiscordQueue( + run: (queue: ChannelIngressQueue) => Promise, +): Promise { + const created = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-discord-handler-")); + const stateDir = await fs.realpath(created); + const queue = createChannelIngressQueueForTests({ + channelId: "discord", + accountId: "default", + stateDir, + }); + try { + return await run(queue); + } finally { + closeOpenClawStateDatabaseForTest(); + await fs.rm(stateDir, { recursive: true, force: true }); + } +} + +function createRawMessage(id: string, channelId = "ch-1"): APIMessage { + return { + id, + channel_id: channelId, + content: "hello", + author: { + id: "user-1", + username: "alice", + discriminator: "0", + avatar: null, + }, + attachments: [], + embeds: [], + mentions: [], + mention_roles: [], + mention_everyone: false, + timestamp: new Date().toISOString(), + edited_timestamp: null, + components: [], + pinned: false, + type: 0, + tts: false, + } as unknown as APIMessage; +} + async function flushQueueWork(): Promise { for (let i = 0; i < 40; i += 1) { await Promise.resolve(); @@ -314,11 +383,31 @@ describe("createDiscordMessageHandler queue behavior", () => { turnAdoptionLifecycle: lifecycle, }); - expect(result).toMatchObject({ kind: "failed-retryable" }); + expect(result).toMatchObject({ kind: "deferred" }); + expect(lifecycle.onCancelled).toHaveBeenCalledTimes(1); expect(lifecycle.onAdopted).not.toHaveBeenCalled(); }); - it("abandons a buffered ingress claim during deactivation", async () => { + it("reports a genuine pre-admission exception only through onFailed", async () => { + preflightDiscordMessageMock.mockReset(); + processDiscordMessageMock.mockReset(); + const failure = new Error("preflight failed"); + preflightDiscordMessageMock.mockRejectedValue(failure); + const handler = createDiscordMessageHandler(createDiscordHandlerParams()); + const lifecycle = createIngressLifecycle(); + + await expect( + handler(createTextMessageData("m-failed") as never, {} as never, { + turnAdoptionLifecycle: lifecycle, + }), + ).resolves.toEqual({ kind: "deferred" }); + + expect(lifecycle.onFailed).toHaveBeenCalledExactlyOnceWith(failure); + expect(lifecycle.onCancelled).not.toHaveBeenCalled(); + expect(lifecycle.onAbandoned).not.toHaveBeenCalled(); + }); + + it("cancels a buffered ingress claim during deactivation", async () => { preflightDiscordMessageMock.mockReset(); processDiscordMessageMock.mockReset(); const params = createDiscordHandlerParams(); @@ -332,11 +421,36 @@ describe("createDiscordMessageHandler queue behavior", () => { await handler.deactivate(); expect(preflightDiscordMessageMock).not.toHaveBeenCalled(); - expect(lifecycle.onAbandoned).toHaveBeenCalledTimes(1); + expect(lifecycle.onCancelled).toHaveBeenCalledTimes(1); + expect(lifecycle.onAbandoned).not.toHaveBeenCalled(); expect(lifecycle.onAdopted).not.toHaveBeenCalled(); }); - it("waits for an active debounce flush and abandons it after shutdown", async () => { + it("settles every buffered claim when cancellation fan-in includes a legacy lifecycle", async () => { + preflightDiscordMessageMock.mockReset(); + processDiscordMessageMock.mockReset(); + const params = createDiscordHandlerParams(); + params.cfg.messages = { inbound: { debounceMs: 60_000 } }; + const handler = createDiscordMessageHandler(params); + const cancellable = createIngressLifecycle(); + const legacy = createIngressLifecycle(); + delete (legacy as Partial).onCancelled; + + await handler(createTextMessageData("m-cancel-modern") as never, {} as never, { + turnAdoptionLifecycle: cancellable, + }); + await handler(createTextMessageData("m-cancel-legacy") as never, {} as never, { + turnAdoptionLifecycle: legacy, + }); + await handler.deactivate(); + + expect(preflightDiscordMessageMock).not.toHaveBeenCalled(); + expect(cancellable.onCancelled).toHaveBeenCalledTimes(1); + expect(legacy.onAbandoned).toHaveBeenCalledTimes(1); + expect(legacy.onAdopted).not.toHaveBeenCalled(); + }); + + it("waits for an active debounce flush and cancels it after shutdown", async () => { preflightDiscordMessageMock.mockReset(); processDiscordMessageMock.mockReset(); const preflightGate = createDeferred(); @@ -360,7 +474,8 @@ describe("createDiscordMessageHandler queue behavior", () => { preflightGate.resolve(); await Promise.all([handling, deactivation]); - expect(lifecycle.onAbandoned).toHaveBeenCalledTimes(1); + expect(lifecycle.onCancelled).toHaveBeenCalledTimes(1); + expect(lifecycle.onAbandoned).not.toHaveBeenCalled(); expect(lifecycle.onAdopted).not.toHaveBeenCalled(); }); @@ -394,6 +509,308 @@ describe("createDiscordMessageHandler queue behavior", () => { expect(stop).toHaveBeenCalledTimes(1); }); + it("dead-letters an exhausted preflight failure and releases its Discord lane", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-01-01T00:00:00.000Z")); + try { + await withDiscordQueue(async (queue) => { + const attempted: string[] = []; + const preflight = vi.fn(async (params: { data: { message?: { id?: string } } }) => { + const id = params.data.message?.id ?? "unknown"; + attempted.push(id); + if (id === "poison") { + throw new Error("deterministic preflight failure"); + } + return null; + }); + const params = createDiscordHandlerParams(); + const handler = createDurableDiscordMessageHandler({ + ...params, + client: {} as never, + testing: { + preflightDiscordMessage: preflight as never, + createIngressMonitor: (monitorParams) => + createDiscordIngressMonitor({ ...monitorParams, queue }), + }, + }); + try { + await handler(createRawMessage("poison", "lane-a") as never, {} as never); + await handler(createRawMessage("follower", "lane-a") as never, {} as never); + await handler(createRawMessage("independent", "lane-b") as never, {} as never); + + for (let attempt = 0; attempt < DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS; attempt += 1) { + await vi.advanceTimersByTimeAsync(3 * 60_000); + } + + await vi.waitFor(() => expect(attempted).toContain("follower")); + expect(attempted.indexOf("independent")).toBeGreaterThanOrEqual(0); + expect(attempted.indexOf("independent")).toBeLessThan(attempted.indexOf("follower")); + expect(attempted.filter((id) => id === "poison")).toHaveLength( + DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS, + ); + await expect(queue.enqueue("poison", {} as DiscordIngressPayload)).resolves.toMatchObject( + { + kind: "failed", + record: { reason: "retry-limit-exceeded" }, + }, + ); + await expect( + queue.enqueue("follower", {} as DiscordIngressPayload), + ).resolves.toMatchObject({ kind: "completed" }); + const runtimeErrors = mockCalls(params.runtime.error as unknown as MockCallSource).map( + ([message]) => String(message), + ); + expect(runtimeErrors.some((message) => message.includes("reached retry limit"))).toBe( + true, + ); + expect(runtimeErrors.join("\n")).not.toContain("hello"); + } finally { + await handler.deactivate(); + } + }); + } finally { + vi.useRealTimers(); + } + }); + + it("preserves retry facts when deactivation cancels a durable Discord claim", async () => { + await withDiscordQueue(async (queue) => { + const raw = createRawMessage("cancelled", "lane-a"); + await queue.enqueue( + "cancelled", + { version: 1, receivedAt: 10, rawMessage: raw }, + { laneKey: "channel:lane-a", receivedAt: 10 }, + ); + const failedClaim = await queue.claim("cancelled", { ownerId: "failed-owner" }); + expect(failedClaim).not.toBeNull(); + if (!failedClaim) { + return; + } + await queue.release(failedClaim, { + lastError: "previous genuine failure", + releasedAt: 20, + }); + const before = (await queue.listPending())[0]; + const firstPreflight = vi.fn(async () => null); + const firstParams = createDiscordHandlerParams(); + firstParams.cfg.messages = { inbound: { debounceMs: 60_000 } }; + const first = createDurableDiscordMessageHandler({ + ...firstParams, + client: {} as never, + testing: { + preflightDiscordMessage: firstPreflight as never, + createIngressMonitor: (monitorParams) => + createDiscordIngressMonitor({ ...monitorParams, queue }), + }, + }); + + await vi.waitFor(async () => expect(await queue.listClaims()).toHaveLength(1)); + await first.deactivate(); + + expect(firstPreflight).not.toHaveBeenCalled(); + expect(await queue.listPending()).toEqual([ + expect.objectContaining({ + id: "cancelled", + attempts: before?.attempts, + lastAttemptAt: before?.lastAttemptAt, + lastError: before?.lastError, + }), + ]); + + const replacementPreflight = vi.fn(async () => null); + const replacementParams = createDiscordHandlerParams(); + const replacement = createDurableDiscordMessageHandler({ + ...replacementParams, + client: {} as never, + testing: { + preflightDiscordMessage: replacementPreflight as never, + createIngressMonitor: (monitorParams) => + createDiscordIngressMonitor({ ...monitorParams, queue }), + }, + }); + try { + await vi.waitFor(() => expect(replacementPreflight).toHaveBeenCalledTimes(1)); + await expect( + queue.enqueue("cancelled", {} as DiscordIngressPayload), + ).resolves.toMatchObject({ kind: "completed" }); + } finally { + await replacement.deactivate(); + } + }); + }); + + it.each(["returns", "throws"] as const)( + "preserves retry facts when a started durable Discord job %s after cancellation", + async (outcome) => { + await withDiscordQueue(async (queue) => { + const id = `started-cancelled-${outcome}`; + const raw = createRawMessage(id, "lane-a"); + await queue.enqueue( + id, + { version: 1, receivedAt: 10, rawMessage: raw }, + { laneKey: "channel:lane-a", receivedAt: 10 }, + ); + const failedClaim = await queue.claim(id, { ownerId: "failed-owner" }); + expect(failedClaim).not.toBeNull(); + if (!failedClaim) { + return; + } + await queue.release(failedClaim, { + lastError: "previous genuine failure", + releasedAt: 20, + }); + const before = (await queue.listPending())[0]; + const processingStarted = createDeferred(); + const finishProcessing = createDeferred(); + let processingSignal: AbortSignal | undefined; + const processDiscordMessage = vi.fn(async (ctx: { abortSignal?: AbortSignal }) => { + processingSignal = ctx.abortSignal; + processingStarted.resolve(); + await finishProcessing.promise; + if (outcome === "throws") { + throw new Error("processing stopped after cancellation"); + } + }); + const params = createDiscordHandlerParams(); + const handler = createDurableDiscordMessageHandler({ + ...params, + client: {} as never, + testing: { + preflightDiscordMessage: (async (preflightParams: { + abortSignal?: AbortSignal; + data: ReturnType; + turnAdoptionLifecycle?: DiscordIngressLifecycle; + }) => ({ + ...createPreflightContextForMessage(preflightParams.data), + abortSignal: preflightParams.abortSignal, + turnAdoptionLifecycle: preflightParams.turnAdoptionLifecycle, + })) as never, + processDiscordMessage: processDiscordMessage as never, + createIngressMonitor: (monitorParams) => + createDiscordIngressMonitor({ ...monitorParams, queue }), + }, + }); + + await processingStarted.promise; + const deactivation = handler.deactivate(); + await vi.waitFor(() => expect(processingSignal?.aborted).toBe(true)); + finishProcessing.resolve(); + await deactivation; + + expect(await queue.listPending()).toEqual([ + expect.objectContaining({ + id, + attempts: before?.attempts, + lastAttemptAt: before?.lastAttemptAt, + lastError: before?.lastError, + }), + ]); + + const recovered = vi.fn(async (_event, lifecycle: DiscordIngressLifecycle) => { + await lifecycle.onAdopted(); + }); + const replacement = createDiscordIngressMonitor({ + accountId: "default", + client: {} as never, + runtime: params.runtime, + queue, + dispatch: recovered, + }); + replacement.start(); + try { + await vi.waitFor(() => expect(recovered).toHaveBeenCalledTimes(1)); + await expect(queue.enqueue(id, {} as DiscordIngressPayload)).resolves.toMatchObject({ + kind: "completed", + }); + } finally { + await replacement.stop(); + } + }); + }, + ); + + it("preserves retry facts when deactivation skips a queued durable Discord job", async () => { + await withDiscordQueue(async (queue) => { + const raw = createRawMessage("queued-cancelled", "lane-a"); + await queue.enqueue( + "queued-cancelled", + { version: 1, receivedAt: 10, rawMessage: raw }, + { laneKey: "channel:lane-a", receivedAt: 10 }, + ); + const failedClaim = await queue.claim("queued-cancelled", { ownerId: "failed-owner" }); + expect(failedClaim).not.toBeNull(); + if (!failedClaim) { + return; + } + await queue.release(failedClaim, { + lastError: "previous genuine failure", + releasedAt: 20, + }); + const before = (await queue.listPending())[0]; + const params = createDiscordHandlerParams(); + const processDiscordMessage = vi.fn(async () => {}); + const messageRunQueue = createDiscordMessageRunQueue({ + runtime: params.runtime, + testing: { processDiscordMessage: processDiscordMessage as never }, + }); + const skipped = createDeferred(); + const monitor = createDiscordIngressMonitor({ + accountId: "default", + client: {} as never, + runtime: params.runtime, + queue, + dispatch: async (_event, lifecycle) => { + const ingress = fanInChannelIngressLifecycles([lifecycle]); + messageRunQueue.enqueue( + buildDiscordInboundJob(await createBaseDiscordMessageContext(), { + ingressSettlement: ingress, + }), + ); + await messageRunQueue.deactivate(); + skipped.resolve(); + return { kind: "deferred" }; + }, + }); + monitor.start(); + try { + await skipped.promise; + await monitor.stop(); + expect(processDiscordMessage).not.toHaveBeenCalled(); + expect(await queue.listPending()).toEqual([ + expect.objectContaining({ + id: "queued-cancelled", + attempts: before?.attempts, + lastAttemptAt: before?.lastAttemptAt, + lastError: before?.lastError, + }), + ]); + } finally { + await monitor.stop(); + await messageRunQueue.deactivate(); + } + + const recovered = vi.fn(async (_event, lifecycle: DiscordIngressLifecycle) => { + await lifecycle.onAdopted(); + }); + const replacement = createDiscordIngressMonitor({ + accountId: "default", + client: {} as never, + runtime: params.runtime, + queue, + dispatch: recovered, + }); + replacement.start(); + try { + await vi.waitFor(() => expect(recovered).toHaveBeenCalledTimes(1)); + await expect( + queue.enqueue("queued-cancelled", {} as DiscordIngressPayload), + ).resolves.toMatchObject({ kind: "completed" }); + } finally { + await replacement.stop(); + } + }); + }); + it("does not abort concurrent runs with a Discord-owned channel timeout", async () => { vi.useFakeTimers(); try { diff --git a/extensions/discord/src/monitor/message-run-queue.ts b/extensions/discord/src/monitor/message-run-queue.ts index f7ca537ca1b3..bb1be5365841 100644 --- a/extensions/discord/src/monitor/message-run-queue.ts +++ b/extensions/discord/src/monitor/message-run-queue.ts @@ -45,21 +45,25 @@ async function processDiscordQueuedMessage(params: { (await loadMessageProcessRuntime()).processDiscordMessage; await processDiscordMessageImpl(materializeDiscordInboundJob(params.job, abortSignal)); if (abortSignal?.aborted) { - await params.job.ingressSettlement?.abandon(abortSignal.reason); + // Cancellation ended ownership before delivery; retain prior retry facts + // so the durable claim can replay under a replacement lifecycle. + await params.job.ingressSettlement?.cancel(); } else { await params.job.ingressSettlement?.settle(); } } catch (error) { - await params.job.ingressSettlement?.abandon(error); + if (abortSignal?.aborted) { + await params.job.ingressSettlement?.cancel(); + } else { + await params.job.ingressSettlement?.abandon(error); + } throw error; } } async function cleanupSkippedDiscordQueuedMessage(params: { job: DiscordInboundJob }) { // A skipped job never reached reply-lane adoption; reopen its durable claim. - await params.job.ingressSettlement?.abandon( - new Error("discord queued run skipped before processing"), - ); + await params.job.ingressSettlement?.cancel(); } export function createDiscordMessageRunQueue( diff --git a/extensions/discord/src/send.creates-thread.chunking.test.ts b/extensions/discord/src/send.creates-thread.chunking.test.ts new file mode 100644 index 000000000000..b285fd15ca24 --- /dev/null +++ b/extensions/discord/src/send.creates-thread.chunking.test.ts @@ -0,0 +1,277 @@ +import { ChannelType, Routes } from "discord-api-types/v10"; +import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures"; +import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { hasDiscordMessageCreateAmbiguity } from "./retry.js"; +import { + makeDiscordRest, + requestBody, + requestPath, + type MockCallSource, +} from "./send.test-harness.js"; + +let createThreadDiscord: typeof import("./send.js").createThreadDiscord; +let DiscordThreadInitialMessageError: typeof import("./send.js").DiscordThreadInitialMessageError; + +const DISCORD_TEST_CFG = { + channels: { + discord: { + accounts: { + default: {}, + }, + }, + }, +}; + +function discordClientOpts(rest: ReturnType["rest"]) { + return { cfg: DISCORD_TEST_CFG, rest, token: "t" }; +} + +const requireRecord = createRequireRecord("object", "expected-label"); + +beforeAll(async () => { + ({ createThreadDiscord, DiscordThreadInitialMessageError } = await import("./send.js")); +}); + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("sendMessageDiscord", () => { + it("keeps forum starter messages within Discord's content limit", async () => { + const { rest, getMock, postMock } = makeDiscordRest(); + getMock.mockResolvedValue({ type: ChannelType.GuildForum }); + postMock.mockResolvedValue({ id: "t1" }); + const content = "a".repeat(2001); + + await createThreadDiscord("chan1", { name: "thread", content }, discordClientOpts(rest)); + + expect(postMock).toHaveBeenCalledTimes(2); + expect(requestBody(postMock as unknown as MockCallSource, 0)).toEqual({ + name: "thread", + message: { content: "a".repeat(2000) }, + }); + expect(requestPath(postMock as unknown as MockCallSource, 1)).toBe( + Routes.channelMessages("t1"), + ); + expect(requestBody(postMock as unknown as MockCallSource, 1)).toMatchObject({ + content: "a", + enforce_nonce: true, + }); + }); + + it("keeps sub-limit multi-line forum content in one starter message", async () => { + const { rest, getMock, postMock } = makeDiscordRest(); + getMock.mockResolvedValue({ type: ChannelType.GuildForum }); + postMock.mockResolvedValue({ id: "t1" }); + const content = Array.from({ length: 18 }, (_, index) => `line ${index + 1}`).join("\n"); + + await createThreadDiscord("chan1", { name: "thread", content }, discordClientOpts(rest)); + + expect(postMock).toHaveBeenCalledTimes(1); + expect(requestBody(postMock as unknown as MockCallSource)).toEqual({ + name: "thread", + message: { content }, + }); + }); + + it("reports a delivered forum starter when a continuation chunk fails", async () => { + const { rest, getMock, postMock } = makeDiscordRest(); + getMock.mockResolvedValue({ type: ChannelType.GuildForum }); + postMock + .mockResolvedValueOnce({ id: "t1", message: { id: "starter1", channel_id: "t1" } }) + .mockRejectedValueOnce(Object.assign(new Error("missing access"), { status: 403 })); + + let thrown: unknown; + try { + await createThreadDiscord( + "chan1", + { name: "thread", content: "a".repeat(2001) }, + discordClientOpts(rest), + ); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(DiscordThreadInitialMessageError); + expect(requireRecord(thrown, "thread initial message error").initialMessageDelivery).toEqual({ + starterMessageDelivered: true, + deliveredChunkCount: 1, + deliveredMessageIds: ["starter1"], + failedChunkDelivery: "not_delivered", + failedChunkIndex: 1, + totalChunkCount: 2, + }); + }); + + it("reports an exhausted ambiguous forum continuation as unknown delivery", async () => { + const { rest, getMock, postMock } = makeDiscordRest(); + getMock.mockResolvedValue({ type: ChannelType.GuildForum }); + const ambiguous = Object.assign(new Error("response lost"), { status: 502 }); + postMock + .mockResolvedValueOnce({ id: "t1", message: { id: "starter1", channel_id: "t1" } }) + .mockRejectedValue(ambiguous); + + let thrown: unknown; + try { + await createThreadDiscord( + "chan1", + { name: "thread", content: "a".repeat(2001) }, + { + ...discordClientOpts(rest), + retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 }, + }, + ); + } catch (error) { + thrown = error; + } + + expect(postMock).toHaveBeenCalledTimes(3); + expect(thrown).toBeInstanceOf(DiscordThreadInitialMessageError); + expect(hasDiscordMessageCreateAmbiguity(thrown)).toBe(true); + expect(requireRecord(thrown, "thread initial message error").message).toContain( + "delivery of the remaining initial content could not be confirmed", + ); + expect(requireRecord(thrown, "thread initial message error").initialMessageDelivery).toEqual({ + starterMessageDelivered: true, + deliveredChunkCount: 1, + deliveredMessageIds: ["starter1"], + failedChunkDelivery: "unknown", + failedChunkIndex: 1, + totalChunkCount: 2, + }); + }); + + it("chunks long initial messages for non-forum threads", async () => { + const { rest, getMock, postMock } = makeDiscordRest(); + getMock.mockResolvedValue({ type: ChannelType.GuildText }); + postMock.mockResolvedValue({ id: "t1" }); + const content = "a".repeat(2001); + + await createThreadDiscord("chan1", { name: "thread", content }, discordClientOpts(rest)); + + expect(postMock).toHaveBeenCalledTimes(3); + expect(requestPath(postMock as unknown as MockCallSource, 1)).toBe( + Routes.channelMessages("t1"), + ); + expect(requestBody(postMock as unknown as MockCallSource, 1)).toMatchObject({ + content: "a".repeat(2000), + enforce_nonce: true, + }); + expect(requestPath(postMock as unknown as MockCallSource, 2)).toBe( + Routes.channelMessages("t1"), + ); + expect(requestBody(postMock as unknown as MockCallSource, 2)).toMatchObject({ + content: "a", + enforce_nonce: true, + }); + }); + + it("keeps sub-limit multi-line non-forum content in one initial message", async () => { + const { rest, getMock, postMock } = makeDiscordRest(); + getMock.mockResolvedValue({ type: ChannelType.GuildText }); + postMock.mockResolvedValue({ id: "t1", channel_id: "t1" }); + const content = Array.from({ length: 18 }, (_, index) => `line ${index + 1}`).join("\n"); + + await createThreadDiscord("chan1", { name: "thread", content }, discordClientOpts(rest)); + + expect(postMock).toHaveBeenCalledTimes(2); + expect(requestBody(postMock as unknown as MockCallSource, 1)).toMatchObject({ content }); + }); + + it("reports delivered non-forum chunks when a later chunk fails", async () => { + const { rest, getMock, postMock } = makeDiscordRest(); + getMock.mockResolvedValue({ type: ChannelType.GuildText }); + postMock + .mockResolvedValueOnce({ id: "t1", name: "thread", type: ChannelType.PublicThread }) + .mockResolvedValueOnce({ id: "msg1", channel_id: "t1" }) + .mockRejectedValueOnce(Object.assign(new Error("missing access"), { status: 403 })); + + let thrown: unknown; + try { + await createThreadDiscord( + "chan1", + { name: "thread", content: "a".repeat(4001) }, + discordClientOpts(rest), + ); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(DiscordThreadInitialMessageError); + expect(requireRecord(thrown, "thread initial message error").initialMessageDelivery).toEqual({ + starterMessageDelivered: false, + deliveredChunkCount: 1, + deliveredMessageIds: ["msg1"], + failedChunkDelivery: "not_delivered", + failedChunkIndex: 1, + totalChunkCount: 3, + }); + }); + + it("reports an exhausted ambiguous non-forum chunk as unknown delivery", async () => { + const { rest, getMock, postMock } = makeDiscordRest(); + getMock.mockResolvedValue({ type: ChannelType.GuildText }); + const ambiguous = Object.assign(new Error("response lost"), { status: 502 }); + postMock + .mockResolvedValueOnce({ id: "t1", name: "thread", type: ChannelType.PublicThread }) + .mockResolvedValueOnce({ id: "msg1", channel_id: "t1" }) + .mockRejectedValue(ambiguous); + + let thrown: unknown; + try { + await createThreadDiscord( + "chan1", + { name: "thread", content: "a".repeat(4001) }, + { + ...discordClientOpts(rest), + retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 }, + }, + ); + } catch (error) { + thrown = error; + } + + expect(postMock).toHaveBeenCalledTimes(4); + expect(thrown).toBeInstanceOf(DiscordThreadInitialMessageError); + expect(hasDiscordMessageCreateAmbiguity(thrown)).toBe(true); + expect(requireRecord(thrown, "thread initial message error").message).toContain( + "delivery of the remaining initial content could not be confirmed", + ); + expect(requireRecord(thrown, "thread initial message error").initialMessageDelivery).toEqual({ + starterMessageDelivered: false, + deliveredChunkCount: 1, + deliveredMessageIds: ["msg1"], + failedChunkDelivery: "unknown", + failedChunkIndex: 1, + totalChunkCount: 3, + }); + }); + + it("retries continuation sends with a stable nonce per chunk", async () => { + const { rest, getMock, postMock } = makeDiscordRest(); + getMock.mockResolvedValue({ type: ChannelType.GuildText }); + postMock + .mockResolvedValueOnce({ id: "t1", name: "thread", type: ChannelType.PublicThread }) + .mockRejectedValueOnce(Object.assign(new Error("bad gateway"), { status: 502 })) + .mockResolvedValueOnce({ id: "msg1", channel_id: "t1" }) + .mockResolvedValueOnce({ id: "msg2", channel_id: "t1" }); + + await createThreadDiscord( + "chan1", + { name: "thread", content: "a".repeat(2001) }, + { + ...discordClientOpts(rest), + retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 }, + }, + ); + + expect(postMock).toHaveBeenCalledTimes(4); + const firstAttempt = requestBody(postMock as unknown as MockCallSource, 1); + const retryAttempt = requestBody(postMock as unknown as MockCallSource, 2); + const nextChunk = requestBody(postMock as unknown as MockCallSource, 3); + expect(firstAttempt.enforce_nonce).toBe(true); + expect(retryAttempt.nonce).toBe(firstAttempt.nonce); + expect(nextChunk.enforce_nonce).toBe(true); + expect(nextChunk.nonce).not.toBe(firstAttempt.nonce); + }); +}); diff --git a/extensions/discord/src/send.creates-thread.test.ts b/extensions/discord/src/send.creates-thread.test.ts index 9fe29c6fed12..e6db36986a60 100644 --- a/extensions/discord/src/send.creates-thread.test.ts +++ b/extensions/discord/src/send.creates-thread.test.ts @@ -4,8 +4,13 @@ import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures"; import { loadWebMediaRaw } from "openclaw/plugin-sdk/web-media"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { RateLimitError } from "./internal/discord.js"; -import { hasDiscordMessageCreateAmbiguity } from "./retry.js"; -import { makeDiscordRest } from "./send.test-harness.js"; +import { + makeDiscordRest, + requestBody, + requestPath, + timerDelayAt, + type MockCallSource, +} from "./send.test-harness.js"; vi.mock("openclaw/plugin-sdk/web-media", async () => { const { discordWebMediaMockFactory } = await import("./send.test-harness.js"); @@ -42,41 +47,8 @@ function discordClientOpts(rest: ReturnType["rest"]) { return { cfg: DISCORD_TEST_CFG, rest, token: "t" }; } -type MockCallSource = { - mock: { - calls: ArrayLike>; - }; -}; - const requireRecord = createRequireRecord("object", "expected-label"); -function mockArg(source: MockCallSource, callIndex: number, argIndex: number, label: string) { - const call = source.mock.calls[callIndex]; - if (!call) { - throw new Error(`expected mock call: ${label}`); - } - return call[argIndex]; -} - -function requestOptions(source: MockCallSource, callIndex = 0) { - return requireRecord( - mockArg(source, callIndex, 1, `request options ${callIndex}`), - "request options", - ); -} - -function requestPath(source: MockCallSource, callIndex = 0) { - return mockArg(source, callIndex, 0, `request path ${callIndex}`); -} - -function requestBody(source: MockCallSource, callIndex = 0) { - return requireRecord(requestOptions(source, callIndex).body, `request body ${callIndex}`); -} - -function timerDelayAt(source: MockCallSource, callIndex = 0) { - return mockArg(source, callIndex, 1, `timer delay ${callIndex}`); -} - function createDiscordForumPayloadHarness(parentType: ChannelType = ChannelType.GuildForum) { const parentId = "700"; const { rest, getMock, postMock } = makeDiscordRest(); @@ -303,110 +275,6 @@ describe("sendMessageDiscord", () => { }); }); - it("keeps forum starter messages within Discord's content limit", async () => { - const { rest, getMock, postMock } = makeDiscordRest(); - getMock.mockResolvedValue({ type: ChannelType.GuildForum }); - postMock.mockResolvedValue({ id: "t1" }); - const content = "a".repeat(2001); - - await createThreadDiscord("chan1", { name: "thread", content }, discordClientOpts(rest)); - - expect(postMock).toHaveBeenCalledTimes(2); - expect(requestBody(postMock as unknown as MockCallSource, 0)).toEqual({ - name: "thread", - message: { content: "a".repeat(2000) }, - }); - expect(requestPath(postMock as unknown as MockCallSource, 1)).toBe( - Routes.channelMessages("t1"), - ); - expect(requestBody(postMock as unknown as MockCallSource, 1)).toMatchObject({ - content: "a", - enforce_nonce: true, - }); - }); - - it("keeps sub-limit multi-line forum content in one starter message", async () => { - const { rest, getMock, postMock } = makeDiscordRest(); - getMock.mockResolvedValue({ type: ChannelType.GuildForum }); - postMock.mockResolvedValue({ id: "t1" }); - const content = Array.from({ length: 18 }, (_, index) => `line ${index + 1}`).join("\n"); - - await createThreadDiscord("chan1", { name: "thread", content }, discordClientOpts(rest)); - - expect(postMock).toHaveBeenCalledTimes(1); - expect(requestBody(postMock as unknown as MockCallSource)).toEqual({ - name: "thread", - message: { content }, - }); - }); - - it("reports a delivered forum starter when a continuation chunk fails", async () => { - const { rest, getMock, postMock } = makeDiscordRest(); - getMock.mockResolvedValue({ type: ChannelType.GuildForum }); - postMock - .mockResolvedValueOnce({ id: "t1", message: { id: "starter1", channel_id: "t1" } }) - .mockRejectedValueOnce(Object.assign(new Error("missing access"), { status: 403 })); - - let thrown: unknown; - try { - await createThreadDiscord( - "chan1", - { name: "thread", content: "a".repeat(2001) }, - discordClientOpts(rest), - ); - } catch (error) { - thrown = error; - } - - expect(thrown).toBeInstanceOf(DiscordThreadInitialMessageError); - expect(requireRecord(thrown, "thread initial message error").initialMessageDelivery).toEqual({ - starterMessageDelivered: true, - deliveredChunkCount: 1, - deliveredMessageIds: ["starter1"], - failedChunkDelivery: "not_delivered", - failedChunkIndex: 1, - totalChunkCount: 2, - }); - }); - - it("reports an exhausted ambiguous forum continuation as unknown delivery", async () => { - const { rest, getMock, postMock } = makeDiscordRest(); - getMock.mockResolvedValue({ type: ChannelType.GuildForum }); - const ambiguous = Object.assign(new Error("response lost"), { status: 502 }); - postMock - .mockResolvedValueOnce({ id: "t1", message: { id: "starter1", channel_id: "t1" } }) - .mockRejectedValue(ambiguous); - - let thrown: unknown; - try { - await createThreadDiscord( - "chan1", - { name: "thread", content: "a".repeat(2001) }, - { - ...discordClientOpts(rest), - retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 }, - }, - ); - } catch (error) { - thrown = error; - } - - expect(postMock).toHaveBeenCalledTimes(3); - expect(thrown).toBeInstanceOf(DiscordThreadInitialMessageError); - expect(hasDiscordMessageCreateAmbiguity(thrown)).toBe(true); - expect(requireRecord(thrown, "thread initial message error").message).toContain( - "delivery of the remaining initial content could not be confirmed", - ); - expect(requireRecord(thrown, "thread initial message error").initialMessageDelivery).toEqual({ - starterMessageDelivered: true, - deliveredChunkCount: 1, - deliveredMessageIds: ["starter1"], - failedChunkDelivery: "unknown", - failedChunkIndex: 1, - totalChunkCount: 2, - }); - }); - it("inherits default_auto_archive_duration for forum threads", async () => { const { rest, getMock, postMock } = makeDiscordRest(); getMock.mockResolvedValue({ @@ -568,140 +436,6 @@ describe("sendMessageDiscord", () => { }); }); - it("chunks long initial messages for non-forum threads", async () => { - const { rest, getMock, postMock } = makeDiscordRest(); - getMock.mockResolvedValue({ type: ChannelType.GuildText }); - postMock.mockResolvedValue({ id: "t1" }); - const content = "a".repeat(2001); - - await createThreadDiscord("chan1", { name: "thread", content }, discordClientOpts(rest)); - - expect(postMock).toHaveBeenCalledTimes(3); - expect(requestPath(postMock as unknown as MockCallSource, 1)).toBe( - Routes.channelMessages("t1"), - ); - expect(requestBody(postMock as unknown as MockCallSource, 1)).toMatchObject({ - content: "a".repeat(2000), - enforce_nonce: true, - }); - expect(requestPath(postMock as unknown as MockCallSource, 2)).toBe( - Routes.channelMessages("t1"), - ); - expect(requestBody(postMock as unknown as MockCallSource, 2)).toMatchObject({ - content: "a", - enforce_nonce: true, - }); - }); - - it("keeps sub-limit multi-line non-forum content in one initial message", async () => { - const { rest, getMock, postMock } = makeDiscordRest(); - getMock.mockResolvedValue({ type: ChannelType.GuildText }); - postMock.mockResolvedValue({ id: "t1", channel_id: "t1" }); - const content = Array.from({ length: 18 }, (_, index) => `line ${index + 1}`).join("\n"); - - await createThreadDiscord("chan1", { name: "thread", content }, discordClientOpts(rest)); - - expect(postMock).toHaveBeenCalledTimes(2); - expect(requestBody(postMock as unknown as MockCallSource, 1)).toMatchObject({ content }); - }); - - it("reports delivered non-forum chunks when a later chunk fails", async () => { - const { rest, getMock, postMock } = makeDiscordRest(); - getMock.mockResolvedValue({ type: ChannelType.GuildText }); - postMock - .mockResolvedValueOnce({ id: "t1", name: "thread", type: ChannelType.PublicThread }) - .mockResolvedValueOnce({ id: "msg1", channel_id: "t1" }) - .mockRejectedValueOnce(Object.assign(new Error("missing access"), { status: 403 })); - - let thrown: unknown; - try { - await createThreadDiscord( - "chan1", - { name: "thread", content: "a".repeat(4001) }, - discordClientOpts(rest), - ); - } catch (error) { - thrown = error; - } - - expect(thrown).toBeInstanceOf(DiscordThreadInitialMessageError); - expect(requireRecord(thrown, "thread initial message error").initialMessageDelivery).toEqual({ - starterMessageDelivered: false, - deliveredChunkCount: 1, - deliveredMessageIds: ["msg1"], - failedChunkDelivery: "not_delivered", - failedChunkIndex: 1, - totalChunkCount: 3, - }); - }); - - it("reports an exhausted ambiguous non-forum chunk as unknown delivery", async () => { - const { rest, getMock, postMock } = makeDiscordRest(); - getMock.mockResolvedValue({ type: ChannelType.GuildText }); - const ambiguous = Object.assign(new Error("response lost"), { status: 502 }); - postMock - .mockResolvedValueOnce({ id: "t1", name: "thread", type: ChannelType.PublicThread }) - .mockResolvedValueOnce({ id: "msg1", channel_id: "t1" }) - .mockRejectedValue(ambiguous); - - let thrown: unknown; - try { - await createThreadDiscord( - "chan1", - { name: "thread", content: "a".repeat(4001) }, - { - ...discordClientOpts(rest), - retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 }, - }, - ); - } catch (error) { - thrown = error; - } - - expect(postMock).toHaveBeenCalledTimes(4); - expect(thrown).toBeInstanceOf(DiscordThreadInitialMessageError); - expect(hasDiscordMessageCreateAmbiguity(thrown)).toBe(true); - expect(requireRecord(thrown, "thread initial message error").message).toContain( - "delivery of the remaining initial content could not be confirmed", - ); - expect(requireRecord(thrown, "thread initial message error").initialMessageDelivery).toEqual({ - starterMessageDelivered: false, - deliveredChunkCount: 1, - deliveredMessageIds: ["msg1"], - failedChunkDelivery: "unknown", - failedChunkIndex: 1, - totalChunkCount: 3, - }); - }); - - it("retries continuation sends with a stable nonce per chunk", async () => { - const { rest, getMock, postMock } = makeDiscordRest(); - getMock.mockResolvedValue({ type: ChannelType.GuildText }); - postMock - .mockResolvedValueOnce({ id: "t1", name: "thread", type: ChannelType.PublicThread }) - .mockRejectedValueOnce(Object.assign(new Error("bad gateway"), { status: 502 })) - .mockResolvedValueOnce({ id: "msg1", channel_id: "t1" }) - .mockResolvedValueOnce({ id: "msg2", channel_id: "t1" }); - - await createThreadDiscord( - "chan1", - { name: "thread", content: "a".repeat(2001) }, - { - ...discordClientOpts(rest), - retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 }, - }, - ); - - expect(postMock).toHaveBeenCalledTimes(4); - const firstAttempt = requestBody(postMock as unknown as MockCallSource, 1); - const retryAttempt = requestBody(postMock as unknown as MockCallSource, 2); - const nextChunk = requestBody(postMock as unknown as MockCallSource, 3); - expect(firstAttempt.enforce_nonce).toBe(true); - expect(retryAttempt.nonce).toBe(firstAttempt.nonce); - expect(nextChunk.enforce_nonce).toBe(true); - expect(nextChunk.nonce).not.toBe(firstAttempt.nonce); - }); - it("keeps created non-forum thread details when initial message send fails", async () => { const { rest, getMock, postMock } = makeDiscordRest(); getMock.mockResolvedValue({ type: ChannelType.GuildText }); diff --git a/extensions/discord/src/send.test-harness.ts b/extensions/discord/src/send.test-harness.ts index 0eacb0108a2e..1627137db10f 100644 --- a/extensions/discord/src/send.test-harness.ts +++ b/extensions/discord/src/send.test-harness.ts @@ -1,6 +1,7 @@ // Discord plugin module implements send harness behavior. import { createServer } from "node:http"; import type { MockFn } from "openclaw/plugin-sdk/plugin-test-runtime"; +import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures"; import { vi } from "vitest"; import { RequestClient } from "./internal/discord.js"; @@ -25,6 +26,41 @@ type DiscordLoopbackRequest = { path: string | undefined; }; +export type MockCallSource = { + mock: { + calls: ArrayLike>; + }; +}; + +const requireRecord = createRequireRecord("object", "expected-label"); + +function mockArg(source: MockCallSource, callIndex: number, argIndex: number, label: string) { + const call = source.mock.calls[callIndex]; + if (!call) { + throw new Error(`expected mock call: ${label}`); + } + return call[argIndex]; +} + +function requestOptions(source: MockCallSource, callIndex = 0) { + return requireRecord( + mockArg(source, callIndex, 1, `request options ${callIndex}`), + "request options", + ); +} + +export function requestPath(source: MockCallSource, callIndex = 0) { + return mockArg(source, callIndex, 0, `request path ${callIndex}`); +} + +export function requestBody(source: MockCallSource, callIndex = 0) { + return requireRecord(requestOptions(source, callIndex).body, `request body ${callIndex}`); +} + +export function timerDelayAt(source: MockCallSource, callIndex = 0) { + return mockArg(source, callIndex, 1, `timer delay ${callIndex}`); +} + export async function createDiscordLoopbackRest(options?: { respond?: (request: DiscordLoopbackRequest) => unknown; }): Promise<{ diff --git a/extensions/feishu/src/monitor.message-handler.ingress.test.ts b/extensions/feishu/src/monitor.message-handler.ingress.test.ts index 1390a6c22be5..25f7308b4702 100644 --- a/extensions/feishu/src/monitor.message-handler.ingress.test.ts +++ b/extensions/feishu/src/monitor.message-handler.ingress.test.ts @@ -1,11 +1,20 @@ -import { createTestInboundDebounceFlush } from "openclaw/plugin-sdk/channel-test-helpers"; -import { createNonExitingRuntimeEnv } from "openclaw/plugin-sdk/plugin-test-runtime"; // Feishu ingress tests cover debounce ownership and constituent claim settlement. +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { createInboundDebouncer } from "openclaw/plugin-sdk/channel-inbound-debounce"; +import { DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS } from "openclaw/plugin-sdk/channel-outbound"; +import { createTestInboundDebounceFlush } from "openclaw/plugin-sdk/channel-test-helpers"; +import { + closeOpenClawStateDatabaseForTest, + createChannelIngressQueueForTests, +} from "openclaw/plugin-sdk/plugin-state-test-runtime"; +import { createNonExitingRuntimeEnv } from "openclaw/plugin-sdk/plugin-test-runtime"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { ClawdbotConfig, PluginRuntime, RuntimeEnv } from "../runtime-api.js"; import * as dedup from "./dedup.js"; import type { FeishuMessageEvent } from "./event-types.js"; -import type { FeishuIngressLifecycle } from "./feishu-ingress.js"; +import { createFeishuDurableIngress, type FeishuIngressLifecycle } from "./feishu-ingress.js"; import { createFeishuMessageReceiveHandler } from "./monitor.message-handler.js"; type MessageReceiveHandlerContext = Parameters[0]; @@ -370,4 +379,144 @@ describe("Feishu durable ingress debounce lifecycle", () => { expect(harness.handleMessage).toHaveBeenCalledTimes(1); expect(second.calls.adopted).not.toHaveBeenCalled(); }); + + it("preserves abandon retry accounting, backoff, threshold, and restart behavior", async () => { + vi.useFakeTimers(); + const now = Date.UTC(2026, 0, 2); + vi.setSystemTime(now); + const created = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-feishu-abandon-")); + const stateDir = await fs.realpath(created); + type Queue = NonNullable[0]["queue"]>; + type Payload = Parameters[1]; + const queue = createChannelIngressQueueForTests({ + channelId: "feishu", + accountId: "default", + stateDir, + }); + const event = { + ...createTextEvent("evt-abandon-retry", "om-abandon-retry", "retry me"), + event_type: "im.message.receive_v1", + }; + const handleMessage = vi.fn(async () => { + throw new Error("Feishu dispatch failed before adoption"); + }); + vi.spyOn(dedup, "claimUnprocessedFeishuMessage").mockImplementation(async () => ({ + kind: "claimed", + handle: createClaim(`retry-${handleMessage.mock.calls.length}`), + })); + + const createIntegratedIngress = () => { + const channelRuntime = { + commands: { isControlCommandMessage: () => false }, + debounce: { + resolveInboundDebounceMs: () => 0, + createInboundDebouncer, + }, + } as unknown as PluginRuntime["channel"]; + const handler = createFeishuMessageReceiveHandler({ + cfg: {} as ClawdbotConfig, + channelRuntime, + accountId: "default", + runtime: createNonExitingRuntimeEnv(), + chatHistories: new Map(), + handleMessage, + resolveDebounceText: () => "retry me", + hasProcessedMessage: vi.fn(async () => false), + getBotOpenId: () => "ou-bot", + resolveIngressLifecycle: (data) => ingress.resolveLifecycle(data), + }); + const ingress = createFeishuDurableIngress({ + accountId: "default", + queue, + dispatcher: { invoke: async (data: unknown) => await handler(data as never) } as never, + runtime: { error: vi.fn(), log: vi.fn() }, + pollIntervalMs: 500, + }); + return ingress; + }; + const pendingAttempt = async (attempts: number) => { + let observed: Awaited>[number] | undefined; + await vi.waitFor(async () => { + const pending = await queue.listPending({ limit: "all" }); + expect(pending).toEqual([ + expect.objectContaining({ + id: "evt-abandon-retry", + attempts, + lastAttemptAt: expect.any(Number), + lastError: "turn-abandoned", + }), + ]); + observed = pending[0]; + }); + const lastAttemptAt = observed?.lastAttemptAt; + if (lastAttemptAt === undefined) { + throw new Error(`Missing Feishu retry timestamp for attempt ${attempts}`); + } + return { ...observed, lastAttemptAt }; + }; + + try { + const first = createIntegratedIngress(); + first.start(); + await first.invokeWebhook(event); + const firstAttempt = await pendingAttempt(1); + expect(handleMessage).toHaveBeenCalledTimes(1); + await first.stop(); + + vi.setSystemTime(firstAttempt.lastAttemptAt + 999); + const blocked = createIntegratedIngress(); + blocked.start(); + await blocked.invokeWebhook(event); + await vi.advanceTimersByTimeAsync(0); + expect(handleMessage).toHaveBeenCalledTimes(1); + await blocked.stop(); + + vi.setSystemTime(firstAttempt.lastAttemptAt + 1_001); + const second = createIntegratedIngress(); + second.start(); + await second.invokeWebhook(event); + const secondAttempt = await pendingAttempt(2); + expect(handleMessage).toHaveBeenCalledTimes(2); + await second.stop(); + + for (let attempt = 3; attempt < DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS; attempt += 1) { + const claim = await queue.claim("evt-abandon-retry", { ownerId: `seed-${attempt}` }); + if (!claim) { + throw new Error(`Expected Feishu seed claim ${attempt}`); + } + await queue.release(claim, { + lastError: "turn-abandoned", + releasedAt: secondAttempt.lastAttemptAt, + }); + } + + vi.setSystemTime(secondAttempt.lastAttemptAt + 64_001); + const threshold = createIntegratedIngress(); + threshold.start(); + await threshold.invokeWebhook(event); + const thresholdAttempt = await pendingAttempt(DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS); + expect(handleMessage).toHaveBeenCalledTimes(3); + await threshold.stop(); + + vi.setSystemTime(thresholdAttempt.lastAttemptAt + 128_001); + const beyond = createIntegratedIngress(); + beyond.start(); + await beyond.invokeWebhook(event); + const beyondAttempt = await pendingAttempt(DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS + 1); + expect(handleMessage).toHaveBeenCalledTimes(4); + await beyond.stop(); + + vi.setSystemTime(beyondAttempt.lastAttemptAt + 1_000); + const blockedRestart = createIntegratedIngress(); + blockedRestart.start(); + await blockedRestart.invokeWebhook(event); + await vi.advanceTimersByTimeAsync(0); + expect(handleMessage).toHaveBeenCalledTimes(4); + await blockedRestart.stop(); + } finally { + closeOpenClawStateDatabaseForTest(); + await fs.rm(stateDir, { recursive: true, force: true }); + vi.useRealTimers(); + } + }); }); diff --git a/extensions/mattermost/src/mattermost/monitor.inbound-system-event.test.ts b/extensions/mattermost/src/mattermost/monitor.inbound-system-event.test.ts index dcb8e3830613..17f0dc7a04d9 100644 --- a/extensions/mattermost/src/mattermost/monitor.inbound-system-event.test.ts +++ b/extensions/mattermost/src/mattermost/monitor.inbound-system-event.test.ts @@ -1,10 +1,20 @@ // Mattermost tests cover monitor.inbound system event plugin behavior. import { once } from "node:events"; +import fs from "node:fs/promises"; import { createServer } from "node:http"; +import os from "node:os"; +import path from "node:path"; import { createChannelPartialDeliveryError } from "openclaw/plugin-sdk/channel-inbound"; import { createInboundDebouncer } from "openclaw/plugin-sdk/channel-inbound-debounce"; -import { createMessageReceiptFromOutboundResults } from "openclaw/plugin-sdk/channel-outbound"; +import { + createMessageReceiptFromOutboundResults, + DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS, +} from "openclaw/plugin-sdk/channel-outbound"; import { createTestInboundDebounceFlush } from "openclaw/plugin-sdk/channel-test-helpers"; +import { + closeOpenClawStateDatabaseForTest, + createChannelIngressQueueForTests, +} from "openclaw/plugin-sdk/plugin-state-test-runtime"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { WebSocketServer } from "ws"; import type { MattermostPost } from "./client.js"; @@ -96,6 +106,7 @@ const mockState = vi.hoisted(() => ({ enqueueSystemEvent: vi.fn(), fetchMattermostMe: vi.fn(), getGlobalHookRunner: vi.fn(), + ingressQueue: undefined as unknown, progressDrafts: [] as Array<{ getSnapshot: () => { lines: readonly unknown[] } }>, registerMattermostMonitorSlashCommands: vi.fn(), registerPluginHttpRoute: vi.fn(), @@ -180,27 +191,36 @@ vi.mock("./monitor-ingress.js", async (importOriginal) => { ...actual, createMattermostIngressMonitor: ( options: Parameters[0], - ) => ({ - receive: async (rawEvent: string) => { - const payload = JSON.parse(rawEvent) as MattermostEventPayload; - const post = - typeof payload.data?.post === "string" - ? (JSON.parse(payload.data.post) as MattermostPost) - : (payload.data?.post as MattermostPost | undefined); - if (payload.event !== "posted" || !post) { - return; - } - await options.dispatch(post, payload, { - abortSignal: new AbortController().signal, - onAdopted: async () => {}, - onDeferred: () => {}, - onAdoptionFinalizing: () => {}, - onAbandoned: async () => {}, + ) => { + if (mockState.ingressQueue) { + return actual.createMattermostIngressMonitor({ + ...options, + queue: mockState.ingressQueue as NonNullable, + pollIntervalMs: 60_000, }); - }, - stop: async () => {}, - waitForIdle: async () => {}, - }), + } + return { + receive: async (rawEvent: string) => { + const payload = JSON.parse(rawEvent) as MattermostEventPayload; + const post = + typeof payload.data?.post === "string" + ? (JSON.parse(payload.data.post) as MattermostPost) + : (payload.data?.post as MattermostPost | undefined); + if (payload.event !== "posted" || !post) { + return; + } + await options.dispatch(post, payload, { + abortSignal: new AbortController().signal, + onAdopted: async () => {}, + onDeferred: () => {}, + onAdoptionFinalizing: () => {}, + onAbandoned: async () => {}, + }); + }, + stop: async () => {}, + waitForIdle: async () => {}, + }; + }, }; }); @@ -534,6 +554,7 @@ describe("mattermost inbound user posts", () => { beforeEach(() => { vi.clearAllMocks(); mockState.abortController = undefined; + mockState.ingressQueue = undefined; mockState.progressDrafts.length = 0; mockState.getGlobalHookRunner.mockReturnValue(null); mockState.runtimeCore = createRuntimeCore(testConfig); @@ -568,6 +589,146 @@ describe("mattermost inbound user posts", () => { }); }); + it("preserves abandon retry accounting, backoff, threshold, and restart behavior", async () => { + vi.useFakeTimers(); + const now = Date.UTC(2026, 0, 2); + vi.setSystemTime(now); + const created = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mattermost-abandon-")); + const stateDir = await fs.realpath(created); + type Payload = { version: 1; receivedAt: number; rawEvent: string }; + const queue = createChannelIngressQueueForTests({ + channelId: "mattermost", + accountId: "default", + stateDir, + }); + mockState.ingressQueue = queue; + mockState.runtimeCore = createRuntimeCore(testConfig, undefined, { + inboundDebounceMs: 0, + createInboundDebouncer, + }); + mockState.dispatchInboundMessage.mockRejectedValue( + new Error("Mattermost dispatch failed before adoption"), + ); + + const activeProviders: Array<{ stop: () => Promise }> = []; + const startProvider = async () => { + const socket = new FakeWebSocket(); + const abortController = new AbortController(); + const monitor = monitorMattermostProvider({ + config: testConfig, + runtime: testRuntime(), + abortSignal: abortController.signal, + webSocketFactory: () => socket, + }); + for (let tick = 0; tick < 20 && socket.openListenerCount === 0; tick += 1) { + await Promise.resolve(); + } + expect(socket.openListenerCount).toBeGreaterThan(0); + socket.emitOpen(); + let stopped = false; + const provider = { + socket, + stop: async () => { + if (stopped) { + return; + } + stopped = true; + abortController.abort(); + socket.emitClose(1000); + await monitor; + }, + }; + activeProviders.push(provider); + return provider; + }; + const send = async (provider: Awaited>) => { + await emitMattermostChannelPost(provider.socket, { + id: "post-abandon-retry", + message: "retry me", + }); + }; + const pendingAttempt = async (attempts: number) => { + let observed: Awaited>[number] | undefined; + await vi.waitFor(async () => { + const pending = await queue.listPending({ limit: "all" }); + expect(pending).toEqual([ + expect.objectContaining({ + id: "post-abandon-retry", + attempts, + lastAttemptAt: expect.any(Number), + lastError: "turn-abandoned", + }), + ]); + observed = pending[0]; + }); + const lastAttemptAt = observed?.lastAttemptAt; + if (lastAttemptAt === undefined) { + throw new Error(`Missing Mattermost retry timestamp for attempt ${attempts}`); + } + return { ...observed, lastAttemptAt }; + }; + + try { + const first = await startProvider(); + await send(first); + const firstAttempt = await pendingAttempt(1); + expect(mockState.dispatchInboundMessage).toHaveBeenCalledTimes(1); + await first.stop(); + + vi.setSystemTime(firstAttempt.lastAttemptAt + 999); + const blocked = await startProvider(); + await send(blocked); + await vi.advanceTimersByTimeAsync(0); + expect(mockState.dispatchInboundMessage).toHaveBeenCalledTimes(1); + await blocked.stop(); + + vi.setSystemTime(firstAttempt.lastAttemptAt + 1_001); + const second = await startProvider(); + await send(second); + const secondAttempt = await pendingAttempt(2); + expect(mockState.dispatchInboundMessage).toHaveBeenCalledTimes(2); + await second.stop(); + + for (let attempt = 3; attempt < DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS; attempt += 1) { + const claim = await queue.claim("post-abandon-retry", { ownerId: `seed-${attempt}` }); + if (!claim) { + throw new Error(`Expected Mattermost seed claim ${attempt}`); + } + await queue.release(claim, { + lastError: "turn-abandoned", + releasedAt: secondAttempt.lastAttemptAt, + }); + } + + vi.setSystemTime(secondAttempt.lastAttemptAt + 64_001); + const threshold = await startProvider(); + await send(threshold); + const thresholdAttempt = await pendingAttempt(DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS); + expect(mockState.dispatchInboundMessage).toHaveBeenCalledTimes(3); + await threshold.stop(); + + vi.setSystemTime(thresholdAttempt.lastAttemptAt + 128_001); + const beyond = await startProvider(); + await send(beyond); + const beyondAttempt = await pendingAttempt(DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS + 1); + expect(mockState.dispatchInboundMessage).toHaveBeenCalledTimes(4); + await beyond.stop(); + + vi.setSystemTime(beyondAttempt.lastAttemptAt + 1_000); + const blockedRestart = await startProvider(); + await send(blockedRestart); + await vi.advanceTimersByTimeAsync(0); + expect(mockState.dispatchInboundMessage).toHaveBeenCalledTimes(4); + await blockedRestart.stop(); + } finally { + await Promise.allSettled(activeProviders.map(async (provider) => await provider.stop())); + mockState.ingressQueue = undefined; + closeOpenClawStateDatabaseForTest(); + await fs.rm(stateDir, { recursive: true, force: true }); + vi.useRealTimers(); + } + }); + it("publishes recovering while API authentication retries, including 401", async () => { const abortController = new AbortController(); const statusSink = vi.fn(); @@ -1879,7 +2040,10 @@ describe("mattermost inbound user posts", () => { chatmode: "onmessage", dmPolicy: "open", groupPolicy: "open", - streaming: { mode: "block", preview: { toolProgress: true } }, + streaming: { + mode: "block", + preview: { toolProgress: true, commandText: "raw" }, + }, }, }, }; diff --git a/extensions/memory-wiki/src/query.test.ts b/extensions/memory-wiki/src/query.test.ts index d3c0f3eb925f..f9016241115b 100644 --- a/extensions/memory-wiki/src/query.test.ts +++ b/extensions/memory-wiki/src/query.test.ts @@ -1785,6 +1785,29 @@ describe("getMemoryWikiPage", () => { ); }); + it("reports a clean error instead of crashing for malformed wiki_get params", async () => { + const { config } = await createQueryVault({ + initialize: true, + config: { search: { backend: "shared", corpus: "memory" } }, + }); + const tool = createWikiGetTool(config, createAppConfig()); + const malformedParams = [ + null, + { path: "sources/example/note.md" }, + { lookup: " " }, + { lookup: 42 }, + ]; + + for (const params of malformedParams) { + const result = await tool.execute("wiki-get-bad-param", params); + + expect(result.details).toEqual({ found: false }); + expect(result.content).toEqual([ + { type: "text", text: "wiki_get requires a non-empty `lookup` path or id." }, + ]); + } + }); + it("normalizes extensionless shared memory lookups before reading", async () => { const { config } = await createQueryVault({ initialize: true, diff --git a/extensions/memory-wiki/src/tool.ts b/extensions/memory-wiki/src/tool.ts index 8be1545fe13c..cb11e6dfe884 100644 --- a/extensions/memory-wiki/src/tool.ts +++ b/extensions/memory-wiki/src/tool.ts @@ -2,6 +2,7 @@ import path from "node:path"; import { optionalFiniteNumberSchema } from "openclaw/plugin-sdk/channel-actions"; import type { OpenClawPluginToolContext } from "openclaw/plugin-sdk/plugin-entry"; +import { asNonArrayRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; import { Type } from "typebox"; import type { AnyAgentTool, OpenClawConfig } from "../api.js"; import { applyMemoryWikiMutation, normalizeMemoryWikiMutationInput } from "./apply.js"; @@ -277,13 +278,20 @@ export function createWikiGetTool( "Read a wiki page by id or relative path, or fall back to the active memory corpus when shared search is enabled.", parameters: WikiGetSchema, execute: async (_toolCallId, rawParams) => { - const params = rawParams as { - lookup: string; + const params = asNonArrayRecord(rawParams) as { + lookup?: string; fromLine?: number; lineCount?: number; backend?: ResolvedMemoryWikiConfig["search"]["backend"]; corpus?: ResolvedMemoryWikiConfig["search"]["corpus"]; }; + const lookup = typeof params.lookup === "string" ? params.lookup.trim() : ""; + if (!lookup) { + return { + content: [{ type: "text", text: "wiki_get requires a non-empty `lookup` path or id." }], + details: { found: false }, + }; + } await syncImportedSourcesIfNeeded(config, appConfig); const result = await getMemoryWikiPage({ config, @@ -292,7 +300,7 @@ export function createWikiGetTool( agentSessionKey: memoryContext.agentSessionKey, sandboxed: memoryContext.sandboxed, conversationRecall: memoryContext.conversationRecall, - lookup: params.lookup, + lookup, fromLine: params.fromLine, lineCount: params.lineCount, ...(params.backend ? { searchBackend: params.backend } : {}), @@ -300,7 +308,7 @@ export function createWikiGetTool( }); if (!result) { return { - content: [{ type: "text", text: `Wiki page not found: ${params.lookup}` }], + content: [{ type: "text", text: `Wiki page not found: ${lookup}` }], details: { found: false }, }; } diff --git a/extensions/msteams/src/monitor-handler/message-handler.ingress-lifecycle.test.ts b/extensions/msteams/src/monitor-handler/message-handler.ingress-lifecycle.test.ts index 662217fb5ccc..1c9c5afc4031 100644 --- a/extensions/msteams/src/monitor-handler/message-handler.ingress-lifecycle.test.ts +++ b/extensions/msteams/src/monitor-handler/message-handler.ingress-lifecycle.test.ts @@ -1,7 +1,16 @@ // Microsoft Teams tests cover durable claim ownership through inbound debounce. +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import { createInboundDebouncer } from "openclaw/plugin-sdk/channel-inbound-debounce"; +import { DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS } from "openclaw/plugin-sdk/channel-outbound"; +import { + closeOpenClawStateDatabaseForTest, + createChannelIngressQueueForTests, +} from "openclaw/plugin-sdk/plugin-state-test-runtime"; import { beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../../runtime-api.js"; +import { createMSTeamsIngress } from "../msteams-ingress.js"; import type { MSTeamsIngressLifecycle } from "../msteams-ingress.js"; import type { MSTeamsTurnContext } from "../sdk-types.js"; import "./message-handler-mock-support.test-support.js"; @@ -156,4 +165,125 @@ describe("Microsoft Teams drain claim ownership", () => { expect(runtimeApiMockState.dispatchReplyWithBufferedBlockDispatcher).not.toHaveBeenCalled(); expect(lifecycle.abandonedCount()).toBe(0); }); + + it("preserves abandon retry accounting, backoff, threshold, and restart behavior", async () => { + vi.useFakeTimers(); + const now = Date.UTC(2026, 0, 2); + vi.setSystemTime(now); + const created = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-msteams-abandon-")); + const stateDir = await fs.realpath(created); + type Queue = NonNullable[0]["queue"]>; + type Payload = Parameters[1]; + const queue = createChannelIngressQueueForTests({ + channelId: "msteams", + accountId: "test-app", + stateDir, + }); + const incoming = directActivity("activity-abandon", "retry me"); + await queue.enqueue( + "activity-abandon", + { version: 1, receivedAt: now - 2 * 24 * 60 * 60_000, rawActivity: JSON.stringify(incoming) }, + { laneKey: "dm-conversation", receivedAt: now - 2 * 24 * 60 * 60_000 }, + ); + const dispatchMock = runtimeApiMockState.dispatchReplyWithBufferedBlockDispatcher; + const priorImplementation = dispatchMock.getMockImplementation(); + dispatchMock.mockRejectedValue(new Error("Microsoft Teams dispatch failed before adoption")); + + const createIntegratedIngress = () => { + const handler = createHandler({ + channels: { msteams: { dmPolicy: "open", allowFrom: ["*"] } }, + } as OpenClawConfig); + return createMSTeamsIngress({ + accountId: "test-app", + queue, + runtime: { error: vi.fn(), log: vi.fn() }, + dispatch: async (activity, lifecycle) => await handler(context(activity), lifecycle), + }); + }; + const expectPendingAttempt = async (attempts: number) => { + let observed: Awaited>[number] | undefined; + await vi.waitFor(async () => { + const pending = await queue.listPending({ limit: "all" }); + expect(pending).toEqual([ + expect.objectContaining({ + id: "activity-abandon", + attempts, + lastAttemptAt: expect.any(Number), + lastError: "turn-abandoned", + }), + ]); + observed = pending[0]; + }); + const lastAttemptAt = observed?.lastAttemptAt; + if (lastAttemptAt === undefined) { + throw new Error(`Missing Microsoft Teams retry timestamp for attempt ${attempts}`); + } + return { ...observed, lastAttemptAt }; + }; + + try { + const first = createIntegratedIngress(); + first.start(); + const firstAttempt = await expectPendingAttempt(1); + expect(dispatchMock).toHaveBeenCalledTimes(1); + await first.stop(); + + vi.setSystemTime(firstAttempt.lastAttemptAt + 999); + const second = createIntegratedIngress(); + second.start(); + await second.accept(incoming); + await vi.advanceTimersByTimeAsync(0); + expect(dispatchMock).toHaveBeenCalledTimes(1); + await second.stop(); + vi.setSystemTime(firstAttempt.lastAttemptAt + 1_001); + const afterBackoff = createIntegratedIngress(); + afterBackoff.start(); + await afterBackoff.accept(incoming); + const secondAttempt = await expectPendingAttempt(2); + expect(dispatchMock).toHaveBeenCalledTimes(2); + await afterBackoff.stop(); + + for (let attempt = 3; attempt < DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS; attempt += 1) { + const claim = await queue.claim("activity-abandon", { ownerId: `seed-${attempt}` }); + if (!claim) { + throw new Error(`Expected Microsoft Teams seed claim ${attempt}`); + } + await queue.release(claim, { + lastError: "turn-abandoned", + releasedAt: secondAttempt.lastAttemptAt, + }); + } + vi.setSystemTime(secondAttempt.lastAttemptAt + 64_001); + const threshold = createIntegratedIngress(); + threshold.start(); + await threshold.accept(incoming); + const thresholdAttempt = await expectPendingAttempt(DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS); + expect(dispatchMock).toHaveBeenCalledTimes(3); + await threshold.stop(); + + vi.setSystemTime(thresholdAttempt.lastAttemptAt + 128_001); + const beyond = createIntegratedIngress(); + beyond.start(); + await beyond.accept(incoming); + const beyondAttempt = await expectPendingAttempt(DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS + 1); + expect(dispatchMock).toHaveBeenCalledTimes(4); + await beyond.stop(); + + vi.setSystemTime(beyondAttempt.lastAttemptAt + 1_000); + const blockedRestart = createIntegratedIngress(); + blockedRestart.start(); + await blockedRestart.accept(incoming); + await vi.advanceTimersByTimeAsync(0); + expect(dispatchMock).toHaveBeenCalledTimes(4); + await blockedRestart.stop(); + } finally { + dispatchMock.mockReset(); + if (priorImplementation) { + dispatchMock.mockImplementation(priorImplementation); + } + closeOpenClawStateDatabaseForTest(); + await fs.rm(stateDir, { recursive: true, force: true }); + vi.useRealTimers(); + } + }); }); diff --git a/extensions/signal/src/monitor/event-handler.reply-session-conflict.test.ts b/extensions/signal/src/monitor/event-handler.reply-session-conflict.test.ts index 81b079006673..00cd07e05580 100644 --- a/extensions/signal/src/monitor/event-handler.reply-session-conflict.test.ts +++ b/extensions/signal/src/monitor/event-handler.reply-session-conflict.test.ts @@ -1,6 +1,15 @@ -import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; // Signal tests cover retry behavior for reply session initialization conflicts. +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS } from "openclaw/plugin-sdk/channel-outbound"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { + closeOpenClawStateDatabaseForTest, + createChannelIngressQueueForTests, +} from "openclaw/plugin-sdk/plugin-state-test-runtime"; import { beforeEach, describe, expect, it, vi } from "vitest"; +import { startSignalIngressMonitor } from "../signal-ingress.js"; import type { SignalEventHandlerDeps } from "./event-handler.types.js"; const [ @@ -326,6 +335,127 @@ describe("signal reply session init conflict retry", () => { } }); + it("preserves durable abandon accounting through backoff, threshold, and restart", async () => { + vi.useFakeTimers(); + const now = Date.UTC(2026, 0, 2); + vi.setSystemTime(now); + const created = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-signal-abandon-")); + const stateDir = await fs.realpath(created); + type Queue = NonNullable[0]["queue"]>; + type Payload = Parameters[1]; + const queue = createChannelIngressQueueForTests({ + channelId: "signal", + accountId: "default", + stateDir, + }); + const timestamp = 1_700_000_000_777; + const event = createSignalReceiveEvent({ + timestamp, + dataMessage: { timestamp, message: "retry through durable ingress", attachments: [] }, + }); + const eventId = JSON.stringify(["number:+15550001111", timestamp]); + dispatchInboundMessageMock.mockRejectedValue(CONFLICT_ERROR); + + const createIntegratedMonitor = async () => { + const tracked = createTrackedTaskHarness(); + const handler = createSignalEventHandler( + createBaseSignalEventHandlerDeps({ + cfg: { messages: { inbound: { debounceMs: 10 } } }, + runTrackedTask: tracked.runTrackedTask, + }), + ); + const monitor = await startSignalIngressMonitor({ + accountId: "default", + queue, + dispatch: async (incoming, lifecycle) => await handler(incoming, lifecycle), + runtime: { error: vi.fn(), log: vi.fn() }, + }); + return { monitor, tracked }; + }; + const finishOuterAttempt = async (tracked: ReturnType) => { + await vi.advanceTimersByTimeAsync(10); + expect(tracked.tasks).toHaveLength(1); + await vi.advanceTimersByTimeAsync(7_000); + await Promise.all(tracked.tasks); + }; + const pendingAttempt = async (attempts: number) => { + const pending = await queue.listPending({ limit: "all" }); + expect(pending).toEqual([ + expect.objectContaining({ + id: eventId, + attempts, + lastAttemptAt: expect.any(Number), + lastError: "turn-abandoned", + }), + ]); + const record = pending[0]; + const lastAttemptAt = record?.lastAttemptAt; + if (lastAttemptAt === undefined) { + throw new Error(`Missing Signal retry timestamp for attempt ${attempts}`); + } + return { ...record, lastAttemptAt }; + }; + + try { + const first = await createIntegratedMonitor(); + await first.monitor.receive(event); + await finishOuterAttempt(first.tracked); + const firstAttempt = await pendingAttempt(1); + expect(dispatchInboundMessageMock).toHaveBeenCalledTimes(4); + await first.monitor.stop(); + + vi.setSystemTime(firstAttempt.lastAttemptAt + 999); + const blocked = await createIntegratedMonitor(); + await vi.advanceTimersByTimeAsync(10); + expect(dispatchInboundMessageMock).toHaveBeenCalledTimes(4); + expect(blocked.tracked.tasks).toHaveLength(0); + await blocked.monitor.stop(); + + vi.setSystemTime(firstAttempt.lastAttemptAt + 1_001); + const second = await createIntegratedMonitor(); + await finishOuterAttempt(second.tracked); + const secondAttempt = await pendingAttempt(2); + expect(dispatchInboundMessageMock).toHaveBeenCalledTimes(8); + await second.monitor.stop(); + + for (let attempt = 3; attempt < DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS; attempt += 1) { + const claim = await queue.claim(eventId, { ownerId: `seed-${attempt}` }); + if (!claim) { + throw new Error(`Expected Signal seed claim ${attempt}`); + } + await queue.release(claim, { + lastError: "turn-abandoned", + releasedAt: secondAttempt.lastAttemptAt, + }); + } + + vi.setSystemTime(secondAttempt.lastAttemptAt + 64_001); + const threshold = await createIntegratedMonitor(); + await finishOuterAttempt(threshold.tracked); + const thresholdAttempt = await pendingAttempt(DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS); + expect(dispatchInboundMessageMock).toHaveBeenCalledTimes(12); + await threshold.monitor.stop(); + + vi.setSystemTime(thresholdAttempt.lastAttemptAt + 128_001); + const beyond = await createIntegratedMonitor(); + await finishOuterAttempt(beyond.tracked); + const beyondAttempt = await pendingAttempt(DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS + 1); + expect(dispatchInboundMessageMock).toHaveBeenCalledTimes(16); + await beyond.monitor.stop(); + + vi.setSystemTime(beyondAttempt.lastAttemptAt + 1_000); + const blockedRestart = await createIntegratedMonitor(); + await vi.advanceTimersByTimeAsync(10); + expect(dispatchInboundMessageMock).toHaveBeenCalledTimes(16); + expect(blockedRestart.tracked.tasks).toHaveLength(0); + await blockedRestart.monitor.stop(); + } finally { + closeOpenClawStateDatabaseForTest(); + await fs.rm(stateDir, { recursive: true, force: true }); + vi.useRealTimers(); + } + }); + it("does not retry non-conflict flush failures", async () => { dispatchInboundMessageMock.mockRejectedValue(new Error("some other dispatch failure")); diff --git a/extensions/telegram/src/bot.create-telegram-bot.test.ts b/extensions/telegram/src/bot.create-telegram-bot.test.ts index 472b8ef389a8..d6cf75bbb9c8 100644 --- a/extensions/telegram/src/bot.create-telegram-bot.test.ts +++ b/extensions/telegram/src/bot.create-telegram-bot.test.ts @@ -2259,7 +2259,7 @@ describe("createTelegramBot", () => { defaults: { model: "openai/gpt-4.1", }, - list: [{ id: "agent-a" }, { id: "agent-b" }], + list: [{ id: "agent-a", default: true }, { id: "agent-b" }], }, channels: { telegram: { dmPolicy: "open", allowFrom: ["*"] }, @@ -3689,7 +3689,7 @@ describe("createTelegramBot", () => { }, }, agents: { - list: [{ id: "agent-a" }, { id: "agent-b" }], + list: [{ id: "agent-a", default: true }, { id: "agent-b" }], }, bindings: [ { @@ -3763,8 +3763,9 @@ describe("createTelegramBot", () => { }, }, agents: { - list: [{ id: "topic-a" }, { id: "topic-b" }], + list: [{ id: "topic-a", default: true }, { id: "topic-b" }], }, + bindings: [{ agentId: "topic-a", match: { channel: "telegram", accountId: "default" } }], }); loadConfig.mockImplementation(configForTopicAgent); @@ -5052,7 +5053,7 @@ describe("createTelegramBot", () => { }, }, agents: { - list: [{ id: "agent-a" }, { id: "agent-b" }], + list: [{ id: "agent-a", default: true }, { id: "agent-b" }], }, bindings: [ { diff --git a/scripts/lib/swift-toolchain.sh b/scripts/lib/swift-toolchain.sh index 30ac5462b068..6c01c555e53b 100644 --- a/scripts/lib/swift-toolchain.sh +++ b/scripts/lib/swift-toolchain.sh @@ -4,7 +4,9 @@ REQUIRED_SWIFT_TOOLS_MAJOR=6 REQUIRED_SWIFT_TOOLS_MINOR=2 require_swift_toolchain() { - if ! xcrun xcodebuild -version >/dev/null 2>&1; then + local xcodebuild_version + if ! xcodebuild_version="$(xcrun xcodebuild -version 2>&1)"; then + printf '%s\n' "$xcodebuild_version" >&2 echo "ERROR: OpenClaw macOS app packaging requires a full Xcode developer directory." >&2 echo " Command Line Tools do not include the required SwiftUI macro plugins." >&2 echo " Use: sudo xcode-select -s /Applications/Xcode.app/Contents/Developer" >&2 diff --git a/scripts/package-openclaw-for-docker.mts b/scripts/package-openclaw-for-docker.mts index 9347666825b7..0e3bfb5dc99a 100644 --- a/scripts/package-openclaw-for-docker.mts +++ b/scripts/package-openclaw-for-docker.mts @@ -1,10 +1,10 @@ #!/usr/bin/env node -// Builds the OpenClaw package artifact used by Docker E2E. -// The script owns the build/inventory/pack sequence so local scheduler, shell -// helpers, and GitHub Actions all prepare the exact same npm tarball. +// Builds the canonical OpenClaw package artifact used by Docker E2E. import { spawn } from "node:child_process"; +import { closeSync, openSync } from "node:fs"; import fs from "node:fs/promises"; import { createRequire } from "node:module"; +import os from "node:os"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV } from "./lib/bundled-plugin-build-entries.mjs"; @@ -32,10 +32,10 @@ const AI_RUNTIME_BACKUP_DIR = ".openclaw-ai-package-backup"; type KillChild = (signal: NodeJS.Signals) => void; type RunOptions = { captureStdout?: boolean; - deferForwardedSignalExit?: boolean; env?: NodeJS.ProcessEnv; killAfterMs?: unknown; maxCapturedStdoutBytes?: number; + stdoutFilePath?: string; timeoutMs?: unknown; }; type CommandRunnerOptions = { @@ -48,15 +48,11 @@ type CommandRunner = ( cwd: string, options: CommandRunnerOptions, ) => Promise; -type CaptureRunnerOptions = { - deferForwardedSignalExit?: boolean; - timeoutMs?: number; -}; type RunImpl = ( command: string, args: string[], cwd: string, - options: CaptureRunnerOptions, + options: RunOptions, ) => Promise; type DocsMapLifecycle = { preparePackageDocsMap: (cwd: string) => Promise; @@ -305,6 +301,13 @@ export function parseArgs(argv: string[]) { } function run(command: string, args: string[], cwd: string, options: RunOptions = {}) { + const setupError = + options.captureStdout && options.stdoutFilePath + ? new Error("captureStdout and stdoutFilePath cannot be combined") + : forwardedSignalExitCode && new ForwardedSignalExitError(forwardedSignalExitCode); + if (setupError) { + return Promise.reject(setupError); + } return new Promise((resolve, reject) => { const resolvedTimeoutMs = resolveOptionalTimerTimeoutMs(options.timeoutMs); const resolvedKillAfterMs = resolvePackageBuildTimeoutMs( @@ -326,14 +329,22 @@ function run(command: string, args: string[], cwd: string, options: RunOptions = : process.platform === "win32" && command === "npm" ? resolveNpmRunner({ env, npmArgs: args }) : { args, command, shell: false }; - const child = spawn(invocation.command, invocation.args, { - cwd, - stdio: ["ignore", "pipe", "pipe"], - env: invocation.env ?? env, - detached: useProcessGroup, - shell: invocation.shell, - windowsVerbatimArguments: invocation.windowsVerbatimArguments, - }); + const stdoutFd = options.stdoutFilePath ? openSync(options.stdoutFilePath, "wx") : undefined; + let child: ReturnType; + try { + child = spawn(invocation.command, invocation.args, { + cwd, + stdio: ["ignore", stdoutFd ?? "pipe", "pipe"], + env: invocation.env ?? env, + detached: useProcessGroup, + shell: invocation.shell, + windowsVerbatimArguments: invocation.windowsVerbatimArguments, + }); + } finally { + if (stdoutFd !== undefined) { + closeSync(stdoutFd); + } + } let timedOut = false; let outputLimitExceeded = false; let stdout = ""; @@ -354,10 +365,6 @@ function run(command: string, args: string[], cwd: string, options: RunOptions = } ACTIVE_CHILD_KILLERS.delete(killChild); if (forwardedSignalExitCode !== undefined && ACTIVE_CHILD_KILLERS.size === 0) { - if (options.deferForwardedSignalExit) { - reject(new ForwardedSignalExitError(forwardedSignalExitCode)); - return; - } process.exit(forwardedSignalExitCode); } if (error) { @@ -423,7 +430,7 @@ function run(command: string, args: string[], cwd: string, options: RunOptions = finish(error, value); }; if (options.captureStdout) { - child.stdout.on("data", (chunk) => { + child.stdout?.on("data", (chunk) => { if (outputLimitExceeded) { return; } @@ -437,10 +444,10 @@ function run(command: string, args: string[], cwd: string, options: RunOptions = stdout += chunkText; stdoutBytes += chunkBytes; }); - } else { - child.stdout.pipe(process.stderr, { end: false }); + } else if (!options.stdoutFilePath) { + child.stdout?.pipe(process.stderr, { end: false }); } - child.stderr.pipe(process.stderr, { end: false }); + child.stderr?.pipe(process.stderr, { end: false }); child.on("error", (error) => finish(error)); child.on("close", (status, signal) => { if (timedOut) { @@ -503,12 +510,12 @@ export async function buildPackageArtifacts( } } -export const runCommandForTest = run; - async function runCapture(command: string, args: string[], cwd: string, options: RunOptions = {}) { - return await run(command, args, cwd, { ...options, captureStdout: true }); + return await run(command, args, cwd, { ...options, captureStdout: !options.stdoutFilePath }); } +export { run as runCommandForTest, runCapture as runCaptureForTest }; + async function newestOpenClawTarball(outputDir: string, packOutput: string) { let fromOutput = ""; try { @@ -740,7 +747,6 @@ export async function prepareBundledAiRuntimePackage( ], sourceDir, { - deferForwardedSignalExit: true, timeoutMs: resolveTimeoutMs( "OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS", DEFAULT_PACKAGE_PACK_TIMEOUT_MS, @@ -859,11 +865,9 @@ async function loadSourcePackageLifecycle( } function packagePreparationRestoreError(error: unknown, restoreError: unknown) { - return new AggregateError( - [error, restoreError], - "Package preparation failed and source artifacts could not be restored.", - { cause: error }, - ); + return new AggregateError([error, restoreError], "Package operation and cleanup both failed.", { + cause: error, + }); } export async function packOpenClawPackageForDocker( @@ -921,6 +925,14 @@ export async function packOpenClawPackageForDocker( console.error("==> Packing OpenClaw package"); // This receipt is the package lifecycle lock; acquire it before touching CHANGELOG.md. await prepareDocsMap(sourcePath); + const deferSignalExit: KillChild = () => {}; + ACTIVE_CHILD_KILLERS.add(deferSignalExit); + const releaseSignalExit = () => { + ACTIVE_CHILD_KILLERS.delete(deferSignalExit); + if (forwardedSignalExitCode !== undefined) { + throw new ForwardedSignalExitError(forwardedSignalExitCode); + } + }; try { await prepareManifest(sourcePath); await prepareChangelog(sourcePath); @@ -933,66 +945,99 @@ export async function packOpenClawPackageForDocker( restoreChangelog, ); } catch (restoreError) { + releaseSignalExit(); throw packagePreparationRestoreError(error, restoreError); } + releaseSignalExit(); throw error; } let packOutput = ""; - let cleanupBundledAiRuntime = async () => {}; + let packageError: unknown; + let packReceiptDir: string | undefined; try { - await cleanPackedOpenClawTarballs(outputPath); - cleanupBundledAiRuntime = await prepareBundledAiRuntime( - sourcePath, + let cleanupBundledAiRuntime = async () => {}; + try { + await cleanPackedOpenClawTarballs(outputPath); + cleanupBundledAiRuntime = await prepareBundledAiRuntime( + sourcePath, + outputPath, + runCaptureImpl, + { + prepareManifest, + restoreManifest, + }, + ); + const packArgs = + packTool === "pnpm" + ? ["pack", "--silent", "--config.ignore-scripts=true", "--pack-destination", outputPath] + : [ + "pack", + ...(packageOptions.packJsonPath ? ["--json"] : []), + "--silent", + "--ignore-scripts", + "--pack-destination", + outputPath, + ]; + if (packTool === "npm" && packageOptions.packJsonPath) { + packReceiptDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-npm-pack-receipt-")); + } + const packReceiptPath = packReceiptDir ? path.join(packReceiptDir, "pack.json") : undefined; + packOutput = await runCaptureImpl(packTool, packArgs, sourcePath, { + stdoutFilePath: packReceiptPath, + timeoutMs: resolveTimeoutMs( + "OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS", + DEFAULT_PACKAGE_PACK_TIMEOUT_MS, + ), + }); + if (packReceiptPath) { + packOutput = await fs.readFile(packReceiptPath, "utf8"); + } + } finally { + try { + await cleanupBundledAiRuntime(); + } finally { + await restorePackageSourceArtifacts( + sourcePath, + restoreDocsMap, + restoreManifest, + restoreChangelog, + ); + } + } + // Scan the emptied pnpm destination instead of trusting its absolute-path output. + let tarball = await newestOpenClawTarball( outputPath, - runCaptureImpl, - { - prepareManifest, - restoreManifest, - }, + packageOptions.pnpmPack ? "" : packOutput, ); - const packArgs = - packTool === "pnpm" - ? ["pack", "--silent", "--config.ignore-scripts=true", "--pack-destination", outputPath] - : [ - "pack", - ...(packageOptions.packJsonPath ? ["--json"] : []), - "--silent", - "--ignore-scripts", - "--pack-destination", - outputPath, - ]; - packOutput = await runCaptureImpl(packTool, packArgs, sourcePath, { - deferForwardedSignalExit: true, - timeoutMs: resolveTimeoutMs( - "OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS", - DEFAULT_PACKAGE_PACK_TIMEOUT_MS, - ), - }); + if (packageOptions.outputName) { + const target = path.join(outputPath, packageOptions.outputName); + if (target !== tarball) { + await fs.rm(target, { force: true }); + await fs.rename(tarball, target); + tarball = target; + } + } + await writePackJson(packOutput, tarball, packageOptions.packJsonPath, sourcePath); + return tarball; + } catch (error) { + packageError = error; + throw error; } finally { try { - await cleanupBundledAiRuntime(); + if (packReceiptDir) { + try { + await fs.rm(packReceiptDir, { force: true, recursive: true }); + } catch (cleanupError) { + // oxlint-disable-next-line eslint/no-unsafe-finally -- Preserve primary and cleanup failures. + throw packageError + ? packagePreparationRestoreError(packageError, cleanupError) + : cleanupError; + } + } } finally { - await restorePackageSourceArtifacts( - sourcePath, - restoreDocsMap, - restoreManifest, - restoreChangelog, - ); + releaseSignalExit(); } } - // pnpm reports an absolute destination path. The directory was emptied before packing, - // so scan that controlled destination instead of accepting a path from command output. - let tarball = await newestOpenClawTarball(outputPath, packageOptions.pnpmPack ? "" : packOutput); - if (packageOptions.outputName) { - const target = path.join(outputPath, packageOptions.outputName); - if (target !== tarball) { - await fs.rm(target, { force: true }); - await fs.rename(tarball, target); - tarball = target; - } - } - await writePackJson(packOutput, tarball, packageOptions.packJsonPath, sourcePath); - return tarball; } export async function writePackageInventoryForDocker( diff --git a/src/channels/message/ingress-drain-lifecycle.test.ts b/src/channels/message/ingress-drain-lifecycle.test.ts new file mode 100644 index 000000000000..9cec5f32693d --- /dev/null +++ b/src/channels/message/ingress-drain-lifecycle.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { bindIngressLifecycleToReplyOptions } from "./ingress-drain-lifecycle.js"; + +describe("channel ingress drain lifecycle", () => { + it("binds only the reply-lane ownership surface", async () => { + const abort = new AbortController(); + const calls: string[] = []; + const bound = bindIngressLifecycleToReplyOptions({ + abortSignal: abort.signal, + onAdoptionFinalizing: () => { + calls.push("finalizing"); + }, + onFailed: () => { + calls.push("failed"); + }, + onCancelled: () => { + calls.push("cancelled"); + }, + onAdopted: () => { + calls.push("adopted"); + }, + onDeferred: () => { + calls.push("deferred"); + }, + onAbandoned: () => { + calls.push("abandoned"); + }, + }); + + expect(bound.turnAdoptionLifecycle).toMatchObject({ + admission: "exclusive", + abortSignal: abort.signal, + }); + expect("onFailed" in bound.turnAdoptionLifecycle).toBe(false); + expect("onCancelled" in bound.turnAdoptionLifecycle).toBe(false); + expect("onAdopted" in bound).toBe(false); + expect(Object.keys(bound)).toEqual(["turnAdoptionLifecycle"]); + bound.turnAdoptionLifecycle.onDeferred(); + await bound.turnAdoptionLifecycle.onAbandoned(); + expect(calls).toEqual(["deferred", "abandoned"]); + calls.length = 0; + bound.turnAdoptionLifecycle.onDeferred(); + await bound.turnAdoptionLifecycle.onAdopted(); + expect(calls).toEqual(["deferred", "adopted"]); + }); +}); diff --git a/src/channels/message/ingress-drain-lifecycle.ts b/src/channels/message/ingress-drain-lifecycle.ts new file mode 100644 index 000000000000..0090847ef3a7 --- /dev/null +++ b/src/channels/message/ingress-drain-lifecycle.ts @@ -0,0 +1,55 @@ +/** Full pre-adoption -> adoption ownership lifecycle for one claimed event. */ +export type ChannelIngressDispatchLifecycle = { + /** Pre-adoption only. After adopt the drain treats this signal as inert. */ + abortSignal: AbortSignal; + /** + * Fires when recovery-relevant session/run state is durable. + * Drain completes (tombstones) the claim here -- never at settle. + */ + onAdopted: () => void | Promise; + /** + * Turn ownership deferred to reply-lane admission (queued followup). + * Claim remains held until adopted or abandoned. + */ + onDeferred: () => void; + /** + * Durable adoption finalization is in progress (e.g. settlement hold while + * committing dedupe). Clears the pre-adoption stall watchdog so a timeout + * settlement cannot race and dead-letter an about-to-complete claim. + * Claim stays held until onAdopted / onAbandoned / fail. + */ + onAdoptionFinalizing: () => void; + /** Deferred work terminally failed after dispatch returned. */ + onFailed?: (error: unknown) => void | Promise; + /** Explicit cancellation before adoption; releases without consuming retry budget. */ + onCancelled?: () => void | Promise; + /** + * Deferred turn finished without ever owning the reply lane. + * Drain releases the claim for retry. + */ + onAbandoned: () => void | Promise; +}; + +/** Maps a drain lifecycle onto the reply-lane ownership surface. */ +export function bindIngressLifecycleToReplyOptions(lifecycle: ChannelIngressDispatchLifecycle): { + turnAdoptionLifecycle: { + admission: "exclusive"; + onAdopted: () => void | Promise; + onDeferred: () => void; + onAbandoned: () => void | Promise; + abortSignal: AbortSignal; + }; +} { + return { + turnAdoptionLifecycle: { + admission: "exclusive", + onAdopted: lifecycle.onAdopted, + onDeferred: lifecycle.onDeferred, + onAbandoned: lifecycle.onAbandoned, + abortSignal: lifecycle.abortSignal, + }, + }; +} + +// onAdoptionFinalizing stays drain-only (not reply-options); channels call it +// via the spooled-replay ALS lifecycle frame during settlement hold. diff --git a/src/channels/message/ingress-drain.cancellation.test.ts b/src/channels/message/ingress-drain.cancellation.test.ts new file mode 100644 index 000000000000..bcb6363f3741 --- /dev/null +++ b/src/channels/message/ingress-drain.cancellation.test.ts @@ -0,0 +1,89 @@ +import { expectDefined } from "@openclaw/normalization-core"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js"; +import { createChannelIngressDrain } from "./ingress-drain.js"; +import { + createTestIngressQueue, + type IngressDrainTestPayload as Payload, + withTempState, +} from "./ingress-drain.test-helpers.js"; + +type ChannelIngressDispatchLifecycle = Parameters< + Parameters[0]["dispatchClaimedEvent"] +>[1]; + +describe("channel ingress drain cancellation", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + closeOpenClawStateDatabaseForTest(); + }); + + it("cancels unadopted work without changing its retry facts", async () => { + await withTempState(async (stateDir) => { + let clock = 100; + const queue = createTestIngressQueue(stateDir, { now: () => clock }); + await queue.enqueue("evt-cancel", { text: "x" }, { laneKey: "l1", receivedAt: 1 }); + const failedClaim = await queue.claim("evt-cancel", { ownerId: "failed-owner" }); + expect(failedClaim).not.toBeNull(); + if (!failedClaim) { + return; + } + await queue.release(failedClaim, { lastError: "previous failure", releasedAt: clock }); + const before = (await queue.listPending())[0]; + for (let cycle = 0; cycle < 3; cycle += 1) { + const lifecycles: ChannelIngressDispatchLifecycle[] = []; + clock += 1; + const drain = createChannelIngressDrain({ + queue, + now: () => clock, + retryPolicy: { baseMs: 0, maxMs: 0 }, + dispatchClaimedEvent: async (_event, lifecycle) => { + lifecycles.push(lifecycle); + return { kind: "deferred" }; + }, + }); + + await drain.drainOnce(); + await vi.waitFor(() => expect(lifecycles).toHaveLength(1)); + await expectDefined( + expectDefined(lifecycles[0], "cancelled lifecycle").onCancelled, + "cancel callback", + )(); + expect(await queue.listPending()).toEqual([ + expect.objectContaining({ + id: "evt-cancel", + attempts: before?.attempts, + lastAttemptAt: before?.lastAttemptAt, + lastError: before?.lastError, + }), + ]); + expect(await queue.listClaims()).toEqual([]); + drain.dispose(); + } + + const terminal = createChannelIngressDrain({ + queue, + now: () => clock, + retryPolicy: { maxAttempts: 2, deadLetterMinAgeMs: 0, baseMs: 0, maxMs: 0 }, + dispatchClaimedEvent: async () => { + throw new Error("final genuine failure"); + }, + }); + await terminal.drainOnce(); + await terminal.waitForIdle(); + expect(await queue.listFailed?.()).toEqual([ + expect.objectContaining({ + id: "evt-cancel", + attempts: 1, + reason: "retry-limit-exceeded", + message: "final genuine failure", + }), + ]); + terminal.dispose(); + }); + }); +}); diff --git a/src/channels/message/ingress-drain.test.ts b/src/channels/message/ingress-drain.test.ts index 2d6e6cc790c3..4baa332b51de 100644 --- a/src/channels/message/ingress-drain.test.ts +++ b/src/channels/message/ingress-drain.test.ts @@ -3,7 +3,6 @@ import { expectDefined } from "@openclaw/normalization-core"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js"; import { - bindIngressLifecycleToReplyOptions, createChannelIngressDrain, DEFAULT_INGRESS_ADOPTION_STALL_MS, isIngressAdoptionLostError, @@ -170,6 +169,84 @@ describe("channel ingress drain", () => { }); }); + it("keeps the lane owned until a dead-letter write commits", async () => { + await withTempState(async (stateDir) => { + const queue = createTestIngressQueue(stateDir); + await queue.enqueue("poison", { text: "bad" }, { laneKey: "shared", receivedAt: 1 }); + await queue.enqueue("follower", { text: "good" }, { laneKey: "shared", receivedAt: 2 }); + const fail = queue.fail.bind(queue); + let failAttempts = 0; + queue.fail = async (...args) => { + failAttempts += 1; + if (failAttempts < 3) { + throw new Error(`transient fail write ${failAttempts}`); + } + return await fail(...args); + }; + const dispatched: string[] = []; + const drain = createChannelIngressDrain({ + queue, + retryPolicy: { maxAttempts: 1, deadLetterMinAgeMs: 0 }, + dispatchClaimedEvent: async (event, lifecycle) => { + dispatched.push(event.id); + if (event.id === "poison") { + throw new Error("poison delivery"); + } + await lifecycle.onAdopted(); + }, + }); + + await drain.drainOnce(); + const idle = drain.waitForIdle(); + await vi.advanceTimersByTimeAsync(0); + expect(failAttempts).toBe(1); + expect(drain.activeLaneKeys()).toEqual(new Set(["shared"])); + expect(await drain.drainOnce()).toEqual({ started: 0 }); + expect(dispatched).toEqual(["poison"]); + + await vi.advanceTimersByTimeAsync(5_000); + await idle; + expect(failAttempts).toBe(3); + expect(await drain.drainOnce()).toEqual({ started: 1 }); + await drain.waitForIdle(); + expect(dispatched).toEqual(["poison", "follower"]); + drain.dispose(); + }); + }); + + it("keeps ownership when every dead-letter write fails", async () => { + await withTempState(async (stateDir) => { + const queue = createTestIngressQueue(stateDir); + await queue.enqueue("poison", { text: "bad" }, { laneKey: "shared", receivedAt: 1 }); + await queue.enqueue("follower", { text: "good" }, { laneKey: "shared", receivedAt: 2 }); + queue.fail = async () => { + throw new Error("persistent fail write"); + }; + const dispatched: string[] = []; + const drain = createChannelIngressDrain({ + queue, + retryPolicy: { maxAttempts: 1, deadLetterMinAgeMs: 0 }, + dispatchClaimedEvent: async (event) => { + dispatched.push(event.id); + throw new Error("poison delivery"); + }, + }); + + await drain.drainOnce(); + const idle = drain.waitForIdle(); + for (let attempt = 0; attempt < 8; attempt += 1) { + await vi.advanceTimersByTimeAsync(180_000); + } + await idle; + + expect(dispatched).toEqual(["poison"]); + expect(drain.activeLaneKeys()).toEqual(new Set(["shared"])); + expect((await queue.listClaims()).map((claim) => claim.id)).toEqual(["poison"]); + expect(await drain.drainOnce()).toEqual({ started: 0 }); + drain.dispose(); + }); + }); + it("holds lanes by default and releases only opted-in deferred lanes", async () => { for (const occupancy of ["hold", "release"] as const) { await withTempState(async (stateDir) => { @@ -378,7 +455,7 @@ describe("channel ingress drain", () => { }); }); - it("abandoned via turnAdoptionLifecycle releases claim with attempt increment", async () => { + it("abandoned reply ownership releases claim with attempt increment", async () => { await withTempState(async (stateDir) => { const queue = createTestIngressQueue(stateDir); await queue.enqueue("evt-q", { text: "x" }, { laneKey: "l1" }); @@ -386,10 +463,9 @@ describe("channel ingress drain", () => { const drain = createChannelIngressDrain({ queue, dispatchClaimedEvent: async (_event, lifecycle) => { - const bound = bindIngressLifecycleToReplyOptions(lifecycle); - bound.turnAdoptionLifecycle.onDeferred(); + lifecycle.onDeferred(); // Never admitted — abandon path releases claim. - await bound.turnAdoptionLifecycle.onAbandoned(); + await lifecycle.onAbandoned(); return { kind: "deferred" }; }, }); @@ -405,7 +481,7 @@ describe("channel ingress drain", () => { }); }); - it("queued deferral→admission completes the claim exactly once via turnAdoptionLifecycle", async () => { + it("queued deferral -> admission completes the claim exactly once", async () => { await withTempState(async (stateDir) => { const queue = createTestIngressQueue(stateDir); await queue.enqueue("evt-admit", { text: "x" }, { laneKey: "l1" }); @@ -414,13 +490,12 @@ describe("channel ingress drain", () => { const drain = createChannelIngressDrain({ queue, dispatchClaimedEvent: async (_event, lifecycle) => { - const bound = bindIngressLifecycleToReplyOptions(lifecycle); // Simulate queue enqueue (defer) then reply-lane admission (adopt). - bound.turnAdoptionLifecycle.onDeferred(); - await bound.turnAdoptionLifecycle.onAdopted(); + lifecycle.onDeferred(); + await lifecycle.onAdopted(); adoptCount += 1; // Second adopt from lifecycle must be a no-op for the claim. - await bound.turnAdoptionLifecycle.onAdopted(); + await lifecycle.onAdopted(); adoptCount += 1; return { kind: "deferred" }; }, @@ -440,105 +515,6 @@ describe("channel ingress drain", () => { }); }); - it("watchdog only guillotines pre-adoption stalls with handler-timeout", async () => { - await withTempState(async (stateDir) => { - let clock = 10_000; - const queue = createTestIngressQueue(stateDir, { now: () => clock }); - await queue.enqueue("evt-stall", { text: "x" }, { laneKey: "l1" }); - - const drain = createChannelIngressDrain({ - queue, - now: () => clock, - adoptionStallTimeoutMs: 5_000, - dispatchClaimedEvent: async () => { - // Never adopt, never return — stall until watchdog. - await new Promise(() => {}); - }, - }); - - await drain.drainOnce(); - clock += 5_000; - await vi.advanceTimersByTimeAsync(5_000); - await drain.waitForIdle(); - - // Failed tombstone, not pending retry. - const reenqueue = await queue.enqueue("evt-stall", { text: "x" }); - expect(reenqueue.kind).toBe("failed"); - if (reenqueue.kind === "failed") { - expect(reenqueue.record.reason).toBe("handler-timeout"); - } - drain.dispose(); - }); - }); - - it("watchdog guillotines deferred phase (timer not cleared by deferral)", async () => { - await withTempState(async (stateDir) => { - let clock = 30_000; - const queue = createTestIngressQueue(stateDir, { now: () => clock }); - await queue.enqueue("evt-def-stall", { text: "x" }, { laneKey: "l1" }); - - const drain = createChannelIngressDrain({ - queue, - now: () => clock, - adoptionStallTimeoutMs: 5_000, - dispatchClaimedEvent: async (_event, lifecycle) => { - lifecycle.onDeferred(); - // Stay deferred without adoption — watchdog must still fire. - await new Promise(() => {}); - }, - }); - - await drain.drainOnce(); - expect(await queue.listClaims()).toHaveLength(1); - clock += 5_000; - await vi.advanceTimersByTimeAsync(5_000); - await drain.waitForIdle(); - - const reenqueue = await queue.enqueue("evt-def-stall", { text: "x" }); - expect(reenqueue.kind).toBe("failed"); - if (reenqueue.kind === "failed") { - expect(reenqueue.record.reason).toBe("handler-timeout"); - } - drain.dispose(); - }); - }); - - it("watchdog does not kill healthy long turns after adoption", async () => { - await withTempState(async (stateDir) => { - let clock = 20_000; - const queue = createTestIngressQueue(stateDir, { now: () => clock }); - await queue.enqueue("evt-long", { text: "x" }, { laneKey: "l1" }); - - let settleResolve!: () => void; - const settleGate = new Promise((resolve) => { - settleResolve = resolve; - }); - - const drain = createChannelIngressDrain({ - queue, - now: () => clock, - adoptionStallTimeoutMs: 1_000, - dispatchClaimedEvent: async (_event, lifecycle) => { - await lifecycle.onAdopted(); - await settleGate; - }, - }); - - await drain.drainOnce(); - await vi.waitFor(async () => { - expect(await queue.listClaims()).toEqual([]); - }); - clock += 60_000; - await vi.advanceTimersByTimeAsync(60_000); - // Still only completed — not failed by watchdog. - const status = await queue.enqueue("evt-long", { text: "x" }); - expect(status.kind).toBe("completed"); - settleResolve(); - await drain.waitForIdle(); - drain.dispose(); - }); - }); - it("supersede tombstones the superseded claim (never re-dispatches)", async () => { await withTempState(async (stateDir) => { const queue = createTestIngressQueue(stateDir); @@ -697,39 +673,37 @@ describe("channel ingress drain", () => { }); }); - it("bindIngressLifecycleToReplyOptions returns only turnAdoptionLifecycle", async () => { - const abort = new AbortController(); - const calls: string[] = []; - const bound = bindIngressLifecycleToReplyOptions({ - abortSignal: abort.signal, - onAdoptionFinalizing: () => { - calls.push("finalizing"); - }, - onFailed: () => { - calls.push("failed"); - }, - onAdopted: () => { - calls.push("adopted"); - }, - onDeferred: () => { - calls.push("deferred"); - }, - onAbandoned: () => { - calls.push("abandoned"); - }, + it("keeps retry-accounted abandonment pending beyond the failure threshold", async () => { + await withTempState(async (stateDir) => { + let clock = 1; + const queue = createTestIngressQueue(stateDir, { now: () => clock }); + await queue.enqueue("abandoned", { text: "x" }, { laneKey: "l", receivedAt: 1 }); + + for (let attempt = 0; attempt < 3; attempt += 1) { + clock += 1; + const drain = createChannelIngressDrain({ + queue, + now: () => clock, + retryPolicy: { maxAttempts: 1, deadLetterMinAgeMs: 0, baseMs: 0, maxMs: 0 }, + dispatchClaimedEvent: async (_event, lifecycle) => { + await lifecycle.onAbandoned(); + return { kind: "deferred" }; + }, + }); + await drain.drainOnce(); + await drain.waitForIdle(); + drain.dispose(); + } + + expect(await queue.listPending()).toEqual([ + expect.objectContaining({ + id: "abandoned", + attempts: 3, + lastError: "turn-abandoned", + }), + ]); + expect(await queue.listFailed?.()).toEqual([]); }); - expect(bound.turnAdoptionLifecycle.abortSignal).toBe(abort.signal); - expect(bound.turnAdoptionLifecycle.admission).toBe("exclusive"); - expect("onFailed" in bound.turnAdoptionLifecycle).toBe(false); - expect("onAdopted" in bound).toBe(false); - expect(Object.keys(bound)).toEqual(["turnAdoptionLifecycle"]); - bound.turnAdoptionLifecycle.onDeferred(); - await bound.turnAdoptionLifecycle.onAbandoned(); - expect(calls).toEqual(["deferred", "abandoned"]); - calls.length = 0; - bound.turnAdoptionLifecycle.onDeferred(); - await bound.turnAdoptionLifecycle.onAdopted(); - expect(calls).toEqual(["deferred", "adopted"]); }); it("refreshes active claims on claimLeaseMs/3 while deferred", async () => { @@ -1141,17 +1115,4 @@ describe("channel ingress drain", () => { drain.dispose(); }); }); - - it("bindIngressLifecycleToReplyOptions marks exclusive admission", () => { - const abort = new AbortController(); - const bound = bindIngressLifecycleToReplyOptions({ - abortSignal: abort.signal, - onAdopted: async () => {}, - onDeferred: () => {}, - onAdoptionFinalizing: () => {}, - onFailed: () => {}, - onAbandoned: () => {}, - }); - expect(bound.turnAdoptionLifecycle.admission).toBe("exclusive"); - }); }); diff --git a/src/channels/message/ingress-drain.ts b/src/channels/message/ingress-drain.ts index b6cdd089f41f..d713e3a1ec42 100644 --- a/src/channels/message/ingress-drain.ts +++ b/src/channels/message/ingress-drain.ts @@ -15,6 +15,7 @@ import { isLiveLocalIngressDrainOwner, registerLiveIngressDrainInstance, } from "./ingress-claim-owner.js"; +import type { ChannelIngressDispatchLifecycle } from "./ingress-drain-lifecycle.js"; import { activeClaimKey, IngressAdoptionLostError, @@ -25,6 +26,7 @@ import { type ChannelIngressDrainDispatchResult, } from "./ingress-drain-state.js"; import { supersedeActiveStatesIfNeeded } from "./ingress-drain-supersede.js"; +export { bindIngressLifecycleToReplyOptions } from "./ingress-drain-lifecycle.js"; export { isIngressAdoptionLostError } from "./ingress-drain-state.js"; import type { ChannelIngressQueue, @@ -46,36 +48,6 @@ export const DEFAULT_INGRESS_ADOPTION_STALL_MS = 5 * 60 * 1000; /** Bounded tombstone write retries — wedged ownership beats silent double-dispatch. */ const INGRESS_TOMBSTONE_RETRY_MAX_ATTEMPTS = 8; -/** Full pre-adoption → adoption ownership lifecycle for one claimed event. */ -type ChannelIngressDispatchLifecycle = { - /** Pre-adoption only. After adopt the drain treats this signal as inert. */ - abortSignal: AbortSignal; - /** - * Fires when recovery-relevant session/run state is durable. - * Drain completes (tombstones) the claim here — never at settle. - */ - onAdopted: () => void | Promise; - /** - * Turn ownership deferred to reply-lane admission (queued followup). - * Claim remains held until adopted or abandoned. - */ - onDeferred: () => void; - /** - * Durable adoption finalization is in progress (e.g. settlement hold while - * committing dedupe). Clears the pre-adoption stall watchdog so a timeout - * settlement cannot race and dead-letter an about-to-complete claim. - * Claim stays held until onAdopted / onAbandoned / fail. - */ - onAdoptionFinalizing: () => void; - /** Deferred work terminally failed after dispatch returned. */ - onFailed?: (error: unknown) => void | Promise; - /** - * Deferred turn finished without ever owning the reply lane. - * Drain releases the claim for retry. - */ - onAbandoned: () => void | Promise; -}; - type DeferredLaneOccupancy = "hold" | "release"; export type CreateChannelIngressDrainOptions< @@ -132,34 +104,6 @@ export type ChannelIngressDrain = { dispose: () => void; }; -/** - * Maps a drain lifecycle onto reply options. - * Single surface: turnAdoptionLifecycle only. - * Marks exclusive admission so collect isolation is not inferred from onAbandoned. - */ -export function bindIngressLifecycleToReplyOptions(lifecycle: ChannelIngressDispatchLifecycle): { - turnAdoptionLifecycle: { - admission: "exclusive"; - onAdopted: () => void | Promise; - onDeferred: () => void; - onAbandoned: () => void | Promise; - abortSignal: AbortSignal; - }; -} { - return { - turnAdoptionLifecycle: { - admission: "exclusive", - onAdopted: lifecycle.onAdopted, - onDeferred: lifecycle.onDeferred, - onAbandoned: lifecycle.onAbandoned, - abortSignal: lifecycle.abortSignal, - }, - }; -} - -// onAdoptionFinalizing stays drain-only (not reply-options); channels call it -// via the spooled-replay ALS lifecycle frame during settlement hold. - /** Creates a channel-agnostic durable ingress drain over an existing queue. */ export function createChannelIngressDrain< TPayload, @@ -347,13 +291,12 @@ export function createChannelIngressDrain< const releaseClaim = async ( claim: ChannelIngressQueueClaim, - lastError?: string, + releaseOptions?: { lastError?: string; recordAttempt?: boolean }, ) => { await commitClaimWriteWithRetry({ claim, label: "release", - write: () => - queue.release(claim, lastError === undefined ? {} : { lastError, releasedAt: now() }), + write: () => queue.release(claim, { ...releaseOptions, releasedAt: now() }), falseMeansReclaimed: false, }); }; @@ -401,7 +344,7 @@ export function createChannelIngressDrain< } const displayId = claim.id.replace(/^0+(?=\d)/, "") || claim.id; log(`spooled update ${displayId} failed; keeping for retry: ${disposition.message}`); - await releaseClaim(claim, disposition.message); + await releaseClaim(claim, { lastError: disposition.message }); }; const createSettleOwner = ( @@ -468,6 +411,24 @@ export function createChannelIngressDrain< state.stallTimer.unref?.(); }; + const releaseUnadopted = async ( + state: ActiveHandlerState, + releaseOptions: { lastError?: string; recordAttempt?: boolean }, + ) => { + if (state.phase !== "deferred" && state.phase !== "dispatching") { + return; + } + if (state.guillotined || state.superseded) { + return; + } + clearStallTimer(state); + await state + .settleOnce(async () => { + await releaseClaim(state.claim, releaseOptions); + }) + .catch(() => undefined); + }; + const createLifecycle = ( state: ActiveHandlerState, ): ChannelIngressDispatchLifecycle => { @@ -528,19 +489,13 @@ export function createChannelIngressDrain< await applyFailureDisposition(state.claim, error); }); }, + onCancelled: async () => { + // Cancellation means ownership ended before delivery, so preserve every + // prior retry fact while reopening the canonical row for replacement. + await releaseUnadopted(state, { recordAttempt: false }); + }, onAbandoned: async () => { - if (state.phase !== "deferred" && state.phase !== "dispatching") { - return; - } - if (state.guillotined || state.superseded) { - return; - } - clearStallTimer(state); - await state - .settleOnce(async () => { - await releaseClaim(state.claim, "turn-abandoned"); - }) - .catch(() => undefined); + await releaseUnadopted(state, { lastError: "turn-abandoned" }); }, }; }; diff --git a/src/channels/message/ingress-drain.watchdog.test.ts b/src/channels/message/ingress-drain.watchdog.test.ts new file mode 100644 index 000000000000..49f5788ab7da --- /dev/null +++ b/src/channels/message/ingress-drain.watchdog.test.ts @@ -0,0 +1,116 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js"; +import { createChannelIngressDrain } from "./ingress-drain.js"; +import { + createTestIngressQueue, + type IngressDrainTestPayload as Payload, + withTempState, +} from "./ingress-drain.test-helpers.js"; + +describe("channel ingress drain watchdog", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + closeOpenClawStateDatabaseForTest(); + }); + + it("guillotines pre-adoption stalls with handler-timeout", async () => { + await withTempState(async (stateDir) => { + let clock = 10_000; + const queue = createTestIngressQueue(stateDir, { now: () => clock }); + await queue.enqueue("evt-stall", { text: "x" }, { laneKey: "l1" }); + + const drain = createChannelIngressDrain({ + queue, + now: () => clock, + adoptionStallTimeoutMs: 5_000, + dispatchClaimedEvent: async () => { + // Never adopt, never return -- stall until watchdog. + await new Promise(() => {}); + }, + }); + + await drain.drainOnce(); + clock += 5_000; + await vi.advanceTimersByTimeAsync(5_000); + await drain.waitForIdle(); + + const reenqueue = await queue.enqueue("evt-stall", { text: "x" }); + expect(reenqueue.kind).toBe("failed"); + if (reenqueue.kind === "failed") { + expect(reenqueue.record.reason).toBe("handler-timeout"); + } + drain.dispose(); + }); + }); + + it("guillotines deferred stalls", async () => { + await withTempState(async (stateDir) => { + let clock = 30_000; + const queue = createTestIngressQueue(stateDir, { now: () => clock }); + await queue.enqueue("evt-def-stall", { text: "x" }, { laneKey: "l1" }); + + const drain = createChannelIngressDrain({ + queue, + now: () => clock, + adoptionStallTimeoutMs: 5_000, + dispatchClaimedEvent: async (_event, lifecycle) => { + lifecycle.onDeferred(); + // Stay deferred without adoption -- watchdog must still fire. + await new Promise(() => {}); + }, + }); + + await drain.drainOnce(); + expect(await queue.listClaims()).toHaveLength(1); + clock += 5_000; + await vi.advanceTimersByTimeAsync(5_000); + await drain.waitForIdle(); + + const reenqueue = await queue.enqueue("evt-def-stall", { text: "x" }); + expect(reenqueue.kind).toBe("failed"); + if (reenqueue.kind === "failed") { + expect(reenqueue.record.reason).toBe("handler-timeout"); + } + drain.dispose(); + }); + }); + + it("does not kill healthy long turns after adoption", async () => { + await withTempState(async (stateDir) => { + let clock = 20_000; + const queue = createTestIngressQueue(stateDir, { now: () => clock }); + await queue.enqueue("evt-long", { text: "x" }, { laneKey: "l1" }); + + let settleResolve!: () => void; + const settleGate = new Promise((resolve) => { + settleResolve = resolve; + }); + + const drain = createChannelIngressDrain({ + queue, + now: () => clock, + adoptionStallTimeoutMs: 1_000, + dispatchClaimedEvent: async (_event, lifecycle) => { + await lifecycle.onAdopted(); + await settleGate; + }, + }); + + await drain.drainOnce(); + await vi.waitFor(async () => { + expect(await queue.listClaims()).toEqual([]); + }); + clock += 60_000; + await vi.advanceTimersByTimeAsync(60_000); + const status = await queue.enqueue("evt-long", { text: "x" }); + expect(status.kind).toBe("completed"); + settleResolve(); + await drain.waitForIdle(); + drain.dispose(); + }); + }); +}); diff --git a/src/channels/message/ingress-monitor.ts b/src/channels/message/ingress-monitor.ts index 07831bdf6423..31a1739d61a4 100644 --- a/src/channels/message/ingress-monitor.ts +++ b/src/channels/message/ingress-monitor.ts @@ -65,6 +65,7 @@ export type ChannelIngressMonitorLifecycle = { onDeferred: () => void; onAdoptionFinalizing: () => void; onFailed?: (error: unknown) => void | Promise; + onCancelled?: () => void | Promise; onAbandoned: () => void | Promise; }; @@ -368,6 +369,16 @@ export function createChannelIngressMonitor void | Promise) => { + handedOff = true; + deferredHandoff = true; + try { + await settle(); + requestDrain(); + } finally { + settleDeferredClaim(); + } + }; const wrappedLifecycle: ChannelIngressMonitorLifecycle = { ...lifecycle, admission: "exclusive", @@ -393,26 +404,9 @@ export function createChannelIngressMonitor { - handedOff = true; - deferredHandoff = true; - try { - await lifecycle.onFailed?.(error); - requestDrain(); - } finally { - settleDeferredClaim(); - } - }, - onAbandoned: async () => { - handedOff = true; - deferredHandoff = true; - try { - await lifecycle.onAbandoned(); - requestDrain(); - } finally { - settleDeferredClaim(); - } - }, + onFailed: (error) => settleDeferredLifecycle(() => lifecycle.onFailed?.(error)), + onCancelled: () => settleDeferredLifecycle(() => lifecycle.onCancelled?.()), + onAbandoned: () => settleDeferredLifecycle(() => lifecycle.onAbandoned()), }; // Adoption can complete before delivery returns; track both lifetimes so stop diff --git a/src/channels/message/ingress-queue.test.ts b/src/channels/message/ingress-queue.test.ts index 79dce21b52ea..8fd517b11c1a 100644 --- a/src/channels/message/ingress-queue.test.ts +++ b/src/channels/message/ingress-queue.test.ts @@ -119,32 +119,66 @@ describe("channel ingress queue", () => { it("keeps channel and account queue identities unambiguous", async () => { await withTempState(async (stateDir) => { const first = createChannelIngressQueue<{ text: string }>({ - channelId: "a", - accountId: "b:c", + channelId: "discord", + accountId: "account-a", stateDir, }); const second = createChannelIngressQueue<{ text: string }>({ - channelId: "a:b", - accountId: "c", + channelId: "discord", + accountId: "account-b", stateDir, }); - expect(await first.enqueue("same-id", { text: "first" })).toMatchObject({ + expect( + await first.enqueue("same-id", { text: "first" }, { laneKey: "channel:same-lane" }), + ).toMatchObject({ kind: "accepted", }); - expect(await second.enqueue("same-id", { text: "second" })).toMatchObject({ + expect( + await second.enqueue("same-id", { text: "second" }, { laneKey: "channel:same-lane" }), + ).toMatchObject({ kind: "accepted", }); - await first.complete("same-id"); + const firstClaim = await first.claim("same-id", { ownerId: "first-worker" }); + expect(firstClaim).not.toBeNull(); + if (!firstClaim) { + return; + } + await first.fail(firstClaim, { reason: "poison", failedAt: 20 }); expect(await first.enqueue("same-id", { text: "first duplicate" })).toMatchObject({ - kind: "completed", + kind: "failed", }); expect(await second.enqueue("same-id", { text: "second duplicate" })).toMatchObject({ kind: "pending", record: { payload: { text: "second" } }, }); + + if (!first.resubmit) { + return; + } + await expect(first.resubmit("same-id", { resubmittedAt: 30 })).resolves.toMatchObject({ + kind: "resubmitted", + record: { attempts: 0, laneKey: "channel:same-lane", payload: { text: "first" } }, + }); + const resubmittedClaim = await first.claim("same-id", { ownerId: "replacement" }); + const secondClaim = await second.claim("same-id", { ownerId: "second-worker" }); + expect(resubmittedClaim).not.toBeNull(); + expect(secondClaim).not.toBeNull(); + if (!resubmittedClaim || !secondClaim) { + return; + } + await first.fail(resubmittedClaim, { reason: "poison-again", failedAt: 40 }); + await second.complete(secondClaim, { completedAt: 40 }); + + expect(await first.prune({ failedTtlMs: 1, now: 42 })).toBe(1); + expect(await first.enqueue("same-id", { text: "fresh after prune" })).toMatchObject({ + kind: "accepted", + }); + expect(await second.enqueue("same-id", { text: "completed duplicate" })).toMatchObject({ + kind: "completed", + }); }); }); diff --git a/src/config/io.compat.test.ts b/src/config/io.compat.test.ts index f4f69a1165cb..119c61960d68 100644 --- a/src/config/io.compat.test.ts +++ b/src/config/io.compat.test.ts @@ -2,10 +2,45 @@ import fs from "node:fs/promises"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { withTempDir } from "../test-utils/temp-dir.js"; import { VERSION } from "../version.js"; -import { createConfigIO } from "./io.js"; +import { createConfigIO } from "./io.factory.js"; import { normalizeExecSafeBinProfilesInConfig } from "./normalize-exec-safe-bin.js"; -import { withTempHome } from "./test-helpers.js"; + +vi.mock("../commands/doctor/shared/legacy-config-compat.js", () => ({ + applyLegacyDoctorMigrations: () => { + throw new Error("config IO compatibility tests must not enter recovery migration"); + }, +})); + +vi.mock("../plugins/gateway-startup-plugin-ids.js", () => ({ + createConfigValidationMetadataPluginIdScope: (params: { + config: { plugins?: { entries?: Record } }; + }) => { + const configuredPluginIds = Object.keys(params.config.plugins?.entries ?? {}).toSorted(); + const removedPluginIds = new Set(["google-antigravity-auth", "google-gemini-cli-auth"]); + if (configuredPluginIds.some((pluginId) => !removedPluginIds.has(pluginId))) { + throw new Error("config IO compatibility tests require real metadata for active plugins"); + } + return { + key: `io-compat:${configuredPluginIds.join(",")}`, + resolve: () => [], + }; + }, +})); + +vi.mock("../plugins/plugin-metadata-snapshot.js", () => ({ + rebasePluginMetadataSnapshotManifestRegistry: () => { + throw new Error("config IO compatibility tests must not materialize metadata snapshots"); + }, + resolvePluginMetadataSnapshot: () => ({ + manifestRegistry: { plugins: [], diagnostics: [] }, + }), +})); + +function withTempHome(run: (home: string) => Promise): Promise { + return withTempDir("openclaw-config-compat-", run); +} async function writeConfig( home: string, @@ -61,71 +96,27 @@ describe("config io paths", () => { }); }); - it.each(["lan", "loopback", "tailnet", "auto", "custom", undefined] as const)( - "keeps canonical gateway bind %s byte-identical during load", - async (bind) => { - await withTempHome(async (home) => { - const configPath = path.join(home, ".openclaw", "openclaw.json"); - await fs.mkdir(path.dirname(configPath), { recursive: true }); - const gateway = { - mode: "local" as const, - ...(bind ? { bind } : {}), - ...(bind === "custom" ? { customBindHost: "127.0.0.1" } : {}), - }; - const raw = `${JSON.stringify({ gateway }, null, 2)}\n`; - await fs.writeFile(configPath, raw, "utf-8"); - const io = createConfigIO({ - configPath, - env: { HOME: home } as NodeJS.ProcessEnv, - homedir: () => home, - }); - - const config = io.loadConfig(); - - expect(config.gateway?.bind).toBe(bind); - await expect(fs.readFile(configPath, "utf-8")).resolves.toBe(raw); - }); - }, - ); - - it("logs validation warnings with real line breaks", async () => { + it("keeps canonical custom gateway bind byte-identical during load", async () => { await withTempHome(async (home) => { const configPath = path.join(home, ".openclaw", "openclaw.json"); await fs.mkdir(path.dirname(configPath), { recursive: true }); - await fs.writeFile( - configPath, - JSON.stringify( - { - plugins: { - entries: { - "google-antigravity-auth": { - enabled: false, - config: { stale: true }, - }, - }, - }, - }, - null, - 2, - ), - ); - const logger = { - error: vi.fn(), - warn: vi.fn(), + const gateway = { + mode: "local" as const, + bind: "custom" as const, + customBindHost: "127.0.0.1", }; - + const raw = `${JSON.stringify({ gateway }, null, 2)}\n`; + await fs.writeFile(configPath, raw, "utf-8"); const io = createConfigIO({ configPath, env: { HOME: home } as NodeJS.ProcessEnv, homedir: () => home, - logger, }); - io.loadConfig(); - expect(logger.warn).toHaveBeenCalledWith( - "Config warnings:\n- plugins.entries.google-antigravity-auth: plugin removed: google-antigravity-auth (stale config entry ignored; remove it from plugins config)", - ); - expect(logger.warn).not.toHaveBeenCalledWith("Config warnings:\\n"); + const config = io.loadConfig(); + + expect(config.gateway).toMatchObject({ bind: "custom", customBindHost: "127.0.0.1" }); + await expect(fs.readFile(configPath, "utf-8")).resolves.toBe(raw); }); }); @@ -155,6 +146,9 @@ describe("config io paths", () => { load(); load(); expect(logger.warn).toHaveBeenCalledTimes(1); + expect(logger.warn).toHaveBeenCalledWith( + "Config warnings:\n- plugins.entries.google-antigravity-auth: plugin removed: google-antigravity-auth (stale config entry ignored; remove it from plugins config)", + ); createConfigIO({ configPath, diff --git a/src/gateway/worker-environments/device-provider.ts b/src/gateway/worker-environments/device-provider.ts index cdab0474b960..53fbeeeabbd2 100644 --- a/src/gateway/worker-environments/device-provider.ts +++ b/src/gateway/worker-environments/device-provider.ts @@ -10,6 +10,7 @@ import type { NodeWorkerSupervisorNodeProof, NodeWorkerSupervisorTransport, } from "../node-registry-private.js"; +import { createNodeWorkerLaunchAdapter } from "./node-launch-adapter.js"; import type { WorkerEnvironmentServiceContract } from "./service-contract.js"; export const DEVICE_WORKER_PROVIDER_ID = "device"; @@ -61,6 +62,7 @@ function deviceLeaseId(deviceId: string, operationId: string): string { /** Core runtime for already-paired node hosts; pairing remains the durable trust owner. */ export function createDeviceWorkerRuntime(options: DeviceWorkerRuntimeOptions) { let nodeTransport: NodeWorkerSupervisorTransport | undefined; + const launchAdapter = createNodeWorkerLaunchAdapter({ getTransport: () => nodeTransport }); const findConnectedNode = async (deviceId: string) => (await nodeTransport?.listCurrentNodes())?.find( (node) => node.nodeId === deviceId && isSessionCapableNode(node), @@ -103,6 +105,7 @@ export function createDeviceWorkerRuntime(options: DeviceWorkerRuntimeOptions) { return { provider, isAvailable, + launchNodeWorker: launchAdapter.launch, bindNodeTransport: (transport: NodeWorkerSupervisorTransport) => { nodeTransport = transport; }, diff --git a/src/gateway/worker-environments/node-launch-adapter.test.ts b/src/gateway/worker-environments/node-launch-adapter.test.ts new file mode 100644 index 000000000000..1b04c7e172f7 --- /dev/null +++ b/src/gateway/worker-environments/node-launch-adapter.test.ts @@ -0,0 +1,431 @@ +import { describe, expect, it, vi } from "vitest"; +import { + GATEWAY_CLIENT_IDS, + GATEWAY_CLIENT_MODES, +} from "../../../packages/gateway-protocol/src/client-info.js"; +import { + WORKER_PROTOCOL_FEATURES, + WORKER_RPC_SET_VERSION, +} from "../../../packages/gateway-protocol/src/schema/worker-admission.js"; +import { NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE } from "../../infra/node-worker-supervisor-dialect.js"; +import { + nodeWorkerPlanHash, + type NodeWorkerLaunchInput, + type NodeWorkerSupervisorReceipt, +} from "../../worker/node-supervisor-protocol.js"; +import type { + NodeWorkerSupervisorNodeProof, + NodeWorkerSupervisorTransport, +} from "../node-registry-private.js"; +import { createNodeWorkerLaunchAdapter } from "./node-launch-adapter.js"; + +const DEVICE_ID = "device-session-host"; + +function nodeProof(connId = "conn-1"): NodeWorkerSupervisorNodeProof { + return { + nodeId: DEVICE_ID, + connId, + pairingIdentity: "identity-1", + pairingGeneration: "generation-1", + clientId: GATEWAY_CLIENT_IDS.NODE_HOST, + clientMode: GATEWAY_CLIENT_MODES.NODE, + protocolFeature: NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE, + commands: ["system.run"], + }; +} + +function launchInput(): NodeWorkerLaunchInput { + const bundleHash = "a".repeat(64); + return { + launchId: "turn-1", + gatewayNamespace: "gateway-1", + bundleHash, + placementGeneration: 4, + descriptor: { + version: 3, + connectionEndpoint: { + kind: "websocket", + url: "wss://gateway.example/__openclaw__/worker", + }, + admission: { + environmentId: "environment-1", + credential: "worker-fixture-value", + sessionId: "session-1", + ownerEpoch: 3, + rpcSetVersion: WORKER_RPC_SET_VERSION, + handshake: { + bundleHash, + openclawVersion: "2026.8.1", + protocolFeatures: [...WORKER_PROTOCOL_FEATURES], + }, + }, + assignment: { + agentId: "agent-1", + operationalRunInstance: { instanceId: "instance-1", runId: "run-1" }, + agentRuntimeIdentityToken: "signed-runtime-token", + runId: "run-1", + turnId: "turn-1", + prompt: "Inspect the workspace.", + suppressPromptTranscript: true, + workspaceDir: "/tmp/openclaw-worker/workspace", + modelRef: { provider: "provider-1", model: "model-1" }, + inferenceOptions: {}, + initialMessages: [], + transcript: { baseLeafId: null, nextSeq: 1 }, + liveEvents: { ackedSeq: 0, nextSeq: 1 }, + toolAuthority: { allowedToolNames: [] }, + }, + }, + }; +} + +function receipt( + input: NodeWorkerLaunchInput, + state: NodeWorkerSupervisorReceipt["state"], +): NodeWorkerSupervisorReceipt { + const identity = { + launchId: input.launchId, + planHash: nodeWorkerPlanHash(input), + environmentId: input.descriptor.admission.environmentId, + sessionId: input.descriptor.admission.sessionId, + ownerEpoch: input.descriptor.admission.ownerEpoch, + placementGeneration: input.placementGeneration, + runId: input.descriptor.assignment.runId, + }; + if (state === "completed") { + return { + ...identity, + state, + resultJson: JSON.stringify({ + status: "completed", + transcriptLeafId: "leaf-1", + transcriptNextSeq: 2, + }), + }; + } + if (state === "failed" || state === "interrupted" || state === "cancelled") { + return { ...identity, state, errorText: `worker ${state}` }; + } + return { ...identity, state }; +} + +function wire(payload: NodeWorkerSupervisorReceipt | null) { + return { ok: true, payloadJSON: JSON.stringify(payload) }; +} + +function transportWith( + invoke: NodeWorkerSupervisorTransport["invoke"], + listCurrentNodes: NodeWorkerSupervisorTransport["listCurrentNodes"] = async () => [nodeProof()], +): NodeWorkerSupervisorTransport { + return { invoke, listCurrentNodes }; +} + +function launchRequest(input = launchInput()) { + return { + deviceId: DEVICE_ID, + input, + isDispatchAuthorized: () => true, + isCancellationAuthorized: () => true, + timeoutMs: 10_000, + }; +} + +describe("node worker launch adapter", () => { + it("launches once, polls status, and returns the exact completed receipt", async () => { + const input = launchInput(); + const invoke = vi.fn(async (request) => + request.command === "worker.supervisor.launch.v1" + ? wire(receipt(input, "running")) + : wire(receipt(input, "completed")), + ); + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => transportWith(invoke), + sleep: async () => {}, + }); + + await expect(adapter.launch(launchRequest(input))).resolves.toEqual( + receipt(input, "completed"), + ); + expect(invoke.mock.calls.map(([request]) => request.command)).toEqual([ + "worker.supervisor.launch.v1", + "worker.supervisor.status.v1", + ]); + }); + + it("reacquires the node and replays the identical launch after ambiguous disconnect", async () => { + const input = launchInput(); + let launchCalls = 0; + const invoke = vi.fn(async (request) => { + if (request.command !== "worker.supervisor.launch.v1") { + throw new Error("unexpected status call"); + } + launchCalls += 1; + request.onDispatchReady?.(`invoke-${launchCalls}`); + return launchCalls === 1 + ? { ok: false, error: { code: "DISCONNECTED", message: "node disconnected" } } + : wire(receipt(input, "completed")); + }); + let listCalls = 0; + const listCurrentNodes = vi.fn(async () => [nodeProof(`conn-${++listCalls}`)]); + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => transportWith(invoke, listCurrentNodes), + sleep: async () => {}, + }); + + await expect(adapter.launch(launchRequest(input))).resolves.toEqual( + receipt(input, "completed"), + ); + expect(invoke).toHaveBeenCalledTimes(2); + expect(invoke.mock.calls[0]?.[0].params).toEqual(input); + expect(invoke.mock.calls[1]?.[0].params).toEqual(input); + expect(invoke.mock.calls[0]?.[0].node.connId).toBe("conn-1"); + expect(invoke.mock.calls[1]?.[0].node.connId).toBe("conn-2"); + }); + + it("snapshots the launch plan before asynchronous node discovery", async () => { + const input = launchInput(); + const expectedInput = structuredClone(input); + const invoke = vi.fn(async (request) => { + expect(request.params).toEqual(expectedInput); + return wire(receipt(expectedInput, "completed")); + }); + const listCurrentNodes = vi.fn(async () => { + input.descriptor.assignment.prompt = "mutated after launch call"; + return [nodeProof()]; + }); + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => transportWith(invoke, listCurrentNodes), + }); + + await expect(adapter.launch(launchRequest(input))).resolves.toEqual( + receipt(expectedInput, "completed"), + ); + expect(input.descriptor.assignment.prompt).toBe("mutated after launch call"); + }); + + it("replays launch when status cannot find the durable receipt", async () => { + const input = launchInput(); + const responses = [ + wire(receipt(input, "running")), + wire(null), + wire(receipt(input, "completed")), + ]; + const invoke = vi.fn(async () => responses.shift()!); + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => transportWith(invoke), + sleep: async () => {}, + }); + + await expect(adapter.launch(launchRequest(input))).resolves.toEqual( + receipt(input, "completed"), + ); + expect(invoke.mock.calls.map(([request]) => request.command)).toEqual([ + "worker.supervisor.launch.v1", + "worker.supervisor.status.v1", + "worker.supervisor.launch.v1", + ]); + }); + + it("cancels before rejecting a post-dispatch identity mismatch", async () => { + const input = launchInput(); + const mismatched = { ...receipt(input, "completed"), environmentId: "environment-other" }; + const invoke = vi.fn(async (request) => { + if (request.command === "worker.supervisor.cancel.v1") { + return wire(receipt(input, "cancelled")); + } + request.onDispatchReady?.("invoke-1"); + return wire(mismatched); + }); + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => transportWith(invoke), + }); + + await expect(adapter.launch(launchRequest(input))).rejects.toThrow( + "node worker supervisor receipt identity mismatch", + ); + expect(invoke.mock.calls.map(([request]) => request.command)).toEqual([ + "worker.supervisor.launch.v1", + "worker.supervisor.cancel.v1", + ]); + }); + + it("retries a timed-out launch RPC within the overall deadline", async () => { + const input = launchInput(); + let launchCalls = 0; + const invoke = vi.fn(async (request) => { + launchCalls += 1; + request.onDispatchReady?.(`invoke-${launchCalls}`); + return launchCalls === 1 + ? await new Promise(() => {}) + : wire(receipt(input, "completed")); + }); + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => transportWith(invoke), + rpcTimeoutMs: 10, + sleep: async () => {}, + }); + + await expect(adapter.launch({ ...launchRequest(input), timeoutMs: 100 })).resolves.toEqual( + receipt(input, "completed"), + ); + expect(invoke).toHaveBeenCalledTimes(2); + }); + + it("durably cancels after caller abort and returns the terminal cancellation receipt", async () => { + const input = launchInput(); + const controller = new AbortController(); + const invoke = vi.fn(async (request) => { + if (request.command === "worker.supervisor.cancel.v1") { + expect(request.params).toEqual( + expect.objectContaining({ + launchId: input.launchId, + planHash: nodeWorkerPlanHash(input), + }), + ); + return wire(receipt(input, "cancelled")); + } + request.onDispatchReady?.("invoke-1"); + return wire(receipt(input, "running")); + }); + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => transportWith(invoke), + sleep: async () => { + controller.abort(); + }, + }); + + await expect( + adapter.launch({ ...launchRequest(input), signal: controller.signal }), + ).resolves.toEqual(receipt(input, "cancelled")); + expect(invoke.mock.calls.at(-1)?.[0].command).toBe("worker.supervisor.cancel.v1"); + }); + + it("keeps cancelling through missing and active receipts until terminal", async () => { + const input = launchInput(); + const controller = new AbortController(); + const cancelResponses = [ + wire(null), + wire(receipt(input, "running")), + wire(receipt(input, "cancelled")), + ]; + const invoke = vi.fn(async (request) => { + if (request.command === "worker.supervisor.cancel.v1") { + return cancelResponses.shift()!; + } + request.onDispatchReady?.("invoke-1"); + return wire(receipt(input, "running")); + }); + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => transportWith(invoke), + pollIntervalMs: 1, + sleep: async () => { + controller.abort(); + }, + }); + + await expect( + adapter.launch({ ...launchRequest(input), signal: controller.signal }), + ).resolves.toEqual(receipt(input, "cancelled")); + expect( + invoke.mock.calls.filter(([request]) => request.command === "worker.supervisor.cancel.v1"), + ).toHaveLength(3); + }); + + it("retries a timed-out cancellation RPC within one cleanup deadline", async () => { + const input = launchInput(); + const controller = new AbortController(); + let cancelCalls = 0; + const invoke = vi.fn(async (request) => { + if (request.command === "worker.supervisor.cancel.v1") { + cancelCalls += 1; + return cancelCalls === 1 + ? await new Promise(() => {}) + : wire(receipt(input, "cancelled")); + } + request.onDispatchReady?.("invoke-1"); + return wire(receipt(input, "running")); + }); + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => transportWith(invoke), + rpcTimeoutMs: 10, + cancellationTimeoutMs: 100, + sleep: async () => { + controller.abort(); + }, + }); + + await expect( + adapter.launch({ ...launchRequest(input), signal: controller.signal }), + ).resolves.toEqual(receipt(input, "cancelled")); + expect(cancelCalls).toBe(2); + }); + + it("uses distinct cancellation authority after dispatch authority closes", async () => { + const input = launchInput(); + let dispatchAuthorized = true; + const cancelAuthorized = vi.fn(() => true); + const invoke = vi.fn(async (request) => { + if (request.command === "worker.supervisor.cancel.v1") { + return wire(receipt(input, "cancelled")); + } + request.onDispatchReady?.("invoke-1"); + return wire(receipt(input, "running")); + }); + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => transportWith(invoke), + sleep: async () => { + dispatchAuthorized = false; + }, + }); + + await expect( + adapter.launch({ + ...launchRequest(input), + isDispatchAuthorized: () => dispatchAuthorized, + isCancellationAuthorized: cancelAuthorized, + }), + ).resolves.toEqual(receipt(input, "cancelled")); + expect(cancelAuthorized).toHaveBeenCalled(); + expect(invoke.mock.calls.at(-1)?.[0].command).toBe("worker.supervisor.cancel.v1"); + }); + + it("bounds node discovery with the overall launch deadline", async () => { + const input = launchInput(); + const invoke = vi.fn(); + const listCurrentNodes = vi.fn(async () => await new Promise(() => {})); + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => transportWith(invoke, listCurrentNodes), + }); + + await expect(adapter.launch({ ...launchRequest(input), timeoutMs: 25 })).rejects.toThrow( + "node worker launch timed out", + ); + expect(invoke).not.toHaveBeenCalled(); + }); + + it("reports unknown cancellation outcome after a hard cancellation deadline", async () => { + const input = launchInput(); + const controller = new AbortController(); + const invoke = vi.fn(async (request) => { + if (request.command === "worker.supervisor.cancel.v1") { + return await new Promise(() => {}); + } + request.onDispatchReady?.("invoke-1"); + return wire(receipt(input, "running")); + }); + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => transportWith(invoke), + cancellationTimeoutMs: 25, + sleep: async () => { + controller.abort(); + }, + }); + + await expect( + adapter.launch({ ...launchRequest(input), signal: controller.signal }), + ).rejects.toThrow("node worker launch failed and cancellation could not be confirmed"); + expect( + invoke.mock.calls.filter(([request]) => request.command === "worker.supervisor.cancel.v1"), + ).toHaveLength(1); + }); +}); diff --git a/src/gateway/worker-environments/node-launch-adapter.ts b/src/gateway/worker-environments/node-launch-adapter.ts new file mode 100644 index 000000000000..56f823b6bf06 --- /dev/null +++ b/src/gateway/worker-environments/node-launch-adapter.ts @@ -0,0 +1,473 @@ +import { sleepWithAbort } from "../../infra/backoff.js"; +import { + NODE_WORKER_SUPERVISOR_CANCEL_COMMAND, + NODE_WORKER_SUPERVISOR_LAUNCH_COMMAND, + NODE_WORKER_SUPERVISOR_STATUS_COMMAND, +} from "../../infra/node-commands.js"; +import { + nodeWorkerPlanHash, + parseNodeWorkerLaunchInput, + parseNodeWorkerSupervisorReceipt, + type NodeWorkerLaunchInput, + type NodeWorkerSupervisorIdentity, + type NodeWorkerSupervisorReceipt, +} from "../../worker/node-supervisor-protocol.js"; +import type { + NodeWorkerSupervisorNodeProof, + NodeWorkerSupervisorTransport, +} from "../node-registry-private.js"; + +const DEFAULT_RPC_TIMEOUT_MS = 30_000; +const DEFAULT_POLL_INTERVAL_MS = 250; +const MAX_RETRY_DELAY_MS = 2_000; +const DEFAULT_CANCELLATION_TIMEOUT_MS = 30_000; + +const RETRYABLE_TRANSPORT_CODES = new Set([ + "DISCONNECTED", + "NOT_CONNECTED", + "PAIRING_CHANGED", + "PRIVATE_DIALECT_UNAVAILABLE", + "ROUTE_CHANGED", + "TIMEOUT", + "UNAVAILABLE", +]); + +type TerminalNodeWorkerSupervisorReceipt = Extract< + NodeWorkerSupervisorReceipt, + { state: "completed" | "failed" | "interrupted" | "cancelled" } +>; + +type DeviceWorkerLaunchRequest = { + deviceId: string; + input: NodeWorkerLaunchInput; + isDispatchAuthorized: () => boolean; + isCancellationAuthorized: () => boolean; + timeoutMs: number; + signal?: AbortSignal; +}; + +type NodeWorkerLaunchAdapterOptions = { + getTransport: () => NodeWorkerSupervisorTransport | undefined; + now?: () => number; + sleep?: (ms: number, signal?: AbortSignal) => Promise; + rpcTimeoutMs?: number; + pollIntervalMs?: number; + cancellationTimeoutMs?: number; +}; + +type OperationDeadline = { + expiresAtMs: number; + signal: AbortSignal; + remainingMs: () => number; + dispose: () => void; +}; + +class NodeWorkerLaunchTransportError extends Error { + constructor( + readonly code: string, + message: string, + ) { + super(message); + } +} + +function isTerminalReceipt( + receipt: NodeWorkerSupervisorReceipt, +): receipt is TerminalNodeWorkerSupervisorReceipt { + return ( + receipt.state === "completed" || + receipt.state === "failed" || + receipt.state === "interrupted" || + receipt.state === "cancelled" + ); +} + +function snapshotLaunchInput(input: NodeWorkerLaunchInput): NodeWorkerLaunchInput { + let encoded: string; + try { + encoded = JSON.stringify(input); + } catch { + throw new Error("node worker launch input is not serializable"); + } + return parseNodeWorkerLaunchInput(encoded); +} + +function expectedIdentity(input: NodeWorkerLaunchInput): NodeWorkerSupervisorIdentity { + if (input.launchId !== input.descriptor.assignment.turnId) { + throw new Error("node worker launch ID must match the durable turn ID"); + } + return { + launchId: input.launchId, + planHash: nodeWorkerPlanHash(input), + environmentId: input.descriptor.admission.environmentId, + sessionId: input.descriptor.admission.sessionId, + ownerEpoch: input.descriptor.admission.ownerEpoch, + placementGeneration: input.placementGeneration, + runId: input.descriptor.assignment.runId, + }; +} + +function receiptMatchesIdentity( + receipt: NodeWorkerSupervisorReceipt, + expected: NodeWorkerSupervisorIdentity, +): boolean { + return ( + receipt.launchId === expected.launchId && + receipt.planHash === expected.planHash && + receipt.environmentId === expected.environmentId && + receipt.sessionId === expected.sessionId && + receipt.ownerEpoch === expected.ownerEpoch && + receipt.placementGeneration === expected.placementGeneration && + receipt.runId === expected.runId + ); +} + +function parseInvokeReceipt( + payloadJSON: string | null | undefined, +): NodeWorkerSupervisorReceipt | null { + if (!payloadJSON) { + throw new Error("node worker supervisor response omitted payload JSON"); + } + let value: unknown; + try { + value = JSON.parse(payloadJSON) as unknown; + } catch { + throw new Error("node worker supervisor response contained malformed JSON"); + } + if (value === null) { + return null; + } + const receipt = parseNodeWorkerSupervisorReceipt(value); + if (!receipt) { + throw new Error("node worker supervisor response violated the private receipt contract"); + } + return receipt; +} + +function signalError(signal: AbortSignal, fallback: string): Error { + return signal.reason instanceof Error ? signal.reason : new Error(fallback); +} + +function raceWithSignal(operation: Promise, signal: AbortSignal): Promise { + if (signal.aborted) { + return Promise.reject(signalError(signal, "node worker operation aborted")); + } + return new Promise((resolve, reject) => { + const onAbort = () => reject(signalError(signal, "node worker operation aborted")); + signal.addEventListener("abort", onAbort, { once: true }); + void operation.then( + (value) => { + signal.removeEventListener("abort", onAbort); + resolve(value); + }, + (error: unknown) => { + signal.removeEventListener("abort", onAbort); + reject(error instanceof Error ? error : new Error("node worker operation failed")); + }, + ); + }); +} + +function createDeadline(params: { + now: () => number; + timeoutMs: number; + signal?: AbortSignal; + label: string; +}): OperationDeadline { + if (!Number.isFinite(params.timeoutMs) || params.timeoutMs <= 0) { + throw new Error(`${params.label} timeout must be a positive finite number`); + } + const controller = new AbortController(); + const timer = setTimeout( + () => controller.abort(new Error(`${params.label} timed out`)), + params.timeoutMs, + ); + timer.unref?.(); + const signal = params.signal + ? AbortSignal.any([params.signal, controller.signal]) + : controller.signal; + const expiresAtMs = params.now() + params.timeoutMs; + return { + expiresAtMs, + signal, + remainingMs: () => Math.max(0, expiresAtMs - params.now()), + dispose: () => clearTimeout(timer), + }; +} + +export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOptions) { + const now = options.now ?? Date.now; + const sleep = options.sleep ?? sleepWithAbort; + const rpcTimeoutMs = options.rpcTimeoutMs ?? DEFAULT_RPC_TIMEOUT_MS; + const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; + const cancellationTimeoutMs = options.cancellationTimeoutMs ?? DEFAULT_CANCELLATION_TIMEOUT_MS; + + const findNode = async (params: { + transport: NodeWorkerSupervisorTransport; + deviceId: string; + signal: AbortSignal; + }): Promise => { + let nodes: readonly NodeWorkerSupervisorNodeProof[]; + try { + nodes = await raceWithSignal(params.transport.listCurrentNodes(), params.signal); + } catch (error) { + if (params.signal.aborted) { + throw error; + } + throw new NodeWorkerLaunchTransportError( + "UNAVAILABLE", + "device worker node discovery is unavailable", + ); + } + const node = nodes.find( + (candidate) => + candidate.nodeId === params.deviceId && candidate.commands.includes("system.run"), + ); + if (!node) { + throw new NodeWorkerLaunchTransportError( + "NOT_CONNECTED", + "device worker node is not currently connected", + ); + } + return node; + }; + + const invoke = async (params: { + deviceId: string; + command: + | typeof NODE_WORKER_SUPERVISOR_LAUNCH_COMMAND + | typeof NODE_WORKER_SUPERVISOR_STATUS_COMMAND + | typeof NODE_WORKER_SUPERVISOR_CANCEL_COMMAND; + payload: unknown; + isAuthorized: () => boolean; + deadline: OperationDeadline; + onDispatchReady?: () => void; + }): Promise => { + if (!params.isAuthorized()) { + throw new NodeWorkerLaunchTransportError( + "APPROVAL_AUTHORITY_CLOSED", + "node worker authority closed", + ); + } + const remainingMs = params.deadline.remainingMs(); + if (remainingMs <= 0 || params.deadline.signal.aborted) { + throw params.deadline.signal.reason ?? new Error("node worker operation timed out"); + } + const transport = options.getTransport(); + if (!transport) { + throw new NodeWorkerLaunchTransportError( + "UNAVAILABLE", + "device worker node transport is unavailable", + ); + } + const rpcController = new AbortController(); + const rpcBudgetMs = Math.max(1, Math.min(rpcTimeoutMs, remainingMs)); + const rpcTimer = setTimeout( + () => rpcController.abort(new Error("node worker RPC timed out")), + rpcBudgetMs, + ); + rpcTimer.unref?.(); + const signal = AbortSignal.any([params.deadline.signal, rpcController.signal]); + try { + const node = await findNode({ transport, deviceId: params.deviceId, signal }); + const operation = transport.invoke({ + node, + command: params.command, + params: params.payload, + timeoutMs: rpcBudgetMs, + signal, + idempotencyKey: + params.command === NODE_WORKER_SUPERVISOR_LAUNCH_COMMAND && + typeof params.payload === "object" && + params.payload !== null && + "launchId" in params.payload && + typeof params.payload.launchId === "string" + ? params.payload.launchId + : undefined, + isDispatchAuthorized: params.isAuthorized, + ...(params.onDispatchReady ? { onDispatchReady: params.onDispatchReady } : {}), + }); + const result = await raceWithSignal(operation, signal); + if (!result.ok) { + const code = result.error?.code ?? "UNAVAILABLE"; + throw new NodeWorkerLaunchTransportError( + code, + `node worker supervisor invocation failed (${code})`, + ); + } + return parseInvokeReceipt(result.payloadJSON); + } catch (error) { + if (rpcController.signal.aborted && !params.deadline.signal.aborted) { + throw new NodeWorkerLaunchTransportError("TIMEOUT", "node worker RPC timed out"); + } + throw error; + } finally { + clearTimeout(rpcTimer); + } + }; + + const validateReceipt = ( + receipt: NodeWorkerSupervisorReceipt, + expected: NodeWorkerSupervisorIdentity, + ): NodeWorkerSupervisorReceipt => { + if (!receiptMatchesIdentity(receipt, expected)) { + throw new Error("node worker supervisor receipt identity mismatch"); + } + return receipt; + }; + + const waitBeforeRetry = async (params: { + delayMs: number; + deadline: OperationDeadline; + }): Promise => { + const remainingMs = params.deadline.remainingMs(); + if (remainingMs <= 0 || params.deadline.signal.aborted) { + throw params.deadline.signal.reason ?? new Error("node worker operation timed out"); + } + await raceWithSignal( + sleep(Math.min(params.delayMs, remainingMs), params.deadline.signal), + params.deadline.signal, + ); + return Math.min(params.delayMs * 2, MAX_RETRY_DELAY_MS); + }; + + const cancelUntilTerminal = async (params: { + request: DeviceWorkerLaunchRequest; + expected: NodeWorkerSupervisorIdentity; + }): Promise => { + const deadline = createDeadline({ + now, + timeoutMs: cancellationTimeoutMs, + label: "node worker cancellation", + }); + let delayMs = pollIntervalMs; + try { + while (!deadline.signal.aborted && deadline.remainingMs() > 0) { + if (!params.request.isCancellationAuthorized()) { + throw new Error("node worker cancellation authority closed before terminal settlement"); + } + try { + const receipt = await invoke({ + deviceId: params.request.deviceId, + command: NODE_WORKER_SUPERVISOR_CANCEL_COMMAND, + payload: params.expected, + isAuthorized: params.request.isCancellationAuthorized, + deadline, + }); + if (receipt) { + const validated = validateReceipt(receipt, params.expected); + if (isTerminalReceipt(validated)) { + return validated; + } + delayMs = pollIntervalMs; + } + } catch (error) { + if (deadline.signal.aborted) { + break; + } + if ( + !(error instanceof NodeWorkerLaunchTransportError) || + !RETRYABLE_TRANSPORT_CODES.has(error.code) + ) { + throw error; + } + } + delayMs = await waitBeforeRetry({ delayMs, deadline }); + } + } finally { + deadline.dispose(); + } + throw new Error("node worker cancellation outcome is unknown after transport loss"); + }; + + const launch = async ( + request: DeviceWorkerLaunchRequest, + ): Promise => { + const input = snapshotLaunchInput(request.input); + const stableRequest = { ...request, input }; + const expected = expectedIdentity(input); + const deadline = createDeadline({ + now, + timeoutMs: request.timeoutMs, + ...(request.signal ? { signal: request.signal } : {}), + label: "node worker launch", + }); + let mayHaveLaunched = false; + let pollStatus = false; + let delayMs = pollIntervalMs; + try { + while (true) { + if (deadline.signal.aborted) { + throw signalError(deadline.signal, "node worker launch aborted"); + } + if (!stableRequest.isDispatchAuthorized()) { + throw new Error("node worker launch authority closed"); + } + try { + const receipt = await invoke({ + deviceId: stableRequest.deviceId, + command: pollStatus + ? NODE_WORKER_SUPERVISOR_STATUS_COMMAND + : NODE_WORKER_SUPERVISOR_LAUNCH_COMMAND, + payload: pollStatus ? { launchId: input.launchId } : input, + isAuthorized: stableRequest.isDispatchAuthorized, + deadline, + ...(!pollStatus + ? { + onDispatchReady: () => { + mayHaveLaunched = true; + }, + } + : {}), + }); + if (!receipt) { + pollStatus = false; + } else { + const validated = validateReceipt(receipt, expected); + mayHaveLaunched = true; + if (isTerminalReceipt(validated)) { + return validated; + } + pollStatus = true; + delayMs = pollIntervalMs; + } + } catch (error) { + if (deadline.signal.aborted || !stableRequest.isDispatchAuthorized()) { + throw error; + } + if ( + !(error instanceof NodeWorkerLaunchTransportError) || + !RETRYABLE_TRANSPORT_CODES.has(error.code) + ) { + throw error; + } + pollStatus = false; + } + delayMs = await waitBeforeRetry({ delayMs, deadline }); + } + } catch (error) { + if (!mayHaveLaunched) { + throw error; + } + let terminal: TerminalNodeWorkerSupervisorReceipt; + try { + terminal = await cancelUntilTerminal({ request: stableRequest, expected }); + } catch (cancelError) { + throw Object.assign( + new Error("node worker launch failed and cancellation could not be confirmed", { + cause: error instanceof Error ? error : new Error("node worker launch failed"), + }), + { cancellationError: cancelError }, + ); + } + if (deadline.signal.aborted || !stableRequest.isDispatchAuthorized()) { + return terminal; + } + throw error; + } finally { + deadline.dispose(); + } + }; + + return { launch }; +} diff --git a/src/plugin-sdk/channel-ingress-runtime.test.ts b/src/plugin-sdk/channel-ingress-runtime.test.ts index 7e1680c876e5..b27706970007 100644 --- a/src/plugin-sdk/channel-ingress-runtime.test.ts +++ b/src/plugin-sdk/channel-ingress-runtime.test.ts @@ -33,10 +33,13 @@ describe("plugin-sdk/channel-ingress-runtime", () => { onDeferred: vi.fn(), onAdoptionFinalizing: vi.fn(), onFailed: vi.fn(async () => {}), + onCancelled: vi.fn(async () => {}), onAbandoned: vi.fn(async () => {}), }); const first = createLifecycle(); const second = createLifecycle(); + const cancellation = fanInChannelIngressLifecycles([first, second]); + await cancellation.lifecycle?.onCancelled?.(); const combined = fanInChannelIngressLifecycles([undefined, first, second]); combined.lifecycle?.onAdoptionFinalizing(); @@ -49,6 +52,8 @@ describe("plugin-sdk/channel-ingress-runtime", () => { expect(second.onAdopted).toHaveBeenCalledOnce(); expect(first.onAbandoned).not.toHaveBeenCalled(); expect(second.onAbandoned).not.toHaveBeenCalled(); + expect(first.onCancelled).toHaveBeenCalledOnce(); + expect(second.onCancelled).toHaveBeenCalledOnce(); }); it("settles or abandons claims that no reply lane adopted", async () => { @@ -71,6 +76,57 @@ describe("plugin-sdk/channel-ingress-runtime", () => { expect(fanInChannelIngressLifecycles([]).lifecycle).toBeUndefined(); }); + it("cancellation-settles every source in mixed capable and legacy fan-in", async () => { + const adopted = vi.fn(async () => {}); + const cancelled = vi.fn(async () => {}); + const legacyAbandoned = vi.fn(async () => {}); + const createLifecycle = ( + onCancelled?: () => Promise, + onAbandoned = vi.fn(async () => {}), + ) => ({ + abortSignal: new AbortController().signal, + onAdopted: adopted, + onDeferred: vi.fn(), + onAdoptionFinalizing: vi.fn(), + onFailed: vi.fn(async () => {}), + onAbandoned, + ...(onCancelled ? { onCancelled } : {}), + }); + const combined = fanInChannelIngressLifecycles([ + createLifecycle(cancelled), + createLifecycle(undefined, legacyAbandoned), + ]); + + expect(combined.lifecycle).not.toHaveProperty("onCancelled"); + await combined.cancel(); + + expect(adopted).not.toHaveBeenCalled(); + expect(cancelled).toHaveBeenCalledOnce(); + expect(legacyAbandoned).toHaveBeenCalledOnce(); + }); + + it("preserves lifecycle receivers while fanning in cancellation", async () => { + const createLifecycle = () => ({ + abortSignal: new AbortController().signal, + cancellationCount: 0, + onAdopted: vi.fn(async () => {}), + onDeferred: vi.fn(), + onAdoptionFinalizing: vi.fn(), + onFailed: vi.fn(async () => {}), + async onCancelled() { + this.cancellationCount += 1; + }, + onAbandoned: vi.fn(async () => {}), + }); + const first = createLifecycle(); + const second = createLifecycle(); + + await fanInChannelIngressLifecycles([first, second]).lifecycle?.onCancelled?.(); + + expect(first.cancellationCount).toBe(1); + expect(second.cancellationCount).toBe(1); + }); + it("can abandon claims after terminal settlement adoption fails", async () => { const abandoned = vi.fn(async () => {}); const combined = fanInChannelIngressLifecycles([ diff --git a/src/plugin-sdk/channel-ingress-runtime.ts b/src/plugin-sdk/channel-ingress-runtime.ts index 3d958ac97912..63efb3b0e590 100644 --- a/src/plugin-sdk/channel-ingress-runtime.ts +++ b/src/plugin-sdk/channel-ingress-runtime.ts @@ -128,11 +128,17 @@ export function fanInChannelIngressLifecycles( lifecycle: ChannelIngressLifecycle | undefined; settle: () => Promise; abandon: (error?: unknown) => Promise; + cancel: () => Promise; } { const lifecycles = inputs.filter((lifecycle) => lifecycle !== undefined); const first = lifecycles[0]; if (!first) { - return { lifecycle: undefined, settle: async () => {}, abandon: async () => {} }; + return { + lifecycle: undefined, + settle: async () => {}, + abandon: async () => {}, + cancel: async () => {}, + }; } let handedOff = false; @@ -147,7 +153,16 @@ export function fanInChannelIngressLifecycles( const failAll = async (error: unknown) => { await Promise.all(lifecycles.map(async (lifecycle) => await lifecycle.onFailed?.(error))); }; - + const supportsCancellation = lifecycles.every((lifecycle) => lifecycle.onCancelled !== undefined); + // Omit aggregate cancellation unless every durable source supports it. Callers + // can then use settle/abandon without an acknowledged-but-unsettled claim. + const cancelAll = async () => { + await Promise.all( + lifecycles.map(async (lifecycle) => + lifecycle.onCancelled ? await lifecycle.onCancelled() : await lifecycle.onAbandoned(), + ), + ); + }; return { lifecycle: { abortSignal: @@ -173,6 +188,14 @@ export function fanInChannelIngressLifecycles( handedOff = true; await failAll(error); }, + ...(supportsCancellation + ? { + onCancelled: async () => { + handedOff = true; + await cancelAll(); + }, + } + : {}), onAbandoned: async () => { handedOff = true; await abandonAll(); @@ -191,5 +214,13 @@ export function fanInChannelIngressLifecycles( await abandonAll(); } }, + // Source-compatible lifecycles predate onCancelled. Settle each source through + // its strongest release callback so mixed fan-in cannot strand a durable claim. + cancel: async () => { + if (!handedOff) { + handedOff = true; + await cancelAll(); + } + }, }; } diff --git a/src/skills/loading/plugin-skills.test.ts b/src/skills/loading/plugin-skills.test.ts index 303232fa2fca..6d79cb10329b 100644 --- a/src/skills/loading/plugin-skills.test.ts +++ b/src/skills/loading/plugin-skills.test.ts @@ -136,6 +136,20 @@ async function setupAcpxAndHelperRegistry() { return { workspaceDir, acpxRoot, helperRoot }; } +function useStableMetadataSnapshot(manifestRegistry: PluginManifestRegistry): void { + const snapshot = { + manifestRegistry, + plugins: manifestRegistry.plugins, + normalizePluginId: (pluginId: string) => + manifestRegistry.plugins.find((plugin) => plugin.legacyPluginIds?.includes(pluginId))?.id ?? + pluginId, + }; + hoisted.loadPluginMetadataSnapshot + .mockReturnValueOnce(snapshot) + .mockReturnValueOnce(snapshot) + .mockReturnValueOnce(snapshot); +} + async function setupPluginOutsideSkills() { const workspaceDir = await tempDirs.make("openclaw-"); const pluginRoot = await tempDirs.make("openclaw-plugin-"); @@ -236,6 +250,57 @@ describe("resolvePluginSkillDirs", () => { expect(dirs).toEqual(expectedDirs({ acpxRoot, helperRoot })); }); + it.each([ + { + name: "unavailable to available", + initiallyAvailable: false, + firstIncludesAcpx: false, + secondIncludesAcpx: true, + }, + { + name: "available to unavailable", + initiallyAvailable: true, + firstIncludesAcpx: true, + secondIncludesAcpx: false, + }, + ])( + "invalidates the memo when ACP changes from $name with stable inputs", + async ({ initiallyAvailable, firstIncludesAcpx, secondIncludesAcpx }) => { + const { workspaceDir, acpxRoot, helperRoot } = await setupAcpxAndHelperRegistry(); + const manifestRegistry = buildRegistry({ acpxRoot, helperRoot }); + useStableMetadataSnapshot(manifestRegistry); + const config = { + acp: { enabled: true }, + plugins: { + entries: { + acpx: { enabled: true }, + helper: { enabled: true }, + }, + }, + } as OpenClawConfig; + if (initiallyAvailable) { + registerHealthyAcpBackend(); + } + + const first = resolvePluginSkillDirs({ workspaceDir, config }); + + if (initiallyAvailable) { + acpRuntimeTesting.resetAcpRuntimeBackendsForTests(); + } else { + registerHealthyAcpBackend(); + } + const second = resolvePluginSkillDirs({ workspaceDir, config }); + + const dirsForState = (includeAcpx: boolean) => [ + ...(includeAcpx ? [path.resolve(acpxRoot, "skills")] : []), + path.resolve(helperRoot, "skills"), + ]; + expect(first).toEqual(dirsForState(firstIncludesAcpx)); + expect(second).toEqual(dirsForState(secondIncludesAcpx)); + expect(resolvePluginSkillDirs({ workspaceDir, config })).toBe(second); + }, + ); + it("rejects plugin skill paths that escape the plugin root", async () => { const { workspaceDir, pluginRoot, outsideSkills } = await setupPluginOutsideSkills(); await fs.mkdir(path.join(pluginRoot, "skills"), { recursive: true }); diff --git a/src/skills/loading/plugin-skills.ts b/src/skills/loading/plugin-skills.ts index 0653f9ca9c73..0578e9bda46c 100644 --- a/src/skills/loading/plugin-skills.ts +++ b/src/skills/loading/plugin-skills.ts @@ -22,12 +22,13 @@ const log = createSubsystemLogger("skills"); type PluginSkillLinkType = "dir" | "junction"; // Plugin metadata is process-stable while the gateway runs, but this resolver sits on the -// per-turn skills-refresh path. The single-slot memo keeps repeat turns from re-walking and -// re-publishing every plugin skill dir; lifecycle clears evict it on plugin reload/install. +// per-turn skills-refresh path. ACP availability changes outside that metadata lifecycle, so it +// stays in the memo identity to prevent stale ACPX skill exposure without repeating directory IO. let pluginSkillDirsMemo: { workspaceDir: string; config: OpenClawConfig | undefined; snapshot: unknown; + acpRuntimeAvailable: boolean; dirs: string[]; } | null = null; @@ -55,16 +56,6 @@ export function resolvePluginSkillDirs(params: { env: process.env, allowWorkspaceScopedCurrent: true, }); - const canMemoize = params.pluginSkillsDir === undefined; - if ( - canMemoize && - pluginSkillDirsMemo && - pluginSkillDirsMemo.workspaceDir === workspaceDir && - pluginSkillDirsMemo.config === params.config && - pluginSkillDirsMemo.snapshot === metadataSnapshot - ) { - return pluginSkillDirsMemo.dirs; - } const registry = metadataSnapshot.manifestRegistry; if (registry.plugins.length === 0) { publishPluginSkills([], { @@ -72,11 +63,22 @@ export function resolvePluginSkillDirs(params: { }); return []; } + const acpRuntimeAvailable = isAcpRuntimeSpawnAvailable({ config }); + const canMemoize = params.pluginSkillsDir === undefined; + if ( + canMemoize && + pluginSkillDirsMemo && + pluginSkillDirsMemo.workspaceDir === workspaceDir && + pluginSkillDirsMemo.config === params.config && + pluginSkillDirsMemo.snapshot === metadataSnapshot && + pluginSkillDirsMemo.acpRuntimeAvailable === acpRuntimeAvailable + ) { + return pluginSkillDirsMemo.dirs; + } const normalizedPlugins = normalizePluginsConfigWithResolver( config.plugins, metadataSnapshot.normalizePluginId, ); - const acpRuntimeAvailable = isAcpRuntimeSpawnAvailable({ config }); const memorySlot = normalizedPlugins.slots.memory; let selectedMemoryPluginId: string | null = null; const seen = new Set(); @@ -147,6 +149,7 @@ export function resolvePluginSkillDirs(params: { workspaceDir, config: params.config, snapshot: metadataSnapshot, + acpRuntimeAvailable, dirs: resolved, }; } diff --git a/test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts b/test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts index fb279da38110..3a4b94fd5448 100644 --- a/test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts +++ b/test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts @@ -16,6 +16,7 @@ import { packOpenClawPackageForDocker, parseArgs, prepareBundledAiRuntimePackage, + runCaptureForTest, runCommandForTest, writePackageInventoryForDocker, } from "../../../../scripts/package-openclaw-for-docker.mts"; @@ -768,14 +769,8 @@ describe("package-openclaw-for-docker", () => { restoreDocsMap: async (cwd: string) => { calls.push(`restore-docs:${cwd}`); }, - runCaptureImpl: async ( - command: string, - args: string[], - cwd: string, - options: { deferForwardedSignalExit?: boolean }, - ) => { + runCaptureImpl: async (command: string, args: string[], cwd: string) => { calls.push(`${command}:${args.join(" ")}:${cwd}`); - expect(options.deferForwardedSignalExit).toBe(true); return "openclaw-2026.5.28.tgz\n"; }, }); @@ -955,61 +950,116 @@ describe("package-openclaw-for-docker", () => { }); it("normalizes npm 12 pack metadata for renamed package artifacts", async () => { + const sourceDir = tempDirs.make("openclaw-docker-pack-source-"); const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-docker-pack-json-")); const packJsonPath = path.join(outputDir, "pack.json"); + const npmPackOutput = JSON.stringify({ + openclaw: { + entryCount: 15_000, + filename: "openclaw-2026.5.28.tgz", + files: Array.from({ length: 15_000 }, (_, index) => ({ + mode: 0o644, + path: `dist/generated/package-entry-${String(index).padStart(5, "0")}.js`, + size: index, + })), + size: 7, + unpackedSize: 7, + version: "2026.5.28", + }, + }); + expect(Buffer.byteLength(npmPackOutput)).toBeGreaterThan(1024 * 1024); + const npmPackOutputPath = path.join(sourceDir, "npm-pack.json"); + fs.writeFileSync(npmPackOutputPath, npmPackOutput); try { - const tarball = await packOpenClawPackageForDocker("/repo", outputDir, { + const tarball = await packOpenClawPackageForDocker(sourceDir, outputDir, { ...skipDocsMapLifecycle, outputName: "openclaw-current.tgz", packJsonPath, prepareBundledAiRuntime: skipBundledAiRuntime, prepareChangelog: async () => {}, restoreChangelog: async () => {}, - runCaptureImpl: async ( - command: string, - args: string[], - _cwd: string, - options: { deferForwardedSignalExit?: boolean }, - ) => { - expect(command).toBe("npm"); - expect(args).toEqual([ - "pack", - "--json", - "--silent", - "--ignore-scripts", - "--pack-destination", - outputDir, - ]); - expect(options.deferForwardedSignalExit).toBe(true); + runCaptureImpl: async (_command, _args, cwd, options) => { fs.writeFileSync(path.join(outputDir, "openclaw-2026.5.28.tgz"), "package"); - return JSON.stringify({ - openclaw: { - entryCount: 1, - filename: "openclaw-2026.5.28.tgz", - size: 7, - unpackedSize: 7, - version: "2026.5.28", - }, - }); + return await runCaptureForTest( + process.execPath, + [ + "-e", + "process.stdout.write(require('node:fs').readFileSync(process.argv[1]))", + npmPackOutputPath, + ], + cwd, + options, + ); }, }); expect(tarball).toBe(path.join(outputDir, "openclaw-current.tgz")); - expect(JSON.parse(fs.readFileSync(packJsonPath, "utf8"))).toEqual([ - { - entryCount: 1, - filename: "openclaw-current.tgz", - size: 7, - unpackedSize: 7, - version: "2026.5.28", - }, - ]); + const packJson = JSON.parse(fs.readFileSync(packJsonPath, "utf8")) as Array<{ + entryCount: number; + filename: string; + files: unknown[]; + }>; + expect(packJson).toHaveLength(1); + expect(packJson[0]).toMatchObject({ + entryCount: 15_000, + filename: "openclaw-current.tgz", + }); + expect(packJson[0]?.files).toHaveLength(15_000); } finally { fs.rmSync(outputDir, { force: true, recursive: true }); } }); + it("cleans receipts without obscuring runner and parse failures", async () => { + const originalRm = fs.promises.rm.bind(fs.promises); + for (const failure of ["runner", "parse"] as const) { + for (const cleanupFails of [false, true]) { + const outputDir = tempDirs.make(`openclaw-docker-pack-${failure}-`); + const cleanupError = new Error("receipt cleanup failed"); + let receiptPath = ""; + const rmSpy = vi.spyOn(fs.promises, "rm").mockImplementation(async (target, options) => { + if (cleanupFails && receiptPath && target === path.dirname(receiptPath)) { + throw cleanupError; + } + return await originalRm(target, options); + }); + try { + const packPromise = packOpenClawPackageForDocker("/repo", outputDir, { + ...skipDocsMapLifecycle, + packJsonPath: path.join(outputDir, "pack.json"), + prepareBundledAiRuntime: skipBundledAiRuntime, + prepareChangelog: async () => {}, + restoreChangelog: async () => {}, + runCaptureImpl: async (_command, _args, _cwd, options) => { + receiptPath = options.stdoutFilePath ?? ""; + if (failure === "runner") throw new Error("npm pack failed"); + fs.writeFileSync(receiptPath, "not json"); + fs.writeFileSync(path.join(outputDir, "openclaw-2026.5.28.tgz"), "package"); + return ""; + }, + }); + const message = + failure === "runner" ? "npm pack failed" : "npm pack --json output was not valid JSON"; + if (cleanupFails) { + await expect(packPromise).rejects.toMatchObject({ + cause: expect.objectContaining({ message }), + errors: [expect.objectContaining({ message }), cleanupError], + message: "Package operation and cleanup both failed.", + }); + } else { + await expect(packPromise).rejects.toThrow(message); + expect(fs.existsSync(receiptPath)).toBe(false); + } + expect(receiptPath).not.toBe(""); + } finally { + rmSpy.mockRestore(); + if (receiptPath) fs.rmSync(path.dirname(receiptPath), { force: true, recursive: true }); + } + } + } + }); + it("rejects path-like npm pack stdout before resolving Docker package tarballs", async () => { for (const filename of [ "../openclaw-2026.6.17.tgz", @@ -1297,6 +1347,46 @@ describe("package-openclaw-for-docker", () => { ).rejects.toThrow(/exceeded captured stdout limit \(1024 bytes\)/u); }); + it("writes exact stdout bytes to a file and rejects capture conflicts", async () => { + const tempDir = tempDirs.make("openclaw-package-stdout-file-"); + const stdoutFilePath = path.join(tempDir, "stdout.bin"); + const expected = Buffer.from([0, 1, 10, 13, 127, 128, 255]); + const output = await runCommandForTest( + process.execPath, + ["-e", `process.stdout.write(Buffer.from(${JSON.stringify([...expected])}))`], + process.cwd(), + { stdoutFilePath }, + ); + + expect(output).toBe(""); + expect(fs.readFileSync(stdoutFilePath)).toEqual(expected); + await expect( + runCommandForTest(process.execPath, ["-e", ""], process.cwd(), { + captureStdout: true, + stdoutFilePath: path.join(tempDir, "conflict.bin"), + }), + ).rejects.toThrow("captureStdout and stdoutFilePath cannot be combined"); + }); + + it("restores source artifacts before exiting after receipt-read termination", async () => { + if (process.platform === "win32") return; + const tempDir = tempDirs.make("openclaw-package-receipt-signal-"); + const markerPath = path.join(tempDir, "restored"); + const scriptUrl = pathToFileURL(path.resolve("scripts/package-openclaw-for-docker.mts")).href; + const runnerScript = ` +import fs from "node:fs"; +const readFile = fs.promises.readFile.bind(fs.promises); +fs.promises.readFile = async (...args) => { if (String(args[0]).endsWith("/pack.json")) { process.kill(process.pid, "SIGTERM"); await new Promise((resolve) => setTimeout(resolve, 50)); } return await readFile(...args); }; +const { packOpenClawPackageForDocker } = await import(${JSON.stringify(scriptUrl)}); +try { + await packOpenClawPackageForDocker("/repo", ${JSON.stringify(tempDir)}, { packJsonPath: "result.json", prepareBundledAiRuntime: async () => async () => {}, prepareChangelog: async () => {}, prepareDocsMap: async () => {}, prepareManifest: async () => {}, restoreChangelog: async () => {}, restoreDocsMap: async () => { fs.writeFileSync(${JSON.stringify(markerPath)}, "done"); }, restoreManifest: async () => {}, runCaptureImpl: async (_command, _args, _cwd, options) => { fs.writeFileSync(options.stdoutFilePath, '[{"filename":"openclaw-2026.5.28.tgz"}]'); fs.writeFileSync(${JSON.stringify(path.join(tempDir, "openclaw-2026.5.28.tgz"))}, "package"); return ""; } }); +} catch (error) { process.exit(error.exitCode ?? 1); } +`; + const runner = spawn(process.execPath, ["--input-type=module", "-e", runnerScript]); + expect(await waitForExit(runner, 5000)).toEqual({ signal: null, status: 143 }); + expect(fs.readFileSync(markerPath, "utf8")).toBe("done"); + }); + it("forwards external termination to active child process groups", async () => { if (process.platform === "win32") { return; diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 9be1fb5ad420..9060e4ba6755 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -4947,6 +4947,13 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" expect(parsedWorkflow.jobs.preflight.outputs.frozen_target).toBe( "${{ steps.manifest.outputs.frozen_target }}", ); + expect(preflightGuards).toContain( + 'if [[ "$FROZEN_TARGET" == "true" ]]; then\n' + + " pnpm dup:check:coverage\n" + + " else\n" + + " pnpm dup:check\n" + + " fi", + ); expect(npmLockGuards).toContain("pnpm deps:npm-lock:check"); expect(preflightGuards).toContain("pnpm deps:patches:check"); expect(preflightGuards).toContain('has_package_script "check:coercion-helpers"'); diff --git a/test/scripts/package-mac-app.test.ts b/test/scripts/package-mac-app.test.ts index 7c024a3d58e1..2c4a7605e2b3 100644 --- a/test/scripts/package-mac-app.test.ts +++ b/test/scripts/package-mac-app.test.ts @@ -62,6 +62,7 @@ function runSwiftToolchainHarness(options: { swiftVersion: string; selectedDeveloperDir: "command-line-tools" | "custom-xcode" | "invalid" | "xcode"; developerDirOverride?: "custom-xcode" | "invalid" | "xcode"; + xcodebuildFailure?: string; }) { const root = tempDirs.make("openclaw-package-swift-root-"); const toolsDir = path.join(root, "tools"); @@ -84,9 +85,14 @@ function runSwiftToolchainHarness(options: { mkdirSync(path.dirname(xcodebuild), { recursive: true }); writeFileSync( xcodebuild, - ["#!/usr/bin/env bash", '[[ "$*" == "-version" ]] || exit 2', "echo 'Xcode 26.0'", ""].join( - "\n", - ), + [ + "#!/usr/bin/env bash", + '[[ "$*" == "-version" ]] || exit 2', + ...(options.xcodebuildFailure + ? [`printf '%s\\n' ${JSON.stringify(options.xcodebuildFailure)} >&2`, "exit 1"] + : ["echo 'Xcode 26.0'"]), + "", + ].join("\n"), "utf8", ); chmodSync(xcodebuild, 0o755); @@ -877,6 +883,23 @@ describe("package-mac-app plist stamping", () => { expect(result.stderr).toContain("requires a full Xcode developer directory"); }); + it("preserves the native Xcode failure before generic selection guidance", () => { + const diagnostic = "xcodebuild: error: SDK metadata is unavailable"; + const result = runSwiftToolchainHarness({ + swiftVersion: "6.2.1", + selectedDeveloperDir: "xcode", + xcodebuildFailure: diagnostic, + }); + + expect(result.status).toBe(1); + const diagnosticIndex = result.stderr.indexOf(diagnostic); + const guidanceIndex = result.stderr.indexOf( + "ERROR: OpenClaw macOS app packaging requires a full Xcode developer directory", + ); + expect(diagnosticIndex).toBeGreaterThanOrEqual(0); + expect(guidanceIndex).toBeGreaterThan(diagnosticIndex); + }); + it("runs Sparkle build metadata derivation from the repository root", () => { const helperBlock = getSparkleBuildHelperBlock(); const tempRoot = tempDirs.make("openclaw-package-sparkle-root-"); diff --git a/ui/src/components/desktop/desktop-panel-state.ts b/ui/src/components/desktop/desktop-panel-state.ts new file mode 100644 index 000000000000..ac02e54b6bf5 --- /dev/null +++ b/ui/src/components/desktop/desktop-panel-state.ts @@ -0,0 +1,31 @@ +import { html, nothing } from "lit"; +import { t } from "../../i18n/index.ts"; + +export type DesktopPanelState = + | "picker" + | "inventory-error" + | "credentials" + | "connecting" + | "connected" + | "disconnected"; + +export function renderDesktopPanelRecovery(props: { + inventoryError: boolean; + reason: string | null; + onRetry: () => void; +}) { + return html` +
+ ${props.inventoryError + ? nothing + : html`
+ ${t("desktop.disconnected", { + reason: props.reason ?? t("desktop.unknownReason"), + })} +
`} + +
+ `; +} diff --git a/ui/src/components/desktop/desktop-panel.ts b/ui/src/components/desktop/desktop-panel.ts index 4af4619d0468..4f07c8a11260 100644 --- a/ui/src/components/desktop/desktop-panel.ts +++ b/ui/src/components/desktop/desktop-panel.ts @@ -23,7 +23,9 @@ import { desktopAppIcon, desktopAppLabel } from "./desktop-app-presentation.ts"; import { DesktopClient, type DesktopConnectionHandle } from "./desktop-client.ts"; import { desktopCredentialRequirement } from "./desktop-panel-credentials.ts"; import { desktopPanelLauncherStyles } from "./desktop-panel-launcher-styles.ts"; +import { type DesktopPanelState, renderDesktopPanelRecovery } from "./desktop-panel-state.ts"; import { desktopPanelStyles } from "./desktop-panel-styles.ts"; +import { desktopSourceForEnvironment } from "./desktop-source.ts"; const CLOSE_GLYPH = svg``; const DOCK_BOTTOM_GLYPH = svg``; @@ -38,7 +40,6 @@ const panelLayout = createDockPanelLayout({ defaultHeight: 420, defaultWidth: 560, }); -type DesktopPanelState = "picker" | "credentials" | "connecting" | "connected" | "disconnected"; type DesktopAppId = WorkerDesktopAppId; type DesktopCredentials = { username?: string; password?: string }; type PendingDesktopConnection = { @@ -49,16 +50,6 @@ type PendingDesktopConnection = { }; type ObservedDesktopConnection = PendingDesktopConnection & { observed: DesktopObserveResult }; -function desktopSourceForEnvironment(environment: Pick): DesktopSource { - if (environment.id === "gateway") { - return { kind: "host" }; - } - if (environment.id.startsWith("node:") && environment.id.length > "node:".length) { - return { kind: "node", nodeId: environment.id.slice("node:".length) }; - } - return { kind: "environment", environmentId: environment.id }; -} - /** `` — dockable RFB access to Gateway desktop sources. */ class OpenClawDesktopPanel extends OpenClawLitElement { @property({ attribute: false }) client: GatewayBrowserClient | null = null; @@ -151,7 +142,7 @@ class OpenClawDesktopPanel extends OpenClawLitElement { const wasOpen = this.dockLayout.open; this.dockLayout.setOpen(true); if (detail?.environmentId) { - void this.connectEnvironment(detail.environmentId, false); + void this.connectRequestedEnvironment(detail.environmentId); } else if (!wasOpen) { void this.refreshEnvironments(); } else if (detail?.open !== true) { @@ -191,24 +182,26 @@ class OpenClawDesktopPanel extends OpenClawLitElement { this.launchErrorText = null; } - private async refreshEnvironments(): Promise { + private async refreshEnvironments(expectedOperationId?: number): Promise { const client = this.client; if (!client || !this.available) { - return; + return false; } - const operationId = ++this.operationId; + const operationId = expectedOperationId ?? ++this.operationId; this.loading = true; this.errorText = null; try { const result = await client.request("environments.list", {}); if (operationId !== this.operationId) { - return; + return false; } this.environments = result.environments.filter((environment) => environment.desktop === true); + return true; } catch (error) { if (operationId === this.operationId) { this.errorText = t("desktop.errors.listFailed", { error: formatUiError(error) }); } + return false; } finally { if (operationId === this.operationId) { this.loading = false; @@ -216,6 +209,22 @@ class OpenClawDesktopPanel extends OpenClawLitElement { } } + private async connectRequestedEnvironment(environmentId: string): Promise { + this.returnToPicker(); + this.environmentId = environmentId; + this.state = "connecting"; + const operationId = this.operationId; + const inventoryLoaded = await this.refreshEnvironments(operationId); + if (operationId !== this.operationId) { + return; + } + if (!inventoryLoaded) { + this.state = "inventory-error"; + return; + } + void this.connectEnvironment(environmentId, false); + } + private async connectEnvironment( environmentId: string, control: boolean, @@ -229,11 +238,11 @@ class OpenClawDesktopPanel extends OpenClawLitElement { this.clearLaunchState(); this.credentials = undefined; this.credentialAuth = undefined; - this.desktopApps = [ - ...(this.environments.find((environment) => environment.id === environmentId)?.worker - ?.desktopApps ?? []), - ]; } + this.desktopApps = [ + ...(this.environments.find((environment) => environment.id === environmentId)?.worker + ?.desktopApps ?? []), + ]; this.disconnectConnection(); const operationId = this.operationId; const environment = this.environments.find((candidate) => candidate.id === environmentId) ?? { @@ -628,27 +637,6 @@ class OpenClawDesktopPanel extends OpenClawLitElement { `; } - private renderDisconnected() { - return html` -
-
- ${t("desktop.disconnected", { - reason: this.disconnectedReason ?? t("desktop.unknownReason"), - })} -
- -
- `; - } - private renderCredentials() { const ardAccount = this.credentialAuth === "ard-account"; return html` @@ -710,10 +698,18 @@ class OpenClawDesktopPanel extends OpenClawLitElement { : nothing} ${this.state === "picker" ? this.renderPicker() - : this.state === "credentials" - ? this.renderCredentials() - : this.state === "disconnected" - ? this.renderDisconnected() + : this.state === "inventory-error" || this.state === "disconnected" + ? renderDesktopPanelRecovery({ + inventoryError: this.state === "inventory-error", + reason: this.disconnectedReason, + onRetry: () => + this.environmentId && + void (this.state === "inventory-error" + ? this.connectRequestedEnvironment(this.environmentId) + : this.connectEnvironment(this.environmentId, this.controlling)), + }) + : this.state === "credentials" + ? this.renderCredentials() : this.renderConnection()} diff --git a/ui/src/components/desktop/desktop-source.ts b/ui/src/components/desktop/desktop-source.ts new file mode 100644 index 000000000000..a888b57d598a --- /dev/null +++ b/ui/src/components/desktop/desktop-source.ts @@ -0,0 +1,13 @@ +import type { DesktopSource, EnvironmentSummary } from "@openclaw/gateway-protocol"; + +export function desktopSourceForEnvironment( + environment: Pick, +): DesktopSource { + if (environment.id === "gateway") { + return { kind: "host" }; + } + if (environment.id.startsWith("node:") && environment.id.length > "node:".length) { + return { kind: "node", nodeId: environment.id.slice("node:".length) }; + } + return { kind: "environment", environmentId: environment.id }; +} diff --git a/ui/src/e2e/desktop-panel.e2e.test.ts b/ui/src/e2e/desktop-panel.e2e.test.ts index b80047503ae6..d09d8b7ae896 100644 --- a/ui/src/e2e/desktop-panel.e2e.test.ts +++ b/ui/src/e2e/desktop-panel.e2e.test.ts @@ -27,6 +27,21 @@ function sessionsList(placement: "local" | "active") { }; } +const workerDesktopEnvironment = { + id: "worker-desktop-1", + type: "worker", + status: "available", + desktop: true, + worker: { + providerId: "crabbox", + state: "attached", + ageMs: 1_000, + attachedSessionIds: ["main"], + tunnelStatus: "connected", + desktopApps: ["browser", "terminal"], + }, +} as const; + async function openPalette(page: import("playwright").Page) { await page.evaluate(() => { window.dispatchEvent(new CustomEvent("openclaw:command-palette-open")); @@ -43,6 +58,16 @@ async function openDesktopPanel(page: import("playwright").Page) { return panel; } +async function openDirectDesktop(page: import("playwright").Page, environmentId: string) { + await page.evaluate((targetEnvironmentId) => { + window.dispatchEvent( + new CustomEvent("openclaw:desktop-toggle", { + detail: { open: true, environmentId: targetEnvironmentId }, + }), + ); + }, environmentId); +} + async function installDesktopClientFake(panel: import("playwright").Locator) { await panel.evaluate((element) => { ( @@ -105,8 +130,185 @@ suite.define(() => { expect(await page.getByRole("option", { name: "Desktop", exact: true }).count()).toBe(1); await page.getByRole("option", { name: "Desktop", exact: true }).click(); - await page.locator("openclaw-desktop-panel section[aria-label='Desktop']").waitFor(); + const panel = page.locator("openclaw-desktop-panel"); + await panel.locator("section[aria-label='Desktop']").waitFor(); + await panel.getByText("Desktop sources", { exact: true }).waitFor(); await gateway.waitForRequest("environments.list"); + expect(await gateway.getRequests("desktop.observe")).toHaveLength(0); + }); + }); + + it("refreshes direct-target inventory before observing the exact worker", async () => { + await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { + const gateway = await installMockGateway(page, { + featureMethods: ["desktop.launch", "desktop.observe", "environments.list"], + methodResponses: { + "sessions.list": sessionsList("active"), + "environments.list": { + environments: [workerDesktopEnvironment], + }, + "desktop.observe": { + transport: "rfb", + wsPath: "/desktop/observe?token=direct", + expiresAtMs: 60_000, + control: false, + }, + }, + }); + await page.goto(`${suite.server.baseUrl}chat`); + const panel = page.locator("openclaw-desktop-panel"); + await installDesktopClientFake(panel); + const requestCount = (await gateway.getRequests()).length; + + await openDirectDesktop(page, "worker-desktop-1"); + + const observeRequest = await gateway.waitForRequest("desktop.observe"); + expect(observeRequest.params).toEqual({ + source: { kind: "environment", environmentId: "worker-desktop-1" }, + control: false, + }); + expect( + (await gateway.getRequests()) + .slice(requestCount) + .filter((request) => ["environments.list", "desktop.observe"].includes(request.method)) + .map((request) => request.method), + ).toEqual(["environments.list", "desktop.observe"]); + expect(await panel.getByText("Desktop sources", { exact: true }).count()).toBe(0); + await panel.getByRole("button", { name: "Browser", exact: true }).waitFor(); + await panel.getByRole("button", { name: "Terminal", exact: true }).waitFor(); + }); + }); + + it("reports an unavailable direct target without showing another source", async () => { + await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { + const gateway = await installMockGateway(page, { + featureMethods: ["desktop.observe", "environments.list"], + methodResponses: { + "sessions.list": sessionsList("active"), + "environments.list": { + environments: [{ id: "gateway", type: "local", status: "available", desktop: true }], + }, + "desktop.observe": { + __mockError: { + code: "UNAVAILABLE", + message: "requested worker desktop is temporarily unavailable", + }, + }, + }, + }); + await page.goto(`${suite.server.baseUrl}chat`); + + await openDirectDesktop(page, "missing-worker"); + + const observeRequest = await gateway.waitForRequest("desktop.observe"); + expect(observeRequest.params).toEqual({ + source: { kind: "environment", environmentId: "missing-worker" }, + control: false, + }); + const panel = page.locator("openclaw-desktop-panel"); + await panel.getByText(/requested worker desktop is temporarily unavailable/).waitFor(); + expect(await panel.getByText("Desktop sources", { exact: true }).count()).toBe(0); + expect(await panel.getByText("This machine", { exact: true }).count()).toBe(0); + }); + }); + + it("shows direct-target inventory failure without observing or falling back", async () => { + await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { + const gateway = await installMockGateway(page, { + featureMethods: ["desktop.observe", "environments.list"], + methodResponses: { + "sessions.list": sessionsList("active"), + "environments.list": { + __mockError: { + code: "UNAVAILABLE", + message: "desktop inventory is temporarily unavailable", + }, + }, + "desktop.observe": { + transport: "rfb", + wsPath: "/desktop/observe?token=degraded", + expiresAtMs: 60_000, + control: false, + }, + }, + }); + await page.goto(`${suite.server.baseUrl}chat`); + await openDirectDesktop(page, "worker-desktop-1"); + + const panel = page.locator("openclaw-desktop-panel"); + await panel.getByRole("alert").filter({ hasText: "inventory" }).waitFor(); + expect(await gateway.getRequests("desktop.observe")).toHaveLength(0); + expect(await panel.getByText("Desktop sources", { exact: true }).count()).toBe(0); + expect(await panel.getByText("This machine", { exact: true }).count()).toBe(0); + + await gateway.setMethodResponse("environments.list", { + environments: [workerDesktopEnvironment], + }); + await installDesktopClientFake(panel); + const requestCount = (await gateway.getRequests()).length; + await panel.getByRole("button", { name: "Retry", exact: true }).click(); + + await expect + .poll(async () => (await gateway.getRequests("environments.list")).length) + .toBe(2); + const observeRequest = await gateway.waitForRequest("desktop.observe"); + expect(observeRequest.params).toEqual({ + source: { kind: "environment", environmentId: "worker-desktop-1" }, + control: false, + }); + expect( + (await gateway.getRequests()) + .slice(requestCount) + .filter((request) => ["environments.list", "desktop.observe"].includes(request.method)) + .map((request) => request.method), + ).toEqual(["environments.list", "desktop.observe"]); + await panel.getByRole("button", { name: "Browser", exact: true }).waitFor(); + await panel.getByRole("button", { name: "Terminal", exact: true }).waitFor(); + expect(await panel.getAttribute("data-connect-count")).toBe("1"); + expect(await panel.getByText("Desktop sources", { exact: true }).count()).toBe(0); + }); + }); + + it("does not observe a direct target after its inventory refresh is closed", async () => { + await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { + const gateway = await installMockGateway(page, { + featureMethods: ["desktop.observe", "environments.list"], + methodResponses: { + "sessions.list": sessionsList("active"), + "environments.list": { environments: [] }, + "desktop.observe": { + transport: "rfb", + wsPath: "/desktop/observe?token=stale", + expiresAtMs: 60_000, + control: false, + }, + }, + }); + await page.goto(`${suite.server.baseUrl}chat`); + await gateway.deferNext("environments.list"); + const inventoryCount = (await gateway.getRequests("environments.list")).length; + + await openDirectDesktop(page, "worker-desktop-1"); + await expect + .poll(async () => (await gateway.getRequests("environments.list")).length) + .toBe(inventoryCount + 1); + await page.evaluate(() => { + window.dispatchEvent( + new CustomEvent("openclaw:desktop-toggle", { detail: { open: false } }), + ); + }); + await gateway.resolveDeferred("environments.list", { environments: [] }); + await page.evaluate( + () => + new Promise((resolve) => { + requestAnimationFrame(() => requestAnimationFrame(() => resolve())); + }), + ); + + expect( + await page.locator("openclaw-desktop-panel section[aria-label='Desktop']").count(), + ).toBe(0); + expect(await gateway.getRequests("desktop.observe")).toHaveLength(0); }); }); diff --git a/ui/src/pages/chat/chat-pane-header.ts b/ui/src/pages/chat/chat-pane-header.ts index 28deb8a45520..aac62b97f69f 100644 --- a/ui/src/pages/chat/chat-pane-header.ts +++ b/ui/src/pages/chat/chat-pane-header.ts @@ -25,7 +25,7 @@ import { } from "../../lib/sessions/session-key.ts"; import { isActiveTask } from "../../lib/tasks/data.ts"; import { renderBoardViewSwitch } from "./board-session-surface.ts"; -import { resolveChatPanePlacement } from "./chat-pane-placement.ts"; +import { resolveChatPaneDesktopTarget, resolveChatPanePlacement } from "./chat-pane-placement.ts"; import { ChatPaneSessionMenu } from "./chat-pane-session-menu.ts"; import { readChatSessionActionAccess } from "./chat-session-action-access.ts"; import { resolveChatAgentId } from "./chat-state-route.ts"; @@ -192,13 +192,19 @@ export abstract class ChatPaneHeader extends ChatPaneSessionMenu { ? { "continue-in-terminal": continueInTerminalDisabledReason } : {}), }; - const desktopPanelAvailable = isDesktopPanelAvailable(this.context.gateway.snapshot); - const openDesktopPanel = () => + const desktopEnvironmentId = resolveChatPaneDesktopTarget(row); + const desktopPanelAvailable = + desktopEnvironmentId !== null && isDesktopPanelAvailable(this.context.gateway.snapshot); + const openDesktopPanel = () => { + if (!desktopEnvironmentId) { + return; + } window.dispatchEvent( new CustomEvent(DESKTOP_PANEL_TOGGLE_EVENT, { - detail: { open: true }, + detail: { open: true, environmentId: desktopEnvironmentId }, }), ); + }; const browserPanelAction = sessionWorkspace.onToggleBrowser ? html`