mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-24 03:15:46 -06:00
Merge remote-tracking branch 'origin/main' into fix/inline-slash-skills
This commit is contained in:
@@ -2435,7 +2435,11 @@ jobs:
|
||||
fi
|
||||
pnpm tool-display:check
|
||||
pnpm check:host-env-policy:swift
|
||||
pnpm dup:check
|
||||
if [[ "$FROZEN_TARGET" == "true" ]]; then
|
||||
pnpm dup:check:coverage
|
||||
else
|
||||
pnpm dup:check
|
||||
fi
|
||||
if has_package_script "check:coercion-helpers"; then
|
||||
pnpm check:coercion-helpers
|
||||
elif [[ "$HISTORICAL_TARGET" == "true" ]]; then
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"a35378b670434316ba33a3ba4e595925be685daee9fb0f3c5989983525b18c50","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"}
|
||||
{"contentHash":"812e818c8d7c013b2287c4502426123222037a4235e130a69481272912062866","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"b9c074a11791688e63213f51099ea21745b0e554f63ed13e694ea34a3e4328ac","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"}
|
||||
{"contentHash":"f639fea8b8ee53626452bdbce156724b09a3a29bb05a134eb8e8f8fb8062d4da","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"d8b7c867b29b4651375455e938ae634e7213e3e8c17444dc2b0ef091e46fda6c","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"}
|
||||
{"contentHash":"02db4abe2f1f4578d438f7afacd68d44114e46b48b9582711e7e10191353e6c5","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"883bbab77caeff122bf8b7505dc9299484fba601e285a42d2c457697b7590b8e","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"}
|
||||
{"contentHash":"6826237a93cc52b6039fddbd4b4e0e00e82b5c83cbadd36a055a50ab0480879c","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"a10b2b86f7fec9a99f951656b882414b73d04b5e16313bf0747e883da50c37df","entrypoint":"channel-inbound","importSpecifier":"openclaw/plugin-sdk/channel-inbound"}
|
||||
{"contentHash":"e0eefe9ad871dbfe2bb8dac626e44ecfcd55c8b35667e9fd041799f2f45ea6ba","entrypoint":"channel-inbound","importSpecifier":"openclaw/plugin-sdk/channel-inbound"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"5ea7b454922c17117a2a34aae8f2bfd9a45666a7268dff223063bde72b379aff","entrypoint":"channel-ingress-runtime","importSpecifier":"openclaw/plugin-sdk/channel-ingress-runtime"}
|
||||
{"contentHash":"6f32863c74ad4ea076404754618d38015ad48281964f1fc24692aae8cd0fcad6","entrypoint":"channel-ingress-runtime","importSpecifier":"openclaw/plugin-sdk/channel-ingress-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"2f85610859f047db1d76efbe52d9e63d8126ce8e944d53d6fdafb4e7f503b5dc","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"}
|
||||
{"contentHash":"bc8881a906f40f0a3ede29eb83efc1e4d3e59b62c00154b2480d85d69cbe4010","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"6b1f575e77dc71e72c4b2575f366e7e9990d391e85364844e0ecec2146e7f902","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"}
|
||||
{"contentHash":"f4b35d03ac9df9788462f3e50b64819ff85aba0245b1a51208a288bd994edb8e","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"d92b60c0b7109446db7d24172281dfd6f79196d6428b8e6fd59e99a3d4d9f016","entrypoint":"channel-pairing","importSpecifier":"openclaw/plugin-sdk/channel-pairing"}
|
||||
{"contentHash":"d1c4d7478b9dcb6c4567cb599839c84b843e005ff39f5d31bc317f26d6745853","entrypoint":"channel-pairing","importSpecifier":"openclaw/plugin-sdk/channel-pairing"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"72d713de6cb87ad8e6d9c49456477367a2d30c8157695ddb66ac57919558272b","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"}
|
||||
{"contentHash":"80b5ab5fedd16c952c83747f4093db46e49640b3cfa1fdaa5a9727ff31edf0a7","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"2ccfe8eb85378f3be3f78f79f2638b00faa943e351ff69464bfed44ed63e09c1","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"}
|
||||
{"contentHash":"edfa6b7a219aef521935ac17a20c5cd74e91bf4988eb3fad5ac75fc2e5015596","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"d23cd5df763235dbe35ede1c23d26735f4c183b03dc216ac212aefe75c0c1f96","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"}
|
||||
{"contentHash":"6490014377554ea6b62c657ab22c53fcec385b4ad24cdfccd95ebd9a79717e59","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"f57a244e027b326090ed91cdc105c0c8c83b24318d99be0ec205fffd33504050","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"}
|
||||
{"contentHash":"bb5122c6ac5f4dfe381493b9d128a303108434edca9434ad5528781d005271b8","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"5008615c80d86383e18527d71442f1501e4f02bdb3faa1494a48edf40fd66f84","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"}
|
||||
{"contentHash":"9060d4011e1249ca8aa0b01f2e2825d71b29eff14dc84f7f702440dc01e73460","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"be5d21b9760636dd2956ac64b786213532e74ebb4cd1cc844a923b7bd5949be7","entrypoint":"plugin-command-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-command-runtime"}
|
||||
{"contentHash":"45c491f92e76d0523e2fafc47f362620e2598a5fdef95aed50ce4e8ff108423d","entrypoint":"plugin-command-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-command-runtime"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"6b73741c889f68cc19a5d1f8703df2eca650fe57803795045cff122d746cbb6a","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"}
|
||||
{"contentHash":"a36c13699a318b3fb1b6701ab3280c45e5a4f5ca982cef29778d10af8a5ae97a","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"15704fc50b81cebceb677a5a69a7bc8c97afd962081c647de58a027bd816f19d","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"}
|
||||
{"contentHash":"bdf0b57a425cac872d156021006cce6813893bee30e77c5ef4c055343697dba0","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"df8043d0630d95bc4c316da28bf675a713e7f1f0c304959d1b4b9c8b30fcbd0d","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"}
|
||||
{"contentHash":"62f6dc31a2b1789b3667fe16c2d465681aa4c01e2cd9b2a6da062fd995235963","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"01918241de10b9b625379aa143ea2b3bee3ebf53369dde648c3402227d3cb5f7","entrypoint":"runtime-store","importSpecifier":"openclaw/plugin-sdk/runtime-store"}
|
||||
{"contentHash":"b33bfa1835dbfedd3fef5d48720d57cf166fd60dd21727f1c18c49162eb2ab87","entrypoint":"runtime-store","importSpecifier":"openclaw/plugin-sdk/runtime-store"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"7bae1f99dfbd5920f9fa2e82abfefdf11e28c3150f15b22ee3fcf115a4dcef19","entrypoint":"session-catalog","importSpecifier":"openclaw/plugin-sdk/session-catalog"}
|
||||
{"contentHash":"b8218ab6c7789147d0ced29d4dfa1487af511a2a91f12808587ee97c702947c9","entrypoint":"session-catalog","importSpecifier":"openclaw/plugin-sdk/session-catalog"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"8b0c71c8abe13163eee28ca65e6afb490d0ae3c16288292143ad021975b96ca6","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"}
|
||||
{"contentHash":"3662752cd7db434787d728355a4fb46e8f4be5b88dece9c29d96225bc02da6db","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"eb0096fd25551a7f4a58dcf6c6b53120f3f1ddc1248b2b962cca60c75f6daa12","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"}
|
||||
{"contentHash":"66ab292503af6befc63d5962f3312a8ebab4ab3bf3a56f47fe4f6a465d7c40b9","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"}
|
||||
|
||||
+16
-16
@@ -13,22 +13,22 @@ Proposal, revision 2. Supersedes revision 1 in place (2026-08-11, operator
|
||||
decision). Implementation in progress; update this table in every PR that
|
||||
advances a milestone.
|
||||
|
||||
| # | Milestone | Status | PRs |
|
||||
| --- | ---------------------------------------------------------- | ----------- | ------------------------------------------- |
|
||||
| 0 | This plan (revision 2) | landed | #122454 |
|
||||
| 1a | Naming: session copy revert | landed | #120667 |
|
||||
| 1b | Naming: devices consolidation | landed | #120689 |
|
||||
| 1c | Cleanup: node-pairing → device-pairing merge | landed | #120726 |
|
||||
| 2 | `openclaw resume` + web Continue in terminal | in progress | #120664, #122870 |
|
||||
| 3 | `openclaw connect` one-paste onboarding + `/j/` join route | in progress | #120768, #122499 |
|
||||
| 4 | Picker: grouping, placement, liveness, enrichment | in progress | #120804, #122531, #122635, #122774, #122923 |
|
||||
| F | Real-wire session boundary harness | landed | #121212 |
|
||||
| 5 | Public worker ingress path | landed | #122578, #122643 |
|
||||
| 6 | Node worker provider (device runners) | in progress | #122683, #122769, #122829, #122939, #123013 |
|
||||
| 7 | Bundle push consent + runner updates | not started | — |
|
||||
| 8 | Stop-and-continue moves | not started | — |
|
||||
| 9 | Deletions (ssh sandbox, openshell, exec-host clones, …) | not started | — |
|
||||
| 10 | Cloud convergence (provisioners run `openclaw connect`) | not started | — |
|
||||
| # | Milestone | Status | PRs |
|
||||
| --- | ---------------------------------------------------------- | ----------- | ---------------------------------------------------- |
|
||||
| 0 | This plan (revision 2) | landed | #122454 |
|
||||
| 1a | Naming: session copy revert | landed | #120667 |
|
||||
| 1b | Naming: devices consolidation | landed | #120689 |
|
||||
| 1c | Cleanup: node-pairing → device-pairing merge | landed | #120726 |
|
||||
| 2 | `openclaw resume` + web Continue in terminal | in progress | #120664, #122870 |
|
||||
| 3 | `openclaw connect` one-paste onboarding + `/j/` join route | in progress | #120768, #122499 |
|
||||
| 4 | Picker: grouping, placement, liveness, enrichment | in progress | #120804, #122531, #122635, #122774, #122923 |
|
||||
| F | Real-wire session boundary harness | landed | #121212 |
|
||||
| 5 | Public worker ingress path | landed | #122578, #122643 |
|
||||
| 6 | Node worker provider (device runners) | in progress | #122683, #122769, #122829, #122939, #123013, #123033 |
|
||||
| 7 | Bundle push consent + runner updates | not started | — |
|
||||
| 8 | Stop-and-continue moves | not started | — |
|
||||
| 9 | Deletions (ssh sandbox, openshell, exec-host clones, …) | not started | — |
|
||||
| 10 | Cloud convergence (provisioners run `openclaw connect`) | not started | — |
|
||||
|
||||
Revision history: revision 1 (2026-08-08) established the session/runner
|
||||
vocabulary, the naming rulings, and the milestone skeleton after a
|
||||
|
||||
@@ -47,13 +47,15 @@ the transport callback instead of dispatching an event that was not made
|
||||
durable. At claim time it decodes the versioned payload, re-runs `inspect`, and
|
||||
rejects an id or lane mismatch before delivery.
|
||||
|
||||
`deliver` receives `onAdopted`, `onDeferred`, `onAdoptionFinalizing`,
|
||||
`onAbandoned`, and `abortSignal`. Returning without an explicit handoff marks a
|
||||
terminal no-dispatch event adopted. `admission` is always `exclusive`. A
|
||||
deferred handoff keeps the claim held, while shutdown or abort leaves unadopted
|
||||
work retryable. The monitor tracks delivery independently from claim settlement
|
||||
because adoption can tombstone a row before the channel's delivery promise
|
||||
returns.
|
||||
`deliver` receives `onAdopted`, `onDeferred`, `onAdoptionFinalizing`, `onFailed`,
|
||||
`onCancelled`, `onAbandoned`, and `abortSignal`. Use `onFailed` for delivery
|
||||
errors, `onCancelled` for explicit pre-adoption cancellation that must preserve
|
||||
retry accounting, and `onAbandoned` when a non-adopted turn should consume a
|
||||
retry attempt. Returning without an explicit handoff marks a terminal
|
||||
no-dispatch event adopted. `admission` is always `exclusive`. A deferred handoff
|
||||
keeps the claim held, while shutdown or abort leaves unadopted work retryable.
|
||||
The monitor tracks delivery independently from claim settlement because
|
||||
adoption can tombstone a row before the channel's delivery promise returns.
|
||||
|
||||
Optional settings include custom append delays, a `drain` option block for
|
||||
advanced drain ordering/concurrency/retry policy, an external `abortSignal`, a
|
||||
|
||||
@@ -123,6 +123,7 @@ describe("buildDiscordInboundJob", () => {
|
||||
const ingressSettlement = {
|
||||
settle: vi.fn(async () => {}),
|
||||
abandon: vi.fn(async () => {}),
|
||||
cancel: vi.fn(async () => {}),
|
||||
};
|
||||
const job = buildDiscordInboundJob(ctx, { ingressSettlement });
|
||||
const overrideAbortController = new AbortController();
|
||||
|
||||
@@ -26,6 +26,7 @@ export type DiscordInboundJob = {
|
||||
ingressSettlement?: {
|
||||
settle: () => Promise<void>;
|
||||
abandon: (error?: unknown) => Promise<void>;
|
||||
cancel: () => Promise<void>;
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@@ -116,6 +116,31 @@ describe("Discord durable ingress", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects unstable message identity before durable allocation", async () => {
|
||||
await withQueue(async (queue) => {
|
||||
const dispatch = vi.fn();
|
||||
const monitor = createDiscordIngressMonitor({
|
||||
accountId: "default",
|
||||
client: {} as never,
|
||||
runtime: runtime(),
|
||||
queue,
|
||||
dispatch,
|
||||
});
|
||||
monitor.start();
|
||||
try {
|
||||
const missingMessageId = { ...createRawMessage("missing"), id: undefined };
|
||||
const missingChannelId = { ...createRawMessage("missing"), channel_id: undefined };
|
||||
|
||||
await expect(monitor.accept(missingMessageId as never)).rejects.toThrow("snowflake");
|
||||
await expect(monitor.accept(missingChannelId as never)).rejects.toThrow("channel_id");
|
||||
expect(await queue.listPending({ limit: "all" })).toEqual([]);
|
||||
expect(dispatch).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
await monitor.stop();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("recovers a claimed row with a fresh drain and dispatches it exactly once", async () => {
|
||||
await withQueue(async (queue) => {
|
||||
const monitors: DiscordIngressMonitor[] = [];
|
||||
|
||||
@@ -3,6 +3,7 @@ import { GatewayDispatchEvents, type APIMessage } from "discord-api-types/v10";
|
||||
import {
|
||||
createChannelIngressError,
|
||||
createChannelIngressMonitor,
|
||||
DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS,
|
||||
type ChannelIngressQueue,
|
||||
type ChannelIngressMonitorDeliveryResult,
|
||||
type ChannelIngressMonitorLifecycle,
|
||||
@@ -147,6 +148,10 @@ export function createDiscordIngressMonitor(params: {
|
||||
},
|
||||
appendRetryDelaysMs: [0],
|
||||
drain: {
|
||||
retryPolicy: {
|
||||
maxAttempts: DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS,
|
||||
deadLetterMinAgeMs: 0,
|
||||
},
|
||||
resolveNonRetryableFailure: (error) => {
|
||||
if (error instanceof DiscordIngressPayloadError) {
|
||||
return { reason: "invalid-event", message: error.message };
|
||||
|
||||
@@ -139,7 +139,7 @@ export function createDiscordMessageDispatcher(
|
||||
}
|
||||
const abortSignal = last.abortSignal;
|
||||
if (abortSignal?.aborted) {
|
||||
await admissionLifecycle.onAbandoned();
|
||||
await ingress.cancel();
|
||||
return;
|
||||
}
|
||||
try {
|
||||
@@ -157,7 +157,7 @@ export function createDiscordMessageDispatcher(
|
||||
turnAdoptionLifecycle: admissionLifecycle,
|
||||
});
|
||||
if (abortSignal?.aborted) {
|
||||
await ingress.abandon(abortSignal.reason);
|
||||
await ingress.cancel();
|
||||
return;
|
||||
}
|
||||
if (!ctx) {
|
||||
@@ -211,7 +211,7 @@ export function createDiscordMessageDispatcher(
|
||||
turnAdoptionLifecycle: admissionLifecycle,
|
||||
});
|
||||
if (abortSignal?.aborted) {
|
||||
await ingress.abandon(abortSignal.reason);
|
||||
await ingress.cancel();
|
||||
return;
|
||||
}
|
||||
if (!ctx) {
|
||||
@@ -232,7 +232,10 @@ export function createDiscordMessageDispatcher(
|
||||
}
|
||||
messageRunQueue.enqueue(buildDiscordInboundJob(ctx, { ingressSettlement: ingress }));
|
||||
} catch (error) {
|
||||
await admissionLifecycle.onAbandoned();
|
||||
if (abortSignal?.aborted) {
|
||||
await ingress.cancel();
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
@@ -244,7 +247,8 @@ export function createDiscordMessageDispatcher(
|
||||
onCancel: (entries) => {
|
||||
for (const entry of entries) {
|
||||
pendingDebounceEntries.delete(entry);
|
||||
const settlement = Promise.resolve(entry.turnAdoptionLifecycle?.onAbandoned())
|
||||
const settlement = fanInChannelIngressLifecycles([entry.turnAdoptionLifecycle])
|
||||
.cancel()
|
||||
.catch((error: unknown) => {
|
||||
params.runtime.error(
|
||||
danger(`discord ingress cancellation settlement failed: ${String(error)}`),
|
||||
@@ -271,6 +275,10 @@ export function createDiscordMessageDispatcher(
|
||||
const reason = dispatcherShutdown.signal.aborted
|
||||
? (dispatcherShutdown.signal.reason ?? new Error("discord dispatcher shut down"))
|
||||
: (options?.abortSignal?.reason ?? new Error("discord dispatch aborted"));
|
||||
if (options?.turnAdoptionLifecycle) {
|
||||
await fanInChannelIngressLifecycles([options.turnAdoptionLifecycle]).cancel();
|
||||
return { kind: "deferred" };
|
||||
}
|
||||
return { kind: "failed-retryable", error: reason };
|
||||
}
|
||||
// Filter bot-own messages before they enter the debounce queue.
|
||||
|
||||
@@ -0,0 +1,263 @@
|
||||
// Discord tests cover durable retry recovery through full handler replacement.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { APIMessage } from "discord-api-types/v10";
|
||||
import {
|
||||
type ChannelIngressQueue,
|
||||
DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS,
|
||||
} from "openclaw/plugin-sdk/channel-outbound";
|
||||
import { createDeferred } from "openclaw/plugin-sdk/extension-shared";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
createChannelIngressQueueForTests,
|
||||
} from "openclaw/plugin-sdk/plugin-state-test-runtime";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createDiscordIngressMonitor } from "./ingress.js";
|
||||
import { createDiscordMessageHandler } from "./message-handler.js";
|
||||
import { createDiscordHandlerParams } from "./message-handler.test-helpers.js";
|
||||
|
||||
type DiscordIngressPayload = {
|
||||
version: 1;
|
||||
receivedAt: number;
|
||||
rawMessage: APIMessage;
|
||||
};
|
||||
|
||||
type DiscordQueue = ChannelIngressQueue<DiscordIngressPayload>;
|
||||
|
||||
function rawMessage(id: string, channelId = "lane-a"): APIMessage {
|
||||
return {
|
||||
id,
|
||||
channel_id: channelId,
|
||||
content: "hello",
|
||||
author: {
|
||||
id: "user-1",
|
||||
username: "alice",
|
||||
discriminator: "0",
|
||||
avatar: null,
|
||||
},
|
||||
attachments: [],
|
||||
embeds: [],
|
||||
mentions: [],
|
||||
mention_roles: [],
|
||||
mention_everyone: false,
|
||||
timestamp: new Date(0).toISOString(),
|
||||
edited_timestamp: null,
|
||||
components: [],
|
||||
pinned: false,
|
||||
type: 0,
|
||||
tts: false,
|
||||
} as unknown as APIMessage;
|
||||
}
|
||||
|
||||
async function withQueue(run: (queue: DiscordQueue) => Promise<void>): Promise<void> {
|
||||
const created = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-discord-recovery-"));
|
||||
const stateDir = await fs.realpath(created);
|
||||
const queue = createChannelIngressQueueForTests<DiscordIngressPayload>({
|
||||
channelId: "discord",
|
||||
accountId: "default",
|
||||
stateDir,
|
||||
});
|
||||
try {
|
||||
await run(queue);
|
||||
} finally {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function seedPendingFailure(params: {
|
||||
queue: DiscordQueue;
|
||||
id: string;
|
||||
attempts: number;
|
||||
laneKey?: string;
|
||||
}): Promise<void> {
|
||||
await params.queue.enqueue(
|
||||
params.id,
|
||||
{ version: 1, receivedAt: 1, rawMessage: rawMessage(params.id) },
|
||||
{ laneKey: params.laneKey ?? "channel:lane-a", receivedAt: 1 },
|
||||
);
|
||||
for (let attempt = 1; attempt <= params.attempts; attempt += 1) {
|
||||
const claim = await params.queue.claim(params.id, { ownerId: `seed-${attempt}` });
|
||||
if (!claim) {
|
||||
throw new Error(`Expected ${params.id} to be claimable for seed attempt ${attempt}`);
|
||||
}
|
||||
await params.queue.release(claim, {
|
||||
lastError: `prior genuine failure ${attempt}`,
|
||||
releasedAt: 10 + attempt,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function retryFacts(queue: DiscordQueue, id: string) {
|
||||
const record = (await queue.listPending({ limit: "all" })).find((entry) => entry.id === id);
|
||||
if (!record) {
|
||||
throw new Error(`Expected pending Discord ingress row ${id}`);
|
||||
}
|
||||
return {
|
||||
attempts: record.attempts,
|
||||
lastAttemptAt: record.lastAttemptAt,
|
||||
lastError: record.lastError,
|
||||
};
|
||||
}
|
||||
|
||||
function createHandler(params: {
|
||||
queue: DiscordQueue;
|
||||
preflight: (input: { data: { message?: { id?: string } } }) => Promise<null>;
|
||||
debounceMs?: number;
|
||||
beforeDispatch?: () => Promise<void>;
|
||||
}) {
|
||||
const handlerParams = createDiscordHandlerParams();
|
||||
handlerParams.cfg.messages = { inbound: { debounceMs: params.debounceMs ?? 0 } };
|
||||
return createDiscordMessageHandler({
|
||||
...handlerParams,
|
||||
client: {} as never,
|
||||
testing: {
|
||||
preflightDiscordMessage: params.preflight as never,
|
||||
createIngressMonitor: (monitorParams) =>
|
||||
createDiscordIngressMonitor({
|
||||
...monitorParams,
|
||||
queue: params.queue,
|
||||
dispatch: params.beforeDispatch
|
||||
? async (event, lifecycle) => {
|
||||
await params.beforeDispatch?.();
|
||||
return await monitorParams.dispatch(event, lifecycle);
|
||||
}
|
||||
: monitorParams.dispatch,
|
||||
}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
describe("Discord durable ingress replacement recovery", () => {
|
||||
it("terminally settles a preexisting exhausted poison row before its follower", async () => {
|
||||
await withQueue(async (queue) => {
|
||||
await seedPendingFailure({
|
||||
queue,
|
||||
id: "poison",
|
||||
attempts: DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS,
|
||||
});
|
||||
await queue.enqueue(
|
||||
"follower",
|
||||
{ version: 1, receivedAt: 2, rawMessage: rawMessage("follower") },
|
||||
{ laneKey: "channel:lane-a", receivedAt: 2 },
|
||||
);
|
||||
const dispatched: string[] = [];
|
||||
const handler = createHandler({
|
||||
queue,
|
||||
preflight: vi.fn(async ({ data }) => {
|
||||
const id = data.message?.id ?? "unknown";
|
||||
dispatched.push(id);
|
||||
if (id === "poison") {
|
||||
throw new Error("recovered poison failure");
|
||||
}
|
||||
return null;
|
||||
}),
|
||||
});
|
||||
try {
|
||||
await vi.waitFor(async () => {
|
||||
await expect(queue.enqueue("poison", {} as DiscordIngressPayload)).resolves.toMatchObject(
|
||||
{ kind: "failed", record: { reason: "retry-limit-exceeded" } },
|
||||
);
|
||||
await expect(
|
||||
queue.enqueue("follower", {} as DiscordIngressPayload),
|
||||
).resolves.toMatchObject({ kind: "completed" });
|
||||
});
|
||||
expect(dispatched).toEqual(["poison", "follower"]);
|
||||
expect(await queue.listPending({ limit: "all" })).toEqual([]);
|
||||
expect(await queue.listClaims()).toEqual([]);
|
||||
} finally {
|
||||
await handler.deactivate();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves retry facts across every Discord cancellation route and replacement", async () => {
|
||||
await withQueue(async (queue) => {
|
||||
await seedPendingFailure({
|
||||
queue,
|
||||
id: "poison",
|
||||
attempts: DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS - 1,
|
||||
});
|
||||
await queue.enqueue(
|
||||
"follower",
|
||||
{ version: 1, receivedAt: 2, rawMessage: rawMessage("follower") },
|
||||
{ laneKey: "channel:lane-a", receivedAt: 2 },
|
||||
);
|
||||
const expectedFacts = await retryFacts(queue, "poison");
|
||||
|
||||
const dispatchEntered = createDeferred<void>();
|
||||
const releaseDispatch = createDeferred<void>();
|
||||
const beforeDispatch = async () => {
|
||||
dispatchEntered.resolve();
|
||||
await releaseDispatch.promise;
|
||||
};
|
||||
const beforeDispatchPreflight = vi.fn(async () => null);
|
||||
const beforeDispatchHandler = createHandler({
|
||||
queue,
|
||||
preflight: beforeDispatchPreflight,
|
||||
beforeDispatch,
|
||||
});
|
||||
await dispatchEntered.promise;
|
||||
const beforeDispatchStop = beforeDispatchHandler.deactivate();
|
||||
await Promise.resolve();
|
||||
releaseDispatch.resolve();
|
||||
await beforeDispatchStop;
|
||||
expect(beforeDispatchPreflight).not.toHaveBeenCalled();
|
||||
expect(await retryFacts(queue, "poison")).toEqual(expectedFacts);
|
||||
|
||||
const bufferedPreflight = vi.fn(async () => null);
|
||||
const bufferedHandler = createHandler({
|
||||
queue,
|
||||
preflight: bufferedPreflight,
|
||||
debounceMs: 60_000,
|
||||
});
|
||||
await vi.waitFor(async () => expect(await queue.listClaims()).toHaveLength(1));
|
||||
await bufferedHandler.deactivate();
|
||||
expect(bufferedPreflight).not.toHaveBeenCalled();
|
||||
expect(await retryFacts(queue, "poison")).toEqual(expectedFacts);
|
||||
|
||||
const preflightEntered = createDeferred<void>();
|
||||
const releasePreflight = createDeferred<void>();
|
||||
const activePreflight = vi.fn(async () => {
|
||||
preflightEntered.resolve();
|
||||
await releasePreflight.promise;
|
||||
return null;
|
||||
});
|
||||
const activeHandler = createHandler({ queue, preflight: activePreflight });
|
||||
await preflightEntered.promise;
|
||||
const activeStop = activeHandler.deactivate();
|
||||
await Promise.resolve();
|
||||
releasePreflight.resolve();
|
||||
await activeStop;
|
||||
expect(activePreflight).toHaveBeenCalledTimes(1);
|
||||
expect(await retryFacts(queue, "poison")).toEqual(expectedFacts);
|
||||
|
||||
const finalDispatches: string[] = [];
|
||||
const replacement = createHandler({
|
||||
queue,
|
||||
preflight: vi.fn(async ({ data }) => {
|
||||
const id = data.message?.id ?? "unknown";
|
||||
finalDispatches.push(id);
|
||||
if (id === "poison") {
|
||||
throw new Error("final genuine failure");
|
||||
}
|
||||
return null;
|
||||
}),
|
||||
});
|
||||
try {
|
||||
await vi.waitFor(async () => {
|
||||
await expect(queue.enqueue("poison", {} as DiscordIngressPayload)).resolves.toMatchObject(
|
||||
{ kind: "failed", record: { reason: "retry-limit-exceeded" } },
|
||||
);
|
||||
await expect(
|
||||
queue.enqueue("follower", {} as DiscordIngressPayload),
|
||||
).resolves.toMatchObject({ kind: "completed" });
|
||||
});
|
||||
expect(finalDispatches).toEqual(["poison", "follower"]);
|
||||
} finally {
|
||||
await replacement.deactivate();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,19 +1,35 @@
|
||||
// Discord tests cover message handler.queue plugin behavior.
|
||||
import { getEventListeners } from "node:events";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { APIMessage } from "discord-api-types/v10";
|
||||
import { fanInChannelIngressLifecycles } from "openclaw/plugin-sdk/channel-ingress-runtime";
|
||||
import {
|
||||
type ChannelIngressQueue,
|
||||
DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS,
|
||||
} from "openclaw/plugin-sdk/channel-outbound";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { createDeferred } from "openclaw/plugin-sdk/extension-shared";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
createChannelIngressQueueForTests,
|
||||
} from "openclaw/plugin-sdk/plugin-state-test-runtime";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { DiscordIngressLifecycle } from "./ingress.js";
|
||||
import { buildDiscordInboundJob } from "./inbound-job.js";
|
||||
import { createDiscordIngressMonitor, type DiscordIngressLifecycle } from "./ingress.js";
|
||||
import { createDiscordMessageHandler as createDurableDiscordMessageHandler } from "./message-handler.js";
|
||||
import {
|
||||
createDiscordMessageHandler,
|
||||
preflightDiscordMessageMock,
|
||||
processDiscordMessageMock,
|
||||
} from "./message-handler.module-test-helpers.js";
|
||||
import { createBaseDiscordMessageContext } from "./message-handler.test-harness.js";
|
||||
import {
|
||||
createDiscordHandlerParams,
|
||||
createDiscordPreflightContext,
|
||||
} from "./message-handler.test-helpers.js";
|
||||
import { createDiscordMessageRunQueue } from "./message-run-queue.js";
|
||||
|
||||
type SetStatusFn = (patch: Record<string, unknown>) => void;
|
||||
type MockCallSource = { mock: { calls: Array<Array<unknown>> } };
|
||||
@@ -35,6 +51,8 @@ function expectStatusPatch(setStatus: MockCallSource, expected: Record<string, u
|
||||
|
||||
function createIngressLifecycle(): DiscordIngressLifecycle & {
|
||||
onAdopted: ReturnType<typeof vi.fn>;
|
||||
onFailed: ReturnType<typeof vi.fn>;
|
||||
onCancelled: ReturnType<typeof vi.fn>;
|
||||
onAbandoned: ReturnType<typeof vi.fn>;
|
||||
} {
|
||||
return {
|
||||
@@ -42,10 +60,61 @@ function createIngressLifecycle(): DiscordIngressLifecycle & {
|
||||
onAdopted: vi.fn(async () => {}),
|
||||
onDeferred: vi.fn(),
|
||||
onAdoptionFinalizing: vi.fn(),
|
||||
onFailed: vi.fn(async () => {}),
|
||||
onCancelled: vi.fn(async () => {}),
|
||||
onAbandoned: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
type DiscordIngressPayload = {
|
||||
version: 1;
|
||||
receivedAt: number;
|
||||
rawMessage: APIMessage;
|
||||
};
|
||||
|
||||
async function withDiscordQueue<T>(
|
||||
run: (queue: ChannelIngressQueue<DiscordIngressPayload>) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const created = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-discord-handler-"));
|
||||
const stateDir = await fs.realpath(created);
|
||||
const queue = createChannelIngressQueueForTests<DiscordIngressPayload>({
|
||||
channelId: "discord",
|
||||
accountId: "default",
|
||||
stateDir,
|
||||
});
|
||||
try {
|
||||
return await run(queue);
|
||||
} finally {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function createRawMessage(id: string, channelId = "ch-1"): APIMessage {
|
||||
return {
|
||||
id,
|
||||
channel_id: channelId,
|
||||
content: "hello",
|
||||
author: {
|
||||
id: "user-1",
|
||||
username: "alice",
|
||||
discriminator: "0",
|
||||
avatar: null,
|
||||
},
|
||||
attachments: [],
|
||||
embeds: [],
|
||||
mentions: [],
|
||||
mention_roles: [],
|
||||
mention_everyone: false,
|
||||
timestamp: new Date().toISOString(),
|
||||
edited_timestamp: null,
|
||||
components: [],
|
||||
pinned: false,
|
||||
type: 0,
|
||||
tts: false,
|
||||
} as unknown as APIMessage;
|
||||
}
|
||||
|
||||
async function flushQueueWork(): Promise<void> {
|
||||
for (let i = 0; i < 40; i += 1) {
|
||||
await Promise.resolve();
|
||||
@@ -314,11 +383,31 @@ describe("createDiscordMessageHandler queue behavior", () => {
|
||||
turnAdoptionLifecycle: lifecycle,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ kind: "failed-retryable" });
|
||||
expect(result).toMatchObject({ kind: "deferred" });
|
||||
expect(lifecycle.onCancelled).toHaveBeenCalledTimes(1);
|
||||
expect(lifecycle.onAdopted).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("abandons a buffered ingress claim during deactivation", async () => {
|
||||
it("reports a genuine pre-admission exception only through onFailed", async () => {
|
||||
preflightDiscordMessageMock.mockReset();
|
||||
processDiscordMessageMock.mockReset();
|
||||
const failure = new Error("preflight failed");
|
||||
preflightDiscordMessageMock.mockRejectedValue(failure);
|
||||
const handler = createDiscordMessageHandler(createDiscordHandlerParams());
|
||||
const lifecycle = createIngressLifecycle();
|
||||
|
||||
await expect(
|
||||
handler(createTextMessageData("m-failed") as never, {} as never, {
|
||||
turnAdoptionLifecycle: lifecycle,
|
||||
}),
|
||||
).resolves.toEqual({ kind: "deferred" });
|
||||
|
||||
expect(lifecycle.onFailed).toHaveBeenCalledExactlyOnceWith(failure);
|
||||
expect(lifecycle.onCancelled).not.toHaveBeenCalled();
|
||||
expect(lifecycle.onAbandoned).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cancels a buffered ingress claim during deactivation", async () => {
|
||||
preflightDiscordMessageMock.mockReset();
|
||||
processDiscordMessageMock.mockReset();
|
||||
const params = createDiscordHandlerParams();
|
||||
@@ -332,11 +421,36 @@ describe("createDiscordMessageHandler queue behavior", () => {
|
||||
await handler.deactivate();
|
||||
|
||||
expect(preflightDiscordMessageMock).not.toHaveBeenCalled();
|
||||
expect(lifecycle.onAbandoned).toHaveBeenCalledTimes(1);
|
||||
expect(lifecycle.onCancelled).toHaveBeenCalledTimes(1);
|
||||
expect(lifecycle.onAbandoned).not.toHaveBeenCalled();
|
||||
expect(lifecycle.onAdopted).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("waits for an active debounce flush and abandons it after shutdown", async () => {
|
||||
it("settles every buffered claim when cancellation fan-in includes a legacy lifecycle", async () => {
|
||||
preflightDiscordMessageMock.mockReset();
|
||||
processDiscordMessageMock.mockReset();
|
||||
const params = createDiscordHandlerParams();
|
||||
params.cfg.messages = { inbound: { debounceMs: 60_000 } };
|
||||
const handler = createDiscordMessageHandler(params);
|
||||
const cancellable = createIngressLifecycle();
|
||||
const legacy = createIngressLifecycle();
|
||||
delete (legacy as Partial<typeof legacy>).onCancelled;
|
||||
|
||||
await handler(createTextMessageData("m-cancel-modern") as never, {} as never, {
|
||||
turnAdoptionLifecycle: cancellable,
|
||||
});
|
||||
await handler(createTextMessageData("m-cancel-legacy") as never, {} as never, {
|
||||
turnAdoptionLifecycle: legacy,
|
||||
});
|
||||
await handler.deactivate();
|
||||
|
||||
expect(preflightDiscordMessageMock).not.toHaveBeenCalled();
|
||||
expect(cancellable.onCancelled).toHaveBeenCalledTimes(1);
|
||||
expect(legacy.onAbandoned).toHaveBeenCalledTimes(1);
|
||||
expect(legacy.onAdopted).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("waits for an active debounce flush and cancels it after shutdown", async () => {
|
||||
preflightDiscordMessageMock.mockReset();
|
||||
processDiscordMessageMock.mockReset();
|
||||
const preflightGate = createDeferred<void>();
|
||||
@@ -360,7 +474,8 @@ describe("createDiscordMessageHandler queue behavior", () => {
|
||||
|
||||
preflightGate.resolve();
|
||||
await Promise.all([handling, deactivation]);
|
||||
expect(lifecycle.onAbandoned).toHaveBeenCalledTimes(1);
|
||||
expect(lifecycle.onCancelled).toHaveBeenCalledTimes(1);
|
||||
expect(lifecycle.onAbandoned).not.toHaveBeenCalled();
|
||||
expect(lifecycle.onAdopted).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -394,6 +509,308 @@ describe("createDiscordMessageHandler queue behavior", () => {
|
||||
expect(stop).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("dead-letters an exhausted preflight failure and releases its Discord lane", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-01-01T00:00:00.000Z"));
|
||||
try {
|
||||
await withDiscordQueue(async (queue) => {
|
||||
const attempted: string[] = [];
|
||||
const preflight = vi.fn(async (params: { data: { message?: { id?: string } } }) => {
|
||||
const id = params.data.message?.id ?? "unknown";
|
||||
attempted.push(id);
|
||||
if (id === "poison") {
|
||||
throw new Error("deterministic preflight failure");
|
||||
}
|
||||
return null;
|
||||
});
|
||||
const params = createDiscordHandlerParams();
|
||||
const handler = createDurableDiscordMessageHandler({
|
||||
...params,
|
||||
client: {} as never,
|
||||
testing: {
|
||||
preflightDiscordMessage: preflight as never,
|
||||
createIngressMonitor: (monitorParams) =>
|
||||
createDiscordIngressMonitor({ ...monitorParams, queue }),
|
||||
},
|
||||
});
|
||||
try {
|
||||
await handler(createRawMessage("poison", "lane-a") as never, {} as never);
|
||||
await handler(createRawMessage("follower", "lane-a") as never, {} as never);
|
||||
await handler(createRawMessage("independent", "lane-b") as never, {} as never);
|
||||
|
||||
for (let attempt = 0; attempt < DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS; attempt += 1) {
|
||||
await vi.advanceTimersByTimeAsync(3 * 60_000);
|
||||
}
|
||||
|
||||
await vi.waitFor(() => expect(attempted).toContain("follower"));
|
||||
expect(attempted.indexOf("independent")).toBeGreaterThanOrEqual(0);
|
||||
expect(attempted.indexOf("independent")).toBeLessThan(attempted.indexOf("follower"));
|
||||
expect(attempted.filter((id) => id === "poison")).toHaveLength(
|
||||
DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS,
|
||||
);
|
||||
await expect(queue.enqueue("poison", {} as DiscordIngressPayload)).resolves.toMatchObject(
|
||||
{
|
||||
kind: "failed",
|
||||
record: { reason: "retry-limit-exceeded" },
|
||||
},
|
||||
);
|
||||
await expect(
|
||||
queue.enqueue("follower", {} as DiscordIngressPayload),
|
||||
).resolves.toMatchObject({ kind: "completed" });
|
||||
const runtimeErrors = mockCalls(params.runtime.error as unknown as MockCallSource).map(
|
||||
([message]) => String(message),
|
||||
);
|
||||
expect(runtimeErrors.some((message) => message.includes("reached retry limit"))).toBe(
|
||||
true,
|
||||
);
|
||||
expect(runtimeErrors.join("\n")).not.toContain("hello");
|
||||
} finally {
|
||||
await handler.deactivate();
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves retry facts when deactivation cancels a durable Discord claim", async () => {
|
||||
await withDiscordQueue(async (queue) => {
|
||||
const raw = createRawMessage("cancelled", "lane-a");
|
||||
await queue.enqueue(
|
||||
"cancelled",
|
||||
{ version: 1, receivedAt: 10, rawMessage: raw },
|
||||
{ laneKey: "channel:lane-a", receivedAt: 10 },
|
||||
);
|
||||
const failedClaim = await queue.claim("cancelled", { ownerId: "failed-owner" });
|
||||
expect(failedClaim).not.toBeNull();
|
||||
if (!failedClaim) {
|
||||
return;
|
||||
}
|
||||
await queue.release(failedClaim, {
|
||||
lastError: "previous genuine failure",
|
||||
releasedAt: 20,
|
||||
});
|
||||
const before = (await queue.listPending())[0];
|
||||
const firstPreflight = vi.fn(async () => null);
|
||||
const firstParams = createDiscordHandlerParams();
|
||||
firstParams.cfg.messages = { inbound: { debounceMs: 60_000 } };
|
||||
const first = createDurableDiscordMessageHandler({
|
||||
...firstParams,
|
||||
client: {} as never,
|
||||
testing: {
|
||||
preflightDiscordMessage: firstPreflight as never,
|
||||
createIngressMonitor: (monitorParams) =>
|
||||
createDiscordIngressMonitor({ ...monitorParams, queue }),
|
||||
},
|
||||
});
|
||||
|
||||
await vi.waitFor(async () => expect(await queue.listClaims()).toHaveLength(1));
|
||||
await first.deactivate();
|
||||
|
||||
expect(firstPreflight).not.toHaveBeenCalled();
|
||||
expect(await queue.listPending()).toEqual([
|
||||
expect.objectContaining({
|
||||
id: "cancelled",
|
||||
attempts: before?.attempts,
|
||||
lastAttemptAt: before?.lastAttemptAt,
|
||||
lastError: before?.lastError,
|
||||
}),
|
||||
]);
|
||||
|
||||
const replacementPreflight = vi.fn(async () => null);
|
||||
const replacementParams = createDiscordHandlerParams();
|
||||
const replacement = createDurableDiscordMessageHandler({
|
||||
...replacementParams,
|
||||
client: {} as never,
|
||||
testing: {
|
||||
preflightDiscordMessage: replacementPreflight as never,
|
||||
createIngressMonitor: (monitorParams) =>
|
||||
createDiscordIngressMonitor({ ...monitorParams, queue }),
|
||||
},
|
||||
});
|
||||
try {
|
||||
await vi.waitFor(() => expect(replacementPreflight).toHaveBeenCalledTimes(1));
|
||||
await expect(
|
||||
queue.enqueue("cancelled", {} as DiscordIngressPayload),
|
||||
).resolves.toMatchObject({ kind: "completed" });
|
||||
} finally {
|
||||
await replacement.deactivate();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["returns", "throws"] as const)(
|
||||
"preserves retry facts when a started durable Discord job %s after cancellation",
|
||||
async (outcome) => {
|
||||
await withDiscordQueue(async (queue) => {
|
||||
const id = `started-cancelled-${outcome}`;
|
||||
const raw = createRawMessage(id, "lane-a");
|
||||
await queue.enqueue(
|
||||
id,
|
||||
{ version: 1, receivedAt: 10, rawMessage: raw },
|
||||
{ laneKey: "channel:lane-a", receivedAt: 10 },
|
||||
);
|
||||
const failedClaim = await queue.claim(id, { ownerId: "failed-owner" });
|
||||
expect(failedClaim).not.toBeNull();
|
||||
if (!failedClaim) {
|
||||
return;
|
||||
}
|
||||
await queue.release(failedClaim, {
|
||||
lastError: "previous genuine failure",
|
||||
releasedAt: 20,
|
||||
});
|
||||
const before = (await queue.listPending())[0];
|
||||
const processingStarted = createDeferred<void>();
|
||||
const finishProcessing = createDeferred<void>();
|
||||
let processingSignal: AbortSignal | undefined;
|
||||
const processDiscordMessage = vi.fn(async (ctx: { abortSignal?: AbortSignal }) => {
|
||||
processingSignal = ctx.abortSignal;
|
||||
processingStarted.resolve();
|
||||
await finishProcessing.promise;
|
||||
if (outcome === "throws") {
|
||||
throw new Error("processing stopped after cancellation");
|
||||
}
|
||||
});
|
||||
const params = createDiscordHandlerParams();
|
||||
const handler = createDurableDiscordMessageHandler({
|
||||
...params,
|
||||
client: {} as never,
|
||||
testing: {
|
||||
preflightDiscordMessage: (async (preflightParams: {
|
||||
abortSignal?: AbortSignal;
|
||||
data: ReturnType<typeof createTextMessageData>;
|
||||
turnAdoptionLifecycle?: DiscordIngressLifecycle;
|
||||
}) => ({
|
||||
...createPreflightContextForMessage(preflightParams.data),
|
||||
abortSignal: preflightParams.abortSignal,
|
||||
turnAdoptionLifecycle: preflightParams.turnAdoptionLifecycle,
|
||||
})) as never,
|
||||
processDiscordMessage: processDiscordMessage as never,
|
||||
createIngressMonitor: (monitorParams) =>
|
||||
createDiscordIngressMonitor({ ...monitorParams, queue }),
|
||||
},
|
||||
});
|
||||
|
||||
await processingStarted.promise;
|
||||
const deactivation = handler.deactivate();
|
||||
await vi.waitFor(() => expect(processingSignal?.aborted).toBe(true));
|
||||
finishProcessing.resolve();
|
||||
await deactivation;
|
||||
|
||||
expect(await queue.listPending()).toEqual([
|
||||
expect.objectContaining({
|
||||
id,
|
||||
attempts: before?.attempts,
|
||||
lastAttemptAt: before?.lastAttemptAt,
|
||||
lastError: before?.lastError,
|
||||
}),
|
||||
]);
|
||||
|
||||
const recovered = vi.fn(async (_event, lifecycle: DiscordIngressLifecycle) => {
|
||||
await lifecycle.onAdopted();
|
||||
});
|
||||
const replacement = createDiscordIngressMonitor({
|
||||
accountId: "default",
|
||||
client: {} as never,
|
||||
runtime: params.runtime,
|
||||
queue,
|
||||
dispatch: recovered,
|
||||
});
|
||||
replacement.start();
|
||||
try {
|
||||
await vi.waitFor(() => expect(recovered).toHaveBeenCalledTimes(1));
|
||||
await expect(queue.enqueue(id, {} as DiscordIngressPayload)).resolves.toMatchObject({
|
||||
kind: "completed",
|
||||
});
|
||||
} finally {
|
||||
await replacement.stop();
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves retry facts when deactivation skips a queued durable Discord job", async () => {
|
||||
await withDiscordQueue(async (queue) => {
|
||||
const raw = createRawMessage("queued-cancelled", "lane-a");
|
||||
await queue.enqueue(
|
||||
"queued-cancelled",
|
||||
{ version: 1, receivedAt: 10, rawMessage: raw },
|
||||
{ laneKey: "channel:lane-a", receivedAt: 10 },
|
||||
);
|
||||
const failedClaim = await queue.claim("queued-cancelled", { ownerId: "failed-owner" });
|
||||
expect(failedClaim).not.toBeNull();
|
||||
if (!failedClaim) {
|
||||
return;
|
||||
}
|
||||
await queue.release(failedClaim, {
|
||||
lastError: "previous genuine failure",
|
||||
releasedAt: 20,
|
||||
});
|
||||
const before = (await queue.listPending())[0];
|
||||
const params = createDiscordHandlerParams();
|
||||
const processDiscordMessage = vi.fn(async () => {});
|
||||
const messageRunQueue = createDiscordMessageRunQueue({
|
||||
runtime: params.runtime,
|
||||
testing: { processDiscordMessage: processDiscordMessage as never },
|
||||
});
|
||||
const skipped = createDeferred<void>();
|
||||
const monitor = createDiscordIngressMonitor({
|
||||
accountId: "default",
|
||||
client: {} as never,
|
||||
runtime: params.runtime,
|
||||
queue,
|
||||
dispatch: async (_event, lifecycle) => {
|
||||
const ingress = fanInChannelIngressLifecycles([lifecycle]);
|
||||
messageRunQueue.enqueue(
|
||||
buildDiscordInboundJob(await createBaseDiscordMessageContext(), {
|
||||
ingressSettlement: ingress,
|
||||
}),
|
||||
);
|
||||
await messageRunQueue.deactivate();
|
||||
skipped.resolve();
|
||||
return { kind: "deferred" };
|
||||
},
|
||||
});
|
||||
monitor.start();
|
||||
try {
|
||||
await skipped.promise;
|
||||
await monitor.stop();
|
||||
expect(processDiscordMessage).not.toHaveBeenCalled();
|
||||
expect(await queue.listPending()).toEqual([
|
||||
expect.objectContaining({
|
||||
id: "queued-cancelled",
|
||||
attempts: before?.attempts,
|
||||
lastAttemptAt: before?.lastAttemptAt,
|
||||
lastError: before?.lastError,
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
await monitor.stop();
|
||||
await messageRunQueue.deactivate();
|
||||
}
|
||||
|
||||
const recovered = vi.fn(async (_event, lifecycle: DiscordIngressLifecycle) => {
|
||||
await lifecycle.onAdopted();
|
||||
});
|
||||
const replacement = createDiscordIngressMonitor({
|
||||
accountId: "default",
|
||||
client: {} as never,
|
||||
runtime: params.runtime,
|
||||
queue,
|
||||
dispatch: recovered,
|
||||
});
|
||||
replacement.start();
|
||||
try {
|
||||
await vi.waitFor(() => expect(recovered).toHaveBeenCalledTimes(1));
|
||||
await expect(
|
||||
queue.enqueue("queued-cancelled", {} as DiscordIngressPayload),
|
||||
).resolves.toMatchObject({ kind: "completed" });
|
||||
} finally {
|
||||
await replacement.stop();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("does not abort concurrent runs with a Discord-owned channel timeout", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
|
||||
@@ -45,21 +45,25 @@ async function processDiscordQueuedMessage(params: {
|
||||
(await loadMessageProcessRuntime()).processDiscordMessage;
|
||||
await processDiscordMessageImpl(materializeDiscordInboundJob(params.job, abortSignal));
|
||||
if (abortSignal?.aborted) {
|
||||
await params.job.ingressSettlement?.abandon(abortSignal.reason);
|
||||
// Cancellation ended ownership before delivery; retain prior retry facts
|
||||
// so the durable claim can replay under a replacement lifecycle.
|
||||
await params.job.ingressSettlement?.cancel();
|
||||
} else {
|
||||
await params.job.ingressSettlement?.settle();
|
||||
}
|
||||
} catch (error) {
|
||||
await params.job.ingressSettlement?.abandon(error);
|
||||
if (abortSignal?.aborted) {
|
||||
await params.job.ingressSettlement?.cancel();
|
||||
} else {
|
||||
await params.job.ingressSettlement?.abandon(error);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function cleanupSkippedDiscordQueuedMessage(params: { job: DiscordInboundJob }) {
|
||||
// A skipped job never reached reply-lane adoption; reopen its durable claim.
|
||||
await params.job.ingressSettlement?.abandon(
|
||||
new Error("discord queued run skipped before processing"),
|
||||
);
|
||||
await params.job.ingressSettlement?.cancel();
|
||||
}
|
||||
|
||||
export function createDiscordMessageRunQueue(
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
import { ChannelType, Routes } from "discord-api-types/v10";
|
||||
import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures";
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { hasDiscordMessageCreateAmbiguity } from "./retry.js";
|
||||
import {
|
||||
makeDiscordRest,
|
||||
requestBody,
|
||||
requestPath,
|
||||
type MockCallSource,
|
||||
} from "./send.test-harness.js";
|
||||
|
||||
let createThreadDiscord: typeof import("./send.js").createThreadDiscord;
|
||||
let DiscordThreadInitialMessageError: typeof import("./send.js").DiscordThreadInitialMessageError;
|
||||
|
||||
const DISCORD_TEST_CFG = {
|
||||
channels: {
|
||||
discord: {
|
||||
accounts: {
|
||||
default: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
function discordClientOpts(rest: ReturnType<typeof makeDiscordRest>["rest"]) {
|
||||
return { cfg: DISCORD_TEST_CFG, rest, token: "t" };
|
||||
}
|
||||
|
||||
const requireRecord = createRequireRecord("object", "expected-label");
|
||||
|
||||
beforeAll(async () => {
|
||||
({ createThreadDiscord, DiscordThreadInitialMessageError } = await import("./send.js"));
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("sendMessageDiscord", () => {
|
||||
it("keeps forum starter messages within Discord's content limit", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildForum });
|
||||
postMock.mockResolvedValue({ id: "t1" });
|
||||
const content = "a".repeat(2001);
|
||||
|
||||
await createThreadDiscord("chan1", { name: "thread", content }, discordClientOpts(rest));
|
||||
|
||||
expect(postMock).toHaveBeenCalledTimes(2);
|
||||
expect(requestBody(postMock as unknown as MockCallSource, 0)).toEqual({
|
||||
name: "thread",
|
||||
message: { content: "a".repeat(2000) },
|
||||
});
|
||||
expect(requestPath(postMock as unknown as MockCallSource, 1)).toBe(
|
||||
Routes.channelMessages("t1"),
|
||||
);
|
||||
expect(requestBody(postMock as unknown as MockCallSource, 1)).toMatchObject({
|
||||
content: "a",
|
||||
enforce_nonce: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps sub-limit multi-line forum content in one starter message", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildForum });
|
||||
postMock.mockResolvedValue({ id: "t1" });
|
||||
const content = Array.from({ length: 18 }, (_, index) => `line ${index + 1}`).join("\n");
|
||||
|
||||
await createThreadDiscord("chan1", { name: "thread", content }, discordClientOpts(rest));
|
||||
|
||||
expect(postMock).toHaveBeenCalledTimes(1);
|
||||
expect(requestBody(postMock as unknown as MockCallSource)).toEqual({
|
||||
name: "thread",
|
||||
message: { content },
|
||||
});
|
||||
});
|
||||
|
||||
it("reports a delivered forum starter when a continuation chunk fails", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildForum });
|
||||
postMock
|
||||
.mockResolvedValueOnce({ id: "t1", message: { id: "starter1", channel_id: "t1" } })
|
||||
.mockRejectedValueOnce(Object.assign(new Error("missing access"), { status: 403 }));
|
||||
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await createThreadDiscord(
|
||||
"chan1",
|
||||
{ name: "thread", content: "a".repeat(2001) },
|
||||
discordClientOpts(rest),
|
||||
);
|
||||
} catch (error) {
|
||||
thrown = error;
|
||||
}
|
||||
|
||||
expect(thrown).toBeInstanceOf(DiscordThreadInitialMessageError);
|
||||
expect(requireRecord(thrown, "thread initial message error").initialMessageDelivery).toEqual({
|
||||
starterMessageDelivered: true,
|
||||
deliveredChunkCount: 1,
|
||||
deliveredMessageIds: ["starter1"],
|
||||
failedChunkDelivery: "not_delivered",
|
||||
failedChunkIndex: 1,
|
||||
totalChunkCount: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("reports an exhausted ambiguous forum continuation as unknown delivery", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildForum });
|
||||
const ambiguous = Object.assign(new Error("response lost"), { status: 502 });
|
||||
postMock
|
||||
.mockResolvedValueOnce({ id: "t1", message: { id: "starter1", channel_id: "t1" } })
|
||||
.mockRejectedValue(ambiguous);
|
||||
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await createThreadDiscord(
|
||||
"chan1",
|
||||
{ name: "thread", content: "a".repeat(2001) },
|
||||
{
|
||||
...discordClientOpts(rest),
|
||||
retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 },
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
thrown = error;
|
||||
}
|
||||
|
||||
expect(postMock).toHaveBeenCalledTimes(3);
|
||||
expect(thrown).toBeInstanceOf(DiscordThreadInitialMessageError);
|
||||
expect(hasDiscordMessageCreateAmbiguity(thrown)).toBe(true);
|
||||
expect(requireRecord(thrown, "thread initial message error").message).toContain(
|
||||
"delivery of the remaining initial content could not be confirmed",
|
||||
);
|
||||
expect(requireRecord(thrown, "thread initial message error").initialMessageDelivery).toEqual({
|
||||
starterMessageDelivered: true,
|
||||
deliveredChunkCount: 1,
|
||||
deliveredMessageIds: ["starter1"],
|
||||
failedChunkDelivery: "unknown",
|
||||
failedChunkIndex: 1,
|
||||
totalChunkCount: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("chunks long initial messages for non-forum threads", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildText });
|
||||
postMock.mockResolvedValue({ id: "t1" });
|
||||
const content = "a".repeat(2001);
|
||||
|
||||
await createThreadDiscord("chan1", { name: "thread", content }, discordClientOpts(rest));
|
||||
|
||||
expect(postMock).toHaveBeenCalledTimes(3);
|
||||
expect(requestPath(postMock as unknown as MockCallSource, 1)).toBe(
|
||||
Routes.channelMessages("t1"),
|
||||
);
|
||||
expect(requestBody(postMock as unknown as MockCallSource, 1)).toMatchObject({
|
||||
content: "a".repeat(2000),
|
||||
enforce_nonce: true,
|
||||
});
|
||||
expect(requestPath(postMock as unknown as MockCallSource, 2)).toBe(
|
||||
Routes.channelMessages("t1"),
|
||||
);
|
||||
expect(requestBody(postMock as unknown as MockCallSource, 2)).toMatchObject({
|
||||
content: "a",
|
||||
enforce_nonce: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps sub-limit multi-line non-forum content in one initial message", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildText });
|
||||
postMock.mockResolvedValue({ id: "t1", channel_id: "t1" });
|
||||
const content = Array.from({ length: 18 }, (_, index) => `line ${index + 1}`).join("\n");
|
||||
|
||||
await createThreadDiscord("chan1", { name: "thread", content }, discordClientOpts(rest));
|
||||
|
||||
expect(postMock).toHaveBeenCalledTimes(2);
|
||||
expect(requestBody(postMock as unknown as MockCallSource, 1)).toMatchObject({ content });
|
||||
});
|
||||
|
||||
it("reports delivered non-forum chunks when a later chunk fails", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildText });
|
||||
postMock
|
||||
.mockResolvedValueOnce({ id: "t1", name: "thread", type: ChannelType.PublicThread })
|
||||
.mockResolvedValueOnce({ id: "msg1", channel_id: "t1" })
|
||||
.mockRejectedValueOnce(Object.assign(new Error("missing access"), { status: 403 }));
|
||||
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await createThreadDiscord(
|
||||
"chan1",
|
||||
{ name: "thread", content: "a".repeat(4001) },
|
||||
discordClientOpts(rest),
|
||||
);
|
||||
} catch (error) {
|
||||
thrown = error;
|
||||
}
|
||||
|
||||
expect(thrown).toBeInstanceOf(DiscordThreadInitialMessageError);
|
||||
expect(requireRecord(thrown, "thread initial message error").initialMessageDelivery).toEqual({
|
||||
starterMessageDelivered: false,
|
||||
deliveredChunkCount: 1,
|
||||
deliveredMessageIds: ["msg1"],
|
||||
failedChunkDelivery: "not_delivered",
|
||||
failedChunkIndex: 1,
|
||||
totalChunkCount: 3,
|
||||
});
|
||||
});
|
||||
|
||||
it("reports an exhausted ambiguous non-forum chunk as unknown delivery", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildText });
|
||||
const ambiguous = Object.assign(new Error("response lost"), { status: 502 });
|
||||
postMock
|
||||
.mockResolvedValueOnce({ id: "t1", name: "thread", type: ChannelType.PublicThread })
|
||||
.mockResolvedValueOnce({ id: "msg1", channel_id: "t1" })
|
||||
.mockRejectedValue(ambiguous);
|
||||
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await createThreadDiscord(
|
||||
"chan1",
|
||||
{ name: "thread", content: "a".repeat(4001) },
|
||||
{
|
||||
...discordClientOpts(rest),
|
||||
retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 },
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
thrown = error;
|
||||
}
|
||||
|
||||
expect(postMock).toHaveBeenCalledTimes(4);
|
||||
expect(thrown).toBeInstanceOf(DiscordThreadInitialMessageError);
|
||||
expect(hasDiscordMessageCreateAmbiguity(thrown)).toBe(true);
|
||||
expect(requireRecord(thrown, "thread initial message error").message).toContain(
|
||||
"delivery of the remaining initial content could not be confirmed",
|
||||
);
|
||||
expect(requireRecord(thrown, "thread initial message error").initialMessageDelivery).toEqual({
|
||||
starterMessageDelivered: false,
|
||||
deliveredChunkCount: 1,
|
||||
deliveredMessageIds: ["msg1"],
|
||||
failedChunkDelivery: "unknown",
|
||||
failedChunkIndex: 1,
|
||||
totalChunkCount: 3,
|
||||
});
|
||||
});
|
||||
|
||||
it("retries continuation sends with a stable nonce per chunk", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildText });
|
||||
postMock
|
||||
.mockResolvedValueOnce({ id: "t1", name: "thread", type: ChannelType.PublicThread })
|
||||
.mockRejectedValueOnce(Object.assign(new Error("bad gateway"), { status: 502 }))
|
||||
.mockResolvedValueOnce({ id: "msg1", channel_id: "t1" })
|
||||
.mockResolvedValueOnce({ id: "msg2", channel_id: "t1" });
|
||||
|
||||
await createThreadDiscord(
|
||||
"chan1",
|
||||
{ name: "thread", content: "a".repeat(2001) },
|
||||
{
|
||||
...discordClientOpts(rest),
|
||||
retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 },
|
||||
},
|
||||
);
|
||||
|
||||
expect(postMock).toHaveBeenCalledTimes(4);
|
||||
const firstAttempt = requestBody(postMock as unknown as MockCallSource, 1);
|
||||
const retryAttempt = requestBody(postMock as unknown as MockCallSource, 2);
|
||||
const nextChunk = requestBody(postMock as unknown as MockCallSource, 3);
|
||||
expect(firstAttempt.enforce_nonce).toBe(true);
|
||||
expect(retryAttempt.nonce).toBe(firstAttempt.nonce);
|
||||
expect(nextChunk.enforce_nonce).toBe(true);
|
||||
expect(nextChunk.nonce).not.toBe(firstAttempt.nonce);
|
||||
});
|
||||
});
|
||||
@@ -4,8 +4,13 @@ import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures";
|
||||
import { loadWebMediaRaw } from "openclaw/plugin-sdk/web-media";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { RateLimitError } from "./internal/discord.js";
|
||||
import { hasDiscordMessageCreateAmbiguity } from "./retry.js";
|
||||
import { makeDiscordRest } from "./send.test-harness.js";
|
||||
import {
|
||||
makeDiscordRest,
|
||||
requestBody,
|
||||
requestPath,
|
||||
timerDelayAt,
|
||||
type MockCallSource,
|
||||
} from "./send.test-harness.js";
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/web-media", async () => {
|
||||
const { discordWebMediaMockFactory } = await import("./send.test-harness.js");
|
||||
@@ -42,41 +47,8 @@ function discordClientOpts(rest: ReturnType<typeof makeDiscordRest>["rest"]) {
|
||||
return { cfg: DISCORD_TEST_CFG, rest, token: "t" };
|
||||
}
|
||||
|
||||
type MockCallSource = {
|
||||
mock: {
|
||||
calls: ArrayLike<ReadonlyArray<unknown>>;
|
||||
};
|
||||
};
|
||||
|
||||
const requireRecord = createRequireRecord("object", "expected-label");
|
||||
|
||||
function mockArg(source: MockCallSource, callIndex: number, argIndex: number, label: string) {
|
||||
const call = source.mock.calls[callIndex];
|
||||
if (!call) {
|
||||
throw new Error(`expected mock call: ${label}`);
|
||||
}
|
||||
return call[argIndex];
|
||||
}
|
||||
|
||||
function requestOptions(source: MockCallSource, callIndex = 0) {
|
||||
return requireRecord(
|
||||
mockArg(source, callIndex, 1, `request options ${callIndex}`),
|
||||
"request options",
|
||||
);
|
||||
}
|
||||
|
||||
function requestPath(source: MockCallSource, callIndex = 0) {
|
||||
return mockArg(source, callIndex, 0, `request path ${callIndex}`);
|
||||
}
|
||||
|
||||
function requestBody(source: MockCallSource, callIndex = 0) {
|
||||
return requireRecord(requestOptions(source, callIndex).body, `request body ${callIndex}`);
|
||||
}
|
||||
|
||||
function timerDelayAt(source: MockCallSource, callIndex = 0) {
|
||||
return mockArg(source, callIndex, 1, `timer delay ${callIndex}`);
|
||||
}
|
||||
|
||||
function createDiscordForumPayloadHarness(parentType: ChannelType = ChannelType.GuildForum) {
|
||||
const parentId = "700";
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
@@ -303,110 +275,6 @@ describe("sendMessageDiscord", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps forum starter messages within Discord's content limit", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildForum });
|
||||
postMock.mockResolvedValue({ id: "t1" });
|
||||
const content = "a".repeat(2001);
|
||||
|
||||
await createThreadDiscord("chan1", { name: "thread", content }, discordClientOpts(rest));
|
||||
|
||||
expect(postMock).toHaveBeenCalledTimes(2);
|
||||
expect(requestBody(postMock as unknown as MockCallSource, 0)).toEqual({
|
||||
name: "thread",
|
||||
message: { content: "a".repeat(2000) },
|
||||
});
|
||||
expect(requestPath(postMock as unknown as MockCallSource, 1)).toBe(
|
||||
Routes.channelMessages("t1"),
|
||||
);
|
||||
expect(requestBody(postMock as unknown as MockCallSource, 1)).toMatchObject({
|
||||
content: "a",
|
||||
enforce_nonce: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps sub-limit multi-line forum content in one starter message", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildForum });
|
||||
postMock.mockResolvedValue({ id: "t1" });
|
||||
const content = Array.from({ length: 18 }, (_, index) => `line ${index + 1}`).join("\n");
|
||||
|
||||
await createThreadDiscord("chan1", { name: "thread", content }, discordClientOpts(rest));
|
||||
|
||||
expect(postMock).toHaveBeenCalledTimes(1);
|
||||
expect(requestBody(postMock as unknown as MockCallSource)).toEqual({
|
||||
name: "thread",
|
||||
message: { content },
|
||||
});
|
||||
});
|
||||
|
||||
it("reports a delivered forum starter when a continuation chunk fails", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildForum });
|
||||
postMock
|
||||
.mockResolvedValueOnce({ id: "t1", message: { id: "starter1", channel_id: "t1" } })
|
||||
.mockRejectedValueOnce(Object.assign(new Error("missing access"), { status: 403 }));
|
||||
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await createThreadDiscord(
|
||||
"chan1",
|
||||
{ name: "thread", content: "a".repeat(2001) },
|
||||
discordClientOpts(rest),
|
||||
);
|
||||
} catch (error) {
|
||||
thrown = error;
|
||||
}
|
||||
|
||||
expect(thrown).toBeInstanceOf(DiscordThreadInitialMessageError);
|
||||
expect(requireRecord(thrown, "thread initial message error").initialMessageDelivery).toEqual({
|
||||
starterMessageDelivered: true,
|
||||
deliveredChunkCount: 1,
|
||||
deliveredMessageIds: ["starter1"],
|
||||
failedChunkDelivery: "not_delivered",
|
||||
failedChunkIndex: 1,
|
||||
totalChunkCount: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("reports an exhausted ambiguous forum continuation as unknown delivery", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildForum });
|
||||
const ambiguous = Object.assign(new Error("response lost"), { status: 502 });
|
||||
postMock
|
||||
.mockResolvedValueOnce({ id: "t1", message: { id: "starter1", channel_id: "t1" } })
|
||||
.mockRejectedValue(ambiguous);
|
||||
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await createThreadDiscord(
|
||||
"chan1",
|
||||
{ name: "thread", content: "a".repeat(2001) },
|
||||
{
|
||||
...discordClientOpts(rest),
|
||||
retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 },
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
thrown = error;
|
||||
}
|
||||
|
||||
expect(postMock).toHaveBeenCalledTimes(3);
|
||||
expect(thrown).toBeInstanceOf(DiscordThreadInitialMessageError);
|
||||
expect(hasDiscordMessageCreateAmbiguity(thrown)).toBe(true);
|
||||
expect(requireRecord(thrown, "thread initial message error").message).toContain(
|
||||
"delivery of the remaining initial content could not be confirmed",
|
||||
);
|
||||
expect(requireRecord(thrown, "thread initial message error").initialMessageDelivery).toEqual({
|
||||
starterMessageDelivered: true,
|
||||
deliveredChunkCount: 1,
|
||||
deliveredMessageIds: ["starter1"],
|
||||
failedChunkDelivery: "unknown",
|
||||
failedChunkIndex: 1,
|
||||
totalChunkCount: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("inherits default_auto_archive_duration for forum threads", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({
|
||||
@@ -568,140 +436,6 @@ describe("sendMessageDiscord", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("chunks long initial messages for non-forum threads", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildText });
|
||||
postMock.mockResolvedValue({ id: "t1" });
|
||||
const content = "a".repeat(2001);
|
||||
|
||||
await createThreadDiscord("chan1", { name: "thread", content }, discordClientOpts(rest));
|
||||
|
||||
expect(postMock).toHaveBeenCalledTimes(3);
|
||||
expect(requestPath(postMock as unknown as MockCallSource, 1)).toBe(
|
||||
Routes.channelMessages("t1"),
|
||||
);
|
||||
expect(requestBody(postMock as unknown as MockCallSource, 1)).toMatchObject({
|
||||
content: "a".repeat(2000),
|
||||
enforce_nonce: true,
|
||||
});
|
||||
expect(requestPath(postMock as unknown as MockCallSource, 2)).toBe(
|
||||
Routes.channelMessages("t1"),
|
||||
);
|
||||
expect(requestBody(postMock as unknown as MockCallSource, 2)).toMatchObject({
|
||||
content: "a",
|
||||
enforce_nonce: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps sub-limit multi-line non-forum content in one initial message", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildText });
|
||||
postMock.mockResolvedValue({ id: "t1", channel_id: "t1" });
|
||||
const content = Array.from({ length: 18 }, (_, index) => `line ${index + 1}`).join("\n");
|
||||
|
||||
await createThreadDiscord("chan1", { name: "thread", content }, discordClientOpts(rest));
|
||||
|
||||
expect(postMock).toHaveBeenCalledTimes(2);
|
||||
expect(requestBody(postMock as unknown as MockCallSource, 1)).toMatchObject({ content });
|
||||
});
|
||||
|
||||
it("reports delivered non-forum chunks when a later chunk fails", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildText });
|
||||
postMock
|
||||
.mockResolvedValueOnce({ id: "t1", name: "thread", type: ChannelType.PublicThread })
|
||||
.mockResolvedValueOnce({ id: "msg1", channel_id: "t1" })
|
||||
.mockRejectedValueOnce(Object.assign(new Error("missing access"), { status: 403 }));
|
||||
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await createThreadDiscord(
|
||||
"chan1",
|
||||
{ name: "thread", content: "a".repeat(4001) },
|
||||
discordClientOpts(rest),
|
||||
);
|
||||
} catch (error) {
|
||||
thrown = error;
|
||||
}
|
||||
|
||||
expect(thrown).toBeInstanceOf(DiscordThreadInitialMessageError);
|
||||
expect(requireRecord(thrown, "thread initial message error").initialMessageDelivery).toEqual({
|
||||
starterMessageDelivered: false,
|
||||
deliveredChunkCount: 1,
|
||||
deliveredMessageIds: ["msg1"],
|
||||
failedChunkDelivery: "not_delivered",
|
||||
failedChunkIndex: 1,
|
||||
totalChunkCount: 3,
|
||||
});
|
||||
});
|
||||
|
||||
it("reports an exhausted ambiguous non-forum chunk as unknown delivery", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildText });
|
||||
const ambiguous = Object.assign(new Error("response lost"), { status: 502 });
|
||||
postMock
|
||||
.mockResolvedValueOnce({ id: "t1", name: "thread", type: ChannelType.PublicThread })
|
||||
.mockResolvedValueOnce({ id: "msg1", channel_id: "t1" })
|
||||
.mockRejectedValue(ambiguous);
|
||||
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await createThreadDiscord(
|
||||
"chan1",
|
||||
{ name: "thread", content: "a".repeat(4001) },
|
||||
{
|
||||
...discordClientOpts(rest),
|
||||
retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 },
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
thrown = error;
|
||||
}
|
||||
|
||||
expect(postMock).toHaveBeenCalledTimes(4);
|
||||
expect(thrown).toBeInstanceOf(DiscordThreadInitialMessageError);
|
||||
expect(hasDiscordMessageCreateAmbiguity(thrown)).toBe(true);
|
||||
expect(requireRecord(thrown, "thread initial message error").message).toContain(
|
||||
"delivery of the remaining initial content could not be confirmed",
|
||||
);
|
||||
expect(requireRecord(thrown, "thread initial message error").initialMessageDelivery).toEqual({
|
||||
starterMessageDelivered: false,
|
||||
deliveredChunkCount: 1,
|
||||
deliveredMessageIds: ["msg1"],
|
||||
failedChunkDelivery: "unknown",
|
||||
failedChunkIndex: 1,
|
||||
totalChunkCount: 3,
|
||||
});
|
||||
});
|
||||
|
||||
it("retries continuation sends with a stable nonce per chunk", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildText });
|
||||
postMock
|
||||
.mockResolvedValueOnce({ id: "t1", name: "thread", type: ChannelType.PublicThread })
|
||||
.mockRejectedValueOnce(Object.assign(new Error("bad gateway"), { status: 502 }))
|
||||
.mockResolvedValueOnce({ id: "msg1", channel_id: "t1" })
|
||||
.mockResolvedValueOnce({ id: "msg2", channel_id: "t1" });
|
||||
|
||||
await createThreadDiscord(
|
||||
"chan1",
|
||||
{ name: "thread", content: "a".repeat(2001) },
|
||||
{
|
||||
...discordClientOpts(rest),
|
||||
retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 },
|
||||
},
|
||||
);
|
||||
|
||||
expect(postMock).toHaveBeenCalledTimes(4);
|
||||
const firstAttempt = requestBody(postMock as unknown as MockCallSource, 1);
|
||||
const retryAttempt = requestBody(postMock as unknown as MockCallSource, 2);
|
||||
const nextChunk = requestBody(postMock as unknown as MockCallSource, 3);
|
||||
expect(firstAttempt.enforce_nonce).toBe(true);
|
||||
expect(retryAttempt.nonce).toBe(firstAttempt.nonce);
|
||||
expect(nextChunk.enforce_nonce).toBe(true);
|
||||
expect(nextChunk.nonce).not.toBe(firstAttempt.nonce);
|
||||
});
|
||||
|
||||
it("keeps created non-forum thread details when initial message send fails", async () => {
|
||||
const { rest, getMock, postMock } = makeDiscordRest();
|
||||
getMock.mockResolvedValue({ type: ChannelType.GuildText });
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// Discord plugin module implements send harness behavior.
|
||||
import { createServer } from "node:http";
|
||||
import type { MockFn } from "openclaw/plugin-sdk/plugin-test-runtime";
|
||||
import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures";
|
||||
import { vi } from "vitest";
|
||||
import { RequestClient } from "./internal/discord.js";
|
||||
|
||||
@@ -25,6 +26,41 @@ type DiscordLoopbackRequest = {
|
||||
path: string | undefined;
|
||||
};
|
||||
|
||||
export type MockCallSource = {
|
||||
mock: {
|
||||
calls: ArrayLike<ReadonlyArray<unknown>>;
|
||||
};
|
||||
};
|
||||
|
||||
const requireRecord = createRequireRecord("object", "expected-label");
|
||||
|
||||
function mockArg(source: MockCallSource, callIndex: number, argIndex: number, label: string) {
|
||||
const call = source.mock.calls[callIndex];
|
||||
if (!call) {
|
||||
throw new Error(`expected mock call: ${label}`);
|
||||
}
|
||||
return call[argIndex];
|
||||
}
|
||||
|
||||
function requestOptions(source: MockCallSource, callIndex = 0) {
|
||||
return requireRecord(
|
||||
mockArg(source, callIndex, 1, `request options ${callIndex}`),
|
||||
"request options",
|
||||
);
|
||||
}
|
||||
|
||||
export function requestPath(source: MockCallSource, callIndex = 0) {
|
||||
return mockArg(source, callIndex, 0, `request path ${callIndex}`);
|
||||
}
|
||||
|
||||
export function requestBody(source: MockCallSource, callIndex = 0) {
|
||||
return requireRecord(requestOptions(source, callIndex).body, `request body ${callIndex}`);
|
||||
}
|
||||
|
||||
export function timerDelayAt(source: MockCallSource, callIndex = 0) {
|
||||
return mockArg(source, callIndex, 1, `timer delay ${callIndex}`);
|
||||
}
|
||||
|
||||
export async function createDiscordLoopbackRest(options?: {
|
||||
respond?: (request: DiscordLoopbackRequest) => unknown;
|
||||
}): Promise<{
|
||||
|
||||
@@ -1,11 +1,20 @@
|
||||
import { createTestInboundDebounceFlush } from "openclaw/plugin-sdk/channel-test-helpers";
|
||||
import { createNonExitingRuntimeEnv } from "openclaw/plugin-sdk/plugin-test-runtime";
|
||||
// Feishu ingress tests cover debounce ownership and constituent claim settlement.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { createInboundDebouncer } from "openclaw/plugin-sdk/channel-inbound-debounce";
|
||||
import { DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS } from "openclaw/plugin-sdk/channel-outbound";
|
||||
import { createTestInboundDebounceFlush } from "openclaw/plugin-sdk/channel-test-helpers";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
createChannelIngressQueueForTests,
|
||||
} from "openclaw/plugin-sdk/plugin-state-test-runtime";
|
||||
import { createNonExitingRuntimeEnv } from "openclaw/plugin-sdk/plugin-test-runtime";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { ClawdbotConfig, PluginRuntime, RuntimeEnv } from "../runtime-api.js";
|
||||
import * as dedup from "./dedup.js";
|
||||
import type { FeishuMessageEvent } from "./event-types.js";
|
||||
import type { FeishuIngressLifecycle } from "./feishu-ingress.js";
|
||||
import { createFeishuDurableIngress, type FeishuIngressLifecycle } from "./feishu-ingress.js";
|
||||
import { createFeishuMessageReceiveHandler } from "./monitor.message-handler.js";
|
||||
|
||||
type MessageReceiveHandlerContext = Parameters<typeof createFeishuMessageReceiveHandler>[0];
|
||||
@@ -370,4 +379,144 @@ describe("Feishu durable ingress debounce lifecycle", () => {
|
||||
expect(harness.handleMessage).toHaveBeenCalledTimes(1);
|
||||
expect(second.calls.adopted).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("preserves abandon retry accounting, backoff, threshold, and restart behavior", async () => {
|
||||
vi.useFakeTimers();
|
||||
const now = Date.UTC(2026, 0, 2);
|
||||
vi.setSystemTime(now);
|
||||
const created = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-feishu-abandon-"));
|
||||
const stateDir = await fs.realpath(created);
|
||||
type Queue = NonNullable<Parameters<typeof createFeishuDurableIngress>[0]["queue"]>;
|
||||
type Payload = Parameters<Queue["enqueue"]>[1];
|
||||
const queue = createChannelIngressQueueForTests<Payload>({
|
||||
channelId: "feishu",
|
||||
accountId: "default",
|
||||
stateDir,
|
||||
});
|
||||
const event = {
|
||||
...createTextEvent("evt-abandon-retry", "om-abandon-retry", "retry me"),
|
||||
event_type: "im.message.receive_v1",
|
||||
};
|
||||
const handleMessage = vi.fn(async () => {
|
||||
throw new Error("Feishu dispatch failed before adoption");
|
||||
});
|
||||
vi.spyOn(dedup, "claimUnprocessedFeishuMessage").mockImplementation(async () => ({
|
||||
kind: "claimed",
|
||||
handle: createClaim(`retry-${handleMessage.mock.calls.length}`),
|
||||
}));
|
||||
|
||||
const createIntegratedIngress = () => {
|
||||
const channelRuntime = {
|
||||
commands: { isControlCommandMessage: () => false },
|
||||
debounce: {
|
||||
resolveInboundDebounceMs: () => 0,
|
||||
createInboundDebouncer,
|
||||
},
|
||||
} as unknown as PluginRuntime["channel"];
|
||||
const handler = createFeishuMessageReceiveHandler({
|
||||
cfg: {} as ClawdbotConfig,
|
||||
channelRuntime,
|
||||
accountId: "default",
|
||||
runtime: createNonExitingRuntimeEnv(),
|
||||
chatHistories: new Map(),
|
||||
handleMessage,
|
||||
resolveDebounceText: () => "retry me",
|
||||
hasProcessedMessage: vi.fn(async () => false),
|
||||
getBotOpenId: () => "ou-bot",
|
||||
resolveIngressLifecycle: (data) => ingress.resolveLifecycle(data),
|
||||
});
|
||||
const ingress = createFeishuDurableIngress({
|
||||
accountId: "default",
|
||||
queue,
|
||||
dispatcher: { invoke: async (data: unknown) => await handler(data as never) } as never,
|
||||
runtime: { error: vi.fn(), log: vi.fn() },
|
||||
pollIntervalMs: 500,
|
||||
});
|
||||
return ingress;
|
||||
};
|
||||
const pendingAttempt = async (attempts: number) => {
|
||||
let observed: Awaited<ReturnType<typeof queue.listPending>>[number] | undefined;
|
||||
await vi.waitFor(async () => {
|
||||
const pending = await queue.listPending({ limit: "all" });
|
||||
expect(pending).toEqual([
|
||||
expect.objectContaining({
|
||||
id: "evt-abandon-retry",
|
||||
attempts,
|
||||
lastAttemptAt: expect.any(Number),
|
||||
lastError: "turn-abandoned",
|
||||
}),
|
||||
]);
|
||||
observed = pending[0];
|
||||
});
|
||||
const lastAttemptAt = observed?.lastAttemptAt;
|
||||
if (lastAttemptAt === undefined) {
|
||||
throw new Error(`Missing Feishu retry timestamp for attempt ${attempts}`);
|
||||
}
|
||||
return { ...observed, lastAttemptAt };
|
||||
};
|
||||
|
||||
try {
|
||||
const first = createIntegratedIngress();
|
||||
first.start();
|
||||
await first.invokeWebhook(event);
|
||||
const firstAttempt = await pendingAttempt(1);
|
||||
expect(handleMessage).toHaveBeenCalledTimes(1);
|
||||
await first.stop();
|
||||
|
||||
vi.setSystemTime(firstAttempt.lastAttemptAt + 999);
|
||||
const blocked = createIntegratedIngress();
|
||||
blocked.start();
|
||||
await blocked.invokeWebhook(event);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(handleMessage).toHaveBeenCalledTimes(1);
|
||||
await blocked.stop();
|
||||
|
||||
vi.setSystemTime(firstAttempt.lastAttemptAt + 1_001);
|
||||
const second = createIntegratedIngress();
|
||||
second.start();
|
||||
await second.invokeWebhook(event);
|
||||
const secondAttempt = await pendingAttempt(2);
|
||||
expect(handleMessage).toHaveBeenCalledTimes(2);
|
||||
await second.stop();
|
||||
|
||||
for (let attempt = 3; attempt < DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS; attempt += 1) {
|
||||
const claim = await queue.claim("evt-abandon-retry", { ownerId: `seed-${attempt}` });
|
||||
if (!claim) {
|
||||
throw new Error(`Expected Feishu seed claim ${attempt}`);
|
||||
}
|
||||
await queue.release(claim, {
|
||||
lastError: "turn-abandoned",
|
||||
releasedAt: secondAttempt.lastAttemptAt,
|
||||
});
|
||||
}
|
||||
|
||||
vi.setSystemTime(secondAttempt.lastAttemptAt + 64_001);
|
||||
const threshold = createIntegratedIngress();
|
||||
threshold.start();
|
||||
await threshold.invokeWebhook(event);
|
||||
const thresholdAttempt = await pendingAttempt(DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS);
|
||||
expect(handleMessage).toHaveBeenCalledTimes(3);
|
||||
await threshold.stop();
|
||||
|
||||
vi.setSystemTime(thresholdAttempt.lastAttemptAt + 128_001);
|
||||
const beyond = createIntegratedIngress();
|
||||
beyond.start();
|
||||
await beyond.invokeWebhook(event);
|
||||
const beyondAttempt = await pendingAttempt(DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS + 1);
|
||||
expect(handleMessage).toHaveBeenCalledTimes(4);
|
||||
await beyond.stop();
|
||||
|
||||
vi.setSystemTime(beyondAttempt.lastAttemptAt + 1_000);
|
||||
const blockedRestart = createIntegratedIngress();
|
||||
blockedRestart.start();
|
||||
await blockedRestart.invokeWebhook(event);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(handleMessage).toHaveBeenCalledTimes(4);
|
||||
await blockedRestart.stop();
|
||||
} finally {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,10 +1,20 @@
|
||||
// Mattermost tests cover monitor.inbound system event plugin behavior.
|
||||
import { once } from "node:events";
|
||||
import fs from "node:fs/promises";
|
||||
import { createServer } from "node:http";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { createChannelPartialDeliveryError } from "openclaw/plugin-sdk/channel-inbound";
|
||||
import { createInboundDebouncer } from "openclaw/plugin-sdk/channel-inbound-debounce";
|
||||
import { createMessageReceiptFromOutboundResults } from "openclaw/plugin-sdk/channel-outbound";
|
||||
import {
|
||||
createMessageReceiptFromOutboundResults,
|
||||
DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS,
|
||||
} from "openclaw/plugin-sdk/channel-outbound";
|
||||
import { createTestInboundDebounceFlush } from "openclaw/plugin-sdk/channel-test-helpers";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
createChannelIngressQueueForTests,
|
||||
} from "openclaw/plugin-sdk/plugin-state-test-runtime";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { WebSocketServer } from "ws";
|
||||
import type { MattermostPost } from "./client.js";
|
||||
@@ -96,6 +106,7 @@ const mockState = vi.hoisted(() => ({
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
fetchMattermostMe: vi.fn(),
|
||||
getGlobalHookRunner: vi.fn(),
|
||||
ingressQueue: undefined as unknown,
|
||||
progressDrafts: [] as Array<{ getSnapshot: () => { lines: readonly unknown[] } }>,
|
||||
registerMattermostMonitorSlashCommands: vi.fn(),
|
||||
registerPluginHttpRoute: vi.fn(),
|
||||
@@ -180,27 +191,36 @@ vi.mock("./monitor-ingress.js", async (importOriginal) => {
|
||||
...actual,
|
||||
createMattermostIngressMonitor: (
|
||||
options: Parameters<typeof actual.createMattermostIngressMonitor>[0],
|
||||
) => ({
|
||||
receive: async (rawEvent: string) => {
|
||||
const payload = JSON.parse(rawEvent) as MattermostEventPayload;
|
||||
const post =
|
||||
typeof payload.data?.post === "string"
|
||||
? (JSON.parse(payload.data.post) as MattermostPost)
|
||||
: (payload.data?.post as MattermostPost | undefined);
|
||||
if (payload.event !== "posted" || !post) {
|
||||
return;
|
||||
}
|
||||
await options.dispatch(post, payload, {
|
||||
abortSignal: new AbortController().signal,
|
||||
onAdopted: async () => {},
|
||||
onDeferred: () => {},
|
||||
onAdoptionFinalizing: () => {},
|
||||
onAbandoned: async () => {},
|
||||
) => {
|
||||
if (mockState.ingressQueue) {
|
||||
return actual.createMattermostIngressMonitor({
|
||||
...options,
|
||||
queue: mockState.ingressQueue as NonNullable<typeof options.queue>,
|
||||
pollIntervalMs: 60_000,
|
||||
});
|
||||
},
|
||||
stop: async () => {},
|
||||
waitForIdle: async () => {},
|
||||
}),
|
||||
}
|
||||
return {
|
||||
receive: async (rawEvent: string) => {
|
||||
const payload = JSON.parse(rawEvent) as MattermostEventPayload;
|
||||
const post =
|
||||
typeof payload.data?.post === "string"
|
||||
? (JSON.parse(payload.data.post) as MattermostPost)
|
||||
: (payload.data?.post as MattermostPost | undefined);
|
||||
if (payload.event !== "posted" || !post) {
|
||||
return;
|
||||
}
|
||||
await options.dispatch(post, payload, {
|
||||
abortSignal: new AbortController().signal,
|
||||
onAdopted: async () => {},
|
||||
onDeferred: () => {},
|
||||
onAdoptionFinalizing: () => {},
|
||||
onAbandoned: async () => {},
|
||||
});
|
||||
},
|
||||
stop: async () => {},
|
||||
waitForIdle: async () => {},
|
||||
};
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
@@ -534,6 +554,7 @@ describe("mattermost inbound user posts", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockState.abortController = undefined;
|
||||
mockState.ingressQueue = undefined;
|
||||
mockState.progressDrafts.length = 0;
|
||||
mockState.getGlobalHookRunner.mockReturnValue(null);
|
||||
mockState.runtimeCore = createRuntimeCore(testConfig);
|
||||
@@ -568,6 +589,146 @@ describe("mattermost inbound user posts", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves abandon retry accounting, backoff, threshold, and restart behavior", async () => {
|
||||
vi.useFakeTimers();
|
||||
const now = Date.UTC(2026, 0, 2);
|
||||
vi.setSystemTime(now);
|
||||
const created = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mattermost-abandon-"));
|
||||
const stateDir = await fs.realpath(created);
|
||||
type Payload = { version: 1; receivedAt: number; rawEvent: string };
|
||||
const queue = createChannelIngressQueueForTests<Payload>({
|
||||
channelId: "mattermost",
|
||||
accountId: "default",
|
||||
stateDir,
|
||||
});
|
||||
mockState.ingressQueue = queue;
|
||||
mockState.runtimeCore = createRuntimeCore(testConfig, undefined, {
|
||||
inboundDebounceMs: 0,
|
||||
createInboundDebouncer,
|
||||
});
|
||||
mockState.dispatchInboundMessage.mockRejectedValue(
|
||||
new Error("Mattermost dispatch failed before adoption"),
|
||||
);
|
||||
|
||||
const activeProviders: Array<{ stop: () => Promise<void> }> = [];
|
||||
const startProvider = async () => {
|
||||
const socket = new FakeWebSocket();
|
||||
const abortController = new AbortController();
|
||||
const monitor = monitorMattermostProvider({
|
||||
config: testConfig,
|
||||
runtime: testRuntime(),
|
||||
abortSignal: abortController.signal,
|
||||
webSocketFactory: () => socket,
|
||||
});
|
||||
for (let tick = 0; tick < 20 && socket.openListenerCount === 0; tick += 1) {
|
||||
await Promise.resolve();
|
||||
}
|
||||
expect(socket.openListenerCount).toBeGreaterThan(0);
|
||||
socket.emitOpen();
|
||||
let stopped = false;
|
||||
const provider = {
|
||||
socket,
|
||||
stop: async () => {
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
stopped = true;
|
||||
abortController.abort();
|
||||
socket.emitClose(1000);
|
||||
await monitor;
|
||||
},
|
||||
};
|
||||
activeProviders.push(provider);
|
||||
return provider;
|
||||
};
|
||||
const send = async (provider: Awaited<ReturnType<typeof startProvider>>) => {
|
||||
await emitMattermostChannelPost(provider.socket, {
|
||||
id: "post-abandon-retry",
|
||||
message: "retry me",
|
||||
});
|
||||
};
|
||||
const pendingAttempt = async (attempts: number) => {
|
||||
let observed: Awaited<ReturnType<typeof queue.listPending>>[number] | undefined;
|
||||
await vi.waitFor(async () => {
|
||||
const pending = await queue.listPending({ limit: "all" });
|
||||
expect(pending).toEqual([
|
||||
expect.objectContaining({
|
||||
id: "post-abandon-retry",
|
||||
attempts,
|
||||
lastAttemptAt: expect.any(Number),
|
||||
lastError: "turn-abandoned",
|
||||
}),
|
||||
]);
|
||||
observed = pending[0];
|
||||
});
|
||||
const lastAttemptAt = observed?.lastAttemptAt;
|
||||
if (lastAttemptAt === undefined) {
|
||||
throw new Error(`Missing Mattermost retry timestamp for attempt ${attempts}`);
|
||||
}
|
||||
return { ...observed, lastAttemptAt };
|
||||
};
|
||||
|
||||
try {
|
||||
const first = await startProvider();
|
||||
await send(first);
|
||||
const firstAttempt = await pendingAttempt(1);
|
||||
expect(mockState.dispatchInboundMessage).toHaveBeenCalledTimes(1);
|
||||
await first.stop();
|
||||
|
||||
vi.setSystemTime(firstAttempt.lastAttemptAt + 999);
|
||||
const blocked = await startProvider();
|
||||
await send(blocked);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(mockState.dispatchInboundMessage).toHaveBeenCalledTimes(1);
|
||||
await blocked.stop();
|
||||
|
||||
vi.setSystemTime(firstAttempt.lastAttemptAt + 1_001);
|
||||
const second = await startProvider();
|
||||
await send(second);
|
||||
const secondAttempt = await pendingAttempt(2);
|
||||
expect(mockState.dispatchInboundMessage).toHaveBeenCalledTimes(2);
|
||||
await second.stop();
|
||||
|
||||
for (let attempt = 3; attempt < DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS; attempt += 1) {
|
||||
const claim = await queue.claim("post-abandon-retry", { ownerId: `seed-${attempt}` });
|
||||
if (!claim) {
|
||||
throw new Error(`Expected Mattermost seed claim ${attempt}`);
|
||||
}
|
||||
await queue.release(claim, {
|
||||
lastError: "turn-abandoned",
|
||||
releasedAt: secondAttempt.lastAttemptAt,
|
||||
});
|
||||
}
|
||||
|
||||
vi.setSystemTime(secondAttempt.lastAttemptAt + 64_001);
|
||||
const threshold = await startProvider();
|
||||
await send(threshold);
|
||||
const thresholdAttempt = await pendingAttempt(DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS);
|
||||
expect(mockState.dispatchInboundMessage).toHaveBeenCalledTimes(3);
|
||||
await threshold.stop();
|
||||
|
||||
vi.setSystemTime(thresholdAttempt.lastAttemptAt + 128_001);
|
||||
const beyond = await startProvider();
|
||||
await send(beyond);
|
||||
const beyondAttempt = await pendingAttempt(DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS + 1);
|
||||
expect(mockState.dispatchInboundMessage).toHaveBeenCalledTimes(4);
|
||||
await beyond.stop();
|
||||
|
||||
vi.setSystemTime(beyondAttempt.lastAttemptAt + 1_000);
|
||||
const blockedRestart = await startProvider();
|
||||
await send(blockedRestart);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(mockState.dispatchInboundMessage).toHaveBeenCalledTimes(4);
|
||||
await blockedRestart.stop();
|
||||
} finally {
|
||||
await Promise.allSettled(activeProviders.map(async (provider) => await provider.stop()));
|
||||
mockState.ingressQueue = undefined;
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("publishes recovering while API authentication retries, including 401", async () => {
|
||||
const abortController = new AbortController();
|
||||
const statusSink = vi.fn();
|
||||
@@ -1879,7 +2040,10 @@ describe("mattermost inbound user posts", () => {
|
||||
chatmode: "onmessage",
|
||||
dmPolicy: "open",
|
||||
groupPolicy: "open",
|
||||
streaming: { mode: "block", preview: { toolProgress: true } },
|
||||
streaming: {
|
||||
mode: "block",
|
||||
preview: { toolProgress: true, commandText: "raw" },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -1785,6 +1785,29 @@ describe("getMemoryWikiPage", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("reports a clean error instead of crashing for malformed wiki_get params", async () => {
|
||||
const { config } = await createQueryVault({
|
||||
initialize: true,
|
||||
config: { search: { backend: "shared", corpus: "memory" } },
|
||||
});
|
||||
const tool = createWikiGetTool(config, createAppConfig());
|
||||
const malformedParams = [
|
||||
null,
|
||||
{ path: "sources/example/note.md" },
|
||||
{ lookup: " " },
|
||||
{ lookup: 42 },
|
||||
];
|
||||
|
||||
for (const params of malformedParams) {
|
||||
const result = await tool.execute("wiki-get-bad-param", params);
|
||||
|
||||
expect(result.details).toEqual({ found: false });
|
||||
expect(result.content).toEqual([
|
||||
{ type: "text", text: "wiki_get requires a non-empty `lookup` path or id." },
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
||||
it("normalizes extensionless shared memory lookups before reading", async () => {
|
||||
const { config } = await createQueryVault({
|
||||
initialize: true,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import path from "node:path";
|
||||
import { optionalFiniteNumberSchema } from "openclaw/plugin-sdk/channel-actions";
|
||||
import type { OpenClawPluginToolContext } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { asNonArrayRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { Type } from "typebox";
|
||||
import type { AnyAgentTool, OpenClawConfig } from "../api.js";
|
||||
import { applyMemoryWikiMutation, normalizeMemoryWikiMutationInput } from "./apply.js";
|
||||
@@ -277,13 +278,20 @@ export function createWikiGetTool(
|
||||
"Read a wiki page by id or relative path, or fall back to the active memory corpus when shared search is enabled.",
|
||||
parameters: WikiGetSchema,
|
||||
execute: async (_toolCallId, rawParams) => {
|
||||
const params = rawParams as {
|
||||
lookup: string;
|
||||
const params = asNonArrayRecord(rawParams) as {
|
||||
lookup?: string;
|
||||
fromLine?: number;
|
||||
lineCount?: number;
|
||||
backend?: ResolvedMemoryWikiConfig["search"]["backend"];
|
||||
corpus?: ResolvedMemoryWikiConfig["search"]["corpus"];
|
||||
};
|
||||
const lookup = typeof params.lookup === "string" ? params.lookup.trim() : "";
|
||||
if (!lookup) {
|
||||
return {
|
||||
content: [{ type: "text", text: "wiki_get requires a non-empty `lookup` path or id." }],
|
||||
details: { found: false },
|
||||
};
|
||||
}
|
||||
await syncImportedSourcesIfNeeded(config, appConfig);
|
||||
const result = await getMemoryWikiPage({
|
||||
config,
|
||||
@@ -292,7 +300,7 @@ export function createWikiGetTool(
|
||||
agentSessionKey: memoryContext.agentSessionKey,
|
||||
sandboxed: memoryContext.sandboxed,
|
||||
conversationRecall: memoryContext.conversationRecall,
|
||||
lookup: params.lookup,
|
||||
lookup,
|
||||
fromLine: params.fromLine,
|
||||
lineCount: params.lineCount,
|
||||
...(params.backend ? { searchBackend: params.backend } : {}),
|
||||
@@ -300,7 +308,7 @@ export function createWikiGetTool(
|
||||
});
|
||||
if (!result) {
|
||||
return {
|
||||
content: [{ type: "text", text: `Wiki page not found: ${params.lookup}` }],
|
||||
content: [{ type: "text", text: `Wiki page not found: ${lookup}` }],
|
||||
details: { found: false },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,7 +1,16 @@
|
||||
// Microsoft Teams tests cover durable claim ownership through inbound debounce.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { createInboundDebouncer } from "openclaw/plugin-sdk/channel-inbound-debounce";
|
||||
import { DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS } from "openclaw/plugin-sdk/channel-outbound";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
createChannelIngressQueueForTests,
|
||||
} from "openclaw/plugin-sdk/plugin-state-test-runtime";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig } from "../../runtime-api.js";
|
||||
import { createMSTeamsIngress } from "../msteams-ingress.js";
|
||||
import type { MSTeamsIngressLifecycle } from "../msteams-ingress.js";
|
||||
import type { MSTeamsTurnContext } from "../sdk-types.js";
|
||||
import "./message-handler-mock-support.test-support.js";
|
||||
@@ -156,4 +165,125 @@ describe("Microsoft Teams drain claim ownership", () => {
|
||||
expect(runtimeApiMockState.dispatchReplyWithBufferedBlockDispatcher).not.toHaveBeenCalled();
|
||||
expect(lifecycle.abandonedCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("preserves abandon retry accounting, backoff, threshold, and restart behavior", async () => {
|
||||
vi.useFakeTimers();
|
||||
const now = Date.UTC(2026, 0, 2);
|
||||
vi.setSystemTime(now);
|
||||
const created = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-msteams-abandon-"));
|
||||
const stateDir = await fs.realpath(created);
|
||||
type Queue = NonNullable<Parameters<typeof createMSTeamsIngress>[0]["queue"]>;
|
||||
type Payload = Parameters<Queue["enqueue"]>[1];
|
||||
const queue = createChannelIngressQueueForTests<Payload>({
|
||||
channelId: "msteams",
|
||||
accountId: "test-app",
|
||||
stateDir,
|
||||
});
|
||||
const incoming = directActivity("activity-abandon", "retry me");
|
||||
await queue.enqueue(
|
||||
"activity-abandon",
|
||||
{ version: 1, receivedAt: now - 2 * 24 * 60 * 60_000, rawActivity: JSON.stringify(incoming) },
|
||||
{ laneKey: "dm-conversation", receivedAt: now - 2 * 24 * 60 * 60_000 },
|
||||
);
|
||||
const dispatchMock = runtimeApiMockState.dispatchReplyWithBufferedBlockDispatcher;
|
||||
const priorImplementation = dispatchMock.getMockImplementation();
|
||||
dispatchMock.mockRejectedValue(new Error("Microsoft Teams dispatch failed before adoption"));
|
||||
|
||||
const createIntegratedIngress = () => {
|
||||
const handler = createHandler({
|
||||
channels: { msteams: { dmPolicy: "open", allowFrom: ["*"] } },
|
||||
} as OpenClawConfig);
|
||||
return createMSTeamsIngress({
|
||||
accountId: "test-app",
|
||||
queue,
|
||||
runtime: { error: vi.fn(), log: vi.fn() },
|
||||
dispatch: async (activity, lifecycle) => await handler(context(activity), lifecycle),
|
||||
});
|
||||
};
|
||||
const expectPendingAttempt = async (attempts: number) => {
|
||||
let observed: Awaited<ReturnType<typeof queue.listPending>>[number] | undefined;
|
||||
await vi.waitFor(async () => {
|
||||
const pending = await queue.listPending({ limit: "all" });
|
||||
expect(pending).toEqual([
|
||||
expect.objectContaining({
|
||||
id: "activity-abandon",
|
||||
attempts,
|
||||
lastAttemptAt: expect.any(Number),
|
||||
lastError: "turn-abandoned",
|
||||
}),
|
||||
]);
|
||||
observed = pending[0];
|
||||
});
|
||||
const lastAttemptAt = observed?.lastAttemptAt;
|
||||
if (lastAttemptAt === undefined) {
|
||||
throw new Error(`Missing Microsoft Teams retry timestamp for attempt ${attempts}`);
|
||||
}
|
||||
return { ...observed, lastAttemptAt };
|
||||
};
|
||||
|
||||
try {
|
||||
const first = createIntegratedIngress();
|
||||
first.start();
|
||||
const firstAttempt = await expectPendingAttempt(1);
|
||||
expect(dispatchMock).toHaveBeenCalledTimes(1);
|
||||
await first.stop();
|
||||
|
||||
vi.setSystemTime(firstAttempt.lastAttemptAt + 999);
|
||||
const second = createIntegratedIngress();
|
||||
second.start();
|
||||
await second.accept(incoming);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(dispatchMock).toHaveBeenCalledTimes(1);
|
||||
await second.stop();
|
||||
vi.setSystemTime(firstAttempt.lastAttemptAt + 1_001);
|
||||
const afterBackoff = createIntegratedIngress();
|
||||
afterBackoff.start();
|
||||
await afterBackoff.accept(incoming);
|
||||
const secondAttempt = await expectPendingAttempt(2);
|
||||
expect(dispatchMock).toHaveBeenCalledTimes(2);
|
||||
await afterBackoff.stop();
|
||||
|
||||
for (let attempt = 3; attempt < DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS; attempt += 1) {
|
||||
const claim = await queue.claim("activity-abandon", { ownerId: `seed-${attempt}` });
|
||||
if (!claim) {
|
||||
throw new Error(`Expected Microsoft Teams seed claim ${attempt}`);
|
||||
}
|
||||
await queue.release(claim, {
|
||||
lastError: "turn-abandoned",
|
||||
releasedAt: secondAttempt.lastAttemptAt,
|
||||
});
|
||||
}
|
||||
vi.setSystemTime(secondAttempt.lastAttemptAt + 64_001);
|
||||
const threshold = createIntegratedIngress();
|
||||
threshold.start();
|
||||
await threshold.accept(incoming);
|
||||
const thresholdAttempt = await expectPendingAttempt(DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS);
|
||||
expect(dispatchMock).toHaveBeenCalledTimes(3);
|
||||
await threshold.stop();
|
||||
|
||||
vi.setSystemTime(thresholdAttempt.lastAttemptAt + 128_001);
|
||||
const beyond = createIntegratedIngress();
|
||||
beyond.start();
|
||||
await beyond.accept(incoming);
|
||||
const beyondAttempt = await expectPendingAttempt(DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS + 1);
|
||||
expect(dispatchMock).toHaveBeenCalledTimes(4);
|
||||
await beyond.stop();
|
||||
|
||||
vi.setSystemTime(beyondAttempt.lastAttemptAt + 1_000);
|
||||
const blockedRestart = createIntegratedIngress();
|
||||
blockedRestart.start();
|
||||
await blockedRestart.accept(incoming);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(dispatchMock).toHaveBeenCalledTimes(4);
|
||||
await blockedRestart.stop();
|
||||
} finally {
|
||||
dispatchMock.mockReset();
|
||||
if (priorImplementation) {
|
||||
dispatchMock.mockImplementation(priorImplementation);
|
||||
}
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,15 @@
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
// Signal tests cover retry behavior for reply session initialization conflicts.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS } from "openclaw/plugin-sdk/channel-outbound";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
createChannelIngressQueueForTests,
|
||||
} from "openclaw/plugin-sdk/plugin-state-test-runtime";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { startSignalIngressMonitor } from "../signal-ingress.js";
|
||||
import type { SignalEventHandlerDeps } from "./event-handler.types.js";
|
||||
|
||||
const [
|
||||
@@ -326,6 +335,127 @@ describe("signal reply session init conflict retry", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves durable abandon accounting through backoff, threshold, and restart", async () => {
|
||||
vi.useFakeTimers();
|
||||
const now = Date.UTC(2026, 0, 2);
|
||||
vi.setSystemTime(now);
|
||||
const created = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-signal-abandon-"));
|
||||
const stateDir = await fs.realpath(created);
|
||||
type Queue = NonNullable<Parameters<typeof startSignalIngressMonitor>[0]["queue"]>;
|
||||
type Payload = Parameters<Queue["enqueue"]>[1];
|
||||
const queue = createChannelIngressQueueForTests<Payload>({
|
||||
channelId: "signal",
|
||||
accountId: "default",
|
||||
stateDir,
|
||||
});
|
||||
const timestamp = 1_700_000_000_777;
|
||||
const event = createSignalReceiveEvent({
|
||||
timestamp,
|
||||
dataMessage: { timestamp, message: "retry through durable ingress", attachments: [] },
|
||||
});
|
||||
const eventId = JSON.stringify(["number:+15550001111", timestamp]);
|
||||
dispatchInboundMessageMock.mockRejectedValue(CONFLICT_ERROR);
|
||||
|
||||
const createIntegratedMonitor = async () => {
|
||||
const tracked = createTrackedTaskHarness();
|
||||
const handler = createSignalEventHandler(
|
||||
createBaseSignalEventHandlerDeps({
|
||||
cfg: { messages: { inbound: { debounceMs: 10 } } },
|
||||
runTrackedTask: tracked.runTrackedTask,
|
||||
}),
|
||||
);
|
||||
const monitor = await startSignalIngressMonitor({
|
||||
accountId: "default",
|
||||
queue,
|
||||
dispatch: async (incoming, lifecycle) => await handler(incoming, lifecycle),
|
||||
runtime: { error: vi.fn(), log: vi.fn() },
|
||||
});
|
||||
return { monitor, tracked };
|
||||
};
|
||||
const finishOuterAttempt = async (tracked: ReturnType<typeof createTrackedTaskHarness>) => {
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
expect(tracked.tasks).toHaveLength(1);
|
||||
await vi.advanceTimersByTimeAsync(7_000);
|
||||
await Promise.all(tracked.tasks);
|
||||
};
|
||||
const pendingAttempt = async (attempts: number) => {
|
||||
const pending = await queue.listPending({ limit: "all" });
|
||||
expect(pending).toEqual([
|
||||
expect.objectContaining({
|
||||
id: eventId,
|
||||
attempts,
|
||||
lastAttemptAt: expect.any(Number),
|
||||
lastError: "turn-abandoned",
|
||||
}),
|
||||
]);
|
||||
const record = pending[0];
|
||||
const lastAttemptAt = record?.lastAttemptAt;
|
||||
if (lastAttemptAt === undefined) {
|
||||
throw new Error(`Missing Signal retry timestamp for attempt ${attempts}`);
|
||||
}
|
||||
return { ...record, lastAttemptAt };
|
||||
};
|
||||
|
||||
try {
|
||||
const first = await createIntegratedMonitor();
|
||||
await first.monitor.receive(event);
|
||||
await finishOuterAttempt(first.tracked);
|
||||
const firstAttempt = await pendingAttempt(1);
|
||||
expect(dispatchInboundMessageMock).toHaveBeenCalledTimes(4);
|
||||
await first.monitor.stop();
|
||||
|
||||
vi.setSystemTime(firstAttempt.lastAttemptAt + 999);
|
||||
const blocked = await createIntegratedMonitor();
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
expect(dispatchInboundMessageMock).toHaveBeenCalledTimes(4);
|
||||
expect(blocked.tracked.tasks).toHaveLength(0);
|
||||
await blocked.monitor.stop();
|
||||
|
||||
vi.setSystemTime(firstAttempt.lastAttemptAt + 1_001);
|
||||
const second = await createIntegratedMonitor();
|
||||
await finishOuterAttempt(second.tracked);
|
||||
const secondAttempt = await pendingAttempt(2);
|
||||
expect(dispatchInboundMessageMock).toHaveBeenCalledTimes(8);
|
||||
await second.monitor.stop();
|
||||
|
||||
for (let attempt = 3; attempt < DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS; attempt += 1) {
|
||||
const claim = await queue.claim(eventId, { ownerId: `seed-${attempt}` });
|
||||
if (!claim) {
|
||||
throw new Error(`Expected Signal seed claim ${attempt}`);
|
||||
}
|
||||
await queue.release(claim, {
|
||||
lastError: "turn-abandoned",
|
||||
releasedAt: secondAttempt.lastAttemptAt,
|
||||
});
|
||||
}
|
||||
|
||||
vi.setSystemTime(secondAttempt.lastAttemptAt + 64_001);
|
||||
const threshold = await createIntegratedMonitor();
|
||||
await finishOuterAttempt(threshold.tracked);
|
||||
const thresholdAttempt = await pendingAttempt(DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS);
|
||||
expect(dispatchInboundMessageMock).toHaveBeenCalledTimes(12);
|
||||
await threshold.monitor.stop();
|
||||
|
||||
vi.setSystemTime(thresholdAttempt.lastAttemptAt + 128_001);
|
||||
const beyond = await createIntegratedMonitor();
|
||||
await finishOuterAttempt(beyond.tracked);
|
||||
const beyondAttempt = await pendingAttempt(DEFAULT_INGRESS_RETRY_MAX_ATTEMPTS + 1);
|
||||
expect(dispatchInboundMessageMock).toHaveBeenCalledTimes(16);
|
||||
await beyond.monitor.stop();
|
||||
|
||||
vi.setSystemTime(beyondAttempt.lastAttemptAt + 1_000);
|
||||
const blockedRestart = await createIntegratedMonitor();
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
expect(dispatchInboundMessageMock).toHaveBeenCalledTimes(16);
|
||||
expect(blockedRestart.tracked.tasks).toHaveLength(0);
|
||||
await blockedRestart.monitor.stop();
|
||||
} finally {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not retry non-conflict flush failures", async () => {
|
||||
dispatchInboundMessageMock.mockRejectedValue(new Error("some other dispatch failure"));
|
||||
|
||||
|
||||
@@ -2259,7 +2259,7 @@ describe("createTelegramBot", () => {
|
||||
defaults: {
|
||||
model: "openai/gpt-4.1",
|
||||
},
|
||||
list: [{ id: "agent-a" }, { id: "agent-b" }],
|
||||
list: [{ id: "agent-a", default: true }, { id: "agent-b" }],
|
||||
},
|
||||
channels: {
|
||||
telegram: { dmPolicy: "open", allowFrom: ["*"] },
|
||||
@@ -3689,7 +3689,7 @@ describe("createTelegramBot", () => {
|
||||
},
|
||||
},
|
||||
agents: {
|
||||
list: [{ id: "agent-a" }, { id: "agent-b" }],
|
||||
list: [{ id: "agent-a", default: true }, { id: "agent-b" }],
|
||||
},
|
||||
bindings: [
|
||||
{
|
||||
@@ -3763,8 +3763,9 @@ describe("createTelegramBot", () => {
|
||||
},
|
||||
},
|
||||
agents: {
|
||||
list: [{ id: "topic-a" }, { id: "topic-b" }],
|
||||
list: [{ id: "topic-a", default: true }, { id: "topic-b" }],
|
||||
},
|
||||
bindings: [{ agentId: "topic-a", match: { channel: "telegram", accountId: "default" } }],
|
||||
});
|
||||
loadConfig.mockImplementation(configForTopicAgent);
|
||||
|
||||
@@ -5052,7 +5053,7 @@ describe("createTelegramBot", () => {
|
||||
},
|
||||
},
|
||||
agents: {
|
||||
list: [{ id: "agent-a" }, { id: "agent-b" }],
|
||||
list: [{ id: "agent-a", default: true }, { id: "agent-b" }],
|
||||
},
|
||||
bindings: [
|
||||
{
|
||||
|
||||
@@ -4,7 +4,9 @@ REQUIRED_SWIFT_TOOLS_MAJOR=6
|
||||
REQUIRED_SWIFT_TOOLS_MINOR=2
|
||||
|
||||
require_swift_toolchain() {
|
||||
if ! xcrun xcodebuild -version >/dev/null 2>&1; then
|
||||
local xcodebuild_version
|
||||
if ! xcodebuild_version="$(xcrun xcodebuild -version 2>&1)"; then
|
||||
printf '%s\n' "$xcodebuild_version" >&2
|
||||
echo "ERROR: OpenClaw macOS app packaging requires a full Xcode developer directory." >&2
|
||||
echo " Command Line Tools do not include the required SwiftUI macro plugins." >&2
|
||||
echo " Use: sudo xcode-select -s /Applications/Xcode.app/Contents/Developer" >&2
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
#!/usr/bin/env node
|
||||
// Builds the OpenClaw package artifact used by Docker E2E.
|
||||
// The script owns the build/inventory/pack sequence so local scheduler, shell
|
||||
// helpers, and GitHub Actions all prepare the exact same npm tarball.
|
||||
// Builds the canonical OpenClaw package artifact used by Docker E2E.
|
||||
import { spawn } from "node:child_process";
|
||||
import { closeSync, openSync } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import { createRequire } from "node:module";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV } from "./lib/bundled-plugin-build-entries.mjs";
|
||||
@@ -32,10 +32,10 @@ const AI_RUNTIME_BACKUP_DIR = ".openclaw-ai-package-backup";
|
||||
type KillChild = (signal: NodeJS.Signals) => void;
|
||||
type RunOptions = {
|
||||
captureStdout?: boolean;
|
||||
deferForwardedSignalExit?: boolean;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
killAfterMs?: unknown;
|
||||
maxCapturedStdoutBytes?: number;
|
||||
stdoutFilePath?: string;
|
||||
timeoutMs?: unknown;
|
||||
};
|
||||
type CommandRunnerOptions = {
|
||||
@@ -48,15 +48,11 @@ type CommandRunner = (
|
||||
cwd: string,
|
||||
options: CommandRunnerOptions,
|
||||
) => Promise<unknown>;
|
||||
type CaptureRunnerOptions = {
|
||||
deferForwardedSignalExit?: boolean;
|
||||
timeoutMs?: number;
|
||||
};
|
||||
type RunImpl = (
|
||||
command: string,
|
||||
args: string[],
|
||||
cwd: string,
|
||||
options: CaptureRunnerOptions,
|
||||
options: RunOptions,
|
||||
) => Promise<string>;
|
||||
type DocsMapLifecycle = {
|
||||
preparePackageDocsMap: (cwd: string) => Promise<unknown>;
|
||||
@@ -305,6 +301,13 @@ export function parseArgs(argv: string[]) {
|
||||
}
|
||||
|
||||
function run(command: string, args: string[], cwd: string, options: RunOptions = {}) {
|
||||
const setupError =
|
||||
options.captureStdout && options.stdoutFilePath
|
||||
? new Error("captureStdout and stdoutFilePath cannot be combined")
|
||||
: forwardedSignalExitCode && new ForwardedSignalExitError(forwardedSignalExitCode);
|
||||
if (setupError) {
|
||||
return Promise.reject(setupError);
|
||||
}
|
||||
return new Promise<string>((resolve, reject) => {
|
||||
const resolvedTimeoutMs = resolveOptionalTimerTimeoutMs(options.timeoutMs);
|
||||
const resolvedKillAfterMs = resolvePackageBuildTimeoutMs(
|
||||
@@ -326,14 +329,22 @@ function run(command: string, args: string[], cwd: string, options: RunOptions =
|
||||
: process.platform === "win32" && command === "npm"
|
||||
? resolveNpmRunner({ env, npmArgs: args })
|
||||
: { args, command, shell: false };
|
||||
const child = spawn(invocation.command, invocation.args, {
|
||||
cwd,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
env: invocation.env ?? env,
|
||||
detached: useProcessGroup,
|
||||
shell: invocation.shell,
|
||||
windowsVerbatimArguments: invocation.windowsVerbatimArguments,
|
||||
});
|
||||
const stdoutFd = options.stdoutFilePath ? openSync(options.stdoutFilePath, "wx") : undefined;
|
||||
let child: ReturnType<typeof spawn>;
|
||||
try {
|
||||
child = spawn(invocation.command, invocation.args, {
|
||||
cwd,
|
||||
stdio: ["ignore", stdoutFd ?? "pipe", "pipe"],
|
||||
env: invocation.env ?? env,
|
||||
detached: useProcessGroup,
|
||||
shell: invocation.shell,
|
||||
windowsVerbatimArguments: invocation.windowsVerbatimArguments,
|
||||
});
|
||||
} finally {
|
||||
if (stdoutFd !== undefined) {
|
||||
closeSync(stdoutFd);
|
||||
}
|
||||
}
|
||||
let timedOut = false;
|
||||
let outputLimitExceeded = false;
|
||||
let stdout = "";
|
||||
@@ -354,10 +365,6 @@ function run(command: string, args: string[], cwd: string, options: RunOptions =
|
||||
}
|
||||
ACTIVE_CHILD_KILLERS.delete(killChild);
|
||||
if (forwardedSignalExitCode !== undefined && ACTIVE_CHILD_KILLERS.size === 0) {
|
||||
if (options.deferForwardedSignalExit) {
|
||||
reject(new ForwardedSignalExitError(forwardedSignalExitCode));
|
||||
return;
|
||||
}
|
||||
process.exit(forwardedSignalExitCode);
|
||||
}
|
||||
if (error) {
|
||||
@@ -423,7 +430,7 @@ function run(command: string, args: string[], cwd: string, options: RunOptions =
|
||||
finish(error, value);
|
||||
};
|
||||
if (options.captureStdout) {
|
||||
child.stdout.on("data", (chunk) => {
|
||||
child.stdout?.on("data", (chunk) => {
|
||||
if (outputLimitExceeded) {
|
||||
return;
|
||||
}
|
||||
@@ -437,10 +444,10 @@ function run(command: string, args: string[], cwd: string, options: RunOptions =
|
||||
stdout += chunkText;
|
||||
stdoutBytes += chunkBytes;
|
||||
});
|
||||
} else {
|
||||
child.stdout.pipe(process.stderr, { end: false });
|
||||
} else if (!options.stdoutFilePath) {
|
||||
child.stdout?.pipe(process.stderr, { end: false });
|
||||
}
|
||||
child.stderr.pipe(process.stderr, { end: false });
|
||||
child.stderr?.pipe(process.stderr, { end: false });
|
||||
child.on("error", (error) => finish(error));
|
||||
child.on("close", (status, signal) => {
|
||||
if (timedOut) {
|
||||
@@ -503,12 +510,12 @@ export async function buildPackageArtifacts(
|
||||
}
|
||||
}
|
||||
|
||||
export const runCommandForTest = run;
|
||||
|
||||
async function runCapture(command: string, args: string[], cwd: string, options: RunOptions = {}) {
|
||||
return await run(command, args, cwd, { ...options, captureStdout: true });
|
||||
return await run(command, args, cwd, { ...options, captureStdout: !options.stdoutFilePath });
|
||||
}
|
||||
|
||||
export { run as runCommandForTest, runCapture as runCaptureForTest };
|
||||
|
||||
async function newestOpenClawTarball(outputDir: string, packOutput: string) {
|
||||
let fromOutput = "";
|
||||
try {
|
||||
@@ -740,7 +747,6 @@ export async function prepareBundledAiRuntimePackage(
|
||||
],
|
||||
sourceDir,
|
||||
{
|
||||
deferForwardedSignalExit: true,
|
||||
timeoutMs: resolveTimeoutMs(
|
||||
"OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS",
|
||||
DEFAULT_PACKAGE_PACK_TIMEOUT_MS,
|
||||
@@ -859,11 +865,9 @@ async function loadSourcePackageLifecycle(
|
||||
}
|
||||
|
||||
function packagePreparationRestoreError(error: unknown, restoreError: unknown) {
|
||||
return new AggregateError(
|
||||
[error, restoreError],
|
||||
"Package preparation failed and source artifacts could not be restored.",
|
||||
{ cause: error },
|
||||
);
|
||||
return new AggregateError([error, restoreError], "Package operation and cleanup both failed.", {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
|
||||
export async function packOpenClawPackageForDocker(
|
||||
@@ -921,6 +925,14 @@ export async function packOpenClawPackageForDocker(
|
||||
console.error("==> Packing OpenClaw package");
|
||||
// This receipt is the package lifecycle lock; acquire it before touching CHANGELOG.md.
|
||||
await prepareDocsMap(sourcePath);
|
||||
const deferSignalExit: KillChild = () => {};
|
||||
ACTIVE_CHILD_KILLERS.add(deferSignalExit);
|
||||
const releaseSignalExit = () => {
|
||||
ACTIVE_CHILD_KILLERS.delete(deferSignalExit);
|
||||
if (forwardedSignalExitCode !== undefined) {
|
||||
throw new ForwardedSignalExitError(forwardedSignalExitCode);
|
||||
}
|
||||
};
|
||||
try {
|
||||
await prepareManifest(sourcePath);
|
||||
await prepareChangelog(sourcePath);
|
||||
@@ -933,66 +945,99 @@ export async function packOpenClawPackageForDocker(
|
||||
restoreChangelog,
|
||||
);
|
||||
} catch (restoreError) {
|
||||
releaseSignalExit();
|
||||
throw packagePreparationRestoreError(error, restoreError);
|
||||
}
|
||||
releaseSignalExit();
|
||||
throw error;
|
||||
}
|
||||
let packOutput = "";
|
||||
let cleanupBundledAiRuntime = async () => {};
|
||||
let packageError: unknown;
|
||||
let packReceiptDir: string | undefined;
|
||||
try {
|
||||
await cleanPackedOpenClawTarballs(outputPath);
|
||||
cleanupBundledAiRuntime = await prepareBundledAiRuntime(
|
||||
sourcePath,
|
||||
let cleanupBundledAiRuntime = async () => {};
|
||||
try {
|
||||
await cleanPackedOpenClawTarballs(outputPath);
|
||||
cleanupBundledAiRuntime = await prepareBundledAiRuntime(
|
||||
sourcePath,
|
||||
outputPath,
|
||||
runCaptureImpl,
|
||||
{
|
||||
prepareManifest,
|
||||
restoreManifest,
|
||||
},
|
||||
);
|
||||
const packArgs =
|
||||
packTool === "pnpm"
|
||||
? ["pack", "--silent", "--config.ignore-scripts=true", "--pack-destination", outputPath]
|
||||
: [
|
||||
"pack",
|
||||
...(packageOptions.packJsonPath ? ["--json"] : []),
|
||||
"--silent",
|
||||
"--ignore-scripts",
|
||||
"--pack-destination",
|
||||
outputPath,
|
||||
];
|
||||
if (packTool === "npm" && packageOptions.packJsonPath) {
|
||||
packReceiptDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-npm-pack-receipt-"));
|
||||
}
|
||||
const packReceiptPath = packReceiptDir ? path.join(packReceiptDir, "pack.json") : undefined;
|
||||
packOutput = await runCaptureImpl(packTool, packArgs, sourcePath, {
|
||||
stdoutFilePath: packReceiptPath,
|
||||
timeoutMs: resolveTimeoutMs(
|
||||
"OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS",
|
||||
DEFAULT_PACKAGE_PACK_TIMEOUT_MS,
|
||||
),
|
||||
});
|
||||
if (packReceiptPath) {
|
||||
packOutput = await fs.readFile(packReceiptPath, "utf8");
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
await cleanupBundledAiRuntime();
|
||||
} finally {
|
||||
await restorePackageSourceArtifacts(
|
||||
sourcePath,
|
||||
restoreDocsMap,
|
||||
restoreManifest,
|
||||
restoreChangelog,
|
||||
);
|
||||
}
|
||||
}
|
||||
// Scan the emptied pnpm destination instead of trusting its absolute-path output.
|
||||
let tarball = await newestOpenClawTarball(
|
||||
outputPath,
|
||||
runCaptureImpl,
|
||||
{
|
||||
prepareManifest,
|
||||
restoreManifest,
|
||||
},
|
||||
packageOptions.pnpmPack ? "" : packOutput,
|
||||
);
|
||||
const packArgs =
|
||||
packTool === "pnpm"
|
||||
? ["pack", "--silent", "--config.ignore-scripts=true", "--pack-destination", outputPath]
|
||||
: [
|
||||
"pack",
|
||||
...(packageOptions.packJsonPath ? ["--json"] : []),
|
||||
"--silent",
|
||||
"--ignore-scripts",
|
||||
"--pack-destination",
|
||||
outputPath,
|
||||
];
|
||||
packOutput = await runCaptureImpl(packTool, packArgs, sourcePath, {
|
||||
deferForwardedSignalExit: true,
|
||||
timeoutMs: resolveTimeoutMs(
|
||||
"OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS",
|
||||
DEFAULT_PACKAGE_PACK_TIMEOUT_MS,
|
||||
),
|
||||
});
|
||||
if (packageOptions.outputName) {
|
||||
const target = path.join(outputPath, packageOptions.outputName);
|
||||
if (target !== tarball) {
|
||||
await fs.rm(target, { force: true });
|
||||
await fs.rename(tarball, target);
|
||||
tarball = target;
|
||||
}
|
||||
}
|
||||
await writePackJson(packOutput, tarball, packageOptions.packJsonPath, sourcePath);
|
||||
return tarball;
|
||||
} catch (error) {
|
||||
packageError = error;
|
||||
throw error;
|
||||
} finally {
|
||||
try {
|
||||
await cleanupBundledAiRuntime();
|
||||
if (packReceiptDir) {
|
||||
try {
|
||||
await fs.rm(packReceiptDir, { force: true, recursive: true });
|
||||
} catch (cleanupError) {
|
||||
// oxlint-disable-next-line eslint/no-unsafe-finally -- Preserve primary and cleanup failures.
|
||||
throw packageError
|
||||
? packagePreparationRestoreError(packageError, cleanupError)
|
||||
: cleanupError;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await restorePackageSourceArtifacts(
|
||||
sourcePath,
|
||||
restoreDocsMap,
|
||||
restoreManifest,
|
||||
restoreChangelog,
|
||||
);
|
||||
releaseSignalExit();
|
||||
}
|
||||
}
|
||||
// pnpm reports an absolute destination path. The directory was emptied before packing,
|
||||
// so scan that controlled destination instead of accepting a path from command output.
|
||||
let tarball = await newestOpenClawTarball(outputPath, packageOptions.pnpmPack ? "" : packOutput);
|
||||
if (packageOptions.outputName) {
|
||||
const target = path.join(outputPath, packageOptions.outputName);
|
||||
if (target !== tarball) {
|
||||
await fs.rm(target, { force: true });
|
||||
await fs.rename(tarball, target);
|
||||
tarball = target;
|
||||
}
|
||||
}
|
||||
await writePackJson(packOutput, tarball, packageOptions.packJsonPath, sourcePath);
|
||||
return tarball;
|
||||
}
|
||||
|
||||
export async function writePackageInventoryForDocker(
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { bindIngressLifecycleToReplyOptions } from "./ingress-drain-lifecycle.js";
|
||||
|
||||
describe("channel ingress drain lifecycle", () => {
|
||||
it("binds only the reply-lane ownership surface", async () => {
|
||||
const abort = new AbortController();
|
||||
const calls: string[] = [];
|
||||
const bound = bindIngressLifecycleToReplyOptions({
|
||||
abortSignal: abort.signal,
|
||||
onAdoptionFinalizing: () => {
|
||||
calls.push("finalizing");
|
||||
},
|
||||
onFailed: () => {
|
||||
calls.push("failed");
|
||||
},
|
||||
onCancelled: () => {
|
||||
calls.push("cancelled");
|
||||
},
|
||||
onAdopted: () => {
|
||||
calls.push("adopted");
|
||||
},
|
||||
onDeferred: () => {
|
||||
calls.push("deferred");
|
||||
},
|
||||
onAbandoned: () => {
|
||||
calls.push("abandoned");
|
||||
},
|
||||
});
|
||||
|
||||
expect(bound.turnAdoptionLifecycle).toMatchObject({
|
||||
admission: "exclusive",
|
||||
abortSignal: abort.signal,
|
||||
});
|
||||
expect("onFailed" in bound.turnAdoptionLifecycle).toBe(false);
|
||||
expect("onCancelled" in bound.turnAdoptionLifecycle).toBe(false);
|
||||
expect("onAdopted" in bound).toBe(false);
|
||||
expect(Object.keys(bound)).toEqual(["turnAdoptionLifecycle"]);
|
||||
bound.turnAdoptionLifecycle.onDeferred();
|
||||
await bound.turnAdoptionLifecycle.onAbandoned();
|
||||
expect(calls).toEqual(["deferred", "abandoned"]);
|
||||
calls.length = 0;
|
||||
bound.turnAdoptionLifecycle.onDeferred();
|
||||
await bound.turnAdoptionLifecycle.onAdopted();
|
||||
expect(calls).toEqual(["deferred", "adopted"]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
/** Full pre-adoption -> adoption ownership lifecycle for one claimed event. */
|
||||
export type ChannelIngressDispatchLifecycle = {
|
||||
/** Pre-adoption only. After adopt the drain treats this signal as inert. */
|
||||
abortSignal: AbortSignal;
|
||||
/**
|
||||
* Fires when recovery-relevant session/run state is durable.
|
||||
* Drain completes (tombstones) the claim here -- never at settle.
|
||||
*/
|
||||
onAdopted: () => void | Promise<void>;
|
||||
/**
|
||||
* Turn ownership deferred to reply-lane admission (queued followup).
|
||||
* Claim remains held until adopted or abandoned.
|
||||
*/
|
||||
onDeferred: () => void;
|
||||
/**
|
||||
* Durable adoption finalization is in progress (e.g. settlement hold while
|
||||
* committing dedupe). Clears the pre-adoption stall watchdog so a timeout
|
||||
* settlement cannot race and dead-letter an about-to-complete claim.
|
||||
* Claim stays held until onAdopted / onAbandoned / fail.
|
||||
*/
|
||||
onAdoptionFinalizing: () => void;
|
||||
/** Deferred work terminally failed after dispatch returned. */
|
||||
onFailed?: (error: unknown) => void | Promise<void>;
|
||||
/** Explicit cancellation before adoption; releases without consuming retry budget. */
|
||||
onCancelled?: () => void | Promise<void>;
|
||||
/**
|
||||
* Deferred turn finished without ever owning the reply lane.
|
||||
* Drain releases the claim for retry.
|
||||
*/
|
||||
onAbandoned: () => void | Promise<void>;
|
||||
};
|
||||
|
||||
/** Maps a drain lifecycle onto the reply-lane ownership surface. */
|
||||
export function bindIngressLifecycleToReplyOptions(lifecycle: ChannelIngressDispatchLifecycle): {
|
||||
turnAdoptionLifecycle: {
|
||||
admission: "exclusive";
|
||||
onAdopted: () => void | Promise<void>;
|
||||
onDeferred: () => void;
|
||||
onAbandoned: () => void | Promise<void>;
|
||||
abortSignal: AbortSignal;
|
||||
};
|
||||
} {
|
||||
return {
|
||||
turnAdoptionLifecycle: {
|
||||
admission: "exclusive",
|
||||
onAdopted: lifecycle.onAdopted,
|
||||
onDeferred: lifecycle.onDeferred,
|
||||
onAbandoned: lifecycle.onAbandoned,
|
||||
abortSignal: lifecycle.abortSignal,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// onAdoptionFinalizing stays drain-only (not reply-options); channels call it
|
||||
// via the spooled-replay ALS lifecycle frame during settlement hold.
|
||||
@@ -0,0 +1,89 @@
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js";
|
||||
import { createChannelIngressDrain } from "./ingress-drain.js";
|
||||
import {
|
||||
createTestIngressQueue,
|
||||
type IngressDrainTestPayload as Payload,
|
||||
withTempState,
|
||||
} from "./ingress-drain.test-helpers.js";
|
||||
|
||||
type ChannelIngressDispatchLifecycle = Parameters<
|
||||
Parameters<typeof createChannelIngressDrain>[0]["dispatchClaimedEvent"]
|
||||
>[1];
|
||||
|
||||
describe("channel ingress drain cancellation", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
});
|
||||
|
||||
it("cancels unadopted work without changing its retry facts", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
let clock = 100;
|
||||
const queue = createTestIngressQueue(stateDir, { now: () => clock });
|
||||
await queue.enqueue("evt-cancel", { text: "x" }, { laneKey: "l1", receivedAt: 1 });
|
||||
const failedClaim = await queue.claim("evt-cancel", { ownerId: "failed-owner" });
|
||||
expect(failedClaim).not.toBeNull();
|
||||
if (!failedClaim) {
|
||||
return;
|
||||
}
|
||||
await queue.release(failedClaim, { lastError: "previous failure", releasedAt: clock });
|
||||
const before = (await queue.listPending())[0];
|
||||
for (let cycle = 0; cycle < 3; cycle += 1) {
|
||||
const lifecycles: ChannelIngressDispatchLifecycle[] = [];
|
||||
clock += 1;
|
||||
const drain = createChannelIngressDrain<Payload>({
|
||||
queue,
|
||||
now: () => clock,
|
||||
retryPolicy: { baseMs: 0, maxMs: 0 },
|
||||
dispatchClaimedEvent: async (_event, lifecycle) => {
|
||||
lifecycles.push(lifecycle);
|
||||
return { kind: "deferred" };
|
||||
},
|
||||
});
|
||||
|
||||
await drain.drainOnce();
|
||||
await vi.waitFor(() => expect(lifecycles).toHaveLength(1));
|
||||
await expectDefined(
|
||||
expectDefined(lifecycles[0], "cancelled lifecycle").onCancelled,
|
||||
"cancel callback",
|
||||
)();
|
||||
expect(await queue.listPending()).toEqual([
|
||||
expect.objectContaining({
|
||||
id: "evt-cancel",
|
||||
attempts: before?.attempts,
|
||||
lastAttemptAt: before?.lastAttemptAt,
|
||||
lastError: before?.lastError,
|
||||
}),
|
||||
]);
|
||||
expect(await queue.listClaims()).toEqual([]);
|
||||
drain.dispose();
|
||||
}
|
||||
|
||||
const terminal = createChannelIngressDrain<Payload>({
|
||||
queue,
|
||||
now: () => clock,
|
||||
retryPolicy: { maxAttempts: 2, deadLetterMinAgeMs: 0, baseMs: 0, maxMs: 0 },
|
||||
dispatchClaimedEvent: async () => {
|
||||
throw new Error("final genuine failure");
|
||||
},
|
||||
});
|
||||
await terminal.drainOnce();
|
||||
await terminal.waitForIdle();
|
||||
expect(await queue.listFailed?.()).toEqual([
|
||||
expect.objectContaining({
|
||||
id: "evt-cancel",
|
||||
attempts: 1,
|
||||
reason: "retry-limit-exceeded",
|
||||
message: "final genuine failure",
|
||||
}),
|
||||
]);
|
||||
terminal.dispose();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -3,7 +3,6 @@ import { expectDefined } from "@openclaw/normalization-core";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js";
|
||||
import {
|
||||
bindIngressLifecycleToReplyOptions,
|
||||
createChannelIngressDrain,
|
||||
DEFAULT_INGRESS_ADOPTION_STALL_MS,
|
||||
isIngressAdoptionLostError,
|
||||
@@ -170,6 +169,84 @@ describe("channel ingress drain", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the lane owned until a dead-letter write commits", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
const queue = createTestIngressQueue(stateDir);
|
||||
await queue.enqueue("poison", { text: "bad" }, { laneKey: "shared", receivedAt: 1 });
|
||||
await queue.enqueue("follower", { text: "good" }, { laneKey: "shared", receivedAt: 2 });
|
||||
const fail = queue.fail.bind(queue);
|
||||
let failAttempts = 0;
|
||||
queue.fail = async (...args) => {
|
||||
failAttempts += 1;
|
||||
if (failAttempts < 3) {
|
||||
throw new Error(`transient fail write ${failAttempts}`);
|
||||
}
|
||||
return await fail(...args);
|
||||
};
|
||||
const dispatched: string[] = [];
|
||||
const drain = createChannelIngressDrain<Payload>({
|
||||
queue,
|
||||
retryPolicy: { maxAttempts: 1, deadLetterMinAgeMs: 0 },
|
||||
dispatchClaimedEvent: async (event, lifecycle) => {
|
||||
dispatched.push(event.id);
|
||||
if (event.id === "poison") {
|
||||
throw new Error("poison delivery");
|
||||
}
|
||||
await lifecycle.onAdopted();
|
||||
},
|
||||
});
|
||||
|
||||
await drain.drainOnce();
|
||||
const idle = drain.waitForIdle();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(failAttempts).toBe(1);
|
||||
expect(drain.activeLaneKeys()).toEqual(new Set(["shared"]));
|
||||
expect(await drain.drainOnce()).toEqual({ started: 0 });
|
||||
expect(dispatched).toEqual(["poison"]);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
await idle;
|
||||
expect(failAttempts).toBe(3);
|
||||
expect(await drain.drainOnce()).toEqual({ started: 1 });
|
||||
await drain.waitForIdle();
|
||||
expect(dispatched).toEqual(["poison", "follower"]);
|
||||
drain.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps ownership when every dead-letter write fails", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
const queue = createTestIngressQueue(stateDir);
|
||||
await queue.enqueue("poison", { text: "bad" }, { laneKey: "shared", receivedAt: 1 });
|
||||
await queue.enqueue("follower", { text: "good" }, { laneKey: "shared", receivedAt: 2 });
|
||||
queue.fail = async () => {
|
||||
throw new Error("persistent fail write");
|
||||
};
|
||||
const dispatched: string[] = [];
|
||||
const drain = createChannelIngressDrain<Payload>({
|
||||
queue,
|
||||
retryPolicy: { maxAttempts: 1, deadLetterMinAgeMs: 0 },
|
||||
dispatchClaimedEvent: async (event) => {
|
||||
dispatched.push(event.id);
|
||||
throw new Error("poison delivery");
|
||||
},
|
||||
});
|
||||
|
||||
await drain.drainOnce();
|
||||
const idle = drain.waitForIdle();
|
||||
for (let attempt = 0; attempt < 8; attempt += 1) {
|
||||
await vi.advanceTimersByTimeAsync(180_000);
|
||||
}
|
||||
await idle;
|
||||
|
||||
expect(dispatched).toEqual(["poison"]);
|
||||
expect(drain.activeLaneKeys()).toEqual(new Set(["shared"]));
|
||||
expect((await queue.listClaims()).map((claim) => claim.id)).toEqual(["poison"]);
|
||||
expect(await drain.drainOnce()).toEqual({ started: 0 });
|
||||
drain.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
it("holds lanes by default and releases only opted-in deferred lanes", async () => {
|
||||
for (const occupancy of ["hold", "release"] as const) {
|
||||
await withTempState(async (stateDir) => {
|
||||
@@ -378,7 +455,7 @@ describe("channel ingress drain", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("abandoned via turnAdoptionLifecycle releases claim with attempt increment", async () => {
|
||||
it("abandoned reply ownership releases claim with attempt increment", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
const queue = createTestIngressQueue(stateDir);
|
||||
await queue.enqueue("evt-q", { text: "x" }, { laneKey: "l1" });
|
||||
@@ -386,10 +463,9 @@ describe("channel ingress drain", () => {
|
||||
const drain = createChannelIngressDrain<Payload>({
|
||||
queue,
|
||||
dispatchClaimedEvent: async (_event, lifecycle) => {
|
||||
const bound = bindIngressLifecycleToReplyOptions(lifecycle);
|
||||
bound.turnAdoptionLifecycle.onDeferred();
|
||||
lifecycle.onDeferred();
|
||||
// Never admitted — abandon path releases claim.
|
||||
await bound.turnAdoptionLifecycle.onAbandoned();
|
||||
await lifecycle.onAbandoned();
|
||||
return { kind: "deferred" };
|
||||
},
|
||||
});
|
||||
@@ -405,7 +481,7 @@ describe("channel ingress drain", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("queued deferral→admission completes the claim exactly once via turnAdoptionLifecycle", async () => {
|
||||
it("queued deferral -> admission completes the claim exactly once", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
const queue = createTestIngressQueue(stateDir);
|
||||
await queue.enqueue("evt-admit", { text: "x" }, { laneKey: "l1" });
|
||||
@@ -414,13 +490,12 @@ describe("channel ingress drain", () => {
|
||||
const drain = createChannelIngressDrain<Payload>({
|
||||
queue,
|
||||
dispatchClaimedEvent: async (_event, lifecycle) => {
|
||||
const bound = bindIngressLifecycleToReplyOptions(lifecycle);
|
||||
// Simulate queue enqueue (defer) then reply-lane admission (adopt).
|
||||
bound.turnAdoptionLifecycle.onDeferred();
|
||||
await bound.turnAdoptionLifecycle.onAdopted();
|
||||
lifecycle.onDeferred();
|
||||
await lifecycle.onAdopted();
|
||||
adoptCount += 1;
|
||||
// Second adopt from lifecycle must be a no-op for the claim.
|
||||
await bound.turnAdoptionLifecycle.onAdopted();
|
||||
await lifecycle.onAdopted();
|
||||
adoptCount += 1;
|
||||
return { kind: "deferred" };
|
||||
},
|
||||
@@ -440,105 +515,6 @@ describe("channel ingress drain", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("watchdog only guillotines pre-adoption stalls with handler-timeout", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
let clock = 10_000;
|
||||
const queue = createTestIngressQueue(stateDir, { now: () => clock });
|
||||
await queue.enqueue("evt-stall", { text: "x" }, { laneKey: "l1" });
|
||||
|
||||
const drain = createChannelIngressDrain<Payload>({
|
||||
queue,
|
||||
now: () => clock,
|
||||
adoptionStallTimeoutMs: 5_000,
|
||||
dispatchClaimedEvent: async () => {
|
||||
// Never adopt, never return — stall until watchdog.
|
||||
await new Promise(() => {});
|
||||
},
|
||||
});
|
||||
|
||||
await drain.drainOnce();
|
||||
clock += 5_000;
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
await drain.waitForIdle();
|
||||
|
||||
// Failed tombstone, not pending retry.
|
||||
const reenqueue = await queue.enqueue("evt-stall", { text: "x" });
|
||||
expect(reenqueue.kind).toBe("failed");
|
||||
if (reenqueue.kind === "failed") {
|
||||
expect(reenqueue.record.reason).toBe("handler-timeout");
|
||||
}
|
||||
drain.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
it("watchdog guillotines deferred phase (timer not cleared by deferral)", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
let clock = 30_000;
|
||||
const queue = createTestIngressQueue(stateDir, { now: () => clock });
|
||||
await queue.enqueue("evt-def-stall", { text: "x" }, { laneKey: "l1" });
|
||||
|
||||
const drain = createChannelIngressDrain<Payload>({
|
||||
queue,
|
||||
now: () => clock,
|
||||
adoptionStallTimeoutMs: 5_000,
|
||||
dispatchClaimedEvent: async (_event, lifecycle) => {
|
||||
lifecycle.onDeferred();
|
||||
// Stay deferred without adoption — watchdog must still fire.
|
||||
await new Promise(() => {});
|
||||
},
|
||||
});
|
||||
|
||||
await drain.drainOnce();
|
||||
expect(await queue.listClaims()).toHaveLength(1);
|
||||
clock += 5_000;
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
await drain.waitForIdle();
|
||||
|
||||
const reenqueue = await queue.enqueue("evt-def-stall", { text: "x" });
|
||||
expect(reenqueue.kind).toBe("failed");
|
||||
if (reenqueue.kind === "failed") {
|
||||
expect(reenqueue.record.reason).toBe("handler-timeout");
|
||||
}
|
||||
drain.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
it("watchdog does not kill healthy long turns after adoption", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
let clock = 20_000;
|
||||
const queue = createTestIngressQueue(stateDir, { now: () => clock });
|
||||
await queue.enqueue("evt-long", { text: "x" }, { laneKey: "l1" });
|
||||
|
||||
let settleResolve!: () => void;
|
||||
const settleGate = new Promise<void>((resolve) => {
|
||||
settleResolve = resolve;
|
||||
});
|
||||
|
||||
const drain = createChannelIngressDrain<Payload>({
|
||||
queue,
|
||||
now: () => clock,
|
||||
adoptionStallTimeoutMs: 1_000,
|
||||
dispatchClaimedEvent: async (_event, lifecycle) => {
|
||||
await lifecycle.onAdopted();
|
||||
await settleGate;
|
||||
},
|
||||
});
|
||||
|
||||
await drain.drainOnce();
|
||||
await vi.waitFor(async () => {
|
||||
expect(await queue.listClaims()).toEqual([]);
|
||||
});
|
||||
clock += 60_000;
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
// Still only completed — not failed by watchdog.
|
||||
const status = await queue.enqueue("evt-long", { text: "x" });
|
||||
expect(status.kind).toBe("completed");
|
||||
settleResolve();
|
||||
await drain.waitForIdle();
|
||||
drain.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
it("supersede tombstones the superseded claim (never re-dispatches)", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
const queue = createTestIngressQueue(stateDir);
|
||||
@@ -697,39 +673,37 @@ describe("channel ingress drain", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("bindIngressLifecycleToReplyOptions returns only turnAdoptionLifecycle", async () => {
|
||||
const abort = new AbortController();
|
||||
const calls: string[] = [];
|
||||
const bound = bindIngressLifecycleToReplyOptions({
|
||||
abortSignal: abort.signal,
|
||||
onAdoptionFinalizing: () => {
|
||||
calls.push("finalizing");
|
||||
},
|
||||
onFailed: () => {
|
||||
calls.push("failed");
|
||||
},
|
||||
onAdopted: () => {
|
||||
calls.push("adopted");
|
||||
},
|
||||
onDeferred: () => {
|
||||
calls.push("deferred");
|
||||
},
|
||||
onAbandoned: () => {
|
||||
calls.push("abandoned");
|
||||
},
|
||||
it("keeps retry-accounted abandonment pending beyond the failure threshold", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
let clock = 1;
|
||||
const queue = createTestIngressQueue(stateDir, { now: () => clock });
|
||||
await queue.enqueue("abandoned", { text: "x" }, { laneKey: "l", receivedAt: 1 });
|
||||
|
||||
for (let attempt = 0; attempt < 3; attempt += 1) {
|
||||
clock += 1;
|
||||
const drain = createChannelIngressDrain<Payload>({
|
||||
queue,
|
||||
now: () => clock,
|
||||
retryPolicy: { maxAttempts: 1, deadLetterMinAgeMs: 0, baseMs: 0, maxMs: 0 },
|
||||
dispatchClaimedEvent: async (_event, lifecycle) => {
|
||||
await lifecycle.onAbandoned();
|
||||
return { kind: "deferred" };
|
||||
},
|
||||
});
|
||||
await drain.drainOnce();
|
||||
await drain.waitForIdle();
|
||||
drain.dispose();
|
||||
}
|
||||
|
||||
expect(await queue.listPending()).toEqual([
|
||||
expect.objectContaining({
|
||||
id: "abandoned",
|
||||
attempts: 3,
|
||||
lastError: "turn-abandoned",
|
||||
}),
|
||||
]);
|
||||
expect(await queue.listFailed?.()).toEqual([]);
|
||||
});
|
||||
expect(bound.turnAdoptionLifecycle.abortSignal).toBe(abort.signal);
|
||||
expect(bound.turnAdoptionLifecycle.admission).toBe("exclusive");
|
||||
expect("onFailed" in bound.turnAdoptionLifecycle).toBe(false);
|
||||
expect("onAdopted" in bound).toBe(false);
|
||||
expect(Object.keys(bound)).toEqual(["turnAdoptionLifecycle"]);
|
||||
bound.turnAdoptionLifecycle.onDeferred();
|
||||
await bound.turnAdoptionLifecycle.onAbandoned();
|
||||
expect(calls).toEqual(["deferred", "abandoned"]);
|
||||
calls.length = 0;
|
||||
bound.turnAdoptionLifecycle.onDeferred();
|
||||
await bound.turnAdoptionLifecycle.onAdopted();
|
||||
expect(calls).toEqual(["deferred", "adopted"]);
|
||||
});
|
||||
|
||||
it("refreshes active claims on claimLeaseMs/3 while deferred", async () => {
|
||||
@@ -1141,17 +1115,4 @@ describe("channel ingress drain", () => {
|
||||
drain.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
it("bindIngressLifecycleToReplyOptions marks exclusive admission", () => {
|
||||
const abort = new AbortController();
|
||||
const bound = bindIngressLifecycleToReplyOptions({
|
||||
abortSignal: abort.signal,
|
||||
onAdopted: async () => {},
|
||||
onDeferred: () => {},
|
||||
onAdoptionFinalizing: () => {},
|
||||
onFailed: () => {},
|
||||
onAbandoned: () => {},
|
||||
});
|
||||
expect(bound.turnAdoptionLifecycle.admission).toBe("exclusive");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
isLiveLocalIngressDrainOwner,
|
||||
registerLiveIngressDrainInstance,
|
||||
} from "./ingress-claim-owner.js";
|
||||
import type { ChannelIngressDispatchLifecycle } from "./ingress-drain-lifecycle.js";
|
||||
import {
|
||||
activeClaimKey,
|
||||
IngressAdoptionLostError,
|
||||
@@ -25,6 +26,7 @@ import {
|
||||
type ChannelIngressDrainDispatchResult,
|
||||
} from "./ingress-drain-state.js";
|
||||
import { supersedeActiveStatesIfNeeded } from "./ingress-drain-supersede.js";
|
||||
export { bindIngressLifecycleToReplyOptions } from "./ingress-drain-lifecycle.js";
|
||||
export { isIngressAdoptionLostError } from "./ingress-drain-state.js";
|
||||
import type {
|
||||
ChannelIngressQueue,
|
||||
@@ -46,36 +48,6 @@ export const DEFAULT_INGRESS_ADOPTION_STALL_MS = 5 * 60 * 1000;
|
||||
/** Bounded tombstone write retries — wedged ownership beats silent double-dispatch. */
|
||||
const INGRESS_TOMBSTONE_RETRY_MAX_ATTEMPTS = 8;
|
||||
|
||||
/** Full pre-adoption → adoption ownership lifecycle for one claimed event. */
|
||||
type ChannelIngressDispatchLifecycle = {
|
||||
/** Pre-adoption only. After adopt the drain treats this signal as inert. */
|
||||
abortSignal: AbortSignal;
|
||||
/**
|
||||
* Fires when recovery-relevant session/run state is durable.
|
||||
* Drain completes (tombstones) the claim here — never at settle.
|
||||
*/
|
||||
onAdopted: () => void | Promise<void>;
|
||||
/**
|
||||
* Turn ownership deferred to reply-lane admission (queued followup).
|
||||
* Claim remains held until adopted or abandoned.
|
||||
*/
|
||||
onDeferred: () => void;
|
||||
/**
|
||||
* Durable adoption finalization is in progress (e.g. settlement hold while
|
||||
* committing dedupe). Clears the pre-adoption stall watchdog so a timeout
|
||||
* settlement cannot race and dead-letter an about-to-complete claim.
|
||||
* Claim stays held until onAdopted / onAbandoned / fail.
|
||||
*/
|
||||
onAdoptionFinalizing: () => void;
|
||||
/** Deferred work terminally failed after dispatch returned. */
|
||||
onFailed?: (error: unknown) => void | Promise<void>;
|
||||
/**
|
||||
* Deferred turn finished without ever owning the reply lane.
|
||||
* Drain releases the claim for retry.
|
||||
*/
|
||||
onAbandoned: () => void | Promise<void>;
|
||||
};
|
||||
|
||||
type DeferredLaneOccupancy = "hold" | "release";
|
||||
|
||||
export type CreateChannelIngressDrainOptions<
|
||||
@@ -132,34 +104,6 @@ export type ChannelIngressDrain = {
|
||||
dispose: () => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* Maps a drain lifecycle onto reply options.
|
||||
* Single surface: turnAdoptionLifecycle only.
|
||||
* Marks exclusive admission so collect isolation is not inferred from onAbandoned.
|
||||
*/
|
||||
export function bindIngressLifecycleToReplyOptions(lifecycle: ChannelIngressDispatchLifecycle): {
|
||||
turnAdoptionLifecycle: {
|
||||
admission: "exclusive";
|
||||
onAdopted: () => void | Promise<void>;
|
||||
onDeferred: () => void;
|
||||
onAbandoned: () => void | Promise<void>;
|
||||
abortSignal: AbortSignal;
|
||||
};
|
||||
} {
|
||||
return {
|
||||
turnAdoptionLifecycle: {
|
||||
admission: "exclusive",
|
||||
onAdopted: lifecycle.onAdopted,
|
||||
onDeferred: lifecycle.onDeferred,
|
||||
onAbandoned: lifecycle.onAbandoned,
|
||||
abortSignal: lifecycle.abortSignal,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// onAdoptionFinalizing stays drain-only (not reply-options); channels call it
|
||||
// via the spooled-replay ALS lifecycle frame during settlement hold.
|
||||
|
||||
/** Creates a channel-agnostic durable ingress drain over an existing queue. */
|
||||
export function createChannelIngressDrain<
|
||||
TPayload,
|
||||
@@ -347,13 +291,12 @@ export function createChannelIngressDrain<
|
||||
|
||||
const releaseClaim = async (
|
||||
claim: ChannelIngressQueueClaim<TPayload, TMetadata>,
|
||||
lastError?: string,
|
||||
releaseOptions?: { lastError?: string; recordAttempt?: boolean },
|
||||
) => {
|
||||
await commitClaimWriteWithRetry({
|
||||
claim,
|
||||
label: "release",
|
||||
write: () =>
|
||||
queue.release(claim, lastError === undefined ? {} : { lastError, releasedAt: now() }),
|
||||
write: () => queue.release(claim, { ...releaseOptions, releasedAt: now() }),
|
||||
falseMeansReclaimed: false,
|
||||
});
|
||||
};
|
||||
@@ -401,7 +344,7 @@ export function createChannelIngressDrain<
|
||||
}
|
||||
const displayId = claim.id.replace(/^0+(?=\d)/, "") || claim.id;
|
||||
log(`spooled update ${displayId} failed; keeping for retry: ${disposition.message}`);
|
||||
await releaseClaim(claim, disposition.message);
|
||||
await releaseClaim(claim, { lastError: disposition.message });
|
||||
};
|
||||
|
||||
const createSettleOwner = (
|
||||
@@ -468,6 +411,24 @@ export function createChannelIngressDrain<
|
||||
state.stallTimer.unref?.();
|
||||
};
|
||||
|
||||
const releaseUnadopted = async (
|
||||
state: ActiveHandlerState<TPayload, TMetadata>,
|
||||
releaseOptions: { lastError?: string; recordAttempt?: boolean },
|
||||
) => {
|
||||
if (state.phase !== "deferred" && state.phase !== "dispatching") {
|
||||
return;
|
||||
}
|
||||
if (state.guillotined || state.superseded) {
|
||||
return;
|
||||
}
|
||||
clearStallTimer(state);
|
||||
await state
|
||||
.settleOnce(async () => {
|
||||
await releaseClaim(state.claim, releaseOptions);
|
||||
})
|
||||
.catch(() => undefined);
|
||||
};
|
||||
|
||||
const createLifecycle = (
|
||||
state: ActiveHandlerState<TPayload, TMetadata>,
|
||||
): ChannelIngressDispatchLifecycle => {
|
||||
@@ -528,19 +489,13 @@ export function createChannelIngressDrain<
|
||||
await applyFailureDisposition(state.claim, error);
|
||||
});
|
||||
},
|
||||
onCancelled: async () => {
|
||||
// Cancellation means ownership ended before delivery, so preserve every
|
||||
// prior retry fact while reopening the canonical row for replacement.
|
||||
await releaseUnadopted(state, { recordAttempt: false });
|
||||
},
|
||||
onAbandoned: async () => {
|
||||
if (state.phase !== "deferred" && state.phase !== "dispatching") {
|
||||
return;
|
||||
}
|
||||
if (state.guillotined || state.superseded) {
|
||||
return;
|
||||
}
|
||||
clearStallTimer(state);
|
||||
await state
|
||||
.settleOnce(async () => {
|
||||
await releaseClaim(state.claim, "turn-abandoned");
|
||||
})
|
||||
.catch(() => undefined);
|
||||
await releaseUnadopted(state, { lastError: "turn-abandoned" });
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js";
|
||||
import { createChannelIngressDrain } from "./ingress-drain.js";
|
||||
import {
|
||||
createTestIngressQueue,
|
||||
type IngressDrainTestPayload as Payload,
|
||||
withTempState,
|
||||
} from "./ingress-drain.test-helpers.js";
|
||||
|
||||
describe("channel ingress drain watchdog", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
});
|
||||
|
||||
it("guillotines pre-adoption stalls with handler-timeout", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
let clock = 10_000;
|
||||
const queue = createTestIngressQueue(stateDir, { now: () => clock });
|
||||
await queue.enqueue("evt-stall", { text: "x" }, { laneKey: "l1" });
|
||||
|
||||
const drain = createChannelIngressDrain<Payload>({
|
||||
queue,
|
||||
now: () => clock,
|
||||
adoptionStallTimeoutMs: 5_000,
|
||||
dispatchClaimedEvent: async () => {
|
||||
// Never adopt, never return -- stall until watchdog.
|
||||
await new Promise(() => {});
|
||||
},
|
||||
});
|
||||
|
||||
await drain.drainOnce();
|
||||
clock += 5_000;
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
await drain.waitForIdle();
|
||||
|
||||
const reenqueue = await queue.enqueue("evt-stall", { text: "x" });
|
||||
expect(reenqueue.kind).toBe("failed");
|
||||
if (reenqueue.kind === "failed") {
|
||||
expect(reenqueue.record.reason).toBe("handler-timeout");
|
||||
}
|
||||
drain.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
it("guillotines deferred stalls", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
let clock = 30_000;
|
||||
const queue = createTestIngressQueue(stateDir, { now: () => clock });
|
||||
await queue.enqueue("evt-def-stall", { text: "x" }, { laneKey: "l1" });
|
||||
|
||||
const drain = createChannelIngressDrain<Payload>({
|
||||
queue,
|
||||
now: () => clock,
|
||||
adoptionStallTimeoutMs: 5_000,
|
||||
dispatchClaimedEvent: async (_event, lifecycle) => {
|
||||
lifecycle.onDeferred();
|
||||
// Stay deferred without adoption -- watchdog must still fire.
|
||||
await new Promise(() => {});
|
||||
},
|
||||
});
|
||||
|
||||
await drain.drainOnce();
|
||||
expect(await queue.listClaims()).toHaveLength(1);
|
||||
clock += 5_000;
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
await drain.waitForIdle();
|
||||
|
||||
const reenqueue = await queue.enqueue("evt-def-stall", { text: "x" });
|
||||
expect(reenqueue.kind).toBe("failed");
|
||||
if (reenqueue.kind === "failed") {
|
||||
expect(reenqueue.record.reason).toBe("handler-timeout");
|
||||
}
|
||||
drain.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
it("does not kill healthy long turns after adoption", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
let clock = 20_000;
|
||||
const queue = createTestIngressQueue(stateDir, { now: () => clock });
|
||||
await queue.enqueue("evt-long", { text: "x" }, { laneKey: "l1" });
|
||||
|
||||
let settleResolve!: () => void;
|
||||
const settleGate = new Promise<void>((resolve) => {
|
||||
settleResolve = resolve;
|
||||
});
|
||||
|
||||
const drain = createChannelIngressDrain<Payload>({
|
||||
queue,
|
||||
now: () => clock,
|
||||
adoptionStallTimeoutMs: 1_000,
|
||||
dispatchClaimedEvent: async (_event, lifecycle) => {
|
||||
await lifecycle.onAdopted();
|
||||
await settleGate;
|
||||
},
|
||||
});
|
||||
|
||||
await drain.drainOnce();
|
||||
await vi.waitFor(async () => {
|
||||
expect(await queue.listClaims()).toEqual([]);
|
||||
});
|
||||
clock += 60_000;
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
const status = await queue.enqueue("evt-long", { text: "x" });
|
||||
expect(status.kind).toBe("completed");
|
||||
settleResolve();
|
||||
await drain.waitForIdle();
|
||||
drain.dispose();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -65,6 +65,7 @@ export type ChannelIngressMonitorLifecycle = {
|
||||
onDeferred: () => void;
|
||||
onAdoptionFinalizing: () => void;
|
||||
onFailed?: (error: unknown) => void | Promise<void>;
|
||||
onCancelled?: () => void | Promise<void>;
|
||||
onAbandoned: () => void | Promise<void>;
|
||||
};
|
||||
|
||||
@@ -368,6 +369,16 @@ export function createChannelIngressMonitor<TRaw, TBody, TStoredPayload, TMetada
|
||||
settleDeferredClaim();
|
||||
}
|
||||
}
|
||||
const settleDeferredLifecycle = async (settle: () => void | Promise<void>) => {
|
||||
handedOff = true;
|
||||
deferredHandoff = true;
|
||||
try {
|
||||
await settle();
|
||||
requestDrain();
|
||||
} finally {
|
||||
settleDeferredClaim();
|
||||
}
|
||||
};
|
||||
const wrappedLifecycle: ChannelIngressMonitorLifecycle = {
|
||||
...lifecycle,
|
||||
admission: "exclusive",
|
||||
@@ -393,26 +404,9 @@ export function createChannelIngressMonitor<TRaw, TBody, TStoredPayload, TMetada
|
||||
deferredHandoff = true;
|
||||
lifecycle.onAdoptionFinalizing();
|
||||
},
|
||||
onFailed: async (error) => {
|
||||
handedOff = true;
|
||||
deferredHandoff = true;
|
||||
try {
|
||||
await lifecycle.onFailed?.(error);
|
||||
requestDrain();
|
||||
} finally {
|
||||
settleDeferredClaim();
|
||||
}
|
||||
},
|
||||
onAbandoned: async () => {
|
||||
handedOff = true;
|
||||
deferredHandoff = true;
|
||||
try {
|
||||
await lifecycle.onAbandoned();
|
||||
requestDrain();
|
||||
} finally {
|
||||
settleDeferredClaim();
|
||||
}
|
||||
},
|
||||
onFailed: (error) => settleDeferredLifecycle(() => lifecycle.onFailed?.(error)),
|
||||
onCancelled: () => settleDeferredLifecycle(() => lifecycle.onCancelled?.()),
|
||||
onAbandoned: () => settleDeferredLifecycle(() => lifecycle.onAbandoned()),
|
||||
};
|
||||
|
||||
// Adoption can complete before delivery returns; track both lifetimes so stop
|
||||
|
||||
@@ -119,32 +119,66 @@ describe("channel ingress queue", () => {
|
||||
it("keeps channel and account queue identities unambiguous", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
const first = createChannelIngressQueue<{ text: string }>({
|
||||
channelId: "a",
|
||||
accountId: "b:c",
|
||||
channelId: "discord",
|
||||
accountId: "account-a",
|
||||
stateDir,
|
||||
});
|
||||
const second = createChannelIngressQueue<{ text: string }>({
|
||||
channelId: "a:b",
|
||||
accountId: "c",
|
||||
channelId: "discord",
|
||||
accountId: "account-b",
|
||||
stateDir,
|
||||
});
|
||||
|
||||
expect(await first.enqueue("same-id", { text: "first" })).toMatchObject({
|
||||
expect(
|
||||
await first.enqueue("same-id", { text: "first" }, { laneKey: "channel:same-lane" }),
|
||||
).toMatchObject({
|
||||
kind: "accepted",
|
||||
});
|
||||
expect(await second.enqueue("same-id", { text: "second" })).toMatchObject({
|
||||
expect(
|
||||
await second.enqueue("same-id", { text: "second" }, { laneKey: "channel:same-lane" }),
|
||||
).toMatchObject({
|
||||
kind: "accepted",
|
||||
});
|
||||
|
||||
await first.complete("same-id");
|
||||
const firstClaim = await first.claim("same-id", { ownerId: "first-worker" });
|
||||
expect(firstClaim).not.toBeNull();
|
||||
if (!firstClaim) {
|
||||
return;
|
||||
}
|
||||
await first.fail(firstClaim, { reason: "poison", failedAt: 20 });
|
||||
|
||||
expect(await first.enqueue("same-id", { text: "first duplicate" })).toMatchObject({
|
||||
kind: "completed",
|
||||
kind: "failed",
|
||||
});
|
||||
expect(await second.enqueue("same-id", { text: "second duplicate" })).toMatchObject({
|
||||
kind: "pending",
|
||||
record: { payload: { text: "second" } },
|
||||
});
|
||||
|
||||
if (!first.resubmit) {
|
||||
return;
|
||||
}
|
||||
await expect(first.resubmit("same-id", { resubmittedAt: 30 })).resolves.toMatchObject({
|
||||
kind: "resubmitted",
|
||||
record: { attempts: 0, laneKey: "channel:same-lane", payload: { text: "first" } },
|
||||
});
|
||||
const resubmittedClaim = await first.claim("same-id", { ownerId: "replacement" });
|
||||
const secondClaim = await second.claim("same-id", { ownerId: "second-worker" });
|
||||
expect(resubmittedClaim).not.toBeNull();
|
||||
expect(secondClaim).not.toBeNull();
|
||||
if (!resubmittedClaim || !secondClaim) {
|
||||
return;
|
||||
}
|
||||
await first.fail(resubmittedClaim, { reason: "poison-again", failedAt: 40 });
|
||||
await second.complete(secondClaim, { completedAt: 40 });
|
||||
|
||||
expect(await first.prune({ failedTtlMs: 1, now: 42 })).toBe(1);
|
||||
expect(await first.enqueue("same-id", { text: "fresh after prune" })).toMatchObject({
|
||||
kind: "accepted",
|
||||
});
|
||||
expect(await second.enqueue("same-id", { text: "completed duplicate" })).toMatchObject({
|
||||
kind: "completed",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -2,10 +2,45 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { withTempDir } from "../test-utils/temp-dir.js";
|
||||
import { VERSION } from "../version.js";
|
||||
import { createConfigIO } from "./io.js";
|
||||
import { createConfigIO } from "./io.factory.js";
|
||||
import { normalizeExecSafeBinProfilesInConfig } from "./normalize-exec-safe-bin.js";
|
||||
import { withTempHome } from "./test-helpers.js";
|
||||
|
||||
vi.mock("../commands/doctor/shared/legacy-config-compat.js", () => ({
|
||||
applyLegacyDoctorMigrations: () => {
|
||||
throw new Error("config IO compatibility tests must not enter recovery migration");
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("../plugins/gateway-startup-plugin-ids.js", () => ({
|
||||
createConfigValidationMetadataPluginIdScope: (params: {
|
||||
config: { plugins?: { entries?: Record<string, unknown> } };
|
||||
}) => {
|
||||
const configuredPluginIds = Object.keys(params.config.plugins?.entries ?? {}).toSorted();
|
||||
const removedPluginIds = new Set(["google-antigravity-auth", "google-gemini-cli-auth"]);
|
||||
if (configuredPluginIds.some((pluginId) => !removedPluginIds.has(pluginId))) {
|
||||
throw new Error("config IO compatibility tests require real metadata for active plugins");
|
||||
}
|
||||
return {
|
||||
key: `io-compat:${configuredPluginIds.join(",")}`,
|
||||
resolve: () => [],
|
||||
};
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("../plugins/plugin-metadata-snapshot.js", () => ({
|
||||
rebasePluginMetadataSnapshotManifestRegistry: () => {
|
||||
throw new Error("config IO compatibility tests must not materialize metadata snapshots");
|
||||
},
|
||||
resolvePluginMetadataSnapshot: () => ({
|
||||
manifestRegistry: { plugins: [], diagnostics: [] },
|
||||
}),
|
||||
}));
|
||||
|
||||
function withTempHome<T>(run: (home: string) => Promise<T>): Promise<T> {
|
||||
return withTempDir("openclaw-config-compat-", run);
|
||||
}
|
||||
|
||||
async function writeConfig(
|
||||
home: string,
|
||||
@@ -61,71 +96,27 @@ describe("config io paths", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["lan", "loopback", "tailnet", "auto", "custom", undefined] as const)(
|
||||
"keeps canonical gateway bind %s byte-identical during load",
|
||||
async (bind) => {
|
||||
await withTempHome(async (home) => {
|
||||
const configPath = path.join(home, ".openclaw", "openclaw.json");
|
||||
await fs.mkdir(path.dirname(configPath), { recursive: true });
|
||||
const gateway = {
|
||||
mode: "local" as const,
|
||||
...(bind ? { bind } : {}),
|
||||
...(bind === "custom" ? { customBindHost: "127.0.0.1" } : {}),
|
||||
};
|
||||
const raw = `${JSON.stringify({ gateway }, null, 2)}\n`;
|
||||
await fs.writeFile(configPath, raw, "utf-8");
|
||||
const io = createConfigIO({
|
||||
configPath,
|
||||
env: { HOME: home } as NodeJS.ProcessEnv,
|
||||
homedir: () => home,
|
||||
});
|
||||
|
||||
const config = io.loadConfig();
|
||||
|
||||
expect(config.gateway?.bind).toBe(bind);
|
||||
await expect(fs.readFile(configPath, "utf-8")).resolves.toBe(raw);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("logs validation warnings with real line breaks", async () => {
|
||||
it("keeps canonical custom gateway bind byte-identical during load", async () => {
|
||||
await withTempHome(async (home) => {
|
||||
const configPath = path.join(home, ".openclaw", "openclaw.json");
|
||||
await fs.mkdir(path.dirname(configPath), { recursive: true });
|
||||
await fs.writeFile(
|
||||
configPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
plugins: {
|
||||
entries: {
|
||||
"google-antigravity-auth": {
|
||||
enabled: false,
|
||||
config: { stale: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
const logger = {
|
||||
error: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
const gateway = {
|
||||
mode: "local" as const,
|
||||
bind: "custom" as const,
|
||||
customBindHost: "127.0.0.1",
|
||||
};
|
||||
|
||||
const raw = `${JSON.stringify({ gateway }, null, 2)}\n`;
|
||||
await fs.writeFile(configPath, raw, "utf-8");
|
||||
const io = createConfigIO({
|
||||
configPath,
|
||||
env: { HOME: home } as NodeJS.ProcessEnv,
|
||||
homedir: () => home,
|
||||
logger,
|
||||
});
|
||||
io.loadConfig();
|
||||
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
"Config warnings:\n- plugins.entries.google-antigravity-auth: plugin removed: google-antigravity-auth (stale config entry ignored; remove it from plugins config)",
|
||||
);
|
||||
expect(logger.warn).not.toHaveBeenCalledWith("Config warnings:\\n");
|
||||
const config = io.loadConfig();
|
||||
|
||||
expect(config.gateway).toMatchObject({ bind: "custom", customBindHost: "127.0.0.1" });
|
||||
await expect(fs.readFile(configPath, "utf-8")).resolves.toBe(raw);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -155,6 +146,9 @@ describe("config io paths", () => {
|
||||
load();
|
||||
load();
|
||||
expect(logger.warn).toHaveBeenCalledTimes(1);
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
"Config warnings:\n- plugins.entries.google-antigravity-auth: plugin removed: google-antigravity-auth (stale config entry ignored; remove it from plugins config)",
|
||||
);
|
||||
|
||||
createConfigIO({
|
||||
configPath,
|
||||
|
||||
@@ -10,6 +10,7 @@ import type {
|
||||
NodeWorkerSupervisorNodeProof,
|
||||
NodeWorkerSupervisorTransport,
|
||||
} from "../node-registry-private.js";
|
||||
import { createNodeWorkerLaunchAdapter } from "./node-launch-adapter.js";
|
||||
import type { WorkerEnvironmentServiceContract } from "./service-contract.js";
|
||||
|
||||
export const DEVICE_WORKER_PROVIDER_ID = "device";
|
||||
@@ -61,6 +62,7 @@ function deviceLeaseId(deviceId: string, operationId: string): string {
|
||||
/** Core runtime for already-paired node hosts; pairing remains the durable trust owner. */
|
||||
export function createDeviceWorkerRuntime(options: DeviceWorkerRuntimeOptions) {
|
||||
let nodeTransport: NodeWorkerSupervisorTransport | undefined;
|
||||
const launchAdapter = createNodeWorkerLaunchAdapter({ getTransport: () => nodeTransport });
|
||||
const findConnectedNode = async (deviceId: string) =>
|
||||
(await nodeTransport?.listCurrentNodes())?.find(
|
||||
(node) => node.nodeId === deviceId && isSessionCapableNode(node),
|
||||
@@ -103,6 +105,7 @@ export function createDeviceWorkerRuntime(options: DeviceWorkerRuntimeOptions) {
|
||||
return {
|
||||
provider,
|
||||
isAvailable,
|
||||
launchNodeWorker: launchAdapter.launch,
|
||||
bindNodeTransport: (transport: NodeWorkerSupervisorTransport) => {
|
||||
nodeTransport = transport;
|
||||
},
|
||||
|
||||
@@ -0,0 +1,431 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
GATEWAY_CLIENT_IDS,
|
||||
GATEWAY_CLIENT_MODES,
|
||||
} from "../../../packages/gateway-protocol/src/client-info.js";
|
||||
import {
|
||||
WORKER_PROTOCOL_FEATURES,
|
||||
WORKER_RPC_SET_VERSION,
|
||||
} from "../../../packages/gateway-protocol/src/schema/worker-admission.js";
|
||||
import { NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE } from "../../infra/node-worker-supervisor-dialect.js";
|
||||
import {
|
||||
nodeWorkerPlanHash,
|
||||
type NodeWorkerLaunchInput,
|
||||
type NodeWorkerSupervisorReceipt,
|
||||
} from "../../worker/node-supervisor-protocol.js";
|
||||
import type {
|
||||
NodeWorkerSupervisorNodeProof,
|
||||
NodeWorkerSupervisorTransport,
|
||||
} from "../node-registry-private.js";
|
||||
import { createNodeWorkerLaunchAdapter } from "./node-launch-adapter.js";
|
||||
|
||||
const DEVICE_ID = "device-session-host";
|
||||
|
||||
function nodeProof(connId = "conn-1"): NodeWorkerSupervisorNodeProof {
|
||||
return {
|
||||
nodeId: DEVICE_ID,
|
||||
connId,
|
||||
pairingIdentity: "identity-1",
|
||||
pairingGeneration: "generation-1",
|
||||
clientId: GATEWAY_CLIENT_IDS.NODE_HOST,
|
||||
clientMode: GATEWAY_CLIENT_MODES.NODE,
|
||||
protocolFeature: NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
|
||||
commands: ["system.run"],
|
||||
};
|
||||
}
|
||||
|
||||
function launchInput(): NodeWorkerLaunchInput {
|
||||
const bundleHash = "a".repeat(64);
|
||||
return {
|
||||
launchId: "turn-1",
|
||||
gatewayNamespace: "gateway-1",
|
||||
bundleHash,
|
||||
placementGeneration: 4,
|
||||
descriptor: {
|
||||
version: 3,
|
||||
connectionEndpoint: {
|
||||
kind: "websocket",
|
||||
url: "wss://gateway.example/__openclaw__/worker",
|
||||
},
|
||||
admission: {
|
||||
environmentId: "environment-1",
|
||||
credential: "worker-fixture-value",
|
||||
sessionId: "session-1",
|
||||
ownerEpoch: 3,
|
||||
rpcSetVersion: WORKER_RPC_SET_VERSION,
|
||||
handshake: {
|
||||
bundleHash,
|
||||
openclawVersion: "2026.8.1",
|
||||
protocolFeatures: [...WORKER_PROTOCOL_FEATURES],
|
||||
},
|
||||
},
|
||||
assignment: {
|
||||
agentId: "agent-1",
|
||||
operationalRunInstance: { instanceId: "instance-1", runId: "run-1" },
|
||||
agentRuntimeIdentityToken: "signed-runtime-token",
|
||||
runId: "run-1",
|
||||
turnId: "turn-1",
|
||||
prompt: "Inspect the workspace.",
|
||||
suppressPromptTranscript: true,
|
||||
workspaceDir: "/tmp/openclaw-worker/workspace",
|
||||
modelRef: { provider: "provider-1", model: "model-1" },
|
||||
inferenceOptions: {},
|
||||
initialMessages: [],
|
||||
transcript: { baseLeafId: null, nextSeq: 1 },
|
||||
liveEvents: { ackedSeq: 0, nextSeq: 1 },
|
||||
toolAuthority: { allowedToolNames: [] },
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function receipt(
|
||||
input: NodeWorkerLaunchInput,
|
||||
state: NodeWorkerSupervisorReceipt["state"],
|
||||
): NodeWorkerSupervisorReceipt {
|
||||
const identity = {
|
||||
launchId: input.launchId,
|
||||
planHash: nodeWorkerPlanHash(input),
|
||||
environmentId: input.descriptor.admission.environmentId,
|
||||
sessionId: input.descriptor.admission.sessionId,
|
||||
ownerEpoch: input.descriptor.admission.ownerEpoch,
|
||||
placementGeneration: input.placementGeneration,
|
||||
runId: input.descriptor.assignment.runId,
|
||||
};
|
||||
if (state === "completed") {
|
||||
return {
|
||||
...identity,
|
||||
state,
|
||||
resultJson: JSON.stringify({
|
||||
status: "completed",
|
||||
transcriptLeafId: "leaf-1",
|
||||
transcriptNextSeq: 2,
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (state === "failed" || state === "interrupted" || state === "cancelled") {
|
||||
return { ...identity, state, errorText: `worker ${state}` };
|
||||
}
|
||||
return { ...identity, state };
|
||||
}
|
||||
|
||||
function wire(payload: NodeWorkerSupervisorReceipt | null) {
|
||||
return { ok: true, payloadJSON: JSON.stringify(payload) };
|
||||
}
|
||||
|
||||
function transportWith(
|
||||
invoke: NodeWorkerSupervisorTransport["invoke"],
|
||||
listCurrentNodes: NodeWorkerSupervisorTransport["listCurrentNodes"] = async () => [nodeProof()],
|
||||
): NodeWorkerSupervisorTransport {
|
||||
return { invoke, listCurrentNodes };
|
||||
}
|
||||
|
||||
function launchRequest(input = launchInput()) {
|
||||
return {
|
||||
deviceId: DEVICE_ID,
|
||||
input,
|
||||
isDispatchAuthorized: () => true,
|
||||
isCancellationAuthorized: () => true,
|
||||
timeoutMs: 10_000,
|
||||
};
|
||||
}
|
||||
|
||||
describe("node worker launch adapter", () => {
|
||||
it("launches once, polls status, and returns the exact completed receipt", async () => {
|
||||
const input = launchInput();
|
||||
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async (request) =>
|
||||
request.command === "worker.supervisor.launch.v1"
|
||||
? wire(receipt(input, "running"))
|
||||
: wire(receipt(input, "completed")),
|
||||
);
|
||||
const adapter = createNodeWorkerLaunchAdapter({
|
||||
getTransport: () => transportWith(invoke),
|
||||
sleep: async () => {},
|
||||
});
|
||||
|
||||
await expect(adapter.launch(launchRequest(input))).resolves.toEqual(
|
||||
receipt(input, "completed"),
|
||||
);
|
||||
expect(invoke.mock.calls.map(([request]) => request.command)).toEqual([
|
||||
"worker.supervisor.launch.v1",
|
||||
"worker.supervisor.status.v1",
|
||||
]);
|
||||
});
|
||||
|
||||
it("reacquires the node and replays the identical launch after ambiguous disconnect", async () => {
|
||||
const input = launchInput();
|
||||
let launchCalls = 0;
|
||||
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async (request) => {
|
||||
if (request.command !== "worker.supervisor.launch.v1") {
|
||||
throw new Error("unexpected status call");
|
||||
}
|
||||
launchCalls += 1;
|
||||
request.onDispatchReady?.(`invoke-${launchCalls}`);
|
||||
return launchCalls === 1
|
||||
? { ok: false, error: { code: "DISCONNECTED", message: "node disconnected" } }
|
||||
: wire(receipt(input, "completed"));
|
||||
});
|
||||
let listCalls = 0;
|
||||
const listCurrentNodes = vi.fn(async () => [nodeProof(`conn-${++listCalls}`)]);
|
||||
const adapter = createNodeWorkerLaunchAdapter({
|
||||
getTransport: () => transportWith(invoke, listCurrentNodes),
|
||||
sleep: async () => {},
|
||||
});
|
||||
|
||||
await expect(adapter.launch(launchRequest(input))).resolves.toEqual(
|
||||
receipt(input, "completed"),
|
||||
);
|
||||
expect(invoke).toHaveBeenCalledTimes(2);
|
||||
expect(invoke.mock.calls[0]?.[0].params).toEqual(input);
|
||||
expect(invoke.mock.calls[1]?.[0].params).toEqual(input);
|
||||
expect(invoke.mock.calls[0]?.[0].node.connId).toBe("conn-1");
|
||||
expect(invoke.mock.calls[1]?.[0].node.connId).toBe("conn-2");
|
||||
});
|
||||
|
||||
it("snapshots the launch plan before asynchronous node discovery", async () => {
|
||||
const input = launchInput();
|
||||
const expectedInput = structuredClone(input);
|
||||
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async (request) => {
|
||||
expect(request.params).toEqual(expectedInput);
|
||||
return wire(receipt(expectedInput, "completed"));
|
||||
});
|
||||
const listCurrentNodes = vi.fn(async () => {
|
||||
input.descriptor.assignment.prompt = "mutated after launch call";
|
||||
return [nodeProof()];
|
||||
});
|
||||
const adapter = createNodeWorkerLaunchAdapter({
|
||||
getTransport: () => transportWith(invoke, listCurrentNodes),
|
||||
});
|
||||
|
||||
await expect(adapter.launch(launchRequest(input))).resolves.toEqual(
|
||||
receipt(expectedInput, "completed"),
|
||||
);
|
||||
expect(input.descriptor.assignment.prompt).toBe("mutated after launch call");
|
||||
});
|
||||
|
||||
it("replays launch when status cannot find the durable receipt", async () => {
|
||||
const input = launchInput();
|
||||
const responses = [
|
||||
wire(receipt(input, "running")),
|
||||
wire(null),
|
||||
wire(receipt(input, "completed")),
|
||||
];
|
||||
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async () => responses.shift()!);
|
||||
const adapter = createNodeWorkerLaunchAdapter({
|
||||
getTransport: () => transportWith(invoke),
|
||||
sleep: async () => {},
|
||||
});
|
||||
|
||||
await expect(adapter.launch(launchRequest(input))).resolves.toEqual(
|
||||
receipt(input, "completed"),
|
||||
);
|
||||
expect(invoke.mock.calls.map(([request]) => request.command)).toEqual([
|
||||
"worker.supervisor.launch.v1",
|
||||
"worker.supervisor.status.v1",
|
||||
"worker.supervisor.launch.v1",
|
||||
]);
|
||||
});
|
||||
|
||||
it("cancels before rejecting a post-dispatch identity mismatch", async () => {
|
||||
const input = launchInput();
|
||||
const mismatched = { ...receipt(input, "completed"), environmentId: "environment-other" };
|
||||
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async (request) => {
|
||||
if (request.command === "worker.supervisor.cancel.v1") {
|
||||
return wire(receipt(input, "cancelled"));
|
||||
}
|
||||
request.onDispatchReady?.("invoke-1");
|
||||
return wire(mismatched);
|
||||
});
|
||||
const adapter = createNodeWorkerLaunchAdapter({
|
||||
getTransport: () => transportWith(invoke),
|
||||
});
|
||||
|
||||
await expect(adapter.launch(launchRequest(input))).rejects.toThrow(
|
||||
"node worker supervisor receipt identity mismatch",
|
||||
);
|
||||
expect(invoke.mock.calls.map(([request]) => request.command)).toEqual([
|
||||
"worker.supervisor.launch.v1",
|
||||
"worker.supervisor.cancel.v1",
|
||||
]);
|
||||
});
|
||||
|
||||
it("retries a timed-out launch RPC within the overall deadline", async () => {
|
||||
const input = launchInput();
|
||||
let launchCalls = 0;
|
||||
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async (request) => {
|
||||
launchCalls += 1;
|
||||
request.onDispatchReady?.(`invoke-${launchCalls}`);
|
||||
return launchCalls === 1
|
||||
? await new Promise<never>(() => {})
|
||||
: wire(receipt(input, "completed"));
|
||||
});
|
||||
const adapter = createNodeWorkerLaunchAdapter({
|
||||
getTransport: () => transportWith(invoke),
|
||||
rpcTimeoutMs: 10,
|
||||
sleep: async () => {},
|
||||
});
|
||||
|
||||
await expect(adapter.launch({ ...launchRequest(input), timeoutMs: 100 })).resolves.toEqual(
|
||||
receipt(input, "completed"),
|
||||
);
|
||||
expect(invoke).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("durably cancels after caller abort and returns the terminal cancellation receipt", async () => {
|
||||
const input = launchInput();
|
||||
const controller = new AbortController();
|
||||
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async (request) => {
|
||||
if (request.command === "worker.supervisor.cancel.v1") {
|
||||
expect(request.params).toEqual(
|
||||
expect.objectContaining({
|
||||
launchId: input.launchId,
|
||||
planHash: nodeWorkerPlanHash(input),
|
||||
}),
|
||||
);
|
||||
return wire(receipt(input, "cancelled"));
|
||||
}
|
||||
request.onDispatchReady?.("invoke-1");
|
||||
return wire(receipt(input, "running"));
|
||||
});
|
||||
const adapter = createNodeWorkerLaunchAdapter({
|
||||
getTransport: () => transportWith(invoke),
|
||||
sleep: async () => {
|
||||
controller.abort();
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
adapter.launch({ ...launchRequest(input), signal: controller.signal }),
|
||||
).resolves.toEqual(receipt(input, "cancelled"));
|
||||
expect(invoke.mock.calls.at(-1)?.[0].command).toBe("worker.supervisor.cancel.v1");
|
||||
});
|
||||
|
||||
it("keeps cancelling through missing and active receipts until terminal", async () => {
|
||||
const input = launchInput();
|
||||
const controller = new AbortController();
|
||||
const cancelResponses = [
|
||||
wire(null),
|
||||
wire(receipt(input, "running")),
|
||||
wire(receipt(input, "cancelled")),
|
||||
];
|
||||
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async (request) => {
|
||||
if (request.command === "worker.supervisor.cancel.v1") {
|
||||
return cancelResponses.shift()!;
|
||||
}
|
||||
request.onDispatchReady?.("invoke-1");
|
||||
return wire(receipt(input, "running"));
|
||||
});
|
||||
const adapter = createNodeWorkerLaunchAdapter({
|
||||
getTransport: () => transportWith(invoke),
|
||||
pollIntervalMs: 1,
|
||||
sleep: async () => {
|
||||
controller.abort();
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
adapter.launch({ ...launchRequest(input), signal: controller.signal }),
|
||||
).resolves.toEqual(receipt(input, "cancelled"));
|
||||
expect(
|
||||
invoke.mock.calls.filter(([request]) => request.command === "worker.supervisor.cancel.v1"),
|
||||
).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("retries a timed-out cancellation RPC within one cleanup deadline", async () => {
|
||||
const input = launchInput();
|
||||
const controller = new AbortController();
|
||||
let cancelCalls = 0;
|
||||
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async (request) => {
|
||||
if (request.command === "worker.supervisor.cancel.v1") {
|
||||
cancelCalls += 1;
|
||||
return cancelCalls === 1
|
||||
? await new Promise<never>(() => {})
|
||||
: wire(receipt(input, "cancelled"));
|
||||
}
|
||||
request.onDispatchReady?.("invoke-1");
|
||||
return wire(receipt(input, "running"));
|
||||
});
|
||||
const adapter = createNodeWorkerLaunchAdapter({
|
||||
getTransport: () => transportWith(invoke),
|
||||
rpcTimeoutMs: 10,
|
||||
cancellationTimeoutMs: 100,
|
||||
sleep: async () => {
|
||||
controller.abort();
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
adapter.launch({ ...launchRequest(input), signal: controller.signal }),
|
||||
).resolves.toEqual(receipt(input, "cancelled"));
|
||||
expect(cancelCalls).toBe(2);
|
||||
});
|
||||
|
||||
it("uses distinct cancellation authority after dispatch authority closes", async () => {
|
||||
const input = launchInput();
|
||||
let dispatchAuthorized = true;
|
||||
const cancelAuthorized = vi.fn(() => true);
|
||||
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async (request) => {
|
||||
if (request.command === "worker.supervisor.cancel.v1") {
|
||||
return wire(receipt(input, "cancelled"));
|
||||
}
|
||||
request.onDispatchReady?.("invoke-1");
|
||||
return wire(receipt(input, "running"));
|
||||
});
|
||||
const adapter = createNodeWorkerLaunchAdapter({
|
||||
getTransport: () => transportWith(invoke),
|
||||
sleep: async () => {
|
||||
dispatchAuthorized = false;
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
adapter.launch({
|
||||
...launchRequest(input),
|
||||
isDispatchAuthorized: () => dispatchAuthorized,
|
||||
isCancellationAuthorized: cancelAuthorized,
|
||||
}),
|
||||
).resolves.toEqual(receipt(input, "cancelled"));
|
||||
expect(cancelAuthorized).toHaveBeenCalled();
|
||||
expect(invoke.mock.calls.at(-1)?.[0].command).toBe("worker.supervisor.cancel.v1");
|
||||
});
|
||||
|
||||
it("bounds node discovery with the overall launch deadline", async () => {
|
||||
const input = launchInput();
|
||||
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>();
|
||||
const listCurrentNodes = vi.fn(async () => await new Promise<never>(() => {}));
|
||||
const adapter = createNodeWorkerLaunchAdapter({
|
||||
getTransport: () => transportWith(invoke, listCurrentNodes),
|
||||
});
|
||||
|
||||
await expect(adapter.launch({ ...launchRequest(input), timeoutMs: 25 })).rejects.toThrow(
|
||||
"node worker launch timed out",
|
||||
);
|
||||
expect(invoke).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reports unknown cancellation outcome after a hard cancellation deadline", async () => {
|
||||
const input = launchInput();
|
||||
const controller = new AbortController();
|
||||
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async (request) => {
|
||||
if (request.command === "worker.supervisor.cancel.v1") {
|
||||
return await new Promise<never>(() => {});
|
||||
}
|
||||
request.onDispatchReady?.("invoke-1");
|
||||
return wire(receipt(input, "running"));
|
||||
});
|
||||
const adapter = createNodeWorkerLaunchAdapter({
|
||||
getTransport: () => transportWith(invoke),
|
||||
cancellationTimeoutMs: 25,
|
||||
sleep: async () => {
|
||||
controller.abort();
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
adapter.launch({ ...launchRequest(input), signal: controller.signal }),
|
||||
).rejects.toThrow("node worker launch failed and cancellation could not be confirmed");
|
||||
expect(
|
||||
invoke.mock.calls.filter(([request]) => request.command === "worker.supervisor.cancel.v1"),
|
||||
).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,473 @@
|
||||
import { sleepWithAbort } from "../../infra/backoff.js";
|
||||
import {
|
||||
NODE_WORKER_SUPERVISOR_CANCEL_COMMAND,
|
||||
NODE_WORKER_SUPERVISOR_LAUNCH_COMMAND,
|
||||
NODE_WORKER_SUPERVISOR_STATUS_COMMAND,
|
||||
} from "../../infra/node-commands.js";
|
||||
import {
|
||||
nodeWorkerPlanHash,
|
||||
parseNodeWorkerLaunchInput,
|
||||
parseNodeWorkerSupervisorReceipt,
|
||||
type NodeWorkerLaunchInput,
|
||||
type NodeWorkerSupervisorIdentity,
|
||||
type NodeWorkerSupervisorReceipt,
|
||||
} from "../../worker/node-supervisor-protocol.js";
|
||||
import type {
|
||||
NodeWorkerSupervisorNodeProof,
|
||||
NodeWorkerSupervisorTransport,
|
||||
} from "../node-registry-private.js";
|
||||
|
||||
const DEFAULT_RPC_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_POLL_INTERVAL_MS = 250;
|
||||
const MAX_RETRY_DELAY_MS = 2_000;
|
||||
const DEFAULT_CANCELLATION_TIMEOUT_MS = 30_000;
|
||||
|
||||
const RETRYABLE_TRANSPORT_CODES = new Set([
|
||||
"DISCONNECTED",
|
||||
"NOT_CONNECTED",
|
||||
"PAIRING_CHANGED",
|
||||
"PRIVATE_DIALECT_UNAVAILABLE",
|
||||
"ROUTE_CHANGED",
|
||||
"TIMEOUT",
|
||||
"UNAVAILABLE",
|
||||
]);
|
||||
|
||||
type TerminalNodeWorkerSupervisorReceipt = Extract<
|
||||
NodeWorkerSupervisorReceipt,
|
||||
{ state: "completed" | "failed" | "interrupted" | "cancelled" }
|
||||
>;
|
||||
|
||||
type DeviceWorkerLaunchRequest = {
|
||||
deviceId: string;
|
||||
input: NodeWorkerLaunchInput;
|
||||
isDispatchAuthorized: () => boolean;
|
||||
isCancellationAuthorized: () => boolean;
|
||||
timeoutMs: number;
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
|
||||
type NodeWorkerLaunchAdapterOptions = {
|
||||
getTransport: () => NodeWorkerSupervisorTransport | undefined;
|
||||
now?: () => number;
|
||||
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
|
||||
rpcTimeoutMs?: number;
|
||||
pollIntervalMs?: number;
|
||||
cancellationTimeoutMs?: number;
|
||||
};
|
||||
|
||||
type OperationDeadline = {
|
||||
expiresAtMs: number;
|
||||
signal: AbortSignal;
|
||||
remainingMs: () => number;
|
||||
dispose: () => void;
|
||||
};
|
||||
|
||||
class NodeWorkerLaunchTransportError extends Error {
|
||||
constructor(
|
||||
readonly code: string,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
function isTerminalReceipt(
|
||||
receipt: NodeWorkerSupervisorReceipt,
|
||||
): receipt is TerminalNodeWorkerSupervisorReceipt {
|
||||
return (
|
||||
receipt.state === "completed" ||
|
||||
receipt.state === "failed" ||
|
||||
receipt.state === "interrupted" ||
|
||||
receipt.state === "cancelled"
|
||||
);
|
||||
}
|
||||
|
||||
function snapshotLaunchInput(input: NodeWorkerLaunchInput): NodeWorkerLaunchInput {
|
||||
let encoded: string;
|
||||
try {
|
||||
encoded = JSON.stringify(input);
|
||||
} catch {
|
||||
throw new Error("node worker launch input is not serializable");
|
||||
}
|
||||
return parseNodeWorkerLaunchInput(encoded);
|
||||
}
|
||||
|
||||
function expectedIdentity(input: NodeWorkerLaunchInput): NodeWorkerSupervisorIdentity {
|
||||
if (input.launchId !== input.descriptor.assignment.turnId) {
|
||||
throw new Error("node worker launch ID must match the durable turn ID");
|
||||
}
|
||||
return {
|
||||
launchId: input.launchId,
|
||||
planHash: nodeWorkerPlanHash(input),
|
||||
environmentId: input.descriptor.admission.environmentId,
|
||||
sessionId: input.descriptor.admission.sessionId,
|
||||
ownerEpoch: input.descriptor.admission.ownerEpoch,
|
||||
placementGeneration: input.placementGeneration,
|
||||
runId: input.descriptor.assignment.runId,
|
||||
};
|
||||
}
|
||||
|
||||
function receiptMatchesIdentity(
|
||||
receipt: NodeWorkerSupervisorReceipt,
|
||||
expected: NodeWorkerSupervisorIdentity,
|
||||
): boolean {
|
||||
return (
|
||||
receipt.launchId === expected.launchId &&
|
||||
receipt.planHash === expected.planHash &&
|
||||
receipt.environmentId === expected.environmentId &&
|
||||
receipt.sessionId === expected.sessionId &&
|
||||
receipt.ownerEpoch === expected.ownerEpoch &&
|
||||
receipt.placementGeneration === expected.placementGeneration &&
|
||||
receipt.runId === expected.runId
|
||||
);
|
||||
}
|
||||
|
||||
function parseInvokeReceipt(
|
||||
payloadJSON: string | null | undefined,
|
||||
): NodeWorkerSupervisorReceipt | null {
|
||||
if (!payloadJSON) {
|
||||
throw new Error("node worker supervisor response omitted payload JSON");
|
||||
}
|
||||
let value: unknown;
|
||||
try {
|
||||
value = JSON.parse(payloadJSON) as unknown;
|
||||
} catch {
|
||||
throw new Error("node worker supervisor response contained malformed JSON");
|
||||
}
|
||||
if (value === null) {
|
||||
return null;
|
||||
}
|
||||
const receipt = parseNodeWorkerSupervisorReceipt(value);
|
||||
if (!receipt) {
|
||||
throw new Error("node worker supervisor response violated the private receipt contract");
|
||||
}
|
||||
return receipt;
|
||||
}
|
||||
|
||||
function signalError(signal: AbortSignal, fallback: string): Error {
|
||||
return signal.reason instanceof Error ? signal.reason : new Error(fallback);
|
||||
}
|
||||
|
||||
function raceWithSignal<T>(operation: Promise<T>, signal: AbortSignal): Promise<T> {
|
||||
if (signal.aborted) {
|
||||
return Promise.reject(signalError(signal, "node worker operation aborted"));
|
||||
}
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const onAbort = () => reject(signalError(signal, "node worker operation aborted"));
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
void operation.then(
|
||||
(value) => {
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
resolve(value);
|
||||
},
|
||||
(error: unknown) => {
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
reject(error instanceof Error ? error : new Error("node worker operation failed"));
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function createDeadline(params: {
|
||||
now: () => number;
|
||||
timeoutMs: number;
|
||||
signal?: AbortSignal;
|
||||
label: string;
|
||||
}): OperationDeadline {
|
||||
if (!Number.isFinite(params.timeoutMs) || params.timeoutMs <= 0) {
|
||||
throw new Error(`${params.label} timeout must be a positive finite number`);
|
||||
}
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(
|
||||
() => controller.abort(new Error(`${params.label} timed out`)),
|
||||
params.timeoutMs,
|
||||
);
|
||||
timer.unref?.();
|
||||
const signal = params.signal
|
||||
? AbortSignal.any([params.signal, controller.signal])
|
||||
: controller.signal;
|
||||
const expiresAtMs = params.now() + params.timeoutMs;
|
||||
return {
|
||||
expiresAtMs,
|
||||
signal,
|
||||
remainingMs: () => Math.max(0, expiresAtMs - params.now()),
|
||||
dispose: () => clearTimeout(timer),
|
||||
};
|
||||
}
|
||||
|
||||
export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOptions) {
|
||||
const now = options.now ?? Date.now;
|
||||
const sleep = options.sleep ?? sleepWithAbort;
|
||||
const rpcTimeoutMs = options.rpcTimeoutMs ?? DEFAULT_RPC_TIMEOUT_MS;
|
||||
const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
||||
const cancellationTimeoutMs = options.cancellationTimeoutMs ?? DEFAULT_CANCELLATION_TIMEOUT_MS;
|
||||
|
||||
const findNode = async (params: {
|
||||
transport: NodeWorkerSupervisorTransport;
|
||||
deviceId: string;
|
||||
signal: AbortSignal;
|
||||
}): Promise<NodeWorkerSupervisorNodeProof> => {
|
||||
let nodes: readonly NodeWorkerSupervisorNodeProof[];
|
||||
try {
|
||||
nodes = await raceWithSignal(params.transport.listCurrentNodes(), params.signal);
|
||||
} catch (error) {
|
||||
if (params.signal.aborted) {
|
||||
throw error;
|
||||
}
|
||||
throw new NodeWorkerLaunchTransportError(
|
||||
"UNAVAILABLE",
|
||||
"device worker node discovery is unavailable",
|
||||
);
|
||||
}
|
||||
const node = nodes.find(
|
||||
(candidate) =>
|
||||
candidate.nodeId === params.deviceId && candidate.commands.includes("system.run"),
|
||||
);
|
||||
if (!node) {
|
||||
throw new NodeWorkerLaunchTransportError(
|
||||
"NOT_CONNECTED",
|
||||
"device worker node is not currently connected",
|
||||
);
|
||||
}
|
||||
return node;
|
||||
};
|
||||
|
||||
const invoke = async (params: {
|
||||
deviceId: string;
|
||||
command:
|
||||
| typeof NODE_WORKER_SUPERVISOR_LAUNCH_COMMAND
|
||||
| typeof NODE_WORKER_SUPERVISOR_STATUS_COMMAND
|
||||
| typeof NODE_WORKER_SUPERVISOR_CANCEL_COMMAND;
|
||||
payload: unknown;
|
||||
isAuthorized: () => boolean;
|
||||
deadline: OperationDeadline;
|
||||
onDispatchReady?: () => void;
|
||||
}): Promise<NodeWorkerSupervisorReceipt | null> => {
|
||||
if (!params.isAuthorized()) {
|
||||
throw new NodeWorkerLaunchTransportError(
|
||||
"APPROVAL_AUTHORITY_CLOSED",
|
||||
"node worker authority closed",
|
||||
);
|
||||
}
|
||||
const remainingMs = params.deadline.remainingMs();
|
||||
if (remainingMs <= 0 || params.deadline.signal.aborted) {
|
||||
throw params.deadline.signal.reason ?? new Error("node worker operation timed out");
|
||||
}
|
||||
const transport = options.getTransport();
|
||||
if (!transport) {
|
||||
throw new NodeWorkerLaunchTransportError(
|
||||
"UNAVAILABLE",
|
||||
"device worker node transport is unavailable",
|
||||
);
|
||||
}
|
||||
const rpcController = new AbortController();
|
||||
const rpcBudgetMs = Math.max(1, Math.min(rpcTimeoutMs, remainingMs));
|
||||
const rpcTimer = setTimeout(
|
||||
() => rpcController.abort(new Error("node worker RPC timed out")),
|
||||
rpcBudgetMs,
|
||||
);
|
||||
rpcTimer.unref?.();
|
||||
const signal = AbortSignal.any([params.deadline.signal, rpcController.signal]);
|
||||
try {
|
||||
const node = await findNode({ transport, deviceId: params.deviceId, signal });
|
||||
const operation = transport.invoke({
|
||||
node,
|
||||
command: params.command,
|
||||
params: params.payload,
|
||||
timeoutMs: rpcBudgetMs,
|
||||
signal,
|
||||
idempotencyKey:
|
||||
params.command === NODE_WORKER_SUPERVISOR_LAUNCH_COMMAND &&
|
||||
typeof params.payload === "object" &&
|
||||
params.payload !== null &&
|
||||
"launchId" in params.payload &&
|
||||
typeof params.payload.launchId === "string"
|
||||
? params.payload.launchId
|
||||
: undefined,
|
||||
isDispatchAuthorized: params.isAuthorized,
|
||||
...(params.onDispatchReady ? { onDispatchReady: params.onDispatchReady } : {}),
|
||||
});
|
||||
const result = await raceWithSignal(operation, signal);
|
||||
if (!result.ok) {
|
||||
const code = result.error?.code ?? "UNAVAILABLE";
|
||||
throw new NodeWorkerLaunchTransportError(
|
||||
code,
|
||||
`node worker supervisor invocation failed (${code})`,
|
||||
);
|
||||
}
|
||||
return parseInvokeReceipt(result.payloadJSON);
|
||||
} catch (error) {
|
||||
if (rpcController.signal.aborted && !params.deadline.signal.aborted) {
|
||||
throw new NodeWorkerLaunchTransportError("TIMEOUT", "node worker RPC timed out");
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(rpcTimer);
|
||||
}
|
||||
};
|
||||
|
||||
const validateReceipt = (
|
||||
receipt: NodeWorkerSupervisorReceipt,
|
||||
expected: NodeWorkerSupervisorIdentity,
|
||||
): NodeWorkerSupervisorReceipt => {
|
||||
if (!receiptMatchesIdentity(receipt, expected)) {
|
||||
throw new Error("node worker supervisor receipt identity mismatch");
|
||||
}
|
||||
return receipt;
|
||||
};
|
||||
|
||||
const waitBeforeRetry = async (params: {
|
||||
delayMs: number;
|
||||
deadline: OperationDeadline;
|
||||
}): Promise<number> => {
|
||||
const remainingMs = params.deadline.remainingMs();
|
||||
if (remainingMs <= 0 || params.deadline.signal.aborted) {
|
||||
throw params.deadline.signal.reason ?? new Error("node worker operation timed out");
|
||||
}
|
||||
await raceWithSignal(
|
||||
sleep(Math.min(params.delayMs, remainingMs), params.deadline.signal),
|
||||
params.deadline.signal,
|
||||
);
|
||||
return Math.min(params.delayMs * 2, MAX_RETRY_DELAY_MS);
|
||||
};
|
||||
|
||||
const cancelUntilTerminal = async (params: {
|
||||
request: DeviceWorkerLaunchRequest;
|
||||
expected: NodeWorkerSupervisorIdentity;
|
||||
}): Promise<TerminalNodeWorkerSupervisorReceipt> => {
|
||||
const deadline = createDeadline({
|
||||
now,
|
||||
timeoutMs: cancellationTimeoutMs,
|
||||
label: "node worker cancellation",
|
||||
});
|
||||
let delayMs = pollIntervalMs;
|
||||
try {
|
||||
while (!deadline.signal.aborted && deadline.remainingMs() > 0) {
|
||||
if (!params.request.isCancellationAuthorized()) {
|
||||
throw new Error("node worker cancellation authority closed before terminal settlement");
|
||||
}
|
||||
try {
|
||||
const receipt = await invoke({
|
||||
deviceId: params.request.deviceId,
|
||||
command: NODE_WORKER_SUPERVISOR_CANCEL_COMMAND,
|
||||
payload: params.expected,
|
||||
isAuthorized: params.request.isCancellationAuthorized,
|
||||
deadline,
|
||||
});
|
||||
if (receipt) {
|
||||
const validated = validateReceipt(receipt, params.expected);
|
||||
if (isTerminalReceipt(validated)) {
|
||||
return validated;
|
||||
}
|
||||
delayMs = pollIntervalMs;
|
||||
}
|
||||
} catch (error) {
|
||||
if (deadline.signal.aborted) {
|
||||
break;
|
||||
}
|
||||
if (
|
||||
!(error instanceof NodeWorkerLaunchTransportError) ||
|
||||
!RETRYABLE_TRANSPORT_CODES.has(error.code)
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
delayMs = await waitBeforeRetry({ delayMs, deadline });
|
||||
}
|
||||
} finally {
|
||||
deadline.dispose();
|
||||
}
|
||||
throw new Error("node worker cancellation outcome is unknown after transport loss");
|
||||
};
|
||||
|
||||
const launch = async (
|
||||
request: DeviceWorkerLaunchRequest,
|
||||
): Promise<TerminalNodeWorkerSupervisorReceipt> => {
|
||||
const input = snapshotLaunchInput(request.input);
|
||||
const stableRequest = { ...request, input };
|
||||
const expected = expectedIdentity(input);
|
||||
const deadline = createDeadline({
|
||||
now,
|
||||
timeoutMs: request.timeoutMs,
|
||||
...(request.signal ? { signal: request.signal } : {}),
|
||||
label: "node worker launch",
|
||||
});
|
||||
let mayHaveLaunched = false;
|
||||
let pollStatus = false;
|
||||
let delayMs = pollIntervalMs;
|
||||
try {
|
||||
while (true) {
|
||||
if (deadline.signal.aborted) {
|
||||
throw signalError(deadline.signal, "node worker launch aborted");
|
||||
}
|
||||
if (!stableRequest.isDispatchAuthorized()) {
|
||||
throw new Error("node worker launch authority closed");
|
||||
}
|
||||
try {
|
||||
const receipt = await invoke({
|
||||
deviceId: stableRequest.deviceId,
|
||||
command: pollStatus
|
||||
? NODE_WORKER_SUPERVISOR_STATUS_COMMAND
|
||||
: NODE_WORKER_SUPERVISOR_LAUNCH_COMMAND,
|
||||
payload: pollStatus ? { launchId: input.launchId } : input,
|
||||
isAuthorized: stableRequest.isDispatchAuthorized,
|
||||
deadline,
|
||||
...(!pollStatus
|
||||
? {
|
||||
onDispatchReady: () => {
|
||||
mayHaveLaunched = true;
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
if (!receipt) {
|
||||
pollStatus = false;
|
||||
} else {
|
||||
const validated = validateReceipt(receipt, expected);
|
||||
mayHaveLaunched = true;
|
||||
if (isTerminalReceipt(validated)) {
|
||||
return validated;
|
||||
}
|
||||
pollStatus = true;
|
||||
delayMs = pollIntervalMs;
|
||||
}
|
||||
} catch (error) {
|
||||
if (deadline.signal.aborted || !stableRequest.isDispatchAuthorized()) {
|
||||
throw error;
|
||||
}
|
||||
if (
|
||||
!(error instanceof NodeWorkerLaunchTransportError) ||
|
||||
!RETRYABLE_TRANSPORT_CODES.has(error.code)
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
pollStatus = false;
|
||||
}
|
||||
delayMs = await waitBeforeRetry({ delayMs, deadline });
|
||||
}
|
||||
} catch (error) {
|
||||
if (!mayHaveLaunched) {
|
||||
throw error;
|
||||
}
|
||||
let terminal: TerminalNodeWorkerSupervisorReceipt;
|
||||
try {
|
||||
terminal = await cancelUntilTerminal({ request: stableRequest, expected });
|
||||
} catch (cancelError) {
|
||||
throw Object.assign(
|
||||
new Error("node worker launch failed and cancellation could not be confirmed", {
|
||||
cause: error instanceof Error ? error : new Error("node worker launch failed"),
|
||||
}),
|
||||
{ cancellationError: cancelError },
|
||||
);
|
||||
}
|
||||
if (deadline.signal.aborted || !stableRequest.isDispatchAuthorized()) {
|
||||
return terminal;
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
deadline.dispose();
|
||||
}
|
||||
};
|
||||
|
||||
return { launch };
|
||||
}
|
||||
@@ -33,10 +33,13 @@ describe("plugin-sdk/channel-ingress-runtime", () => {
|
||||
onDeferred: vi.fn(),
|
||||
onAdoptionFinalizing: vi.fn(),
|
||||
onFailed: vi.fn(async () => {}),
|
||||
onCancelled: vi.fn(async () => {}),
|
||||
onAbandoned: vi.fn(async () => {}),
|
||||
});
|
||||
const first = createLifecycle();
|
||||
const second = createLifecycle();
|
||||
const cancellation = fanInChannelIngressLifecycles([first, second]);
|
||||
await cancellation.lifecycle?.onCancelled?.();
|
||||
const combined = fanInChannelIngressLifecycles([undefined, first, second]);
|
||||
|
||||
combined.lifecycle?.onAdoptionFinalizing();
|
||||
@@ -49,6 +52,8 @@ describe("plugin-sdk/channel-ingress-runtime", () => {
|
||||
expect(second.onAdopted).toHaveBeenCalledOnce();
|
||||
expect(first.onAbandoned).not.toHaveBeenCalled();
|
||||
expect(second.onAbandoned).not.toHaveBeenCalled();
|
||||
expect(first.onCancelled).toHaveBeenCalledOnce();
|
||||
expect(second.onCancelled).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("settles or abandons claims that no reply lane adopted", async () => {
|
||||
@@ -71,6 +76,57 @@ describe("plugin-sdk/channel-ingress-runtime", () => {
|
||||
expect(fanInChannelIngressLifecycles([]).lifecycle).toBeUndefined();
|
||||
});
|
||||
|
||||
it("cancellation-settles every source in mixed capable and legacy fan-in", async () => {
|
||||
const adopted = vi.fn(async () => {});
|
||||
const cancelled = vi.fn(async () => {});
|
||||
const legacyAbandoned = vi.fn(async () => {});
|
||||
const createLifecycle = (
|
||||
onCancelled?: () => Promise<void>,
|
||||
onAbandoned = vi.fn(async () => {}),
|
||||
) => ({
|
||||
abortSignal: new AbortController().signal,
|
||||
onAdopted: adopted,
|
||||
onDeferred: vi.fn(),
|
||||
onAdoptionFinalizing: vi.fn(),
|
||||
onFailed: vi.fn(async () => {}),
|
||||
onAbandoned,
|
||||
...(onCancelled ? { onCancelled } : {}),
|
||||
});
|
||||
const combined = fanInChannelIngressLifecycles([
|
||||
createLifecycle(cancelled),
|
||||
createLifecycle(undefined, legacyAbandoned),
|
||||
]);
|
||||
|
||||
expect(combined.lifecycle).not.toHaveProperty("onCancelled");
|
||||
await combined.cancel();
|
||||
|
||||
expect(adopted).not.toHaveBeenCalled();
|
||||
expect(cancelled).toHaveBeenCalledOnce();
|
||||
expect(legacyAbandoned).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("preserves lifecycle receivers while fanning in cancellation", async () => {
|
||||
const createLifecycle = () => ({
|
||||
abortSignal: new AbortController().signal,
|
||||
cancellationCount: 0,
|
||||
onAdopted: vi.fn(async () => {}),
|
||||
onDeferred: vi.fn(),
|
||||
onAdoptionFinalizing: vi.fn(),
|
||||
onFailed: vi.fn(async () => {}),
|
||||
async onCancelled() {
|
||||
this.cancellationCount += 1;
|
||||
},
|
||||
onAbandoned: vi.fn(async () => {}),
|
||||
});
|
||||
const first = createLifecycle();
|
||||
const second = createLifecycle();
|
||||
|
||||
await fanInChannelIngressLifecycles([first, second]).lifecycle?.onCancelled?.();
|
||||
|
||||
expect(first.cancellationCount).toBe(1);
|
||||
expect(second.cancellationCount).toBe(1);
|
||||
});
|
||||
|
||||
it("can abandon claims after terminal settlement adoption fails", async () => {
|
||||
const abandoned = vi.fn(async () => {});
|
||||
const combined = fanInChannelIngressLifecycles([
|
||||
|
||||
@@ -128,11 +128,17 @@ export function fanInChannelIngressLifecycles(
|
||||
lifecycle: ChannelIngressLifecycle | undefined;
|
||||
settle: () => Promise<void>;
|
||||
abandon: (error?: unknown) => Promise<void>;
|
||||
cancel: () => Promise<void>;
|
||||
} {
|
||||
const lifecycles = inputs.filter((lifecycle) => lifecycle !== undefined);
|
||||
const first = lifecycles[0];
|
||||
if (!first) {
|
||||
return { lifecycle: undefined, settle: async () => {}, abandon: async () => {} };
|
||||
return {
|
||||
lifecycle: undefined,
|
||||
settle: async () => {},
|
||||
abandon: async () => {},
|
||||
cancel: async () => {},
|
||||
};
|
||||
}
|
||||
|
||||
let handedOff = false;
|
||||
@@ -147,7 +153,16 @@ export function fanInChannelIngressLifecycles(
|
||||
const failAll = async (error: unknown) => {
|
||||
await Promise.all(lifecycles.map(async (lifecycle) => await lifecycle.onFailed?.(error)));
|
||||
};
|
||||
|
||||
const supportsCancellation = lifecycles.every((lifecycle) => lifecycle.onCancelled !== undefined);
|
||||
// Omit aggregate cancellation unless every durable source supports it. Callers
|
||||
// can then use settle/abandon without an acknowledged-but-unsettled claim.
|
||||
const cancelAll = async () => {
|
||||
await Promise.all(
|
||||
lifecycles.map(async (lifecycle) =>
|
||||
lifecycle.onCancelled ? await lifecycle.onCancelled() : await lifecycle.onAbandoned(),
|
||||
),
|
||||
);
|
||||
};
|
||||
return {
|
||||
lifecycle: {
|
||||
abortSignal:
|
||||
@@ -173,6 +188,14 @@ export function fanInChannelIngressLifecycles(
|
||||
handedOff = true;
|
||||
await failAll(error);
|
||||
},
|
||||
...(supportsCancellation
|
||||
? {
|
||||
onCancelled: async () => {
|
||||
handedOff = true;
|
||||
await cancelAll();
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
onAbandoned: async () => {
|
||||
handedOff = true;
|
||||
await abandonAll();
|
||||
@@ -191,5 +214,13 @@ export function fanInChannelIngressLifecycles(
|
||||
await abandonAll();
|
||||
}
|
||||
},
|
||||
// Source-compatible lifecycles predate onCancelled. Settle each source through
|
||||
// its strongest release callback so mixed fan-in cannot strand a durable claim.
|
||||
cancel: async () => {
|
||||
if (!handedOff) {
|
||||
handedOff = true;
|
||||
await cancelAll();
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -136,6 +136,20 @@ async function setupAcpxAndHelperRegistry() {
|
||||
return { workspaceDir, acpxRoot, helperRoot };
|
||||
}
|
||||
|
||||
function useStableMetadataSnapshot(manifestRegistry: PluginManifestRegistry): void {
|
||||
const snapshot = {
|
||||
manifestRegistry,
|
||||
plugins: manifestRegistry.plugins,
|
||||
normalizePluginId: (pluginId: string) =>
|
||||
manifestRegistry.plugins.find((plugin) => plugin.legacyPluginIds?.includes(pluginId))?.id ??
|
||||
pluginId,
|
||||
};
|
||||
hoisted.loadPluginMetadataSnapshot
|
||||
.mockReturnValueOnce(snapshot)
|
||||
.mockReturnValueOnce(snapshot)
|
||||
.mockReturnValueOnce(snapshot);
|
||||
}
|
||||
|
||||
async function setupPluginOutsideSkills() {
|
||||
const workspaceDir = await tempDirs.make("openclaw-");
|
||||
const pluginRoot = await tempDirs.make("openclaw-plugin-");
|
||||
@@ -236,6 +250,57 @@ describe("resolvePluginSkillDirs", () => {
|
||||
expect(dirs).toEqual(expectedDirs({ acpxRoot, helperRoot }));
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "unavailable to available",
|
||||
initiallyAvailable: false,
|
||||
firstIncludesAcpx: false,
|
||||
secondIncludesAcpx: true,
|
||||
},
|
||||
{
|
||||
name: "available to unavailable",
|
||||
initiallyAvailable: true,
|
||||
firstIncludesAcpx: true,
|
||||
secondIncludesAcpx: false,
|
||||
},
|
||||
])(
|
||||
"invalidates the memo when ACP changes from $name with stable inputs",
|
||||
async ({ initiallyAvailable, firstIncludesAcpx, secondIncludesAcpx }) => {
|
||||
const { workspaceDir, acpxRoot, helperRoot } = await setupAcpxAndHelperRegistry();
|
||||
const manifestRegistry = buildRegistry({ acpxRoot, helperRoot });
|
||||
useStableMetadataSnapshot(manifestRegistry);
|
||||
const config = {
|
||||
acp: { enabled: true },
|
||||
plugins: {
|
||||
entries: {
|
||||
acpx: { enabled: true },
|
||||
helper: { enabled: true },
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
if (initiallyAvailable) {
|
||||
registerHealthyAcpBackend();
|
||||
}
|
||||
|
||||
const first = resolvePluginSkillDirs({ workspaceDir, config });
|
||||
|
||||
if (initiallyAvailable) {
|
||||
acpRuntimeTesting.resetAcpRuntimeBackendsForTests();
|
||||
} else {
|
||||
registerHealthyAcpBackend();
|
||||
}
|
||||
const second = resolvePluginSkillDirs({ workspaceDir, config });
|
||||
|
||||
const dirsForState = (includeAcpx: boolean) => [
|
||||
...(includeAcpx ? [path.resolve(acpxRoot, "skills")] : []),
|
||||
path.resolve(helperRoot, "skills"),
|
||||
];
|
||||
expect(first).toEqual(dirsForState(firstIncludesAcpx));
|
||||
expect(second).toEqual(dirsForState(secondIncludesAcpx));
|
||||
expect(resolvePluginSkillDirs({ workspaceDir, config })).toBe(second);
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects plugin skill paths that escape the plugin root", async () => {
|
||||
const { workspaceDir, pluginRoot, outsideSkills } = await setupPluginOutsideSkills();
|
||||
await fs.mkdir(path.join(pluginRoot, "skills"), { recursive: true });
|
||||
|
||||
@@ -22,12 +22,13 @@ const log = createSubsystemLogger("skills");
|
||||
type PluginSkillLinkType = "dir" | "junction";
|
||||
|
||||
// Plugin metadata is process-stable while the gateway runs, but this resolver sits on the
|
||||
// per-turn skills-refresh path. The single-slot memo keeps repeat turns from re-walking and
|
||||
// re-publishing every plugin skill dir; lifecycle clears evict it on plugin reload/install.
|
||||
// per-turn skills-refresh path. ACP availability changes outside that metadata lifecycle, so it
|
||||
// stays in the memo identity to prevent stale ACPX skill exposure without repeating directory IO.
|
||||
let pluginSkillDirsMemo: {
|
||||
workspaceDir: string;
|
||||
config: OpenClawConfig | undefined;
|
||||
snapshot: unknown;
|
||||
acpRuntimeAvailable: boolean;
|
||||
dirs: string[];
|
||||
} | null = null;
|
||||
|
||||
@@ -55,16 +56,6 @@ export function resolvePluginSkillDirs(params: {
|
||||
env: process.env,
|
||||
allowWorkspaceScopedCurrent: true,
|
||||
});
|
||||
const canMemoize = params.pluginSkillsDir === undefined;
|
||||
if (
|
||||
canMemoize &&
|
||||
pluginSkillDirsMemo &&
|
||||
pluginSkillDirsMemo.workspaceDir === workspaceDir &&
|
||||
pluginSkillDirsMemo.config === params.config &&
|
||||
pluginSkillDirsMemo.snapshot === metadataSnapshot
|
||||
) {
|
||||
return pluginSkillDirsMemo.dirs;
|
||||
}
|
||||
const registry = metadataSnapshot.manifestRegistry;
|
||||
if (registry.plugins.length === 0) {
|
||||
publishPluginSkills([], {
|
||||
@@ -72,11 +63,22 @@ export function resolvePluginSkillDirs(params: {
|
||||
});
|
||||
return [];
|
||||
}
|
||||
const acpRuntimeAvailable = isAcpRuntimeSpawnAvailable({ config });
|
||||
const canMemoize = params.pluginSkillsDir === undefined;
|
||||
if (
|
||||
canMemoize &&
|
||||
pluginSkillDirsMemo &&
|
||||
pluginSkillDirsMemo.workspaceDir === workspaceDir &&
|
||||
pluginSkillDirsMemo.config === params.config &&
|
||||
pluginSkillDirsMemo.snapshot === metadataSnapshot &&
|
||||
pluginSkillDirsMemo.acpRuntimeAvailable === acpRuntimeAvailable
|
||||
) {
|
||||
return pluginSkillDirsMemo.dirs;
|
||||
}
|
||||
const normalizedPlugins = normalizePluginsConfigWithResolver(
|
||||
config.plugins,
|
||||
metadataSnapshot.normalizePluginId,
|
||||
);
|
||||
const acpRuntimeAvailable = isAcpRuntimeSpawnAvailable({ config });
|
||||
const memorySlot = normalizedPlugins.slots.memory;
|
||||
let selectedMemoryPluginId: string | null = null;
|
||||
const seen = new Set<string>();
|
||||
@@ -147,6 +149,7 @@ export function resolvePluginSkillDirs(params: {
|
||||
workspaceDir,
|
||||
config: params.config,
|
||||
snapshot: metadataSnapshot,
|
||||
acpRuntimeAvailable,
|
||||
dirs: resolved,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
packOpenClawPackageForDocker,
|
||||
parseArgs,
|
||||
prepareBundledAiRuntimePackage,
|
||||
runCaptureForTest,
|
||||
runCommandForTest,
|
||||
writePackageInventoryForDocker,
|
||||
} from "../../../../scripts/package-openclaw-for-docker.mts";
|
||||
@@ -768,14 +769,8 @@ describe("package-openclaw-for-docker", () => {
|
||||
restoreDocsMap: async (cwd: string) => {
|
||||
calls.push(`restore-docs:${cwd}`);
|
||||
},
|
||||
runCaptureImpl: async (
|
||||
command: string,
|
||||
args: string[],
|
||||
cwd: string,
|
||||
options: { deferForwardedSignalExit?: boolean },
|
||||
) => {
|
||||
runCaptureImpl: async (command: string, args: string[], cwd: string) => {
|
||||
calls.push(`${command}:${args.join(" ")}:${cwd}`);
|
||||
expect(options.deferForwardedSignalExit).toBe(true);
|
||||
return "openclaw-2026.5.28.tgz\n";
|
||||
},
|
||||
});
|
||||
@@ -955,61 +950,116 @@ describe("package-openclaw-for-docker", () => {
|
||||
});
|
||||
|
||||
it("normalizes npm 12 pack metadata for renamed package artifacts", async () => {
|
||||
const sourceDir = tempDirs.make("openclaw-docker-pack-source-");
|
||||
const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-docker-pack-json-"));
|
||||
const packJsonPath = path.join(outputDir, "pack.json");
|
||||
const npmPackOutput = JSON.stringify({
|
||||
openclaw: {
|
||||
entryCount: 15_000,
|
||||
filename: "openclaw-2026.5.28.tgz",
|
||||
files: Array.from({ length: 15_000 }, (_, index) => ({
|
||||
mode: 0o644,
|
||||
path: `dist/generated/package-entry-${String(index).padStart(5, "0")}.js`,
|
||||
size: index,
|
||||
})),
|
||||
size: 7,
|
||||
unpackedSize: 7,
|
||||
version: "2026.5.28",
|
||||
},
|
||||
});
|
||||
expect(Buffer.byteLength(npmPackOutput)).toBeGreaterThan(1024 * 1024);
|
||||
const npmPackOutputPath = path.join(sourceDir, "npm-pack.json");
|
||||
fs.writeFileSync(npmPackOutputPath, npmPackOutput);
|
||||
|
||||
try {
|
||||
const tarball = await packOpenClawPackageForDocker("/repo", outputDir, {
|
||||
const tarball = await packOpenClawPackageForDocker(sourceDir, outputDir, {
|
||||
...skipDocsMapLifecycle,
|
||||
outputName: "openclaw-current.tgz",
|
||||
packJsonPath,
|
||||
prepareBundledAiRuntime: skipBundledAiRuntime,
|
||||
prepareChangelog: async () => {},
|
||||
restoreChangelog: async () => {},
|
||||
runCaptureImpl: async (
|
||||
command: string,
|
||||
args: string[],
|
||||
_cwd: string,
|
||||
options: { deferForwardedSignalExit?: boolean },
|
||||
) => {
|
||||
expect(command).toBe("npm");
|
||||
expect(args).toEqual([
|
||||
"pack",
|
||||
"--json",
|
||||
"--silent",
|
||||
"--ignore-scripts",
|
||||
"--pack-destination",
|
||||
outputDir,
|
||||
]);
|
||||
expect(options.deferForwardedSignalExit).toBe(true);
|
||||
runCaptureImpl: async (_command, _args, cwd, options) => {
|
||||
fs.writeFileSync(path.join(outputDir, "openclaw-2026.5.28.tgz"), "package");
|
||||
return JSON.stringify({
|
||||
openclaw: {
|
||||
entryCount: 1,
|
||||
filename: "openclaw-2026.5.28.tgz",
|
||||
size: 7,
|
||||
unpackedSize: 7,
|
||||
version: "2026.5.28",
|
||||
},
|
||||
});
|
||||
return await runCaptureForTest(
|
||||
process.execPath,
|
||||
[
|
||||
"-e",
|
||||
"process.stdout.write(require('node:fs').readFileSync(process.argv[1]))",
|
||||
npmPackOutputPath,
|
||||
],
|
||||
cwd,
|
||||
options,
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
expect(tarball).toBe(path.join(outputDir, "openclaw-current.tgz"));
|
||||
expect(JSON.parse(fs.readFileSync(packJsonPath, "utf8"))).toEqual([
|
||||
{
|
||||
entryCount: 1,
|
||||
filename: "openclaw-current.tgz",
|
||||
size: 7,
|
||||
unpackedSize: 7,
|
||||
version: "2026.5.28",
|
||||
},
|
||||
]);
|
||||
const packJson = JSON.parse(fs.readFileSync(packJsonPath, "utf8")) as Array<{
|
||||
entryCount: number;
|
||||
filename: string;
|
||||
files: unknown[];
|
||||
}>;
|
||||
expect(packJson).toHaveLength(1);
|
||||
expect(packJson[0]).toMatchObject({
|
||||
entryCount: 15_000,
|
||||
filename: "openclaw-current.tgz",
|
||||
});
|
||||
expect(packJson[0]?.files).toHaveLength(15_000);
|
||||
} finally {
|
||||
fs.rmSync(outputDir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("cleans receipts without obscuring runner and parse failures", async () => {
|
||||
const originalRm = fs.promises.rm.bind(fs.promises);
|
||||
for (const failure of ["runner", "parse"] as const) {
|
||||
for (const cleanupFails of [false, true]) {
|
||||
const outputDir = tempDirs.make(`openclaw-docker-pack-${failure}-`);
|
||||
const cleanupError = new Error("receipt cleanup failed");
|
||||
let receiptPath = "";
|
||||
const rmSpy = vi.spyOn(fs.promises, "rm").mockImplementation(async (target, options) => {
|
||||
if (cleanupFails && receiptPath && target === path.dirname(receiptPath)) {
|
||||
throw cleanupError;
|
||||
}
|
||||
return await originalRm(target, options);
|
||||
});
|
||||
try {
|
||||
const packPromise = packOpenClawPackageForDocker("/repo", outputDir, {
|
||||
...skipDocsMapLifecycle,
|
||||
packJsonPath: path.join(outputDir, "pack.json"),
|
||||
prepareBundledAiRuntime: skipBundledAiRuntime,
|
||||
prepareChangelog: async () => {},
|
||||
restoreChangelog: async () => {},
|
||||
runCaptureImpl: async (_command, _args, _cwd, options) => {
|
||||
receiptPath = options.stdoutFilePath ?? "";
|
||||
if (failure === "runner") throw new Error("npm pack failed");
|
||||
fs.writeFileSync(receiptPath, "not json");
|
||||
fs.writeFileSync(path.join(outputDir, "openclaw-2026.5.28.tgz"), "package");
|
||||
return "";
|
||||
},
|
||||
});
|
||||
const message =
|
||||
failure === "runner" ? "npm pack failed" : "npm pack --json output was not valid JSON";
|
||||
if (cleanupFails) {
|
||||
await expect(packPromise).rejects.toMatchObject({
|
||||
cause: expect.objectContaining({ message }),
|
||||
errors: [expect.objectContaining({ message }), cleanupError],
|
||||
message: "Package operation and cleanup both failed.",
|
||||
});
|
||||
} else {
|
||||
await expect(packPromise).rejects.toThrow(message);
|
||||
expect(fs.existsSync(receiptPath)).toBe(false);
|
||||
}
|
||||
expect(receiptPath).not.toBe("");
|
||||
} finally {
|
||||
rmSpy.mockRestore();
|
||||
if (receiptPath) fs.rmSync(path.dirname(receiptPath), { force: true, recursive: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects path-like npm pack stdout before resolving Docker package tarballs", async () => {
|
||||
for (const filename of [
|
||||
"../openclaw-2026.6.17.tgz",
|
||||
@@ -1297,6 +1347,46 @@ describe("package-openclaw-for-docker", () => {
|
||||
).rejects.toThrow(/exceeded captured stdout limit \(1024 bytes\)/u);
|
||||
});
|
||||
|
||||
it("writes exact stdout bytes to a file and rejects capture conflicts", async () => {
|
||||
const tempDir = tempDirs.make("openclaw-package-stdout-file-");
|
||||
const stdoutFilePath = path.join(tempDir, "stdout.bin");
|
||||
const expected = Buffer.from([0, 1, 10, 13, 127, 128, 255]);
|
||||
const output = await runCommandForTest(
|
||||
process.execPath,
|
||||
["-e", `process.stdout.write(Buffer.from(${JSON.stringify([...expected])}))`],
|
||||
process.cwd(),
|
||||
{ stdoutFilePath },
|
||||
);
|
||||
|
||||
expect(output).toBe("");
|
||||
expect(fs.readFileSync(stdoutFilePath)).toEqual(expected);
|
||||
await expect(
|
||||
runCommandForTest(process.execPath, ["-e", ""], process.cwd(), {
|
||||
captureStdout: true,
|
||||
stdoutFilePath: path.join(tempDir, "conflict.bin"),
|
||||
}),
|
||||
).rejects.toThrow("captureStdout and stdoutFilePath cannot be combined");
|
||||
});
|
||||
|
||||
it("restores source artifacts before exiting after receipt-read termination", async () => {
|
||||
if (process.platform === "win32") return;
|
||||
const tempDir = tempDirs.make("openclaw-package-receipt-signal-");
|
||||
const markerPath = path.join(tempDir, "restored");
|
||||
const scriptUrl = pathToFileURL(path.resolve("scripts/package-openclaw-for-docker.mts")).href;
|
||||
const runnerScript = `
|
||||
import fs from "node:fs";
|
||||
const readFile = fs.promises.readFile.bind(fs.promises);
|
||||
fs.promises.readFile = async (...args) => { if (String(args[0]).endsWith("/pack.json")) { process.kill(process.pid, "SIGTERM"); await new Promise((resolve) => setTimeout(resolve, 50)); } return await readFile(...args); };
|
||||
const { packOpenClawPackageForDocker } = await import(${JSON.stringify(scriptUrl)});
|
||||
try {
|
||||
await packOpenClawPackageForDocker("/repo", ${JSON.stringify(tempDir)}, { packJsonPath: "result.json", prepareBundledAiRuntime: async () => async () => {}, prepareChangelog: async () => {}, prepareDocsMap: async () => {}, prepareManifest: async () => {}, restoreChangelog: async () => {}, restoreDocsMap: async () => { fs.writeFileSync(${JSON.stringify(markerPath)}, "done"); }, restoreManifest: async () => {}, runCaptureImpl: async (_command, _args, _cwd, options) => { fs.writeFileSync(options.stdoutFilePath, '[{"filename":"openclaw-2026.5.28.tgz"}]'); fs.writeFileSync(${JSON.stringify(path.join(tempDir, "openclaw-2026.5.28.tgz"))}, "package"); return ""; } });
|
||||
} catch (error) { process.exit(error.exitCode ?? 1); }
|
||||
`;
|
||||
const runner = spawn(process.execPath, ["--input-type=module", "-e", runnerScript]);
|
||||
expect(await waitForExit(runner, 5000)).toEqual({ signal: null, status: 143 });
|
||||
expect(fs.readFileSync(markerPath, "utf8")).toBe("done");
|
||||
});
|
||||
|
||||
it("forwards external termination to active child process groups", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
|
||||
@@ -4947,6 +4947,13 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}"
|
||||
expect(parsedWorkflow.jobs.preflight.outputs.frozen_target).toBe(
|
||||
"${{ steps.manifest.outputs.frozen_target }}",
|
||||
);
|
||||
expect(preflightGuards).toContain(
|
||||
'if [[ "$FROZEN_TARGET" == "true" ]]; then\n' +
|
||||
" pnpm dup:check:coverage\n" +
|
||||
" else\n" +
|
||||
" pnpm dup:check\n" +
|
||||
" fi",
|
||||
);
|
||||
expect(npmLockGuards).toContain("pnpm deps:npm-lock:check");
|
||||
expect(preflightGuards).toContain("pnpm deps:patches:check");
|
||||
expect(preflightGuards).toContain('has_package_script "check:coercion-helpers"');
|
||||
|
||||
@@ -62,6 +62,7 @@ function runSwiftToolchainHarness(options: {
|
||||
swiftVersion: string;
|
||||
selectedDeveloperDir: "command-line-tools" | "custom-xcode" | "invalid" | "xcode";
|
||||
developerDirOverride?: "custom-xcode" | "invalid" | "xcode";
|
||||
xcodebuildFailure?: string;
|
||||
}) {
|
||||
const root = tempDirs.make("openclaw-package-swift-root-");
|
||||
const toolsDir = path.join(root, "tools");
|
||||
@@ -84,9 +85,14 @@ function runSwiftToolchainHarness(options: {
|
||||
mkdirSync(path.dirname(xcodebuild), { recursive: true });
|
||||
writeFileSync(
|
||||
xcodebuild,
|
||||
["#!/usr/bin/env bash", '[[ "$*" == "-version" ]] || exit 2', "echo 'Xcode 26.0'", ""].join(
|
||||
"\n",
|
||||
),
|
||||
[
|
||||
"#!/usr/bin/env bash",
|
||||
'[[ "$*" == "-version" ]] || exit 2',
|
||||
...(options.xcodebuildFailure
|
||||
? [`printf '%s\\n' ${JSON.stringify(options.xcodebuildFailure)} >&2`, "exit 1"]
|
||||
: ["echo 'Xcode 26.0'"]),
|
||||
"",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
chmodSync(xcodebuild, 0o755);
|
||||
@@ -877,6 +883,23 @@ describe("package-mac-app plist stamping", () => {
|
||||
expect(result.stderr).toContain("requires a full Xcode developer directory");
|
||||
});
|
||||
|
||||
it("preserves the native Xcode failure before generic selection guidance", () => {
|
||||
const diagnostic = "xcodebuild: error: SDK metadata is unavailable";
|
||||
const result = runSwiftToolchainHarness({
|
||||
swiftVersion: "6.2.1",
|
||||
selectedDeveloperDir: "xcode",
|
||||
xcodebuildFailure: diagnostic,
|
||||
});
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
const diagnosticIndex = result.stderr.indexOf(diagnostic);
|
||||
const guidanceIndex = result.stderr.indexOf(
|
||||
"ERROR: OpenClaw macOS app packaging requires a full Xcode developer directory",
|
||||
);
|
||||
expect(diagnosticIndex).toBeGreaterThanOrEqual(0);
|
||||
expect(guidanceIndex).toBeGreaterThan(diagnosticIndex);
|
||||
});
|
||||
|
||||
it("runs Sparkle build metadata derivation from the repository root", () => {
|
||||
const helperBlock = getSparkleBuildHelperBlock();
|
||||
const tempRoot = tempDirs.make("openclaw-package-sparkle-root-");
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import { html, nothing } from "lit";
|
||||
import { t } from "../../i18n/index.ts";
|
||||
|
||||
export type DesktopPanelState =
|
||||
| "picker"
|
||||
| "inventory-error"
|
||||
| "credentials"
|
||||
| "connecting"
|
||||
| "connected"
|
||||
| "disconnected";
|
||||
|
||||
export function renderDesktopPanelRecovery(props: {
|
||||
inventoryError: boolean;
|
||||
reason: string | null;
|
||||
onRetry: () => void;
|
||||
}) {
|
||||
return html`
|
||||
<div class="desktop-status">
|
||||
${props.inventoryError
|
||||
? nothing
|
||||
: html`<div>
|
||||
${t("desktop.disconnected", {
|
||||
reason: props.reason ?? t("desktop.unknownReason"),
|
||||
})}
|
||||
</div>`}
|
||||
<button class="desktop-button desktop-button--primary" type="button" @click=${props.onRetry}>
|
||||
${t(props.inventoryError ? "common.retry" : "desktop.reconnect")}
|
||||
</button>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
@@ -23,7 +23,9 @@ import { desktopAppIcon, desktopAppLabel } from "./desktop-app-presentation.ts";
|
||||
import { DesktopClient, type DesktopConnectionHandle } from "./desktop-client.ts";
|
||||
import { desktopCredentialRequirement } from "./desktop-panel-credentials.ts";
|
||||
import { desktopPanelLauncherStyles } from "./desktop-panel-launcher-styles.ts";
|
||||
import { type DesktopPanelState, renderDesktopPanelRecovery } from "./desktop-panel-state.ts";
|
||||
import { desktopPanelStyles } from "./desktop-panel-styles.ts";
|
||||
import { desktopSourceForEnvironment } from "./desktop-source.ts";
|
||||
|
||||
const CLOSE_GLYPH = svg`<svg viewBox="0 0 16 16" width="12" height="12" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round"><path d="M4 4l8 8M12 4l-8 8" /></svg>`;
|
||||
const DOCK_BOTTOM_GLYPH = svg`<svg viewBox="0 0 16 16" width="13" height="13" fill="none" stroke="currentColor" stroke-width="1.3"><rect x="2" y="2.5" width="12" height="11" rx="1.5" /><path d="M2 10h12" /></svg>`;
|
||||
@@ -38,7 +40,6 @@ const panelLayout = createDockPanelLayout({
|
||||
defaultHeight: 420,
|
||||
defaultWidth: 560,
|
||||
});
|
||||
type DesktopPanelState = "picker" | "credentials" | "connecting" | "connected" | "disconnected";
|
||||
type DesktopAppId = WorkerDesktopAppId;
|
||||
type DesktopCredentials = { username?: string; password?: string };
|
||||
type PendingDesktopConnection = {
|
||||
@@ -49,16 +50,6 @@ type PendingDesktopConnection = {
|
||||
};
|
||||
type ObservedDesktopConnection = PendingDesktopConnection & { observed: DesktopObserveResult };
|
||||
|
||||
function desktopSourceForEnvironment(environment: Pick<EnvironmentSummary, "id">): DesktopSource {
|
||||
if (environment.id === "gateway") {
|
||||
return { kind: "host" };
|
||||
}
|
||||
if (environment.id.startsWith("node:") && environment.id.length > "node:".length) {
|
||||
return { kind: "node", nodeId: environment.id.slice("node:".length) };
|
||||
}
|
||||
return { kind: "environment", environmentId: environment.id };
|
||||
}
|
||||
|
||||
/** `<openclaw-desktop-panel>` — dockable RFB access to Gateway desktop sources. */
|
||||
class OpenClawDesktopPanel extends OpenClawLitElement {
|
||||
@property({ attribute: false }) client: GatewayBrowserClient | null = null;
|
||||
@@ -151,7 +142,7 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
|
||||
const wasOpen = this.dockLayout.open;
|
||||
this.dockLayout.setOpen(true);
|
||||
if (detail?.environmentId) {
|
||||
void this.connectEnvironment(detail.environmentId, false);
|
||||
void this.connectRequestedEnvironment(detail.environmentId);
|
||||
} else if (!wasOpen) {
|
||||
void this.refreshEnvironments();
|
||||
} else if (detail?.open !== true) {
|
||||
@@ -191,24 +182,26 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
|
||||
this.launchErrorText = null;
|
||||
}
|
||||
|
||||
private async refreshEnvironments(): Promise<void> {
|
||||
private async refreshEnvironments(expectedOperationId?: number): Promise<boolean> {
|
||||
const client = this.client;
|
||||
if (!client || !this.available) {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
const operationId = ++this.operationId;
|
||||
const operationId = expectedOperationId ?? ++this.operationId;
|
||||
this.loading = true;
|
||||
this.errorText = null;
|
||||
try {
|
||||
const result = await client.request<EnvironmentsListResult>("environments.list", {});
|
||||
if (operationId !== this.operationId) {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
this.environments = result.environments.filter((environment) => environment.desktop === true);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (operationId === this.operationId) {
|
||||
this.errorText = t("desktop.errors.listFailed", { error: formatUiError(error) });
|
||||
}
|
||||
return false;
|
||||
} finally {
|
||||
if (operationId === this.operationId) {
|
||||
this.loading = false;
|
||||
@@ -216,6 +209,22 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
|
||||
}
|
||||
}
|
||||
|
||||
private async connectRequestedEnvironment(environmentId: string): Promise<void> {
|
||||
this.returnToPicker();
|
||||
this.environmentId = environmentId;
|
||||
this.state = "connecting";
|
||||
const operationId = this.operationId;
|
||||
const inventoryLoaded = await this.refreshEnvironments(operationId);
|
||||
if (operationId !== this.operationId) {
|
||||
return;
|
||||
}
|
||||
if (!inventoryLoaded) {
|
||||
this.state = "inventory-error";
|
||||
return;
|
||||
}
|
||||
void this.connectEnvironment(environmentId, false);
|
||||
}
|
||||
|
||||
private async connectEnvironment(
|
||||
environmentId: string,
|
||||
control: boolean,
|
||||
@@ -229,11 +238,11 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
|
||||
this.clearLaunchState();
|
||||
this.credentials = undefined;
|
||||
this.credentialAuth = undefined;
|
||||
this.desktopApps = [
|
||||
...(this.environments.find((environment) => environment.id === environmentId)?.worker
|
||||
?.desktopApps ?? []),
|
||||
];
|
||||
}
|
||||
this.desktopApps = [
|
||||
...(this.environments.find((environment) => environment.id === environmentId)?.worker
|
||||
?.desktopApps ?? []),
|
||||
];
|
||||
this.disconnectConnection();
|
||||
const operationId = this.operationId;
|
||||
const environment = this.environments.find((candidate) => candidate.id === environmentId) ?? {
|
||||
@@ -628,27 +637,6 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
|
||||
`;
|
||||
}
|
||||
|
||||
private renderDisconnected() {
|
||||
return html`
|
||||
<div class="desktop-status">
|
||||
<div>
|
||||
${t("desktop.disconnected", {
|
||||
reason: this.disconnectedReason ?? t("desktop.unknownReason"),
|
||||
})}
|
||||
</div>
|
||||
<button
|
||||
class="desktop-button desktop-button--primary"
|
||||
type="button"
|
||||
@click=${() =>
|
||||
this.environmentId &&
|
||||
void this.connectEnvironment(this.environmentId, this.controlling)}
|
||||
>
|
||||
${t("desktop.reconnect")}
|
||||
</button>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
private renderCredentials() {
|
||||
const ardAccount = this.credentialAuth === "ard-account";
|
||||
return html`
|
||||
@@ -710,10 +698,18 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
|
||||
: nothing}
|
||||
${this.state === "picker"
|
||||
? this.renderPicker()
|
||||
: this.state === "credentials"
|
||||
? this.renderCredentials()
|
||||
: this.state === "disconnected"
|
||||
? this.renderDisconnected()
|
||||
: this.state === "inventory-error" || this.state === "disconnected"
|
||||
? renderDesktopPanelRecovery({
|
||||
inventoryError: this.state === "inventory-error",
|
||||
reason: this.disconnectedReason,
|
||||
onRetry: () =>
|
||||
this.environmentId &&
|
||||
void (this.state === "inventory-error"
|
||||
? this.connectRequestedEnvironment(this.environmentId)
|
||||
: this.connectEnvironment(this.environmentId, this.controlling)),
|
||||
})
|
||||
: this.state === "credentials"
|
||||
? this.renderCredentials()
|
||||
: this.renderConnection()}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
import type { DesktopSource, EnvironmentSummary } from "@openclaw/gateway-protocol";
|
||||
|
||||
export function desktopSourceForEnvironment(
|
||||
environment: Pick<EnvironmentSummary, "id">,
|
||||
): DesktopSource {
|
||||
if (environment.id === "gateway") {
|
||||
return { kind: "host" };
|
||||
}
|
||||
if (environment.id.startsWith("node:") && environment.id.length > "node:".length) {
|
||||
return { kind: "node", nodeId: environment.id.slice("node:".length) };
|
||||
}
|
||||
return { kind: "environment", environmentId: environment.id };
|
||||
}
|
||||
@@ -27,6 +27,21 @@ function sessionsList(placement: "local" | "active") {
|
||||
};
|
||||
}
|
||||
|
||||
const workerDesktopEnvironment = {
|
||||
id: "worker-desktop-1",
|
||||
type: "worker",
|
||||
status: "available",
|
||||
desktop: true,
|
||||
worker: {
|
||||
providerId: "crabbox",
|
||||
state: "attached",
|
||||
ageMs: 1_000,
|
||||
attachedSessionIds: ["main"],
|
||||
tunnelStatus: "connected",
|
||||
desktopApps: ["browser", "terminal"],
|
||||
},
|
||||
} as const;
|
||||
|
||||
async function openPalette(page: import("playwright").Page) {
|
||||
await page.evaluate(() => {
|
||||
window.dispatchEvent(new CustomEvent("openclaw:command-palette-open"));
|
||||
@@ -43,6 +58,16 @@ async function openDesktopPanel(page: import("playwright").Page) {
|
||||
return panel;
|
||||
}
|
||||
|
||||
async function openDirectDesktop(page: import("playwright").Page, environmentId: string) {
|
||||
await page.evaluate((targetEnvironmentId) => {
|
||||
window.dispatchEvent(
|
||||
new CustomEvent("openclaw:desktop-toggle", {
|
||||
detail: { open: true, environmentId: targetEnvironmentId },
|
||||
}),
|
||||
);
|
||||
}, environmentId);
|
||||
}
|
||||
|
||||
async function installDesktopClientFake(panel: import("playwright").Locator) {
|
||||
await panel.evaluate((element) => {
|
||||
(
|
||||
@@ -105,8 +130,185 @@ suite.define(() => {
|
||||
expect(await page.getByRole("option", { name: "Desktop", exact: true }).count()).toBe(1);
|
||||
|
||||
await page.getByRole("option", { name: "Desktop", exact: true }).click();
|
||||
await page.locator("openclaw-desktop-panel section[aria-label='Desktop']").waitFor();
|
||||
const panel = page.locator("openclaw-desktop-panel");
|
||||
await panel.locator("section[aria-label='Desktop']").waitFor();
|
||||
await panel.getByText("Desktop sources", { exact: true }).waitFor();
|
||||
await gateway.waitForRequest("environments.list");
|
||||
expect(await gateway.getRequests("desktop.observe")).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
it("refreshes direct-target inventory before observing the exact worker", async () => {
|
||||
await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => {
|
||||
const gateway = await installMockGateway(page, {
|
||||
featureMethods: ["desktop.launch", "desktop.observe", "environments.list"],
|
||||
methodResponses: {
|
||||
"sessions.list": sessionsList("active"),
|
||||
"environments.list": {
|
||||
environments: [workerDesktopEnvironment],
|
||||
},
|
||||
"desktop.observe": {
|
||||
transport: "rfb",
|
||||
wsPath: "/desktop/observe?token=direct",
|
||||
expiresAtMs: 60_000,
|
||||
control: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
await page.goto(`${suite.server.baseUrl}chat`);
|
||||
const panel = page.locator("openclaw-desktop-panel");
|
||||
await installDesktopClientFake(panel);
|
||||
const requestCount = (await gateway.getRequests()).length;
|
||||
|
||||
await openDirectDesktop(page, "worker-desktop-1");
|
||||
|
||||
const observeRequest = await gateway.waitForRequest("desktop.observe");
|
||||
expect(observeRequest.params).toEqual({
|
||||
source: { kind: "environment", environmentId: "worker-desktop-1" },
|
||||
control: false,
|
||||
});
|
||||
expect(
|
||||
(await gateway.getRequests())
|
||||
.slice(requestCount)
|
||||
.filter((request) => ["environments.list", "desktop.observe"].includes(request.method))
|
||||
.map((request) => request.method),
|
||||
).toEqual(["environments.list", "desktop.observe"]);
|
||||
expect(await panel.getByText("Desktop sources", { exact: true }).count()).toBe(0);
|
||||
await panel.getByRole("button", { name: "Browser", exact: true }).waitFor();
|
||||
await panel.getByRole("button", { name: "Terminal", exact: true }).waitFor();
|
||||
});
|
||||
});
|
||||
|
||||
it("reports an unavailable direct target without showing another source", async () => {
|
||||
await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => {
|
||||
const gateway = await installMockGateway(page, {
|
||||
featureMethods: ["desktop.observe", "environments.list"],
|
||||
methodResponses: {
|
||||
"sessions.list": sessionsList("active"),
|
||||
"environments.list": {
|
||||
environments: [{ id: "gateway", type: "local", status: "available", desktop: true }],
|
||||
},
|
||||
"desktop.observe": {
|
||||
__mockError: {
|
||||
code: "UNAVAILABLE",
|
||||
message: "requested worker desktop is temporarily unavailable",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
await page.goto(`${suite.server.baseUrl}chat`);
|
||||
|
||||
await openDirectDesktop(page, "missing-worker");
|
||||
|
||||
const observeRequest = await gateway.waitForRequest("desktop.observe");
|
||||
expect(observeRequest.params).toEqual({
|
||||
source: { kind: "environment", environmentId: "missing-worker" },
|
||||
control: false,
|
||||
});
|
||||
const panel = page.locator("openclaw-desktop-panel");
|
||||
await panel.getByText(/requested worker desktop is temporarily unavailable/).waitFor();
|
||||
expect(await panel.getByText("Desktop sources", { exact: true }).count()).toBe(0);
|
||||
expect(await panel.getByText("This machine", { exact: true }).count()).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
it("shows direct-target inventory failure without observing or falling back", async () => {
|
||||
await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => {
|
||||
const gateway = await installMockGateway(page, {
|
||||
featureMethods: ["desktop.observe", "environments.list"],
|
||||
methodResponses: {
|
||||
"sessions.list": sessionsList("active"),
|
||||
"environments.list": {
|
||||
__mockError: {
|
||||
code: "UNAVAILABLE",
|
||||
message: "desktop inventory is temporarily unavailable",
|
||||
},
|
||||
},
|
||||
"desktop.observe": {
|
||||
transport: "rfb",
|
||||
wsPath: "/desktop/observe?token=degraded",
|
||||
expiresAtMs: 60_000,
|
||||
control: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
await page.goto(`${suite.server.baseUrl}chat`);
|
||||
await openDirectDesktop(page, "worker-desktop-1");
|
||||
|
||||
const panel = page.locator("openclaw-desktop-panel");
|
||||
await panel.getByRole("alert").filter({ hasText: "inventory" }).waitFor();
|
||||
expect(await gateway.getRequests("desktop.observe")).toHaveLength(0);
|
||||
expect(await panel.getByText("Desktop sources", { exact: true }).count()).toBe(0);
|
||||
expect(await panel.getByText("This machine", { exact: true }).count()).toBe(0);
|
||||
|
||||
await gateway.setMethodResponse("environments.list", {
|
||||
environments: [workerDesktopEnvironment],
|
||||
});
|
||||
await installDesktopClientFake(panel);
|
||||
const requestCount = (await gateway.getRequests()).length;
|
||||
await panel.getByRole("button", { name: "Retry", exact: true }).click();
|
||||
|
||||
await expect
|
||||
.poll(async () => (await gateway.getRequests("environments.list")).length)
|
||||
.toBe(2);
|
||||
const observeRequest = await gateway.waitForRequest("desktop.observe");
|
||||
expect(observeRequest.params).toEqual({
|
||||
source: { kind: "environment", environmentId: "worker-desktop-1" },
|
||||
control: false,
|
||||
});
|
||||
expect(
|
||||
(await gateway.getRequests())
|
||||
.slice(requestCount)
|
||||
.filter((request) => ["environments.list", "desktop.observe"].includes(request.method))
|
||||
.map((request) => request.method),
|
||||
).toEqual(["environments.list", "desktop.observe"]);
|
||||
await panel.getByRole("button", { name: "Browser", exact: true }).waitFor();
|
||||
await panel.getByRole("button", { name: "Terminal", exact: true }).waitFor();
|
||||
expect(await panel.getAttribute("data-connect-count")).toBe("1");
|
||||
expect(await panel.getByText("Desktop sources", { exact: true }).count()).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
it("does not observe a direct target after its inventory refresh is closed", async () => {
|
||||
await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => {
|
||||
const gateway = await installMockGateway(page, {
|
||||
featureMethods: ["desktop.observe", "environments.list"],
|
||||
methodResponses: {
|
||||
"sessions.list": sessionsList("active"),
|
||||
"environments.list": { environments: [] },
|
||||
"desktop.observe": {
|
||||
transport: "rfb",
|
||||
wsPath: "/desktop/observe?token=stale",
|
||||
expiresAtMs: 60_000,
|
||||
control: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
await page.goto(`${suite.server.baseUrl}chat`);
|
||||
await gateway.deferNext("environments.list");
|
||||
const inventoryCount = (await gateway.getRequests("environments.list")).length;
|
||||
|
||||
await openDirectDesktop(page, "worker-desktop-1");
|
||||
await expect
|
||||
.poll(async () => (await gateway.getRequests("environments.list")).length)
|
||||
.toBe(inventoryCount + 1);
|
||||
await page.evaluate(() => {
|
||||
window.dispatchEvent(
|
||||
new CustomEvent("openclaw:desktop-toggle", { detail: { open: false } }),
|
||||
);
|
||||
});
|
||||
await gateway.resolveDeferred("environments.list", { environments: [] });
|
||||
await page.evaluate(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => resolve()));
|
||||
}),
|
||||
);
|
||||
|
||||
expect(
|
||||
await page.locator("openclaw-desktop-panel section[aria-label='Desktop']").count(),
|
||||
).toBe(0);
|
||||
expect(await gateway.getRequests("desktop.observe")).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ import {
|
||||
} from "../../lib/sessions/session-key.ts";
|
||||
import { isActiveTask } from "../../lib/tasks/data.ts";
|
||||
import { renderBoardViewSwitch } from "./board-session-surface.ts";
|
||||
import { resolveChatPanePlacement } from "./chat-pane-placement.ts";
|
||||
import { resolveChatPaneDesktopTarget, resolveChatPanePlacement } from "./chat-pane-placement.ts";
|
||||
import { ChatPaneSessionMenu } from "./chat-pane-session-menu.ts";
|
||||
import { readChatSessionActionAccess } from "./chat-session-action-access.ts";
|
||||
import { resolveChatAgentId } from "./chat-state-route.ts";
|
||||
@@ -192,13 +192,19 @@ export abstract class ChatPaneHeader extends ChatPaneSessionMenu {
|
||||
? { "continue-in-terminal": continueInTerminalDisabledReason }
|
||||
: {}),
|
||||
};
|
||||
const desktopPanelAvailable = isDesktopPanelAvailable(this.context.gateway.snapshot);
|
||||
const openDesktopPanel = () =>
|
||||
const desktopEnvironmentId = resolveChatPaneDesktopTarget(row);
|
||||
const desktopPanelAvailable =
|
||||
desktopEnvironmentId !== null && isDesktopPanelAvailable(this.context.gateway.snapshot);
|
||||
const openDesktopPanel = () => {
|
||||
if (!desktopEnvironmentId) {
|
||||
return;
|
||||
}
|
||||
window.dispatchEvent(
|
||||
new CustomEvent<DesktopPanelToggleDetail>(DESKTOP_PANEL_TOGGLE_EVENT, {
|
||||
detail: { open: true },
|
||||
detail: { open: true, environmentId: desktopEnvironmentId },
|
||||
}),
|
||||
);
|
||||
};
|
||||
const browserPanelAction = sessionWorkspace.onToggleBrowser
|
||||
? html`<openclaw-tooltip .content=${t("browser.toggle")}>
|
||||
<button
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
|
||||
import type { GatewayBrowserClient } from "../../api/gateway.ts";
|
||||
import type { GatewaySessionRow } from "../../api/types.ts";
|
||||
import type { ApplicationGatewaySnapshot } from "../../app/context.ts";
|
||||
@@ -5,10 +6,27 @@ import {
|
||||
requestCloudWorkerStop,
|
||||
resolveCloudWorkerStopAction,
|
||||
} from "../../components/cloud-worker-stop.ts";
|
||||
import { isCloudWorkerPlacementState } from "../../components/session-row-badges.ts";
|
||||
import { t } from "../../i18n/index.ts";
|
||||
import { readSessionMethodAccess } from "../../lib/session-method-access.ts";
|
||||
import { parseAgentSessionKey } from "../../lib/sessions/session-key.ts";
|
||||
|
||||
export function resolveChatPaneDesktopTarget(
|
||||
session: GatewaySessionRow | undefined,
|
||||
): string | null {
|
||||
if (!session) {
|
||||
return null;
|
||||
}
|
||||
const placement = session.placement;
|
||||
if (isCloudWorkerPlacementState(placement?.state)) {
|
||||
return "environmentId" in placement
|
||||
? (normalizeOptionalString(placement.environmentId) ?? null)
|
||||
: null;
|
||||
}
|
||||
const execNode = normalizeOptionalString(session.execNode);
|
||||
return execNode ? `node:${execNode}` : "gateway";
|
||||
}
|
||||
|
||||
export function resolveChatPanePlacement(params: {
|
||||
gatewaySnapshot: ApplicationGatewaySnapshot;
|
||||
reclaimingKey: string | null;
|
||||
|
||||
@@ -168,7 +168,7 @@ describe("chat pane terminal action", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("renders desktop controls for local sessions when the source RPC is available", () => {
|
||||
it("targets the selected session from the desktop controls", () => {
|
||||
const client = { request: vi.fn() } as unknown as GatewayBrowserClient;
|
||||
const { pane, state } = createTestChatPane({ client, sessions: {} as SessionCapability });
|
||||
const localSession = {
|
||||
@@ -176,12 +176,8 @@ describe("chat pane terminal action", () => {
|
||||
kind: "direct",
|
||||
updatedAt: 0,
|
||||
} satisfies GatewaySessionRow;
|
||||
const cloudSession = {
|
||||
...localSession,
|
||||
placement: { state: "active" } as GatewaySessionRow["placement"],
|
||||
} satisfies GatewaySessionRow;
|
||||
const container = document.createElement("div");
|
||||
const renderHeader = (session: GatewaySessionRow) => {
|
||||
const renderHeader = (session: GatewaySessionRow | undefined) => {
|
||||
render(
|
||||
pane.renderPaneHeader(
|
||||
createSessionWorkspaceProps(state),
|
||||
@@ -202,30 +198,80 @@ describe("chat pane terminal action", () => {
|
||||
?.panelActions.map((action) => action.id) ?? [];
|
||||
const snapshot = pane.context.gateway.snapshot;
|
||||
snapshot.hello = desktopHello([], ["operator.admin"]);
|
||||
renderHeader(cloudSession);
|
||||
renderHeader(localSession);
|
||||
expect(container.querySelector('[aria-label="Toggle desktop panel"]')).toBeNull();
|
||||
|
||||
snapshot.hello = desktopHello(["desktop.observe"], ["operator.admin"]);
|
||||
renderHeader(localSession);
|
||||
expect(container.querySelector('[aria-label="Toggle desktop panel"]')).not.toBeNull();
|
||||
expect(panelActionIds()).toContain("desktop");
|
||||
|
||||
const events: CustomEvent<DesktopPanelToggleDetail>[] = [];
|
||||
const listener = (event: Event) => events.push(event as CustomEvent<DesktopPanelToggleDetail>);
|
||||
window.addEventListener(DESKTOP_PANEL_TOGGLE_EVENT, listener);
|
||||
try {
|
||||
renderHeader(localSession);
|
||||
const button = container.querySelector<HTMLButtonElement>(
|
||||
'[aria-label="Toggle desktop panel"]',
|
||||
);
|
||||
expect(button).not.toBeNull();
|
||||
expect(panelActionIds()).toContain("desktop");
|
||||
button?.click();
|
||||
expect(events).toHaveLength(1);
|
||||
expect(events[0]?.detail).toEqual({ open: true });
|
||||
const targetCases: Array<{
|
||||
name: string;
|
||||
session: GatewaySessionRow;
|
||||
environmentId: string;
|
||||
}> = [
|
||||
{ name: "local", session: localSession, environmentId: "gateway" },
|
||||
{
|
||||
name: "cloud",
|
||||
session: {
|
||||
...localSession,
|
||||
execNode: "stale-node",
|
||||
placement: {
|
||||
state: "active",
|
||||
environmentId: "worker-desktop-1",
|
||||
} as GatewaySessionRow["placement"],
|
||||
},
|
||||
environmentId: "worker-desktop-1",
|
||||
},
|
||||
{
|
||||
name: "node",
|
||||
session: { ...localSession, execNode: " paired-node " },
|
||||
environmentId: "node:paired-node",
|
||||
},
|
||||
{
|
||||
name: "reclaimed",
|
||||
session: {
|
||||
...localSession,
|
||||
execNode: " reclaimed-node ",
|
||||
placement: {
|
||||
state: "reclaimed",
|
||||
environmentId: "former-worker",
|
||||
} as GatewaySessionRow["placement"],
|
||||
},
|
||||
environmentId: "node:reclaimed-node",
|
||||
},
|
||||
];
|
||||
for (const testCase of targetCases) {
|
||||
renderHeader(testCase.session);
|
||||
const button = container.querySelector<HTMLButtonElement>(
|
||||
'[aria-label="Toggle desktop panel"]',
|
||||
);
|
||||
expect(button, testCase.name).not.toBeNull();
|
||||
expect(panelActionIds(), testCase.name).toContain("desktop");
|
||||
button?.click();
|
||||
expect(events.at(-1)?.detail, testCase.name).toEqual({
|
||||
open: true,
|
||||
environmentId: testCase.environmentId,
|
||||
});
|
||||
}
|
||||
|
||||
const eventCount = events.length;
|
||||
renderHeader({
|
||||
...localSession,
|
||||
execNode: "must-not-fall-back",
|
||||
placement: { state: "requested" } as GatewaySessionRow["placement"],
|
||||
});
|
||||
expect(container.querySelector('[aria-label="Toggle desktop panel"]')).toBeNull();
|
||||
expect(panelActionIds()).not.toContain("desktop");
|
||||
expect(events).toHaveLength(eventCount);
|
||||
|
||||
renderHeader(undefined);
|
||||
expect(container.querySelector('[aria-label="Toggle desktop panel"]')).toBeNull();
|
||||
expect(panelActionIds()).not.toContain("desktop");
|
||||
|
||||
snapshot.hello = desktopHello(["desktop.observe"], ["operator.read"]);
|
||||
renderHeader(cloudSession);
|
||||
renderHeader(localSession);
|
||||
expect(container.querySelector('[aria-label="Toggle desktop panel"]')).toBeNull();
|
||||
} finally {
|
||||
window.removeEventListener(DESKTOP_PANEL_TOGGLE_EVENT, listener);
|
||||
|
||||
Reference in New Issue
Block a user