ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(gateway): avoid stale wake delays after node re-pairing (#109647)

Browse Source 
* fix(gateway): clear wake state on node removal

* fix(gateway): invalidate wake flows on node removal

* fix(gateway): preserve active wakes on disconnect

* fix(gateway): share node removal runtime cleanup

* fix(gateway): clean up pending node work reliably

* fix(gateway): bind node work to pairing generations

* fix(gateway): fence stale node pairing sessions

* fix(gateway): tighten node pairing ownership

* fix(gateway): close node pairing mutation races

* fix(gateway): bind node sessions to pairing generation

* fix(gateway): harden node pairing generation guards

* fix(gateway): close node pairing generation races

* fix(gateway): guard APNs transport ownership

* fix(gateway): revoke reapproved node transports

* fix(gateway): guard retired node lifecycles

* fix(gateway): preserve pending work across reconnects

* fix(gateway): bind node subscriptions to pairing generations

* fix(gateway): settle subscription fanout failures

* fix(gateway): fence pairing generation ownership

* fix(gateway): fence asynchronous node events

* fix(gateway): fence watch results and voice wake fanout

* fix(gateway): guard voice wake generation fanout

* fix(gateway): carry authenticated snapshot generation

* fix(gateway): bind node state to pairing generation

* fix(gateway): align generation guards with current base

* fix(gateway): enforce pairing-owned node lifecycle

* fix(gateway): fence detached voice session writes

* chore(deps): bump proxyline to 0.3.4

* fix(gateway): close pairing lifecycle races

Co-authored-by: zhang-guiping <zhang.guiping@xydigit.com>

* fix(gateway): distinguish pending action mutation

Co-authored-by: zhang-guiping <zhang.guiping@xydigit.com>

* test(gateway): align lifecycle coverage

Co-authored-by: zhang-guiping <zhang.guiping@xydigit.com>

* test(skills): split remote reconciliation coverage

Co-authored-by: zhang-guiping <zhang.guiping@xydigit.com>

* style: format pairing lifecycle changes

Co-authored-by: zhang-guiping <zhang.guiping@xydigit.com>

---------

Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
xingzhou
2026-07-21 20:10:08 +08:00
committed by GitHub
parent b2aa21612d
commit 186c25f7f7
95 changed files with 9242 additions and 1840 deletions
Download Patch File Download Diff File Expand all files Collapse all files
npm-shrinkwrap.json
Generated
+5 -5
View File
@@ -24,7 +24,7 @@
"@modelcontextprotocol/sdk": "1.29.0",
"@mozilla/readability": "0.6.0",
"@openclaw/fs-safe": "0.4.4",
"@openclaw/proxyline": "0.3.3",
"@openclaw/proxyline": "0.3.4",
"@silvia-odwyer/photon-node": "0.3.4",
"chalk": "5.6.2",
"chokidar": "5.0.0",
@@ -447,15 +447,15 @@
}
},
"node_modules/@openclaw/proxyline": {
"version": "0.3.3",
"resolved": "https://registry.npmjs.org/@openclaw/proxyline/-/proxyline-0.3.3.tgz",
"integrity": "sha512-sftHnW69NHQqLjCxBTvQ8f/eQl+peZ5pHCBQtuTWBbeuYRHZ0/GXVTmw/O/YKsShMbqPWhJB0UYtPPdvCUSS8w==",
"version": "0.3.4",
"resolved": "https://registry.npmjs.org/@openclaw/proxyline/-/proxyline-0.3.4.tgz",
"integrity": "sha512-WuLE282PW7vLienDNFTjdSI/BCYDAepvr5siOTYABQhqFo8qbP9MgmJzZnwmskUqiYugGzle6gvaRbJTkwbZ8A==",
"license": "MIT",
"engines": {
"node": ">=22.19.0"
},
"peerDependencies": {
"undici": ">=8.3.0 <9"
"undici": ">=8.5.0 <9"
}
},
"node_modules/@opentelemetry/semantic-conventions": {
package.json
+1 -1
View File
@@ -1895,7 +1895,7 @@
"@mozilla/readability": "0.6.0",
"@openclaw/ai": "workspace:*",
"@openclaw/fs-safe": "0.4.4",
"@openclaw/proxyline": "0.3.3",
"@openclaw/proxyline": "0.3.4",
"@silvia-odwyer/photon-node": "0.3.4",
"chalk": "5.6.2",
"chokidar": "5.0.0",
pnpm-lock.yaml
Generated
+6 -6
View File
@@ -86,8 +86,8 @@ importers:
specifier: 0.4.4
version: 0.4.4
'@openclaw/proxyline':
specifier: 0.3.3
version: 0.3.3(undici@8.5.0)
specifier: 0.3.4
version: 0.3.4(undici@8.5.0)
'@silvia-odwyer/photon-node':
specifier: 0.3.4
version: 0.3.4
@@ -3712,11 +3712,11 @@ packages:
node-pty:
optional: true
'@openclaw/proxyline@0.3.3':
resolution: {integrity: sha512-sftHnW69NHQqLjCxBTvQ8f/eQl+peZ5pHCBQtuTWBbeuYRHZ0/GXVTmw/O/YKsShMbqPWhJB0UYtPPdvCUSS8w==}
'@openclaw/proxyline@0.3.4':
resolution: {integrity: sha512-WuLE282PW7vLienDNFTjdSI/BCYDAepvr5siOTYABQhqFo8qbP9MgmJzZnwmskUqiYugGzle6gvaRbJTkwbZ8A==}
engines: {node: '>=22.19.0'}
peerDependencies:
undici: '>=8.3.0 <9'
undici: '>=8.5.0 <9'
'@openclaw/uirouter@0.1.0':
resolution: {integrity: sha512-w5tNj2FIukVJqJ1wt5wiDbrI6DI4tOkUbtqnnU5Fl4EgnRKFJtfHI3WrWTWTTJlXbbrwGSMptyrSZmQVdRo83Q==}
@@ -10087,7 +10087,7 @@ snapshots:
dependencies:
ghostty-web: 0.4.0
'@openclaw/proxyline@0.3.3(undici@8.5.0)':
'@openclaw/proxyline@0.3.4(undici@8.5.0)':
dependencies:
undici: 8.5.0
pnpm-workspace.yaml
+1 -1
View File
@@ -11,7 +11,7 @@ minimumReleaseAgeExclude:
- "@openclaw/crabline@0.1.11"
- "@openclaw/fs-safe@0.4.4"
- "@openclaw/libterminal@0.3.2"
- "@openclaw/proxyline@0.3.3"
- "@openclaw/proxyline@0.3.4"
- "@openclaw/uirouter@0.1.0"
- "acpx"
- "tokenjuice"
src/agents/embedded-agent-runner/compact.ts
+2 -2
View File
@@ -19,7 +19,7 @@ import {
} from "../../gateway/session-compaction-checkpoints.js";
import {
formatActiveNodeContextLabel,
getActiveNodeContext,
getCurrentActiveNodeContext,
} from "../../infra/active-node-context.js";
import { resolveDiagnosticModelContentCapturePolicy } from "../../infra/diagnostic-llm-content.js";
import {
@@ -1317,7 +1317,7 @@ async function compactEmbeddedAgentSessionDirectOnce(
agentId: sessionAgentId,
}),
}),
activeNode: formatActiveNodeContextLabel(getActiveNodeContext()),
activeNode: formatActiveNodeContextLabel(getCurrentActiveNodeContext()),
};
const sandboxInfoExecPolicy = resolveEmbeddedSandboxInfoExecPolicy({
config: params.config,
src/agents/system-prompt-params.test.ts
+11
View File
@@ -42,6 +42,17 @@ describe("buildSystemPromptParams", () => {
expect(runtimeInfo.activeNode).toBe("mac-123");
});
it("omits an active node that fails current-generation validation", () => {
setActiveNodeContext(
{ nodeId: "mac-123", pairingGeneration: "generation-a" },
{ isCurrent: () => false },
);
const { runtimeInfo } = buildParams({});
expect(runtimeInfo.activeNode).toBeUndefined();
});
it("detects repo root from workspaceDir", async () => {
const temp = await makeTempDir("workspace");
const repoRoot = path.join(temp, "repo");
src/agents/system-prompt-params.ts
+3 -2
View File
@@ -10,7 +10,7 @@ import type { ChatType } from "../channels/chat-type.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import {
formatActiveNodeContextLabel,
getActiveNodeContext,
getCurrentActiveNodeContext,
} from "../infra/active-node-context.js";
import { findGitRoot } from "../infra/git-root.js";
import type { ActiveProcessSessionReference } from "./bash-process-references.js";
@@ -68,7 +68,8 @@ export function buildSystemPromptParams(params: {
runtimeInfo: {
agentId: params.agentId,
...params.runtime,
activeNode: formatActiveNodeContextLabel(getActiveNodeContext()) ?? params.runtime.activeNode,
activeNode:
formatActiveNodeContextLabel(getCurrentActiveNodeContext()) ?? params.runtime.activeNode,
repoRoot,
},
userTimezone,
src/gateway/device-pairing-prune.test.ts
+85 -4
View File
@@ -1,13 +1,26 @@
// Covers gateway-side cleanup when silent pairing supersedes stale sibling records.
import { afterAll, beforeAll, describe, expect, test } from "vitest";
import { afterAll, afterEach, beforeAll, describe, expect, test } from "vitest";
import {
approveDevicePairing,
listDevicePairing,
removePairedDeviceRole,
requestDevicePairing,
} from "../infra/device-pairing.js";
import { approveNodePairing, listNodePairing, requestNodePairing } from "../infra/node-pairing.js";
import { loadApnsRegistration, registerApnsRegistration } from "../infra/push-apns.js";
import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js";
import { pruneSupersededSilentPairingsAfterApproval } from "./device-pairing-prune.js";
import { drainNodePendingWork, enqueueNodePendingWork } from "./node-pending-work.js";
import { enqueuePendingNodeAction, listPendingNodeActions } from "./node-runtime-state.js";
import {
captureNodeWakeLifecycle,
runNodeWakeAttempt,
runNodeWakeNudgeAttempt,
} from "./node-wake-state.js";
import {
getNodeWakeStateSnapshot,
resetNodeWakeStateForTest,
} from "./node-wake-state.test-support.js";
const suiteRootTracker = createSuiteTempRootTracker({ prefix: "openclaw-gateway-pairing-prune-" });
@@ -19,6 +32,8 @@ function createPruneContext(params?: { connectedDeviceIds?: string[] }) {
const invalidated: string[] = [];
const disconnected: string[] = [];
const logs: string[] = [];
const warnings: string[] = [];
const clearedSurfaces: string[] = [];
const connected = new Set(params?.connectedDeviceIds ?? []);
const context: PruneContext = {
broadcast: (event, payload) => {
@@ -26,6 +41,7 @@ function createPruneContext(params?: { connectedDeviceIds?: string[] }) {
},
logGateway: {
info: (message: string) => logs.push(message),
warn: (message: string) => warnings.push(message),
},
hasConnectedClientsForDevice: (deviceId: string) => connected.has(deviceId),
invalidateClientsForDevice: (deviceId: string) => {
@@ -34,8 +50,14 @@ function createPruneContext(params?: { connectedDeviceIds?: string[] }) {
disconnectClientsForDevice: (deviceId: string) => {
disconnected.push(deviceId);
},
nodeRegistry: {
updateSurface: (nodeId: string) => {
clearedSurfaces.push(nodeId);
return null;
},
},
};
return { broadcasts, invalidated, disconnected, logs, context };
return { broadcasts, invalidated, disconnected, logs, warnings, clearedSurfaces, context };
}
async function pairSilentDevice(params: {
@@ -79,6 +101,10 @@ describe("pruneSupersededSilentPairingsAfterApproval", () => {
await suiteRootTracker.cleanup();
});
afterEach(() => {
resetNodeWakeStateForTest();
});
test("retires stale node siblings across both pairing stores", async () => {
const baseDir = await suiteRootTracker.make("case");
await pairSilentDevice({
@@ -102,6 +128,39 @@ describe("pruneSupersededSilentPairingsAfterApproval", () => {
clientMode: "node",
displayName: "megaclaw",
});
await registerApnsRegistration({
nodeId: "node-stale",
transport: "direct",
token: "ABCD1234ABCD1234ABCD1234ABCD1234",
topic: "ai.openclaw.ios",
environment: "sandbox",
baseDir,
});
await runNodeWakeAttempt({
nodeId: "node-stale",
force: true,
throttleMs: 60_000,
attempt: async (markAttempted) => {
markAttempted();
return { available: true, throttled: false, path: "sent", durationMs: 1 };
},
});
await runNodeWakeNudgeAttempt({
nodeId: "node-stale",
throttleMs: 60_000,
throttled: () => ({ sent: false, throttled: true, reason: "throttled", durationMs: 0 }),
attempt: async () => ({ sent: true, throttled: false, reason: "sent", durationMs: 1 }),
});
enqueueNodePendingWork({ nodeId: "node-stale", type: "location.request" });
enqueuePendingNodeAction({
nodeId: "node-stale",
pairingGeneration: "generation-1",
command: "camera.capture",
idempotencyKey: "idem-1",
ttlMs: 60_000,
maxPerNode: 10,
});
const wakeLifecycle = captureNodeWakeLifecycle("node-stale");
const harness = createPruneContext();
const pruned = await pruneSupersededSilentPairingsAfterApproval({
@@ -116,8 +175,15 @@ describe("pruneSupersededSilentPairingsAfterApproval", () => {
expect(devices.paired.map((device) => device.deviceId)).toEqual(["node-anchor"]);
const nodes = await listNodePairing(baseDir);
expect(nodes.paired).toHaveLength(0);
expect(getNodeWakeStateSnapshot("node-stale")).toBeUndefined();
expect(wakeLifecycle.aborted).toBe(true);
expect(drainNodePendingWork("node-stale", { includeDefaultStatus: false }).items).toEqual([]);
expect(listPendingNodeActions({ nodeId: "node-stale", ttlMs: 60_000 })).toEqual([]);
await expect(loadApnsRegistration("node-stale", baseDir)).resolves.toBeNull();
expect(harness.invalidated).toEqual(["node-stale"]);
expect(harness.disconnected).toEqual(["node-stale"]);
expect(harness.clearedSurfaces).toEqual(["node-stale"]);
expect(harness.warnings).toEqual([]);
expect(harness.broadcasts).toEqual([
{
event: "node.pair.resolved",
@@ -126,15 +192,29 @@ describe("pruneSupersededSilentPairingsAfterApproval", () => {
]);
});
test("keeps connected siblings and emits no node broadcast for operator-only prunes", async () => {
test("keeps connected siblings and clears APNs for operator-only full prunes", async () => {
const baseDir = await suiteRootTracker.make("case");
await pairSilentDevice({
baseDir,
deviceId: "cli-stale",
roles: ["operator"],
roles: ["operator", "node"],
clientId: "cli",
clientMode: "cli",
});
await registerApnsRegistration({
nodeId: "cli-stale",
transport: "direct",
token: "ABCD1234ABCD1234ABCD1234ABCD1234",
topic: "ai.openclaw.ios",
environment: "sandbox",
baseDir,
});
await expect(
removePairedDeviceRole({ deviceId: "cli-stale", role: "node", baseDir }),
).resolves.toEqual({ deviceId: "cli-stale", role: "node", removedDevice: false });
await expect(loadApnsRegistration("cli-stale", baseDir)).resolves.toMatchObject({
nodeId: "cli-stale",
});
await pairSilentDevice({
baseDir,
deviceId: "cli-live",
@@ -164,6 +244,7 @@ describe("pruneSupersededSilentPairingsAfterApproval", () => {
"cli-anchor",
"cli-live",
]);
await expect(loadApnsRegistration("cli-stale", baseDir)).resolves.toBeNull();
expect(harness.broadcasts).toEqual([]);
expect(harness.disconnected).toEqual(["cli-stale"]);
});
src/gateway/device-pairing-prune.ts
+8 -5
View File
@@ -3,6 +3,7 @@ import {
pruneSupersededSilentPairedDevices,
type PrunedSupersededPairedDevice,
} from "../infra/device-pairing.js";
import { clearRemovedNodeRuntimeState } from "./node-runtime-state.js";
import type { GatewayRequestContext } from "./server-methods/types.js";
type PruneContext = Pick<
@@ -12,7 +13,8 @@ type PruneContext = Pick<
| "invalidateClientsForDevice"
| "disconnectClientsForDevice"
> & {
logGateway: Pick<GatewayRequestContext["logGateway"], "info">;
logGateway: Pick<GatewayRequestContext["logGateway"], "info" | "warn">;
nodeRegistry: Pick<GatewayRequestContext["nodeRegistry"], "updateSurface">;
};
/**
@@ -38,13 +40,14 @@ export async function pruneSupersededSilentPairingsAfterApproval(params: {
context.logGateway.info(
`device pairing pruned superseded silent pairing device=${entry.deviceId} roles=${entry.roles.join(",") || "none"}`,
);
if (entry.roles.includes("node")) {
// Persistent pruning is a node removal owner too. Clear disconnected
// queues, wake lifecycles, and runtime metadata before session teardown.
clearRemovedNodeRuntimeState({ nodeId: entry.deviceId, context });
}
// Invalidate before disconnect so buffered frames from a racing reconnect
// fail authorization, mirroring device.pair.remove ordering.
context.invalidateClientsForDevice?.(entry.deviceId, { reason: "device-pair-removed" });
// The node surface lives on the pruned device record, so dropping the
// record retired it too; tell node list consumers. Pruned devices are
// offline (connected ones are skipped), so there is no live node session
// or queued action state to clear.
if (entry.roles.includes("node")) {
context.broadcast(
"node.pair.resolved",
src/gateway/gateway-misc.test.ts
+25 -92
View File
@@ -4,7 +4,6 @@ import * as fs from "node:fs/promises";
import type { IncomingMessage, ServerResponse } from "node:http";
import * as os from "node:os";
import * as path from "node:path";
import { expectDefined } from "@openclaw/normalization-core";
import { beforeAll, beforeEach, describe, expect, it, test, vi } from "vitest";
import {
GATEWAY_CLIENT_CAPS,
@@ -27,19 +26,13 @@ import {
DEFAULT_DANGEROUS_NODE_COMMANDS,
resolveNodeCommandAllowlist,
} from "./node-command-policy.js";
import type { SerializedEventPayload } from "./node-registry.js";
import { createGatewayBroadcaster } from "./server-broadcast.js";
import {
createSessionEventSubscriberRegistry,
createSessionMessageSubscriberRegistry,
} from "./server-chat-state.js";
import { createSessionMessageSubscriberRegistry } from "./server-chat-state.js";
import { createChatRunRegistry } from "./server-chat.js";
import { MAX_BUFFERED_BYTES } from "./server-constants.js";
import { handleNodeInvokeResult } from "./server-methods/nodes.handlers.invoke-result.js";
import type { GatewayClient as GatewayMethodClient } from "./server-methods/types.js";
import type { GatewayRequestContext, RespondFn } from "./server-methods/types.js";
import { createGatewayNodeSessionRuntime } from "./server-node-session-runtime.js";
import { createNodeSubscriptionManager } from "./server-node-subscriptions.js";
import { formatError, normalizeVoiceWakeTriggers } from "./server-utils.js";
import type { GatewayWsClient } from "./server/ws-types.js";
@@ -163,6 +156,16 @@ describe("GatewayClient", () => {
expect(opts?.maxPayload).toBe(25 * 1024 * 1024);
});
test("uses the admitted pairing identity for shared-auth connect failures", async () => {
const source = await fs.readFile(
new URL("./server/ws-connection/connect-session.ts", import.meta.url),
"utf8",
);
expect(source).toContain("deviceId: admittedNodePairing.identity.nodeId");
expect(source).not.toContain("deviceId: authenticatedNodePairing.nodeId");
});
test("does not pass an explicit direct agent for loopback control-plane WebSocket connections", () => {
expectNoGatewayClientAgent({ url: "ws://127.0.0.1:1" });
});
@@ -421,6 +424,20 @@ describe("gateway broadcaster", () => {
expect(workerSocket.send).not.toHaveBeenCalled();
});
it("skips locally invalidated clients before generic broadcast delivery", () => {
const socket = makeRecordingSocket();
const client = makeOperatorWsClient("c-invalidated", socket, ["operator.read"]);
client.invalidated = true;
const { broadcast, broadcastToConnIds } = createGatewayBroadcaster({
clients: new Set([client]),
});
broadcast("heartbeat", { ts: 1 });
broadcastToConnIds("heartbeat", { ts: 2 }, new Set([client.connId]));
expect(socket.send).not.toHaveBeenCalled();
});
it("delivers scoped client events only for gateway-owned session subscriptions", () => {
const legacySocket = makeRecordingSocket();
const firstSocket = makeRecordingSocket();
@@ -677,8 +694,6 @@ describe("gateway broadcaster", () => {
"update.available",
]);
expectSentEvents(nodeSocket, [
"voicewake.changed",
"voicewake.routing.changed",
"heartbeat",
"presence",
"health",
@@ -892,88 +907,6 @@ describe("late-arriving invoke results", () => {
});
});
describe("node subscription manager", () => {
test("routes events to subscribed nodes", () => {
const manager = createNodeSubscriptionManager();
const sent: Array<{
nodeId: string;
event: string;
payloadJSON?: SerializedEventPayload | null;
}> = [];
const sendEvent = (evt: {
nodeId: string;
event: string;
payloadJSON?: SerializedEventPayload | null;
}) => sent.push(evt);
manager.subscribe("node-a", "main");
manager.subscribe("node-b", "main");
manager.sendToSession("main", "chat", { ok: true }, sendEvent);
expect(sent).toHaveLength(2);
expect(sent.map((s) => s.nodeId).toSorted()).toEqual(["node-a", "node-b"]);
expect(expectDefined(sent[0], "sent[0] test invariant").event).toBe("chat");
});
test("runtime forwards subscribed node payload json without parsing it again", () => {
const frames: string[] = [];
const socket: TestSocket = {
bufferedAmount: 0,
send: vi.fn((payload: string) => frames.push(payload)),
close: vi.fn(),
};
const parseSpy = vi.spyOn(JSON, "parse");
try {
const runtime = createGatewayNodeSessionRuntime({
broadcast: vi.fn(),
sessionEventSubscribers: createSessionEventSubscriberRegistry(),
sessionMessageSubscribers: createSessionMessageSubscriberRegistry(),
});
runtime.nodeRegistry.register(
makeGatewayWsClient("conn-node-a", socket, {
role: "node",
scopes: [],
client: {
id: "node-client",
version: "1.0.0",
platform: "macos",
mode: "node",
},
device: { id: "node-a" },
} as unknown as GatewayWsClient["connect"]),
{},
);
runtime.nodeSubscribe("node-a", "main");
runtime.nodeSendToSession("main", "chat", { ok: true });
expect(parseSpy).not.toHaveBeenCalled();
} finally {
parseSpy.mockRestore();
}
expect(JSON.parse(frames[0] ?? "{}")).toEqual({
type: "event",
event: "chat",
payload: { ok: true },
});
});
test("unsubscribeAll clears session mappings", () => {
const manager = createNodeSubscriptionManager();
const sent: string[] = [];
const sendEvent = (evt: { nodeId: string; event: string }) =>
sent.push(`${evt.nodeId}:${evt.event}`);
manager.subscribe("node-a", "main");
manager.subscribe("node-a", "secondary");
manager.unsubscribeAll("node-a");
manager.sendToSession("main", "tick", {}, sendEvent);
manager.sendToSession("secondary", "tick", {}, sendEvent);
expect(sent).toStrictEqual([]);
});
});
describe("resolveNodeCommandAllowlist", () => {
function expectAllowed(allow: { has: (cmd: string) => boolean }, commands: string[]) {
for (const cmd of commands) {
src/gateway/local-request-context.ts
+1 -1
View File
@@ -154,7 +154,7 @@ function createLocalGatewayRequestContext(
nodeSubscribe: () => {},
nodeUnsubscribe: () => {},
nodeUnsubscribeAll: () => {},
hasConnectedTalkNode: () => false,
hasConnectedTalkNode: async () => false,
nodeRegistry: new NodeRegistry(),
agentRunSeq: new Map(),
chatAbortControllers: new Map(),
src/gateway/node-agent-cli-runtime.test.ts
+7
View File
@@ -32,6 +32,7 @@ describe("invokeNodeClaudeCliRun", () => {
mocks.get.mockReturnValue({
connId: "conn-1",
nodeId: "node-1",
pairingGeneration: "generation-1",
commands: ["agent.cli.claude.run.v1"],
});
});
@@ -75,5 +76,11 @@ describe("invokeNodeClaudeCliRun", () => {
).resolves.toEqual({ ok: true });
expect(mocks.resolveNodeCommandAllowlist).toHaveBeenCalledOnce();
expect(mocks.invoke).toHaveBeenCalledOnce();
expect(mocks.invoke).toHaveBeenCalledWith(
expect.objectContaining({
expectedConnId: "conn-1",
expectedPairingGeneration: "generation-1",
}),
);
});
});
src/gateway/node-agent-cli-runtime.ts
+1
View File
@@ -59,6 +59,7 @@ export async function invokeNodeClaudeCliRun(params: {
return await context.nodeRegistry.invoke({
nodeId: params.nodeId,
expectedConnId: node.connId,
...(node.pairingGeneration ? { expectedPairingGeneration: node.pairingGeneration } : {}),
command: NODE_AGENT_CLI_CLAUDE_RUN_COMMAND,
params: {
argv: params.argv,
src/gateway/node-connection-notifications.test.ts
+39 -8
View File
@@ -17,6 +17,7 @@ function node(
return {
nodeId,
connId: `conn-${nodeId}`,
pairingIdentity: `identity-${nodeId}`,
displayName: nodeId,
platform: "darwin",
commands: ["system.notify"],
@@ -25,9 +26,17 @@ function node(
} as NodeSession;
}
function registry<T extends object>(params: T): T {
testRegistries.push(params);
return params;
function registry<T extends { listConnected: () => NodeSession[] }>(params: T): T {
const value = params as T & {
listCurrentConnected?: () => Promise<NodeSession[]>;
isConnectionCurrentPairingState?: (connId: string) => Promise<boolean>;
};
const listCurrentConnected = value.listCurrentConnected ?? (async () => value.listConnected());
value.listCurrentConnected = listCurrentConnected;
value.isConnectionCurrentPairingState ??= async (connId) =>
(await listCurrentConnected()).some((entry) => entry.connId === connId);
testRegistries.push(value);
return value;
}
function schedule(registryValue: object, source: NodeSession): void {
@@ -152,7 +161,7 @@ describe("node connection notification routing", () => {
expect(invoke.mock.calls[0]?.[0]).toMatchObject({ nodeId: "desk" });
});
it("resolves the current source session when replacement races persistence", async () => {
it("drops a source alert when its exact connection is replaced without taking ownership", async () => {
vi.useFakeTimers();
const oldSource = node("new-node");
const replacement = {
@@ -171,11 +180,33 @@ describe("node connection notification routing", () => {
connected = [replacement, desk];
await vi.advanceTimersByTimeAsync(PRIMARY_DELAY_MS);
expect(invoke).toHaveBeenCalledTimes(1);
expect(invoke.mock.calls[0]?.[0]).toMatchObject({
nodeId: "desk",
params: { body: "Replacement Mac connected to OpenClaw." },
expect(invoke).not.toHaveBeenCalled();
});
it("drops a delayed alert when the source pairing generation is rotated", async () => {
vi.useFakeTimers();
const source = {
...node("new-node"),
pairingIdentity: "identity-a",
pairingGeneration: "generation-a",
};
const desk = node("desk", { lastActiveAtMs: 100 });
let currentPairingGeneration = "generation-a";
const invoke = vi.fn(async () => ({ ok: true }));
const registryValue = registry({
listConnected: () => [source, desk],
listCurrentConnected: async () =>
currentPairingGeneration === source.pairingGeneration ? [source, desk] : [desk],
isConnectionCurrentPairingState: async (connId: string) =>
connId === source.connId && currentPairingGeneration === source.pairingGeneration,
invoke,
});
schedule(registryValue, source);
currentPairingGeneration = "generation-b";
await vi.advanceTimersByTimeAsync(PRIMARY_DELAY_MS + FALLBACK_DELAY_MS);
expect(invoke).not.toHaveBeenCalled();
});
it("cancels the first-connection claim when the node is gone at delivery", async () => {
src/gateway/node-connection-notifications.ts
+56 -13
View File
@@ -4,7 +4,10 @@ import { normalizeOptionalString } from "@openclaw/normalization-core/string-coe
import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
import type { NodeRegistry, NodeSession } from "./node-registry.js";
type NotificationRegistry = Pick<NodeRegistry, "listConnected" | "invoke">;
type NotificationRegistry = Pick<
NodeRegistry,
"listCurrentConnected" | "isConnectionCurrentPairingState" | "invoke"
>;
type RouterOptions = {
primaryDelayMs?: number;
@@ -13,6 +16,9 @@ type RouterOptions = {
type PendingConnectionAlert = {
nodeId: string;
connId: string;
pairingIdentity?: string;
pairingGeneration?: string;
timer?: ReturnType<typeof setTimeout>;
};
@@ -64,7 +70,12 @@ class NodeConnectionNotificationRouter {
if (previous?.timer) {
clearTimeout(previous.timer);
}
const pending: PendingConnectionAlert = { nodeId: source.nodeId };
const pending: PendingConnectionAlert = {
nodeId: source.nodeId,
connId: source.connId,
pairingIdentity: source.pairingIdentity,
pairingGeneration: source.pairingGeneration,
};
this.pendingByNodeId.set(source.nodeId, pending);
this.armTimer(pending, this.primaryDelayMs, () => this.deliverPrimary(pending));
}
@@ -79,16 +90,17 @@ class NodeConnectionNotificationRouter {
}
private async deliverPrimary(pending: PendingConnectionAlert): Promise<void> {
const source = this.currentSource(pending);
const connected = await this.registry.listCurrentConnected();
const source = this.currentSource(pending, connected);
if (!source) {
this.finishAlert(pending);
return;
}
const primary = this.notificationTargets()
const primary = this.notificationTargets(connected)
.filter((node) => node.lastActiveAtMs !== undefined)
.toSorted(compareActivity)
.at(0);
const delivered = primary ? await this.notify(primary, source) : false;
const delivered = primary ? await this.notify(primary, source, pending) : false;
if (!this.attemptIsCurrent(pending)) {
return;
}
@@ -105,23 +117,35 @@ class NodeConnectionNotificationRouter {
pending: PendingConnectionAlert,
attemptedConnId?: string,
): Promise<void> {
const source = this.currentSource(pending);
const connected = await this.registry.listCurrentConnected();
const source = this.currentSource(pending, connected);
if (!source) {
this.finishAlert(pending);
return;
}
const targets = this.notificationTargets().filter((node) => node.connId !== attemptedConnId);
await Promise.all(targets.map(async (node) => await this.notify(node, source)));
const targets = this.notificationTargets(connected).filter(
(node) => node.connId !== attemptedConnId,
);
await Promise.all(targets.map(async (node) => await this.notify(node, source, pending)));
if (this.attemptIsCurrent(pending)) {
this.finishAlert(pending);
}
}
private currentSource(pending: PendingConnectionAlert): NodeSession | undefined {
private currentSource(
pending: PendingConnectionAlert,
connected: readonly NodeSession[],
): NodeSession | undefined {
if (!this.attemptIsCurrent(pending)) {
return undefined;
}
return this.registry.listConnected().find((node) => node.nodeId === pending.nodeId);
return connected.find(
(node) =>
node.nodeId === pending.nodeId &&
node.connId === pending.connId &&
node.pairingIdentity === pending.pairingIdentity &&
node.pairingGeneration === pending.pairingGeneration,
);
}
private attemptIsCurrent(pending: PendingConnectionAlert): boolean {
@@ -139,15 +163,34 @@ class NodeConnectionNotificationRouter {
}
}
private notificationTargets(): NodeSession[] {
return this.registry.listConnected().filter(isMacNotificationNode);
private notificationTargets(connected: readonly NodeSession[]): NodeSession[] {
return connected.filter(isMacNotificationNode);
}
private async notify(target: NodeSession, source: NodeSession): Promise<boolean> {
private async sourceIsCurrent(pending: PendingConnectionAlert): Promise<boolean> {
if (!this.attemptIsCurrent(pending)) {
return false;
}
const connected = await this.registry.listCurrentConnected();
if (!this.currentSource(pending, connected)) {
return false;
}
return await this.registry.isConnectionCurrentPairingState(pending.connId);
}
private async notify(
target: NodeSession,
source: NodeSession,
pending: PendingConnectionAlert,
): Promise<boolean> {
try {
if (!(await this.sourceIsCurrent(pending)) || !this.attemptIsCurrent(pending)) {
return false;
}
const result = await this.registry.invoke({
nodeId: target.nodeId,
expectedConnId: target.connId,
expectedPairingGeneration: target.pairingGeneration,
command: "system.notify",
params: {
title: "Node connected",
src/gateway/node-invoke-plugin-policy.test.ts
+47
View File
@@ -307,6 +307,53 @@ describe("applyPluginNodeInvokePolicy", () => {
expect(invoke).not.toHaveBeenCalled();
});
it("rejects plugin transport dispatch after invocation ownership changes", async () => {
setDangerousDemoCommandRegistry([
createDemoPolicy((ctx: OpenClawPluginNodeInvokePolicyContext) => ctx.invokeNode()),
]);
const { context, invoke } = createContext();
const result = await applyPluginNodeInvokePolicy({
context,
client: null,
nodeSession: createNodeSession(),
command: DEMO_COMMAND,
params: DEMO_PARAMS,
isInvocationCurrent: async () => false,
});
expect(result).toMatchObject({
ok: false,
code: "PAIRING_CHANGED",
details: { nodeCommandDispatched: false },
});
expect(invoke).not.toHaveBeenCalled();
});
it("rejects plugin transport dispatch through an invalidated node session", async () => {
setDangerousDemoCommandRegistry([
createDemoPolicy((ctx: OpenClawPluginNodeInvokePolicyContext) => ctx.invokeNode()),
]);
const nodeSession = createNodeSession();
nodeSession.client.invalidated = true;
const { context, invoke } = createContext({ nodeSession });
const result = await applyPluginNodeInvokePolicy({
context,
client: null,
nodeSession,
command: DEMO_COMMAND,
params: DEMO_PARAMS,
});
expect(result).toMatchObject({
ok: false,
code: "PAIRING_CHANGED",
details: { nodeCommandDispatched: false },
});
expect(invoke).not.toHaveBeenCalled();
});
it("overrides plugin dispatch claims with the actual pre-dispatch state", async () => {
setDangerousDemoCommandRegistry([
createDemoPolicy(async () => ({
src/gateway/node-invoke-plugin-policy.ts
+24 -1
View File
@@ -219,6 +219,7 @@ export async function applyPluginNodeInvokePolicy(params: {
};
timeoutMs?: number;
idempotencyKey?: string;
isInvocationCurrent?: () => boolean | Promise<boolean>;
}): Promise<OpenClawPluginNodeInvokePolicyResult | null> {
const registry = getActivePluginGatewayNodePolicyRegistry();
// Route metadata is authority-bearing: only a signed agent-runtime caller may nominate it.
@@ -247,7 +248,19 @@ export async function applyPluginNodeInvokePolicy(params: {
): Promise<OpenClawPluginNodeInvokeTransportResult> => {
// Policies invoke the real node through this narrowed transport wrapper so
// they can retry/override params without getting direct registry access.
const currentNode = params.context.nodeRegistry.get(params.nodeSession.nodeId);
if (params.isInvocationCurrent && !(await params.isInvocationCurrent())) {
return {
ok: false,
code: "PAIRING_CHANGED",
message: "node pairing changed before dispatch",
};
}
const currentNode = params.nodeSession.pairingGeneration
? params.context.nodeRegistry.getForPairingGeneration(
params.nodeSession.nodeId,
params.nodeSession.pairingGeneration,
)
: params.context.nodeRegistry.get(params.nodeSession.nodeId);
if (!currentNode || currentNode.connId !== params.nodeSession.connId) {
return {
ok: false,
@@ -255,6 +268,13 @@ export async function applyPluginNodeInvokePolicy(params: {
message: "node connection changed before dispatch",
};
}
if (currentNode.client.invalidated === true) {
return {
ok: false,
code: "PAIRING_CHANGED",
message: "node pairing changed before dispatch",
};
}
const currentConfig = params.context.getRuntimeConfig();
const allowlist = resolveNodeCommandAllowlist(currentConfig, {
...currentNode,
@@ -279,6 +299,9 @@ export async function applyPluginNodeInvokePolicy(params: {
const res = await params.context.nodeRegistry.invoke({
nodeId: params.nodeSession.nodeId,
expectedConnId: params.nodeSession.connId,
...(params.nodeSession.pairingGeneration
? { expectedPairingGeneration: params.nodeSession.pairingGeneration }
: {}),
command: params.command,
params: override.params ?? params.params,
timeoutMs: override.timeoutMs ?? params.timeoutMs,
src/gateway/node-pending-work.test.ts
+97 -1
View File
@@ -2,7 +2,12 @@
* Node pending-work tracking tests.
*/
import { describe, expect, it, vi } from "vitest";
import { drainNodePendingWork, enqueueNodePendingWork } from "./node-pending-work.js";
import {
clearNodePendingWork,
drainNodePendingWork,
enqueueNodePendingWork,
removeNodePendingWorkItem,
} from "./node-pending-work.js";
describe("node pending work", () => {
it("returns a baseline status request even when no explicit work is queued", () => {
@@ -33,6 +38,97 @@ describe("node pending work", () => {
drainNodePendingWork("node-2");
});
it("clears explicit work when a pairing is removed", () => {
enqueueNodePendingWork({ nodeId: "node-removed", type: "location.request" });
expect(clearNodePendingWork(" node-removed ")).toBe(true);
expect(drainNodePendingWork("node-removed").items.map((item) => item.id)).toEqual([
"baseline-status",
]);
expect(clearNodePendingWork("node-removed")).toBe(false);
});
it("keeps explicit work isolated from a replacement pairing generation", () => {
enqueueNodePendingWork({
nodeId: "node-replaced",
type: "location.request",
pairingGeneration: "generation-1",
});
const drained = drainNodePendingWork("node-replaced", {
pairingGeneration: "generation-2",
});
expect(drained.items.map((item) => item.id)).toEqual(["baseline-status"]);
expect(
drainNodePendingWork("node-replaced", { pairingGeneration: "generation-1" }).items.map(
(item) => item.type,
),
).toEqual(["location.request", "status.request"]);
});
it("prunes expired work from retired generations on current-generation access", () => {
enqueueNodePendingWork({
nodeId: "node-retired-generation",
type: "location.request",
expiresInMs: 1_000,
pairingGeneration: "generation-1",
});
drainNodePendingWork("node-retired-generation", {
pairingGeneration: "generation-2",
nowMs: Date.now() + 2_000,
});
expect(clearNodePendingWork("node-retired-generation", "generation-1")).toBe(false);
});
it("does not let a stale drain delete replacement-generation work", () => {
enqueueNodePendingWork({
nodeId: "node-stale-drain",
type: "location.request",
pairingGeneration: "generation-2",
});
expect(
drainNodePendingWork("node-stale-drain", { pairingGeneration: "generation-1" }).items.map(
(item) => item.id,
),
).toEqual(["baseline-status"]);
expect(
drainNodePendingWork("node-stale-drain", { pairingGeneration: "generation-2" }).items.map(
(item) => item.type,
),
).toEqual(["location.request", "status.request"]);
});
it("rolls back only the exact item owned by one enqueue", () => {
const location = enqueueNodePendingWork({
nodeId: "node-item-rollback",
type: "location.request",
pairingGeneration: "generation-1",
});
enqueueNodePendingWork({
nodeId: "node-item-rollback",
type: "status.request",
pairingGeneration: "generation-1",
});
expect(
removeNodePendingWorkItem({
nodeId: "node-item-rollback",
itemId: location.item.id,
pairingGeneration: "generation-1",
}),
).toBe(true);
expect(
drainNodePendingWork("node-item-rollback", {
pairingGeneration: "generation-1",
includeDefaultStatus: false,
}).items.map((item) => item.type),
).toEqual(["status.request"]);
});
it("keeps hasMore true when the baseline status item is deferred by maxItems", () => {
enqueueNodePendingWork({ nodeId: "node-3", type: "location.request" });
src/gateway/node-pending-work.ts
+90 -10
View File
@@ -30,12 +30,14 @@ type NodePendingWorkItem = {
type NodePendingWorkState = {
revision: number;
itemsById: Map<string, NodePendingWorkItem>;
pairingGeneration?: string;
};
type DrainOptions = {
maxItems?: number;
includeDefaultStatus?: boolean;
nowMs?: number;
pairingGeneration?: string;
};
type DrainResult = {
@@ -56,16 +58,22 @@ const PRIORITY_RANK: Record<NodePendingWorkPriority, number> = {
default: 1,
};
const stateByNodeId = new Map<string, NodePendingWorkState>();
const stateByNodeId = new Map<string, Map<string | undefined, NodePendingWorkState>>();
function getOrCreateState(nodeId: string): NodePendingWorkState {
let state = stateByNodeId.get(nodeId);
function getOrCreateState(nodeId: string, pairingGeneration?: string): NodePendingWorkState {
let states = stateByNodeId.get(nodeId);
if (!states) {
states = new Map();
stateByNodeId.set(nodeId, states);
}
let state = states.get(pairingGeneration);
if (!state) {
state = {
revision: 0,
itemsById: new Map(),
...(pairingGeneration ? { pairingGeneration } : {}),
};
stateByNodeId.set(nodeId, state);
states.set(pairingGeneration, state);
}
return state;
}
@@ -92,12 +100,43 @@ function pruneExpired(state: NodePendingWorkState, nowMs: number): boolean {
return changed;
}
function pruneStateIfEmpty(nodeId: string, state: NodePendingWorkState) {
if (state.itemsById.size === 0) {
function pruneExpiredRetiredGenerations(
nodeId: string,
currentPairingGeneration: string | undefined,
nowMs: number,
): void {
const states = stateByNodeId.get(nodeId);
if (!states) {
return;
}
for (const [pairingGeneration, state] of states) {
if (pairingGeneration === currentPairingGeneration) {
continue;
}
pruneExpired(state, nowMs);
if (state.itemsById.size === 0) {
states.delete(pairingGeneration);
}
}
if (states.size === 0) {
stateByNodeId.delete(nodeId);
}
}
function pruneStateIfEmpty(
nodeId: string,
pairingGeneration: string | undefined,
state: NodePendingWorkState,
) {
if (state.itemsById.size === 0) {
const states = stateByNodeId.get(nodeId);
states?.delete(pairingGeneration);
if (states?.size === 0) {
stateByNodeId.delete(nodeId);
}
}
}
function sortedItems(state: NodePendingWorkState): NodePendingWorkItem[] {
// Higher priority wins, then older work, then id for deterministic paging.
return [...state.itemsById.values()].toSorted((a, b) => {
@@ -136,6 +175,7 @@ export function enqueueNodePendingWork(params: {
priority?: NodePendingWorkPriority;
expiresInMs?: number;
payload?: Record<string, unknown>;
pairingGeneration?: string;
}): { revision: number; item: NodePendingWorkItem; deduped: boolean } {
const nodeId = params.nodeId.trim();
if (!nodeId) {
@@ -143,7 +183,10 @@ export function enqueueNodePendingWork(params: {
}
const rawNowMs = Date.now();
const nowMs = resolveDateTimestampMs(rawNowMs);
const state = getOrCreateState(nodeId);
// Generation changes stop touching old buckets, so sweep their TTLs from
// every active-generation access instead of retaining retired work forever.
pruneExpiredRetiredGenerations(nodeId, params.pairingGeneration, nowMs);
const state = getOrCreateState(nodeId, params.pairingGeneration);
pruneExpired(state, nowMs);
// Keep one outstanding item per type so repeated status/location requests
// collapse until the node has a chance to drain them.
@@ -164,6 +207,42 @@ export function enqueueNodePendingWork(params: {
return { revision: state.revision, item, deduped: false };
}
/** Clears explicit pending work owned by a removed node pairing. */
export function clearNodePendingWork(nodeId: string, pairingGeneration?: string): boolean {
const normalizedNodeId = nodeId.trim();
if (!normalizedNodeId) {
return false;
}
if (pairingGeneration === undefined) {
return stateByNodeId.delete(normalizedNodeId);
}
const states = stateByNodeId.get(normalizedNodeId);
const deleted = states?.delete(pairingGeneration) ?? false;
if (states?.size === 0) {
stateByNodeId.delete(normalizedNodeId);
}
return deleted;
}
/** Removes one exact item without disturbing concurrent work in the same generation. */
export function removeNodePendingWorkItem(params: {
nodeId: string;
itemId: string;
pairingGeneration?: string;
}): boolean {
const normalizedNodeId = params.nodeId.trim();
if (!normalizedNodeId || !params.itemId) {
return false;
}
const state = stateByNodeId.get(normalizedNodeId)?.get(params.pairingGeneration);
if (!state || !state.itemsById.delete(params.itemId)) {
return false;
}
state.revision += 1;
pruneStateIfEmpty(normalizedNodeId, params.pairingGeneration, state);
return true;
}
/** Drains pending work for a node, including a baseline status request unless disabled. */
export function drainNodePendingWork(nodeId: string, opts: DrainOptions = {}): DrainResult {
const normalizedNodeId = nodeId.trim();
@@ -171,10 +250,11 @@ export function drainNodePendingWork(nodeId: string, opts: DrainOptions = {}): D
return { revision: 0, items: [], hasMore: false };
}
const nowMs = resolveDateTimestampMs(opts.nowMs ?? Date.now());
const state = stateByNodeId.get(normalizedNodeId);
pruneExpiredRetiredGenerations(normalizedNodeId, opts.pairingGeneration, nowMs);
const state = stateByNodeId.get(normalizedNodeId)?.get(opts.pairingGeneration);
if (state) {
pruneExpired(state, nowMs);
pruneStateIfEmpty(normalizedNodeId, state);
pruneStateIfEmpty(normalizedNodeId, opts.pairingGeneration, state);
}
const revision = state?.revision ?? 0;
const maxItems = Math.min(MAX_ITEMS, Math.max(1, Math.trunc(opts.maxItems ?? DEFAULT_MAX_ITEMS)));
@@ -194,7 +274,7 @@ export function drainNodePendingWork(nodeId: string, opts: DrainOptions = {}): D
}
}
state.revision += 1;
pruneStateIfEmpty(normalizedNodeId, state);
pruneStateIfEmpty(normalizedNodeId, opts.pairingGeneration, state);
}
return {
revision: state?.revision ?? revision,
src/gateway/node-registry.test.ts
+566 -40
View File
@@ -7,7 +7,7 @@ import {
MAX_TIMER_TIMEOUT_MS,
} from "@openclaw/normalization-core/number-coercion";
import { afterEach, describe, expect, it, vi } from "vitest";
import { getActiveNodeContext, setActiveNodeContext } from "../infra/active-node-context.js";
import { getCurrentActiveNodeContext, setActiveNodeContext } from "../infra/active-node-context.js";
import { onDiagnosticEvent, resetDiagnosticEventsForTest } from "../infra/diagnostic-events.js";
import { createEmptyPluginRegistry } from "../plugins/registry-empty.js";
import { listConnectedNodePluginTools } from "./node-plugin-tool-snapshot.js";
@@ -98,6 +98,15 @@ function makeClient(
};
}
function registerNodeSession(
registry: NodeRegistry,
client: GatewayWsClient,
opts: Partial<Parameters<NodeRegistry["register"]>[1]> = {},
) {
const { pairingIdentity = "identity-a", ...registration } = opts;
return registry.register(client, { ...registration, pairingIdentity });
}
function registerDemoNodePluginTool(params: {
name: string;
command: string;
@@ -151,7 +160,7 @@ function makeConnectivitySocket(emitPong: boolean) {
function registerNode(registry: NodeRegistry, opts: Parameters<typeof makeClient>[3] = {}) {
const frames: string[] = [];
registry.register(makeClient("conn-1", "node-1", frames, opts), {});
registerNodeSession(registry, makeClient("conn-1", "node-1", frames, opts), {});
return frames;
}
@@ -218,6 +227,334 @@ function authorizeSystemRun(registry: NodeRegistry, overrides: Partial<SystemRun
}
describe("gateway/node-registry", () => {
it("rejects registration without an authenticated pairing identity", () => {
const registry = new NodeRegistry();
const client = makeClient("conn-unbound", "node-unbound");
expect(() => registry.register(client, {} as never)).toThrow(
"node session registration requires pairing identity",
);
expect(registry.listConnected()).toEqual([]);
});
it("rejects dispatch through an invalidated node connection", async () => {
const registry = new NodeRegistry();
const frames: string[] = [];
const client = makeClient("conn-invalidated", "node-invalidated", frames);
registerNodeSession(registry, client, {});
client.invalidated = true;
expect(registry.get("node-invalidated")).toBeUndefined();
expect(registry.listConnected()).toEqual([]);
expect(registry.sendEvent("node-invalidated", "node.test", { ok: true })).toBe(false);
await expect(
registry.invoke({ nodeId: "node-invalidated", command: "system.run" }),
).resolves.toMatchObject({
ok: false,
error: { code: "PAIRING_CHANGED" },
});
expect(frames).toEqual([]);
});
it("rejects generation-mismatched lookup and dispatch without invalidating the session", async () => {
const registry = new NodeRegistry();
const frames: string[] = [];
const client = makeClient("conn-old-generation", "node-generation", frames);
registerNodeSession(registry, client, { pairingGeneration: "generation-a" });
expect(registry.getForPairingGeneration("node-generation", "generation-b")).toBeUndefined();
expect(client.invalidated).not.toBe(true);
await expect(
registry.invoke({
nodeId: "node-generation",
expectedPairingGeneration: "generation-b",
command: "system.run",
}),
).resolves.toMatchObject({
ok: false,
error: { code: "PAIRING_CHANGED" },
});
expect(client.invalidated).not.toBe(true);
expect(frames).toEqual([]);
});
it("does not let a stale operation invalidate the valid replacement generation", () => {
const registry = new NodeRegistry();
const frames: string[] = [];
const replacement = makeClient("conn-replacement", "node-replaced", frames);
registerNodeSession(registry, replacement, { pairingGeneration: "generation-b" });
expect(registry.getForPairingGeneration("node-replaced", "generation-a")).toBeUndefined();
expect(replacement.invalidated).not.toBe(true);
expect(registry.getForPairingGeneration("node-replaced", "generation-b")?.connId).toBe(
"conn-replacement",
);
});
it("revalidates the persistent generation immediately before dispatch", async () => {
const frames: string[] = [];
const resolveCurrentPairingState = vi.fn().mockResolvedValue({
identity: "identity-a",
generation: "generation-b",
});
const registry = new NodeRegistry({ resolveCurrentPairingState });
const client = makeClient("conn-generation", "node-generation", frames);
registerNodeSession(registry, client, {
pairingIdentity: "identity-a",
pairingGeneration: "generation-a",
});
await expect(
registry.invoke({
nodeId: "node-generation",
expectedConnId: "conn-generation",
expectedPairingGeneration: "generation-a",
command: "system.run",
}),
).resolves.toMatchObject({
ok: false,
error: { code: "PAIRING_CHANGED" },
});
expect(resolveCurrentPairingState).toHaveBeenCalledWith("node-generation");
expect(frames).toEqual([]);
});
it("revalidates persistent generation ownership for inbound node RPCs", async () => {
const resolveCurrentPairingState = vi.fn().mockResolvedValue({
identity: "identity-a",
generation: "generation-a",
});
const registry = createNodeRegistry({ resolveCurrentPairingState });
const client = makeClient("conn-generation", "node-generation");
registerNodeSession(registry, client, {
pairingIdentity: "identity-a",
pairingGeneration: "generation-a",
});
await expect(registry.isConnectionCurrentPairingState("conn-generation")).resolves.toBe(true);
resolveCurrentPairingState.mockResolvedValue({
identity: "identity-a",
generation: "generation-b",
});
await expect(registry.isConnectionCurrentPairingState("conn-generation")).resolves.toBe(false);
expect(client.invalidated).toBe(true);
expect(resolveCurrentPairingState).toHaveBeenCalledWith("node-generation");
});
it("removes an externally replaced session from connected and active projections", async () => {
let currentPairingGeneration = "generation-a";
const onPairingInvalidated = vi.fn();
const registry = createNodeRegistry({
resolveCurrentPairingState: async () => ({
identity: "identity-a",
generation: currentPairingGeneration,
}),
onPairingInvalidated,
});
const client = makeClient("conn-generation", "node-generation", [], {
permissions: { accessibility: true },
});
registerNodeSession(registry, client, {
pairingIdentity: "identity-a",
pairingGeneration: "generation-a",
});
registry.updatePresenceActivity({
nodeId: "node-generation",
connId: "conn-generation",
idleSeconds: 0,
});
expect(registry.getActiveNode()?.nodeId).toBe("node-generation");
currentPairingGeneration = "generation-b";
await expect(registry.listCurrentConnected()).resolves.toEqual([]);
expect(registry.getActiveNode()).toBeUndefined();
expect(getCurrentActiveNodeContext()).toBeNull();
expect(client.invalidated).toBe(true);
expect(onPairingInvalidated).toHaveBeenCalledWith({
nodeId: "node-generation",
connId: "conn-generation",
});
});
it("does not invalidate a session promoted while persistent generation is loading", async () => {
let resolveLookup: ((value: { identity: string; generation: string }) => void) | undefined;
const resolveCurrentPairingState = vi.fn(
() =>
new Promise<{ identity: string; generation: string }>((resolve) => {
resolveLookup = resolve;
}),
);
const onPairingInvalidated = vi.fn();
const registry = createNodeRegistry({
resolveCurrentPairingState,
onPairingInvalidated,
});
const client = makeClient("conn-generation", "node-generation");
registerNodeSession(registry, client, {
pairingIdentity: "identity-a",
pairingGeneration: "generation-a",
});
const connected = registry.listCurrentConnected();
expect(resolveCurrentPairingState).toHaveBeenCalledWith("node-generation");
expect(
registry.updateSurface(
"node-generation",
{ commands: [] },
{
expectedConnId: "conn-generation",
expectedPairingIdentity: "identity-a",
expectedPairingGeneration: "generation-a",
nextPairingGeneration: "generation-b",
},
),
).not.toBeNull();
resolveLookup?.({ identity: "identity-a", generation: "generation-a" });
await expect(connected).resolves.toEqual([]);
expect(registry.get("node-generation")?.pairingGeneration).toBe("generation-b");
expect(client.invalidated).not.toBe(true);
expect(onPairingInvalidated).not.toHaveBeenCalled();
});
it("revalidates the active node at the prompt projection boundary", () => {
let currentPairingGeneration = "generation-a";
const registry = createNodeRegistry({
isPairingStateCurrent: (_nodeId, expected) =>
expected.identity === "identity-a" && expected.generation === currentPairingGeneration,
});
registerNodeSession(
registry,
makeClient("conn-generation", "node-generation", [], {
permissions: { accessibility: true },
}),
{ pairingIdentity: "identity-a", pairingGeneration: "generation-a" },
);
registry.updatePresenceActivity({
nodeId: "node-generation",
connId: "conn-generation",
idleSeconds: 0,
});
expect(getCurrentActiveNodeContext()).toMatchObject({
nodeId: "node-generation",
pairingGeneration: "generation-a",
});
currentPairingGeneration = "generation-b";
expect(getCurrentActiveNodeContext()).toBeNull();
});
it("filters an already-loaded pairing snapshot without invalidating a newer session", () => {
const registry = createNodeRegistry();
const client = makeClient("conn-generation", "node-generation");
registerNodeSession(registry, client, {
pairingIdentity: "identity-b",
pairingGeneration: "generation-b",
});
expect(
registry.listConnectedForPairingStates(
new Map([["node-generation", { identity: "identity-a", generation: "generation-a" }]]),
),
).toEqual([]);
expect(client.invalidated).not.toBe(true);
});
it("distinguishes a pending surface from a missing paired-device row", () => {
const registry = createNodeRegistry();
const client = makeClient("conn-pending-surface", "node-pending-surface");
registerNodeSession(registry, client, { pairingIdentity: "identity-a" });
expect(
registry.listConnectedForPairingStates(
new Map([["node-pending-surface", { identity: "identity-a" }]]),
),
).toHaveLength(1);
expect(registry.listConnectedForPairingStates(new Map())).toEqual([]);
expect(client.invalidated).not.toBe(true);
});
it("fails closed without invalidating a session when pairing persistence is unavailable", async () => {
const registry = createNodeRegistry({
resolveCurrentPairingState: async () => {
throw new Error("pairing store unavailable");
},
});
const client = makeClient("conn-generation", "node-generation");
registerNodeSession(registry, client, {
pairingIdentity: "identity-a",
pairingGeneration: "generation-a",
});
await expect(registry.listCurrentConnected()).resolves.toEqual([]);
expect(client.invalidated).not.toBe(true);
expect(registry.listConnected()).toHaveLength(1);
});
it("reconciles stale persistent generations synchronously for prompt projections", () => {
let currentPairingGeneration = "generation-a";
const onPairingInvalidated = vi.fn();
const registry = createNodeRegistry({
isPairingStateCurrent: (_nodeId, expected) =>
expected.generation === currentPairingGeneration,
onPairingInvalidated,
});
const client = makeClient("conn-generation", "node-generation");
registerNodeSession(registry, client, { pairingGeneration: "generation-a" });
expect(registry.listCurrentConnectedSync()).toHaveLength(1);
currentPairingGeneration = "generation-b";
expect(registry.listCurrentConnectedSync()).toEqual([]);
expect(client.invalidated).toBe(true);
expect(onPairingInvalidated).toHaveBeenCalledWith({
nodeId: "node-generation",
connId: "conn-generation",
});
});
it("fails closed synchronously when pairing persistence is unavailable", () => {
const registry = createNodeRegistry({
isPairingStateCurrent: () => {
throw new Error("pairing store unavailable");
},
});
const client = makeClient("conn-generation", "node-generation");
registerNodeSession(registry, client, { pairingGeneration: "generation-a" });
expect(registry.listCurrentConnectedSync()).toEqual([]);
expect(client.invalidated).not.toBe(true);
expect(registry.listConnected()).toHaveLength(1);
});
it("invalidates a generation-less session after external pairing deletion", async () => {
let currentPairingState: { identity: string } | undefined = { identity: "identity-a" };
const registry = createNodeRegistry({
resolveCurrentPairingState: async () => currentPairingState,
});
const client = makeClient("conn-pending-surface", "node-pending-surface");
registerNodeSession(registry, client, { pairingIdentity: "identity-a" });
await expect(registry.listCurrentConnected()).resolves.toHaveLength(1);
currentPairingState = undefined;
await expect(registry.listCurrentConnected()).resolves.toEqual([]);
expect(client.invalidated).toBe(true);
});
it("invalidates a generation-less session synchronously after pairing deletion", () => {
let pairingExists = true;
const registry = createNodeRegistry({
isPairingStateCurrent: (_nodeId, expected) =>
pairingExists && expected.identity === "identity-a",
});
const client = makeClient("conn-pending-surface", "node-pending-surface");
registerNodeSession(registry, client, { pairingIdentity: "identity-a" });
expect(registry.listCurrentConnectedSync()).toHaveLength(1);
pairingExists = false;
expect(registry.listCurrentConnectedSync()).toEqual([]);
expect(client.invalidated).toBe(true);
});
it("routes ordered input to the pending invoke connection and rejects unknown invokes", async () => {
const registry = new NodeRegistry();
const frames = registerNode(registry);
@@ -297,14 +634,16 @@ describe("gateway/node-registry", () => {
it("ranks connected nodes by gateway-derived input activity", () => {
const registry = createTestNodeRegistry();
registry.register(
registerNodeSession(
registry,
makeClient("conn-1", "node-1", [], {
displayName: "Desk Mac",
permissions: { accessibility: true },
}),
{},
);
registry.register(
registerNodeSession(
registry,
makeClient("conn-2", "node-2", [], {
displayName: "Laptop",
permissions: { accessibility: true },
@@ -328,15 +667,16 @@ describe("gateway/node-registry", () => {
});
expect(registry.getActiveNode()?.nodeId).toBe("node-2");
expect(getActiveNodeContext()).toEqual({ nodeId: "node-2" });
expect(getCurrentActiveNodeContext()).toEqual({ nodeId: "node-2" });
expect(registry.unregister("conn-2")).toBe("node-2");
expect(registry.getActiveNode()?.nodeId).toBe("node-1");
expect(getActiveNodeContext()).toEqual({ nodeId: "node-1" });
expect(getCurrentActiveNodeContext()).toEqual({ nodeId: "node-1" });
});
it("recomputes active context when a same-id connection replaces reported presence", () => {
const registry = createTestNodeRegistry();
registry.register(
registerNodeSession(
registry,
makeClient("conn-old", "node-1", [], { permissions: { accessibility: true } }),
{},
);
@@ -347,20 +687,22 @@ describe("gateway/node-registry", () => {
observedAtMs: 100_000,
});
registry.register(
registerNodeSession(
registry,
makeClient("conn-new", "node-1", [], { permissions: { accessibility: true } }),
{},
);
expect(registry.getActiveNode()).toBeUndefined();
expect(getActiveNodeContext()).toBeNull();
expect(getCurrentActiveNodeContext()).toBeNull();
expect(registry.unregister("conn-old")).toBeNull();
expect(getActiveNodeContext()).toBeNull();
expect(getCurrentActiveNodeContext()).toBeNull();
});
it("rejects presence updates from stale node connections", () => {
const registry = createTestNodeRegistry();
registry.register(
registerNodeSession(
registry,
makeClient("conn-new", "node-1", [], { permissions: { accessibility: true } }),
{},
);
@@ -378,7 +720,8 @@ describe("gateway/node-registry", () => {
it("does not advance a bounded estimate on saturated idle keepalives", () => {
const registry = createTestNodeRegistry();
registry.register(
registerNodeSession(
registry,
makeClient("conn-1", "node-1", [], { permissions: { accessibility: true } }),
{},
);
@@ -404,7 +747,8 @@ describe("gateway/node-registry", () => {
it("clears reported presence when Accessibility permission is removed", () => {
const registry = createTestNodeRegistry();
registry.register(
registerNodeSession(
registry,
makeClient("conn-1", "node-1", [], {
permissions: { accessibility: true },
declaredPermissions: { accessibility: true },
@@ -423,12 +767,13 @@ describe("gateway/node-registry", () => {
expect(registry.get("node-1")?.lastActiveAtMs).toBeUndefined();
expect(registry.get("node-1")?.presenceUpdatedAtMs).toBeUndefined();
expect(registry.getActiveNode()).toBeUndefined();
expect(getActiveNodeContext()).toBeNull();
expect(getCurrentActiveNodeContext()).toBeNull();
});
it("checks node websocket connectivity with ping/pong", async () => {
const registry = createTestNodeRegistry();
registry.register(
registerNodeSession(
registry,
makeClient("conn-1", "node-1", [], {
socket: makeConnectivitySocket(true),
}),
@@ -440,7 +785,8 @@ describe("gateway/node-registry", () => {
it("reports stale node websocket connectivity before invoke timeout", async () => {
const registry = createTestNodeRegistry();
registry.register(
registerNodeSession(
registry,
makeClient("conn-1", "node-1", [], {
socket: makeConnectivitySocket(false),
}),
@@ -460,7 +806,7 @@ describe("gateway/node-registry", () => {
const oldFrames: string[] = [];
const newClient = makeClient("conn-new", "node-1");
registry.register(makeClient("conn-old", "node-1", oldFrames), {});
registerNodeSession(registry, makeClient("conn-old", "node-1", oldFrames), {});
const oldInvoke = registry.invoke({
nodeId: "node-1",
command: "system.run",
@@ -468,7 +814,7 @@ describe("gateway/node-registry", () => {
});
const oldDisconnected = oldInvoke.catch((err: unknown) => err);
const oldRequest = JSON.parse(oldFrames[0] ?? "{}") as { payload?: { id?: string } };
const newSession = registry.register(newClient, {});
const newSession = registerNodeSession(registry, newClient, {});
expect(
registry.handleInvokeResult({
@@ -486,8 +832,8 @@ describe("gateway/node-registry", () => {
it("rejects invoke when the node connection changed before dispatch", async () => {
const registry = createNodeRegistry();
const replacementFrames: string[] = [];
registry.register(makeClient("conn-old", "node-1"), {});
registry.register(makeClient("conn-new", "node-1", replacementFrames), {});
registerNodeSession(registry, makeClient("conn-old", "node-1"), {});
registerNodeSession(registry, makeClient("conn-new", "node-1", replacementFrames), {});
await expect(
registry.invoke({
@@ -1270,7 +1616,7 @@ describe("gateway/node-registry", () => {
it("sends raw event payload JSON without changing the envelope shape", () => {
const registry = createTestNodeRegistry();
const frames: string[] = [];
registry.register(makeClient("conn-1", "node-1", frames), {});
registerNodeSession(registry, makeClient("conn-1", "node-1", frames), {});
const payload = serializeEventPayload({ foo: "bar" });
const nullPayload = serializeEventPayload(null);
const falsePayload = serializeEventPayload(false);
@@ -1309,6 +1655,60 @@ describe("gateway/node-registry", () => {
]);
});
it("drops a delayed voice-wake snapshot after persistent generation changes", async () => {
let resolveCurrent!: (state: { identity: string; generation?: string } | undefined) => void;
const currentPairingState = new Promise<{ identity: string; generation?: string } | undefined>(
(resolve) => {
resolveCurrent = resolve;
},
);
const resolveCurrentPairingState = vi.fn(() => currentPairingState);
const registry = createNodeRegistry({ resolveCurrentPairingState });
const frames: string[] = [];
registerNodeSession(registry, makeClient("conn-1", "node-1", frames), {
pairingIdentity: "identity-a",
pairingGeneration: "generation-a",
});
const send = registry.sendEventRawForPairingGeneration(
"node-1",
"generation-a",
"voicewake.changed",
serializeEventPayload({ triggers: ["openclaw"] }),
);
await vi.waitFor(() => expect(resolveCurrentPairingState).toHaveBeenCalledTimes(1));
resolveCurrent({ identity: "identity-a", generation: "generation-b" });
await expect(send).resolves.toBe(false);
expect(frames).toEqual([]);
});
it("drops a delayed command-free snapshot after pairing identity deletion", async () => {
let resolveCurrent!: (state: { identity: string } | undefined) => void;
const currentPairingState = new Promise<{ identity: string } | undefined>((resolve) => {
resolveCurrent = resolve;
});
const registry = createNodeRegistry({
resolveCurrentPairingState: async () => await currentPairingState,
});
const frames: string[] = [];
registerNodeSession(registry, makeClient("conn-1", "node-1", frames), {
pairingIdentity: "identity-a",
});
const send = registry.sendEventForPairingIdentity({
nodeId: "node-1",
connId: "conn-1",
pairingIdentity: "identity-a",
event: "voicewake.changed",
payload: { triggers: ["openclaw"] },
});
resolveCurrent(undefined);
await expect(send).resolves.toBe(false);
expect(frames).toEqual([]);
});
it("rejects raw event sends when the node socket buffer is saturated", () => {
resetDiagnosticEventsForTest();
const diagnosticEvents: unknown[] = [];
@@ -1319,7 +1719,8 @@ describe("gateway/node-registry", () => {
send: vi.fn(),
close: vi.fn(),
};
registry.register(
registerNodeSession(
registry,
makeClient("conn-1", "node-1", [], {
socket: socket as unknown as GatewayWsClient["socket"],
}),
@@ -1359,7 +1760,7 @@ describe("gateway/node-registry", () => {
declaredPermissions: { microphone: true, camera: false },
});
const session = registry.register(client, {});
const session = registerNodeSession(registry, client, {});
expect(session.caps).toEqual([]);
expect(session.commands).toEqual([]);
@@ -1376,6 +1777,103 @@ describe("gateway/node-registry", () => {
expect((client.connect as { commands?: string[] }).commands).toEqual(["talk.ptt.start"]);
});
it("advances the exact live session with its approved surface generation", () => {
const registry = createTestNodeRegistry();
const client = makeClient("conn-1", "node-1", [], {
declaredCommands: ["device.info"],
});
registerNodeSession(registry, client, {
pairingIdentity: "identity-a",
pairingGeneration: "generation-a",
});
const updated = registry.updateSurface(
"node-1",
{ commands: ["device.info"] },
{
expectedConnId: "conn-1",
expectedPairingIdentity: "identity-a",
expectedPairingGeneration: "generation-a",
nextPairingGeneration: "generation-b",
},
);
expect(updated?.pairingGeneration).toBe("generation-b");
expect(
registry.updateSurface(
"node-1",
{ commands: [] },
{
expectedConnId: "conn-stale",
expectedPairingIdentity: "identity-a",
expectedPairingGeneration: "generation-a",
nextPairingGeneration: "generation-c",
},
),
).toBeNull();
expect(registry.get("node-1")?.commands).toEqual(["device.info"]);
expect(registry.get("node-1")?.pairingGeneration).toBe("generation-b");
});
it("rebinds active-node presence when a live session advances generations", () => {
const registry = createTestNodeRegistry();
const client = makeClient("conn-1", "node-1", [], {
permissions: { accessibility: true },
declaredPermissions: { accessibility: true },
});
registerNodeSession(registry, client, {
pairingIdentity: "identity-a",
pairingGeneration: "generation-a",
});
registry.updatePresenceActivity({
nodeId: "node-1",
connId: "conn-1",
idleSeconds: 0,
observedAtMs: 100_000,
});
registry.updateSurface(
"node-1",
{ commands: [], permissions: { accessibility: true } },
{
expectedConnId: "conn-1",
expectedPairingIdentity: "identity-a",
expectedPairingGeneration: "generation-a",
nextPairingGeneration: "generation-b",
},
);
expect(getCurrentActiveNodeContext()).toMatchObject({
nodeId: "node-1",
pairingGeneration: "generation-b",
});
});
it("does not promote a generation-less session from a retired pairing identity", () => {
const registry = createTestNodeRegistry();
const client = makeClient("conn-1", "node-1", [], {
declaredCommands: ["device.info"],
});
registerNodeSession(registry, client, { pairingIdentity: "identity-a" });
expect(
registry.updateSurface(
"node-1",
{ commands: ["device.info"] },
{
expectedConnId: "conn-1",
expectedPairingIdentity: "identity-b",
nextPairingGeneration: "generation-b",
},
),
).toBeNull();
expect(registry.get("node-1")).toMatchObject({
pairingIdentity: "identity-a",
commands: [],
});
expect(registry.get("node-1")?.pairingGeneration).toBeUndefined();
});
it("keeps node-hosted plugin tools inside the approved command surface", () => {
registerDemoNodePluginTool({ name: "demo_echo", command: "demo.echo" });
const registry = createTestNodeRegistry();
@@ -1383,7 +1881,7 @@ describe("gateway/node-registry", () => {
commands: ["demo.echo"],
});
const session = registry.register(client, {});
const session = registerNodeSession(registry, client, {});
publishNodePluginTools(registry, [
{
pluginId: "demo",
@@ -1413,6 +1911,30 @@ describe("gateway/node-registry", () => {
expect(listConnectedNodePluginTools()).toEqual([]);
});
it("retires node-hosted plugin tools immediately when a connection is invalidated", () => {
registerDemoNodePluginTool({ name: "demo_echo", command: "demo.echo" });
const registry = createTestNodeRegistry();
const client = makeClient("conn-1", "node-1", [], { commands: ["demo.echo"] });
registerNodeSession(registry, client, {});
publishNodePluginTools(registry, [
{
pluginId: "demo",
name: "demo_echo",
description: "Echo through the node",
command: "demo.echo",
},
]);
expect(listConnectedNodePluginTools()).toHaveLength(1);
expect(registry.invalidateConnectionForPairingChange("conn-1", "device-token-revoked")).toBe(
true,
);
expect(client.invalidated).toBe(true);
expect(client.invalidatedReason).toBe("device-token-revoked");
expect(listConnectedNodePluginTools()).toEqual([]);
});
it("keeps dangerous node-hosted plugin tools once explicitly approved", () => {
registerDemoNodePluginTool({
name: "demo_dangerous",
@@ -1424,7 +1946,7 @@ describe("gateway/node-registry", () => {
commands: ["demo.dangerous"],
});
const session = registry.register(client, {});
const session = registerNodeSession(registry, client, {});
publishNodePluginTools(registry, [
{
pluginId: "demo",
@@ -1447,7 +1969,7 @@ describe("gateway/node-registry", () => {
commands: ["demo.echo"],
});
const session = registry.register(client, {});
const session = registerNodeSession(registry, client, {});
publishNodePluginTools(registry, [
{
pluginId: "demo",
@@ -1475,7 +1997,7 @@ describe("gateway/node-registry", () => {
commands: ["system.run"],
});
const session = registry.register(client, {});
const session = registerNodeSession(registry, client, {});
publishNodePluginTools(registry, [
{
pluginId: "demo",
@@ -1512,7 +2034,7 @@ describe("gateway/node-registry", () => {
commands: ["demo.echo"],
});
const session = registry.register(client, {});
const session = registerNodeSession(registry, client, {});
publishNodePluginTools(registry, [
{
pluginId: "demo",
@@ -1548,7 +2070,7 @@ describe("gateway/node-registry", () => {
declaredCommands: ["demo.echo"],
});
const session = registry.register(client, {});
const session = registerNodeSession(registry, client, {});
publishNodePluginTools(registry, [
{
pluginId: "demo",
@@ -1573,7 +2095,8 @@ describe("gateway/node-registry", () => {
it("enriches published node tools after matching plugin descriptors load", () => {
const registry = createTestNodeRegistry();
registry.register(
registerNodeSession(
registry,
makeClient("conn-1", "node-1", [], {
commands: ["demo.echo"],
}),
@@ -1602,7 +2125,8 @@ describe("gateway/node-registry", () => {
it("ignores published node tools when gateway publication is disabled", () => {
const registry = createNodeRegistry({ nodePluginToolsEnabled: false });
registry.register(
registerNodeSession(
registry,
makeClient("conn-1", "node-1", [], {
commands: ["demo.echo"],
}),
@@ -1626,13 +2150,15 @@ describe("gateway/node-registry", () => {
it("ignores node plugin tool updates from stale connections", () => {
registerDemoNodePluginTool({ name: "demo_echo", command: "demo.echo" });
const registry = createTestNodeRegistry();
registry.register(
registerNodeSession(
registry,
makeClient("conn-old", "node-1", [], {
commands: ["demo.echo"],
}),
{},
);
registry.register(
registerNodeSession(
registry,
makeClient("conn-new", "node-1", [], {
commands: ["demo.echo"],
}),
@@ -1655,7 +2181,7 @@ describe("gateway/node-registry", () => {
it("stores bounded node-hosted skill updates on the current session", () => {
const registry = createTestNodeRegistry();
const session = registry.register(makeClient("conn-1", "node-1"), {});
const session = registerNodeSession(registry, makeClient("conn-1", "node-1"), {});
const updated = publishNodeSkills(registry, [
nodeSkill("release-helper"),
@@ -1668,7 +2194,7 @@ describe("gateway/node-registry", () => {
it("enforces node skill count and total-content caps", () => {
const registry = createTestNodeRegistry();
registry.register(makeClient("conn-1", "node-1"), {});
registerNodeSession(registry, makeClient("conn-1", "node-1"), {});
const countUpdate = publishNodeSkills(
registry,
@@ -1689,12 +2215,12 @@ describe("gateway/node-registry", () => {
it("ignores node skills when publication is disabled or the connection is stale", () => {
const disabled = createNodeRegistry({ nodeSkillsEnabled: false });
disabled.register(makeClient("conn-1", "node-1"), {});
registerNodeSession(disabled, makeClient("conn-1", "node-1"), {});
expect(publishNodeSkills(disabled, [nodeSkill("disabled")])?.nodeSkills).toEqual([]);
const registry = createTestNodeRegistry();
registry.register(makeClient("conn-old", "node-1"), {});
registry.register(makeClient("conn-new", "node-1"), {});
registerNodeSession(registry, makeClient("conn-old", "node-1"), {});
registerNodeSession(registry, makeClient("conn-new", "node-1"), {});
expect(publishNodeSkills(registry, [nodeSkill("stale")], "conn-old")).toBeNull();
expect(registry.get("node-1")?.nodeSkills).toEqual([]);
});
@@ -1706,7 +2232,7 @@ describe("gateway/node-registry", () => {
declaredPermissions: { camera: false },
});
registry.register(client, {});
registerNodeSession(registry, client, {});
const updated = registry.updateSurface("node-1", {
caps: [],
commands: [],
@@ -1730,7 +2256,7 @@ describe("gateway/node-registry", () => {
sessionCommandsCeiling: ["device.info"],
});
registry.register(client, {});
registerNodeSession(registry, client, {});
const updated = registry.updateSurface("node-1", {
caps: ["canvas", "device"],
commands: ["canvas.snapshot", "device.info"],
src/gateway/node-registry.ts
+505 -34
View File
@@ -16,6 +16,7 @@ import type {
} from "../../packages/gateway-protocol/src/schema/nodes.js";
import { setActiveNodeContext } from "../infra/active-node-context.js";
import { NODE_MCP_TOOLS_CALL_COMMAND } from "../infra/node-commands.js";
import type { NodePairingBinding } from "../infra/node-pairing-state.js";
import { logRejectedLargePayload } from "../logging/diagnostic-payload.js";
import { normalizeString } from "./node-normalize.js";
import {
@@ -42,6 +43,10 @@ import type { GatewayWsClient } from "./server/ws-types.js";
export type NodeSession = {
nodeId: string;
connId: string;
/** Persistent device key and node-token identity authenticated for this connection. */
pairingIdentity?: string;
/** Persistent pairing generation authenticated before this session was registered. */
pairingGeneration?: string;
client: GatewayWsClient;
clientId?: string;
clientMode?: string;
@@ -70,6 +75,20 @@ export type NodeSession = {
presenceUpdatedAtMs?: number;
};
type PairingBoundNodeSession = NodeSession & { pairingIdentity: string };
type PairingBoundNodeSessionLease = {
session: PairingBoundNodeSession;
nodeId: string;
connId: string;
binding: NodePairingBinding;
};
type PairingLeaseResolution =
| { status: "current"; session: PairingBoundNodeSession }
| { status: "stale"; presenceInvalidated: boolean }
| { status: "unavailable" };
/** Authorized system.run event window bound to one node connection. */
type AuthorizedSystemRunEvent = PendingSystemRunEvent & {
nodeId: string;
@@ -164,12 +183,49 @@ export type NodeEventTransport = {
checkConnectivity?: (timeoutMs: number) => Promise<NodeConnectivityResult>;
};
type NodePairingStateSnapshot = NodePairingBinding;
type NodeSessionRegistrationOptions = {
remoteIp?: string | undefined;
pairingIdentity: string;
pairingGeneration?: string | undefined;
};
function pairingBindingForSession(node: PairingBoundNodeSession): NodePairingBinding {
return {
identity: node.pairingIdentity,
...(node.pairingGeneration ? { generation: node.pairingGeneration } : {}),
};
}
function pairingStateMatchesBinding(
binding: NodePairingBinding,
current: NodePairingStateSnapshot | undefined,
): boolean {
if (!current) {
return false;
}
if (binding.identity !== current.identity) {
return false;
}
return !binding.generation || binding.generation === current.generation;
}
export type NodeRegistryOptions = {
listRegisteredNodePluginToolCommands?:
| (() => readonly RegisteredNodePluginToolCommand[] | undefined)
| undefined;
nodePluginToolsEnabled?: boolean;
nodeSkillsEnabled?: boolean;
resolveCurrentPairingState?: (nodeId: string) => Promise<NodePairingStateSnapshot | undefined>;
isPairingStateCurrent?: (nodeId: string, expected: NodePairingBinding) => boolean;
onPairingGenerationChanged?: (params: {
nodeId: string;
previousPairingGeneration: string;
nextPairingGeneration: string;
preserveSessionState: boolean;
}) => void;
onPairingInvalidated?: (params: { nodeId: string; connId: string }) => void;
};
/** Serialize an event payload once so fanout can reuse the same JSON string. */
@@ -193,7 +249,7 @@ function isSerializedEventPayload(value: unknown): value is SerializedEventPaylo
/** Registry of currently connected Gateway nodes. */
export class NodeRegistry {
private nodesById = new Map<string, NodeSession>();
private nodesById = new Map<string, PairingBoundNodeSession>();
private nodesByConn = new Map<string, string>();
private eventTransportsByConn = new Map<string, NodeEventTransport>();
private pendingInvokes = new Map<string, PendingInvoke>();
@@ -245,9 +301,78 @@ export class NodeRegistry {
},
});
private authorizedSystemRunEvents = new Map<string, AuthorizedSystemRunEvent>();
private pairingGenerationEventChains = new Map<string, Promise<void>>();
constructor(private readonly options: NodeRegistryOptions = {}) {}
private listConnectedSessions(): PairingBoundNodeSession[] {
return [...this.nodesById.values()].filter((node) => node.client.invalidated !== true);
}
private capturePairingLease(node: PairingBoundNodeSession): PairingBoundNodeSessionLease {
return {
session: node,
nodeId: node.nodeId,
connId: node.connId,
binding: pairingBindingForSession(node),
};
}
private currentSessionForLease(
lease: PairingBoundNodeSessionLease,
): PairingBoundNodeSession | undefined {
const current = this.nodesById.get(lease.nodeId);
return current === lease.session &&
current.connId === lease.connId &&
current.pairingIdentity === lease.binding.identity &&
current.pairingGeneration === lease.binding.generation &&
current.client.invalidated !== true
? current
: undefined;
}
private settlePairingLease(params: {
lease: PairingBoundNodeSessionLease;
isCurrent: boolean;
invalidateStale: boolean;
}): PairingLeaseResolution {
const current = this.currentSessionForLease(params.lease);
if (!current) {
return { status: "stale", presenceInvalidated: false };
}
if (params.isCurrent) {
return { status: "current", session: current };
}
const presenceInvalidated = params.invalidateStale
? this.invalidateSessionForPairingChange(current)
: false;
return { status: "stale", presenceInvalidated };
}
private async resolvePairingLease(
lease: PairingBoundNodeSessionLease,
options: { invalidateStale: boolean },
): Promise<PairingLeaseResolution> {
const resolveCurrentPairingState = this.options.resolveCurrentPairingState;
if (!resolveCurrentPairingState) {
const current = this.currentSessionForLease(lease);
return current
? { status: "current", session: current }
: { status: "stale", presenceInvalidated: false };
}
let currentPairingState: NodePairingStateSnapshot | undefined;
try {
currentPairingState = await resolveCurrentPairingState(lease.nodeId);
} catch {
return { status: "unavailable" };
}
return this.settlePairingLease({
lease,
isCurrent: pairingStateMatchesBinding(lease.binding, currentPairingState),
invalidateStale: options.invalidateStale,
});
}
private normalizePluginToolDescriptors(params: {
nodeId: string;
tools?: readonly NodePluginToolDescriptor[];
@@ -285,14 +410,14 @@ export class NodeRegistry {
}
/** Register a websocket client as the current connection for its node id. */
register(client: GatewayWsClient, opts: { remoteIp?: string | undefined }) {
register(client: GatewayWsClient, opts: NodeSessionRegistrationOptions) {
return this.registerSession(client, opts);
}
/** Register a node whose events are delivered by an HTTP polling transport. */
registerTransport(
client: GatewayWsClient,
opts: { remoteIp?: string | undefined },
opts: NodeSessionRegistrationOptions,
transport: NodeEventTransport,
) {
return this.registerSession(client, opts, transport);
@@ -300,11 +425,15 @@ export class NodeRegistry {
private registerSession(
client: GatewayWsClient,
opts: { remoteIp?: string | undefined },
opts: NodeSessionRegistrationOptions,
transport?: NodeEventTransport,
) {
if (!opts.pairingIdentity) {
throw new Error("node session registration requires pairing identity");
}
const connect = client.connect;
const nodeId = connect.device?.id ?? connect.client.id;
const previousPairingGeneration = this.nodesById.get(nodeId)?.pairingGeneration;
const caps = Array.isArray(connect.caps) ? connect.caps : [];
const declaredCaps = Array.isArray((connect as { declaredCaps?: string[] }).declaredCaps)
? ((connect as { declaredCaps?: string[] }).declaredCaps ?? [])
@@ -346,9 +475,11 @@ export class NodeRegistry {
const declaredNodePluginTools: NodePluginToolDescriptor[] = [];
const nodePluginTools: NodePluginToolDescriptor[] = [];
const nodeSkills: NodeSkillDescriptor[] = [];
const session: NodeSession = {
const session: PairingBoundNodeSession = {
nodeId,
connId: client.connId,
pairingIdentity: opts.pairingIdentity,
...(opts.pairingGeneration ? { pairingGeneration: opts.pairingGeneration } : {}),
client,
clientId: connect.client.id,
clientMode: connect.client.mode,
@@ -377,6 +508,18 @@ export class NodeRegistry {
const replacesPresence = this.nodesById.get(nodeId)?.lastActiveAtMs !== undefined;
this.nodesById.set(nodeId, session);
this.nodesByConn.set(client.connId, nodeId);
if (
previousPairingGeneration &&
session.pairingGeneration &&
previousPairingGeneration !== session.pairingGeneration
) {
this.options.onPairingGenerationChanged?.({
nodeId,
previousPairingGeneration,
nextPairingGeneration: session.pairingGeneration,
preserveSessionState: false,
});
}
if (transport) {
this.eventTransportsByConn.set(client.connId, transport);
} else {
@@ -423,12 +566,153 @@ export class NodeRegistry {
/** List connected node sessions. */
listConnected(): NodeSession[] {
return [...this.nodesById.values()];
return this.listConnectedSessions();
}
/** Filter connected sessions against an already-loaded pairing-state snapshot. */
listConnectedForPairingStates(
currentPairingStates: ReadonlyMap<string, NodePairingStateSnapshot>,
): NodeSession[] {
return this.listConnectedSessions().filter((node) => {
const current = currentPairingStates.get(node.nodeId);
return pairingStateMatchesBinding(pairingBindingForSession(node), current);
});
}
/** Reconcile connected sessions through the synchronous persistent-pairing owner. */
listCurrentConnectedSync(): NodeSession[] {
const isPairingStateCurrent = this.options.isPairingStateCurrent;
if (!isPairingStateCurrent) {
return this.listConnected();
}
const connected: NodeSession[] = [];
let invalidatedPresence = false;
for (const candidate of this.listConnectedSessions()) {
const lease = this.capturePairingLease(candidate);
let isCurrent: boolean;
try {
isCurrent = isPairingStateCurrent(candidate.nodeId, lease.binding);
} catch {
continue;
}
const resolution = this.settlePairingLease({
lease,
isCurrent,
invalidateStale: true,
});
if (resolution.status === "current") {
connected.push(resolution.session);
} else if (resolution.status === "stale") {
invalidatedPresence ||= resolution.presenceInvalidated;
}
}
if (invalidatedPresence) {
this.publishActiveNodeContext();
}
return connected;
}
/** Resolve persistent pairing state before projecting connected sessions. */
async listCurrentConnected(): Promise<NodeSession[]> {
const resolved = await Promise.all(
this.listConnectedSessions().map((node) =>
this.resolvePairingLease(this.capturePairingLease(node), { invalidateStale: true }),
),
);
const connected: NodeSession[] = [];
let invalidatedPresence = false;
for (const result of resolved) {
if (result.status === "current") {
connected.push(result.session);
} else if (result.status === "stale") {
invalidatedPresence ||= result.presenceInvalidated;
}
}
if (invalidatedPresence) {
this.publishActiveNodeContext();
}
return connected;
}
private invalidateSessionForPairingChange(
node: NodeSession,
reason = "device-pairing-changed",
): boolean {
if (this.nodesById.get(node.nodeId) !== node || node.client.invalidated === true) {
return false;
}
node.client.invalidated = true;
node.client.invalidatedReason ??= reason;
removeConnectedNodePluginTools(node.nodeId);
this.invokeStreams.handleDisconnect(node.connId);
for (const [key, event] of this.authorizedSystemRunEvents) {
if (event.connId === node.connId) {
this.authorizedSystemRunEvents.delete(key);
}
}
this.options.onPairingInvalidated?.({ nodeId: node.nodeId, connId: node.connId });
return node.lastActiveAtMs !== undefined;
}
/** Immediately retires one exact transport after its persisted pairing authority changes. */
invalidateConnectionForPairingChange(connId: string, reason = "device-pairing-changed"): boolean {
const nodeId = this.nodesByConn.get(connId);
const node = nodeId ? this.nodesById.get(nodeId) : undefined;
if (!node || node.connId !== connId) {
return false;
}
const invalidatedPresence = this.invalidateSessionForPairingChange(node, reason);
if (invalidatedPresence) {
this.publishActiveNodeContext();
}
return node.client.invalidated === true;
}
/** Return a connected node session by node id. */
get(nodeId: string): NodeSession | undefined {
return this.nodesById.get(nodeId);
return this.getRegisteredSession(nodeId);
}
private getRegisteredSession(nodeId: string): PairingBoundNodeSession | undefined {
const node = this.nodesById.get(nodeId);
return node?.client.invalidated === true ? undefined : node;
}
/** Return only the session authenticated for the requested persistent pairing generation. */
getForPairingGeneration(nodeId: string, pairingGeneration: string): NodeSession | undefined {
return this.getRegisteredSessionForPairingGeneration(nodeId, pairingGeneration);
}
private getRegisteredSessionForPairingGeneration(
nodeId: string,
pairingGeneration: string,
): PairingBoundNodeSession | undefined {
const node = this.getRegisteredSession(nodeId);
// A mismatch alone does not reveal whether the session or the requesting
// operation is stale, so lookup must not revoke either generation.
return node?.pairingGeneration === pairingGeneration ? node : undefined;
}
/** Revalidates that one inbound node connection still owns its persisted pairing state. */
async isConnectionCurrentPairingState(connId: string): Promise<boolean> {
const nodeId = this.nodesByConn.get(connId);
const initial = nodeId ? this.nodesById.get(nodeId) : undefined;
if (
!nodeId ||
!initial ||
initial.connId !== connId ||
initial.client.invalidated === true ||
!this.options.resolveCurrentPairingState
) {
return false;
}
const resolution = await this.resolvePairingLease(this.capturePairingLease(initial), {
invalidateStale: true,
});
if (resolution.status === "stale" && resolution.presenceInvalidated) {
this.publishActiveNodeContext();
}
return resolution.status === "current";
}
/** Updates recent input activity for the exact authenticated node connection. */
@@ -459,9 +743,11 @@ export class NodeRegistry {
}
/** Returns the connected node with the freshest reported local input. */
getActiveNode(): NodeSession | undefined {
getActiveNode(
connectedNodes: readonly NodeSession[] = this.listConnected(),
): NodeSession | undefined {
let active: NodeSession | undefined;
for (const node of this.nodesById.values()) {
for (const node of connectedNodes) {
if (node.lastActiveAtMs === undefined) {
continue;
}
@@ -478,8 +764,30 @@ export class NodeRegistry {
}
private publishActiveNodeContext(): void {
const active = this.getActiveNode();
setActiveNodeContext(active ? { nodeId: active.nodeId } : null);
const active = this.getActiveNode(this.listConnectedSessions()) as
| PairingBoundNodeSession
| undefined;
const lease = active ? this.capturePairingLease(active) : undefined;
setActiveNodeContext(
active
? {
nodeId: active.nodeId,
...(active.pairingGeneration ? { pairingGeneration: active.pairingGeneration } : {}),
}
: null,
lease
? {
isCurrent: () => {
if (!this.currentSessionForLease(lease)) {
return false;
}
return this.options.isPairingStateCurrent
? this.options.isPairingStateCurrent(lease.nodeId, lease.binding)
: true;
},
}
: undefined,
);
}
/** Probe websocket liveness with ping/pong when the socket supports it. */
@@ -611,9 +919,22 @@ export class NodeRegistry {
commands: readonly string[];
permissions?: Record<string, boolean> | undefined;
},
generationTransition?: {
expectedConnId: string;
expectedPairingIdentity: string;
expectedPairingGeneration?: string;
nextPairingGeneration: string;
},
): NodeSession | null {
const node = this.nodesById.get(nodeId);
if (!node) {
if (
!node ||
node.client.invalidated === true ||
(generationTransition !== undefined &&
(node.connId !== generationTransition.expectedConnId ||
node.pairingIdentity !== generationTransition.expectedPairingIdentity ||
node.pairingGeneration !== generationTransition.expectedPairingGeneration))
) {
return null;
}
@@ -638,29 +959,46 @@ export class NodeRegistry {
node.permissions = undefined;
(node.client.connect as { permissions?: Record<string, boolean> }).permissions = undefined;
this.clearPresenceIfAccessibilityUnavailable(node);
return node;
} else {
const declared = node.declaredPermissions ?? {};
const nextEntries: Array<[string, boolean]> = [];
for (const [key, declaredValue] of Object.entries(declared)) {
if (!declaredValue) {
nextEntries.push([key, false]);
continue;
}
const approvedValue = surface.permissions?.[key];
if (approvedValue) {
nextEntries.push([key, true]);
continue;
}
if (approvedValue !== undefined) {
nextEntries.push([key, false]);
}
}
const nextPermissions =
nextEntries.length > 0 ? Object.fromEntries(nextEntries) : undefined;
node.permissions = nextPermissions;
(node.client.connect as { permissions?: Record<string, boolean> }).permissions =
nextPermissions;
this.clearPresenceIfAccessibilityUnavailable(node);
}
const declared = node.declaredPermissions ?? {};
const nextEntries: Array<[string, boolean]> = [];
for (const [key, declaredValue] of Object.entries(declared)) {
if (!declaredValue) {
nextEntries.push([key, false]);
continue;
}
const approvedValue = surface.permissions?.[key];
if (approvedValue) {
nextEntries.push([key, true]);
continue;
}
if (approvedValue !== undefined) {
nextEntries.push([key, false]);
}
}
if (generationTransition) {
const previousPairingGeneration = node.pairingGeneration;
node.pairingGeneration = generationTransition.nextPairingGeneration;
if (previousPairingGeneration) {
this.options.onPairingGenerationChanged?.({
nodeId,
previousPairingGeneration,
nextPairingGeneration: generationTransition.nextPairingGeneration,
preserveSessionState: true,
});
}
const nextPermissions = nextEntries.length > 0 ? Object.fromEntries(nextEntries) : undefined;
node.permissions = nextPermissions;
(node.client.connect as { permissions?: Record<string, boolean> }).permissions =
nextPermissions;
this.clearPresenceIfAccessibilityUnavailable(node);
// Active-node leases capture the pairing generation, so a promoted live
// session must republish its lease even when its presence is unchanged.
this.publishActiveNodeContext();
}
return node;
@@ -678,6 +1016,7 @@ export class NodeRegistry {
async invoke(params: {
nodeId: string;
expectedConnId?: string;
expectedPairingGeneration?: string;
command: string;
params?: unknown;
timeoutMs?: number;
@@ -693,19 +1032,65 @@ export class NodeRegistry {
if (params.signal?.aborted) {
return { ok: false, error: { code: "ABORTED", message: "node invoke cancelled" } };
}
const node = this.nodesById.get(params.nodeId);
let node = this.nodesById.get(params.nodeId);
if (!node) {
return {
ok: false,
error: { code: "NOT_CONNECTED", message: "node not connected" },
};
}
if (node.client.invalidated === true) {
return {
ok: false,
error: { code: "PAIRING_CHANGED", message: "node pairing changed before dispatch" },
};
}
const expectedPairingGeneration = params.expectedPairingGeneration ?? node.pairingGeneration;
if (this.options.resolveCurrentPairingState && !expectedPairingGeneration) {
return {
ok: false,
error: { code: "PAIRING_CHANGED", message: "node pairing generation unavailable" },
};
}
if (expectedPairingGeneration && node.pairingGeneration !== expectedPairingGeneration) {
return {
ok: false,
error: { code: "PAIRING_CHANGED", message: "node pairing changed before dispatch" },
};
}
if (params.expectedConnId && node.connId !== params.expectedConnId) {
return {
ok: false,
error: { code: "ROUTE_CHANGED", message: "node connection changed before dispatch" },
};
}
if (expectedPairingGeneration && this.options.resolveCurrentPairingState) {
const resolution = await this.resolvePairingLease(this.capturePairingLease(node), {
invalidateStale: false,
});
if (resolution.status === "unavailable") {
return {
ok: false,
error: {
code: "UNAVAILABLE",
message: "node pairing state unavailable before dispatch",
},
};
}
if (resolution.status !== "current") {
return {
ok: false,
error: { code: "PAIRING_CHANGED", message: "node pairing changed before dispatch" },
};
}
node = resolution.session;
if (params.expectedConnId && node.connId !== params.expectedConnId) {
return {
ok: false,
error: { code: "ROUTE_CHANGED", message: "node connection changed before dispatch" },
};
}
}
const requestId = randomUUID();
const invokeParams = normalizeSystemRunInvokeParams({
command: params.command,
@@ -958,7 +1343,90 @@ export class NodeRegistry {
return this.sendEventRawInternal(node, event, payloadJSON);
}
/** Sends command-free events only to the exact authenticated pairing connection. */
async sendEventForPairingIdentity(params: {
nodeId: string;
connId: string;
pairingIdentity: string;
event: string;
payload?: unknown;
}): Promise<boolean> {
const initial = this.nodesById.get(params.nodeId);
if (
!initial ||
initial.connId !== params.connId ||
initial.pairingIdentity !== params.pairingIdentity ||
initial.client.invalidated === true ||
!this.options.resolveCurrentPairingState
) {
return false;
}
const resolution = await this.resolvePairingLease(this.capturePairingLease(initial), {
invalidateStale: true,
});
if (resolution.status !== "current") {
if (resolution.status === "stale" && resolution.presenceInvalidated) {
this.publishActiveNodeContext();
}
return false;
}
return this.sendEventToSession(resolution.session, params.event, params.payload);
}
/** Sends only to a session that still owns the requested persistent pairing generation. */
async sendEventRawForPairingGeneration(
nodeId: string,
pairingGeneration: string,
event: string,
payloadJSON?: SerializedEventPayload | null,
): Promise<boolean> {
const previous = this.pairingGenerationEventChains.get(nodeId) ?? Promise.resolve();
const send = previous.then(() =>
this.sendEventRawForPairingGenerationNow(nodeId, pairingGeneration, event, payloadJSON),
);
const tail = send.then(
() => undefined,
() => undefined,
);
this.pairingGenerationEventChains.set(nodeId, tail);
try {
return await send;
} finally {
if (this.pairingGenerationEventChains.get(nodeId) === tail) {
this.pairingGenerationEventChains.delete(nodeId);
}
}
}
private async sendEventRawForPairingGenerationNow(
nodeId: string,
pairingGeneration: string,
event: string,
payloadJSON?: SerializedEventPayload | null,
): Promise<boolean> {
let node = this.getRegisteredSessionForPairingGeneration(nodeId, pairingGeneration);
if (!node) {
return false;
}
if (this.options.resolveCurrentPairingState) {
const resolution = await this.resolvePairingLease(this.capturePairingLease(node), {
invalidateStale: true,
});
if (resolution.status !== "current") {
if (resolution.status === "stale" && resolution.presenceInvalidated) {
this.publishActiveNodeContext();
}
return false;
}
node = resolution.session;
}
return this.sendEventRawInternal(node, event, payloadJSON);
}
private sendEventInternal(node: NodeSession, event: string, payload: unknown): boolean {
if (node.client.invalidated === true) {
return false;
}
const eventTransport = this.eventTransportsByConn.get(node.connId);
if (eventTransport) {
return eventTransport.send(event, payload);
@@ -985,6 +1453,9 @@ export class NodeRegistry {
event: string,
payloadJSON?: SerializedEventPayload | null,
): boolean {
if (node.client.invalidated === true) {
return false;
}
if (
payloadJSON !== null &&
payloadJSON !== undefined &&
src/gateway/node-runtime-state.ts
+185
View File
@@ -0,0 +1,185 @@
// Process-local node state shared by node and full-device pairing removal.
import { randomUUID } from "node:crypto";
import { removeRemoteNodeInfo } from "../skills/runtime/remote.js";
import { clearNodePendingWork } from "./node-pending-work.js";
import type { NodeRegistry } from "./node-registry.js";
import { invalidateNodeWakeState } from "./node-wake-state.js";
export type PendingNodeAction = {
id: string;
nodeId: string;
pairingGeneration: string;
command: string;
paramsJSON?: string;
idempotencyKey: string;
enqueuedAtMs: number;
};
const pendingNodeActionsById = new Map<string, PendingNodeAction[]>();
function prunePendingNodeActions(params: {
nodeId: string;
nowMs: number;
ttlMs: number;
pairingGeneration?: string;
}): PendingNodeAction[] {
const queue = pendingNodeActionsById.get(params.nodeId) ?? [];
const minTimestampMs = params.nowMs - params.ttlMs;
const live = queue.filter((entry) => entry.enqueuedAtMs >= minTimestampMs);
if (live.length === 0) {
pendingNodeActionsById.delete(params.nodeId);
return [];
}
pendingNodeActionsById.set(params.nodeId, live);
return params.pairingGeneration
? live.filter((entry) => entry.pairingGeneration === params.pairingGeneration)
: live;
}
export function replacePendingNodeActionsForGeneration(params: {
nodeId: string;
pairingGeneration: string;
replacement: PendingNodeAction[];
ttlMs: number;
nowMs?: number;
}): void {
const live = prunePendingNodeActions({
nodeId: params.nodeId,
nowMs: params.nowMs ?? Date.now(),
ttlMs: params.ttlMs,
});
const next = [
...live.filter((entry) => entry.pairingGeneration !== params.pairingGeneration),
...params.replacement,
];
if (next.length === 0) {
pendingNodeActionsById.delete(params.nodeId);
return;
}
pendingNodeActionsById.set(params.nodeId, next);
}
export function enqueuePendingNodeAction(params: {
nodeId: string;
pairingGeneration: string;
command: string;
paramsJSON?: string;
idempotencyKey: string;
ttlMs: number;
maxPerNode: number;
nowMs?: number;
}): { action: PendingNodeAction; created: boolean } {
const nowMs = params.nowMs ?? Date.now();
const queue = prunePendingNodeActions({
nodeId: params.nodeId,
nowMs,
ttlMs: params.ttlMs,
pairingGeneration: params.pairingGeneration,
});
const existing = queue.find((entry) => entry.idempotencyKey === params.idempotencyKey);
if (existing) {
return { action: existing, created: false };
}
const action: PendingNodeAction = {
id: randomUUID(),
nodeId: params.nodeId,
pairingGeneration: params.pairingGeneration,
command: params.command,
paramsJSON: params.paramsJSON,
idempotencyKey: params.idempotencyKey,
enqueuedAtMs: nowMs,
};
queue.push(action);
if (queue.length > params.maxPerNode) {
queue.splice(0, queue.length - params.maxPerNode);
}
replacePendingNodeActionsForGeneration({
nodeId: params.nodeId,
pairingGeneration: params.pairingGeneration,
replacement: queue,
ttlMs: params.ttlMs,
nowMs,
});
return { action, created: true };
}
export function listPendingNodeActions(params: {
nodeId: string;
pairingGeneration?: string;
ttlMs: number;
nowMs?: number;
}): PendingNodeAction[] {
return prunePendingNodeActions({
nodeId: params.nodeId,
nowMs: params.nowMs ?? Date.now(),
ttlMs: params.ttlMs,
pairingGeneration: params.pairingGeneration,
});
}
export function acknowledgePendingNodeActions(params: {
nodeId: string;
pairingGeneration: string;
ids: readonly string[];
ttlMs: number;
}): PendingNodeAction[] {
const pending = prunePendingNodeActions({
nodeId: params.nodeId,
pairingGeneration: params.pairingGeneration,
nowMs: Date.now(),
ttlMs: params.ttlMs,
});
if (params.ids.length === 0) {
return pending;
}
const ids = new Set(params.ids);
const remaining = pending.filter((entry) => !ids.has(entry.id));
replacePendingNodeActionsForGeneration({
...params,
replacement: remaining,
});
return remaining;
}
export function removePendingNodeAction(params: {
nodeId: string;
pairingGeneration: string;
actionId: string;
ttlMs: number;
}): void {
const pending = prunePendingNodeActions({
nodeId: params.nodeId,
pairingGeneration: params.pairingGeneration,
nowMs: Date.now(),
ttlMs: params.ttlMs,
});
const remaining = pending.filter((entry) => entry.id !== params.actionId);
if (remaining.length === pending.length) {
return;
}
replacePendingNodeActionsForGeneration({
...params,
replacement: remaining,
});
}
function clearPendingNodeActions(nodeId: string): void {
pendingNodeActionsById.delete(nodeId);
}
export function clearRemovedNodeRuntimeState(params: {
nodeId: string;
context: {
nodeRegistry: Pick<NodeRegistry, "updateSurface">;
};
}) {
clearPendingNodeActions(params.nodeId);
clearNodePendingWork(params.nodeId);
invalidateNodeWakeState(params.nodeId);
params.context.nodeRegistry.updateSurface(params.nodeId, {
caps: [],
commands: [],
permissions: undefined,
});
removeRemoteNodeInfo(params.nodeId);
}
src/gateway/node-wake-state-store.ts
+27
View File
@@ -0,0 +1,27 @@
type StoredNodeWakeAttempt = {
available: boolean;
throttled: boolean;
path: "throttled" | "no-registration" | "no-auth" | "sent" | "send-error" | "invalidated";
durationMs: number;
apnsStatus?: number;
apnsReason?: string;
};
export type NodeWakeOwnerState = {
nodeId: string;
stateKey: string;
lastWakeAtMs: number;
inFlightWake?: Promise<StoredNodeWakeAttempt>;
lastNudgeAtMs: number;
lifecycle?: {
controller: AbortController;
users: number;
};
};
export const nodeWakeStateByOwner = new Map<string, NodeWakeOwnerState>();
export const nodeWakeOwnerBySignal = new WeakMap<AbortSignal, NodeWakeOwnerState>();
export function nodeWakeStateKey(nodeId: string, pairingGeneration?: string): string {
return JSON.stringify([nodeId.trim(), pairingGeneration?.trim() || null]);
}
src/gateway/node-wake-state.test-support.ts
+31
View File
@@ -0,0 +1,31 @@
import { nodeWakeStateByOwner, nodeWakeStateKey } from "./node-wake-state-store.js";
/** Read-only test projection; callers cannot mutate lifecycle ownership. */
export function getNodeWakeStateSnapshot(
nodeId: string,
pairingGeneration?: string,
):
| {
lastWakeAtMs: number;
wakeInFlight: boolean;
lastNudgeAtMs: number;
lifecycleUsers: number;
}
| undefined {
const owner = nodeWakeStateByOwner.get(nodeWakeStateKey(nodeId, pairingGeneration));
return owner
? {
lastWakeAtMs: owner.lastWakeAtMs,
wakeInFlight: owner.inFlightWake !== undefined,
lastNudgeAtMs: owner.lastNudgeAtMs,
lifecycleUsers: owner.lifecycle?.users ?? 0,
}
: undefined;
}
export function resetNodeWakeStateForTest(): void {
for (const owner of nodeWakeStateByOwner.values()) {
owner.lifecycle?.controller.abort();
}
nodeWakeStateByOwner.clear();
}
src/gateway/node-wake-state.test.ts
+145
View File
@@ -0,0 +1,145 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
captureNodeWakeLifecycle,
clearNodeWakeState,
invalidateNodeWakeState,
isNodeWakeLifecycleCurrent,
releaseNodeWakeLifecycle,
runNodeWakeAttempt,
runNodeWakeNudgeAttempt,
type NodeWakeAttempt,
} from "./node-wake-state.js";
import {
getNodeWakeStateSnapshot,
resetNodeWakeStateForTest,
} from "./node-wake-state.test-support.js";
const sentWake: NodeWakeAttempt = {
available: true,
throttled: false,
path: "sent",
durationMs: 1,
};
beforeEach(() => {
resetNodeWakeStateForTest();
});
describe("node wake lifecycle ownership", () => {
it("isolates generations and invalidates every generation for a removed node", () => {
const generationOne = captureNodeWakeLifecycle("node-1", "generation-1");
const generationTwo = captureNodeWakeLifecycle("node-1", "generation-2");
expect(isNodeWakeLifecycleCurrent("node-1", generationOne, "generation-1")).toBe(true);
expect(isNodeWakeLifecycleCurrent("node-1", generationOne, "generation-2")).toBe(false);
expect(isNodeWakeLifecycleCurrent("node-1", generationTwo, "generation-2")).toBe(true);
invalidateNodeWakeState("node-1");
expect(generationOne.aborted).toBe(true);
expect(generationTwo.aborted).toBe(true);
expect(getNodeWakeStateSnapshot("node-1", "generation-1")).toBeUndefined();
expect(getNodeWakeStateSnapshot("node-1", "generation-2")).toBeUndefined();
});
it("releases an idle lifecycle without retaining owner state", () => {
const lifecycle = captureNodeWakeLifecycle("node-idle");
releaseNodeWakeLifecycle("node-idle", lifecycle);
expect(lifecycle.aborted).toBe(true);
expect(getNodeWakeStateSnapshot("node-idle")).toBeUndefined();
});
it("clears throttle state without aborting an active lifecycle", async () => {
const lifecycle = captureNodeWakeLifecycle("node-active");
await runNodeWakeAttempt({
nodeId: "node-active",
force: true,
throttleMs: 1_000,
attempt: async (markAttempted) => {
markAttempted();
return sentWake;
},
});
clearNodeWakeState("node-active");
expect(lifecycle.aborted).toBe(false);
expect(getNodeWakeStateSnapshot("node-active")?.lastWakeAtMs).toBe(0);
releaseNodeWakeLifecycle("node-active", lifecycle);
});
});
describe("node wake coordination", () => {
it("deduplicates concurrent wake attempts for one generation", async () => {
let finish: ((attempt: NodeWakeAttempt) => void) | undefined;
const attempt = vi.fn(
() =>
new Promise<NodeWakeAttempt>((resolve) => {
finish = resolve;
}),
);
const params = {
nodeId: "node-1",
pairingGeneration: "generation-1",
force: false,
throttleMs: 1_000,
attempt,
};
const first = runNodeWakeAttempt(params);
const second = runNodeWakeAttempt(params);
expect(attempt).toHaveBeenCalledOnce();
finish?.(sentWake);
await expect(first).resolves.toEqual(sentWake);
await expect(second).resolves.toEqual(sentWake);
});
it("throttles only after transport admission marks a real wake attempt", async () => {
await runNodeWakeAttempt({
nodeId: "node-1",
force: false,
throttleMs: 60_000,
attempt: async (markAttempted) => {
markAttempted();
return sentWake;
},
});
const second = await runNodeWakeAttempt({
nodeId: "node-1",
force: false,
throttleMs: 60_000,
attempt: async () => sentWake,
});
expect(second).toEqual({
available: true,
throttled: true,
path: "throttled",
durationMs: 0,
});
});
it("tracks reconnect-nudge throttle independently from wake throttle", async () => {
const sent = await runNodeWakeNudgeAttempt({
nodeId: "node-1",
throttleMs: 60_000,
throttled: () => ({ sent: false, throttled: true, reason: "throttled", durationMs: 0 }),
attempt: async () => ({ sent: true, throttled: false, reason: "sent", durationMs: 1 }),
});
const throttled = await runNodeWakeNudgeAttempt({
nodeId: "node-1",
throttleMs: 60_000,
throttled: () => ({ sent: false, throttled: true, reason: "throttled", durationMs: 0 }),
attempt: async () => ({ sent: true, throttled: false, reason: "sent", durationMs: 1 }),
});
expect(sent.reason).toBe("sent");
expect(throttled.reason).toBe("throttled");
expect(getNodeWakeStateSnapshot("node-1")?.lastWakeAtMs).toBe(0);
expect(getNodeWakeStateSnapshot("node-1")?.lastNudgeAtMs).toBeGreaterThan(0);
});
});
src/gateway/node-wake-state.ts
+194
View File
@@ -0,0 +1,194 @@
// Gateway-owned APNs wake, reconnect nudge, and cancellation state.
import {
nodeWakeOwnerBySignal,
nodeWakeStateByOwner,
nodeWakeStateKey,
type NodeWakeOwnerState,
} from "./node-wake-state-store.js";
export const NODE_WAKE_RECONNECT_WAIT_MS = 3_000;
export const NODE_WAKE_RECONNECT_RETRY_WAIT_MS = 12_000;
export const NODE_WAKE_RECONNECT_POLL_MS = 150;
export type NodeWakeAttempt = {
available: boolean;
throttled: boolean;
path: "throttled" | "no-registration" | "no-auth" | "sent" | "send-error" | "invalidated";
durationMs: number;
apnsStatus?: number;
apnsReason?: string;
};
export type NodeWakeNudgeAttempt = {
sent: boolean;
throttled: boolean;
reason:
| "throttled"
| "no-registration"
| "no-auth"
| "send-error"
| "apns-not-ok"
| "sent"
| "invalidated";
durationMs: number;
apnsStatus?: number;
apnsReason?: string;
};
export type NodeWakeLifecycle = AbortSignal;
function getOrCreateNodeWakeOwner(nodeId: string, pairingGeneration?: string): NodeWakeOwnerState {
const normalizedNodeId = nodeId.trim();
const stateKey = nodeWakeStateKey(normalizedNodeId, pairingGeneration);
const existing = nodeWakeStateByOwner.get(stateKey);
if (existing) {
return existing;
}
const created: NodeWakeOwnerState = {
nodeId: normalizedNodeId,
stateKey,
lastWakeAtMs: 0,
lastNudgeAtMs: 0,
};
nodeWakeStateByOwner.set(stateKey, created);
return created;
}
function deleteIdleNodeWakeOwner(owner: NodeWakeOwnerState): void {
if (
owner.lifecycle?.users ||
owner.inFlightWake ||
owner.lastWakeAtMs > 0 ||
owner.lastNudgeAtMs > 0
) {
return;
}
owner.lifecycle?.controller.abort();
if (owner.lifecycle) {
nodeWakeOwnerBySignal.delete(owner.lifecycle.controller.signal);
}
nodeWakeStateByOwner.delete(owner.stateKey);
}
export function captureNodeWakeLifecycle(
nodeId: string,
pairingGeneration?: string,
): NodeWakeLifecycle {
const owner = getOrCreateNodeWakeOwner(nodeId, pairingGeneration);
if (!owner.lifecycle || owner.lifecycle.controller.signal.aborted) {
owner.lifecycle = { controller: new AbortController(), users: 0 };
nodeWakeOwnerBySignal.set(owner.lifecycle.controller.signal, owner);
}
owner.lifecycle.users += 1;
return owner.lifecycle.controller.signal;
}
export function isNodeWakeLifecycleCurrent(
nodeId: string,
lifecycle: NodeWakeLifecycle,
pairingGeneration?: string,
): boolean {
const owner = nodeWakeOwnerBySignal.get(lifecycle);
const expectedStateKey = nodeWakeStateKey(nodeId, pairingGeneration);
return (
!lifecycle.aborted &&
owner?.nodeId === nodeId.trim() &&
owner.stateKey === expectedStateKey &&
nodeWakeStateByOwner.get(expectedStateKey) === owner &&
owner.lifecycle?.controller.signal === lifecycle
);
}
export function releaseNodeWakeLifecycle(nodeId: string, lifecycle: NodeWakeLifecycle): void {
const owner = nodeWakeOwnerBySignal.get(lifecycle);
if (
owner?.nodeId !== nodeId.trim() ||
nodeWakeStateByOwner.get(owner.stateKey) !== owner ||
owner.lifecycle?.controller.signal !== lifecycle
) {
return;
}
owner.lifecycle.users = Math.max(0, owner.lifecycle.users - 1);
deleteIdleNodeWakeOwner(owner);
}
/** Owns wake dedupe and throttle state while the caller owns APNs policy and I/O. */
export async function runNodeWakeAttempt(params: {
nodeId: string;
pairingGeneration?: string;
force: boolean;
throttleMs: number;
attempt: (markAttempted: () => void) => Promise<NodeWakeAttempt>;
}): Promise<NodeWakeAttempt> {
const owner = getOrCreateNodeWakeOwner(params.nodeId, params.pairingGeneration);
if (owner.inFlightWake) {
return await owner.inFlightWake;
}
if (
!params.force &&
owner.lastWakeAtMs > 0 &&
Date.now() - owner.lastWakeAtMs < params.throttleMs
) {
return { available: true, throttled: true, path: "throttled", durationMs: 0 };
}
const attempt = params.attempt(() => {
owner.lastWakeAtMs = Date.now();
});
owner.inFlightWake = attempt;
try {
return await attempt;
} finally {
if (owner.inFlightWake === attempt) {
owner.inFlightWake = undefined;
}
deleteIdleNodeWakeOwner(owner);
}
}
/** Owns reconnect-nudge throttling while the caller owns APNs policy and I/O. */
export async function runNodeWakeNudgeAttempt(params: {
nodeId: string;
pairingGeneration?: string;
throttleMs: number;
throttled: () => NodeWakeNudgeAttempt;
attempt: () => Promise<NodeWakeNudgeAttempt>;
}): Promise<NodeWakeNudgeAttempt> {
const owner = getOrCreateNodeWakeOwner(params.nodeId, params.pairingGeneration);
if (owner.lastNudgeAtMs > 0 && Date.now() - owner.lastNudgeAtMs < params.throttleMs) {
return params.throttled();
}
const result = await params.attempt();
if (result.reason === "sent") {
owner.lastNudgeAtMs = Date.now();
}
deleteIdleNodeWakeOwner(owner);
return result;
}
export function clearNodeWakeState(nodeId: string): void {
const normalizedNodeId = nodeId.trim();
for (const owner of nodeWakeStateByOwner.values()) {
if (owner.nodeId !== normalizedNodeId) {
continue;
}
owner.lastWakeAtMs = 0;
owner.inFlightWake = undefined;
owner.lastNudgeAtMs = 0;
deleteIdleNodeWakeOwner(owner);
}
}
export function invalidateNodeWakeState(nodeId: string): void {
const normalizedNodeId = nodeId.trim();
for (const owner of nodeWakeStateByOwner.values()) {
if (owner.nodeId !== normalizedNodeId) {
continue;
}
owner.lifecycle?.controller.abort();
if (owner.lifecycle) {
nodeWakeOwnerBySignal.delete(owner.lifecycle.controller.signal);
}
nodeWakeStateByOwner.delete(owner.stateKey);
}
}
src/gateway/server-broadcast.ts
+4 -6
View File
@@ -80,11 +80,6 @@ const EVENT_SCOPE_GUARDS: Record<string, string[]> = {
"terminal.exit": [ADMIN_SCOPE],
};
// Events that node-role sessions must receive even when the event's operator
// scope would otherwise reject non-operator roles. Nodes act on these updates
// (e.g. reconfiguring wake-word triggers).
const NODE_ALLOWED_EVENTS = new Set<string>(["voicewake.changed", "voicewake.routing.changed"]);
// Opt-in scoped clients never receive session-bearing broadcasts without an
// authoritative registry key, including malformed/sessionless agent events.
const SESSION_SUBSCRIPTION_EVENTS = new Set([
@@ -143,7 +138,7 @@ function hasEventScope(
}
const role = client.connect.role ?? "operator";
if (role !== "operator") {
return role === "node" && NODE_ALLOWED_EVENTS.has(event);
return false;
}
const scopes = Array.isArray(client.connect.scopes) ? client.connect.scopes : [];
if (scopes.includes(ADMIN_SCOPE)) {
@@ -209,6 +204,9 @@ export function createGatewayBroadcaster(params: {
return frameBase;
};
for (const c of params.clients) {
if (c.invalidated === true) {
continue;
}
if (targetConnIds && !targetConnIds.has(c.connId)) {
continue;
}
src/gateway/server-methods.authorization.test.ts
+45
View File
@@ -84,6 +84,51 @@ describe("gateway method authorization", () => {
});
});
it("rejects every node RPC when its connection no longer owns the pairing generation", async () => {
const handler = vi.fn<GatewayRequestHandler>(({ respond }) => respond(true, { ok: true }));
const respond = vi.fn();
const isConnectionCurrentPairingState = vi.fn().mockResolvedValue(false);
await handleGatewayRequest({
req: { type: "req", id: "req-node-stale", method: "node.event", params: { event: "test" } },
respond,
client: {
connId: "conn-node-stale",
connect: {
role: "node",
scopes: [],
device: {
id: "node-stale",
publicKey: "public-key",
signature: "signature",
signedAt: 1,
nonce: "nonce",
},
client: { id: "node-host", version: "1", platform: "test", mode: "node" },
minProtocol: 1,
maxProtocol: 1,
},
} as Parameters<typeof handleGatewayRequest>[0]["client"],
isWebchatConnect: () => false,
context: {
logGateway: { warn: vi.fn() },
nodeRegistry: { isConnectionCurrentPairingState },
} as unknown as Parameters<typeof handleGatewayRequest>[0]["context"],
extraHandlers: { "node.event": handler },
});
expect(isConnectionCurrentPairingState).toHaveBeenCalledWith("conn-node-stale");
expect(handler).not.toHaveBeenCalled();
expect(respond).toHaveBeenCalledWith(
false,
undefined,
expect.objectContaining({
code: "UNAVAILABLE",
details: { code: "PAIRING_CHANGED" },
}),
);
});
async function dispatchProfileMutation(params: {
authenticatedUserId?: string;
profileId: string;
src/gateway/server-methods.ts
+14
View File
@@ -914,6 +914,20 @@ export async function handleGatewayRequest(
respond(false, undefined, authError);
return;
}
if (
client?.connect.role === "node" &&
(!client.connId || !(await context.nodeRegistry.isConnectionCurrentPairingState(client.connId)))
) {
respond(
false,
undefined,
errorShape(ErrorCodes.UNAVAILABLE, "node pairing changed before request dispatch", {
retryable: true,
details: { code: "PAIRING_CHANGED" },
}),
);
return;
}
if (context.unavailableGatewayMethods?.has(req.method)) {
// During startup, methods can be listed before their runtime is ready. Return the protocol
// retry shape so clients can back off without treating startup as a permanent unknown method.
src/gateway/server-methods/devices.test.ts
+164
View File
@@ -8,6 +8,17 @@ import {
resetDiagnosticEventsForTest,
type DiagnosticSecurityEvent,
} from "../../infra/diagnostic-events.js";
import { drainNodePendingWork, enqueueNodePendingWork } from "../node-pending-work.js";
import {
captureNodeWakeLifecycle,
releaseNodeWakeLifecycle,
runNodeWakeAttempt,
runNodeWakeNudgeAttempt,
} from "../node-wake-state.js";
import {
getNodeWakeStateSnapshot,
resetNodeWakeStateForTest,
} from "../node-wake-state.test-support.js";
import { deviceHandlers } from "./devices.js";
import type { GatewayRequestHandlerOptions } from "./types.js";
@@ -88,6 +99,9 @@ function createOptions(
info: vi.fn(),
warn: vi.fn(),
},
nodeRegistry: {
updateSurface: vi.fn(),
},
},
...overrides,
} as unknown as GatewayRequestHandlerOptions;
@@ -145,12 +159,60 @@ function captureSecurityEvents(): {
return { events, stop };
}
async function seedNodeWakeState(nodeId: string): Promise<void> {
await runNodeWakeAttempt({
nodeId,
force: true,
throttleMs: 60_000,
attempt: async (markAttempted) => {
markAttempted();
return { available: true, throttled: false, path: "sent", durationMs: 1 };
},
});
await runNodeWakeNudgeAttempt({
nodeId,
throttleMs: 60_000,
throttled: () => ({ sent: false, throttled: true, reason: "throttled", durationMs: 0 }),
attempt: async () => ({ sent: true, throttled: false, reason: "sent", durationMs: 1 }),
});
}
describe("deviceHandlers", () => {
beforeEach(() => {
resetDiagnosticEventsForTest();
resetNodeWakeStateForTest();
vi.clearAllMocks();
});
it("clears and invalidates node runtime state after removing a full device pairing", async () => {
const nodeId = "disconnected-node-device";
removePairedDeviceMock.mockResolvedValue({ deviceId: nodeId });
await seedNodeWakeState(nodeId);
enqueueNodePendingWork({ nodeId, type: "location.request" });
const wakeLifecycle = captureNodeWakeLifecycle(nodeId);
const opts = createOptions("device.pair.remove", { deviceId: nodeId });
await expectDefined(
deviceHandlers["device.pair.remove"],
'deviceHandlers["device.pair.remove"] test invariant',
)(opts);
expect(getNodeWakeStateSnapshot(nodeId)).toBeUndefined();
expect(wakeLifecycle.aborted).toBe(true);
expect(drainNodePendingWork(nodeId).items.map((item) => item.id)).toEqual(["baseline-status"]);
const nodeRegistry = opts.context.nodeRegistry as unknown as {
updateSurface: ReturnType<typeof vi.fn>;
};
expect(nodeRegistry.updateSurface).toHaveBeenCalledWith(nodeId, {
caps: [],
commands: [],
permissions: undefined,
});
expect(opts.context.invalidateClientsForDevice).toHaveBeenCalledWith(nodeId, {
reason: "device-pair-removed",
});
});
it("disconnects active clients after removing a paired device", async () => {
removePairedDeviceMock.mockResolvedValue({ deviceId: "device-1", removedAtMs: 123 });
const opts = createOptions("device.pair.remove", { deviceId: " device-1 " });
@@ -477,6 +539,70 @@ describe("deviceHandlers", () => {
);
});
it("invalidates an in-flight node wake when the node token rotates", async () => {
rotateDeviceTokenMock.mockResolvedValue({
ok: true,
entry: {
token: "new-node-token",
role: "node",
scopes: [],
createdAtMs: 456,
rotatedAtMs: 789,
},
});
const lifecycle = captureNodeWakeLifecycle("device-1");
const opts = createOptions(
"device.token.rotate",
{ deviceId: "device-1", role: "node" },
{ client: createClient(["operator.admin"], "admin-device", { isDeviceTokenAuth: true }) },
);
await expectDefined(
deviceHandlers["device.token.rotate"],
'deviceHandlers["device.token.rotate"] test invariant',
)(opts);
expect(lifecycle.aborted).toBe(true);
});
it("invalidates an in-flight node wake when the node token is revoked", async () => {
revokeDeviceTokenMock.mockResolvedValue({
ok: true,
entry: { role: "node", revokedAtMs: 789 },
});
const lifecycle = captureNodeWakeLifecycle("device-1");
const opts = createOptions(
"device.token.revoke",
{ deviceId: "device-1", role: "node" },
{ client: createClient(["operator.admin"], "admin-device", { isDeviceTokenAuth: true }) },
);
await expectDefined(
deviceHandlers["device.token.revoke"],
'deviceHandlers["device.token.revoke"] test invariant',
)(opts);
expect(lifecycle.aborted).toBe(true);
});
it("keeps node wake ownership across unrelated operator token rotation", async () => {
mockRotateOperatorTokenSuccess();
const lifecycle = captureNodeWakeLifecycle("device-1");
const opts = createOptions("device.token.rotate", {
deviceId: "device-1",
role: "operator",
scopes: ["operator.pairing"],
});
await expectDefined(
deviceHandlers["device.token.rotate"],
'deviceHandlers["device.token.rotate"] test invariant',
)(opts);
expect(lifecycle.aborted).toBe(false);
releaseNodeWakeLifecycle("device-1", lifecycle);
});
it("invalidates affected clients synchronously before responding to device.token.rotate", async () => {
mockPairedOperatorDevice();
mockRotateOperatorTokenSuccess();
@@ -1054,6 +1180,44 @@ describe("deviceHandlers", () => {
expect(serialized).not.toContain("pk-2");
});
it("retires the previous node generation before returning reapproval success", async () => {
approveDevicePairingMock.mockResolvedValue({
status: "approved",
requestId: "req-node-repair",
nodePairingGenerationChanged: true,
device: {
deviceId: "node-repaired",
publicKey: "replacement-key",
role: "node",
roles: ["node"],
approvedAtMs: 200,
createdAtMs: 100,
},
});
const lifecycle = captureNodeWakeLifecycle("node-repaired");
const opts = createOptions("device.pair.approve", { requestId: "req-node-repair" });
const respond = vi.mocked(opts.respond);
const invalidate = vi.mocked(opts.context.invalidateClientsForDevice!);
const disconnect = vi.mocked(opts.context.disconnectClientsForDevice!);
respond.mockImplementation(() => {
expect(lifecycle.aborted).toBe(true);
expect(invalidate).toHaveBeenCalledWith("node-repaired", {
role: "node",
reason: "device-pairing-reapproved",
});
expect(disconnect).not.toHaveBeenCalled();
});
await expectDefined(
deviceHandlers["device.pair.approve"],
'deviceHandlers["device.pair.approve"] test invariant',
)(opts);
await Promise.resolve();
expect(respond).toHaveBeenCalledTimes(1);
expect(disconnect).toHaveBeenCalledWith("node-repaired", { role: "node" });
});
it("allows approving the caller device from a non-admin device session", async () => {
getPendingDevicePairingMock.mockResolvedValue({
requestId: "req-1",
src/gateway/server-methods/devices.ts
+30 -9
View File
@@ -28,6 +28,8 @@ import {
updatePairedDeviceMetadata,
} from "../../infra/device-pairing.js";
import type { DiagnosticSecurityEventInput } from "../../infra/diagnostic-events.js";
import { clearRemovedNodeRuntimeState } from "../node-runtime-state.js";
import { invalidateNodeWakeState } from "../node-wake-state.js";
import {
deniesCrossDeviceManagement,
deniesDeviceTokenRoleManagement,
@@ -329,6 +331,16 @@ export const deviceHandlers: GatewayRequestHandlers = {
);
return;
}
const normalizedDeviceId = approved.device.deviceId.trim();
if (approved.nodePairingGenerationChanged) {
invalidateNodeWakeState(normalizedDeviceId);
// Mark the retired node generation before publishing success so buffered
// node RPCs cannot retain authority while transport teardown is deferred.
context.invalidateClientsForDevice?.(normalizedDeviceId, {
role: "node",
reason: "device-pairing-reapproved",
});
}
context.logGateway.info(
`device pairing approved device=${approved.device.deviceId} role=${approved.device.role ?? "unknown"}`,
);
@@ -354,6 +366,11 @@ export const deviceHandlers: GatewayRequestHandlers = {
{ dropIfSlow: true },
);
respond(true, { requestId, device: redactPairedDevice(approved.device) }, undefined);
if (approved.nodePairingGenerationChanged) {
queueMicrotask(() => {
context.disconnectClientsForDevice?.(normalizedDeviceId, { role: "node" });
});
}
},
"device.pair.reject": async ({ params, respond, context, client }) => {
if (!validateDevicePairRejectParams(params)) {
@@ -481,6 +498,10 @@ export const deviceHandlers: GatewayRequestHandlers = {
respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, "unknown deviceId"));
return;
}
clearRemovedNodeRuntimeState({ nodeId: removed.deviceId, context });
context.invalidateClientsForDevice?.(removed.deviceId, {
reason: "device-pair-removed",
});
context.logGateway.info(`device pairing removed device=${removed.deviceId}`);
emitDevicePairingLifecycleSecurityEvent({
action: "device.pairing.removed",
@@ -489,13 +510,6 @@ export const deviceHandlers: GatewayRequestHandlers = {
targetDeviceId: removed.deviceId,
controlId: "device.pair.remove",
});
// Mark affected clients invalid *before* responding so any RPCs already
// pipelined into their WS socket buffer are rejected at the per-request
// dispatch check, closing the race between queueMicrotask-scheduled
// disconnect and inflight frames.
context.invalidateClientsForDevice?.(removed.deviceId, {
reason: "device-pair-removed",
});
respond(true, removed, undefined);
queueMicrotask(() => {
context.disconnectClientsForDevice?.(removed.deviceId);
@@ -671,6 +685,7 @@ export const deviceHandlers: GatewayRequestHandlers = {
return;
}
const entry = rotated.entry;
const normalizedDeviceId = deviceId.trim();
context.logGateway.info(
`device token rotated device=${deviceId} role=${entry.role} scopes=${entry.scopes.join(",")}`,
);
@@ -683,11 +698,14 @@ export const deviceHandlers: GatewayRequestHandlers = {
role: entry.role,
scopeCount: entry.scopes.length,
});
if (entry.role === "node") {
invalidateNodeWakeState(normalizedDeviceId);
}
// Mark affected clients invalid *before* responding so any RPCs already
// pipelined into their WS socket buffer are rejected at the per-request
// dispatch check, closing the race between queueMicrotask-scheduled
// disconnect and inflight frames.
context.invalidateClientsForDevice?.(deviceId.trim(), {
context.invalidateClientsForDevice?.(normalizedDeviceId, {
role: entry.role,
reason: "device-token-rotated",
});
@@ -703,7 +721,7 @@ export const deviceHandlers: GatewayRequestHandlers = {
undefined,
);
queueMicrotask(() => {
context.disconnectClientsForDevice?.(deviceId.trim(), { role: entry.role });
context.disconnectClientsForDevice?.(normalizedDeviceId, { role: entry.role });
});
},
"device.token.revoke": async ({ params, respond, context, client }) => {
@@ -798,6 +816,9 @@ export const deviceHandlers: GatewayRequestHandlers = {
controlId: "device.token.revoke",
role: entry.role,
});
if (entry.role === "node") {
invalidateNodeWakeState(normalizedDeviceId);
}
// Mark affected clients invalid *before* responding so any RPCs already
// pipelined into their WS socket buffer are rejected at the per-request
// dispatch check, closing the race between queueMicrotask-scheduled
src/gateway/server-methods/environments.test.ts
+2 -1
View File
@@ -11,6 +11,7 @@ import { environmentsHandlers, summarizeWorkerEnvironment } from "./environments
vi.mock("../../infra/device-pairing.js", () => ({
listDevicePairing: vi.fn(),
resolveNodePairingState: vi.fn(),
}));
vi.mock("../../infra/node-pairing.js", () => ({
@@ -41,7 +42,7 @@ function mockContext(
warn: vi.fn(),
},
nodeRegistry: {
listConnected: () => [
listConnectedForPairingStates: () => [
{
nodeId: "node-live",
connId: "conn-live",
src/gateway/server-methods/environments.ts
+12 -2
View File
@@ -8,7 +8,7 @@ import {
validateEnvironmentsListParams,
validateEnvironmentsStatusParams,
} from "../../../packages/gateway-protocol/src/index.js";
import { listDevicePairing } from "../../infra/device-pairing.js";
import { listDevicePairing, resolveNodePairingState } from "../../infra/device-pairing.js";
import { listNodePairing } from "../../infra/node-pairing.js";
import type { NodeListNode } from "../../shared/node-list-types.js";
import { createKnownNodeCatalog, listKnownNodes } from "../node-catalog.js";
@@ -84,10 +84,20 @@ export function summarizeWorkerEnvironment(
}
async function listEnvironments(context: GatewayRequestContext): Promise<EnvironmentSummary[]> {
const [devices, nodes] = await Promise.all([listDevicePairing(), listNodePairing()]);
const currentPairingStates = new Map<string, { identity: string; generation?: string }>();
for (const device of devices.paired) {
const state = resolveNodePairingState(device);
if (state) {
currentPairingStates.set(state.identity.nodeId, {
identity: state.identity.key,
...(state.generation ? { generation: state.generation.key } : {}),
});
}
}
const catalog = createKnownNodeCatalog({
pairedDevices: devices.paired,
pairedNodes: nodes.paired,
connectedNodes: context.nodeRegistry.listConnected(),
connectedNodes: context.nodeRegistry.listConnectedForPairingStates(currentPairingStates),
});
return [GATEWAY_ENVIRONMENT, ...listKnownNodes(catalog).map(summarizeNodeEnvironment)];
}
src/gateway/server-methods/exec-approvals.test.ts
+11
View File
@@ -251,6 +251,7 @@ describe("exec approvals gateway methods", () => {
get: () => ({
nodeId: "node-1",
connId: "conn-1",
pairingGeneration: "generation-1",
platform: "windows",
deviceFamily: "Windows",
declaredCommands: [testCase.command],
@@ -309,6 +310,7 @@ describe("exec approvals gateway methods", () => {
get: () => ({
nodeId: "node-1",
connId: "conn-1",
pairingGeneration: "generation-1",
clientId: "openclaw-macos",
clientMode: "node",
platform: "macOS 26.5.2",
@@ -323,6 +325,8 @@ describe("exec approvals gateway methods", () => {
expect(invoke).toHaveBeenCalledWith({
nodeId: "node-1",
expectedConnId: "conn-1",
expectedPairingGeneration: "generation-1",
command,
params: { includeResolvedDefaults: true },
});
@@ -396,6 +400,7 @@ describe("exec approvals gateway methods", () => {
get: () => ({
nodeId: "node-1",
connId: "conn-1",
pairingGeneration: "generation-1",
clientId: identity.clientId,
clientMode: identity.clientMode,
platform: identity.platform,
@@ -410,6 +415,8 @@ describe("exec approvals gateway methods", () => {
expect(invoke).toHaveBeenCalledWith({
nodeId: "node-1",
expectedConnId: "conn-1",
expectedPairingGeneration: "generation-1",
command,
params: {},
});
@@ -452,6 +459,7 @@ describe("exec approvals gateway methods", () => {
get: () => ({
nodeId: "windows-node",
connId: "conn-1",
pairingGeneration: "generation-1",
platform: "windows",
deviceFamily: "Windows",
declaredCommands: [command],
@@ -464,6 +472,8 @@ describe("exec approvals gateway methods", () => {
expect(invoke).toHaveBeenCalledWith({
nodeId: "windows-node",
expectedConnId: "conn-1",
expectedPairingGeneration: "generation-1",
command,
params: {
defaultAction: "deny",
@@ -498,6 +508,7 @@ describe("exec approvals gateway methods", () => {
get: () => ({
nodeId: "windows-node",
connId: "conn-1",
pairingGeneration: "generation-1",
platform: "windows",
deviceFamily: "Windows",
declaredCommands: [command],
src/gateway/server-methods/exec-approvals.ts
+8
View File
@@ -166,6 +166,14 @@ async function respondWithExecApprovalsNodePayload<TParams extends { nodeId: str
await respondUnavailableOnThrow(params.respond, async () => {
const res = await params.context.nodeRegistry.invoke({
nodeId,
...(nodeSession
? {
expectedConnId: nodeSession.connId,
...(nodeSession.pairingGeneration
? { expectedPairingGeneration: nodeSession.pairingGeneration }
: {}),
}
: {}),
command: params.command,
params: params.commandParams(parsedParams, nodeSession),
});
src/gateway/server-methods/fs.test.ts
+2
View File
@@ -122,6 +122,7 @@ describe("fs.listDir", () => {
nodeRegistry: {
get: vi.fn().mockReturnValue({
connId: "conn-1",
pairingGeneration: "generation-1",
nodeId: "macbook",
platform: "macos",
deviceFamily: "Mac",
@@ -141,6 +142,7 @@ describe("fs.listDir", () => {
expect(invoke).toHaveBeenCalledWith({
nodeId: "macbook",
expectedConnId: "conn-1",
expectedPairingGeneration: "generation-1",
command: "fs.listDir",
params: {},
});
src/gateway/server-methods/fs.ts
+1
View File
@@ -69,6 +69,7 @@ export const fsHandlers: GatewayRequestHandlers = {
const result = await context.nodeRegistry.invoke({
nodeId: params.nodeId,
expectedConnId: node.connId,
...(node.pairingGeneration ? { expectedPairingGeneration: node.pairingGeneration } : {}),
command: NODE_FS_LIST_DIR_COMMAND,
params: params.path ? { path: params.path } : {},
});
src/gateway/server-methods/nodes-pending.test.ts
+296 -7
View File
@@ -7,21 +7,39 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import { nodePendingHandlers } from "./nodes-pending.js";
const mocks = vi.hoisted(() => ({
captureNodePairingGeneration: vi.fn(),
captureNodeWakeLifecycle: vi.fn(),
drainNodePendingWork: vi.fn(),
enqueueNodePendingWork: vi.fn(),
isNodePairingGenerationCurrent: vi.fn(),
isNodeWakeLifecycleCurrent: vi.fn(),
maybeWakeNodeWithApns: vi.fn(),
maybeSendNodeWakeNudge: vi.fn(),
removeNodePendingWorkItem: vi.fn(),
releaseNodeWakeLifecycle: vi.fn(),
waitForNodeReconnect: vi.fn(),
}));
vi.mock("../node-pending-work.js", () => ({
drainNodePendingWork: mocks.drainNodePendingWork,
enqueueNodePendingWork: mocks.enqueueNodePendingWork,
removeNodePendingWorkItem: mocks.removeNodePendingWorkItem,
}));
vi.mock("../../infra/node-pairing-state.js", () => ({
captureNodePairingGeneration: mocks.captureNodePairingGeneration,
isNodePairingGenerationCurrent: mocks.isNodePairingGenerationCurrent,
}));
vi.mock("../node-wake-state.js", () => ({
NODE_WAKE_RECONNECT_WAIT_MS: 3_000,
NODE_WAKE_RECONNECT_RETRY_WAIT_MS: 12_000,
captureNodeWakeLifecycle: mocks.captureNodeWakeLifecycle,
isNodeWakeLifecycleCurrent: mocks.isNodeWakeLifecycleCurrent,
releaseNodeWakeLifecycle: mocks.releaseNodeWakeLifecycle,
}));
vi.mock("./nodes.js", () => ({
NODE_WAKE_RECONNECT_WAIT_MS: 3_000,
NODE_WAKE_RECONNECT_RETRY_WAIT_MS: 12_000,
maybeWakeNodeWithApns: mocks.maybeWakeNodeWithApns,
maybeSendNodeWakeNudge: mocks.maybeSendNodeWakeNudge,
waitForNodeReconnect: mocks.waitForNodeReconnect,
@@ -41,6 +59,7 @@ function makeContext(overrides?: Partial<Record<string, unknown>>) {
return {
nodeRegistry: {
get: vi.fn(() => undefined),
getForPairingGeneration: vi.fn(() => undefined),
},
logGateway: {
info: vi.fn(),
@@ -57,10 +76,21 @@ function respondCall(respond: ReturnType<typeof vi.fn>): RespondCall | undefined
describe("node.pending handlers", () => {
beforeEach(() => {
mocks.captureNodePairingGeneration.mockReset().mockImplementation(async (nodeId: string) => ({
nodeId,
key: `generation:${nodeId}:1`,
}));
mocks.captureNodeWakeLifecycle.mockReset();
mocks.drainNodePendingWork.mockReset();
mocks.enqueueNodePendingWork.mockReset();
mocks.isNodePairingGenerationCurrent.mockReset().mockResolvedValue(true);
mocks.isNodeWakeLifecycleCurrent
.mockReset()
.mockImplementation((_nodeId: string, lifecycle: AbortSignal) => !lifecycle.aborted);
mocks.maybeWakeNodeWithApns.mockReset();
mocks.maybeSendNodeWakeNudge.mockReset();
mocks.removeNodePendingWorkItem.mockReset();
mocks.releaseNodeWakeLifecycle.mockReset();
mocks.waitForNodeReconnect.mockReset();
});
@@ -71,6 +101,12 @@ describe("node.pending handlers", () => {
hasMore: false,
});
const respond = vi.fn();
const context = makeContext({
nodeRegistry: {
get: vi.fn(() => undefined),
getForPairingGeneration: vi.fn(() => ({ connId: "conn-ios-node-1" })),
},
});
await expectDefined(
nodePendingHandlers["node.pending.drain"],
@@ -78,8 +114,11 @@ describe("node.pending handlers", () => {
)({
params: { maxItems: 3 },
respond: respond as never,
client: { connect: { device: { id: "ios-node-1" } } } as never,
context: makeContext() as never,
client: {
connId: "conn-ios-node-1",
connect: { device: { id: "ios-node-1" } },
} as never,
context: context as never,
req: { type: "req", id: "req-node-pending-drain", method: "node.pending.drain" },
isWebchatConnect: () => false,
});
@@ -87,6 +126,7 @@ describe("node.pending handlers", () => {
expect(mocks.drainNodePendingWork).toHaveBeenCalledWith("ios-node-1", {
maxItems: 3,
includeDefaultStatus: true,
pairingGeneration: "generation:ios-node-1:1",
});
expect(respond).toHaveBeenCalledWith(
true,
@@ -120,7 +160,111 @@ describe("node.pending handlers", () => {
expect(call?.[2]?.message).toContain("connected device identity");
});
it("enqueues pending work and wakes a disconnected node once", async () => {
it("rejects a changed pairing before draining its pending work", async () => {
mocks.isNodePairingGenerationCurrent.mockResolvedValue(false);
const respond = vi.fn();
const context = makeContext({
nodeRegistry: {
get: vi.fn(() => undefined),
getForPairingGeneration: vi.fn(() => ({ connId: "conn-stale-drain" })),
},
});
await expectDefined(
nodePendingHandlers["node.pending.drain"],
'nodePendingHandlers["node.pending.drain"] test invariant',
)({
params: {},
respond: respond as never,
client: {
connId: "conn-stale-drain",
connect: { device: { id: "ios-node-stale-drain" } },
} as never,
context: context as never,
req: { type: "req", id: "req-node-stale-drain", method: "node.pending.drain" },
isWebchatConnect: () => false,
});
expect(respondCall(respond)).toMatchObject([
false,
undefined,
{ details: { code: "PAIRING_CHANGED" } },
]);
expect(mocks.drainNodePendingWork).not.toHaveBeenCalled();
});
it("rejects a same-generation reconnect before destructively draining", async () => {
let currentConnId = "conn-original";
mocks.isNodePairingGenerationCurrent.mockImplementation(async () => {
currentConnId = "conn-replacement";
return true;
});
const respond = vi.fn();
const context = makeContext({
nodeRegistry: {
get: vi.fn(() => undefined),
getForPairingGeneration: vi.fn(() => ({ connId: currentConnId })),
},
});
await expectDefined(
nodePendingHandlers["node.pending.drain"],
'nodePendingHandlers["node.pending.drain"] test invariant',
)({
params: {},
respond: respond as never,
client: {
connId: "conn-original",
connect: { device: { id: "ios-node-reconnected" } },
} as never,
context: context as never,
req: { type: "req", id: "req-node-reconnected-drain", method: "node.pending.drain" },
isWebchatConnect: () => false,
});
expect(mocks.drainNodePendingWork).not.toHaveBeenCalled();
expect(respondCall(respond)).toMatchObject([
false,
undefined,
{ details: { code: "PAIRING_CHANGED" } },
]);
});
it("rejects a prior-generation socket before it drains replacement work", async () => {
const respond = vi.fn();
const context = makeContext({
nodeRegistry: {
get: vi.fn(() => undefined),
getForPairingGeneration: vi.fn(() => ({ connId: "conn-replacement" })),
},
});
await expectDefined(
nodePendingHandlers["node.pending.drain"],
'nodePendingHandlers["node.pending.drain"] test invariant',
)({
params: {},
respond: respond as never,
client: {
connId: "conn-prior-generation",
connect: { device: { id: "ios-node-replaced" } },
} as never,
context: context as never,
req: { type: "req", id: "req-node-prior-drain", method: "node.pending.drain" },
isWebchatConnect: () => false,
});
expect(mocks.drainNodePendingWork).not.toHaveBeenCalled();
expect(respondCall(respond)).toMatchObject([
false,
undefined,
{ details: { code: "PAIRING_CHANGED" } },
]);
});
it("normalizes the target identity before queueing and waking a disconnected node", async () => {
const wakeLifecycle = new AbortController().signal;
mocks.captureNodeWakeLifecycle.mockReturnValue(wakeLifecycle);
mocks.enqueueNodePendingWork.mockReturnValue({
revision: 4,
deduped: false,
@@ -147,7 +291,8 @@ describe("node.pending handlers", () => {
});
const context = makeContext({
nodeRegistry: {
get: vi.fn(() => (connected ? { nodeId: "ios-node-2" } : undefined)),
get: vi.fn(() => undefined),
getForPairingGeneration: vi.fn(() => (connected ? { nodeId: "ios-node-2" } : undefined)),
},
});
const respond = vi.fn();
@@ -157,7 +302,7 @@ describe("node.pending handlers", () => {
'nodePendingHandlers["node.pending.enqueue"] test invariant',
)({
params: {
nodeId: "ios-node-2",
nodeId: " ios-node-2 ",
type: "location.request",
priority: "high",
},
@@ -173,17 +318,34 @@ describe("node.pending handlers", () => {
type: "location.request",
priority: "high",
expiresInMs: undefined,
pairingGeneration: "generation:ios-node-2:1",
});
expect(context.nodeRegistry.getForPairingGeneration).toHaveBeenCalledWith(
"ios-node-2",
"generation:ios-node-2:1",
);
expect(context.nodeRegistry.getForPairingGeneration).not.toHaveBeenCalledWith(
" ios-node-2 ",
expect.anything(),
);
expect(mocks.maybeWakeNodeWithApns).toHaveBeenCalledWith("ios-node-2", {
wakeReason: "node.pending",
cfg: {},
lifecycle: wakeLifecycle,
generation: {
nodeId: "ios-node-2",
key: "generation:ios-node-2:1",
},
});
expect(mocks.waitForNodeReconnect).toHaveBeenCalledWith({
nodeId: "ios-node-2",
context,
timeoutMs: 3_000,
lifecycle: wakeLifecycle,
pairingGeneration: "generation:ios-node-2:1",
});
expect(mocks.maybeSendNodeWakeNudge).not.toHaveBeenCalled();
expect(mocks.releaseNodeWakeLifecycle).toHaveBeenCalledWith("ios-node-2", wakeLifecycle);
const call = respondCall(respond) as
| [boolean, { nodeId?: string; revision?: number; wakeTriggered?: boolean }, unknown?]
| undefined;
@@ -193,4 +355,131 @@ describe("node.pending handlers", () => {
expect(call?.[1]?.wakeTriggered).toBe(true);
expect(call?.[2]).toBeUndefined();
});
it("returns unavailable when pairing removal invalidates an enqueued item", async () => {
const lifecycleController = new AbortController();
const wakeLifecycle = lifecycleController.signal;
mocks.captureNodeWakeLifecycle.mockReturnValue(wakeLifecycle);
mocks.enqueueNodePendingWork.mockReturnValue({
revision: 5,
deduped: false,
item: {
id: "pending-invalidated",
type: "location.request",
priority: "default",
createdAtMs: 100,
expiresAtMs: null,
},
});
mocks.maybeWakeNodeWithApns.mockImplementation(async () =>
wakeLifecycle.aborted
? {
available: false,
throttled: false,
path: "invalidated",
durationMs: 0,
}
: {
available: true,
throttled: false,
path: "sent",
durationMs: 1,
},
);
mocks.waitForNodeReconnect.mockImplementation(async () => {
lifecycleController.abort();
return false;
});
mocks.maybeSendNodeWakeNudge.mockResolvedValue({
sent: false,
throttled: false,
reason: "invalidated",
durationMs: 0,
});
const context = makeContext();
const respond = vi.fn();
await expectDefined(
nodePendingHandlers["node.pending.enqueue"],
'nodePendingHandlers["node.pending.enqueue"] test invariant',
)({
params: { nodeId: " ios-node-invalidated ", type: "location.request" },
respond: respond as never,
client: null,
context: context as never,
req: { type: "req", id: "req-node-pending-invalidated", method: "node.pending.enqueue" },
isWebchatConnect: () => false,
});
expect(mocks.captureNodeWakeLifecycle).toHaveBeenCalledWith(
"ios-node-invalidated",
"generation:ios-node-invalidated:1",
);
expect(mocks.maybeWakeNodeWithApns).toHaveBeenCalledTimes(1);
for (const call of mocks.maybeWakeNodeWithApns.mock.calls) {
expect(call[0]).toBe("ios-node-invalidated");
expect(call[1]).toMatchObject({ lifecycle: wakeLifecycle });
}
expect(mocks.waitForNodeReconnect).toHaveBeenCalledWith({
nodeId: "ios-node-invalidated",
context,
timeoutMs: 3_000,
lifecycle: wakeLifecycle,
pairingGeneration: "generation:ios-node-invalidated:1",
});
expect(mocks.maybeSendNodeWakeNudge).not.toHaveBeenCalled();
expect(mocks.releaseNodeWakeLifecycle).toHaveBeenCalledWith(
"ios-node-invalidated",
wakeLifecycle,
);
expect(mocks.removeNodePendingWorkItem).toHaveBeenCalledWith({
nodeId: "ios-node-invalidated",
itemId: "pending-invalidated",
pairingGeneration: "generation:ios-node-invalidated:1",
});
const call = respondCall(respond);
expect(call?.[0]).toBe(false);
expect(call?.[2]).toMatchObject({
message: "node pairing changed while pending work was active",
details: { code: "PAIRING_CHANGED" },
});
});
it("does not remove replacement work when an invalidated enqueue reused it", async () => {
const wakeLifecycle = new AbortController().signal;
mocks.captureNodeWakeLifecycle.mockReturnValue(wakeLifecycle);
mocks.enqueueNodePendingWork.mockReturnValue({
revision: 9,
deduped: true,
item: {
id: "replacement-work",
type: "location.request",
priority: "normal",
createdAtMs: 200,
expiresAtMs: null,
},
});
mocks.isNodePairingGenerationCurrent.mockResolvedValueOnce(true).mockResolvedValueOnce(false);
const respond = vi.fn();
await expectDefined(
nodePendingHandlers["node.pending.enqueue"],
'nodePendingHandlers["node.pending.enqueue"] test invariant',
)({
params: { nodeId: "node-replacement", type: "location.request", wake: false },
respond: respond as never,
client: null,
context: makeContext() as never,
req: { type: "req", id: "req-node-replacement", method: "node.pending.enqueue" },
isWebchatConnect: () => false,
});
expect(mocks.enqueueNodePendingWork).toHaveBeenCalledTimes(1);
expect(mocks.removeNodePendingWorkItem).not.toHaveBeenCalled();
expect(respondCall(respond)).toMatchObject([
false,
undefined,
{ details: { code: "PAIRING_CHANGED" } },
]);
});
});
src/gateway/server-methods/nodes-pending.ts
+194 -85
View File
@@ -6,22 +6,52 @@ import {
validateNodePendingDrainParams,
validateNodePendingEnqueueParams,
} from "../../../packages/gateway-protocol/src/index.js";
import {
captureNodePairingGeneration,
isNodePairingGenerationCurrent,
type NodePairingGeneration,
} from "../../infra/node-pairing-state.js";
import {
drainNodePendingWork,
enqueueNodePendingWork,
removeNodePendingWorkItem,
type NodePendingWorkPriority,
type NodePendingWorkType,
} from "../node-pending-work.js";
import { respondInvalidParams, respondUnavailableOnThrow } from "./nodes.helpers.js";
import {
maybeSendNodeWakeNudge,
maybeWakeNodeWithApns,
captureNodeWakeLifecycle,
isNodeWakeLifecycleCurrent,
NODE_WAKE_RECONNECT_RETRY_WAIT_MS,
NODE_WAKE_RECONNECT_WAIT_MS,
waitForNodeReconnect,
} from "./nodes.js";
releaseNodeWakeLifecycle,
} from "../node-wake-state.js";
import { respondInvalidParams, respondUnavailableOnThrow } from "./nodes.helpers.js";
import { maybeSendNodeWakeNudge, maybeWakeNodeWithApns, waitForNodeReconnect } from "./nodes.js";
import type { RespondFn } from "./shared-types.js";
import type { GatewayRequestHandlers } from "./types.js";
function respondPairingChanged(respond: RespondFn) {
respond(
false,
undefined,
errorShape(ErrorCodes.UNAVAILABLE, "node pairing changed while pending work was active", {
retryable: true,
details: { code: "PAIRING_CHANGED" },
}),
);
}
async function isPendingGenerationCurrent(params: {
nodeId: string;
generation: NodePairingGeneration;
lifecycle: AbortSignal;
}): Promise<boolean> {
return (
isNodeWakeLifecycleCurrent(params.nodeId, params.lifecycle, params.generation.key) &&
(await isNodePairingGenerationCurrent(params.generation))
);
}
function resolveClientNodeId(
client: { connect?: { device?: { id?: string }; client?: { id?: string } } } | null,
): string | null {
@@ -32,7 +62,7 @@ function resolveClientNodeId(
/** Gateway handlers for queueing work until a paired node reconnects. */
export const nodePendingHandlers: GatewayRequestHandlers = {
"node.pending.drain": async ({ params, respond, client }) => {
"node.pending.drain": async ({ params, respond, client, context }) => {
if (!validateNodePendingDrainParams(params)) {
respondInvalidParams({
respond,
@@ -53,12 +83,27 @@ export const nodePendingHandlers: GatewayRequestHandlers = {
);
return;
}
const p = params as { maxItems?: number };
const drained = drainNodePendingWork(nodeId, {
maxItems: p.maxItems,
includeDefaultStatus: true,
await respondUnavailableOnThrow(respond, async () => {
const generation = await captureNodePairingGeneration(nodeId);
if (!generation || !(await isNodePairingGenerationCurrent(generation))) {
respondPairingChanged(respond);
return;
}
// Draining deletes work, so the authenticated caller must still be the
// registry session that owns the persisted generation.
const session = context.nodeRegistry.getForPairingGeneration(nodeId, generation.key);
if (!client?.connId || session?.connId !== client.connId) {
respondPairingChanged(respond);
return;
}
const p = params as { maxItems?: number };
const drained = drainNodePendingWork(nodeId, {
maxItems: p.maxItems,
includeDefaultStatus: true,
pairingGeneration: generation.key,
});
respond(true, { nodeId, ...drained }, undefined);
});
respond(true, { nodeId, ...drained }, undefined);
},
"node.pending.enqueue": async ({ params, respond, context }) => {
if (!validateNodePendingEnqueueParams(params)) {
@@ -77,95 +122,159 @@ export const nodePendingHandlers: GatewayRequestHandlers = {
wake?: boolean;
};
await respondUnavailableOnThrow(respond, async () => {
const queued = enqueueNodePendingWork({
nodeId: p.nodeId,
type: p.type,
priority: p.priority,
expiresInMs: p.expiresInMs,
});
let wakeTriggered = false;
if (p.wake !== false && !queued.deduped && !context.nodeRegistry.get(p.nodeId)) {
const wakeReqId = queued.item.id;
context.logGateway.info(
`node pending wake start node=${p.nodeId} req=${wakeReqId} type=${queued.item.type}`,
);
const cfg = context.getRuntimeConfig();
const wake = await maybeWakeNodeWithApns(p.nodeId, {
wakeReason: "node.pending",
cfg,
});
context.logGateway.info(
`node pending wake stage=wake1 node=${p.nodeId} req=${wakeReqId} ` +
`available=${wake.available} throttled=${wake.throttled} ` +
`path=${wake.path} durationMs=${wake.durationMs} ` +
`apnsStatus=${wake.apnsStatus ?? -1} apnsReason=${wake.apnsReason ?? "-"}`,
);
wakeTriggered = wake.available;
if (wake.available) {
// Give the first wake a short reconnect window before forcing a
// second wake; this keeps normal APNs delivery cheap and quiet.
const reconnected = await waitForNodeReconnect({
nodeId: p.nodeId,
context,
timeoutMs: NODE_WAKE_RECONNECT_WAIT_MS,
});
context.logGateway.info(
`node pending wake stage=wait1 node=${p.nodeId} req=${wakeReqId} ` +
`reconnected=${reconnected} timeoutMs=${NODE_WAKE_RECONNECT_WAIT_MS}`,
);
const nodeId = p.nodeId.trim();
const generation = await captureNodePairingGeneration(nodeId);
if (!generation) {
respondPairingChanged(respond);
return;
}
const wakeLifecycle = captureNodeWakeLifecycle(nodeId, generation.key);
try {
if (!(await isPendingGenerationCurrent({ nodeId, generation, lifecycle: wakeLifecycle }))) {
respondPairingChanged(respond);
return;
}
if (!context.nodeRegistry.get(p.nodeId) && wake.available) {
// A forced retry is only useful after the first wake was deliverable
// but the node still has not reattached to the Gateway.
const retryWake = await maybeWakeNodeWithApns(p.nodeId, {
force: true,
const queued = enqueueNodePendingWork({
nodeId,
type: p.type,
priority: p.priority,
expiresInMs: p.expiresInMs,
pairingGeneration: generation.key,
});
let wakeTriggered = false;
if (
p.wake !== false &&
!queued.deduped &&
!context.nodeRegistry.getForPairingGeneration(nodeId, generation.key)
) {
const wakeReqId = queued.item.id;
context.logGateway.info(
`node pending wake start node=${nodeId} req=${wakeReqId} type=${queued.item.type}`,
);
const cfg = context.getRuntimeConfig();
const wake = await maybeWakeNodeWithApns(nodeId, {
wakeReason: "node.pending",
cfg,
lifecycle: wakeLifecycle,
generation,
});
context.logGateway.info(
`node pending wake stage=wake2 node=${p.nodeId} req=${wakeReqId} force=true ` +
`available=${retryWake.available} throttled=${retryWake.throttled} ` +
`path=${retryWake.path} durationMs=${retryWake.durationMs} ` +
`apnsStatus=${retryWake.apnsStatus ?? -1} apnsReason=${retryWake.apnsReason ?? "-"}`,
`node pending wake stage=wake1 node=${nodeId} req=${wakeReqId} ` +
`available=${wake.available} throttled=${wake.throttled} ` +
`path=${wake.path} durationMs=${wake.durationMs} ` +
`apnsStatus=${wake.apnsStatus ?? -1} apnsReason=${wake.apnsReason ?? "-"}`,
);
if (retryWake.available) {
wakeTriggered = wake.available;
if (wake.available) {
// Give the first wake a short reconnect window before forcing a
// second wake; this keeps normal APNs delivery cheap and quiet.
const reconnected = await waitForNodeReconnect({
nodeId: p.nodeId,
nodeId,
context,
timeoutMs: NODE_WAKE_RECONNECT_RETRY_WAIT_MS,
timeoutMs: NODE_WAKE_RECONNECT_WAIT_MS,
lifecycle: wakeLifecycle,
pairingGeneration: generation.key,
});
context.logGateway.info(
`node pending wake stage=wait2 node=${p.nodeId} req=${wakeReqId} ` +
`reconnected=${reconnected} timeoutMs=${NODE_WAKE_RECONNECT_RETRY_WAIT_MS}`,
`node pending wake stage=wait1 node=${nodeId} req=${wakeReqId} ` +
`reconnected=${reconnected} timeoutMs=${NODE_WAKE_RECONNECT_WAIT_MS}`,
);
}
if (
(await isPendingGenerationCurrent({
nodeId,
generation,
lifecycle: wakeLifecycle,
})) &&
!context.nodeRegistry.getForPairingGeneration(nodeId, generation.key) &&
wake.available
) {
// A forced retry is only useful after the first wake was deliverable
// but the node still has not reattached to the Gateway.
const retryWake = await maybeWakeNodeWithApns(nodeId, {
force: true,
wakeReason: "node.pending",
cfg,
lifecycle: wakeLifecycle,
generation,
});
context.logGateway.info(
`node pending wake stage=wake2 node=${nodeId} req=${wakeReqId} force=true ` +
`available=${retryWake.available} throttled=${retryWake.throttled} ` +
`path=${retryWake.path} durationMs=${retryWake.durationMs} ` +
`apnsStatus=${retryWake.apnsStatus ?? -1} apnsReason=${retryWake.apnsReason ?? "-"}`,
);
if (retryWake.available) {
const reconnected = await waitForNodeReconnect({
nodeId,
context,
timeoutMs: NODE_WAKE_RECONNECT_RETRY_WAIT_MS,
lifecycle: wakeLifecycle,
pairingGeneration: generation.key,
});
context.logGateway.info(
`node pending wake stage=wait2 node=${nodeId} req=${wakeReqId} ` +
`reconnected=${reconnected} timeoutMs=${NODE_WAKE_RECONNECT_RETRY_WAIT_MS}`,
);
}
}
if (
(await isPendingGenerationCurrent({
nodeId,
generation,
lifecycle: wakeLifecycle,
})) &&
!context.nodeRegistry.getForPairingGeneration(nodeId, generation.key)
) {
const nudge = await maybeSendNodeWakeNudge(nodeId, {
cfg,
lifecycle: wakeLifecycle,
generation,
});
context.logGateway.info(
`node pending wake nudge node=${nodeId} req=${wakeReqId} sent=${nudge.sent} ` +
`throttled=${nudge.throttled} reason=${nudge.reason} durationMs=${nudge.durationMs} ` +
`apnsStatus=${nudge.apnsStatus ?? -1} apnsReason=${nudge.apnsReason ?? "-"}`,
);
context.logGateway.warn(
`node pending wake done node=${nodeId} req=${wakeReqId} connected=false reason=not_connected`,
);
} else if (
await isPendingGenerationCurrent({
nodeId,
generation,
lifecycle: wakeLifecycle,
})
) {
context.logGateway.info(
`node pending wake done node=${nodeId} req=${wakeReqId} connected=true`,
);
}
}
if (!context.nodeRegistry.get(p.nodeId)) {
const nudge = await maybeSendNodeWakeNudge(p.nodeId, { cfg });
context.logGateway.info(
`node pending wake nudge node=${p.nodeId} req=${wakeReqId} sent=${nudge.sent} ` +
`throttled=${nudge.throttled} reason=${nudge.reason} durationMs=${nudge.durationMs} ` +
`apnsStatus=${nudge.apnsStatus ?? -1} apnsReason=${nudge.apnsReason ?? "-"}`,
);
context.logGateway.warn(
`node pending wake done node=${p.nodeId} req=${wakeReqId} connected=false reason=not_connected`,
);
} else {
context.logGateway.info(
`node pending wake done node=${p.nodeId} req=${wakeReqId} connected=true`,
);
if (!(await isPendingGenerationCurrent({ nodeId, generation, lifecycle: wakeLifecycle }))) {
if (!queued.deduped) {
removeNodePendingWorkItem({
nodeId,
itemId: queued.item.id,
pairingGeneration: generation.key,
});
}
respondPairingChanged(respond);
return;
}
respond(
true,
{
nodeId,
revision: queued.revision,
queued: queued.item,
wakeTriggered,
},
undefined,
);
} finally {
releaseNodeWakeLifecycle(nodeId, wakeLifecycle);
}
respond(
true,
{
nodeId: p.nodeId,
revision: queued.revision,
queued: queued.item,
wakeTriggered,
},
undefined,
);
});
},
};
src/gateway/server-methods/nodes-wake-state.test.ts
-107
View File
@@ -1,107 +0,0 @@
// Tests for node wake state tracking and testing seam.
import { beforeEach, describe, expect, it } from "vitest";
import {
NODE_WAKE_RECONNECT_WAIT_MS,
NODE_WAKE_RECONNECT_RETRY_WAIT_MS,
NODE_WAKE_RECONNECT_POLL_MS,
clearNodeWakeState,
nodeWakeById,
nodeWakeNudgeById,
} from "./nodes-wake-state.js";
beforeEach(() => {
nodeWakeById.clear();
nodeWakeNudgeById.clear();
});
describe("constants", () => {
it("exports expected wait/poll constants", () => {
expect(NODE_WAKE_RECONNECT_WAIT_MS).toBe(3_000);
expect(NODE_WAKE_RECONNECT_RETRY_WAIT_MS).toBe(12_000);
expect(NODE_WAKE_RECONNECT_POLL_MS).toBe(150);
});
});
describe("nodeWakeById", () => {
it("starts empty", () => {
expect(nodeWakeById.size).toBe(0);
});
it("stores NodeWakeState entries", () => {
const now = Date.now();
nodeWakeById.set("node-1", { lastWakeAtMs: now });
expect(nodeWakeById.size).toBe(1);
expect(nodeWakeById.get("node-1")?.lastWakeAtMs).toBe(now);
});
it("stores multiple entries", () => {
nodeWakeById.set("a", { lastWakeAtMs: 100 });
nodeWakeById.set("b", { lastWakeAtMs: 200 });
nodeWakeById.set("c", { lastWakeAtMs: 300 });
expect(nodeWakeById.size).toBe(3);
});
it("overwrites existing entry when key is reused", () => {
nodeWakeById.set("node-1", { lastWakeAtMs: 100 });
nodeWakeById.set("node-1", { lastWakeAtMs: 200 });
expect(nodeWakeById.size).toBe(1);
});
it("supports inFlight promise property", () => {
const promise = Promise.resolve({
available: true,
throttled: false,
path: "sent" as const,
durationMs: 50,
});
nodeWakeById.set("node-1", { lastWakeAtMs: Date.now(), inFlight: promise });
expect(nodeWakeById.get("node-1")?.inFlight).toBe(promise);
});
});
describe("nodeWakeNudgeById", () => {
it("starts empty", () => {
expect(nodeWakeNudgeById.size).toBe(0);
});
it("stores nudge timestamps", () => {
nodeWakeNudgeById.set("node-1", 1000);
expect(nodeWakeNudgeById.size).toBe(1);
expect(nodeWakeNudgeById.get("node-1")).toBe(1000);
});
it("independently tracked from nodeWakeById", () => {
nodeWakeById.set("node-1", { lastWakeAtMs: 500 });
nodeWakeNudgeById.set("node-1", 1000);
expect(nodeWakeById.size).toBe(1);
expect(nodeWakeNudgeById.size).toBe(1);
});
});
describe("clearNodeWakeState", () => {
it("removes the wake entry and nudge for the given node", () => {
nodeWakeById.set("node-1", { lastWakeAtMs: 100 });
nodeWakeNudgeById.set("node-1", 200);
clearNodeWakeState("node-1");
expect(nodeWakeById.has("node-1")).toBe(false);
expect(nodeWakeNudgeById.has("node-1")).toBe(false);
});
it("is a no-op when the node id does not exist", () => {
expect(() => clearNodeWakeState("ghost")).not.toThrow();
expect(nodeWakeById.size).toBe(0);
expect(nodeWakeNudgeById.size).toBe(0);
});
it("only removes the specified node, leaving others intact", () => {
nodeWakeById.set("a", { lastWakeAtMs: 1 });
nodeWakeById.set("b", { lastWakeAtMs: 2 });
nodeWakeNudgeById.set("a", 10);
nodeWakeNudgeById.set("b", 20);
clearNodeWakeState("a");
expect(nodeWakeById.has("a")).toBe(false);
expect(nodeWakeById.has("b")).toBe(true);
expect(nodeWakeNudgeById.has("a")).toBe(false);
expect(nodeWakeNudgeById.has("b")).toBe(true);
});
});
src/gateway/server-methods/nodes-wake-state.ts
-27
View File
@@ -1,27 +0,0 @@
// Node wake state tracks APNs wake attempts and reconnect nudges for paired
// nodes, plus a narrow read-only testing seam.
export const NODE_WAKE_RECONNECT_WAIT_MS = 3_000;
export const NODE_WAKE_RECONNECT_RETRY_WAIT_MS = 12_000;
export const NODE_WAKE_RECONNECT_POLL_MS = 150;
export type NodeWakeAttempt = {
available: boolean;
throttled: boolean;
path: "throttled" | "no-registration" | "no-auth" | "sent" | "send-error";
durationMs: number;
apnsStatus?: number;
apnsReason?: string;
};
type NodeWakeState = {
lastWakeAtMs: number;
inFlight?: Promise<NodeWakeAttempt>;
};
export const nodeWakeById = new Map<string, NodeWakeState>();
export const nodeWakeNudgeById = new Map<string, number>();
export function clearNodeWakeState(nodeId: string): void {
nodeWakeById.delete(nodeId);
nodeWakeNudgeById.delete(nodeId);
}
src/gateway/server-methods/nodes.invoke-wake.test.ts
+651 -14
View File
@@ -5,9 +5,17 @@ import { expectDefined } from "@openclaw/normalization-core";
import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { ErrorCodes } from "../../../packages/gateway-protocol/src/index.js";
import {
captureNodeWakeLifecycle,
clearNodeWakeState,
invalidateNodeWakeState,
} from "../node-wake-state.js";
import {
getNodeWakeStateSnapshot,
resetNodeWakeStateForTest,
} from "../node-wake-state.test-support.js";
import { expectRecordFields, requireRecord } from "../test-helpers.assertions.js";
import {
clearNodeWakeState,
maybeSendNodeWakeNudge,
maybeWakeNodeWithApns,
nodeHandlers,
@@ -30,7 +38,9 @@ type MockNodeConfig = {
};
const mocks = vi.hoisted(() => ({
captureNodePairingGeneration: vi.fn(),
getRuntimeConfig: vi.fn(() => ({})),
isNodePairingGenerationCurrent: vi.fn(),
resolveNodeCommandAllowlist: vi.fn<(cfg: MockNodeConfig) => Set<string>>(() => new Set()),
isNodeCommandAllowed: vi.fn<
(params: MockNodeCommandPolicyParams) => { ok: true } | { ok: false; reason: string }
@@ -56,6 +66,11 @@ vi.mock("../../config/io.js", () => ({
getRuntimeConfig: mocks.getRuntimeConfig,
}));
vi.mock("../../infra/node-pairing-state.js", () => ({
captureNodePairingGeneration: mocks.captureNodePairingGeneration,
isNodePairingGenerationCurrent: mocks.isNodePairingGenerationCurrent,
}));
vi.mock("../node-command-policy.js", () => ({
DEFAULT_DANGEROUS_NODE_COMMANDS: ["sms.send", "sms.search"],
resolveNodeCommandAllowlist: mocks.resolveNodeCommandAllowlist,
@@ -105,9 +120,12 @@ type MockCallSource = {
type TestNodeSession = {
nodeId: string;
connId?: string;
pairingGeneration?: string;
commands: string[];
declaredCommands?: string[];
platform?: string;
client?: { invalidated?: boolean };
};
function requireString(value: unknown, label: string): string {
@@ -326,12 +344,17 @@ function makeNodeInvokeParams(overrides?: Partial<Record<string, unknown>>) {
async function invokeNode(params: {
nodeRegistry: {
get: (nodeId: string) => TestNodeSession | undefined;
getForPairingGeneration?: (
nodeId: string,
pairingGeneration: string,
) => TestNodeSession | undefined;
invoke: (payload: {
nodeId: string;
command: string;
params?: unknown;
timeoutMs?: number;
idempotencyKey?: string;
expectedPairingGeneration?: string;
}) => Promise<{
ok: boolean;
payload?: unknown;
@@ -347,6 +370,12 @@ async function invokeNode(params: {
info: vi.fn(),
warn: vi.fn(),
};
const nodeRegistry = {
...params.nodeRegistry,
getForPairingGeneration:
params.nodeRegistry.getForPairingGeneration ??
((nodeId: string, _pairingGeneration: string) => params.nodeRegistry.get(nodeId)),
};
await expectDefined(
nodeHandlers["node.invoke"],
'nodeHandlers["node.invoke"] test invariant',
@@ -354,7 +383,7 @@ async function invokeNode(params: {
params: makeNodeInvokeParams(params.requestParams),
respond: respond as never,
context: {
nodeRegistry: params.nodeRegistry,
nodeRegistry,
execApprovalManager: undefined,
logGateway,
getRuntimeConfig: () => mocks.getRuntimeConfig(),
@@ -387,6 +416,7 @@ function createOperatorClient(params?: { scopes?: string[]; pluginRuntimeOwnerId
function createNodeClient(nodeId: string, commands?: string[]) {
return {
connId: `conn:${nodeId}`,
connect: {
...(commands ? { commands } : {}),
role: "node" as const,
@@ -429,32 +459,57 @@ function createMissingNodeRegistry() {
};
}
async function pullPending(nodeId: string, commands?: string[]) {
async function pullPending(
nodeId: string,
commands?: string[],
sessionConnId: string | null = `conn:${nodeId}`,
) {
const respond = vi.fn();
const client = createNodeClient(nodeId, commands);
await expectDefined(
nodeHandlers["node.pending.pull"],
'nodeHandlers["node.pending.pull"] test invariant',
)({
params: {},
respond: respond as never,
context: { getRuntimeConfig: () => mocks.getRuntimeConfig() } as never,
client: createNodeClient(nodeId, commands) as never,
context: {
getRuntimeConfig: () => mocks.getRuntimeConfig(),
nodeRegistry: {
getForPairingGeneration: vi.fn(() =>
sessionConnId === null ? undefined : { connId: sessionConnId },
),
},
} as never,
client: client as never,
req: { type: "req", id: "req-node-pending", method: "node.pending.pull" },
isWebchatConnect: () => false,
});
return respond;
}
async function ackPending(nodeId: string, ids: string[], commands?: string[]) {
async function ackPending(
nodeId: string,
ids: string[],
commands?: string[],
sessionConnId: string | null = `conn:${nodeId}`,
) {
const respond = vi.fn();
const client = createNodeClient(nodeId, commands);
await expectDefined(
nodeHandlers["node.pending.ack"],
'nodeHandlers["node.pending.ack"] test invariant',
)({
params: { ids },
respond: respond as never,
context: { getRuntimeConfig: () => mocks.getRuntimeConfig() } as never,
client: createNodeClient(nodeId, commands) as never,
context: {
getRuntimeConfig: () => mocks.getRuntimeConfig(),
nodeRegistry: {
getForPairingGeneration: vi.fn(() =>
sessionConnId === null ? undefined : { connId: sessionConnId },
),
},
} as never,
client: client as never,
req: { type: "req", id: "req-node-pending-ack", method: "node.pending.ack" },
isWebchatConnect: () => false,
});
@@ -649,6 +704,11 @@ describe("plugin surface refresh", () => {
describe("node.invoke APNs wake path", () => {
beforeEach(() => {
resetNodeWakeStateForTest();
mocks.captureNodePairingGeneration.mockReset().mockImplementation(async (nodeId: string) => ({
nodeId,
key: `generation:${nodeId}:1`,
}));
mocks.getRuntimeConfig.mockClear();
mocks.getRuntimeConfig.mockReturnValue({});
mocks.resolveNodeCommandAllowlist.mockClear();
@@ -659,6 +719,7 @@ describe("node.invoke APNs wake path", () => {
mocks.isForegroundRestrictedPluginNodeCommand.mockImplementation((command: string) =>
command.startsWith("canvas."),
);
mocks.isNodePairingGenerationCurrent.mockReset().mockResolvedValue(true);
mocks.sanitizeNodeInvokeParamsForForwarding.mockClear();
mocks.sanitizeNodeInvokeParamsForForwarding.mockImplementation(
({ rawParams }: { rawParams: unknown }) => ({ ok: true, params: rawParams }),
@@ -926,6 +987,32 @@ describe("node.invoke APNs wake path", () => {
expect(nodeRegistry.invoke).not.toHaveBeenCalled();
});
it("releases idle wake state when an unregistered node reconnects during invoke", async () => {
const nodeId = "ios-node-reconnect-without-registration";
mocks.loadApnsRegistration.mockResolvedValue(null);
const session: TestNodeSession = { nodeId, commands: ["camera.capture"] };
let lookupCount = 0;
const nodeRegistry = {
get: vi.fn(() => {
lookupCount += 1;
return lookupCount === 1 ? undefined : session;
}),
invoke: vi.fn().mockResolvedValue({
ok: true,
payload: { ok: true },
}),
};
const respond = await invokeNode({
nodeRegistry,
requestParams: { nodeId, idempotencyKey: "idem-reconnect-without-registration" },
});
expect(firstRespondCall(respond)[0]).toBe(true);
expect(nodeRegistry.invoke).toHaveBeenCalledTimes(1);
expect(getNodeWakeStateSnapshot(nodeId)).toBeUndefined();
});
it("does not throttle repeated relay wake attempts when relay config is missing", async () => {
mocks.loadApnsRegistration.mockResolvedValue(relayRegistration("ios-node-relay-no-auth"));
mocks.resolveApnsRelayConfigFromEnv.mockReturnValue({
@@ -942,6 +1029,69 @@ describe("node.invoke APNs wake path", () => {
expect(mocks.sendApnsBackgroundWake).not.toHaveBeenCalled();
});
it("does not share an in-flight wake with a replacement pairing generation", async () => {
const nodeId = "ios-node-replacement-in-flight";
const generationOne = { nodeId, key: "generation-1" };
const generationTwo = { nodeId, key: "generation-2" };
let resolveFirstRegistration!: (value: null) => void;
mocks.loadApnsRegistration
.mockImplementationOnce(
() =>
new Promise<null>((resolve) => {
resolveFirstRegistration = resolve;
}),
)
.mockResolvedValueOnce(null);
const firstWake = maybeWakeNodeWithApns(nodeId, { generation: generationOne });
await vi.waitFor(() => expect(mocks.loadApnsRegistration).toHaveBeenCalledTimes(1));
await expect(
maybeWakeNodeWithApns(nodeId, { generation: generationTwo }),
).resolves.toMatchObject({ path: "no-registration", available: false });
expect(mocks.loadApnsRegistration).toHaveBeenCalledTimes(2);
resolveFirstRegistration(null);
await expect(firstWake).resolves.toMatchObject({ path: "no-registration", available: false });
invalidateNodeWakeState(nodeId);
});
it("does not share wake or nudge throttles with a replacement pairing generation", async () => {
const nodeId = "ios-node-replacement-throttles";
const generationOne = { nodeId, key: "generation-1" };
const generationTwo = { nodeId, key: "generation-2" };
mockDirectWakeConfig(nodeId);
mocks.sendApnsAlert.mockResolvedValue({
ok: true,
status: 200,
tokenSuffix: "1234abcd",
topic: "ai.openclaw.ios",
environment: "sandbox",
transport: "direct",
});
await expect(
maybeWakeNodeWithApns(nodeId, { generation: generationOne }),
).resolves.toMatchObject({ path: "sent", throttled: false });
await expect(
maybeWakeNodeWithApns(nodeId, { generation: generationTwo }),
).resolves.toMatchObject({ path: "sent", throttled: false });
await expect(
maybeSendNodeWakeNudge(nodeId, { generation: generationOne }),
).resolves.toMatchObject({ reason: "sent", throttled: false });
await expect(
maybeSendNodeWakeNudge(nodeId, { generation: generationTwo }),
).resolves.toMatchObject({ reason: "sent", throttled: false });
expect(mocks.sendApnsBackgroundWake).toHaveBeenCalledTimes(2);
expect(mocks.sendApnsAlert).toHaveBeenCalledTimes(2);
expect(getNodeWakeStateSnapshot(nodeId, generationOne.key)?.lastWakeAtMs).toBeGreaterThan(0);
expect(getNodeWakeStateSnapshot(nodeId, generationTwo.key)?.lastWakeAtMs).toBeGreaterThan(0);
expect(getNodeWakeStateSnapshot(nodeId, generationOne.key)?.lastNudgeAtMs).toBeGreaterThan(0);
expect(getNodeWakeStateSnapshot(nodeId, generationTwo.key)?.lastNudgeAtMs).toBeGreaterThan(0);
invalidateNodeWakeState(nodeId);
});
it("clears wake and nudge throttle state when a node disconnects", async () => {
mockDirectWakeConfig("ios-node-clear-wake");
mocks.sendApnsAlert.mockResolvedValue({
@@ -1152,6 +1302,7 @@ describe("node.invoke APNs wake path", () => {
vi.setSystemTime(0);
const nodeRegistry = {
get: vi.fn(() => undefined),
getForPairingGeneration: vi.fn(() => undefined),
};
const reconnectPromise = waitForNodeReconnect({
@@ -1169,13 +1320,16 @@ describe("node.invoke APNs wake path", () => {
it("broadcasts canonical Talk capture events for successful PTT node commands", async () => {
const respond = vi.fn();
const broadcast = vi.fn();
const nodeSession = {
nodeId: "android-talk-node",
pairingGeneration: "generation:android-talk-node:1",
commands: ["talk.ptt.start"],
capabilities: ["talk"],
platform: "android",
};
const nodeRegistry = {
get: vi.fn(() => ({
nodeId: "android-talk-node",
commands: ["talk.ptt.start"],
capabilities: ["talk"],
platform: "android",
})),
get: vi.fn(() => nodeSession),
getForPairingGeneration: vi.fn(() => nodeSession),
invoke: vi.fn().mockResolvedValue({
ok: true,
payloadJSON: '{"captureId":"capture-1"}',
@@ -1282,6 +1436,29 @@ describe("node.invoke APNs wake path", () => {
expect(mocks.clearApnsRegistrationIfCurrent).not.toHaveBeenCalled();
});
it("rejects an invoke admitted after pairing removal without waking or dispatching", async () => {
const nodeId = "ios-node-removed-before-invoke";
mocks.captureNodePairingGeneration.mockResolvedValueOnce(null);
const nodeRegistry = createMissingNodeRegistry();
const respond = await invokeNode({
nodeRegistry,
requestParams: { nodeId, idempotencyKey: "idem-removed-before-invoke" },
});
expect(mocks.loadApnsRegistration).not.toHaveBeenCalled();
expect(mocks.sendApnsBackgroundWake).not.toHaveBeenCalled();
expect(nodeRegistry.invoke).not.toHaveBeenCalled();
expect(firstRespondCall(respond)).toMatchObject([
false,
undefined,
{
message: "node pairing changed while invocation was active",
details: { code: "PAIRING_CHANGED" },
},
]);
});
it("forces one retry wake when the first wake still fails to reconnect", async () => {
vi.useFakeTimers();
mockDirectWakeConfig("ios-node-throttle");
@@ -1299,6 +1476,339 @@ describe("node.invoke APNs wake path", () => {
expect(nodeRegistry.invoke).not.toHaveBeenCalled();
});
it("does not recreate wake state after removal invalidates an in-flight invoke", async () => {
vi.useFakeTimers();
const nodeId = "ios-node-remove-during-wake";
mockDirectWakeConfig(nodeId);
mocks.sendApnsAlert.mockResolvedValue({
ok: true,
status: 200,
tokenSuffix: "1234abcd",
topic: "ai.openclaw.ios",
environment: "sandbox",
transport: "direct",
});
const nodeRegistry = createMissingNodeRegistry();
const invokePromise = invokeNode({
nodeRegistry,
requestParams: { nodeId, idempotencyKey: "idem-remove-during-wake" },
});
await vi.advanceTimersByTimeAsync(0);
expect(mocks.sendApnsBackgroundWake).toHaveBeenCalledTimes(1);
invalidateNodeWakeState(nodeId);
await vi.advanceTimersByTimeAsync(20_000);
const respond = await invokePromise;
expect(mocks.sendApnsBackgroundWake).toHaveBeenCalledTimes(1);
expect(mocks.sendApnsAlert).not.toHaveBeenCalled();
expect(getNodeWakeStateSnapshot(nodeId)).toBeUndefined();
expect(nodeRegistry.invoke).not.toHaveBeenCalled();
const call = firstRespondCall(respond);
expect(call[0]).toBe(false);
expect(call[2]).toMatchObject({
message: "node pairing changed while invocation was active",
details: { code: "PAIRING_CHANGED" },
});
});
it("revalidates pairing generation after direct wake auth resolves", async () => {
const nodeId = "ios-node-generation-change-during-wake-auth";
const generation = { nodeId, key: "generation-1" };
const lifecycle = captureNodeWakeLifecycle(nodeId, generation.key);
let pairingCurrent = true;
let resolveAuth!: (value: {
ok: true;
value: { teamId: string; keyId: string; privateKey: string };
}) => void;
mocks.isNodePairingGenerationCurrent.mockImplementation(async () => pairingCurrent);
mocks.loadApnsRegistration.mockResolvedValue(directRegistration(nodeId));
mocks.resolveApnsAuthConfigFromEnv.mockReturnValue(
new Promise((resolve) => {
resolveAuth = resolve;
}),
);
const wakePromise = maybeWakeNodeWithApns(nodeId, { lifecycle, generation });
await vi.waitFor(() => expect(mocks.resolveApnsAuthConfigFromEnv).toHaveBeenCalledTimes(1));
pairingCurrent = false;
resolveAuth({
ok: true,
value: {
teamId: "TEAM123",
keyId: "KEY123",
privateKey: "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----", // pragma: allowlist secret
},
});
await expect(wakePromise).resolves.toMatchObject({ path: "invalidated", available: false });
expect(mocks.sendApnsBackgroundWake).not.toHaveBeenCalled();
invalidateNodeWakeState(nodeId);
});
it("passes lifecycle and persistent currentness into direct APNs transport", async () => {
const nodeId = "ios-node-transport-generation-guard";
const generation = { nodeId, key: "generation-1" };
const lifecycle = captureNodeWakeLifecycle(nodeId, generation.key);
let pairingCurrent = true;
mocks.isNodePairingGenerationCurrent.mockImplementation(async () => pairingCurrent);
mocks.loadApnsRegistration.mockResolvedValue(directRegistration(nodeId));
mocks.resolveApnsAuthConfigFromEnv.mockResolvedValue({
ok: true,
value: {
teamId: "TEAM123",
keyId: "KEY123",
privateKey: "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----", // pragma: allowlist secret
},
});
await maybeWakeNodeWithApns(nodeId, { lifecycle, generation });
const transport = requireRecord(
mockArg(mocks.sendApnsBackgroundWake, 0, 0),
"guarded APNs transport",
);
expect(transport.signal).toBe(lifecycle);
expect(transport.isCurrent).toBeTypeOf("function");
pairingCurrent = false;
await expect((transport.isCurrent as () => Promise<boolean>)()).resolves.toBe(false);
invalidateNodeWakeState(nodeId);
});
it("revalidates pairing generation after direct nudge auth resolves", async () => {
const nodeId = "ios-node-generation-change-during-nudge-auth";
const generation = { nodeId, key: "generation-1" };
const lifecycle = captureNodeWakeLifecycle(nodeId, generation.key);
let pairingCurrent = true;
let resolveAuth!: (value: {
ok: true;
value: { teamId: string; keyId: string; privateKey: string };
}) => void;
mocks.isNodePairingGenerationCurrent.mockImplementation(async () => pairingCurrent);
mocks.loadApnsRegistration.mockResolvedValue(directRegistration(nodeId));
mocks.resolveApnsAuthConfigFromEnv.mockReturnValue(
new Promise((resolve) => {
resolveAuth = resolve;
}),
);
const nudgePromise = maybeSendNodeWakeNudge(nodeId, { lifecycle, generation });
await vi.waitFor(() => expect(mocks.resolveApnsAuthConfigFromEnv).toHaveBeenCalledTimes(1));
pairingCurrent = false;
resolveAuth({
ok: true,
value: {
teamId: "TEAM123",
keyId: "KEY123",
privateKey: "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----", // pragma: allowlist secret
},
});
await expect(nudgePromise).resolves.toMatchObject({ reason: "invalidated", sent: false });
expect(mocks.sendApnsAlert).not.toHaveBeenCalled();
invalidateNodeWakeState(nodeId);
});
it("does not dispatch an admitted invoke after the pairing generation is replaced", async () => {
vi.useFakeTimers();
const nodeId = "ios-node-replacement-after-remove";
mockDirectWakeConfig(nodeId);
let pairingCurrent = true;
mocks.isNodePairingGenerationCurrent.mockImplementation(async () => pairingCurrent);
const replacementSession: TestNodeSession = {
nodeId,
connId: "replacement-conn",
commands: ["camera.capture"],
platform: "iOS 26.4.0",
};
let replacementConnected = false;
const nodeRegistry = {
get: vi.fn(() => (replacementConnected ? replacementSession : undefined)),
invoke: vi.fn().mockResolvedValue({ ok: true, payload: { replacement: true } }),
};
const invokePromise = invokeNode({
nodeRegistry,
requestParams: { nodeId, idempotencyKey: "idem-replacement-after-remove" },
});
await vi.advanceTimersByTimeAsync(0);
expect(mocks.sendApnsBackgroundWake).toHaveBeenCalledTimes(1);
pairingCurrent = false;
replacementConnected = true;
await vi.advanceTimersByTimeAsync(20_000);
const respond = await invokePromise;
expect(nodeRegistry.invoke).not.toHaveBeenCalled();
const call = firstRespondCall(respond);
expect(call[0]).toBe(false);
expect(call[2]).toMatchObject({
message: "node pairing changed while invocation was active",
details: { code: "PAIRING_CHANGED" },
});
});
it("does not dispatch through an invalidated old node session", async () => {
const nodeId = "ios-node-invalidated-session";
mocks.loadApnsRegistration.mockResolvedValue(null);
const invalidatedSession: TestNodeSession = {
nodeId,
connId: "old-conn",
commands: ["camera.capture"],
platform: "iOS 26.4.0",
client: { invalidated: true },
};
const nodeRegistry = {
get: vi.fn(() => invalidatedSession),
invoke: vi.fn().mockResolvedValue({ ok: true, payload: { delivered: true } }),
};
const respond = await invokeNode({
nodeRegistry,
requestParams: { nodeId, idempotencyKey: "idem-invalidated-session" },
});
expect(nodeRegistry.invoke).not.toHaveBeenCalled();
expect(firstRespondCall(respond)).toMatchObject([
false,
undefined,
{ details: { code: "NOT_CONNECTED" } },
]);
});
it("does not dispatch current-generation work to a prior-generation session", async () => {
const nodeId = "ios-node-prior-generation-session";
mocks.loadApnsRegistration.mockResolvedValue(null);
const oldSession: TestNodeSession = {
nodeId,
connId: "old-generation-conn",
pairingGeneration: `generation:${nodeId}:0`,
commands: ["camera.capture"],
platform: "iOS 26.4.0",
};
const nodeRegistry = {
get: vi.fn(() => oldSession),
getForPairingGeneration: vi.fn((_requestedNodeId: string, generation: string) =>
generation === oldSession.pairingGeneration ? oldSession : undefined,
),
invoke: vi.fn().mockResolvedValue({ ok: true, payload: { delivered: true } }),
};
const respond = await invokeNode({
nodeRegistry,
requestParams: { nodeId, idempotencyKey: "idem-prior-generation-session" },
});
expect(nodeRegistry.getForPairingGeneration).toHaveBeenCalledWith(
nodeId,
`generation:${nodeId}:1`,
);
expect(nodeRegistry.invoke).not.toHaveBeenCalled();
expect(firstRespondCall(respond)).toMatchObject([
false,
undefined,
{ details: { code: "NOT_CONNECTED" } },
]);
});
it("does not queue foreground work when pairing changes during node dispatch", async () => {
const nodeId = "ios-node-replaced-during-dispatch";
let pairingCurrent = true;
mocks.isNodePairingGenerationCurrent.mockImplementation(async () => pairingCurrent);
const nodeRegistry = createForegroundUnavailableNodeRegistry({
nodeId,
commands: ["canvas.navigate"],
platform: "iOS 26.4.0",
});
nodeRegistry.invoke.mockImplementation(async () => {
pairingCurrent = false;
return {
ok: false,
error: {
code: "NODE_BACKGROUND_UNAVAILABLE",
message: "NODE_BACKGROUND_UNAVAILABLE: canvas commands require foreground",
},
};
});
const respond = await invokeNode({
nodeRegistry,
requestParams: {
nodeId,
command: "canvas.navigate",
idempotencyKey: "idem-replaced-during-dispatch",
},
});
expect(nodeRegistry.invoke).toHaveBeenCalledTimes(1);
expect(mocks.loadApnsRegistration).not.toHaveBeenCalled();
expect(firstRespondCall(respond)).toMatchObject([
false,
undefined,
{
message: "node pairing changed while invocation was active",
details: { code: "PAIRING_CHANGED" },
},
]);
});
it("keeps a foreground-recovery wake alive across an ordinary disconnect cleanup", async () => {
const nodeId = "ios-node-disconnect-during-foreground-wake";
const registration = directRegistration(nodeId);
let resolveRegistration!: (value: typeof registration) => void;
mocks.loadApnsRegistration.mockReturnValue(
new Promise((resolve) => {
resolveRegistration = resolve;
}),
);
mocks.resolveApnsAuthConfigFromEnv.mockResolvedValue({
ok: true,
value: {
teamId: "TEAM123",
keyId: "KEY123",
privateKey: "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----", // pragma: allowlist secret
},
});
mocks.sendApnsBackgroundWake.mockResolvedValue({
ok: true,
status: 200,
tokenSuffix: "1234abcd",
topic: "ai.openclaw.ios",
environment: "sandbox",
transport: "direct",
});
const nodeRegistry = createForegroundUnavailableNodeRegistry({
nodeId,
commands: ["canvas.navigate"],
platform: "iOS 26.4.0",
});
const invokePromise = invokeNode({
nodeRegistry,
requestParams: {
nodeId,
command: "canvas.navigate",
params: { url: "http://example.com/" },
idempotencyKey: "idem-disconnect-during-foreground-wake",
},
});
await vi.waitFor(() => expect(mocks.loadApnsRegistration).toHaveBeenCalledTimes(1));
clearNodeWakeState(nodeId);
resolveRegistration(registration);
const respond = await invokePromise;
expect(mocks.sendApnsBackgroundWake).toHaveBeenCalledTimes(1);
const call = firstRespondCall(respond);
const details = requireRecord(call[2]?.details, "queued foreground details");
expectRecordFields(details.wake, "queued foreground wake", {
path: "sent",
available: true,
throttled: false,
});
});
it("queues iOS foreground-only command failures and keeps them until acked", async () => {
mocks.loadApnsRegistration.mockResolvedValue(null);
@@ -1427,6 +1937,133 @@ describe("node.invoke APNs wake path", () => {
});
});
it("does not expose queued foreground actions to a replacement pairing generation", async () => {
const nodeId = "ios-node-replaced-before-pull";
mocks.loadApnsRegistration.mockResolvedValue(null);
const nodeRegistry = createForegroundUnavailableNodeRegistry({
nodeId,
commands: ["canvas.navigate"],
platform: "iOS 26.4.0",
});
await invokeNode({
nodeRegistry,
requestParams: {
nodeId,
command: "canvas.navigate",
idempotencyKey: "idem-replaced-before-pull",
},
});
mocks.captureNodePairingGeneration.mockResolvedValue({
nodeId,
key: `generation:${nodeId}:2`,
});
const pullRespond = await pullPending(nodeId, ["canvas.navigate"]);
expectRecordFields(
requireRespondPayload(firstRespondCall(pullRespond), "replacement pull response"),
"replacement pull payload",
{ nodeId, actions: [] },
);
});
it("does not let a prior-generation session pull or ack current-generation actions", async () => {
const nodeId = "ios-node-prior-generation-pending";
mocks.loadApnsRegistration.mockResolvedValue(null);
const nodeRegistry = createForegroundUnavailableNodeRegistry({
nodeId,
commands: ["canvas.navigate"],
platform: "iOS 26.4.0",
});
await invokeNode({
nodeRegistry,
requestParams: {
nodeId,
command: "canvas.navigate",
idempotencyKey: "idem-prior-generation-pending",
},
});
expect(firstRespondCall(await pullPending(nodeId, ["canvas.navigate"], null))).toMatchObject([
false,
undefined,
{ details: { code: "PAIRING_CHANGED" } },
]);
const currentPullPayload = requireRespondPayload(
firstRespondCall(await pullPending(nodeId, ["canvas.navigate"])),
"current-generation pull response",
);
const queuedActionId = requireString(
(currentPullPayload.actions as Array<{ id?: string }> | undefined)?.[0]?.id,
"current-generation queued action id",
);
expect(
firstRespondCall(await ackPending(nodeId, [queuedActionId], ["canvas.navigate"], null)),
).toMatchObject([false, undefined, { details: { code: "PAIRING_CHANGED" } }]);
expect(
(
requireRespondPayload(
firstRespondCall(await pullPending(nodeId, ["canvas.navigate"])),
"post-stale-ack pull response",
).actions as unknown[] | undefined
)?.length,
).toBe(1);
});
it("does not let a stale foreground pull delete replacement-generation actions", async () => {
const nodeId = "ios-node-stale-pull";
mocks.loadApnsRegistration.mockResolvedValue(null);
const nodeRegistry = createForegroundUnavailableNodeRegistry({
nodeId,
commands: ["canvas.navigate"],
platform: "iOS 26.4.0",
});
await invokeNode({
nodeRegistry,
requestParams: {
nodeId,
command: "canvas.navigate",
idempotencyKey: "idem-stale-generation",
},
});
mocks.captureNodePairingGeneration.mockResolvedValue({
nodeId,
key: `generation:${nodeId}:2`,
});
await invokeNode({
nodeRegistry,
requestParams: {
nodeId,
command: "canvas.navigate",
idempotencyKey: "idem-replacement-generation",
},
});
mocks.captureNodePairingGeneration.mockResolvedValue({
nodeId,
key: `generation:${nodeId}:1`,
});
const stalePullPayload = requireRespondPayload(
firstRespondCall(await pullPending(nodeId, ["canvas.navigate"])),
"stale generation pull response",
);
expect((stalePullPayload.actions as unknown[] | undefined)?.length).toBe(1);
mocks.captureNodePairingGeneration.mockResolvedValue({
nodeId,
key: `generation:${nodeId}:2`,
});
const replacementPullPayload = requireRespondPayload(
firstRespondCall(await pullPending(nodeId, ["canvas.navigate"])),
"replacement generation pull response",
);
expect((replacementPullPayload.actions as unknown[] | undefined)?.length).toBe(1);
});
it("dedupes queued foreground actions by idempotency key", async () => {
mocks.loadApnsRegistration.mockResolvedValue(null);
src/gateway/server-methods/nodes.test.ts
+400
View File
@@ -5,6 +5,7 @@ import {
listDevicePairing,
requestDevicePairing,
revokeDeviceToken,
rotateDeviceToken,
withPairedDeviceRecords,
} from "../../infra/device-pairing.js";
import {
@@ -12,16 +13,50 @@ import {
resetDiagnosticEventsForTest,
type DiagnosticSecurityEvent,
} from "../../infra/diagnostic-events.js";
import {
captureNodePairingGeneration,
captureNodePairingState,
} from "../../infra/node-pairing-state.js";
import { approveNodePairing, requestNodePairing } from "../../infra/node-pairing.js";
import { loadApnsRegistration, registerApnsRegistration } from "../../infra/push-apns.js";
import { resetRemoteNodeSkillsForTests } from "../../skills/runtime/remote-skills.test-support.js";
import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js";
import {
createOpenClawTestState,
type OpenClawTestState,
} from "../../test-utils/openclaw-test-state.js";
import { drainNodePendingWork, enqueueNodePendingWork } from "../node-pending-work.js";
import {
captureNodeWakeLifecycle,
runNodeWakeAttempt,
runNodeWakeNudgeAttempt,
} from "../node-wake-state.js";
import {
getNodeWakeStateSnapshot,
resetNodeWakeStateForTest,
} from "../node-wake-state.test-support.js";
import { nodeHandlers } from "./nodes.js";
import type { GatewayRequestHandlerOptions } from "./types.js";
const createdStates: OpenClawTestState[] = [];
const pairingGenerationHooks = vi.hoisted(() => ({
beforeCapture: vi.fn<(nodeId: string) => Promise<void> | void>(),
}));
vi.mock("../../infra/node-pairing-state.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../../infra/node-pairing-state.js")>();
return {
...actual,
captureNodePairingState: async (nodeId: string) => {
await pairingGenerationHooks.beforeCapture(nodeId);
return await actual.captureNodePairingState(nodeId);
},
captureNodePairingGeneration: async (nodeId: string) => {
await pairingGenerationHooks.beforeCapture(nodeId);
return await actual.captureNodePairingGeneration(nodeId);
},
};
});
async function createState(label: string): Promise<OpenClawTestState> {
const state = await createOpenClawTestState({ label, layout: "state-only" });
@@ -29,10 +64,31 @@ async function createState(label: string): Promise<OpenClawTestState> {
return state;
}
async function seedNodeWakeState(nodeId: string): Promise<void> {
await runNodeWakeAttempt({
nodeId,
force: true,
throttleMs: 60_000,
attempt: async (markAttempted) => {
markAttempted();
return { available: true, throttled: false, path: "sent", durationMs: 1 };
},
});
await runNodeWakeNudgeAttempt({
nodeId,
throttleMs: 60_000,
throttled: () => ({ sent: false, throttled: true, reason: "throttled", durationMs: 0 }),
attempt: async () => ({ sent: true, throttled: false, reason: "sent", durationMs: 1 }),
});
}
afterEach(async () => {
resetDiagnosticEventsForTest();
resetRemoteNodeSkillsForTests();
resetNodeWakeStateForTest();
pairingGenerationHooks.beforeCapture.mockReset();
vi.clearAllMocks();
closeOpenClawStateDatabaseForTest();
while (createdStates.length > 0) {
await createdStates.pop()?.cleanup();
}
@@ -55,6 +111,7 @@ function createContext() {
return {
broadcast: vi.fn(),
disconnectClientsForDevice: vi.fn(),
getRuntimeConfig: vi.fn(() => ({})),
invalidateClientsForDevice: vi.fn(),
logGateway: {
debug: vi.fn(),
@@ -63,7 +120,10 @@ function createContext() {
warn: vi.fn(),
},
nodeRegistry: {
get: vi.fn(),
listConnected: vi.fn(() => []),
listConnectedForPairingStates: vi.fn(() => []),
getActiveNode: vi.fn(),
updateSurface: vi.fn(),
updateNodeSkills: vi.fn(),
},
@@ -209,7 +269,335 @@ async function readPaired(stateDir: string): Promise<Record<string, unknown>> {
return Object.fromEntries(paired.map((device) => [device.deviceId, device]));
}
describe("nodeHandlers node.pair.approve", () => {
it("promotes the first surface only for the same authenticated pairing identity", async () => {
const state = await createState("node-approve-promotes-pending-identity");
const nodeId = "node-pending-identity-stable";
await pairAndroidNodeDevice(state.stateDir, nodeId);
const pending = await requestNodePairing(
{
nodeId,
platform: "android",
deviceFamily: "Android",
clientId: "openclaw-android",
clientMode: "node",
displayName: "Galaxy A54 5G pending",
},
state.stateDir,
);
const pendingState = await captureNodePairingState(nodeId);
expect(pendingState?.generation).toBeNull();
const { context, opts } = createOptions({ requestId: pending.request.requestId });
context.nodeRegistry.get.mockReturnValue({
nodeId,
connId: "conn-pending-identity-stable",
pairingIdentity: pendingState?.identity.key,
});
await expectDefined(
nodeHandlers["node.pair.approve"],
'nodeHandlers["node.pair.approve"] test invariant',
)(opts);
const approvedState = await captureNodePairingState(nodeId);
expect(approvedState?.identity.key).toBe(pendingState?.identity.key);
expect(approvedState?.generation).not.toBeNull();
expect(context.nodeRegistry.updateSurface).toHaveBeenCalledWith(
nodeId,
expect.objectContaining({ commands: expect.any(Array) }),
{
expectedConnId: "conn-pending-identity-stable",
expectedPairingIdentity: pendingState?.identity.key,
nextPairingGeneration: approvedState?.generation?.key,
},
);
expect(opts.respond).toHaveBeenCalledWith(
true,
expect.objectContaining({ node: expect.objectContaining({ nodeId }) }),
undefined,
);
});
it("invalidates an in-flight wake when node-surface approval rotates generation", async () => {
const state = await createState("node-approve-invalidates-wake");
const nodeId = "node-surface-reapproval";
await pairAndroidNodeDevice(state.stateDir, nodeId);
await approveNodeSurface(state.stateDir, nodeId);
const previousGeneration = await captureNodePairingGeneration(nodeId);
const previousState = await captureNodePairingState(nodeId);
expect(previousGeneration).not.toBeNull();
expect(previousState).not.toBeNull();
const pending = await requestNodePairing(
{
nodeId,
platform: "android",
deviceFamily: "Android",
clientId: "openclaw-android",
clientMode: "node",
displayName: "Galaxy A54 5G reapproved",
},
state.stateDir,
);
const lifecycle = captureNodeWakeLifecycle(nodeId);
const { context, opts } = createOptions({ requestId: pending.request.requestId });
context.nodeRegistry.get.mockReturnValue({
nodeId,
connId: "conn-surface-reapproval",
pairingIdentity: previousState?.identity.key,
pairingGeneration: previousGeneration?.key,
});
await expectDefined(
nodeHandlers["node.pair.approve"],
'nodeHandlers["node.pair.approve"] test invariant',
)(opts);
expect(lifecycle.aborted).toBe(true);
const nextGeneration = await captureNodePairingGeneration(nodeId);
expect(nextGeneration?.key).not.toBe(previousGeneration?.key);
expect(context.nodeRegistry.updateSurface).toHaveBeenCalledWith(
nodeId,
expect.objectContaining({ commands: expect.any(Array) }),
{
expectedConnId: "conn-surface-reapproval",
expectedPairingIdentity: previousState?.identity.key,
expectedPairingGeneration: previousGeneration?.key,
nextPairingGeneration: nextGeneration?.key,
},
);
expect(opts.respond).toHaveBeenCalledWith(
true,
expect.objectContaining({ node: expect.objectContaining({ nodeId }) }),
undefined,
);
});
it("does not promote a session authenticated before external device reapproval", async () => {
const state = await createState("node-approve-rejects-stale-live-session");
const nodeId = "node-stale-surface-session";
await pairAndroidNodeDevice(state.stateDir, nodeId);
await approveNodeSurface(state.stateDir, nodeId);
const staleGeneration = await captureNodePairingGeneration(nodeId);
const staleState = await captureNodePairingState(nodeId);
expect(staleGeneration).not.toBeNull();
expect(staleState).not.toBeNull();
const pending = await requestNodePairing(
{
nodeId,
platform: "android",
deviceFamily: "Android",
clientId: "openclaw-android",
clientMode: "node",
displayName: "Galaxy A54 5G reapproved",
},
state.stateDir,
);
await pairAndroidNodeDevice(state.stateDir, nodeId);
const currentGeneration = await captureNodePairingGeneration(nodeId);
expect(currentGeneration?.key).not.toBe(staleGeneration?.key);
const { context, opts } = createOptions({ requestId: pending.request.requestId });
context.nodeRegistry.get.mockReturnValue({
nodeId,
connId: "conn-authenticated-before-reapproval",
pairingIdentity: staleState?.identity.key,
pairingGeneration: staleGeneration?.key,
});
await expectDefined(
nodeHandlers["node.pair.approve"],
'nodeHandlers["node.pair.approve"] test invariant',
)(opts);
expect(context.nodeRegistry.updateSurface).not.toHaveBeenCalled();
expect(opts.respond).toHaveBeenCalledWith(
true,
expect.objectContaining({ node: expect.objectContaining({ nodeId }) }),
undefined,
);
});
it("does not promote an old session when reapproval wins after surface approval commits", async () => {
const state = await createState("node-approve-fences-post-approval-repair");
const nodeId = "node-post-approval-repair";
await pairAndroidNodeDevice(state.stateDir, nodeId);
await approveNodeSurface(state.stateDir, nodeId);
const staleGeneration = await captureNodePairingGeneration(nodeId);
const staleState = await captureNodePairingState(nodeId);
expect(staleGeneration).not.toBeNull();
expect(staleState).not.toBeNull();
const pending = await requestNodePairing(
{
nodeId,
platform: "android",
deviceFamily: "Android",
clientId: "openclaw-android",
clientMode: "node",
displayName: "Galaxy A54 5G surface refresh",
},
state.stateDir,
);
let captureCount = 0;
pairingGenerationHooks.beforeCapture.mockImplementation(async (capturedNodeId) => {
if (capturedNodeId !== nodeId) {
return;
}
captureCount += 1;
if (captureCount === 2) {
await pairAndroidNodeDevice(state.stateDir, nodeId);
}
});
const { context, opts } = createOptions({ requestId: pending.request.requestId });
context.nodeRegistry.get.mockReturnValue({
nodeId,
connId: "conn-authenticated-before-post-approval-repair",
pairingIdentity: staleState?.identity.key,
pairingGeneration: staleGeneration?.key,
});
await expectDefined(
nodeHandlers["node.pair.approve"],
'nodeHandlers["node.pair.approve"] test invariant',
)(opts);
const currentGeneration = await captureNodePairingGeneration(nodeId);
expect(currentGeneration?.key).not.toBe(staleGeneration?.key);
expect(context.nodeRegistry.updateSurface).not.toHaveBeenCalled();
expect(opts.respond).toHaveBeenCalledWith(
true,
expect.objectContaining({ node: expect.objectContaining({ nodeId }) }),
undefined,
);
});
it("does not promote a generation-less session after external node-token rotation", async () => {
const state = await createState("node-approve-fences-pending-identity");
const nodeId = "node-pending-identity-rotation";
await pairAndroidNodeDevice(state.stateDir, nodeId);
const pending = await requestNodePairing(
{
nodeId,
platform: "android",
deviceFamily: "Android",
clientId: "openclaw-android",
clientMode: "node",
displayName: "Galaxy A54 5G pending",
},
state.stateDir,
);
const staleState = await captureNodePairingState(nodeId);
expect(staleState?.generation).toBeNull();
const rotated = await rotateDeviceToken({
deviceId: nodeId,
role: "node",
scopes: [],
baseDir: state.stateDir,
});
expect(rotated.ok).toBe(true);
const currentState = await captureNodePairingState(nodeId);
expect(currentState?.generation).toBeNull();
expect(currentState?.identity.key).not.toBe(staleState?.identity.key);
const { context, opts } = createOptions({ requestId: pending.request.requestId });
context.nodeRegistry.get.mockReturnValue({
nodeId,
connId: "conn-authenticated-before-token-rotation",
pairingIdentity: staleState?.identity.key,
});
await expectDefined(
nodeHandlers["node.pair.approve"],
'nodeHandlers["node.pair.approve"] test invariant',
)(opts);
expect(context.nodeRegistry.updateSurface).not.toHaveBeenCalled();
expect(opts.respond).toHaveBeenCalledWith(
true,
expect.objectContaining({ node: expect.objectContaining({ nodeId }) }),
undefined,
);
});
});
describe("nodeHandlers node.pair.remove", () => {
it("clears and invalidates wake state when removing a disconnected device-backed node", async () => {
const state = await createState("node-remove-clears-wake-state");
const nodeId = "disconnected-ios-node";
await pairAndroidNodeDevice(state.stateDir, nodeId);
await registerApnsRegistration({
nodeId,
transport: "direct",
token: "ABCD1234ABCD1234ABCD1234ABCD1234",
topic: "ai.openclaw.ios",
environment: "sandbox",
});
await seedNodeWakeState(nodeId);
enqueueNodePendingWork({ nodeId, type: "location.request" });
const wakeLifecycle = captureNodeWakeLifecycle(nodeId);
const { opts } = createOptions({ nodeId });
await expectDefined(
nodeHandlers["node.pair.remove"],
'nodeHandlers["node.pair.remove"] test invariant',
)(opts);
await Promise.resolve();
expect(opts.respond).toHaveBeenCalledWith(true, { nodeId }, undefined);
expect(getNodeWakeStateSnapshot(nodeId)).toBeUndefined();
expect(wakeLifecycle.aborted).toBe(true);
expect(drainNodePendingWork(nodeId).items.map((item) => item.id)).toEqual(["baseline-status"]);
await expect(loadApnsRegistration(nodeId)).resolves.toBeNull();
});
it("preserves an APNs registration created after node-role removal commits", async () => {
const state = await createState("node-remove-apns-registration-race");
const nodeId = "ios-node-registration-race";
await pairAndroidNodeDevice(state.stateDir, nodeId);
await registerApnsRegistration({
nodeId,
transport: "direct",
token: "ABCD1234ABCD1234ABCD1234ABCD1234",
topic: "ai.openclaw.ios",
environment: "sandbox",
});
const { context, opts } = createOptions({ nodeId });
let replacementWrite: Promise<unknown> | undefined;
context.invalidateClientsForDevice.mockImplementation(() => {
replacementWrite = (async () => {
await pairAndroidNodeDevice(state.stateDir, nodeId);
await approveNodeSurface(state.stateDir, nodeId);
const replacementGeneration = await captureNodePairingGeneration(nodeId);
if (!replacementGeneration) {
throw new Error("expected replacement pairing generation");
}
return await registerApnsRegistration({
nodeId,
transport: "direct",
token: "DCBA4321DCBA4321DCBA4321DCBA4321",
topic: "ai.openclaw.ios",
environment: "sandbox",
expectedPairingGeneration: replacementGeneration.key,
});
})();
});
await expectDefined(
nodeHandlers["node.pair.remove"],
'nodeHandlers["node.pair.remove"] test invariant',
)(opts);
await replacementWrite;
await expect(loadApnsRegistration(nodeId)).resolves.toMatchObject({
nodeId,
transport: "direct",
token: "dcba4321dcba4321dcba4321dcba4321",
});
});
it("removes Android device-backed node rows from the paired-device store", async () => {
const state = await createState("node-remove-android-device-backed");
const nodeId = "android-node-1";
@@ -359,6 +747,13 @@ describe("nodeHandlers node.pair.remove", () => {
const state = await createState("node-remove-mixed-role-device");
const nodeId = "mixed-role-android-node-1";
await pairMixedRoleAndroidDevice(state.stateDir, nodeId);
await registerApnsRegistration({
nodeId,
transport: "direct",
token: "ABCD1234ABCD1234ABCD1234ABCD1234",
topic: "ai.openclaw.ios",
environment: "sandbox",
});
const before = await readPaired(state.stateDir);
expect(
@@ -387,6 +782,11 @@ describe("nodeHandlers node.pair.remove", () => {
expect(
Object.hasOwn((after[nodeId] as { tokens?: Record<string, unknown> }).tokens ?? {}, "node"),
).toBe(false);
await expect(loadApnsRegistration(nodeId)).resolves.toMatchObject({
nodeId,
transport: "direct",
token: "abcd1234abcd1234abcd1234abcd1234",
});
expect(context.invalidateClientsForDevice).toHaveBeenCalledWith(nodeId, {
role: "node",
reason: "device-pair-removed",
src/gateway/server-methods/nodes.ts
+919 -599
View File
File diff suppressed because it is too large Load Diff
src/gateway/server-methods/nodes.wake-leak.test.ts
+10 -5
View File
@@ -12,18 +12,21 @@ const mocks = vi.hoisted(() => ({
vi.mock("../../infra/push-apns.js", () => mocks);
import { nodeWakeById } from "./nodes-wake-state.js";
import {
getNodeWakeStateSnapshot,
resetNodeWakeStateForTest,
} from "../node-wake-state.test-support.js";
import { maybeWakeNodeWithApns } from "./nodes.js";
describe("maybeWakeNodeWithApns no-registration cleanup", () => {
beforeEach(() => {
nodeWakeById.clear();
resetNodeWakeStateForTest();
vi.clearAllMocks();
mocks.loadApnsRegistration.mockResolvedValue(null);
});
afterEach(() => {
nodeWakeById.clear();
resetNodeWakeStateForTest();
});
it("does not retain state for unregistered node ids", async () => {
@@ -35,11 +38,13 @@ describe("maybeWakeNodeWithApns no-registration cleanup", () => {
});
}
expect(nodeWakeById.size).toBe(0);
for (let index = 0; index < 50; index += 1) {
expect(getNodeWakeStateSnapshot(`unregistered-node-${index}`)).toBeUndefined();
}
});
it("cleans up after a single no-registration result", async () => {
await maybeWakeNodeWithApns("stale-node-id");
expect(nodeWakeById.has("stale-node-id")).toBe(false);
expect(getNodeWakeStateSnapshot("stale-node-id")).toBeUndefined();
});
});
src/gateway/server-methods/shared-types.ts
+3 -3
View File
@@ -198,10 +198,10 @@ export type GatewayRequestContext = {
broadcastToConnIds: GatewayBroadcastToConnIdsFn;
nodeSendToSession: (sessionKey: string, event: string, payload: unknown) => void;
nodeSendToAllSubscribed: (event: string, payload: unknown) => void;
nodeSubscribe: (nodeId: string, sessionKey: string) => void;
nodeUnsubscribe: (nodeId: string, sessionKey: string) => void;
nodeSubscribe: (nodeId: string, sessionKey: string, connId?: string) => void;
nodeUnsubscribe: (nodeId: string, sessionKey: string, connId?: string) => void;
nodeUnsubscribeAll: (nodeId: string) => void;
hasConnectedTalkNode: () => boolean;
hasConnectedTalkNode: () => Promise<boolean>;
isConnectionActive?: (connId: string) => boolean;
hasExecApprovalClients?: (excludeConnId?: string) => boolean;
/** Instance-local native approval subscribers; never derived from a network client. */
src/gateway/server-methods/talk.ts
+2 -2
View File
@@ -863,8 +863,8 @@ export const talkHandlers: GatewayRequestHandlers = {
respond(false, undefined, talkSpeakError("synthesis_failed", formatForLog(err)));
}
},
"talk.mode": ({ params, respond, context, client, isWebchatConnect }) => {
if (client && isWebchatConnect(client.connect) && !context.hasConnectedTalkNode()) {
"talk.mode": async ({ params, respond, context, client, isWebchatConnect }) => {
if (client && isWebchatConnect(client.connect) && !(await context.hasConnectedTalkNode())) {
respond(
false,
undefined,
src/gateway/server-methods/terminal.test.ts
+13 -2
View File
@@ -571,7 +571,12 @@ describe("terminal gateway policy", () => {
paramsJSON: JSON.stringify({ threadId: "thread" }),
}),
});
const node = { nodeId: "node-1", connId: "conn-node", commands: [command] };
const node = {
nodeId: "node-1",
connId: "conn-node",
pairingGeneration: "generation-node",
commands: [command],
};
const invoke = vi.fn((rawParams: unknown) => {
const params = rawParams as { onInvokeId?: (id: string) => void };
params.onInvokeId?.("invoke-1");
@@ -610,7 +615,11 @@ describe("terminal gateway policy", () => {
)?.at(0);
await openRequest?.createBackend?.();
expect(invoke).toHaveBeenCalledWith(
expect.objectContaining({ nodeId: "node-1", expectedConnId: "conn-node" }),
expect.objectContaining({
nodeId: "node-1",
expectedConnId: "conn-node",
expectedPairingGeneration: "generation-node",
}),
);
});
@@ -795,6 +804,7 @@ describe("terminal gateway policy", () => {
const node = {
nodeId: "node-1",
connId: "conn-node",
pairingGeneration: "generation-node",
commands: [command, uploadCommand],
};
const invoke = vi.fn(async () => ({
@@ -824,6 +834,7 @@ describe("terminal gateway policy", () => {
expect(invoke).toHaveBeenCalledWith({
nodeId: "node-1",
expectedConnId: "conn-node",
expectedPairingGeneration: "generation-node",
command: uploadCommand,
params: { name: "report.pdf", contentBase64: "dGVzdA==" },
timeoutMs: 120_000,
src/gateway/server-methods/terminal.ts
+4
View File
@@ -87,6 +87,9 @@ async function stageNodeTerminalUpload(
const result = await context.nodeRegistry.invoke({
nodeId,
expectedConnId: access.node.connId,
...(access.node.pairingGeneration
? { expectedPairingGeneration: access.node.pairingGeneration }
: {}),
command: NODE_TERMINAL_UPLOAD_COMMAND,
params: file,
timeoutMs: 120_000,
@@ -290,6 +293,7 @@ export const terminalHandlers: GatewayRequestHandlers = {
registry: context.nodeRegistry,
nodeId: relay.plan.nodeId,
expectedConnId: access.node.connId,
expectedPairingGeneration: access.node.pairingGeneration,
command: relay.plan.command,
params: relay.params,
});
src/gateway/server-node-events-types.ts
+2 -2
View File
@@ -12,8 +12,8 @@ export type NodeEventContext = {
deps: CliDeps;
broadcast: (event: string, payload: unknown, opts?: { dropIfSlow?: boolean }) => void;
nodeSendToSession: (sessionKey: string, event: string, payload: unknown) => void;
nodeSubscribe: (nodeId: string, sessionKey: string) => void;
nodeUnsubscribe: (nodeId: string, sessionKey: string) => void;
nodeSubscribe: (nodeId: string, sessionKey: string, connId?: string) => void | Promise<void>;
nodeUnsubscribe: (nodeId: string, sessionKey: string, connId?: string) => void | Promise<void>;
broadcastVoiceWakeChanged: (triggers: string[]) => void;
addChatRun: (sessionId: string, entry: ChatRunRegistration) => void;
removeChatRun: (
src/gateway/server-node-events.runtime.ts
+4 -1
View File
@@ -13,7 +13,10 @@ export { loadOrCreateProcessDeviceIdentity } from "../infra/device-identity.js";
export { requestHeartbeat } from "../infra/heartbeat-wake.js";
export { buildOutboundSessionContext } from "../infra/outbound/session-context.js";
export { resolveOutboundTarget } from "../infra/outbound/targets.js";
export { registerApnsRegistration } from "../infra/push-apns.js";
export {
ApnsRegistrationPairingChangedError,
registerApnsRegistration,
} from "../infra/push-apns.js";
export { enqueueSystemEvent } from "../infra/system-events.js";
export { deleteMediaBuffer } from "../media/store.js";
export { normalizeMainKey } from "../routing/session-key.js";
src/gateway/server-node-events.test.ts
+530 -61
View File
@@ -83,10 +83,16 @@ const sanitizeInboundSystemTagsMock = vi.hoisted(() =>
.replace(/^(\s*)System:(?=\s|$)/gim, "$1System (untrusted):"),
),
);
const updatePairedDeviceMetadataMock = vi.hoisted(() => vi.fn().mockResolvedValue(true));
const updatePairedDevicePresenceMock = vi.hoisted(() => vi.fn().mockResolvedValue(true));
const runtimeMocks = vi.hoisted(() => ({
agentCommandFromIngress: ingressAgentCommandMock,
ApnsRegistrationPairingChangedError: class ApnsRegistrationPairingChangedError extends Error {
constructor() {
super("node pairing changed before APNs registration");
this.name = "ApnsRegistrationPairingChangedError";
}
},
buildOutboundSessionContext: vi.fn(({ sessionKey }: { sessionKey: string }) => ({
key: sessionKey,
agentId: "main",
@@ -151,7 +157,7 @@ const runtimeMocks = vi.hoisted(() => ({
vi.mock("./server-node-events.runtime.js", () => runtimeMocks);
vi.mock("../infra/device-pairing.js", () => ({
updatePairedDeviceMetadata: updatePairedDeviceMetadataMock,
updatePairedDevicePresence: updatePairedDevicePresenceMock,
}));
import type { CliDeps } from "../cli/deps.js";
import type { HealthSummary } from "../commands/health.js";
@@ -321,13 +327,21 @@ function expectPresencePersistCall(
reason: string,
): void {
expect(mock).toHaveBeenCalledTimes(1);
const [actualDeviceId, metadata] = mockCall(mock) ?? [];
const [actualDeviceId, metadata, generation] = mockCall(mock) ?? [];
expect(actualDeviceId).toBe(deviceId);
expectFields(metadata, { lastSeenReason: reason });
expect(generation).toEqual({ nodeId: deviceId, key: `${deviceId}-generation` });
const lastSeenAtMs = (metadata as { lastSeenAtMs?: unknown } | undefined)?.lastSeenAtMs;
expect(typeof lastSeenAtMs).toBe("number");
}
function presenceConnection(deviceId: string, generation = `${deviceId}-generation`) {
return {
deviceId,
pairingGeneration: { nodeId: deviceId, key: generation },
};
}
describe("node exec events", () => {
beforeEach(() => {
enqueueSystemEventMock.mockClear();
@@ -340,8 +354,8 @@ describe("node exec events", () => {
persistInboundImagesForTranscriptMock.mockResolvedValue([]);
normalizeChannelIdVi.mockImplementation((channel?: string | null) => channel ?? null);
sanitizeInboundSystemTagsMock.mockClear();
updatePairedDeviceMetadataMock.mockClear();
updatePairedDeviceMetadataMock.mockResolvedValue(true);
updatePairedDevicePresenceMock.mockClear();
updatePairedDevicePresenceMock.mockResolvedValue(true);
});
it("enqueues exec.started events", async () => {
@@ -395,7 +409,9 @@ describe("node exec events", () => {
it("keeps a node run authorized from exec.started through exec.finished", async () => {
const registry = new NodeRegistry();
const frames: string[] = [];
registry.register(makeNodeClient("conn-1", "node-1", frames), {});
registry.register(makeNodeClient("conn-1", "node-1", frames), {
pairingIdentity: "identity-a",
});
const invoke = registry.invoke({
nodeId: "node-1",
command: "system.run",
@@ -763,14 +779,19 @@ describe("node exec events", () => {
it("stores direct APNs registrations from node events", async () => {
const ctx = buildCtx();
await handleNodeEvent(ctx, "node-direct", {
event: "push.apns.register",
payloadJSON: JSON.stringify({
token: "abcd1234abcd1234abcd1234abcd1234",
topic: "ai.openclaw.ios",
environment: "sandbox",
}),
});
await handleNodeEvent(
ctx,
"node-direct",
{
event: "push.apns.register",
payloadJSON: JSON.stringify({
token: "abcd1234abcd1234abcd1234abcd1234",
topic: "ai.openclaw.ios",
environment: "sandbox",
}),
},
{ resolveApnsRegistrationGeneration: () => "generation-node-direct" },
);
expect(registerApnsRegistrationVi).toHaveBeenCalledWith({
nodeId: "node-direct",
@@ -778,25 +799,31 @@ describe("node exec events", () => {
token: "abcd1234abcd1234abcd1234abcd1234",
topic: "ai.openclaw.ios",
environment: "sandbox",
expectedPairingGeneration: "generation-node-direct",
});
});
it("stores relay APNs registrations from node events", async () => {
const ctx = buildCtx();
await handleNodeEvent(ctx, "node-relay", {
event: "push.apns.register",
payloadJSON: JSON.stringify({
transport: "relay",
relayHandle: "relay-handle-123",
sendGrant: "send-grant-123",
gatewayDeviceId: "gateway-device-1",
installationId: "install-123",
topic: "ai.openclaw.ios",
environment: "production",
distribution: "official",
tokenDebugSuffix: "abcd1234",
}),
});
await handleNodeEvent(
ctx,
"node-relay",
{
event: "push.apns.register",
payloadJSON: JSON.stringify({
transport: "relay",
relayHandle: "relay-handle-123",
sendGrant: "send-grant-123",
gatewayDeviceId: "gateway-device-1",
installationId: "install-123",
topic: "ai.openclaw.ios",
environment: "production",
distribution: "official",
tokenDebugSuffix: "abcd1234",
}),
},
{ resolveApnsRegistrationGeneration: () => "generation-node-relay" },
);
expect(registerApnsRegistrationVi).toHaveBeenCalledWith({
nodeId: "node-relay",
@@ -808,25 +835,31 @@ describe("node exec events", () => {
environment: "production",
distribution: "official",
tokenDebugSuffix: "abcd1234",
expectedPairingGeneration: "generation-node-relay",
});
});
it("stores sandbox relay APNs registrations from node events", async () => {
const ctx = buildCtx();
await handleNodeEvent(ctx, "node-relay-sandbox", {
event: "push.apns.register",
payloadJSON: JSON.stringify({
transport: "relay",
relayHandle: "relay-handle-123",
sendGrant: "send-grant-123",
gatewayDeviceId: "gateway-device-1",
installationId: "install-123",
topic: "ai.openclaw.ios",
environment: "sandbox",
distribution: "official",
tokenDebugSuffix: "abcd1234",
}),
});
await handleNodeEvent(
ctx,
"node-relay-sandbox",
{
event: "push.apns.register",
payloadJSON: JSON.stringify({
transport: "relay",
relayHandle: "relay-handle-123",
sendGrant: "send-grant-123",
gatewayDeviceId: "gateway-device-1",
installationId: "install-123",
topic: "ai.openclaw.ios",
environment: "sandbox",
distribution: "official",
tokenDebugSuffix: "abcd1234",
}),
},
{ resolveApnsRegistrationGeneration: () => "generation-node-relay-sandbox" },
);
expect(registerApnsRegistrationVi).toHaveBeenCalledWith({
nodeId: "node-relay-sandbox",
@@ -838,6 +871,7 @@ describe("node exec events", () => {
environment: "sandbox",
distribution: "official",
tokenDebugSuffix: "abcd1234",
expectedPairingGeneration: "generation-node-relay-sandbox",
});
});
@@ -859,6 +893,61 @@ describe("node exec events", () => {
expect(registerApnsRegistrationVi).not.toHaveBeenCalled();
});
it("rejects APNs registration after the source pairing session is invalidated", async () => {
const warn = vi.fn();
const ctx: NodeEventContext = { ...buildCtx(), logGateway: { warn } };
const result = await handleNodeEvent(
ctx,
"node-invalidated-register",
{
event: "push.apns.register",
payloadJSON: JSON.stringify({
token: "abcd1234abcd1234abcd1234abcd1234",
topic: "ai.openclaw.ios",
environment: "sandbox",
}),
},
{ resolveApnsRegistrationGeneration: async () => null },
);
expect(result).toEqual({
ok: true,
event: "push.apns.register",
handled: false,
reason: "pairing_changed",
});
expect(registerApnsRegistrationVi).not.toHaveBeenCalled();
expect(warn).toHaveBeenCalledWith(
"push apns register rejected node=node-invalidated-register: stale or invalidated pairing session",
);
});
it("returns pairing changed when APNs registration loses ownership in its transaction", async () => {
registerApnsRegistrationVi.mockRejectedValueOnce(
new runtimeMocks.ApnsRegistrationPairingChangedError(),
);
const result = await handleNodeEvent(
buildCtx(),
"node-transaction-invalidated-register",
{
event: "push.apns.register",
payloadJSON: JSON.stringify({
token: "abcd1234abcd1234abcd1234abcd1234",
topic: "ai.openclaw.ios",
environment: "sandbox",
}),
},
{ resolveApnsRegistrationGeneration: async () => "generation-before-transaction" },
);
expect(result).toEqual({
ok: true,
event: "push.apns.register",
handled: false,
reason: "pairing_changed",
});
});
});
describe("voice transcript events", () => {
@@ -874,7 +963,7 @@ describe("voice transcript events", () => {
});
});
it("dedupes repeated transcript payloads for the same session", async () => {
it("dedupes repeated transcript agent dispatches for the same session", async () => {
const addChatRun = vi.fn();
const ctx = buildCtx();
ctx.addChatRun = addChatRun;
@@ -898,6 +987,258 @@ describe("voice transcript events", () => {
expect(canonicalizeSessionEntryAliasesMock).toHaveBeenCalledTimes(1);
});
it("persists only the accepted replay session ID when identical new-session events race", async () => {
const addChatRun = vi.fn();
const ctx = buildCtx();
ctx.addChatRun = addChatRun;
loadSessionEntryMock.mockImplementation((sessionKey: string) => ({
...buildSessionLookup(sessionKey),
entry: undefined,
}));
let persistedEntry: { sessionId?: string } | undefined;
canonicalizeSessionEntryAliasesMock.mockImplementation(async ({ target, update }) => {
const entry = update ? await update(undefined) : undefined;
persistedEntry = entry;
return { canonicalKey: target.canonicalKey, entry };
});
const detachedChecksStarted = createDeferred();
const detachedAdmission = createDeferred<boolean>();
let checkCount = 0;
const isConnectionCurrent = vi.fn(() => {
checkCount += 1;
if (checkCount === 1 || checkCount === 3) {
return true;
}
if (checkCount === 4) {
detachedChecksStarted.resolve();
}
return detachedAdmission.promise;
});
const payload = {
text: "one command for a new session",
sessionKey: "voice-new-session-replay-race",
};
await handleNodeEvent(
ctx,
"node-new-session-replay",
{
event: "voice.transcript",
payloadJSON: JSON.stringify(payload),
},
{ isConnectionCurrent },
);
await handleNodeEvent(
ctx,
"node-new-session-replay",
{
event: "voice.transcript",
payloadJSON: JSON.stringify(payload),
},
{ isConnectionCurrent },
);
await detachedChecksStarted.promise;
detachedAdmission.resolve(true);
await waitForFast(() => expect(agentCommandMock).toHaveBeenCalledTimes(1));
expect(canonicalizeSessionEntryAliasesMock).toHaveBeenCalledTimes(1);
expect(addChatRun).toHaveBeenCalledTimes(1);
const dispatched = mockCallArg(agentCommandMock) as { sessionId?: unknown };
expect(persistedEntry?.sessionId).toBe(dispatched.sessionId);
});
it("uses receipt time when delayed identical transcript admissions finish together", async () => {
const addChatRun = vi.fn();
const ctx = buildCtx();
ctx.addChatRun = addChatRun;
let now = 1_000;
const nowSpy = vi.spyOn(Date, "now").mockImplementation(() => now);
const detachedChecksStarted = createDeferred();
const detachedAdmission = createDeferred<boolean>();
let checkCount = 0;
const isConnectionCurrent = vi.fn(() => {
checkCount += 1;
if (checkCount === 1 || checkCount === 3) {
return true;
}
if (checkCount === 4) {
detachedChecksStarted.resolve();
}
return detachedAdmission.promise;
});
const payload = {
text: "repeat after the replay window",
sessionKey: "voice-delayed-admission-window",
};
try {
await handleNodeEvent(
ctx,
"node-delayed-admission",
{
event: "voice.transcript",
payloadJSON: JSON.stringify(payload),
},
{ isConnectionCurrent },
);
now = 3_000;
await handleNodeEvent(
ctx,
"node-delayed-admission",
{
event: "voice.transcript",
payloadJSON: JSON.stringify(payload),
},
{ isConnectionCurrent },
);
await detachedChecksStarted.promise;
now = 10_000;
detachedAdmission.resolve(true);
await waitForFast(() => expect(agentCommandMock).toHaveBeenCalledTimes(2));
expect(addChatRun).toHaveBeenCalledTimes(2);
expect(canonicalizeSessionEntryAliasesMock).toHaveBeenCalledTimes(2);
} finally {
nowSpy.mockRestore();
}
});
it("allows a current replay after rejecting the same transcript from a stale connection", async () => {
const addChatRun = vi.fn();
const ctx = buildCtx();
ctx.addChatRun = addChatRun;
const detachedChecksStarted = createDeferred();
const staleAdmission = createDeferred<boolean>();
let checkCount = 0;
const isConnectionCurrent = vi.fn(() => {
checkCount += 1;
if (checkCount === 1) {
return true;
}
if (checkCount === 2) {
detachedChecksStarted.resolve();
}
return staleAdmission.promise;
});
const payload = {
text: "replay after reconnect",
sessionKey: "voice-stale-replay-session",
};
await handleNodeEvent(
ctx,
"node-stale-voice",
{
event: "voice.transcript",
payloadJSON: JSON.stringify(payload),
},
{ isConnectionCurrent },
);
await detachedChecksStarted.promise;
await handleNodeEvent(
ctx,
"node-current-voice",
{
event: "voice.transcript",
payloadJSON: JSON.stringify(payload),
},
{ isConnectionCurrent: () => true },
);
expect(addChatRun).not.toHaveBeenCalled();
expect(agentCommandMock).not.toHaveBeenCalled();
staleAdmission.resolve(false);
await waitForFast(() => expect(agentCommandMock).toHaveBeenCalledTimes(1));
await waitForFast(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
expect(addChatRun).toHaveBeenCalledTimes(1);
expect(canonicalizeSessionEntryAliasesMock).toHaveBeenCalledTimes(1);
});
it("rechecks a queued replay after an earlier stale reservation is released", async () => {
const addChatRun = vi.fn();
const ctx = buildCtx();
ctx.addChatRun = addChatRun;
const staleCheckStarted = createDeferred();
const staleAdmission = createDeferred<boolean>();
let staleCheckCount = 0;
const isStaleConnectionCurrent = vi.fn(() => {
staleCheckCount += 1;
if (staleCheckCount === 1) {
return true;
}
staleCheckStarted.resolve();
return staleAdmission.promise;
});
let replayCurrent = true;
const isReplayConnectionCurrent = vi.fn(() => replayCurrent);
const payload = {
text: "invalidate while queued",
sessionKey: "voice-queued-replay-currentness",
};
await handleNodeEvent(
ctx,
"node-stale-queued-voice",
{
event: "voice.transcript",
payloadJSON: JSON.stringify(payload),
},
{ isConnectionCurrent: isStaleConnectionCurrent },
);
await staleCheckStarted.promise;
await handleNodeEvent(
ctx,
"node-replay-invalidated-while-queued",
{
event: "voice.transcript",
payloadJSON: JSON.stringify(payload),
},
{ isConnectionCurrent: isReplayConnectionCurrent },
);
await waitForFast(() => expect(isReplayConnectionCurrent).toHaveBeenCalledTimes(2));
replayCurrent = false;
staleAdmission.resolve(false);
await waitForFast(() => expect(isReplayConnectionCurrent).toHaveBeenCalledTimes(3));
await waitForFast(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
expect(agentCommandMock).not.toHaveBeenCalled();
expect(addChatRun).not.toHaveBeenCalled();
expect(canonicalizeSessionEntryAliasesMock).not.toHaveBeenCalled();
});
it("skips the detached session-store touch after voice admission loses ownership", async () => {
const addChatRun = vi.fn();
const ctx = buildCtx();
ctx.addChatRun = addChatRun;
let checkCount = 0;
const isConnectionCurrent = vi.fn(() => {
checkCount += 1;
return checkCount <= 3;
});
await handleNodeEvent(
ctx,
"node-stale-after-voice-admission",
{
event: "voice.transcript",
payloadJSON: JSON.stringify({
text: "do not persist stale voice ownership",
sessionKey: "voice-detached-store-currentness",
}),
},
{ isConnectionCurrent },
);
await waitForFast(() => expect(isConnectionCurrent).toHaveBeenCalledTimes(4));
await waitForFast(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
expect(agentCommandMock).toHaveBeenCalledTimes(1);
expect(addChatRun).toHaveBeenCalledTimes(1);
expect(canonicalizeSessionEntryAliasesMock).not.toHaveBeenCalled();
});
it("rejects a missing harness-owned session before touching the store", async () => {
const sessionKey = "agent:main:harness:codex:supervision:missing-voice";
loadSessionEntryMock.mockReturnValueOnce({
@@ -1352,7 +1693,9 @@ describe("agent request events", () => {
runtimeMocks.resolveSessionAgentId.mockClear();
runtimeMocks.resolveSessionModelRef.mockClear();
runtimeMocks.resolveGatewayModelSupportsImages.mockClear();
persistInboundImagesForTranscriptMock.mockClear();
persistInboundImagesForTranscriptMock.mockReset();
persistInboundImagesForTranscriptMock.mockResolvedValue([]);
runtimeMocks.deleteMediaBuffer.mockClear();
canonicalizeSessionEntryAliasesMock.mockClear();
loadSessionEntryMock.mockClear();
normalizeChannelIdVi.mockClear();
@@ -1450,6 +1793,83 @@ describe("agent request events", () => {
expectSuspendReady("receipt-delivery-ready");
});
it("does not launch agent work when pairing changes during model lookup", async () => {
const modelCatalog =
createDeferred<Awaited<ReturnType<NodeEventContext["loadGatewayModelCatalog"]>>>();
const ctx = buildCtx();
ctx.loadGatewayModelCatalog = vi.fn(() => modelCatalog.promise);
let connectionCurrent = true;
const isConnectionCurrent = vi.fn(async () => connectionCurrent);
const request = handleNodeEvent(
ctx,
"node-revoked-during-model-lookup",
{
event: "agent.request",
payloadJSON: JSON.stringify({
message: "describe this image",
sessionKey: "agent:main:revoked-during-model-lookup",
attachments: [{ type: "image", mimeType: "image/png", content: "AAAA" }],
deliver: true,
receipt: true,
channel: "telegram",
to: "123",
}),
},
{ isConnectionCurrent },
);
await waitForFast(() => expect(ctx.loadGatewayModelCatalog).toHaveBeenCalledTimes(1));
connectionCurrent = false;
modelCatalog.resolve([]);
await expect(request).resolves.toEqual({
ok: true,
event: "agent.request",
handled: false,
reason: "pairing_changed",
});
expect(parseMessageWithAttachmentsMock).not.toHaveBeenCalled();
expect(canonicalizeSessionEntryAliasesMock).not.toHaveBeenCalled();
expect(sendDurableMessageBatchMock).not.toHaveBeenCalled();
expect(persistInboundImagesForTranscriptMock).not.toHaveBeenCalled();
expect(agentCommandMock).not.toHaveBeenCalled();
});
it("cleans persisted transcript media when detached agent admission is revoked", async () => {
persistInboundImagesForTranscriptMock.mockResolvedValueOnce([
{
id: "saved-after-admission",
path: "/media/inbound/saved-after-admission.png",
size: 5,
contentType: "image/png",
},
]);
let currentnessChecks = 0;
const isConnectionCurrent = vi.fn(async () => {
currentnessChecks += 1;
return currentnessChecks < 6;
});
await handleNodeEvent(
buildCtx(),
"node-revoked-before-detached-start",
{
event: "agent.request",
payloadJSON: JSON.stringify({
message: "do not retain this media",
sessionKey: "agent:main:revoked-before-detached-start",
}),
},
{ isConnectionCurrent },
);
await waitForFast(() => {
expect(runtimeMocks.deleteMediaBuffer).toHaveBeenCalledWith("saved-after-admission");
});
expect(agentCommandMock).not.toHaveBeenCalled();
});
it.each([
["wrong owner", { agentHarnessId: "other", modelSelectionLocked: true }],
["missing session id", { agentHarnessId: "codex", modelSelectionLocked: true, sessionId: "" }],
@@ -1670,8 +2090,8 @@ describe("agent request events", () => {
});
beforeEach(() => {
updatePairedDeviceMetadataMock.mockClear();
updatePairedDeviceMetadataMock.mockResolvedValue(true);
updatePairedDevicePresenceMock.mockClear();
updatePairedDevicePresenceMock.mockResolvedValue(true);
});
it("persists authenticated node presence alive events", async () => {
@@ -1683,7 +2103,7 @@ describe("agent request events", () => {
event: "node.presence.alive",
payloadJSON: JSON.stringify({ trigger: "bg_app_refresh", sentAtMs: 123 }),
},
{ deviceId: "ios-presence-persist" },
presenceConnection("ios-presence-persist"),
);
expect(result).toEqual({
@@ -1693,7 +2113,7 @@ describe("agent request events", () => {
reason: "persisted",
});
expectPresencePersistCall(
updatePairedDeviceMetadataMock,
updatePairedDevicePresenceMock,
"ios-presence-persist",
"bg_app_refresh",
);
@@ -1712,11 +2132,31 @@ describe("agent request events", () => {
handled: false,
reason: "missing_device_identity",
});
expect(updatePairedDeviceMetadataMock).not.toHaveBeenCalled();
expect(updatePairedDevicePresenceMock).not.toHaveBeenCalled();
});
it("does not throttle unknown node presence alive identities", async () => {
updatePairedDeviceMetadataMock.mockResolvedValue(false);
it("rejects node presence alive events without the authenticated pairing generation", async () => {
const result = await handleNodeEvent(
buildCtx(),
"ios-presence-missing-generation",
{
event: "node.presence.alive",
payloadJSON: JSON.stringify({ trigger: "silent_push" }),
},
{ deviceId: "ios-presence-missing-generation" },
);
expect(result).toEqual({
ok: true,
event: "node.presence.alive",
handled: false,
reason: "pairing_changed",
});
expect(updatePairedDevicePresenceMock).not.toHaveBeenCalled();
});
it("does not throttle stale node presence alive generations", async () => {
updatePairedDevicePresenceMock.mockResolvedValue(false);
const ctx = buildCtx();
const result = await handleNodeEvent(
ctx,
@@ -1725,18 +2165,18 @@ describe("agent request events", () => {
event: "node.presence.alive",
payloadJSON: JSON.stringify({ trigger: "silent_push" }),
},
{ deviceId: "ios-presence-unpaired" },
presenceConnection("ios-presence-unpaired"),
);
expect(result).toEqual({
ok: true,
event: "node.presence.alive",
handled: false,
reason: "unpaired",
reason: "pairing_changed",
});
updatePairedDeviceMetadataMock.mockClear();
updatePairedDeviceMetadataMock.mockResolvedValue(true);
updatePairedDevicePresenceMock.mockClear();
updatePairedDevicePresenceMock.mockResolvedValue(true);
const retry = await handleNodeEvent(
ctx,
"ios-presence-unpaired",
@@ -1744,7 +2184,7 @@ describe("agent request events", () => {
event: "node.presence.alive",
payloadJSON: JSON.stringify({ trigger: "silent_push" }),
},
{ deviceId: "ios-presence-unpaired" },
presenceConnection("ios-presence-unpaired"),
);
expect(retry).toEqual({
ok: true,
@@ -1752,7 +2192,7 @@ describe("agent request events", () => {
handled: true,
reason: "persisted",
});
expect(updatePairedDeviceMetadataMock).toHaveBeenCalledTimes(1);
expect(updatePairedDevicePresenceMock).toHaveBeenCalledTimes(1);
});
it("throttles repeated node presence alive persistence per device", async () => {
@@ -1761,7 +2201,7 @@ describe("agent request events", () => {
event: "node.presence.alive" as const,
payloadJSON: JSON.stringify({ trigger: "silent_push" }),
};
const connection = { deviceId: "ios-presence-throttle" };
const connection = presenceConnection("ios-presence-throttle");
await handleNodeEvent(ctx, "ios-presence-throttle", event, connection);
const result = await handleNodeEvent(ctx, "ios-presence-throttle", event, connection);
@@ -1772,7 +2212,36 @@ describe("agent request events", () => {
handled: true,
reason: "throttled",
});
expect(updatePairedDeviceMetadataMock).toHaveBeenCalledTimes(1);
expect(updatePairedDevicePresenceMock).toHaveBeenCalledTimes(1);
});
it("does not throttle the first presence update from a replacement generation", async () => {
const ctx = buildCtx();
const event = {
event: "node.presence.alive" as const,
payloadJSON: JSON.stringify({ trigger: "silent_push" }),
};
await handleNodeEvent(
ctx,
"ios-presence-replacement",
event,
presenceConnection("ios-presence-replacement", "generation-a"),
);
const result = await handleNodeEvent(
ctx,
"ios-presence-replacement",
event,
presenceConnection("ios-presence-replacement", "generation-b"),
);
expect(result).toEqual({
ok: true,
event: "node.presence.alive",
handled: true,
reason: "persisted",
});
expect(updatePairedDevicePresenceMock).toHaveBeenCalledTimes(2);
});
it("updates authenticated accessibility-backed node activity without a system event", async () => {
@@ -1850,11 +2319,11 @@ describe("agent request events", () => {
event: "node.presence.alive",
payloadJSON: JSON.stringify({ trigger: "x".repeat(4096) }),
},
{ deviceId: "ios-presence-normalize" },
presenceConnection("ios-presence-normalize"),
);
expectPresencePersistCall(
updatePairedDeviceMetadataMock,
updatePairedDevicePresenceMock,
"ios-presence-normalize",
"background",
);
src/gateway/server-node-events.ts
+356 -88
View File
@@ -8,7 +8,7 @@ import {
import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
import { validateNodePresenceActivityPayload } from "../../packages/gateway-protocol/src/index.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { updatePairedDeviceMetadata } from "../infra/device-pairing.js";
import { updatePairedDevicePresence, type NodePairingGeneration } from "../infra/device-pairing.js";
import { formatErrorMessage } from "../infra/errors.js";
import {
resolveEventSessionKeyForPolicy,
@@ -26,6 +26,7 @@ import {
import type { NodeEvent, NodeEventContext } from "./server-node-events-types.js";
import {
agentCommandFromIngress,
ApnsRegistrationPairingChangedError,
buildOutboundSessionContext,
createOutboundSendDeps,
defaultRuntime,
@@ -62,6 +63,18 @@ const NODE_PRESENCE_PERSIST_MIN_INTERVAL_MS = 60_000;
const MAX_RECENT_NODE_PRESENCE_KEYS = 1024;
const recentVoiceTranscripts = new Map<string, { fingerprint: string; ts: number }>();
type VoiceTranscriptReservationAdmission = { work: Promise<unknown> } | null;
type VoiceTranscriptReservation = {
fingerprint: string;
receivedAt: number;
status: "pending" | "ready" | "checking" | "rejected";
isConnectionCurrent?: () => boolean | Promise<boolean>;
start?: () => Promise<unknown>;
resolve: (admission: VoiceTranscriptReservationAdmission) => void;
rejectDecision: (reason: unknown) => void;
decision: Promise<VoiceTranscriptReservationAdmission>;
};
const pendingVoiceTranscriptReservations = new Map<string, VoiceTranscriptReservation[]>();
const recentExecFinishedRuns = new Map<string, number>();
const recentNodePresencePersistAt = new Map<string, number>();
@@ -82,12 +95,19 @@ function dispatchNodeAgentCommand(
ctx: NodeEventContext,
nodeId: string,
input: NodeAgentCommandInput,
isConnectionCurrent?: () => boolean | Promise<boolean>,
onAdmissionRejected?: () => void | Promise<void>,
): void {
// The node RPC can finish before the agent starts its own session admission.
// Reserve a root now so suspension cannot acknowledge and then strand the turn.
void runWithGatewayIndependentRootWorkContinuation(() =>
agentCommandFromIngress(input, defaultRuntime, ctx.deps),
).catch((err: unknown) => {
// Reserve a root now so suspension cannot acknowledge and then strand the turn,
// but recheck the admitted connection before agent work actually starts.
void runWithGatewayIndependentRootWorkContinuation(async () => {
if (isConnectionCurrent && !(await isConnectionCurrent())) {
await onAdmissionRejected?.();
return;
}
await agentCommandFromIngress(input, defaultRuntime, ctx.deps);
}).catch((err: unknown) => {
ctx.logGateway.warn(`agent failed node=${nodeId}: ${formatForLog(err)}`);
});
}
@@ -164,6 +184,125 @@ function shouldDropDuplicateVoiceTranscript(params: {
return false;
}
function reserveVoiceTranscript(params: {
sessionKey: string;
fingerprint: string;
receivedAt: number;
}): {
admit: (params: {
isConnectionCurrent?: () => boolean | Promise<boolean>;
start: () => Promise<unknown>;
}) => Promise<VoiceTranscriptReservationAdmission>;
reject: () => void;
} {
// Resolve reservations in receipt order so delayed currentness checks cannot
// change the dedupe window, while rejected connections leave no committed state.
let resolveDecision: (admission: VoiceTranscriptReservationAdmission) => void = () => {};
let rejectDecision: (reason: unknown) => void = () => {};
const decision = new Promise<VoiceTranscriptReservationAdmission>((resolve, reject) => {
resolveDecision = resolve;
rejectDecision = reject;
});
const reservation: VoiceTranscriptReservation = {
fingerprint: params.fingerprint,
receivedAt: params.receivedAt,
status: "pending",
resolve: resolveDecision,
rejectDecision,
decision,
};
const queue = pendingVoiceTranscriptReservations.get(params.sessionKey) ?? [];
queue.push(reservation);
pendingVoiceTranscriptReservations.set(params.sessionKey, queue);
const drain = () => {
while (queue[0]?.status === "rejected") {
const next = queue.shift();
if (!next) {
break;
}
next.resolve(null);
}
const next = queue[0];
if (!next) {
pendingVoiceTranscriptReservations.delete(params.sessionKey);
return;
}
if (next.status !== "ready") {
return;
}
next.status = "checking";
void (async () => {
try {
const isCurrent = next.isConnectionCurrent ? await next.isConnectionCurrent() : true;
const admission =
isCurrent &&
!shouldDropDuplicateVoiceTranscript({
sessionKey: params.sessionKey,
fingerprint: next.fingerprint,
now: next.receivedAt,
}) &&
next.start
? { work: next.start() }
: null;
queue.shift();
next.resolve(admission);
} catch (err) {
queue.shift();
next.rejectDecision(err);
}
drain();
})();
};
const settle = (status: "ready" | "rejected") => {
if (reservation.status !== "pending") {
return;
}
reservation.status = status;
drain();
};
return {
admit: ({ isConnectionCurrent, start }) => {
reservation.isConnectionCurrent = isConnectionCurrent;
reservation.start = start;
settle("ready");
return reservation.decision;
},
reject: () => settle("rejected"),
};
}
function dispatchReservedVoiceAgentCommand(params: {
ctx: NodeEventContext;
nodeId: string;
input: NodeAgentCommandInput;
reservation: ReturnType<typeof reserveVoiceTranscript>;
isConnectionCurrent?: () => boolean | Promise<boolean>;
onStart: () => void;
}): void {
void runWithGatewayIndependentRootWorkContinuation(async () => {
if (params.isConnectionCurrent && !(await params.isConnectionCurrent())) {
params.reservation.reject();
return;
}
const admission = await params.reservation.admit({
isConnectionCurrent: params.isConnectionCurrent,
start: () => {
params.onStart();
return agentCommandFromIngress(params.input, defaultRuntime, params.ctx.deps);
},
});
if (!admission) {
return;
}
await admission.work;
}).catch((err: unknown) => {
params.reservation.reject();
params.ctx.logGateway.warn(`agent failed node=${params.nodeId}: ${formatForLog(err)}`);
});
}
function shouldDropDuplicateExecFinished(params: {
sessionKey: string;
runId: string;
@@ -296,23 +435,59 @@ function queueSessionStoreTouch(params: {
entry: LoadedSessionEntry["entry"];
sessionId: string;
now: number;
isConnectionCurrent?: () => boolean | Promise<boolean>;
}) {
// Voice dispatch intentionally does not wait for persistence, but a host
// snapshot must not race the accepted write after its node RPC returns.
void runWithGatewayIndependentRootWorkContinuation(() =>
touchSessionStore({
void runWithGatewayIndependentRootWorkContinuation(async () => {
if (params.isConnectionCurrent && !(await params.isConnectionCurrent())) {
return;
}
await touchSessionStore({
storePath: params.storePath,
canonicalKey: params.canonicalKey,
storeKeys: params.storeKeys,
entry: params.entry,
sessionId: params.sessionId,
now: params.now,
}),
).catch((err: unknown) => {
});
}).catch((err: unknown) => {
params.ctx.logGateway.warn("voice session-store update failed: " + formatForLog(err));
});
}
async function isNodeEventConnectionCurrent(opts?: {
isConnectionCurrent?: () => boolean | Promise<boolean>;
}): Promise<boolean> {
if (!opts?.isConnectionCurrent) {
return true;
}
try {
return await opts.isConnectionCurrent();
} catch {
return false;
}
}
function pairingChangedResult(event: string): NodeEventHandleResult {
return { ok: true, event, handled: false, reason: "pairing_changed" };
}
async function cleanupNodeEventMedia(
ids: Iterable<string>,
ctx: Pick<NodeEventContext, "logGateway">,
): Promise<void> {
for (const id of ids) {
try {
await deleteMediaBuffer(id);
} catch (cleanupErr) {
ctx.logGateway.warn(
`Failed to cleanup orphaned media ${id}: ${formatErrorMessage(cleanupErr)}`,
);
}
}
}
function parseSessionKeyFromPayloadJSON(payloadJSON: string): string | null {
let payload: unknown;
try {
@@ -383,8 +558,18 @@ export const handleNodeEvent = async (
ctx: NodeEventContext,
nodeId: string,
evt: NodeEvent,
opts?: { connId?: string; deviceId?: string; presenceAllowed?: boolean },
opts?: {
connId?: string;
deviceId?: string;
pairingGeneration?: NodePairingGeneration;
presenceAllowed?: boolean;
isConnectionCurrent?: () => boolean | Promise<boolean>;
resolveApnsRegistrationGeneration?: () => string | null | Promise<string | null>;
},
): Promise<NodeEventHandleResult | undefined> => {
if (!(await isNodeEventConnectionCurrent(opts))) {
return pairingChangedResult(evt.event);
}
switch (evt.event) {
case "voice.transcript": {
const obj = parsePayloadObject(evt.payloadJSON);
@@ -406,44 +591,55 @@ export const handleNodeEvent = async (
if (resolveAgentHarnessSessionContextError(canonicalKey, entry)) {
return undefined;
}
const now = Date.now();
const receivedAt = Date.now();
const fingerprint = resolveVoiceTranscriptFingerprint(obj, text);
if (shouldDropDuplicateVoiceTranscript({ sessionKey: canonicalKey, fingerprint, now })) {
return undefined;
}
const sessionId = entry?.sessionId ?? randomUUID();
queueSessionStoreTouch({
ctx,
storePath,
canonicalKey,
storeKeys,
entry,
sessionId,
now,
});
const runId = randomUUID();
// Ensure chat UI clients refresh when this run completes (even though it wasn't started via chat.send).
// This maps agent bus events (keyed by per-turn runId) to chat events (keyed by clientRunId).
ctx.addChatRun(runId, {
const transcriptReservation = reserveVoiceTranscript({
sessionKey: canonicalKey,
clientRunId: `voice-${randomUUID()}`,
fingerprint,
receivedAt,
});
dispatchNodeAgentCommand(ctx, nodeId, {
runId,
message: text,
sessionId,
sessionKey: canonicalKey,
thinking: "low",
deliver: false,
messageChannel: "node",
inputProvenance: {
kind: "external_user",
sourceChannel: "voice",
sourceTool: "gateway.voice.transcript",
dispatchReservedVoiceAgentCommand({
ctx,
nodeId,
input: {
runId,
message: text,
sessionId,
sessionKey: canonicalKey,
thinking: "low",
deliver: false,
messageChannel: "node",
inputProvenance: {
kind: "external_user",
sourceChannel: "voice",
sourceTool: "gateway.voice.transcript",
},
allowModelOverride: false,
},
reservation: transcriptReservation,
isConnectionCurrent: opts?.isConnectionCurrent,
onStart: () => {
queueSessionStoreTouch({
ctx,
storePath,
canonicalKey,
storeKeys,
entry,
sessionId,
now: receivedAt,
isConnectionCurrent: opts?.isConnectionCurrent,
});
// Ensure chat UI clients refresh when this run completes (even though it wasn't started via chat.send).
// This maps agent bus events (keyed by per-turn runId) to chat events (keyed by clientRunId).
ctx.addChatRun(runId, {
sessionKey: canonicalKey,
clientRunId: `voice-${randomUUID()}`,
});
},
allowModelOverride: false,
});
return undefined;
}
@@ -508,6 +704,9 @@ export const handleNodeEvent = async (
provider: modelRef.provider,
model: modelRef.model,
});
if (!(await isNodeEventConnectionCurrent(opts))) {
return pairingChangedResult(evt.event);
}
try {
const parsed = await parseMessageWithAttachments(message, normalizedAttachments, {
maxBytes: resolveChatAttachmentMaxBytes(cfg),
@@ -518,6 +717,13 @@ export const handleNodeEvent = async (
// explicitly rather than saving them where the agent cannot reach them.
acceptNonImage: false,
});
if (!(await isNodeEventConnectionCurrent(opts))) {
await cleanupNodeEventMedia(
(parsed.offloadedRefs ?? []).map((ref) => ref.id),
ctx,
);
return pairingChangedResult(evt.event);
}
message = parsed.message.trim();
images = parsed.images;
imageOrder = parsed.imageOrder;
@@ -527,15 +733,10 @@ export const handleNodeEvent = async (
`agent.request message exceeds limit after attachment parsing (length=${message.length})`,
);
if (parsed.offloadedRefs && parsed.offloadedRefs.length > 0) {
for (const ref of parsed.offloadedRefs) {
try {
await deleteMediaBuffer(ref.id);
} catch (cleanupErr) {
ctx.logGateway.warn(
`Failed to cleanup orphaned media ${ref.id}: ${formatErrorMessage(cleanupErr)}`,
);
}
}
await cleanupNodeEventMedia(
parsed.offloadedRefs.map((ref) => ref.id),
ctx,
);
}
return undefined;
}
@@ -560,6 +761,13 @@ export const handleNodeEvent = async (
const now = Date.now();
const sessionId = entry?.sessionId ?? randomUUID();
if (!(await isNodeEventConnectionCurrent(opts))) {
await cleanupNodeEventMedia(
(offloadedRefs ?? []).map((ref) => ref.id),
ctx,
);
return pairingChangedResult(evt.event);
}
await touchSessionStore({
storePath,
canonicalKey,
@@ -568,6 +776,13 @@ export const handleNodeEvent = async (
sessionId,
now,
});
if (!(await isNodeEventConnectionCurrent(opts))) {
await cleanupNodeEventMedia(
(offloadedRefs ?? []).map((ref) => ref.id),
ctx,
);
return pairingChangedResult(evt.event);
}
if (deliverRequested && (!channel || !to)) {
const entryChannel =
@@ -591,19 +806,48 @@ export const handleNodeEvent = async (
);
}
if (!(await isNodeEventConnectionCurrent(opts))) {
await cleanupNodeEventMedia(
(offloadedRefs ?? []).map((ref) => ref.id),
ctx,
);
return pairingChangedResult(evt.event);
}
const persistedTranscriptMedia = await persistInboundImagesForTranscript({
images,
imageOrder,
offloadedRefs,
log: ctx.logGateway,
logContext: "agent.request",
});
if (!(await isNodeEventConnectionCurrent(opts))) {
await cleanupNodeEventMedia(
persistedTranscriptMedia.map((media) => media.id),
ctx,
);
return pairingChangedResult(evt.event);
}
const transcriptMedia = persistedTranscriptMedia.map((media) => ({
path: media.path,
contentType: media.contentType,
}));
if (wantsReceipt && deliveryChannel && deliveryTo) {
// Delivery stays detached from agent startup, but remains part of the
// accepted node request until the durable send settles.
void runWithGatewayIndependentRootWorkContinuation(() =>
sendReceiptAck({
void runWithGatewayIndependentRootWorkContinuation(async () => {
if (!(await isNodeEventConnectionCurrent(opts))) {
return;
}
await sendReceiptAck({
cfg,
deps: ctx.deps,
sessionKey: canonicalKey,
channel: deliveryChannel,
to: deliveryTo,
text: receiptText,
}),
).catch((err: unknown) => {
});
}).catch((err: unknown) => {
ctx.logGateway.warn(`agent receipt failed node=${nodeId}: ${formatForLog(err)}`);
});
} else if (wantsReceipt) {
@@ -612,33 +856,33 @@ export const handleNodeEvent = async (
);
}
const transcriptMedia = (
await persistInboundImagesForTranscript({
dispatchNodeAgentCommand(
ctx,
nodeId,
{
runId: sessionId,
message,
images,
imageOrder,
offloadedRefs,
log: ctx.logGateway,
logContext: "agent.request",
})
).map((media) => ({ path: media.path, contentType: media.contentType }));
dispatchNodeAgentCommand(ctx, nodeId, {
runId: sessionId,
message,
images,
imageOrder,
...(transcriptMedia.length > 0 ? { transcriptMessage, transcriptMedia } : {}),
sessionId,
sessionKey: canonicalKey,
thinking: link?.thinking ?? undefined,
deliver,
to: deliveryTo,
channel: deliveryChannel,
timeout:
typeof link?.timeoutSeconds === "number" ? link.timeoutSeconds.toString() : undefined,
messageChannel: "node",
allowModelOverride: false,
});
...(transcriptMedia.length > 0 ? { transcriptMessage, transcriptMedia } : {}),
sessionId,
sessionKey: canonicalKey,
thinking: link?.thinking ?? undefined,
deliver,
to: deliveryTo,
channel: deliveryChannel,
timeout:
typeof link?.timeoutSeconds === "number" ? link.timeoutSeconds.toString() : undefined,
messageChannel: "node",
allowModelOverride: false,
},
opts?.isConnectionCurrent,
() =>
cleanupNodeEventMedia(
persistedTranscriptMedia.map((media) => media.id),
ctx,
),
);
return undefined;
}
case "notifications.changed": {
@@ -705,7 +949,7 @@ export const handleNodeEvent = async (
if (!sessionKey) {
return undefined;
}
ctx.nodeSubscribe(nodeId, sessionKey);
await ctx.nodeSubscribe(nodeId, sessionKey, opts?.connId);
return undefined;
}
case "chat.unsubscribe": {
@@ -716,7 +960,7 @@ export const handleNodeEvent = async (
if (!sessionKey) {
return undefined;
}
ctx.nodeUnsubscribe(nodeId, sessionKey);
await ctx.nodeUnsubscribe(nodeId, sessionKey, opts?.connId);
return undefined;
}
case "exec.started":
@@ -847,6 +1091,13 @@ export const handleNodeEvent = async (
const topic = typeof obj.topic === "string" ? obj.topic : "";
const environment = obj.environment;
try {
const expectedPairingGeneration = await opts?.resolveApnsRegistrationGeneration?.();
if (!expectedPairingGeneration) {
ctx.logGateway.warn(
`push apns register rejected node=${nodeId}: stale or invalidated pairing session`,
);
return pairingChangedResult(evt.event);
}
if (transport === "relay") {
const gatewayDeviceId = normalizeOptionalString(obj.gatewayDeviceId) ?? "";
const currentGatewayDeviceId = loadOrCreateProcessDeviceIdentity().deviceId;
@@ -867,6 +1118,7 @@ export const handleNodeEvent = async (
distribution: obj.distribution,
relayOrigin: obj.relayOrigin,
tokenDebugSuffix: obj.tokenDebugSuffix,
expectedPairingGeneration,
});
} else {
await registerApnsRegistration({
@@ -875,9 +1127,16 @@ export const handleNodeEvent = async (
token: typeof obj.token === "string" ? obj.token : "",
topic,
environment,
expectedPairingGeneration,
});
}
} catch (err) {
if (err instanceof ApnsRegistrationPairingChangedError) {
ctx.logGateway.warn(
`push apns register rejected node=${nodeId}: stale or invalidated pairing session`,
);
return pairingChangedResult(evt.event);
}
ctx.logGateway.warn(`push apns register failed node=${nodeId}: ${formatForLog(err)}`);
}
return undefined;
@@ -911,8 +1170,13 @@ export const handleNodeEvent = async (
if (!deviceId) {
return { ok: true, event: evt.event, handled: false, reason: "missing_device_identity" };
}
const pairingGeneration = opts?.pairingGeneration;
if (!pairingGeneration || pairingGeneration.nodeId !== deviceId) {
return pairingChangedResult(evt.event);
}
const now = Date.now();
const lastPersistedAt = recentNodePresencePersistAt.get(deviceId) ?? 0;
const presenceOwnerKey = `${deviceId}\0${pairingGeneration.key}`;
const lastPersistedAt = recentNodePresencePersistAt.get(presenceOwnerKey) ?? 0;
if (now - lastPersistedAt < NODE_PRESENCE_PERSIST_MIN_INTERVAL_MS) {
return { ok: true, event: evt.event, handled: true, reason: "throttled" };
}
@@ -921,14 +1185,18 @@ export const handleNodeEvent = async (
try {
// Node last-seen lives on the device record; node.pair.list projects
// it from there, so one write covers both surfaces.
const deviceUpdated = await updatePairedDeviceMetadata(deviceId, {
lastSeenAtMs: now,
lastSeenReason,
});
const deviceUpdated = await updatePairedDevicePresence(
deviceId,
{
lastSeenAtMs: now,
lastSeenReason,
},
pairingGeneration,
);
if (!deviceUpdated) {
return { ok: true, event: evt.event, handled: false, reason: "unpaired" };
return pairingChangedResult(evt.event);
}
recentNodePresencePersistAt.set(deviceId, now);
recentNodePresencePersistAt.set(presenceOwnerKey, now);
pruneBoundedTimestampMap(recentNodePresencePersistAt, {
now,
ttlMs: NODE_PRESENCE_PERSIST_MIN_INTERVAL_MS * 10,
src/gateway/server-node-session-runtime.test.ts
+219
View File
@@ -0,0 +1,219 @@
import { describe, expect, test, vi } from "vitest";
import {
createSessionEventSubscriberRegistry,
createSessionMessageSubscriberRegistry,
} from "./server-chat-state.js";
import { createGatewayNodeSessionRuntime } from "./server-node-session-runtime.js";
import type { GatewayWsClient } from "./server/ws-types.js";
type TestSocket = {
bufferedAmount: number;
send: (payload: string) => void;
close: (code?: number, reason?: string) => void;
};
function makeGatewayWsClient(connId: string, socket: TestSocket): GatewayWsClient {
return {
socket: socket as unknown as GatewayWsClient["socket"],
connId,
usesSharedGatewayAuth: false,
connect: {
role: "node",
scopes: [],
client: {
id: "node-client",
version: "1.0.0",
platform: "macos",
mode: "node",
},
device: { id: "node-a" },
} as unknown as GatewayWsClient["connect"],
};
}
function createRuntime(
resolveCurrentPairingGeneration: () => Promise<string>,
broadcast = vi.fn(),
isPairingStateCurrent: NonNullable<
Parameters<typeof createGatewayNodeSessionRuntime>[0]["isPairingStateCurrent"]
> = (_nodeId, expected) =>
expected.identity === "identity-a" && expected.generation === "generation-a",
) {
return createGatewayNodeSessionRuntime({
broadcast,
resolveCurrentPairingState: async () => ({
identity: "identity-a",
generation: await resolveCurrentPairingGeneration(),
}),
isPairingStateCurrent,
sessionEventSubscribers: createSessionEventSubscriberRegistry(),
sessionMessageSubscribers: createSessionMessageSubscriberRegistry(),
});
}
function registerNode(
runtime: ReturnType<typeof createRuntime>,
connId: string,
pairingGeneration: string,
frames: string[],
) {
const socket: TestSocket = {
bufferedAmount: 0,
send: vi.fn((payload: string) => frames.push(payload)),
close: vi.fn(),
};
runtime.nodeRegistry.register(makeGatewayWsClient(connId, socket), {
pairingIdentity: "identity-a",
pairingGeneration,
});
}
describe("gateway node session runtime", () => {
test("forwards subscribed payload json without parsing it again", async () => {
const frames: string[] = [];
const runtime = createRuntime(async () => "generation-a");
registerNode(runtime, "conn-node-a", "generation-a", frames);
runtime.nodeSubscribe("node-a", "main", "conn-node-a");
const parseSpy = vi.spyOn(JSON, "parse");
try {
runtime.nodeSendToSession("main", "chat", { ok: true });
await vi.waitFor(() => expect(frames).toHaveLength(1));
expect(parseSpy).not.toHaveBeenCalled();
} finally {
parseSpy.mockRestore();
}
expect(JSON.parse(frames[0] ?? "{}")).toEqual({
type: "event",
event: "chat",
payload: { ok: true },
});
});
test("fences voice-wake updates by pairing generation while retaining operator broadcasts", async () => {
let currentPairingGeneration = "generation-a";
const resolveCurrentPairingGeneration = vi.fn(async () => currentPairingGeneration);
const broadcast = vi.fn();
const runtime = createRuntime(
resolveCurrentPairingGeneration,
broadcast,
(_nodeId, expected) =>
expected.identity === "identity-a" && expected.generation === currentPairingGeneration,
);
const frames: string[] = [];
registerNode(runtime, "conn-node-a", "generation-a", frames);
const send = vi.spyOn(runtime.nodeRegistry, "sendEventRawForPairingGeneration");
const routing = {
version: 1 as const,
defaultTarget: { mode: "current" as const },
routes: [],
updatedAtMs: 1,
};
runtime.broadcastVoiceWakeChanged(["openclaw"]);
runtime.broadcastVoiceWakeRoutingChanged(routing);
await vi.waitFor(() => expect(frames).toHaveLength(2));
currentPairingGeneration = "generation-b";
runtime.broadcastVoiceWakeChanged(["retired"]);
runtime.broadcastVoiceWakeRoutingChanged({ ...routing, updatedAtMs: 2 });
await vi.waitFor(() => expect(send).toHaveBeenCalledTimes(4));
expect(frames.map((frame) => JSON.parse(frame))).toEqual([
{ type: "event", event: "voicewake.changed", payload: { triggers: ["openclaw"] } },
{ type: "event", event: "voicewake.routing.changed", payload: { config: routing } },
]);
expect(broadcast).toHaveBeenCalledTimes(4);
});
test("fences generation-less voice-wake updates by authenticated pairing identity", async () => {
let pairingExists = true;
const broadcast = vi.fn();
const runtime = createGatewayNodeSessionRuntime({
broadcast,
resolveCurrentPairingState: async () =>
pairingExists ? { identity: "identity-a" } : undefined,
isPairingStateCurrent: (_nodeId, expected) =>
pairingExists && expected.identity === "identity-a",
sessionEventSubscribers: createSessionEventSubscriberRegistry(),
sessionMessageSubscribers: createSessionMessageSubscriberRegistry(),
});
const frames: string[] = [];
const socket: TestSocket = {
bufferedAmount: 0,
send: vi.fn((payload: string) => frames.push(payload)),
close: vi.fn(),
};
const client = makeGatewayWsClient("conn-node-a", socket);
runtime.nodeRegistry.register(client, { pairingIdentity: "identity-a" });
const send = vi.spyOn(runtime.nodeRegistry, "sendEventRawForPairingGeneration");
runtime.broadcastVoiceWakeChanged(["openclaw"]);
await vi.waitFor(() => expect(frames).toHaveLength(1));
pairingExists = false;
runtime.broadcastVoiceWakeChanged(["retired"]);
await vi.waitFor(() => expect(client.invalidated).toBe(true));
expect(send).not.toHaveBeenCalled();
expect(frames.map((frame) => JSON.parse(frame))).toEqual([
{ type: "event", event: "voicewake.changed", payload: { triggers: ["openclaw"] } },
]);
expect(client.invalidated).toBe(true);
expect(broadcast).toHaveBeenCalledTimes(2);
});
test("does not inherit subscriptions across a replacement pairing generation", async () => {
let currentPairingGeneration = "generation-a";
const runtime = createRuntime(async () => currentPairingGeneration);
const originalFrames: string[] = [];
registerNode(runtime, "conn-original", "generation-a", originalFrames);
runtime.nodeSubscribe("node-a", "main", "conn-original");
runtime.nodeSendToSession("main", "chat", { seq: 1 });
await vi.waitFor(() => expect(originalFrames).toHaveLength(1));
currentPairingGeneration = "generation-b";
runtime.nodeSendToSession("main", "chat", { seq: 2 });
await vi.waitFor(() => expect(runtime.nodeRegistry.get("node-a")).toBeUndefined());
expect(originalFrames).toHaveLength(1);
const replacementFrames: string[] = [];
registerNode(runtime, "conn-replacement", "generation-b", replacementFrames);
runtime.nodeSubscribe("node-a", "retired", "conn-original");
runtime.nodeSendToSession("retired", "chat", { seq: 3 });
expect(replacementFrames).toHaveLength(0);
runtime.nodeSubscribe("node-a", "main", "conn-replacement");
runtime.nodeSendToSession("main", "chat", { seq: 4 });
await vi.waitFor(() => expect(replacementFrames).toHaveLength(1));
const reconnectFrames: string[] = [];
registerNode(runtime, "conn-reconnect", "generation-b", reconnectFrames);
runtime.nodeSendToSession("main", "chat", { seq: 5 });
await vi.waitFor(() => expect(reconnectFrames).toHaveLength(1));
});
test("preserves subscriptions for an exact live pairing generation promotion", async () => {
let currentPairingGeneration = "generation-a";
const runtime = createRuntime(async () => currentPairingGeneration);
const frames: string[] = [];
registerNode(runtime, "conn-node-a", "generation-a", frames);
runtime.nodeSubscribe("node-a", "main", "conn-node-a");
currentPairingGeneration = "generation-b";
expect(
runtime.nodeRegistry.updateSurface(
"node-a",
{ commands: [] },
{
expectedConnId: "conn-node-a",
expectedPairingIdentity: "identity-a",
expectedPairingGeneration: "generation-a",
nextPairingGeneration: "generation-b",
},
),
).not.toBeNull();
runtime.nodeSendToSession("main", "chat", { ok: true });
await vi.waitFor(() => expect(frames).toHaveLength(1));
});
});
src/gateway/server-node-session-runtime.ts
+84 -8
View File
@@ -1,7 +1,13 @@
import {
isNodePairingBindingCurrent,
resolveCurrentNodePairingBinding,
} from "../infra/node-pairing-state.js";
import type { VoiceWakeRoutingConfig } from "../infra/voicewake-routing.js";
// Gateway node session runtime factory.
// Creates node registry, subscription, and voice-wake fanout state.
import {
NodeRegistry,
serializeEventPayload,
type NodeRegistryOptions,
type SerializedEventPayload,
} from "./node-registry.js";
@@ -20,33 +26,102 @@ export function createGatewayNodeSessionRuntime(params: {
listRegisteredNodePluginToolCommands?: NodeRegistryOptions["listRegisteredNodePluginToolCommands"];
nodePluginToolsEnabled?: boolean;
nodeSkillsEnabled?: boolean;
resolveCurrentPairingState?: NodeRegistryOptions["resolveCurrentPairingState"];
isPairingStateCurrent?: NodeRegistryOptions["isPairingStateCurrent"];
onPairingInvalidated?: NodeRegistryOptions["onPairingInvalidated"];
sessionEventSubscribers: SessionEventSubscriberRegistry;
sessionMessageSubscribers: SessionMessageSubscriberRegistry;
}) {
const nodeSubscriptions = createNodeSubscriptionManager();
const nodeRegistry = new NodeRegistry({
listRegisteredNodePluginToolCommands: params.listRegisteredNodePluginToolCommands,
nodePluginToolsEnabled: params.nodePluginToolsEnabled,
nodeSkillsEnabled: params.nodeSkillsEnabled,
resolveCurrentPairingState:
params.resolveCurrentPairingState ?? resolveCurrentNodePairingBinding,
isPairingStateCurrent: params.isPairingStateCurrent ?? isNodePairingBindingCurrent,
onPairingInvalidated: params.onPairingInvalidated,
onPairingGenerationChanged: (change) => {
nodeSubscriptions.updatePairingGeneration({
...change,
preserveSubscriptions: change.preserveSessionState,
});
},
});
const nodePresenceTimers = new Map<string, ReturnType<typeof setInterval>>();
const nodeSubscriptions = createNodeSubscriptionManager();
const sessionEventSubscribers = params.sessionEventSubscribers;
const sessionMessageSubscribers = params.sessionMessageSubscribers;
const nodeSendEvent = (opts: {
nodeId: string;
pairingGeneration: string;
event: string;
payloadJSON?: SerializedEventPayload | null;
}) => {
nodeRegistry.sendEventRaw(opts.nodeId, opts.event, opts.payloadJSON ?? null);
return nodeRegistry.sendEventRawForPairingGeneration(
opts.nodeId,
opts.pairingGeneration,
opts.event,
opts.payloadJSON ?? null,
);
};
// Session fanout goes through the subscription manager so node reconnects and
// explicit unsubscribes keep both node->session indexes in sync.
const nodeSendToSession = (sessionKey: string, event: string, payload: unknown) =>
nodeSubscriptions.sendToSession(sessionKey, event, payload, nodeSendEvent);
const nodeSendToAllSubscribed = (event: string, payload: unknown) =>
nodeSubscriptions.sendToAllSubscribed(event, payload, nodeSendEvent);
const nodeSendToSession = (sessionKey: string, event: string, payload: unknown) => {
void nodeSubscriptions.sendToSession(sessionKey, event, payload, nodeSendEvent);
};
const nodeSendToAllSubscribed = (event: string, payload: unknown) => {
void nodeSubscriptions.sendToAllSubscribed(event, payload, nodeSendEvent);
};
const resolveSubscriptionGeneration = (nodeId: string, connId?: string) => {
const node = nodeRegistry.get(nodeId);
return connId && node?.connId === connId ? node.pairingGeneration : undefined;
};
const nodeSubscribe = (nodeId: string, sessionKey: string, connId?: string) => {
const pairingGeneration = resolveSubscriptionGeneration(nodeId, connId);
if (pairingGeneration) {
nodeSubscriptions.subscribe(nodeId, pairingGeneration, sessionKey);
}
};
const nodeUnsubscribe = (nodeId: string, sessionKey: string, connId?: string) => {
const pairingGeneration = resolveSubscriptionGeneration(nodeId, connId);
if (pairingGeneration) {
nodeSubscriptions.unsubscribe(nodeId, pairingGeneration, sessionKey);
}
};
const sendVoiceWakeEventToCurrentNodes = (event: string, payload: unknown) => {
const payloadJSON = serializeEventPayload(payload);
for (const node of nodeRegistry.listConnected()) {
const pairingGeneration = node.pairingGeneration;
if (!pairingGeneration) {
// Pending first-surface sessions have no command authority yet, but
// their authenticated pairing identity still fences compatibility broadcasts.
if (node.pairingIdentity) {
void nodeRegistry
.sendEventForPairingIdentity({
nodeId: node.nodeId,
connId: node.connId,
pairingIdentity: node.pairingIdentity,
event,
payload,
})
.catch(() => undefined);
}
continue;
}
// Voice-wake broadcasts are fire-and-forget, but each node send still
// resolves persistent generation before crossing its transport.
void nodeRegistry
.sendEventRawForPairingGeneration(node.nodeId, pairingGeneration, event, payloadJSON)
.catch(() => undefined);
}
};
const broadcastVoiceWakeChanged = (triggers: string[]) => {
params.broadcast("voicewake.changed", { triggers }, { dropIfSlow: true });
sendVoiceWakeEventToCurrentNodes("voicewake.changed", { triggers });
};
const broadcastVoiceWakeRoutingChanged = (config: VoiceWakeRoutingConfig) => {
params.broadcast("voicewake.routing.changed", { config }, { dropIfSlow: true });
sendVoiceWakeEventToCurrentNodes("voicewake.routing.changed", { config });
};
const hasTalkNodeConnected = () => hasConnectedTalkNode(nodeRegistry);
@@ -57,10 +132,11 @@ export function createGatewayNodeSessionRuntime(params: {
sessionMessageSubscribers,
nodeSendToSession,
nodeSendToAllSubscribed,
nodeSubscribe: nodeSubscriptions.subscribe,
nodeUnsubscribe: nodeSubscriptions.unsubscribe,
nodeSubscribe,
nodeUnsubscribe,
nodeUnsubscribeAll: nodeSubscriptions.unsubscribeAll,
broadcastVoiceWakeChanged,
broadcastVoiceWakeRoutingChanged,
hasTalkNodeConnected,
};
}
src/gateway/server-node-subscriptions.test.ts
+74
View File
@@ -0,0 +1,74 @@
import { describe, expect, test, vi } from "vitest";
import type { SerializedEventPayload } from "./node-registry.js";
import { createNodeSubscriptionManager } from "./server-node-subscriptions.js";
describe("node subscription manager", () => {
test("routes events with each subscribed node pairing generation", async () => {
const manager = createNodeSubscriptionManager();
const sent: Array<{
nodeId: string;
pairingGeneration: string;
event: string;
payloadJSON?: SerializedEventPayload | null;
}> = [];
manager.subscribe("node-a", "generation-a", "main");
manager.subscribe("node-b", "generation-b", "main");
await manager.sendToSession("main", "chat", { ok: true }, (event) => {
sent.push(event);
});
expect(sent).toHaveLength(2);
expect(sent.map((event) => event.nodeId).toSorted()).toEqual(["node-a", "node-b"]);
expect(sent.map((event) => event.pairingGeneration).toSorted()).toEqual([
"generation-a",
"generation-b",
]);
});
test("unsubscribeAll clears both subscription indexes", async () => {
const manager = createNodeSubscriptionManager();
const sent: string[] = [];
const sendEvent = (event: { nodeId: string; event: string }) => {
sent.push(`${event.nodeId}:${event.event}`);
};
manager.subscribe("node-a", "generation-a", "main");
manager.subscribe("node-a", "generation-a", "secondary");
manager.unsubscribeAll("node-a");
await manager.sendToSession("main", "tick", {}, sendEvent);
await manager.sendToSession("secondary", "tick", {}, sendEvent);
expect(sent).toStrictEqual([]);
});
test("settles sender failures without rejecting fire-and-forget fanout", async () => {
const manager = createNodeSubscriptionManager();
const sent: string[] = [];
manager.subscribe("node-a", "generation-a", "main");
manager.subscribe("node-b", "generation-b", "main");
await expect(
manager.sendToSession("main", "tick", {}, ({ nodeId }) => {
if (nodeId === "node-a") {
throw new Error("transport failed");
}
sent.push(nodeId);
}),
).resolves.toBeUndefined();
expect(sent).toStrictEqual(["node-b"]);
});
test("drops unserializable payloads without rejecting fanout", async () => {
const manager = createNodeSubscriptionManager();
const sendEvent = vi.fn();
manager.subscribe("node-a", "generation-a", "main");
await expect(
manager.sendToSession("main", "tick", { invalid: 1n }, sendEvent),
).resolves.toBeUndefined();
expect(sendEvent).not.toHaveBeenCalled();
});
});
src/gateway/server-node-subscriptions.ts
+162 -61
View File
@@ -6,107 +6,175 @@ import { serializeEventPayload, type SerializedEventPayload } from "./node-regis
// events can fan out by session and all node cleanup paths remove reverse links.
type NodeSendEventFn = (opts: {
nodeId: string;
pairingGeneration: string;
event: string;
payloadJSON?: SerializedEventPayload | null;
}) => void;
}) => void | Promise<unknown>;
type NodeListConnectedFn = () => Array<{ nodeId: string }>;
type NodeListConnectedFn = () => Array<{ nodeId: string; pairingGeneration?: string }>;
type NodeSubscriptionManager = {
subscribe: (nodeId: string, sessionKey: string) => void;
unsubscribe: (nodeId: string, sessionKey: string) => void;
unsubscribeAll: (nodeId: string) => void;
subscribe: (nodeId: string, pairingGeneration: string, sessionKey: string) => void;
unsubscribe: (nodeId: string, pairingGeneration: string, sessionKey: string) => void;
unsubscribeAll: (nodeId: string, pairingGeneration?: string) => void;
updatePairingGeneration: (params: {
nodeId: string;
previousPairingGeneration: string;
nextPairingGeneration: string;
preserveSubscriptions: boolean;
}) => void;
sendToSession: (
sessionKey: string,
event: string,
payload: unknown,
sendEvent?: NodeSendEventFn | null,
) => void;
) => Promise<void>;
sendToAllSubscribed: (
event: string,
payload: unknown,
sendEvent?: NodeSendEventFn | null,
) => void;
) => Promise<void>;
sendToAllConnected: (
event: string,
payload: unknown,
listConnected?: NodeListConnectedFn | null,
sendEvent?: NodeSendEventFn | null,
) => void;
) => Promise<void>;
clear: () => void;
};
/** Manages node subscriptions to gateway session events. */
export function createNodeSubscriptionManager(): NodeSubscriptionManager {
const nodeSubscriptions = new Map<string, Set<string>>();
const sessionSubscribers = new Map<string, Set<string>>();
const nodeSubscriptions = new Map<
string,
{ pairingGeneration: string; sessionKeys: Set<string> }
>();
const sessionSubscribers = new Map<string, Map<string, string>>();
const toPayloadJSON = (payload: unknown) => serializeEventPayload(payload);
const subscribe = (nodeId: string, sessionKey: string) => {
const normalizedNodeId = nodeId.trim();
const normalizedSessionKey = sessionKey.trim();
if (!normalizedNodeId || !normalizedSessionKey) {
return;
const toPayloadJSON = (payload: unknown): SerializedEventPayload | null | undefined => {
try {
return serializeEventPayload(payload);
} catch {
return undefined;
}
let nodeSet = nodeSubscriptions.get(normalizedNodeId);
if (!nodeSet) {
nodeSet = new Set<string>();
nodeSubscriptions.set(normalizedNodeId, nodeSet);
}
if (nodeSet.has(normalizedSessionKey)) {
return;
}
nodeSet.add(normalizedSessionKey);
let sessionSet = sessionSubscribers.get(normalizedSessionKey);
if (!sessionSet) {
sessionSet = new Set<string>();
sessionSubscribers.set(normalizedSessionKey, sessionSet);
}
sessionSet.add(normalizedNodeId);
};
const unsubscribe = (nodeId: string, sessionKey: string) => {
const settleFanout = async (sends: Array<() => void | Promise<unknown>>): Promise<void> => {
// Public gateway callers intentionally fire-and-forget fanout. Settle every
// sender so one transport failure cannot become an unhandled rejection.
await Promise.allSettled(sends.map((send) => Promise.resolve().then(send)));
};
const subscribe = (nodeId: string, pairingGeneration: string, sessionKey: string) => {
const normalizedNodeId = nodeId.trim();
const normalizedPairingGeneration = pairingGeneration.trim();
const normalizedSessionKey = sessionKey.trim();
if (!normalizedNodeId || !normalizedSessionKey) {
if (!normalizedNodeId || !normalizedPairingGeneration || !normalizedSessionKey) {
return;
}
const nodeSet = nodeSubscriptions.get(normalizedNodeId);
nodeSet?.delete(normalizedSessionKey);
if (nodeSet?.size === 0) {
let nodeEntry = nodeSubscriptions.get(normalizedNodeId);
if (nodeEntry?.pairingGeneration !== normalizedPairingGeneration) {
unsubscribeAll(normalizedNodeId);
nodeEntry = undefined;
}
if (!nodeEntry) {
nodeEntry = {
pairingGeneration: normalizedPairingGeneration,
sessionKeys: new Set<string>(),
};
nodeSubscriptions.set(normalizedNodeId, nodeEntry);
}
if (nodeEntry.sessionKeys.has(normalizedSessionKey)) {
return;
}
nodeEntry.sessionKeys.add(normalizedSessionKey);
let sessionMap = sessionSubscribers.get(normalizedSessionKey);
if (!sessionMap) {
sessionMap = new Map<string, string>();
sessionSubscribers.set(normalizedSessionKey, sessionMap);
}
sessionMap.set(normalizedNodeId, normalizedPairingGeneration);
};
const unsubscribe = (nodeId: string, pairingGeneration: string, sessionKey: string) => {
const normalizedNodeId = nodeId.trim();
const normalizedPairingGeneration = pairingGeneration.trim();
const normalizedSessionKey = sessionKey.trim();
if (!normalizedNodeId || !normalizedPairingGeneration || !normalizedSessionKey) {
return;
}
const nodeEntry = nodeSubscriptions.get(normalizedNodeId);
if (nodeEntry?.pairingGeneration !== normalizedPairingGeneration) {
return;
}
nodeEntry.sessionKeys.delete(normalizedSessionKey);
if (nodeEntry.sessionKeys.size === 0) {
nodeSubscriptions.delete(normalizedNodeId);
}
const sessionSet = sessionSubscribers.get(normalizedSessionKey);
sessionSet?.delete(normalizedNodeId);
if (sessionSet?.size === 0) {
const sessionMap = sessionSubscribers.get(normalizedSessionKey);
if (sessionMap?.get(normalizedNodeId) === normalizedPairingGeneration) {
sessionMap.delete(normalizedNodeId);
}
if (sessionMap?.size === 0) {
sessionSubscribers.delete(normalizedSessionKey);
}
};
const unsubscribeAll = (nodeId: string) => {
function unsubscribeAll(nodeId: string, pairingGeneration?: string) {
const normalizedNodeId = nodeId.trim();
const nodeSet = nodeSubscriptions.get(normalizedNodeId);
if (!nodeSet) {
const nodeEntry = nodeSubscriptions.get(normalizedNodeId);
if (
!nodeEntry ||
(pairingGeneration !== undefined && nodeEntry.pairingGeneration !== pairingGeneration.trim())
) {
return;
}
// Remove reverse session indexes before deleting the node index so session
// fanout cannot retain disconnected node ids.
for (const sessionKey of nodeSet) {
const sessionSet = sessionSubscribers.get(sessionKey);
sessionSet?.delete(normalizedNodeId);
if (sessionSet?.size === 0) {
for (const sessionKey of nodeEntry.sessionKeys) {
const sessionMap = sessionSubscribers.get(sessionKey);
if (sessionMap?.get(normalizedNodeId) === nodeEntry.pairingGeneration) {
sessionMap.delete(normalizedNodeId);
}
if (sessionMap?.size === 0) {
sessionSubscribers.delete(sessionKey);
}
}
nodeSubscriptions.delete(normalizedNodeId);
}
const updatePairingGeneration = (params: {
nodeId: string;
previousPairingGeneration: string;
nextPairingGeneration: string;
preserveSubscriptions: boolean;
}) => {
const normalizedNodeId = params.nodeId.trim();
const previousPairingGeneration = params.previousPairingGeneration.trim();
const nextPairingGeneration = params.nextPairingGeneration.trim();
const nodeEntry = nodeSubscriptions.get(normalizedNodeId);
if (
!nodeEntry ||
!previousPairingGeneration ||
nodeEntry.pairingGeneration !== previousPairingGeneration
) {
return;
}
if (!params.preserveSubscriptions || !nextPairingGeneration) {
unsubscribeAll(normalizedNodeId, previousPairingGeneration);
return;
}
nodeEntry.pairingGeneration = nextPairingGeneration;
for (const sessionKey of nodeEntry.sessionKeys) {
sessionSubscribers.get(sessionKey)?.set(normalizedNodeId, nextPairingGeneration);
}
};
const sendToSession = (
const sendToSession = async (
sessionKey: string,
event: string,
payload: unknown,
@@ -116,20 +184,27 @@ export function createNodeSubscriptionManager(): NodeSubscriptionManager {
if (!normalizedSessionKey || !sendEvent) {
return;
}
const subs = sessionSubscribers.get(normalizedSessionKey);
if (!subs || subs.size === 0) {
const subscribers = sessionSubscribers.get(normalizedSessionKey);
if (!subscribers || subscribers.size === 0) {
return;
}
const payloadJSON = toPayloadJSON(payload);
if (payloadJSON === undefined) {
return;
}
// Serialize once per event and reuse across all subscribed nodes to keep
// fanout deterministic and avoid repeated JSON conversion.
for (const nodeId of subs) {
sendEvent({ nodeId, event, payloadJSON });
}
await settleFanout(
[...subscribers].map(
([nodeId, pairingGeneration]) =>
() =>
sendEvent({ nodeId, pairingGeneration, event, payloadJSON }),
),
);
};
const sendToAllSubscribed = (
const sendToAllSubscribed = async (
event: string,
payload: unknown,
sendEvent?: NodeSendEventFn | null,
@@ -138,12 +213,24 @@ export function createNodeSubscriptionManager(): NodeSubscriptionManager {
return;
}
const payloadJSON = toPayloadJSON(payload);
for (const nodeId of nodeSubscriptions.keys()) {
sendEvent({ nodeId, event, payloadJSON });
if (payloadJSON === undefined) {
return;
}
await settleFanout(
[...nodeSubscriptions].map(
([nodeId, subscription]) =>
() =>
sendEvent({
nodeId,
pairingGeneration: subscription.pairingGeneration,
event,
payloadJSON,
}),
),
);
};
const sendToAllConnected = (
const sendToAllConnected = async (
event: string,
payload: unknown,
listConnected?: NodeListConnectedFn | null,
@@ -153,9 +240,22 @@ export function createNodeSubscriptionManager(): NodeSubscriptionManager {
return;
}
const payloadJSON = toPayloadJSON(payload);
for (const node of listConnected()) {
sendEvent({ nodeId: node.nodeId, event, payloadJSON });
if (payloadJSON === undefined) {
return;
}
await settleFanout(
listConnected().map(
(node) => () =>
node.pairingGeneration
? sendEvent({
nodeId: node.nodeId,
pairingGeneration: node.pairingGeneration,
event,
payloadJSON,
})
: undefined,
),
);
};
const clear = () => {
@@ -167,6 +267,7 @@ export function createNodeSubscriptionManager(): NodeSubscriptionManager {
subscribe,
unsubscribe,
unsubscribeAll,
updatePairingGeneration,
sendToSession,
sendToAllSubscribed,
sendToAllConnected,
src/gateway/server-request-context.test.ts
+12 -3
View File
@@ -53,10 +53,10 @@ function makeContextParams(
nodeSubscribe: vi.fn(),
nodeUnsubscribe: vi.fn(),
nodeUnsubscribeAll: vi.fn(),
hasConnectedTalkNode: vi.fn(() => false),
hasConnectedTalkNode: vi.fn(async () => false),
clients: new Set(),
enforceSharedGatewayAuthGenerationForConfigWrite: vi.fn(),
nodeRegistry: {} as never,
nodeRegistry: { invalidateConnectionForPairingChange: vi.fn() } as never,
agentRunSeq: new Map(),
chatAbortControllers: new Map(),
chatQueuedTurns: new Map(),
@@ -297,9 +297,14 @@ describe("createGatewayRequestContext", () => {
};
const clients = new Set([target, unrelated]) as never;
const invalidateDeviceTransports = vi.fn();
const invalidateConnectionForPairingChange = vi.fn();
const context = createGatewayRequestContext(
makeContextParams({ clients, invalidateDeviceTransports }),
makeContextParams({
clients,
invalidateDeviceTransports,
nodeRegistry: { invalidateConnectionForPairingChange } as never,
}),
);
context.invalidateClientsForDevice?.("device-1", { reason: "device-token-rotated" });
@@ -308,6 +313,10 @@ describe("createGatewayRequestContext", () => {
"device-token-rotated",
);
expect(target.socket.close).not.toHaveBeenCalled();
expect(invalidateConnectionForPairingChange).toHaveBeenCalledWith(
"conn-target",
"device-token-rotated",
);
expect((unrelated as { invalidated?: boolean }).invalidated).toBeUndefined();
expect(unrelated.socket.close).not.toHaveBeenCalled();
src/gateway/server-request-context.ts
+5 -2
View File
@@ -250,8 +250,11 @@ export function createGatewayRequestContext(
if (opts?.role && gatewayClient.connect.role !== opts.role) {
continue;
}
// Marking is separate from socket close so already-buffered requests
// fail authorization even if transport teardown has not completed.
// Retire node-owned projections and pending invokes synchronously; socket
// close remains separate so already-buffered requests fail authorization.
if (gatewayClient.connId) {
params.nodeRegistry.invalidateConnectionForPairingChange(gatewayClient.connId, reason);
}
gatewayClient.invalidated = true;
gatewayClient.invalidatedReason = reason;
}
src/gateway/server-talk-nodes.test.ts
+10 -8
View File
@@ -7,7 +7,7 @@ import { hasConnectedTalkNode } from "./server-talk-nodes.js";
function registryWith(nodes: Array<Partial<NodeSession>>): NodeRegistry {
return {
listConnected: () =>
listCurrentConnected: async () =>
nodes.map((node, index) => ({
nodeId: `node-${index}`,
connId: `conn-${index}`,
@@ -22,16 +22,18 @@ function registryWith(nodes: Array<Partial<NodeSession>>): NodeRegistry {
}
describe("hasConnectedTalkNode", () => {
it("uses explicit talk capability instead of platform names", () => {
expect(
it("uses explicit talk capability instead of platform names", async () => {
await expect(
hasConnectedTalkNode(registryWith([{ platform: "android", caps: ["device"], commands: [] }])),
).toBe(false);
expect(hasConnectedTalkNode(registryWith([{ platform: "linux", caps: ["talk"] }]))).toBe(true);
).resolves.toBe(false);
await expect(
hasConnectedTalkNode(registryWith([{ platform: "linux", caps: ["talk"] }])),
).resolves.toBe(true);
});
it("accepts nodes that declare talk command support", () => {
expect(
it("accepts nodes that declare talk command support", async () => {
await expect(
hasConnectedTalkNode(registryWith([{ platform: "custom", commands: ["talk.ptt.start"] }])),
).toBe(true);
).resolves.toBe(true);
});
});
src/gateway/server-talk-nodes.ts
+2 -2
View File
@@ -9,8 +9,8 @@ const TALK_CAPABILITY = "talk";
const TALK_COMMAND_PREFIX = "talk.";
/** Returns true when any connected node can handle talk routing. */
export function hasConnectedTalkNode(registry: NodeRegistry): boolean {
return registry.listConnected().some(isTalkCapableNode);
export async function hasConnectedTalkNode(registry: NodeRegistry): Promise<boolean> {
return (await registry.listCurrentConnected()).some(isTalkCapableNode);
}
function isTalkCapableNode(node: NodeSession): boolean {
src/gateway/server.impl.ts
+13 -7
View File
@@ -59,7 +59,6 @@ import {
} from "../infra/restart.js";
import { enqueueSystemEvent } from "../infra/system-events.js";
import { upsertPresence } from "../infra/system-presence.js";
import type { VoiceWakeRoutingConfig } from "../infra/voicewake-routing.js";
import { withDiagnosticPhase } from "../logging/diagnostic-phase.js";
import { startDiagnosticHeartbeat, stopDiagnosticHeartbeat } from "../logging/diagnostic.js";
import { createSubsystemLogger, runtimeForLogger } from "../logging/subsystem.js";
@@ -80,7 +79,11 @@ import {
} from "../secrets/runtime-state.js";
import { createLazyRuntimeModule } from "../shared/lazy-runtime.js";
import { createLazyPromise } from "../shared/lazy-runtime.js";
import { recordRemoteNodeInfo, removeRemoteNodeInfo } from "../skills/runtime/remote.js";
import {
recordRemoteNodeInfo,
removeRemoteNodeInfo,
removeRemoteNodeInfoForConnection,
} from "../skills/runtime/remote.js";
import { createAuthRateLimiter, type AuthRateLimiter } from "./auth-rate-limit.js";
import { resolveGatewayAuth } from "./auth.js";
import type { RestartRecoveryCandidate } from "./chat-abort.js";
@@ -102,6 +105,7 @@ import {
import { isLoopbackHost } from "./net.js";
import { disposeNodeConnectionNotifications } from "./node-connection-notifications.js";
import { createNodeReapprovalCoordinator } from "./node-reapproval-coordinator.js";
import { clearNodeWakeState } from "./node-wake-state.js";
import {
mergeActivationSectionsIntoRuntimeConfig,
resolveGatewayReloadPluginActivationCandidate,
@@ -128,7 +132,6 @@ import type { GatewayInstanceRuntime } from "./server-instance-runtime.types.js"
import { applyGatewayLaneConcurrency, resolveGatewayLaneConcurrency } from "./server-lanes.js";
import { createGatewayServerLiveState, type GatewayServerLiveState } from "./server-live-state.js";
import { GATEWAY_EVENTS } from "./server-methods-list.js";
import { clearNodeWakeState } from "./server-methods/nodes-wake-state.js";
import type { GatewayRequestContext, GatewayRequestHandlers } from "./server-methods/types.js";
import { setFallbackGatewayContextResolver } from "./server-plugins.js";
import type { GatewayPluginReloadResult } from "./server-reload-handlers.js";
@@ -1311,6 +1314,7 @@ export async function startGatewayServer(
nodeUnsubscribe,
nodeUnsubscribeAll,
broadcastVoiceWakeChanged,
broadcastVoiceWakeRoutingChanged,
hasTalkNodeConnected,
} = createGatewayNodeSessionRuntime({
broadcast,
@@ -1319,6 +1323,11 @@ export async function startGatewayServer(
listRegisteredNodePluginToolCommands: () => pluginRegistry.nodeHostCommands,
nodePluginToolsEnabled: cfgAtStart.gateway?.nodes?.pluginTools?.enabled !== false,
nodeSkillsEnabled: cfgAtStart.gateway?.nodes?.skills?.enabled !== false,
onPairingInvalidated: ({ nodeId, connId }) => {
upsertPresence(nodeId, { reason: "disconnect" });
broadcastPresenceSnapshot({ broadcast, incrementPresenceVersion, getHealthVersion });
removeRemoteNodeInfoForConnection(nodeId, connId);
},
});
const { createWatchNodeHttpRuntime } = await import("./watch-node-http.js");
const watchNodeHttpRuntime = createWatchNodeHttpRuntime({
@@ -1351,6 +1360,7 @@ export async function startGatewayServer(
deviceFamily: session.deviceFamily,
commands: session.commands,
remoteIp: session.remoteIp,
pairingGeneration: session.pairingGeneration,
});
},
onNodeDisconnected: (nodeId) => {
@@ -1573,10 +1583,6 @@ export async function startGatewayServer(
clearFallbackGatewayContextForServer();
}
};
const broadcastVoiceWakeRoutingChanged = (config: VoiceWakeRoutingConfig) => {
broadcast("voicewake.routing.changed", { config }, { dropIfSlow: true });
};
try {
const earlyRuntime = await startupTrace.measure("runtime.early", () =>
loadGatewayStartupEarlyModule().then(({ startGatewayEarlyRuntime }) =>
src/gateway/server.roles-allowlist-update.test.ts
+5 -3
View File
@@ -502,7 +502,7 @@ describe("gateway node command allowlist", () => {
path: path.join(os.tmpdir(), `openclaw-node-allowed-${Date.now()}-${Math.random()}.sqlite`),
});
systemClient = await connectNodeClientWithPairing({
systemClient = await connectNodeClientWithNodePairing({
port,
commands: ["system.run"],
instanceId: "node-system-run",
@@ -521,7 +521,7 @@ describe("gateway node command allowlist", () => {
await systemClient.stopAndWait();
await waitForConnectedCount(0);
emptyClient = await connectNodeClientWithPairing({
emptyClient = await connectNodeClientWithNodePairing({
port,
commands: [],
instanceId: "node-empty",
@@ -611,7 +611,9 @@ describe("gateway node command allowlist", () => {
const nodeId = await findConnectedNodeIdByDisplayName(displayName);
await expectPendingPairingCommands(nodeId, ["canvas.snapshot", "system.run"]);
await expectCanvasSnapshotDenied(nodeId, "pending-node-canvas");
const denied = await invokeCanvasSnapshot(nodeId, "pending-node-canvas");
expect(denied.ok).toBe(false);
expect(denied.error?.details).toMatchObject({ code: "PAIRING_CHANGED" });
} finally {
await nodeClient?.stopAndWait();
}
src/gateway/server/ws-connection.ts
+1 -1
View File
@@ -19,13 +19,13 @@ import { resolveHostedPluginSurfaceUrl } from "../hosted-plugin-surface-url.js";
import type { GatewayMethodRegistry } from "../methods/registry.js";
import { isLoopbackAddress } from "../net.js";
import type { NodeReapprovalCoordinator } from "../node-reapproval-coordinator.js";
import { clearNodeWakeState } from "../node-wake-state.js";
import type { PluginNodeCapabilitySurface } from "../plugin-node-capability.js";
import {
MAX_BUFFERED_BYTES,
MAX_PAYLOAD_BYTES,
MAX_PREAUTH_PAYLOAD_BYTES,
} from "../server-constants.js";
import { clearNodeWakeState } from "../server-methods/nodes-wake-state.js";
import type { GatewayRequestContext, GatewayRequestHandlers } from "../server-methods/types.js";
import { formatError } from "../server-utils.js";
import { formatForLog, logWs } from "../ws-log.js";
src/gateway/server/ws-connection/connect-hello.ts
+15 -6
View File
@@ -189,20 +189,29 @@ export async function sendGatewayHello(
const requestContext = buildRequestContext();
const nodeId = connectParams.device?.id ?? connectParams.client.id;
const nodeSession = requestContext.nodeRegistry.get(nodeId);
// Claim by the authenticated node id only. A replacement may register while
// persistence waits; the router transfers the pending alert by node identity.
if (nodeSession?.connId === connId) {
const pairingGeneration = nodeSession?.pairingGeneration;
if (nodeSession?.connId === connId && pairingGeneration) {
try {
const connection = await recordPairedNodeConnection(
nodeSession.nodeId,
nodeSession.connectedAtMs,
undefined,
{ nodeId: nodeSession.nodeId, key: pairingGeneration },
);
if (!connection.recorded) {
logGateway.warn(`failed to record last connect for ${nodeSession.nodeId}: not paired`);
} else {
scheduleNodeConnectionNotification(requestContext.nodeRegistry, nodeSession, {
isFirstConnection: connection.firstConnection,
});
const currentSession = requestContext.nodeRegistry.getForPairingGeneration(
nodeSession.nodeId,
pairingGeneration,
);
// A rapid same-generation reconnect may take over the durable
// first-connection claim; generation lookup excludes stale replacements.
if (currentSession) {
scheduleNodeConnectionNotification(requestContext.nodeRegistry, currentSession, {
isFirstConnection: connection.firstConnection,
});
}
}
} catch (err) {
logGateway.warn(
src/gateway/server/ws-connection/connect-session.ts
+100 -7
View File
@@ -9,6 +9,11 @@ import {
import { ConnectErrorDetailCodes } from "../../../../packages/gateway-protocol/src/connect-error-details.js";
import { ErrorCodes, PROTOCOL_VERSION } from "../../../../packages/gateway-protocol/src/index.js";
import { getRuntimeConfig } from "../../../config/io.js";
import {
captureAuthenticatedNodePairingState,
type NodePairingGeneration,
type NodePairingIdentity,
} from "../../../infra/node-pairing-state.js";
import { upsertPresence } from "../../../infra/system-presence.js";
import { loadVoiceWakeRoutingConfig } from "../../../infra/voicewake-routing.js";
import { loadVoiceWakeConfig } from "../../../infra/voicewake.js";
@@ -22,6 +27,7 @@ import {
import { resolveRuntimeServiceVersion } from "../../../version.js";
import { verifyAgentRuntimeIdentityToken } from "../../agent-runtime-identity-token.js";
import { APPROVALS_SCOPE } from "../../method-scopes.js";
import { serializeEventPayload } from "../../node-registry.js";
import { isOperatorApprovalRuntimeToken } from "../../operator-approval-runtime-token.js";
import {
buildPluginNodeCapabilityScopedHostUrl,
@@ -47,6 +53,12 @@ import type {
/** Match production release versions (YYYY.M.PATCH or YYYY.M.PATCH-beta.N). */
const RELEASED_VERSION_RE = /^\d{4}\.\d+\.\d+/;
type AuthenticatedNodePairingAdmission = {
authenticated: { nodeId: string; publicKey: string; token: string };
identity: NodePairingIdentity;
generation?: NodePairingGeneration;
};
function isReleasedVersion(version: string): boolean {
return RELEASED_VERSION_RE.test(version);
}
@@ -108,6 +120,8 @@ export async function attachAuthenticatedGatewayConnect(
role,
scopes,
device,
devicePublicKey,
deviceToken,
authResult,
authMethod,
pairingLocality,
@@ -118,6 +132,46 @@ export async function attachAuthenticatedGatewayConnect(
return;
}
let nodePairingAdmission: AuthenticatedNodePairingAdmission | undefined;
if (role === "node") {
const nodeId = device?.id ?? connectParams.client.id;
const authenticatedNodeToken =
authMethod === "device-token"
? normalizeOptionalString(connectParams.auth?.deviceToken ?? connectParams.auth?.token)
: deviceToken?.token;
if (!device || !devicePublicKey || !authenticatedNodeToken) {
const message = "authenticated node pairing identity unavailable";
markHandshakeFailure("node-pairing-generation-changed", {});
sendHandshakeErrorResponse(ErrorCodes.NOT_PAIRED, message);
await releasePendingNodePairingCleanup();
close(1008, truncateCloseReason(message));
return;
}
const authenticatedNodePairing = {
nodeId,
publicKey: devicePublicKey,
token: authenticatedNodeToken,
};
const admittedPairingState =
await captureAuthenticatedNodePairingState(authenticatedNodePairing);
if (!admittedPairingState) {
const message = "node pairing changed during connect";
markHandshakeFailure(
"node-pairing-generation-changed",
device?.id ? { deviceId: device.id } : {},
);
sendHandshakeErrorResponse(ErrorCodes.NOT_PAIRED, message);
await releasePendingNodePairingCleanup();
close(1008, truncateCloseReason(message));
return;
}
nodePairingAdmission = {
authenticated: authenticatedNodePairing,
identity: admittedPairingState.identity,
...(admittedPairingState.generation ? { generation: admittedPairingState.generation } : {}),
};
}
// Presence lists user-visible clients/nodes. Ephemeral control-plane connections
// (CLI, backend RPC probes, tests) churn for the full TTL and stay excluded.
const shouldTrackPresence = !isEphemeralGatewayClient(connectParams.client);
@@ -306,6 +360,27 @@ export async function attachAuthenticatedGatewayConnect(
}
}
const admittedNodePairing = role === "node" ? nodePairingAdmission : undefined;
if (admittedNodePairing) {
const currentPairingState = await captureAuthenticatedNodePairingState(
admittedNodePairing.authenticated,
);
if (
!currentPairingState ||
currentPairingState.identity.key !== admittedNodePairing.identity.key ||
currentPairingState.generation?.key !== admittedNodePairing.generation?.key
) {
const message = "node pairing changed during connect";
markHandshakeFailure("node-pairing-generation-changed", {
deviceId: admittedNodePairing.identity.nodeId,
});
sendHandshakeErrorResponse(ErrorCodes.NOT_PAIRED, message);
await releasePendingNodePairingCleanup();
close(1008, truncateCloseReason(message));
return;
}
}
if (!setClient(nextClient)) {
await releasePendingNodePairingCleanup();
setCloseCause("connect-aborted-before-register", {
@@ -375,10 +450,13 @@ export async function attachAuthenticatedGatewayConnect(
});
incrementPresenceVersion();
}
if (role === "node") {
if (admittedNodePairing) {
const pairingGeneration = admittedNodePairing.generation?.key;
const requestContext = buildRequestContext();
const nodeSession = requestContext.nodeRegistry.register(nextClient, {
remoteIp: reportedClientIp,
pairingIdentity: admittedNodePairing.identity.key,
...(pairingGeneration ? { pairingGeneration } : {}),
});
recordRemoteNodeInfo({
nodeId: nodeSession.nodeId,
@@ -388,6 +466,7 @@ export async function attachAuthenticatedGatewayConnect(
deviceFamily: nodeSession.deviceFamily,
commands: nodeSession.commands,
remoteIp: nodeSession.remoteIp,
pairingGeneration: nodeSession.pairingGeneration,
});
runDetachedConnectWork(
async () => {
@@ -405,12 +484,28 @@ export async function attachAuthenticatedGatewayConnect(
(err) =>
logGateway.warn(`remote bin probe failed for ${nodeSession.nodeId}: ${formatForLog(err)}`),
);
const sendConnectSnapshot = async (event: string, payload: unknown) => {
if (pairingGeneration) {
await requestContext.nodeRegistry.sendEventRawForPairingGeneration(
nodeSession.nodeId,
pairingGeneration,
event,
serializeEventPayload(payload),
);
return;
}
await requestContext.nodeRegistry.sendEventForPairingIdentity({
nodeId: nodeSession.nodeId,
connId: nodeSession.connId,
pairingIdentity: admittedNodePairing.identity.key,
event,
payload,
});
};
runDetachedConnectWork(
async () => {
const cfg = await loadVoiceWakeConfig();
requestContext.nodeRegistry.sendEvent(nodeSession.nodeId, "voicewake.changed", {
triggers: cfg.triggers,
});
await sendConnectSnapshot("voicewake.changed", { triggers: cfg.triggers });
},
(err) =>
logGateway.warn(
@@ -420,9 +515,7 @@ export async function attachAuthenticatedGatewayConnect(
runDetachedConnectWork(
async () => {
const routing = await loadVoiceWakeRoutingConfig();
requestContext.nodeRegistry.sendEvent(nodeSession.nodeId, "voicewake.routing.changed", {
config: routing,
});
await sendConnectSnapshot("voicewake.routing.changed", { config: routing });
},
(err) =>
logGateway.warn(
src/gateway/terminal/node-relay.test.ts
+6 -1
View File
@@ -103,6 +103,7 @@ describe("createNodeRelayBackend", () => {
registry,
nodeId: "node-1",
expectedConnId: "conn-authorized",
expectedPairingGeneration: "generation-authorized",
command: "codex.terminal.resume.v1",
params: {},
});
@@ -110,7 +111,11 @@ describe("createNodeRelayBackend", () => {
backend.onExit(exit);
expect(invoke).toHaveBeenCalledWith(
expect.objectContaining({ nodeId: "node-1", expectedConnId: "conn-authorized" }),
expect.objectContaining({
nodeId: "node-1",
expectedConnId: "conn-authorized",
expectedPairingGeneration: "generation-authorized",
}),
);
await vi.waitFor(() =>
expect(exit).toHaveBeenCalledWith({
src/gateway/terminal/node-relay.ts
+4
View File
@@ -65,6 +65,7 @@ export async function createNodeRelayBackend(params: {
registry: NodeRegistry;
nodeId: string;
expectedConnId: string;
expectedPairingGeneration?: string;
command: string;
params: Record<string, unknown>;
}): Promise<TerminalBackend> {
@@ -82,6 +83,9 @@ export async function createNodeRelayBackend(params: {
.invoke({
nodeId: params.nodeId,
expectedConnId: params.expectedConnId,
...(params.expectedPairingGeneration
? { expectedPairingGeneration: params.expectedPairingGeneration }
: {}),
command: params.command,
params: params.params,
timeoutMs: 0,
src/gateway/test-helpers.server.ts
+1 -1
View File
@@ -1239,7 +1239,7 @@ export async function rpcReq<T extends Record<string, unknown>>(
id: string;
ok: boolean;
payload?: T | null | undefined;
error?: { message?: string; code?: string };
error?: { message?: string; code?: string; details?: unknown };
}>(
ws,
(o) => {
src/gateway/watch-node-http.test.ts
+172 -3
View File
@@ -1,4 +1,4 @@
import { createServer, type Server } from "node:http";
import { createServer, request as httpRequest, type ClientRequest, type Server } from "node:http";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
@@ -13,7 +13,13 @@ import {
publicKeyRawBase64UrlFromPem,
signDevicePayload,
} from "../infra/device-identity.js";
import { revokeDeviceToken } from "../infra/device-pairing.js";
import {
approveDevicePairing,
getPairedDevice,
requestDevicePairing,
resolveNodePairingState,
revokeDeviceToken,
} from "../infra/device-pairing.js";
import { listNodePairing } from "../infra/node-pairing.js";
import { NODE_PAIRING_SETUP_BOOTSTRAP_PROFILE } from "../shared/device-bootstrap-profile.js";
import { createTrackedTempDirs } from "../test-utils/tracked-temp-dirs.js";
@@ -102,7 +108,17 @@ async function startRuntime(
config?: OpenClawConfig;
},
) {
const nodeRegistry = new NodeRegistry();
const nodeRegistry = new NodeRegistry({
resolveCurrentPairingState: async (nodeId) => {
const state = resolveNodePairingState(await getPairedDevice(nodeId, baseDir));
return state
? {
identity: state.identity.key,
...(state.generation ? { generation: state.generation.key } : {}),
}
: undefined;
},
});
const broadcasts: Array<{ event: string; payload: unknown }> = [];
const connectedNodes: string[] = [];
const disconnectedNodes: Array<{ nodeId: string; reason: string }> = [];
@@ -164,6 +180,37 @@ async function readJson(response: Response): Promise<Record<string, unknown>> {
return (await response.json()) as Record<string, unknown>;
}
function startPartialJsonRequest(params: { url: string; authorization: string }): {
request: ClientRequest;
response: Promise<{ statusCode: number; body: string }>;
} {
let request!: ClientRequest;
const response = new Promise<{ statusCode: number; body: string }>((resolve, reject) => {
request = httpRequest(
params.url,
{
method: "POST",
headers: {
authorization: params.authorization,
"content-type": "application/json",
},
},
(result) => {
const chunks: Buffer[] = [];
result.on("data", (chunk: Buffer) => chunks.push(chunk));
result.once("end", () => {
resolve({
statusCode: result.statusCode ?? 0,
body: Buffer.concat(chunks).toString("utf8"),
});
});
},
);
request.once("error", reject);
});
return { request, response };
}
async function waitForLastConnectedMetadata(baseDir: string, nodeId: string): Promise<void> {
await vi.waitFor(async () => {
const paired = (await listNodePairing(baseDir)).paired.find((entry) => entry.nodeId === nodeId);
@@ -359,6 +406,128 @@ describe("watch node HTTP transport", () => {
expect(disconnectedNodes).toHaveLength(1);
});
it("rejects an HTTP node session after an external reapproval changes its generation", async () => {
const baseDir = await tempDirs.make("openclaw-watch-node-reapproval-");
const identity = loadOrCreateDeviceIdentity({
path: path.join(baseDir, "watch-identity.sqlite"),
});
const issued = await issueDeviceBootstrapToken({
baseDir,
profile: NODE_PAIRING_SETUP_BOOTSTRAP_PROFILE,
});
const { nodeRegistry, disconnectedNodes, runtime, baseUrl } = await startRuntime(baseDir);
const challenge = await readJson(await fetch(`${baseUrl}/challenge`));
const connectResponse = await fetch(`${baseUrl}/connect`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(
makeConnectParams({
identity,
nonce: String(challenge.nonce),
bootstrapToken: issued.token,
}),
),
});
const connected = await readJson(connectResponse);
const paired = await getPairedDevice(identity.deviceId, baseDir);
const repair = await requestDevicePairing(
{
deviceId: identity.deviceId,
publicKey: paired?.publicKey ?? "",
role: "node",
roles: ["node"],
scopes: [],
},
baseDir,
);
await approveDevicePairing(repair.request.requestId, { callerScopes: [] }, baseDir);
const stalePoll = await fetch(`${baseUrl}/poll`, {
method: "POST",
headers: { authorization: `Bearer ${String(connected.sessionToken)}` },
});
expect(stalePoll.status).toBe(401);
expect(nodeRegistry.get(identity.deviceId)).toBeUndefined();
expect(disconnectedNodes).toContainEqual({
nodeId: identity.deviceId,
reason: "node pairing changed",
});
runtime.close();
});
it("rejects an invoke result when pairing changes during body upload", async () => {
const baseDir = await tempDirs.make("openclaw-watch-node-result-generation-");
const identity = loadOrCreateDeviceIdentity({
path: path.join(baseDir, "watch-identity.sqlite"),
});
const issued = await issueDeviceBootstrapToken({
baseDir,
profile: NODE_PAIRING_SETUP_BOOTSTRAP_PROFILE,
});
const { nodeRegistry, disconnectedNodes, runtime, baseUrl } = await startRuntime(baseDir);
const challenge = await readJson(await fetch(`${baseUrl}/challenge`));
const connectResponse = await fetch(`${baseUrl}/connect`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(
makeConnectParams({
identity,
nonce: String(challenge.nonce),
bootstrapToken: issued.token,
}),
),
});
const connected = await readJson(connectResponse);
const invoke = nodeRegistry.invoke({
nodeId: identity.deviceId,
command: "device.info",
timeoutMs: 2_000,
});
const invokeAfterDisconnect = invoke.catch((error: unknown) => error);
const pollResponse = await fetch(`${baseUrl}/poll`, {
method: "POST",
headers: { authorization: `Bearer ${String(connected.sessionToken)}` },
});
const polled = await readJson(pollResponse);
const event = polled.event as { payload: { id: string } };
const currentCheck = vi.spyOn(nodeRegistry, "isConnectionCurrentPairingState");
currentCheck.mockClear();
const partial = startPartialJsonRequest({
url: `${baseUrl}/result`,
authorization: `Bearer ${String(connected.sessionToken)}`,
});
partial.request.write(`{"id":${JSON.stringify(event.payload.id)},"ok":`);
await vi.waitFor(() => expect(currentCheck).toHaveBeenCalledTimes(1));
const paired = await getPairedDevice(identity.deviceId, baseDir);
const repair = await requestDevicePairing(
{
deviceId: identity.deviceId,
publicKey: paired?.publicKey ?? "",
role: "node",
roles: ["node"],
scopes: [],
},
baseDir,
);
await approveDevicePairing(repair.request.requestId, { callerScopes: [] }, baseDir);
partial.request.end(`true,"payloadJSON":"{\\"model\\":\\"stale\\"}"}`);
const resultResponse = await partial.response;
expect(resultResponse.statusCode).toBe(401);
expect(JSON.parse(resultResponse.body)).toMatchObject({
error: { type: "unauthorized" },
});
expect(currentCheck).toHaveBeenCalledTimes(2);
expect(nodeRegistry.get(identity.deviceId)).toBeUndefined();
expect(disconnectedNodes).toContainEqual({
nodeId: identity.deviceId,
reason: "node pairing changed",
});
await expect(invokeAfterDisconnect).resolves.toBeInstanceOf(Error);
runtime.close();
});
it("rejects empty shadow credentials without consuming the challenge", async () => {
const baseDir = await tempDirs.make("openclaw-watch-node-auth-fields-");
const identity = loadOrCreateDeviceIdentity({
src/gateway/watch-node-http.ts
+39 -7
View File
@@ -31,6 +31,7 @@ import {
requestDevicePairing,
verifyDeviceToken,
} from "../infra/device-pairing.js";
import { captureAuthenticatedNodePairingState } from "../infra/node-pairing-state.js";
import {
approveNodePairing,
beginNodePairingConnect,
@@ -439,7 +440,10 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions)
},
});
const getSession = (req: IncomingMessage, res: ServerResponse): WatchNodeSession | null => {
const getSession = async (
req: IncomingMessage,
res: ServerResponse,
): Promise<WatchNodeSession | null> => {
const token = readBearerToken(req);
const session = token ? sessionsByToken.get(token) : undefined;
if (!session) {
@@ -451,6 +455,11 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions)
sendUnauthorized(res);
return null;
}
if (!(await options.nodeRegistry.isConnectionCurrentPairingState(session.connId))) {
closeSession(session, "node pairing changed");
sendUnauthorized(res);
return null;
}
touchSession(session);
return session;
};
@@ -801,6 +810,17 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions)
sendUnauthorized(res);
return;
}
const nodePairingState = await captureAuthenticatedNodePairingState({
nodeId: derivedDeviceId,
publicKey,
token: issuedDeviceToken,
baseDir: options.pairingBaseDir,
});
const nodePairingGeneration = nodePairingState?.generation;
if (!nodePairingState || !nodePairingGeneration) {
sendUnauthorized(res);
return;
}
if (closed || responseLifecycle.isAborted()) {
if (revokedBootstrapTokenRecord) {
await restoreDeviceBootstrapToken({
@@ -846,7 +866,11 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions)
};
const nodeSession = options.nodeRegistry.registerTransport(
client,
{ remoteIp: clientIp },
{
remoteIp: clientIp,
pairingIdentity: nodePairingState.identity.key,
pairingGeneration: nodePairingGeneration.key,
},
createTransport(session),
);
sessionsByToken.set(session.token, session);
@@ -906,6 +930,7 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions)
session.nodeId,
nodeSession.connectedAtMs,
options.pairingBaseDir,
nodePairingGeneration,
).catch((error: unknown) =>
options.onError?.("watch node last-connect metadata update failed", error),
);
@@ -933,7 +958,7 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions)
sendMethodNotAllowed(res);
return;
}
const session = getSession(req, res);
const session = await getSession(req, res);
if (!session) {
return;
}
@@ -969,12 +994,12 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions)
});
};
const handleDisconnect = (req: IncomingMessage, res: ServerResponse) => {
const handleDisconnect = async (req: IncomingMessage, res: ServerResponse) => {
if ((req.method ?? "").toUpperCase() !== "POST") {
sendMethodNotAllowed(res);
return;
}
const session = getSession(req, res);
const session = await getSession(req, res);
if (!session) {
return;
}
@@ -987,7 +1012,7 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions)
sendMethodNotAllowed(res);
return;
}
const session = getSession(req, res);
const session = await getSession(req, res);
if (!session) {
return;
}
@@ -1005,6 +1030,13 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions)
...(typeof body.error.message === "string" ? { message: body.error.message } : {}),
}
: null;
// Body upload can yield long enough for an external pairing mutation.
// Recheck the exact HTTP transport before committing its invoke result.
if (!(await options.nodeRegistry.isConnectionCurrentPairingState(session.connId))) {
closeSession(session, "node pairing changed");
sendUnauthorized(res);
return;
}
const accepted = options.nodeRegistry.handleInvokeResult({
id: body.id,
nodeId: session.nodeId,
@@ -1035,7 +1067,7 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions)
await handleConnect(req, res);
return true;
case DISCONNECT_PATH:
handleDisconnect(req, res);
await handleDisconnect(req, res);
return true;
case POLL_PATH:
await handlePoll(req, res);
src/infra/active-node-context.ts
+32 -7
View File
@@ -1,18 +1,43 @@
/** Stable active-node identity projected into the dynamic model runtime line. */
type ActiveNodeContext = {
nodeId: string;
pairingGeneration?: string;
};
let activeNodeContext: ActiveNodeContext | null = null;
type ActiveNodeContextState = ActiveNodeContext & {
isCurrent?: () => boolean;
};
/** Publishes the gateway's current active-node choice without volatile timestamps. */
export function setActiveNodeContext(next: ActiveNodeContext | null): void {
activeNodeContext = next ? { ...next } : null;
let activeNodeContext: ActiveNodeContextState | null = null;
function snapshotActiveNodeContext(context: ActiveNodeContextState): ActiveNodeContext {
return {
nodeId: context.nodeId,
...(context.pairingGeneration ? { pairingGeneration: context.pairingGeneration } : {}),
};
}
/** Returns a defensive snapshot for prompt construction. */
export function getActiveNodeContext(): ActiveNodeContext | null {
return activeNodeContext ? { ...activeNodeContext } : null;
/** Publishes the gateway's current active-node choice without volatile timestamps. */
export function setActiveNodeContext(
next: ActiveNodeContext | null,
options?: { isCurrent?: () => boolean },
): void {
activeNodeContext = next ? { ...next, ...options } : null;
}
/** Revalidates the published node before projecting it into an agent prompt. */
export function getCurrentActiveNodeContext(): ActiveNodeContext | null {
if (!activeNodeContext) {
return null;
}
try {
if (activeNodeContext.isCurrent && !activeNodeContext.isCurrent()) {
return null;
}
} catch {
return null;
}
return snapshotActiveNodeContext(activeNodeContext);
}
/** Formats the stable authenticated id; node-controlled labels stay out of prompt text. */
src/infra/device-pairing-store.ts
+116 -1
View File
@@ -14,6 +14,7 @@ import type {
import {
openOpenClawStateDatabase,
runOpenClawStateWriteTransaction,
type OpenClawStateDatabase,
type OpenClawStateDatabaseOptions,
} from "../state/openclaw-state-db.js";
import type {
@@ -25,7 +26,12 @@ import type {
PairedDeviceNodeSurface,
PairedDevicePendingNodeSurface,
} from "./device-pairing.types.js";
import { executeSqliteQuerySync, getNodeSqliteKysely } from "./kysely-sync.js";
import {
executeSqliteQuerySync,
executeSqliteQueryTakeFirstSync,
getNodeSqliteKysely,
} from "./kysely-sync.js";
import { clearApnsRegistrationFromDatabase } from "./push-apns-store-transaction.js";
type DevicePairingStoreState = {
pendingById: Record<string, DevicePairingPendingRecord>;
@@ -34,6 +40,19 @@ type DevicePairingStoreState = {
type DevicePairingStoreTarget = "pending" | "paired" | "both";
type PairedDeviceNodeSurfaceUpdate<T> =
| { value: T; persist: false }
| { value: T; persist: true; nodeSurface: PairedDeviceNodeSurface };
type PairedDevicePresenceUpdate<T> =
| { value: T; persist: false }
| {
value: T;
persist: true;
lastSeenAtMs: number;
lastSeenReason: string;
};
/** Route an explicit pairing base dir (tests, alternate state roots) to that dir's DB. */
function resolveDevicePairingStateDbOptions(baseDir?: string): OpenClawStateDatabaseOptions {
return baseDir ? { env: { ...process.env, OPENCLAW_STATE_DIR: baseDir } } : {};
@@ -245,11 +264,104 @@ export function loadDevicePairingStoreState(baseDir?: string): DevicePairingStor
return { pendingById, pairedByDeviceId };
}
/** Load one paired-device row without materializing either pairing table. */
export function loadPairedDevicePairingStoreRecord(
deviceId: string,
baseDir?: string,
): PairedDevice | null {
const { db } = openOpenClawStateDatabase(resolveDevicePairingStateDbOptions(baseDir));
return loadPairedDevicePairingStoreRecordFromDatabase(db, deviceId);
}
/** Load one paired-device row from an existing shared-state transaction. */
export function loadPairedDevicePairingStoreRecordFromDatabase(
db: OpenClawStateDatabase["db"],
deviceId: string,
): PairedDevice | null {
const normalizedDeviceId = deviceId.trim();
if (!normalizedDeviceId) {
return null;
}
const kysely = getNodeSqliteKysely<OpenClawStateKyselyDatabase>(db);
const row = executeSqliteQueryTakeFirstSync(
db,
kysely
.selectFrom("device_pairing_paired")
.selectAll()
.where("device_id", "=", normalizedDeviceId),
);
return row ? fromPairedRow(row) : null;
}
/** Read, validate, and update one paired node surface in a single cross-process transaction. */
export function updatePairedDeviceNodeSurfaceInTransaction<T>(
deviceId: string,
baseDir: string | undefined,
update: (device: PairedDevice | null) => PairedDeviceNodeSurfaceUpdate<T>,
): T {
return runOpenClawStateWriteTransaction(({ db }) => {
const normalizedDeviceId = deviceId.trim();
const device = normalizedDeviceId
? loadPairedDevicePairingStoreRecordFromDatabase(db, normalizedDeviceId)
: null;
const result = update(device);
if (!result.persist) {
return result.value;
}
if (!device) {
throw new Error("cannot update a missing paired-device node surface");
}
const kysely = getNodeSqliteKysely<OpenClawStateKyselyDatabase>(db);
executeSqliteQuerySync(
db,
kysely
.updateTable("device_pairing_paired")
.set({ node_surface_json: toJsonColumn(result.nodeSurface) })
.where("device_id", "=", normalizedDeviceId),
);
return result.value;
}, resolveDevicePairingStateDbOptions(baseDir));
}
/** Read, validate, and update one paired-device presence row in one transaction. */
export function updatePairedDevicePresenceInTransaction<T>(
deviceId: string,
baseDir: string | undefined,
update: (device: PairedDevice | null) => PairedDevicePresenceUpdate<T>,
): T {
return runOpenClawStateWriteTransaction(({ db }) => {
const normalizedDeviceId = deviceId.trim();
const device = normalizedDeviceId
? loadPairedDevicePairingStoreRecordFromDatabase(db, normalizedDeviceId)
: null;
const result = update(device);
if (!result.persist) {
return result.value;
}
if (!device) {
throw new Error("cannot update presence for a missing paired device");
}
const kysely = getNodeSqliteKysely<OpenClawStateKyselyDatabase>(db);
executeSqliteQuerySync(
db,
kysely
.updateTable("device_pairing_paired")
.set({
last_seen_at_ms: result.lastSeenAtMs,
last_seen_reason: result.lastSeenReason,
})
.where("device_id", "=", normalizedDeviceId),
);
return result.value;
}, resolveDevicePairingStateDbOptions(baseDir));
}
/** Replace the pending and/or paired table contents with the given snapshot. */
export function persistDevicePairingStoreState(
state: DevicePairingStoreState,
baseDir: string | undefined,
target: DevicePairingStoreTarget,
options?: { clearApnsNodeIds?: readonly string[] },
): void {
runOpenClawStateWriteTransaction(({ db }) => {
const kysely = getNodeSqliteKysely<OpenClawStateKyselyDatabase>(db);
@@ -267,6 +379,9 @@ export function persistDevicePairingStoreState(
executeSqliteQuerySync(db, kysely.insertInto("device_pairing_paired").values(rows));
}
}
for (const nodeId of new Set(options?.clearApnsNodeIds ?? [])) {
clearApnsRegistrationFromDatabase(db, nodeId);
}
}, resolveDevicePairingStateDbOptions(baseDir));
}
src/infra/device-pairing.test.ts
+157
View File
@@ -23,13 +23,17 @@ import {
removePairedDevice,
requestDevicePairing,
rejectDevicePairing,
resolveNodePairingGeneration,
revokeDeviceToken,
rotateDeviceToken,
updatePairedDeviceMetadata,
updatePairedDevicePresence,
verifyDeviceToken,
withPairedDeviceRecords,
type PairedDevice,
} from "./device-pairing.js";
import { approveNodePairing, requestNodePairing, updatePairedNodeBins } from "./node-pairing.js";
import { loadApnsRegistration, registerApnsRegistration } from "./push-apns.js";
type RotateDeviceTokenResult = Awaited<ReturnType<typeof rotateDeviceToken>>;
@@ -1004,6 +1008,51 @@ describe("device pairing tokens", () => {
});
});
test("stale node presence cannot update a replacement pairing generation", async () => {
const baseDir = await makeDevicePairingDir();
await setupPairedNodeDevice(baseDir);
const nodePairing = await requestNodePairing(
{ nodeId: "node-1", platform: "darwin", commands: ["system.run"] },
baseDir,
);
await approveNodePairing(
nodePairing.request.requestId,
{ callerScopes: ["operator.pairing", "operator.admin"] },
baseDir,
);
const original = resolveNodePairingGeneration(await getPairedDevice("node-1", baseDir));
expect(original).not.toBeNull();
if (!original) {
throw new Error("expected original node pairing generation");
}
await expect(updatePairedNodeBins("node-1", ["retired-bin"], original, baseDir)).resolves.toBe(
true,
);
const rotated = await rotateDeviceToken({
deviceId: "node-1",
role: "node",
scopes: [],
baseDir,
});
expect(rotated.ok).toBe(true);
const replacement = resolveNodePairingGeneration(await getPairedDevice("node-1", baseDir));
expect(replacement?.key).not.toBe(original.key);
await expect(
updatePairedDevicePresence(
"node-1",
{ lastSeenAtMs: 4321, lastSeenReason: "bg_app_refresh" },
original,
baseDir,
),
).resolves.toBe(false);
const paired = await getPairedDevice("node-1", baseDir);
expect(paired?.nodeSurface?.bins).toBeUndefined();
expect(paired?.lastSeenAtMs).toBeUndefined();
expect(paired?.lastSeenReason).toBeUndefined();
});
test("approval access metadata initializes paired device last-seen fields", async () => {
const baseDir = await makeDevicePairingDir();
const request = await requestDevicePairing(
@@ -2148,14 +2197,122 @@ describe("device pairing tokens", () => {
test("removes paired devices by device id", async () => {
const baseDir = await makeDevicePairingDir();
await setupPairedOperatorDevice(baseDir, ["operator.read"]);
await registerApnsRegistration({
nodeId: "device-1",
transport: "direct",
token: "ABCD1234ABCD1234ABCD1234ABCD1234",
topic: "ai.openclaw.ios",
environment: "sandbox",
baseDir,
});
const removed = await removePairedDevice("device-1", baseDir);
expect(removed).toEqual({ deviceId: "device-1" });
await expect(getPairedDevice("device-1", baseDir)).resolves.toBeNull();
await expect(loadApnsRegistration("device-1", baseDir)).resolves.toBeNull();
await expect(removePairedDevice("device-1", baseDir)).resolves.toBeNull();
});
test("clears APNs only when a node reapproval changes installation identity", async () => {
const baseDir = await makeDevicePairingDir();
await setupPairedNodeDevice(baseDir);
const nodePairing = await requestNodePairing({ nodeId: "node-1" }, baseDir);
await approveNodePairing(
nodePairing.request.requestId,
{ callerScopes: ["operator.pairing"] },
baseDir,
);
await registerApnsRegistration({
nodeId: "node-1",
transport: "direct",
token: "ABCD1234ABCD1234ABCD1234ABCD1234",
topic: "ai.openclaw.ios",
environment: "sandbox",
baseDir,
});
const sameInstallationRepair = await requestDevicePairing(
{
deviceId: "node-1",
publicKey: "public-key-node-1",
role: "node",
scopes: [],
},
baseDir,
);
await expect(
approveDevicePairing(sameInstallationRepair.request.requestId, { callerScopes: [] }, baseDir),
).resolves.toMatchObject({
status: "approved",
nodePairingGenerationChanged: true,
});
await expect(loadApnsRegistration("node-1", baseDir)).resolves.toMatchObject({
token: "abcd1234abcd1234abcd1234abcd1234",
});
const replacementRepair = await requestDevicePairing(
{
deviceId: "node-1",
publicKey: "public-key-node-1-replacement",
role: "node",
scopes: [],
},
baseDir,
);
await expect(
approveDevicePairing(replacementRepair.request.requestId, { callerScopes: [] }, baseDir),
).resolves.toMatchObject({
status: "approved",
nodePairingGenerationChanged: true,
});
await expect(loadApnsRegistration("node-1", baseDir)).resolves.toBeNull();
});
test("clears generation-owned node bins on public-key replacement", async () => {
const baseDir = await makeDevicePairingDir();
await setupPairedNodeDevice(baseDir);
const nodePairing = await requestNodePairing(
{ nodeId: "node-1", platform: "darwin", commands: ["system.run", "system.which"] },
baseDir,
);
await approveNodePairing(
nodePairing.request.requestId,
{ callerScopes: ["operator.pairing", "operator.admin"] },
baseDir,
);
const previousGeneration = resolveNodePairingGeneration(
await getPairedDevice("node-1", baseDir),
);
if (!previousGeneration) {
throw new Error("expected previous node pairing generation");
}
await expect(
updatePairedNodeBins("node-1", ["retired-bin"], previousGeneration, baseDir),
).resolves.toBe(true);
const replacement = await requestDevicePairing(
{
deviceId: "node-1",
publicKey: "public-key-node-1-replacement",
role: "node",
scopes: [],
},
baseDir,
);
await expect(
approveDevicePairing(replacement.request.requestId, { callerScopes: [] }, baseDir),
).resolves.toMatchObject({
status: "approved",
nodePairingGenerationChanged: true,
});
const paired = await getPairedDevice("node-1", baseDir);
expect(resolveNodePairingGeneration(paired)?.key).not.toBe(previousGeneration.key);
expect(paired?.nodeSurface?.commands).toEqual(["system.run", "system.which"]);
expect(paired?.nodeSurface?.bins).toBeUndefined();
});
test("removing a paired device clears pending requests for that device only", async () => {
const baseDir = await makeDevicePairingDir();
await setupPairedOperatorDevice(baseDir, ["operator.read"]);
src/infra/device-pairing.ts
+194 -11
View File
@@ -1,5 +1,5 @@
// Manages device pairing requests, approvals, and token issuance.
import { randomUUID } from "node:crypto";
import { createHash, randomUUID } from "node:crypto";
import { expectDefined } from "@openclaw/normalization-core";
import { normalizeUniqueSingleOrTrimmedStringList } from "@openclaw/normalization-core/string-normalization";
import { normalizeDeviceAuthScopes } from "../shared/device-auth.js";
@@ -16,7 +16,9 @@ import {
import { revokeDeviceBootstrapTokensForDevice } from "./device-bootstrap.js";
import {
loadDevicePairingStoreState,
loadPairedDevicePairingStoreRecord,
persistDevicePairingStoreState as persistState,
updatePairedDevicePresenceInTransaction,
} from "./device-pairing-store.js";
import type {
DeviceAuthToken,
@@ -35,6 +37,21 @@ export type {
PairedDevicePendingNodeSurface,
} from "./device-pairing.types.js";
export type NodePairingGeneration = {
nodeId: string;
key: string;
};
export type NodePairingIdentity = {
nodeId: string;
key: string;
};
export type NodePairingState = {
identity: NodePairingIdentity;
generation: NodePairingGeneration | null;
};
/** Pending request summary returned when a replacement supersedes older requests. */
type DevicePairingSupersededRequest = Pick<DevicePairingPendingRequest, "requestId" | "deviceId">;
@@ -118,7 +135,13 @@ type DevicePairingForbiddenResult = {
/** Pairing approval outcome: approved, forbidden with reason, or request not found. */
type ApproveDevicePairingResult =
| { status: "approved"; requestId: string; device: PairedDevice }
| {
status: "approved";
requestId: string;
device: PairedDevice;
/** Existing connected node transports must be retired before success is returned. */
nodePairingGenerationChanged?: true;
}
| DevicePairingForbiddenResult
| null;
@@ -262,6 +285,81 @@ export function hasEffectivePairedDeviceRole(
return listEffectivePairedDeviceRoles(device).includes(normalized);
}
/** Resolve the authenticated node pairing independently of surface approval. */
function resolveNodePairingIdentity(device: PairedDevice | null): NodePairingIdentity | null {
if (!device || !hasEffectivePairedDeviceRole(device, "node")) {
return null;
}
const nodeToken = device.tokens?.node;
if (!nodeToken) {
return null;
}
const key = createHash("sha256")
.update(
[
device.publicKey,
device.createdAtMs,
nodeToken.token,
nodeToken.createdAtMs,
nodeToken.rotatedAtMs ?? "",
nodeToken.revokedAtMs ?? "",
].join("\0"),
)
.digest("hex");
return { nodeId: device.deviceId, key };
}
/** Resolve the durable node-owned identity used to admit asynchronous work. */
export function resolveNodePairingGeneration(
device: PairedDevice | null,
): NodePairingGeneration | null {
if (!device || !hasEffectivePairedDeviceRole(device, "node") || !device.nodeSurface) {
return null;
}
const nodeToken = device.tokens?.node;
const nodeSurface = device.nodeSurface;
// Device-wide approval also changes for unrelated operator upgrades, so only
// node-owned identity participates in the generation.
const key = createHash("sha256")
.update(
[
device.publicKey,
device.createdAtMs,
nodeToken?.token ?? "",
nodeToken?.revokedAtMs ?? "",
nodeSurface.createdAtMs,
nodeSurface.approvedAtMs,
].join("\0"),
)
.digest("hex");
return { nodeId: device.deviceId, key };
}
/** Clear node-surface cache state when its owning pairing generation changes. */
export function clearNodePairingGenerationBins(
device: PairedDevice,
previousGeneration: NodePairingGeneration | null,
): void {
const nextGeneration = resolveNodePairingGeneration(device);
if (
previousGeneration?.key === nextGeneration?.key ||
!device.nodeSurface ||
device.nodeSurface.bins === undefined
) {
return;
}
delete device.nodeSurface.bins;
}
/** Resolve connection identity and optional approved surface generation from one row. */
export function resolveNodePairingState(device: PairedDevice | null): NodePairingState | null {
const identity = resolveNodePairingIdentity(device);
if (!identity) {
return null;
}
return { identity, generation: resolveNodePairingGeneration(device) };
}
function mergeScopes(...items: Array<string[] | undefined>): string[] | undefined {
const scopes = new Set<string>();
let sawExplicitScopeList = false;
@@ -645,8 +743,11 @@ export async function getPairedDevice(
deviceId: string,
baseDir?: string,
): Promise<PairedDevice | null> {
const state = await loadState(baseDir);
return state.pairedByDeviceId[normalizeDeviceId(deviceId)] ?? null;
const device = loadPairedDevicePairingStoreRecord(normalizeDeviceId(deviceId), baseDir);
if (device?.pendingNodeSurface && Date.now() - device.pendingNodeSurface.ts > PENDING_TTL_MS) {
delete device.pendingNodeSurface;
}
return device;
}
/** Return one pending pairing request by request id. */
@@ -870,6 +971,7 @@ export async function approveDevicePairing(
existing?.approvedScopes ?? existing?.scopes,
pending.scopes,
);
const previousNodeGeneration = resolveNodePairingGeneration(existing ?? null);
const tokens = existing?.tokens ? { ...existing.tokens } : {};
const nextTokenScopesByRole = new Map<string, string[]>();
for (const roleForToken of requestedRoles) {
@@ -928,10 +1030,28 @@ export async function approveDevicePairing(
approvedVia: options?.approvedVia ?? "owner",
accessMetadata: options?.accessMetadata,
});
const nextNodeGeneration = resolveNodePairingGeneration(device);
const nodePairingGenerationChanged = Boolean(
previousNodeGeneration && previousNodeGeneration.key !== nextNodeGeneration?.key,
);
clearNodePairingGenerationBins(device, previousNodeGeneration);
const installationIdentityChanged = Boolean(
existing && existing.publicKey !== device.publicKey,
);
delete state.pendingById[requestId];
state.pairedByDeviceId[device.deviceId] = device;
persistState(state, baseDir, "both");
return { status: "approved", requestId, device };
persistState(
state,
baseDir,
"both",
installationIdentityChanged ? { clearApnsNodeIds: [device.deviceId] } : undefined,
);
return {
status: "approved",
requestId,
device,
...(nodePairingGenerationChanged ? { nodePairingGenerationChanged: true as const } : {}),
};
});
}
@@ -1003,6 +1123,7 @@ export async function approveBootstrapDevicePairing(
);
const roles = mergeRoles(existing?.roles, existing?.role, pending.roles, pending.role);
const nextApprovedScopes = mergeScopes(preservedExistingScopes, grantedScopes);
const previousNodeGeneration = resolveNodePairingGeneration(existing ?? null);
const tokens = existing?.tokens ? { ...existing.tokens } : {};
for (const roleForToken of grantedRoles) {
const existingToken = tokens[roleForToken];
@@ -1029,10 +1150,28 @@ export async function approveBootstrapDevicePairing(
approvedVia: "bootstrap",
accessMetadata: options?.accessMetadata,
});
const nextNodeGeneration = resolveNodePairingGeneration(device);
const nodePairingGenerationChanged = Boolean(
previousNodeGeneration && previousNodeGeneration.key !== nextNodeGeneration?.key,
);
clearNodePairingGenerationBins(device, previousNodeGeneration);
const installationIdentityChanged = Boolean(
existing && existing.publicKey !== device.publicKey,
);
delete state.pendingById[requestId];
state.pairedByDeviceId[device.deviceId] = device;
persistState(state, baseDir, "both");
return { status: "approved", requestId, device };
persistState(
state,
baseDir,
"both",
installationIdentityChanged ? { clearApnsNodeIds: [device.deviceId] } : undefined,
);
return {
status: "approved",
requestId,
device,
...(nodePairingGenerationChanged ? { nodePairingGenerationChanged: true as const } : {}),
};
});
}
@@ -1075,7 +1214,7 @@ export async function removePairedDevice(
delete state.pendingById[requestId];
}
}
persistState(state, baseDir, "both");
persistState(state, baseDir, "both", { clearApnsNodeIds: [normalized] });
return { deviceId: normalized };
});
}
@@ -1164,7 +1303,9 @@ export async function pruneSupersededSilentPairedDevices(params: {
if (removed.length === 0) {
return [];
}
persistState(state, params.baseDir, "both");
persistState(state, params.baseDir, "both", {
clearApnsNodeIds: removed.map((entry) => entry.deviceId),
});
return removed;
});
}
@@ -1194,7 +1335,9 @@ export async function removePairedDeviceRole(params: {
}
}
delete state.pairedByDeviceId[normalizedDeviceId];
persistState(state, params.baseDir, "both");
persistState(state, params.baseDir, "both", {
clearApnsNodeIds: [normalizedDeviceId],
});
return { deviceId: normalizedDeviceId, role, removedDevice: true };
}
@@ -1297,6 +1440,40 @@ export async function updatePairedDeviceMetadata(
});
}
/** Update paired-device presence only while the authenticated node generation still owns it. */
export async function updatePairedDevicePresence(
deviceId: string,
patch: { lastSeenAtMs: number; lastSeenReason: string },
expectedPairingGeneration: NodePairingGeneration,
baseDir?: string,
): Promise<boolean> {
return await withLock(async () => {
const updated = updatePairedDevicePresenceInTransaction<boolean>(
deviceId,
baseDir,
(device) => {
const currentPairingGeneration = resolveNodePairingGeneration(device);
if (
!device ||
expectedPairingGeneration.nodeId !== device.deviceId ||
currentPairingGeneration?.key !== expectedPairingGeneration.key
) {
return { value: false, persist: false };
}
return {
value: true,
persist: true,
lastSeenAtMs: patch.lastSeenAtMs,
lastSeenReason: patch.lastSeenReason,
};
},
);
// The row-scoped transaction owns cross-process generation validation. Keep
// the outer lock so local full-snapshot writers cannot replay older presence.
return updated;
});
}
/** Summarize token metadata without exposing bearer token strings. */
export function summarizeDeviceTokens(
tokens: Record<string, DeviceAuthToken> | undefined,
@@ -1405,6 +1582,7 @@ export async function ensureDeviceToken(params: {
return null;
}
const { device, role, tokens, existing } = context;
const previousNodeGeneration = resolveNodePairingGeneration(device);
const approvedScopes = resolveApprovedDeviceScopeBaseline(device);
if (
!scopesWithinApprovedDeviceBaseline({
@@ -1441,6 +1619,7 @@ export async function ensureDeviceToken(params: {
});
tokens[role] = next;
device.tokens = tokens;
clearNodePairingGenerationBins(device, previousNodeGeneration);
state.pairedByDeviceId[device.deviceId] = device;
persistState(state, params.baseDir, "paired");
return next;
@@ -1494,6 +1673,7 @@ export async function rotateDeviceToken(params: {
return { ok: false, reason: "unknown-device-or-role" };
}
const { device, role, tokens, existing } = context;
const previousNodeGeneration = resolveNodePairingGeneration(device);
const requestedScopes = normalizeDeviceAuthScopes(
params.scopes ?? existing?.scopes ?? device.scopes,
);
@@ -1531,6 +1711,7 @@ export async function rotateDeviceToken(params: {
});
tokens[role] = next;
device.tokens = tokens;
clearNodePairingGenerationBins(device, previousNodeGeneration);
state.pairedByDeviceId[device.deviceId] = device;
persistState(state, params.baseDir, "paired");
return { ok: true, entry: next };
@@ -1555,6 +1736,7 @@ export async function revokeDeviceToken(params: {
return { ok: false, reason: "unknown-device-or-role" };
}
const { device, role, tokens, existing } = context;
const previousNodeGeneration = resolveNodePairingGeneration(device);
const targetScopes = normalizeDeviceAuthScopes(
Array.isArray(existing.scopes) ? existing.scopes : device.scopes,
);
@@ -1571,6 +1753,7 @@ export async function revokeDeviceToken(params: {
const entry = { ...existing, revokedAtMs: Date.now() };
tokens[role] = entry;
device.tokens = tokens;
clearNodePairingGenerationBins(device, previousNodeGeneration);
state.pairedByDeviceId[device.deviceId] = device;
persistState(state, params.baseDir, "paired");
return { ok: true, entry };
src/infra/node-pairing-state.test.ts
+224
View File
@@ -0,0 +1,224 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { PairedDevice } from "./device-pairing.js";
import {
captureAuthenticatedNodePairingState,
captureNodePairingGeneration,
isNodePairingGenerationCurrent,
} from "./node-pairing-state.js";
const mocks = vi.hoisted(() => ({
getPairedDevice: vi.fn(),
}));
vi.mock("./device-pairing.js", async () => {
const actual = await vi.importActual<typeof import("./device-pairing.js")>("./device-pairing.js");
return { ...actual, getPairedDevice: mocks.getPairedDevice };
});
function pairedNode(overrides: Partial<PairedDevice> = {}): PairedDevice {
return {
deviceId: "node-1",
publicKey: "public-key-1",
role: "node",
roles: ["node", "operator"],
tokens: {
node: {
token: "node-token-1",
role: "node",
scopes: [],
createdAtMs: 150,
},
operator: {
token: "operator-token-1",
role: "operator",
scopes: ["operator.pairing"],
createdAtMs: 151,
},
},
createdAtMs: 100,
approvedAtMs: 200,
nodeSurface: {
createdAtMs: 300,
approvedAtMs: 400,
},
...overrides,
};
}
describe("node pairing generation", () => {
beforeEach(() => {
mocks.getPairedDevice.mockReset();
});
it("binds work to node-role token and node-surface approval identity", async () => {
const original = pairedNode();
mocks.getPairedDevice.mockResolvedValueOnce(original);
const generation = await captureNodePairingGeneration(original.deviceId);
expect(generation).toEqual({
nodeId: "node-1",
key: expect.stringMatching(/^[a-f0-9]{64}$/u),
});
mocks.getPairedDevice.mockResolvedValueOnce(
pairedNode({
tokens: {
...original.tokens,
node: { ...original.tokens!.node!, token: "node-token-2", rotatedAtMs: 500 },
},
}),
);
await expect(isNodePairingGenerationCurrent(generation!)).resolves.toBe(false);
});
it("binds connected sessions to the public key and node token used at authentication", async () => {
const original = pairedNode();
mocks.getPairedDevice
.mockResolvedValueOnce(original)
.mockResolvedValueOnce({ ...original, publicKey: "replacement-public-key" })
.mockResolvedValueOnce(
pairedNode({
tokens: {
...original.tokens,
node: { ...original.tokens!.node!, token: "replacement-node-token" },
},
}),
);
await expect(
captureAuthenticatedNodePairingState({
nodeId: original.deviceId,
publicKey: original.publicKey,
token: original.tokens!.node!.token,
}),
).resolves.toMatchObject({ generation: { nodeId: original.deviceId } });
await expect(
captureAuthenticatedNodePairingState({
nodeId: original.deviceId,
publicKey: original.publicKey,
token: original.tokens!.node!.token,
}),
).resolves.toBeNull();
await expect(
captureAuthenticatedNodePairingState({
nodeId: original.deviceId,
publicKey: original.publicKey,
token: original.tokens!.node!.token,
}),
).resolves.toBeNull();
});
it("keeps authenticated pairing identity while first surface approval is pending", async () => {
const original = pairedNode({ nodeSurface: undefined });
mocks.getPairedDevice.mockResolvedValueOnce(original);
await expect(
captureAuthenticatedNodePairingState({
nodeId: original.deviceId,
publicKey: original.publicKey,
token: original.tokens!.node!.token,
}),
).resolves.toEqual({
identity: {
nodeId: original.deviceId,
key: expect.stringMatching(/^[a-f0-9]{64}$/u),
},
generation: null,
});
});
it("keeps pairing identity stable when the pending surface is approved", async () => {
const pending = pairedNode({ nodeSurface: undefined });
const approved = pairedNode();
mocks.getPairedDevice.mockResolvedValueOnce(pending).mockResolvedValueOnce(approved);
const params = {
nodeId: pending.deviceId,
publicKey: pending.publicKey,
token: pending.tokens!.node!.token,
};
const pendingState = await captureAuthenticatedNodePairingState(params);
const approvedState = await captureAuthenticatedNodePairingState(params);
expect(pendingState?.generation).toBeNull();
expect(approvedState?.generation).not.toBeNull();
expect(approvedState?.identity.key).toBe(pendingState?.identity.key);
});
it("keeps node work current across unrelated operator approval", async () => {
const original = pairedNode();
mocks.getPairedDevice.mockResolvedValueOnce(original);
const generation = await captureNodePairingGeneration(original.deviceId);
mocks.getPairedDevice.mockResolvedValueOnce(
pairedNode({
approvedAtMs: 201,
tokens: {
...original.tokens,
operator: {
...original.tokens!.operator!,
token: "operator-token-2",
rotatedAtMs: 501,
},
},
}),
);
await expect(isNodePairingGenerationCurrent(generation!)).resolves.toBe(true);
});
it("invalidates node work when the node surface is reapproved", async () => {
const original = pairedNode();
mocks.getPairedDevice.mockResolvedValueOnce(original);
const generation = await captureNodePairingGeneration(original.deviceId);
mocks.getPairedDevice.mockResolvedValueOnce(
pairedNode({ nodeSurface: { createdAtMs: 300, approvedAtMs: 401 } }),
);
await expect(isNodePairingGenerationCurrent(generation!)).resolves.toBe(false);
});
it("invalidates node work when its effective node token is revoked", async () => {
const original = pairedNode();
mocks.getPairedDevice.mockResolvedValueOnce(original);
const generation = await captureNodePairingGeneration(original.deviceId);
mocks.getPairedDevice.mockResolvedValueOnce(
pairedNode({
tokens: {
...original.tokens,
node: { ...original.tokens!.node!, revokedAtMs: 501 },
},
}),
);
await expect(isNodePairingGenerationCurrent(generation!)).resolves.toBe(false);
});
it("rejects admission without an active node token or approved node surface", async () => {
const original = pairedNode();
mocks.getPairedDevice
.mockResolvedValueOnce(pairedNode({ tokens: undefined }))
.mockResolvedValueOnce(
pairedNode({
tokens: {
...original.tokens,
node: { ...original.tokens!.node!, revokedAtMs: 501 },
},
}),
)
.mockResolvedValueOnce(pairedNode({ nodeSurface: undefined }));
await expect(captureNodePairingGeneration("node-1")).resolves.toBeNull();
await expect(captureNodePairingGeneration("node-1")).resolves.toBeNull();
await expect(captureNodePairingGeneration("node-1")).resolves.toBeNull();
});
it("rejects admission without a durable approved node role", async () => {
mocks.getPairedDevice.mockResolvedValueOnce(
pairedNode({ role: "operator", roles: ["operator"] }),
);
await expect(captureNodePairingGeneration("node-1")).resolves.toBeNull();
});
});
src/infra/node-pairing-state.ts
+86
View File
@@ -0,0 +1,86 @@
import { loadPairedDevicePairingStoreRecord } from "./device-pairing-store.js";
import {
getPairedDevice,
hasEffectivePairedDeviceRole,
resolveNodePairingGeneration,
resolveNodePairingState,
type NodePairingGeneration,
type NodePairingState,
} from "./device-pairing.js";
export type { NodePairingGeneration, NodePairingIdentity } from "./device-pairing.js";
export type NodePairingBinding = {
identity: string;
generation?: string;
};
function toNodePairingBinding(state: NodePairingState | null): NodePairingBinding | undefined {
return state
? {
identity: state.identity.key,
...(state.generation ? { generation: state.generation.key } : {}),
}
: undefined;
}
/** Captures the persistent authenticated pairing and optional approved surface. */
export async function captureNodePairingState(
nodeId: string,
baseDir?: string,
): Promise<NodePairingState | null> {
return resolveNodePairingState(await getPairedDevice(nodeId, baseDir));
}
/** Registry projection of the current persistent pairing owner. */
export async function resolveCurrentNodePairingBinding(
nodeId: string,
): Promise<NodePairingBinding | undefined> {
return toNodePairingBinding(await captureNodePairingState(nodeId));
}
/** Synchronous registry projection for non-yielding process-local reads. */
export function isNodePairingBindingCurrent(nodeId: string, expected: NodePairingBinding): boolean {
const current = toNodePairingBinding(
resolveNodePairingState(loadPairedDevicePairingStoreRecord(nodeId)),
);
return Boolean(
current &&
current.identity === expected.identity &&
(!expected.generation || current.generation === expected.generation),
);
}
/** Captures the persistent node pairing generation admitted for new work. */
export async function captureNodePairingGeneration(
nodeId: string,
): Promise<NodePairingGeneration | null> {
return (await captureNodePairingState(nodeId))?.generation ?? null;
}
/** Binds a connected session to the exact device key and node token it authenticated with. */
export async function captureAuthenticatedNodePairingState(params: {
nodeId: string;
publicKey: string;
token: string;
baseDir?: string;
}): Promise<NodePairingState | null> {
const device = await getPairedDevice(params.nodeId, params.baseDir);
if (
!device ||
device.publicKey !== params.publicKey ||
device.tokens?.node?.token !== params.token ||
!hasEffectivePairedDeviceRole(device, "node")
) {
return null;
}
return resolveNodePairingState(device);
}
/** Revalidates that asynchronous work still belongs to the admitted pairing. */
export async function isNodePairingGenerationCurrent(
generation: NodePairingGeneration,
): Promise<boolean> {
const current = resolveNodePairingGeneration(await getPairedDevice(generation.nodeId));
return current?.key === generation.key;
}
src/infra/node-pairing.test.ts
+138 -3
View File
@@ -1,7 +1,14 @@
// Tests node capability-surface approvals stored on paired device records.
import { afterAll, beforeAll, describe, expect, test } from "vitest";
import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js";
import { approveDevicePairing, requestDevicePairing } from "./device-pairing.js";
import { createDeferred } from "../test-utils/deferred.js";
import {
approveDevicePairing,
getPairedDevice,
requestDevicePairing,
resolveNodePairingGeneration,
withPairedDeviceRecords,
} from "./device-pairing.js";
import {
approveNodePairing,
beginNodePairingConnect,
@@ -572,13 +579,21 @@ describe("node surface approvals", () => {
test("updates remote skill bins and reports missing nodes", async () => {
await withNodePairingDir(async (baseDir) => {
await setupPairedNode(baseDir);
const generation = resolveNodePairingGeneration(await getPairedDevice("node-1", baseDir));
if (!generation) {
throw new Error("expected node pairing generation");
}
await expect(recordPairedNodeConnection("node-1", 1_234, baseDir)).resolves.toEqual({
recorded: true,
firstConnection: true,
});
await expect(updatePairedNodeBins("node-1", ["ffmpeg"], baseDir)).resolves.toBe(true);
await expect(updatePairedNodeBins("missing", ["ffmpeg"], baseDir)).resolves.toBe(false);
await expect(updatePairedNodeBins("node-1", ["ffmpeg"], generation, baseDir)).resolves.toBe(
true,
);
await expect(updatePairedNodeBins("missing", ["ffmpeg"], generation, baseDir)).resolves.toBe(
false,
);
const pairedNode = await findPairedNode("node-1", baseDir);
expect(pairedNode?.lastConnectedAtMs).toBe(1_234);
@@ -586,6 +601,47 @@ describe("node surface approvals", () => {
});
});
test("rejects retired-generation bins after node-surface reapproval", async () => {
await withNodePairingDir(async (baseDir) => {
await setupPairedNode(baseDir);
const previousGeneration = resolveNodePairingGeneration(
await getPairedDevice("node-1", baseDir),
);
if (!previousGeneration) {
throw new Error("expected previous node pairing generation");
}
await expect(
updatePairedNodeBins("node-1", ["retired-bin"], previousGeneration, baseDir),
).resolves.toBe(true);
const pending = await requestNodePairing(
{ nodeId: "node-1", platform: "darwin", commands: ["system.run", "system.which"] },
baseDir,
);
await approveNodePairing(
pending.request.requestId,
{ callerScopes: ["operator.pairing", "operator.admin"] },
baseDir,
);
const currentGeneration = resolveNodePairingGeneration(
await getPairedDevice("node-1", baseDir),
);
if (!currentGeneration) {
throw new Error("expected current node pairing generation");
}
expect(currentGeneration.key).not.toBe(previousGeneration.key);
expect((await findPairedNode("node-1", baseDir))?.bins).toBeUndefined();
await expect(
updatePairedNodeBins("node-1", ["stale-write"], previousGeneration, baseDir),
).resolves.toBe(false);
await expect(
updatePairedNodeBins("node-1", ["current-bin"], currentGeneration, baseDir),
).resolves.toBe(true);
expect((await findPairedNode("node-1", baseDir))?.bins).toEqual(["current-bin"]);
});
});
test("atomically grants one first-connection claim across concurrent handshakes", async () => {
await withNodePairingDir(async (baseDir) => {
await setupPairedNode(baseDir);
@@ -614,6 +670,85 @@ describe("node surface approvals", () => {
});
});
test("serializes connection metadata with locked node-surface mutations", async () => {
await withNodePairingDir(async (baseDir) => {
await setupPairedNode(baseDir);
const snapshotLoaded = createDeferred();
const releaseMutation = createDeferred();
const lockedMutation = withPairedDeviceRecords(baseDir, async (pairedByDeviceId) => {
const device = pairedByDeviceId["node-1"];
if (!device?.nodeSurface) {
throw new Error("expected paired node surface");
}
device.nodeSurface = { ...device.nodeSurface, bins: ["ffmpeg"] };
snapshotLoaded.resolve();
await releaseMutation.promise;
return { value: true, persist: true };
});
await snapshotLoaded.promise;
let connectionSettled = false;
const connection = recordPairedNodeConnection("node-1", 1_234, baseDir).then((result) => {
connectionSettled = true;
return result;
});
await Promise.resolve();
expect(connectionSettled).toBe(false);
releaseMutation.resolve();
await expect(lockedMutation).resolves.toBe(true);
await expect(connection).resolves.toEqual({ recorded: true, firstConnection: true });
expect(await findPairedNode("node-1", baseDir)).toMatchObject({
bins: ["ffmpeg"],
lastConnectedAtMs: 1_234,
});
});
});
test("rejects connection metadata from a retired pairing generation", async () => {
await withNodePairingDir(async (baseDir) => {
await setupPairedNode(baseDir);
const previousGeneration = resolveNodePairingGeneration(
await getPairedDevice("node-1", baseDir),
);
if (!previousGeneration) {
throw new Error("expected initial node pairing generation");
}
const pending = await requestNodePairing(
{
nodeId: "node-1",
platform: "darwin",
commands: ["system.run", "canvas.snapshot"],
},
baseDir,
);
await approveNodePairing(
pending.request.requestId,
{ callerScopes: ["operator.pairing", "operator.admin", "operator.write"] },
baseDir,
);
const currentGeneration = resolveNodePairingGeneration(
await getPairedDevice("node-1", baseDir),
);
if (!currentGeneration) {
throw new Error("expected replacement node pairing generation");
}
expect(currentGeneration.key).not.toBe(previousGeneration.key);
await expect(
recordPairedNodeConnection("node-1", 1_000, baseDir, previousGeneration),
).resolves.toEqual({ recorded: false });
expect((await findPairedNode("node-1", baseDir))?.lastConnectedAtMs).toBeUndefined();
await expect(
recordPairedNodeConnection("node-1", 2_000, baseDir, currentGeneration),
).resolves.toEqual({ recorded: true, firstConnection: true });
expect((await findPairedNode("node-1", baseDir))?.lastConnectedAtMs).toBe(2_000);
});
});
test("keeps the approved node surface across a device pairing re-approval", async () => {
await withNodePairingDir(async (baseDir) => {
await setupPairedNode(baseDir);
src/infra/node-pairing.ts
+114 -34
View File
@@ -8,8 +8,13 @@
import { randomUUID } from "node:crypto";
import { normalizeArrayBackedTrimmedStringList } from "@openclaw/normalization-core/string-normalization";
import { resolveMissingRequestedScope } from "../shared/operator-scope-compat.js";
import { updatePairedDeviceNodeSurfaceInTransaction } from "./device-pairing-store.js";
import {
clearNodePairingGenerationBins,
resolveNodePairingGeneration,
resolveNodePairingState,
withPairedDeviceRecords,
type NodePairingGeneration,
type PairedDevice,
type PairedDevicePendingNodeSurface,
} from "./device-pairing.js";
@@ -87,6 +92,10 @@ type NodePairingList = {
paired: NodePairingPairedNode[];
};
type NodePairingListWithGeneration = Omit<NodePairingList, "paired"> & {
paired: Array<NodePairingPairedNode & { pairingGeneration?: string }>;
};
const OPERATOR_ROLE = "operator";
const activeCleanupRevisionClaims = new Map<string, Set<number>>();
@@ -149,11 +158,17 @@ function toPendingEntry(
};
}
function toPairedNode(device: PairedDevice): NodePairingPairedNode | null {
function toPairedNode(
device: PairedDevice,
options?: { includePairingGeneration?: boolean },
): NodePairingPairedNode | null {
const surface = device.nodeSurface;
if (!surface) {
return null;
}
const pairingGeneration = options?.includePairingGeneration
? resolveNodePairingGeneration(device)?.key
: undefined;
return {
nodeId: device.deviceId,
clientId: device.clientId,
@@ -172,6 +187,7 @@ function toPairedNode(device: PairedDevice): NodePairingPairedNode | null {
permissions: surface.permissions,
remoteIp: device.remoteIp,
bins: surface.bins,
...(pairingGeneration ? { pairingGeneration } : {}),
createdAtMs: surface.createdAtMs,
approvedAtMs: surface.approvedAtMs,
lastConnectedAtMs: surface.lastConnectedAtMs,
@@ -322,7 +338,15 @@ function pendingHasActiveCleanupClaim(
return (activeCleanupRevisionClaims.get(key)?.size ?? 0) > 0;
}
export async function listNodePairing(baseDir?: string): Promise<NodePairingList> {
export function listNodePairing(baseDir?: string): Promise<NodePairingList>;
export function listNodePairing(
baseDir: string | undefined,
options: { includePairingGeneration: true },
): Promise<NodePairingListWithGeneration>;
export async function listNodePairing(
baseDir?: string,
options?: { includePairingGeneration?: boolean },
): Promise<NodePairingList | NodePairingListWithGeneration> {
return await withPairedDeviceRecords(baseDir, (pairedByDeviceId) => {
const pending: NodePairingPendingEntry[] = [];
const paired: NodePairingPairedNode[] = [];
@@ -330,7 +354,7 @@ export async function listNodePairing(baseDir?: string): Promise<NodePairingList
if (device.pendingNodeSurface) {
pending.push(toPendingEntry(device, device.pendingNodeSurface));
}
const node = toPairedNode(device);
const node = toPairedNode(device, options);
if (node) {
paired.push(node);
}
@@ -485,7 +509,13 @@ export async function reusePendingNodePairingForReconnect(
});
}
type ApprovedNodePairingResult = { requestId: string; node: NodePairingPairedNode };
type ApprovedNodePairingResult = {
requestId: string;
node: NodePairingPairedNode;
pairingIdentity: string;
nextPairingGeneration: string;
previousPairingGeneration?: string;
};
type ForbiddenNodePairingResult = { status: "forbidden"; missingScope: string };
type ApproveNodePairingResult = ApprovedNodePairingResult | ForbiddenNodePairingResult | null;
@@ -525,7 +555,8 @@ export async function approveNodePairing(
return { value: { status: "forbidden" as const, missingScope }, persist: false };
}
const now = Date.now();
const previousPairingGeneration = resolveNodePairingGeneration(device);
const now = Math.max(Date.now(), (device.nodeSurface?.approvedAtMs ?? -1) + 1);
device.nodeSurface = {
displayName: pending.displayName,
version: pending.version,
@@ -541,11 +572,28 @@ export async function approveNodePairing(
lastConnectedAtMs: device.nodeSurface?.lastConnectedAtMs,
};
delete device.pendingNodeSurface;
const nextPairingState = resolveNodePairingState(device);
const nextPairingGeneration = nextPairingState?.generation?.key;
if (!nextPairingState || !nextPairingGeneration) {
return { value: null, persist: false };
}
clearNodePairingGenerationBins(device, previousPairingGeneration);
const node = toPairedNode(device);
if (!node) {
return { value: null, persist: false };
}
return { value: { requestId, node }, persist: true };
return {
value: {
requestId,
node,
pairingIdentity: nextPairingState.identity.key,
nextPairingGeneration,
...(previousPairingGeneration
? { previousPairingGeneration: previousPairingGeneration.key }
: {}),
},
persist: true,
};
});
}
@@ -582,18 +630,31 @@ export async function getPendingNodePairing(
export async function updatePairedNodeBins(
nodeId: string,
bins: string[],
expectedPairingGeneration: NodePairingGeneration,
baseDir?: string,
): Promise<boolean> {
return await withPairedDeviceRecords(baseDir, (pairedByDeviceId) => {
const device = nodeSurfaceDevice(pairedByDeviceId, nodeId);
if (!device?.nodeSurface) {
return { value: false, persist: false };
}
device.nodeSurface = {
...device.nodeSurface,
bins,
};
return { value: true, persist: true };
return await withPairedDeviceRecords<boolean>(baseDir, () => {
const value = updatePairedDeviceNodeSurfaceInTransaction<boolean>(nodeId, baseDir, (device) => {
const currentPairingGeneration = resolveNodePairingGeneration(device);
if (
!device?.nodeSurface ||
expectedPairingGeneration.nodeId !== device.deviceId ||
currentPairingGeneration?.key !== expectedPairingGeneration.key
) {
return { value: false, persist: false };
}
return {
value: true,
persist: true,
nodeSurface: {
...device.nodeSurface,
bins,
},
};
});
// The row-scoped transaction owns cross-process generation validation, while
// this lock prevents a local full-snapshot writer from replaying retired bins.
return { value, persist: false };
});
}
@@ -606,25 +667,44 @@ export async function recordPairedNodeConnection(
nodeId: string,
connectedAtMs: number,
baseDir?: string,
expectedPairingGeneration?: NodePairingGeneration,
): Promise<RecordPairedNodeConnectionResult> {
return await withPairedDeviceRecords<RecordPairedNodeConnectionResult>(
baseDir,
(pairedByDeviceId) => {
const device = nodeSurfaceDevice(pairedByDeviceId, nodeId);
if (!device?.nodeSurface) {
return { value: { recorded: false }, persist: false };
}
// Read and write under the pairing lock. Concurrent rehandshakes must not
// both claim the same node's first connection and schedule duplicate alerts.
const firstConnection = device.nodeSurface.lastConnectedAtMs === undefined;
const previousConnectedAtMs = device.nodeSurface.lastConnectedAtMs ?? connectedAtMs;
device.nodeSurface = {
...device.nodeSurface,
lastConnectedAtMs: Math.max(previousConnectedAtMs, connectedAtMs),
};
return { value: { recorded: true, firstConnection }, persist: true };
},
);
return await withPairedDeviceRecords<RecordPairedNodeConnectionResult>(baseDir, () => {
const value = updatePairedDeviceNodeSurfaceInTransaction<RecordPairedNodeConnectionResult>(
nodeId,
baseDir,
(device) => {
if (!device?.nodeSurface) {
return { value: { recorded: false }, persist: false };
}
if (expectedPairingGeneration) {
const currentPairingGeneration = resolveNodePairingGeneration(device);
if (
expectedPairingGeneration.nodeId !== device.deviceId ||
currentPairingGeneration?.key !== expectedPairingGeneration.key
) {
return { value: { recorded: false }, persist: false };
}
}
// Read and write under the pairing lock. Concurrent rehandshakes must not
// both claim the same node's first connection and schedule duplicate alerts.
const firstConnection = device.nodeSurface.lastConnectedAtMs === undefined;
const previousConnectedAtMs = device.nodeSurface.lastConnectedAtMs ?? connectedAtMs;
return {
value: { recorded: true, firstConnection },
persist: true,
nodeSurface: {
...device.nodeSurface,
lastConnectedAtMs: Math.max(previousConnectedAtMs, connectedAtMs),
},
};
},
);
// The row-scoped transaction owns cross-process generation validation, while
// this outer shared lock prevents local full-snapshot writers from replaying
// node-surface state loaded before the connection metadata commit.
return { value, persist: false };
});
}
/** Rename a paired node display name while preserving approval metadata. */
src/infra/push-apns-auth.ts
+122
View File
@@ -0,0 +1,122 @@
// Resolves APNs provider credentials and owns provider-token signing/cache state.
import { createHash, createPrivateKey, sign as signJwt } from "node:crypto";
import fs from "node:fs/promises";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { formatErrorMessage } from "./errors.js";
/** Direct APNs provider authentication used to mint ES256 bearer tokens. */
export type ApnsAuthConfig = {
teamId: string;
keyId: string;
privateKey: string;
};
type ApnsAuthConfigResolution = { ok: true; value: ApnsAuthConfig } | { ok: false; error: string };
const APNS_JWT_TTL_MS = 50 * 60 * 1000;
let cachedJwt: { cacheKey: string; token: string; expiresAtMs: number } | null = null;
function toBase64UrlBytes(value: Uint8Array): string {
return Buffer.from(value)
.toString("base64")
.replace(/\+/g, "-")
.replace(/\//g, "_")
.replace(/=+$/g, "");
}
function toBase64UrlJson(value: object): string {
return toBase64UrlBytes(Buffer.from(JSON.stringify(value)));
}
function getJwtCacheKey(auth: ApnsAuthConfig): string {
const keyHash = createHash("sha256").update(auth.privateKey).digest("hex");
return `${auth.teamId}:${auth.keyId}:${keyHash}`;
}
export function getApnsBearerToken(auth: ApnsAuthConfig, nowMs: number = Date.now()): string {
const cacheKey = getJwtCacheKey(auth);
if (cachedJwt && cachedJwt.cacheKey === cacheKey && nowMs < cachedJwt.expiresAtMs) {
return cachedJwt.token;
}
// APNs provider tokens are valid for one hour. Cache for slightly less so
// bursty wake/approval pushes avoid repeated ECDSA signing.
const iat = Math.floor(nowMs / 1000);
const header = toBase64UrlJson({ alg: "ES256", kid: auth.keyId, typ: "JWT" });
const payload = toBase64UrlJson({ iss: auth.teamId, iat });
const signingInput = `${header}.${payload}`;
const signature = signJwt("sha256", Buffer.from(signingInput, "utf8"), {
key: createPrivateKey(auth.privateKey),
dsaEncoding: "ieee-p1363",
});
const token = `${signingInput}.${toBase64UrlBytes(signature)}`;
cachedJwt = {
cacheKey,
token,
expiresAtMs: nowMs + APNS_JWT_TTL_MS,
};
return token;
}
function normalizePrivateKey(value: string): string {
return value.trim().replace(/\\n/g, "\n");
}
function normalizeNonEmptyString(value: string | undefined): string | null {
const trimmed = normalizeOptionalString(value) ?? "";
return trimmed.length > 0 ? trimmed : null;
}
/** Resolves direct APNs provider auth from env, accepting inline or file-backed keys. */
export async function resolveApnsAuthConfigFromEnv(
env: NodeJS.ProcessEnv = process.env,
): Promise<ApnsAuthConfigResolution> {
const teamId = normalizeNonEmptyString(env.OPENCLAW_APNS_TEAM_ID);
const keyId = normalizeNonEmptyString(env.OPENCLAW_APNS_KEY_ID);
if (!teamId || !keyId) {
return {
ok: false,
error: "APNs auth missing: set OPENCLAW_APNS_TEAM_ID and OPENCLAW_APNS_KEY_ID",
};
}
const inlineKeyRaw =
normalizeNonEmptyString(env.OPENCLAW_APNS_PRIVATE_KEY_P8) ??
normalizeNonEmptyString(env.OPENCLAW_APNS_PRIVATE_KEY);
if (inlineKeyRaw) {
return {
ok: true,
value: {
teamId,
keyId,
privateKey: normalizePrivateKey(inlineKeyRaw),
},
};
}
const keyPath = normalizeNonEmptyString(env.OPENCLAW_APNS_PRIVATE_KEY_PATH);
if (!keyPath) {
return {
ok: false,
error:
"APNs private key missing: set OPENCLAW_APNS_PRIVATE_KEY_P8 or OPENCLAW_APNS_PRIVATE_KEY_PATH",
};
}
try {
const privateKey = normalizePrivateKey(await fs.readFile(keyPath, "utf8"));
return {
ok: true,
value: {
teamId,
keyId,
privateKey,
},
};
} catch (err) {
return {
ok: false,
error: `failed reading OPENCLAW_APNS_PRIVATE_KEY_PATH (${keyPath}): ${formatErrorMessage(err)}`,
};
}
}
src/infra/push-apns-cancellation.test.ts
+67
View File
@@ -0,0 +1,67 @@
import { generateKeyPairSync } from "node:crypto";
import { EventEmitter } from "node:events";
import http2 from "node:http2";
import { describe, expect, it, vi } from "vitest";
import { APNS_HTTP2_CANCEL_CODE } from "./push-apns-http2.js";
import { sendApnsBackgroundWake } from "./push-apns.js";
const testAuthPrivateKey = generateKeyPairSync("ec", {
namedCurve: "prime256v1",
}).privateKey.export({ format: "pem", type: "pkcs8" });
describe("APNs cancellation", () => {
it("cancels the active stream when pairing ownership is revoked", async () => {
const request = Object.assign(new EventEmitter(), {
destroyed: false,
setTimeout: vi.fn(),
close: vi.fn(),
end: vi.fn(),
});
request.close.mockImplementation(() => {
request.destroyed = true;
request.emit("close");
});
const session = Object.assign(new EventEmitter(), {
close: vi.fn(),
destroy: vi.fn(),
request: vi.fn(() => request),
});
session.close.mockImplementation(() => session.emit("close"));
const connect = vi
.spyOn(http2, "connect")
.mockReturnValue(session as unknown as http2.ClientHttp2Session);
const controller = new AbortController();
try {
const sending = sendApnsBackgroundWake({
registration: {
nodeId: "ios-node-cancelled-stream",
transport: "direct",
token: "ABCD1234ABCD1234ABCD1234ABCD1234",
topic: "ai.openclaw.ios",
environment: "production",
updatedAtMs: 1,
},
nodeId: "ios-node-cancelled-stream",
wakeReason: "node.invoke",
auth: {
teamId: "TEAM123",
keyId: "KEY123",
privateKey: testAuthPrivateKey,
},
signal: controller.signal,
isCurrent: vi.fn().mockResolvedValue(true),
});
await vi.waitFor(() => expect(request.end).toHaveBeenCalledTimes(1));
controller.abort(new Error("pairing removed"));
await expect(sending).rejects.toThrow("pairing removed");
expect(request.close).toHaveBeenCalledWith(APNS_HTTP2_CANCEL_CODE);
expect(session.close).toHaveBeenCalledTimes(1);
expect(session.destroy).not.toHaveBeenCalled();
} finally {
connect.mockRestore();
}
});
});
src/infra/push-apns-http2.test.ts
+39 -15
View File
@@ -86,7 +86,10 @@ const {
}),
request: vi.fn(() => fakeRequestLocal),
});
const fakeProxySocketLocal = { destroy: vi.fn() };
const fakeProxySocketLocal = Object.assign(new FakeEmitter(), {
destroy: vi.fn(),
unshift: vi.fn(),
});
const fakeTlsSocketLocal = Object.assign(new FakeEmitter(), {
encrypted: true,
alpnProtocol: "h2" as string | false,
@@ -163,7 +166,9 @@ describe("connectApnsHttp2Session", () => {
tunnelSpy.mockClear();
tlsConnectSpy.mockClear();
setTargetTlsEvent("secureConnect");
fakeProxySocket.reset();
fakeProxySocket.destroy.mockClear();
fakeProxySocket.unshift.mockClear();
fakeRequest.reset();
fakeRequest.setEncoding.mockClear();
fakeRequest.end.mockClear();
@@ -196,6 +201,22 @@ describe("connectApnsHttp2Session", () => {
expect(connectSpy).toHaveBeenCalledWith("https://api.sandbox.push.apple.com");
});
it("rejects an already invalidated direct APNs setup before opening a session", async () => {
const { connectApnsHttp2Session } = await import("./push-apns-http2.js");
const controller = new AbortController();
controller.abort(new Error("pairing removed"));
await expect(
connectApnsHttp2Session({
authority: "https://api.sandbox.push.apple.com",
timeoutMs: 10_000,
signal: controller.signal,
}),
).rejects.toThrow("pairing removed");
expect(connectSpy).not.toHaveBeenCalled();
});
it("normalizes the default APNs HTTPS port", async () => {
const { connectApnsHttp2Session } = await import("./push-apns-http2.js");
@@ -227,30 +248,33 @@ describe("connectApnsHttp2Session", () => {
});
it("uses an HTTP CONNECT tunnel when managed proxy is active", async () => {
const registration = registerActiveManagedProxyUrl(new URL("https://proxy.example:8443"), {
loopbackMode: "gateway-only",
proxyTls: { ca: "active-proxy-ca" },
});
const registration = registerActiveManagedProxyUrl(
new URL("https://user:pass@proxy.example:8443"),
{
loopbackMode: "gateway-only",
proxyTls: { ca: "active-proxy-ca" },
},
);
const { connectApnsHttp2Session } = await import("./push-apns-http2.js");
const controller = new AbortController();
const session = await connectApnsHttp2Session({
authority: "https://api.push.apple.com",
timeoutMs: 10_000,
signal: controller.signal,
});
stopActiveManagedProxyRegistration(registration);
expect(session).toBe(fakeSession);
const tunnelCall = lastTunnelCall();
const proxyUrl = tunnelCall.proxyUrl;
expect(proxyUrl).toBeInstanceOf(URL);
if (!(proxyUrl instanceof URL)) {
throw new Error("expected active managed proxy URL");
}
expect(proxyUrl.href).toBe("https://proxy.example:8443/");
expect(tunnelCall.proxyTls).toEqual({ ca: "active-proxy-ca" });
expect(tunnelCall.targetHost).toBe("api.push.apple.com");
expect(tunnelCall.targetPort).toBe(443);
expect(tunnelCall.timeoutMs).toBe(10_000);
expect(tunnelCall).toMatchObject({
targetHost: "api.push.apple.com",
targetPort: 443,
timeoutMs: 10_000,
proxyTls: { ca: "active-proxy-ca" },
});
expect(String(tunnelCall.proxyUrl)).toBe("https://user:pass@proxy.example:8443/");
expect(tunnelCall).toMatchObject({ signal: controller.signal });
expect(tlsConnectSpy).toHaveBeenCalledWith({
socket: fakeProxySocket,
servername: "api.push.apple.com",
src/infra/push-apns-http2.ts
+28
View File
@@ -37,6 +37,7 @@ type ApnsResponseBodyCapture = {
type ConnectApnsHttp2SessionParams = {
authority: string;
timeoutMs: number;
signal?: AbortSignal;
};
/** Parameters for validating APNs reachability through an explicit proxy. */
@@ -55,6 +56,10 @@ type ProbeApnsHttp2ReachabilityViaProxyResult = {
responseHeaders: Record<string, string>;
};
function apnsAbortError(signal: AbortSignal): Error {
return signal.reason instanceof Error ? signal.reason : new Error("APNs send invalidated");
}
function assertApnsAuthority(authority: string): ApnsAuthority {
let parsed: URL;
try {
@@ -106,6 +111,7 @@ async function openApnsTlsTunnel(params: {
targetHost: string;
targetPort: number;
timeoutMs: number;
signal?: AbortSignal;
}): Promise<tls.TLSSocket> {
// CONNECT ignores URL paths. Strip path metadata before Proxyline sees it so
// tokens embedded in a configured proxy URL cannot surface in errors.
@@ -117,9 +123,19 @@ async function openApnsTlsTunnel(params: {
targetHost: params.targetHost,
targetPort: params.targetPort,
timeoutMs: params.timeoutMs,
...(params.signal ? { signal: params.signal } : {}),
});
const abortController = new AbortController();
const abortFromCaller = () => {
if (params.signal) {
abortController.abort(apnsAbortError(params.signal));
}
};
params.signal?.addEventListener("abort", abortFromCaller, { once: true });
if (params.signal?.aborted) {
abortFromCaller();
}
let targetTlsSocket: tls.TLSSocket | undefined;
let timeout: NodeJS.Timeout | undefined;
try {
@@ -157,6 +173,7 @@ async function openApnsTlsTunnel(params: {
if (timeout) {
clearTimeout(timeout);
}
params.signal?.removeEventListener("abort", abortFromCaller);
abortController.abort();
}
}
@@ -166,6 +183,7 @@ async function openProxiedApnsHttp2Session(params: {
proxyUrl: ActiveManagedProxyUrl;
proxyTls?: ManagedProxyTlsOptions;
timeoutMs: number;
signal?: AbortSignal;
}): Promise<http2.ClientHttp2Session> {
const apnsHost = new URL(params.authority).hostname;
const tlsSocket = await openApnsTlsTunnel({
@@ -174,8 +192,14 @@ async function openProxiedApnsHttp2Session(params: {
targetHost: apnsHost,
targetPort: 443,
timeoutMs: params.timeoutMs,
...(params.signal ? { signal: params.signal } : {}),
});
if (params.signal?.aborted) {
tlsSocket.destroy();
throw apnsAbortError(params.signal);
}
// The CONNECT helper already completed the target TLS handshake; reuse that
// socket so the session cannot open a separate direct route.
return http2.connect(params.authority, {
@@ -191,6 +215,9 @@ export async function connectApnsHttp2Session(
const timeoutMs = resolveApnsHttp2TimeoutMs(params.timeoutMs);
const proxyUrl = getActiveManagedProxyUrl();
if (!proxyUrl) {
if (params.signal?.aborted) {
throw apnsAbortError(params.signal);
}
return http2.connect(authority);
}
@@ -199,6 +226,7 @@ export async function connectApnsHttp2Session(
proxyUrl,
proxyTls: getActiveManagedProxyTlsOptions(),
timeoutMs,
...(params.signal ? { signal: params.signal } : {}),
});
}
src/infra/push-apns-store-transaction.ts
+75
View File
@@ -0,0 +1,75 @@
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
import type { OpenClawStateDatabase } from "../state/openclaw-state-db.js";
import {
executeSqliteQuerySync,
executeSqliteQueryTakeFirstSync,
getNodeSqliteKysely,
} from "./kysely-sync.js";
type ApnsRegistrationDatabase = Pick<
OpenClawStateKyselyDatabase,
"apns_registrations" | "apns_registration_tombstones"
>;
/** Advances a registration/tombstone version without reusing an observed owner version. */
export function nextApnsRegistrationVersion(
nodeId: string,
previousVersions: readonly number[],
): number {
let latest = -1;
for (const version of previousVersions) {
if (!Number.isSafeInteger(version) || version < 0) {
throw new Error(`invalid APNs registration version for node ${nodeId}`);
}
latest = Math.max(latest, version);
}
if (latest === Number.MAX_SAFE_INTEGER) {
throw new Error(`APNs registration version exhausted for node ${nodeId}`);
}
return Math.max(Date.now(), latest + 1);
}
/** Tombstones and deletes one APNs owner inside the caller's shared-state transaction. */
export function clearApnsRegistrationFromDatabase(
db: OpenClawStateDatabase["db"],
nodeId: string,
): boolean {
const normalizedNodeId = nodeId.trim();
if (!normalizedNodeId) {
return false;
}
const stateDb = getNodeSqliteKysely<ApnsRegistrationDatabase>(db);
const currentRow = executeSqliteQueryTakeFirstSync(
db,
stateDb
.selectFrom("apns_registrations")
.select("updated_at_ms")
.where("node_id", "=", normalizedNodeId),
);
const tombstone = executeSqliteQueryTakeFirstSync(
db,
stateDb
.selectFrom("apns_registration_tombstones")
.select("deleted_at_ms")
.where("node_id", "=", normalizedNodeId),
);
const previousVersions = [currentRow?.updated_at_ms, tombstone?.deleted_at_ms].filter(
(version): version is number => version !== undefined,
);
const deletedAtMs = nextApnsRegistrationVersion(normalizedNodeId, previousVersions);
// Tombstone even an empty row so a retired source cannot restore ownership.
executeSqliteQuerySync(
db,
stateDb
.insertInto("apns_registration_tombstones")
.values({ node_id: normalizedNodeId, deleted_at_ms: deletedAtMs })
.onConflict((conflict) =>
conflict.column("node_id").doUpdateSet({ deleted_at_ms: deletedAtMs }),
),
);
executeSqliteQuerySync(
db,
stateDb.deleteFrom("apns_registrations").where("node_id", "=", normalizedNodeId),
);
return currentRow !== undefined;
}
src/infra/push-apns-store.ts
+22 -14
View File
@@ -10,11 +10,14 @@ import {
runOpenClawStateWriteTransaction,
type OpenClawStateDatabaseOptions,
} from "../state/openclaw-state-db.js";
import { loadPairedDevicePairingStoreRecordFromDatabase } from "./device-pairing-store.js";
import { resolveNodePairingGeneration } from "./device-pairing.js";
import {
executeSqliteQuerySync,
executeSqliteQueryTakeFirstSync,
getNodeSqliteKysely,
} from "./kysely-sync.js";
import { nextApnsRegistrationVersion } from "./push-apns-store-transaction.js";
import {
normalizeApnsRelayBaseUrl,
normalizePersistedApnsRelayBaseUrl,
@@ -48,12 +51,20 @@ export type RelayApnsRegistration = {
/** Stored APNs registration for either direct device tokens or official relay handles. */
export type ApnsRegistration = DirectApnsRegistration | RelayApnsRegistration;
export class ApnsRegistrationPairingChangedError extends Error {
constructor() {
super("node pairing changed before APNs registration");
this.name = "ApnsRegistrationPairingChangedError";
}
}
type RegisterDirectApnsParams = {
nodeId: string;
transport?: "direct";
token: string;
topic: string;
environment?: unknown;
expectedPairingGeneration?: string;
baseDir?: string;
};
@@ -68,6 +79,7 @@ type RegisterRelayApnsParams = {
distribution?: unknown;
relayOrigin?: unknown;
tokenDebugSuffix?: unknown;
expectedPairingGeneration?: string;
baseDir?: string;
};
@@ -426,20 +438,6 @@ function apnsRegistrationsEqual(left: ApnsRegistration, right: ApnsRegistration)
);
}
function nextApnsRegistrationVersion(nodeId: string, previousVersions: readonly number[]): number {
let latest = -1;
for (const version of previousVersions) {
if (!Number.isSafeInteger(version) || version < 0) {
throw new Error(`invalid APNs registration version for node ${nodeId}`);
}
latest = Math.max(latest, version);
}
if (latest === Number.MAX_SAFE_INTEGER) {
throw new Error(`APNs registration version exhausted for node ${nodeId}`);
}
return Math.max(Date.now(), latest + 1);
}
/** Persists a validated direct or relay APNs registration for one node id. */
export async function registerApnsRegistration(
params: RegisterApnsParams,
@@ -507,6 +505,16 @@ export async function registerApnsRegistration(
}
return runOpenClawStateWriteTransaction(({ db }) => {
if (params.expectedPairingGeneration) {
// The Gateway admission check happens before this transaction. Reread the
// pairing here so removal and APNs ownership cannot commit out of order.
const pairing = resolveNodePairingGeneration(
loadPairedDevicePairingStoreRecordFromDatabase(db, nodeId),
);
if (pairing?.key !== params.expectedPairingGeneration) {
throw new ApnsRegistrationPairingChangedError();
}
}
const stateDb = getNodeSqliteKysely<ApnsRegistrationDatabase>(db);
const current = executeSqliteQueryTakeFirstSync(
db,
src/infra/push-apns.relay.test.ts
+33
View File
@@ -214,6 +214,39 @@ describe("push-apns.relay", () => {
});
describe("sendApnsRelayPush", () => {
it("revalidates ownership before relay fetch and combines the lifecycle signal", async () => {
const controller = new AbortController();
const isCurrent = vi.fn().mockResolvedValue(true);
const fetchMock = vi.fn().mockResolvedValue(new Response("", { status: 202 }));
vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch);
await sendApnsRelayPush({
...createRelayPushParams(),
signal: controller.signal,
isCurrent,
});
expect(isCurrent).toHaveBeenCalledTimes(2);
const fetchOptions = firstMockCall(fetchMock)?.[1] as { signal?: AbortSignal } | undefined;
expect(fetchOptions?.signal?.aborted).toBe(false);
controller.abort(new Error("pairing removed"));
expect(fetchOptions?.signal?.aborted).toBe(true);
});
it("does not start relay transport when persistent ownership changed", async () => {
const fetchMock = vi.fn();
vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch);
await expect(
sendApnsRelayPush({
...createRelayPushParams(),
isCurrent: vi.fn().mockResolvedValue(false),
}),
).rejects.toThrow("APNs send invalidated");
expect(fetchMock).not.toHaveBeenCalled();
});
it("signs relay payloads and forwards the request through the injected sender", async () => {
vi.spyOn(Date, "now").mockReturnValue(123_456_789);
const sender = vi.fn().mockResolvedValue({
src/infra/push-apns.relay.ts
+31 -1
View File
@@ -57,6 +57,8 @@ export type ApnsRelayRequestSender = (params: {
pushType: ApnsRelayPushType;
priority: "10" | "5";
payload: object;
signal?: AbortSignal;
isCurrent?: () => Promise<boolean>;
}) => Promise<ApnsRelayPushResponse>;
/** Hosted APNs relay origin used only when registrations prove they were minted there. */
@@ -76,6 +78,24 @@ function normalizeNonEmptyString(value: string | undefined): string | null {
return trimmed.length > 0 ? trimmed : null;
}
function throwIfApnsRelaySendAborted(signal: AbortSignal | undefined): void {
if (!signal?.aborted) {
return;
}
throw signal.reason instanceof Error ? signal.reason : new Error("APNs send invalidated");
}
async function requireCurrentApnsRelaySend(params: {
signal?: AbortSignal;
isCurrent?: () => Promise<boolean>;
}): Promise<void> {
throwIfApnsRelaySendAborted(params.signal);
if (params.isCurrent && !(await params.isCurrent())) {
throw new Error("APNs send invalidated");
}
throwIfApnsRelaySendAborted(params.signal);
}
function normalizeTimeoutMs(value: string | number | undefined): number {
const raw =
typeof value === "number"
@@ -279,7 +299,12 @@ async function sendApnsRelayRequest(params: {
pushType: ApnsRelayPushType;
priority: "10" | "5";
payload: object;
signal?: AbortSignal;
isCurrent?: () => Promise<boolean>;
}): Promise<ApnsRelayPushResponse> {
await requireCurrentApnsRelaySend(params);
const timeoutSignal = AbortSignal.timeout(params.relayConfig.timeoutMs);
const signal = params.signal ? AbortSignal.any([params.signal, timeoutSignal]) : timeoutSignal;
const response = await fetch(`${params.relayConfig.baseUrl}/v1/push/send`, {
method: "POST",
redirect: "manual",
@@ -291,7 +316,7 @@ async function sendApnsRelayRequest(params: {
[GATEWAY_SIGNED_AT_HEADER]: String(params.signedAtMs),
},
body: params.bodyJson,
signal: AbortSignal.timeout(params.relayConfig.timeoutMs),
signal,
});
// Do not follow relay redirects; grants and signatures are scoped to the configured relay origin.
if (response.status >= 300 && response.status < 400) {
@@ -353,7 +378,10 @@ export async function sendApnsRelayPush(params: {
payload: object;
gatewayIdentity?: Pick<DeviceIdentity, "deviceId" | "privateKeyPem">;
requestSender?: ApnsRelayRequestSender;
signal?: AbortSignal;
isCurrent?: () => Promise<boolean>;
}): Promise<ApnsRelayPushResponse> {
await requireCurrentApnsRelaySend(params);
const sender = params.requestSender ?? sendApnsRelayRequest;
const gatewayIdentity = params.gatewayIdentity ?? loadOrCreateProcessDeviceIdentity();
const signedAtMs = Date.now();
@@ -382,5 +410,7 @@ export async function sendApnsRelayPush(params: {
pushType: params.pushType,
priority: params.priority,
payload: params.payload,
...(params.signal ? { signal: params.signal } : {}),
...(params.isCurrent ? { isCurrent: params.isCurrent } : {}),
});
}
src/infra/push-apns.store.test.ts
+92
View File
@@ -9,8 +9,11 @@ import {
runOpenClawStateWriteTransaction,
} from "../state/openclaw-state-db.js";
import { createTrackedTempDirs } from "../test-utils/tracked-temp-dirs.js";
import { persistDevicePairingStoreState } from "./device-pairing-store.js";
import { resolveNodePairingGeneration, type PairedDevice } from "./device-pairing.js";
import { executeSqliteQuerySync, getNodeSqliteKysely } from "./kysely-sync.js";
import {
ApnsRegistrationPairingChangedError,
clearApnsRegistrationIfCurrent,
loadApnsRegistration,
loadApnsRegistrations,
@@ -32,6 +35,7 @@ async function registerDirectApnsRegistration(params: {
token?: string;
topic?: string;
environment?: unknown;
expectedPairingGeneration?: string;
baseDir: string;
}) {
return await registerApnsRegistration({
@@ -244,6 +248,94 @@ describe("push APNs registration store", () => {
await expect(loadApnsRegistration("ios-node-1", baseDir)).resolves.toEqual(replacement);
});
it("rejects a stale registration after the expected pairing generation is removed", async () => {
const baseDir = await makeTempDir();
const nodeId = "ios-node-generation-guard";
const pairedDevice: PairedDevice = {
deviceId: nodeId,
publicKey: "public-key-generation-a",
role: "node",
roles: ["node"],
tokens: {
node: {
token: "node-token-generation-a",
role: "node",
scopes: [],
createdAtMs: 100,
},
},
nodeSurface: {
createdAtMs: 200,
approvedAtMs: 300,
},
createdAtMs: 50,
approvedAtMs: 300,
};
persistDevicePairingStoreState(
{ pendingById: {}, pairedByDeviceId: { [nodeId]: pairedDevice } },
baseDir,
"paired",
);
const generation = resolveNodePairingGeneration(pairedDevice);
if (!generation) {
throw new Error("expected node pairing generation");
}
await registerDirectApnsRegistration({
nodeId,
expectedPairingGeneration: generation.key,
baseDir,
});
persistDevicePairingStoreState({ pendingById: {}, pairedByDeviceId: {} }, baseDir, "paired", {
clearApnsNodeIds: [nodeId],
});
await expect(loadApnsRegistration(nodeId, baseDir)).resolves.toBeNull();
const replacementDevice: PairedDevice = {
...pairedDevice,
publicKey: "public-key-generation-b",
tokens: {
node: {
token: "node-token-generation-b",
role: "node",
scopes: [],
createdAtMs: 101,
},
},
nodeSurface: {
...pairedDevice.nodeSurface,
createdAtMs: 200,
approvedAtMs: 301,
},
};
persistDevicePairingStoreState(
{ pendingById: {}, pairedByDeviceId: { [nodeId]: replacementDevice } },
baseDir,
"paired",
);
const replacementGeneration = resolveNodePairingGeneration(replacementDevice);
if (!replacementGeneration) {
throw new Error("expected replacement node pairing generation");
}
const replacement = await registerDirectApnsRegistration({
nodeId,
token: "DCBA4321DCBA4321DCBA4321DCBA4321",
expectedPairingGeneration: replacementGeneration.key,
baseDir,
});
await expect(loadApnsRegistration(nodeId, baseDir)).resolves.toEqual(replacement);
await expect(
registerDirectApnsRegistration({
nodeId,
token: "ABCD1234ABCD1234ABCD1234ABCD1234",
expectedPairingGeneration: generation.key,
baseDir,
}),
).rejects.toBeInstanceOf(ApnsRegistrationPairingChangedError);
await expect(loadApnsRegistration(nodeId, baseDir)).resolves.toEqual(replacement);
});
it("rejects invalid direct and relay inputs", async () => {
const baseDir = await makeTempDir();
const oversized = "x".repeat(257);
src/infra/push-apns.test.ts
+90
View File
@@ -1,10 +1,12 @@
// Tests APNS push signing and request construction.
import { generateKeyPairSync } from "node:crypto";
import { EventEmitter } from "node:events";
import { createServer, type Server as HttpServer } from "node:http";
import http2 from "node:http2";
import net from "node:net";
import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion";
import { afterEach, describe, expect, it, vi } from "vitest";
import { createDeferred } from "../test-utils/deferred.js";
import { startProxy, stopProxy, type ProxyHandle } from "./net/proxy/proxy-lifecycle.js";
import {
appendApnsResponseBodyCapture,
@@ -510,6 +512,94 @@ describe("push APNs send semantics", () => {
expect(result.transport).toBe("direct");
});
it("guards direct wake transport with current ownership and lifecycle", async () => {
const { send, registration, auth } = createDirectApnsSendFixture({
nodeId: "ios-node-guarded-wake",
environment: "production",
sendResult: {
status: 200,
apnsId: "apns-guarded-wake-id",
body: "",
},
});
const controller = new AbortController();
const isCurrent = vi.fn().mockResolvedValue(true);
await sendApnsBackgroundWake({
registration,
nodeId: "ios-node-guarded-wake",
wakeReason: "node.invoke",
auth,
requestSender: send,
signal: controller.signal,
isCurrent,
});
expect(isCurrent).toHaveBeenCalledTimes(1);
const sent = requireSendRequest(send);
expect(sent.signal).toBe(controller.signal);
expect(sent.isCurrent).toBe(isCurrent);
await expect(
sendApnsBackgroundWake({
registration,
nodeId: "ios-node-guarded-wake",
wakeReason: "node.invoke",
auth,
requestSender: send,
isCurrent: vi.fn().mockResolvedValue(false),
}),
).rejects.toThrow("APNs send invalidated");
expect(send).toHaveBeenCalledTimes(1);
});
it("owns direct session errors while persistent currentness is pending", async () => {
const session = Object.assign(new EventEmitter(), {
close: vi.fn(),
destroy: vi.fn(),
request: vi.fn(),
});
session.destroy.mockImplementation(() => session.emit("close"));
const connect = vi
.spyOn(http2, "connect")
.mockReturnValue(session as unknown as http2.ClientHttp2Session);
const currentness = createDeferred<boolean>();
const isCurrent = vi
.fn<() => Promise<boolean>>()
.mockResolvedValueOnce(true)
.mockReturnValueOnce(currentness.promise);
const { registration, auth } = createDirectApnsSendFixture({
nodeId: "ios-node-session-error",
environment: "production",
sendResult: { status: 200, apnsId: "unused", body: "" },
});
try {
const sending = sendApnsBackgroundWake({
registration,
nodeId: "ios-node-session-error",
wakeReason: "node.invoke",
auth,
isCurrent,
});
await vi.waitFor(() => {
expect(connect).toHaveBeenCalledTimes(1);
expect(isCurrent).toHaveBeenCalledTimes(2);
});
expect(session.listenerCount("error")).toBeGreaterThan(0);
session.emit("error", new Error("APNs connection failed"));
currentness.resolve(true);
await expect(sending).rejects.toThrow("APNs connection failed");
expect(session.destroy).toHaveBeenCalledTimes(1);
expect(session.request).not.toHaveBeenCalled();
} finally {
currentness.resolve(false);
connect.mockRestore();
}
});
it("sends exec approval alert pushes with generic modal-only metadata", async () => {
const { send, registration, auth } = createDirectApnsSendFixture({
nodeId: "ios-node-approval-alert",
src/infra/push-apns.ts
+141 -155
View File
@@ -1,11 +1,10 @@
// Manages APNs registration state and direct/relay push sending.
import { createHash, createPrivateKey, sign as signJwt } from "node:crypto";
import fs from "node:fs/promises";
import { resolveTimerTimeoutMs } from "@openclaw/normalization-core/number-coercion";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
import type { DeviceIdentity } from "./device-identity.js";
import { formatErrorMessage, toErrorObject } from "./errors.js";
import { toErrorObject } from "./errors.js";
import { getApnsBearerToken, type ApnsAuthConfig } from "./push-apns-auth.js";
import {
APNS_HTTP2_CANCEL_CODE,
appendApnsResponseBodyCapture,
@@ -40,6 +39,7 @@ import {
} from "./push-apns.relay.js";
export {
ApnsRegistrationPairingChangedError,
clearApnsRegistrationIfCurrent,
loadApnsRegistration,
loadApnsRegistrations,
@@ -47,18 +47,11 @@ export {
registerApnsRegistration,
} from "./push-apns-store.js";
export type { ApnsRegistration } from "./push-apns-store.js";
export { resolveApnsAuthConfigFromEnv } from "./push-apns-auth.js";
export type { ApnsAuthConfig } from "./push-apns-auth.js";
type ApnsTransport = "direct" | "relay";
/** Direct APNs provider authentication used to mint ES256 bearer tokens. */
export type ApnsAuthConfig = {
teamId: string;
keyId: string;
privateKey: string;
};
type ApnsAuthConfigResolution = { ok: true; value: ApnsAuthConfig } | { ok: false; error: string };
/** Normalized APNs push result returned to gateway push/nodes methods. */
type ApnsPushResult = {
ok: boolean;
@@ -88,16 +81,33 @@ type ApnsRequestParams = {
timeoutMs: number;
pushType: ApnsPushType;
priority: "10" | "5";
signal?: AbortSignal;
isCurrent?: () => Promise<boolean>;
};
type ApnsRequestResponse = { status: number; apnsId?: string; body: string };
type ApnsRequestSender = (params: ApnsRequestParams) => Promise<ApnsRequestResponse>;
const APNS_JWT_TTL_MS = 50 * 60 * 1000;
const DEFAULT_APNS_TIMEOUT_MS = 10_000;
let cachedJwt: { cacheKey: string; token: string; expiresAtMs: number } | null = null;
function throwIfApnsSendAborted(signal: AbortSignal | undefined): void {
if (!signal?.aborted) {
return;
}
throw signal.reason instanceof Error ? signal.reason : new Error("APNs send invalidated");
}
async function requireCurrentApnsSend(params: {
signal?: AbortSignal;
isCurrent?: () => Promise<boolean>;
}): Promise<void> {
throwIfApnsSendAborted(params.signal);
if (params.isCurrent && !(await params.isCurrent())) {
throw new Error("APNs send invalidated");
}
throwIfApnsSendAborted(params.signal);
}
function parseReason(body: string): string | undefined {
const trimmed = body.trim();
@@ -114,57 +124,6 @@ function parseReason(body: string): string | undefined {
}
}
function toBase64UrlBytes(value: Uint8Array): string {
return Buffer.from(value)
.toString("base64")
.replace(/\+/g, "-")
.replace(/\//g, "_")
.replace(/=+$/g, "");
}
function toBase64UrlJson(value: object): string {
return toBase64UrlBytes(Buffer.from(JSON.stringify(value)));
}
function getJwtCacheKey(auth: ApnsAuthConfig): string {
const keyHash = createHash("sha256").update(auth.privateKey).digest("hex");
return `${auth.teamId}:${auth.keyId}:${keyHash}`;
}
function getApnsBearerToken(auth: ApnsAuthConfig, nowMs: number = Date.now()): string {
const cacheKey = getJwtCacheKey(auth);
if (cachedJwt && cachedJwt.cacheKey === cacheKey && nowMs < cachedJwt.expiresAtMs) {
return cachedJwt.token;
}
// APNs provider tokens are valid for one hour. Cache for slightly less so
// bursty wake/approval pushes avoid repeated ECDSA signing.
const iat = Math.floor(nowMs / 1000);
const header = toBase64UrlJson({ alg: "ES256", kid: auth.keyId, typ: "JWT" });
const payload = toBase64UrlJson({ iss: auth.teamId, iat });
const signingInput = `${header}.${payload}`;
const signature = signJwt("sha256", Buffer.from(signingInput, "utf8"), {
key: createPrivateKey(auth.privateKey),
dsaEncoding: "ieee-p1363",
});
const token = `${signingInput}.${toBase64UrlBytes(signature)}`;
cachedJwt = {
cacheKey,
token,
expiresAtMs: nowMs + APNS_JWT_TTL_MS,
};
return token;
}
function normalizePrivateKey(value: string): string {
return value.trim().replace(/\\n/g, "\n");
}
function normalizeNonEmptyString(value: string | undefined): string | null {
const trimmed = normalizeOptionalString(value) ?? "";
return trimmed.length > 0 ? trimmed : null;
}
/** Returns true for APNs responses that mean the direct device token is no longer usable. */
function shouldInvalidateApnsRegistration(result: { status: number; reason?: string }): boolean {
if (result.status === 410) {
@@ -191,60 +150,6 @@ export function shouldClearStoredApnsRegistration(params: {
return shouldInvalidateApnsRegistration(params.result);
}
/** Resolves direct APNs provider auth from env, accepting inline or file-backed keys. */
export async function resolveApnsAuthConfigFromEnv(
env: NodeJS.ProcessEnv = process.env,
): Promise<ApnsAuthConfigResolution> {
const teamId = normalizeNonEmptyString(env.OPENCLAW_APNS_TEAM_ID);
const keyId = normalizeNonEmptyString(env.OPENCLAW_APNS_KEY_ID);
if (!teamId || !keyId) {
return {
ok: false,
error: "APNs auth missing: set OPENCLAW_APNS_TEAM_ID and OPENCLAW_APNS_KEY_ID",
};
}
const inlineKeyRaw =
normalizeNonEmptyString(env.OPENCLAW_APNS_PRIVATE_KEY_P8) ??
normalizeNonEmptyString(env.OPENCLAW_APNS_PRIVATE_KEY);
if (inlineKeyRaw) {
return {
ok: true,
value: {
teamId,
keyId,
privateKey: normalizePrivateKey(inlineKeyRaw),
},
};
}
const keyPath = normalizeNonEmptyString(env.OPENCLAW_APNS_PRIVATE_KEY_PATH);
if (!keyPath) {
return {
ok: false,
error:
"APNs private key missing: set OPENCLAW_APNS_PRIVATE_KEY_P8 or OPENCLAW_APNS_PRIVATE_KEY_PATH",
};
}
try {
const privateKey = normalizePrivateKey(await fs.readFile(keyPath, "utf8"));
return {
ok: true,
value: {
teamId,
keyId,
privateKey,
},
};
} catch (err) {
const message = formatErrorMessage(err);
return {
ok: false,
error: `failed reading OPENCLAW_APNS_PRIVATE_KEY_PATH (${keyPath}): ${message}`,
};
}
}
async function sendApnsRequest(params: {
token: string;
topic: string;
@@ -254,6 +159,8 @@ async function sendApnsRequest(params: {
timeoutMs: number;
pushType: ApnsPushType;
priority: "10" | "5";
signal?: AbortSignal;
isCurrent?: () => Promise<boolean>;
}): Promise<ApnsRequestResponse> {
const authority =
params.environment === "production"
@@ -266,66 +173,124 @@ async function sendApnsRequest(params: {
const client = await connectApnsHttp2Session({
authority,
timeoutMs: params.timeoutMs,
...(params.signal ? { signal: params.signal } : {}),
});
// Connection failures can arrive while the persistent ownership check is
// yielding. Keep a consuming owner until the session closes, while the
// request-specific listener below still rejects the active send.
const consumeSessionError = () => undefined;
client.on("error", consumeSessionError);
client.once("close", () => client.off("error", consumeSessionError));
return await new Promise((resolve, reject) => {
let settled = false;
const fail = (err: unknown) => {
let activeRequest: ReturnType<typeof client.request> | undefined;
const cleanup = () => {
client.off("error", fail);
params.signal?.removeEventListener("abort", onAbort);
};
const fail = (err: unknown, options?: { cancelRequest?: boolean }) => {
if (settled) {
return;
}
settled = true;
client.destroy();
cleanup();
if (options?.cancelRequest && activeRequest && !activeRequest.destroyed) {
activeRequest.close(APNS_HTTP2_CANCEL_CODE);
client.close();
} else {
client.destroy();
}
reject(toErrorObject(err, "Non-Error rejection"));
};
const onAbort = () =>
fail(
params.signal?.reason instanceof Error
? params.signal.reason
: new Error("APNs send invalidated"),
{ cancelRequest: true },
);
const finish = (result: { status: number; apnsId?: string; body: string }) => {
if (settled) {
return;
}
settled = true;
cleanup();
client.close();
resolve(result);
};
client.once("error", (err) => fail(err));
const startRequest = async () => {
try {
await requireCurrentApnsSend(params);
if (settled) {
return;
}
if (params.signal?.aborted) {
onAbort();
return;
}
const req = client.request({
":method": "POST",
":path": requestPath,
authorization: `bearer ${params.bearerToken}`,
"apns-topic": params.topic,
"apns-push-type": params.pushType,
"apns-priority": params.priority,
"apns-expiration": "0",
"content-type": "application/json",
"content-length": Buffer.byteLength(body).toString(),
});
const req = client.request({
":method": "POST",
":path": requestPath,
authorization: `bearer ${params.bearerToken}`,
"apns-topic": params.topic,
"apns-push-type": params.pushType,
"apns-priority": params.priority,
"apns-expiration": "0",
"content-type": "application/json",
"content-length": Buffer.byteLength(body).toString(),
});
activeRequest = req;
let statusCode = 0;
let apnsId: string | undefined;
const responseBody = createApnsResponseBodyCapture();
let statusCode = 0;
let apnsId: string | undefined;
const responseBody = createApnsResponseBodyCapture();
req.setTimeout(params.timeoutMs, () => {
req.close(APNS_HTTP2_CANCEL_CODE);
fail(new Error(`APNs request timed out after ${params.timeoutMs}ms`));
});
req.on("response", (headers) => {
const statusHeader = headers[":status"];
statusCode = statusHeader ?? 0;
const idHeader = headers["apns-id"];
if (typeof idHeader === "string" && idHeader.trim().length > 0) {
apnsId = idHeader.trim();
req.setTimeout(params.timeoutMs, () => {
fail(new Error(`APNs request timed out after ${params.timeoutMs}ms`), {
cancelRequest: true,
});
});
req.on("response", (headers) => {
const statusHeader = headers[":status"];
statusCode = statusHeader ?? 0;
const idHeader = headers["apns-id"];
if (typeof idHeader === "string" && idHeader.trim().length > 0) {
apnsId = idHeader.trim();
}
});
req.on("data", (chunk) => {
appendApnsResponseBodyCapture(responseBody, chunk);
});
req.on("end", () => {
finish({
status: statusCode,
apnsId,
body: getApnsResponseBodyCaptureText(responseBody),
});
});
req.on("error", (err) => fail(err));
if (params.signal?.aborted) {
onAbort();
return;
}
req.end(body);
} catch (error) {
fail(error);
}
});
req.on("data", (chunk) => {
appendApnsResponseBodyCapture(responseBody, chunk);
});
req.on("end", () => {
finish({ status: statusCode, apnsId, body: getApnsResponseBodyCaptureText(responseBody) });
});
req.on("error", (err) => fail(err));
};
req.end(body);
client.once("error", fail);
params.signal?.addEventListener("abort", onAbort, { once: true });
if (params.signal?.aborted) {
onAbort();
return;
}
void startRequest();
});
}
@@ -411,11 +376,14 @@ async function sendDirectApnsPush(params: {
requestSender?: ApnsRequestSender;
pushType: ApnsPushType;
priority: "10" | "5";
signal?: AbortSignal;
isCurrent?: () => Promise<boolean>;
}): Promise<ApnsPushResult> {
const { token, topic, environment, bearerToken } = resolveDirectSendContext({
auth: params.auth,
registration: params.registration,
});
await requireCurrentApnsSend(params);
const sender = params.requestSender ?? sendApnsRequest;
const response = await sender({
token,
@@ -426,6 +394,8 @@ async function sendDirectApnsPush(params: {
timeoutMs: resolveApnsTimeoutMs(params.timeoutMs),
pushType: params.pushType,
priority: params.priority,
...(params.signal ? { signal: params.signal } : {}),
...(params.isCurrent ? { isCurrent: params.isCurrent } : {}),
});
return toPushResult({
registration: params.registration,
@@ -442,6 +412,8 @@ async function sendRelayApnsPush(params: {
priority: "10" | "5";
gatewayIdentity?: Pick<DeviceIdentity, "deviceId" | "privateKeyPem">;
requestSender?: ApnsRelayRequestSender;
signal?: AbortSignal;
isCurrent?: () => Promise<boolean>;
}): Promise<ApnsPushResult> {
const response = await sendApnsRelayPush({
relayConfig: params.relayConfig,
@@ -452,6 +424,8 @@ async function sendRelayApnsPush(params: {
priority: params.priority,
gatewayIdentity: params.gatewayIdentity,
requestSender: params.requestSender,
...(params.signal ? { signal: params.signal } : {}),
...(params.isCurrent ? { isCurrent: params.isCurrent } : {}),
});
return toPushResult({ registration: params.registration, response });
}
@@ -461,6 +435,8 @@ type ApnsAlertCommonParams = {
title: string;
body: string;
timeoutMs?: number;
signal?: AbortSignal;
isCurrent?: () => Promise<boolean>;
};
type DirectApnsAlertParams = ApnsAlertCommonParams & {
@@ -484,6 +460,8 @@ type ApnsBackgroundWakeCommonParams = {
nodeId: string;
wakeReason?: string;
timeoutMs?: number;
signal?: AbortSignal;
isCurrent?: () => Promise<boolean>;
};
type DirectApnsBackgroundWakeParams = ApnsBackgroundWakeCommonParams & {
@@ -554,6 +532,8 @@ export async function sendApnsAlert(
priority: "10",
gatewayIdentity: relayParams.relayGatewayIdentity,
requestSender: relayParams.relayRequestSender,
...(relayParams.signal ? { signal: relayParams.signal } : {}),
...(relayParams.isCurrent ? { isCurrent: relayParams.isCurrent } : {}),
});
}
const directParams = params as DirectApnsAlertParams;
@@ -565,6 +545,8 @@ export async function sendApnsAlert(
requestSender: directParams.requestSender,
pushType: "alert",
priority: "10",
...(directParams.signal ? { signal: directParams.signal } : {}),
...(directParams.isCurrent ? { isCurrent: directParams.isCurrent } : {}),
});
}
@@ -587,6 +569,8 @@ export async function sendApnsBackgroundWake(
priority: "5",
gatewayIdentity: relayParams.relayGatewayIdentity,
requestSender: relayParams.relayRequestSender,
...(relayParams.signal ? { signal: relayParams.signal } : {}),
...(relayParams.isCurrent ? { isCurrent: relayParams.isCurrent } : {}),
});
}
const directParams = params as DirectApnsBackgroundWakeParams;
@@ -598,6 +582,8 @@ export async function sendApnsBackgroundWake(
requestSender: directParams.requestSender,
pushType: "background",
priority: "5",
...(directParams.signal ? { signal: directParams.signal } : {}),
...(directParams.isCurrent ? { isCurrent: directParams.isCurrent } : {}),
});
}
src/skills/runtime/remote-probe-utils.ts
+116
View File
@@ -0,0 +1,116 @@
// Pure platform and payload helpers for remote skill binary probes.
import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { normalizeStringEntries } from "@openclaw/normalization-core/string-normalization";
import type { SkillEntry } from "../types.js";
export function extractErrorMessage(err: unknown): string | undefined {
if (!err) {
return undefined;
}
if (typeof err === "string") {
return err;
}
if (err instanceof Error) {
return err.message;
}
if (typeof err === "object" && "message" in err && typeof err.message === "string") {
return err.message;
}
if (typeof err === "number" || typeof err === "boolean" || typeof err === "bigint") {
return String(err);
}
if (typeof err === "symbol") {
return err.toString();
}
if (typeof err === "object") {
try {
return JSON.stringify(err);
} catch {
return undefined;
}
}
return undefined;
}
export function isMacPlatform(platform?: string, deviceFamily?: string): boolean {
const platformNorm = normalizeLowercaseStringOrEmpty(platform);
const familyNorm = normalizeLowercaseStringOrEmpty(deviceFamily);
return platformNorm.includes("mac") || platformNorm.includes("darwin") || familyNorm === "mac";
}
export function supportsSystemRun(commands?: string[]): boolean {
return Array.isArray(commands) && commands.includes("system.run");
}
export function supportsSystemWhich(commands?: string[]): boolean {
return Array.isArray(commands) && commands.includes("system.which");
}
export function collectRequiredBins(entries: SkillEntry[], targetPlatform: string): string[] {
const bins = new Set<string>();
for (const entry of entries) {
const os = entry.metadata?.os ?? [];
if (os.length > 0 && !os.includes(targetPlatform)) {
continue;
}
for (const bin of [
...(entry.metadata?.requires?.bins ?? []),
...(entry.metadata?.requires?.anyBins ?? []),
]) {
if (bin.trim()) {
bins.add(bin.trim());
}
}
}
return [...bins];
}
export function buildBinProbeScript(bins: string[]): string {
const escaped = bins.map((bin) => `'${bin.replace(/'/g, `'\\''`)}'`).join(" ");
return `for b in ${escaped}; do if command -v "$b" >/dev/null 2>&1; then echo "$b"; fi; done`;
}
export function parseBinProbePayload(
payloadJSON: string | null | undefined,
payload?: unknown,
): string[] {
if (!payloadJSON && !payload) {
return [];
}
try {
const parsed = payloadJSON
? (JSON.parse(payloadJSON) as { stdout?: unknown; bins?: unknown })
: (payload as { stdout?: unknown; bins?: unknown });
if (Array.isArray(parsed.bins)) {
return normalizeStringEntries(parsed.bins);
}
if (parsed.bins && typeof parsed.bins === "object") {
return Object.entries(parsed.bins)
.filter(([, resolvedPath]) => normalizeOptionalString(resolvedPath) !== undefined)
.map(([bin]) => normalizeOptionalString(bin) ?? "")
.filter(Boolean);
}
if (typeof parsed.stdout === "string") {
return parsed.stdout
.split(/\r?\n/)
.map((line) => normalizeOptionalString(line) ?? "")
.filter(Boolean);
}
} catch {
return [];
}
return [];
}
export function areBinSetsEqual(a: Set<string> | undefined, b: Set<string>): boolean {
if (!a || a.size !== b.size) {
return false;
}
for (const bin of b) {
if (!a.has(bin)) {
return false;
}
}
return true;
}
src/skills/runtime/remote-reconciliation.test.ts
+78
View File
@@ -0,0 +1,78 @@
import { randomUUID } from "node:crypto";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { NodeRegistry } from "../../gateway/node-registry.js";
import { mergeRemoteNodeSkillEntries, replaceRemoteNodeSkills } from "./remote-skills.js";
import {
getRemoteSkillEligibility,
recordRemoteNodeBins,
recordRemoteNodeInfo,
removeRemoteNodeInfo,
setSkillsRemoteRegistry,
} from "./remote.js";
const TEST_PAIRING_GENERATION = "generation-test";
afterEach(() => {
setSkillsRemoteRegistry(null);
vi.restoreAllMocks();
});
describe("remote projection reconciliation", () => {
it("rejects legacy-generation and connectionless node projections", () => {
const legacyNodeId = `node-${randomUUID()}`;
const missingConnectionNodeId = `node-${randomUUID()}`;
const bin = `bin-${randomUUID()}`;
const legacySkillName = `skill-${randomUUID()}`;
const missingConnectionSkillName = `skill-${randomUUID()}`;
const listCurrentConnectedSync = vi.fn(() => []);
try {
setSkillsRemoteRegistry({
listCurrentConnectedSync,
} as unknown as NodeRegistry);
recordRemoteNodeInfo({
nodeId: legacyNodeId,
connId: "conn-retired",
displayName: "Legacy Mac",
platform: "darwin",
commands: ["system.run"],
});
replaceRemoteNodeSkills({
nodeId: legacyNodeId,
displayName: "Legacy Mac",
skills: [
{
name: legacySkillName,
description: "Legacy remote skill",
content: `---\nname: ${legacySkillName}\ndescription: Legacy remote skill\n---\n`,
},
],
});
recordRemoteNodeInfo({
nodeId: missingConnectionNodeId,
pairingGeneration: TEST_PAIRING_GENERATION,
displayName: "Retired Mac",
platform: "darwin",
commands: ["system.run", "system.which"],
});
recordRemoteNodeBins(missingConnectionNodeId, [bin], TEST_PAIRING_GENERATION);
replaceRemoteNodeSkills({
nodeId: missingConnectionNodeId,
displayName: "Retired Mac",
skills: [
{
name: missingConnectionSkillName,
description: "Retired remote skill",
content: `---\nname: ${missingConnectionSkillName}\ndescription: Retired remote skill\n---\n`,
},
],
});
expect(getRemoteSkillEligibility()?.hasBin(bin) ?? false).toBe(false);
expect(mergeRemoteNodeSkillEntries([], { canExec: true })).toEqual([]);
expect(listCurrentConnectedSync).toHaveBeenCalled();
} finally {
removeRemoteNodeInfo(legacyNodeId);
removeRemoteNodeInfo(missingConnectionNodeId);
}
});
});
src/skills/runtime/remote-skills.ts
+20 -1
View File
@@ -22,6 +22,18 @@ type RemoteSkillNode = {
const remoteSkillNodes = new Map<string, RemoteSkillNode>();
const log = createSubsystemLogger("gateway/skills-remote");
let reconcileRemoteSkillConnections: (() => ReadonlySet<string> | undefined) | null = null;
function remoteConnectionKey(nodeId: string, connId: string): string {
return `${nodeId}\0${connId}`;
}
/** Installs the gateway-owned persistent-generation reconciliation boundary. */
export function setRemoteSkillConnectionReconciler(
reconcile: (() => ReadonlySet<string> | undefined) | null,
): void {
reconcileRemoteSkillConnections = reconcile;
}
function prepareNodeSkills(
nodeId: string,
@@ -168,8 +180,14 @@ export function mergeRemoteNodeSkillEntries(
if (options?.canExec !== true) {
return [...localEntries];
}
const currentConnections = reconcileRemoteSkillConnections?.();
const connectedNodes = [...remoteSkillNodes.values()].filter(
(node) => node.connected && node.canExec,
(node) =>
node.connected &&
node.canExec &&
(!currentConnections ||
(node.connId !== undefined &&
currentConnections.has(remoteConnectionKey(node.nodeId, node.connId)))),
);
let boundNodeId: string | undefined;
if (options.node) {
@@ -247,6 +265,7 @@ export function mergeRemoteNodeSkillEntries(
function resetRemoteNodeSkillsForTests(): void {
remoteSkillNodes.clear();
reconcileRemoteSkillConnections = null;
}
if (process.env.VITEST || process.env.NODE_ENV === "test") {
src/skills/runtime/remote.test.ts
+285 -43
View File
@@ -8,16 +8,53 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js";
import type { NodeRegistry } from "../../gateway/node-registry.js";
import { getSkillsSnapshotVersion } from "./refresh-state.js";
import { resetSkillsRefreshForTest } from "./refresh.test-support.js";
import { mergeRemoteNodeSkillEntries, replaceRemoteNodeSkills } from "./remote-skills.js";
vi.mock("../../infra/node-pairing.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../../infra/node-pairing.js")>();
return { ...actual, updatePairedNodeBins: vi.fn(async () => true) };
});
import {
getRemoteSkillEligibility,
recordRemoteNodeBins,
recordRemoteNodeInfo,
removeRemoteNodeInfo,
removeRemoteNodeInfoForConnection,
refreshRemoteBinsForConnectedNodes,
refreshRemoteNodeBins,
setSkillsRemoteRegistry,
} from "./remote.js";
const TEST_PAIRING_GENERATION = "generation-test";
function testRemoteSession(
nodeId: string,
overrides?: Partial<NonNullable<ReturnType<NodeRegistry["get"]>>>,
): NonNullable<ReturnType<NodeRegistry["get"]>> {
return {
nodeId,
connId: `conn-${nodeId}`,
pairingGeneration: TEST_PAIRING_GENERATION,
platform: "darwin",
commands: ["system.run", "system.which"],
...overrides,
} as NonNullable<ReturnType<NodeRegistry["get"]>>;
}
function setTestSkillsRemoteRegistry(
nodeIds: string | readonly string[],
registry: Partial<NodeRegistry> & Pick<NodeRegistry, "get">,
): void {
const ids = typeof nodeIds === "string" ? [nodeIds] : nodeIds;
setSkillsRemoteRegistry({
...registry,
listCurrentConnectedSync:
registry.listCurrentConnectedSync ??
(() => ids.flatMap((nodeId) => (registry.get(nodeId) ? [registry.get(nodeId)!] : []))),
} as unknown as NodeRegistry);
}
function createRemoteSkillWorkspace(bin: string): { cfg: OpenClawConfig; workspaceDir: string } {
const workspaceDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-remote-skills-"));
const skillDir = path.join(workspaceDir, "skills", "remote-skill");
@@ -49,6 +86,8 @@ function createRemoteSkillWorkspace(bin: string): { cfg: OpenClawConfig; workspa
function recordRemoteMacWithSystemWhich(nodeId: string): void {
recordRemoteNodeInfo({
nodeId,
connId: `conn-${nodeId}`,
pairingGeneration: TEST_PAIRING_GENERATION,
displayName: "Remote Mac",
platform: "darwin",
commands: ["system.run", "system.which"],
@@ -71,7 +110,7 @@ describe("skills-remote", () => {
platform: "darwin",
commands: ["system.run"],
});
recordRemoteNodeBins(nodeId, [bin]);
recordRemoteNodeBins(nodeId, [bin], TEST_PAIRING_GENERATION);
expect(getRemoteSkillEligibility()?.hasBin(bin)).toBe(true);
@@ -86,6 +125,74 @@ describe("skills-remote", () => {
expect(removeRemoteNodeInfo(nodeId)).toBeUndefined();
});
it("preserves bins across reconnects only within one pairing generation", () => {
const nodeId = `node-${randomUUID()}`;
const retiredBin = `bin-${randomUUID()}`;
try {
recordRemoteNodeInfo({
nodeId,
connId: "conn-a",
pairingGeneration: "generation-a",
platform: "darwin",
commands: ["system.run"],
});
recordRemoteNodeBins(nodeId, [retiredBin], "generation-a");
recordRemoteNodeInfo({
nodeId,
connId: "conn-a-reconnect",
pairingGeneration: "generation-a",
platform: "darwin",
commands: ["system.run"],
});
expect(getRemoteSkillEligibility()?.hasBin(retiredBin)).toBe(true);
recordRemoteNodeInfo({
nodeId,
connId: "conn-b",
pairingGeneration: "generation-b",
platform: "darwin",
commands: ["system.run"],
});
expect(getRemoteSkillEligibility()?.hasBin(retiredBin) ?? false).toBe(false);
} finally {
removeRemoteNodeInfo(nodeId);
}
});
it("removes remote projections only for the exact invalidated connection", () => {
const nodeId = `node-${randomUUID()}`;
const bin = `bin-${randomUUID()}`;
const skillName = `skill-${randomUUID()}`;
recordRemoteNodeInfo({
nodeId,
connId: "conn-current",
displayName: "Remote Mac",
platform: "darwin",
commands: ["system.run"],
});
recordRemoteNodeBins(nodeId, [bin], TEST_PAIRING_GENERATION);
replaceRemoteNodeSkills({
nodeId,
displayName: "Remote Mac",
skills: [
{
name: skillName,
description: "Remote test skill",
content: `---\nname: ${skillName}\ndescription: Remote test skill\n---\n`,
},
],
});
const listRemoteSkillNames = () =>
mergeRemoteNodeSkillEntries([], { canExec: true }).map((entry) => entry.skill.name);
expect(removeRemoteNodeInfoForConnection(nodeId, "conn-retired")).toBe(false);
expect(getRemoteSkillEligibility()?.hasBin(bin)).toBe(true);
expect(listRemoteSkillNames()).toContain(skillName);
expect(removeRemoteNodeInfoForConnection(nodeId, "conn-current")).toBe(true);
expect(getRemoteSkillEligibility()?.hasBin(bin) ?? false).toBe(false);
expect(listRemoteSkillNames()).not.toContain(skillName);
});
it("bumps the skills snapshot version when an eligible remote node disconnects", async () => {
await resetSkillsRefreshForTest();
const workspaceDir = `/tmp/ws-${randomUUID()}`;
@@ -115,7 +222,7 @@ describe("skills-remote", () => {
platform: "linux",
commands: ["system.run"],
});
recordRemoteNodeBins(linuxNodeId, [bin]);
recordRemoteNodeBins(linuxNodeId, [bin], TEST_PAIRING_GENERATION);
recordRemoteNodeInfo({
nodeId: noRunNodeId,
@@ -123,7 +230,7 @@ describe("skills-remote", () => {
platform: "darwin",
commands: ["system.which"],
});
recordRemoteNodeBins(noRunNodeId, [bin]);
recordRemoteNodeBins(noRunNodeId, [bin], TEST_PAIRING_GENERATION);
expect(getRemoteSkillEligibility()).toBeUndefined();
} finally {
@@ -144,14 +251,14 @@ describe("skills-remote", () => {
platform: "darwin",
commands: ["system.run"],
});
recordRemoteNodeBins(nodeA, [binA]);
recordRemoteNodeBins(nodeA, [binA], TEST_PAIRING_GENERATION);
recordRemoteNodeInfo({
nodeId: nodeB,
platform: "macOS",
commands: ["system.run"],
});
recordRemoteNodeBins(nodeB, [binB]);
recordRemoteNodeBins(nodeB, [binB], TEST_PAIRING_GENERATION);
const eligibility = getRemoteSkillEligibility();
expect(eligibility?.platforms).toEqual(["darwin"]);
@@ -175,7 +282,7 @@ describe("skills-remote", () => {
platform: "darwin",
commands: ["system.run"],
});
recordRemoteNodeBins(nodeId, [bin]);
recordRemoteNodeBins(nodeId, [bin], TEST_PAIRING_GENERATION);
const eligibility = getRemoteSkillEligibility({ advertiseExecNode: false });
@@ -190,7 +297,7 @@ describe("skills-remote", () => {
const nodeId = `node-${randomUUID()}`;
const bin = `bin-${randomUUID()}`;
try {
recordRemoteNodeBins(nodeId, [bin]);
recordRemoteNodeBins(nodeId, [bin], TEST_PAIRING_GENERATION);
expect(getRemoteSkillEligibility()?.hasBin(bin) ?? false).toBe(false);
} finally {
@@ -205,9 +312,8 @@ describe("skills-remote", () => {
const { cfg, workspaceDir } = createRemoteSkillWorkspace(bin);
try {
const invokeCalls: string[] = [];
setSkillsRemoteRegistry({
listConnected: () => [],
get: () => undefined,
setTestSkillsRemoteRegistry(nodeId, {
get: () => testRemoteSession(nodeId),
invoke: async (params: { command: string }) => {
invokeCalls.push(params.command);
return {
@@ -217,7 +323,7 @@ describe("skills-remote", () => {
},
} as unknown as NodeRegistry);
recordRemoteMacWithSystemWhich(nodeId);
recordRemoteNodeBins(nodeId, [bin]);
recordRemoteNodeBins(nodeId, [bin], TEST_PAIRING_GENERATION);
const before = getSkillsSnapshotVersion(workspaceDir);
await refreshRemoteNodeBins({
@@ -264,9 +370,8 @@ describe("skills-remote", () => {
},
} satisfies OpenClawConfig;
const invokeCalls: string[] = [];
setSkillsRemoteRegistry({
listConnected: () => [],
get: () => undefined,
setTestSkillsRemoteRegistry(nodeId, {
get: () => testRemoteSession(nodeId),
checkConnectivity: async () => ({
ok: false,
error: { code: "TIMEOUT", message: "node connectivity probe timed out" },
@@ -281,11 +386,12 @@ describe("skills-remote", () => {
} as unknown as NodeRegistry);
recordRemoteNodeInfo({
nodeId,
pairingGeneration: TEST_PAIRING_GENERATION,
displayName: "Remote Mac",
platform: "darwin",
commands: ["system.run", "system.which"],
});
recordRemoteNodeBins(nodeId, [bin]);
recordRemoteNodeBins(nodeId, [bin], TEST_PAIRING_GENERATION);
const before = getSkillsSnapshotVersion(workspaceDir);
await refreshRemoteNodeBins({
@@ -334,12 +440,12 @@ describe("skills-remote", () => {
let connId = "conn-old";
const connectivityCalls: string[] = [];
const invokeCalls: string[] = [];
setSkillsRemoteRegistry({
listConnected: () => [],
setTestSkillsRemoteRegistry(nodeId, {
get: () =>
({
nodeId,
connId,
pairingGeneration: TEST_PAIRING_GENERATION,
platform: "darwin",
commands: ["system.run", "system.which"],
}) as unknown as ReturnType<NodeRegistry["get"]>,
@@ -347,6 +453,14 @@ describe("skills-remote", () => {
connectivityCalls.push(connId);
if (connectivityCalls.length === 1) {
connId = "conn-new";
recordRemoteNodeInfo({
nodeId,
connId,
pairingGeneration: TEST_PAIRING_GENERATION,
displayName: "Remote Mac",
platform: "darwin",
commands: ["system.run", "system.which"],
});
return {
ok: false,
error: { code: "TIMEOUT", message: "node connectivity probe timed out" },
@@ -364,6 +478,8 @@ describe("skills-remote", () => {
} as unknown as NodeRegistry);
recordRemoteNodeInfo({
nodeId,
connId,
pairingGeneration: TEST_PAIRING_GENERATION,
displayName: "Remote Mac",
platform: "darwin",
commands: ["system.run", "system.which"],
@@ -393,9 +509,8 @@ describe("skills-remote", () => {
let invokeCount = 0;
let releaseProbe: (() => void) | undefined;
const probeStarted = new Promise<void>((resolve) => {
setSkillsRemoteRegistry({
listConnected: () => [],
get: () => undefined,
setTestSkillsRemoteRegistry(nodeId, {
get: () => testRemoteSession(nodeId),
invoke: async () => {
invokeCount += 1;
resolve();
@@ -432,6 +547,7 @@ describe("skills-remote", () => {
} satisfies OpenClawConfig;
recordRemoteNodeInfo({
nodeId,
pairingGeneration: TEST_PAIRING_GENERATION,
displayName: "Remote Mac",
platform: "darwin",
commands: ["system.run", "system.which"],
@@ -473,9 +589,8 @@ describe("skills-remote", () => {
const nowSpy = vi.spyOn(Date, "now").mockReturnValue(1_000_000);
let invokeCount = 0;
try {
setSkillsRemoteRegistry({
listConnected: () => [],
get: () => undefined,
setTestSkillsRemoteRegistry(nodeId, {
get: () => testRemoteSession(nodeId),
invoke: async () => {
invokeCount += 1;
return { ok: true, payload: { bins: [bin] } };
@@ -528,9 +643,8 @@ describe("skills-remote", () => {
vi.spyOn(Date, "now").mockReturnValue(2_000_000);
let invokeCount = 0;
try {
setSkillsRemoteRegistry({
listConnected: () => [],
get: () => undefined,
setTestSkillsRemoteRegistry(nodeId, {
get: () => testRemoteSession(nodeId),
invoke: async () => {
invokeCount += 1;
return { ok: false, error: { code: "TIMEOUT", message: "node invoke timed out" } };
@@ -566,9 +680,8 @@ describe("skills-remote", () => {
const checkConnectivity = vi.fn(async () => ({ ok: true as const }));
const invoke = vi.fn(async () => ({ ok: true as const, payload: { bins: [bin] } }));
try {
setSkillsRemoteRegistry({
listConnected: () => [],
get: () => undefined,
setTestSkillsRemoteRegistry(nodeId, {
get: () => testRemoteSession(nodeId),
checkConnectivity,
invoke,
} as unknown as NodeRegistry);
@@ -612,12 +725,12 @@ describe("skills-remote", () => {
.mockImplementationOnce(async () => await firstInvoke)
.mockResolvedValueOnce({ ok: true as const, payload: { bins: [bin] } });
try {
setSkillsRemoteRegistry({
listConnected: () => [],
setTestSkillsRemoteRegistry(nodeId, {
get: () =>
({
nodeId,
connId,
pairingGeneration: TEST_PAIRING_GENERATION,
platform: "darwin",
commands: ["system.run", "system.which"],
}) as unknown as ReturnType<NodeRegistry["get"]>,
@@ -626,6 +739,7 @@ describe("skills-remote", () => {
recordRemoteNodeInfo({
nodeId,
connId,
pairingGeneration: TEST_PAIRING_GENERATION,
platform: "darwin",
commands: ["system.run", "system.which"],
});
@@ -644,6 +758,7 @@ describe("skills-remote", () => {
recordRemoteNodeInfo({
nodeId,
connId,
pairingGeneration: TEST_PAIRING_GENERATION,
platform: "darwin",
commands: ["system.run", "system.which"],
});
@@ -661,20 +776,97 @@ describe("skills-remote", () => {
}
});
it("does not carry a retired generation probe into its replacement", async () => {
const nodeId = `node-${randomUUID()}`;
const retiredBin = `bin-${randomUUID()}`;
const currentBin = `bin-${randomUUID()}`;
const { cfg, workspaceDir } = createRemoteSkillWorkspace(currentBin);
let session = {
nodeId,
connId: "conn-a",
pairingGeneration: "generation-a",
platform: "darwin",
commands: ["system.run", "system.which"],
} as NonNullable<ReturnType<NodeRegistry["get"]>>;
let resolveRetiredProbe:
| ((value: Awaited<ReturnType<NodeRegistry["invoke"]>>) => void)
| undefined;
const retiredProbe = new Promise<Awaited<ReturnType<NodeRegistry["invoke"]>>>((resolve) => {
resolveRetiredProbe = resolve;
});
const invoke = vi
.fn()
.mockImplementationOnce(async () => await retiredProbe)
.mockResolvedValueOnce({ ok: true as const, payload: { bins: [currentBin] } });
try {
setTestSkillsRemoteRegistry(nodeId, {
get: () => session,
invoke,
} as unknown as NodeRegistry);
recordRemoteNodeInfo({
nodeId,
connId: session.connId,
pairingGeneration: session.pairingGeneration,
platform: session.platform,
commands: session.commands,
});
const firstRefresh = refreshRemoteNodeBins({
nodeId,
platform: session.platform,
commands: session.commands,
cfg,
});
await vi.waitFor(() => expect(invoke).toHaveBeenCalledTimes(1));
session = { ...session, connId: "conn-b", pairingGeneration: "generation-b" };
recordRemoteNodeInfo({
nodeId,
connId: session.connId,
pairingGeneration: session.pairingGeneration,
platform: session.platform,
commands: session.commands,
});
recordRemoteNodeBins(nodeId, [currentBin], "generation-b");
const replacementRefresh = refreshRemoteNodeBins({
nodeId,
platform: session.platform,
commands: session.commands,
cfg,
});
resolveRetiredProbe?.({ ok: true, payload: { bins: [retiredBin] } });
await Promise.all([firstRefresh, replacementRefresh]);
expect(invoke).toHaveBeenNthCalledWith(
1,
expect.objectContaining({ expectedPairingGeneration: "generation-a" }),
);
expect(invoke).toHaveBeenNthCalledWith(
2,
expect.objectContaining({ expectedPairingGeneration: "generation-b" }),
);
expect(getRemoteSkillEligibility()?.hasBin(retiredBin) ?? false).toBe(false);
expect(getRemoteSkillEligibility()?.hasBin(currentBin)).toBe(true);
} finally {
removeRemoteNodeInfo(nodeId);
fs.rmSync(workspaceDir, { recursive: true, force: true });
}
});
it("uses the approved live command surface after the connect readiness delay", async () => {
vi.useFakeTimers();
const nodeId = `node-${randomUUID()}`;
const bin = `bin-${randomUUID()}`;
const { cfg, workspaceDir } = createRemoteSkillWorkspace(bin);
let commands: string[] = [];
let pairingGeneration: string | undefined;
const invoke = vi.fn(async () => ({ ok: true as const, payload: { bins: [bin] } }));
try {
setSkillsRemoteRegistry({
listConnected: () => [],
setTestSkillsRemoteRegistry(nodeId, {
get: () =>
({
nodeId,
connId: "conn-current",
pairingGeneration,
platform: "darwin",
commands,
}) as unknown as ReturnType<NodeRegistry["get"]>,
@@ -691,6 +883,14 @@ describe("skills-remote", () => {
readinessDelayMs: 5_000,
});
commands = ["system.run", "system.which"];
pairingGeneration = TEST_PAIRING_GENERATION;
recordRemoteNodeInfo({
nodeId,
connId: "conn-current",
pairingGeneration,
platform: "darwin",
commands,
});
const approvalRefresh = refreshRemoteNodeBins({
nodeId,
platform: "darwin",
@@ -717,9 +917,8 @@ describe("skills-remote", () => {
const { cfg, workspaceDir } = createRemoteSkillWorkspace(bin);
try {
const invokeCalls: string[] = [];
setSkillsRemoteRegistry({
listConnected: () => [],
get: () => undefined,
setTestSkillsRemoteRegistry(nodeId, {
get: () => testRemoteSession(nodeId),
invoke: async (params: { command: string }) => {
invokeCalls.push(params.command);
return {
@@ -758,12 +957,8 @@ describe("skills-remote", () => {
const { cfg, workspaceDir } = createRemoteSkillWorkspace(bin);
try {
const invokeCalls: string[] = [];
setSkillsRemoteRegistry({
listConnected: () => [
{ nodeId: nodeA, platform: "darwin", commands: ["system.run", "system.which"] },
{ nodeId: nodeB, platform: "darwin", commands: ["system.run", "system.which"] },
],
get: () => undefined,
setTestSkillsRemoteRegistry([nodeA, nodeB], {
get: (nodeId: string) => testRemoteSession(nodeId),
checkConnectivity: (nodeId: string) => {
if (nodeId === nodeA) {
throw new Error("simulated connectivity failure");
@@ -777,7 +972,7 @@ describe("skills-remote", () => {
} as unknown as NodeRegistry);
recordRemoteMacWithSystemWhich(nodeA);
recordRemoteMacWithSystemWhich(nodeB);
recordRemoteNodeBins(nodeA, ["stale-bin"]);
recordRemoteNodeBins(nodeA, ["stale-bin"], TEST_PAIRING_GENERATION);
await expect(refreshRemoteBinsForConnectedNodes(cfg)).resolves.toBeUndefined();
@@ -790,4 +985,51 @@ describe("skills-remote", () => {
fs.rmSync(workspaceDir, { recursive: true, force: true });
}
});
it("refreshes bins only for generation-current connected sessions", async () => {
await resetSkillsRefreshForTest();
const staleNodeId = `node-${randomUUID()}`;
const currentNodeId = `node-${randomUUID()}`;
const bin = `bin-${randomUUID()}`;
const { cfg, workspaceDir } = createRemoteSkillWorkspace(bin);
const staleSession = testRemoteSession(staleNodeId, { connId: "conn-stale" });
const currentSession = testRemoteSession(currentNodeId, { connId: "conn-current" });
const invoke = vi.fn(async () => ({
ok: false as const,
error: { code: "TIMEOUT", message: "node invoke timed out" },
}));
const listCurrentConnectedSync = vi.fn(() => [currentSession]);
try {
setSkillsRemoteRegistry({
listConnected: () => [staleSession, currentSession],
listCurrentConnectedSync,
get: (nodeId: string) => (nodeId === currentNodeId ? currentSession : staleSession),
invoke,
} as unknown as NodeRegistry);
recordRemoteNodeInfo({
nodeId: staleNodeId,
connId: "conn-stale",
pairingGeneration: TEST_PAIRING_GENERATION,
platform: "darwin",
commands: ["system.run", "system.which"],
});
recordRemoteNodeInfo({
nodeId: currentNodeId,
connId: "conn-current",
pairingGeneration: TEST_PAIRING_GENERATION,
platform: "darwin",
commands: ["system.run", "system.which"],
});
await refreshRemoteBinsForConnectedNodes(cfg);
expect(listCurrentConnectedSync).toHaveBeenCalled();
expect(invoke).toHaveBeenCalledTimes(1);
expect(invoke).toHaveBeenCalledWith(expect.objectContaining({ nodeId: currentNodeId }));
} finally {
removeRemoteNodeInfo(staleNodeId);
removeRemoteNodeInfo(currentNodeId);
fs.rmSync(workspaceDir, { recursive: true, force: true });
}
});
});
src/skills/runtime/remote.ts
+179 -188
View File
@@ -1,18 +1,28 @@
// Remote skill runtime helpers send skill refresh and snapshot state across remotes.
import {
normalizeLowercaseStringOrEmpty,
normalizeOptionalString,
} from "@openclaw/normalization-core/string-coerce";
import { normalizeStringEntries } from "@openclaw/normalization-core/string-normalization";
import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
import { listAgentWorkspaceDirs } from "../../agents/workspace-dirs.js";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import type { NodeRegistry } from "../../gateway/node-registry.js";
import type { NodeRegistry, NodeSession } from "../../gateway/node-registry.js";
import { listNodePairing, updatePairedNodeBins } from "../../infra/node-pairing.js";
import { createSubsystemLogger } from "../../logging/subsystem.js";
import { loadWorkspaceSkillEntries } from "../loading/workspace.js";
import type { SkillEligibilityContext, SkillEntry } from "../types.js";
import type { SkillEligibilityContext } from "../types.js";
import { bumpSkillsSnapshotVersion } from "./refresh-state.js";
import { recordRemoteSkillNodeInfo, removeRemoteNodeSkills } from "./remote-skills.js";
import {
areBinSetsEqual,
buildBinProbeScript,
collectRequiredBins,
extractErrorMessage,
isMacPlatform,
parseBinProbePayload,
supportsSystemRun,
supportsSystemWhich,
} from "./remote-probe-utils.js";
import {
recordRemoteSkillNodeInfo,
removeRemoteNodeSkills,
setRemoteSkillConnectionReconciler,
} from "./remote-skills.js";
type RemoteNodeRecord = {
nodeId: string;
@@ -22,12 +32,14 @@ type RemoteNodeRecord = {
deviceFamily?: string;
commands?: string[];
bins: Set<string>;
pairingGeneration?: string;
connected: boolean;
remoteIp?: string;
};
type RemoteNodeProbeState = {
signature: string;
pairingGeneration: string;
nextProbeAfterMs: number;
failedProbeCount: number;
bins?: Set<string>;
@@ -36,8 +48,11 @@ type RemoteNodeProbeState = {
const log = createSubsystemLogger("gateway/skills-remote");
const remoteNodes = new Map<string, RemoteNodeRecord>();
const remoteNodeProbeStates = new Map<string, RemoteNodeProbeState>();
type RemoteBinProbeInflight = {
type RemoteNodeOwner = {
connId?: string;
pairingGeneration: string;
};
type RemoteBinProbeInflight = RemoteNodeOwner & {
promise: Promise<void>;
};
@@ -55,35 +70,6 @@ function describeNode(nodeId: string): string {
return ip ? `${base} @ ${ip}` : base;
}
function extractErrorMessage(err: unknown): string | undefined {
if (!err) {
return undefined;
}
if (typeof err === "string") {
return err;
}
if (err instanceof Error) {
return err.message;
}
if (typeof err === "object" && "message" in err && typeof err.message === "string") {
return err.message;
}
if (typeof err === "number" || typeof err === "boolean" || typeof err === "bigint") {
return String(err);
}
if (typeof err === "symbol") {
return err.toString();
}
if (typeof err === "object") {
try {
return JSON.stringify(err);
} catch {
return undefined;
}
}
return undefined;
}
type RemoteBinProbeLogContext = {
command?: string;
timeoutMs?: number;
@@ -138,42 +124,33 @@ function logRemoteBinProbeFailure(
log.warn(`remote bin probe error (${label}; ${details}): ${message ?? "unknown"}`);
}
function isMacPlatform(platform?: string, deviceFamily?: string): boolean {
const platformNorm = normalizeLowercaseStringOrEmpty(platform);
const familyNorm = normalizeLowercaseStringOrEmpty(deviceFamily);
if (platformNorm.includes("mac")) {
return true;
}
if (platformNorm.includes("darwin")) {
return true;
}
if (familyNorm === "mac") {
return true;
}
return false;
}
function supportsSystemRun(commands?: string[]): boolean {
return Array.isArray(commands) && commands.includes("system.run");
}
function supportsSystemWhich(commands?: string[]): boolean {
return Array.isArray(commands) && commands.includes("system.which");
}
function upsertNode(record: {
nodeId: string;
connId?: string;
displayName?: string;
platform?: string;
deviceFamily?: string;
commands?: string[];
remoteIp?: string;
bins?: string[];
connected?: boolean;
}) {
function upsertNode(
record: {
nodeId: string;
connId?: string;
displayName?: string;
platform?: string;
deviceFamily?: string;
commands?: string[];
remoteIp?: string;
bins?: string[];
pairingGeneration?: string;
connected?: boolean;
},
options?: { pairingGenerationAuthoritative?: boolean },
) {
const existing = remoteNodes.get(record.nodeId);
const bins = new Set<string>(record.bins ?? existing?.bins ?? []);
const pairingGeneration = options?.pairingGenerationAuthoritative
? record.pairingGeneration
: (record.pairingGeneration ?? existing?.pairingGeneration);
const pairingGenerationChanged = Boolean(
options?.pairingGenerationAuthoritative &&
existing &&
existing.pairingGeneration !== record.pairingGeneration,
);
const bins = new Set<string>(
record.bins ?? (pairingGenerationChanged ? [] : (existing?.bins ?? [])),
);
remoteNodes.set(record.nodeId, {
nodeId: record.nodeId,
connId: record.connId ?? existing?.connId,
@@ -183,6 +160,7 @@ function upsertNode(record: {
commands: record.commands ?? existing?.commands,
remoteIp: record.remoteIp ?? existing?.remoteIp,
bins,
...(pairingGeneration ? { pairingGeneration } : {}),
connected: record.connected ?? existing?.connected ?? false,
});
}
@@ -214,44 +192,59 @@ function buildRemoteProbeSignature(params: {
function shouldSkipRemoteNodeProbe(params: {
state: RemoteNodeProbeState | undefined;
pairingGeneration: string;
signature: string;
nowMs: number;
}): boolean {
return (
params.state?.signature === params.signature && params.nowMs < params.state.nextProbeAfterMs
params.state?.pairingGeneration === params.pairingGeneration &&
params.state.signature === params.signature &&
params.nowMs < params.state.nextProbeAfterMs
);
}
function restoreCachedRemoteNodeBins(nodeId: string): boolean {
const node = remoteNodes.get(nodeId);
const cachedBins = remoteNodeProbeStates.get(nodeId)?.bins;
if (!node || !cachedBins || areBinSetsEqual(node.bins, cachedBins)) {
const state = remoteNodeProbeStates.get(nodeId);
const cachedBins = state?.bins;
if (
!node ||
state?.pairingGeneration !== node.pairingGeneration ||
!cachedBins ||
areBinSetsEqual(node.bins, cachedBins)
) {
return false;
}
node.bins = new Set(cachedBins);
return true;
}
function isCurrentRemoteNodeConnection(nodeId: string, connId?: string): boolean {
if (!connId) {
return true;
}
function sameRemoteNodeOwner(left: RemoteNodeOwner, right: RemoteNodeOwner): boolean {
return left.connId === right.connId && left.pairingGeneration === right.pairingGeneration;
}
function isCurrentRemoteNodeOwner(nodeId: string, owner: RemoteNodeOwner): boolean {
const current = remoteNodes.get(nodeId);
return Boolean(current && (!current.connId || current.connId === connId));
return Boolean(
current &&
current.pairingGeneration === owner.pairingGeneration &&
(!owner.connId || !current.connId || current.connId === owner.connId),
);
}
function markRemoteNodeProbeSuccess(params: {
nodeId: string;
connId?: string;
owner: RemoteNodeOwner;
signature: string;
nowMs: number;
bins: string[];
}): boolean {
if (!isCurrentRemoteNodeConnection(params.nodeId, params.connId)) {
if (!isCurrentRemoteNodeOwner(params.nodeId, params.owner)) {
return false;
}
remoteNodeProbeStates.set(params.nodeId, {
signature: params.signature,
pairingGeneration: params.owner.pairingGeneration,
nextProbeAfterMs: params.nowMs + REMOTE_BIN_PROBE_SUCCESS_TTL_MS,
failedProbeCount: 0,
bins: new Set(params.bins),
@@ -261,11 +254,11 @@ function markRemoteNodeProbeSuccess(params: {
function markRemoteNodeProbeFailure(params: {
nodeId: string;
connId?: string;
owner: RemoteNodeOwner;
signature: string;
nowMs: number;
}): boolean {
if (!isCurrentRemoteNodeConnection(params.nodeId, params.connId)) {
if (!isCurrentRemoteNodeOwner(params.nodeId, params.owner)) {
return false;
}
const existing = remoteNodeProbeStates.get(params.nodeId);
@@ -277,14 +270,33 @@ function markRemoteNodeProbeFailure(params: {
);
remoteNodeProbeStates.set(params.nodeId, {
signature: params.signature,
pairingGeneration: params.owner.pairingGeneration,
nextProbeAfterMs: params.nowMs + backoffMs,
failedProbeCount,
});
return true;
}
function remoteConnectionKey(nodeId: string, connId: string): string {
return `${nodeId}\0${connId}`;
}
function listCurrentRemoteSessions(): NodeSession[] {
return remoteRegistry?.listCurrentConnectedSync() ?? [];
}
function listCurrentRemoteConnectionKeys(): ReadonlySet<string> | undefined {
if (!remoteRegistry) {
return undefined;
}
return new Set(
listCurrentRemoteSessions().map((node) => remoteConnectionKey(node.nodeId, node.connId)),
);
}
export function setSkillsRemoteRegistry(registry: NodeRegistry | null) {
remoteRegistry = registry;
setRemoteSkillConnectionReconciler(registry ? () => listCurrentRemoteConnectionKeys() : null);
if (!registry) {
remoteNodeProbeStates.clear();
}
@@ -292,19 +304,26 @@ export function setSkillsRemoteRegistry(registry: NodeRegistry | null) {
export async function primeRemoteSkillsCache() {
try {
const list = await listNodePairing();
const { paired } = await listNodePairing(undefined, { includePairingGeneration: true });
let sawMac = false;
for (const node of list.paired) {
upsertNode({
nodeId: node.nodeId,
displayName: node.displayName,
platform: node.platform,
deviceFamily: node.deviceFamily,
commands: node.commands,
remoteIp: node.remoteIp,
bins: node.bins,
connected: false,
});
for (const node of paired) {
if (!node.pairingGeneration) {
continue;
}
upsertNode(
{
nodeId: node.nodeId,
displayName: node.displayName,
platform: node.platform,
deviceFamily: node.deviceFamily,
commands: node.commands,
remoteIp: node.remoteIp,
bins: node.bins,
pairingGeneration: node.pairingGeneration,
connected: false,
},
{ pairingGenerationAuthoritative: true },
);
if (
node.bins &&
node.bins.length > 0 &&
@@ -330,16 +349,21 @@ export function recordRemoteNodeInfo(node: {
deviceFamily?: string;
commands?: string[];
remoteIp?: string;
pairingGeneration?: string;
}) {
const existing = remoteNodes.get(node.nodeId);
const pairingGenerationChanged = Boolean(
existing && existing.pairingGeneration !== node.pairingGeneration,
);
if (
node.connId &&
existing?.connId !== node.connId &&
!remoteNodeProbeStates.get(node.nodeId)?.bins
pairingGenerationChanged ||
(node.connId &&
existing?.connId !== node.connId &&
!remoteNodeProbeStates.get(node.nodeId)?.bins)
) {
remoteNodeProbeStates.delete(node.nodeId);
}
upsertNode({ ...node, connected: true });
upsertNode({ ...node, connected: true }, { pairingGenerationAuthoritative: true });
recordRemoteSkillNodeInfo({
nodeId: node.nodeId,
connId: node.connId,
@@ -348,8 +372,8 @@ export function recordRemoteNodeInfo(node: {
});
}
export function recordRemoteNodeBins(nodeId: string, bins: string[]) {
upsertNode({ nodeId, bins });
export function recordRemoteNodeBins(nodeId: string, bins: string[], pairingGeneration: string) {
upsertNode({ nodeId, bins, pairingGeneration });
}
export function removeRemoteNodeInfo(nodeId: string) {
@@ -371,75 +395,12 @@ export function removeRemoteNodeInfo(nodeId: string) {
}
}
function collectRequiredBins(entries: SkillEntry[], targetPlatform: string): string[] {
const bins = new Set<string>();
for (const entry of entries) {
const os = entry.metadata?.os ?? [];
if (os.length > 0 && !os.includes(targetPlatform)) {
continue;
}
const required = entry.metadata?.requires?.bins ?? [];
const anyBins = entry.metadata?.requires?.anyBins ?? [];
for (const bin of required) {
if (bin.trim()) {
bins.add(bin.trim());
}
}
for (const bin of anyBins) {
if (bin.trim()) {
bins.add(bin.trim());
}
}
}
return [...bins];
}
function buildBinProbeScript(bins: string[]): string {
const escaped = bins.map((bin) => `'${bin.replace(/'/g, `'\\''`)}'`).join(" ");
return `for b in ${escaped}; do if command -v "$b" >/dev/null 2>&1; then echo "$b"; fi; done`;
}
function parseBinProbePayload(payloadJSON: string | null | undefined, payload?: unknown): string[] {
if (!payloadJSON && !payload) {
return [];
}
try {
const parsed = payloadJSON
? (JSON.parse(payloadJSON) as { stdout?: unknown; bins?: unknown })
: (payload as { stdout?: unknown; bins?: unknown });
if (Array.isArray(parsed.bins)) {
return normalizeStringEntries(parsed.bins);
}
if (parsed.bins && typeof parsed.bins === "object") {
return Object.entries(parsed.bins)
.filter(([, resolvedPath]) => normalizeOptionalString(resolvedPath) !== undefined)
.map(([bin]) => normalizeOptionalString(bin) ?? "")
.filter(Boolean);
}
if (typeof parsed.stdout === "string") {
return parsed.stdout
.split(/\r?\n/)
.map((line) => normalizeOptionalString(line) ?? "")
.filter(Boolean);
}
} catch {
return [];
}
return [];
}
function areBinSetsEqual(a: Set<string> | undefined, b: Set<string>): boolean {
if (!a) {
/** Remove remote projections only while they still belong to the invalidated connection. */
export function removeRemoteNodeInfoForConnection(nodeId: string, connId: string): boolean {
if (remoteNodes.get(nodeId)?.connId !== connId) {
return false;
}
if (a.size !== b.size) {
return false;
}
for (const bin of b) {
if (!a.has(bin)) {
return false;
}
}
removeRemoteNodeInfo(nodeId);
return true;
}
@@ -452,16 +413,23 @@ export async function refreshRemoteNodeBins(params: {
timeoutMs?: number;
readinessDelayMs?: number;
}) {
const connId = remoteRegistry?.get(params.nodeId)?.connId;
const session = remoteRegistry?.get(params.nodeId);
if (!session?.pairingGeneration) {
return;
}
const owner: RemoteNodeOwner = {
connId: session.connId,
pairingGeneration: session.pairingGeneration,
};
const existing = remoteBinProbeInflight.get(params.nodeId);
if (existing) {
await existing.promise;
if (existing.connId === connId) {
if (sameRemoteNodeOwner(existing, owner)) {
return;
}
}
const inflight: RemoteBinProbeInflight = {
connId,
...owner,
promise: Promise.resolve(),
};
const run = refreshRemoteNodeBinsUncoalesced(params).finally(() => {
@@ -495,7 +463,13 @@ async function refreshRemoteNodeBinsUncoalesced(params: {
// Pairing can replace the command surface while the connect-time readiness
// delay is pending. Probe the live session so that approval refresh is not lost.
const liveSession = remoteRegistry.get(params.nodeId);
const probeConnId = liveSession?.connId;
if (!liveSession?.pairingGeneration) {
return;
}
const probeOwner: RemoteNodeOwner = {
connId: liveSession.connId,
pairingGeneration: liveSession.pairingGeneration,
};
const platform = liveSession?.platform ?? params.platform;
const deviceFamily = liveSession?.deviceFamily ?? params.deviceFamily;
const commands = liveSession?.commands ?? params.commands;
@@ -534,6 +508,7 @@ async function refreshRemoteNodeBinsUncoalesced(params: {
if (
shouldSkipRemoteNodeProbe({
state: remoteNodeProbeStates.get(params.nodeId),
pairingGeneration: probeOwner.pairingGeneration,
signature: probeSignature,
nowMs,
})
@@ -553,7 +528,7 @@ async function refreshRemoteNodeBinsUncoalesced(params: {
} catch (err) {
const recorded = markRemoteNodeProbeFailure({
nodeId: params.nodeId,
connId: probeConnId,
owner: probeOwner,
signature: probeSignature,
nowMs: Date.now(),
});
@@ -591,7 +566,7 @@ async function refreshRemoteNodeBinsUncoalesced(params: {
}
const recorded = markRemoteNodeProbeFailure({
nodeId: params.nodeId,
connId: probeConnId,
owner: probeOwner,
signature: probeSignature,
nowMs: Date.now(),
});
@@ -620,12 +595,14 @@ async function refreshRemoteNodeBinsUncoalesced(params: {
canWhich
? {
nodeId: params.nodeId,
expectedPairingGeneration: probeOwner.pairingGeneration,
command,
params: { bins: binsList },
timeoutMs,
}
: {
nodeId: params.nodeId,
expectedPairingGeneration: probeOwner.pairingGeneration,
command,
params: {
command: ["/bin/sh", "-lc", buildBinProbeScript(binsList)],
@@ -636,7 +613,7 @@ async function refreshRemoteNodeBinsUncoalesced(params: {
if (!res.ok) {
const recorded = markRemoteNodeProbeFailure({
nodeId: params.nodeId,
connId: probeConnId,
owner: probeOwner,
signature: probeSignature,
nowMs: Date.now(),
});
@@ -651,9 +628,24 @@ async function refreshRemoteNodeBinsUncoalesced(params: {
return;
}
const bins = parseBinProbePayload(res.payloadJSON, res.payload);
if (!isCurrentRemoteNodeOwner(params.nodeId, probeOwner)) {
return;
}
const existingBins = remoteNodes.get(params.nodeId)?.bins;
const nextBins = new Set(bins);
const hasChanged = !areBinSetsEqual(existingBins, nextBins);
if (hasChanged) {
const persisted = await updatePairedNodeBins(params.nodeId, bins, {
nodeId: params.nodeId,
key: probeOwner.pairingGeneration,
});
if (!persisted) {
return;
}
}
const recorded = markRemoteNodeProbeSuccess({
nodeId: params.nodeId,
connId: probeConnId,
owner: probeOwner,
signature: probeSignature,
nowMs: Date.now(),
bins,
@@ -661,19 +653,14 @@ async function refreshRemoteNodeBinsUncoalesced(params: {
if (!recorded) {
return;
}
const existingBins = remoteNodes.get(params.nodeId)?.bins;
const nextBins = new Set(bins);
const hasChanged = !areBinSetsEqual(existingBins, nextBins);
recordRemoteNodeBins(params.nodeId, bins);
if (!hasChanged) {
return;
recordRemoteNodeBins(params.nodeId, bins, probeOwner.pairingGeneration);
if (hasChanged) {
bumpSkillsSnapshotVersion({ reason: "remote-node" });
}
await updatePairedNodeBins(params.nodeId, bins);
bumpSkillsSnapshotVersion({ reason: "remote-node" });
} catch (err) {
const recorded = markRemoteNodeProbeFailure({
nodeId: params.nodeId,
connId: probeConnId,
owner: probeOwner,
signature: probeSignature,
nowMs: Date.now(),
});
@@ -691,9 +678,13 @@ async function refreshRemoteNodeBinsUncoalesced(params: {
export function getRemoteSkillEligibility(options?: {
advertiseExecNode?: boolean;
}): SkillEligibilityContext["remote"] | undefined {
const currentConnections = listCurrentRemoteConnectionKeys();
const macNodes = [...remoteNodes.values()].filter(
(node) =>
node.connected &&
(!currentConnections ||
(node.connId !== undefined &&
currentConnections.has(remoteConnectionKey(node.nodeId, node.connId)))) &&
isMacPlatform(node.platform, node.deviceFamily) &&
supportsSystemRun(node.commands),
);
@@ -725,7 +716,7 @@ export async function refreshRemoteBinsForConnectedNodes(cfg: OpenClawConfig) {
if (!remoteRegistry) {
return;
}
const connected = remoteRegistry.listConnected();
const connected = listCurrentRemoteSessions();
for (const node of connected) {
try {
await refreshRemoteNodeBins({