diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 019ffc493ff1..0f0d2a90176e 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -24,7 +24,7 @@ "@modelcontextprotocol/sdk": "1.29.0", "@mozilla/readability": "0.6.0", "@openclaw/fs-safe": "0.4.4", - "@openclaw/proxyline": "0.3.3", + "@openclaw/proxyline": "0.3.4", "@silvia-odwyer/photon-node": "0.3.4", "chalk": "5.6.2", "chokidar": "5.0.0", @@ -447,15 +447,15 @@ } }, "node_modules/@openclaw/proxyline": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/@openclaw/proxyline/-/proxyline-0.3.3.tgz", - "integrity": "sha512-sftHnW69NHQqLjCxBTvQ8f/eQl+peZ5pHCBQtuTWBbeuYRHZ0/GXVTmw/O/YKsShMbqPWhJB0UYtPPdvCUSS8w==", + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@openclaw/proxyline/-/proxyline-0.3.4.tgz", + "integrity": "sha512-WuLE282PW7vLienDNFTjdSI/BCYDAepvr5siOTYABQhqFo8qbP9MgmJzZnwmskUqiYugGzle6gvaRbJTkwbZ8A==", "license": "MIT", "engines": { "node": ">=22.19.0" }, "peerDependencies": { - "undici": ">=8.3.0 <9" + "undici": ">=8.5.0 <9" } }, "node_modules/@opentelemetry/semantic-conventions": { diff --git a/package.json b/package.json index 43f54643e2c3..acefdca04701 100644 --- a/package.json +++ b/package.json @@ -1895,7 +1895,7 @@ "@mozilla/readability": "0.6.0", "@openclaw/ai": "workspace:*", "@openclaw/fs-safe": "0.4.4", - "@openclaw/proxyline": "0.3.3", + "@openclaw/proxyline": "0.3.4", "@silvia-odwyer/photon-node": "0.3.4", "chalk": "5.6.2", "chokidar": "5.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b93f72a0e9ef..c8ccdcd212df 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -86,8 +86,8 @@ importers: specifier: 0.4.4 version: 0.4.4 '@openclaw/proxyline': - specifier: 0.3.3 - version: 0.3.3(undici@8.5.0) + specifier: 0.3.4 + version: 0.3.4(undici@8.5.0) '@silvia-odwyer/photon-node': specifier: 0.3.4 version: 0.3.4 @@ -3712,11 +3712,11 @@ packages: node-pty: optional: true - '@openclaw/proxyline@0.3.3': - resolution: {integrity: sha512-sftHnW69NHQqLjCxBTvQ8f/eQl+peZ5pHCBQtuTWBbeuYRHZ0/GXVTmw/O/YKsShMbqPWhJB0UYtPPdvCUSS8w==} + '@openclaw/proxyline@0.3.4': + resolution: {integrity: sha512-WuLE282PW7vLienDNFTjdSI/BCYDAepvr5siOTYABQhqFo8qbP9MgmJzZnwmskUqiYugGzle6gvaRbJTkwbZ8A==} engines: {node: '>=22.19.0'} peerDependencies: - undici: '>=8.3.0 <9' + undici: '>=8.5.0 <9' '@openclaw/uirouter@0.1.0': resolution: {integrity: sha512-w5tNj2FIukVJqJ1wt5wiDbrI6DI4tOkUbtqnnU5Fl4EgnRKFJtfHI3WrWTWTTJlXbbrwGSMptyrSZmQVdRo83Q==} @@ -10087,7 +10087,7 @@ snapshots: dependencies: ghostty-web: 0.4.0 - '@openclaw/proxyline@0.3.3(undici@8.5.0)': + '@openclaw/proxyline@0.3.4(undici@8.5.0)': dependencies: undici: 8.5.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 14af08a56122..220eb0adf8f7 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -11,7 +11,7 @@ minimumReleaseAgeExclude: - "@openclaw/crabline@0.1.11" - "@openclaw/fs-safe@0.4.4" - "@openclaw/libterminal@0.3.2" - - "@openclaw/proxyline@0.3.3" + - "@openclaw/proxyline@0.3.4" - "@openclaw/uirouter@0.1.0" - "acpx" - "tokenjuice" diff --git a/src/agents/embedded-agent-runner/compact.ts b/src/agents/embedded-agent-runner/compact.ts index be52a05d3a49..13d61dce2fcc 100644 --- a/src/agents/embedded-agent-runner/compact.ts +++ b/src/agents/embedded-agent-runner/compact.ts @@ -19,7 +19,7 @@ import { } from "../../gateway/session-compaction-checkpoints.js"; import { formatActiveNodeContextLabel, - getActiveNodeContext, + getCurrentActiveNodeContext, } from "../../infra/active-node-context.js"; import { resolveDiagnosticModelContentCapturePolicy } from "../../infra/diagnostic-llm-content.js"; import { @@ -1317,7 +1317,7 @@ async function compactEmbeddedAgentSessionDirectOnce( agentId: sessionAgentId, }), }), - activeNode: formatActiveNodeContextLabel(getActiveNodeContext()), + activeNode: formatActiveNodeContextLabel(getCurrentActiveNodeContext()), }; const sandboxInfoExecPolicy = resolveEmbeddedSandboxInfoExecPolicy({ config: params.config, diff --git a/src/agents/system-prompt-params.test.ts b/src/agents/system-prompt-params.test.ts index f3d919a91694..a8f38d176e57 100644 --- a/src/agents/system-prompt-params.test.ts +++ b/src/agents/system-prompt-params.test.ts @@ -42,6 +42,17 @@ describe("buildSystemPromptParams", () => { expect(runtimeInfo.activeNode).toBe("mac-123"); }); + it("omits an active node that fails current-generation validation", () => { + setActiveNodeContext( + { nodeId: "mac-123", pairingGeneration: "generation-a" }, + { isCurrent: () => false }, + ); + + const { runtimeInfo } = buildParams({}); + + expect(runtimeInfo.activeNode).toBeUndefined(); + }); + it("detects repo root from workspaceDir", async () => { const temp = await makeTempDir("workspace"); const repoRoot = path.join(temp, "repo"); diff --git a/src/agents/system-prompt-params.ts b/src/agents/system-prompt-params.ts index 678c5c4a6e0d..92490247e622 100644 --- a/src/agents/system-prompt-params.ts +++ b/src/agents/system-prompt-params.ts @@ -10,7 +10,7 @@ import type { ChatType } from "../channels/chat-type.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { formatActiveNodeContextLabel, - getActiveNodeContext, + getCurrentActiveNodeContext, } from "../infra/active-node-context.js"; import { findGitRoot } from "../infra/git-root.js"; import type { ActiveProcessSessionReference } from "./bash-process-references.js"; @@ -68,7 +68,8 @@ export function buildSystemPromptParams(params: { runtimeInfo: { agentId: params.agentId, ...params.runtime, - activeNode: formatActiveNodeContextLabel(getActiveNodeContext()) ?? params.runtime.activeNode, + activeNode: + formatActiveNodeContextLabel(getCurrentActiveNodeContext()) ?? params.runtime.activeNode, repoRoot, }, userTimezone, diff --git a/src/gateway/device-pairing-prune.test.ts b/src/gateway/device-pairing-prune.test.ts index 2ca8e128ca97..a503b17d0e3f 100644 --- a/src/gateway/device-pairing-prune.test.ts +++ b/src/gateway/device-pairing-prune.test.ts @@ -1,13 +1,26 @@ // Covers gateway-side cleanup when silent pairing supersedes stale sibling records. -import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { afterAll, afterEach, beforeAll, describe, expect, test } from "vitest"; import { approveDevicePairing, listDevicePairing, + removePairedDeviceRole, requestDevicePairing, } from "../infra/device-pairing.js"; import { approveNodePairing, listNodePairing, requestNodePairing } from "../infra/node-pairing.js"; +import { loadApnsRegistration, registerApnsRegistration } from "../infra/push-apns.js"; import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js"; import { pruneSupersededSilentPairingsAfterApproval } from "./device-pairing-prune.js"; +import { drainNodePendingWork, enqueueNodePendingWork } from "./node-pending-work.js"; +import { enqueuePendingNodeAction, listPendingNodeActions } from "./node-runtime-state.js"; +import { + captureNodeWakeLifecycle, + runNodeWakeAttempt, + runNodeWakeNudgeAttempt, +} from "./node-wake-state.js"; +import { + getNodeWakeStateSnapshot, + resetNodeWakeStateForTest, +} from "./node-wake-state.test-support.js"; const suiteRootTracker = createSuiteTempRootTracker({ prefix: "openclaw-gateway-pairing-prune-" }); @@ -19,6 +32,8 @@ function createPruneContext(params?: { connectedDeviceIds?: string[] }) { const invalidated: string[] = []; const disconnected: string[] = []; const logs: string[] = []; + const warnings: string[] = []; + const clearedSurfaces: string[] = []; const connected = new Set(params?.connectedDeviceIds ?? []); const context: PruneContext = { broadcast: (event, payload) => { @@ -26,6 +41,7 @@ function createPruneContext(params?: { connectedDeviceIds?: string[] }) { }, logGateway: { info: (message: string) => logs.push(message), + warn: (message: string) => warnings.push(message), }, hasConnectedClientsForDevice: (deviceId: string) => connected.has(deviceId), invalidateClientsForDevice: (deviceId: string) => { @@ -34,8 +50,14 @@ function createPruneContext(params?: { connectedDeviceIds?: string[] }) { disconnectClientsForDevice: (deviceId: string) => { disconnected.push(deviceId); }, + nodeRegistry: { + updateSurface: (nodeId: string) => { + clearedSurfaces.push(nodeId); + return null; + }, + }, }; - return { broadcasts, invalidated, disconnected, logs, context }; + return { broadcasts, invalidated, disconnected, logs, warnings, clearedSurfaces, context }; } async function pairSilentDevice(params: { @@ -79,6 +101,10 @@ describe("pruneSupersededSilentPairingsAfterApproval", () => { await suiteRootTracker.cleanup(); }); + afterEach(() => { + resetNodeWakeStateForTest(); + }); + test("retires stale node siblings across both pairing stores", async () => { const baseDir = await suiteRootTracker.make("case"); await pairSilentDevice({ @@ -102,6 +128,39 @@ describe("pruneSupersededSilentPairingsAfterApproval", () => { clientMode: "node", displayName: "megaclaw", }); + await registerApnsRegistration({ + nodeId: "node-stale", + transport: "direct", + token: "ABCD1234ABCD1234ABCD1234ABCD1234", + topic: "ai.openclaw.ios", + environment: "sandbox", + baseDir, + }); + await runNodeWakeAttempt({ + nodeId: "node-stale", + force: true, + throttleMs: 60_000, + attempt: async (markAttempted) => { + markAttempted(); + return { available: true, throttled: false, path: "sent", durationMs: 1 }; + }, + }); + await runNodeWakeNudgeAttempt({ + nodeId: "node-stale", + throttleMs: 60_000, + throttled: () => ({ sent: false, throttled: true, reason: "throttled", durationMs: 0 }), + attempt: async () => ({ sent: true, throttled: false, reason: "sent", durationMs: 1 }), + }); + enqueueNodePendingWork({ nodeId: "node-stale", type: "location.request" }); + enqueuePendingNodeAction({ + nodeId: "node-stale", + pairingGeneration: "generation-1", + command: "camera.capture", + idempotencyKey: "idem-1", + ttlMs: 60_000, + maxPerNode: 10, + }); + const wakeLifecycle = captureNodeWakeLifecycle("node-stale"); const harness = createPruneContext(); const pruned = await pruneSupersededSilentPairingsAfterApproval({ @@ -116,8 +175,15 @@ describe("pruneSupersededSilentPairingsAfterApproval", () => { expect(devices.paired.map((device) => device.deviceId)).toEqual(["node-anchor"]); const nodes = await listNodePairing(baseDir); expect(nodes.paired).toHaveLength(0); + expect(getNodeWakeStateSnapshot("node-stale")).toBeUndefined(); + expect(wakeLifecycle.aborted).toBe(true); + expect(drainNodePendingWork("node-stale", { includeDefaultStatus: false }).items).toEqual([]); + expect(listPendingNodeActions({ nodeId: "node-stale", ttlMs: 60_000 })).toEqual([]); + await expect(loadApnsRegistration("node-stale", baseDir)).resolves.toBeNull(); expect(harness.invalidated).toEqual(["node-stale"]); expect(harness.disconnected).toEqual(["node-stale"]); + expect(harness.clearedSurfaces).toEqual(["node-stale"]); + expect(harness.warnings).toEqual([]); expect(harness.broadcasts).toEqual([ { event: "node.pair.resolved", @@ -126,15 +192,29 @@ describe("pruneSupersededSilentPairingsAfterApproval", () => { ]); }); - test("keeps connected siblings and emits no node broadcast for operator-only prunes", async () => { + test("keeps connected siblings and clears APNs for operator-only full prunes", async () => { const baseDir = await suiteRootTracker.make("case"); await pairSilentDevice({ baseDir, deviceId: "cli-stale", - roles: ["operator"], + roles: ["operator", "node"], clientId: "cli", clientMode: "cli", }); + await registerApnsRegistration({ + nodeId: "cli-stale", + transport: "direct", + token: "ABCD1234ABCD1234ABCD1234ABCD1234", + topic: "ai.openclaw.ios", + environment: "sandbox", + baseDir, + }); + await expect( + removePairedDeviceRole({ deviceId: "cli-stale", role: "node", baseDir }), + ).resolves.toEqual({ deviceId: "cli-stale", role: "node", removedDevice: false }); + await expect(loadApnsRegistration("cli-stale", baseDir)).resolves.toMatchObject({ + nodeId: "cli-stale", + }); await pairSilentDevice({ baseDir, deviceId: "cli-live", @@ -164,6 +244,7 @@ describe("pruneSupersededSilentPairingsAfterApproval", () => { "cli-anchor", "cli-live", ]); + await expect(loadApnsRegistration("cli-stale", baseDir)).resolves.toBeNull(); expect(harness.broadcasts).toEqual([]); expect(harness.disconnected).toEqual(["cli-stale"]); }); diff --git a/src/gateway/device-pairing-prune.ts b/src/gateway/device-pairing-prune.ts index 1e056d5101bc..dddd5b762fd5 100644 --- a/src/gateway/device-pairing-prune.ts +++ b/src/gateway/device-pairing-prune.ts @@ -3,6 +3,7 @@ import { pruneSupersededSilentPairedDevices, type PrunedSupersededPairedDevice, } from "../infra/device-pairing.js"; +import { clearRemovedNodeRuntimeState } from "./node-runtime-state.js"; import type { GatewayRequestContext } from "./server-methods/types.js"; type PruneContext = Pick< @@ -12,7 +13,8 @@ type PruneContext = Pick< | "invalidateClientsForDevice" | "disconnectClientsForDevice" > & { - logGateway: Pick; + logGateway: Pick; + nodeRegistry: Pick; }; /** @@ -38,13 +40,14 @@ export async function pruneSupersededSilentPairingsAfterApproval(params: { context.logGateway.info( `device pairing pruned superseded silent pairing device=${entry.deviceId} roles=${entry.roles.join(",") || "none"}`, ); + if (entry.roles.includes("node")) { + // Persistent pruning is a node removal owner too. Clear disconnected + // queues, wake lifecycles, and runtime metadata before session teardown. + clearRemovedNodeRuntimeState({ nodeId: entry.deviceId, context }); + } // Invalidate before disconnect so buffered frames from a racing reconnect // fail authorization, mirroring device.pair.remove ordering. context.invalidateClientsForDevice?.(entry.deviceId, { reason: "device-pair-removed" }); - // The node surface lives on the pruned device record, so dropping the - // record retired it too; tell node list consumers. Pruned devices are - // offline (connected ones are skipped), so there is no live node session - // or queued action state to clear. if (entry.roles.includes("node")) { context.broadcast( "node.pair.resolved", diff --git a/src/gateway/gateway-misc.test.ts b/src/gateway/gateway-misc.test.ts index d5b489fab5e7..356591ae271f 100644 --- a/src/gateway/gateway-misc.test.ts +++ b/src/gateway/gateway-misc.test.ts @@ -4,7 +4,6 @@ import * as fs from "node:fs/promises"; import type { IncomingMessage, ServerResponse } from "node:http"; import * as os from "node:os"; import * as path from "node:path"; -import { expectDefined } from "@openclaw/normalization-core"; import { beforeAll, beforeEach, describe, expect, it, test, vi } from "vitest"; import { GATEWAY_CLIENT_CAPS, @@ -27,19 +26,13 @@ import { DEFAULT_DANGEROUS_NODE_COMMANDS, resolveNodeCommandAllowlist, } from "./node-command-policy.js"; -import type { SerializedEventPayload } from "./node-registry.js"; import { createGatewayBroadcaster } from "./server-broadcast.js"; -import { - createSessionEventSubscriberRegistry, - createSessionMessageSubscriberRegistry, -} from "./server-chat-state.js"; +import { createSessionMessageSubscriberRegistry } from "./server-chat-state.js"; import { createChatRunRegistry } from "./server-chat.js"; import { MAX_BUFFERED_BYTES } from "./server-constants.js"; import { handleNodeInvokeResult } from "./server-methods/nodes.handlers.invoke-result.js"; import type { GatewayClient as GatewayMethodClient } from "./server-methods/types.js"; import type { GatewayRequestContext, RespondFn } from "./server-methods/types.js"; -import { createGatewayNodeSessionRuntime } from "./server-node-session-runtime.js"; -import { createNodeSubscriptionManager } from "./server-node-subscriptions.js"; import { formatError, normalizeVoiceWakeTriggers } from "./server-utils.js"; import type { GatewayWsClient } from "./server/ws-types.js"; @@ -163,6 +156,16 @@ describe("GatewayClient", () => { expect(opts?.maxPayload).toBe(25 * 1024 * 1024); }); + test("uses the admitted pairing identity for shared-auth connect failures", async () => { + const source = await fs.readFile( + new URL("./server/ws-connection/connect-session.ts", import.meta.url), + "utf8", + ); + + expect(source).toContain("deviceId: admittedNodePairing.identity.nodeId"); + expect(source).not.toContain("deviceId: authenticatedNodePairing.nodeId"); + }); + test("does not pass an explicit direct agent for loopback control-plane WebSocket connections", () => { expectNoGatewayClientAgent({ url: "ws://127.0.0.1:1" }); }); @@ -421,6 +424,20 @@ describe("gateway broadcaster", () => { expect(workerSocket.send).not.toHaveBeenCalled(); }); + it("skips locally invalidated clients before generic broadcast delivery", () => { + const socket = makeRecordingSocket(); + const client = makeOperatorWsClient("c-invalidated", socket, ["operator.read"]); + client.invalidated = true; + const { broadcast, broadcastToConnIds } = createGatewayBroadcaster({ + clients: new Set([client]), + }); + + broadcast("heartbeat", { ts: 1 }); + broadcastToConnIds("heartbeat", { ts: 2 }, new Set([client.connId])); + + expect(socket.send).not.toHaveBeenCalled(); + }); + it("delivers scoped client events only for gateway-owned session subscriptions", () => { const legacySocket = makeRecordingSocket(); const firstSocket = makeRecordingSocket(); @@ -677,8 +694,6 @@ describe("gateway broadcaster", () => { "update.available", ]); expectSentEvents(nodeSocket, [ - "voicewake.changed", - "voicewake.routing.changed", "heartbeat", "presence", "health", @@ -892,88 +907,6 @@ describe("late-arriving invoke results", () => { }); }); -describe("node subscription manager", () => { - test("routes events to subscribed nodes", () => { - const manager = createNodeSubscriptionManager(); - const sent: Array<{ - nodeId: string; - event: string; - payloadJSON?: SerializedEventPayload | null; - }> = []; - const sendEvent = (evt: { - nodeId: string; - event: string; - payloadJSON?: SerializedEventPayload | null; - }) => sent.push(evt); - - manager.subscribe("node-a", "main"); - manager.subscribe("node-b", "main"); - manager.sendToSession("main", "chat", { ok: true }, sendEvent); - - expect(sent).toHaveLength(2); - expect(sent.map((s) => s.nodeId).toSorted()).toEqual(["node-a", "node-b"]); - expect(expectDefined(sent[0], "sent[0] test invariant").event).toBe("chat"); - }); - - test("runtime forwards subscribed node payload json without parsing it again", () => { - const frames: string[] = []; - const socket: TestSocket = { - bufferedAmount: 0, - send: vi.fn((payload: string) => frames.push(payload)), - close: vi.fn(), - }; - const parseSpy = vi.spyOn(JSON, "parse"); - try { - const runtime = createGatewayNodeSessionRuntime({ - broadcast: vi.fn(), - sessionEventSubscribers: createSessionEventSubscriberRegistry(), - sessionMessageSubscribers: createSessionMessageSubscriberRegistry(), - }); - runtime.nodeRegistry.register( - makeGatewayWsClient("conn-node-a", socket, { - role: "node", - scopes: [], - client: { - id: "node-client", - version: "1.0.0", - platform: "macos", - mode: "node", - }, - device: { id: "node-a" }, - } as unknown as GatewayWsClient["connect"]), - {}, - ); - runtime.nodeSubscribe("node-a", "main"); - - runtime.nodeSendToSession("main", "chat", { ok: true }); - - expect(parseSpy).not.toHaveBeenCalled(); - } finally { - parseSpy.mockRestore(); - } - expect(JSON.parse(frames[0] ?? "{}")).toEqual({ - type: "event", - event: "chat", - payload: { ok: true }, - }); - }); - - test("unsubscribeAll clears session mappings", () => { - const manager = createNodeSubscriptionManager(); - const sent: string[] = []; - const sendEvent = (evt: { nodeId: string; event: string }) => - sent.push(`${evt.nodeId}:${evt.event}`); - - manager.subscribe("node-a", "main"); - manager.subscribe("node-a", "secondary"); - manager.unsubscribeAll("node-a"); - manager.sendToSession("main", "tick", {}, sendEvent); - manager.sendToSession("secondary", "tick", {}, sendEvent); - - expect(sent).toStrictEqual([]); - }); -}); - describe("resolveNodeCommandAllowlist", () => { function expectAllowed(allow: { has: (cmd: string) => boolean }, commands: string[]) { for (const cmd of commands) { diff --git a/src/gateway/local-request-context.ts b/src/gateway/local-request-context.ts index 9e1054291cf9..2404034b50a0 100644 --- a/src/gateway/local-request-context.ts +++ b/src/gateway/local-request-context.ts @@ -154,7 +154,7 @@ function createLocalGatewayRequestContext( nodeSubscribe: () => {}, nodeUnsubscribe: () => {}, nodeUnsubscribeAll: () => {}, - hasConnectedTalkNode: () => false, + hasConnectedTalkNode: async () => false, nodeRegistry: new NodeRegistry(), agentRunSeq: new Map(), chatAbortControllers: new Map(), diff --git a/src/gateway/node-agent-cli-runtime.test.ts b/src/gateway/node-agent-cli-runtime.test.ts index 09f16d10f4ad..10443e59e239 100644 --- a/src/gateway/node-agent-cli-runtime.test.ts +++ b/src/gateway/node-agent-cli-runtime.test.ts @@ -32,6 +32,7 @@ describe("invokeNodeClaudeCliRun", () => { mocks.get.mockReturnValue({ connId: "conn-1", nodeId: "node-1", + pairingGeneration: "generation-1", commands: ["agent.cli.claude.run.v1"], }); }); @@ -75,5 +76,11 @@ describe("invokeNodeClaudeCliRun", () => { ).resolves.toEqual({ ok: true }); expect(mocks.resolveNodeCommandAllowlist).toHaveBeenCalledOnce(); expect(mocks.invoke).toHaveBeenCalledOnce(); + expect(mocks.invoke).toHaveBeenCalledWith( + expect.objectContaining({ + expectedConnId: "conn-1", + expectedPairingGeneration: "generation-1", + }), + ); }); }); diff --git a/src/gateway/node-agent-cli-runtime.ts b/src/gateway/node-agent-cli-runtime.ts index 3039b1d7ecd5..5bcf3aab9629 100644 --- a/src/gateway/node-agent-cli-runtime.ts +++ b/src/gateway/node-agent-cli-runtime.ts @@ -59,6 +59,7 @@ export async function invokeNodeClaudeCliRun(params: { return await context.nodeRegistry.invoke({ nodeId: params.nodeId, expectedConnId: node.connId, + ...(node.pairingGeneration ? { expectedPairingGeneration: node.pairingGeneration } : {}), command: NODE_AGENT_CLI_CLAUDE_RUN_COMMAND, params: { argv: params.argv, diff --git a/src/gateway/node-connection-notifications.test.ts b/src/gateway/node-connection-notifications.test.ts index ce6647cf07a3..8c7e1eb847fb 100644 --- a/src/gateway/node-connection-notifications.test.ts +++ b/src/gateway/node-connection-notifications.test.ts @@ -17,6 +17,7 @@ function node( return { nodeId, connId: `conn-${nodeId}`, + pairingIdentity: `identity-${nodeId}`, displayName: nodeId, platform: "darwin", commands: ["system.notify"], @@ -25,9 +26,17 @@ function node( } as NodeSession; } -function registry(params: T): T { - testRegistries.push(params); - return params; +function registry NodeSession[] }>(params: T): T { + const value = params as T & { + listCurrentConnected?: () => Promise; + isConnectionCurrentPairingState?: (connId: string) => Promise; + }; + const listCurrentConnected = value.listCurrentConnected ?? (async () => value.listConnected()); + value.listCurrentConnected = listCurrentConnected; + value.isConnectionCurrentPairingState ??= async (connId) => + (await listCurrentConnected()).some((entry) => entry.connId === connId); + testRegistries.push(value); + return value; } function schedule(registryValue: object, source: NodeSession): void { @@ -152,7 +161,7 @@ describe("node connection notification routing", () => { expect(invoke.mock.calls[0]?.[0]).toMatchObject({ nodeId: "desk" }); }); - it("resolves the current source session when replacement races persistence", async () => { + it("drops a source alert when its exact connection is replaced without taking ownership", async () => { vi.useFakeTimers(); const oldSource = node("new-node"); const replacement = { @@ -171,11 +180,33 @@ describe("node connection notification routing", () => { connected = [replacement, desk]; await vi.advanceTimersByTimeAsync(PRIMARY_DELAY_MS); - expect(invoke).toHaveBeenCalledTimes(1); - expect(invoke.mock.calls[0]?.[0]).toMatchObject({ - nodeId: "desk", - params: { body: "Replacement Mac connected to OpenClaw." }, + expect(invoke).not.toHaveBeenCalled(); + }); + + it("drops a delayed alert when the source pairing generation is rotated", async () => { + vi.useFakeTimers(); + const source = { + ...node("new-node"), + pairingIdentity: "identity-a", + pairingGeneration: "generation-a", + }; + const desk = node("desk", { lastActiveAtMs: 100 }); + let currentPairingGeneration = "generation-a"; + const invoke = vi.fn(async () => ({ ok: true })); + const registryValue = registry({ + listConnected: () => [source, desk], + listCurrentConnected: async () => + currentPairingGeneration === source.pairingGeneration ? [source, desk] : [desk], + isConnectionCurrentPairingState: async (connId: string) => + connId === source.connId && currentPairingGeneration === source.pairingGeneration, + invoke, }); + + schedule(registryValue, source); + currentPairingGeneration = "generation-b"; + await vi.advanceTimersByTimeAsync(PRIMARY_DELAY_MS + FALLBACK_DELAY_MS); + + expect(invoke).not.toHaveBeenCalled(); }); it("cancels the first-connection claim when the node is gone at delivery", async () => { diff --git a/src/gateway/node-connection-notifications.ts b/src/gateway/node-connection-notifications.ts index 7bd676fe73c7..b8c735835d57 100644 --- a/src/gateway/node-connection-notifications.ts +++ b/src/gateway/node-connection-notifications.ts @@ -4,7 +4,10 @@ import { normalizeOptionalString } from "@openclaw/normalization-core/string-coe import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import type { NodeRegistry, NodeSession } from "./node-registry.js"; -type NotificationRegistry = Pick; +type NotificationRegistry = Pick< + NodeRegistry, + "listCurrentConnected" | "isConnectionCurrentPairingState" | "invoke" +>; type RouterOptions = { primaryDelayMs?: number; @@ -13,6 +16,9 @@ type RouterOptions = { type PendingConnectionAlert = { nodeId: string; + connId: string; + pairingIdentity?: string; + pairingGeneration?: string; timer?: ReturnType; }; @@ -64,7 +70,12 @@ class NodeConnectionNotificationRouter { if (previous?.timer) { clearTimeout(previous.timer); } - const pending: PendingConnectionAlert = { nodeId: source.nodeId }; + const pending: PendingConnectionAlert = { + nodeId: source.nodeId, + connId: source.connId, + pairingIdentity: source.pairingIdentity, + pairingGeneration: source.pairingGeneration, + }; this.pendingByNodeId.set(source.nodeId, pending); this.armTimer(pending, this.primaryDelayMs, () => this.deliverPrimary(pending)); } @@ -79,16 +90,17 @@ class NodeConnectionNotificationRouter { } private async deliverPrimary(pending: PendingConnectionAlert): Promise { - const source = this.currentSource(pending); + const connected = await this.registry.listCurrentConnected(); + const source = this.currentSource(pending, connected); if (!source) { this.finishAlert(pending); return; } - const primary = this.notificationTargets() + const primary = this.notificationTargets(connected) .filter((node) => node.lastActiveAtMs !== undefined) .toSorted(compareActivity) .at(0); - const delivered = primary ? await this.notify(primary, source) : false; + const delivered = primary ? await this.notify(primary, source, pending) : false; if (!this.attemptIsCurrent(pending)) { return; } @@ -105,23 +117,35 @@ class NodeConnectionNotificationRouter { pending: PendingConnectionAlert, attemptedConnId?: string, ): Promise { - const source = this.currentSource(pending); + const connected = await this.registry.listCurrentConnected(); + const source = this.currentSource(pending, connected); if (!source) { this.finishAlert(pending); return; } - const targets = this.notificationTargets().filter((node) => node.connId !== attemptedConnId); - await Promise.all(targets.map(async (node) => await this.notify(node, source))); + const targets = this.notificationTargets(connected).filter( + (node) => node.connId !== attemptedConnId, + ); + await Promise.all(targets.map(async (node) => await this.notify(node, source, pending))); if (this.attemptIsCurrent(pending)) { this.finishAlert(pending); } } - private currentSource(pending: PendingConnectionAlert): NodeSession | undefined { + private currentSource( + pending: PendingConnectionAlert, + connected: readonly NodeSession[], + ): NodeSession | undefined { if (!this.attemptIsCurrent(pending)) { return undefined; } - return this.registry.listConnected().find((node) => node.nodeId === pending.nodeId); + return connected.find( + (node) => + node.nodeId === pending.nodeId && + node.connId === pending.connId && + node.pairingIdentity === pending.pairingIdentity && + node.pairingGeneration === pending.pairingGeneration, + ); } private attemptIsCurrent(pending: PendingConnectionAlert): boolean { @@ -139,15 +163,34 @@ class NodeConnectionNotificationRouter { } } - private notificationTargets(): NodeSession[] { - return this.registry.listConnected().filter(isMacNotificationNode); + private notificationTargets(connected: readonly NodeSession[]): NodeSession[] { + return connected.filter(isMacNotificationNode); } - private async notify(target: NodeSession, source: NodeSession): Promise { + private async sourceIsCurrent(pending: PendingConnectionAlert): Promise { + if (!this.attemptIsCurrent(pending)) { + return false; + } + const connected = await this.registry.listCurrentConnected(); + if (!this.currentSource(pending, connected)) { + return false; + } + return await this.registry.isConnectionCurrentPairingState(pending.connId); + } + + private async notify( + target: NodeSession, + source: NodeSession, + pending: PendingConnectionAlert, + ): Promise { try { + if (!(await this.sourceIsCurrent(pending)) || !this.attemptIsCurrent(pending)) { + return false; + } const result = await this.registry.invoke({ nodeId: target.nodeId, expectedConnId: target.connId, + expectedPairingGeneration: target.pairingGeneration, command: "system.notify", params: { title: "Node connected", diff --git a/src/gateway/node-invoke-plugin-policy.test.ts b/src/gateway/node-invoke-plugin-policy.test.ts index ea6df751ca2b..4b8b3d509a27 100644 --- a/src/gateway/node-invoke-plugin-policy.test.ts +++ b/src/gateway/node-invoke-plugin-policy.test.ts @@ -307,6 +307,53 @@ describe("applyPluginNodeInvokePolicy", () => { expect(invoke).not.toHaveBeenCalled(); }); + it("rejects plugin transport dispatch after invocation ownership changes", async () => { + setDangerousDemoCommandRegistry([ + createDemoPolicy((ctx: OpenClawPluginNodeInvokePolicyContext) => ctx.invokeNode()), + ]); + const { context, invoke } = createContext(); + + const result = await applyPluginNodeInvokePolicy({ + context, + client: null, + nodeSession: createNodeSession(), + command: DEMO_COMMAND, + params: DEMO_PARAMS, + isInvocationCurrent: async () => false, + }); + + expect(result).toMatchObject({ + ok: false, + code: "PAIRING_CHANGED", + details: { nodeCommandDispatched: false }, + }); + expect(invoke).not.toHaveBeenCalled(); + }); + + it("rejects plugin transport dispatch through an invalidated node session", async () => { + setDangerousDemoCommandRegistry([ + createDemoPolicy((ctx: OpenClawPluginNodeInvokePolicyContext) => ctx.invokeNode()), + ]); + const nodeSession = createNodeSession(); + nodeSession.client.invalidated = true; + const { context, invoke } = createContext({ nodeSession }); + + const result = await applyPluginNodeInvokePolicy({ + context, + client: null, + nodeSession, + command: DEMO_COMMAND, + params: DEMO_PARAMS, + }); + + expect(result).toMatchObject({ + ok: false, + code: "PAIRING_CHANGED", + details: { nodeCommandDispatched: false }, + }); + expect(invoke).not.toHaveBeenCalled(); + }); + it("overrides plugin dispatch claims with the actual pre-dispatch state", async () => { setDangerousDemoCommandRegistry([ createDemoPolicy(async () => ({ diff --git a/src/gateway/node-invoke-plugin-policy.ts b/src/gateway/node-invoke-plugin-policy.ts index 8c470fc8ea73..bf1b8b52e1c6 100644 --- a/src/gateway/node-invoke-plugin-policy.ts +++ b/src/gateway/node-invoke-plugin-policy.ts @@ -219,6 +219,7 @@ export async function applyPluginNodeInvokePolicy(params: { }; timeoutMs?: number; idempotencyKey?: string; + isInvocationCurrent?: () => boolean | Promise; }): Promise { const registry = getActivePluginGatewayNodePolicyRegistry(); // Route metadata is authority-bearing: only a signed agent-runtime caller may nominate it. @@ -247,7 +248,19 @@ export async function applyPluginNodeInvokePolicy(params: { ): Promise => { // Policies invoke the real node through this narrowed transport wrapper so // they can retry/override params without getting direct registry access. - const currentNode = params.context.nodeRegistry.get(params.nodeSession.nodeId); + if (params.isInvocationCurrent && !(await params.isInvocationCurrent())) { + return { + ok: false, + code: "PAIRING_CHANGED", + message: "node pairing changed before dispatch", + }; + } + const currentNode = params.nodeSession.pairingGeneration + ? params.context.nodeRegistry.getForPairingGeneration( + params.nodeSession.nodeId, + params.nodeSession.pairingGeneration, + ) + : params.context.nodeRegistry.get(params.nodeSession.nodeId); if (!currentNode || currentNode.connId !== params.nodeSession.connId) { return { ok: false, @@ -255,6 +268,13 @@ export async function applyPluginNodeInvokePolicy(params: { message: "node connection changed before dispatch", }; } + if (currentNode.client.invalidated === true) { + return { + ok: false, + code: "PAIRING_CHANGED", + message: "node pairing changed before dispatch", + }; + } const currentConfig = params.context.getRuntimeConfig(); const allowlist = resolveNodeCommandAllowlist(currentConfig, { ...currentNode, @@ -279,6 +299,9 @@ export async function applyPluginNodeInvokePolicy(params: { const res = await params.context.nodeRegistry.invoke({ nodeId: params.nodeSession.nodeId, expectedConnId: params.nodeSession.connId, + ...(params.nodeSession.pairingGeneration + ? { expectedPairingGeneration: params.nodeSession.pairingGeneration } + : {}), command: params.command, params: override.params ?? params.params, timeoutMs: override.timeoutMs ?? params.timeoutMs, diff --git a/src/gateway/node-pending-work.test.ts b/src/gateway/node-pending-work.test.ts index c6f29949be42..4e24cde109c0 100644 --- a/src/gateway/node-pending-work.test.ts +++ b/src/gateway/node-pending-work.test.ts @@ -2,7 +2,12 @@ * Node pending-work tracking tests. */ import { describe, expect, it, vi } from "vitest"; -import { drainNodePendingWork, enqueueNodePendingWork } from "./node-pending-work.js"; +import { + clearNodePendingWork, + drainNodePendingWork, + enqueueNodePendingWork, + removeNodePendingWorkItem, +} from "./node-pending-work.js"; describe("node pending work", () => { it("returns a baseline status request even when no explicit work is queued", () => { @@ -33,6 +38,97 @@ describe("node pending work", () => { drainNodePendingWork("node-2"); }); + it("clears explicit work when a pairing is removed", () => { + enqueueNodePendingWork({ nodeId: "node-removed", type: "location.request" }); + + expect(clearNodePendingWork(" node-removed ")).toBe(true); + expect(drainNodePendingWork("node-removed").items.map((item) => item.id)).toEqual([ + "baseline-status", + ]); + expect(clearNodePendingWork("node-removed")).toBe(false); + }); + + it("keeps explicit work isolated from a replacement pairing generation", () => { + enqueueNodePendingWork({ + nodeId: "node-replaced", + type: "location.request", + pairingGeneration: "generation-1", + }); + + const drained = drainNodePendingWork("node-replaced", { + pairingGeneration: "generation-2", + }); + + expect(drained.items.map((item) => item.id)).toEqual(["baseline-status"]); + expect( + drainNodePendingWork("node-replaced", { pairingGeneration: "generation-1" }).items.map( + (item) => item.type, + ), + ).toEqual(["location.request", "status.request"]); + }); + + it("prunes expired work from retired generations on current-generation access", () => { + enqueueNodePendingWork({ + nodeId: "node-retired-generation", + type: "location.request", + expiresInMs: 1_000, + pairingGeneration: "generation-1", + }); + + drainNodePendingWork("node-retired-generation", { + pairingGeneration: "generation-2", + nowMs: Date.now() + 2_000, + }); + + expect(clearNodePendingWork("node-retired-generation", "generation-1")).toBe(false); + }); + + it("does not let a stale drain delete replacement-generation work", () => { + enqueueNodePendingWork({ + nodeId: "node-stale-drain", + type: "location.request", + pairingGeneration: "generation-2", + }); + + expect( + drainNodePendingWork("node-stale-drain", { pairingGeneration: "generation-1" }).items.map( + (item) => item.id, + ), + ).toEqual(["baseline-status"]); + expect( + drainNodePendingWork("node-stale-drain", { pairingGeneration: "generation-2" }).items.map( + (item) => item.type, + ), + ).toEqual(["location.request", "status.request"]); + }); + + it("rolls back only the exact item owned by one enqueue", () => { + const location = enqueueNodePendingWork({ + nodeId: "node-item-rollback", + type: "location.request", + pairingGeneration: "generation-1", + }); + enqueueNodePendingWork({ + nodeId: "node-item-rollback", + type: "status.request", + pairingGeneration: "generation-1", + }); + + expect( + removeNodePendingWorkItem({ + nodeId: "node-item-rollback", + itemId: location.item.id, + pairingGeneration: "generation-1", + }), + ).toBe(true); + expect( + drainNodePendingWork("node-item-rollback", { + pairingGeneration: "generation-1", + includeDefaultStatus: false, + }).items.map((item) => item.type), + ).toEqual(["status.request"]); + }); + it("keeps hasMore true when the baseline status item is deferred by maxItems", () => { enqueueNodePendingWork({ nodeId: "node-3", type: "location.request" }); diff --git a/src/gateway/node-pending-work.ts b/src/gateway/node-pending-work.ts index 553903a16d22..1032dec44744 100644 --- a/src/gateway/node-pending-work.ts +++ b/src/gateway/node-pending-work.ts @@ -30,12 +30,14 @@ type NodePendingWorkItem = { type NodePendingWorkState = { revision: number; itemsById: Map; + pairingGeneration?: string; }; type DrainOptions = { maxItems?: number; includeDefaultStatus?: boolean; nowMs?: number; + pairingGeneration?: string; }; type DrainResult = { @@ -56,16 +58,22 @@ const PRIORITY_RANK: Record = { default: 1, }; -const stateByNodeId = new Map(); +const stateByNodeId = new Map>(); -function getOrCreateState(nodeId: string): NodePendingWorkState { - let state = stateByNodeId.get(nodeId); +function getOrCreateState(nodeId: string, pairingGeneration?: string): NodePendingWorkState { + let states = stateByNodeId.get(nodeId); + if (!states) { + states = new Map(); + stateByNodeId.set(nodeId, states); + } + let state = states.get(pairingGeneration); if (!state) { state = { revision: 0, itemsById: new Map(), + ...(pairingGeneration ? { pairingGeneration } : {}), }; - stateByNodeId.set(nodeId, state); + states.set(pairingGeneration, state); } return state; } @@ -92,12 +100,43 @@ function pruneExpired(state: NodePendingWorkState, nowMs: number): boolean { return changed; } -function pruneStateIfEmpty(nodeId: string, state: NodePendingWorkState) { - if (state.itemsById.size === 0) { +function pruneExpiredRetiredGenerations( + nodeId: string, + currentPairingGeneration: string | undefined, + nowMs: number, +): void { + const states = stateByNodeId.get(nodeId); + if (!states) { + return; + } + for (const [pairingGeneration, state] of states) { + if (pairingGeneration === currentPairingGeneration) { + continue; + } + pruneExpired(state, nowMs); + if (state.itemsById.size === 0) { + states.delete(pairingGeneration); + } + } + if (states.size === 0) { stateByNodeId.delete(nodeId); } } +function pruneStateIfEmpty( + nodeId: string, + pairingGeneration: string | undefined, + state: NodePendingWorkState, +) { + if (state.itemsById.size === 0) { + const states = stateByNodeId.get(nodeId); + states?.delete(pairingGeneration); + if (states?.size === 0) { + stateByNodeId.delete(nodeId); + } + } +} + function sortedItems(state: NodePendingWorkState): NodePendingWorkItem[] { // Higher priority wins, then older work, then id for deterministic paging. return [...state.itemsById.values()].toSorted((a, b) => { @@ -136,6 +175,7 @@ export function enqueueNodePendingWork(params: { priority?: NodePendingWorkPriority; expiresInMs?: number; payload?: Record; + pairingGeneration?: string; }): { revision: number; item: NodePendingWorkItem; deduped: boolean } { const nodeId = params.nodeId.trim(); if (!nodeId) { @@ -143,7 +183,10 @@ export function enqueueNodePendingWork(params: { } const rawNowMs = Date.now(); const nowMs = resolveDateTimestampMs(rawNowMs); - const state = getOrCreateState(nodeId); + // Generation changes stop touching old buckets, so sweep their TTLs from + // every active-generation access instead of retaining retired work forever. + pruneExpiredRetiredGenerations(nodeId, params.pairingGeneration, nowMs); + const state = getOrCreateState(nodeId, params.pairingGeneration); pruneExpired(state, nowMs); // Keep one outstanding item per type so repeated status/location requests // collapse until the node has a chance to drain them. @@ -164,6 +207,42 @@ export function enqueueNodePendingWork(params: { return { revision: state.revision, item, deduped: false }; } +/** Clears explicit pending work owned by a removed node pairing. */ +export function clearNodePendingWork(nodeId: string, pairingGeneration?: string): boolean { + const normalizedNodeId = nodeId.trim(); + if (!normalizedNodeId) { + return false; + } + if (pairingGeneration === undefined) { + return stateByNodeId.delete(normalizedNodeId); + } + const states = stateByNodeId.get(normalizedNodeId); + const deleted = states?.delete(pairingGeneration) ?? false; + if (states?.size === 0) { + stateByNodeId.delete(normalizedNodeId); + } + return deleted; +} + +/** Removes one exact item without disturbing concurrent work in the same generation. */ +export function removeNodePendingWorkItem(params: { + nodeId: string; + itemId: string; + pairingGeneration?: string; +}): boolean { + const normalizedNodeId = params.nodeId.trim(); + if (!normalizedNodeId || !params.itemId) { + return false; + } + const state = stateByNodeId.get(normalizedNodeId)?.get(params.pairingGeneration); + if (!state || !state.itemsById.delete(params.itemId)) { + return false; + } + state.revision += 1; + pruneStateIfEmpty(normalizedNodeId, params.pairingGeneration, state); + return true; +} + /** Drains pending work for a node, including a baseline status request unless disabled. */ export function drainNodePendingWork(nodeId: string, opts: DrainOptions = {}): DrainResult { const normalizedNodeId = nodeId.trim(); @@ -171,10 +250,11 @@ export function drainNodePendingWork(nodeId: string, opts: DrainOptions = {}): D return { revision: 0, items: [], hasMore: false }; } const nowMs = resolveDateTimestampMs(opts.nowMs ?? Date.now()); - const state = stateByNodeId.get(normalizedNodeId); + pruneExpiredRetiredGenerations(normalizedNodeId, opts.pairingGeneration, nowMs); + const state = stateByNodeId.get(normalizedNodeId)?.get(opts.pairingGeneration); if (state) { pruneExpired(state, nowMs); - pruneStateIfEmpty(normalizedNodeId, state); + pruneStateIfEmpty(normalizedNodeId, opts.pairingGeneration, state); } const revision = state?.revision ?? 0; const maxItems = Math.min(MAX_ITEMS, Math.max(1, Math.trunc(opts.maxItems ?? DEFAULT_MAX_ITEMS))); @@ -194,7 +274,7 @@ export function drainNodePendingWork(nodeId: string, opts: DrainOptions = {}): D } } state.revision += 1; - pruneStateIfEmpty(normalizedNodeId, state); + pruneStateIfEmpty(normalizedNodeId, opts.pairingGeneration, state); } return { revision: state?.revision ?? revision, diff --git a/src/gateway/node-registry.test.ts b/src/gateway/node-registry.test.ts index d7b391a67c81..b614d329730a 100644 --- a/src/gateway/node-registry.test.ts +++ b/src/gateway/node-registry.test.ts @@ -7,7 +7,7 @@ import { MAX_TIMER_TIMEOUT_MS, } from "@openclaw/normalization-core/number-coercion"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { getActiveNodeContext, setActiveNodeContext } from "../infra/active-node-context.js"; +import { getCurrentActiveNodeContext, setActiveNodeContext } from "../infra/active-node-context.js"; import { onDiagnosticEvent, resetDiagnosticEventsForTest } from "../infra/diagnostic-events.js"; import { createEmptyPluginRegistry } from "../plugins/registry-empty.js"; import { listConnectedNodePluginTools } from "./node-plugin-tool-snapshot.js"; @@ -98,6 +98,15 @@ function makeClient( }; } +function registerNodeSession( + registry: NodeRegistry, + client: GatewayWsClient, + opts: Partial[1]> = {}, +) { + const { pairingIdentity = "identity-a", ...registration } = opts; + return registry.register(client, { ...registration, pairingIdentity }); +} + function registerDemoNodePluginTool(params: { name: string; command: string; @@ -151,7 +160,7 @@ function makeConnectivitySocket(emitPong: boolean) { function registerNode(registry: NodeRegistry, opts: Parameters[3] = {}) { const frames: string[] = []; - registry.register(makeClient("conn-1", "node-1", frames, opts), {}); + registerNodeSession(registry, makeClient("conn-1", "node-1", frames, opts), {}); return frames; } @@ -218,6 +227,334 @@ function authorizeSystemRun(registry: NodeRegistry, overrides: Partial { + it("rejects registration without an authenticated pairing identity", () => { + const registry = new NodeRegistry(); + const client = makeClient("conn-unbound", "node-unbound"); + + expect(() => registry.register(client, {} as never)).toThrow( + "node session registration requires pairing identity", + ); + expect(registry.listConnected()).toEqual([]); + }); + + it("rejects dispatch through an invalidated node connection", async () => { + const registry = new NodeRegistry(); + const frames: string[] = []; + const client = makeClient("conn-invalidated", "node-invalidated", frames); + registerNodeSession(registry, client, {}); + client.invalidated = true; + + expect(registry.get("node-invalidated")).toBeUndefined(); + expect(registry.listConnected()).toEqual([]); + expect(registry.sendEvent("node-invalidated", "node.test", { ok: true })).toBe(false); + await expect( + registry.invoke({ nodeId: "node-invalidated", command: "system.run" }), + ).resolves.toMatchObject({ + ok: false, + error: { code: "PAIRING_CHANGED" }, + }); + expect(frames).toEqual([]); + }); + + it("rejects generation-mismatched lookup and dispatch without invalidating the session", async () => { + const registry = new NodeRegistry(); + const frames: string[] = []; + const client = makeClient("conn-old-generation", "node-generation", frames); + registerNodeSession(registry, client, { pairingGeneration: "generation-a" }); + + expect(registry.getForPairingGeneration("node-generation", "generation-b")).toBeUndefined(); + expect(client.invalidated).not.toBe(true); + await expect( + registry.invoke({ + nodeId: "node-generation", + expectedPairingGeneration: "generation-b", + command: "system.run", + }), + ).resolves.toMatchObject({ + ok: false, + error: { code: "PAIRING_CHANGED" }, + }); + expect(client.invalidated).not.toBe(true); + expect(frames).toEqual([]); + }); + + it("does not let a stale operation invalidate the valid replacement generation", () => { + const registry = new NodeRegistry(); + const frames: string[] = []; + const replacement = makeClient("conn-replacement", "node-replaced", frames); + registerNodeSession(registry, replacement, { pairingGeneration: "generation-b" }); + + expect(registry.getForPairingGeneration("node-replaced", "generation-a")).toBeUndefined(); + expect(replacement.invalidated).not.toBe(true); + expect(registry.getForPairingGeneration("node-replaced", "generation-b")?.connId).toBe( + "conn-replacement", + ); + }); + + it("revalidates the persistent generation immediately before dispatch", async () => { + const frames: string[] = []; + const resolveCurrentPairingState = vi.fn().mockResolvedValue({ + identity: "identity-a", + generation: "generation-b", + }); + const registry = new NodeRegistry({ resolveCurrentPairingState }); + const client = makeClient("conn-generation", "node-generation", frames); + registerNodeSession(registry, client, { + pairingIdentity: "identity-a", + pairingGeneration: "generation-a", + }); + + await expect( + registry.invoke({ + nodeId: "node-generation", + expectedConnId: "conn-generation", + expectedPairingGeneration: "generation-a", + command: "system.run", + }), + ).resolves.toMatchObject({ + ok: false, + error: { code: "PAIRING_CHANGED" }, + }); + expect(resolveCurrentPairingState).toHaveBeenCalledWith("node-generation"); + expect(frames).toEqual([]); + }); + + it("revalidates persistent generation ownership for inbound node RPCs", async () => { + const resolveCurrentPairingState = vi.fn().mockResolvedValue({ + identity: "identity-a", + generation: "generation-a", + }); + const registry = createNodeRegistry({ resolveCurrentPairingState }); + const client = makeClient("conn-generation", "node-generation"); + registerNodeSession(registry, client, { + pairingIdentity: "identity-a", + pairingGeneration: "generation-a", + }); + + await expect(registry.isConnectionCurrentPairingState("conn-generation")).resolves.toBe(true); + resolveCurrentPairingState.mockResolvedValue({ + identity: "identity-a", + generation: "generation-b", + }); + await expect(registry.isConnectionCurrentPairingState("conn-generation")).resolves.toBe(false); + expect(client.invalidated).toBe(true); + expect(resolveCurrentPairingState).toHaveBeenCalledWith("node-generation"); + }); + + it("removes an externally replaced session from connected and active projections", async () => { + let currentPairingGeneration = "generation-a"; + const onPairingInvalidated = vi.fn(); + const registry = createNodeRegistry({ + resolveCurrentPairingState: async () => ({ + identity: "identity-a", + generation: currentPairingGeneration, + }), + onPairingInvalidated, + }); + const client = makeClient("conn-generation", "node-generation", [], { + permissions: { accessibility: true }, + }); + registerNodeSession(registry, client, { + pairingIdentity: "identity-a", + pairingGeneration: "generation-a", + }); + registry.updatePresenceActivity({ + nodeId: "node-generation", + connId: "conn-generation", + idleSeconds: 0, + }); + expect(registry.getActiveNode()?.nodeId).toBe("node-generation"); + + currentPairingGeneration = "generation-b"; + await expect(registry.listCurrentConnected()).resolves.toEqual([]); + expect(registry.getActiveNode()).toBeUndefined(); + expect(getCurrentActiveNodeContext()).toBeNull(); + expect(client.invalidated).toBe(true); + expect(onPairingInvalidated).toHaveBeenCalledWith({ + nodeId: "node-generation", + connId: "conn-generation", + }); + }); + + it("does not invalidate a session promoted while persistent generation is loading", async () => { + let resolveLookup: ((value: { identity: string; generation: string }) => void) | undefined; + const resolveCurrentPairingState = vi.fn( + () => + new Promise<{ identity: string; generation: string }>((resolve) => { + resolveLookup = resolve; + }), + ); + const onPairingInvalidated = vi.fn(); + const registry = createNodeRegistry({ + resolveCurrentPairingState, + onPairingInvalidated, + }); + const client = makeClient("conn-generation", "node-generation"); + registerNodeSession(registry, client, { + pairingIdentity: "identity-a", + pairingGeneration: "generation-a", + }); + + const connected = registry.listCurrentConnected(); + expect(resolveCurrentPairingState).toHaveBeenCalledWith("node-generation"); + expect( + registry.updateSurface( + "node-generation", + { commands: [] }, + { + expectedConnId: "conn-generation", + expectedPairingIdentity: "identity-a", + expectedPairingGeneration: "generation-a", + nextPairingGeneration: "generation-b", + }, + ), + ).not.toBeNull(); + resolveLookup?.({ identity: "identity-a", generation: "generation-a" }); + + await expect(connected).resolves.toEqual([]); + expect(registry.get("node-generation")?.pairingGeneration).toBe("generation-b"); + expect(client.invalidated).not.toBe(true); + expect(onPairingInvalidated).not.toHaveBeenCalled(); + }); + + it("revalidates the active node at the prompt projection boundary", () => { + let currentPairingGeneration = "generation-a"; + const registry = createNodeRegistry({ + isPairingStateCurrent: (_nodeId, expected) => + expected.identity === "identity-a" && expected.generation === currentPairingGeneration, + }); + registerNodeSession( + registry, + makeClient("conn-generation", "node-generation", [], { + permissions: { accessibility: true }, + }), + { pairingIdentity: "identity-a", pairingGeneration: "generation-a" }, + ); + registry.updatePresenceActivity({ + nodeId: "node-generation", + connId: "conn-generation", + idleSeconds: 0, + }); + + expect(getCurrentActiveNodeContext()).toMatchObject({ + nodeId: "node-generation", + pairingGeneration: "generation-a", + }); + currentPairingGeneration = "generation-b"; + expect(getCurrentActiveNodeContext()).toBeNull(); + }); + + it("filters an already-loaded pairing snapshot without invalidating a newer session", () => { + const registry = createNodeRegistry(); + const client = makeClient("conn-generation", "node-generation"); + registerNodeSession(registry, client, { + pairingIdentity: "identity-b", + pairingGeneration: "generation-b", + }); + + expect( + registry.listConnectedForPairingStates( + new Map([["node-generation", { identity: "identity-a", generation: "generation-a" }]]), + ), + ).toEqual([]); + expect(client.invalidated).not.toBe(true); + }); + + it("distinguishes a pending surface from a missing paired-device row", () => { + const registry = createNodeRegistry(); + const client = makeClient("conn-pending-surface", "node-pending-surface"); + registerNodeSession(registry, client, { pairingIdentity: "identity-a" }); + + expect( + registry.listConnectedForPairingStates( + new Map([["node-pending-surface", { identity: "identity-a" }]]), + ), + ).toHaveLength(1); + expect(registry.listConnectedForPairingStates(new Map())).toEqual([]); + expect(client.invalidated).not.toBe(true); + }); + + it("fails closed without invalidating a session when pairing persistence is unavailable", async () => { + const registry = createNodeRegistry({ + resolveCurrentPairingState: async () => { + throw new Error("pairing store unavailable"); + }, + }); + const client = makeClient("conn-generation", "node-generation"); + registerNodeSession(registry, client, { + pairingIdentity: "identity-a", + pairingGeneration: "generation-a", + }); + + await expect(registry.listCurrentConnected()).resolves.toEqual([]); + expect(client.invalidated).not.toBe(true); + expect(registry.listConnected()).toHaveLength(1); + }); + + it("reconciles stale persistent generations synchronously for prompt projections", () => { + let currentPairingGeneration = "generation-a"; + const onPairingInvalidated = vi.fn(); + const registry = createNodeRegistry({ + isPairingStateCurrent: (_nodeId, expected) => + expected.generation === currentPairingGeneration, + onPairingInvalidated, + }); + const client = makeClient("conn-generation", "node-generation"); + registerNodeSession(registry, client, { pairingGeneration: "generation-a" }); + + expect(registry.listCurrentConnectedSync()).toHaveLength(1); + currentPairingGeneration = "generation-b"; + expect(registry.listCurrentConnectedSync()).toEqual([]); + expect(client.invalidated).toBe(true); + expect(onPairingInvalidated).toHaveBeenCalledWith({ + nodeId: "node-generation", + connId: "conn-generation", + }); + }); + + it("fails closed synchronously when pairing persistence is unavailable", () => { + const registry = createNodeRegistry({ + isPairingStateCurrent: () => { + throw new Error("pairing store unavailable"); + }, + }); + const client = makeClient("conn-generation", "node-generation"); + registerNodeSession(registry, client, { pairingGeneration: "generation-a" }); + + expect(registry.listCurrentConnectedSync()).toEqual([]); + expect(client.invalidated).not.toBe(true); + expect(registry.listConnected()).toHaveLength(1); + }); + + it("invalidates a generation-less session after external pairing deletion", async () => { + let currentPairingState: { identity: string } | undefined = { identity: "identity-a" }; + const registry = createNodeRegistry({ + resolveCurrentPairingState: async () => currentPairingState, + }); + const client = makeClient("conn-pending-surface", "node-pending-surface"); + registerNodeSession(registry, client, { pairingIdentity: "identity-a" }); + + await expect(registry.listCurrentConnected()).resolves.toHaveLength(1); + currentPairingState = undefined; + await expect(registry.listCurrentConnected()).resolves.toEqual([]); + expect(client.invalidated).toBe(true); + }); + + it("invalidates a generation-less session synchronously after pairing deletion", () => { + let pairingExists = true; + const registry = createNodeRegistry({ + isPairingStateCurrent: (_nodeId, expected) => + pairingExists && expected.identity === "identity-a", + }); + const client = makeClient("conn-pending-surface", "node-pending-surface"); + registerNodeSession(registry, client, { pairingIdentity: "identity-a" }); + + expect(registry.listCurrentConnectedSync()).toHaveLength(1); + pairingExists = false; + expect(registry.listCurrentConnectedSync()).toEqual([]); + expect(client.invalidated).toBe(true); + }); + it("routes ordered input to the pending invoke connection and rejects unknown invokes", async () => { const registry = new NodeRegistry(); const frames = registerNode(registry); @@ -297,14 +634,16 @@ describe("gateway/node-registry", () => { it("ranks connected nodes by gateway-derived input activity", () => { const registry = createTestNodeRegistry(); - registry.register( + registerNodeSession( + registry, makeClient("conn-1", "node-1", [], { displayName: "Desk Mac", permissions: { accessibility: true }, }), {}, ); - registry.register( + registerNodeSession( + registry, makeClient("conn-2", "node-2", [], { displayName: "Laptop", permissions: { accessibility: true }, @@ -328,15 +667,16 @@ describe("gateway/node-registry", () => { }); expect(registry.getActiveNode()?.nodeId).toBe("node-2"); - expect(getActiveNodeContext()).toEqual({ nodeId: "node-2" }); + expect(getCurrentActiveNodeContext()).toEqual({ nodeId: "node-2" }); expect(registry.unregister("conn-2")).toBe("node-2"); expect(registry.getActiveNode()?.nodeId).toBe("node-1"); - expect(getActiveNodeContext()).toEqual({ nodeId: "node-1" }); + expect(getCurrentActiveNodeContext()).toEqual({ nodeId: "node-1" }); }); it("recomputes active context when a same-id connection replaces reported presence", () => { const registry = createTestNodeRegistry(); - registry.register( + registerNodeSession( + registry, makeClient("conn-old", "node-1", [], { permissions: { accessibility: true } }), {}, ); @@ -347,20 +687,22 @@ describe("gateway/node-registry", () => { observedAtMs: 100_000, }); - registry.register( + registerNodeSession( + registry, makeClient("conn-new", "node-1", [], { permissions: { accessibility: true } }), {}, ); expect(registry.getActiveNode()).toBeUndefined(); - expect(getActiveNodeContext()).toBeNull(); + expect(getCurrentActiveNodeContext()).toBeNull(); expect(registry.unregister("conn-old")).toBeNull(); - expect(getActiveNodeContext()).toBeNull(); + expect(getCurrentActiveNodeContext()).toBeNull(); }); it("rejects presence updates from stale node connections", () => { const registry = createTestNodeRegistry(); - registry.register( + registerNodeSession( + registry, makeClient("conn-new", "node-1", [], { permissions: { accessibility: true } }), {}, ); @@ -378,7 +720,8 @@ describe("gateway/node-registry", () => { it("does not advance a bounded estimate on saturated idle keepalives", () => { const registry = createTestNodeRegistry(); - registry.register( + registerNodeSession( + registry, makeClient("conn-1", "node-1", [], { permissions: { accessibility: true } }), {}, ); @@ -404,7 +747,8 @@ describe("gateway/node-registry", () => { it("clears reported presence when Accessibility permission is removed", () => { const registry = createTestNodeRegistry(); - registry.register( + registerNodeSession( + registry, makeClient("conn-1", "node-1", [], { permissions: { accessibility: true }, declaredPermissions: { accessibility: true }, @@ -423,12 +767,13 @@ describe("gateway/node-registry", () => { expect(registry.get("node-1")?.lastActiveAtMs).toBeUndefined(); expect(registry.get("node-1")?.presenceUpdatedAtMs).toBeUndefined(); expect(registry.getActiveNode()).toBeUndefined(); - expect(getActiveNodeContext()).toBeNull(); + expect(getCurrentActiveNodeContext()).toBeNull(); }); it("checks node websocket connectivity with ping/pong", async () => { const registry = createTestNodeRegistry(); - registry.register( + registerNodeSession( + registry, makeClient("conn-1", "node-1", [], { socket: makeConnectivitySocket(true), }), @@ -440,7 +785,8 @@ describe("gateway/node-registry", () => { it("reports stale node websocket connectivity before invoke timeout", async () => { const registry = createTestNodeRegistry(); - registry.register( + registerNodeSession( + registry, makeClient("conn-1", "node-1", [], { socket: makeConnectivitySocket(false), }), @@ -460,7 +806,7 @@ describe("gateway/node-registry", () => { const oldFrames: string[] = []; const newClient = makeClient("conn-new", "node-1"); - registry.register(makeClient("conn-old", "node-1", oldFrames), {}); + registerNodeSession(registry, makeClient("conn-old", "node-1", oldFrames), {}); const oldInvoke = registry.invoke({ nodeId: "node-1", command: "system.run", @@ -468,7 +814,7 @@ describe("gateway/node-registry", () => { }); const oldDisconnected = oldInvoke.catch((err: unknown) => err); const oldRequest = JSON.parse(oldFrames[0] ?? "{}") as { payload?: { id?: string } }; - const newSession = registry.register(newClient, {}); + const newSession = registerNodeSession(registry, newClient, {}); expect( registry.handleInvokeResult({ @@ -486,8 +832,8 @@ describe("gateway/node-registry", () => { it("rejects invoke when the node connection changed before dispatch", async () => { const registry = createNodeRegistry(); const replacementFrames: string[] = []; - registry.register(makeClient("conn-old", "node-1"), {}); - registry.register(makeClient("conn-new", "node-1", replacementFrames), {}); + registerNodeSession(registry, makeClient("conn-old", "node-1"), {}); + registerNodeSession(registry, makeClient("conn-new", "node-1", replacementFrames), {}); await expect( registry.invoke({ @@ -1270,7 +1616,7 @@ describe("gateway/node-registry", () => { it("sends raw event payload JSON without changing the envelope shape", () => { const registry = createTestNodeRegistry(); const frames: string[] = []; - registry.register(makeClient("conn-1", "node-1", frames), {}); + registerNodeSession(registry, makeClient("conn-1", "node-1", frames), {}); const payload = serializeEventPayload({ foo: "bar" }); const nullPayload = serializeEventPayload(null); const falsePayload = serializeEventPayload(false); @@ -1309,6 +1655,60 @@ describe("gateway/node-registry", () => { ]); }); + it("drops a delayed voice-wake snapshot after persistent generation changes", async () => { + let resolveCurrent!: (state: { identity: string; generation?: string } | undefined) => void; + const currentPairingState = new Promise<{ identity: string; generation?: string } | undefined>( + (resolve) => { + resolveCurrent = resolve; + }, + ); + const resolveCurrentPairingState = vi.fn(() => currentPairingState); + const registry = createNodeRegistry({ resolveCurrentPairingState }); + const frames: string[] = []; + registerNodeSession(registry, makeClient("conn-1", "node-1", frames), { + pairingIdentity: "identity-a", + pairingGeneration: "generation-a", + }); + + const send = registry.sendEventRawForPairingGeneration( + "node-1", + "generation-a", + "voicewake.changed", + serializeEventPayload({ triggers: ["openclaw"] }), + ); + await vi.waitFor(() => expect(resolveCurrentPairingState).toHaveBeenCalledTimes(1)); + resolveCurrent({ identity: "identity-a", generation: "generation-b" }); + + await expect(send).resolves.toBe(false); + expect(frames).toEqual([]); + }); + + it("drops a delayed command-free snapshot after pairing identity deletion", async () => { + let resolveCurrent!: (state: { identity: string } | undefined) => void; + const currentPairingState = new Promise<{ identity: string } | undefined>((resolve) => { + resolveCurrent = resolve; + }); + const registry = createNodeRegistry({ + resolveCurrentPairingState: async () => await currentPairingState, + }); + const frames: string[] = []; + registerNodeSession(registry, makeClient("conn-1", "node-1", frames), { + pairingIdentity: "identity-a", + }); + + const send = registry.sendEventForPairingIdentity({ + nodeId: "node-1", + connId: "conn-1", + pairingIdentity: "identity-a", + event: "voicewake.changed", + payload: { triggers: ["openclaw"] }, + }); + resolveCurrent(undefined); + + await expect(send).resolves.toBe(false); + expect(frames).toEqual([]); + }); + it("rejects raw event sends when the node socket buffer is saturated", () => { resetDiagnosticEventsForTest(); const diagnosticEvents: unknown[] = []; @@ -1319,7 +1719,8 @@ describe("gateway/node-registry", () => { send: vi.fn(), close: vi.fn(), }; - registry.register( + registerNodeSession( + registry, makeClient("conn-1", "node-1", [], { socket: socket as unknown as GatewayWsClient["socket"], }), @@ -1359,7 +1760,7 @@ describe("gateway/node-registry", () => { declaredPermissions: { microphone: true, camera: false }, }); - const session = registry.register(client, {}); + const session = registerNodeSession(registry, client, {}); expect(session.caps).toEqual([]); expect(session.commands).toEqual([]); @@ -1376,6 +1777,103 @@ describe("gateway/node-registry", () => { expect((client.connect as { commands?: string[] }).commands).toEqual(["talk.ptt.start"]); }); + it("advances the exact live session with its approved surface generation", () => { + const registry = createTestNodeRegistry(); + const client = makeClient("conn-1", "node-1", [], { + declaredCommands: ["device.info"], + }); + registerNodeSession(registry, client, { + pairingIdentity: "identity-a", + pairingGeneration: "generation-a", + }); + + const updated = registry.updateSurface( + "node-1", + { commands: ["device.info"] }, + { + expectedConnId: "conn-1", + expectedPairingIdentity: "identity-a", + expectedPairingGeneration: "generation-a", + nextPairingGeneration: "generation-b", + }, + ); + + expect(updated?.pairingGeneration).toBe("generation-b"); + expect( + registry.updateSurface( + "node-1", + { commands: [] }, + { + expectedConnId: "conn-stale", + expectedPairingIdentity: "identity-a", + expectedPairingGeneration: "generation-a", + nextPairingGeneration: "generation-c", + }, + ), + ).toBeNull(); + expect(registry.get("node-1")?.commands).toEqual(["device.info"]); + expect(registry.get("node-1")?.pairingGeneration).toBe("generation-b"); + }); + + it("rebinds active-node presence when a live session advances generations", () => { + const registry = createTestNodeRegistry(); + const client = makeClient("conn-1", "node-1", [], { + permissions: { accessibility: true }, + declaredPermissions: { accessibility: true }, + }); + registerNodeSession(registry, client, { + pairingIdentity: "identity-a", + pairingGeneration: "generation-a", + }); + registry.updatePresenceActivity({ + nodeId: "node-1", + connId: "conn-1", + idleSeconds: 0, + observedAtMs: 100_000, + }); + + registry.updateSurface( + "node-1", + { commands: [], permissions: { accessibility: true } }, + { + expectedConnId: "conn-1", + expectedPairingIdentity: "identity-a", + expectedPairingGeneration: "generation-a", + nextPairingGeneration: "generation-b", + }, + ); + + expect(getCurrentActiveNodeContext()).toMatchObject({ + nodeId: "node-1", + pairingGeneration: "generation-b", + }); + }); + + it("does not promote a generation-less session from a retired pairing identity", () => { + const registry = createTestNodeRegistry(); + const client = makeClient("conn-1", "node-1", [], { + declaredCommands: ["device.info"], + }); + registerNodeSession(registry, client, { pairingIdentity: "identity-a" }); + + expect( + registry.updateSurface( + "node-1", + { commands: ["device.info"] }, + { + expectedConnId: "conn-1", + expectedPairingIdentity: "identity-b", + nextPairingGeneration: "generation-b", + }, + ), + ).toBeNull(); + expect(registry.get("node-1")).toMatchObject({ + pairingIdentity: "identity-a", + commands: [], + }); + expect(registry.get("node-1")?.pairingGeneration).toBeUndefined(); + }); + it("keeps node-hosted plugin tools inside the approved command surface", () => { registerDemoNodePluginTool({ name: "demo_echo", command: "demo.echo" }); const registry = createTestNodeRegistry(); @@ -1383,7 +1881,7 @@ describe("gateway/node-registry", () => { commands: ["demo.echo"], }); - const session = registry.register(client, {}); + const session = registerNodeSession(registry, client, {}); publishNodePluginTools(registry, [ { pluginId: "demo", @@ -1413,6 +1911,30 @@ describe("gateway/node-registry", () => { expect(listConnectedNodePluginTools()).toEqual([]); }); + it("retires node-hosted plugin tools immediately when a connection is invalidated", () => { + registerDemoNodePluginTool({ name: "demo_echo", command: "demo.echo" }); + const registry = createTestNodeRegistry(); + const client = makeClient("conn-1", "node-1", [], { commands: ["demo.echo"] }); + registerNodeSession(registry, client, {}); + publishNodePluginTools(registry, [ + { + pluginId: "demo", + name: "demo_echo", + description: "Echo through the node", + command: "demo.echo", + }, + ]); + expect(listConnectedNodePluginTools()).toHaveLength(1); + + expect(registry.invalidateConnectionForPairingChange("conn-1", "device-token-revoked")).toBe( + true, + ); + + expect(client.invalidated).toBe(true); + expect(client.invalidatedReason).toBe("device-token-revoked"); + expect(listConnectedNodePluginTools()).toEqual([]); + }); + it("keeps dangerous node-hosted plugin tools once explicitly approved", () => { registerDemoNodePluginTool({ name: "demo_dangerous", @@ -1424,7 +1946,7 @@ describe("gateway/node-registry", () => { commands: ["demo.dangerous"], }); - const session = registry.register(client, {}); + const session = registerNodeSession(registry, client, {}); publishNodePluginTools(registry, [ { pluginId: "demo", @@ -1447,7 +1969,7 @@ describe("gateway/node-registry", () => { commands: ["demo.echo"], }); - const session = registry.register(client, {}); + const session = registerNodeSession(registry, client, {}); publishNodePluginTools(registry, [ { pluginId: "demo", @@ -1475,7 +1997,7 @@ describe("gateway/node-registry", () => { commands: ["system.run"], }); - const session = registry.register(client, {}); + const session = registerNodeSession(registry, client, {}); publishNodePluginTools(registry, [ { pluginId: "demo", @@ -1512,7 +2034,7 @@ describe("gateway/node-registry", () => { commands: ["demo.echo"], }); - const session = registry.register(client, {}); + const session = registerNodeSession(registry, client, {}); publishNodePluginTools(registry, [ { pluginId: "demo", @@ -1548,7 +2070,7 @@ describe("gateway/node-registry", () => { declaredCommands: ["demo.echo"], }); - const session = registry.register(client, {}); + const session = registerNodeSession(registry, client, {}); publishNodePluginTools(registry, [ { pluginId: "demo", @@ -1573,7 +2095,8 @@ describe("gateway/node-registry", () => { it("enriches published node tools after matching plugin descriptors load", () => { const registry = createTestNodeRegistry(); - registry.register( + registerNodeSession( + registry, makeClient("conn-1", "node-1", [], { commands: ["demo.echo"], }), @@ -1602,7 +2125,8 @@ describe("gateway/node-registry", () => { it("ignores published node tools when gateway publication is disabled", () => { const registry = createNodeRegistry({ nodePluginToolsEnabled: false }); - registry.register( + registerNodeSession( + registry, makeClient("conn-1", "node-1", [], { commands: ["demo.echo"], }), @@ -1626,13 +2150,15 @@ describe("gateway/node-registry", () => { it("ignores node plugin tool updates from stale connections", () => { registerDemoNodePluginTool({ name: "demo_echo", command: "demo.echo" }); const registry = createTestNodeRegistry(); - registry.register( + registerNodeSession( + registry, makeClient("conn-old", "node-1", [], { commands: ["demo.echo"], }), {}, ); - registry.register( + registerNodeSession( + registry, makeClient("conn-new", "node-1", [], { commands: ["demo.echo"], }), @@ -1655,7 +2181,7 @@ describe("gateway/node-registry", () => { it("stores bounded node-hosted skill updates on the current session", () => { const registry = createTestNodeRegistry(); - const session = registry.register(makeClient("conn-1", "node-1"), {}); + const session = registerNodeSession(registry, makeClient("conn-1", "node-1"), {}); const updated = publishNodeSkills(registry, [ nodeSkill("release-helper"), @@ -1668,7 +2194,7 @@ describe("gateway/node-registry", () => { it("enforces node skill count and total-content caps", () => { const registry = createTestNodeRegistry(); - registry.register(makeClient("conn-1", "node-1"), {}); + registerNodeSession(registry, makeClient("conn-1", "node-1"), {}); const countUpdate = publishNodeSkills( registry, @@ -1689,12 +2215,12 @@ describe("gateway/node-registry", () => { it("ignores node skills when publication is disabled or the connection is stale", () => { const disabled = createNodeRegistry({ nodeSkillsEnabled: false }); - disabled.register(makeClient("conn-1", "node-1"), {}); + registerNodeSession(disabled, makeClient("conn-1", "node-1"), {}); expect(publishNodeSkills(disabled, [nodeSkill("disabled")])?.nodeSkills).toEqual([]); const registry = createTestNodeRegistry(); - registry.register(makeClient("conn-old", "node-1"), {}); - registry.register(makeClient("conn-new", "node-1"), {}); + registerNodeSession(registry, makeClient("conn-old", "node-1"), {}); + registerNodeSession(registry, makeClient("conn-new", "node-1"), {}); expect(publishNodeSkills(registry, [nodeSkill("stale")], "conn-old")).toBeNull(); expect(registry.get("node-1")?.nodeSkills).toEqual([]); }); @@ -1706,7 +2232,7 @@ describe("gateway/node-registry", () => { declaredPermissions: { camera: false }, }); - registry.register(client, {}); + registerNodeSession(registry, client, {}); const updated = registry.updateSurface("node-1", { caps: [], commands: [], @@ -1730,7 +2256,7 @@ describe("gateway/node-registry", () => { sessionCommandsCeiling: ["device.info"], }); - registry.register(client, {}); + registerNodeSession(registry, client, {}); const updated = registry.updateSurface("node-1", { caps: ["canvas", "device"], commands: ["canvas.snapshot", "device.info"], diff --git a/src/gateway/node-registry.ts b/src/gateway/node-registry.ts index af8ef92135fe..eeca634b74e6 100644 --- a/src/gateway/node-registry.ts +++ b/src/gateway/node-registry.ts @@ -16,6 +16,7 @@ import type { } from "../../packages/gateway-protocol/src/schema/nodes.js"; import { setActiveNodeContext } from "../infra/active-node-context.js"; import { NODE_MCP_TOOLS_CALL_COMMAND } from "../infra/node-commands.js"; +import type { NodePairingBinding } from "../infra/node-pairing-state.js"; import { logRejectedLargePayload } from "../logging/diagnostic-payload.js"; import { normalizeString } from "./node-normalize.js"; import { @@ -42,6 +43,10 @@ import type { GatewayWsClient } from "./server/ws-types.js"; export type NodeSession = { nodeId: string; connId: string; + /** Persistent device key and node-token identity authenticated for this connection. */ + pairingIdentity?: string; + /** Persistent pairing generation authenticated before this session was registered. */ + pairingGeneration?: string; client: GatewayWsClient; clientId?: string; clientMode?: string; @@ -70,6 +75,20 @@ export type NodeSession = { presenceUpdatedAtMs?: number; }; +type PairingBoundNodeSession = NodeSession & { pairingIdentity: string }; + +type PairingBoundNodeSessionLease = { + session: PairingBoundNodeSession; + nodeId: string; + connId: string; + binding: NodePairingBinding; +}; + +type PairingLeaseResolution = + | { status: "current"; session: PairingBoundNodeSession } + | { status: "stale"; presenceInvalidated: boolean } + | { status: "unavailable" }; + /** Authorized system.run event window bound to one node connection. */ type AuthorizedSystemRunEvent = PendingSystemRunEvent & { nodeId: string; @@ -164,12 +183,49 @@ export type NodeEventTransport = { checkConnectivity?: (timeoutMs: number) => Promise; }; +type NodePairingStateSnapshot = NodePairingBinding; + +type NodeSessionRegistrationOptions = { + remoteIp?: string | undefined; + pairingIdentity: string; + pairingGeneration?: string | undefined; +}; + +function pairingBindingForSession(node: PairingBoundNodeSession): NodePairingBinding { + return { + identity: node.pairingIdentity, + ...(node.pairingGeneration ? { generation: node.pairingGeneration } : {}), + }; +} + +function pairingStateMatchesBinding( + binding: NodePairingBinding, + current: NodePairingStateSnapshot | undefined, +): boolean { + if (!current) { + return false; + } + if (binding.identity !== current.identity) { + return false; + } + return !binding.generation || binding.generation === current.generation; +} + export type NodeRegistryOptions = { listRegisteredNodePluginToolCommands?: | (() => readonly RegisteredNodePluginToolCommand[] | undefined) | undefined; nodePluginToolsEnabled?: boolean; nodeSkillsEnabled?: boolean; + resolveCurrentPairingState?: (nodeId: string) => Promise; + isPairingStateCurrent?: (nodeId: string, expected: NodePairingBinding) => boolean; + onPairingGenerationChanged?: (params: { + nodeId: string; + previousPairingGeneration: string; + nextPairingGeneration: string; + preserveSessionState: boolean; + }) => void; + onPairingInvalidated?: (params: { nodeId: string; connId: string }) => void; }; /** Serialize an event payload once so fanout can reuse the same JSON string. */ @@ -193,7 +249,7 @@ function isSerializedEventPayload(value: unknown): value is SerializedEventPaylo /** Registry of currently connected Gateway nodes. */ export class NodeRegistry { - private nodesById = new Map(); + private nodesById = new Map(); private nodesByConn = new Map(); private eventTransportsByConn = new Map(); private pendingInvokes = new Map(); @@ -245,9 +301,78 @@ export class NodeRegistry { }, }); private authorizedSystemRunEvents = new Map(); + private pairingGenerationEventChains = new Map>(); constructor(private readonly options: NodeRegistryOptions = {}) {} + private listConnectedSessions(): PairingBoundNodeSession[] { + return [...this.nodesById.values()].filter((node) => node.client.invalidated !== true); + } + + private capturePairingLease(node: PairingBoundNodeSession): PairingBoundNodeSessionLease { + return { + session: node, + nodeId: node.nodeId, + connId: node.connId, + binding: pairingBindingForSession(node), + }; + } + + private currentSessionForLease( + lease: PairingBoundNodeSessionLease, + ): PairingBoundNodeSession | undefined { + const current = this.nodesById.get(lease.nodeId); + return current === lease.session && + current.connId === lease.connId && + current.pairingIdentity === lease.binding.identity && + current.pairingGeneration === lease.binding.generation && + current.client.invalidated !== true + ? current + : undefined; + } + + private settlePairingLease(params: { + lease: PairingBoundNodeSessionLease; + isCurrent: boolean; + invalidateStale: boolean; + }): PairingLeaseResolution { + const current = this.currentSessionForLease(params.lease); + if (!current) { + return { status: "stale", presenceInvalidated: false }; + } + if (params.isCurrent) { + return { status: "current", session: current }; + } + const presenceInvalidated = params.invalidateStale + ? this.invalidateSessionForPairingChange(current) + : false; + return { status: "stale", presenceInvalidated }; + } + + private async resolvePairingLease( + lease: PairingBoundNodeSessionLease, + options: { invalidateStale: boolean }, + ): Promise { + const resolveCurrentPairingState = this.options.resolveCurrentPairingState; + if (!resolveCurrentPairingState) { + const current = this.currentSessionForLease(lease); + return current + ? { status: "current", session: current } + : { status: "stale", presenceInvalidated: false }; + } + let currentPairingState: NodePairingStateSnapshot | undefined; + try { + currentPairingState = await resolveCurrentPairingState(lease.nodeId); + } catch { + return { status: "unavailable" }; + } + return this.settlePairingLease({ + lease, + isCurrent: pairingStateMatchesBinding(lease.binding, currentPairingState), + invalidateStale: options.invalidateStale, + }); + } + private normalizePluginToolDescriptors(params: { nodeId: string; tools?: readonly NodePluginToolDescriptor[]; @@ -285,14 +410,14 @@ export class NodeRegistry { } /** Register a websocket client as the current connection for its node id. */ - register(client: GatewayWsClient, opts: { remoteIp?: string | undefined }) { + register(client: GatewayWsClient, opts: NodeSessionRegistrationOptions) { return this.registerSession(client, opts); } /** Register a node whose events are delivered by an HTTP polling transport. */ registerTransport( client: GatewayWsClient, - opts: { remoteIp?: string | undefined }, + opts: NodeSessionRegistrationOptions, transport: NodeEventTransport, ) { return this.registerSession(client, opts, transport); @@ -300,11 +425,15 @@ export class NodeRegistry { private registerSession( client: GatewayWsClient, - opts: { remoteIp?: string | undefined }, + opts: NodeSessionRegistrationOptions, transport?: NodeEventTransport, ) { + if (!opts.pairingIdentity) { + throw new Error("node session registration requires pairing identity"); + } const connect = client.connect; const nodeId = connect.device?.id ?? connect.client.id; + const previousPairingGeneration = this.nodesById.get(nodeId)?.pairingGeneration; const caps = Array.isArray(connect.caps) ? connect.caps : []; const declaredCaps = Array.isArray((connect as { declaredCaps?: string[] }).declaredCaps) ? ((connect as { declaredCaps?: string[] }).declaredCaps ?? []) @@ -346,9 +475,11 @@ export class NodeRegistry { const declaredNodePluginTools: NodePluginToolDescriptor[] = []; const nodePluginTools: NodePluginToolDescriptor[] = []; const nodeSkills: NodeSkillDescriptor[] = []; - const session: NodeSession = { + const session: PairingBoundNodeSession = { nodeId, connId: client.connId, + pairingIdentity: opts.pairingIdentity, + ...(opts.pairingGeneration ? { pairingGeneration: opts.pairingGeneration } : {}), client, clientId: connect.client.id, clientMode: connect.client.mode, @@ -377,6 +508,18 @@ export class NodeRegistry { const replacesPresence = this.nodesById.get(nodeId)?.lastActiveAtMs !== undefined; this.nodesById.set(nodeId, session); this.nodesByConn.set(client.connId, nodeId); + if ( + previousPairingGeneration && + session.pairingGeneration && + previousPairingGeneration !== session.pairingGeneration + ) { + this.options.onPairingGenerationChanged?.({ + nodeId, + previousPairingGeneration, + nextPairingGeneration: session.pairingGeneration, + preserveSessionState: false, + }); + } if (transport) { this.eventTransportsByConn.set(client.connId, transport); } else { @@ -423,12 +566,153 @@ export class NodeRegistry { /** List connected node sessions. */ listConnected(): NodeSession[] { - return [...this.nodesById.values()]; + return this.listConnectedSessions(); + } + + /** Filter connected sessions against an already-loaded pairing-state snapshot. */ + listConnectedForPairingStates( + currentPairingStates: ReadonlyMap, + ): NodeSession[] { + return this.listConnectedSessions().filter((node) => { + const current = currentPairingStates.get(node.nodeId); + return pairingStateMatchesBinding(pairingBindingForSession(node), current); + }); + } + + /** Reconcile connected sessions through the synchronous persistent-pairing owner. */ + listCurrentConnectedSync(): NodeSession[] { + const isPairingStateCurrent = this.options.isPairingStateCurrent; + if (!isPairingStateCurrent) { + return this.listConnected(); + } + const connected: NodeSession[] = []; + let invalidatedPresence = false; + for (const candidate of this.listConnectedSessions()) { + const lease = this.capturePairingLease(candidate); + let isCurrent: boolean; + try { + isCurrent = isPairingStateCurrent(candidate.nodeId, lease.binding); + } catch { + continue; + } + const resolution = this.settlePairingLease({ + lease, + isCurrent, + invalidateStale: true, + }); + if (resolution.status === "current") { + connected.push(resolution.session); + } else if (resolution.status === "stale") { + invalidatedPresence ||= resolution.presenceInvalidated; + } + } + if (invalidatedPresence) { + this.publishActiveNodeContext(); + } + return connected; + } + + /** Resolve persistent pairing state before projecting connected sessions. */ + async listCurrentConnected(): Promise { + const resolved = await Promise.all( + this.listConnectedSessions().map((node) => + this.resolvePairingLease(this.capturePairingLease(node), { invalidateStale: true }), + ), + ); + const connected: NodeSession[] = []; + let invalidatedPresence = false; + for (const result of resolved) { + if (result.status === "current") { + connected.push(result.session); + } else if (result.status === "stale") { + invalidatedPresence ||= result.presenceInvalidated; + } + } + if (invalidatedPresence) { + this.publishActiveNodeContext(); + } + return connected; + } + + private invalidateSessionForPairingChange( + node: NodeSession, + reason = "device-pairing-changed", + ): boolean { + if (this.nodesById.get(node.nodeId) !== node || node.client.invalidated === true) { + return false; + } + node.client.invalidated = true; + node.client.invalidatedReason ??= reason; + removeConnectedNodePluginTools(node.nodeId); + this.invokeStreams.handleDisconnect(node.connId); + for (const [key, event] of this.authorizedSystemRunEvents) { + if (event.connId === node.connId) { + this.authorizedSystemRunEvents.delete(key); + } + } + this.options.onPairingInvalidated?.({ nodeId: node.nodeId, connId: node.connId }); + return node.lastActiveAtMs !== undefined; + } + + /** Immediately retires one exact transport after its persisted pairing authority changes. */ + invalidateConnectionForPairingChange(connId: string, reason = "device-pairing-changed"): boolean { + const nodeId = this.nodesByConn.get(connId); + const node = nodeId ? this.nodesById.get(nodeId) : undefined; + if (!node || node.connId !== connId) { + return false; + } + const invalidatedPresence = this.invalidateSessionForPairingChange(node, reason); + if (invalidatedPresence) { + this.publishActiveNodeContext(); + } + return node.client.invalidated === true; } /** Return a connected node session by node id. */ get(nodeId: string): NodeSession | undefined { - return this.nodesById.get(nodeId); + return this.getRegisteredSession(nodeId); + } + + private getRegisteredSession(nodeId: string): PairingBoundNodeSession | undefined { + const node = this.nodesById.get(nodeId); + return node?.client.invalidated === true ? undefined : node; + } + + /** Return only the session authenticated for the requested persistent pairing generation. */ + getForPairingGeneration(nodeId: string, pairingGeneration: string): NodeSession | undefined { + return this.getRegisteredSessionForPairingGeneration(nodeId, pairingGeneration); + } + + private getRegisteredSessionForPairingGeneration( + nodeId: string, + pairingGeneration: string, + ): PairingBoundNodeSession | undefined { + const node = this.getRegisteredSession(nodeId); + // A mismatch alone does not reveal whether the session or the requesting + // operation is stale, so lookup must not revoke either generation. + return node?.pairingGeneration === pairingGeneration ? node : undefined; + } + + /** Revalidates that one inbound node connection still owns its persisted pairing state. */ + async isConnectionCurrentPairingState(connId: string): Promise { + const nodeId = this.nodesByConn.get(connId); + const initial = nodeId ? this.nodesById.get(nodeId) : undefined; + if ( + !nodeId || + !initial || + initial.connId !== connId || + initial.client.invalidated === true || + !this.options.resolveCurrentPairingState + ) { + return false; + } + const resolution = await this.resolvePairingLease(this.capturePairingLease(initial), { + invalidateStale: true, + }); + if (resolution.status === "stale" && resolution.presenceInvalidated) { + this.publishActiveNodeContext(); + } + return resolution.status === "current"; } /** Updates recent input activity for the exact authenticated node connection. */ @@ -459,9 +743,11 @@ export class NodeRegistry { } /** Returns the connected node with the freshest reported local input. */ - getActiveNode(): NodeSession | undefined { + getActiveNode( + connectedNodes: readonly NodeSession[] = this.listConnected(), + ): NodeSession | undefined { let active: NodeSession | undefined; - for (const node of this.nodesById.values()) { + for (const node of connectedNodes) { if (node.lastActiveAtMs === undefined) { continue; } @@ -478,8 +764,30 @@ export class NodeRegistry { } private publishActiveNodeContext(): void { - const active = this.getActiveNode(); - setActiveNodeContext(active ? { nodeId: active.nodeId } : null); + const active = this.getActiveNode(this.listConnectedSessions()) as + | PairingBoundNodeSession + | undefined; + const lease = active ? this.capturePairingLease(active) : undefined; + setActiveNodeContext( + active + ? { + nodeId: active.nodeId, + ...(active.pairingGeneration ? { pairingGeneration: active.pairingGeneration } : {}), + } + : null, + lease + ? { + isCurrent: () => { + if (!this.currentSessionForLease(lease)) { + return false; + } + return this.options.isPairingStateCurrent + ? this.options.isPairingStateCurrent(lease.nodeId, lease.binding) + : true; + }, + } + : undefined, + ); } /** Probe websocket liveness with ping/pong when the socket supports it. */ @@ -611,9 +919,22 @@ export class NodeRegistry { commands: readonly string[]; permissions?: Record | undefined; }, + generationTransition?: { + expectedConnId: string; + expectedPairingIdentity: string; + expectedPairingGeneration?: string; + nextPairingGeneration: string; + }, ): NodeSession | null { const node = this.nodesById.get(nodeId); - if (!node) { + if ( + !node || + node.client.invalidated === true || + (generationTransition !== undefined && + (node.connId !== generationTransition.expectedConnId || + node.pairingIdentity !== generationTransition.expectedPairingIdentity || + node.pairingGeneration !== generationTransition.expectedPairingGeneration)) + ) { return null; } @@ -638,29 +959,46 @@ export class NodeRegistry { node.permissions = undefined; (node.client.connect as { permissions?: Record }).permissions = undefined; this.clearPresenceIfAccessibilityUnavailable(node); - return node; + } else { + const declared = node.declaredPermissions ?? {}; + const nextEntries: Array<[string, boolean]> = []; + for (const [key, declaredValue] of Object.entries(declared)) { + if (!declaredValue) { + nextEntries.push([key, false]); + continue; + } + const approvedValue = surface.permissions?.[key]; + if (approvedValue) { + nextEntries.push([key, true]); + continue; + } + if (approvedValue !== undefined) { + nextEntries.push([key, false]); + } + } + const nextPermissions = + nextEntries.length > 0 ? Object.fromEntries(nextEntries) : undefined; + node.permissions = nextPermissions; + (node.client.connect as { permissions?: Record }).permissions = + nextPermissions; + this.clearPresenceIfAccessibilityUnavailable(node); } - const declared = node.declaredPermissions ?? {}; - const nextEntries: Array<[string, boolean]> = []; - for (const [key, declaredValue] of Object.entries(declared)) { - if (!declaredValue) { - nextEntries.push([key, false]); - continue; - } - const approvedValue = surface.permissions?.[key]; - if (approvedValue) { - nextEntries.push([key, true]); - continue; - } - if (approvedValue !== undefined) { - nextEntries.push([key, false]); - } + } + + if (generationTransition) { + const previousPairingGeneration = node.pairingGeneration; + node.pairingGeneration = generationTransition.nextPairingGeneration; + if (previousPairingGeneration) { + this.options.onPairingGenerationChanged?.({ + nodeId, + previousPairingGeneration, + nextPairingGeneration: generationTransition.nextPairingGeneration, + preserveSessionState: true, + }); } - const nextPermissions = nextEntries.length > 0 ? Object.fromEntries(nextEntries) : undefined; - node.permissions = nextPermissions; - (node.client.connect as { permissions?: Record }).permissions = - nextPermissions; - this.clearPresenceIfAccessibilityUnavailable(node); + // Active-node leases capture the pairing generation, so a promoted live + // session must republish its lease even when its presence is unchanged. + this.publishActiveNodeContext(); } return node; @@ -678,6 +1016,7 @@ export class NodeRegistry { async invoke(params: { nodeId: string; expectedConnId?: string; + expectedPairingGeneration?: string; command: string; params?: unknown; timeoutMs?: number; @@ -693,19 +1032,65 @@ export class NodeRegistry { if (params.signal?.aborted) { return { ok: false, error: { code: "ABORTED", message: "node invoke cancelled" } }; } - const node = this.nodesById.get(params.nodeId); + let node = this.nodesById.get(params.nodeId); if (!node) { return { ok: false, error: { code: "NOT_CONNECTED", message: "node not connected" }, }; } + if (node.client.invalidated === true) { + return { + ok: false, + error: { code: "PAIRING_CHANGED", message: "node pairing changed before dispatch" }, + }; + } + const expectedPairingGeneration = params.expectedPairingGeneration ?? node.pairingGeneration; + if (this.options.resolveCurrentPairingState && !expectedPairingGeneration) { + return { + ok: false, + error: { code: "PAIRING_CHANGED", message: "node pairing generation unavailable" }, + }; + } + if (expectedPairingGeneration && node.pairingGeneration !== expectedPairingGeneration) { + return { + ok: false, + error: { code: "PAIRING_CHANGED", message: "node pairing changed before dispatch" }, + }; + } if (params.expectedConnId && node.connId !== params.expectedConnId) { return { ok: false, error: { code: "ROUTE_CHANGED", message: "node connection changed before dispatch" }, }; } + if (expectedPairingGeneration && this.options.resolveCurrentPairingState) { + const resolution = await this.resolvePairingLease(this.capturePairingLease(node), { + invalidateStale: false, + }); + if (resolution.status === "unavailable") { + return { + ok: false, + error: { + code: "UNAVAILABLE", + message: "node pairing state unavailable before dispatch", + }, + }; + } + if (resolution.status !== "current") { + return { + ok: false, + error: { code: "PAIRING_CHANGED", message: "node pairing changed before dispatch" }, + }; + } + node = resolution.session; + if (params.expectedConnId && node.connId !== params.expectedConnId) { + return { + ok: false, + error: { code: "ROUTE_CHANGED", message: "node connection changed before dispatch" }, + }; + } + } const requestId = randomUUID(); const invokeParams = normalizeSystemRunInvokeParams({ command: params.command, @@ -958,7 +1343,90 @@ export class NodeRegistry { return this.sendEventRawInternal(node, event, payloadJSON); } + /** Sends command-free events only to the exact authenticated pairing connection. */ + async sendEventForPairingIdentity(params: { + nodeId: string; + connId: string; + pairingIdentity: string; + event: string; + payload?: unknown; + }): Promise { + const initial = this.nodesById.get(params.nodeId); + if ( + !initial || + initial.connId !== params.connId || + initial.pairingIdentity !== params.pairingIdentity || + initial.client.invalidated === true || + !this.options.resolveCurrentPairingState + ) { + return false; + } + const resolution = await this.resolvePairingLease(this.capturePairingLease(initial), { + invalidateStale: true, + }); + if (resolution.status !== "current") { + if (resolution.status === "stale" && resolution.presenceInvalidated) { + this.publishActiveNodeContext(); + } + return false; + } + return this.sendEventToSession(resolution.session, params.event, params.payload); + } + + /** Sends only to a session that still owns the requested persistent pairing generation. */ + async sendEventRawForPairingGeneration( + nodeId: string, + pairingGeneration: string, + event: string, + payloadJSON?: SerializedEventPayload | null, + ): Promise { + const previous = this.pairingGenerationEventChains.get(nodeId) ?? Promise.resolve(); + const send = previous.then(() => + this.sendEventRawForPairingGenerationNow(nodeId, pairingGeneration, event, payloadJSON), + ); + const tail = send.then( + () => undefined, + () => undefined, + ); + this.pairingGenerationEventChains.set(nodeId, tail); + try { + return await send; + } finally { + if (this.pairingGenerationEventChains.get(nodeId) === tail) { + this.pairingGenerationEventChains.delete(nodeId); + } + } + } + + private async sendEventRawForPairingGenerationNow( + nodeId: string, + pairingGeneration: string, + event: string, + payloadJSON?: SerializedEventPayload | null, + ): Promise { + let node = this.getRegisteredSessionForPairingGeneration(nodeId, pairingGeneration); + if (!node) { + return false; + } + if (this.options.resolveCurrentPairingState) { + const resolution = await this.resolvePairingLease(this.capturePairingLease(node), { + invalidateStale: true, + }); + if (resolution.status !== "current") { + if (resolution.status === "stale" && resolution.presenceInvalidated) { + this.publishActiveNodeContext(); + } + return false; + } + node = resolution.session; + } + return this.sendEventRawInternal(node, event, payloadJSON); + } + private sendEventInternal(node: NodeSession, event: string, payload: unknown): boolean { + if (node.client.invalidated === true) { + return false; + } const eventTransport = this.eventTransportsByConn.get(node.connId); if (eventTransport) { return eventTransport.send(event, payload); @@ -985,6 +1453,9 @@ export class NodeRegistry { event: string, payloadJSON?: SerializedEventPayload | null, ): boolean { + if (node.client.invalidated === true) { + return false; + } if ( payloadJSON !== null && payloadJSON !== undefined && diff --git a/src/gateway/node-runtime-state.ts b/src/gateway/node-runtime-state.ts new file mode 100644 index 000000000000..7594cea67258 --- /dev/null +++ b/src/gateway/node-runtime-state.ts @@ -0,0 +1,185 @@ +// Process-local node state shared by node and full-device pairing removal. +import { randomUUID } from "node:crypto"; +import { removeRemoteNodeInfo } from "../skills/runtime/remote.js"; +import { clearNodePendingWork } from "./node-pending-work.js"; +import type { NodeRegistry } from "./node-registry.js"; +import { invalidateNodeWakeState } from "./node-wake-state.js"; + +export type PendingNodeAction = { + id: string; + nodeId: string; + pairingGeneration: string; + command: string; + paramsJSON?: string; + idempotencyKey: string; + enqueuedAtMs: number; +}; + +const pendingNodeActionsById = new Map(); + +function prunePendingNodeActions(params: { + nodeId: string; + nowMs: number; + ttlMs: number; + pairingGeneration?: string; +}): PendingNodeAction[] { + const queue = pendingNodeActionsById.get(params.nodeId) ?? []; + const minTimestampMs = params.nowMs - params.ttlMs; + const live = queue.filter((entry) => entry.enqueuedAtMs >= minTimestampMs); + if (live.length === 0) { + pendingNodeActionsById.delete(params.nodeId); + return []; + } + pendingNodeActionsById.set(params.nodeId, live); + return params.pairingGeneration + ? live.filter((entry) => entry.pairingGeneration === params.pairingGeneration) + : live; +} + +export function replacePendingNodeActionsForGeneration(params: { + nodeId: string; + pairingGeneration: string; + replacement: PendingNodeAction[]; + ttlMs: number; + nowMs?: number; +}): void { + const live = prunePendingNodeActions({ + nodeId: params.nodeId, + nowMs: params.nowMs ?? Date.now(), + ttlMs: params.ttlMs, + }); + const next = [ + ...live.filter((entry) => entry.pairingGeneration !== params.pairingGeneration), + ...params.replacement, + ]; + if (next.length === 0) { + pendingNodeActionsById.delete(params.nodeId); + return; + } + pendingNodeActionsById.set(params.nodeId, next); +} + +export function enqueuePendingNodeAction(params: { + nodeId: string; + pairingGeneration: string; + command: string; + paramsJSON?: string; + idempotencyKey: string; + ttlMs: number; + maxPerNode: number; + nowMs?: number; +}): { action: PendingNodeAction; created: boolean } { + const nowMs = params.nowMs ?? Date.now(); + const queue = prunePendingNodeActions({ + nodeId: params.nodeId, + nowMs, + ttlMs: params.ttlMs, + pairingGeneration: params.pairingGeneration, + }); + const existing = queue.find((entry) => entry.idempotencyKey === params.idempotencyKey); + if (existing) { + return { action: existing, created: false }; + } + const action: PendingNodeAction = { + id: randomUUID(), + nodeId: params.nodeId, + pairingGeneration: params.pairingGeneration, + command: params.command, + paramsJSON: params.paramsJSON, + idempotencyKey: params.idempotencyKey, + enqueuedAtMs: nowMs, + }; + queue.push(action); + if (queue.length > params.maxPerNode) { + queue.splice(0, queue.length - params.maxPerNode); + } + replacePendingNodeActionsForGeneration({ + nodeId: params.nodeId, + pairingGeneration: params.pairingGeneration, + replacement: queue, + ttlMs: params.ttlMs, + nowMs, + }); + return { action, created: true }; +} + +export function listPendingNodeActions(params: { + nodeId: string; + pairingGeneration?: string; + ttlMs: number; + nowMs?: number; +}): PendingNodeAction[] { + return prunePendingNodeActions({ + nodeId: params.nodeId, + nowMs: params.nowMs ?? Date.now(), + ttlMs: params.ttlMs, + pairingGeneration: params.pairingGeneration, + }); +} + +export function acknowledgePendingNodeActions(params: { + nodeId: string; + pairingGeneration: string; + ids: readonly string[]; + ttlMs: number; +}): PendingNodeAction[] { + const pending = prunePendingNodeActions({ + nodeId: params.nodeId, + pairingGeneration: params.pairingGeneration, + nowMs: Date.now(), + ttlMs: params.ttlMs, + }); + if (params.ids.length === 0) { + return pending; + } + const ids = new Set(params.ids); + const remaining = pending.filter((entry) => !ids.has(entry.id)); + replacePendingNodeActionsForGeneration({ + ...params, + replacement: remaining, + }); + return remaining; +} + +export function removePendingNodeAction(params: { + nodeId: string; + pairingGeneration: string; + actionId: string; + ttlMs: number; +}): void { + const pending = prunePendingNodeActions({ + nodeId: params.nodeId, + pairingGeneration: params.pairingGeneration, + nowMs: Date.now(), + ttlMs: params.ttlMs, + }); + const remaining = pending.filter((entry) => entry.id !== params.actionId); + if (remaining.length === pending.length) { + return; + } + replacePendingNodeActionsForGeneration({ + ...params, + replacement: remaining, + }); +} + +function clearPendingNodeActions(nodeId: string): void { + pendingNodeActionsById.delete(nodeId); +} + +export function clearRemovedNodeRuntimeState(params: { + nodeId: string; + context: { + nodeRegistry: Pick; + }; +}) { + clearPendingNodeActions(params.nodeId); + clearNodePendingWork(params.nodeId); + invalidateNodeWakeState(params.nodeId); + params.context.nodeRegistry.updateSurface(params.nodeId, { + caps: [], + commands: [], + permissions: undefined, + }); + removeRemoteNodeInfo(params.nodeId); +} diff --git a/src/gateway/node-wake-state-store.ts b/src/gateway/node-wake-state-store.ts new file mode 100644 index 000000000000..4c9f8e4d1a08 --- /dev/null +++ b/src/gateway/node-wake-state-store.ts @@ -0,0 +1,27 @@ +type StoredNodeWakeAttempt = { + available: boolean; + throttled: boolean; + path: "throttled" | "no-registration" | "no-auth" | "sent" | "send-error" | "invalidated"; + durationMs: number; + apnsStatus?: number; + apnsReason?: string; +}; + +export type NodeWakeOwnerState = { + nodeId: string; + stateKey: string; + lastWakeAtMs: number; + inFlightWake?: Promise; + lastNudgeAtMs: number; + lifecycle?: { + controller: AbortController; + users: number; + }; +}; + +export const nodeWakeStateByOwner = new Map(); +export const nodeWakeOwnerBySignal = new WeakMap(); + +export function nodeWakeStateKey(nodeId: string, pairingGeneration?: string): string { + return JSON.stringify([nodeId.trim(), pairingGeneration?.trim() || null]); +} diff --git a/src/gateway/node-wake-state.test-support.ts b/src/gateway/node-wake-state.test-support.ts new file mode 100644 index 000000000000..2a7c5a5fd391 --- /dev/null +++ b/src/gateway/node-wake-state.test-support.ts @@ -0,0 +1,31 @@ +import { nodeWakeStateByOwner, nodeWakeStateKey } from "./node-wake-state-store.js"; + +/** Read-only test projection; callers cannot mutate lifecycle ownership. */ +export function getNodeWakeStateSnapshot( + nodeId: string, + pairingGeneration?: string, +): + | { + lastWakeAtMs: number; + wakeInFlight: boolean; + lastNudgeAtMs: number; + lifecycleUsers: number; + } + | undefined { + const owner = nodeWakeStateByOwner.get(nodeWakeStateKey(nodeId, pairingGeneration)); + return owner + ? { + lastWakeAtMs: owner.lastWakeAtMs, + wakeInFlight: owner.inFlightWake !== undefined, + lastNudgeAtMs: owner.lastNudgeAtMs, + lifecycleUsers: owner.lifecycle?.users ?? 0, + } + : undefined; +} + +export function resetNodeWakeStateForTest(): void { + for (const owner of nodeWakeStateByOwner.values()) { + owner.lifecycle?.controller.abort(); + } + nodeWakeStateByOwner.clear(); +} diff --git a/src/gateway/node-wake-state.test.ts b/src/gateway/node-wake-state.test.ts new file mode 100644 index 000000000000..42f51dee0ec9 --- /dev/null +++ b/src/gateway/node-wake-state.test.ts @@ -0,0 +1,145 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + captureNodeWakeLifecycle, + clearNodeWakeState, + invalidateNodeWakeState, + isNodeWakeLifecycleCurrent, + releaseNodeWakeLifecycle, + runNodeWakeAttempt, + runNodeWakeNudgeAttempt, + type NodeWakeAttempt, +} from "./node-wake-state.js"; +import { + getNodeWakeStateSnapshot, + resetNodeWakeStateForTest, +} from "./node-wake-state.test-support.js"; + +const sentWake: NodeWakeAttempt = { + available: true, + throttled: false, + path: "sent", + durationMs: 1, +}; + +beforeEach(() => { + resetNodeWakeStateForTest(); +}); + +describe("node wake lifecycle ownership", () => { + it("isolates generations and invalidates every generation for a removed node", () => { + const generationOne = captureNodeWakeLifecycle("node-1", "generation-1"); + const generationTwo = captureNodeWakeLifecycle("node-1", "generation-2"); + + expect(isNodeWakeLifecycleCurrent("node-1", generationOne, "generation-1")).toBe(true); + expect(isNodeWakeLifecycleCurrent("node-1", generationOne, "generation-2")).toBe(false); + expect(isNodeWakeLifecycleCurrent("node-1", generationTwo, "generation-2")).toBe(true); + + invalidateNodeWakeState("node-1"); + + expect(generationOne.aborted).toBe(true); + expect(generationTwo.aborted).toBe(true); + expect(getNodeWakeStateSnapshot("node-1", "generation-1")).toBeUndefined(); + expect(getNodeWakeStateSnapshot("node-1", "generation-2")).toBeUndefined(); + }); + + it("releases an idle lifecycle without retaining owner state", () => { + const lifecycle = captureNodeWakeLifecycle("node-idle"); + + releaseNodeWakeLifecycle("node-idle", lifecycle); + + expect(lifecycle.aborted).toBe(true); + expect(getNodeWakeStateSnapshot("node-idle")).toBeUndefined(); + }); + + it("clears throttle state without aborting an active lifecycle", async () => { + const lifecycle = captureNodeWakeLifecycle("node-active"); + await runNodeWakeAttempt({ + nodeId: "node-active", + force: true, + throttleMs: 1_000, + attempt: async (markAttempted) => { + markAttempted(); + return sentWake; + }, + }); + + clearNodeWakeState("node-active"); + + expect(lifecycle.aborted).toBe(false); + expect(getNodeWakeStateSnapshot("node-active")?.lastWakeAtMs).toBe(0); + releaseNodeWakeLifecycle("node-active", lifecycle); + }); +}); + +describe("node wake coordination", () => { + it("deduplicates concurrent wake attempts for one generation", async () => { + let finish: ((attempt: NodeWakeAttempt) => void) | undefined; + const attempt = vi.fn( + () => + new Promise((resolve) => { + finish = resolve; + }), + ); + const params = { + nodeId: "node-1", + pairingGeneration: "generation-1", + force: false, + throttleMs: 1_000, + attempt, + }; + + const first = runNodeWakeAttempt(params); + const second = runNodeWakeAttempt(params); + expect(attempt).toHaveBeenCalledOnce(); + finish?.(sentWake); + + await expect(first).resolves.toEqual(sentWake); + await expect(second).resolves.toEqual(sentWake); + }); + + it("throttles only after transport admission marks a real wake attempt", async () => { + await runNodeWakeAttempt({ + nodeId: "node-1", + force: false, + throttleMs: 60_000, + attempt: async (markAttempted) => { + markAttempted(); + return sentWake; + }, + }); + + const second = await runNodeWakeAttempt({ + nodeId: "node-1", + force: false, + throttleMs: 60_000, + attempt: async () => sentWake, + }); + + expect(second).toEqual({ + available: true, + throttled: true, + path: "throttled", + durationMs: 0, + }); + }); + + it("tracks reconnect-nudge throttle independently from wake throttle", async () => { + const sent = await runNodeWakeNudgeAttempt({ + nodeId: "node-1", + throttleMs: 60_000, + throttled: () => ({ sent: false, throttled: true, reason: "throttled", durationMs: 0 }), + attempt: async () => ({ sent: true, throttled: false, reason: "sent", durationMs: 1 }), + }); + const throttled = await runNodeWakeNudgeAttempt({ + nodeId: "node-1", + throttleMs: 60_000, + throttled: () => ({ sent: false, throttled: true, reason: "throttled", durationMs: 0 }), + attempt: async () => ({ sent: true, throttled: false, reason: "sent", durationMs: 1 }), + }); + + expect(sent.reason).toBe("sent"); + expect(throttled.reason).toBe("throttled"); + expect(getNodeWakeStateSnapshot("node-1")?.lastWakeAtMs).toBe(0); + expect(getNodeWakeStateSnapshot("node-1")?.lastNudgeAtMs).toBeGreaterThan(0); + }); +}); diff --git a/src/gateway/node-wake-state.ts b/src/gateway/node-wake-state.ts new file mode 100644 index 000000000000..8867d1da8545 --- /dev/null +++ b/src/gateway/node-wake-state.ts @@ -0,0 +1,194 @@ +// Gateway-owned APNs wake, reconnect nudge, and cancellation state. +import { + nodeWakeOwnerBySignal, + nodeWakeStateByOwner, + nodeWakeStateKey, + type NodeWakeOwnerState, +} from "./node-wake-state-store.js"; + +export const NODE_WAKE_RECONNECT_WAIT_MS = 3_000; +export const NODE_WAKE_RECONNECT_RETRY_WAIT_MS = 12_000; +export const NODE_WAKE_RECONNECT_POLL_MS = 150; + +export type NodeWakeAttempt = { + available: boolean; + throttled: boolean; + path: "throttled" | "no-registration" | "no-auth" | "sent" | "send-error" | "invalidated"; + durationMs: number; + apnsStatus?: number; + apnsReason?: string; +}; + +export type NodeWakeNudgeAttempt = { + sent: boolean; + throttled: boolean; + reason: + | "throttled" + | "no-registration" + | "no-auth" + | "send-error" + | "apns-not-ok" + | "sent" + | "invalidated"; + durationMs: number; + apnsStatus?: number; + apnsReason?: string; +}; + +export type NodeWakeLifecycle = AbortSignal; + +function getOrCreateNodeWakeOwner(nodeId: string, pairingGeneration?: string): NodeWakeOwnerState { + const normalizedNodeId = nodeId.trim(); + const stateKey = nodeWakeStateKey(normalizedNodeId, pairingGeneration); + const existing = nodeWakeStateByOwner.get(stateKey); + if (existing) { + return existing; + } + const created: NodeWakeOwnerState = { + nodeId: normalizedNodeId, + stateKey, + lastWakeAtMs: 0, + lastNudgeAtMs: 0, + }; + nodeWakeStateByOwner.set(stateKey, created); + return created; +} + +function deleteIdleNodeWakeOwner(owner: NodeWakeOwnerState): void { + if ( + owner.lifecycle?.users || + owner.inFlightWake || + owner.lastWakeAtMs > 0 || + owner.lastNudgeAtMs > 0 + ) { + return; + } + owner.lifecycle?.controller.abort(); + if (owner.lifecycle) { + nodeWakeOwnerBySignal.delete(owner.lifecycle.controller.signal); + } + nodeWakeStateByOwner.delete(owner.stateKey); +} + +export function captureNodeWakeLifecycle( + nodeId: string, + pairingGeneration?: string, +): NodeWakeLifecycle { + const owner = getOrCreateNodeWakeOwner(nodeId, pairingGeneration); + if (!owner.lifecycle || owner.lifecycle.controller.signal.aborted) { + owner.lifecycle = { controller: new AbortController(), users: 0 }; + nodeWakeOwnerBySignal.set(owner.lifecycle.controller.signal, owner); + } + owner.lifecycle.users += 1; + return owner.lifecycle.controller.signal; +} + +export function isNodeWakeLifecycleCurrent( + nodeId: string, + lifecycle: NodeWakeLifecycle, + pairingGeneration?: string, +): boolean { + const owner = nodeWakeOwnerBySignal.get(lifecycle); + const expectedStateKey = nodeWakeStateKey(nodeId, pairingGeneration); + return ( + !lifecycle.aborted && + owner?.nodeId === nodeId.trim() && + owner.stateKey === expectedStateKey && + nodeWakeStateByOwner.get(expectedStateKey) === owner && + owner.lifecycle?.controller.signal === lifecycle + ); +} + +export function releaseNodeWakeLifecycle(nodeId: string, lifecycle: NodeWakeLifecycle): void { + const owner = nodeWakeOwnerBySignal.get(lifecycle); + if ( + owner?.nodeId !== nodeId.trim() || + nodeWakeStateByOwner.get(owner.stateKey) !== owner || + owner.lifecycle?.controller.signal !== lifecycle + ) { + return; + } + owner.lifecycle.users = Math.max(0, owner.lifecycle.users - 1); + deleteIdleNodeWakeOwner(owner); +} + +/** Owns wake dedupe and throttle state while the caller owns APNs policy and I/O. */ +export async function runNodeWakeAttempt(params: { + nodeId: string; + pairingGeneration?: string; + force: boolean; + throttleMs: number; + attempt: (markAttempted: () => void) => Promise; +}): Promise { + const owner = getOrCreateNodeWakeOwner(params.nodeId, params.pairingGeneration); + if (owner.inFlightWake) { + return await owner.inFlightWake; + } + if ( + !params.force && + owner.lastWakeAtMs > 0 && + Date.now() - owner.lastWakeAtMs < params.throttleMs + ) { + return { available: true, throttled: true, path: "throttled", durationMs: 0 }; + } + + const attempt = params.attempt(() => { + owner.lastWakeAtMs = Date.now(); + }); + owner.inFlightWake = attempt; + try { + return await attempt; + } finally { + if (owner.inFlightWake === attempt) { + owner.inFlightWake = undefined; + } + deleteIdleNodeWakeOwner(owner); + } +} + +/** Owns reconnect-nudge throttling while the caller owns APNs policy and I/O. */ +export async function runNodeWakeNudgeAttempt(params: { + nodeId: string; + pairingGeneration?: string; + throttleMs: number; + throttled: () => NodeWakeNudgeAttempt; + attempt: () => Promise; +}): Promise { + const owner = getOrCreateNodeWakeOwner(params.nodeId, params.pairingGeneration); + if (owner.lastNudgeAtMs > 0 && Date.now() - owner.lastNudgeAtMs < params.throttleMs) { + return params.throttled(); + } + const result = await params.attempt(); + if (result.reason === "sent") { + owner.lastNudgeAtMs = Date.now(); + } + deleteIdleNodeWakeOwner(owner); + return result; +} + +export function clearNodeWakeState(nodeId: string): void { + const normalizedNodeId = nodeId.trim(); + for (const owner of nodeWakeStateByOwner.values()) { + if (owner.nodeId !== normalizedNodeId) { + continue; + } + owner.lastWakeAtMs = 0; + owner.inFlightWake = undefined; + owner.lastNudgeAtMs = 0; + deleteIdleNodeWakeOwner(owner); + } +} + +export function invalidateNodeWakeState(nodeId: string): void { + const normalizedNodeId = nodeId.trim(); + for (const owner of nodeWakeStateByOwner.values()) { + if (owner.nodeId !== normalizedNodeId) { + continue; + } + owner.lifecycle?.controller.abort(); + if (owner.lifecycle) { + nodeWakeOwnerBySignal.delete(owner.lifecycle.controller.signal); + } + nodeWakeStateByOwner.delete(owner.stateKey); + } +} diff --git a/src/gateway/server-broadcast.ts b/src/gateway/server-broadcast.ts index 4f21654fbcb7..8e29b70f4ab4 100644 --- a/src/gateway/server-broadcast.ts +++ b/src/gateway/server-broadcast.ts @@ -80,11 +80,6 @@ const EVENT_SCOPE_GUARDS: Record = { "terminal.exit": [ADMIN_SCOPE], }; -// Events that node-role sessions must receive even when the event's operator -// scope would otherwise reject non-operator roles. Nodes act on these updates -// (e.g. reconfiguring wake-word triggers). -const NODE_ALLOWED_EVENTS = new Set(["voicewake.changed", "voicewake.routing.changed"]); - // Opt-in scoped clients never receive session-bearing broadcasts without an // authoritative registry key, including malformed/sessionless agent events. const SESSION_SUBSCRIPTION_EVENTS = new Set([ @@ -143,7 +138,7 @@ function hasEventScope( } const role = client.connect.role ?? "operator"; if (role !== "operator") { - return role === "node" && NODE_ALLOWED_EVENTS.has(event); + return false; } const scopes = Array.isArray(client.connect.scopes) ? client.connect.scopes : []; if (scopes.includes(ADMIN_SCOPE)) { @@ -209,6 +204,9 @@ export function createGatewayBroadcaster(params: { return frameBase; }; for (const c of params.clients) { + if (c.invalidated === true) { + continue; + } if (targetConnIds && !targetConnIds.has(c.connId)) { continue; } diff --git a/src/gateway/server-methods.authorization.test.ts b/src/gateway/server-methods.authorization.test.ts index f736e11c79ca..9cb5898e4923 100644 --- a/src/gateway/server-methods.authorization.test.ts +++ b/src/gateway/server-methods.authorization.test.ts @@ -84,6 +84,51 @@ describe("gateway method authorization", () => { }); }); + it("rejects every node RPC when its connection no longer owns the pairing generation", async () => { + const handler = vi.fn(({ respond }) => respond(true, { ok: true })); + const respond = vi.fn(); + const isConnectionCurrentPairingState = vi.fn().mockResolvedValue(false); + + await handleGatewayRequest({ + req: { type: "req", id: "req-node-stale", method: "node.event", params: { event: "test" } }, + respond, + client: { + connId: "conn-node-stale", + connect: { + role: "node", + scopes: [], + device: { + id: "node-stale", + publicKey: "public-key", + signature: "signature", + signedAt: 1, + nonce: "nonce", + }, + client: { id: "node-host", version: "1", platform: "test", mode: "node" }, + minProtocol: 1, + maxProtocol: 1, + }, + } as Parameters[0]["client"], + isWebchatConnect: () => false, + context: { + logGateway: { warn: vi.fn() }, + nodeRegistry: { isConnectionCurrentPairingState }, + } as unknown as Parameters[0]["context"], + extraHandlers: { "node.event": handler }, + }); + + expect(isConnectionCurrentPairingState).toHaveBeenCalledWith("conn-node-stale"); + expect(handler).not.toHaveBeenCalled(); + expect(respond).toHaveBeenCalledWith( + false, + undefined, + expect.objectContaining({ + code: "UNAVAILABLE", + details: { code: "PAIRING_CHANGED" }, + }), + ); + }); + async function dispatchProfileMutation(params: { authenticatedUserId?: string; profileId: string; diff --git a/src/gateway/server-methods.ts b/src/gateway/server-methods.ts index 5e8b6d948cdc..d4b4e46799eb 100644 --- a/src/gateway/server-methods.ts +++ b/src/gateway/server-methods.ts @@ -914,6 +914,20 @@ export async function handleGatewayRequest( respond(false, undefined, authError); return; } + if ( + client?.connect.role === "node" && + (!client.connId || !(await context.nodeRegistry.isConnectionCurrentPairingState(client.connId))) + ) { + respond( + false, + undefined, + errorShape(ErrorCodes.UNAVAILABLE, "node pairing changed before request dispatch", { + retryable: true, + details: { code: "PAIRING_CHANGED" }, + }), + ); + return; + } if (context.unavailableGatewayMethods?.has(req.method)) { // During startup, methods can be listed before their runtime is ready. Return the protocol // retry shape so clients can back off without treating startup as a permanent unknown method. diff --git a/src/gateway/server-methods/devices.test.ts b/src/gateway/server-methods/devices.test.ts index 33811f7c2d77..d0bebf54478a 100644 --- a/src/gateway/server-methods/devices.test.ts +++ b/src/gateway/server-methods/devices.test.ts @@ -8,6 +8,17 @@ import { resetDiagnosticEventsForTest, type DiagnosticSecurityEvent, } from "../../infra/diagnostic-events.js"; +import { drainNodePendingWork, enqueueNodePendingWork } from "../node-pending-work.js"; +import { + captureNodeWakeLifecycle, + releaseNodeWakeLifecycle, + runNodeWakeAttempt, + runNodeWakeNudgeAttempt, +} from "../node-wake-state.js"; +import { + getNodeWakeStateSnapshot, + resetNodeWakeStateForTest, +} from "../node-wake-state.test-support.js"; import { deviceHandlers } from "./devices.js"; import type { GatewayRequestHandlerOptions } from "./types.js"; @@ -88,6 +99,9 @@ function createOptions( info: vi.fn(), warn: vi.fn(), }, + nodeRegistry: { + updateSurface: vi.fn(), + }, }, ...overrides, } as unknown as GatewayRequestHandlerOptions; @@ -145,12 +159,60 @@ function captureSecurityEvents(): { return { events, stop }; } +async function seedNodeWakeState(nodeId: string): Promise { + await runNodeWakeAttempt({ + nodeId, + force: true, + throttleMs: 60_000, + attempt: async (markAttempted) => { + markAttempted(); + return { available: true, throttled: false, path: "sent", durationMs: 1 }; + }, + }); + await runNodeWakeNudgeAttempt({ + nodeId, + throttleMs: 60_000, + throttled: () => ({ sent: false, throttled: true, reason: "throttled", durationMs: 0 }), + attempt: async () => ({ sent: true, throttled: false, reason: "sent", durationMs: 1 }), + }); +} + describe("deviceHandlers", () => { beforeEach(() => { resetDiagnosticEventsForTest(); + resetNodeWakeStateForTest(); vi.clearAllMocks(); }); + it("clears and invalidates node runtime state after removing a full device pairing", async () => { + const nodeId = "disconnected-node-device"; + removePairedDeviceMock.mockResolvedValue({ deviceId: nodeId }); + await seedNodeWakeState(nodeId); + enqueueNodePendingWork({ nodeId, type: "location.request" }); + const wakeLifecycle = captureNodeWakeLifecycle(nodeId); + const opts = createOptions("device.pair.remove", { deviceId: nodeId }); + + await expectDefined( + deviceHandlers["device.pair.remove"], + 'deviceHandlers["device.pair.remove"] test invariant', + )(opts); + + expect(getNodeWakeStateSnapshot(nodeId)).toBeUndefined(); + expect(wakeLifecycle.aborted).toBe(true); + expect(drainNodePendingWork(nodeId).items.map((item) => item.id)).toEqual(["baseline-status"]); + const nodeRegistry = opts.context.nodeRegistry as unknown as { + updateSurface: ReturnType; + }; + expect(nodeRegistry.updateSurface).toHaveBeenCalledWith(nodeId, { + caps: [], + commands: [], + permissions: undefined, + }); + expect(opts.context.invalidateClientsForDevice).toHaveBeenCalledWith(nodeId, { + reason: "device-pair-removed", + }); + }); + it("disconnects active clients after removing a paired device", async () => { removePairedDeviceMock.mockResolvedValue({ deviceId: "device-1", removedAtMs: 123 }); const opts = createOptions("device.pair.remove", { deviceId: " device-1 " }); @@ -477,6 +539,70 @@ describe("deviceHandlers", () => { ); }); + it("invalidates an in-flight node wake when the node token rotates", async () => { + rotateDeviceTokenMock.mockResolvedValue({ + ok: true, + entry: { + token: "new-node-token", + role: "node", + scopes: [], + createdAtMs: 456, + rotatedAtMs: 789, + }, + }); + const lifecycle = captureNodeWakeLifecycle("device-1"); + const opts = createOptions( + "device.token.rotate", + { deviceId: "device-1", role: "node" }, + { client: createClient(["operator.admin"], "admin-device", { isDeviceTokenAuth: true }) }, + ); + + await expectDefined( + deviceHandlers["device.token.rotate"], + 'deviceHandlers["device.token.rotate"] test invariant', + )(opts); + + expect(lifecycle.aborted).toBe(true); + }); + + it("invalidates an in-flight node wake when the node token is revoked", async () => { + revokeDeviceTokenMock.mockResolvedValue({ + ok: true, + entry: { role: "node", revokedAtMs: 789 }, + }); + const lifecycle = captureNodeWakeLifecycle("device-1"); + const opts = createOptions( + "device.token.revoke", + { deviceId: "device-1", role: "node" }, + { client: createClient(["operator.admin"], "admin-device", { isDeviceTokenAuth: true }) }, + ); + + await expectDefined( + deviceHandlers["device.token.revoke"], + 'deviceHandlers["device.token.revoke"] test invariant', + )(opts); + + expect(lifecycle.aborted).toBe(true); + }); + + it("keeps node wake ownership across unrelated operator token rotation", async () => { + mockRotateOperatorTokenSuccess(); + const lifecycle = captureNodeWakeLifecycle("device-1"); + const opts = createOptions("device.token.rotate", { + deviceId: "device-1", + role: "operator", + scopes: ["operator.pairing"], + }); + + await expectDefined( + deviceHandlers["device.token.rotate"], + 'deviceHandlers["device.token.rotate"] test invariant', + )(opts); + + expect(lifecycle.aborted).toBe(false); + releaseNodeWakeLifecycle("device-1", lifecycle); + }); + it("invalidates affected clients synchronously before responding to device.token.rotate", async () => { mockPairedOperatorDevice(); mockRotateOperatorTokenSuccess(); @@ -1054,6 +1180,44 @@ describe("deviceHandlers", () => { expect(serialized).not.toContain("pk-2"); }); + it("retires the previous node generation before returning reapproval success", async () => { + approveDevicePairingMock.mockResolvedValue({ + status: "approved", + requestId: "req-node-repair", + nodePairingGenerationChanged: true, + device: { + deviceId: "node-repaired", + publicKey: "replacement-key", + role: "node", + roles: ["node"], + approvedAtMs: 200, + createdAtMs: 100, + }, + }); + const lifecycle = captureNodeWakeLifecycle("node-repaired"); + const opts = createOptions("device.pair.approve", { requestId: "req-node-repair" }); + const respond = vi.mocked(opts.respond); + const invalidate = vi.mocked(opts.context.invalidateClientsForDevice!); + const disconnect = vi.mocked(opts.context.disconnectClientsForDevice!); + respond.mockImplementation(() => { + expect(lifecycle.aborted).toBe(true); + expect(invalidate).toHaveBeenCalledWith("node-repaired", { + role: "node", + reason: "device-pairing-reapproved", + }); + expect(disconnect).not.toHaveBeenCalled(); + }); + + await expectDefined( + deviceHandlers["device.pair.approve"], + 'deviceHandlers["device.pair.approve"] test invariant', + )(opts); + await Promise.resolve(); + + expect(respond).toHaveBeenCalledTimes(1); + expect(disconnect).toHaveBeenCalledWith("node-repaired", { role: "node" }); + }); + it("allows approving the caller device from a non-admin device session", async () => { getPendingDevicePairingMock.mockResolvedValue({ requestId: "req-1", diff --git a/src/gateway/server-methods/devices.ts b/src/gateway/server-methods/devices.ts index abff04a88a3d..77988477489f 100644 --- a/src/gateway/server-methods/devices.ts +++ b/src/gateway/server-methods/devices.ts @@ -28,6 +28,8 @@ import { updatePairedDeviceMetadata, } from "../../infra/device-pairing.js"; import type { DiagnosticSecurityEventInput } from "../../infra/diagnostic-events.js"; +import { clearRemovedNodeRuntimeState } from "../node-runtime-state.js"; +import { invalidateNodeWakeState } from "../node-wake-state.js"; import { deniesCrossDeviceManagement, deniesDeviceTokenRoleManagement, @@ -329,6 +331,16 @@ export const deviceHandlers: GatewayRequestHandlers = { ); return; } + const normalizedDeviceId = approved.device.deviceId.trim(); + if (approved.nodePairingGenerationChanged) { + invalidateNodeWakeState(normalizedDeviceId); + // Mark the retired node generation before publishing success so buffered + // node RPCs cannot retain authority while transport teardown is deferred. + context.invalidateClientsForDevice?.(normalizedDeviceId, { + role: "node", + reason: "device-pairing-reapproved", + }); + } context.logGateway.info( `device pairing approved device=${approved.device.deviceId} role=${approved.device.role ?? "unknown"}`, ); @@ -354,6 +366,11 @@ export const deviceHandlers: GatewayRequestHandlers = { { dropIfSlow: true }, ); respond(true, { requestId, device: redactPairedDevice(approved.device) }, undefined); + if (approved.nodePairingGenerationChanged) { + queueMicrotask(() => { + context.disconnectClientsForDevice?.(normalizedDeviceId, { role: "node" }); + }); + } }, "device.pair.reject": async ({ params, respond, context, client }) => { if (!validateDevicePairRejectParams(params)) { @@ -481,6 +498,10 @@ export const deviceHandlers: GatewayRequestHandlers = { respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, "unknown deviceId")); return; } + clearRemovedNodeRuntimeState({ nodeId: removed.deviceId, context }); + context.invalidateClientsForDevice?.(removed.deviceId, { + reason: "device-pair-removed", + }); context.logGateway.info(`device pairing removed device=${removed.deviceId}`); emitDevicePairingLifecycleSecurityEvent({ action: "device.pairing.removed", @@ -489,13 +510,6 @@ export const deviceHandlers: GatewayRequestHandlers = { targetDeviceId: removed.deviceId, controlId: "device.pair.remove", }); - // Mark affected clients invalid *before* responding so any RPCs already - // pipelined into their WS socket buffer are rejected at the per-request - // dispatch check, closing the race between queueMicrotask-scheduled - // disconnect and inflight frames. - context.invalidateClientsForDevice?.(removed.deviceId, { - reason: "device-pair-removed", - }); respond(true, removed, undefined); queueMicrotask(() => { context.disconnectClientsForDevice?.(removed.deviceId); @@ -671,6 +685,7 @@ export const deviceHandlers: GatewayRequestHandlers = { return; } const entry = rotated.entry; + const normalizedDeviceId = deviceId.trim(); context.logGateway.info( `device token rotated device=${deviceId} role=${entry.role} scopes=${entry.scopes.join(",")}`, ); @@ -683,11 +698,14 @@ export const deviceHandlers: GatewayRequestHandlers = { role: entry.role, scopeCount: entry.scopes.length, }); + if (entry.role === "node") { + invalidateNodeWakeState(normalizedDeviceId); + } // Mark affected clients invalid *before* responding so any RPCs already // pipelined into their WS socket buffer are rejected at the per-request // dispatch check, closing the race between queueMicrotask-scheduled // disconnect and inflight frames. - context.invalidateClientsForDevice?.(deviceId.trim(), { + context.invalidateClientsForDevice?.(normalizedDeviceId, { role: entry.role, reason: "device-token-rotated", }); @@ -703,7 +721,7 @@ export const deviceHandlers: GatewayRequestHandlers = { undefined, ); queueMicrotask(() => { - context.disconnectClientsForDevice?.(deviceId.trim(), { role: entry.role }); + context.disconnectClientsForDevice?.(normalizedDeviceId, { role: entry.role }); }); }, "device.token.revoke": async ({ params, respond, context, client }) => { @@ -798,6 +816,9 @@ export const deviceHandlers: GatewayRequestHandlers = { controlId: "device.token.revoke", role: entry.role, }); + if (entry.role === "node") { + invalidateNodeWakeState(normalizedDeviceId); + } // Mark affected clients invalid *before* responding so any RPCs already // pipelined into their WS socket buffer are rejected at the per-request // dispatch check, closing the race between queueMicrotask-scheduled diff --git a/src/gateway/server-methods/environments.test.ts b/src/gateway/server-methods/environments.test.ts index ca7214dde355..a1d9fb2af213 100644 --- a/src/gateway/server-methods/environments.test.ts +++ b/src/gateway/server-methods/environments.test.ts @@ -11,6 +11,7 @@ import { environmentsHandlers, summarizeWorkerEnvironment } from "./environments vi.mock("../../infra/device-pairing.js", () => ({ listDevicePairing: vi.fn(), + resolveNodePairingState: vi.fn(), })); vi.mock("../../infra/node-pairing.js", () => ({ @@ -41,7 +42,7 @@ function mockContext( warn: vi.fn(), }, nodeRegistry: { - listConnected: () => [ + listConnectedForPairingStates: () => [ { nodeId: "node-live", connId: "conn-live", diff --git a/src/gateway/server-methods/environments.ts b/src/gateway/server-methods/environments.ts index 94dc2ece98e0..6911078beca2 100644 --- a/src/gateway/server-methods/environments.ts +++ b/src/gateway/server-methods/environments.ts @@ -8,7 +8,7 @@ import { validateEnvironmentsListParams, validateEnvironmentsStatusParams, } from "../../../packages/gateway-protocol/src/index.js"; -import { listDevicePairing } from "../../infra/device-pairing.js"; +import { listDevicePairing, resolveNodePairingState } from "../../infra/device-pairing.js"; import { listNodePairing } from "../../infra/node-pairing.js"; import type { NodeListNode } from "../../shared/node-list-types.js"; import { createKnownNodeCatalog, listKnownNodes } from "../node-catalog.js"; @@ -84,10 +84,20 @@ export function summarizeWorkerEnvironment( } async function listEnvironments(context: GatewayRequestContext): Promise { const [devices, nodes] = await Promise.all([listDevicePairing(), listNodePairing()]); + const currentPairingStates = new Map(); + for (const device of devices.paired) { + const state = resolveNodePairingState(device); + if (state) { + currentPairingStates.set(state.identity.nodeId, { + identity: state.identity.key, + ...(state.generation ? { generation: state.generation.key } : {}), + }); + } + } const catalog = createKnownNodeCatalog({ pairedDevices: devices.paired, pairedNodes: nodes.paired, - connectedNodes: context.nodeRegistry.listConnected(), + connectedNodes: context.nodeRegistry.listConnectedForPairingStates(currentPairingStates), }); return [GATEWAY_ENVIRONMENT, ...listKnownNodes(catalog).map(summarizeNodeEnvironment)]; } diff --git a/src/gateway/server-methods/exec-approvals.test.ts b/src/gateway/server-methods/exec-approvals.test.ts index b13e2f5f1e61..3d6aaf9faf2f 100644 --- a/src/gateway/server-methods/exec-approvals.test.ts +++ b/src/gateway/server-methods/exec-approvals.test.ts @@ -251,6 +251,7 @@ describe("exec approvals gateway methods", () => { get: () => ({ nodeId: "node-1", connId: "conn-1", + pairingGeneration: "generation-1", platform: "windows", deviceFamily: "Windows", declaredCommands: [testCase.command], @@ -309,6 +310,7 @@ describe("exec approvals gateway methods", () => { get: () => ({ nodeId: "node-1", connId: "conn-1", + pairingGeneration: "generation-1", clientId: "openclaw-macos", clientMode: "node", platform: "macOS 26.5.2", @@ -323,6 +325,8 @@ describe("exec approvals gateway methods", () => { expect(invoke).toHaveBeenCalledWith({ nodeId: "node-1", + expectedConnId: "conn-1", + expectedPairingGeneration: "generation-1", command, params: { includeResolvedDefaults: true }, }); @@ -396,6 +400,7 @@ describe("exec approvals gateway methods", () => { get: () => ({ nodeId: "node-1", connId: "conn-1", + pairingGeneration: "generation-1", clientId: identity.clientId, clientMode: identity.clientMode, platform: identity.platform, @@ -410,6 +415,8 @@ describe("exec approvals gateway methods", () => { expect(invoke).toHaveBeenCalledWith({ nodeId: "node-1", + expectedConnId: "conn-1", + expectedPairingGeneration: "generation-1", command, params: {}, }); @@ -452,6 +459,7 @@ describe("exec approvals gateway methods", () => { get: () => ({ nodeId: "windows-node", connId: "conn-1", + pairingGeneration: "generation-1", platform: "windows", deviceFamily: "Windows", declaredCommands: [command], @@ -464,6 +472,8 @@ describe("exec approvals gateway methods", () => { expect(invoke).toHaveBeenCalledWith({ nodeId: "windows-node", + expectedConnId: "conn-1", + expectedPairingGeneration: "generation-1", command, params: { defaultAction: "deny", @@ -498,6 +508,7 @@ describe("exec approvals gateway methods", () => { get: () => ({ nodeId: "windows-node", connId: "conn-1", + pairingGeneration: "generation-1", platform: "windows", deviceFamily: "Windows", declaredCommands: [command], diff --git a/src/gateway/server-methods/exec-approvals.ts b/src/gateway/server-methods/exec-approvals.ts index 0c9969369936..306c9cc4c5aa 100644 --- a/src/gateway/server-methods/exec-approvals.ts +++ b/src/gateway/server-methods/exec-approvals.ts @@ -166,6 +166,14 @@ async function respondWithExecApprovalsNodePayload { const res = await params.context.nodeRegistry.invoke({ nodeId, + ...(nodeSession + ? { + expectedConnId: nodeSession.connId, + ...(nodeSession.pairingGeneration + ? { expectedPairingGeneration: nodeSession.pairingGeneration } + : {}), + } + : {}), command: params.command, params: params.commandParams(parsedParams, nodeSession), }); diff --git a/src/gateway/server-methods/fs.test.ts b/src/gateway/server-methods/fs.test.ts index 4d40879c7da4..e0ce4efc7b72 100644 --- a/src/gateway/server-methods/fs.test.ts +++ b/src/gateway/server-methods/fs.test.ts @@ -122,6 +122,7 @@ describe("fs.listDir", () => { nodeRegistry: { get: vi.fn().mockReturnValue({ connId: "conn-1", + pairingGeneration: "generation-1", nodeId: "macbook", platform: "macos", deviceFamily: "Mac", @@ -141,6 +142,7 @@ describe("fs.listDir", () => { expect(invoke).toHaveBeenCalledWith({ nodeId: "macbook", expectedConnId: "conn-1", + expectedPairingGeneration: "generation-1", command: "fs.listDir", params: {}, }); diff --git a/src/gateway/server-methods/fs.ts b/src/gateway/server-methods/fs.ts index 937df99de5cc..cd972c43479a 100644 --- a/src/gateway/server-methods/fs.ts +++ b/src/gateway/server-methods/fs.ts @@ -69,6 +69,7 @@ export const fsHandlers: GatewayRequestHandlers = { const result = await context.nodeRegistry.invoke({ nodeId: params.nodeId, expectedConnId: node.connId, + ...(node.pairingGeneration ? { expectedPairingGeneration: node.pairingGeneration } : {}), command: NODE_FS_LIST_DIR_COMMAND, params: params.path ? { path: params.path } : {}, }); diff --git a/src/gateway/server-methods/nodes-pending.test.ts b/src/gateway/server-methods/nodes-pending.test.ts index 2fbda6ddd617..e1bdce11e399 100644 --- a/src/gateway/server-methods/nodes-pending.test.ts +++ b/src/gateway/server-methods/nodes-pending.test.ts @@ -7,21 +7,39 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { nodePendingHandlers } from "./nodes-pending.js"; const mocks = vi.hoisted(() => ({ + captureNodePairingGeneration: vi.fn(), + captureNodeWakeLifecycle: vi.fn(), drainNodePendingWork: vi.fn(), enqueueNodePendingWork: vi.fn(), + isNodePairingGenerationCurrent: vi.fn(), + isNodeWakeLifecycleCurrent: vi.fn(), maybeWakeNodeWithApns: vi.fn(), maybeSendNodeWakeNudge: vi.fn(), + removeNodePendingWorkItem: vi.fn(), + releaseNodeWakeLifecycle: vi.fn(), waitForNodeReconnect: vi.fn(), })); vi.mock("../node-pending-work.js", () => ({ drainNodePendingWork: mocks.drainNodePendingWork, enqueueNodePendingWork: mocks.enqueueNodePendingWork, + removeNodePendingWorkItem: mocks.removeNodePendingWorkItem, +})); + +vi.mock("../../infra/node-pairing-state.js", () => ({ + captureNodePairingGeneration: mocks.captureNodePairingGeneration, + isNodePairingGenerationCurrent: mocks.isNodePairingGenerationCurrent, +})); + +vi.mock("../node-wake-state.js", () => ({ + NODE_WAKE_RECONNECT_WAIT_MS: 3_000, + NODE_WAKE_RECONNECT_RETRY_WAIT_MS: 12_000, + captureNodeWakeLifecycle: mocks.captureNodeWakeLifecycle, + isNodeWakeLifecycleCurrent: mocks.isNodeWakeLifecycleCurrent, + releaseNodeWakeLifecycle: mocks.releaseNodeWakeLifecycle, })); vi.mock("./nodes.js", () => ({ - NODE_WAKE_RECONNECT_WAIT_MS: 3_000, - NODE_WAKE_RECONNECT_RETRY_WAIT_MS: 12_000, maybeWakeNodeWithApns: mocks.maybeWakeNodeWithApns, maybeSendNodeWakeNudge: mocks.maybeSendNodeWakeNudge, waitForNodeReconnect: mocks.waitForNodeReconnect, @@ -41,6 +59,7 @@ function makeContext(overrides?: Partial>) { return { nodeRegistry: { get: vi.fn(() => undefined), + getForPairingGeneration: vi.fn(() => undefined), }, logGateway: { info: vi.fn(), @@ -57,10 +76,21 @@ function respondCall(respond: ReturnType): RespondCall | undefined describe("node.pending handlers", () => { beforeEach(() => { + mocks.captureNodePairingGeneration.mockReset().mockImplementation(async (nodeId: string) => ({ + nodeId, + key: `generation:${nodeId}:1`, + })); + mocks.captureNodeWakeLifecycle.mockReset(); mocks.drainNodePendingWork.mockReset(); mocks.enqueueNodePendingWork.mockReset(); + mocks.isNodePairingGenerationCurrent.mockReset().mockResolvedValue(true); + mocks.isNodeWakeLifecycleCurrent + .mockReset() + .mockImplementation((_nodeId: string, lifecycle: AbortSignal) => !lifecycle.aborted); mocks.maybeWakeNodeWithApns.mockReset(); mocks.maybeSendNodeWakeNudge.mockReset(); + mocks.removeNodePendingWorkItem.mockReset(); + mocks.releaseNodeWakeLifecycle.mockReset(); mocks.waitForNodeReconnect.mockReset(); }); @@ -71,6 +101,12 @@ describe("node.pending handlers", () => { hasMore: false, }); const respond = vi.fn(); + const context = makeContext({ + nodeRegistry: { + get: vi.fn(() => undefined), + getForPairingGeneration: vi.fn(() => ({ connId: "conn-ios-node-1" })), + }, + }); await expectDefined( nodePendingHandlers["node.pending.drain"], @@ -78,8 +114,11 @@ describe("node.pending handlers", () => { )({ params: { maxItems: 3 }, respond: respond as never, - client: { connect: { device: { id: "ios-node-1" } } } as never, - context: makeContext() as never, + client: { + connId: "conn-ios-node-1", + connect: { device: { id: "ios-node-1" } }, + } as never, + context: context as never, req: { type: "req", id: "req-node-pending-drain", method: "node.pending.drain" }, isWebchatConnect: () => false, }); @@ -87,6 +126,7 @@ describe("node.pending handlers", () => { expect(mocks.drainNodePendingWork).toHaveBeenCalledWith("ios-node-1", { maxItems: 3, includeDefaultStatus: true, + pairingGeneration: "generation:ios-node-1:1", }); expect(respond).toHaveBeenCalledWith( true, @@ -120,7 +160,111 @@ describe("node.pending handlers", () => { expect(call?.[2]?.message).toContain("connected device identity"); }); - it("enqueues pending work and wakes a disconnected node once", async () => { + it("rejects a changed pairing before draining its pending work", async () => { + mocks.isNodePairingGenerationCurrent.mockResolvedValue(false); + const respond = vi.fn(); + const context = makeContext({ + nodeRegistry: { + get: vi.fn(() => undefined), + getForPairingGeneration: vi.fn(() => ({ connId: "conn-stale-drain" })), + }, + }); + + await expectDefined( + nodePendingHandlers["node.pending.drain"], + 'nodePendingHandlers["node.pending.drain"] test invariant', + )({ + params: {}, + respond: respond as never, + client: { + connId: "conn-stale-drain", + connect: { device: { id: "ios-node-stale-drain" } }, + } as never, + context: context as never, + req: { type: "req", id: "req-node-stale-drain", method: "node.pending.drain" }, + isWebchatConnect: () => false, + }); + + expect(respondCall(respond)).toMatchObject([ + false, + undefined, + { details: { code: "PAIRING_CHANGED" } }, + ]); + expect(mocks.drainNodePendingWork).not.toHaveBeenCalled(); + }); + + it("rejects a same-generation reconnect before destructively draining", async () => { + let currentConnId = "conn-original"; + mocks.isNodePairingGenerationCurrent.mockImplementation(async () => { + currentConnId = "conn-replacement"; + return true; + }); + const respond = vi.fn(); + const context = makeContext({ + nodeRegistry: { + get: vi.fn(() => undefined), + getForPairingGeneration: vi.fn(() => ({ connId: currentConnId })), + }, + }); + + await expectDefined( + nodePendingHandlers["node.pending.drain"], + 'nodePendingHandlers["node.pending.drain"] test invariant', + )({ + params: {}, + respond: respond as never, + client: { + connId: "conn-original", + connect: { device: { id: "ios-node-reconnected" } }, + } as never, + context: context as never, + req: { type: "req", id: "req-node-reconnected-drain", method: "node.pending.drain" }, + isWebchatConnect: () => false, + }); + + expect(mocks.drainNodePendingWork).not.toHaveBeenCalled(); + expect(respondCall(respond)).toMatchObject([ + false, + undefined, + { details: { code: "PAIRING_CHANGED" } }, + ]); + }); + + it("rejects a prior-generation socket before it drains replacement work", async () => { + const respond = vi.fn(); + const context = makeContext({ + nodeRegistry: { + get: vi.fn(() => undefined), + getForPairingGeneration: vi.fn(() => ({ connId: "conn-replacement" })), + }, + }); + + await expectDefined( + nodePendingHandlers["node.pending.drain"], + 'nodePendingHandlers["node.pending.drain"] test invariant', + )({ + params: {}, + respond: respond as never, + client: { + connId: "conn-prior-generation", + connect: { device: { id: "ios-node-replaced" } }, + } as never, + context: context as never, + req: { type: "req", id: "req-node-prior-drain", method: "node.pending.drain" }, + isWebchatConnect: () => false, + }); + + expect(mocks.drainNodePendingWork).not.toHaveBeenCalled(); + expect(respondCall(respond)).toMatchObject([ + false, + undefined, + { details: { code: "PAIRING_CHANGED" } }, + ]); + }); + + it("normalizes the target identity before queueing and waking a disconnected node", async () => { + const wakeLifecycle = new AbortController().signal; + mocks.captureNodeWakeLifecycle.mockReturnValue(wakeLifecycle); mocks.enqueueNodePendingWork.mockReturnValue({ revision: 4, deduped: false, @@ -147,7 +291,8 @@ describe("node.pending handlers", () => { }); const context = makeContext({ nodeRegistry: { - get: vi.fn(() => (connected ? { nodeId: "ios-node-2" } : undefined)), + get: vi.fn(() => undefined), + getForPairingGeneration: vi.fn(() => (connected ? { nodeId: "ios-node-2" } : undefined)), }, }); const respond = vi.fn(); @@ -157,7 +302,7 @@ describe("node.pending handlers", () => { 'nodePendingHandlers["node.pending.enqueue"] test invariant', )({ params: { - nodeId: "ios-node-2", + nodeId: " ios-node-2 ", type: "location.request", priority: "high", }, @@ -173,17 +318,34 @@ describe("node.pending handlers", () => { type: "location.request", priority: "high", expiresInMs: undefined, + pairingGeneration: "generation:ios-node-2:1", }); + expect(context.nodeRegistry.getForPairingGeneration).toHaveBeenCalledWith( + "ios-node-2", + "generation:ios-node-2:1", + ); + expect(context.nodeRegistry.getForPairingGeneration).not.toHaveBeenCalledWith( + " ios-node-2 ", + expect.anything(), + ); expect(mocks.maybeWakeNodeWithApns).toHaveBeenCalledWith("ios-node-2", { wakeReason: "node.pending", cfg: {}, + lifecycle: wakeLifecycle, + generation: { + nodeId: "ios-node-2", + key: "generation:ios-node-2:1", + }, }); expect(mocks.waitForNodeReconnect).toHaveBeenCalledWith({ nodeId: "ios-node-2", context, timeoutMs: 3_000, + lifecycle: wakeLifecycle, + pairingGeneration: "generation:ios-node-2:1", }); expect(mocks.maybeSendNodeWakeNudge).not.toHaveBeenCalled(); + expect(mocks.releaseNodeWakeLifecycle).toHaveBeenCalledWith("ios-node-2", wakeLifecycle); const call = respondCall(respond) as | [boolean, { nodeId?: string; revision?: number; wakeTriggered?: boolean }, unknown?] | undefined; @@ -193,4 +355,131 @@ describe("node.pending handlers", () => { expect(call?.[1]?.wakeTriggered).toBe(true); expect(call?.[2]).toBeUndefined(); }); + + it("returns unavailable when pairing removal invalidates an enqueued item", async () => { + const lifecycleController = new AbortController(); + const wakeLifecycle = lifecycleController.signal; + mocks.captureNodeWakeLifecycle.mockReturnValue(wakeLifecycle); + mocks.enqueueNodePendingWork.mockReturnValue({ + revision: 5, + deduped: false, + item: { + id: "pending-invalidated", + type: "location.request", + priority: "default", + createdAtMs: 100, + expiresAtMs: null, + }, + }); + mocks.maybeWakeNodeWithApns.mockImplementation(async () => + wakeLifecycle.aborted + ? { + available: false, + throttled: false, + path: "invalidated", + durationMs: 0, + } + : { + available: true, + throttled: false, + path: "sent", + durationMs: 1, + }, + ); + mocks.waitForNodeReconnect.mockImplementation(async () => { + lifecycleController.abort(); + return false; + }); + mocks.maybeSendNodeWakeNudge.mockResolvedValue({ + sent: false, + throttled: false, + reason: "invalidated", + durationMs: 0, + }); + const context = makeContext(); + const respond = vi.fn(); + + await expectDefined( + nodePendingHandlers["node.pending.enqueue"], + 'nodePendingHandlers["node.pending.enqueue"] test invariant', + )({ + params: { nodeId: " ios-node-invalidated ", type: "location.request" }, + respond: respond as never, + client: null, + context: context as never, + req: { type: "req", id: "req-node-pending-invalidated", method: "node.pending.enqueue" }, + isWebchatConnect: () => false, + }); + + expect(mocks.captureNodeWakeLifecycle).toHaveBeenCalledWith( + "ios-node-invalidated", + "generation:ios-node-invalidated:1", + ); + expect(mocks.maybeWakeNodeWithApns).toHaveBeenCalledTimes(1); + for (const call of mocks.maybeWakeNodeWithApns.mock.calls) { + expect(call[0]).toBe("ios-node-invalidated"); + expect(call[1]).toMatchObject({ lifecycle: wakeLifecycle }); + } + expect(mocks.waitForNodeReconnect).toHaveBeenCalledWith({ + nodeId: "ios-node-invalidated", + context, + timeoutMs: 3_000, + lifecycle: wakeLifecycle, + pairingGeneration: "generation:ios-node-invalidated:1", + }); + expect(mocks.maybeSendNodeWakeNudge).not.toHaveBeenCalled(); + expect(mocks.releaseNodeWakeLifecycle).toHaveBeenCalledWith( + "ios-node-invalidated", + wakeLifecycle, + ); + expect(mocks.removeNodePendingWorkItem).toHaveBeenCalledWith({ + nodeId: "ios-node-invalidated", + itemId: "pending-invalidated", + pairingGeneration: "generation:ios-node-invalidated:1", + }); + const call = respondCall(respond); + expect(call?.[0]).toBe(false); + expect(call?.[2]).toMatchObject({ + message: "node pairing changed while pending work was active", + details: { code: "PAIRING_CHANGED" }, + }); + }); + + it("does not remove replacement work when an invalidated enqueue reused it", async () => { + const wakeLifecycle = new AbortController().signal; + mocks.captureNodeWakeLifecycle.mockReturnValue(wakeLifecycle); + mocks.enqueueNodePendingWork.mockReturnValue({ + revision: 9, + deduped: true, + item: { + id: "replacement-work", + type: "location.request", + priority: "normal", + createdAtMs: 200, + expiresAtMs: null, + }, + }); + mocks.isNodePairingGenerationCurrent.mockResolvedValueOnce(true).mockResolvedValueOnce(false); + const respond = vi.fn(); + + await expectDefined( + nodePendingHandlers["node.pending.enqueue"], + 'nodePendingHandlers["node.pending.enqueue"] test invariant', + )({ + params: { nodeId: "node-replacement", type: "location.request", wake: false }, + respond: respond as never, + client: null, + context: makeContext() as never, + req: { type: "req", id: "req-node-replacement", method: "node.pending.enqueue" }, + isWebchatConnect: () => false, + }); + + expect(mocks.enqueueNodePendingWork).toHaveBeenCalledTimes(1); + expect(mocks.removeNodePendingWorkItem).not.toHaveBeenCalled(); + expect(respondCall(respond)).toMatchObject([ + false, + undefined, + { details: { code: "PAIRING_CHANGED" } }, + ]); + }); }); diff --git a/src/gateway/server-methods/nodes-pending.ts b/src/gateway/server-methods/nodes-pending.ts index ae6e175f2364..3e79874e416b 100644 --- a/src/gateway/server-methods/nodes-pending.ts +++ b/src/gateway/server-methods/nodes-pending.ts @@ -6,22 +6,52 @@ import { validateNodePendingDrainParams, validateNodePendingEnqueueParams, } from "../../../packages/gateway-protocol/src/index.js"; +import { + captureNodePairingGeneration, + isNodePairingGenerationCurrent, + type NodePairingGeneration, +} from "../../infra/node-pairing-state.js"; import { drainNodePendingWork, enqueueNodePendingWork, + removeNodePendingWorkItem, type NodePendingWorkPriority, type NodePendingWorkType, } from "../node-pending-work.js"; -import { respondInvalidParams, respondUnavailableOnThrow } from "./nodes.helpers.js"; import { - maybeSendNodeWakeNudge, - maybeWakeNodeWithApns, + captureNodeWakeLifecycle, + isNodeWakeLifecycleCurrent, NODE_WAKE_RECONNECT_RETRY_WAIT_MS, NODE_WAKE_RECONNECT_WAIT_MS, - waitForNodeReconnect, -} from "./nodes.js"; + releaseNodeWakeLifecycle, +} from "../node-wake-state.js"; +import { respondInvalidParams, respondUnavailableOnThrow } from "./nodes.helpers.js"; +import { maybeSendNodeWakeNudge, maybeWakeNodeWithApns, waitForNodeReconnect } from "./nodes.js"; +import type { RespondFn } from "./shared-types.js"; import type { GatewayRequestHandlers } from "./types.js"; +function respondPairingChanged(respond: RespondFn) { + respond( + false, + undefined, + errorShape(ErrorCodes.UNAVAILABLE, "node pairing changed while pending work was active", { + retryable: true, + details: { code: "PAIRING_CHANGED" }, + }), + ); +} + +async function isPendingGenerationCurrent(params: { + nodeId: string; + generation: NodePairingGeneration; + lifecycle: AbortSignal; +}): Promise { + return ( + isNodeWakeLifecycleCurrent(params.nodeId, params.lifecycle, params.generation.key) && + (await isNodePairingGenerationCurrent(params.generation)) + ); +} + function resolveClientNodeId( client: { connect?: { device?: { id?: string }; client?: { id?: string } } } | null, ): string | null { @@ -32,7 +62,7 @@ function resolveClientNodeId( /** Gateway handlers for queueing work until a paired node reconnects. */ export const nodePendingHandlers: GatewayRequestHandlers = { - "node.pending.drain": async ({ params, respond, client }) => { + "node.pending.drain": async ({ params, respond, client, context }) => { if (!validateNodePendingDrainParams(params)) { respondInvalidParams({ respond, @@ -53,12 +83,27 @@ export const nodePendingHandlers: GatewayRequestHandlers = { ); return; } - const p = params as { maxItems?: number }; - const drained = drainNodePendingWork(nodeId, { - maxItems: p.maxItems, - includeDefaultStatus: true, + await respondUnavailableOnThrow(respond, async () => { + const generation = await captureNodePairingGeneration(nodeId); + if (!generation || !(await isNodePairingGenerationCurrent(generation))) { + respondPairingChanged(respond); + return; + } + // Draining deletes work, so the authenticated caller must still be the + // registry session that owns the persisted generation. + const session = context.nodeRegistry.getForPairingGeneration(nodeId, generation.key); + if (!client?.connId || session?.connId !== client.connId) { + respondPairingChanged(respond); + return; + } + const p = params as { maxItems?: number }; + const drained = drainNodePendingWork(nodeId, { + maxItems: p.maxItems, + includeDefaultStatus: true, + pairingGeneration: generation.key, + }); + respond(true, { nodeId, ...drained }, undefined); }); - respond(true, { nodeId, ...drained }, undefined); }, "node.pending.enqueue": async ({ params, respond, context }) => { if (!validateNodePendingEnqueueParams(params)) { @@ -77,95 +122,159 @@ export const nodePendingHandlers: GatewayRequestHandlers = { wake?: boolean; }; await respondUnavailableOnThrow(respond, async () => { - const queued = enqueueNodePendingWork({ - nodeId: p.nodeId, - type: p.type, - priority: p.priority, - expiresInMs: p.expiresInMs, - }); - let wakeTriggered = false; - if (p.wake !== false && !queued.deduped && !context.nodeRegistry.get(p.nodeId)) { - const wakeReqId = queued.item.id; - context.logGateway.info( - `node pending wake start node=${p.nodeId} req=${wakeReqId} type=${queued.item.type}`, - ); - const cfg = context.getRuntimeConfig(); - const wake = await maybeWakeNodeWithApns(p.nodeId, { - wakeReason: "node.pending", - cfg, - }); - context.logGateway.info( - `node pending wake stage=wake1 node=${p.nodeId} req=${wakeReqId} ` + - `available=${wake.available} throttled=${wake.throttled} ` + - `path=${wake.path} durationMs=${wake.durationMs} ` + - `apnsStatus=${wake.apnsStatus ?? -1} apnsReason=${wake.apnsReason ?? "-"}`, - ); - wakeTriggered = wake.available; - if (wake.available) { - // Give the first wake a short reconnect window before forcing a - // second wake; this keeps normal APNs delivery cheap and quiet. - const reconnected = await waitForNodeReconnect({ - nodeId: p.nodeId, - context, - timeoutMs: NODE_WAKE_RECONNECT_WAIT_MS, - }); - context.logGateway.info( - `node pending wake stage=wait1 node=${p.nodeId} req=${wakeReqId} ` + - `reconnected=${reconnected} timeoutMs=${NODE_WAKE_RECONNECT_WAIT_MS}`, - ); + const nodeId = p.nodeId.trim(); + const generation = await captureNodePairingGeneration(nodeId); + if (!generation) { + respondPairingChanged(respond); + return; + } + const wakeLifecycle = captureNodeWakeLifecycle(nodeId, generation.key); + try { + if (!(await isPendingGenerationCurrent({ nodeId, generation, lifecycle: wakeLifecycle }))) { + respondPairingChanged(respond); + return; } - if (!context.nodeRegistry.get(p.nodeId) && wake.available) { - // A forced retry is only useful after the first wake was deliverable - // but the node still has not reattached to the Gateway. - const retryWake = await maybeWakeNodeWithApns(p.nodeId, { - force: true, + const queued = enqueueNodePendingWork({ + nodeId, + type: p.type, + priority: p.priority, + expiresInMs: p.expiresInMs, + pairingGeneration: generation.key, + }); + let wakeTriggered = false; + if ( + p.wake !== false && + !queued.deduped && + !context.nodeRegistry.getForPairingGeneration(nodeId, generation.key) + ) { + const wakeReqId = queued.item.id; + context.logGateway.info( + `node pending wake start node=${nodeId} req=${wakeReqId} type=${queued.item.type}`, + ); + const cfg = context.getRuntimeConfig(); + const wake = await maybeWakeNodeWithApns(nodeId, { wakeReason: "node.pending", cfg, + lifecycle: wakeLifecycle, + generation, }); context.logGateway.info( - `node pending wake stage=wake2 node=${p.nodeId} req=${wakeReqId} force=true ` + - `available=${retryWake.available} throttled=${retryWake.throttled} ` + - `path=${retryWake.path} durationMs=${retryWake.durationMs} ` + - `apnsStatus=${retryWake.apnsStatus ?? -1} apnsReason=${retryWake.apnsReason ?? "-"}`, + `node pending wake stage=wake1 node=${nodeId} req=${wakeReqId} ` + + `available=${wake.available} throttled=${wake.throttled} ` + + `path=${wake.path} durationMs=${wake.durationMs} ` + + `apnsStatus=${wake.apnsStatus ?? -1} apnsReason=${wake.apnsReason ?? "-"}`, ); - if (retryWake.available) { + wakeTriggered = wake.available; + if (wake.available) { + // Give the first wake a short reconnect window before forcing a + // second wake; this keeps normal APNs delivery cheap and quiet. const reconnected = await waitForNodeReconnect({ - nodeId: p.nodeId, + nodeId, context, - timeoutMs: NODE_WAKE_RECONNECT_RETRY_WAIT_MS, + timeoutMs: NODE_WAKE_RECONNECT_WAIT_MS, + lifecycle: wakeLifecycle, + pairingGeneration: generation.key, }); context.logGateway.info( - `node pending wake stage=wait2 node=${p.nodeId} req=${wakeReqId} ` + - `reconnected=${reconnected} timeoutMs=${NODE_WAKE_RECONNECT_RETRY_WAIT_MS}`, + `node pending wake stage=wait1 node=${nodeId} req=${wakeReqId} ` + + `reconnected=${reconnected} timeoutMs=${NODE_WAKE_RECONNECT_WAIT_MS}`, + ); + } + if ( + (await isPendingGenerationCurrent({ + nodeId, + generation, + lifecycle: wakeLifecycle, + })) && + !context.nodeRegistry.getForPairingGeneration(nodeId, generation.key) && + wake.available + ) { + // A forced retry is only useful after the first wake was deliverable + // but the node still has not reattached to the Gateway. + const retryWake = await maybeWakeNodeWithApns(nodeId, { + force: true, + wakeReason: "node.pending", + cfg, + lifecycle: wakeLifecycle, + generation, + }); + context.logGateway.info( + `node pending wake stage=wake2 node=${nodeId} req=${wakeReqId} force=true ` + + `available=${retryWake.available} throttled=${retryWake.throttled} ` + + `path=${retryWake.path} durationMs=${retryWake.durationMs} ` + + `apnsStatus=${retryWake.apnsStatus ?? -1} apnsReason=${retryWake.apnsReason ?? "-"}`, + ); + if (retryWake.available) { + const reconnected = await waitForNodeReconnect({ + nodeId, + context, + timeoutMs: NODE_WAKE_RECONNECT_RETRY_WAIT_MS, + lifecycle: wakeLifecycle, + pairingGeneration: generation.key, + }); + context.logGateway.info( + `node pending wake stage=wait2 node=${nodeId} req=${wakeReqId} ` + + `reconnected=${reconnected} timeoutMs=${NODE_WAKE_RECONNECT_RETRY_WAIT_MS}`, + ); + } + } + if ( + (await isPendingGenerationCurrent({ + nodeId, + generation, + lifecycle: wakeLifecycle, + })) && + !context.nodeRegistry.getForPairingGeneration(nodeId, generation.key) + ) { + const nudge = await maybeSendNodeWakeNudge(nodeId, { + cfg, + lifecycle: wakeLifecycle, + generation, + }); + context.logGateway.info( + `node pending wake nudge node=${nodeId} req=${wakeReqId} sent=${nudge.sent} ` + + `throttled=${nudge.throttled} reason=${nudge.reason} durationMs=${nudge.durationMs} ` + + `apnsStatus=${nudge.apnsStatus ?? -1} apnsReason=${nudge.apnsReason ?? "-"}`, + ); + context.logGateway.warn( + `node pending wake done node=${nodeId} req=${wakeReqId} connected=false reason=not_connected`, + ); + } else if ( + await isPendingGenerationCurrent({ + nodeId, + generation, + lifecycle: wakeLifecycle, + }) + ) { + context.logGateway.info( + `node pending wake done node=${nodeId} req=${wakeReqId} connected=true`, ); } } - if (!context.nodeRegistry.get(p.nodeId)) { - const nudge = await maybeSendNodeWakeNudge(p.nodeId, { cfg }); - context.logGateway.info( - `node pending wake nudge node=${p.nodeId} req=${wakeReqId} sent=${nudge.sent} ` + - `throttled=${nudge.throttled} reason=${nudge.reason} durationMs=${nudge.durationMs} ` + - `apnsStatus=${nudge.apnsStatus ?? -1} apnsReason=${nudge.apnsReason ?? "-"}`, - ); - context.logGateway.warn( - `node pending wake done node=${p.nodeId} req=${wakeReqId} connected=false reason=not_connected`, - ); - } else { - context.logGateway.info( - `node pending wake done node=${p.nodeId} req=${wakeReqId} connected=true`, - ); + if (!(await isPendingGenerationCurrent({ nodeId, generation, lifecycle: wakeLifecycle }))) { + if (!queued.deduped) { + removeNodePendingWorkItem({ + nodeId, + itemId: queued.item.id, + pairingGeneration: generation.key, + }); + } + respondPairingChanged(respond); + return; } + respond( + true, + { + nodeId, + revision: queued.revision, + queued: queued.item, + wakeTriggered, + }, + undefined, + ); + } finally { + releaseNodeWakeLifecycle(nodeId, wakeLifecycle); } - respond( - true, - { - nodeId: p.nodeId, - revision: queued.revision, - queued: queued.item, - wakeTriggered, - }, - undefined, - ); }); }, }; diff --git a/src/gateway/server-methods/nodes-wake-state.test.ts b/src/gateway/server-methods/nodes-wake-state.test.ts deleted file mode 100644 index b4ed666fb4ab..000000000000 --- a/src/gateway/server-methods/nodes-wake-state.test.ts +++ /dev/null @@ -1,107 +0,0 @@ -// Tests for node wake state tracking and testing seam. -import { beforeEach, describe, expect, it } from "vitest"; -import { - NODE_WAKE_RECONNECT_WAIT_MS, - NODE_WAKE_RECONNECT_RETRY_WAIT_MS, - NODE_WAKE_RECONNECT_POLL_MS, - clearNodeWakeState, - nodeWakeById, - nodeWakeNudgeById, -} from "./nodes-wake-state.js"; - -beforeEach(() => { - nodeWakeById.clear(); - nodeWakeNudgeById.clear(); -}); - -describe("constants", () => { - it("exports expected wait/poll constants", () => { - expect(NODE_WAKE_RECONNECT_WAIT_MS).toBe(3_000); - expect(NODE_WAKE_RECONNECT_RETRY_WAIT_MS).toBe(12_000); - expect(NODE_WAKE_RECONNECT_POLL_MS).toBe(150); - }); -}); - -describe("nodeWakeById", () => { - it("starts empty", () => { - expect(nodeWakeById.size).toBe(0); - }); - - it("stores NodeWakeState entries", () => { - const now = Date.now(); - nodeWakeById.set("node-1", { lastWakeAtMs: now }); - expect(nodeWakeById.size).toBe(1); - expect(nodeWakeById.get("node-1")?.lastWakeAtMs).toBe(now); - }); - - it("stores multiple entries", () => { - nodeWakeById.set("a", { lastWakeAtMs: 100 }); - nodeWakeById.set("b", { lastWakeAtMs: 200 }); - nodeWakeById.set("c", { lastWakeAtMs: 300 }); - expect(nodeWakeById.size).toBe(3); - }); - - it("overwrites existing entry when key is reused", () => { - nodeWakeById.set("node-1", { lastWakeAtMs: 100 }); - nodeWakeById.set("node-1", { lastWakeAtMs: 200 }); - expect(nodeWakeById.size).toBe(1); - }); - - it("supports inFlight promise property", () => { - const promise = Promise.resolve({ - available: true, - throttled: false, - path: "sent" as const, - durationMs: 50, - }); - nodeWakeById.set("node-1", { lastWakeAtMs: Date.now(), inFlight: promise }); - expect(nodeWakeById.get("node-1")?.inFlight).toBe(promise); - }); -}); - -describe("nodeWakeNudgeById", () => { - it("starts empty", () => { - expect(nodeWakeNudgeById.size).toBe(0); - }); - - it("stores nudge timestamps", () => { - nodeWakeNudgeById.set("node-1", 1000); - expect(nodeWakeNudgeById.size).toBe(1); - expect(nodeWakeNudgeById.get("node-1")).toBe(1000); - }); - - it("independently tracked from nodeWakeById", () => { - nodeWakeById.set("node-1", { lastWakeAtMs: 500 }); - nodeWakeNudgeById.set("node-1", 1000); - expect(nodeWakeById.size).toBe(1); - expect(nodeWakeNudgeById.size).toBe(1); - }); -}); - -describe("clearNodeWakeState", () => { - it("removes the wake entry and nudge for the given node", () => { - nodeWakeById.set("node-1", { lastWakeAtMs: 100 }); - nodeWakeNudgeById.set("node-1", 200); - clearNodeWakeState("node-1"); - expect(nodeWakeById.has("node-1")).toBe(false); - expect(nodeWakeNudgeById.has("node-1")).toBe(false); - }); - - it("is a no-op when the node id does not exist", () => { - expect(() => clearNodeWakeState("ghost")).not.toThrow(); - expect(nodeWakeById.size).toBe(0); - expect(nodeWakeNudgeById.size).toBe(0); - }); - - it("only removes the specified node, leaving others intact", () => { - nodeWakeById.set("a", { lastWakeAtMs: 1 }); - nodeWakeById.set("b", { lastWakeAtMs: 2 }); - nodeWakeNudgeById.set("a", 10); - nodeWakeNudgeById.set("b", 20); - clearNodeWakeState("a"); - expect(nodeWakeById.has("a")).toBe(false); - expect(nodeWakeById.has("b")).toBe(true); - expect(nodeWakeNudgeById.has("a")).toBe(false); - expect(nodeWakeNudgeById.has("b")).toBe(true); - }); -}); diff --git a/src/gateway/server-methods/nodes-wake-state.ts b/src/gateway/server-methods/nodes-wake-state.ts deleted file mode 100644 index beaef22c5883..000000000000 --- a/src/gateway/server-methods/nodes-wake-state.ts +++ /dev/null @@ -1,27 +0,0 @@ -// Node wake state tracks APNs wake attempts and reconnect nudges for paired -// nodes, plus a narrow read-only testing seam. -export const NODE_WAKE_RECONNECT_WAIT_MS = 3_000; -export const NODE_WAKE_RECONNECT_RETRY_WAIT_MS = 12_000; -export const NODE_WAKE_RECONNECT_POLL_MS = 150; - -export type NodeWakeAttempt = { - available: boolean; - throttled: boolean; - path: "throttled" | "no-registration" | "no-auth" | "sent" | "send-error"; - durationMs: number; - apnsStatus?: number; - apnsReason?: string; -}; - -type NodeWakeState = { - lastWakeAtMs: number; - inFlight?: Promise; -}; - -export const nodeWakeById = new Map(); -export const nodeWakeNudgeById = new Map(); - -export function clearNodeWakeState(nodeId: string): void { - nodeWakeById.delete(nodeId); - nodeWakeNudgeById.delete(nodeId); -} diff --git a/src/gateway/server-methods/nodes.invoke-wake.test.ts b/src/gateway/server-methods/nodes.invoke-wake.test.ts index ffa1516e8273..6dca39da54d8 100644 --- a/src/gateway/server-methods/nodes.invoke-wake.test.ts +++ b/src/gateway/server-methods/nodes.invoke-wake.test.ts @@ -5,9 +5,17 @@ import { expectDefined } from "@openclaw/normalization-core"; import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { ErrorCodes } from "../../../packages/gateway-protocol/src/index.js"; +import { + captureNodeWakeLifecycle, + clearNodeWakeState, + invalidateNodeWakeState, +} from "../node-wake-state.js"; +import { + getNodeWakeStateSnapshot, + resetNodeWakeStateForTest, +} from "../node-wake-state.test-support.js"; import { expectRecordFields, requireRecord } from "../test-helpers.assertions.js"; import { - clearNodeWakeState, maybeSendNodeWakeNudge, maybeWakeNodeWithApns, nodeHandlers, @@ -30,7 +38,9 @@ type MockNodeConfig = { }; const mocks = vi.hoisted(() => ({ + captureNodePairingGeneration: vi.fn(), getRuntimeConfig: vi.fn(() => ({})), + isNodePairingGenerationCurrent: vi.fn(), resolveNodeCommandAllowlist: vi.fn<(cfg: MockNodeConfig) => Set>(() => new Set()), isNodeCommandAllowed: vi.fn< (params: MockNodeCommandPolicyParams) => { ok: true } | { ok: false; reason: string } @@ -56,6 +66,11 @@ vi.mock("../../config/io.js", () => ({ getRuntimeConfig: mocks.getRuntimeConfig, })); +vi.mock("../../infra/node-pairing-state.js", () => ({ + captureNodePairingGeneration: mocks.captureNodePairingGeneration, + isNodePairingGenerationCurrent: mocks.isNodePairingGenerationCurrent, +})); + vi.mock("../node-command-policy.js", () => ({ DEFAULT_DANGEROUS_NODE_COMMANDS: ["sms.send", "sms.search"], resolveNodeCommandAllowlist: mocks.resolveNodeCommandAllowlist, @@ -105,9 +120,12 @@ type MockCallSource = { type TestNodeSession = { nodeId: string; + connId?: string; + pairingGeneration?: string; commands: string[]; declaredCommands?: string[]; platform?: string; + client?: { invalidated?: boolean }; }; function requireString(value: unknown, label: string): string { @@ -326,12 +344,17 @@ function makeNodeInvokeParams(overrides?: Partial>) { async function invokeNode(params: { nodeRegistry: { get: (nodeId: string) => TestNodeSession | undefined; + getForPairingGeneration?: ( + nodeId: string, + pairingGeneration: string, + ) => TestNodeSession | undefined; invoke: (payload: { nodeId: string; command: string; params?: unknown; timeoutMs?: number; idempotencyKey?: string; + expectedPairingGeneration?: string; }) => Promise<{ ok: boolean; payload?: unknown; @@ -347,6 +370,12 @@ async function invokeNode(params: { info: vi.fn(), warn: vi.fn(), }; + const nodeRegistry = { + ...params.nodeRegistry, + getForPairingGeneration: + params.nodeRegistry.getForPairingGeneration ?? + ((nodeId: string, _pairingGeneration: string) => params.nodeRegistry.get(nodeId)), + }; await expectDefined( nodeHandlers["node.invoke"], 'nodeHandlers["node.invoke"] test invariant', @@ -354,7 +383,7 @@ async function invokeNode(params: { params: makeNodeInvokeParams(params.requestParams), respond: respond as never, context: { - nodeRegistry: params.nodeRegistry, + nodeRegistry, execApprovalManager: undefined, logGateway, getRuntimeConfig: () => mocks.getRuntimeConfig(), @@ -387,6 +416,7 @@ function createOperatorClient(params?: { scopes?: string[]; pluginRuntimeOwnerId function createNodeClient(nodeId: string, commands?: string[]) { return { + connId: `conn:${nodeId}`, connect: { ...(commands ? { commands } : {}), role: "node" as const, @@ -429,32 +459,57 @@ function createMissingNodeRegistry() { }; } -async function pullPending(nodeId: string, commands?: string[]) { +async function pullPending( + nodeId: string, + commands?: string[], + sessionConnId: string | null = `conn:${nodeId}`, +) { const respond = vi.fn(); + const client = createNodeClient(nodeId, commands); await expectDefined( nodeHandlers["node.pending.pull"], 'nodeHandlers["node.pending.pull"] test invariant', )({ params: {}, respond: respond as never, - context: { getRuntimeConfig: () => mocks.getRuntimeConfig() } as never, - client: createNodeClient(nodeId, commands) as never, + context: { + getRuntimeConfig: () => mocks.getRuntimeConfig(), + nodeRegistry: { + getForPairingGeneration: vi.fn(() => + sessionConnId === null ? undefined : { connId: sessionConnId }, + ), + }, + } as never, + client: client as never, req: { type: "req", id: "req-node-pending", method: "node.pending.pull" }, isWebchatConnect: () => false, }); return respond; } -async function ackPending(nodeId: string, ids: string[], commands?: string[]) { +async function ackPending( + nodeId: string, + ids: string[], + commands?: string[], + sessionConnId: string | null = `conn:${nodeId}`, +) { const respond = vi.fn(); + const client = createNodeClient(nodeId, commands); await expectDefined( nodeHandlers["node.pending.ack"], 'nodeHandlers["node.pending.ack"] test invariant', )({ params: { ids }, respond: respond as never, - context: { getRuntimeConfig: () => mocks.getRuntimeConfig() } as never, - client: createNodeClient(nodeId, commands) as never, + context: { + getRuntimeConfig: () => mocks.getRuntimeConfig(), + nodeRegistry: { + getForPairingGeneration: vi.fn(() => + sessionConnId === null ? undefined : { connId: sessionConnId }, + ), + }, + } as never, + client: client as never, req: { type: "req", id: "req-node-pending-ack", method: "node.pending.ack" }, isWebchatConnect: () => false, }); @@ -649,6 +704,11 @@ describe("plugin surface refresh", () => { describe("node.invoke APNs wake path", () => { beforeEach(() => { + resetNodeWakeStateForTest(); + mocks.captureNodePairingGeneration.mockReset().mockImplementation(async (nodeId: string) => ({ + nodeId, + key: `generation:${nodeId}:1`, + })); mocks.getRuntimeConfig.mockClear(); mocks.getRuntimeConfig.mockReturnValue({}); mocks.resolveNodeCommandAllowlist.mockClear(); @@ -659,6 +719,7 @@ describe("node.invoke APNs wake path", () => { mocks.isForegroundRestrictedPluginNodeCommand.mockImplementation((command: string) => command.startsWith("canvas."), ); + mocks.isNodePairingGenerationCurrent.mockReset().mockResolvedValue(true); mocks.sanitizeNodeInvokeParamsForForwarding.mockClear(); mocks.sanitizeNodeInvokeParamsForForwarding.mockImplementation( ({ rawParams }: { rawParams: unknown }) => ({ ok: true, params: rawParams }), @@ -926,6 +987,32 @@ describe("node.invoke APNs wake path", () => { expect(nodeRegistry.invoke).not.toHaveBeenCalled(); }); + it("releases idle wake state when an unregistered node reconnects during invoke", async () => { + const nodeId = "ios-node-reconnect-without-registration"; + mocks.loadApnsRegistration.mockResolvedValue(null); + const session: TestNodeSession = { nodeId, commands: ["camera.capture"] }; + let lookupCount = 0; + const nodeRegistry = { + get: vi.fn(() => { + lookupCount += 1; + return lookupCount === 1 ? undefined : session; + }), + invoke: vi.fn().mockResolvedValue({ + ok: true, + payload: { ok: true }, + }), + }; + + const respond = await invokeNode({ + nodeRegistry, + requestParams: { nodeId, idempotencyKey: "idem-reconnect-without-registration" }, + }); + + expect(firstRespondCall(respond)[0]).toBe(true); + expect(nodeRegistry.invoke).toHaveBeenCalledTimes(1); + expect(getNodeWakeStateSnapshot(nodeId)).toBeUndefined(); + }); + it("does not throttle repeated relay wake attempts when relay config is missing", async () => { mocks.loadApnsRegistration.mockResolvedValue(relayRegistration("ios-node-relay-no-auth")); mocks.resolveApnsRelayConfigFromEnv.mockReturnValue({ @@ -942,6 +1029,69 @@ describe("node.invoke APNs wake path", () => { expect(mocks.sendApnsBackgroundWake).not.toHaveBeenCalled(); }); + it("does not share an in-flight wake with a replacement pairing generation", async () => { + const nodeId = "ios-node-replacement-in-flight"; + const generationOne = { nodeId, key: "generation-1" }; + const generationTwo = { nodeId, key: "generation-2" }; + let resolveFirstRegistration!: (value: null) => void; + mocks.loadApnsRegistration + .mockImplementationOnce( + () => + new Promise((resolve) => { + resolveFirstRegistration = resolve; + }), + ) + .mockResolvedValueOnce(null); + + const firstWake = maybeWakeNodeWithApns(nodeId, { generation: generationOne }); + await vi.waitFor(() => expect(mocks.loadApnsRegistration).toHaveBeenCalledTimes(1)); + + await expect( + maybeWakeNodeWithApns(nodeId, { generation: generationTwo }), + ).resolves.toMatchObject({ path: "no-registration", available: false }); + expect(mocks.loadApnsRegistration).toHaveBeenCalledTimes(2); + + resolveFirstRegistration(null); + await expect(firstWake).resolves.toMatchObject({ path: "no-registration", available: false }); + invalidateNodeWakeState(nodeId); + }); + + it("does not share wake or nudge throttles with a replacement pairing generation", async () => { + const nodeId = "ios-node-replacement-throttles"; + const generationOne = { nodeId, key: "generation-1" }; + const generationTwo = { nodeId, key: "generation-2" }; + mockDirectWakeConfig(nodeId); + mocks.sendApnsAlert.mockResolvedValue({ + ok: true, + status: 200, + tokenSuffix: "1234abcd", + topic: "ai.openclaw.ios", + environment: "sandbox", + transport: "direct", + }); + + await expect( + maybeWakeNodeWithApns(nodeId, { generation: generationOne }), + ).resolves.toMatchObject({ path: "sent", throttled: false }); + await expect( + maybeWakeNodeWithApns(nodeId, { generation: generationTwo }), + ).resolves.toMatchObject({ path: "sent", throttled: false }); + await expect( + maybeSendNodeWakeNudge(nodeId, { generation: generationOne }), + ).resolves.toMatchObject({ reason: "sent", throttled: false }); + await expect( + maybeSendNodeWakeNudge(nodeId, { generation: generationTwo }), + ).resolves.toMatchObject({ reason: "sent", throttled: false }); + + expect(mocks.sendApnsBackgroundWake).toHaveBeenCalledTimes(2); + expect(mocks.sendApnsAlert).toHaveBeenCalledTimes(2); + expect(getNodeWakeStateSnapshot(nodeId, generationOne.key)?.lastWakeAtMs).toBeGreaterThan(0); + expect(getNodeWakeStateSnapshot(nodeId, generationTwo.key)?.lastWakeAtMs).toBeGreaterThan(0); + expect(getNodeWakeStateSnapshot(nodeId, generationOne.key)?.lastNudgeAtMs).toBeGreaterThan(0); + expect(getNodeWakeStateSnapshot(nodeId, generationTwo.key)?.lastNudgeAtMs).toBeGreaterThan(0); + invalidateNodeWakeState(nodeId); + }); + it("clears wake and nudge throttle state when a node disconnects", async () => { mockDirectWakeConfig("ios-node-clear-wake"); mocks.sendApnsAlert.mockResolvedValue({ @@ -1152,6 +1302,7 @@ describe("node.invoke APNs wake path", () => { vi.setSystemTime(0); const nodeRegistry = { get: vi.fn(() => undefined), + getForPairingGeneration: vi.fn(() => undefined), }; const reconnectPromise = waitForNodeReconnect({ @@ -1169,13 +1320,16 @@ describe("node.invoke APNs wake path", () => { it("broadcasts canonical Talk capture events for successful PTT node commands", async () => { const respond = vi.fn(); const broadcast = vi.fn(); + const nodeSession = { + nodeId: "android-talk-node", + pairingGeneration: "generation:android-talk-node:1", + commands: ["talk.ptt.start"], + capabilities: ["talk"], + platform: "android", + }; const nodeRegistry = { - get: vi.fn(() => ({ - nodeId: "android-talk-node", - commands: ["talk.ptt.start"], - capabilities: ["talk"], - platform: "android", - })), + get: vi.fn(() => nodeSession), + getForPairingGeneration: vi.fn(() => nodeSession), invoke: vi.fn().mockResolvedValue({ ok: true, payloadJSON: '{"captureId":"capture-1"}', @@ -1282,6 +1436,29 @@ describe("node.invoke APNs wake path", () => { expect(mocks.clearApnsRegistrationIfCurrent).not.toHaveBeenCalled(); }); + it("rejects an invoke admitted after pairing removal without waking or dispatching", async () => { + const nodeId = "ios-node-removed-before-invoke"; + mocks.captureNodePairingGeneration.mockResolvedValueOnce(null); + const nodeRegistry = createMissingNodeRegistry(); + + const respond = await invokeNode({ + nodeRegistry, + requestParams: { nodeId, idempotencyKey: "idem-removed-before-invoke" }, + }); + + expect(mocks.loadApnsRegistration).not.toHaveBeenCalled(); + expect(mocks.sendApnsBackgroundWake).not.toHaveBeenCalled(); + expect(nodeRegistry.invoke).not.toHaveBeenCalled(); + expect(firstRespondCall(respond)).toMatchObject([ + false, + undefined, + { + message: "node pairing changed while invocation was active", + details: { code: "PAIRING_CHANGED" }, + }, + ]); + }); + it("forces one retry wake when the first wake still fails to reconnect", async () => { vi.useFakeTimers(); mockDirectWakeConfig("ios-node-throttle"); @@ -1299,6 +1476,339 @@ describe("node.invoke APNs wake path", () => { expect(nodeRegistry.invoke).not.toHaveBeenCalled(); }); + it("does not recreate wake state after removal invalidates an in-flight invoke", async () => { + vi.useFakeTimers(); + const nodeId = "ios-node-remove-during-wake"; + mockDirectWakeConfig(nodeId); + mocks.sendApnsAlert.mockResolvedValue({ + ok: true, + status: 200, + tokenSuffix: "1234abcd", + topic: "ai.openclaw.ios", + environment: "sandbox", + transport: "direct", + }); + const nodeRegistry = createMissingNodeRegistry(); + + const invokePromise = invokeNode({ + nodeRegistry, + requestParams: { nodeId, idempotencyKey: "idem-remove-during-wake" }, + }); + await vi.advanceTimersByTimeAsync(0); + expect(mocks.sendApnsBackgroundWake).toHaveBeenCalledTimes(1); + + invalidateNodeWakeState(nodeId); + await vi.advanceTimersByTimeAsync(20_000); + const respond = await invokePromise; + + expect(mocks.sendApnsBackgroundWake).toHaveBeenCalledTimes(1); + expect(mocks.sendApnsAlert).not.toHaveBeenCalled(); + expect(getNodeWakeStateSnapshot(nodeId)).toBeUndefined(); + expect(nodeRegistry.invoke).not.toHaveBeenCalled(); + const call = firstRespondCall(respond); + expect(call[0]).toBe(false); + expect(call[2]).toMatchObject({ + message: "node pairing changed while invocation was active", + details: { code: "PAIRING_CHANGED" }, + }); + }); + + it("revalidates pairing generation after direct wake auth resolves", async () => { + const nodeId = "ios-node-generation-change-during-wake-auth"; + const generation = { nodeId, key: "generation-1" }; + const lifecycle = captureNodeWakeLifecycle(nodeId, generation.key); + let pairingCurrent = true; + let resolveAuth!: (value: { + ok: true; + value: { teamId: string; keyId: string; privateKey: string }; + }) => void; + mocks.isNodePairingGenerationCurrent.mockImplementation(async () => pairingCurrent); + mocks.loadApnsRegistration.mockResolvedValue(directRegistration(nodeId)); + mocks.resolveApnsAuthConfigFromEnv.mockReturnValue( + new Promise((resolve) => { + resolveAuth = resolve; + }), + ); + + const wakePromise = maybeWakeNodeWithApns(nodeId, { lifecycle, generation }); + await vi.waitFor(() => expect(mocks.resolveApnsAuthConfigFromEnv).toHaveBeenCalledTimes(1)); + pairingCurrent = false; + resolveAuth({ + ok: true, + value: { + teamId: "TEAM123", + keyId: "KEY123", + privateKey: "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----", // pragma: allowlist secret + }, + }); + + await expect(wakePromise).resolves.toMatchObject({ path: "invalidated", available: false }); + expect(mocks.sendApnsBackgroundWake).not.toHaveBeenCalled(); + invalidateNodeWakeState(nodeId); + }); + + it("passes lifecycle and persistent currentness into direct APNs transport", async () => { + const nodeId = "ios-node-transport-generation-guard"; + const generation = { nodeId, key: "generation-1" }; + const lifecycle = captureNodeWakeLifecycle(nodeId, generation.key); + let pairingCurrent = true; + mocks.isNodePairingGenerationCurrent.mockImplementation(async () => pairingCurrent); + mocks.loadApnsRegistration.mockResolvedValue(directRegistration(nodeId)); + mocks.resolveApnsAuthConfigFromEnv.mockResolvedValue({ + ok: true, + value: { + teamId: "TEAM123", + keyId: "KEY123", + privateKey: "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----", // pragma: allowlist secret + }, + }); + + await maybeWakeNodeWithApns(nodeId, { lifecycle, generation }); + + const transport = requireRecord( + mockArg(mocks.sendApnsBackgroundWake, 0, 0), + "guarded APNs transport", + ); + expect(transport.signal).toBe(lifecycle); + expect(transport.isCurrent).toBeTypeOf("function"); + pairingCurrent = false; + await expect((transport.isCurrent as () => Promise)()).resolves.toBe(false); + invalidateNodeWakeState(nodeId); + }); + + it("revalidates pairing generation after direct nudge auth resolves", async () => { + const nodeId = "ios-node-generation-change-during-nudge-auth"; + const generation = { nodeId, key: "generation-1" }; + const lifecycle = captureNodeWakeLifecycle(nodeId, generation.key); + let pairingCurrent = true; + let resolveAuth!: (value: { + ok: true; + value: { teamId: string; keyId: string; privateKey: string }; + }) => void; + mocks.isNodePairingGenerationCurrent.mockImplementation(async () => pairingCurrent); + mocks.loadApnsRegistration.mockResolvedValue(directRegistration(nodeId)); + mocks.resolveApnsAuthConfigFromEnv.mockReturnValue( + new Promise((resolve) => { + resolveAuth = resolve; + }), + ); + + const nudgePromise = maybeSendNodeWakeNudge(nodeId, { lifecycle, generation }); + await vi.waitFor(() => expect(mocks.resolveApnsAuthConfigFromEnv).toHaveBeenCalledTimes(1)); + pairingCurrent = false; + resolveAuth({ + ok: true, + value: { + teamId: "TEAM123", + keyId: "KEY123", + privateKey: "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----", // pragma: allowlist secret + }, + }); + + await expect(nudgePromise).resolves.toMatchObject({ reason: "invalidated", sent: false }); + expect(mocks.sendApnsAlert).not.toHaveBeenCalled(); + invalidateNodeWakeState(nodeId); + }); + + it("does not dispatch an admitted invoke after the pairing generation is replaced", async () => { + vi.useFakeTimers(); + const nodeId = "ios-node-replacement-after-remove"; + mockDirectWakeConfig(nodeId); + let pairingCurrent = true; + mocks.isNodePairingGenerationCurrent.mockImplementation(async () => pairingCurrent); + const replacementSession: TestNodeSession = { + nodeId, + connId: "replacement-conn", + commands: ["camera.capture"], + platform: "iOS 26.4.0", + }; + let replacementConnected = false; + const nodeRegistry = { + get: vi.fn(() => (replacementConnected ? replacementSession : undefined)), + invoke: vi.fn().mockResolvedValue({ ok: true, payload: { replacement: true } }), + }; + + const invokePromise = invokeNode({ + nodeRegistry, + requestParams: { nodeId, idempotencyKey: "idem-replacement-after-remove" }, + }); + await vi.advanceTimersByTimeAsync(0); + expect(mocks.sendApnsBackgroundWake).toHaveBeenCalledTimes(1); + + pairingCurrent = false; + replacementConnected = true; + await vi.advanceTimersByTimeAsync(20_000); + const respond = await invokePromise; + + expect(nodeRegistry.invoke).not.toHaveBeenCalled(); + const call = firstRespondCall(respond); + expect(call[0]).toBe(false); + expect(call[2]).toMatchObject({ + message: "node pairing changed while invocation was active", + details: { code: "PAIRING_CHANGED" }, + }); + }); + + it("does not dispatch through an invalidated old node session", async () => { + const nodeId = "ios-node-invalidated-session"; + mocks.loadApnsRegistration.mockResolvedValue(null); + const invalidatedSession: TestNodeSession = { + nodeId, + connId: "old-conn", + commands: ["camera.capture"], + platform: "iOS 26.4.0", + client: { invalidated: true }, + }; + const nodeRegistry = { + get: vi.fn(() => invalidatedSession), + invoke: vi.fn().mockResolvedValue({ ok: true, payload: { delivered: true } }), + }; + + const respond = await invokeNode({ + nodeRegistry, + requestParams: { nodeId, idempotencyKey: "idem-invalidated-session" }, + }); + + expect(nodeRegistry.invoke).not.toHaveBeenCalled(); + expect(firstRespondCall(respond)).toMatchObject([ + false, + undefined, + { details: { code: "NOT_CONNECTED" } }, + ]); + }); + + it("does not dispatch current-generation work to a prior-generation session", async () => { + const nodeId = "ios-node-prior-generation-session"; + mocks.loadApnsRegistration.mockResolvedValue(null); + const oldSession: TestNodeSession = { + nodeId, + connId: "old-generation-conn", + pairingGeneration: `generation:${nodeId}:0`, + commands: ["camera.capture"], + platform: "iOS 26.4.0", + }; + const nodeRegistry = { + get: vi.fn(() => oldSession), + getForPairingGeneration: vi.fn((_requestedNodeId: string, generation: string) => + generation === oldSession.pairingGeneration ? oldSession : undefined, + ), + invoke: vi.fn().mockResolvedValue({ ok: true, payload: { delivered: true } }), + }; + + const respond = await invokeNode({ + nodeRegistry, + requestParams: { nodeId, idempotencyKey: "idem-prior-generation-session" }, + }); + + expect(nodeRegistry.getForPairingGeneration).toHaveBeenCalledWith( + nodeId, + `generation:${nodeId}:1`, + ); + expect(nodeRegistry.invoke).not.toHaveBeenCalled(); + expect(firstRespondCall(respond)).toMatchObject([ + false, + undefined, + { details: { code: "NOT_CONNECTED" } }, + ]); + }); + + it("does not queue foreground work when pairing changes during node dispatch", async () => { + const nodeId = "ios-node-replaced-during-dispatch"; + let pairingCurrent = true; + mocks.isNodePairingGenerationCurrent.mockImplementation(async () => pairingCurrent); + const nodeRegistry = createForegroundUnavailableNodeRegistry({ + nodeId, + commands: ["canvas.navigate"], + platform: "iOS 26.4.0", + }); + nodeRegistry.invoke.mockImplementation(async () => { + pairingCurrent = false; + return { + ok: false, + error: { + code: "NODE_BACKGROUND_UNAVAILABLE", + message: "NODE_BACKGROUND_UNAVAILABLE: canvas commands require foreground", + }, + }; + }); + + const respond = await invokeNode({ + nodeRegistry, + requestParams: { + nodeId, + command: "canvas.navigate", + idempotencyKey: "idem-replaced-during-dispatch", + }, + }); + + expect(nodeRegistry.invoke).toHaveBeenCalledTimes(1); + expect(mocks.loadApnsRegistration).not.toHaveBeenCalled(); + expect(firstRespondCall(respond)).toMatchObject([ + false, + undefined, + { + message: "node pairing changed while invocation was active", + details: { code: "PAIRING_CHANGED" }, + }, + ]); + }); + + it("keeps a foreground-recovery wake alive across an ordinary disconnect cleanup", async () => { + const nodeId = "ios-node-disconnect-during-foreground-wake"; + const registration = directRegistration(nodeId); + let resolveRegistration!: (value: typeof registration) => void; + mocks.loadApnsRegistration.mockReturnValue( + new Promise((resolve) => { + resolveRegistration = resolve; + }), + ); + mocks.resolveApnsAuthConfigFromEnv.mockResolvedValue({ + ok: true, + value: { + teamId: "TEAM123", + keyId: "KEY123", + privateKey: "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----", // pragma: allowlist secret + }, + }); + mocks.sendApnsBackgroundWake.mockResolvedValue({ + ok: true, + status: 200, + tokenSuffix: "1234abcd", + topic: "ai.openclaw.ios", + environment: "sandbox", + transport: "direct", + }); + const nodeRegistry = createForegroundUnavailableNodeRegistry({ + nodeId, + commands: ["canvas.navigate"], + platform: "iOS 26.4.0", + }); + + const invokePromise = invokeNode({ + nodeRegistry, + requestParams: { + nodeId, + command: "canvas.navigate", + params: { url: "http://example.com/" }, + idempotencyKey: "idem-disconnect-during-foreground-wake", + }, + }); + await vi.waitFor(() => expect(mocks.loadApnsRegistration).toHaveBeenCalledTimes(1)); + + clearNodeWakeState(nodeId); + resolveRegistration(registration); + const respond = await invokePromise; + + expect(mocks.sendApnsBackgroundWake).toHaveBeenCalledTimes(1); + const call = firstRespondCall(respond); + const details = requireRecord(call[2]?.details, "queued foreground details"); + expectRecordFields(details.wake, "queued foreground wake", { + path: "sent", + available: true, + throttled: false, + }); + }); + it("queues iOS foreground-only command failures and keeps them until acked", async () => { mocks.loadApnsRegistration.mockResolvedValue(null); @@ -1427,6 +1937,133 @@ describe("node.invoke APNs wake path", () => { }); }); + it("does not expose queued foreground actions to a replacement pairing generation", async () => { + const nodeId = "ios-node-replaced-before-pull"; + mocks.loadApnsRegistration.mockResolvedValue(null); + const nodeRegistry = createForegroundUnavailableNodeRegistry({ + nodeId, + commands: ["canvas.navigate"], + platform: "iOS 26.4.0", + }); + + await invokeNode({ + nodeRegistry, + requestParams: { + nodeId, + command: "canvas.navigate", + idempotencyKey: "idem-replaced-before-pull", + }, + }); + mocks.captureNodePairingGeneration.mockResolvedValue({ + nodeId, + key: `generation:${nodeId}:2`, + }); + + const pullRespond = await pullPending(nodeId, ["canvas.navigate"]); + expectRecordFields( + requireRespondPayload(firstRespondCall(pullRespond), "replacement pull response"), + "replacement pull payload", + { nodeId, actions: [] }, + ); + }); + + it("does not let a prior-generation session pull or ack current-generation actions", async () => { + const nodeId = "ios-node-prior-generation-pending"; + mocks.loadApnsRegistration.mockResolvedValue(null); + const nodeRegistry = createForegroundUnavailableNodeRegistry({ + nodeId, + commands: ["canvas.navigate"], + platform: "iOS 26.4.0", + }); + + await invokeNode({ + nodeRegistry, + requestParams: { + nodeId, + command: "canvas.navigate", + idempotencyKey: "idem-prior-generation-pending", + }, + }); + + expect(firstRespondCall(await pullPending(nodeId, ["canvas.navigate"], null))).toMatchObject([ + false, + undefined, + { details: { code: "PAIRING_CHANGED" } }, + ]); + + const currentPullPayload = requireRespondPayload( + firstRespondCall(await pullPending(nodeId, ["canvas.navigate"])), + "current-generation pull response", + ); + const queuedActionId = requireString( + (currentPullPayload.actions as Array<{ id?: string }> | undefined)?.[0]?.id, + "current-generation queued action id", + ); + + expect( + firstRespondCall(await ackPending(nodeId, [queuedActionId], ["canvas.navigate"], null)), + ).toMatchObject([false, undefined, { details: { code: "PAIRING_CHANGED" } }]); + expect( + ( + requireRespondPayload( + firstRespondCall(await pullPending(nodeId, ["canvas.navigate"])), + "post-stale-ack pull response", + ).actions as unknown[] | undefined + )?.length, + ).toBe(1); + }); + + it("does not let a stale foreground pull delete replacement-generation actions", async () => { + const nodeId = "ios-node-stale-pull"; + mocks.loadApnsRegistration.mockResolvedValue(null); + const nodeRegistry = createForegroundUnavailableNodeRegistry({ + nodeId, + commands: ["canvas.navigate"], + platform: "iOS 26.4.0", + }); + + await invokeNode({ + nodeRegistry, + requestParams: { + nodeId, + command: "canvas.navigate", + idempotencyKey: "idem-stale-generation", + }, + }); + mocks.captureNodePairingGeneration.mockResolvedValue({ + nodeId, + key: `generation:${nodeId}:2`, + }); + await invokeNode({ + nodeRegistry, + requestParams: { + nodeId, + command: "canvas.navigate", + idempotencyKey: "idem-replacement-generation", + }, + }); + + mocks.captureNodePairingGeneration.mockResolvedValue({ + nodeId, + key: `generation:${nodeId}:1`, + }); + const stalePullPayload = requireRespondPayload( + firstRespondCall(await pullPending(nodeId, ["canvas.navigate"])), + "stale generation pull response", + ); + expect((stalePullPayload.actions as unknown[] | undefined)?.length).toBe(1); + + mocks.captureNodePairingGeneration.mockResolvedValue({ + nodeId, + key: `generation:${nodeId}:2`, + }); + const replacementPullPayload = requireRespondPayload( + firstRespondCall(await pullPending(nodeId, ["canvas.navigate"])), + "replacement generation pull response", + ); + expect((replacementPullPayload.actions as unknown[] | undefined)?.length).toBe(1); + }); + it("dedupes queued foreground actions by idempotency key", async () => { mocks.loadApnsRegistration.mockResolvedValue(null); diff --git a/src/gateway/server-methods/nodes.test.ts b/src/gateway/server-methods/nodes.test.ts index a50074505f34..6ed3fcbb0d46 100644 --- a/src/gateway/server-methods/nodes.test.ts +++ b/src/gateway/server-methods/nodes.test.ts @@ -5,6 +5,7 @@ import { listDevicePairing, requestDevicePairing, revokeDeviceToken, + rotateDeviceToken, withPairedDeviceRecords, } from "../../infra/device-pairing.js"; import { @@ -12,16 +13,50 @@ import { resetDiagnosticEventsForTest, type DiagnosticSecurityEvent, } from "../../infra/diagnostic-events.js"; +import { + captureNodePairingGeneration, + captureNodePairingState, +} from "../../infra/node-pairing-state.js"; import { approveNodePairing, requestNodePairing } from "../../infra/node-pairing.js"; +import { loadApnsRegistration, registerApnsRegistration } from "../../infra/push-apns.js"; import { resetRemoteNodeSkillsForTests } from "../../skills/runtime/remote-skills.test-support.js"; +import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js"; import { createOpenClawTestState, type OpenClawTestState, } from "../../test-utils/openclaw-test-state.js"; +import { drainNodePendingWork, enqueueNodePendingWork } from "../node-pending-work.js"; +import { + captureNodeWakeLifecycle, + runNodeWakeAttempt, + runNodeWakeNudgeAttempt, +} from "../node-wake-state.js"; +import { + getNodeWakeStateSnapshot, + resetNodeWakeStateForTest, +} from "../node-wake-state.test-support.js"; import { nodeHandlers } from "./nodes.js"; import type { GatewayRequestHandlerOptions } from "./types.js"; const createdStates: OpenClawTestState[] = []; +const pairingGenerationHooks = vi.hoisted(() => ({ + beforeCapture: vi.fn<(nodeId: string) => Promise | void>(), +})); + +vi.mock("../../infra/node-pairing-state.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + captureNodePairingState: async (nodeId: string) => { + await pairingGenerationHooks.beforeCapture(nodeId); + return await actual.captureNodePairingState(nodeId); + }, + captureNodePairingGeneration: async (nodeId: string) => { + await pairingGenerationHooks.beforeCapture(nodeId); + return await actual.captureNodePairingGeneration(nodeId); + }, + }; +}); async function createState(label: string): Promise { const state = await createOpenClawTestState({ label, layout: "state-only" }); @@ -29,10 +64,31 @@ async function createState(label: string): Promise { return state; } +async function seedNodeWakeState(nodeId: string): Promise { + await runNodeWakeAttempt({ + nodeId, + force: true, + throttleMs: 60_000, + attempt: async (markAttempted) => { + markAttempted(); + return { available: true, throttled: false, path: "sent", durationMs: 1 }; + }, + }); + await runNodeWakeNudgeAttempt({ + nodeId, + throttleMs: 60_000, + throttled: () => ({ sent: false, throttled: true, reason: "throttled", durationMs: 0 }), + attempt: async () => ({ sent: true, throttled: false, reason: "sent", durationMs: 1 }), + }); +} + afterEach(async () => { resetDiagnosticEventsForTest(); resetRemoteNodeSkillsForTests(); + resetNodeWakeStateForTest(); + pairingGenerationHooks.beforeCapture.mockReset(); vi.clearAllMocks(); + closeOpenClawStateDatabaseForTest(); while (createdStates.length > 0) { await createdStates.pop()?.cleanup(); } @@ -55,6 +111,7 @@ function createContext() { return { broadcast: vi.fn(), disconnectClientsForDevice: vi.fn(), + getRuntimeConfig: vi.fn(() => ({})), invalidateClientsForDevice: vi.fn(), logGateway: { debug: vi.fn(), @@ -63,7 +120,10 @@ function createContext() { warn: vi.fn(), }, nodeRegistry: { + get: vi.fn(), listConnected: vi.fn(() => []), + listConnectedForPairingStates: vi.fn(() => []), + getActiveNode: vi.fn(), updateSurface: vi.fn(), updateNodeSkills: vi.fn(), }, @@ -209,7 +269,335 @@ async function readPaired(stateDir: string): Promise> { return Object.fromEntries(paired.map((device) => [device.deviceId, device])); } +describe("nodeHandlers node.pair.approve", () => { + it("promotes the first surface only for the same authenticated pairing identity", async () => { + const state = await createState("node-approve-promotes-pending-identity"); + const nodeId = "node-pending-identity-stable"; + await pairAndroidNodeDevice(state.stateDir, nodeId); + const pending = await requestNodePairing( + { + nodeId, + platform: "android", + deviceFamily: "Android", + clientId: "openclaw-android", + clientMode: "node", + displayName: "Galaxy A54 5G pending", + }, + state.stateDir, + ); + const pendingState = await captureNodePairingState(nodeId); + expect(pendingState?.generation).toBeNull(); + + const { context, opts } = createOptions({ requestId: pending.request.requestId }); + context.nodeRegistry.get.mockReturnValue({ + nodeId, + connId: "conn-pending-identity-stable", + pairingIdentity: pendingState?.identity.key, + }); + + await expectDefined( + nodeHandlers["node.pair.approve"], + 'nodeHandlers["node.pair.approve"] test invariant', + )(opts); + + const approvedState = await captureNodePairingState(nodeId); + expect(approvedState?.identity.key).toBe(pendingState?.identity.key); + expect(approvedState?.generation).not.toBeNull(); + expect(context.nodeRegistry.updateSurface).toHaveBeenCalledWith( + nodeId, + expect.objectContaining({ commands: expect.any(Array) }), + { + expectedConnId: "conn-pending-identity-stable", + expectedPairingIdentity: pendingState?.identity.key, + nextPairingGeneration: approvedState?.generation?.key, + }, + ); + expect(opts.respond).toHaveBeenCalledWith( + true, + expect.objectContaining({ node: expect.objectContaining({ nodeId }) }), + undefined, + ); + }); + + it("invalidates an in-flight wake when node-surface approval rotates generation", async () => { + const state = await createState("node-approve-invalidates-wake"); + const nodeId = "node-surface-reapproval"; + await pairAndroidNodeDevice(state.stateDir, nodeId); + await approveNodeSurface(state.stateDir, nodeId); + const previousGeneration = await captureNodePairingGeneration(nodeId); + const previousState = await captureNodePairingState(nodeId); + expect(previousGeneration).not.toBeNull(); + expect(previousState).not.toBeNull(); + const pending = await requestNodePairing( + { + nodeId, + platform: "android", + deviceFamily: "Android", + clientId: "openclaw-android", + clientMode: "node", + displayName: "Galaxy A54 5G reapproved", + }, + state.stateDir, + ); + const lifecycle = captureNodeWakeLifecycle(nodeId); + const { context, opts } = createOptions({ requestId: pending.request.requestId }); + context.nodeRegistry.get.mockReturnValue({ + nodeId, + connId: "conn-surface-reapproval", + pairingIdentity: previousState?.identity.key, + pairingGeneration: previousGeneration?.key, + }); + + await expectDefined( + nodeHandlers["node.pair.approve"], + 'nodeHandlers["node.pair.approve"] test invariant', + )(opts); + + expect(lifecycle.aborted).toBe(true); + const nextGeneration = await captureNodePairingGeneration(nodeId); + expect(nextGeneration?.key).not.toBe(previousGeneration?.key); + expect(context.nodeRegistry.updateSurface).toHaveBeenCalledWith( + nodeId, + expect.objectContaining({ commands: expect.any(Array) }), + { + expectedConnId: "conn-surface-reapproval", + expectedPairingIdentity: previousState?.identity.key, + expectedPairingGeneration: previousGeneration?.key, + nextPairingGeneration: nextGeneration?.key, + }, + ); + expect(opts.respond).toHaveBeenCalledWith( + true, + expect.objectContaining({ node: expect.objectContaining({ nodeId }) }), + undefined, + ); + }); + + it("does not promote a session authenticated before external device reapproval", async () => { + const state = await createState("node-approve-rejects-stale-live-session"); + const nodeId = "node-stale-surface-session"; + await pairAndroidNodeDevice(state.stateDir, nodeId); + await approveNodeSurface(state.stateDir, nodeId); + const staleGeneration = await captureNodePairingGeneration(nodeId); + const staleState = await captureNodePairingState(nodeId); + expect(staleGeneration).not.toBeNull(); + expect(staleState).not.toBeNull(); + const pending = await requestNodePairing( + { + nodeId, + platform: "android", + deviceFamily: "Android", + clientId: "openclaw-android", + clientMode: "node", + displayName: "Galaxy A54 5G reapproved", + }, + state.stateDir, + ); + await pairAndroidNodeDevice(state.stateDir, nodeId); + const currentGeneration = await captureNodePairingGeneration(nodeId); + expect(currentGeneration?.key).not.toBe(staleGeneration?.key); + + const { context, opts } = createOptions({ requestId: pending.request.requestId }); + context.nodeRegistry.get.mockReturnValue({ + nodeId, + connId: "conn-authenticated-before-reapproval", + pairingIdentity: staleState?.identity.key, + pairingGeneration: staleGeneration?.key, + }); + + await expectDefined( + nodeHandlers["node.pair.approve"], + 'nodeHandlers["node.pair.approve"] test invariant', + )(opts); + + expect(context.nodeRegistry.updateSurface).not.toHaveBeenCalled(); + expect(opts.respond).toHaveBeenCalledWith( + true, + expect.objectContaining({ node: expect.objectContaining({ nodeId }) }), + undefined, + ); + }); + + it("does not promote an old session when reapproval wins after surface approval commits", async () => { + const state = await createState("node-approve-fences-post-approval-repair"); + const nodeId = "node-post-approval-repair"; + await pairAndroidNodeDevice(state.stateDir, nodeId); + await approveNodeSurface(state.stateDir, nodeId); + const staleGeneration = await captureNodePairingGeneration(nodeId); + const staleState = await captureNodePairingState(nodeId); + expect(staleGeneration).not.toBeNull(); + expect(staleState).not.toBeNull(); + const pending = await requestNodePairing( + { + nodeId, + platform: "android", + deviceFamily: "Android", + clientId: "openclaw-android", + clientMode: "node", + displayName: "Galaxy A54 5G surface refresh", + }, + state.stateDir, + ); + let captureCount = 0; + pairingGenerationHooks.beforeCapture.mockImplementation(async (capturedNodeId) => { + if (capturedNodeId !== nodeId) { + return; + } + captureCount += 1; + if (captureCount === 2) { + await pairAndroidNodeDevice(state.stateDir, nodeId); + } + }); + + const { context, opts } = createOptions({ requestId: pending.request.requestId }); + context.nodeRegistry.get.mockReturnValue({ + nodeId, + connId: "conn-authenticated-before-post-approval-repair", + pairingIdentity: staleState?.identity.key, + pairingGeneration: staleGeneration?.key, + }); + + await expectDefined( + nodeHandlers["node.pair.approve"], + 'nodeHandlers["node.pair.approve"] test invariant', + )(opts); + + const currentGeneration = await captureNodePairingGeneration(nodeId); + expect(currentGeneration?.key).not.toBe(staleGeneration?.key); + expect(context.nodeRegistry.updateSurface).not.toHaveBeenCalled(); + expect(opts.respond).toHaveBeenCalledWith( + true, + expect.objectContaining({ node: expect.objectContaining({ nodeId }) }), + undefined, + ); + }); + + it("does not promote a generation-less session after external node-token rotation", async () => { + const state = await createState("node-approve-fences-pending-identity"); + const nodeId = "node-pending-identity-rotation"; + await pairAndroidNodeDevice(state.stateDir, nodeId); + const pending = await requestNodePairing( + { + nodeId, + platform: "android", + deviceFamily: "Android", + clientId: "openclaw-android", + clientMode: "node", + displayName: "Galaxy A54 5G pending", + }, + state.stateDir, + ); + const staleState = await captureNodePairingState(nodeId); + expect(staleState?.generation).toBeNull(); + + const rotated = await rotateDeviceToken({ + deviceId: nodeId, + role: "node", + scopes: [], + baseDir: state.stateDir, + }); + expect(rotated.ok).toBe(true); + const currentState = await captureNodePairingState(nodeId); + expect(currentState?.generation).toBeNull(); + expect(currentState?.identity.key).not.toBe(staleState?.identity.key); + + const { context, opts } = createOptions({ requestId: pending.request.requestId }); + context.nodeRegistry.get.mockReturnValue({ + nodeId, + connId: "conn-authenticated-before-token-rotation", + pairingIdentity: staleState?.identity.key, + }); + + await expectDefined( + nodeHandlers["node.pair.approve"], + 'nodeHandlers["node.pair.approve"] test invariant', + )(opts); + + expect(context.nodeRegistry.updateSurface).not.toHaveBeenCalled(); + expect(opts.respond).toHaveBeenCalledWith( + true, + expect.objectContaining({ node: expect.objectContaining({ nodeId }) }), + undefined, + ); + }); +}); + describe("nodeHandlers node.pair.remove", () => { + it("clears and invalidates wake state when removing a disconnected device-backed node", async () => { + const state = await createState("node-remove-clears-wake-state"); + const nodeId = "disconnected-ios-node"; + await pairAndroidNodeDevice(state.stateDir, nodeId); + await registerApnsRegistration({ + nodeId, + transport: "direct", + token: "ABCD1234ABCD1234ABCD1234ABCD1234", + topic: "ai.openclaw.ios", + environment: "sandbox", + }); + await seedNodeWakeState(nodeId); + enqueueNodePendingWork({ nodeId, type: "location.request" }); + const wakeLifecycle = captureNodeWakeLifecycle(nodeId); + + const { opts } = createOptions({ nodeId }); + await expectDefined( + nodeHandlers["node.pair.remove"], + 'nodeHandlers["node.pair.remove"] test invariant', + )(opts); + await Promise.resolve(); + + expect(opts.respond).toHaveBeenCalledWith(true, { nodeId }, undefined); + expect(getNodeWakeStateSnapshot(nodeId)).toBeUndefined(); + expect(wakeLifecycle.aborted).toBe(true); + expect(drainNodePendingWork(nodeId).items.map((item) => item.id)).toEqual(["baseline-status"]); + await expect(loadApnsRegistration(nodeId)).resolves.toBeNull(); + }); + + it("preserves an APNs registration created after node-role removal commits", async () => { + const state = await createState("node-remove-apns-registration-race"); + const nodeId = "ios-node-registration-race"; + await pairAndroidNodeDevice(state.stateDir, nodeId); + await registerApnsRegistration({ + nodeId, + transport: "direct", + token: "ABCD1234ABCD1234ABCD1234ABCD1234", + topic: "ai.openclaw.ios", + environment: "sandbox", + }); + + const { context, opts } = createOptions({ nodeId }); + let replacementWrite: Promise | undefined; + context.invalidateClientsForDevice.mockImplementation(() => { + replacementWrite = (async () => { + await pairAndroidNodeDevice(state.stateDir, nodeId); + await approveNodeSurface(state.stateDir, nodeId); + const replacementGeneration = await captureNodePairingGeneration(nodeId); + if (!replacementGeneration) { + throw new Error("expected replacement pairing generation"); + } + return await registerApnsRegistration({ + nodeId, + transport: "direct", + token: "DCBA4321DCBA4321DCBA4321DCBA4321", + topic: "ai.openclaw.ios", + environment: "sandbox", + expectedPairingGeneration: replacementGeneration.key, + }); + })(); + }); + + await expectDefined( + nodeHandlers["node.pair.remove"], + 'nodeHandlers["node.pair.remove"] test invariant', + )(opts); + await replacementWrite; + + await expect(loadApnsRegistration(nodeId)).resolves.toMatchObject({ + nodeId, + transport: "direct", + token: "dcba4321dcba4321dcba4321dcba4321", + }); + }); + it("removes Android device-backed node rows from the paired-device store", async () => { const state = await createState("node-remove-android-device-backed"); const nodeId = "android-node-1"; @@ -359,6 +747,13 @@ describe("nodeHandlers node.pair.remove", () => { const state = await createState("node-remove-mixed-role-device"); const nodeId = "mixed-role-android-node-1"; await pairMixedRoleAndroidDevice(state.stateDir, nodeId); + await registerApnsRegistration({ + nodeId, + transport: "direct", + token: "ABCD1234ABCD1234ABCD1234ABCD1234", + topic: "ai.openclaw.ios", + environment: "sandbox", + }); const before = await readPaired(state.stateDir); expect( @@ -387,6 +782,11 @@ describe("nodeHandlers node.pair.remove", () => { expect( Object.hasOwn((after[nodeId] as { tokens?: Record }).tokens ?? {}, "node"), ).toBe(false); + await expect(loadApnsRegistration(nodeId)).resolves.toMatchObject({ + nodeId, + transport: "direct", + token: "abcd1234abcd1234abcd1234abcd1234", + }); expect(context.invalidateClientsForDevice).toHaveBeenCalledWith(nodeId, { role: "node", reason: "device-pair-removed", diff --git a/src/gateway/server-methods/nodes.ts b/src/gateway/server-methods/nodes.ts index dc88d0f6884a..f6d6a55c4357 100644 --- a/src/gateway/server-methods/nodes.ts +++ b/src/gateway/server-methods/nodes.ts @@ -32,9 +32,16 @@ import { listApprovedPairedDeviceRoles, listDevicePairing, removePairedDeviceRole, + resolveNodePairingState, } from "../../infra/device-pairing.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { isAdminOnlyNodeInvokeCommand } from "../../infra/node-commands.js"; +import { + captureNodePairingGeneration, + captureNodePairingState, + isNodePairingGenerationCurrent, + type NodePairingGeneration, +} from "../../infra/node-pairing-state.js"; import { approveNodePairing, getPendingNodePairing, @@ -53,11 +60,7 @@ import { } from "../../infra/push-apns.js"; import type { NodeListNode } from "../../shared/node-list-types.js"; import { replaceRemoteNodeSkills } from "../../skills/runtime/remote-skills.js"; -import { - recordRemoteNodeInfo, - refreshRemoteNodeBins, - removeRemoteNodeInfo, -} from "../../skills/runtime/remote.js"; +import { recordRemoteNodeInfo, refreshRemoteNodeBins } from "../../skills/runtime/remote.js"; import { isForbiddenBrowserProxyMutation } from "../node-browser-proxy-policy.js"; import { createKnownNodeCatalog, getKnownNode, listKnownNodes } from "../node-catalog.js"; import { @@ -70,6 +73,29 @@ import { import { applyPluginNodeInvokePolicy } from "../node-invoke-plugin-policy.js"; import { sanitizeNodeInvokeParamsForForwarding } from "../node-invoke-sanitize.js"; import type { NodeSession } from "../node-registry.js"; +import { + acknowledgePendingNodeActions, + clearRemovedNodeRuntimeState, + enqueuePendingNodeAction, + listPendingNodeActions, + removePendingNodeAction, + replacePendingNodeActionsForGeneration, + type PendingNodeAction, +} from "../node-runtime-state.js"; +import { + captureNodeWakeLifecycle, + invalidateNodeWakeState, + isNodeWakeLifecycleCurrent, + NODE_WAKE_RECONNECT_POLL_MS, + NODE_WAKE_RECONNECT_RETRY_WAIT_MS, + NODE_WAKE_RECONNECT_WAIT_MS, + releaseNodeWakeLifecycle, + runNodeWakeAttempt, + runNodeWakeNudgeAttempt, + type NodeWakeAttempt, + type NodeWakeLifecycle, + type NodeWakeNudgeAttempt, +} from "../node-wake-state.js"; import { ADMIN_SCOPE, PAIRING_SCOPE } from "../operator-scopes.js"; import { hasAuthorizedClientPluginNodeCapabilityUrl, @@ -87,14 +113,6 @@ import { import { emitDeviceManagementSecurityEvent } from "./device-management-security.js"; import { buildNodeCommandRejectionHint } from "./node-command-rejection-hint.js"; import { nodeInvokePolicy } from "./nodes-policy.js"; -import { - NODE_WAKE_RECONNECT_POLL_MS, - NODE_WAKE_RECONNECT_RETRY_WAIT_MS, - NODE_WAKE_RECONNECT_WAIT_MS, - nodeWakeById, - nodeWakeNudgeById, - type NodeWakeAttempt, -} from "./nodes-wake-state.js"; import { handleNodeInvokeProgress } from "./nodes.handlers.invoke-progress.js"; import { handleNodeInvokeResult } from "./nodes.handlers.invoke-result.js"; import { @@ -106,12 +124,6 @@ import { import type { GatewayClient, GatewayRequestContext, RespondFn } from "./shared-types.js"; import type { GatewayRequestHandler, GatewayRequestHandlers } from "./types.js"; -export { - clearNodeWakeState, - NODE_WAKE_RECONNECT_RETRY_WAIT_MS, - NODE_WAKE_RECONNECT_WAIT_MS, -} from "./nodes-wake-state.js"; - const TALK_PTT_COMMANDS = new Set([ "talk.ptt.start", "talk.ptt.stop", @@ -120,27 +132,6 @@ const TALK_PTT_COMMANDS = new Set([ ]); const talkPttEventSeqBySessionId = new Map(); -type NodeWakeNudgeAttempt = { - sent: boolean; - throttled: boolean; - reason: "throttled" | "no-registration" | "no-auth" | "send-error" | "apns-not-ok" | "sent"; - durationMs: number; - apnsStatus?: number; - apnsReason?: string; -}; - -type PendingNodeAction = { - id: string; - nodeId: string; - command: string; - paramsJSON?: string; - idempotencyKey: string; - - enqueuedAtMs: number; -}; - -const pendingNodeActionsById = new Map(); - function safeNodeReadProjection( node: NodeListNode, ownDeviceId: string | undefined, @@ -191,6 +182,23 @@ function listNodesForClient(params: { return nodes.map((node) => safeNodeReadProjection(node, ownDeviceId)).filter(isVisibleNode); } +function listCurrentConnectedNodes( + context: GatewayRequestContext, + pairedDevices: Awaited>["paired"], +): NodeSession[] { + const currentPairingStates = new Map(); + for (const device of pairedDevices) { + const state = resolveNodePairingState(device); + if (state) { + currentPairingStates.set(state.identity.nodeId, { + identity: state.identity.key, + ...(state.generation ? { generation: state.generation.key } : {}), + }); + } + } + return context.nodeRegistry.listConnectedForPairingStates(currentPairingStates); +} + function normalizePluginSurfaceRefreshParams( params: unknown, ): { surface: string; observedUrl?: string } | undefined { @@ -356,29 +364,49 @@ function shouldQueueAsPendingForegroundAction(params: { return code === "NODE_BACKGROUND_UNAVAILABLE" || message.includes("BACKGROUND_UNAVAILABLE"); } -function prunePendingNodeActions(nodeId: string, nowMs: number): PendingNodeAction[] { - const queue = pendingNodeActionsById.get(nodeId) ?? []; - const minTimestampMs = nowMs - nodeInvokePolicy.pendingActionTtlMs; - const live = queue.filter((entry) => entry.enqueuedAtMs >= minTimestampMs); - if (live.length === 0) { - pendingNodeActionsById.delete(nodeId); - return []; - } - pendingNodeActionsById.set(nodeId, live); - return live; +function respondPairingChanged(respond: RespondFn) { + respond( + false, + undefined, + errorShape(ErrorCodes.UNAVAILABLE, "node pairing changed while invocation was active", { + retryable: true, + details: { code: "PAIRING_CHANGED" }, + }), + ); } -function clearRemovedNodeRuntimeState(params: { +async function isNodePairingWorkCurrent(params: { nodeId: string; - context: Pick; -}) { - pendingNodeActionsById.delete(params.nodeId); - params.context.nodeRegistry.updateSurface(params.nodeId, { - caps: [], - commands: [], - permissions: undefined, - }); - removeRemoteNodeInfo(params.nodeId); + generation: NodePairingGeneration; + lifecycle: NodeWakeLifecycle; +}): Promise { + if (!isNodeWakeLifecycleCurrent(params.nodeId, params.lifecycle, params.generation.key)) { + return false; + } + if (!(await isNodePairingGenerationCurrent(params.generation))) { + return false; + } + // Pairing mutation owners invalidate the lifecycle after persistence. Check + // it again because the keyed generation lookup may yield before side effects. + return isNodeWakeLifecycleCurrent(params.nodeId, params.lifecycle, params.generation.key); +} + +async function isNodePushAttemptCurrent(params: { + nodeId: string; + lifecycle: NodeWakeLifecycle; + generation?: NodePairingGeneration; +}): Promise { + return params.generation + ? isNodePairingWorkCurrent({ + nodeId: params.nodeId, + generation: params.generation, + lifecycle: params.lifecycle, + }) + : isNodeWakeLifecycleCurrent(params.nodeId, params.lifecycle); +} + +function resolveDispatchableNodeSession(session: NodeSession | undefined): NodeSession | undefined { + return session?.client?.invalidated === true ? undefined : session; } function broadcastRemovedNodePairing(params: { @@ -486,7 +514,11 @@ async function removePairedDeviceBackedNode(params: { "disconnectClientsForDevice" | "invalidateClientsForDevice" | "logGateway" >; }): Promise< - | { status: "removed"; nodeId: string; disconnectDeviceId: string } + | { + status: "removed"; + nodeId: string; + disconnectDeviceId: string; + } | { status: "denied"; message: string } | { status: "unknown" } > { @@ -544,41 +576,11 @@ async function removePairedDeviceBackedNode(params: { role: "node", reason: "device-pair-removed", }); - return { status: "removed", nodeId: removed.deviceId, disconnectDeviceId: removed.deviceId }; -} - -function enqueuePendingNodeAction(params: { - nodeId: string; - command: string; - paramsJSON?: string; - idempotencyKey: string; -}): PendingNodeAction { - const nowMs = Date.now(); - const queue = prunePendingNodeActions(params.nodeId, nowMs); - const existing = queue.find((entry) => entry.idempotencyKey === params.idempotencyKey); - if (existing) { - // Keep retries idempotent so callers do not create duplicate foreground - // actions while the node is still backgrounded. - return existing; - } - const entry: PendingNodeAction = { - id: randomUUID(), - nodeId: params.nodeId, - command: params.command, - paramsJSON: params.paramsJSON, - idempotencyKey: params.idempotencyKey, - enqueuedAtMs: nowMs, + return { + status: "removed", + nodeId: removed.deviceId, + disconnectDeviceId: removed.deviceId, }; - queue.push(entry); - if (queue.length > nodeInvokePolicy.pendingActionMaxPerNode) { - queue.splice(0, queue.length - nodeInvokePolicy.pendingActionMaxPerNode); - } - pendingNodeActionsById.set(params.nodeId, queue); - return entry; -} - -function listPendingNodeActions(nodeId: string): PendingNodeAction[] { - return prunePendingNodeActions(nodeId, Date.now()); } function refreshConnectedNodeSurfaceCaches(params: { @@ -596,6 +598,7 @@ function refreshConnectedNodeSurfaceCaches(params: { deviceFamily: nodeSession.deviceFamily, commands: nodeSession.commands, remoteIp: nodeSession.remoteIp, + pairingGeneration: nodeSession.pairingGeneration, }); void refreshRemoteNodeBins({ nodeId: nodeSession.nodeId, @@ -612,10 +615,15 @@ function refreshConnectedNodeSurfaceCaches(params: { function resolveAllowedPendingNodeActions(params: { nodeId: string; + pairingGeneration: string; client: { connect?: ConnectParams | null } | null; cfg: OpenClawConfig; }): PendingNodeAction[] { - const pending = listPendingNodeActions(params.nodeId); + const pending = listPendingNodeActions({ + nodeId: params.nodeId, + pairingGeneration: params.pairingGeneration, + ttlMs: nodeInvokePolicy.pendingActionTtlMs, + }); if (pending.length === 0) { return pending; } @@ -638,28 +646,34 @@ function resolveAllowedPendingNodeActions(params: { return result.ok; }); if (allowed.length !== pending.length) { - if (allowed.length === 0) { - pendingNodeActionsById.delete(params.nodeId); - } else { - pendingNodeActionsById.set(params.nodeId, allowed); - } + replacePendingNodeActionsForGeneration({ + nodeId: params.nodeId, + pairingGeneration: params.pairingGeneration, + replacement: allowed, + ttlMs: nodeInvokePolicy.pendingActionTtlMs, + }); } return allowed; } -function ackPendingNodeActions(nodeId: string, ids: string[]): PendingNodeAction[] { +function ackPendingNodeActions( + nodeId: string, + ids: string[], + pairingGeneration: string, +): PendingNodeAction[] { if (ids.length === 0) { - return listPendingNodeActions(nodeId); + return listPendingNodeActions({ + nodeId, + pairingGeneration, + ttlMs: nodeInvokePolicy.pendingActionTtlMs, + }); } - const pending = prunePendingNodeActions(nodeId, Date.now()); - const idSet = new Set(ids); - const remaining = pending.filter((entry) => !idSet.has(entry.id)); - if (remaining.length === 0) { - pendingNodeActionsById.delete(nodeId); - return []; - } - pendingNodeActionsById.set(nodeId, remaining); - return remaining; + return acknowledgePendingNodeActions({ + nodeId, + pairingGeneration, + ids, + ttlMs: nodeInvokePolicy.pendingActionTtlMs, + }); } function toPendingParamsJSON(params: unknown): string | undefined { @@ -738,127 +752,148 @@ function emitTalkPttNodeEvent(params: { export async function maybeWakeNodeWithApns( nodeId: string, - opts?: { force?: boolean; wakeReason?: string; cfg?: OpenClawConfig }, + opts?: { + force?: boolean; + wakeReason?: string; + cfg?: OpenClawConfig; + lifecycle?: NodeWakeLifecycle; + generation?: NodePairingGeneration; + }, ): Promise { - const state = nodeWakeById.get(nodeId) ?? { lastWakeAtMs: 0 }; - nodeWakeById.set(nodeId, state); - - if (state.inFlight) { - return await state.inFlight; - } - - const now = Date.now(); - const force = opts?.force === true; - if ( - !force && - state.lastWakeAtMs > 0 && - now - state.lastWakeAtMs < nodeInvokePolicy.wakeThrottleMs - ) { - return { available: true, throttled: true, path: "throttled", durationMs: 0 }; - } - - state.inFlight = (async () => { - const startedAtMs = Date.now(); - const withDuration = (attempt: Omit): NodeWakeAttempt => ({ - ...attempt, - durationMs: Math.max(0, Date.now() - startedAtMs), - }); - - try { - const registration = await loadApnsRegistration(nodeId); - if (!registration) { - // Avoid leaking the state entry we speculatively set at the top of - // maybeWakeNodeWithApns: this nodeId has no APNs registration, so the - // throttle bookkeeping we just created will never be touched by the - // WS-close cleanup path (clearNodeWakeState is only called for - // registered nodes in ws-connection.ts). - nodeWakeById.delete(nodeId); - return withDuration({ available: false, throttled: false, path: "no-registration" }); - } - - let wakeResult; - if (registration.transport === "relay") { - const relay = resolveRelayNodePushConfig(opts?.cfg ?? getRuntimeConfig(), registration); - if (!relay.ok) { - return withDuration({ - available: false, - throttled: false, - path: "no-auth", - apnsReason: relay.error, - }); - } - state.lastWakeAtMs = Date.now(); - wakeResult = await sendApnsBackgroundWake({ - registration, - nodeId, - wakeReason: opts?.wakeReason ?? "node.invoke", - relayConfig: relay.relayConfig, - }); - } else { - const auth = await resolveDirectNodePushConfig(); - if (!auth.ok) { - return withDuration({ - available: false, - throttled: false, - path: "no-auth", - apnsReason: auth.error, - }); - } - state.lastWakeAtMs = Date.now(); - wakeResult = await sendApnsBackgroundWake({ - registration, - nodeId, - wakeReason: opts?.wakeReason ?? "node.invoke", - auth: auth.auth, - }); - } - await clearStaleApnsRegistrationIfNeeded(registration, nodeId, wakeResult); - if (!wakeResult.ok) { - return withDuration({ - available: true, - throttled: false, - path: "send-error", - apnsStatus: wakeResult.status, - apnsReason: wakeResult.reason, - }); - } - return withDuration({ - available: true, - throttled: false, - path: "sent", - apnsStatus: wakeResult.status, - apnsReason: wakeResult.reason, - }); - } catch (err) { - // Best-effort wake only. - const message = formatErrorMessage(err); - if (state.lastWakeAtMs === 0) { - return withDuration({ - available: false, - throttled: false, - path: "send-error", - apnsReason: message, - }); - } - return withDuration({ - available: true, - throttled: false, - path: "send-error", - apnsReason: message, - }); - } - })(); - + const lifecycleProvided = opts?.lifecycle !== undefined; + const pairingGeneration = opts?.generation?.key; + const lifecycle = opts?.lifecycle ?? captureNodeWakeLifecycle(nodeId, pairingGeneration); + const isAttemptCurrent = () => + isNodePushAttemptCurrent({ nodeId, lifecycle, generation: opts?.generation }); try { - return await state.inFlight; + if (!(await isAttemptCurrent())) { + return { available: false, throttled: false, path: "invalidated", durationMs: 0 }; + } + const result = await runNodeWakeAttempt({ + nodeId, + pairingGeneration, + force: opts?.force === true, + throttleMs: nodeInvokePolicy.wakeThrottleMs, + attempt: async (markAttempted) => { + const startedAtMs = Date.now(); + let attempted = false; + const withDuration = (attempt: Omit): NodeWakeAttempt => ({ + ...attempt, + durationMs: Math.max(0, Date.now() - startedAtMs), + }); + const markWakeAttempted = () => { + attempted = true; + markAttempted(); + }; + + try { + if (!(await isAttemptCurrent())) { + return withDuration({ available: false, throttled: false, path: "invalidated" }); + } + const registration = await loadApnsRegistration(nodeId); + if (!(await isAttemptCurrent())) { + return withDuration({ available: false, throttled: false, path: "invalidated" }); + } + if (!registration) { + return withDuration({ available: false, throttled: false, path: "no-registration" }); + } + + let wakeResult; + if (registration.transport === "relay") { + const relay = resolveRelayNodePushConfig(opts?.cfg ?? getRuntimeConfig(), registration); + if (!relay.ok) { + return withDuration({ + available: false, + throttled: false, + path: "no-auth", + apnsReason: relay.error, + }); + } + if (!(await isAttemptCurrent())) { + return withDuration({ available: false, throttled: false, path: "invalidated" }); + } + markWakeAttempted(); + wakeResult = await sendApnsBackgroundWake({ + registration, + nodeId, + wakeReason: opts?.wakeReason ?? "node.invoke", + relayConfig: relay.relayConfig, + signal: lifecycle, + isCurrent: isAttemptCurrent, + }); + } else { + const auth = await resolveDirectNodePushConfig(); + if (!auth.ok) { + return withDuration({ + available: false, + throttled: false, + path: "no-auth", + apnsReason: auth.error, + }); + } + if (!(await isAttemptCurrent())) { + return withDuration({ available: false, throttled: false, path: "invalidated" }); + } + markWakeAttempted(); + wakeResult = await sendApnsBackgroundWake({ + registration, + nodeId, + wakeReason: opts?.wakeReason ?? "node.invoke", + auth: auth.auth, + signal: lifecycle, + isCurrent: isAttemptCurrent, + }); + } + if (!(await isAttemptCurrent())) { + return withDuration({ available: false, throttled: false, path: "invalidated" }); + } + await clearStaleApnsRegistrationIfNeeded(registration, nodeId, wakeResult); + if (!wakeResult.ok) { + return withDuration({ + available: true, + throttled: false, + path: "send-error", + apnsStatus: wakeResult.status, + apnsReason: wakeResult.reason, + }); + } + return withDuration({ + available: true, + throttled: false, + path: "sent", + apnsStatus: wakeResult.status, + apnsReason: wakeResult.reason, + }); + } catch (err) { + if (!(await isAttemptCurrent())) { + return withDuration({ available: false, throttled: false, path: "invalidated" }); + } + return withDuration({ + available: attempted, + throttled: false, + path: "send-error", + apnsReason: formatErrorMessage(err), + }); + } + }, + }); + return (await isAttemptCurrent()) + ? result + : { available: false, throttled: false, path: "invalidated", durationMs: 0 }; } finally { - state.inFlight = undefined; + if (!lifecycleProvided) { + releaseNodeWakeLifecycle(nodeId, lifecycle); + } } } export async function maybeSendNodeWakeNudge( nodeId: string, - opts?: { cfg?: OpenClawConfig }, + opts?: { + cfg?: OpenClawConfig; + lifecycle?: NodeWakeLifecycle; + generation?: NodePairingGeneration; + }, ): Promise { const startedAtMs = Date.now(); const withDuration = ( @@ -867,99 +902,162 @@ export async function maybeSendNodeWakeNudge( ...attempt, durationMs: Math.max(0, Date.now() - startedAtMs), }); - - const lastNudgeAtMs = nodeWakeNudgeById.get(nodeId) ?? 0; - if (lastNudgeAtMs > 0 && Date.now() - lastNudgeAtMs < nodeInvokePolicy.wakeNudgeThrottleMs) { - return withDuration({ sent: false, throttled: true, reason: "throttled" }); - } - - const registration = await loadApnsRegistration(nodeId); - if (!registration) { - return withDuration({ sent: false, throttled: false, reason: "no-registration" }); - } + const lifecycleProvided = opts?.lifecycle !== undefined; + const pairingGeneration = opts?.generation?.key; + const lifecycle = opts?.lifecycle ?? captureNodeWakeLifecycle(nodeId, pairingGeneration); + const isAttemptCurrent = () => + isNodePushAttemptCurrent({ nodeId, lifecycle, generation: opts?.generation }); try { - let result; - if (registration.transport === "relay") { - const relay = resolveRelayNodePushConfig(opts?.cfg ?? getRuntimeConfig(), registration); - if (!relay.ok) { - return withDuration({ - sent: false, - throttled: false, - reason: "no-auth", - apnsReason: relay.error, - }); - } - result = await sendApnsAlert({ - registration, - nodeId, - title: "OpenClaw needs a quick reopen", - body: "Tap to reopen OpenClaw and restore the node connection.", - relayConfig: relay.relayConfig, - }); - } else { - const auth = await resolveDirectNodePushConfig(); - if (!auth.ok) { - return withDuration({ - sent: false, - throttled: false, - reason: "no-auth", - apnsReason: auth.error, - }); - } - result = await sendApnsAlert({ - registration, - nodeId, - title: "OpenClaw needs a quick reopen", - body: "Tap to reopen OpenClaw and restore the node connection.", - auth: auth.auth, - }); + if (!(await isAttemptCurrent())) { + return withDuration({ sent: false, throttled: false, reason: "invalidated" }); } - await clearStaleApnsRegistrationIfNeeded(registration, nodeId, result); - if (!result.ok) { - return withDuration({ - sent: false, - throttled: false, - reason: "apns-not-ok", - apnsStatus: result.status, - apnsReason: result.reason, - }); + return await runNodeWakeNudgeAttempt({ + nodeId, + pairingGeneration, + throttleMs: nodeInvokePolicy.wakeNudgeThrottleMs, + throttled: () => withDuration({ sent: false, throttled: true, reason: "throttled" }), + attempt: async () => { + const registration = await loadApnsRegistration(nodeId); + if (!(await isAttemptCurrent())) { + return withDuration({ sent: false, throttled: false, reason: "invalidated" }); + } + if (!registration) { + return withDuration({ sent: false, throttled: false, reason: "no-registration" }); + } + try { + let result; + if (registration.transport === "relay") { + const relay = resolveRelayNodePushConfig(opts?.cfg ?? getRuntimeConfig(), registration); + if (!relay.ok) { + return withDuration({ + sent: false, + throttled: false, + reason: "no-auth", + apnsReason: relay.error, + }); + } + if (!(await isAttemptCurrent())) { + return withDuration({ sent: false, throttled: false, reason: "invalidated" }); + } + result = await sendApnsAlert({ + registration, + nodeId, + title: "OpenClaw needs a quick reopen", + body: "Tap to reopen OpenClaw and restore the node connection.", + relayConfig: relay.relayConfig, + signal: lifecycle, + isCurrent: isAttemptCurrent, + }); + } else { + const auth = await resolveDirectNodePushConfig(); + if (!auth.ok) { + return withDuration({ + sent: false, + throttled: false, + reason: "no-auth", + apnsReason: auth.error, + }); + } + if (!(await isAttemptCurrent())) { + return withDuration({ sent: false, throttled: false, reason: "invalidated" }); + } + result = await sendApnsAlert({ + registration, + nodeId, + title: "OpenClaw needs a quick reopen", + body: "Tap to reopen OpenClaw and restore the node connection.", + auth: auth.auth, + signal: lifecycle, + isCurrent: isAttemptCurrent, + }); + } + if (!(await isAttemptCurrent())) { + return withDuration({ sent: result.ok, throttled: false, reason: "invalidated" }); + } + await clearStaleApnsRegistrationIfNeeded(registration, nodeId, result); + if (!(await isAttemptCurrent())) { + return withDuration({ sent: result.ok, throttled: false, reason: "invalidated" }); + } + return result.ok + ? withDuration({ + sent: true, + throttled: false, + reason: "sent", + apnsStatus: result.status, + apnsReason: result.reason, + }) + : withDuration({ + sent: false, + throttled: false, + reason: "apns-not-ok", + apnsStatus: result.status, + apnsReason: result.reason, + }); + } catch (err) { + if (!(await isAttemptCurrent())) { + return withDuration({ sent: false, throttled: false, reason: "invalidated" }); + } + return withDuration({ + sent: false, + throttled: false, + reason: "send-error", + apnsReason: formatErrorMessage(err), + }); + } + }, + }); + } finally { + if (!lifecycleProvided) { + releaseNodeWakeLifecycle(nodeId, lifecycle); } - nodeWakeNudgeById.set(nodeId, Date.now()); - return withDuration({ - sent: true, - throttled: false, - reason: "sent", - apnsStatus: result.status, - apnsReason: result.reason, - }); - } catch (err) { - const message = formatErrorMessage(err); - return withDuration({ - sent: false, - throttled: false, - reason: "send-error", - apnsReason: message, - }); } } export async function waitForNodeReconnect(params: { nodeId: string; - context: { nodeRegistry: { get: (nodeId: string) => unknown } }; + context: { + nodeRegistry: { + get: (nodeId: string) => NodeSession | undefined; + getForPairingGeneration: ( + nodeId: string, + pairingGeneration: string, + ) => NodeSession | undefined; + }; + }; timeoutMs?: number; pollMs?: number; + lifecycle?: NodeWakeLifecycle; + pairingGeneration?: string; }): Promise { const timeoutMs = resolveTimerTimeoutMs(params.timeoutMs, NODE_WAKE_RECONNECT_WAIT_MS, 250); const pollMs = resolveTimerTimeoutMs(params.pollMs, NODE_WAKE_RECONNECT_POLL_MS, 50); const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { - if (params.context.nodeRegistry.get(params.nodeId)) { + if ( + params.lifecycle && + !isNodeWakeLifecycleCurrent(params.nodeId, params.lifecycle, params.pairingGeneration) + ) { + return false; + } + const session = params.pairingGeneration + ? params.context.nodeRegistry.getForPairingGeneration(params.nodeId, params.pairingGeneration) + : params.context.nodeRegistry.get(params.nodeId); + if (resolveDispatchableNodeSession(session)) { return true; } await delayMs(pollMs); } - return Boolean(params.context.nodeRegistry.get(params.nodeId)); + if ( + params.lifecycle && + !isNodeWakeLifecycleCurrent(params.nodeId, params.lifecycle, params.pairingGeneration) + ) { + return false; + } + const session = params.pairingGeneration + ? params.context.nodeRegistry.getForPairingGeneration(params.nodeId, params.pairingGeneration) + : params.context.nodeRegistry.get(params.nodeId); + return Boolean(resolveDispatchableNodeSession(session)); } export const nodeHandlers: GatewayRequestHandlers = { @@ -1011,6 +1109,13 @@ export const nodeHandlers: GatewayRequestHandlers = { ) { return; } + const pendingApproval = await getPendingNodePairing(requestId); + const pairingStateBeforeApproval = pendingApproval + ? await captureNodePairingState(pendingApproval.nodeId) + : null; + const sessionBeforeApproval = pendingApproval + ? context.nodeRegistry.get(pendingApproval.nodeId) + : undefined; const approved = await approveNodePairing(requestId, { callerScopes }); if (!approved) { respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, "unknown requestId")); @@ -1035,6 +1140,9 @@ export const nodeHandlers: GatewayRequestHandlers = { return; } const approvedNode = approved.node; + // Surface approval rotates the persistent generation. Abort any wake + // already admitted under the prior command surface before it can send. + invalidateNodeWakeState(approvedNode.nodeId); const cfg = context.getRuntimeConfig(); // Pairing allowlist, matching connect-time reconciliation: approved // dangerous surfaces (e.g. computer.act) stay on the live session so a @@ -1050,11 +1158,39 @@ export const nodeHandlers: GatewayRequestHandlers = { declaredCommands: approvedNode.commands ?? [], allowlist: currentAllowlist, }); - const updatedNode = context.nodeRegistry.updateSurface(approvedNode.nodeId, { - caps: approvedNode.caps ?? [], - commands: currentAllowedCommands, - permissions: approvedNode.permissions, - }); + // Only the exact generation committed by this approval may inherit the + // authenticated live session. A later re-pair must reconnect instead. + const persistedApprovedState = await captureNodePairingState(approvedNode.nodeId); + const previousGenerationKey = pairingStateBeforeApproval?.generation?.key; + const liveSessionOwnsPreviousPairingState = Boolean( + sessionBeforeApproval && + pairingStateBeforeApproval?.identity.key === approved.pairingIdentity && + sessionBeforeApproval.pairingIdentity === approved.pairingIdentity && + approved.previousPairingGeneration === previousGenerationKey && + sessionBeforeApproval.pairingGeneration === previousGenerationKey, + ); + const updatedNode = + liveSessionOwnsPreviousPairingState && + sessionBeforeApproval && + persistedApprovedState?.identity.key === approved.pairingIdentity && + persistedApprovedState.generation?.key === approved.nextPairingGeneration + ? context.nodeRegistry.updateSurface( + approvedNode.nodeId, + { + caps: approvedNode.caps ?? [], + commands: currentAllowedCommands, + permissions: approvedNode.permissions, + }, + { + expectedConnId: sessionBeforeApproval.connId, + expectedPairingIdentity: approved.pairingIdentity, + ...(previousGenerationKey + ? { expectedPairingGeneration: previousGenerationKey } + : {}), + nextPairingGeneration: approved.nextPairingGeneration, + }, + ) + : null; if (updatedNode) { refreshConnectedNodeSurfaceCaches({ context, nodeSession: updatedNode, cfg }); } @@ -1068,7 +1204,7 @@ export const nodeHandlers: GatewayRequestHandlers = { }, { dropIfSlow: true }, ); - respond(true, approved, undefined); + respond(true, { requestId: approved.requestId, node: approvedNode }, undefined); }); }, "node.pair.reject": async ({ params, respond, context, client }) => { @@ -1209,14 +1345,15 @@ export const nodeHandlers: GatewayRequestHandlers = { listDevicePairing(), listNodePairing(), ]); + const connectedNodes = listCurrentConnectedNodes(context, devicePairing.paired); const nodes = listNodesForClient({ client, pairedDevices: devicePairing.paired, pairedNodes: nodePairing.paired, pendingNodes: nodePairing.pending, - connectedNodes: context.nodeRegistry.listConnected(), + connectedNodes, }); - const activeNodeId = context.nodeRegistry.getActiveNode()?.nodeId; + const activeNodeId = context.nodeRegistry.getActiveNode(connectedNodes)?.nodeId; const nodesWithPresence = activeNodeId ? nodes.map((node) => (node.nodeId === activeNodeId ? { ...node, active: true } : node)) : nodes; @@ -1243,11 +1380,12 @@ export const nodeHandlers: GatewayRequestHandlers = { listDevicePairing(), listNodePairing(), ]); + const connectedNodes = listCurrentConnectedNodes(context, devicePairing.paired); const catalog = createKnownNodeCatalog({ pairedDevices: devicePairing.paired, pairedNodes: nodePairing.paired, pendingNodes: nodePairing.pending, - connectedNodes: context.nodeRegistry.listConnected(), + connectedNodes, }); const catalogNode = getKnownNode(catalog, id); const node = @@ -1265,7 +1403,9 @@ export const nodeHandlers: GatewayRequestHandlers = { { ts: Date.now(), ...node, - ...(context.nodeRegistry.getActiveNode()?.nodeId === id ? { active: true } : {}), + ...(context.nodeRegistry.getActiveNode(connectedNodes)?.nodeId === id + ? { active: true } + : {}), }, undefined, ); @@ -1344,26 +1484,43 @@ export const nodeHandlers: GatewayRequestHandlers = { return; } - const pending = resolveAllowedPendingNodeActions({ - nodeId: trimmedNodeId, - client, - cfg: context.getRuntimeConfig(), - }); - respond( - true, - { + await respondUnavailableOnThrow(respond, async () => { + const generation = await captureNodePairingGeneration(trimmedNodeId); + if (!generation) { + respondPairingChanged(respond); + return; + } + const session = context.nodeRegistry.getForPairingGeneration(trimmedNodeId, generation.key); + if (!session || session.connId !== client?.connId) { + respondPairingChanged(respond); + return; + } + const pending = resolveAllowedPendingNodeActions({ nodeId: trimmedNodeId, - actions: pending.map((entry) => ({ - id: entry.id, - command: entry.command, - paramsJSON: entry.paramsJSON ?? null, - enqueuedAtMs: entry.enqueuedAtMs, - })), - }, - undefined, - ); + pairingGeneration: generation.key, + client, + cfg: context.getRuntimeConfig(), + }); + if (!(await isNodePairingGenerationCurrent(generation))) { + respondPairingChanged(respond); + return; + } + respond( + true, + { + nodeId: trimmedNodeId, + actions: pending.map((entry) => ({ + id: entry.id, + command: entry.command, + paramsJSON: entry.paramsJSON ?? null, + enqueuedAtMs: entry.enqueuedAtMs, + })), + }, + undefined, + ); + }); }, - "node.pending.ack": async ({ params, respond, client }) => { + "node.pending.ack": async ({ params, respond, client, context }) => { if (!validateNodePendingAckParams(params)) { respondInvalidParams({ respond, @@ -1378,17 +1535,33 @@ export const nodeHandlers: GatewayRequestHandlers = { respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, "nodeId required")); return; } - const ackIds = normalizeUniqueTrimmedStringList(params.ids); - const remaining = ackPendingNodeActions(trimmedNodeId, ackIds); - respond( - true, - { - nodeId: trimmedNodeId, - ackedIds: ackIds, - remainingCount: remaining.length, - }, - undefined, - ); + await respondUnavailableOnThrow(respond, async () => { + const generation = await captureNodePairingGeneration(trimmedNodeId); + if (!generation) { + respondPairingChanged(respond); + return; + } + const session = context.nodeRegistry.getForPairingGeneration(trimmedNodeId, generation.key); + if (!session || session.connId !== client?.connId) { + respondPairingChanged(respond); + return; + } + const ackIds = normalizeUniqueTrimmedStringList(params.ids); + const remaining = ackPendingNodeActions(trimmedNodeId, ackIds, generation.key); + if (!(await isNodePairingGenerationCurrent(generation))) { + respondPairingChanged(respond); + return; + } + respond( + true, + { + nodeId: trimmedNodeId, + ackedIds: ackIds, + remainingCount: remaining.length, + }, + undefined, + ); + }); }, "node.invoke": async ({ params, respond, context, client, req }) => { if (!validateNodeInvokeParams(params)) { @@ -1461,172 +1634,370 @@ export const nodeHandlers: GatewayRequestHandlers = { return; } await respondUnavailableOnThrow(respond, async () => { - const cfg = context.getRuntimeConfig(); - let nodeSession = context.nodeRegistry.get(nodeId); - if (!nodeSession) { - const wakeReqId = req.id; - const wakeFlowStartedAtMs = Date.now(); - context.logGateway.info( - `node wake start node=${nodeId} req=${wakeReqId} command=${command}`, - ); + const generation = await captureNodePairingGeneration(nodeId); + if (!generation) { + respondPairingChanged(respond); + return; + } + const wakeLifecycle = captureNodeWakeLifecycle(nodeId, generation.key); + try { + const continuePairingWork = async (): Promise => { + if (await isNodePairingWorkCurrent({ nodeId, generation, lifecycle: wakeLifecycle })) { + return true; + } + respondPairingChanged(respond); + return false; + }; - const wake = await maybeWakeNodeWithApns(nodeId, { cfg }); - context.logGateway.info( - `node wake stage=wake1 node=${nodeId} req=${wakeReqId} ` + - `available=${wake.available} throttled=${wake.throttled} ` + - `path=${wake.path} durationMs=${wake.durationMs} ` + - `apnsStatus=${wake.apnsStatus ?? -1} apnsReason=${wake.apnsReason ?? "-"}`, + const cfg = context.getRuntimeConfig(); + let nodeSession = resolveDispatchableNodeSession( + context.nodeRegistry.getForPairingGeneration(nodeId, generation.key), ); - if (wake.available) { - const waitStartedAtMs = Date.now(); - const waitTimeoutMs = NODE_WAKE_RECONNECT_WAIT_MS; - const reconnected = await waitForNodeReconnect({ - nodeId, - context, - timeoutMs: waitTimeoutMs, + if (!nodeSession) { + const wakeReqId = req.id; + const wakeFlowStartedAtMs = Date.now(); + context.logGateway.info( + `node wake start node=${nodeId} req=${wakeReqId} command=${command}`, + ); + + const wake = await maybeWakeNodeWithApns(nodeId, { + cfg, + lifecycle: wakeLifecycle, + generation, }); - const waitDurationMs = Math.max(0, Date.now() - waitStartedAtMs); context.logGateway.info( - `node wake stage=wait1 node=${nodeId} req=${wakeReqId} ` + - `reconnected=${reconnected} timeoutMs=${waitTimeoutMs} durationMs=${waitDurationMs}`, + `node wake stage=wake1 node=${nodeId} req=${wakeReqId} ` + + `available=${wake.available} throttled=${wake.throttled} ` + + `path=${wake.path} durationMs=${wake.durationMs} ` + + `apnsStatus=${wake.apnsStatus ?? -1} apnsReason=${wake.apnsReason ?? "-"}`, ); - } - nodeSession = context.nodeRegistry.get(nodeId); - if (!nodeSession && wake.available) { - const retryWake = await maybeWakeNodeWithApns(nodeId, { force: true, cfg }); - context.logGateway.info( - `node wake stage=wake2 node=${nodeId} req=${wakeReqId} force=true ` + - `available=${retryWake.available} throttled=${retryWake.throttled} ` + - `path=${retryWake.path} durationMs=${retryWake.durationMs} ` + - `apnsStatus=${retryWake.apnsStatus ?? -1} apnsReason=${retryWake.apnsReason ?? "-"}`, - ); - if (retryWake.available) { + if (wake.available) { const waitStartedAtMs = Date.now(); - const waitTimeoutMs = NODE_WAKE_RECONNECT_RETRY_WAIT_MS; + const waitTimeoutMs = NODE_WAKE_RECONNECT_WAIT_MS; const reconnected = await waitForNodeReconnect({ nodeId, context, timeoutMs: waitTimeoutMs, + lifecycle: wakeLifecycle, + pairingGeneration: generation.key, }); const waitDurationMs = Math.max(0, Date.now() - waitStartedAtMs); context.logGateway.info( - `node wake stage=wait2 node=${nodeId} req=${wakeReqId} ` + + `node wake stage=wait1 node=${nodeId} req=${wakeReqId} ` + `reconnected=${reconnected} timeoutMs=${waitTimeoutMs} durationMs=${waitDurationMs}`, ); } - nodeSession = context.nodeRegistry.get(nodeId); - } - if (!nodeSession) { + if (!(await continuePairingWork())) { + return; + } + nodeSession = resolveDispatchableNodeSession( + context.nodeRegistry.getForPairingGeneration(nodeId, generation.key), + ); + if (!nodeSession && wake.available) { + const retryWake = await maybeWakeNodeWithApns(nodeId, { + force: true, + cfg, + lifecycle: wakeLifecycle, + generation, + }); + context.logGateway.info( + `node wake stage=wake2 node=${nodeId} req=${wakeReqId} force=true ` + + `available=${retryWake.available} throttled=${retryWake.throttled} ` + + `path=${retryWake.path} durationMs=${retryWake.durationMs} ` + + `apnsStatus=${retryWake.apnsStatus ?? -1} apnsReason=${retryWake.apnsReason ?? "-"}`, + ); + if (retryWake.available) { + const waitStartedAtMs = Date.now(); + const waitTimeoutMs = NODE_WAKE_RECONNECT_RETRY_WAIT_MS; + const reconnected = await waitForNodeReconnect({ + nodeId, + context, + timeoutMs: waitTimeoutMs, + lifecycle: wakeLifecycle, + pairingGeneration: generation.key, + }); + const waitDurationMs = Math.max(0, Date.now() - waitStartedAtMs); + context.logGateway.info( + `node wake stage=wait2 node=${nodeId} req=${wakeReqId} ` + + `reconnected=${reconnected} timeoutMs=${waitTimeoutMs} durationMs=${waitDurationMs}`, + ); + } + if (!(await continuePairingWork())) { + return; + } + nodeSession = resolveDispatchableNodeSession( + context.nodeRegistry.getForPairingGeneration(nodeId, generation.key), + ); + } + if (!nodeSession) { + const totalDurationMs = Math.max(0, Date.now() - wakeFlowStartedAtMs); + const nudge = await maybeSendNodeWakeNudge(nodeId, { + cfg, + lifecycle: wakeLifecycle, + generation, + }); + if (!(await continuePairingWork())) { + return; + } + context.logGateway.info( + `node wake nudge node=${nodeId} req=${wakeReqId} sent=${nudge.sent} ` + + `throttled=${nudge.throttled} reason=${nudge.reason} durationMs=${nudge.durationMs} ` + + `apnsStatus=${nudge.apnsStatus ?? -1} apnsReason=${nudge.apnsReason ?? "-"}`, + ); + context.logGateway.warn( + `node wake done node=${nodeId} req=${wakeReqId} connected=false ` + + `reason=not_connected totalMs=${totalDurationMs}`, + ); + respond( + false, + undefined, + errorShape(ErrorCodes.UNAVAILABLE, "node not connected", { + details: { code: "NOT_CONNECTED" }, + }), + ); + return; + } + const totalDurationMs = Math.max(0, Date.now() - wakeFlowStartedAtMs); - const nudge = await maybeSendNodeWakeNudge(nodeId, { cfg }); context.logGateway.info( - `node wake nudge node=${nodeId} req=${wakeReqId} sent=${nudge.sent} ` + - `throttled=${nudge.throttled} reason=${nudge.reason} durationMs=${nudge.durationMs} ` + - `apnsStatus=${nudge.apnsStatus ?? -1} apnsReason=${nudge.apnsReason ?? "-"}`, + `node wake done node=${nodeId} req=${wakeReqId} connected=true totalMs=${totalDurationMs}`, ); - context.logGateway.warn( - `node wake done node=${nodeId} req=${wakeReqId} connected=false ` + - `reason=not_connected totalMs=${totalDurationMs}`, - ); - respond( - false, - undefined, - errorShape(ErrorCodes.UNAVAILABLE, "node not connected", { - details: { code: "NOT_CONNECTED" }, - }), - ); - return; } - - const totalDurationMs = Math.max(0, Date.now() - wakeFlowStartedAtMs); - context.logGateway.info( - `node wake done node=${nodeId} req=${wakeReqId} connected=true totalMs=${totalDurationMs}`, - ); - } - // A reload may revoke authority for an in-flight request, but it must not - // retroactively grant one that was denied when admitted before node wake. - for (const authorizationCfg of [cfg, context.getRuntimeConfig()]) { - const allowlist = resolveNodeCommandAllowlist(authorizationCfg, { - ...nodeSession, - approvedCommands: nodeSession.commands, - }); - const allowed = isNodeCommandAllowed({ - command, - declaredCommands: nodeSession.commands, - allowlist, - }); - if (!allowed.ok) { - const hint = buildNodeCommandRejectionHint( - allowed.reason, + // A reload may revoke authority for an in-flight request, but it must not + // retroactively grant one that was denied when admitted before node wake. + for (const authorizationCfg of [cfg, context.getRuntimeConfig()]) { + const allowlist = resolveNodeCommandAllowlist(authorizationCfg, { + ...nodeSession, + approvedCommands: nodeSession.commands, + }); + const allowed = isNodeCommandAllowed({ command, - nodeSession, - authorizationCfg, - ); - respond( - false, - undefined, - errorShape(ErrorCodes.INVALID_REQUEST, hint, { - details: { reason: allowed.reason, command }, - }), - ); - return; + declaredCommands: nodeSession.commands, + allowlist, + }); + if (!allowed.ok) { + const hint = buildNodeCommandRejectionHint( + allowed.reason, + command, + nodeSession, + authorizationCfg, + ); + respond( + false, + undefined, + errorShape(ErrorCodes.INVALID_REQUEST, hint, { + details: { reason: allowed.reason, command }, + }), + ); + return; + } } - } - const forwardedParams = sanitizeNodeInvokeParamsForForwarding({ - nodeId, - command, - rawParams: p.params, - client, - execApprovalManager: context.execApprovalManager, - }); - if (!forwardedParams.ok) { - respond( - false, - undefined, - errorShape(ErrorCodes.INVALID_REQUEST, forwardedParams.message, { - details: forwardedParams.details ?? null, - }), - ); - return; - } - const policyResult = await applyPluginNodeInvokePolicy({ - context, - client, - nodeSession, - command, - params: forwardedParams.params, - turnSource: { - channel: p.turnSourceChannel, - to: p.turnSourceTo, - accountId: p.turnSourceAccountId, - threadId: p.turnSourceThreadId, - }, - timeoutMs: p.timeoutMs, - idempotencyKey: p.idempotencyKey, - }); - if (policyResult) { - // Plugin policies can satisfy an invocation without crossing the raw - // node command channel; still emit mirrored Talk events for UI state. - if (!policyResult.ok) { - const errorCode = policyResult.unavailable - ? ErrorCodes.UNAVAILABLE - : ErrorCodes.INVALID_REQUEST; + const forwardedParams = sanitizeNodeInvokeParamsForForwarding({ + nodeId, + command, + rawParams: p.params, + client, + execApprovalManager: context.execApprovalManager, + }); + if (!forwardedParams.ok) { respond( false, undefined, - errorShape(errorCode, policyResult.message, { - details: { - ...policyResult.details, - ...(policyResult.code ? { code: policyResult.code } : {}), - }, + errorShape(ErrorCodes.INVALID_REQUEST, forwardedParams.message, { + details: forwardedParams.details ?? null, }), ); return; } - const payload = policyResult.payloadJSON - ? safeParseJson(policyResult.payloadJSON) - : policyResult.payload; + const policyResult = await applyPluginNodeInvokePolicy({ + context, + client, + nodeSession, + command, + params: forwardedParams.params, + turnSource: { + channel: p.turnSourceChannel, + to: p.turnSourceTo, + accountId: p.turnSourceAccountId, + threadId: p.turnSourceThreadId, + }, + timeoutMs: p.timeoutMs, + idempotencyKey: p.idempotencyKey, + isInvocationCurrent: () => + isNodePairingWorkCurrent({ nodeId, generation, lifecycle: wakeLifecycle }), + }); + if (!(await continuePairingWork())) { + return; + } + if (policyResult) { + // Plugin policies can satisfy an invocation without crossing the raw + // node command channel; still emit mirrored Talk events for UI state. + if (!policyResult.ok) { + const errorCode = policyResult.unavailable + ? ErrorCodes.UNAVAILABLE + : ErrorCodes.INVALID_REQUEST; + respond( + false, + undefined, + errorShape(errorCode, policyResult.message, { + details: { + ...policyResult.details, + ...(policyResult.code ? { code: policyResult.code } : {}), + }, + }), + ); + return; + } + const payload = policyResult.payloadJSON + ? safeParseJson(policyResult.payloadJSON) + : policyResult.payload; + emitTalkPttNodeEvent({ + context, + nodeId, + command, + payload, + }); + respond( + true, + { + ok: true, + nodeId, + command, + payload: policyResult.payload, + payloadJSON: policyResult.payloadJSON ?? null, + }, + undefined, + ); + return; + } + const dispatchSession = resolveDispatchableNodeSession( + context.nodeRegistry.getForPairingGeneration(nodeId, generation.key), + ); + if (!dispatchSession || dispatchSession.connId !== nodeSession.connId) { + respond( + false, + undefined, + errorShape(ErrorCodes.UNAVAILABLE, "node connection changed before dispatch", { + retryable: true, + details: { code: "ROUTE_CHANGED" }, + }), + ); + return; + } + const dispatchCfg = context.getRuntimeConfig(); + const dispatchAllowlist = resolveNodeCommandAllowlist(dispatchCfg, { + ...dispatchSession, + approvedCommands: dispatchSession.commands, + }); + const dispatchAllowed = isNodeCommandAllowed({ + command, + declaredCommands: dispatchSession.commands, + allowlist: dispatchAllowlist, + }); + if (!dispatchAllowed.ok) { + respond( + false, + undefined, + errorShape( + ErrorCodes.INVALID_REQUEST, + buildNodeCommandRejectionHint( + dispatchAllowed.reason, + command, + dispatchSession, + dispatchCfg, + ), + { details: { reason: dispatchAllowed.reason, command } }, + ), + ); + return; + } + const res = await context.nodeRegistry.invoke({ + nodeId, + expectedConnId: nodeSession.connId, + expectedPairingGeneration: generation.key, + command, + params: forwardedParams.params, + timeoutMs: p.timeoutMs, + idempotencyKey: p.idempotencyKey, + ...(sessionKey ? { sessionKey } : {}), + }); + if (!(await continuePairingWork())) { + return; + } + if (!res.ok) { + if ( + shouldQueueAsPendingForegroundAction({ + platform: nodeSession.platform, + command, + error: res.error, + }) + ) { + // Foreground-only iOS commands become pullable pending actions instead + // of failing permanently while the device is locked/backgrounded. + const paramsJSON = toPendingParamsJSON(forwardedParams.params); + const queued = enqueuePendingNodeAction({ + nodeId, + pairingGeneration: generation.key, + command, + paramsJSON, + idempotencyKey: p.idempotencyKey, + ttlMs: nodeInvokePolicy.pendingActionTtlMs, + maxPerNode: nodeInvokePolicy.pendingActionMaxPerNode, + }); + const wake = await maybeWakeNodeWithApns(nodeId, { + cfg, + lifecycle: wakeLifecycle, + generation, + }); + if (!(await continuePairingWork())) { + if (queued.created) { + removePendingNodeAction({ + nodeId, + pairingGeneration: generation.key, + actionId: queued.action.id, + ttlMs: nodeInvokePolicy.pendingActionTtlMs, + }); + } + return; + } + context.logGateway.info( + `node pending queued node=${nodeId} req=${req.id} command=${command} ` + + `queuedId=${queued.action.id} wakePath=${wake.path} wakeAvailable=${wake.available}`, + ); + respond( + false, + undefined, + errorShape( + ErrorCodes.UNAVAILABLE, + "node command queued until iOS returns to foreground", + { + retryable: true, + details: { + code: "QUEUED_UNTIL_FOREGROUND", + queuedActionId: queued.action.id, + nodeId, + command, + wake: { + path: wake.path, + available: wake.available, + throttled: wake.throttled, + apnsStatus: wake.apnsStatus, + apnsReason: wake.apnsReason, + }, + nodeError: res.error ?? null, + }, + }, + ), + ); + return; + } + if (!respondUnavailableOnNodeInvokeError(respond, res)) { + return; + } + return; + } + const payload = res.payloadJSON ? safeParseJson(res.payloadJSON) : res.payload; emitTalkPttNodeEvent({ context, nodeId, @@ -1639,133 +2010,14 @@ export const nodeHandlers: GatewayRequestHandlers = { ok: true, nodeId, command, - payload: policyResult.payload, - payloadJSON: policyResult.payloadJSON ?? null, + payload, + payloadJSON: res.payloadJSON ?? null, }, undefined, ); - return; + } finally { + releaseNodeWakeLifecycle(nodeId, wakeLifecycle); } - const dispatchSession = context.nodeRegistry.get(nodeId); - if (!dispatchSession || dispatchSession.connId !== nodeSession.connId) { - respond( - false, - undefined, - errorShape(ErrorCodes.UNAVAILABLE, "node connection changed before dispatch", { - retryable: true, - details: { code: "ROUTE_CHANGED" }, - }), - ); - return; - } - const dispatchCfg = context.getRuntimeConfig(); - const dispatchAllowlist = resolveNodeCommandAllowlist(dispatchCfg, { - ...dispatchSession, - approvedCommands: dispatchSession.commands, - }); - const dispatchAllowed = isNodeCommandAllowed({ - command, - declaredCommands: dispatchSession.commands, - allowlist: dispatchAllowlist, - }); - if (!dispatchAllowed.ok) { - respond( - false, - undefined, - errorShape( - ErrorCodes.INVALID_REQUEST, - buildNodeCommandRejectionHint( - dispatchAllowed.reason, - command, - dispatchSession, - dispatchCfg, - ), - { details: { reason: dispatchAllowed.reason, command } }, - ), - ); - return; - } - const res = await context.nodeRegistry.invoke({ - nodeId, - expectedConnId: nodeSession.connId, - command, - params: forwardedParams.params, - timeoutMs: p.timeoutMs, - idempotencyKey: p.idempotencyKey, - ...(sessionKey ? { sessionKey } : {}), - }); - if (!res.ok) { - if ( - shouldQueueAsPendingForegroundAction({ - platform: nodeSession.platform, - command, - error: res.error, - }) - ) { - // Foreground-only iOS commands become pullable pending actions instead - // of failing permanently while the device is locked/backgrounded. - const paramsJSON = toPendingParamsJSON(forwardedParams.params); - const queued = enqueuePendingNodeAction({ - nodeId, - command, - paramsJSON, - idempotencyKey: p.idempotencyKey, - }); - const wake = await maybeWakeNodeWithApns(nodeId, { cfg }); - context.logGateway.info( - `node pending queued node=${nodeId} req=${req.id} command=${command} ` + - `queuedId=${queued.id} wakePath=${wake.path} wakeAvailable=${wake.available}`, - ); - respond( - false, - undefined, - errorShape( - ErrorCodes.UNAVAILABLE, - "node command queued until iOS returns to foreground", - { - retryable: true, - details: { - code: "QUEUED_UNTIL_FOREGROUND", - queuedActionId: queued.id, - nodeId, - command, - wake: { - path: wake.path, - available: wake.available, - throttled: wake.throttled, - apnsStatus: wake.apnsStatus, - apnsReason: wake.apnsReason, - }, - nodeError: res.error ?? null, - }, - }, - ), - ); - return; - } - if (!respondUnavailableOnNodeInvokeError(respond, res)) { - return; - } - return; - } - const payload = res.payloadJSON ? safeParseJson(res.payloadJSON) : res.payload; - emitTalkPttNodeEvent({ - context, - nodeId, - command, - payload, - }); - respond( - true, - { - ok: true, - nodeId, - command, - payload, - payloadJSON: res.payloadJSON ?? null, - }, - undefined, - ); }); }, "node.invoke.progress": handleNodeInvokeProgress, @@ -1787,9 +2039,31 @@ export const nodeHandlers: GatewayRequestHandlers = { ? JSON.stringify(p.payload) : null; await respondUnavailableOnThrow(respond, async () => { - const { handleNodeEvent } = await import("../server-node-events.js"); const nodeId = client?.connect?.device?.id ?? client?.connect?.client?.id ?? "node"; const nodeSession = context.nodeRegistry.get(nodeId); + const eventConnId = client?.connId; + const eventPairingGeneration = nodeSession?.pairingGeneration; + const isEventConnectionCurrent = async (): Promise => { + if (!eventConnId || !eventPairingGeneration) { + return false; + } + const before = resolveDispatchableNodeSession( + context.nodeRegistry.getForPairingGeneration(nodeId, eventPairingGeneration), + ); + if (!before || before.connId !== eventConnId) { + return false; + } + if (!(await context.nodeRegistry.isConnectionCurrentPairingState(eventConnId))) { + return false; + } + const after = resolveDispatchableNodeSession( + context.nodeRegistry.getForPairingGeneration(nodeId, eventPairingGeneration), + ); + return after?.connId === eventConnId; + }; + const { handleNodeEvent } = await import("../server-node-events.js"); + const apnsGeneration = + p.event === "push.apns.register" ? await captureNodePairingGeneration(nodeId) : null; const presenceAllowed = nodeSession !== undefined && nodeSession.connId === client?.connId && @@ -1798,8 +2072,28 @@ export const nodeHandlers: GatewayRequestHandlers = { deps: context.deps, broadcast: context.broadcast, nodeSendToSession: context.nodeSendToSession, - nodeSubscribe: context.nodeSubscribe, - nodeUnsubscribe: context.nodeUnsubscribe, + nodeSubscribe: async (subscriptionNodeId, sessionKey, subscriptionConnId) => { + if ( + subscriptionNodeId !== nodeId || + !subscriptionConnId || + subscriptionConnId !== client?.connId || + !(await isEventConnectionCurrent()) + ) { + return; + } + context.nodeSubscribe(subscriptionNodeId, sessionKey, subscriptionConnId); + }, + nodeUnsubscribe: async (subscriptionNodeId, sessionKey, subscriptionConnId) => { + if ( + subscriptionNodeId !== nodeId || + !subscriptionConnId || + subscriptionConnId !== client?.connId || + !(await isEventConnectionCurrent()) + ) { + return; + } + context.nodeUnsubscribe(subscriptionNodeId, sessionKey, subscriptionConnId); + }, broadcastVoiceWakeChanged: context.broadcastVoiceWakeChanged, addChatRun: context.addChatRun, removeChatRun: context.removeChatRun, @@ -1841,9 +2135,35 @@ export const nodeHandlers: GatewayRequestHandlers = { { connId: client?.connId, deviceId: client?.connect?.device?.id, + pairingGeneration: eventPairingGeneration + ? { nodeId, key: eventPairingGeneration } + : undefined, presenceAllowed, + isConnectionCurrent: isEventConnectionCurrent, + resolveApnsRegistrationGeneration: async () => { + if (!apnsGeneration || !client?.connId) { + return null; + } + const before = resolveDispatchableNodeSession( + context.nodeRegistry.getForPairingGeneration(nodeId, apnsGeneration.key), + ); + if (!before || before.connId !== client.connId) { + return null; + } + if (!(await isNodePairingGenerationCurrent(apnsGeneration))) { + return null; + } + const after = resolveDispatchableNodeSession( + context.nodeRegistry.getForPairingGeneration(nodeId, apnsGeneration.key), + ); + return after?.connId === client.connId ? apnsGeneration.key : null; + }, }, ); + if (result?.reason === "pairing_changed") { + respondPairingChanged(respond); + return; + } respond(true, result ?? { ok: true }, undefined); }); }, diff --git a/src/gateway/server-methods/nodes.wake-leak.test.ts b/src/gateway/server-methods/nodes.wake-leak.test.ts index 543b31f1de36..9c8ed94d1267 100644 --- a/src/gateway/server-methods/nodes.wake-leak.test.ts +++ b/src/gateway/server-methods/nodes.wake-leak.test.ts @@ -12,18 +12,21 @@ const mocks = vi.hoisted(() => ({ vi.mock("../../infra/push-apns.js", () => mocks); -import { nodeWakeById } from "./nodes-wake-state.js"; +import { + getNodeWakeStateSnapshot, + resetNodeWakeStateForTest, +} from "../node-wake-state.test-support.js"; import { maybeWakeNodeWithApns } from "./nodes.js"; describe("maybeWakeNodeWithApns no-registration cleanup", () => { beforeEach(() => { - nodeWakeById.clear(); + resetNodeWakeStateForTest(); vi.clearAllMocks(); mocks.loadApnsRegistration.mockResolvedValue(null); }); afterEach(() => { - nodeWakeById.clear(); + resetNodeWakeStateForTest(); }); it("does not retain state for unregistered node ids", async () => { @@ -35,11 +38,13 @@ describe("maybeWakeNodeWithApns no-registration cleanup", () => { }); } - expect(nodeWakeById.size).toBe(0); + for (let index = 0; index < 50; index += 1) { + expect(getNodeWakeStateSnapshot(`unregistered-node-${index}`)).toBeUndefined(); + } }); it("cleans up after a single no-registration result", async () => { await maybeWakeNodeWithApns("stale-node-id"); - expect(nodeWakeById.has("stale-node-id")).toBe(false); + expect(getNodeWakeStateSnapshot("stale-node-id")).toBeUndefined(); }); }); diff --git a/src/gateway/server-methods/shared-types.ts b/src/gateway/server-methods/shared-types.ts index 1f50c5a18d77..9629c0fc0745 100644 --- a/src/gateway/server-methods/shared-types.ts +++ b/src/gateway/server-methods/shared-types.ts @@ -198,10 +198,10 @@ export type GatewayRequestContext = { broadcastToConnIds: GatewayBroadcastToConnIdsFn; nodeSendToSession: (sessionKey: string, event: string, payload: unknown) => void; nodeSendToAllSubscribed: (event: string, payload: unknown) => void; - nodeSubscribe: (nodeId: string, sessionKey: string) => void; - nodeUnsubscribe: (nodeId: string, sessionKey: string) => void; + nodeSubscribe: (nodeId: string, sessionKey: string, connId?: string) => void; + nodeUnsubscribe: (nodeId: string, sessionKey: string, connId?: string) => void; nodeUnsubscribeAll: (nodeId: string) => void; - hasConnectedTalkNode: () => boolean; + hasConnectedTalkNode: () => Promise; isConnectionActive?: (connId: string) => boolean; hasExecApprovalClients?: (excludeConnId?: string) => boolean; /** Instance-local native approval subscribers; never derived from a network client. */ diff --git a/src/gateway/server-methods/talk.ts b/src/gateway/server-methods/talk.ts index 776030286fe1..db53831fde4c 100644 --- a/src/gateway/server-methods/talk.ts +++ b/src/gateway/server-methods/talk.ts @@ -863,8 +863,8 @@ export const talkHandlers: GatewayRequestHandlers = { respond(false, undefined, talkSpeakError("synthesis_failed", formatForLog(err))); } }, - "talk.mode": ({ params, respond, context, client, isWebchatConnect }) => { - if (client && isWebchatConnect(client.connect) && !context.hasConnectedTalkNode()) { + "talk.mode": async ({ params, respond, context, client, isWebchatConnect }) => { + if (client && isWebchatConnect(client.connect) && !(await context.hasConnectedTalkNode())) { respond( false, undefined, diff --git a/src/gateway/server-methods/terminal.test.ts b/src/gateway/server-methods/terminal.test.ts index 3b3fdefe1912..28cae7f8cf60 100644 --- a/src/gateway/server-methods/terminal.test.ts +++ b/src/gateway/server-methods/terminal.test.ts @@ -571,7 +571,12 @@ describe("terminal gateway policy", () => { paramsJSON: JSON.stringify({ threadId: "thread" }), }), }); - const node = { nodeId: "node-1", connId: "conn-node", commands: [command] }; + const node = { + nodeId: "node-1", + connId: "conn-node", + pairingGeneration: "generation-node", + commands: [command], + }; const invoke = vi.fn((rawParams: unknown) => { const params = rawParams as { onInvokeId?: (id: string) => void }; params.onInvokeId?.("invoke-1"); @@ -610,7 +615,11 @@ describe("terminal gateway policy", () => { )?.at(0); await openRequest?.createBackend?.(); expect(invoke).toHaveBeenCalledWith( - expect.objectContaining({ nodeId: "node-1", expectedConnId: "conn-node" }), + expect.objectContaining({ + nodeId: "node-1", + expectedConnId: "conn-node", + expectedPairingGeneration: "generation-node", + }), ); }); @@ -795,6 +804,7 @@ describe("terminal gateway policy", () => { const node = { nodeId: "node-1", connId: "conn-node", + pairingGeneration: "generation-node", commands: [command, uploadCommand], }; const invoke = vi.fn(async () => ({ @@ -824,6 +834,7 @@ describe("terminal gateway policy", () => { expect(invoke).toHaveBeenCalledWith({ nodeId: "node-1", expectedConnId: "conn-node", + expectedPairingGeneration: "generation-node", command: uploadCommand, params: { name: "report.pdf", contentBase64: "dGVzdA==" }, timeoutMs: 120_000, diff --git a/src/gateway/server-methods/terminal.ts b/src/gateway/server-methods/terminal.ts index eeb54f7f07da..9d938435acfe 100644 --- a/src/gateway/server-methods/terminal.ts +++ b/src/gateway/server-methods/terminal.ts @@ -87,6 +87,9 @@ async function stageNodeTerminalUpload( const result = await context.nodeRegistry.invoke({ nodeId, expectedConnId: access.node.connId, + ...(access.node.pairingGeneration + ? { expectedPairingGeneration: access.node.pairingGeneration } + : {}), command: NODE_TERMINAL_UPLOAD_COMMAND, params: file, timeoutMs: 120_000, @@ -290,6 +293,7 @@ export const terminalHandlers: GatewayRequestHandlers = { registry: context.nodeRegistry, nodeId: relay.plan.nodeId, expectedConnId: access.node.connId, + expectedPairingGeneration: access.node.pairingGeneration, command: relay.plan.command, params: relay.params, }); diff --git a/src/gateway/server-node-events-types.ts b/src/gateway/server-node-events-types.ts index b814cdb0a338..e176069b0c02 100644 --- a/src/gateway/server-node-events-types.ts +++ b/src/gateway/server-node-events-types.ts @@ -12,8 +12,8 @@ export type NodeEventContext = { deps: CliDeps; broadcast: (event: string, payload: unknown, opts?: { dropIfSlow?: boolean }) => void; nodeSendToSession: (sessionKey: string, event: string, payload: unknown) => void; - nodeSubscribe: (nodeId: string, sessionKey: string) => void; - nodeUnsubscribe: (nodeId: string, sessionKey: string) => void; + nodeSubscribe: (nodeId: string, sessionKey: string, connId?: string) => void | Promise; + nodeUnsubscribe: (nodeId: string, sessionKey: string, connId?: string) => void | Promise; broadcastVoiceWakeChanged: (triggers: string[]) => void; addChatRun: (sessionId: string, entry: ChatRunRegistration) => void; removeChatRun: ( diff --git a/src/gateway/server-node-events.runtime.ts b/src/gateway/server-node-events.runtime.ts index ee2d1f466c8c..3f36e1ed9172 100644 --- a/src/gateway/server-node-events.runtime.ts +++ b/src/gateway/server-node-events.runtime.ts @@ -13,7 +13,10 @@ export { loadOrCreateProcessDeviceIdentity } from "../infra/device-identity.js"; export { requestHeartbeat } from "../infra/heartbeat-wake.js"; export { buildOutboundSessionContext } from "../infra/outbound/session-context.js"; export { resolveOutboundTarget } from "../infra/outbound/targets.js"; -export { registerApnsRegistration } from "../infra/push-apns.js"; +export { + ApnsRegistrationPairingChangedError, + registerApnsRegistration, +} from "../infra/push-apns.js"; export { enqueueSystemEvent } from "../infra/system-events.js"; export { deleteMediaBuffer } from "../media/store.js"; export { normalizeMainKey } from "../routing/session-key.js"; diff --git a/src/gateway/server-node-events.test.ts b/src/gateway/server-node-events.test.ts index f63ac3551853..d3268e2beb36 100644 --- a/src/gateway/server-node-events.test.ts +++ b/src/gateway/server-node-events.test.ts @@ -83,10 +83,16 @@ const sanitizeInboundSystemTagsMock = vi.hoisted(() => .replace(/^(\s*)System:(?=\s|$)/gim, "$1System (untrusted):"), ), ); -const updatePairedDeviceMetadataMock = vi.hoisted(() => vi.fn().mockResolvedValue(true)); +const updatePairedDevicePresenceMock = vi.hoisted(() => vi.fn().mockResolvedValue(true)); const runtimeMocks = vi.hoisted(() => ({ agentCommandFromIngress: ingressAgentCommandMock, + ApnsRegistrationPairingChangedError: class ApnsRegistrationPairingChangedError extends Error { + constructor() { + super("node pairing changed before APNs registration"); + this.name = "ApnsRegistrationPairingChangedError"; + } + }, buildOutboundSessionContext: vi.fn(({ sessionKey }: { sessionKey: string }) => ({ key: sessionKey, agentId: "main", @@ -151,7 +157,7 @@ const runtimeMocks = vi.hoisted(() => ({ vi.mock("./server-node-events.runtime.js", () => runtimeMocks); vi.mock("../infra/device-pairing.js", () => ({ - updatePairedDeviceMetadata: updatePairedDeviceMetadataMock, + updatePairedDevicePresence: updatePairedDevicePresenceMock, })); import type { CliDeps } from "../cli/deps.js"; import type { HealthSummary } from "../commands/health.js"; @@ -321,13 +327,21 @@ function expectPresencePersistCall( reason: string, ): void { expect(mock).toHaveBeenCalledTimes(1); - const [actualDeviceId, metadata] = mockCall(mock) ?? []; + const [actualDeviceId, metadata, generation] = mockCall(mock) ?? []; expect(actualDeviceId).toBe(deviceId); expectFields(metadata, { lastSeenReason: reason }); + expect(generation).toEqual({ nodeId: deviceId, key: `${deviceId}-generation` }); const lastSeenAtMs = (metadata as { lastSeenAtMs?: unknown } | undefined)?.lastSeenAtMs; expect(typeof lastSeenAtMs).toBe("number"); } +function presenceConnection(deviceId: string, generation = `${deviceId}-generation`) { + return { + deviceId, + pairingGeneration: { nodeId: deviceId, key: generation }, + }; +} + describe("node exec events", () => { beforeEach(() => { enqueueSystemEventMock.mockClear(); @@ -340,8 +354,8 @@ describe("node exec events", () => { persistInboundImagesForTranscriptMock.mockResolvedValue([]); normalizeChannelIdVi.mockImplementation((channel?: string | null) => channel ?? null); sanitizeInboundSystemTagsMock.mockClear(); - updatePairedDeviceMetadataMock.mockClear(); - updatePairedDeviceMetadataMock.mockResolvedValue(true); + updatePairedDevicePresenceMock.mockClear(); + updatePairedDevicePresenceMock.mockResolvedValue(true); }); it("enqueues exec.started events", async () => { @@ -395,7 +409,9 @@ describe("node exec events", () => { it("keeps a node run authorized from exec.started through exec.finished", async () => { const registry = new NodeRegistry(); const frames: string[] = []; - registry.register(makeNodeClient("conn-1", "node-1", frames), {}); + registry.register(makeNodeClient("conn-1", "node-1", frames), { + pairingIdentity: "identity-a", + }); const invoke = registry.invoke({ nodeId: "node-1", command: "system.run", @@ -763,14 +779,19 @@ describe("node exec events", () => { it("stores direct APNs registrations from node events", async () => { const ctx = buildCtx(); - await handleNodeEvent(ctx, "node-direct", { - event: "push.apns.register", - payloadJSON: JSON.stringify({ - token: "abcd1234abcd1234abcd1234abcd1234", - topic: "ai.openclaw.ios", - environment: "sandbox", - }), - }); + await handleNodeEvent( + ctx, + "node-direct", + { + event: "push.apns.register", + payloadJSON: JSON.stringify({ + token: "abcd1234abcd1234abcd1234abcd1234", + topic: "ai.openclaw.ios", + environment: "sandbox", + }), + }, + { resolveApnsRegistrationGeneration: () => "generation-node-direct" }, + ); expect(registerApnsRegistrationVi).toHaveBeenCalledWith({ nodeId: "node-direct", @@ -778,25 +799,31 @@ describe("node exec events", () => { token: "abcd1234abcd1234abcd1234abcd1234", topic: "ai.openclaw.ios", environment: "sandbox", + expectedPairingGeneration: "generation-node-direct", }); }); it("stores relay APNs registrations from node events", async () => { const ctx = buildCtx(); - await handleNodeEvent(ctx, "node-relay", { - event: "push.apns.register", - payloadJSON: JSON.stringify({ - transport: "relay", - relayHandle: "relay-handle-123", - sendGrant: "send-grant-123", - gatewayDeviceId: "gateway-device-1", - installationId: "install-123", - topic: "ai.openclaw.ios", - environment: "production", - distribution: "official", - tokenDebugSuffix: "abcd1234", - }), - }); + await handleNodeEvent( + ctx, + "node-relay", + { + event: "push.apns.register", + payloadJSON: JSON.stringify({ + transport: "relay", + relayHandle: "relay-handle-123", + sendGrant: "send-grant-123", + gatewayDeviceId: "gateway-device-1", + installationId: "install-123", + topic: "ai.openclaw.ios", + environment: "production", + distribution: "official", + tokenDebugSuffix: "abcd1234", + }), + }, + { resolveApnsRegistrationGeneration: () => "generation-node-relay" }, + ); expect(registerApnsRegistrationVi).toHaveBeenCalledWith({ nodeId: "node-relay", @@ -808,25 +835,31 @@ describe("node exec events", () => { environment: "production", distribution: "official", tokenDebugSuffix: "abcd1234", + expectedPairingGeneration: "generation-node-relay", }); }); it("stores sandbox relay APNs registrations from node events", async () => { const ctx = buildCtx(); - await handleNodeEvent(ctx, "node-relay-sandbox", { - event: "push.apns.register", - payloadJSON: JSON.stringify({ - transport: "relay", - relayHandle: "relay-handle-123", - sendGrant: "send-grant-123", - gatewayDeviceId: "gateway-device-1", - installationId: "install-123", - topic: "ai.openclaw.ios", - environment: "sandbox", - distribution: "official", - tokenDebugSuffix: "abcd1234", - }), - }); + await handleNodeEvent( + ctx, + "node-relay-sandbox", + { + event: "push.apns.register", + payloadJSON: JSON.stringify({ + transport: "relay", + relayHandle: "relay-handle-123", + sendGrant: "send-grant-123", + gatewayDeviceId: "gateway-device-1", + installationId: "install-123", + topic: "ai.openclaw.ios", + environment: "sandbox", + distribution: "official", + tokenDebugSuffix: "abcd1234", + }), + }, + { resolveApnsRegistrationGeneration: () => "generation-node-relay-sandbox" }, + ); expect(registerApnsRegistrationVi).toHaveBeenCalledWith({ nodeId: "node-relay-sandbox", @@ -838,6 +871,7 @@ describe("node exec events", () => { environment: "sandbox", distribution: "official", tokenDebugSuffix: "abcd1234", + expectedPairingGeneration: "generation-node-relay-sandbox", }); }); @@ -859,6 +893,61 @@ describe("node exec events", () => { expect(registerApnsRegistrationVi).not.toHaveBeenCalled(); }); + + it("rejects APNs registration after the source pairing session is invalidated", async () => { + const warn = vi.fn(); + const ctx: NodeEventContext = { ...buildCtx(), logGateway: { warn } }; + const result = await handleNodeEvent( + ctx, + "node-invalidated-register", + { + event: "push.apns.register", + payloadJSON: JSON.stringify({ + token: "abcd1234abcd1234abcd1234abcd1234", + topic: "ai.openclaw.ios", + environment: "sandbox", + }), + }, + { resolveApnsRegistrationGeneration: async () => null }, + ); + + expect(result).toEqual({ + ok: true, + event: "push.apns.register", + handled: false, + reason: "pairing_changed", + }); + expect(registerApnsRegistrationVi).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + "push apns register rejected node=node-invalidated-register: stale or invalidated pairing session", + ); + }); + + it("returns pairing changed when APNs registration loses ownership in its transaction", async () => { + registerApnsRegistrationVi.mockRejectedValueOnce( + new runtimeMocks.ApnsRegistrationPairingChangedError(), + ); + const result = await handleNodeEvent( + buildCtx(), + "node-transaction-invalidated-register", + { + event: "push.apns.register", + payloadJSON: JSON.stringify({ + token: "abcd1234abcd1234abcd1234abcd1234", + topic: "ai.openclaw.ios", + environment: "sandbox", + }), + }, + { resolveApnsRegistrationGeneration: async () => "generation-before-transaction" }, + ); + + expect(result).toEqual({ + ok: true, + event: "push.apns.register", + handled: false, + reason: "pairing_changed", + }); + }); }); describe("voice transcript events", () => { @@ -874,7 +963,7 @@ describe("voice transcript events", () => { }); }); - it("dedupes repeated transcript payloads for the same session", async () => { + it("dedupes repeated transcript agent dispatches for the same session", async () => { const addChatRun = vi.fn(); const ctx = buildCtx(); ctx.addChatRun = addChatRun; @@ -898,6 +987,258 @@ describe("voice transcript events", () => { expect(canonicalizeSessionEntryAliasesMock).toHaveBeenCalledTimes(1); }); + it("persists only the accepted replay session ID when identical new-session events race", async () => { + const addChatRun = vi.fn(); + const ctx = buildCtx(); + ctx.addChatRun = addChatRun; + loadSessionEntryMock.mockImplementation((sessionKey: string) => ({ + ...buildSessionLookup(sessionKey), + entry: undefined, + })); + let persistedEntry: { sessionId?: string } | undefined; + canonicalizeSessionEntryAliasesMock.mockImplementation(async ({ target, update }) => { + const entry = update ? await update(undefined) : undefined; + persistedEntry = entry; + return { canonicalKey: target.canonicalKey, entry }; + }); + const detachedChecksStarted = createDeferred(); + const detachedAdmission = createDeferred(); + let checkCount = 0; + const isConnectionCurrent = vi.fn(() => { + checkCount += 1; + if (checkCount === 1 || checkCount === 3) { + return true; + } + if (checkCount === 4) { + detachedChecksStarted.resolve(); + } + return detachedAdmission.promise; + }); + const payload = { + text: "one command for a new session", + sessionKey: "voice-new-session-replay-race", + }; + + await handleNodeEvent( + ctx, + "node-new-session-replay", + { + event: "voice.transcript", + payloadJSON: JSON.stringify(payload), + }, + { isConnectionCurrent }, + ); + await handleNodeEvent( + ctx, + "node-new-session-replay", + { + event: "voice.transcript", + payloadJSON: JSON.stringify(payload), + }, + { isConnectionCurrent }, + ); + await detachedChecksStarted.promise; + detachedAdmission.resolve(true); + await waitForFast(() => expect(agentCommandMock).toHaveBeenCalledTimes(1)); + + expect(canonicalizeSessionEntryAliasesMock).toHaveBeenCalledTimes(1); + expect(addChatRun).toHaveBeenCalledTimes(1); + const dispatched = mockCallArg(agentCommandMock) as { sessionId?: unknown }; + expect(persistedEntry?.sessionId).toBe(dispatched.sessionId); + }); + + it("uses receipt time when delayed identical transcript admissions finish together", async () => { + const addChatRun = vi.fn(); + const ctx = buildCtx(); + ctx.addChatRun = addChatRun; + let now = 1_000; + const nowSpy = vi.spyOn(Date, "now").mockImplementation(() => now); + const detachedChecksStarted = createDeferred(); + const detachedAdmission = createDeferred(); + let checkCount = 0; + const isConnectionCurrent = vi.fn(() => { + checkCount += 1; + if (checkCount === 1 || checkCount === 3) { + return true; + } + if (checkCount === 4) { + detachedChecksStarted.resolve(); + } + return detachedAdmission.promise; + }); + const payload = { + text: "repeat after the replay window", + sessionKey: "voice-delayed-admission-window", + }; + + try { + await handleNodeEvent( + ctx, + "node-delayed-admission", + { + event: "voice.transcript", + payloadJSON: JSON.stringify(payload), + }, + { isConnectionCurrent }, + ); + now = 3_000; + await handleNodeEvent( + ctx, + "node-delayed-admission", + { + event: "voice.transcript", + payloadJSON: JSON.stringify(payload), + }, + { isConnectionCurrent }, + ); + await detachedChecksStarted.promise; + now = 10_000; + detachedAdmission.resolve(true); + await waitForFast(() => expect(agentCommandMock).toHaveBeenCalledTimes(2)); + + expect(addChatRun).toHaveBeenCalledTimes(2); + expect(canonicalizeSessionEntryAliasesMock).toHaveBeenCalledTimes(2); + } finally { + nowSpy.mockRestore(); + } + }); + + it("allows a current replay after rejecting the same transcript from a stale connection", async () => { + const addChatRun = vi.fn(); + const ctx = buildCtx(); + ctx.addChatRun = addChatRun; + const detachedChecksStarted = createDeferred(); + const staleAdmission = createDeferred(); + let checkCount = 0; + const isConnectionCurrent = vi.fn(() => { + checkCount += 1; + if (checkCount === 1) { + return true; + } + if (checkCount === 2) { + detachedChecksStarted.resolve(); + } + return staleAdmission.promise; + }); + const payload = { + text: "replay after reconnect", + sessionKey: "voice-stale-replay-session", + }; + + await handleNodeEvent( + ctx, + "node-stale-voice", + { + event: "voice.transcript", + payloadJSON: JSON.stringify(payload), + }, + { isConnectionCurrent }, + ); + await detachedChecksStarted.promise; + await handleNodeEvent( + ctx, + "node-current-voice", + { + event: "voice.transcript", + payloadJSON: JSON.stringify(payload), + }, + { isConnectionCurrent: () => true }, + ); + + expect(addChatRun).not.toHaveBeenCalled(); + expect(agentCommandMock).not.toHaveBeenCalled(); + + staleAdmission.resolve(false); + await waitForFast(() => expect(agentCommandMock).toHaveBeenCalledTimes(1)); + await waitForFast(() => expect(getActiveGatewayRootWorkCount()).toBe(0)); + + expect(addChatRun).toHaveBeenCalledTimes(1); + expect(canonicalizeSessionEntryAliasesMock).toHaveBeenCalledTimes(1); + }); + + it("rechecks a queued replay after an earlier stale reservation is released", async () => { + const addChatRun = vi.fn(); + const ctx = buildCtx(); + ctx.addChatRun = addChatRun; + const staleCheckStarted = createDeferred(); + const staleAdmission = createDeferred(); + let staleCheckCount = 0; + const isStaleConnectionCurrent = vi.fn(() => { + staleCheckCount += 1; + if (staleCheckCount === 1) { + return true; + } + staleCheckStarted.resolve(); + return staleAdmission.promise; + }); + let replayCurrent = true; + const isReplayConnectionCurrent = vi.fn(() => replayCurrent); + const payload = { + text: "invalidate while queued", + sessionKey: "voice-queued-replay-currentness", + }; + + await handleNodeEvent( + ctx, + "node-stale-queued-voice", + { + event: "voice.transcript", + payloadJSON: JSON.stringify(payload), + }, + { isConnectionCurrent: isStaleConnectionCurrent }, + ); + await staleCheckStarted.promise; + await handleNodeEvent( + ctx, + "node-replay-invalidated-while-queued", + { + event: "voice.transcript", + payloadJSON: JSON.stringify(payload), + }, + { isConnectionCurrent: isReplayConnectionCurrent }, + ); + await waitForFast(() => expect(isReplayConnectionCurrent).toHaveBeenCalledTimes(2)); + + replayCurrent = false; + staleAdmission.resolve(false); + await waitForFast(() => expect(isReplayConnectionCurrent).toHaveBeenCalledTimes(3)); + await waitForFast(() => expect(getActiveGatewayRootWorkCount()).toBe(0)); + + expect(agentCommandMock).not.toHaveBeenCalled(); + expect(addChatRun).not.toHaveBeenCalled(); + expect(canonicalizeSessionEntryAliasesMock).not.toHaveBeenCalled(); + }); + + it("skips the detached session-store touch after voice admission loses ownership", async () => { + const addChatRun = vi.fn(); + const ctx = buildCtx(); + ctx.addChatRun = addChatRun; + let checkCount = 0; + const isConnectionCurrent = vi.fn(() => { + checkCount += 1; + return checkCount <= 3; + }); + + await handleNodeEvent( + ctx, + "node-stale-after-voice-admission", + { + event: "voice.transcript", + payloadJSON: JSON.stringify({ + text: "do not persist stale voice ownership", + sessionKey: "voice-detached-store-currentness", + }), + }, + { isConnectionCurrent }, + ); + await waitForFast(() => expect(isConnectionCurrent).toHaveBeenCalledTimes(4)); + await waitForFast(() => expect(getActiveGatewayRootWorkCount()).toBe(0)); + + expect(agentCommandMock).toHaveBeenCalledTimes(1); + expect(addChatRun).toHaveBeenCalledTimes(1); + expect(canonicalizeSessionEntryAliasesMock).not.toHaveBeenCalled(); + }); + it("rejects a missing harness-owned session before touching the store", async () => { const sessionKey = "agent:main:harness:codex:supervision:missing-voice"; loadSessionEntryMock.mockReturnValueOnce({ @@ -1352,7 +1693,9 @@ describe("agent request events", () => { runtimeMocks.resolveSessionAgentId.mockClear(); runtimeMocks.resolveSessionModelRef.mockClear(); runtimeMocks.resolveGatewayModelSupportsImages.mockClear(); - persistInboundImagesForTranscriptMock.mockClear(); + persistInboundImagesForTranscriptMock.mockReset(); + persistInboundImagesForTranscriptMock.mockResolvedValue([]); + runtimeMocks.deleteMediaBuffer.mockClear(); canonicalizeSessionEntryAliasesMock.mockClear(); loadSessionEntryMock.mockClear(); normalizeChannelIdVi.mockClear(); @@ -1450,6 +1793,83 @@ describe("agent request events", () => { expectSuspendReady("receipt-delivery-ready"); }); + it("does not launch agent work when pairing changes during model lookup", async () => { + const modelCatalog = + createDeferred>>(); + const ctx = buildCtx(); + ctx.loadGatewayModelCatalog = vi.fn(() => modelCatalog.promise); + let connectionCurrent = true; + const isConnectionCurrent = vi.fn(async () => connectionCurrent); + + const request = handleNodeEvent( + ctx, + "node-revoked-during-model-lookup", + { + event: "agent.request", + payloadJSON: JSON.stringify({ + message: "describe this image", + sessionKey: "agent:main:revoked-during-model-lookup", + attachments: [{ type: "image", mimeType: "image/png", content: "AAAA" }], + deliver: true, + receipt: true, + channel: "telegram", + to: "123", + }), + }, + { isConnectionCurrent }, + ); + + await waitForFast(() => expect(ctx.loadGatewayModelCatalog).toHaveBeenCalledTimes(1)); + connectionCurrent = false; + modelCatalog.resolve([]); + + await expect(request).resolves.toEqual({ + ok: true, + event: "agent.request", + handled: false, + reason: "pairing_changed", + }); + expect(parseMessageWithAttachmentsMock).not.toHaveBeenCalled(); + expect(canonicalizeSessionEntryAliasesMock).not.toHaveBeenCalled(); + expect(sendDurableMessageBatchMock).not.toHaveBeenCalled(); + expect(persistInboundImagesForTranscriptMock).not.toHaveBeenCalled(); + expect(agentCommandMock).not.toHaveBeenCalled(); + }); + + it("cleans persisted transcript media when detached agent admission is revoked", async () => { + persistInboundImagesForTranscriptMock.mockResolvedValueOnce([ + { + id: "saved-after-admission", + path: "/media/inbound/saved-after-admission.png", + size: 5, + contentType: "image/png", + }, + ]); + let currentnessChecks = 0; + const isConnectionCurrent = vi.fn(async () => { + currentnessChecks += 1; + return currentnessChecks < 6; + }); + + await handleNodeEvent( + buildCtx(), + "node-revoked-before-detached-start", + { + event: "agent.request", + payloadJSON: JSON.stringify({ + message: "do not retain this media", + sessionKey: "agent:main:revoked-before-detached-start", + }), + }, + { isConnectionCurrent }, + ); + + await waitForFast(() => { + expect(runtimeMocks.deleteMediaBuffer).toHaveBeenCalledWith("saved-after-admission"); + }); + expect(agentCommandMock).not.toHaveBeenCalled(); + }); + it.each([ ["wrong owner", { agentHarnessId: "other", modelSelectionLocked: true }], ["missing session id", { agentHarnessId: "codex", modelSelectionLocked: true, sessionId: "" }], @@ -1670,8 +2090,8 @@ describe("agent request events", () => { }); beforeEach(() => { - updatePairedDeviceMetadataMock.mockClear(); - updatePairedDeviceMetadataMock.mockResolvedValue(true); + updatePairedDevicePresenceMock.mockClear(); + updatePairedDevicePresenceMock.mockResolvedValue(true); }); it("persists authenticated node presence alive events", async () => { @@ -1683,7 +2103,7 @@ describe("agent request events", () => { event: "node.presence.alive", payloadJSON: JSON.stringify({ trigger: "bg_app_refresh", sentAtMs: 123 }), }, - { deviceId: "ios-presence-persist" }, + presenceConnection("ios-presence-persist"), ); expect(result).toEqual({ @@ -1693,7 +2113,7 @@ describe("agent request events", () => { reason: "persisted", }); expectPresencePersistCall( - updatePairedDeviceMetadataMock, + updatePairedDevicePresenceMock, "ios-presence-persist", "bg_app_refresh", ); @@ -1712,11 +2132,31 @@ describe("agent request events", () => { handled: false, reason: "missing_device_identity", }); - expect(updatePairedDeviceMetadataMock).not.toHaveBeenCalled(); + expect(updatePairedDevicePresenceMock).not.toHaveBeenCalled(); }); - it("does not throttle unknown node presence alive identities", async () => { - updatePairedDeviceMetadataMock.mockResolvedValue(false); + it("rejects node presence alive events without the authenticated pairing generation", async () => { + const result = await handleNodeEvent( + buildCtx(), + "ios-presence-missing-generation", + { + event: "node.presence.alive", + payloadJSON: JSON.stringify({ trigger: "silent_push" }), + }, + { deviceId: "ios-presence-missing-generation" }, + ); + + expect(result).toEqual({ + ok: true, + event: "node.presence.alive", + handled: false, + reason: "pairing_changed", + }); + expect(updatePairedDevicePresenceMock).not.toHaveBeenCalled(); + }); + + it("does not throttle stale node presence alive generations", async () => { + updatePairedDevicePresenceMock.mockResolvedValue(false); const ctx = buildCtx(); const result = await handleNodeEvent( ctx, @@ -1725,18 +2165,18 @@ describe("agent request events", () => { event: "node.presence.alive", payloadJSON: JSON.stringify({ trigger: "silent_push" }), }, - { deviceId: "ios-presence-unpaired" }, + presenceConnection("ios-presence-unpaired"), ); expect(result).toEqual({ ok: true, event: "node.presence.alive", handled: false, - reason: "unpaired", + reason: "pairing_changed", }); - updatePairedDeviceMetadataMock.mockClear(); - updatePairedDeviceMetadataMock.mockResolvedValue(true); + updatePairedDevicePresenceMock.mockClear(); + updatePairedDevicePresenceMock.mockResolvedValue(true); const retry = await handleNodeEvent( ctx, "ios-presence-unpaired", @@ -1744,7 +2184,7 @@ describe("agent request events", () => { event: "node.presence.alive", payloadJSON: JSON.stringify({ trigger: "silent_push" }), }, - { deviceId: "ios-presence-unpaired" }, + presenceConnection("ios-presence-unpaired"), ); expect(retry).toEqual({ ok: true, @@ -1752,7 +2192,7 @@ describe("agent request events", () => { handled: true, reason: "persisted", }); - expect(updatePairedDeviceMetadataMock).toHaveBeenCalledTimes(1); + expect(updatePairedDevicePresenceMock).toHaveBeenCalledTimes(1); }); it("throttles repeated node presence alive persistence per device", async () => { @@ -1761,7 +2201,7 @@ describe("agent request events", () => { event: "node.presence.alive" as const, payloadJSON: JSON.stringify({ trigger: "silent_push" }), }; - const connection = { deviceId: "ios-presence-throttle" }; + const connection = presenceConnection("ios-presence-throttle"); await handleNodeEvent(ctx, "ios-presence-throttle", event, connection); const result = await handleNodeEvent(ctx, "ios-presence-throttle", event, connection); @@ -1772,7 +2212,36 @@ describe("agent request events", () => { handled: true, reason: "throttled", }); - expect(updatePairedDeviceMetadataMock).toHaveBeenCalledTimes(1); + expect(updatePairedDevicePresenceMock).toHaveBeenCalledTimes(1); + }); + + it("does not throttle the first presence update from a replacement generation", async () => { + const ctx = buildCtx(); + const event = { + event: "node.presence.alive" as const, + payloadJSON: JSON.stringify({ trigger: "silent_push" }), + }; + + await handleNodeEvent( + ctx, + "ios-presence-replacement", + event, + presenceConnection("ios-presence-replacement", "generation-a"), + ); + const result = await handleNodeEvent( + ctx, + "ios-presence-replacement", + event, + presenceConnection("ios-presence-replacement", "generation-b"), + ); + + expect(result).toEqual({ + ok: true, + event: "node.presence.alive", + handled: true, + reason: "persisted", + }); + expect(updatePairedDevicePresenceMock).toHaveBeenCalledTimes(2); }); it("updates authenticated accessibility-backed node activity without a system event", async () => { @@ -1850,11 +2319,11 @@ describe("agent request events", () => { event: "node.presence.alive", payloadJSON: JSON.stringify({ trigger: "x".repeat(4096) }), }, - { deviceId: "ios-presence-normalize" }, + presenceConnection("ios-presence-normalize"), ); expectPresencePersistCall( - updatePairedDeviceMetadataMock, + updatePairedDevicePresenceMock, "ios-presence-normalize", "background", ); diff --git a/src/gateway/server-node-events.ts b/src/gateway/server-node-events.ts index 9c78fcd3e75e..11917af6bccc 100644 --- a/src/gateway/server-node-events.ts +++ b/src/gateway/server-node-events.ts @@ -8,7 +8,7 @@ import { import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import { validateNodePresenceActivityPayload } from "../../packages/gateway-protocol/src/index.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { updatePairedDeviceMetadata } from "../infra/device-pairing.js"; +import { updatePairedDevicePresence, type NodePairingGeneration } from "../infra/device-pairing.js"; import { formatErrorMessage } from "../infra/errors.js"; import { resolveEventSessionKeyForPolicy, @@ -26,6 +26,7 @@ import { import type { NodeEvent, NodeEventContext } from "./server-node-events-types.js"; import { agentCommandFromIngress, + ApnsRegistrationPairingChangedError, buildOutboundSessionContext, createOutboundSendDeps, defaultRuntime, @@ -62,6 +63,18 @@ const NODE_PRESENCE_PERSIST_MIN_INTERVAL_MS = 60_000; const MAX_RECENT_NODE_PRESENCE_KEYS = 1024; const recentVoiceTranscripts = new Map(); +type VoiceTranscriptReservationAdmission = { work: Promise } | null; +type VoiceTranscriptReservation = { + fingerprint: string; + receivedAt: number; + status: "pending" | "ready" | "checking" | "rejected"; + isConnectionCurrent?: () => boolean | Promise; + start?: () => Promise; + resolve: (admission: VoiceTranscriptReservationAdmission) => void; + rejectDecision: (reason: unknown) => void; + decision: Promise; +}; +const pendingVoiceTranscriptReservations = new Map(); const recentExecFinishedRuns = new Map(); const recentNodePresencePersistAt = new Map(); @@ -82,12 +95,19 @@ function dispatchNodeAgentCommand( ctx: NodeEventContext, nodeId: string, input: NodeAgentCommandInput, + isConnectionCurrent?: () => boolean | Promise, + onAdmissionRejected?: () => void | Promise, ): void { // The node RPC can finish before the agent starts its own session admission. - // Reserve a root now so suspension cannot acknowledge and then strand the turn. - void runWithGatewayIndependentRootWorkContinuation(() => - agentCommandFromIngress(input, defaultRuntime, ctx.deps), - ).catch((err: unknown) => { + // Reserve a root now so suspension cannot acknowledge and then strand the turn, + // but recheck the admitted connection before agent work actually starts. + void runWithGatewayIndependentRootWorkContinuation(async () => { + if (isConnectionCurrent && !(await isConnectionCurrent())) { + await onAdmissionRejected?.(); + return; + } + await agentCommandFromIngress(input, defaultRuntime, ctx.deps); + }).catch((err: unknown) => { ctx.logGateway.warn(`agent failed node=${nodeId}: ${formatForLog(err)}`); }); } @@ -164,6 +184,125 @@ function shouldDropDuplicateVoiceTranscript(params: { return false; } +function reserveVoiceTranscript(params: { + sessionKey: string; + fingerprint: string; + receivedAt: number; +}): { + admit: (params: { + isConnectionCurrent?: () => boolean | Promise; + start: () => Promise; + }) => Promise; + reject: () => void; +} { + // Resolve reservations in receipt order so delayed currentness checks cannot + // change the dedupe window, while rejected connections leave no committed state. + let resolveDecision: (admission: VoiceTranscriptReservationAdmission) => void = () => {}; + let rejectDecision: (reason: unknown) => void = () => {}; + const decision = new Promise((resolve, reject) => { + resolveDecision = resolve; + rejectDecision = reject; + }); + const reservation: VoiceTranscriptReservation = { + fingerprint: params.fingerprint, + receivedAt: params.receivedAt, + status: "pending", + resolve: resolveDecision, + rejectDecision, + decision, + }; + const queue = pendingVoiceTranscriptReservations.get(params.sessionKey) ?? []; + queue.push(reservation); + pendingVoiceTranscriptReservations.set(params.sessionKey, queue); + + const drain = () => { + while (queue[0]?.status === "rejected") { + const next = queue.shift(); + if (!next) { + break; + } + next.resolve(null); + } + const next = queue[0]; + if (!next) { + pendingVoiceTranscriptReservations.delete(params.sessionKey); + return; + } + if (next.status !== "ready") { + return; + } + next.status = "checking"; + void (async () => { + try { + const isCurrent = next.isConnectionCurrent ? await next.isConnectionCurrent() : true; + const admission = + isCurrent && + !shouldDropDuplicateVoiceTranscript({ + sessionKey: params.sessionKey, + fingerprint: next.fingerprint, + now: next.receivedAt, + }) && + next.start + ? { work: next.start() } + : null; + queue.shift(); + next.resolve(admission); + } catch (err) { + queue.shift(); + next.rejectDecision(err); + } + drain(); + })(); + }; + const settle = (status: "ready" | "rejected") => { + if (reservation.status !== "pending") { + return; + } + reservation.status = status; + drain(); + }; + + return { + admit: ({ isConnectionCurrent, start }) => { + reservation.isConnectionCurrent = isConnectionCurrent; + reservation.start = start; + settle("ready"); + return reservation.decision; + }, + reject: () => settle("rejected"), + }; +} + +function dispatchReservedVoiceAgentCommand(params: { + ctx: NodeEventContext; + nodeId: string; + input: NodeAgentCommandInput; + reservation: ReturnType; + isConnectionCurrent?: () => boolean | Promise; + onStart: () => void; +}): void { + void runWithGatewayIndependentRootWorkContinuation(async () => { + if (params.isConnectionCurrent && !(await params.isConnectionCurrent())) { + params.reservation.reject(); + return; + } + const admission = await params.reservation.admit({ + isConnectionCurrent: params.isConnectionCurrent, + start: () => { + params.onStart(); + return agentCommandFromIngress(params.input, defaultRuntime, params.ctx.deps); + }, + }); + if (!admission) { + return; + } + await admission.work; + }).catch((err: unknown) => { + params.reservation.reject(); + params.ctx.logGateway.warn(`agent failed node=${params.nodeId}: ${formatForLog(err)}`); + }); +} + function shouldDropDuplicateExecFinished(params: { sessionKey: string; runId: string; @@ -296,23 +435,59 @@ function queueSessionStoreTouch(params: { entry: LoadedSessionEntry["entry"]; sessionId: string; now: number; + isConnectionCurrent?: () => boolean | Promise; }) { // Voice dispatch intentionally does not wait for persistence, but a host // snapshot must not race the accepted write after its node RPC returns. - void runWithGatewayIndependentRootWorkContinuation(() => - touchSessionStore({ + void runWithGatewayIndependentRootWorkContinuation(async () => { + if (params.isConnectionCurrent && !(await params.isConnectionCurrent())) { + return; + } + await touchSessionStore({ storePath: params.storePath, canonicalKey: params.canonicalKey, storeKeys: params.storeKeys, entry: params.entry, sessionId: params.sessionId, now: params.now, - }), - ).catch((err: unknown) => { + }); + }).catch((err: unknown) => { params.ctx.logGateway.warn("voice session-store update failed: " + formatForLog(err)); }); } +async function isNodeEventConnectionCurrent(opts?: { + isConnectionCurrent?: () => boolean | Promise; +}): Promise { + if (!opts?.isConnectionCurrent) { + return true; + } + try { + return await opts.isConnectionCurrent(); + } catch { + return false; + } +} + +function pairingChangedResult(event: string): NodeEventHandleResult { + return { ok: true, event, handled: false, reason: "pairing_changed" }; +} + +async function cleanupNodeEventMedia( + ids: Iterable, + ctx: Pick, +): Promise { + for (const id of ids) { + try { + await deleteMediaBuffer(id); + } catch (cleanupErr) { + ctx.logGateway.warn( + `Failed to cleanup orphaned media ${id}: ${formatErrorMessage(cleanupErr)}`, + ); + } + } +} + function parseSessionKeyFromPayloadJSON(payloadJSON: string): string | null { let payload: unknown; try { @@ -383,8 +558,18 @@ export const handleNodeEvent = async ( ctx: NodeEventContext, nodeId: string, evt: NodeEvent, - opts?: { connId?: string; deviceId?: string; presenceAllowed?: boolean }, + opts?: { + connId?: string; + deviceId?: string; + pairingGeneration?: NodePairingGeneration; + presenceAllowed?: boolean; + isConnectionCurrent?: () => boolean | Promise; + resolveApnsRegistrationGeneration?: () => string | null | Promise; + }, ): Promise => { + if (!(await isNodeEventConnectionCurrent(opts))) { + return pairingChangedResult(evt.event); + } switch (evt.event) { case "voice.transcript": { const obj = parsePayloadObject(evt.payloadJSON); @@ -406,44 +591,55 @@ export const handleNodeEvent = async ( if (resolveAgentHarnessSessionContextError(canonicalKey, entry)) { return undefined; } - const now = Date.now(); + const receivedAt = Date.now(); const fingerprint = resolveVoiceTranscriptFingerprint(obj, text); - if (shouldDropDuplicateVoiceTranscript({ sessionKey: canonicalKey, fingerprint, now })) { - return undefined; - } const sessionId = entry?.sessionId ?? randomUUID(); - queueSessionStoreTouch({ - ctx, - storePath, - canonicalKey, - storeKeys, - entry, - sessionId, - now, - }); const runId = randomUUID(); - - // Ensure chat UI clients refresh when this run completes (even though it wasn't started via chat.send). - // This maps agent bus events (keyed by per-turn runId) to chat events (keyed by clientRunId). - ctx.addChatRun(runId, { + const transcriptReservation = reserveVoiceTranscript({ sessionKey: canonicalKey, - clientRunId: `voice-${randomUUID()}`, + fingerprint, + receivedAt, }); - dispatchNodeAgentCommand(ctx, nodeId, { - runId, - message: text, - sessionId, - sessionKey: canonicalKey, - thinking: "low", - deliver: false, - messageChannel: "node", - inputProvenance: { - kind: "external_user", - sourceChannel: "voice", - sourceTool: "gateway.voice.transcript", + dispatchReservedVoiceAgentCommand({ + ctx, + nodeId, + input: { + runId, + message: text, + sessionId, + sessionKey: canonicalKey, + thinking: "low", + deliver: false, + messageChannel: "node", + inputProvenance: { + kind: "external_user", + sourceChannel: "voice", + sourceTool: "gateway.voice.transcript", + }, + allowModelOverride: false, + }, + reservation: transcriptReservation, + isConnectionCurrent: opts?.isConnectionCurrent, + onStart: () => { + queueSessionStoreTouch({ + ctx, + storePath, + canonicalKey, + storeKeys, + entry, + sessionId, + now: receivedAt, + isConnectionCurrent: opts?.isConnectionCurrent, + }); + + // Ensure chat UI clients refresh when this run completes (even though it wasn't started via chat.send). + // This maps agent bus events (keyed by per-turn runId) to chat events (keyed by clientRunId). + ctx.addChatRun(runId, { + sessionKey: canonicalKey, + clientRunId: `voice-${randomUUID()}`, + }); }, - allowModelOverride: false, }); return undefined; } @@ -508,6 +704,9 @@ export const handleNodeEvent = async ( provider: modelRef.provider, model: modelRef.model, }); + if (!(await isNodeEventConnectionCurrent(opts))) { + return pairingChangedResult(evt.event); + } try { const parsed = await parseMessageWithAttachments(message, normalizedAttachments, { maxBytes: resolveChatAttachmentMaxBytes(cfg), @@ -518,6 +717,13 @@ export const handleNodeEvent = async ( // explicitly rather than saving them where the agent cannot reach them. acceptNonImage: false, }); + if (!(await isNodeEventConnectionCurrent(opts))) { + await cleanupNodeEventMedia( + (parsed.offloadedRefs ?? []).map((ref) => ref.id), + ctx, + ); + return pairingChangedResult(evt.event); + } message = parsed.message.trim(); images = parsed.images; imageOrder = parsed.imageOrder; @@ -527,15 +733,10 @@ export const handleNodeEvent = async ( `agent.request message exceeds limit after attachment parsing (length=${message.length})`, ); if (parsed.offloadedRefs && parsed.offloadedRefs.length > 0) { - for (const ref of parsed.offloadedRefs) { - try { - await deleteMediaBuffer(ref.id); - } catch (cleanupErr) { - ctx.logGateway.warn( - `Failed to cleanup orphaned media ${ref.id}: ${formatErrorMessage(cleanupErr)}`, - ); - } - } + await cleanupNodeEventMedia( + parsed.offloadedRefs.map((ref) => ref.id), + ctx, + ); } return undefined; } @@ -560,6 +761,13 @@ export const handleNodeEvent = async ( const now = Date.now(); const sessionId = entry?.sessionId ?? randomUUID(); + if (!(await isNodeEventConnectionCurrent(opts))) { + await cleanupNodeEventMedia( + (offloadedRefs ?? []).map((ref) => ref.id), + ctx, + ); + return pairingChangedResult(evt.event); + } await touchSessionStore({ storePath, canonicalKey, @@ -568,6 +776,13 @@ export const handleNodeEvent = async ( sessionId, now, }); + if (!(await isNodeEventConnectionCurrent(opts))) { + await cleanupNodeEventMedia( + (offloadedRefs ?? []).map((ref) => ref.id), + ctx, + ); + return pairingChangedResult(evt.event); + } if (deliverRequested && (!channel || !to)) { const entryChannel = @@ -591,19 +806,48 @@ export const handleNodeEvent = async ( ); } + if (!(await isNodeEventConnectionCurrent(opts))) { + await cleanupNodeEventMedia( + (offloadedRefs ?? []).map((ref) => ref.id), + ctx, + ); + return pairingChangedResult(evt.event); + } + const persistedTranscriptMedia = await persistInboundImagesForTranscript({ + images, + imageOrder, + offloadedRefs, + log: ctx.logGateway, + logContext: "agent.request", + }); + if (!(await isNodeEventConnectionCurrent(opts))) { + await cleanupNodeEventMedia( + persistedTranscriptMedia.map((media) => media.id), + ctx, + ); + return pairingChangedResult(evt.event); + } + const transcriptMedia = persistedTranscriptMedia.map((media) => ({ + path: media.path, + contentType: media.contentType, + })); + if (wantsReceipt && deliveryChannel && deliveryTo) { // Delivery stays detached from agent startup, but remains part of the // accepted node request until the durable send settles. - void runWithGatewayIndependentRootWorkContinuation(() => - sendReceiptAck({ + void runWithGatewayIndependentRootWorkContinuation(async () => { + if (!(await isNodeEventConnectionCurrent(opts))) { + return; + } + await sendReceiptAck({ cfg, deps: ctx.deps, sessionKey: canonicalKey, channel: deliveryChannel, to: deliveryTo, text: receiptText, - }), - ).catch((err: unknown) => { + }); + }).catch((err: unknown) => { ctx.logGateway.warn(`agent receipt failed node=${nodeId}: ${formatForLog(err)}`); }); } else if (wantsReceipt) { @@ -612,33 +856,33 @@ export const handleNodeEvent = async ( ); } - const transcriptMedia = ( - await persistInboundImagesForTranscript({ + dispatchNodeAgentCommand( + ctx, + nodeId, + { + runId: sessionId, + message, images, imageOrder, - offloadedRefs, - log: ctx.logGateway, - logContext: "agent.request", - }) - ).map((media) => ({ path: media.path, contentType: media.contentType })); - - dispatchNodeAgentCommand(ctx, nodeId, { - runId: sessionId, - message, - images, - imageOrder, - ...(transcriptMedia.length > 0 ? { transcriptMessage, transcriptMedia } : {}), - sessionId, - sessionKey: canonicalKey, - thinking: link?.thinking ?? undefined, - deliver, - to: deliveryTo, - channel: deliveryChannel, - timeout: - typeof link?.timeoutSeconds === "number" ? link.timeoutSeconds.toString() : undefined, - messageChannel: "node", - allowModelOverride: false, - }); + ...(transcriptMedia.length > 0 ? { transcriptMessage, transcriptMedia } : {}), + sessionId, + sessionKey: canonicalKey, + thinking: link?.thinking ?? undefined, + deliver, + to: deliveryTo, + channel: deliveryChannel, + timeout: + typeof link?.timeoutSeconds === "number" ? link.timeoutSeconds.toString() : undefined, + messageChannel: "node", + allowModelOverride: false, + }, + opts?.isConnectionCurrent, + () => + cleanupNodeEventMedia( + persistedTranscriptMedia.map((media) => media.id), + ctx, + ), + ); return undefined; } case "notifications.changed": { @@ -705,7 +949,7 @@ export const handleNodeEvent = async ( if (!sessionKey) { return undefined; } - ctx.nodeSubscribe(nodeId, sessionKey); + await ctx.nodeSubscribe(nodeId, sessionKey, opts?.connId); return undefined; } case "chat.unsubscribe": { @@ -716,7 +960,7 @@ export const handleNodeEvent = async ( if (!sessionKey) { return undefined; } - ctx.nodeUnsubscribe(nodeId, sessionKey); + await ctx.nodeUnsubscribe(nodeId, sessionKey, opts?.connId); return undefined; } case "exec.started": @@ -847,6 +1091,13 @@ export const handleNodeEvent = async ( const topic = typeof obj.topic === "string" ? obj.topic : ""; const environment = obj.environment; try { + const expectedPairingGeneration = await opts?.resolveApnsRegistrationGeneration?.(); + if (!expectedPairingGeneration) { + ctx.logGateway.warn( + `push apns register rejected node=${nodeId}: stale or invalidated pairing session`, + ); + return pairingChangedResult(evt.event); + } if (transport === "relay") { const gatewayDeviceId = normalizeOptionalString(obj.gatewayDeviceId) ?? ""; const currentGatewayDeviceId = loadOrCreateProcessDeviceIdentity().deviceId; @@ -867,6 +1118,7 @@ export const handleNodeEvent = async ( distribution: obj.distribution, relayOrigin: obj.relayOrigin, tokenDebugSuffix: obj.tokenDebugSuffix, + expectedPairingGeneration, }); } else { await registerApnsRegistration({ @@ -875,9 +1127,16 @@ export const handleNodeEvent = async ( token: typeof obj.token === "string" ? obj.token : "", topic, environment, + expectedPairingGeneration, }); } } catch (err) { + if (err instanceof ApnsRegistrationPairingChangedError) { + ctx.logGateway.warn( + `push apns register rejected node=${nodeId}: stale or invalidated pairing session`, + ); + return pairingChangedResult(evt.event); + } ctx.logGateway.warn(`push apns register failed node=${nodeId}: ${formatForLog(err)}`); } return undefined; @@ -911,8 +1170,13 @@ export const handleNodeEvent = async ( if (!deviceId) { return { ok: true, event: evt.event, handled: false, reason: "missing_device_identity" }; } + const pairingGeneration = opts?.pairingGeneration; + if (!pairingGeneration || pairingGeneration.nodeId !== deviceId) { + return pairingChangedResult(evt.event); + } const now = Date.now(); - const lastPersistedAt = recentNodePresencePersistAt.get(deviceId) ?? 0; + const presenceOwnerKey = `${deviceId}\0${pairingGeneration.key}`; + const lastPersistedAt = recentNodePresencePersistAt.get(presenceOwnerKey) ?? 0; if (now - lastPersistedAt < NODE_PRESENCE_PERSIST_MIN_INTERVAL_MS) { return { ok: true, event: evt.event, handled: true, reason: "throttled" }; } @@ -921,14 +1185,18 @@ export const handleNodeEvent = async ( try { // Node last-seen lives on the device record; node.pair.list projects // it from there, so one write covers both surfaces. - const deviceUpdated = await updatePairedDeviceMetadata(deviceId, { - lastSeenAtMs: now, - lastSeenReason, - }); + const deviceUpdated = await updatePairedDevicePresence( + deviceId, + { + lastSeenAtMs: now, + lastSeenReason, + }, + pairingGeneration, + ); if (!deviceUpdated) { - return { ok: true, event: evt.event, handled: false, reason: "unpaired" }; + return pairingChangedResult(evt.event); } - recentNodePresencePersistAt.set(deviceId, now); + recentNodePresencePersistAt.set(presenceOwnerKey, now); pruneBoundedTimestampMap(recentNodePresencePersistAt, { now, ttlMs: NODE_PRESENCE_PERSIST_MIN_INTERVAL_MS * 10, diff --git a/src/gateway/server-node-session-runtime.test.ts b/src/gateway/server-node-session-runtime.test.ts new file mode 100644 index 000000000000..773d4c3093fb --- /dev/null +++ b/src/gateway/server-node-session-runtime.test.ts @@ -0,0 +1,219 @@ +import { describe, expect, test, vi } from "vitest"; +import { + createSessionEventSubscriberRegistry, + createSessionMessageSubscriberRegistry, +} from "./server-chat-state.js"; +import { createGatewayNodeSessionRuntime } from "./server-node-session-runtime.js"; +import type { GatewayWsClient } from "./server/ws-types.js"; + +type TestSocket = { + bufferedAmount: number; + send: (payload: string) => void; + close: (code?: number, reason?: string) => void; +}; + +function makeGatewayWsClient(connId: string, socket: TestSocket): GatewayWsClient { + return { + socket: socket as unknown as GatewayWsClient["socket"], + connId, + usesSharedGatewayAuth: false, + connect: { + role: "node", + scopes: [], + client: { + id: "node-client", + version: "1.0.0", + platform: "macos", + mode: "node", + }, + device: { id: "node-a" }, + } as unknown as GatewayWsClient["connect"], + }; +} + +function createRuntime( + resolveCurrentPairingGeneration: () => Promise, + broadcast = vi.fn(), + isPairingStateCurrent: NonNullable< + Parameters[0]["isPairingStateCurrent"] + > = (_nodeId, expected) => + expected.identity === "identity-a" && expected.generation === "generation-a", +) { + return createGatewayNodeSessionRuntime({ + broadcast, + resolveCurrentPairingState: async () => ({ + identity: "identity-a", + generation: await resolveCurrentPairingGeneration(), + }), + isPairingStateCurrent, + sessionEventSubscribers: createSessionEventSubscriberRegistry(), + sessionMessageSubscribers: createSessionMessageSubscriberRegistry(), + }); +} + +function registerNode( + runtime: ReturnType, + connId: string, + pairingGeneration: string, + frames: string[], +) { + const socket: TestSocket = { + bufferedAmount: 0, + send: vi.fn((payload: string) => frames.push(payload)), + close: vi.fn(), + }; + runtime.nodeRegistry.register(makeGatewayWsClient(connId, socket), { + pairingIdentity: "identity-a", + pairingGeneration, + }); +} + +describe("gateway node session runtime", () => { + test("forwards subscribed payload json without parsing it again", async () => { + const frames: string[] = []; + const runtime = createRuntime(async () => "generation-a"); + registerNode(runtime, "conn-node-a", "generation-a", frames); + runtime.nodeSubscribe("node-a", "main", "conn-node-a"); + + const parseSpy = vi.spyOn(JSON, "parse"); + try { + runtime.nodeSendToSession("main", "chat", { ok: true }); + await vi.waitFor(() => expect(frames).toHaveLength(1)); + expect(parseSpy).not.toHaveBeenCalled(); + } finally { + parseSpy.mockRestore(); + } + expect(JSON.parse(frames[0] ?? "{}")).toEqual({ + type: "event", + event: "chat", + payload: { ok: true }, + }); + }); + + test("fences voice-wake updates by pairing generation while retaining operator broadcasts", async () => { + let currentPairingGeneration = "generation-a"; + const resolveCurrentPairingGeneration = vi.fn(async () => currentPairingGeneration); + const broadcast = vi.fn(); + const runtime = createRuntime( + resolveCurrentPairingGeneration, + broadcast, + (_nodeId, expected) => + expected.identity === "identity-a" && expected.generation === currentPairingGeneration, + ); + const frames: string[] = []; + registerNode(runtime, "conn-node-a", "generation-a", frames); + const send = vi.spyOn(runtime.nodeRegistry, "sendEventRawForPairingGeneration"); + const routing = { + version: 1 as const, + defaultTarget: { mode: "current" as const }, + routes: [], + updatedAtMs: 1, + }; + + runtime.broadcastVoiceWakeChanged(["openclaw"]); + runtime.broadcastVoiceWakeRoutingChanged(routing); + await vi.waitFor(() => expect(frames).toHaveLength(2)); + + currentPairingGeneration = "generation-b"; + runtime.broadcastVoiceWakeChanged(["retired"]); + runtime.broadcastVoiceWakeRoutingChanged({ ...routing, updatedAtMs: 2 }); + await vi.waitFor(() => expect(send).toHaveBeenCalledTimes(4)); + + expect(frames.map((frame) => JSON.parse(frame))).toEqual([ + { type: "event", event: "voicewake.changed", payload: { triggers: ["openclaw"] } }, + { type: "event", event: "voicewake.routing.changed", payload: { config: routing } }, + ]); + expect(broadcast).toHaveBeenCalledTimes(4); + }); + + test("fences generation-less voice-wake updates by authenticated pairing identity", async () => { + let pairingExists = true; + const broadcast = vi.fn(); + const runtime = createGatewayNodeSessionRuntime({ + broadcast, + resolveCurrentPairingState: async () => + pairingExists ? { identity: "identity-a" } : undefined, + isPairingStateCurrent: (_nodeId, expected) => + pairingExists && expected.identity === "identity-a", + sessionEventSubscribers: createSessionEventSubscriberRegistry(), + sessionMessageSubscribers: createSessionMessageSubscriberRegistry(), + }); + const frames: string[] = []; + const socket: TestSocket = { + bufferedAmount: 0, + send: vi.fn((payload: string) => frames.push(payload)), + close: vi.fn(), + }; + const client = makeGatewayWsClient("conn-node-a", socket); + runtime.nodeRegistry.register(client, { pairingIdentity: "identity-a" }); + const send = vi.spyOn(runtime.nodeRegistry, "sendEventRawForPairingGeneration"); + + runtime.broadcastVoiceWakeChanged(["openclaw"]); + await vi.waitFor(() => expect(frames).toHaveLength(1)); + pairingExists = false; + runtime.broadcastVoiceWakeChanged(["retired"]); + await vi.waitFor(() => expect(client.invalidated).toBe(true)); + + expect(send).not.toHaveBeenCalled(); + expect(frames.map((frame) => JSON.parse(frame))).toEqual([ + { type: "event", event: "voicewake.changed", payload: { triggers: ["openclaw"] } }, + ]); + expect(client.invalidated).toBe(true); + expect(broadcast).toHaveBeenCalledTimes(2); + }); + + test("does not inherit subscriptions across a replacement pairing generation", async () => { + let currentPairingGeneration = "generation-a"; + const runtime = createRuntime(async () => currentPairingGeneration); + + const originalFrames: string[] = []; + registerNode(runtime, "conn-original", "generation-a", originalFrames); + runtime.nodeSubscribe("node-a", "main", "conn-original"); + runtime.nodeSendToSession("main", "chat", { seq: 1 }); + await vi.waitFor(() => expect(originalFrames).toHaveLength(1)); + + currentPairingGeneration = "generation-b"; + runtime.nodeSendToSession("main", "chat", { seq: 2 }); + await vi.waitFor(() => expect(runtime.nodeRegistry.get("node-a")).toBeUndefined()); + expect(originalFrames).toHaveLength(1); + + const replacementFrames: string[] = []; + registerNode(runtime, "conn-replacement", "generation-b", replacementFrames); + runtime.nodeSubscribe("node-a", "retired", "conn-original"); + runtime.nodeSendToSession("retired", "chat", { seq: 3 }); + expect(replacementFrames).toHaveLength(0); + + runtime.nodeSubscribe("node-a", "main", "conn-replacement"); + runtime.nodeSendToSession("main", "chat", { seq: 4 }); + await vi.waitFor(() => expect(replacementFrames).toHaveLength(1)); + + const reconnectFrames: string[] = []; + registerNode(runtime, "conn-reconnect", "generation-b", reconnectFrames); + runtime.nodeSendToSession("main", "chat", { seq: 5 }); + await vi.waitFor(() => expect(reconnectFrames).toHaveLength(1)); + }); + + test("preserves subscriptions for an exact live pairing generation promotion", async () => { + let currentPairingGeneration = "generation-a"; + const runtime = createRuntime(async () => currentPairingGeneration); + const frames: string[] = []; + registerNode(runtime, "conn-node-a", "generation-a", frames); + runtime.nodeSubscribe("node-a", "main", "conn-node-a"); + + currentPairingGeneration = "generation-b"; + expect( + runtime.nodeRegistry.updateSurface( + "node-a", + { commands: [] }, + { + expectedConnId: "conn-node-a", + expectedPairingIdentity: "identity-a", + expectedPairingGeneration: "generation-a", + nextPairingGeneration: "generation-b", + }, + ), + ).not.toBeNull(); + runtime.nodeSendToSession("main", "chat", { ok: true }); + await vi.waitFor(() => expect(frames).toHaveLength(1)); + }); +}); diff --git a/src/gateway/server-node-session-runtime.ts b/src/gateway/server-node-session-runtime.ts index 919cbe1ccbf6..a9c705337fff 100644 --- a/src/gateway/server-node-session-runtime.ts +++ b/src/gateway/server-node-session-runtime.ts @@ -1,7 +1,13 @@ +import { + isNodePairingBindingCurrent, + resolveCurrentNodePairingBinding, +} from "../infra/node-pairing-state.js"; +import type { VoiceWakeRoutingConfig } from "../infra/voicewake-routing.js"; // Gateway node session runtime factory. // Creates node registry, subscription, and voice-wake fanout state. import { NodeRegistry, + serializeEventPayload, type NodeRegistryOptions, type SerializedEventPayload, } from "./node-registry.js"; @@ -20,33 +26,102 @@ export function createGatewayNodeSessionRuntime(params: { listRegisteredNodePluginToolCommands?: NodeRegistryOptions["listRegisteredNodePluginToolCommands"]; nodePluginToolsEnabled?: boolean; nodeSkillsEnabled?: boolean; + resolveCurrentPairingState?: NodeRegistryOptions["resolveCurrentPairingState"]; + isPairingStateCurrent?: NodeRegistryOptions["isPairingStateCurrent"]; + onPairingInvalidated?: NodeRegistryOptions["onPairingInvalidated"]; sessionEventSubscribers: SessionEventSubscriberRegistry; sessionMessageSubscribers: SessionMessageSubscriberRegistry; }) { + const nodeSubscriptions = createNodeSubscriptionManager(); const nodeRegistry = new NodeRegistry({ listRegisteredNodePluginToolCommands: params.listRegisteredNodePluginToolCommands, nodePluginToolsEnabled: params.nodePluginToolsEnabled, nodeSkillsEnabled: params.nodeSkillsEnabled, + resolveCurrentPairingState: + params.resolveCurrentPairingState ?? resolveCurrentNodePairingBinding, + isPairingStateCurrent: params.isPairingStateCurrent ?? isNodePairingBindingCurrent, + onPairingInvalidated: params.onPairingInvalidated, + onPairingGenerationChanged: (change) => { + nodeSubscriptions.updatePairingGeneration({ + ...change, + preserveSubscriptions: change.preserveSessionState, + }); + }, }); const nodePresenceTimers = new Map>(); - const nodeSubscriptions = createNodeSubscriptionManager(); const sessionEventSubscribers = params.sessionEventSubscribers; const sessionMessageSubscribers = params.sessionMessageSubscribers; const nodeSendEvent = (opts: { nodeId: string; + pairingGeneration: string; event: string; payloadJSON?: SerializedEventPayload | null; }) => { - nodeRegistry.sendEventRaw(opts.nodeId, opts.event, opts.payloadJSON ?? null); + return nodeRegistry.sendEventRawForPairingGeneration( + opts.nodeId, + opts.pairingGeneration, + opts.event, + opts.payloadJSON ?? null, + ); }; // Session fanout goes through the subscription manager so node reconnects and // explicit unsubscribes keep both node->session indexes in sync. - const nodeSendToSession = (sessionKey: string, event: string, payload: unknown) => - nodeSubscriptions.sendToSession(sessionKey, event, payload, nodeSendEvent); - const nodeSendToAllSubscribed = (event: string, payload: unknown) => - nodeSubscriptions.sendToAllSubscribed(event, payload, nodeSendEvent); + const nodeSendToSession = (sessionKey: string, event: string, payload: unknown) => { + void nodeSubscriptions.sendToSession(sessionKey, event, payload, nodeSendEvent); + }; + const nodeSendToAllSubscribed = (event: string, payload: unknown) => { + void nodeSubscriptions.sendToAllSubscribed(event, payload, nodeSendEvent); + }; + const resolveSubscriptionGeneration = (nodeId: string, connId?: string) => { + const node = nodeRegistry.get(nodeId); + return connId && node?.connId === connId ? node.pairingGeneration : undefined; + }; + const nodeSubscribe = (nodeId: string, sessionKey: string, connId?: string) => { + const pairingGeneration = resolveSubscriptionGeneration(nodeId, connId); + if (pairingGeneration) { + nodeSubscriptions.subscribe(nodeId, pairingGeneration, sessionKey); + } + }; + const nodeUnsubscribe = (nodeId: string, sessionKey: string, connId?: string) => { + const pairingGeneration = resolveSubscriptionGeneration(nodeId, connId); + if (pairingGeneration) { + nodeSubscriptions.unsubscribe(nodeId, pairingGeneration, sessionKey); + } + }; + const sendVoiceWakeEventToCurrentNodes = (event: string, payload: unknown) => { + const payloadJSON = serializeEventPayload(payload); + for (const node of nodeRegistry.listConnected()) { + const pairingGeneration = node.pairingGeneration; + if (!pairingGeneration) { + // Pending first-surface sessions have no command authority yet, but + // their authenticated pairing identity still fences compatibility broadcasts. + if (node.pairingIdentity) { + void nodeRegistry + .sendEventForPairingIdentity({ + nodeId: node.nodeId, + connId: node.connId, + pairingIdentity: node.pairingIdentity, + event, + payload, + }) + .catch(() => undefined); + } + continue; + } + // Voice-wake broadcasts are fire-and-forget, but each node send still + // resolves persistent generation before crossing its transport. + void nodeRegistry + .sendEventRawForPairingGeneration(node.nodeId, pairingGeneration, event, payloadJSON) + .catch(() => undefined); + } + }; const broadcastVoiceWakeChanged = (triggers: string[]) => { params.broadcast("voicewake.changed", { triggers }, { dropIfSlow: true }); + sendVoiceWakeEventToCurrentNodes("voicewake.changed", { triggers }); + }; + const broadcastVoiceWakeRoutingChanged = (config: VoiceWakeRoutingConfig) => { + params.broadcast("voicewake.routing.changed", { config }, { dropIfSlow: true }); + sendVoiceWakeEventToCurrentNodes("voicewake.routing.changed", { config }); }; const hasTalkNodeConnected = () => hasConnectedTalkNode(nodeRegistry); @@ -57,10 +132,11 @@ export function createGatewayNodeSessionRuntime(params: { sessionMessageSubscribers, nodeSendToSession, nodeSendToAllSubscribed, - nodeSubscribe: nodeSubscriptions.subscribe, - nodeUnsubscribe: nodeSubscriptions.unsubscribe, + nodeSubscribe, + nodeUnsubscribe, nodeUnsubscribeAll: nodeSubscriptions.unsubscribeAll, broadcastVoiceWakeChanged, + broadcastVoiceWakeRoutingChanged, hasTalkNodeConnected, }; } diff --git a/src/gateway/server-node-subscriptions.test.ts b/src/gateway/server-node-subscriptions.test.ts new file mode 100644 index 000000000000..2578d443e315 --- /dev/null +++ b/src/gateway/server-node-subscriptions.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, test, vi } from "vitest"; +import type { SerializedEventPayload } from "./node-registry.js"; +import { createNodeSubscriptionManager } from "./server-node-subscriptions.js"; + +describe("node subscription manager", () => { + test("routes events with each subscribed node pairing generation", async () => { + const manager = createNodeSubscriptionManager(); + const sent: Array<{ + nodeId: string; + pairingGeneration: string; + event: string; + payloadJSON?: SerializedEventPayload | null; + }> = []; + + manager.subscribe("node-a", "generation-a", "main"); + manager.subscribe("node-b", "generation-b", "main"); + await manager.sendToSession("main", "chat", { ok: true }, (event) => { + sent.push(event); + }); + + expect(sent).toHaveLength(2); + expect(sent.map((event) => event.nodeId).toSorted()).toEqual(["node-a", "node-b"]); + expect(sent.map((event) => event.pairingGeneration).toSorted()).toEqual([ + "generation-a", + "generation-b", + ]); + }); + + test("unsubscribeAll clears both subscription indexes", async () => { + const manager = createNodeSubscriptionManager(); + const sent: string[] = []; + const sendEvent = (event: { nodeId: string; event: string }) => { + sent.push(`${event.nodeId}:${event.event}`); + }; + + manager.subscribe("node-a", "generation-a", "main"); + manager.subscribe("node-a", "generation-a", "secondary"); + manager.unsubscribeAll("node-a"); + await manager.sendToSession("main", "tick", {}, sendEvent); + await manager.sendToSession("secondary", "tick", {}, sendEvent); + + expect(sent).toStrictEqual([]); + }); + + test("settles sender failures without rejecting fire-and-forget fanout", async () => { + const manager = createNodeSubscriptionManager(); + const sent: string[] = []; + + manager.subscribe("node-a", "generation-a", "main"); + manager.subscribe("node-b", "generation-b", "main"); + await expect( + manager.sendToSession("main", "tick", {}, ({ nodeId }) => { + if (nodeId === "node-a") { + throw new Error("transport failed"); + } + sent.push(nodeId); + }), + ).resolves.toBeUndefined(); + + expect(sent).toStrictEqual(["node-b"]); + }); + + test("drops unserializable payloads without rejecting fanout", async () => { + const manager = createNodeSubscriptionManager(); + const sendEvent = vi.fn(); + + manager.subscribe("node-a", "generation-a", "main"); + await expect( + manager.sendToSession("main", "tick", { invalid: 1n }, sendEvent), + ).resolves.toBeUndefined(); + + expect(sendEvent).not.toHaveBeenCalled(); + }); +}); diff --git a/src/gateway/server-node-subscriptions.ts b/src/gateway/server-node-subscriptions.ts index 71534d4548a4..f8864fb4ccf4 100644 --- a/src/gateway/server-node-subscriptions.ts +++ b/src/gateway/server-node-subscriptions.ts @@ -6,107 +6,175 @@ import { serializeEventPayload, type SerializedEventPayload } from "./node-regis // events can fan out by session and all node cleanup paths remove reverse links. type NodeSendEventFn = (opts: { nodeId: string; + pairingGeneration: string; event: string; payloadJSON?: SerializedEventPayload | null; -}) => void; +}) => void | Promise; -type NodeListConnectedFn = () => Array<{ nodeId: string }>; +type NodeListConnectedFn = () => Array<{ nodeId: string; pairingGeneration?: string }>; type NodeSubscriptionManager = { - subscribe: (nodeId: string, sessionKey: string) => void; - unsubscribe: (nodeId: string, sessionKey: string) => void; - unsubscribeAll: (nodeId: string) => void; + subscribe: (nodeId: string, pairingGeneration: string, sessionKey: string) => void; + unsubscribe: (nodeId: string, pairingGeneration: string, sessionKey: string) => void; + unsubscribeAll: (nodeId: string, pairingGeneration?: string) => void; + updatePairingGeneration: (params: { + nodeId: string; + previousPairingGeneration: string; + nextPairingGeneration: string; + preserveSubscriptions: boolean; + }) => void; sendToSession: ( sessionKey: string, event: string, payload: unknown, sendEvent?: NodeSendEventFn | null, - ) => void; + ) => Promise; sendToAllSubscribed: ( event: string, payload: unknown, sendEvent?: NodeSendEventFn | null, - ) => void; + ) => Promise; sendToAllConnected: ( event: string, payload: unknown, listConnected?: NodeListConnectedFn | null, sendEvent?: NodeSendEventFn | null, - ) => void; + ) => Promise; clear: () => void; }; /** Manages node subscriptions to gateway session events. */ export function createNodeSubscriptionManager(): NodeSubscriptionManager { - const nodeSubscriptions = new Map>(); - const sessionSubscribers = new Map>(); + const nodeSubscriptions = new Map< + string, + { pairingGeneration: string; sessionKeys: Set } + >(); + const sessionSubscribers = new Map>(); - const toPayloadJSON = (payload: unknown) => serializeEventPayload(payload); - - const subscribe = (nodeId: string, sessionKey: string) => { - const normalizedNodeId = nodeId.trim(); - const normalizedSessionKey = sessionKey.trim(); - if (!normalizedNodeId || !normalizedSessionKey) { - return; + const toPayloadJSON = (payload: unknown): SerializedEventPayload | null | undefined => { + try { + return serializeEventPayload(payload); + } catch { + return undefined; } - - let nodeSet = nodeSubscriptions.get(normalizedNodeId); - if (!nodeSet) { - nodeSet = new Set(); - nodeSubscriptions.set(normalizedNodeId, nodeSet); - } - if (nodeSet.has(normalizedSessionKey)) { - return; - } - nodeSet.add(normalizedSessionKey); - - let sessionSet = sessionSubscribers.get(normalizedSessionKey); - if (!sessionSet) { - sessionSet = new Set(); - sessionSubscribers.set(normalizedSessionKey, sessionSet); - } - sessionSet.add(normalizedNodeId); }; - const unsubscribe = (nodeId: string, sessionKey: string) => { + const settleFanout = async (sends: Array<() => void | Promise>): Promise => { + // Public gateway callers intentionally fire-and-forget fanout. Settle every + // sender so one transport failure cannot become an unhandled rejection. + await Promise.allSettled(sends.map((send) => Promise.resolve().then(send))); + }; + + const subscribe = (nodeId: string, pairingGeneration: string, sessionKey: string) => { const normalizedNodeId = nodeId.trim(); + const normalizedPairingGeneration = pairingGeneration.trim(); const normalizedSessionKey = sessionKey.trim(); - if (!normalizedNodeId || !normalizedSessionKey) { + if (!normalizedNodeId || !normalizedPairingGeneration || !normalizedSessionKey) { return; } - const nodeSet = nodeSubscriptions.get(normalizedNodeId); - nodeSet?.delete(normalizedSessionKey); - if (nodeSet?.size === 0) { + let nodeEntry = nodeSubscriptions.get(normalizedNodeId); + if (nodeEntry?.pairingGeneration !== normalizedPairingGeneration) { + unsubscribeAll(normalizedNodeId); + nodeEntry = undefined; + } + if (!nodeEntry) { + nodeEntry = { + pairingGeneration: normalizedPairingGeneration, + sessionKeys: new Set(), + }; + nodeSubscriptions.set(normalizedNodeId, nodeEntry); + } + if (nodeEntry.sessionKeys.has(normalizedSessionKey)) { + return; + } + nodeEntry.sessionKeys.add(normalizedSessionKey); + + let sessionMap = sessionSubscribers.get(normalizedSessionKey); + if (!sessionMap) { + sessionMap = new Map(); + sessionSubscribers.set(normalizedSessionKey, sessionMap); + } + sessionMap.set(normalizedNodeId, normalizedPairingGeneration); + }; + + const unsubscribe = (nodeId: string, pairingGeneration: string, sessionKey: string) => { + const normalizedNodeId = nodeId.trim(); + const normalizedPairingGeneration = pairingGeneration.trim(); + const normalizedSessionKey = sessionKey.trim(); + if (!normalizedNodeId || !normalizedPairingGeneration || !normalizedSessionKey) { + return; + } + + const nodeEntry = nodeSubscriptions.get(normalizedNodeId); + if (nodeEntry?.pairingGeneration !== normalizedPairingGeneration) { + return; + } + nodeEntry.sessionKeys.delete(normalizedSessionKey); + if (nodeEntry.sessionKeys.size === 0) { nodeSubscriptions.delete(normalizedNodeId); } - const sessionSet = sessionSubscribers.get(normalizedSessionKey); - sessionSet?.delete(normalizedNodeId); - if (sessionSet?.size === 0) { + const sessionMap = sessionSubscribers.get(normalizedSessionKey); + if (sessionMap?.get(normalizedNodeId) === normalizedPairingGeneration) { + sessionMap.delete(normalizedNodeId); + } + if (sessionMap?.size === 0) { sessionSubscribers.delete(normalizedSessionKey); } }; - const unsubscribeAll = (nodeId: string) => { + function unsubscribeAll(nodeId: string, pairingGeneration?: string) { const normalizedNodeId = nodeId.trim(); - const nodeSet = nodeSubscriptions.get(normalizedNodeId); - if (!nodeSet) { + const nodeEntry = nodeSubscriptions.get(normalizedNodeId); + if ( + !nodeEntry || + (pairingGeneration !== undefined && nodeEntry.pairingGeneration !== pairingGeneration.trim()) + ) { return; } // Remove reverse session indexes before deleting the node index so session // fanout cannot retain disconnected node ids. - for (const sessionKey of nodeSet) { - const sessionSet = sessionSubscribers.get(sessionKey); - sessionSet?.delete(normalizedNodeId); - if (sessionSet?.size === 0) { + for (const sessionKey of nodeEntry.sessionKeys) { + const sessionMap = sessionSubscribers.get(sessionKey); + if (sessionMap?.get(normalizedNodeId) === nodeEntry.pairingGeneration) { + sessionMap.delete(normalizedNodeId); + } + if (sessionMap?.size === 0) { sessionSubscribers.delete(sessionKey); } } nodeSubscriptions.delete(normalizedNodeId); + } + + const updatePairingGeneration = (params: { + nodeId: string; + previousPairingGeneration: string; + nextPairingGeneration: string; + preserveSubscriptions: boolean; + }) => { + const normalizedNodeId = params.nodeId.trim(); + const previousPairingGeneration = params.previousPairingGeneration.trim(); + const nextPairingGeneration = params.nextPairingGeneration.trim(); + const nodeEntry = nodeSubscriptions.get(normalizedNodeId); + if ( + !nodeEntry || + !previousPairingGeneration || + nodeEntry.pairingGeneration !== previousPairingGeneration + ) { + return; + } + if (!params.preserveSubscriptions || !nextPairingGeneration) { + unsubscribeAll(normalizedNodeId, previousPairingGeneration); + return; + } + nodeEntry.pairingGeneration = nextPairingGeneration; + for (const sessionKey of nodeEntry.sessionKeys) { + sessionSubscribers.get(sessionKey)?.set(normalizedNodeId, nextPairingGeneration); + } }; - const sendToSession = ( + const sendToSession = async ( sessionKey: string, event: string, payload: unknown, @@ -116,20 +184,27 @@ export function createNodeSubscriptionManager(): NodeSubscriptionManager { if (!normalizedSessionKey || !sendEvent) { return; } - const subs = sessionSubscribers.get(normalizedSessionKey); - if (!subs || subs.size === 0) { + const subscribers = sessionSubscribers.get(normalizedSessionKey); + if (!subscribers || subscribers.size === 0) { return; } const payloadJSON = toPayloadJSON(payload); + if (payloadJSON === undefined) { + return; + } // Serialize once per event and reuse across all subscribed nodes to keep // fanout deterministic and avoid repeated JSON conversion. - for (const nodeId of subs) { - sendEvent({ nodeId, event, payloadJSON }); - } + await settleFanout( + [...subscribers].map( + ([nodeId, pairingGeneration]) => + () => + sendEvent({ nodeId, pairingGeneration, event, payloadJSON }), + ), + ); }; - const sendToAllSubscribed = ( + const sendToAllSubscribed = async ( event: string, payload: unknown, sendEvent?: NodeSendEventFn | null, @@ -138,12 +213,24 @@ export function createNodeSubscriptionManager(): NodeSubscriptionManager { return; } const payloadJSON = toPayloadJSON(payload); - for (const nodeId of nodeSubscriptions.keys()) { - sendEvent({ nodeId, event, payloadJSON }); + if (payloadJSON === undefined) { + return; } + await settleFanout( + [...nodeSubscriptions].map( + ([nodeId, subscription]) => + () => + sendEvent({ + nodeId, + pairingGeneration: subscription.pairingGeneration, + event, + payloadJSON, + }), + ), + ); }; - const sendToAllConnected = ( + const sendToAllConnected = async ( event: string, payload: unknown, listConnected?: NodeListConnectedFn | null, @@ -153,9 +240,22 @@ export function createNodeSubscriptionManager(): NodeSubscriptionManager { return; } const payloadJSON = toPayloadJSON(payload); - for (const node of listConnected()) { - sendEvent({ nodeId: node.nodeId, event, payloadJSON }); + if (payloadJSON === undefined) { + return; } + await settleFanout( + listConnected().map( + (node) => () => + node.pairingGeneration + ? sendEvent({ + nodeId: node.nodeId, + pairingGeneration: node.pairingGeneration, + event, + payloadJSON, + }) + : undefined, + ), + ); }; const clear = () => { @@ -167,6 +267,7 @@ export function createNodeSubscriptionManager(): NodeSubscriptionManager { subscribe, unsubscribe, unsubscribeAll, + updatePairingGeneration, sendToSession, sendToAllSubscribed, sendToAllConnected, diff --git a/src/gateway/server-request-context.test.ts b/src/gateway/server-request-context.test.ts index add170b9c01e..f36bd727adb3 100644 --- a/src/gateway/server-request-context.test.ts +++ b/src/gateway/server-request-context.test.ts @@ -53,10 +53,10 @@ function makeContextParams( nodeSubscribe: vi.fn(), nodeUnsubscribe: vi.fn(), nodeUnsubscribeAll: vi.fn(), - hasConnectedTalkNode: vi.fn(() => false), + hasConnectedTalkNode: vi.fn(async () => false), clients: new Set(), enforceSharedGatewayAuthGenerationForConfigWrite: vi.fn(), - nodeRegistry: {} as never, + nodeRegistry: { invalidateConnectionForPairingChange: vi.fn() } as never, agentRunSeq: new Map(), chatAbortControllers: new Map(), chatQueuedTurns: new Map(), @@ -297,9 +297,14 @@ describe("createGatewayRequestContext", () => { }; const clients = new Set([target, unrelated]) as never; const invalidateDeviceTransports = vi.fn(); + const invalidateConnectionForPairingChange = vi.fn(); const context = createGatewayRequestContext( - makeContextParams({ clients, invalidateDeviceTransports }), + makeContextParams({ + clients, + invalidateDeviceTransports, + nodeRegistry: { invalidateConnectionForPairingChange } as never, + }), ); context.invalidateClientsForDevice?.("device-1", { reason: "device-token-rotated" }); @@ -308,6 +313,10 @@ describe("createGatewayRequestContext", () => { "device-token-rotated", ); expect(target.socket.close).not.toHaveBeenCalled(); + expect(invalidateConnectionForPairingChange).toHaveBeenCalledWith( + "conn-target", + "device-token-rotated", + ); expect((unrelated as { invalidated?: boolean }).invalidated).toBeUndefined(); expect(unrelated.socket.close).not.toHaveBeenCalled(); diff --git a/src/gateway/server-request-context.ts b/src/gateway/server-request-context.ts index 5f6188868e4b..a82cac47a3e8 100644 --- a/src/gateway/server-request-context.ts +++ b/src/gateway/server-request-context.ts @@ -250,8 +250,11 @@ export function createGatewayRequestContext( if (opts?.role && gatewayClient.connect.role !== opts.role) { continue; } - // Marking is separate from socket close so already-buffered requests - // fail authorization even if transport teardown has not completed. + // Retire node-owned projections and pending invokes synchronously; socket + // close remains separate so already-buffered requests fail authorization. + if (gatewayClient.connId) { + params.nodeRegistry.invalidateConnectionForPairingChange(gatewayClient.connId, reason); + } gatewayClient.invalidated = true; gatewayClient.invalidatedReason = reason; } diff --git a/src/gateway/server-talk-nodes.test.ts b/src/gateway/server-talk-nodes.test.ts index ef3fa38add88..eec32b740faa 100644 --- a/src/gateway/server-talk-nodes.test.ts +++ b/src/gateway/server-talk-nodes.test.ts @@ -7,7 +7,7 @@ import { hasConnectedTalkNode } from "./server-talk-nodes.js"; function registryWith(nodes: Array>): NodeRegistry { return { - listConnected: () => + listCurrentConnected: async () => nodes.map((node, index) => ({ nodeId: `node-${index}`, connId: `conn-${index}`, @@ -22,16 +22,18 @@ function registryWith(nodes: Array>): NodeRegistry { } describe("hasConnectedTalkNode", () => { - it("uses explicit talk capability instead of platform names", () => { - expect( + it("uses explicit talk capability instead of platform names", async () => { + await expect( hasConnectedTalkNode(registryWith([{ platform: "android", caps: ["device"], commands: [] }])), - ).toBe(false); - expect(hasConnectedTalkNode(registryWith([{ platform: "linux", caps: ["talk"] }]))).toBe(true); + ).resolves.toBe(false); + await expect( + hasConnectedTalkNode(registryWith([{ platform: "linux", caps: ["talk"] }])), + ).resolves.toBe(true); }); - it("accepts nodes that declare talk command support", () => { - expect( + it("accepts nodes that declare talk command support", async () => { + await expect( hasConnectedTalkNode(registryWith([{ platform: "custom", commands: ["talk.ptt.start"] }])), - ).toBe(true); + ).resolves.toBe(true); }); }); diff --git a/src/gateway/server-talk-nodes.ts b/src/gateway/server-talk-nodes.ts index 44091bc15f94..e1693a729d4b 100644 --- a/src/gateway/server-talk-nodes.ts +++ b/src/gateway/server-talk-nodes.ts @@ -9,8 +9,8 @@ const TALK_CAPABILITY = "talk"; const TALK_COMMAND_PREFIX = "talk."; /** Returns true when any connected node can handle talk routing. */ -export function hasConnectedTalkNode(registry: NodeRegistry): boolean { - return registry.listConnected().some(isTalkCapableNode); +export async function hasConnectedTalkNode(registry: NodeRegistry): Promise { + return (await registry.listCurrentConnected()).some(isTalkCapableNode); } function isTalkCapableNode(node: NodeSession): boolean { diff --git a/src/gateway/server.impl.ts b/src/gateway/server.impl.ts index 3eaa6e377437..a6a5dd94da1f 100644 --- a/src/gateway/server.impl.ts +++ b/src/gateway/server.impl.ts @@ -59,7 +59,6 @@ import { } from "../infra/restart.js"; import { enqueueSystemEvent } from "../infra/system-events.js"; import { upsertPresence } from "../infra/system-presence.js"; -import type { VoiceWakeRoutingConfig } from "../infra/voicewake-routing.js"; import { withDiagnosticPhase } from "../logging/diagnostic-phase.js"; import { startDiagnosticHeartbeat, stopDiagnosticHeartbeat } from "../logging/diagnostic.js"; import { createSubsystemLogger, runtimeForLogger } from "../logging/subsystem.js"; @@ -80,7 +79,11 @@ import { } from "../secrets/runtime-state.js"; import { createLazyRuntimeModule } from "../shared/lazy-runtime.js"; import { createLazyPromise } from "../shared/lazy-runtime.js"; -import { recordRemoteNodeInfo, removeRemoteNodeInfo } from "../skills/runtime/remote.js"; +import { + recordRemoteNodeInfo, + removeRemoteNodeInfo, + removeRemoteNodeInfoForConnection, +} from "../skills/runtime/remote.js"; import { createAuthRateLimiter, type AuthRateLimiter } from "./auth-rate-limit.js"; import { resolveGatewayAuth } from "./auth.js"; import type { RestartRecoveryCandidate } from "./chat-abort.js"; @@ -102,6 +105,7 @@ import { import { isLoopbackHost } from "./net.js"; import { disposeNodeConnectionNotifications } from "./node-connection-notifications.js"; import { createNodeReapprovalCoordinator } from "./node-reapproval-coordinator.js"; +import { clearNodeWakeState } from "./node-wake-state.js"; import { mergeActivationSectionsIntoRuntimeConfig, resolveGatewayReloadPluginActivationCandidate, @@ -128,7 +132,6 @@ import type { GatewayInstanceRuntime } from "./server-instance-runtime.types.js" import { applyGatewayLaneConcurrency, resolveGatewayLaneConcurrency } from "./server-lanes.js"; import { createGatewayServerLiveState, type GatewayServerLiveState } from "./server-live-state.js"; import { GATEWAY_EVENTS } from "./server-methods-list.js"; -import { clearNodeWakeState } from "./server-methods/nodes-wake-state.js"; import type { GatewayRequestContext, GatewayRequestHandlers } from "./server-methods/types.js"; import { setFallbackGatewayContextResolver } from "./server-plugins.js"; import type { GatewayPluginReloadResult } from "./server-reload-handlers.js"; @@ -1311,6 +1314,7 @@ export async function startGatewayServer( nodeUnsubscribe, nodeUnsubscribeAll, broadcastVoiceWakeChanged, + broadcastVoiceWakeRoutingChanged, hasTalkNodeConnected, } = createGatewayNodeSessionRuntime({ broadcast, @@ -1319,6 +1323,11 @@ export async function startGatewayServer( listRegisteredNodePluginToolCommands: () => pluginRegistry.nodeHostCommands, nodePluginToolsEnabled: cfgAtStart.gateway?.nodes?.pluginTools?.enabled !== false, nodeSkillsEnabled: cfgAtStart.gateway?.nodes?.skills?.enabled !== false, + onPairingInvalidated: ({ nodeId, connId }) => { + upsertPresence(nodeId, { reason: "disconnect" }); + broadcastPresenceSnapshot({ broadcast, incrementPresenceVersion, getHealthVersion }); + removeRemoteNodeInfoForConnection(nodeId, connId); + }, }); const { createWatchNodeHttpRuntime } = await import("./watch-node-http.js"); const watchNodeHttpRuntime = createWatchNodeHttpRuntime({ @@ -1351,6 +1360,7 @@ export async function startGatewayServer( deviceFamily: session.deviceFamily, commands: session.commands, remoteIp: session.remoteIp, + pairingGeneration: session.pairingGeneration, }); }, onNodeDisconnected: (nodeId) => { @@ -1573,10 +1583,6 @@ export async function startGatewayServer( clearFallbackGatewayContextForServer(); } }; - const broadcastVoiceWakeRoutingChanged = (config: VoiceWakeRoutingConfig) => { - broadcast("voicewake.routing.changed", { config }, { dropIfSlow: true }); - }; - try { const earlyRuntime = await startupTrace.measure("runtime.early", () => loadGatewayStartupEarlyModule().then(({ startGatewayEarlyRuntime }) => diff --git a/src/gateway/server.roles-allowlist-update.test.ts b/src/gateway/server.roles-allowlist-update.test.ts index 00e47f75d4a4..c44862c8b014 100644 --- a/src/gateway/server.roles-allowlist-update.test.ts +++ b/src/gateway/server.roles-allowlist-update.test.ts @@ -502,7 +502,7 @@ describe("gateway node command allowlist", () => { path: path.join(os.tmpdir(), `openclaw-node-allowed-${Date.now()}-${Math.random()}.sqlite`), }); - systemClient = await connectNodeClientWithPairing({ + systemClient = await connectNodeClientWithNodePairing({ port, commands: ["system.run"], instanceId: "node-system-run", @@ -521,7 +521,7 @@ describe("gateway node command allowlist", () => { await systemClient.stopAndWait(); await waitForConnectedCount(0); - emptyClient = await connectNodeClientWithPairing({ + emptyClient = await connectNodeClientWithNodePairing({ port, commands: [], instanceId: "node-empty", @@ -611,7 +611,9 @@ describe("gateway node command allowlist", () => { const nodeId = await findConnectedNodeIdByDisplayName(displayName); await expectPendingPairingCommands(nodeId, ["canvas.snapshot", "system.run"]); - await expectCanvasSnapshotDenied(nodeId, "pending-node-canvas"); + const denied = await invokeCanvasSnapshot(nodeId, "pending-node-canvas"); + expect(denied.ok).toBe(false); + expect(denied.error?.details).toMatchObject({ code: "PAIRING_CHANGED" }); } finally { await nodeClient?.stopAndWait(); } diff --git a/src/gateway/server/ws-connection.ts b/src/gateway/server/ws-connection.ts index 15a105a58e74..f5feb236d831 100644 --- a/src/gateway/server/ws-connection.ts +++ b/src/gateway/server/ws-connection.ts @@ -19,13 +19,13 @@ import { resolveHostedPluginSurfaceUrl } from "../hosted-plugin-surface-url.js"; import type { GatewayMethodRegistry } from "../methods/registry.js"; import { isLoopbackAddress } from "../net.js"; import type { NodeReapprovalCoordinator } from "../node-reapproval-coordinator.js"; +import { clearNodeWakeState } from "../node-wake-state.js"; import type { PluginNodeCapabilitySurface } from "../plugin-node-capability.js"; import { MAX_BUFFERED_BYTES, MAX_PAYLOAD_BYTES, MAX_PREAUTH_PAYLOAD_BYTES, } from "../server-constants.js"; -import { clearNodeWakeState } from "../server-methods/nodes-wake-state.js"; import type { GatewayRequestContext, GatewayRequestHandlers } from "../server-methods/types.js"; import { formatError } from "../server-utils.js"; import { formatForLog, logWs } from "../ws-log.js"; diff --git a/src/gateway/server/ws-connection/connect-hello.ts b/src/gateway/server/ws-connection/connect-hello.ts index efa1c6755cca..bd4058cc644a 100644 --- a/src/gateway/server/ws-connection/connect-hello.ts +++ b/src/gateway/server/ws-connection/connect-hello.ts @@ -189,20 +189,29 @@ export async function sendGatewayHello( const requestContext = buildRequestContext(); const nodeId = connectParams.device?.id ?? connectParams.client.id; const nodeSession = requestContext.nodeRegistry.get(nodeId); - // Claim by the authenticated node id only. A replacement may register while - // persistence waits; the router transfers the pending alert by node identity. - if (nodeSession?.connId === connId) { + const pairingGeneration = nodeSession?.pairingGeneration; + if (nodeSession?.connId === connId && pairingGeneration) { try { const connection = await recordPairedNodeConnection( nodeSession.nodeId, nodeSession.connectedAtMs, + undefined, + { nodeId: nodeSession.nodeId, key: pairingGeneration }, ); if (!connection.recorded) { logGateway.warn(`failed to record last connect for ${nodeSession.nodeId}: not paired`); } else { - scheduleNodeConnectionNotification(requestContext.nodeRegistry, nodeSession, { - isFirstConnection: connection.firstConnection, - }); + const currentSession = requestContext.nodeRegistry.getForPairingGeneration( + nodeSession.nodeId, + pairingGeneration, + ); + // A rapid same-generation reconnect may take over the durable + // first-connection claim; generation lookup excludes stale replacements. + if (currentSession) { + scheduleNodeConnectionNotification(requestContext.nodeRegistry, currentSession, { + isFirstConnection: connection.firstConnection, + }); + } } } catch (err) { logGateway.warn( diff --git a/src/gateway/server/ws-connection/connect-session.ts b/src/gateway/server/ws-connection/connect-session.ts index 19b95536bb17..b7fa7713eeba 100644 --- a/src/gateway/server/ws-connection/connect-session.ts +++ b/src/gateway/server/ws-connection/connect-session.ts @@ -9,6 +9,11 @@ import { import { ConnectErrorDetailCodes } from "../../../../packages/gateway-protocol/src/connect-error-details.js"; import { ErrorCodes, PROTOCOL_VERSION } from "../../../../packages/gateway-protocol/src/index.js"; import { getRuntimeConfig } from "../../../config/io.js"; +import { + captureAuthenticatedNodePairingState, + type NodePairingGeneration, + type NodePairingIdentity, +} from "../../../infra/node-pairing-state.js"; import { upsertPresence } from "../../../infra/system-presence.js"; import { loadVoiceWakeRoutingConfig } from "../../../infra/voicewake-routing.js"; import { loadVoiceWakeConfig } from "../../../infra/voicewake.js"; @@ -22,6 +27,7 @@ import { import { resolveRuntimeServiceVersion } from "../../../version.js"; import { verifyAgentRuntimeIdentityToken } from "../../agent-runtime-identity-token.js"; import { APPROVALS_SCOPE } from "../../method-scopes.js"; +import { serializeEventPayload } from "../../node-registry.js"; import { isOperatorApprovalRuntimeToken } from "../../operator-approval-runtime-token.js"; import { buildPluginNodeCapabilityScopedHostUrl, @@ -47,6 +53,12 @@ import type { /** Match production release versions (YYYY.M.PATCH or YYYY.M.PATCH-beta.N). */ const RELEASED_VERSION_RE = /^\d{4}\.\d+\.\d+/; +type AuthenticatedNodePairingAdmission = { + authenticated: { nodeId: string; publicKey: string; token: string }; + identity: NodePairingIdentity; + generation?: NodePairingGeneration; +}; + function isReleasedVersion(version: string): boolean { return RELEASED_VERSION_RE.test(version); } @@ -108,6 +120,8 @@ export async function attachAuthenticatedGatewayConnect( role, scopes, device, + devicePublicKey, + deviceToken, authResult, authMethod, pairingLocality, @@ -118,6 +132,46 @@ export async function attachAuthenticatedGatewayConnect( return; } + let nodePairingAdmission: AuthenticatedNodePairingAdmission | undefined; + if (role === "node") { + const nodeId = device?.id ?? connectParams.client.id; + const authenticatedNodeToken = + authMethod === "device-token" + ? normalizeOptionalString(connectParams.auth?.deviceToken ?? connectParams.auth?.token) + : deviceToken?.token; + if (!device || !devicePublicKey || !authenticatedNodeToken) { + const message = "authenticated node pairing identity unavailable"; + markHandshakeFailure("node-pairing-generation-changed", {}); + sendHandshakeErrorResponse(ErrorCodes.NOT_PAIRED, message); + await releasePendingNodePairingCleanup(); + close(1008, truncateCloseReason(message)); + return; + } + const authenticatedNodePairing = { + nodeId, + publicKey: devicePublicKey, + token: authenticatedNodeToken, + }; + const admittedPairingState = + await captureAuthenticatedNodePairingState(authenticatedNodePairing); + if (!admittedPairingState) { + const message = "node pairing changed during connect"; + markHandshakeFailure( + "node-pairing-generation-changed", + device?.id ? { deviceId: device.id } : {}, + ); + sendHandshakeErrorResponse(ErrorCodes.NOT_PAIRED, message); + await releasePendingNodePairingCleanup(); + close(1008, truncateCloseReason(message)); + return; + } + nodePairingAdmission = { + authenticated: authenticatedNodePairing, + identity: admittedPairingState.identity, + ...(admittedPairingState.generation ? { generation: admittedPairingState.generation } : {}), + }; + } + // Presence lists user-visible clients/nodes. Ephemeral control-plane connections // (CLI, backend RPC probes, tests) churn for the full TTL and stay excluded. const shouldTrackPresence = !isEphemeralGatewayClient(connectParams.client); @@ -306,6 +360,27 @@ export async function attachAuthenticatedGatewayConnect( } } + const admittedNodePairing = role === "node" ? nodePairingAdmission : undefined; + if (admittedNodePairing) { + const currentPairingState = await captureAuthenticatedNodePairingState( + admittedNodePairing.authenticated, + ); + if ( + !currentPairingState || + currentPairingState.identity.key !== admittedNodePairing.identity.key || + currentPairingState.generation?.key !== admittedNodePairing.generation?.key + ) { + const message = "node pairing changed during connect"; + markHandshakeFailure("node-pairing-generation-changed", { + deviceId: admittedNodePairing.identity.nodeId, + }); + sendHandshakeErrorResponse(ErrorCodes.NOT_PAIRED, message); + await releasePendingNodePairingCleanup(); + close(1008, truncateCloseReason(message)); + return; + } + } + if (!setClient(nextClient)) { await releasePendingNodePairingCleanup(); setCloseCause("connect-aborted-before-register", { @@ -375,10 +450,13 @@ export async function attachAuthenticatedGatewayConnect( }); incrementPresenceVersion(); } - if (role === "node") { + if (admittedNodePairing) { + const pairingGeneration = admittedNodePairing.generation?.key; const requestContext = buildRequestContext(); const nodeSession = requestContext.nodeRegistry.register(nextClient, { remoteIp: reportedClientIp, + pairingIdentity: admittedNodePairing.identity.key, + ...(pairingGeneration ? { pairingGeneration } : {}), }); recordRemoteNodeInfo({ nodeId: nodeSession.nodeId, @@ -388,6 +466,7 @@ export async function attachAuthenticatedGatewayConnect( deviceFamily: nodeSession.deviceFamily, commands: nodeSession.commands, remoteIp: nodeSession.remoteIp, + pairingGeneration: nodeSession.pairingGeneration, }); runDetachedConnectWork( async () => { @@ -405,12 +484,28 @@ export async function attachAuthenticatedGatewayConnect( (err) => logGateway.warn(`remote bin probe failed for ${nodeSession.nodeId}: ${formatForLog(err)}`), ); + const sendConnectSnapshot = async (event: string, payload: unknown) => { + if (pairingGeneration) { + await requestContext.nodeRegistry.sendEventRawForPairingGeneration( + nodeSession.nodeId, + pairingGeneration, + event, + serializeEventPayload(payload), + ); + return; + } + await requestContext.nodeRegistry.sendEventForPairingIdentity({ + nodeId: nodeSession.nodeId, + connId: nodeSession.connId, + pairingIdentity: admittedNodePairing.identity.key, + event, + payload, + }); + }; runDetachedConnectWork( async () => { const cfg = await loadVoiceWakeConfig(); - requestContext.nodeRegistry.sendEvent(nodeSession.nodeId, "voicewake.changed", { - triggers: cfg.triggers, - }); + await sendConnectSnapshot("voicewake.changed", { triggers: cfg.triggers }); }, (err) => logGateway.warn( @@ -420,9 +515,7 @@ export async function attachAuthenticatedGatewayConnect( runDetachedConnectWork( async () => { const routing = await loadVoiceWakeRoutingConfig(); - requestContext.nodeRegistry.sendEvent(nodeSession.nodeId, "voicewake.routing.changed", { - config: routing, - }); + await sendConnectSnapshot("voicewake.routing.changed", { config: routing }); }, (err) => logGateway.warn( diff --git a/src/gateway/terminal/node-relay.test.ts b/src/gateway/terminal/node-relay.test.ts index b83b1a7f7c84..4d3525b5a58a 100644 --- a/src/gateway/terminal/node-relay.test.ts +++ b/src/gateway/terminal/node-relay.test.ts @@ -103,6 +103,7 @@ describe("createNodeRelayBackend", () => { registry, nodeId: "node-1", expectedConnId: "conn-authorized", + expectedPairingGeneration: "generation-authorized", command: "codex.terminal.resume.v1", params: {}, }); @@ -110,7 +111,11 @@ describe("createNodeRelayBackend", () => { backend.onExit(exit); expect(invoke).toHaveBeenCalledWith( - expect.objectContaining({ nodeId: "node-1", expectedConnId: "conn-authorized" }), + expect.objectContaining({ + nodeId: "node-1", + expectedConnId: "conn-authorized", + expectedPairingGeneration: "generation-authorized", + }), ); await vi.waitFor(() => expect(exit).toHaveBeenCalledWith({ diff --git a/src/gateway/terminal/node-relay.ts b/src/gateway/terminal/node-relay.ts index 6004159d93a4..235c6819c86f 100644 --- a/src/gateway/terminal/node-relay.ts +++ b/src/gateway/terminal/node-relay.ts @@ -65,6 +65,7 @@ export async function createNodeRelayBackend(params: { registry: NodeRegistry; nodeId: string; expectedConnId: string; + expectedPairingGeneration?: string; command: string; params: Record; }): Promise { @@ -82,6 +83,9 @@ export async function createNodeRelayBackend(params: { .invoke({ nodeId: params.nodeId, expectedConnId: params.expectedConnId, + ...(params.expectedPairingGeneration + ? { expectedPairingGeneration: params.expectedPairingGeneration } + : {}), command: params.command, params: params.params, timeoutMs: 0, diff --git a/src/gateway/test-helpers.server.ts b/src/gateway/test-helpers.server.ts index 38a6389e2ddc..12187a6d75a0 100644 --- a/src/gateway/test-helpers.server.ts +++ b/src/gateway/test-helpers.server.ts @@ -1239,7 +1239,7 @@ export async function rpcReq>( id: string; ok: boolean; payload?: T | null | undefined; - error?: { message?: string; code?: string }; + error?: { message?: string; code?: string; details?: unknown }; }>( ws, (o) => { diff --git a/src/gateway/watch-node-http.test.ts b/src/gateway/watch-node-http.test.ts index b733f353e029..13dae3095732 100644 --- a/src/gateway/watch-node-http.test.ts +++ b/src/gateway/watch-node-http.test.ts @@ -1,4 +1,4 @@ -import { createServer, type Server } from "node:http"; +import { createServer, request as httpRequest, type ClientRequest, type Server } from "node:http"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { @@ -13,7 +13,13 @@ import { publicKeyRawBase64UrlFromPem, signDevicePayload, } from "../infra/device-identity.js"; -import { revokeDeviceToken } from "../infra/device-pairing.js"; +import { + approveDevicePairing, + getPairedDevice, + requestDevicePairing, + resolveNodePairingState, + revokeDeviceToken, +} from "../infra/device-pairing.js"; import { listNodePairing } from "../infra/node-pairing.js"; import { NODE_PAIRING_SETUP_BOOTSTRAP_PROFILE } from "../shared/device-bootstrap-profile.js"; import { createTrackedTempDirs } from "../test-utils/tracked-temp-dirs.js"; @@ -102,7 +108,17 @@ async function startRuntime( config?: OpenClawConfig; }, ) { - const nodeRegistry = new NodeRegistry(); + const nodeRegistry = new NodeRegistry({ + resolveCurrentPairingState: async (nodeId) => { + const state = resolveNodePairingState(await getPairedDevice(nodeId, baseDir)); + return state + ? { + identity: state.identity.key, + ...(state.generation ? { generation: state.generation.key } : {}), + } + : undefined; + }, + }); const broadcasts: Array<{ event: string; payload: unknown }> = []; const connectedNodes: string[] = []; const disconnectedNodes: Array<{ nodeId: string; reason: string }> = []; @@ -164,6 +180,37 @@ async function readJson(response: Response): Promise> { return (await response.json()) as Record; } +function startPartialJsonRequest(params: { url: string; authorization: string }): { + request: ClientRequest; + response: Promise<{ statusCode: number; body: string }>; +} { + let request!: ClientRequest; + const response = new Promise<{ statusCode: number; body: string }>((resolve, reject) => { + request = httpRequest( + params.url, + { + method: "POST", + headers: { + authorization: params.authorization, + "content-type": "application/json", + }, + }, + (result) => { + const chunks: Buffer[] = []; + result.on("data", (chunk: Buffer) => chunks.push(chunk)); + result.once("end", () => { + resolve({ + statusCode: result.statusCode ?? 0, + body: Buffer.concat(chunks).toString("utf8"), + }); + }); + }, + ); + request.once("error", reject); + }); + return { request, response }; +} + async function waitForLastConnectedMetadata(baseDir: string, nodeId: string): Promise { await vi.waitFor(async () => { const paired = (await listNodePairing(baseDir)).paired.find((entry) => entry.nodeId === nodeId); @@ -359,6 +406,128 @@ describe("watch node HTTP transport", () => { expect(disconnectedNodes).toHaveLength(1); }); + it("rejects an HTTP node session after an external reapproval changes its generation", async () => { + const baseDir = await tempDirs.make("openclaw-watch-node-reapproval-"); + const identity = loadOrCreateDeviceIdentity({ + path: path.join(baseDir, "watch-identity.sqlite"), + }); + const issued = await issueDeviceBootstrapToken({ + baseDir, + profile: NODE_PAIRING_SETUP_BOOTSTRAP_PROFILE, + }); + const { nodeRegistry, disconnectedNodes, runtime, baseUrl } = await startRuntime(baseDir); + const challenge = await readJson(await fetch(`${baseUrl}/challenge`)); + const connectResponse = await fetch(`${baseUrl}/connect`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify( + makeConnectParams({ + identity, + nonce: String(challenge.nonce), + bootstrapToken: issued.token, + }), + ), + }); + const connected = await readJson(connectResponse); + const paired = await getPairedDevice(identity.deviceId, baseDir); + const repair = await requestDevicePairing( + { + deviceId: identity.deviceId, + publicKey: paired?.publicKey ?? "", + role: "node", + roles: ["node"], + scopes: [], + }, + baseDir, + ); + await approveDevicePairing(repair.request.requestId, { callerScopes: [] }, baseDir); + + const stalePoll = await fetch(`${baseUrl}/poll`, { + method: "POST", + headers: { authorization: `Bearer ${String(connected.sessionToken)}` }, + }); + expect(stalePoll.status).toBe(401); + expect(nodeRegistry.get(identity.deviceId)).toBeUndefined(); + expect(disconnectedNodes).toContainEqual({ + nodeId: identity.deviceId, + reason: "node pairing changed", + }); + runtime.close(); + }); + + it("rejects an invoke result when pairing changes during body upload", async () => { + const baseDir = await tempDirs.make("openclaw-watch-node-result-generation-"); + const identity = loadOrCreateDeviceIdentity({ + path: path.join(baseDir, "watch-identity.sqlite"), + }); + const issued = await issueDeviceBootstrapToken({ + baseDir, + profile: NODE_PAIRING_SETUP_BOOTSTRAP_PROFILE, + }); + const { nodeRegistry, disconnectedNodes, runtime, baseUrl } = await startRuntime(baseDir); + const challenge = await readJson(await fetch(`${baseUrl}/challenge`)); + const connectResponse = await fetch(`${baseUrl}/connect`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify( + makeConnectParams({ + identity, + nonce: String(challenge.nonce), + bootstrapToken: issued.token, + }), + ), + }); + const connected = await readJson(connectResponse); + const invoke = nodeRegistry.invoke({ + nodeId: identity.deviceId, + command: "device.info", + timeoutMs: 2_000, + }); + const invokeAfterDisconnect = invoke.catch((error: unknown) => error); + const pollResponse = await fetch(`${baseUrl}/poll`, { + method: "POST", + headers: { authorization: `Bearer ${String(connected.sessionToken)}` }, + }); + const polled = await readJson(pollResponse); + const event = polled.event as { payload: { id: string } }; + const currentCheck = vi.spyOn(nodeRegistry, "isConnectionCurrentPairingState"); + currentCheck.mockClear(); + const partial = startPartialJsonRequest({ + url: `${baseUrl}/result`, + authorization: `Bearer ${String(connected.sessionToken)}`, + }); + partial.request.write(`{"id":${JSON.stringify(event.payload.id)},"ok":`); + await vi.waitFor(() => expect(currentCheck).toHaveBeenCalledTimes(1)); + + const paired = await getPairedDevice(identity.deviceId, baseDir); + const repair = await requestDevicePairing( + { + deviceId: identity.deviceId, + publicKey: paired?.publicKey ?? "", + role: "node", + roles: ["node"], + scopes: [], + }, + baseDir, + ); + await approveDevicePairing(repair.request.requestId, { callerScopes: [] }, baseDir); + partial.request.end(`true,"payloadJSON":"{\\"model\\":\\"stale\\"}"}`); + + const resultResponse = await partial.response; + expect(resultResponse.statusCode).toBe(401); + expect(JSON.parse(resultResponse.body)).toMatchObject({ + error: { type: "unauthorized" }, + }); + expect(currentCheck).toHaveBeenCalledTimes(2); + expect(nodeRegistry.get(identity.deviceId)).toBeUndefined(); + expect(disconnectedNodes).toContainEqual({ + nodeId: identity.deviceId, + reason: "node pairing changed", + }); + await expect(invokeAfterDisconnect).resolves.toBeInstanceOf(Error); + runtime.close(); + }); + it("rejects empty shadow credentials without consuming the challenge", async () => { const baseDir = await tempDirs.make("openclaw-watch-node-auth-fields-"); const identity = loadOrCreateDeviceIdentity({ diff --git a/src/gateway/watch-node-http.ts b/src/gateway/watch-node-http.ts index ed274924f8a1..dd6017e47b60 100644 --- a/src/gateway/watch-node-http.ts +++ b/src/gateway/watch-node-http.ts @@ -31,6 +31,7 @@ import { requestDevicePairing, verifyDeviceToken, } from "../infra/device-pairing.js"; +import { captureAuthenticatedNodePairingState } from "../infra/node-pairing-state.js"; import { approveNodePairing, beginNodePairingConnect, @@ -439,7 +440,10 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions) }, }); - const getSession = (req: IncomingMessage, res: ServerResponse): WatchNodeSession | null => { + const getSession = async ( + req: IncomingMessage, + res: ServerResponse, + ): Promise => { const token = readBearerToken(req); const session = token ? sessionsByToken.get(token) : undefined; if (!session) { @@ -451,6 +455,11 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions) sendUnauthorized(res); return null; } + if (!(await options.nodeRegistry.isConnectionCurrentPairingState(session.connId))) { + closeSession(session, "node pairing changed"); + sendUnauthorized(res); + return null; + } touchSession(session); return session; }; @@ -801,6 +810,17 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions) sendUnauthorized(res); return; } + const nodePairingState = await captureAuthenticatedNodePairingState({ + nodeId: derivedDeviceId, + publicKey, + token: issuedDeviceToken, + baseDir: options.pairingBaseDir, + }); + const nodePairingGeneration = nodePairingState?.generation; + if (!nodePairingState || !nodePairingGeneration) { + sendUnauthorized(res); + return; + } if (closed || responseLifecycle.isAborted()) { if (revokedBootstrapTokenRecord) { await restoreDeviceBootstrapToken({ @@ -846,7 +866,11 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions) }; const nodeSession = options.nodeRegistry.registerTransport( client, - { remoteIp: clientIp }, + { + remoteIp: clientIp, + pairingIdentity: nodePairingState.identity.key, + pairingGeneration: nodePairingGeneration.key, + }, createTransport(session), ); sessionsByToken.set(session.token, session); @@ -906,6 +930,7 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions) session.nodeId, nodeSession.connectedAtMs, options.pairingBaseDir, + nodePairingGeneration, ).catch((error: unknown) => options.onError?.("watch node last-connect metadata update failed", error), ); @@ -933,7 +958,7 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions) sendMethodNotAllowed(res); return; } - const session = getSession(req, res); + const session = await getSession(req, res); if (!session) { return; } @@ -969,12 +994,12 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions) }); }; - const handleDisconnect = (req: IncomingMessage, res: ServerResponse) => { + const handleDisconnect = async (req: IncomingMessage, res: ServerResponse) => { if ((req.method ?? "").toUpperCase() !== "POST") { sendMethodNotAllowed(res); return; } - const session = getSession(req, res); + const session = await getSession(req, res); if (!session) { return; } @@ -987,7 +1012,7 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions) sendMethodNotAllowed(res); return; } - const session = getSession(req, res); + const session = await getSession(req, res); if (!session) { return; } @@ -1005,6 +1030,13 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions) ...(typeof body.error.message === "string" ? { message: body.error.message } : {}), } : null; + // Body upload can yield long enough for an external pairing mutation. + // Recheck the exact HTTP transport before committing its invoke result. + if (!(await options.nodeRegistry.isConnectionCurrentPairingState(session.connId))) { + closeSession(session, "node pairing changed"); + sendUnauthorized(res); + return; + } const accepted = options.nodeRegistry.handleInvokeResult({ id: body.id, nodeId: session.nodeId, @@ -1035,7 +1067,7 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions) await handleConnect(req, res); return true; case DISCONNECT_PATH: - handleDisconnect(req, res); + await handleDisconnect(req, res); return true; case POLL_PATH: await handlePoll(req, res); diff --git a/src/infra/active-node-context.ts b/src/infra/active-node-context.ts index 6a740d96ed5a..188f55f489c4 100644 --- a/src/infra/active-node-context.ts +++ b/src/infra/active-node-context.ts @@ -1,18 +1,43 @@ /** Stable active-node identity projected into the dynamic model runtime line. */ type ActiveNodeContext = { nodeId: string; + pairingGeneration?: string; }; -let activeNodeContext: ActiveNodeContext | null = null; +type ActiveNodeContextState = ActiveNodeContext & { + isCurrent?: () => boolean; +}; -/** Publishes the gateway's current active-node choice without volatile timestamps. */ -export function setActiveNodeContext(next: ActiveNodeContext | null): void { - activeNodeContext = next ? { ...next } : null; +let activeNodeContext: ActiveNodeContextState | null = null; + +function snapshotActiveNodeContext(context: ActiveNodeContextState): ActiveNodeContext { + return { + nodeId: context.nodeId, + ...(context.pairingGeneration ? { pairingGeneration: context.pairingGeneration } : {}), + }; } -/** Returns a defensive snapshot for prompt construction. */ -export function getActiveNodeContext(): ActiveNodeContext | null { - return activeNodeContext ? { ...activeNodeContext } : null; +/** Publishes the gateway's current active-node choice without volatile timestamps. */ +export function setActiveNodeContext( + next: ActiveNodeContext | null, + options?: { isCurrent?: () => boolean }, +): void { + activeNodeContext = next ? { ...next, ...options } : null; +} + +/** Revalidates the published node before projecting it into an agent prompt. */ +export function getCurrentActiveNodeContext(): ActiveNodeContext | null { + if (!activeNodeContext) { + return null; + } + try { + if (activeNodeContext.isCurrent && !activeNodeContext.isCurrent()) { + return null; + } + } catch { + return null; + } + return snapshotActiveNodeContext(activeNodeContext); } /** Formats the stable authenticated id; node-controlled labels stay out of prompt text. */ diff --git a/src/infra/device-pairing-store.ts b/src/infra/device-pairing-store.ts index a4397893f5a2..588368ba2482 100644 --- a/src/infra/device-pairing-store.ts +++ b/src/infra/device-pairing-store.ts @@ -14,6 +14,7 @@ import type { import { openOpenClawStateDatabase, runOpenClawStateWriteTransaction, + type OpenClawStateDatabase, type OpenClawStateDatabaseOptions, } from "../state/openclaw-state-db.js"; import type { @@ -25,7 +26,12 @@ import type { PairedDeviceNodeSurface, PairedDevicePendingNodeSurface, } from "./device-pairing.types.js"; -import { executeSqliteQuerySync, getNodeSqliteKysely } from "./kysely-sync.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; +import { clearApnsRegistrationFromDatabase } from "./push-apns-store-transaction.js"; type DevicePairingStoreState = { pendingById: Record; @@ -34,6 +40,19 @@ type DevicePairingStoreState = { type DevicePairingStoreTarget = "pending" | "paired" | "both"; +type PairedDeviceNodeSurfaceUpdate = + | { value: T; persist: false } + | { value: T; persist: true; nodeSurface: PairedDeviceNodeSurface }; + +type PairedDevicePresenceUpdate = + | { value: T; persist: false } + | { + value: T; + persist: true; + lastSeenAtMs: number; + lastSeenReason: string; + }; + /** Route an explicit pairing base dir (tests, alternate state roots) to that dir's DB. */ function resolveDevicePairingStateDbOptions(baseDir?: string): OpenClawStateDatabaseOptions { return baseDir ? { env: { ...process.env, OPENCLAW_STATE_DIR: baseDir } } : {}; @@ -245,11 +264,104 @@ export function loadDevicePairingStoreState(baseDir?: string): DevicePairingStor return { pendingById, pairedByDeviceId }; } +/** Load one paired-device row without materializing either pairing table. */ +export function loadPairedDevicePairingStoreRecord( + deviceId: string, + baseDir?: string, +): PairedDevice | null { + const { db } = openOpenClawStateDatabase(resolveDevicePairingStateDbOptions(baseDir)); + return loadPairedDevicePairingStoreRecordFromDatabase(db, deviceId); +} + +/** Load one paired-device row from an existing shared-state transaction. */ +export function loadPairedDevicePairingStoreRecordFromDatabase( + db: OpenClawStateDatabase["db"], + deviceId: string, +): PairedDevice | null { + const normalizedDeviceId = deviceId.trim(); + if (!normalizedDeviceId) { + return null; + } + const kysely = getNodeSqliteKysely(db); + const row = executeSqliteQueryTakeFirstSync( + db, + kysely + .selectFrom("device_pairing_paired") + .selectAll() + .where("device_id", "=", normalizedDeviceId), + ); + return row ? fromPairedRow(row) : null; +} + +/** Read, validate, and update one paired node surface in a single cross-process transaction. */ +export function updatePairedDeviceNodeSurfaceInTransaction( + deviceId: string, + baseDir: string | undefined, + update: (device: PairedDevice | null) => PairedDeviceNodeSurfaceUpdate, +): T { + return runOpenClawStateWriteTransaction(({ db }) => { + const normalizedDeviceId = deviceId.trim(); + const device = normalizedDeviceId + ? loadPairedDevicePairingStoreRecordFromDatabase(db, normalizedDeviceId) + : null; + const result = update(device); + if (!result.persist) { + return result.value; + } + if (!device) { + throw new Error("cannot update a missing paired-device node surface"); + } + const kysely = getNodeSqliteKysely(db); + executeSqliteQuerySync( + db, + kysely + .updateTable("device_pairing_paired") + .set({ node_surface_json: toJsonColumn(result.nodeSurface) }) + .where("device_id", "=", normalizedDeviceId), + ); + return result.value; + }, resolveDevicePairingStateDbOptions(baseDir)); +} + +/** Read, validate, and update one paired-device presence row in one transaction. */ +export function updatePairedDevicePresenceInTransaction( + deviceId: string, + baseDir: string | undefined, + update: (device: PairedDevice | null) => PairedDevicePresenceUpdate, +): T { + return runOpenClawStateWriteTransaction(({ db }) => { + const normalizedDeviceId = deviceId.trim(); + const device = normalizedDeviceId + ? loadPairedDevicePairingStoreRecordFromDatabase(db, normalizedDeviceId) + : null; + const result = update(device); + if (!result.persist) { + return result.value; + } + if (!device) { + throw new Error("cannot update presence for a missing paired device"); + } + const kysely = getNodeSqliteKysely(db); + executeSqliteQuerySync( + db, + kysely + .updateTable("device_pairing_paired") + .set({ + last_seen_at_ms: result.lastSeenAtMs, + last_seen_reason: result.lastSeenReason, + }) + .where("device_id", "=", normalizedDeviceId), + ); + return result.value; + }, resolveDevicePairingStateDbOptions(baseDir)); +} + /** Replace the pending and/or paired table contents with the given snapshot. */ export function persistDevicePairingStoreState( state: DevicePairingStoreState, baseDir: string | undefined, target: DevicePairingStoreTarget, + options?: { clearApnsNodeIds?: readonly string[] }, ): void { runOpenClawStateWriteTransaction(({ db }) => { const kysely = getNodeSqliteKysely(db); @@ -267,6 +379,9 @@ export function persistDevicePairingStoreState( executeSqliteQuerySync(db, kysely.insertInto("device_pairing_paired").values(rows)); } } + for (const nodeId of new Set(options?.clearApnsNodeIds ?? [])) { + clearApnsRegistrationFromDatabase(db, nodeId); + } }, resolveDevicePairingStateDbOptions(baseDir)); } diff --git a/src/infra/device-pairing.test.ts b/src/infra/device-pairing.test.ts index 7d309d6ffdf0..9b6c260436a5 100644 --- a/src/infra/device-pairing.test.ts +++ b/src/infra/device-pairing.test.ts @@ -23,13 +23,17 @@ import { removePairedDevice, requestDevicePairing, rejectDevicePairing, + resolveNodePairingGeneration, revokeDeviceToken, rotateDeviceToken, updatePairedDeviceMetadata, + updatePairedDevicePresence, verifyDeviceToken, withPairedDeviceRecords, type PairedDevice, } from "./device-pairing.js"; +import { approveNodePairing, requestNodePairing, updatePairedNodeBins } from "./node-pairing.js"; +import { loadApnsRegistration, registerApnsRegistration } from "./push-apns.js"; type RotateDeviceTokenResult = Awaited>; @@ -1004,6 +1008,51 @@ describe("device pairing tokens", () => { }); }); + test("stale node presence cannot update a replacement pairing generation", async () => { + const baseDir = await makeDevicePairingDir(); + await setupPairedNodeDevice(baseDir); + const nodePairing = await requestNodePairing( + { nodeId: "node-1", platform: "darwin", commands: ["system.run"] }, + baseDir, + ); + await approveNodePairing( + nodePairing.request.requestId, + { callerScopes: ["operator.pairing", "operator.admin"] }, + baseDir, + ); + const original = resolveNodePairingGeneration(await getPairedDevice("node-1", baseDir)); + expect(original).not.toBeNull(); + if (!original) { + throw new Error("expected original node pairing generation"); + } + await expect(updatePairedNodeBins("node-1", ["retired-bin"], original, baseDir)).resolves.toBe( + true, + ); + + const rotated = await rotateDeviceToken({ + deviceId: "node-1", + role: "node", + scopes: [], + baseDir, + }); + expect(rotated.ok).toBe(true); + const replacement = resolveNodePairingGeneration(await getPairedDevice("node-1", baseDir)); + expect(replacement?.key).not.toBe(original.key); + + await expect( + updatePairedDevicePresence( + "node-1", + { lastSeenAtMs: 4321, lastSeenReason: "bg_app_refresh" }, + original, + baseDir, + ), + ).resolves.toBe(false); + const paired = await getPairedDevice("node-1", baseDir); + expect(paired?.nodeSurface?.bins).toBeUndefined(); + expect(paired?.lastSeenAtMs).toBeUndefined(); + expect(paired?.lastSeenReason).toBeUndefined(); + }); + test("approval access metadata initializes paired device last-seen fields", async () => { const baseDir = await makeDevicePairingDir(); const request = await requestDevicePairing( @@ -2148,14 +2197,122 @@ describe("device pairing tokens", () => { test("removes paired devices by device id", async () => { const baseDir = await makeDevicePairingDir(); await setupPairedOperatorDevice(baseDir, ["operator.read"]); + await registerApnsRegistration({ + nodeId: "device-1", + transport: "direct", + token: "ABCD1234ABCD1234ABCD1234ABCD1234", + topic: "ai.openclaw.ios", + environment: "sandbox", + baseDir, + }); const removed = await removePairedDevice("device-1", baseDir); expect(removed).toEqual({ deviceId: "device-1" }); await expect(getPairedDevice("device-1", baseDir)).resolves.toBeNull(); + await expect(loadApnsRegistration("device-1", baseDir)).resolves.toBeNull(); await expect(removePairedDevice("device-1", baseDir)).resolves.toBeNull(); }); + test("clears APNs only when a node reapproval changes installation identity", async () => { + const baseDir = await makeDevicePairingDir(); + await setupPairedNodeDevice(baseDir); + const nodePairing = await requestNodePairing({ nodeId: "node-1" }, baseDir); + await approveNodePairing( + nodePairing.request.requestId, + { callerScopes: ["operator.pairing"] }, + baseDir, + ); + await registerApnsRegistration({ + nodeId: "node-1", + transport: "direct", + token: "ABCD1234ABCD1234ABCD1234ABCD1234", + topic: "ai.openclaw.ios", + environment: "sandbox", + baseDir, + }); + + const sameInstallationRepair = await requestDevicePairing( + { + deviceId: "node-1", + publicKey: "public-key-node-1", + role: "node", + scopes: [], + }, + baseDir, + ); + await expect( + approveDevicePairing(sameInstallationRepair.request.requestId, { callerScopes: [] }, baseDir), + ).resolves.toMatchObject({ + status: "approved", + nodePairingGenerationChanged: true, + }); + await expect(loadApnsRegistration("node-1", baseDir)).resolves.toMatchObject({ + token: "abcd1234abcd1234abcd1234abcd1234", + }); + + const replacementRepair = await requestDevicePairing( + { + deviceId: "node-1", + publicKey: "public-key-node-1-replacement", + role: "node", + scopes: [], + }, + baseDir, + ); + await expect( + approveDevicePairing(replacementRepair.request.requestId, { callerScopes: [] }, baseDir), + ).resolves.toMatchObject({ + status: "approved", + nodePairingGenerationChanged: true, + }); + await expect(loadApnsRegistration("node-1", baseDir)).resolves.toBeNull(); + }); + + test("clears generation-owned node bins on public-key replacement", async () => { + const baseDir = await makeDevicePairingDir(); + await setupPairedNodeDevice(baseDir); + const nodePairing = await requestNodePairing( + { nodeId: "node-1", platform: "darwin", commands: ["system.run", "system.which"] }, + baseDir, + ); + await approveNodePairing( + nodePairing.request.requestId, + { callerScopes: ["operator.pairing", "operator.admin"] }, + baseDir, + ); + const previousGeneration = resolveNodePairingGeneration( + await getPairedDevice("node-1", baseDir), + ); + if (!previousGeneration) { + throw new Error("expected previous node pairing generation"); + } + await expect( + updatePairedNodeBins("node-1", ["retired-bin"], previousGeneration, baseDir), + ).resolves.toBe(true); + + const replacement = await requestDevicePairing( + { + deviceId: "node-1", + publicKey: "public-key-node-1-replacement", + role: "node", + scopes: [], + }, + baseDir, + ); + await expect( + approveDevicePairing(replacement.request.requestId, { callerScopes: [] }, baseDir), + ).resolves.toMatchObject({ + status: "approved", + nodePairingGenerationChanged: true, + }); + + const paired = await getPairedDevice("node-1", baseDir); + expect(resolveNodePairingGeneration(paired)?.key).not.toBe(previousGeneration.key); + expect(paired?.nodeSurface?.commands).toEqual(["system.run", "system.which"]); + expect(paired?.nodeSurface?.bins).toBeUndefined(); + }); + test("removing a paired device clears pending requests for that device only", async () => { const baseDir = await makeDevicePairingDir(); await setupPairedOperatorDevice(baseDir, ["operator.read"]); diff --git a/src/infra/device-pairing.ts b/src/infra/device-pairing.ts index 50c93d5de6c0..be92ddc409e4 100644 --- a/src/infra/device-pairing.ts +++ b/src/infra/device-pairing.ts @@ -1,5 +1,5 @@ // Manages device pairing requests, approvals, and token issuance. -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { expectDefined } from "@openclaw/normalization-core"; import { normalizeUniqueSingleOrTrimmedStringList } from "@openclaw/normalization-core/string-normalization"; import { normalizeDeviceAuthScopes } from "../shared/device-auth.js"; @@ -16,7 +16,9 @@ import { import { revokeDeviceBootstrapTokensForDevice } from "./device-bootstrap.js"; import { loadDevicePairingStoreState, + loadPairedDevicePairingStoreRecord, persistDevicePairingStoreState as persistState, + updatePairedDevicePresenceInTransaction, } from "./device-pairing-store.js"; import type { DeviceAuthToken, @@ -35,6 +37,21 @@ export type { PairedDevicePendingNodeSurface, } from "./device-pairing.types.js"; +export type NodePairingGeneration = { + nodeId: string; + key: string; +}; + +export type NodePairingIdentity = { + nodeId: string; + key: string; +}; + +export type NodePairingState = { + identity: NodePairingIdentity; + generation: NodePairingGeneration | null; +}; + /** Pending request summary returned when a replacement supersedes older requests. */ type DevicePairingSupersededRequest = Pick; @@ -118,7 +135,13 @@ type DevicePairingForbiddenResult = { /** Pairing approval outcome: approved, forbidden with reason, or request not found. */ type ApproveDevicePairingResult = - | { status: "approved"; requestId: string; device: PairedDevice } + | { + status: "approved"; + requestId: string; + device: PairedDevice; + /** Existing connected node transports must be retired before success is returned. */ + nodePairingGenerationChanged?: true; + } | DevicePairingForbiddenResult | null; @@ -262,6 +285,81 @@ export function hasEffectivePairedDeviceRole( return listEffectivePairedDeviceRoles(device).includes(normalized); } +/** Resolve the authenticated node pairing independently of surface approval. */ +function resolveNodePairingIdentity(device: PairedDevice | null): NodePairingIdentity | null { + if (!device || !hasEffectivePairedDeviceRole(device, "node")) { + return null; + } + const nodeToken = device.tokens?.node; + if (!nodeToken) { + return null; + } + const key = createHash("sha256") + .update( + [ + device.publicKey, + device.createdAtMs, + nodeToken.token, + nodeToken.createdAtMs, + nodeToken.rotatedAtMs ?? "", + nodeToken.revokedAtMs ?? "", + ].join("\0"), + ) + .digest("hex"); + return { nodeId: device.deviceId, key }; +} + +/** Resolve the durable node-owned identity used to admit asynchronous work. */ +export function resolveNodePairingGeneration( + device: PairedDevice | null, +): NodePairingGeneration | null { + if (!device || !hasEffectivePairedDeviceRole(device, "node") || !device.nodeSurface) { + return null; + } + const nodeToken = device.tokens?.node; + const nodeSurface = device.nodeSurface; + // Device-wide approval also changes for unrelated operator upgrades, so only + // node-owned identity participates in the generation. + const key = createHash("sha256") + .update( + [ + device.publicKey, + device.createdAtMs, + nodeToken?.token ?? "", + nodeToken?.revokedAtMs ?? "", + nodeSurface.createdAtMs, + nodeSurface.approvedAtMs, + ].join("\0"), + ) + .digest("hex"); + return { nodeId: device.deviceId, key }; +} + +/** Clear node-surface cache state when its owning pairing generation changes. */ +export function clearNodePairingGenerationBins( + device: PairedDevice, + previousGeneration: NodePairingGeneration | null, +): void { + const nextGeneration = resolveNodePairingGeneration(device); + if ( + previousGeneration?.key === nextGeneration?.key || + !device.nodeSurface || + device.nodeSurface.bins === undefined + ) { + return; + } + delete device.nodeSurface.bins; +} + +/** Resolve connection identity and optional approved surface generation from one row. */ +export function resolveNodePairingState(device: PairedDevice | null): NodePairingState | null { + const identity = resolveNodePairingIdentity(device); + if (!identity) { + return null; + } + return { identity, generation: resolveNodePairingGeneration(device) }; +} + function mergeScopes(...items: Array): string[] | undefined { const scopes = new Set(); let sawExplicitScopeList = false; @@ -645,8 +743,11 @@ export async function getPairedDevice( deviceId: string, baseDir?: string, ): Promise { - const state = await loadState(baseDir); - return state.pairedByDeviceId[normalizeDeviceId(deviceId)] ?? null; + const device = loadPairedDevicePairingStoreRecord(normalizeDeviceId(deviceId), baseDir); + if (device?.pendingNodeSurface && Date.now() - device.pendingNodeSurface.ts > PENDING_TTL_MS) { + delete device.pendingNodeSurface; + } + return device; } /** Return one pending pairing request by request id. */ @@ -870,6 +971,7 @@ export async function approveDevicePairing( existing?.approvedScopes ?? existing?.scopes, pending.scopes, ); + const previousNodeGeneration = resolveNodePairingGeneration(existing ?? null); const tokens = existing?.tokens ? { ...existing.tokens } : {}; const nextTokenScopesByRole = new Map(); for (const roleForToken of requestedRoles) { @@ -928,10 +1030,28 @@ export async function approveDevicePairing( approvedVia: options?.approvedVia ?? "owner", accessMetadata: options?.accessMetadata, }); + const nextNodeGeneration = resolveNodePairingGeneration(device); + const nodePairingGenerationChanged = Boolean( + previousNodeGeneration && previousNodeGeneration.key !== nextNodeGeneration?.key, + ); + clearNodePairingGenerationBins(device, previousNodeGeneration); + const installationIdentityChanged = Boolean( + existing && existing.publicKey !== device.publicKey, + ); delete state.pendingById[requestId]; state.pairedByDeviceId[device.deviceId] = device; - persistState(state, baseDir, "both"); - return { status: "approved", requestId, device }; + persistState( + state, + baseDir, + "both", + installationIdentityChanged ? { clearApnsNodeIds: [device.deviceId] } : undefined, + ); + return { + status: "approved", + requestId, + device, + ...(nodePairingGenerationChanged ? { nodePairingGenerationChanged: true as const } : {}), + }; }); } @@ -1003,6 +1123,7 @@ export async function approveBootstrapDevicePairing( ); const roles = mergeRoles(existing?.roles, existing?.role, pending.roles, pending.role); const nextApprovedScopes = mergeScopes(preservedExistingScopes, grantedScopes); + const previousNodeGeneration = resolveNodePairingGeneration(existing ?? null); const tokens = existing?.tokens ? { ...existing.tokens } : {}; for (const roleForToken of grantedRoles) { const existingToken = tokens[roleForToken]; @@ -1029,10 +1150,28 @@ export async function approveBootstrapDevicePairing( approvedVia: "bootstrap", accessMetadata: options?.accessMetadata, }); + const nextNodeGeneration = resolveNodePairingGeneration(device); + const nodePairingGenerationChanged = Boolean( + previousNodeGeneration && previousNodeGeneration.key !== nextNodeGeneration?.key, + ); + clearNodePairingGenerationBins(device, previousNodeGeneration); + const installationIdentityChanged = Boolean( + existing && existing.publicKey !== device.publicKey, + ); delete state.pendingById[requestId]; state.pairedByDeviceId[device.deviceId] = device; - persistState(state, baseDir, "both"); - return { status: "approved", requestId, device }; + persistState( + state, + baseDir, + "both", + installationIdentityChanged ? { clearApnsNodeIds: [device.deviceId] } : undefined, + ); + return { + status: "approved", + requestId, + device, + ...(nodePairingGenerationChanged ? { nodePairingGenerationChanged: true as const } : {}), + }; }); } @@ -1075,7 +1214,7 @@ export async function removePairedDevice( delete state.pendingById[requestId]; } } - persistState(state, baseDir, "both"); + persistState(state, baseDir, "both", { clearApnsNodeIds: [normalized] }); return { deviceId: normalized }; }); } @@ -1164,7 +1303,9 @@ export async function pruneSupersededSilentPairedDevices(params: { if (removed.length === 0) { return []; } - persistState(state, params.baseDir, "both"); + persistState(state, params.baseDir, "both", { + clearApnsNodeIds: removed.map((entry) => entry.deviceId), + }); return removed; }); } @@ -1194,7 +1335,9 @@ export async function removePairedDeviceRole(params: { } } delete state.pairedByDeviceId[normalizedDeviceId]; - persistState(state, params.baseDir, "both"); + persistState(state, params.baseDir, "both", { + clearApnsNodeIds: [normalizedDeviceId], + }); return { deviceId: normalizedDeviceId, role, removedDevice: true }; } @@ -1297,6 +1440,40 @@ export async function updatePairedDeviceMetadata( }); } +/** Update paired-device presence only while the authenticated node generation still owns it. */ +export async function updatePairedDevicePresence( + deviceId: string, + patch: { lastSeenAtMs: number; lastSeenReason: string }, + expectedPairingGeneration: NodePairingGeneration, + baseDir?: string, +): Promise { + return await withLock(async () => { + const updated = updatePairedDevicePresenceInTransaction( + deviceId, + baseDir, + (device) => { + const currentPairingGeneration = resolveNodePairingGeneration(device); + if ( + !device || + expectedPairingGeneration.nodeId !== device.deviceId || + currentPairingGeneration?.key !== expectedPairingGeneration.key + ) { + return { value: false, persist: false }; + } + return { + value: true, + persist: true, + lastSeenAtMs: patch.lastSeenAtMs, + lastSeenReason: patch.lastSeenReason, + }; + }, + ); + // The row-scoped transaction owns cross-process generation validation. Keep + // the outer lock so local full-snapshot writers cannot replay older presence. + return updated; + }); +} + /** Summarize token metadata without exposing bearer token strings. */ export function summarizeDeviceTokens( tokens: Record | undefined, @@ -1405,6 +1582,7 @@ export async function ensureDeviceToken(params: { return null; } const { device, role, tokens, existing } = context; + const previousNodeGeneration = resolveNodePairingGeneration(device); const approvedScopes = resolveApprovedDeviceScopeBaseline(device); if ( !scopesWithinApprovedDeviceBaseline({ @@ -1441,6 +1619,7 @@ export async function ensureDeviceToken(params: { }); tokens[role] = next; device.tokens = tokens; + clearNodePairingGenerationBins(device, previousNodeGeneration); state.pairedByDeviceId[device.deviceId] = device; persistState(state, params.baseDir, "paired"); return next; @@ -1494,6 +1673,7 @@ export async function rotateDeviceToken(params: { return { ok: false, reason: "unknown-device-or-role" }; } const { device, role, tokens, existing } = context; + const previousNodeGeneration = resolveNodePairingGeneration(device); const requestedScopes = normalizeDeviceAuthScopes( params.scopes ?? existing?.scopes ?? device.scopes, ); @@ -1531,6 +1711,7 @@ export async function rotateDeviceToken(params: { }); tokens[role] = next; device.tokens = tokens; + clearNodePairingGenerationBins(device, previousNodeGeneration); state.pairedByDeviceId[device.deviceId] = device; persistState(state, params.baseDir, "paired"); return { ok: true, entry: next }; @@ -1555,6 +1736,7 @@ export async function revokeDeviceToken(params: { return { ok: false, reason: "unknown-device-or-role" }; } const { device, role, tokens, existing } = context; + const previousNodeGeneration = resolveNodePairingGeneration(device); const targetScopes = normalizeDeviceAuthScopes( Array.isArray(existing.scopes) ? existing.scopes : device.scopes, ); @@ -1571,6 +1753,7 @@ export async function revokeDeviceToken(params: { const entry = { ...existing, revokedAtMs: Date.now() }; tokens[role] = entry; device.tokens = tokens; + clearNodePairingGenerationBins(device, previousNodeGeneration); state.pairedByDeviceId[device.deviceId] = device; persistState(state, params.baseDir, "paired"); return { ok: true, entry }; diff --git a/src/infra/node-pairing-state.test.ts b/src/infra/node-pairing-state.test.ts new file mode 100644 index 000000000000..a6c1f4f0b465 --- /dev/null +++ b/src/infra/node-pairing-state.test.ts @@ -0,0 +1,224 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { PairedDevice } from "./device-pairing.js"; +import { + captureAuthenticatedNodePairingState, + captureNodePairingGeneration, + isNodePairingGenerationCurrent, +} from "./node-pairing-state.js"; + +const mocks = vi.hoisted(() => ({ + getPairedDevice: vi.fn(), +})); + +vi.mock("./device-pairing.js", async () => { + const actual = await vi.importActual("./device-pairing.js"); + return { ...actual, getPairedDevice: mocks.getPairedDevice }; +}); + +function pairedNode(overrides: Partial = {}): PairedDevice { + return { + deviceId: "node-1", + publicKey: "public-key-1", + role: "node", + roles: ["node", "operator"], + tokens: { + node: { + token: "node-token-1", + role: "node", + scopes: [], + createdAtMs: 150, + }, + operator: { + token: "operator-token-1", + role: "operator", + scopes: ["operator.pairing"], + createdAtMs: 151, + }, + }, + createdAtMs: 100, + approvedAtMs: 200, + nodeSurface: { + createdAtMs: 300, + approvedAtMs: 400, + }, + ...overrides, + }; +} + +describe("node pairing generation", () => { + beforeEach(() => { + mocks.getPairedDevice.mockReset(); + }); + + it("binds work to node-role token and node-surface approval identity", async () => { + const original = pairedNode(); + mocks.getPairedDevice.mockResolvedValueOnce(original); + + const generation = await captureNodePairingGeneration(original.deviceId); + + expect(generation).toEqual({ + nodeId: "node-1", + key: expect.stringMatching(/^[a-f0-9]{64}$/u), + }); + mocks.getPairedDevice.mockResolvedValueOnce( + pairedNode({ + tokens: { + ...original.tokens, + node: { ...original.tokens!.node!, token: "node-token-2", rotatedAtMs: 500 }, + }, + }), + ); + await expect(isNodePairingGenerationCurrent(generation!)).resolves.toBe(false); + }); + + it("binds connected sessions to the public key and node token used at authentication", async () => { + const original = pairedNode(); + mocks.getPairedDevice + .mockResolvedValueOnce(original) + .mockResolvedValueOnce({ ...original, publicKey: "replacement-public-key" }) + .mockResolvedValueOnce( + pairedNode({ + tokens: { + ...original.tokens, + node: { ...original.tokens!.node!, token: "replacement-node-token" }, + }, + }), + ); + + await expect( + captureAuthenticatedNodePairingState({ + nodeId: original.deviceId, + publicKey: original.publicKey, + token: original.tokens!.node!.token, + }), + ).resolves.toMatchObject({ generation: { nodeId: original.deviceId } }); + await expect( + captureAuthenticatedNodePairingState({ + nodeId: original.deviceId, + publicKey: original.publicKey, + token: original.tokens!.node!.token, + }), + ).resolves.toBeNull(); + await expect( + captureAuthenticatedNodePairingState({ + nodeId: original.deviceId, + publicKey: original.publicKey, + token: original.tokens!.node!.token, + }), + ).resolves.toBeNull(); + }); + + it("keeps authenticated pairing identity while first surface approval is pending", async () => { + const original = pairedNode({ nodeSurface: undefined }); + mocks.getPairedDevice.mockResolvedValueOnce(original); + + await expect( + captureAuthenticatedNodePairingState({ + nodeId: original.deviceId, + publicKey: original.publicKey, + token: original.tokens!.node!.token, + }), + ).resolves.toEqual({ + identity: { + nodeId: original.deviceId, + key: expect.stringMatching(/^[a-f0-9]{64}$/u), + }, + generation: null, + }); + }); + + it("keeps pairing identity stable when the pending surface is approved", async () => { + const pending = pairedNode({ nodeSurface: undefined }); + const approved = pairedNode(); + mocks.getPairedDevice.mockResolvedValueOnce(pending).mockResolvedValueOnce(approved); + + const params = { + nodeId: pending.deviceId, + publicKey: pending.publicKey, + token: pending.tokens!.node!.token, + }; + const pendingState = await captureAuthenticatedNodePairingState(params); + const approvedState = await captureAuthenticatedNodePairingState(params); + + expect(pendingState?.generation).toBeNull(); + expect(approvedState?.generation).not.toBeNull(); + expect(approvedState?.identity.key).toBe(pendingState?.identity.key); + }); + + it("keeps node work current across unrelated operator approval", async () => { + const original = pairedNode(); + mocks.getPairedDevice.mockResolvedValueOnce(original); + const generation = await captureNodePairingGeneration(original.deviceId); + + mocks.getPairedDevice.mockResolvedValueOnce( + pairedNode({ + approvedAtMs: 201, + tokens: { + ...original.tokens, + operator: { + ...original.tokens!.operator!, + token: "operator-token-2", + rotatedAtMs: 501, + }, + }, + }), + ); + + await expect(isNodePairingGenerationCurrent(generation!)).resolves.toBe(true); + }); + + it("invalidates node work when the node surface is reapproved", async () => { + const original = pairedNode(); + mocks.getPairedDevice.mockResolvedValueOnce(original); + const generation = await captureNodePairingGeneration(original.deviceId); + + mocks.getPairedDevice.mockResolvedValueOnce( + pairedNode({ nodeSurface: { createdAtMs: 300, approvedAtMs: 401 } }), + ); + await expect(isNodePairingGenerationCurrent(generation!)).resolves.toBe(false); + }); + + it("invalidates node work when its effective node token is revoked", async () => { + const original = pairedNode(); + mocks.getPairedDevice.mockResolvedValueOnce(original); + const generation = await captureNodePairingGeneration(original.deviceId); + + mocks.getPairedDevice.mockResolvedValueOnce( + pairedNode({ + tokens: { + ...original.tokens, + node: { ...original.tokens!.node!, revokedAtMs: 501 }, + }, + }), + ); + + await expect(isNodePairingGenerationCurrent(generation!)).resolves.toBe(false); + }); + + it("rejects admission without an active node token or approved node surface", async () => { + const original = pairedNode(); + mocks.getPairedDevice + .mockResolvedValueOnce(pairedNode({ tokens: undefined })) + .mockResolvedValueOnce( + pairedNode({ + tokens: { + ...original.tokens, + node: { ...original.tokens!.node!, revokedAtMs: 501 }, + }, + }), + ) + .mockResolvedValueOnce(pairedNode({ nodeSurface: undefined })); + + await expect(captureNodePairingGeneration("node-1")).resolves.toBeNull(); + await expect(captureNodePairingGeneration("node-1")).resolves.toBeNull(); + await expect(captureNodePairingGeneration("node-1")).resolves.toBeNull(); + }); + + it("rejects admission without a durable approved node role", async () => { + mocks.getPairedDevice.mockResolvedValueOnce( + pairedNode({ role: "operator", roles: ["operator"] }), + ); + + await expect(captureNodePairingGeneration("node-1")).resolves.toBeNull(); + }); +}); diff --git a/src/infra/node-pairing-state.ts b/src/infra/node-pairing-state.ts new file mode 100644 index 000000000000..eedb6fcb41c3 --- /dev/null +++ b/src/infra/node-pairing-state.ts @@ -0,0 +1,86 @@ +import { loadPairedDevicePairingStoreRecord } from "./device-pairing-store.js"; +import { + getPairedDevice, + hasEffectivePairedDeviceRole, + resolveNodePairingGeneration, + resolveNodePairingState, + type NodePairingGeneration, + type NodePairingState, +} from "./device-pairing.js"; + +export type { NodePairingGeneration, NodePairingIdentity } from "./device-pairing.js"; + +export type NodePairingBinding = { + identity: string; + generation?: string; +}; + +function toNodePairingBinding(state: NodePairingState | null): NodePairingBinding | undefined { + return state + ? { + identity: state.identity.key, + ...(state.generation ? { generation: state.generation.key } : {}), + } + : undefined; +} + +/** Captures the persistent authenticated pairing and optional approved surface. */ +export async function captureNodePairingState( + nodeId: string, + baseDir?: string, +): Promise { + return resolveNodePairingState(await getPairedDevice(nodeId, baseDir)); +} + +/** Registry projection of the current persistent pairing owner. */ +export async function resolveCurrentNodePairingBinding( + nodeId: string, +): Promise { + return toNodePairingBinding(await captureNodePairingState(nodeId)); +} + +/** Synchronous registry projection for non-yielding process-local reads. */ +export function isNodePairingBindingCurrent(nodeId: string, expected: NodePairingBinding): boolean { + const current = toNodePairingBinding( + resolveNodePairingState(loadPairedDevicePairingStoreRecord(nodeId)), + ); + return Boolean( + current && + current.identity === expected.identity && + (!expected.generation || current.generation === expected.generation), + ); +} + +/** Captures the persistent node pairing generation admitted for new work. */ +export async function captureNodePairingGeneration( + nodeId: string, +): Promise { + return (await captureNodePairingState(nodeId))?.generation ?? null; +} + +/** Binds a connected session to the exact device key and node token it authenticated with. */ +export async function captureAuthenticatedNodePairingState(params: { + nodeId: string; + publicKey: string; + token: string; + baseDir?: string; +}): Promise { + const device = await getPairedDevice(params.nodeId, params.baseDir); + if ( + !device || + device.publicKey !== params.publicKey || + device.tokens?.node?.token !== params.token || + !hasEffectivePairedDeviceRole(device, "node") + ) { + return null; + } + return resolveNodePairingState(device); +} + +/** Revalidates that asynchronous work still belongs to the admitted pairing. */ +export async function isNodePairingGenerationCurrent( + generation: NodePairingGeneration, +): Promise { + const current = resolveNodePairingGeneration(await getPairedDevice(generation.nodeId)); + return current?.key === generation.key; +} diff --git a/src/infra/node-pairing.test.ts b/src/infra/node-pairing.test.ts index a13596fd6b78..90a8137a434d 100644 --- a/src/infra/node-pairing.test.ts +++ b/src/infra/node-pairing.test.ts @@ -1,7 +1,14 @@ // Tests node capability-surface approvals stored on paired device records. import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js"; -import { approveDevicePairing, requestDevicePairing } from "./device-pairing.js"; +import { createDeferred } from "../test-utils/deferred.js"; +import { + approveDevicePairing, + getPairedDevice, + requestDevicePairing, + resolveNodePairingGeneration, + withPairedDeviceRecords, +} from "./device-pairing.js"; import { approveNodePairing, beginNodePairingConnect, @@ -572,13 +579,21 @@ describe("node surface approvals", () => { test("updates remote skill bins and reports missing nodes", async () => { await withNodePairingDir(async (baseDir) => { await setupPairedNode(baseDir); + const generation = resolveNodePairingGeneration(await getPairedDevice("node-1", baseDir)); + if (!generation) { + throw new Error("expected node pairing generation"); + } await expect(recordPairedNodeConnection("node-1", 1_234, baseDir)).resolves.toEqual({ recorded: true, firstConnection: true, }); - await expect(updatePairedNodeBins("node-1", ["ffmpeg"], baseDir)).resolves.toBe(true); - await expect(updatePairedNodeBins("missing", ["ffmpeg"], baseDir)).resolves.toBe(false); + await expect(updatePairedNodeBins("node-1", ["ffmpeg"], generation, baseDir)).resolves.toBe( + true, + ); + await expect(updatePairedNodeBins("missing", ["ffmpeg"], generation, baseDir)).resolves.toBe( + false, + ); const pairedNode = await findPairedNode("node-1", baseDir); expect(pairedNode?.lastConnectedAtMs).toBe(1_234); @@ -586,6 +601,47 @@ describe("node surface approvals", () => { }); }); + test("rejects retired-generation bins after node-surface reapproval", async () => { + await withNodePairingDir(async (baseDir) => { + await setupPairedNode(baseDir); + const previousGeneration = resolveNodePairingGeneration( + await getPairedDevice("node-1", baseDir), + ); + if (!previousGeneration) { + throw new Error("expected previous node pairing generation"); + } + await expect( + updatePairedNodeBins("node-1", ["retired-bin"], previousGeneration, baseDir), + ).resolves.toBe(true); + + const pending = await requestNodePairing( + { nodeId: "node-1", platform: "darwin", commands: ["system.run", "system.which"] }, + baseDir, + ); + await approveNodePairing( + pending.request.requestId, + { callerScopes: ["operator.pairing", "operator.admin"] }, + baseDir, + ); + + const currentGeneration = resolveNodePairingGeneration( + await getPairedDevice("node-1", baseDir), + ); + if (!currentGeneration) { + throw new Error("expected current node pairing generation"); + } + expect(currentGeneration.key).not.toBe(previousGeneration.key); + expect((await findPairedNode("node-1", baseDir))?.bins).toBeUndefined(); + await expect( + updatePairedNodeBins("node-1", ["stale-write"], previousGeneration, baseDir), + ).resolves.toBe(false); + await expect( + updatePairedNodeBins("node-1", ["current-bin"], currentGeneration, baseDir), + ).resolves.toBe(true); + expect((await findPairedNode("node-1", baseDir))?.bins).toEqual(["current-bin"]); + }); + }); + test("atomically grants one first-connection claim across concurrent handshakes", async () => { await withNodePairingDir(async (baseDir) => { await setupPairedNode(baseDir); @@ -614,6 +670,85 @@ describe("node surface approvals", () => { }); }); + test("serializes connection metadata with locked node-surface mutations", async () => { + await withNodePairingDir(async (baseDir) => { + await setupPairedNode(baseDir); + + const snapshotLoaded = createDeferred(); + const releaseMutation = createDeferred(); + const lockedMutation = withPairedDeviceRecords(baseDir, async (pairedByDeviceId) => { + const device = pairedByDeviceId["node-1"]; + if (!device?.nodeSurface) { + throw new Error("expected paired node surface"); + } + device.nodeSurface = { ...device.nodeSurface, bins: ["ffmpeg"] }; + snapshotLoaded.resolve(); + await releaseMutation.promise; + return { value: true, persist: true }; + }); + await snapshotLoaded.promise; + + let connectionSettled = false; + const connection = recordPairedNodeConnection("node-1", 1_234, baseDir).then((result) => { + connectionSettled = true; + return result; + }); + await Promise.resolve(); + expect(connectionSettled).toBe(false); + + releaseMutation.resolve(); + await expect(lockedMutation).resolves.toBe(true); + await expect(connection).resolves.toEqual({ recorded: true, firstConnection: true }); + expect(await findPairedNode("node-1", baseDir)).toMatchObject({ + bins: ["ffmpeg"], + lastConnectedAtMs: 1_234, + }); + }); + }); + + test("rejects connection metadata from a retired pairing generation", async () => { + await withNodePairingDir(async (baseDir) => { + await setupPairedNode(baseDir); + const previousGeneration = resolveNodePairingGeneration( + await getPairedDevice("node-1", baseDir), + ); + if (!previousGeneration) { + throw new Error("expected initial node pairing generation"); + } + + const pending = await requestNodePairing( + { + nodeId: "node-1", + platform: "darwin", + commands: ["system.run", "canvas.snapshot"], + }, + baseDir, + ); + await approveNodePairing( + pending.request.requestId, + { callerScopes: ["operator.pairing", "operator.admin", "operator.write"] }, + baseDir, + ); + const currentGeneration = resolveNodePairingGeneration( + await getPairedDevice("node-1", baseDir), + ); + if (!currentGeneration) { + throw new Error("expected replacement node pairing generation"); + } + expect(currentGeneration.key).not.toBe(previousGeneration.key); + + await expect( + recordPairedNodeConnection("node-1", 1_000, baseDir, previousGeneration), + ).resolves.toEqual({ recorded: false }); + expect((await findPairedNode("node-1", baseDir))?.lastConnectedAtMs).toBeUndefined(); + + await expect( + recordPairedNodeConnection("node-1", 2_000, baseDir, currentGeneration), + ).resolves.toEqual({ recorded: true, firstConnection: true }); + expect((await findPairedNode("node-1", baseDir))?.lastConnectedAtMs).toBe(2_000); + }); + }); + test("keeps the approved node surface across a device pairing re-approval", async () => { await withNodePairingDir(async (baseDir) => { await setupPairedNode(baseDir); diff --git a/src/infra/node-pairing.ts b/src/infra/node-pairing.ts index 88463d9d6ac5..901b1919901a 100644 --- a/src/infra/node-pairing.ts +++ b/src/infra/node-pairing.ts @@ -8,8 +8,13 @@ import { randomUUID } from "node:crypto"; import { normalizeArrayBackedTrimmedStringList } from "@openclaw/normalization-core/string-normalization"; import { resolveMissingRequestedScope } from "../shared/operator-scope-compat.js"; +import { updatePairedDeviceNodeSurfaceInTransaction } from "./device-pairing-store.js"; import { + clearNodePairingGenerationBins, + resolveNodePairingGeneration, + resolveNodePairingState, withPairedDeviceRecords, + type NodePairingGeneration, type PairedDevice, type PairedDevicePendingNodeSurface, } from "./device-pairing.js"; @@ -87,6 +92,10 @@ type NodePairingList = { paired: NodePairingPairedNode[]; }; +type NodePairingListWithGeneration = Omit & { + paired: Array; +}; + const OPERATOR_ROLE = "operator"; const activeCleanupRevisionClaims = new Map>(); @@ -149,11 +158,17 @@ function toPendingEntry( }; } -function toPairedNode(device: PairedDevice): NodePairingPairedNode | null { +function toPairedNode( + device: PairedDevice, + options?: { includePairingGeneration?: boolean }, +): NodePairingPairedNode | null { const surface = device.nodeSurface; if (!surface) { return null; } + const pairingGeneration = options?.includePairingGeneration + ? resolveNodePairingGeneration(device)?.key + : undefined; return { nodeId: device.deviceId, clientId: device.clientId, @@ -172,6 +187,7 @@ function toPairedNode(device: PairedDevice): NodePairingPairedNode | null { permissions: surface.permissions, remoteIp: device.remoteIp, bins: surface.bins, + ...(pairingGeneration ? { pairingGeneration } : {}), createdAtMs: surface.createdAtMs, approvedAtMs: surface.approvedAtMs, lastConnectedAtMs: surface.lastConnectedAtMs, @@ -322,7 +338,15 @@ function pendingHasActiveCleanupClaim( return (activeCleanupRevisionClaims.get(key)?.size ?? 0) > 0; } -export async function listNodePairing(baseDir?: string): Promise { +export function listNodePairing(baseDir?: string): Promise; +export function listNodePairing( + baseDir: string | undefined, + options: { includePairingGeneration: true }, +): Promise; +export async function listNodePairing( + baseDir?: string, + options?: { includePairingGeneration?: boolean }, +): Promise { return await withPairedDeviceRecords(baseDir, (pairedByDeviceId) => { const pending: NodePairingPendingEntry[] = []; const paired: NodePairingPairedNode[] = []; @@ -330,7 +354,7 @@ export async function listNodePairing(baseDir?: string): Promise { - return await withPairedDeviceRecords(baseDir, (pairedByDeviceId) => { - const device = nodeSurfaceDevice(pairedByDeviceId, nodeId); - if (!device?.nodeSurface) { - return { value: false, persist: false }; - } - device.nodeSurface = { - ...device.nodeSurface, - bins, - }; - return { value: true, persist: true }; + return await withPairedDeviceRecords(baseDir, () => { + const value = updatePairedDeviceNodeSurfaceInTransaction(nodeId, baseDir, (device) => { + const currentPairingGeneration = resolveNodePairingGeneration(device); + if ( + !device?.nodeSurface || + expectedPairingGeneration.nodeId !== device.deviceId || + currentPairingGeneration?.key !== expectedPairingGeneration.key + ) { + return { value: false, persist: false }; + } + return { + value: true, + persist: true, + nodeSurface: { + ...device.nodeSurface, + bins, + }, + }; + }); + // The row-scoped transaction owns cross-process generation validation, while + // this lock prevents a local full-snapshot writer from replaying retired bins. + return { value, persist: false }; }); } @@ -606,25 +667,44 @@ export async function recordPairedNodeConnection( nodeId: string, connectedAtMs: number, baseDir?: string, + expectedPairingGeneration?: NodePairingGeneration, ): Promise { - return await withPairedDeviceRecords( - baseDir, - (pairedByDeviceId) => { - const device = nodeSurfaceDevice(pairedByDeviceId, nodeId); - if (!device?.nodeSurface) { - return { value: { recorded: false }, persist: false }; - } - // Read and write under the pairing lock. Concurrent rehandshakes must not - // both claim the same node's first connection and schedule duplicate alerts. - const firstConnection = device.nodeSurface.lastConnectedAtMs === undefined; - const previousConnectedAtMs = device.nodeSurface.lastConnectedAtMs ?? connectedAtMs; - device.nodeSurface = { - ...device.nodeSurface, - lastConnectedAtMs: Math.max(previousConnectedAtMs, connectedAtMs), - }; - return { value: { recorded: true, firstConnection }, persist: true }; - }, - ); + return await withPairedDeviceRecords(baseDir, () => { + const value = updatePairedDeviceNodeSurfaceInTransaction( + nodeId, + baseDir, + (device) => { + if (!device?.nodeSurface) { + return { value: { recorded: false }, persist: false }; + } + if (expectedPairingGeneration) { + const currentPairingGeneration = resolveNodePairingGeneration(device); + if ( + expectedPairingGeneration.nodeId !== device.deviceId || + currentPairingGeneration?.key !== expectedPairingGeneration.key + ) { + return { value: { recorded: false }, persist: false }; + } + } + // Read and write under the pairing lock. Concurrent rehandshakes must not + // both claim the same node's first connection and schedule duplicate alerts. + const firstConnection = device.nodeSurface.lastConnectedAtMs === undefined; + const previousConnectedAtMs = device.nodeSurface.lastConnectedAtMs ?? connectedAtMs; + return { + value: { recorded: true, firstConnection }, + persist: true, + nodeSurface: { + ...device.nodeSurface, + lastConnectedAtMs: Math.max(previousConnectedAtMs, connectedAtMs), + }, + }; + }, + ); + // The row-scoped transaction owns cross-process generation validation, while + // this outer shared lock prevents local full-snapshot writers from replaying + // node-surface state loaded before the connection metadata commit. + return { value, persist: false }; + }); } /** Rename a paired node display name while preserving approval metadata. */ diff --git a/src/infra/push-apns-auth.ts b/src/infra/push-apns-auth.ts new file mode 100644 index 000000000000..f9b9c1241803 --- /dev/null +++ b/src/infra/push-apns-auth.ts @@ -0,0 +1,122 @@ +// Resolves APNs provider credentials and owns provider-token signing/cache state. +import { createHash, createPrivateKey, sign as signJwt } from "node:crypto"; +import fs from "node:fs/promises"; +import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; +import { formatErrorMessage } from "./errors.js"; + +/** Direct APNs provider authentication used to mint ES256 bearer tokens. */ +export type ApnsAuthConfig = { + teamId: string; + keyId: string; + privateKey: string; +}; + +type ApnsAuthConfigResolution = { ok: true; value: ApnsAuthConfig } | { ok: false; error: string }; + +const APNS_JWT_TTL_MS = 50 * 60 * 1000; + +let cachedJwt: { cacheKey: string; token: string; expiresAtMs: number } | null = null; + +function toBase64UrlBytes(value: Uint8Array): string { + return Buffer.from(value) + .toString("base64") + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=+$/g, ""); +} + +function toBase64UrlJson(value: object): string { + return toBase64UrlBytes(Buffer.from(JSON.stringify(value))); +} + +function getJwtCacheKey(auth: ApnsAuthConfig): string { + const keyHash = createHash("sha256").update(auth.privateKey).digest("hex"); + return `${auth.teamId}:${auth.keyId}:${keyHash}`; +} + +export function getApnsBearerToken(auth: ApnsAuthConfig, nowMs: number = Date.now()): string { + const cacheKey = getJwtCacheKey(auth); + if (cachedJwt && cachedJwt.cacheKey === cacheKey && nowMs < cachedJwt.expiresAtMs) { + return cachedJwt.token; + } + + // APNs provider tokens are valid for one hour. Cache for slightly less so + // bursty wake/approval pushes avoid repeated ECDSA signing. + const iat = Math.floor(nowMs / 1000); + const header = toBase64UrlJson({ alg: "ES256", kid: auth.keyId, typ: "JWT" }); + const payload = toBase64UrlJson({ iss: auth.teamId, iat }); + const signingInput = `${header}.${payload}`; + const signature = signJwt("sha256", Buffer.from(signingInput, "utf8"), { + key: createPrivateKey(auth.privateKey), + dsaEncoding: "ieee-p1363", + }); + const token = `${signingInput}.${toBase64UrlBytes(signature)}`; + cachedJwt = { + cacheKey, + token, + expiresAtMs: nowMs + APNS_JWT_TTL_MS, + }; + return token; +} + +function normalizePrivateKey(value: string): string { + return value.trim().replace(/\\n/g, "\n"); +} + +function normalizeNonEmptyString(value: string | undefined): string | null { + const trimmed = normalizeOptionalString(value) ?? ""; + return trimmed.length > 0 ? trimmed : null; +} + +/** Resolves direct APNs provider auth from env, accepting inline or file-backed keys. */ +export async function resolveApnsAuthConfigFromEnv( + env: NodeJS.ProcessEnv = process.env, +): Promise { + const teamId = normalizeNonEmptyString(env.OPENCLAW_APNS_TEAM_ID); + const keyId = normalizeNonEmptyString(env.OPENCLAW_APNS_KEY_ID); + if (!teamId || !keyId) { + return { + ok: false, + error: "APNs auth missing: set OPENCLAW_APNS_TEAM_ID and OPENCLAW_APNS_KEY_ID", + }; + } + + const inlineKeyRaw = + normalizeNonEmptyString(env.OPENCLAW_APNS_PRIVATE_KEY_P8) ?? + normalizeNonEmptyString(env.OPENCLAW_APNS_PRIVATE_KEY); + if (inlineKeyRaw) { + return { + ok: true, + value: { + teamId, + keyId, + privateKey: normalizePrivateKey(inlineKeyRaw), + }, + }; + } + + const keyPath = normalizeNonEmptyString(env.OPENCLAW_APNS_PRIVATE_KEY_PATH); + if (!keyPath) { + return { + ok: false, + error: + "APNs private key missing: set OPENCLAW_APNS_PRIVATE_KEY_P8 or OPENCLAW_APNS_PRIVATE_KEY_PATH", + }; + } + try { + const privateKey = normalizePrivateKey(await fs.readFile(keyPath, "utf8")); + return { + ok: true, + value: { + teamId, + keyId, + privateKey, + }, + }; + } catch (err) { + return { + ok: false, + error: `failed reading OPENCLAW_APNS_PRIVATE_KEY_PATH (${keyPath}): ${formatErrorMessage(err)}`, + }; + } +} diff --git a/src/infra/push-apns-cancellation.test.ts b/src/infra/push-apns-cancellation.test.ts new file mode 100644 index 000000000000..530e948cbb95 --- /dev/null +++ b/src/infra/push-apns-cancellation.test.ts @@ -0,0 +1,67 @@ +import { generateKeyPairSync } from "node:crypto"; +import { EventEmitter } from "node:events"; +import http2 from "node:http2"; +import { describe, expect, it, vi } from "vitest"; +import { APNS_HTTP2_CANCEL_CODE } from "./push-apns-http2.js"; +import { sendApnsBackgroundWake } from "./push-apns.js"; + +const testAuthPrivateKey = generateKeyPairSync("ec", { + namedCurve: "prime256v1", +}).privateKey.export({ format: "pem", type: "pkcs8" }); + +describe("APNs cancellation", () => { + it("cancels the active stream when pairing ownership is revoked", async () => { + const request = Object.assign(new EventEmitter(), { + destroyed: false, + setTimeout: vi.fn(), + close: vi.fn(), + end: vi.fn(), + }); + request.close.mockImplementation(() => { + request.destroyed = true; + request.emit("close"); + }); + const session = Object.assign(new EventEmitter(), { + close: vi.fn(), + destroy: vi.fn(), + request: vi.fn(() => request), + }); + session.close.mockImplementation(() => session.emit("close")); + const connect = vi + .spyOn(http2, "connect") + .mockReturnValue(session as unknown as http2.ClientHttp2Session); + const controller = new AbortController(); + + try { + const sending = sendApnsBackgroundWake({ + registration: { + nodeId: "ios-node-cancelled-stream", + transport: "direct", + token: "ABCD1234ABCD1234ABCD1234ABCD1234", + topic: "ai.openclaw.ios", + environment: "production", + updatedAtMs: 1, + }, + nodeId: "ios-node-cancelled-stream", + wakeReason: "node.invoke", + auth: { + teamId: "TEAM123", + keyId: "KEY123", + privateKey: testAuthPrivateKey, + }, + signal: controller.signal, + isCurrent: vi.fn().mockResolvedValue(true), + }); + await vi.waitFor(() => expect(request.end).toHaveBeenCalledTimes(1)); + + controller.abort(new Error("pairing removed")); + + await expect(sending).rejects.toThrow("pairing removed"); + expect(request.close).toHaveBeenCalledWith(APNS_HTTP2_CANCEL_CODE); + expect(session.close).toHaveBeenCalledTimes(1); + expect(session.destroy).not.toHaveBeenCalled(); + } finally { + connect.mockRestore(); + } + }); +}); diff --git a/src/infra/push-apns-http2.test.ts b/src/infra/push-apns-http2.test.ts index 7941728dc40d..ad2fefbbb6e5 100644 --- a/src/infra/push-apns-http2.test.ts +++ b/src/infra/push-apns-http2.test.ts @@ -86,7 +86,10 @@ const { }), request: vi.fn(() => fakeRequestLocal), }); - const fakeProxySocketLocal = { destroy: vi.fn() }; + const fakeProxySocketLocal = Object.assign(new FakeEmitter(), { + destroy: vi.fn(), + unshift: vi.fn(), + }); const fakeTlsSocketLocal = Object.assign(new FakeEmitter(), { encrypted: true, alpnProtocol: "h2" as string | false, @@ -163,7 +166,9 @@ describe("connectApnsHttp2Session", () => { tunnelSpy.mockClear(); tlsConnectSpy.mockClear(); setTargetTlsEvent("secureConnect"); + fakeProxySocket.reset(); fakeProxySocket.destroy.mockClear(); + fakeProxySocket.unshift.mockClear(); fakeRequest.reset(); fakeRequest.setEncoding.mockClear(); fakeRequest.end.mockClear(); @@ -196,6 +201,22 @@ describe("connectApnsHttp2Session", () => { expect(connectSpy).toHaveBeenCalledWith("https://api.sandbox.push.apple.com"); }); + it("rejects an already invalidated direct APNs setup before opening a session", async () => { + const { connectApnsHttp2Session } = await import("./push-apns-http2.js"); + const controller = new AbortController(); + controller.abort(new Error("pairing removed")); + + await expect( + connectApnsHttp2Session({ + authority: "https://api.sandbox.push.apple.com", + timeoutMs: 10_000, + signal: controller.signal, + }), + ).rejects.toThrow("pairing removed"); + + expect(connectSpy).not.toHaveBeenCalled(); + }); + it("normalizes the default APNs HTTPS port", async () => { const { connectApnsHttp2Session } = await import("./push-apns-http2.js"); @@ -227,30 +248,33 @@ describe("connectApnsHttp2Session", () => { }); it("uses an HTTP CONNECT tunnel when managed proxy is active", async () => { - const registration = registerActiveManagedProxyUrl(new URL("https://proxy.example:8443"), { - loopbackMode: "gateway-only", - proxyTls: { ca: "active-proxy-ca" }, - }); + const registration = registerActiveManagedProxyUrl( + new URL("https://user:pass@proxy.example:8443"), + { + loopbackMode: "gateway-only", + proxyTls: { ca: "active-proxy-ca" }, + }, + ); const { connectApnsHttp2Session } = await import("./push-apns-http2.js"); + const controller = new AbortController(); const session = await connectApnsHttp2Session({ authority: "https://api.push.apple.com", timeoutMs: 10_000, + signal: controller.signal, }); stopActiveManagedProxyRegistration(registration); expect(session).toBe(fakeSession); const tunnelCall = lastTunnelCall(); - const proxyUrl = tunnelCall.proxyUrl; - expect(proxyUrl).toBeInstanceOf(URL); - if (!(proxyUrl instanceof URL)) { - throw new Error("expected active managed proxy URL"); - } - expect(proxyUrl.href).toBe("https://proxy.example:8443/"); - expect(tunnelCall.proxyTls).toEqual({ ca: "active-proxy-ca" }); - expect(tunnelCall.targetHost).toBe("api.push.apple.com"); - expect(tunnelCall.targetPort).toBe(443); - expect(tunnelCall.timeoutMs).toBe(10_000); + expect(tunnelCall).toMatchObject({ + targetHost: "api.push.apple.com", + targetPort: 443, + timeoutMs: 10_000, + proxyTls: { ca: "active-proxy-ca" }, + }); + expect(String(tunnelCall.proxyUrl)).toBe("https://user:pass@proxy.example:8443/"); + expect(tunnelCall).toMatchObject({ signal: controller.signal }); expect(tlsConnectSpy).toHaveBeenCalledWith({ socket: fakeProxySocket, servername: "api.push.apple.com", diff --git a/src/infra/push-apns-http2.ts b/src/infra/push-apns-http2.ts index a8d239419f4e..719a31a87b34 100644 --- a/src/infra/push-apns-http2.ts +++ b/src/infra/push-apns-http2.ts @@ -37,6 +37,7 @@ type ApnsResponseBodyCapture = { type ConnectApnsHttp2SessionParams = { authority: string; timeoutMs: number; + signal?: AbortSignal; }; /** Parameters for validating APNs reachability through an explicit proxy. */ @@ -55,6 +56,10 @@ type ProbeApnsHttp2ReachabilityViaProxyResult = { responseHeaders: Record; }; +function apnsAbortError(signal: AbortSignal): Error { + return signal.reason instanceof Error ? signal.reason : new Error("APNs send invalidated"); +} + function assertApnsAuthority(authority: string): ApnsAuthority { let parsed: URL; try { @@ -106,6 +111,7 @@ async function openApnsTlsTunnel(params: { targetHost: string; targetPort: number; timeoutMs: number; + signal?: AbortSignal; }): Promise { // CONNECT ignores URL paths. Strip path metadata before Proxyline sees it so // tokens embedded in a configured proxy URL cannot surface in errors. @@ -117,9 +123,19 @@ async function openApnsTlsTunnel(params: { targetHost: params.targetHost, targetPort: params.targetPort, timeoutMs: params.timeoutMs, + ...(params.signal ? { signal: params.signal } : {}), }); const abortController = new AbortController(); + const abortFromCaller = () => { + if (params.signal) { + abortController.abort(apnsAbortError(params.signal)); + } + }; + params.signal?.addEventListener("abort", abortFromCaller, { once: true }); + if (params.signal?.aborted) { + abortFromCaller(); + } let targetTlsSocket: tls.TLSSocket | undefined; let timeout: NodeJS.Timeout | undefined; try { @@ -157,6 +173,7 @@ async function openApnsTlsTunnel(params: { if (timeout) { clearTimeout(timeout); } + params.signal?.removeEventListener("abort", abortFromCaller); abortController.abort(); } } @@ -166,6 +183,7 @@ async function openProxiedApnsHttp2Session(params: { proxyUrl: ActiveManagedProxyUrl; proxyTls?: ManagedProxyTlsOptions; timeoutMs: number; + signal?: AbortSignal; }): Promise { const apnsHost = new URL(params.authority).hostname; const tlsSocket = await openApnsTlsTunnel({ @@ -174,8 +192,14 @@ async function openProxiedApnsHttp2Session(params: { targetHost: apnsHost, targetPort: 443, timeoutMs: params.timeoutMs, + ...(params.signal ? { signal: params.signal } : {}), }); + if (params.signal?.aborted) { + tlsSocket.destroy(); + throw apnsAbortError(params.signal); + } + // The CONNECT helper already completed the target TLS handshake; reuse that // socket so the session cannot open a separate direct route. return http2.connect(params.authority, { @@ -191,6 +215,9 @@ export async function connectApnsHttp2Session( const timeoutMs = resolveApnsHttp2TimeoutMs(params.timeoutMs); const proxyUrl = getActiveManagedProxyUrl(); if (!proxyUrl) { + if (params.signal?.aborted) { + throw apnsAbortError(params.signal); + } return http2.connect(authority); } @@ -199,6 +226,7 @@ export async function connectApnsHttp2Session( proxyUrl, proxyTls: getActiveManagedProxyTlsOptions(), timeoutMs, + ...(params.signal ? { signal: params.signal } : {}), }); } diff --git a/src/infra/push-apns-store-transaction.ts b/src/infra/push-apns-store-transaction.ts new file mode 100644 index 000000000000..496a3e27e108 --- /dev/null +++ b/src/infra/push-apns-store-transaction.ts @@ -0,0 +1,75 @@ +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import type { OpenClawStateDatabase } from "../state/openclaw-state-db.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; + +type ApnsRegistrationDatabase = Pick< + OpenClawStateKyselyDatabase, + "apns_registrations" | "apns_registration_tombstones" +>; + +/** Advances a registration/tombstone version without reusing an observed owner version. */ +export function nextApnsRegistrationVersion( + nodeId: string, + previousVersions: readonly number[], +): number { + let latest = -1; + for (const version of previousVersions) { + if (!Number.isSafeInteger(version) || version < 0) { + throw new Error(`invalid APNs registration version for node ${nodeId}`); + } + latest = Math.max(latest, version); + } + if (latest === Number.MAX_SAFE_INTEGER) { + throw new Error(`APNs registration version exhausted for node ${nodeId}`); + } + return Math.max(Date.now(), latest + 1); +} + +/** Tombstones and deletes one APNs owner inside the caller's shared-state transaction. */ +export function clearApnsRegistrationFromDatabase( + db: OpenClawStateDatabase["db"], + nodeId: string, +): boolean { + const normalizedNodeId = nodeId.trim(); + if (!normalizedNodeId) { + return false; + } + const stateDb = getNodeSqliteKysely(db); + const currentRow = executeSqliteQueryTakeFirstSync( + db, + stateDb + .selectFrom("apns_registrations") + .select("updated_at_ms") + .where("node_id", "=", normalizedNodeId), + ); + const tombstone = executeSqliteQueryTakeFirstSync( + db, + stateDb + .selectFrom("apns_registration_tombstones") + .select("deleted_at_ms") + .where("node_id", "=", normalizedNodeId), + ); + const previousVersions = [currentRow?.updated_at_ms, tombstone?.deleted_at_ms].filter( + (version): version is number => version !== undefined, + ); + const deletedAtMs = nextApnsRegistrationVersion(normalizedNodeId, previousVersions); + // Tombstone even an empty row so a retired source cannot restore ownership. + executeSqliteQuerySync( + db, + stateDb + .insertInto("apns_registration_tombstones") + .values({ node_id: normalizedNodeId, deleted_at_ms: deletedAtMs }) + .onConflict((conflict) => + conflict.column("node_id").doUpdateSet({ deleted_at_ms: deletedAtMs }), + ), + ); + executeSqliteQuerySync( + db, + stateDb.deleteFrom("apns_registrations").where("node_id", "=", normalizedNodeId), + ); + return currentRow !== undefined; +} diff --git a/src/infra/push-apns-store.ts b/src/infra/push-apns-store.ts index e2fd51dcfc4b..b2dfb48568c0 100644 --- a/src/infra/push-apns-store.ts +++ b/src/infra/push-apns-store.ts @@ -10,11 +10,14 @@ import { runOpenClawStateWriteTransaction, type OpenClawStateDatabaseOptions, } from "../state/openclaw-state-db.js"; +import { loadPairedDevicePairingStoreRecordFromDatabase } from "./device-pairing-store.js"; +import { resolveNodePairingGeneration } from "./device-pairing.js"; import { executeSqliteQuerySync, executeSqliteQueryTakeFirstSync, getNodeSqliteKysely, } from "./kysely-sync.js"; +import { nextApnsRegistrationVersion } from "./push-apns-store-transaction.js"; import { normalizeApnsRelayBaseUrl, normalizePersistedApnsRelayBaseUrl, @@ -48,12 +51,20 @@ export type RelayApnsRegistration = { /** Stored APNs registration for either direct device tokens or official relay handles. */ export type ApnsRegistration = DirectApnsRegistration | RelayApnsRegistration; +export class ApnsRegistrationPairingChangedError extends Error { + constructor() { + super("node pairing changed before APNs registration"); + this.name = "ApnsRegistrationPairingChangedError"; + } +} + type RegisterDirectApnsParams = { nodeId: string; transport?: "direct"; token: string; topic: string; environment?: unknown; + expectedPairingGeneration?: string; baseDir?: string; }; @@ -68,6 +79,7 @@ type RegisterRelayApnsParams = { distribution?: unknown; relayOrigin?: unknown; tokenDebugSuffix?: unknown; + expectedPairingGeneration?: string; baseDir?: string; }; @@ -426,20 +438,6 @@ function apnsRegistrationsEqual(left: ApnsRegistration, right: ApnsRegistration) ); } -function nextApnsRegistrationVersion(nodeId: string, previousVersions: readonly number[]): number { - let latest = -1; - for (const version of previousVersions) { - if (!Number.isSafeInteger(version) || version < 0) { - throw new Error(`invalid APNs registration version for node ${nodeId}`); - } - latest = Math.max(latest, version); - } - if (latest === Number.MAX_SAFE_INTEGER) { - throw new Error(`APNs registration version exhausted for node ${nodeId}`); - } - return Math.max(Date.now(), latest + 1); -} - /** Persists a validated direct or relay APNs registration for one node id. */ export async function registerApnsRegistration( params: RegisterApnsParams, @@ -507,6 +505,16 @@ export async function registerApnsRegistration( } return runOpenClawStateWriteTransaction(({ db }) => { + if (params.expectedPairingGeneration) { + // The Gateway admission check happens before this transaction. Reread the + // pairing here so removal and APNs ownership cannot commit out of order. + const pairing = resolveNodePairingGeneration( + loadPairedDevicePairingStoreRecordFromDatabase(db, nodeId), + ); + if (pairing?.key !== params.expectedPairingGeneration) { + throw new ApnsRegistrationPairingChangedError(); + } + } const stateDb = getNodeSqliteKysely(db); const current = executeSqliteQueryTakeFirstSync( db, diff --git a/src/infra/push-apns.relay.test.ts b/src/infra/push-apns.relay.test.ts index a3a9025e760f..95af0c98d01d 100644 --- a/src/infra/push-apns.relay.test.ts +++ b/src/infra/push-apns.relay.test.ts @@ -214,6 +214,39 @@ describe("push-apns.relay", () => { }); describe("sendApnsRelayPush", () => { + it("revalidates ownership before relay fetch and combines the lifecycle signal", async () => { + const controller = new AbortController(); + const isCurrent = vi.fn().mockResolvedValue(true); + const fetchMock = vi.fn().mockResolvedValue(new Response("", { status: 202 })); + vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch); + + await sendApnsRelayPush({ + ...createRelayPushParams(), + signal: controller.signal, + isCurrent, + }); + + expect(isCurrent).toHaveBeenCalledTimes(2); + const fetchOptions = firstMockCall(fetchMock)?.[1] as { signal?: AbortSignal } | undefined; + expect(fetchOptions?.signal?.aborted).toBe(false); + controller.abort(new Error("pairing removed")); + expect(fetchOptions?.signal?.aborted).toBe(true); + }); + + it("does not start relay transport when persistent ownership changed", async () => { + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch); + + await expect( + sendApnsRelayPush({ + ...createRelayPushParams(), + isCurrent: vi.fn().mockResolvedValue(false), + }), + ).rejects.toThrow("APNs send invalidated"); + + expect(fetchMock).not.toHaveBeenCalled(); + }); + it("signs relay payloads and forwards the request through the injected sender", async () => { vi.spyOn(Date, "now").mockReturnValue(123_456_789); const sender = vi.fn().mockResolvedValue({ diff --git a/src/infra/push-apns.relay.ts b/src/infra/push-apns.relay.ts index 96c9d36580e0..352ab83d329f 100644 --- a/src/infra/push-apns.relay.ts +++ b/src/infra/push-apns.relay.ts @@ -57,6 +57,8 @@ export type ApnsRelayRequestSender = (params: { pushType: ApnsRelayPushType; priority: "10" | "5"; payload: object; + signal?: AbortSignal; + isCurrent?: () => Promise; }) => Promise; /** Hosted APNs relay origin used only when registrations prove they were minted there. */ @@ -76,6 +78,24 @@ function normalizeNonEmptyString(value: string | undefined): string | null { return trimmed.length > 0 ? trimmed : null; } +function throwIfApnsRelaySendAborted(signal: AbortSignal | undefined): void { + if (!signal?.aborted) { + return; + } + throw signal.reason instanceof Error ? signal.reason : new Error("APNs send invalidated"); +} + +async function requireCurrentApnsRelaySend(params: { + signal?: AbortSignal; + isCurrent?: () => Promise; +}): Promise { + throwIfApnsRelaySendAborted(params.signal); + if (params.isCurrent && !(await params.isCurrent())) { + throw new Error("APNs send invalidated"); + } + throwIfApnsRelaySendAborted(params.signal); +} + function normalizeTimeoutMs(value: string | number | undefined): number { const raw = typeof value === "number" @@ -279,7 +299,12 @@ async function sendApnsRelayRequest(params: { pushType: ApnsRelayPushType; priority: "10" | "5"; payload: object; + signal?: AbortSignal; + isCurrent?: () => Promise; }): Promise { + await requireCurrentApnsRelaySend(params); + const timeoutSignal = AbortSignal.timeout(params.relayConfig.timeoutMs); + const signal = params.signal ? AbortSignal.any([params.signal, timeoutSignal]) : timeoutSignal; const response = await fetch(`${params.relayConfig.baseUrl}/v1/push/send`, { method: "POST", redirect: "manual", @@ -291,7 +316,7 @@ async function sendApnsRelayRequest(params: { [GATEWAY_SIGNED_AT_HEADER]: String(params.signedAtMs), }, body: params.bodyJson, - signal: AbortSignal.timeout(params.relayConfig.timeoutMs), + signal, }); // Do not follow relay redirects; grants and signatures are scoped to the configured relay origin. if (response.status >= 300 && response.status < 400) { @@ -353,7 +378,10 @@ export async function sendApnsRelayPush(params: { payload: object; gatewayIdentity?: Pick; requestSender?: ApnsRelayRequestSender; + signal?: AbortSignal; + isCurrent?: () => Promise; }): Promise { + await requireCurrentApnsRelaySend(params); const sender = params.requestSender ?? sendApnsRelayRequest; const gatewayIdentity = params.gatewayIdentity ?? loadOrCreateProcessDeviceIdentity(); const signedAtMs = Date.now(); @@ -382,5 +410,7 @@ export async function sendApnsRelayPush(params: { pushType: params.pushType, priority: params.priority, payload: params.payload, + ...(params.signal ? { signal: params.signal } : {}), + ...(params.isCurrent ? { isCurrent: params.isCurrent } : {}), }); } diff --git a/src/infra/push-apns.store.test.ts b/src/infra/push-apns.store.test.ts index 55789af8114a..c6491d25a1fa 100644 --- a/src/infra/push-apns.store.test.ts +++ b/src/infra/push-apns.store.test.ts @@ -9,8 +9,11 @@ import { runOpenClawStateWriteTransaction, } from "../state/openclaw-state-db.js"; import { createTrackedTempDirs } from "../test-utils/tracked-temp-dirs.js"; +import { persistDevicePairingStoreState } from "./device-pairing-store.js"; +import { resolveNodePairingGeneration, type PairedDevice } from "./device-pairing.js"; import { executeSqliteQuerySync, getNodeSqliteKysely } from "./kysely-sync.js"; import { + ApnsRegistrationPairingChangedError, clearApnsRegistrationIfCurrent, loadApnsRegistration, loadApnsRegistrations, @@ -32,6 +35,7 @@ async function registerDirectApnsRegistration(params: { token?: string; topic?: string; environment?: unknown; + expectedPairingGeneration?: string; baseDir: string; }) { return await registerApnsRegistration({ @@ -244,6 +248,94 @@ describe("push APNs registration store", () => { await expect(loadApnsRegistration("ios-node-1", baseDir)).resolves.toEqual(replacement); }); + it("rejects a stale registration after the expected pairing generation is removed", async () => { + const baseDir = await makeTempDir(); + const nodeId = "ios-node-generation-guard"; + const pairedDevice: PairedDevice = { + deviceId: nodeId, + publicKey: "public-key-generation-a", + role: "node", + roles: ["node"], + tokens: { + node: { + token: "node-token-generation-a", + role: "node", + scopes: [], + createdAtMs: 100, + }, + }, + nodeSurface: { + createdAtMs: 200, + approvedAtMs: 300, + }, + createdAtMs: 50, + approvedAtMs: 300, + }; + persistDevicePairingStoreState( + { pendingById: {}, pairedByDeviceId: { [nodeId]: pairedDevice } }, + baseDir, + "paired", + ); + const generation = resolveNodePairingGeneration(pairedDevice); + if (!generation) { + throw new Error("expected node pairing generation"); + } + + await registerDirectApnsRegistration({ + nodeId, + expectedPairingGeneration: generation.key, + baseDir, + }); + persistDevicePairingStoreState({ pendingById: {}, pairedByDeviceId: {} }, baseDir, "paired", { + clearApnsNodeIds: [nodeId], + }); + await expect(loadApnsRegistration(nodeId, baseDir)).resolves.toBeNull(); + + const replacementDevice: PairedDevice = { + ...pairedDevice, + publicKey: "public-key-generation-b", + tokens: { + node: { + token: "node-token-generation-b", + role: "node", + scopes: [], + createdAtMs: 101, + }, + }, + nodeSurface: { + ...pairedDevice.nodeSurface, + createdAtMs: 200, + approvedAtMs: 301, + }, + }; + persistDevicePairingStoreState( + { pendingById: {}, pairedByDeviceId: { [nodeId]: replacementDevice } }, + baseDir, + "paired", + ); + const replacementGeneration = resolveNodePairingGeneration(replacementDevice); + if (!replacementGeneration) { + throw new Error("expected replacement node pairing generation"); + } + const replacement = await registerDirectApnsRegistration({ + nodeId, + token: "DCBA4321DCBA4321DCBA4321DCBA4321", + expectedPairingGeneration: replacementGeneration.key, + baseDir, + }); + await expect(loadApnsRegistration(nodeId, baseDir)).resolves.toEqual(replacement); + + await expect( + registerDirectApnsRegistration({ + nodeId, + token: "ABCD1234ABCD1234ABCD1234ABCD1234", + expectedPairingGeneration: generation.key, + baseDir, + }), + ).rejects.toBeInstanceOf(ApnsRegistrationPairingChangedError); + await expect(loadApnsRegistration(nodeId, baseDir)).resolves.toEqual(replacement); + }); + it("rejects invalid direct and relay inputs", async () => { const baseDir = await makeTempDir(); const oversized = "x".repeat(257); diff --git a/src/infra/push-apns.test.ts b/src/infra/push-apns.test.ts index 79e28fddbd19..f89cdc7e0a94 100644 --- a/src/infra/push-apns.test.ts +++ b/src/infra/push-apns.test.ts @@ -1,10 +1,12 @@ // Tests APNS push signing and request construction. import { generateKeyPairSync } from "node:crypto"; +import { EventEmitter } from "node:events"; import { createServer, type Server as HttpServer } from "node:http"; import http2 from "node:http2"; import net from "node:net"; import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { createDeferred } from "../test-utils/deferred.js"; import { startProxy, stopProxy, type ProxyHandle } from "./net/proxy/proxy-lifecycle.js"; import { appendApnsResponseBodyCapture, @@ -510,6 +512,94 @@ describe("push APNs send semantics", () => { expect(result.transport).toBe("direct"); }); + it("guards direct wake transport with current ownership and lifecycle", async () => { + const { send, registration, auth } = createDirectApnsSendFixture({ + nodeId: "ios-node-guarded-wake", + environment: "production", + sendResult: { + status: 200, + apnsId: "apns-guarded-wake-id", + body: "", + }, + }); + const controller = new AbortController(); + const isCurrent = vi.fn().mockResolvedValue(true); + + await sendApnsBackgroundWake({ + registration, + nodeId: "ios-node-guarded-wake", + wakeReason: "node.invoke", + auth, + requestSender: send, + signal: controller.signal, + isCurrent, + }); + + expect(isCurrent).toHaveBeenCalledTimes(1); + const sent = requireSendRequest(send); + expect(sent.signal).toBe(controller.signal); + expect(sent.isCurrent).toBe(isCurrent); + + await expect( + sendApnsBackgroundWake({ + registration, + nodeId: "ios-node-guarded-wake", + wakeReason: "node.invoke", + auth, + requestSender: send, + isCurrent: vi.fn().mockResolvedValue(false), + }), + ).rejects.toThrow("APNs send invalidated"); + expect(send).toHaveBeenCalledTimes(1); + }); + + it("owns direct session errors while persistent currentness is pending", async () => { + const session = Object.assign(new EventEmitter(), { + close: vi.fn(), + destroy: vi.fn(), + request: vi.fn(), + }); + session.destroy.mockImplementation(() => session.emit("close")); + const connect = vi + .spyOn(http2, "connect") + .mockReturnValue(session as unknown as http2.ClientHttp2Session); + const currentness = createDeferred(); + const isCurrent = vi + .fn<() => Promise>() + .mockResolvedValueOnce(true) + .mockReturnValueOnce(currentness.promise); + const { registration, auth } = createDirectApnsSendFixture({ + nodeId: "ios-node-session-error", + environment: "production", + sendResult: { status: 200, apnsId: "unused", body: "" }, + }); + + try { + const sending = sendApnsBackgroundWake({ + registration, + nodeId: "ios-node-session-error", + wakeReason: "node.invoke", + auth, + isCurrent, + }); + await vi.waitFor(() => { + expect(connect).toHaveBeenCalledTimes(1); + expect(isCurrent).toHaveBeenCalledTimes(2); + }); + + expect(session.listenerCount("error")).toBeGreaterThan(0); + session.emit("error", new Error("APNs connection failed")); + currentness.resolve(true); + + await expect(sending).rejects.toThrow("APNs connection failed"); + expect(session.destroy).toHaveBeenCalledTimes(1); + expect(session.request).not.toHaveBeenCalled(); + } finally { + currentness.resolve(false); + connect.mockRestore(); + } + }); + it("sends exec approval alert pushes with generic modal-only metadata", async () => { const { send, registration, auth } = createDirectApnsSendFixture({ nodeId: "ios-node-approval-alert", diff --git a/src/infra/push-apns.ts b/src/infra/push-apns.ts index 8d5cf94aba8c..176062ea68cc 100644 --- a/src/infra/push-apns.ts +++ b/src/infra/push-apns.ts @@ -1,11 +1,10 @@ // Manages APNs registration state and direct/relay push sending. -import { createHash, createPrivateKey, sign as signJwt } from "node:crypto"; -import fs from "node:fs/promises"; import { resolveTimerTimeoutMs } from "@openclaw/normalization-core/number-coercion"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import type { DeviceIdentity } from "./device-identity.js"; -import { formatErrorMessage, toErrorObject } from "./errors.js"; +import { toErrorObject } from "./errors.js"; +import { getApnsBearerToken, type ApnsAuthConfig } from "./push-apns-auth.js"; import { APNS_HTTP2_CANCEL_CODE, appendApnsResponseBodyCapture, @@ -40,6 +39,7 @@ import { } from "./push-apns.relay.js"; export { + ApnsRegistrationPairingChangedError, clearApnsRegistrationIfCurrent, loadApnsRegistration, loadApnsRegistrations, @@ -47,18 +47,11 @@ export { registerApnsRegistration, } from "./push-apns-store.js"; export type { ApnsRegistration } from "./push-apns-store.js"; +export { resolveApnsAuthConfigFromEnv } from "./push-apns-auth.js"; +export type { ApnsAuthConfig } from "./push-apns-auth.js"; type ApnsTransport = "direct" | "relay"; -/** Direct APNs provider authentication used to mint ES256 bearer tokens. */ -export type ApnsAuthConfig = { - teamId: string; - keyId: string; - privateKey: string; -}; - -type ApnsAuthConfigResolution = { ok: true; value: ApnsAuthConfig } | { ok: false; error: string }; - /** Normalized APNs push result returned to gateway push/nodes methods. */ type ApnsPushResult = { ok: boolean; @@ -88,16 +81,33 @@ type ApnsRequestParams = { timeoutMs: number; pushType: ApnsPushType; priority: "10" | "5"; + signal?: AbortSignal; + isCurrent?: () => Promise; }; type ApnsRequestResponse = { status: number; apnsId?: string; body: string }; type ApnsRequestSender = (params: ApnsRequestParams) => Promise; -const APNS_JWT_TTL_MS = 50 * 60 * 1000; const DEFAULT_APNS_TIMEOUT_MS = 10_000; -let cachedJwt: { cacheKey: string; token: string; expiresAtMs: number } | null = null; +function throwIfApnsSendAborted(signal: AbortSignal | undefined): void { + if (!signal?.aborted) { + return; + } + throw signal.reason instanceof Error ? signal.reason : new Error("APNs send invalidated"); +} + +async function requireCurrentApnsSend(params: { + signal?: AbortSignal; + isCurrent?: () => Promise; +}): Promise { + throwIfApnsSendAborted(params.signal); + if (params.isCurrent && !(await params.isCurrent())) { + throw new Error("APNs send invalidated"); + } + throwIfApnsSendAborted(params.signal); +} function parseReason(body: string): string | undefined { const trimmed = body.trim(); @@ -114,57 +124,6 @@ function parseReason(body: string): string | undefined { } } -function toBase64UrlBytes(value: Uint8Array): string { - return Buffer.from(value) - .toString("base64") - .replace(/\+/g, "-") - .replace(/\//g, "_") - .replace(/=+$/g, ""); -} - -function toBase64UrlJson(value: object): string { - return toBase64UrlBytes(Buffer.from(JSON.stringify(value))); -} - -function getJwtCacheKey(auth: ApnsAuthConfig): string { - const keyHash = createHash("sha256").update(auth.privateKey).digest("hex"); - return `${auth.teamId}:${auth.keyId}:${keyHash}`; -} - -function getApnsBearerToken(auth: ApnsAuthConfig, nowMs: number = Date.now()): string { - const cacheKey = getJwtCacheKey(auth); - if (cachedJwt && cachedJwt.cacheKey === cacheKey && nowMs < cachedJwt.expiresAtMs) { - return cachedJwt.token; - } - - // APNs provider tokens are valid for one hour. Cache for slightly less so - // bursty wake/approval pushes avoid repeated ECDSA signing. - const iat = Math.floor(nowMs / 1000); - const header = toBase64UrlJson({ alg: "ES256", kid: auth.keyId, typ: "JWT" }); - const payload = toBase64UrlJson({ iss: auth.teamId, iat }); - const signingInput = `${header}.${payload}`; - const signature = signJwt("sha256", Buffer.from(signingInput, "utf8"), { - key: createPrivateKey(auth.privateKey), - dsaEncoding: "ieee-p1363", - }); - const token = `${signingInput}.${toBase64UrlBytes(signature)}`; - cachedJwt = { - cacheKey, - token, - expiresAtMs: nowMs + APNS_JWT_TTL_MS, - }; - return token; -} - -function normalizePrivateKey(value: string): string { - return value.trim().replace(/\\n/g, "\n"); -} - -function normalizeNonEmptyString(value: string | undefined): string | null { - const trimmed = normalizeOptionalString(value) ?? ""; - return trimmed.length > 0 ? trimmed : null; -} - /** Returns true for APNs responses that mean the direct device token is no longer usable. */ function shouldInvalidateApnsRegistration(result: { status: number; reason?: string }): boolean { if (result.status === 410) { @@ -191,60 +150,6 @@ export function shouldClearStoredApnsRegistration(params: { return shouldInvalidateApnsRegistration(params.result); } -/** Resolves direct APNs provider auth from env, accepting inline or file-backed keys. */ -export async function resolveApnsAuthConfigFromEnv( - env: NodeJS.ProcessEnv = process.env, -): Promise { - const teamId = normalizeNonEmptyString(env.OPENCLAW_APNS_TEAM_ID); - const keyId = normalizeNonEmptyString(env.OPENCLAW_APNS_KEY_ID); - if (!teamId || !keyId) { - return { - ok: false, - error: "APNs auth missing: set OPENCLAW_APNS_TEAM_ID and OPENCLAW_APNS_KEY_ID", - }; - } - - const inlineKeyRaw = - normalizeNonEmptyString(env.OPENCLAW_APNS_PRIVATE_KEY_P8) ?? - normalizeNonEmptyString(env.OPENCLAW_APNS_PRIVATE_KEY); - if (inlineKeyRaw) { - return { - ok: true, - value: { - teamId, - keyId, - privateKey: normalizePrivateKey(inlineKeyRaw), - }, - }; - } - - const keyPath = normalizeNonEmptyString(env.OPENCLAW_APNS_PRIVATE_KEY_PATH); - if (!keyPath) { - return { - ok: false, - error: - "APNs private key missing: set OPENCLAW_APNS_PRIVATE_KEY_P8 or OPENCLAW_APNS_PRIVATE_KEY_PATH", - }; - } - try { - const privateKey = normalizePrivateKey(await fs.readFile(keyPath, "utf8")); - return { - ok: true, - value: { - teamId, - keyId, - privateKey, - }, - }; - } catch (err) { - const message = formatErrorMessage(err); - return { - ok: false, - error: `failed reading OPENCLAW_APNS_PRIVATE_KEY_PATH (${keyPath}): ${message}`, - }; - } -} - async function sendApnsRequest(params: { token: string; topic: string; @@ -254,6 +159,8 @@ async function sendApnsRequest(params: { timeoutMs: number; pushType: ApnsPushType; priority: "10" | "5"; + signal?: AbortSignal; + isCurrent?: () => Promise; }): Promise { const authority = params.environment === "production" @@ -266,66 +173,124 @@ async function sendApnsRequest(params: { const client = await connectApnsHttp2Session({ authority, timeoutMs: params.timeoutMs, + ...(params.signal ? { signal: params.signal } : {}), }); + // Connection failures can arrive while the persistent ownership check is + // yielding. Keep a consuming owner until the session closes, while the + // request-specific listener below still rejects the active send. + const consumeSessionError = () => undefined; + client.on("error", consumeSessionError); + client.once("close", () => client.off("error", consumeSessionError)); + return await new Promise((resolve, reject) => { let settled = false; - const fail = (err: unknown) => { + let activeRequest: ReturnType | undefined; + const cleanup = () => { + client.off("error", fail); + params.signal?.removeEventListener("abort", onAbort); + }; + const fail = (err: unknown, options?: { cancelRequest?: boolean }) => { if (settled) { return; } settled = true; - client.destroy(); + cleanup(); + if (options?.cancelRequest && activeRequest && !activeRequest.destroyed) { + activeRequest.close(APNS_HTTP2_CANCEL_CODE); + client.close(); + } else { + client.destroy(); + } reject(toErrorObject(err, "Non-Error rejection")); }; + const onAbort = () => + fail( + params.signal?.reason instanceof Error + ? params.signal.reason + : new Error("APNs send invalidated"), + { cancelRequest: true }, + ); const finish = (result: { status: number; apnsId?: string; body: string }) => { if (settled) { return; } settled = true; + cleanup(); client.close(); resolve(result); }; - client.once("error", (err) => fail(err)); + const startRequest = async () => { + try { + await requireCurrentApnsSend(params); + if (settled) { + return; + } + if (params.signal?.aborted) { + onAbort(); + return; + } - const req = client.request({ - ":method": "POST", - ":path": requestPath, - authorization: `bearer ${params.bearerToken}`, - "apns-topic": params.topic, - "apns-push-type": params.pushType, - "apns-priority": params.priority, - "apns-expiration": "0", - "content-type": "application/json", - "content-length": Buffer.byteLength(body).toString(), - }); + const req = client.request({ + ":method": "POST", + ":path": requestPath, + authorization: `bearer ${params.bearerToken}`, + "apns-topic": params.topic, + "apns-push-type": params.pushType, + "apns-priority": params.priority, + "apns-expiration": "0", + "content-type": "application/json", + "content-length": Buffer.byteLength(body).toString(), + }); + activeRequest = req; - let statusCode = 0; - let apnsId: string | undefined; - const responseBody = createApnsResponseBodyCapture(); + let statusCode = 0; + let apnsId: string | undefined; + const responseBody = createApnsResponseBodyCapture(); - req.setTimeout(params.timeoutMs, () => { - req.close(APNS_HTTP2_CANCEL_CODE); - fail(new Error(`APNs request timed out after ${params.timeoutMs}ms`)); - }); - req.on("response", (headers) => { - const statusHeader = headers[":status"]; - statusCode = statusHeader ?? 0; - const idHeader = headers["apns-id"]; - if (typeof idHeader === "string" && idHeader.trim().length > 0) { - apnsId = idHeader.trim(); + req.setTimeout(params.timeoutMs, () => { + fail(new Error(`APNs request timed out after ${params.timeoutMs}ms`), { + cancelRequest: true, + }); + }); + req.on("response", (headers) => { + const statusHeader = headers[":status"]; + statusCode = statusHeader ?? 0; + const idHeader = headers["apns-id"]; + if (typeof idHeader === "string" && idHeader.trim().length > 0) { + apnsId = idHeader.trim(); + } + }); + req.on("data", (chunk) => { + appendApnsResponseBodyCapture(responseBody, chunk); + }); + req.on("end", () => { + finish({ + status: statusCode, + apnsId, + body: getApnsResponseBodyCaptureText(responseBody), + }); + }); + req.on("error", (err) => fail(err)); + + if (params.signal?.aborted) { + onAbort(); + return; + } + req.end(body); + } catch (error) { + fail(error); } - }); - req.on("data", (chunk) => { - appendApnsResponseBodyCapture(responseBody, chunk); - }); - req.on("end", () => { - finish({ status: statusCode, apnsId, body: getApnsResponseBodyCaptureText(responseBody) }); - }); - req.on("error", (err) => fail(err)); + }; - req.end(body); + client.once("error", fail); + params.signal?.addEventListener("abort", onAbort, { once: true }); + if (params.signal?.aborted) { + onAbort(); + return; + } + void startRequest(); }); } @@ -411,11 +376,14 @@ async function sendDirectApnsPush(params: { requestSender?: ApnsRequestSender; pushType: ApnsPushType; priority: "10" | "5"; + signal?: AbortSignal; + isCurrent?: () => Promise; }): Promise { const { token, topic, environment, bearerToken } = resolveDirectSendContext({ auth: params.auth, registration: params.registration, }); + await requireCurrentApnsSend(params); const sender = params.requestSender ?? sendApnsRequest; const response = await sender({ token, @@ -426,6 +394,8 @@ async function sendDirectApnsPush(params: { timeoutMs: resolveApnsTimeoutMs(params.timeoutMs), pushType: params.pushType, priority: params.priority, + ...(params.signal ? { signal: params.signal } : {}), + ...(params.isCurrent ? { isCurrent: params.isCurrent } : {}), }); return toPushResult({ registration: params.registration, @@ -442,6 +412,8 @@ async function sendRelayApnsPush(params: { priority: "10" | "5"; gatewayIdentity?: Pick; requestSender?: ApnsRelayRequestSender; + signal?: AbortSignal; + isCurrent?: () => Promise; }): Promise { const response = await sendApnsRelayPush({ relayConfig: params.relayConfig, @@ -452,6 +424,8 @@ async function sendRelayApnsPush(params: { priority: params.priority, gatewayIdentity: params.gatewayIdentity, requestSender: params.requestSender, + ...(params.signal ? { signal: params.signal } : {}), + ...(params.isCurrent ? { isCurrent: params.isCurrent } : {}), }); return toPushResult({ registration: params.registration, response }); } @@ -461,6 +435,8 @@ type ApnsAlertCommonParams = { title: string; body: string; timeoutMs?: number; + signal?: AbortSignal; + isCurrent?: () => Promise; }; type DirectApnsAlertParams = ApnsAlertCommonParams & { @@ -484,6 +460,8 @@ type ApnsBackgroundWakeCommonParams = { nodeId: string; wakeReason?: string; timeoutMs?: number; + signal?: AbortSignal; + isCurrent?: () => Promise; }; type DirectApnsBackgroundWakeParams = ApnsBackgroundWakeCommonParams & { @@ -554,6 +532,8 @@ export async function sendApnsAlert( priority: "10", gatewayIdentity: relayParams.relayGatewayIdentity, requestSender: relayParams.relayRequestSender, + ...(relayParams.signal ? { signal: relayParams.signal } : {}), + ...(relayParams.isCurrent ? { isCurrent: relayParams.isCurrent } : {}), }); } const directParams = params as DirectApnsAlertParams; @@ -565,6 +545,8 @@ export async function sendApnsAlert( requestSender: directParams.requestSender, pushType: "alert", priority: "10", + ...(directParams.signal ? { signal: directParams.signal } : {}), + ...(directParams.isCurrent ? { isCurrent: directParams.isCurrent } : {}), }); } @@ -587,6 +569,8 @@ export async function sendApnsBackgroundWake( priority: "5", gatewayIdentity: relayParams.relayGatewayIdentity, requestSender: relayParams.relayRequestSender, + ...(relayParams.signal ? { signal: relayParams.signal } : {}), + ...(relayParams.isCurrent ? { isCurrent: relayParams.isCurrent } : {}), }); } const directParams = params as DirectApnsBackgroundWakeParams; @@ -598,6 +582,8 @@ export async function sendApnsBackgroundWake( requestSender: directParams.requestSender, pushType: "background", priority: "5", + ...(directParams.signal ? { signal: directParams.signal } : {}), + ...(directParams.isCurrent ? { isCurrent: directParams.isCurrent } : {}), }); } diff --git a/src/skills/runtime/remote-probe-utils.ts b/src/skills/runtime/remote-probe-utils.ts new file mode 100644 index 000000000000..a2ac67113125 --- /dev/null +++ b/src/skills/runtime/remote-probe-utils.ts @@ -0,0 +1,116 @@ +// Pure platform and payload helpers for remote skill binary probes. +import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; +import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; +import { normalizeStringEntries } from "@openclaw/normalization-core/string-normalization"; +import type { SkillEntry } from "../types.js"; + +export function extractErrorMessage(err: unknown): string | undefined { + if (!err) { + return undefined; + } + if (typeof err === "string") { + return err; + } + if (err instanceof Error) { + return err.message; + } + if (typeof err === "object" && "message" in err && typeof err.message === "string") { + return err.message; + } + if (typeof err === "number" || typeof err === "boolean" || typeof err === "bigint") { + return String(err); + } + if (typeof err === "symbol") { + return err.toString(); + } + if (typeof err === "object") { + try { + return JSON.stringify(err); + } catch { + return undefined; + } + } + return undefined; +} + +export function isMacPlatform(platform?: string, deviceFamily?: string): boolean { + const platformNorm = normalizeLowercaseStringOrEmpty(platform); + const familyNorm = normalizeLowercaseStringOrEmpty(deviceFamily); + return platformNorm.includes("mac") || platformNorm.includes("darwin") || familyNorm === "mac"; +} + +export function supportsSystemRun(commands?: string[]): boolean { + return Array.isArray(commands) && commands.includes("system.run"); +} + +export function supportsSystemWhich(commands?: string[]): boolean { + return Array.isArray(commands) && commands.includes("system.which"); +} + +export function collectRequiredBins(entries: SkillEntry[], targetPlatform: string): string[] { + const bins = new Set(); + for (const entry of entries) { + const os = entry.metadata?.os ?? []; + if (os.length > 0 && !os.includes(targetPlatform)) { + continue; + } + for (const bin of [ + ...(entry.metadata?.requires?.bins ?? []), + ...(entry.metadata?.requires?.anyBins ?? []), + ]) { + if (bin.trim()) { + bins.add(bin.trim()); + } + } + } + return [...bins]; +} + +export function buildBinProbeScript(bins: string[]): string { + const escaped = bins.map((bin) => `'${bin.replace(/'/g, `'\\''`)}'`).join(" "); + return `for b in ${escaped}; do if command -v "$b" >/dev/null 2>&1; then echo "$b"; fi; done`; +} + +export function parseBinProbePayload( + payloadJSON: string | null | undefined, + payload?: unknown, +): string[] { + if (!payloadJSON && !payload) { + return []; + } + try { + const parsed = payloadJSON + ? (JSON.parse(payloadJSON) as { stdout?: unknown; bins?: unknown }) + : (payload as { stdout?: unknown; bins?: unknown }); + if (Array.isArray(parsed.bins)) { + return normalizeStringEntries(parsed.bins); + } + if (parsed.bins && typeof parsed.bins === "object") { + return Object.entries(parsed.bins) + .filter(([, resolvedPath]) => normalizeOptionalString(resolvedPath) !== undefined) + .map(([bin]) => normalizeOptionalString(bin) ?? "") + .filter(Boolean); + } + if (typeof parsed.stdout === "string") { + return parsed.stdout + .split(/\r?\n/) + .map((line) => normalizeOptionalString(line) ?? "") + .filter(Boolean); + } + } catch { + return []; + } + return []; +} + +export function areBinSetsEqual(a: Set | undefined, b: Set): boolean { + if (!a || a.size !== b.size) { + return false; + } + for (const bin of b) { + if (!a.has(bin)) { + return false; + } + } + return true; +} diff --git a/src/skills/runtime/remote-reconciliation.test.ts b/src/skills/runtime/remote-reconciliation.test.ts new file mode 100644 index 000000000000..5ddeea6ddef4 --- /dev/null +++ b/src/skills/runtime/remote-reconciliation.test.ts @@ -0,0 +1,78 @@ +import { randomUUID } from "node:crypto"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { NodeRegistry } from "../../gateway/node-registry.js"; +import { mergeRemoteNodeSkillEntries, replaceRemoteNodeSkills } from "./remote-skills.js"; +import { + getRemoteSkillEligibility, + recordRemoteNodeBins, + recordRemoteNodeInfo, + removeRemoteNodeInfo, + setSkillsRemoteRegistry, +} from "./remote.js"; + +const TEST_PAIRING_GENERATION = "generation-test"; + +afterEach(() => { + setSkillsRemoteRegistry(null); + vi.restoreAllMocks(); +}); + +describe("remote projection reconciliation", () => { + it("rejects legacy-generation and connectionless node projections", () => { + const legacyNodeId = `node-${randomUUID()}`; + const missingConnectionNodeId = `node-${randomUUID()}`; + const bin = `bin-${randomUUID()}`; + const legacySkillName = `skill-${randomUUID()}`; + const missingConnectionSkillName = `skill-${randomUUID()}`; + const listCurrentConnectedSync = vi.fn(() => []); + try { + setSkillsRemoteRegistry({ + listCurrentConnectedSync, + } as unknown as NodeRegistry); + recordRemoteNodeInfo({ + nodeId: legacyNodeId, + connId: "conn-retired", + displayName: "Legacy Mac", + platform: "darwin", + commands: ["system.run"], + }); + replaceRemoteNodeSkills({ + nodeId: legacyNodeId, + displayName: "Legacy Mac", + skills: [ + { + name: legacySkillName, + description: "Legacy remote skill", + content: `---\nname: ${legacySkillName}\ndescription: Legacy remote skill\n---\n`, + }, + ], + }); + recordRemoteNodeInfo({ + nodeId: missingConnectionNodeId, + pairingGeneration: TEST_PAIRING_GENERATION, + displayName: "Retired Mac", + platform: "darwin", + commands: ["system.run", "system.which"], + }); + recordRemoteNodeBins(missingConnectionNodeId, [bin], TEST_PAIRING_GENERATION); + replaceRemoteNodeSkills({ + nodeId: missingConnectionNodeId, + displayName: "Retired Mac", + skills: [ + { + name: missingConnectionSkillName, + description: "Retired remote skill", + content: `---\nname: ${missingConnectionSkillName}\ndescription: Retired remote skill\n---\n`, + }, + ], + }); + + expect(getRemoteSkillEligibility()?.hasBin(bin) ?? false).toBe(false); + expect(mergeRemoteNodeSkillEntries([], { canExec: true })).toEqual([]); + expect(listCurrentConnectedSync).toHaveBeenCalled(); + } finally { + removeRemoteNodeInfo(legacyNodeId); + removeRemoteNodeInfo(missingConnectionNodeId); + } + }); +}); diff --git a/src/skills/runtime/remote-skills.ts b/src/skills/runtime/remote-skills.ts index c7e6493fd976..34f92a652994 100644 --- a/src/skills/runtime/remote-skills.ts +++ b/src/skills/runtime/remote-skills.ts @@ -22,6 +22,18 @@ type RemoteSkillNode = { const remoteSkillNodes = new Map(); const log = createSubsystemLogger("gateway/skills-remote"); +let reconcileRemoteSkillConnections: (() => ReadonlySet | undefined) | null = null; + +function remoteConnectionKey(nodeId: string, connId: string): string { + return `${nodeId}\0${connId}`; +} + +/** Installs the gateway-owned persistent-generation reconciliation boundary. */ +export function setRemoteSkillConnectionReconciler( + reconcile: (() => ReadonlySet | undefined) | null, +): void { + reconcileRemoteSkillConnections = reconcile; +} function prepareNodeSkills( nodeId: string, @@ -168,8 +180,14 @@ export function mergeRemoteNodeSkillEntries( if (options?.canExec !== true) { return [...localEntries]; } + const currentConnections = reconcileRemoteSkillConnections?.(); const connectedNodes = [...remoteSkillNodes.values()].filter( - (node) => node.connected && node.canExec, + (node) => + node.connected && + node.canExec && + (!currentConnections || + (node.connId !== undefined && + currentConnections.has(remoteConnectionKey(node.nodeId, node.connId)))), ); let boundNodeId: string | undefined; if (options.node) { @@ -247,6 +265,7 @@ export function mergeRemoteNodeSkillEntries( function resetRemoteNodeSkillsForTests(): void { remoteSkillNodes.clear(); + reconcileRemoteSkillConnections = null; } if (process.env.VITEST || process.env.NODE_ENV === "test") { diff --git a/src/skills/runtime/remote.test.ts b/src/skills/runtime/remote.test.ts index 947e882746fe..7e1b67313a5d 100644 --- a/src/skills/runtime/remote.test.ts +++ b/src/skills/runtime/remote.test.ts @@ -8,16 +8,53 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js"; import type { NodeRegistry } from "../../gateway/node-registry.js"; import { getSkillsSnapshotVersion } from "./refresh-state.js"; import { resetSkillsRefreshForTest } from "./refresh.test-support.js"; +import { mergeRemoteNodeSkillEntries, replaceRemoteNodeSkills } from "./remote-skills.js"; + +vi.mock("../../infra/node-pairing.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, updatePairedNodeBins: vi.fn(async () => true) }; +}); + import { getRemoteSkillEligibility, recordRemoteNodeBins, recordRemoteNodeInfo, removeRemoteNodeInfo, + removeRemoteNodeInfoForConnection, refreshRemoteBinsForConnectedNodes, refreshRemoteNodeBins, setSkillsRemoteRegistry, } from "./remote.js"; +const TEST_PAIRING_GENERATION = "generation-test"; + +function testRemoteSession( + nodeId: string, + overrides?: Partial>>, +): NonNullable> { + return { + nodeId, + connId: `conn-${nodeId}`, + pairingGeneration: TEST_PAIRING_GENERATION, + platform: "darwin", + commands: ["system.run", "system.which"], + ...overrides, + } as NonNullable>; +} + +function setTestSkillsRemoteRegistry( + nodeIds: string | readonly string[], + registry: Partial & Pick, +): void { + const ids = typeof nodeIds === "string" ? [nodeIds] : nodeIds; + setSkillsRemoteRegistry({ + ...registry, + listCurrentConnectedSync: + registry.listCurrentConnectedSync ?? + (() => ids.flatMap((nodeId) => (registry.get(nodeId) ? [registry.get(nodeId)!] : []))), + } as unknown as NodeRegistry); +} + function createRemoteSkillWorkspace(bin: string): { cfg: OpenClawConfig; workspaceDir: string } { const workspaceDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-remote-skills-")); const skillDir = path.join(workspaceDir, "skills", "remote-skill"); @@ -49,6 +86,8 @@ function createRemoteSkillWorkspace(bin: string): { cfg: OpenClawConfig; workspa function recordRemoteMacWithSystemWhich(nodeId: string): void { recordRemoteNodeInfo({ nodeId, + connId: `conn-${nodeId}`, + pairingGeneration: TEST_PAIRING_GENERATION, displayName: "Remote Mac", platform: "darwin", commands: ["system.run", "system.which"], @@ -71,7 +110,7 @@ describe("skills-remote", () => { platform: "darwin", commands: ["system.run"], }); - recordRemoteNodeBins(nodeId, [bin]); + recordRemoteNodeBins(nodeId, [bin], TEST_PAIRING_GENERATION); expect(getRemoteSkillEligibility()?.hasBin(bin)).toBe(true); @@ -86,6 +125,74 @@ describe("skills-remote", () => { expect(removeRemoteNodeInfo(nodeId)).toBeUndefined(); }); + it("preserves bins across reconnects only within one pairing generation", () => { + const nodeId = `node-${randomUUID()}`; + const retiredBin = `bin-${randomUUID()}`; + try { + recordRemoteNodeInfo({ + nodeId, + connId: "conn-a", + pairingGeneration: "generation-a", + platform: "darwin", + commands: ["system.run"], + }); + recordRemoteNodeBins(nodeId, [retiredBin], "generation-a"); + recordRemoteNodeInfo({ + nodeId, + connId: "conn-a-reconnect", + pairingGeneration: "generation-a", + platform: "darwin", + commands: ["system.run"], + }); + expect(getRemoteSkillEligibility()?.hasBin(retiredBin)).toBe(true); + + recordRemoteNodeInfo({ + nodeId, + connId: "conn-b", + pairingGeneration: "generation-b", + platform: "darwin", + commands: ["system.run"], + }); + expect(getRemoteSkillEligibility()?.hasBin(retiredBin) ?? false).toBe(false); + } finally { + removeRemoteNodeInfo(nodeId); + } + }); + + it("removes remote projections only for the exact invalidated connection", () => { + const nodeId = `node-${randomUUID()}`; + const bin = `bin-${randomUUID()}`; + const skillName = `skill-${randomUUID()}`; + recordRemoteNodeInfo({ + nodeId, + connId: "conn-current", + displayName: "Remote Mac", + platform: "darwin", + commands: ["system.run"], + }); + recordRemoteNodeBins(nodeId, [bin], TEST_PAIRING_GENERATION); + replaceRemoteNodeSkills({ + nodeId, + displayName: "Remote Mac", + skills: [ + { + name: skillName, + description: "Remote test skill", + content: `---\nname: ${skillName}\ndescription: Remote test skill\n---\n`, + }, + ], + }); + const listRemoteSkillNames = () => + mergeRemoteNodeSkillEntries([], { canExec: true }).map((entry) => entry.skill.name); + + expect(removeRemoteNodeInfoForConnection(nodeId, "conn-retired")).toBe(false); + expect(getRemoteSkillEligibility()?.hasBin(bin)).toBe(true); + expect(listRemoteSkillNames()).toContain(skillName); + expect(removeRemoteNodeInfoForConnection(nodeId, "conn-current")).toBe(true); + expect(getRemoteSkillEligibility()?.hasBin(bin) ?? false).toBe(false); + expect(listRemoteSkillNames()).not.toContain(skillName); + }); + it("bumps the skills snapshot version when an eligible remote node disconnects", async () => { await resetSkillsRefreshForTest(); const workspaceDir = `/tmp/ws-${randomUUID()}`; @@ -115,7 +222,7 @@ describe("skills-remote", () => { platform: "linux", commands: ["system.run"], }); - recordRemoteNodeBins(linuxNodeId, [bin]); + recordRemoteNodeBins(linuxNodeId, [bin], TEST_PAIRING_GENERATION); recordRemoteNodeInfo({ nodeId: noRunNodeId, @@ -123,7 +230,7 @@ describe("skills-remote", () => { platform: "darwin", commands: ["system.which"], }); - recordRemoteNodeBins(noRunNodeId, [bin]); + recordRemoteNodeBins(noRunNodeId, [bin], TEST_PAIRING_GENERATION); expect(getRemoteSkillEligibility()).toBeUndefined(); } finally { @@ -144,14 +251,14 @@ describe("skills-remote", () => { platform: "darwin", commands: ["system.run"], }); - recordRemoteNodeBins(nodeA, [binA]); + recordRemoteNodeBins(nodeA, [binA], TEST_PAIRING_GENERATION); recordRemoteNodeInfo({ nodeId: nodeB, platform: "macOS", commands: ["system.run"], }); - recordRemoteNodeBins(nodeB, [binB]); + recordRemoteNodeBins(nodeB, [binB], TEST_PAIRING_GENERATION); const eligibility = getRemoteSkillEligibility(); expect(eligibility?.platforms).toEqual(["darwin"]); @@ -175,7 +282,7 @@ describe("skills-remote", () => { platform: "darwin", commands: ["system.run"], }); - recordRemoteNodeBins(nodeId, [bin]); + recordRemoteNodeBins(nodeId, [bin], TEST_PAIRING_GENERATION); const eligibility = getRemoteSkillEligibility({ advertiseExecNode: false }); @@ -190,7 +297,7 @@ describe("skills-remote", () => { const nodeId = `node-${randomUUID()}`; const bin = `bin-${randomUUID()}`; try { - recordRemoteNodeBins(nodeId, [bin]); + recordRemoteNodeBins(nodeId, [bin], TEST_PAIRING_GENERATION); expect(getRemoteSkillEligibility()?.hasBin(bin) ?? false).toBe(false); } finally { @@ -205,9 +312,8 @@ describe("skills-remote", () => { const { cfg, workspaceDir } = createRemoteSkillWorkspace(bin); try { const invokeCalls: string[] = []; - setSkillsRemoteRegistry({ - listConnected: () => [], - get: () => undefined, + setTestSkillsRemoteRegistry(nodeId, { + get: () => testRemoteSession(nodeId), invoke: async (params: { command: string }) => { invokeCalls.push(params.command); return { @@ -217,7 +323,7 @@ describe("skills-remote", () => { }, } as unknown as NodeRegistry); recordRemoteMacWithSystemWhich(nodeId); - recordRemoteNodeBins(nodeId, [bin]); + recordRemoteNodeBins(nodeId, [bin], TEST_PAIRING_GENERATION); const before = getSkillsSnapshotVersion(workspaceDir); await refreshRemoteNodeBins({ @@ -264,9 +370,8 @@ describe("skills-remote", () => { }, } satisfies OpenClawConfig; const invokeCalls: string[] = []; - setSkillsRemoteRegistry({ - listConnected: () => [], - get: () => undefined, + setTestSkillsRemoteRegistry(nodeId, { + get: () => testRemoteSession(nodeId), checkConnectivity: async () => ({ ok: false, error: { code: "TIMEOUT", message: "node connectivity probe timed out" }, @@ -281,11 +386,12 @@ describe("skills-remote", () => { } as unknown as NodeRegistry); recordRemoteNodeInfo({ nodeId, + pairingGeneration: TEST_PAIRING_GENERATION, displayName: "Remote Mac", platform: "darwin", commands: ["system.run", "system.which"], }); - recordRemoteNodeBins(nodeId, [bin]); + recordRemoteNodeBins(nodeId, [bin], TEST_PAIRING_GENERATION); const before = getSkillsSnapshotVersion(workspaceDir); await refreshRemoteNodeBins({ @@ -334,12 +440,12 @@ describe("skills-remote", () => { let connId = "conn-old"; const connectivityCalls: string[] = []; const invokeCalls: string[] = []; - setSkillsRemoteRegistry({ - listConnected: () => [], + setTestSkillsRemoteRegistry(nodeId, { get: () => ({ nodeId, connId, + pairingGeneration: TEST_PAIRING_GENERATION, platform: "darwin", commands: ["system.run", "system.which"], }) as unknown as ReturnType, @@ -347,6 +453,14 @@ describe("skills-remote", () => { connectivityCalls.push(connId); if (connectivityCalls.length === 1) { connId = "conn-new"; + recordRemoteNodeInfo({ + nodeId, + connId, + pairingGeneration: TEST_PAIRING_GENERATION, + displayName: "Remote Mac", + platform: "darwin", + commands: ["system.run", "system.which"], + }); return { ok: false, error: { code: "TIMEOUT", message: "node connectivity probe timed out" }, @@ -364,6 +478,8 @@ describe("skills-remote", () => { } as unknown as NodeRegistry); recordRemoteNodeInfo({ nodeId, + connId, + pairingGeneration: TEST_PAIRING_GENERATION, displayName: "Remote Mac", platform: "darwin", commands: ["system.run", "system.which"], @@ -393,9 +509,8 @@ describe("skills-remote", () => { let invokeCount = 0; let releaseProbe: (() => void) | undefined; const probeStarted = new Promise((resolve) => { - setSkillsRemoteRegistry({ - listConnected: () => [], - get: () => undefined, + setTestSkillsRemoteRegistry(nodeId, { + get: () => testRemoteSession(nodeId), invoke: async () => { invokeCount += 1; resolve(); @@ -432,6 +547,7 @@ describe("skills-remote", () => { } satisfies OpenClawConfig; recordRemoteNodeInfo({ nodeId, + pairingGeneration: TEST_PAIRING_GENERATION, displayName: "Remote Mac", platform: "darwin", commands: ["system.run", "system.which"], @@ -473,9 +589,8 @@ describe("skills-remote", () => { const nowSpy = vi.spyOn(Date, "now").mockReturnValue(1_000_000); let invokeCount = 0; try { - setSkillsRemoteRegistry({ - listConnected: () => [], - get: () => undefined, + setTestSkillsRemoteRegistry(nodeId, { + get: () => testRemoteSession(nodeId), invoke: async () => { invokeCount += 1; return { ok: true, payload: { bins: [bin] } }; @@ -528,9 +643,8 @@ describe("skills-remote", () => { vi.spyOn(Date, "now").mockReturnValue(2_000_000); let invokeCount = 0; try { - setSkillsRemoteRegistry({ - listConnected: () => [], - get: () => undefined, + setTestSkillsRemoteRegistry(nodeId, { + get: () => testRemoteSession(nodeId), invoke: async () => { invokeCount += 1; return { ok: false, error: { code: "TIMEOUT", message: "node invoke timed out" } }; @@ -566,9 +680,8 @@ describe("skills-remote", () => { const checkConnectivity = vi.fn(async () => ({ ok: true as const })); const invoke = vi.fn(async () => ({ ok: true as const, payload: { bins: [bin] } })); try { - setSkillsRemoteRegistry({ - listConnected: () => [], - get: () => undefined, + setTestSkillsRemoteRegistry(nodeId, { + get: () => testRemoteSession(nodeId), checkConnectivity, invoke, } as unknown as NodeRegistry); @@ -612,12 +725,12 @@ describe("skills-remote", () => { .mockImplementationOnce(async () => await firstInvoke) .mockResolvedValueOnce({ ok: true as const, payload: { bins: [bin] } }); try { - setSkillsRemoteRegistry({ - listConnected: () => [], + setTestSkillsRemoteRegistry(nodeId, { get: () => ({ nodeId, connId, + pairingGeneration: TEST_PAIRING_GENERATION, platform: "darwin", commands: ["system.run", "system.which"], }) as unknown as ReturnType, @@ -626,6 +739,7 @@ describe("skills-remote", () => { recordRemoteNodeInfo({ nodeId, connId, + pairingGeneration: TEST_PAIRING_GENERATION, platform: "darwin", commands: ["system.run", "system.which"], }); @@ -644,6 +758,7 @@ describe("skills-remote", () => { recordRemoteNodeInfo({ nodeId, connId, + pairingGeneration: TEST_PAIRING_GENERATION, platform: "darwin", commands: ["system.run", "system.which"], }); @@ -661,20 +776,97 @@ describe("skills-remote", () => { } }); + it("does not carry a retired generation probe into its replacement", async () => { + const nodeId = `node-${randomUUID()}`; + const retiredBin = `bin-${randomUUID()}`; + const currentBin = `bin-${randomUUID()}`; + const { cfg, workspaceDir } = createRemoteSkillWorkspace(currentBin); + let session = { + nodeId, + connId: "conn-a", + pairingGeneration: "generation-a", + platform: "darwin", + commands: ["system.run", "system.which"], + } as NonNullable>; + let resolveRetiredProbe: + | ((value: Awaited>) => void) + | undefined; + const retiredProbe = new Promise>>((resolve) => { + resolveRetiredProbe = resolve; + }); + const invoke = vi + .fn() + .mockImplementationOnce(async () => await retiredProbe) + .mockResolvedValueOnce({ ok: true as const, payload: { bins: [currentBin] } }); + try { + setTestSkillsRemoteRegistry(nodeId, { + get: () => session, + invoke, + } as unknown as NodeRegistry); + recordRemoteNodeInfo({ + nodeId, + connId: session.connId, + pairingGeneration: session.pairingGeneration, + platform: session.platform, + commands: session.commands, + }); + const firstRefresh = refreshRemoteNodeBins({ + nodeId, + platform: session.platform, + commands: session.commands, + cfg, + }); + await vi.waitFor(() => expect(invoke).toHaveBeenCalledTimes(1)); + + session = { ...session, connId: "conn-b", pairingGeneration: "generation-b" }; + recordRemoteNodeInfo({ + nodeId, + connId: session.connId, + pairingGeneration: session.pairingGeneration, + platform: session.platform, + commands: session.commands, + }); + recordRemoteNodeBins(nodeId, [currentBin], "generation-b"); + const replacementRefresh = refreshRemoteNodeBins({ + nodeId, + platform: session.platform, + commands: session.commands, + cfg, + }); + resolveRetiredProbe?.({ ok: true, payload: { bins: [retiredBin] } }); + await Promise.all([firstRefresh, replacementRefresh]); + + expect(invoke).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ expectedPairingGeneration: "generation-a" }), + ); + expect(invoke).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ expectedPairingGeneration: "generation-b" }), + ); + expect(getRemoteSkillEligibility()?.hasBin(retiredBin) ?? false).toBe(false); + expect(getRemoteSkillEligibility()?.hasBin(currentBin)).toBe(true); + } finally { + removeRemoteNodeInfo(nodeId); + fs.rmSync(workspaceDir, { recursive: true, force: true }); + } + }); + it("uses the approved live command surface after the connect readiness delay", async () => { vi.useFakeTimers(); const nodeId = `node-${randomUUID()}`; const bin = `bin-${randomUUID()}`; const { cfg, workspaceDir } = createRemoteSkillWorkspace(bin); let commands: string[] = []; + let pairingGeneration: string | undefined; const invoke = vi.fn(async () => ({ ok: true as const, payload: { bins: [bin] } })); try { - setSkillsRemoteRegistry({ - listConnected: () => [], + setTestSkillsRemoteRegistry(nodeId, { get: () => ({ nodeId, connId: "conn-current", + pairingGeneration, platform: "darwin", commands, }) as unknown as ReturnType, @@ -691,6 +883,14 @@ describe("skills-remote", () => { readinessDelayMs: 5_000, }); commands = ["system.run", "system.which"]; + pairingGeneration = TEST_PAIRING_GENERATION; + recordRemoteNodeInfo({ + nodeId, + connId: "conn-current", + pairingGeneration, + platform: "darwin", + commands, + }); const approvalRefresh = refreshRemoteNodeBins({ nodeId, platform: "darwin", @@ -717,9 +917,8 @@ describe("skills-remote", () => { const { cfg, workspaceDir } = createRemoteSkillWorkspace(bin); try { const invokeCalls: string[] = []; - setSkillsRemoteRegistry({ - listConnected: () => [], - get: () => undefined, + setTestSkillsRemoteRegistry(nodeId, { + get: () => testRemoteSession(nodeId), invoke: async (params: { command: string }) => { invokeCalls.push(params.command); return { @@ -758,12 +957,8 @@ describe("skills-remote", () => { const { cfg, workspaceDir } = createRemoteSkillWorkspace(bin); try { const invokeCalls: string[] = []; - setSkillsRemoteRegistry({ - listConnected: () => [ - { nodeId: nodeA, platform: "darwin", commands: ["system.run", "system.which"] }, - { nodeId: nodeB, platform: "darwin", commands: ["system.run", "system.which"] }, - ], - get: () => undefined, + setTestSkillsRemoteRegistry([nodeA, nodeB], { + get: (nodeId: string) => testRemoteSession(nodeId), checkConnectivity: (nodeId: string) => { if (nodeId === nodeA) { throw new Error("simulated connectivity failure"); @@ -777,7 +972,7 @@ describe("skills-remote", () => { } as unknown as NodeRegistry); recordRemoteMacWithSystemWhich(nodeA); recordRemoteMacWithSystemWhich(nodeB); - recordRemoteNodeBins(nodeA, ["stale-bin"]); + recordRemoteNodeBins(nodeA, ["stale-bin"], TEST_PAIRING_GENERATION); await expect(refreshRemoteBinsForConnectedNodes(cfg)).resolves.toBeUndefined(); @@ -790,4 +985,51 @@ describe("skills-remote", () => { fs.rmSync(workspaceDir, { recursive: true, force: true }); } }); + + it("refreshes bins only for generation-current connected sessions", async () => { + await resetSkillsRefreshForTest(); + const staleNodeId = `node-${randomUUID()}`; + const currentNodeId = `node-${randomUUID()}`; + const bin = `bin-${randomUUID()}`; + const { cfg, workspaceDir } = createRemoteSkillWorkspace(bin); + const staleSession = testRemoteSession(staleNodeId, { connId: "conn-stale" }); + const currentSession = testRemoteSession(currentNodeId, { connId: "conn-current" }); + const invoke = vi.fn(async () => ({ + ok: false as const, + error: { code: "TIMEOUT", message: "node invoke timed out" }, + })); + const listCurrentConnectedSync = vi.fn(() => [currentSession]); + try { + setSkillsRemoteRegistry({ + listConnected: () => [staleSession, currentSession], + listCurrentConnectedSync, + get: (nodeId: string) => (nodeId === currentNodeId ? currentSession : staleSession), + invoke, + } as unknown as NodeRegistry); + recordRemoteNodeInfo({ + nodeId: staleNodeId, + connId: "conn-stale", + pairingGeneration: TEST_PAIRING_GENERATION, + platform: "darwin", + commands: ["system.run", "system.which"], + }); + recordRemoteNodeInfo({ + nodeId: currentNodeId, + connId: "conn-current", + pairingGeneration: TEST_PAIRING_GENERATION, + platform: "darwin", + commands: ["system.run", "system.which"], + }); + + await refreshRemoteBinsForConnectedNodes(cfg); + + expect(listCurrentConnectedSync).toHaveBeenCalled(); + expect(invoke).toHaveBeenCalledTimes(1); + expect(invoke).toHaveBeenCalledWith(expect.objectContaining({ nodeId: currentNodeId })); + } finally { + removeRemoteNodeInfo(staleNodeId); + removeRemoteNodeInfo(currentNodeId); + fs.rmSync(workspaceDir, { recursive: true, force: true }); + } + }); }); diff --git a/src/skills/runtime/remote.ts b/src/skills/runtime/remote.ts index 597083b8bf62..934c2c314727 100644 --- a/src/skills/runtime/remote.ts +++ b/src/skills/runtime/remote.ts @@ -1,18 +1,28 @@ // Remote skill runtime helpers send skill refresh and snapshot state across remotes. -import { - normalizeLowercaseStringOrEmpty, - normalizeOptionalString, -} from "@openclaw/normalization-core/string-coerce"; -import { normalizeStringEntries } from "@openclaw/normalization-core/string-normalization"; +import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; import { listAgentWorkspaceDirs } from "../../agents/workspace-dirs.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; -import type { NodeRegistry } from "../../gateway/node-registry.js"; +import type { NodeRegistry, NodeSession } from "../../gateway/node-registry.js"; import { listNodePairing, updatePairedNodeBins } from "../../infra/node-pairing.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; import { loadWorkspaceSkillEntries } from "../loading/workspace.js"; -import type { SkillEligibilityContext, SkillEntry } from "../types.js"; +import type { SkillEligibilityContext } from "../types.js"; import { bumpSkillsSnapshotVersion } from "./refresh-state.js"; -import { recordRemoteSkillNodeInfo, removeRemoteNodeSkills } from "./remote-skills.js"; +import { + areBinSetsEqual, + buildBinProbeScript, + collectRequiredBins, + extractErrorMessage, + isMacPlatform, + parseBinProbePayload, + supportsSystemRun, + supportsSystemWhich, +} from "./remote-probe-utils.js"; +import { + recordRemoteSkillNodeInfo, + removeRemoteNodeSkills, + setRemoteSkillConnectionReconciler, +} from "./remote-skills.js"; type RemoteNodeRecord = { nodeId: string; @@ -22,12 +32,14 @@ type RemoteNodeRecord = { deviceFamily?: string; commands?: string[]; bins: Set; + pairingGeneration?: string; connected: boolean; remoteIp?: string; }; type RemoteNodeProbeState = { signature: string; + pairingGeneration: string; nextProbeAfterMs: number; failedProbeCount: number; bins?: Set; @@ -36,8 +48,11 @@ type RemoteNodeProbeState = { const log = createSubsystemLogger("gateway/skills-remote"); const remoteNodes = new Map(); const remoteNodeProbeStates = new Map(); -type RemoteBinProbeInflight = { +type RemoteNodeOwner = { connId?: string; + pairingGeneration: string; +}; +type RemoteBinProbeInflight = RemoteNodeOwner & { promise: Promise; }; @@ -55,35 +70,6 @@ function describeNode(nodeId: string): string { return ip ? `${base} @ ${ip}` : base; } -function extractErrorMessage(err: unknown): string | undefined { - if (!err) { - return undefined; - } - if (typeof err === "string") { - return err; - } - if (err instanceof Error) { - return err.message; - } - if (typeof err === "object" && "message" in err && typeof err.message === "string") { - return err.message; - } - if (typeof err === "number" || typeof err === "boolean" || typeof err === "bigint") { - return String(err); - } - if (typeof err === "symbol") { - return err.toString(); - } - if (typeof err === "object") { - try { - return JSON.stringify(err); - } catch { - return undefined; - } - } - return undefined; -} - type RemoteBinProbeLogContext = { command?: string; timeoutMs?: number; @@ -138,42 +124,33 @@ function logRemoteBinProbeFailure( log.warn(`remote bin probe error (${label}; ${details}): ${message ?? "unknown"}`); } -function isMacPlatform(platform?: string, deviceFamily?: string): boolean { - const platformNorm = normalizeLowercaseStringOrEmpty(platform); - const familyNorm = normalizeLowercaseStringOrEmpty(deviceFamily); - if (platformNorm.includes("mac")) { - return true; - } - if (platformNorm.includes("darwin")) { - return true; - } - if (familyNorm === "mac") { - return true; - } - return false; -} - -function supportsSystemRun(commands?: string[]): boolean { - return Array.isArray(commands) && commands.includes("system.run"); -} - -function supportsSystemWhich(commands?: string[]): boolean { - return Array.isArray(commands) && commands.includes("system.which"); -} - -function upsertNode(record: { - nodeId: string; - connId?: string; - displayName?: string; - platform?: string; - deviceFamily?: string; - commands?: string[]; - remoteIp?: string; - bins?: string[]; - connected?: boolean; -}) { +function upsertNode( + record: { + nodeId: string; + connId?: string; + displayName?: string; + platform?: string; + deviceFamily?: string; + commands?: string[]; + remoteIp?: string; + bins?: string[]; + pairingGeneration?: string; + connected?: boolean; + }, + options?: { pairingGenerationAuthoritative?: boolean }, +) { const existing = remoteNodes.get(record.nodeId); - const bins = new Set(record.bins ?? existing?.bins ?? []); + const pairingGeneration = options?.pairingGenerationAuthoritative + ? record.pairingGeneration + : (record.pairingGeneration ?? existing?.pairingGeneration); + const pairingGenerationChanged = Boolean( + options?.pairingGenerationAuthoritative && + existing && + existing.pairingGeneration !== record.pairingGeneration, + ); + const bins = new Set( + record.bins ?? (pairingGenerationChanged ? [] : (existing?.bins ?? [])), + ); remoteNodes.set(record.nodeId, { nodeId: record.nodeId, connId: record.connId ?? existing?.connId, @@ -183,6 +160,7 @@ function upsertNode(record: { commands: record.commands ?? existing?.commands, remoteIp: record.remoteIp ?? existing?.remoteIp, bins, + ...(pairingGeneration ? { pairingGeneration } : {}), connected: record.connected ?? existing?.connected ?? false, }); } @@ -214,44 +192,59 @@ function buildRemoteProbeSignature(params: { function shouldSkipRemoteNodeProbe(params: { state: RemoteNodeProbeState | undefined; + pairingGeneration: string; signature: string; nowMs: number; }): boolean { return ( - params.state?.signature === params.signature && params.nowMs < params.state.nextProbeAfterMs + params.state?.pairingGeneration === params.pairingGeneration && + params.state.signature === params.signature && + params.nowMs < params.state.nextProbeAfterMs ); } function restoreCachedRemoteNodeBins(nodeId: string): boolean { const node = remoteNodes.get(nodeId); - const cachedBins = remoteNodeProbeStates.get(nodeId)?.bins; - if (!node || !cachedBins || areBinSetsEqual(node.bins, cachedBins)) { + const state = remoteNodeProbeStates.get(nodeId); + const cachedBins = state?.bins; + if ( + !node || + state?.pairingGeneration !== node.pairingGeneration || + !cachedBins || + areBinSetsEqual(node.bins, cachedBins) + ) { return false; } node.bins = new Set(cachedBins); return true; } -function isCurrentRemoteNodeConnection(nodeId: string, connId?: string): boolean { - if (!connId) { - return true; - } +function sameRemoteNodeOwner(left: RemoteNodeOwner, right: RemoteNodeOwner): boolean { + return left.connId === right.connId && left.pairingGeneration === right.pairingGeneration; +} + +function isCurrentRemoteNodeOwner(nodeId: string, owner: RemoteNodeOwner): boolean { const current = remoteNodes.get(nodeId); - return Boolean(current && (!current.connId || current.connId === connId)); + return Boolean( + current && + current.pairingGeneration === owner.pairingGeneration && + (!owner.connId || !current.connId || current.connId === owner.connId), + ); } function markRemoteNodeProbeSuccess(params: { nodeId: string; - connId?: string; + owner: RemoteNodeOwner; signature: string; nowMs: number; bins: string[]; }): boolean { - if (!isCurrentRemoteNodeConnection(params.nodeId, params.connId)) { + if (!isCurrentRemoteNodeOwner(params.nodeId, params.owner)) { return false; } remoteNodeProbeStates.set(params.nodeId, { signature: params.signature, + pairingGeneration: params.owner.pairingGeneration, nextProbeAfterMs: params.nowMs + REMOTE_BIN_PROBE_SUCCESS_TTL_MS, failedProbeCount: 0, bins: new Set(params.bins), @@ -261,11 +254,11 @@ function markRemoteNodeProbeSuccess(params: { function markRemoteNodeProbeFailure(params: { nodeId: string; - connId?: string; + owner: RemoteNodeOwner; signature: string; nowMs: number; }): boolean { - if (!isCurrentRemoteNodeConnection(params.nodeId, params.connId)) { + if (!isCurrentRemoteNodeOwner(params.nodeId, params.owner)) { return false; } const existing = remoteNodeProbeStates.get(params.nodeId); @@ -277,14 +270,33 @@ function markRemoteNodeProbeFailure(params: { ); remoteNodeProbeStates.set(params.nodeId, { signature: params.signature, + pairingGeneration: params.owner.pairingGeneration, nextProbeAfterMs: params.nowMs + backoffMs, failedProbeCount, }); return true; } +function remoteConnectionKey(nodeId: string, connId: string): string { + return `${nodeId}\0${connId}`; +} + +function listCurrentRemoteSessions(): NodeSession[] { + return remoteRegistry?.listCurrentConnectedSync() ?? []; +} + +function listCurrentRemoteConnectionKeys(): ReadonlySet | undefined { + if (!remoteRegistry) { + return undefined; + } + return new Set( + listCurrentRemoteSessions().map((node) => remoteConnectionKey(node.nodeId, node.connId)), + ); +} + export function setSkillsRemoteRegistry(registry: NodeRegistry | null) { remoteRegistry = registry; + setRemoteSkillConnectionReconciler(registry ? () => listCurrentRemoteConnectionKeys() : null); if (!registry) { remoteNodeProbeStates.clear(); } @@ -292,19 +304,26 @@ export function setSkillsRemoteRegistry(registry: NodeRegistry | null) { export async function primeRemoteSkillsCache() { try { - const list = await listNodePairing(); + const { paired } = await listNodePairing(undefined, { includePairingGeneration: true }); let sawMac = false; - for (const node of list.paired) { - upsertNode({ - nodeId: node.nodeId, - displayName: node.displayName, - platform: node.platform, - deviceFamily: node.deviceFamily, - commands: node.commands, - remoteIp: node.remoteIp, - bins: node.bins, - connected: false, - }); + for (const node of paired) { + if (!node.pairingGeneration) { + continue; + } + upsertNode( + { + nodeId: node.nodeId, + displayName: node.displayName, + platform: node.platform, + deviceFamily: node.deviceFamily, + commands: node.commands, + remoteIp: node.remoteIp, + bins: node.bins, + pairingGeneration: node.pairingGeneration, + connected: false, + }, + { pairingGenerationAuthoritative: true }, + ); if ( node.bins && node.bins.length > 0 && @@ -330,16 +349,21 @@ export function recordRemoteNodeInfo(node: { deviceFamily?: string; commands?: string[]; remoteIp?: string; + pairingGeneration?: string; }) { const existing = remoteNodes.get(node.nodeId); + const pairingGenerationChanged = Boolean( + existing && existing.pairingGeneration !== node.pairingGeneration, + ); if ( - node.connId && - existing?.connId !== node.connId && - !remoteNodeProbeStates.get(node.nodeId)?.bins + pairingGenerationChanged || + (node.connId && + existing?.connId !== node.connId && + !remoteNodeProbeStates.get(node.nodeId)?.bins) ) { remoteNodeProbeStates.delete(node.nodeId); } - upsertNode({ ...node, connected: true }); + upsertNode({ ...node, connected: true }, { pairingGenerationAuthoritative: true }); recordRemoteSkillNodeInfo({ nodeId: node.nodeId, connId: node.connId, @@ -348,8 +372,8 @@ export function recordRemoteNodeInfo(node: { }); } -export function recordRemoteNodeBins(nodeId: string, bins: string[]) { - upsertNode({ nodeId, bins }); +export function recordRemoteNodeBins(nodeId: string, bins: string[], pairingGeneration: string) { + upsertNode({ nodeId, bins, pairingGeneration }); } export function removeRemoteNodeInfo(nodeId: string) { @@ -371,75 +395,12 @@ export function removeRemoteNodeInfo(nodeId: string) { } } -function collectRequiredBins(entries: SkillEntry[], targetPlatform: string): string[] { - const bins = new Set(); - for (const entry of entries) { - const os = entry.metadata?.os ?? []; - if (os.length > 0 && !os.includes(targetPlatform)) { - continue; - } - const required = entry.metadata?.requires?.bins ?? []; - const anyBins = entry.metadata?.requires?.anyBins ?? []; - for (const bin of required) { - if (bin.trim()) { - bins.add(bin.trim()); - } - } - for (const bin of anyBins) { - if (bin.trim()) { - bins.add(bin.trim()); - } - } - } - return [...bins]; -} - -function buildBinProbeScript(bins: string[]): string { - const escaped = bins.map((bin) => `'${bin.replace(/'/g, `'\\''`)}'`).join(" "); - return `for b in ${escaped}; do if command -v "$b" >/dev/null 2>&1; then echo "$b"; fi; done`; -} - -function parseBinProbePayload(payloadJSON: string | null | undefined, payload?: unknown): string[] { - if (!payloadJSON && !payload) { - return []; - } - try { - const parsed = payloadJSON - ? (JSON.parse(payloadJSON) as { stdout?: unknown; bins?: unknown }) - : (payload as { stdout?: unknown; bins?: unknown }); - if (Array.isArray(parsed.bins)) { - return normalizeStringEntries(parsed.bins); - } - if (parsed.bins && typeof parsed.bins === "object") { - return Object.entries(parsed.bins) - .filter(([, resolvedPath]) => normalizeOptionalString(resolvedPath) !== undefined) - .map(([bin]) => normalizeOptionalString(bin) ?? "") - .filter(Boolean); - } - if (typeof parsed.stdout === "string") { - return parsed.stdout - .split(/\r?\n/) - .map((line) => normalizeOptionalString(line) ?? "") - .filter(Boolean); - } - } catch { - return []; - } - return []; -} - -function areBinSetsEqual(a: Set | undefined, b: Set): boolean { - if (!a) { +/** Remove remote projections only while they still belong to the invalidated connection. */ +export function removeRemoteNodeInfoForConnection(nodeId: string, connId: string): boolean { + if (remoteNodes.get(nodeId)?.connId !== connId) { return false; } - if (a.size !== b.size) { - return false; - } - for (const bin of b) { - if (!a.has(bin)) { - return false; - } - } + removeRemoteNodeInfo(nodeId); return true; } @@ -452,16 +413,23 @@ export async function refreshRemoteNodeBins(params: { timeoutMs?: number; readinessDelayMs?: number; }) { - const connId = remoteRegistry?.get(params.nodeId)?.connId; + const session = remoteRegistry?.get(params.nodeId); + if (!session?.pairingGeneration) { + return; + } + const owner: RemoteNodeOwner = { + connId: session.connId, + pairingGeneration: session.pairingGeneration, + }; const existing = remoteBinProbeInflight.get(params.nodeId); if (existing) { await existing.promise; - if (existing.connId === connId) { + if (sameRemoteNodeOwner(existing, owner)) { return; } } const inflight: RemoteBinProbeInflight = { - connId, + ...owner, promise: Promise.resolve(), }; const run = refreshRemoteNodeBinsUncoalesced(params).finally(() => { @@ -495,7 +463,13 @@ async function refreshRemoteNodeBinsUncoalesced(params: { // Pairing can replace the command surface while the connect-time readiness // delay is pending. Probe the live session so that approval refresh is not lost. const liveSession = remoteRegistry.get(params.nodeId); - const probeConnId = liveSession?.connId; + if (!liveSession?.pairingGeneration) { + return; + } + const probeOwner: RemoteNodeOwner = { + connId: liveSession.connId, + pairingGeneration: liveSession.pairingGeneration, + }; const platform = liveSession?.platform ?? params.platform; const deviceFamily = liveSession?.deviceFamily ?? params.deviceFamily; const commands = liveSession?.commands ?? params.commands; @@ -534,6 +508,7 @@ async function refreshRemoteNodeBinsUncoalesced(params: { if ( shouldSkipRemoteNodeProbe({ state: remoteNodeProbeStates.get(params.nodeId), + pairingGeneration: probeOwner.pairingGeneration, signature: probeSignature, nowMs, }) @@ -553,7 +528,7 @@ async function refreshRemoteNodeBinsUncoalesced(params: { } catch (err) { const recorded = markRemoteNodeProbeFailure({ nodeId: params.nodeId, - connId: probeConnId, + owner: probeOwner, signature: probeSignature, nowMs: Date.now(), }); @@ -591,7 +566,7 @@ async function refreshRemoteNodeBinsUncoalesced(params: { } const recorded = markRemoteNodeProbeFailure({ nodeId: params.nodeId, - connId: probeConnId, + owner: probeOwner, signature: probeSignature, nowMs: Date.now(), }); @@ -620,12 +595,14 @@ async function refreshRemoteNodeBinsUncoalesced(params: { canWhich ? { nodeId: params.nodeId, + expectedPairingGeneration: probeOwner.pairingGeneration, command, params: { bins: binsList }, timeoutMs, } : { nodeId: params.nodeId, + expectedPairingGeneration: probeOwner.pairingGeneration, command, params: { command: ["/bin/sh", "-lc", buildBinProbeScript(binsList)], @@ -636,7 +613,7 @@ async function refreshRemoteNodeBinsUncoalesced(params: { if (!res.ok) { const recorded = markRemoteNodeProbeFailure({ nodeId: params.nodeId, - connId: probeConnId, + owner: probeOwner, signature: probeSignature, nowMs: Date.now(), }); @@ -651,9 +628,24 @@ async function refreshRemoteNodeBinsUncoalesced(params: { return; } const bins = parseBinProbePayload(res.payloadJSON, res.payload); + if (!isCurrentRemoteNodeOwner(params.nodeId, probeOwner)) { + return; + } + const existingBins = remoteNodes.get(params.nodeId)?.bins; + const nextBins = new Set(bins); + const hasChanged = !areBinSetsEqual(existingBins, nextBins); + if (hasChanged) { + const persisted = await updatePairedNodeBins(params.nodeId, bins, { + nodeId: params.nodeId, + key: probeOwner.pairingGeneration, + }); + if (!persisted) { + return; + } + } const recorded = markRemoteNodeProbeSuccess({ nodeId: params.nodeId, - connId: probeConnId, + owner: probeOwner, signature: probeSignature, nowMs: Date.now(), bins, @@ -661,19 +653,14 @@ async function refreshRemoteNodeBinsUncoalesced(params: { if (!recorded) { return; } - const existingBins = remoteNodes.get(params.nodeId)?.bins; - const nextBins = new Set(bins); - const hasChanged = !areBinSetsEqual(existingBins, nextBins); - recordRemoteNodeBins(params.nodeId, bins); - if (!hasChanged) { - return; + recordRemoteNodeBins(params.nodeId, bins, probeOwner.pairingGeneration); + if (hasChanged) { + bumpSkillsSnapshotVersion({ reason: "remote-node" }); } - await updatePairedNodeBins(params.nodeId, bins); - bumpSkillsSnapshotVersion({ reason: "remote-node" }); } catch (err) { const recorded = markRemoteNodeProbeFailure({ nodeId: params.nodeId, - connId: probeConnId, + owner: probeOwner, signature: probeSignature, nowMs: Date.now(), }); @@ -691,9 +678,13 @@ async function refreshRemoteNodeBinsUncoalesced(params: { export function getRemoteSkillEligibility(options?: { advertiseExecNode?: boolean; }): SkillEligibilityContext["remote"] | undefined { + const currentConnections = listCurrentRemoteConnectionKeys(); const macNodes = [...remoteNodes.values()].filter( (node) => node.connected && + (!currentConnections || + (node.connId !== undefined && + currentConnections.has(remoteConnectionKey(node.nodeId, node.connId)))) && isMacPlatform(node.platform, node.deviceFamily) && supportsSystemRun(node.commands), ); @@ -725,7 +716,7 @@ export async function refreshRemoteBinsForConnectedNodes(cfg: OpenClawConfig) { if (!remoteRegistry) { return; } - const connected = remoteRegistry.listConnected(); + const connected = listCurrentRemoteSessions(); for (const node of connected) { try { await refreshRemoteNodeBins({