diff --git a/config/assertion-safety-baseline.txt b/config/assertion-safety-baseline.txt index fda1717c78cd..ec30bb235ee4 100644 --- a/config/assertion-safety-baseline.txt +++ b/config/assertion-safety-baseline.txt @@ -3479,11 +3479,10 @@ src/node-host/invoke-payload.ts 5 src/node-host/invoke.ts 7 src/node-host/mcp.ts 3 src/node-host/node-worker-bundle-installer.ts 1 -src/node-host/node-worker-supervisor.ts 1 src/node-host/node-worker-transfer-client.ts 3 src/node-host/node-worker-transfer-http.ts 2 src/node-host/node-worker-tree-control.ts 2 -src/node-host/node-worker-workspace.ts 7 +src/node-host/node-worker-workspace.ts 6 src/node-host/plugin-node-host.ts 1 src/node-host/pty-command.ts 11 src/node-host/runtime.ts 2 @@ -3979,7 +3978,7 @@ src/worker/node-workspace-protocol.ts 3 src/worker/worker-command.runtime.ts 1 src/worker/worker-connection-admission.ts 2 src/worker/worker-connection-frames.ts 4 -src/worker/worker-process.ts 2 +src/worker/worker-process.ts 1 src/worker/worker-session-tools.ts 4 src/worker/workspace-rsync-receiver.ts 1 ui/src/api/gateway.ts 1 diff --git a/extensions/memory-core/src/memory/broker-entry.test.ts b/extensions/memory-core/src/memory/broker-entry.test.ts index f8e835d2c9f1..1bad886714bd 100644 --- a/extensions/memory-core/src/memory/broker-entry.test.ts +++ b/extensions/memory-core/src/memory/broker-entry.test.ts @@ -1,28 +1,47 @@ +import type { MemoryBrokerAuthorizationBinding } from "openclaw/plugin-sdk/memory-broker-runtime"; import { describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ authorize: vi.fn(), + export: vi.fn(), + import: vi.fn(), + read: vi.fn(), + recoverPendingWrites: vi.fn(), search: vi.fn(), status: vi.fn(), + sync: vi.fn(), + virtualFile: vi.fn(), + virtualView: vi.fn(), + write: vi.fn(), })); vi.mock("./scoped-memory-runtime.js", () => ({ builtinScopedMemoryAuthorizedRuntime: { authorize: mocks.authorize, + exportAuthorized: mocks.export, + importAuthorized: mocks.import, + readAuthorized: mocks.read, searchAuthorized: mocks.search, statusAuthorized: mocks.status, + syncAuthorized: mocks.sync, + writeAuthorized: mocks.write, }, - builtinScopedMemoryVirtualView: {}, + builtinScopedMemoryVirtualView: { + materializeAuthorizedVirtualView: mocks.virtualView, + readAuthorizedVirtualFile: mocks.virtualFile, + }, + recoverBuiltinScopedMemoryPendingWrites: mocks.recoverPendingWrites, })); -const { createMemoryBrokerHandler } = await import("./broker-entry.js"); +const { createMemoryBrokerHandler, initializeMemoryBroker } = await import("./broker-entry.js"); -const binding = { +const binding: MemoryBrokerAuthorizationBinding = { agentId: "agent-a", sessionId: "session-a", runId: "run-a", contextFingerprint: "context-a", subjectRevision: "subject-a", + actor: { kind: "principal", actorKind: "human", principalId: "alice" }, actorRevision: "actor-a", // Without a delegation, Gateway uses hostFactsRevision for the capability snapshot. capabilitySnapshotId: "policy-a", @@ -36,12 +55,25 @@ const context = { runId: binding.runId, contextFingerprint: binding.contextFingerprint, subjectRevision: binding.subjectRevision, - actor: { evidenceRevision: binding.actorRevision }, + actor: { + kind: binding.actor.kind, + actorKind: binding.actor.actorKind, + principalId: binding.actor.principalId, + evidenceRevision: binding.actorRevision, + }, delivery: { deliveryRevision: binding.deliveryRevision }, hostFactsRevision: binding.policyRevision, }; describe("memory-core broker entry", () => { + it("recovers only the configured agent set before the broker exposes its socket", () => { + mocks.recoverPendingWrites.mockClear(); + + initializeMemoryBroker({ agentIds: ["main", "work"] }); + + expect(mocks.recoverPendingWrites).toHaveBeenCalledWith(["main", "work"]); + }); + it("rejects a client-provided context that does not exactly match the Gateway binding", async () => { const handler = createMemoryBrokerHandler(); @@ -58,8 +90,27 @@ describe("memory-core broker entry", () => { expect(mocks.authorize).not.toHaveBeenCalled(); }); + it("rejects an actor principal substitution that keeps the same evidence revision", async () => { + const handler = createMemoryBrokerHandler(); + + await expect( + handler({ + binding, + request: { + method: "memory.authorize", + payload: { + context: { ...context, actor: { ...context.actor, principalId: "bob" } }, + }, + }, + signal: new AbortController().signal, + }), + ).rejects.toThrow("memory broker binding is unavailable"); + expect(mocks.authorize).not.toHaveBeenCalled(); + }); + it("dispatches a bound operation only after its plan policy revision matches", async () => { const handler = createMemoryBrokerHandler(); + const controller = new AbortController(); const plan = { memoryPolicyRevision: binding.policyRevision }; mocks.status.mockResolvedValue({ version: 1, value: { backend: "builtin" } }); @@ -70,10 +121,10 @@ describe("memory-core broker entry", () => { method: "memory.status", payload: { context, plan }, }, - signal: new AbortController().signal, + signal: controller.signal, }), ).resolves.toEqual({ version: 1, value: { backend: "builtin" } }); - expect(mocks.status).toHaveBeenCalledWith({ context, plan }); + expect(mocks.status).toHaveBeenCalledWith({ context, plan, signal: controller.signal }); }); it("forwards the broker cancellation signal into content search", async () => { @@ -101,4 +152,77 @@ describe("memory-core broker entry", () => { signal: controller.signal, }); }); + + it("forwards the broker cancellation signal into every authorized operation", async () => { + const handler = createMemoryBrokerHandler(); + const controller = new AbortController(); + const plan = { memoryPolicyRevision: binding.policyRevision }; + const readContext = { ...context, operation: "read" as const }; + const writeContext = { ...context, operation: "append" as const }; + mocks.read.mockResolvedValue({ version: 1, value: { text: "ok" } }); + mocks.virtualView.mockResolvedValue({ version: 1, files: [] }); + mocks.virtualFile.mockResolvedValue({ version: 1, value: { text: "ok" } }); + mocks.write.mockResolvedValue({ status: "committed" }); + mocks.import.mockResolvedValue({ status: "committed" }); + mocks.sync.mockResolvedValue({ version: 1, value: { status: "completed" } }); + mocks.export.mockResolvedValue({ version: 1, value: {} }); + + await Promise.all([ + handler({ + binding, + request: { method: "memory.read", payload: { context: readContext, plan, handle: {} } }, + signal: controller.signal, + }), + handler({ + binding, + request: { method: "memory.virtual-view", payload: { context: readContext, plan } }, + signal: controller.signal, + }), + handler({ + binding, + request: { + method: "memory.virtual-file", + payload: { context: readContext, plan, view: {}, virtualPath: "private/1.md" }, + }, + signal: controller.signal, + }), + handler({ + binding, + request: { method: "memory.write", payload: { context: writeContext, plan, mutation: {} } }, + signal: controller.signal, + }), + handler({ + binding, + request: { + method: "memory.import", + payload: { context: writeContext, plan, mutation: { kind: "import" } }, + }, + signal: controller.signal, + }), + handler({ + binding, + request: { method: "memory.sync", payload: { context: writeContext, plan } }, + signal: controller.signal, + }), + handler({ + binding, + request: { method: "memory.export", payload: { context: writeContext, plan, handles: [] } }, + signal: controller.signal, + }), + ]); + + for (const operation of [ + mocks.read, + mocks.virtualView, + mocks.virtualFile, + mocks.write, + mocks.import, + mocks.sync, + mocks.export, + ]) { + expect(operation).toHaveBeenCalledWith( + expect.objectContaining({ signal: controller.signal }), + ); + } + }); }); diff --git a/extensions/memory-core/src/memory/broker-entry.ts b/extensions/memory-core/src/memory/broker-entry.ts index 8e31ef76ec33..c4f9cd95500f 100644 --- a/extensions/memory-core/src/memory/broker-entry.ts +++ b/extensions/memory-core/src/memory/broker-entry.ts @@ -9,10 +9,12 @@ import type { import type { MemoryBrokerAuthorizationBinding, MemoryBrokerHandler, + MemoryBrokerStartupContext, } from "openclaw/plugin-sdk/memory-broker-runtime"; import { builtinScopedMemoryAuthorizedRuntime, builtinScopedMemoryVirtualView, + recoverBuiltinScopedMemoryPendingWrites, } from "./scoped-memory-runtime.js"; type BrokerPayload = Readonly<{ @@ -37,6 +39,27 @@ function asPayload(value: unknown): BrokerPayload { return value as BrokerPayload; } +function hasBoundActor(params: { + binding: MemoryBrokerAuthorizationBinding["actor"]; + context: MemoryAccessContext; +}): boolean { + const { binding, context } = params; + if (context.actor.kind !== binding.kind) { + return false; + } + if (binding.kind === "principal") { + return ( + context.actor.kind === "principal" && + context.actor.actorKind === binding.actorKind && + context.actor.principalId === binding.principalId + ); + } + return ( + context.actor.kind === "unattributed" && + context.actor.transportAuditRef === binding.transportAuditRef + ); +} + function assertBoundContext(params: { binding: MemoryBrokerAuthorizationBinding; context: MemoryAccessContext; @@ -52,6 +75,7 @@ function assertBoundContext(params: { context.runId !== binding.runId || context.contextFingerprint !== binding.contextFingerprint || context.subjectRevision !== binding.subjectRevision || + !hasBoundActor({ binding: binding.actor, context }) || context.actor.evidenceRevision !== binding.actorRevision || capabilitySnapshotId !== binding.capabilitySnapshotId || policyRevision !== binding.policyRevision || @@ -75,6 +99,11 @@ function readPlan(payload: BrokerPayload): AuthorizedMemoryPlan { return payload.plan; } +/** Complete selected-memory recovery before the broker child exposes its authenticated socket. */ +export function initializeMemoryBroker(context: MemoryBrokerStartupContext): void { + recoverBuiltinScopedMemoryPendingWrites(context.agentIds); +} + /** * This entry is loaded only by the Gateway-owned broker child. It owns the selected runtime's * process-local plan and handle maps, so Gateway and workers can retain only opaque DTOs. @@ -115,6 +144,7 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler { handle: payload.handle, ...(Number.isSafeInteger(payload.from) ? { from: payload.from } : {}), ...(Number.isSafeInteger(payload.lines) ? { lines: payload.lines } : {}), + signal, }); } case "memory.virtual-view": { @@ -125,6 +155,7 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler { return await builtinScopedMemoryVirtualView.materializeAuthorizedVirtualView({ context, plan: readPlan(payload) as never, + signal, }); } case "memory.virtual-file": { @@ -141,6 +172,7 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler { plan: readPlan(payload) as never, view: payload.view, virtualPath: payload.virtualPath, + signal, }); } case "memory.write": @@ -151,6 +183,7 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler { context: payload.context, plan: readPlan(payload), mutation: payload.mutation, + signal, } as never); case "memory.import": if (!payload.mutation || payload.mutation.kind !== "import") { @@ -160,11 +193,13 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler { context: payload.context, plan: readPlan(payload), mutation: payload.mutation, + signal, } as never); case "memory.sync": return await builtinScopedMemoryAuthorizedRuntime.syncAuthorized({ context: payload.context, plan: readPlan(payload), + signal, } as never); case "memory.export": if (!Array.isArray(payload.handles)) { @@ -174,11 +209,13 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler { context: payload.context, plan: readPlan(payload), handles: payload.handles, + signal, } as never); case "memory.status": return await builtinScopedMemoryAuthorizedRuntime.statusAuthorized({ context: payload.context, plan: readPlan(payload), + signal, } as never); default: throw new Error("memory broker operation is unavailable"); diff --git a/extensions/memory-core/src/memory/scoped-memory-runtime.test.ts b/extensions/memory-core/src/memory/scoped-memory-runtime.test.ts index 5c8244fd019a..c5eb40c151b7 100644 --- a/extensions/memory-core/src/memory/scoped-memory-runtime.test.ts +++ b/extensions/memory-core/src/memory/scoped-memory-runtime.test.ts @@ -1,12 +1,17 @@ +import { spawnSync } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; import fs from "node:fs"; +import net from "node:net"; import os from "node:os"; import path from "node:path"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import type { OpenClawPluginApi } from "openclaw/plugin-sdk/core"; import type { AuthorizedMemoryPlan, MemoryAccessContext, MemoryContentAccessContext, } from "openclaw/plugin-sdk/memory-authorization"; +import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api"; import { closeOpenClawAgentDatabasesForTest } from "openclaw/plugin-sdk/sqlite-runtime-testing"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { @@ -21,7 +26,7 @@ import { consumeAdmittedChannelMemoryIdentityFromContext, createChannelMemoryIdentityAdmission, } from "../../../../src/channels/message-access/memory-identity-admission.js"; -import { appendSqliteTranscriptMessage } from "../../../../src/config/sessions/session-accessor.sqlite-transcript-write.js"; +import { appendTranscriptMessage } from "../../../../src/config/sessions/session-accessor.sqlite-transcript-write.js"; import { readAuthorizedTranscriptDerivation } from "../../../../src/config/sessions/session-transcript-memory-policy.js"; import { withOwnedSessionTranscriptWrites } from "../../../../src/config/sessions/transcript-write-context.js"; import { @@ -29,7 +34,12 @@ import { resetAgentRunRegistryForTest, } from "../../../../src/infra/agent-run-registry.js"; import { startMemoryBrokerProcess } from "../../../../src/memory-broker/process.js"; +import { requestJsonlSocket } from "../../../../src/infra/jsonl-socket.js"; import { admitMemoryAuthorizationReadRuntime } from "../../../../src/plugins/memory-authorization-runtime.js"; +import { + closeBrokeredMemoryRuntimes, + withBrokeredMemoryMaintenance, +} from "../../../../src/plugins/memory-broker-runtime.js"; import { resetMemoryIsolationCutoverForTest } from "../../../../src/plugins/memory-cutover.js"; import { persistMemoryRunExposureBeforeContentInDatabase, @@ -56,6 +66,8 @@ import { openOpenClawStateDatabase, } from "../../../../src/state/openclaw-state-db.js"; import { ensureProfileForEmail } from "../../../../src/state/user-profiles.js"; +import { verifyNodeWorkerContainerProjectionIsolation } from "../../../../test/helpers/node-worker-container-projection-isolation.js"; +import memoryCorePlugin from "../../index.js"; import { MEMORY_CORE_AUTHORIZATION_CAPABILITIES } from "../authorization.js"; import { createMemoryRuntime } from "../runtime-provider.js"; import { resolveScopedMemoryArtifactBase, withScopedMemoryDatabase } from "./scoped-memory-db.js"; @@ -63,6 +75,7 @@ import { builtinScopedMemoryConformanceAdapter } from "./scoped-memory-policy.js import { createBuiltinScopedMemoryResource, readBuiltinScopedMemoryRevisionSnapshot, + resolveBuiltinScopedMemoryArtifactPath, setBuiltinScopedMemoryRevisionLifecycle, } from "./scoped-memory-resources.js"; import { @@ -86,6 +99,62 @@ vi.mock("../../../../src/auto-reply/reply/dispatch-from-config.js", () => ({ const { dispatchInboundMessage } = await import("../../../../src/auto-reply/dispatch.js"); +const dockerAvailable = + spawnSync("docker", ["info"], { stdio: "ignore", timeout: 3_000 }).status === 0; + +function constrainedSandboxConfig(params: { + image: string; + prefix: string; + workspaceRoot: string; +}): OpenClawConfig { + return { + agents: { + defaults: { + skipBootstrap: true, + sandbox: { + mode: "all", + backend: "docker", + scope: "session", + workspaceAccess: "none", + workspaceRoot: params.workspaceRoot, + docker: { image: params.image, containerPrefix: params.prefix }, + browser: { enabled: false }, + prune: { idleHours: 0, maxAgeDays: 0 }, + }, + }, + }, + }; +} + +function resolveSelectedMemoryBrokerChildPid(): number { + const processList = spawnSync("ps", ["-axo", "pid=,ppid=,command="], { encoding: "utf8" }); + if (processList.status !== 0) { + throw new Error("fixture failed to inspect the selected memory broker child"); + } + const child = processList.stdout + .split("\n") + .map((line) => /^\s*(\d+)\s+(\d+)\s+(.*)$/u.exec(line)) + .find( + (match) => + match?.[2] === String(process.pid) && + /(?:^|[\\/])memory-broker[\\/]child\.(?:ts|js)(?:\s|$)/u.test(match[3]), + ); + if (!child?.[1]) { + throw new Error("fixture failed to locate the selected memory broker child"); + } + return Number(child[1]); +} + +function resolveNewMemoryBrokerSocketPath(existingDirectories: ReadonlySet): string { + const directory = fs + .readdirSync(os.tmpdir()) + .find((name) => name.startsWith("openclaw-memory-broker-") && !existingDirectories.has(name)); + if (!directory) { + throw new Error("fixture failed to locate the selected memory broker socket"); + } + return path.join(os.tmpdir(), directory, "broker.sock"); +} + describe("builtin scoped authorized runtime", () => { let stateDir = ""; @@ -94,7 +163,8 @@ describe("builtin scoped authorized runtime", () => { vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); }); - afterEach(() => { + afterEach(async () => { + await closeBrokeredMemoryRuntimes(); dispatchReplyFromConfig.mockReset(); resetBuiltinScopedMemoryAuthorizedRuntimeForTest(); resetMemoryIsolationCutoverForTest(); @@ -129,11 +199,45 @@ describe("builtin scoped authorized runtime", () => { authorizationConformance: builtinScopedMemoryConformanceAdapter, virtualView: builtinScopedMemoryVirtualView, runtime: { ...createMemoryRuntime(), ...builtinScopedMemoryAuthorizedRuntime }, + broker: { + version: 1, + kind: "local-child", + moduleUrl: new URL("./broker-entry.ts", import.meta.url).href, + }, }, }); setActivePluginRegistry(registry); } + function installDefaultMemoryCoreSelectedRuntime() { + const registry = createEmptyPluginRegistry(); + registry.plugins.push({ id: "memory-core", memorySlotSelected: true } as never); + const runtime = { + llm: { acquireLocalService: async () => undefined }, + state: { + openKeyedStore: () => ({ + lookup: () => undefined, + register: () => undefined, + delete: () => undefined, + list: () => [], + }), + }, + } as unknown as OpenClawPluginApi["runtime"]; + memoryCorePlugin.register( + createTestPluginApi({ + runtime, + registerMemoryCapability(capability) { + registry.memoryCapabilities.push({ pluginId: "memory-core", capability }); + }, + }), + ); + if (registry.memoryCapabilities.length !== 1) { + throw new Error("fixture failed to register the default memory-core capability"); + } + setActivePluginRegistry(registry); + return registry.memoryCapabilities[0]!.capability; + } + function createSession(params: { sessionKey: string; sessionId: string; @@ -543,18 +647,61 @@ describe("builtin scoped authorized runtime", () => { it("keeps the selected runtime plan state in a real broker child across IPC", async () => { createPrivateResource("alice", "BROKER_CHILD_SERIALIZED_PLAN_ONLY"); + const startupRecovered = createWriteRecoveryFixture({ + principalId: "alice", + content: "BROKER_STARTUP_RECOVERY_SENTINEL", + placement: "stage", + }); const context = createContext("alice"); const broker = await startMemoryBrokerProcess({ brokerId: "memory-core-test-broker", handlerModuleUrl: new URL("./broker-entry.ts", import.meta.url).href, + agentIds: ["main"], }); try { + // The child reached ready only after its plugin startup hook recovered the staged revision. + // This assertion occurs before the first broker request, so authorize cannot hide deferred + // recovery as request-time repair. + withScopedMemoryDatabase("main", (database) => { + expect( + database + .prepare( + `SELECT revision.lifecycle_state, intent.state + FROM memory_resource_revisions AS revision + JOIN memory_write_intents AS intent ON intent.pending_revision_id = revision.revision_id + WHERE revision.revision_id = ?`, + ) + .get(startupRecovered.revisionId), + ).toEqual({ lifecycle_state: "active", state: "active" }); + expect( + database + .prepare("SELECT text FROM memory_scoped_chunks WHERE revision_id = ?") + .all(startupRecovered.revisionId), + ).toEqual([{ text: "BROKER_STARTUP_RECOVERY_SENTINEL" }]); + }); + expect(fs.existsSync(path.join(startupRecovered.directory, startupRecovered.stageLocator))).toBe( + false, + ); + expect(fs.existsSync(path.join(startupRecovered.directory, startupRecovered.finalLocator))).toBe( + true, + ); const authorizationBinding = { agentId: context.agentId, sessionId: context.sessionId, runId: context.runId, contextFingerprint: context.contextFingerprint, subjectRevision: context.subjectRevision, + actor: + context.actor.kind === "principal" + ? { + kind: "principal" as const, + actorKind: context.actor.actorKind, + principalId: context.actor.principalId, + } + : { + kind: "unattributed" as const, + transportAuditRef: context.actor.transportAuditRef, + }, actorRevision: context.actor.evidenceRevision, capabilitySnapshotId: context.delegation?.capabilitySnapshotId ?? context.hostFactsRevision, policyRevision: context.hostFactsRevision, @@ -591,6 +738,65 @@ describe("builtin scoped authorized runtime", () => { }), ).resolves.toBeUndefined(); + await expect( + broker.client.request({ + binding: { ...authorizationBinding, policyRevision: plan.memoryPolicyRevision }, + method: "memory.search", + payload: { + context: { ...context, sessionId: "replayed-session" }, + plan, + query: "BROKER_CHILD_SERIALIZED_PLAN_ONLY", + limit: 10, + }, + expiresAtMs: Date.parse(plan.expiresAt), + }), + ).resolves.toBeUndefined(); + + await expect( + broker.client.request({ + binding: { ...authorizationBinding, policyRevision: plan.memoryPolicyRevision }, + method: "memory.search", + payload: { + context: { ...context, agentId: "replayed-agent" }, + plan, + query: "BROKER_CHILD_SERIALIZED_PLAN_ONLY", + limit: 10, + }, + expiresAtMs: Date.parse(plan.expiresAt), + }), + ).resolves.toBeUndefined(); + + await expect( + broker.client.request({ + binding: { ...authorizationBinding, policyRevision: plan.memoryPolicyRevision }, + method: "memory.search", + payload: { + context: { + ...context, + actor: { ...context.actor, principalId: "mallory" }, + }, + plan, + query: "BROKER_CHILD_SERIALIZED_PLAN_ONLY", + limit: 10, + }, + expiresAtMs: Date.parse(plan.expiresAt), + }), + ).resolves.toBeUndefined(); + + await expect( + broker.client.request({ + binding: { ...authorizationBinding, policyRevision: plan.memoryPolicyRevision }, + method: "memory.search", + payload: { + context: { ...context, hostFactsRevision: "revoked-capability-snapshot" }, + plan, + query: "BROKER_CHILD_SERIALIZED_PLAN_ONLY", + limit: 10, + }, + expiresAtMs: Date.parse(plan.expiresAt), + }), + ).resolves.toBeUndefined(); + await expect( broker.client.request({ binding: { ...authorizationBinding, policyRevision: "stale-policy-revision" }, @@ -1230,7 +1436,7 @@ describe("builtin scoped authorized runtime", () => { withTranscriptWrite: async (run) => await run(), }, async () => { - await appendSqliteTranscriptMessage( + await appendTranscriptMessage( { agentId: context.agentId, sessionId: context.sessionId, @@ -1360,6 +1566,484 @@ describe("builtin scoped authorized runtime", () => { }); }); + it("returns intentional unavailability while maintenance quiesces the real selected broker", async () => { + const session = { sessionKey: "agent:main:direct:alice", sessionId: "alice-session" }; + const alicePrincipalId = createVerifiedDirectSession({ name: "alice", ...session }); + createPrivateResource(alicePrincipalId, "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL"); + markCutOver(); + installBuiltinSelectedRuntime(); + + const host = createAuthorizedMemoryReadHost({ + agentId: "main", + ...session, + deliveryContext: { channel: "telegram", accountId: "default", to: "alice" }, + }); + if (!host) { + throw new Error("fixture failed to build an authorized memory host"); + } + + await expect( + host.search({ query: "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL", limit: 1 }), + ).resolves.toMatchObject({ + results: [{ snippet: "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL" }], + }); + + let enterMaintenance: (() => void) | undefined; + let releaseMaintenance: (() => void) | undefined; + const maintenanceEntered = new Promise((resolve) => { + enterMaintenance = resolve; + }); + const maintenanceRelease = new Promise((resolve) => { + releaseMaintenance = resolve; + }); + const maintenance = withBrokeredMemoryMaintenance(async () => { + enterMaintenance?.(); + await maintenanceRelease; + }); + await maintenanceEntered; + + await expect( + host.search({ query: "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL", limit: 1 }), + ).resolves.toEqual({ + disabled: true, + unavailable: true, + error: "memory unavailable", + }); + + releaseMaintenance?.(); + await maintenance; + await expect( + host.search({ query: "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL", limit: 1 }), + ).resolves.toMatchObject({ + results: [{ snippet: "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL" }], + }); + }); + + it.runIf(process.platform !== "win32")( + "restarts the default selected broker only after Gateway reauthorizes and rejects a pre-crash envelope", + async () => { + const session = { sessionKey: "agent:main:direct:alice", sessionId: "alice-session" }; + const alicePrincipalId = createVerifiedDirectSession({ name: "alice", ...session }); + createPrivateResource(alicePrincipalId, "SELECTED_BROKER_CRASH_RESTART_SENTINEL"); + markCutOver(); + installDefaultMemoryCoreSelectedRuntime(); + + const brokerDirectoriesBefore = new Set( + fs.readdirSync(os.tmpdir()).filter((name) => name.startsWith("openclaw-memory-broker-")), + ); + const createHost = () => { + const host = createAuthorizedMemoryReadHost({ + agentId: "main", + ...session, + deliveryContext: { channel: "telegram", accountId: "default", to: "alice" }, + }); + if (!host) { + throw new Error("fixture failed to build an authorized memory host"); + } + return host; + }; + const host = createHost(); + + const writes = vi.spyOn(net.Socket.prototype, "write"); + let staleRequestLine: string | undefined; + try { + await expect( + host.search({ query: "SELECTED_BROKER_CRASH_RESTART_SENTINEL", limit: 1 }), + ).resolves.toMatchObject({ + results: [{ snippet: "SELECTED_BROKER_CRASH_RESTART_SENTINEL" }], + }); + staleRequestLine = writes.mock.calls + .map(([chunk]) => (typeof chunk === "string" ? chunk.trim() : undefined)) + .find((line) => { + if (!line) { + return false; + } + try { + const frame = JSON.parse(line) as { + envelope?: unknown; + request?: { method?: unknown }; + }; + return frame.envelope !== undefined && frame.request?.method === "memory.authorize"; + } catch { + return false; + } + }); + } finally { + writes.mockRestore(); + } + if (!staleRequestLine) { + throw new Error("fixture failed to capture a signed selected-broker authorization frame"); + } + + const firstSocketPath = resolveNewMemoryBrokerSocketPath(brokerDirectoriesBefore); + expect(fs.existsSync(firstSocketPath)).toBe(true); + process.kill(resolveSelectedMemoryBrokerChildPid(), "SIGKILL"); + await vi.waitFor(() => expect(fs.existsSync(firstSocketPath)).toBe(false), { + timeout: 2_000, + interval: 20, + }); + + await vi.waitFor( + async () => { + await expect( + createHost().search({ query: "SELECTED_BROKER_CRASH_RESTART_SENTINEL", limit: 1 }), + ).resolves.toMatchObject({ + results: [{ snippet: "SELECTED_BROKER_CRASH_RESTART_SENTINEL" }], + }); + }, + { timeout: 2_000, interval: 20 }, + ); + + const replacementSocketPath = resolveNewMemoryBrokerSocketPath( + new Set([...brokerDirectoriesBefore, path.basename(path.dirname(firstSocketPath))]), + ); + expect(replacementSocketPath).not.toBe(firstSocketPath); + await expect( + requestJsonlSocket({ + socketPath: replacementSocketPath, + requestLine: staleRequestLine, + timeoutMs: 1_000, + keepWriteOpen: true, + accept: (response) => + response && typeof response === "object" && !Array.isArray(response) + ? (response as { ok: boolean; error?: string }) + : undefined, + }), + ).resolves.toEqual({ ok: false, error: "unauthorized" }); + }, + ); + + it.runIf(process.platform !== "win32")( + "rotates the selected broker epoch across controlled Gateway shutdown before reauthorization", + async () => { + const session = { sessionKey: "agent:main:direct:alice", sessionId: "alice-session" }; + const alicePrincipalId = createVerifiedDirectSession({ name: "alice", ...session }); + createPrivateResource(alicePrincipalId, "SELECTED_BROKER_CONTROLLED_RESTART_SENTINEL"); + markCutOver(); + installDefaultMemoryCoreSelectedRuntime(); + + const brokerDirectoriesBefore = new Set( + fs.readdirSync(os.tmpdir()).filter((name) => name.startsWith("openclaw-memory-broker-")), + ); + const createHost = () => { + const host = createAuthorizedMemoryReadHost({ + agentId: "main", + ...session, + deliveryContext: { channel: "telegram", accountId: "default", to: "alice" }, + }); + if (!host) { + throw new Error("fixture failed to build an authorized memory host"); + } + return host; + }; + + const writes = vi.spyOn(net.Socket.prototype, "write"); + let preShutdownRequestLine: string | undefined; + try { + await expect( + createHost().search({ query: "SELECTED_BROKER_CONTROLLED_RESTART_SENTINEL", limit: 1 }), + ).resolves.toMatchObject({ + results: [{ snippet: "SELECTED_BROKER_CONTROLLED_RESTART_SENTINEL" }], + }); + preShutdownRequestLine = writes.mock.calls + .map(([chunk]) => (typeof chunk === "string" ? chunk.trim() : undefined)) + .find((line) => { + if (!line) { + return false; + } + try { + const frame = JSON.parse(line) as { + envelope?: unknown; + request?: { method?: unknown }; + }; + return frame.envelope !== undefined && frame.request?.method === "memory.authorize"; + } catch { + return false; + } + }); + } finally { + writes.mockRestore(); + } + if (!preShutdownRequestLine) { + throw new Error("fixture failed to capture a signed pre-shutdown authorization frame"); + } + + const firstSocketPath = resolveNewMemoryBrokerSocketPath(brokerDirectoriesBefore); + await closeBrokeredMemoryRuntimes(); + await vi.waitFor(() => expect(fs.existsSync(firstSocketPath)).toBe(false), { + timeout: 2_000, + interval: 20, + }); + + await expect( + createHost().search({ query: "SELECTED_BROKER_CONTROLLED_RESTART_SENTINEL", limit: 1 }), + ).resolves.toMatchObject({ + results: [{ snippet: "SELECTED_BROKER_CONTROLLED_RESTART_SENTINEL" }], + }); + const replacementSocketPath = resolveNewMemoryBrokerSocketPath( + new Set([...brokerDirectoriesBefore, path.basename(path.dirname(firstSocketPath))]), + ); + await expect( + requestJsonlSocket({ + socketPath: replacementSocketPath, + requestLine: preShutdownRequestLine, + timeoutMs: 1_000, + keepWriteOpen: true, + accept: (response) => + response && typeof response === "object" && !Array.isArray(response) + ? (response as { ok: boolean; error?: string }) + : undefined, + }), + ).resolves.toEqual({ ok: false, error: "unauthorized" }); + }, + ); + + it.runIf(dockerAvailable && process.env.OPENCLAW_PROCESS_ISOLATION_E2E === "1")( + "keeps real memory-core artifacts behind its broker when a malicious model-facing tool runs in a constrained process", + async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-memory-core-container-e2e-")); + const workspaceDir = path.join(root, "workspace"); + const image = process.env.OPENCLAW_SANDBOX_TEST_IMAGE ?? "openclaw-sandbox:bookworm-slim"; + const session = { sessionKey: "agent:main:direct:alice", sessionId: "alice-session" }; + const bobSession = { sessionKey: "agent:main:direct:bob", sessionId: "bob-session" }; + const disposeProjections: Array<() => Promise> = []; + const runtimeIds: string[] = []; + try { + fs.mkdirSync(workspaceDir, { recursive: true }); + vi.stubEnv("OPENCLAW_MEMORY_BROKER_TEST_SECRET", "gateway-only-test-secret"); + const alicePrincipalId = createVerifiedDirectSession({ name: "alice", ...session }); + const aliceResource = createPrivateResource( + alicePrincipalId, + "ALICE_REAL_BROKER_ARTIFACT_SENTINEL", + ); + const bobPrincipalId = createVerifiedDirectSession({ + name: "bob", + ...bobSession, + }); + const bobResource = createPrivateResource( + bobPrincipalId, + "BOB_REAL_BROKER_ARTIFACT_SENTINEL", + ); + markCutOver(); + const capability = installDefaultMemoryCoreSelectedRuntime(); + expect(capability.broker).toMatchObject({ version: 1, kind: "local-child" }); + + const brokerDirectoriesBefore = new Set( + fs.readdirSync(os.tmpdir()).filter((name) => name.startsWith("openclaw-memory-broker-")), + ); + const host = createAuthorizedMemoryReadHost({ + agentId: "main", + ...session, + deliveryContext: { channel: "telegram", accountId: "default", to: "alice" }, + }); + if (!host) { + throw new Error("fixture failed to construct the selected memory-core host"); + } + const indexed = await host.search({ query: "REAL_BROKER_ARTIFACT_SENTINEL", limit: 10 }); + expect(indexed).toMatchObject({ + results: [{ path: "memory/MEMORY.md", snippet: "ALICE_REAL_BROKER_ARTIFACT_SENTINEL" }], + }); + expect(JSON.stringify(indexed)).not.toContain("BOB_REAL_BROKER_ARTIFACT_SENTINEL"); + const broker = await resolveAuthorizedMemoryVirtualFileBroker(host); + const virtualFile = broker?.view.files[0]; + if (!broker || !virtualFile) { + throw new Error("fixture failed to materialize a broker-backed virtual view"); + } + await expect(broker.readFile(virtualFile.virtualPath)).resolves.toBe( + "ALICE_REAL_BROKER_ARTIFACT_SENTINEL", + ); + + const brokerDirectory = fs + .readdirSync(os.tmpdir()) + .find( + (name) => + name.startsWith("openclaw-memory-broker-") && !brokerDirectoriesBefore.has(name), + ); + if (!brokerDirectory) { + throw new Error("fixture failed to locate the selected memory-core broker socket"); + } + const brokerSocketPath = path.join(os.tmpdir(), brokerDirectory, "broker.sock"); + expect(fs.existsSync(brokerSocketPath)).toBe(true); + + const artifactPathFor = (revision: { revisionId: string; artifactLocator: string }) => { + let artifactPath: string | undefined; + withScopedMemoryDatabase("main", (database, databasePath) => { + const row = database + .prepare( + `SELECT root.path_key + FROM memory_resource_revisions AS revision + JOIN memory_resources AS resource ON resource.resource_id = revision.resource_id + JOIN memory_stores AS store ON store.store_id = resource.store_id + JOIN memory_storage_roots AS root ON root.storage_root_id = store.storage_root_id + WHERE revision.revision_id = ?`, + ) + .get(revision.revisionId) as { path_key?: string } | undefined; + if (row?.path_key) { + artifactPath = resolveBuiltinScopedMemoryArtifactPath({ + databasePath, + pathKey: row.path_key, + artifactLocator: revision.artifactLocator, + }); + } + }); + if (!artifactPath) { + throw new Error("fixture failed to locate a scoped memory artifact"); + } + return artifactPath; + }; + const aliceArtifactPath = artifactPathFor(aliceResource); + const bobArtifactPath = artifactPathFor(bobResource); + const agentDatabasePath = path.join( + stateDir, + "agents", + "main", + "agent", + "openclaw-agent.sqlite", + ); + expect(fs.existsSync(aliceArtifactPath)).toBe(true); + expect(fs.existsSync(bobArtifactPath)).toBe(true); + expect(fs.existsSync(agentDatabasePath)).toBe(true); + + const [{ resolveSandboxContext }, { stageAuthorizedVirtualProjectionMountPlan }] = + await Promise.all([ + import("../../../../src/agents/sandbox/context.js"), + import("../../../../src/agents/sandbox/authorized-virtual-projection-staging.js"), + ]); + const sandbox = await resolveSandboxContext({ + agentId: "main", + config: constrainedSandboxConfig({ + image, + prefix: `oc-qa-memory-core-${process.pid}-`, + workspaceRoot: path.join(root, "sandboxes"), + }), + sessionKey: session.sessionKey, + workspaceDir, + prepareAuthorizedVirtualProjectionMountPlan: async ({ agentWorkspaceDir }) => { + const staged = await stageAuthorizedVirtualProjectionMountPlan({ + agentWorkspaceDir, + broker, + }); + return staged; + }, + }); + if (!sandbox?.backend) { + throw new Error("fixture failed to start the constrained model-tool process"); + } + runtimeIds.push(sandbox.runtimeId); + if (sandbox.disposeAuthorizedVirtualProjectionMountPlan) { + disposeProjections.push(sandbox.disposeAuthorizedVirtualProjectionMountPlan); + } + + const result = await sandbox.backend.runShellCommand({ + script: [ + 'test "$(cat "$1")" = "$2"', + 'test "$(find /memory -type f -print | sort)" = "$1"', + 'test ! -e "$3"', + 'test ! -e "$4"', + 'test ! -e "$5"', + 'test ! -e "$6"', + 'test ! -e "$7"', + 'test -z "${OPENCLAW_MEMORY_BROKER_TEST_SECRET:-}"', + '! find /proc -path "*/fd/*" -lname "$3" -print -quit 2>/dev/null | grep -q .', + '! find /proc -path "*/fd/*" -lname "$5" -print -quit 2>/dev/null | grep -q .', + '! printf denied > "$1"', + ].join(" && "), + args: [ + `/memory/${virtualFile.virtualPath}`, + "ALICE_REAL_BROKER_ARTIFACT_SENTINEL", + aliceArtifactPath, + bobArtifactPath, + agentDatabasePath, + stateDir, + brokerSocketPath, + ], + }); + expect(result.code, result.stderr.toString()).toBe(0); + + await verifyNodeWorkerContainerProjectionIsolation({ + root: path.join(root, "node-worker-projection"), + broker, + outsideArtifactPath: aliceArtifactPath, + outsideArtifactContents: "ALICE_REAL_BROKER_ARTIFACT_SENTINEL", + issuedVirtualPath: virtualFile.virtualPath, + issuedContents: "ALICE_REAL_BROKER_ARTIFACT_SENTINEL", + forbiddenEnvironmentVariable: "OPENCLAW_MEMORY_BROKER_TEST_SECRET", + }); + + const bobHost = createAuthorizedMemoryReadHost({ + agentId: "main", + ...bobSession, + deliveryContext: { channel: "telegram", accountId: "default", to: "bob" }, + }); + const bobBroker = await resolveAuthorizedMemoryVirtualFileBroker(bobHost); + const bobVirtualFile = bobBroker?.view.files[0]; + if (!bobHost || !bobBroker || !bobVirtualFile) { + throw new Error("fixture failed to materialize Bob's selected memory-core virtual view"); + } + await expect( + bobHost.search({ query: "REAL_BROKER_ARTIFACT_SENTINEL", limit: 10 }), + ).resolves.toMatchObject({ + results: [{ path: "memory/MEMORY.md", snippet: "BOB_REAL_BROKER_ARTIFACT_SENTINEL" }], + }); + await expect(bobBroker.readFile(bobVirtualFile.virtualPath)).resolves.toBe( + "BOB_REAL_BROKER_ARTIFACT_SENTINEL", + ); + const bobSandbox = await resolveSandboxContext({ + agentId: "main", + config: constrainedSandboxConfig({ + image, + prefix: `oc-qa-memory-core-${process.pid}-`, + workspaceRoot: path.join(root, "sandboxes"), + }), + sessionKey: bobSession.sessionKey, + workspaceDir, + prepareAuthorizedVirtualProjectionMountPlan: async ({ agentWorkspaceDir }) => { + return await stageAuthorizedVirtualProjectionMountPlan({ + agentWorkspaceDir, + broker: bobBroker, + }); + }, + }); + if (!bobSandbox?.backend) { + throw new Error("fixture failed to start Bob's constrained model-tool process"); + } + runtimeIds.push(bobSandbox.runtimeId); + if (bobSandbox.disposeAuthorizedVirtualProjectionMountPlan) { + disposeProjections.push(bobSandbox.disposeAuthorizedVirtualProjectionMountPlan); + } + expect(bobSandbox.runtimeId).not.toBe(sandbox.runtimeId); + const bobResult = await bobSandbox.backend.runShellCommand({ + script: [ + 'test "$(cat "$1")" = "$2"', + 'test "$(find /memory -type f -print | sort)" = "$1"', + '! grep -R -F "$3" /memory', + '! printf denied > "$1"', + ].join(" && "), + args: [ + `/memory/${bobVirtualFile.virtualPath}`, + "BOB_REAL_BROKER_ARTIFACT_SENTINEL", + "ALICE_REAL_BROKER_ARTIFACT_SENTINEL", + ], + }); + expect(bobResult.code, bobResult.stderr.toString()).toBe(0); + } finally { + if (runtimeIds.length > 0) { + const [{ execDocker }, { removeSandboxContainer }] = await Promise.all([ + import("../../../../src/agents/sandbox/docker.js"), + import("../../../../src/agents/sandbox/manage.js"), + ]); + for (const runtimeId of runtimeIds) { + await removeSandboxContainer(runtimeId); + await execDocker(["rm", "-f", runtimeId], { allowFailure: true }); + } + } + await Promise.all(disposeProjections.map((disposeProjection) => disposeProjection())); + fs.rmSync(root, { recursive: true, force: true }); + } + }, + 120_000, + ); + it("invalidates materialized virtual views after binding revocation and plan expiry", async () => { const session = { sessionKey: "agent:main:direct:alice", sessionId: "alice-session" }; const alicePrincipalId = createVerifiedDirectSession({ name: "alice", ...session }); @@ -2017,6 +2701,131 @@ describe("builtin scoped authorized runtime", () => { }); }); + it("quarantines cancelled writes before activation so recovery cannot publish them", async () => { + const principalId = "cancelled-writer"; + const store = createBuiltinScopedMemoryStore({ + agentId: "main", + scopeKind: "user", + audienceKind: "user", + audienceId: principalId, + authorityKind: "user", + authorityOwnerId: principalId, + defaultCapabilities: ["append"], + actor: { kind: "human", id: principalId }, + reason: "cancellation fixture", + }); + const context = { + ...createContext(principalId), + operation: "append" as const, + } satisfies MemoryAccessContext; + const plan = await builtinScopedMemoryAuthorizedRuntime.authorize(context); + const preAborted = new AbortController(); + preAborted.abort(); + + await expect( + builtinScopedMemoryAuthorizedRuntime.writeAuthorized({ + context, + plan, + mutation: { + version: 1, + kind: "remember", + mutationId: "cancelled-before-stage", + idempotencyKey: "cancelled-before-stage-request", + content: "CANCELLED_BEFORE_STAGE_SENTINEL", + contentType: "markdown", + }, + signal: preAborted.signal, + }), + ).rejects.toThrow(); + + withScopedMemoryDatabase("main", (database) => { + expect( + database + .prepare("SELECT count(*) AS count FROM memory_resources WHERE store_id = ?") + .get(store.storeId), + ).toEqual({ count: 0 }); + expect( + database + .prepare("SELECT count(*) AS count FROM memory_write_intents WHERE store_id = ?") + .get(store.storeId), + ).toEqual({ count: 0 }); + }); + + // This test-only signal becomes aborted at the exact synchronous fence after final rename. + // It proves a late pre-activation disconnect leaves durable intent/artifact state quarantined. + let abortedReads = 0; + const abortAfterFinalRename = { + get aborted() { + abortedReads += 1; + return abortedReads > 2; + }, + throwIfAborted() { + if (abortedReads > 2) { + throw new Error("authorized write cancelled after final rename"); + } + }, + } as AbortSignal; + await expect( + builtinScopedMemoryAuthorizedRuntime.writeAuthorized({ + context, + plan, + mutation: { + version: 1, + kind: "remember", + mutationId: "cancelled-after-rename", + idempotencyKey: "cancelled-after-rename-request", + content: "CANCELLED_AFTER_RENAME_SENTINEL", + contentType: "markdown", + }, + signal: abortAfterFinalRename, + }), + ).rejects.toThrow("cancelled after final rename"); + + let revisionId = ""; + let directory = ""; + withScopedMemoryDatabase("main", (database, databasePath) => { + const intent = database + .prepare( + `SELECT intent.pending_revision_id, revision.lifecycle_state, intent.state, root.path_key + FROM memory_write_intents AS intent + JOIN memory_resource_revisions AS revision ON revision.revision_id = intent.pending_revision_id + JOIN memory_stores AS store ON store.store_id = intent.store_id + JOIN memory_storage_roots AS root ON root.storage_root_id = store.storage_root_id + WHERE intent.idempotency_key = ?`, + ) + .get("cancelled-after-rename-request") as { + pending_revision_id: string; + lifecycle_state: string; + path_key: string; + state: string; + }; + revisionId = intent.pending_revision_id; + directory = path.join(resolveScopedMemoryArtifactBase(databasePath), intent.path_key); + expect(intent).toMatchObject({ lifecycle_state: "quarantined", state: "quarantined" }); + expect( + database + .prepare("SELECT * FROM memory_scoped_chunks WHERE revision_id = ?") + .all(revisionId), + ).toEqual([]); + }); + expect(fs.readdirSync(path.join(path.dirname(directory), ".quarantine"))).not.toEqual([]); + + // A later authorization runs recovery. It must retain the tombstone rather than promote it. + await builtinScopedMemoryAuthorizedRuntime.authorize(context); + withScopedMemoryDatabase("main", (database) => { + expect( + database + .prepare( + `SELECT revision.lifecycle_state, intent.state + FROM memory_resource_revisions AS revision + JOIN memory_write_intents AS intent ON intent.pending_revision_id = revision.revision_id + WHERE revision.revision_id = ?`, + ) + .get(revisionId), + ).toEqual({ lifecycle_state: "quarantined", state: "quarantined" }); + }); + }); + it("recovers each interrupted write boundary without exposing a pending revision", async () => { const stageOnly = createWriteRecoveryFixture({ principalId: "recovery-stage-only", diff --git a/extensions/memory-core/src/memory/scoped-memory-runtime.ts b/extensions/memory-core/src/memory/scoped-memory-runtime.ts index 8057fde7d0e9..2fd41a00a286 100644 --- a/extensions/memory-core/src/memory/scoped-memory-runtime.ts +++ b/extensions/memory-core/src/memory/scoped-memory-runtime.ts @@ -466,7 +466,9 @@ function readPlan(params: { function materializeAuthorizedVirtualView(params: { context: MemoryContentAccessContext<"read">; plan: AuthorizedMemoryPlan & Readonly<{ operation: "read" }>; + signal?: AbortSignal; }): AuthorizedMemoryVirtualView | undefined { + throwIfAuthorizedMemoryOperationAborted(params.signal); pruneExpiredVirtualViews(); const state = readPlan(params); if (!state || state.stores.length !== state.plan.mounts.length) { @@ -483,6 +485,7 @@ function materializeAuthorizedVirtualView(params: { ); const revisionByVirtualPath = new Map(); const files = state.stores.flatMap((store, index) => { + throwIfAuthorizedMemoryOperationAborted(params.signal); const root = roots[index]!; const rows = withScopedMemoryDatabase( params.context.agentId, @@ -504,6 +507,7 @@ function materializeAuthorizedVirtualView(params: { }>, ); return rows.flatMap((row, ordinal) => { + throwIfAuthorizedMemoryOperationAborted(params.signal); const virtualPath = `${root.virtualRoot}/${ordinal + 1}.md`; revisionByVirtualPath.set(virtualPath, row.revision_id); return [ @@ -542,7 +546,9 @@ function readAuthorizedVirtualFile(params: { plan: AuthorizedMemoryPlan & Readonly<{ operation: "read" }>; view: AuthorizedMemoryVirtualView; virtualPath: string; + signal?: AbortSignal; }): AuthorizedMemoryResultEnvelope { + throwIfAuthorizedMemoryOperationAborted(params.signal); pruneExpiredVirtualViews(); const state = readPlan(params); const allocation = virtualViews.get(params.view.viewId); @@ -571,6 +577,7 @@ function readAuthorizedVirtualFile(params: { if (!snapshot) { throw new Error("authorized memory virtual view is unavailable"); } + throwIfAuthorizedMemoryOperationAborted(params.signal); return createEnvelope({ state, context: params.context, @@ -779,6 +786,14 @@ function syncDirectory(directory: string): void { } } +/** + * Broker cancellation is an authorization fence: check it before any recovery, + * filesystem staging, or durable write so cancelled work cannot be recovered as a new revision. + */ +function throwIfAuthorizedMemoryOperationAborted(signal?: AbortSignal): void { + signal?.throwIfAborted(); +} + function readVerifiedFile(params: { pathname: string; contentHash: string; @@ -1360,6 +1375,34 @@ function quarantineWriteIntent(params: { }); } +/** + * A cancelled write may already have durable pending rows and staged/final bytes. Quarantine both + * before surfacing the abort so recovery never promotes the cancelled revision on a later request. + */ +function throwIfPendingWriteAborted(params: { + signal?: AbortSignal; + database: Parameters>[0]; + directory: string; + intentId: string; + revisionId: string; + stagedPath: string; + finalPath: string; +}): void { + if (!params.signal?.aborted) { + return; + } + quarantineWriteIntent({ + database: params.database, + intentId: params.intentId, + revisionId: params.revisionId, + nowMs: Date.now(), + reasonCode: "authorized-write-cancelled-before-activation", + }); + quarantineArtifact({ directory: params.directory, pathname: params.stagedPath }); + quarantineArtifact({ directory: params.directory, pathname: params.finalPath }); + params.signal.throwIfAborted(); +} + function indexRecoveredRevision(params: { database: Parameters>[0]; intentId: string; @@ -1709,16 +1752,32 @@ function recoverPendingWrites(agentId: string): void { }); } +/** + * The selected broker calls this before accepting its first socket request. Recovery is explicit + * at the broker lifecycle boundary so a replaced child cannot defer pending-write repair until a + * later caller happens to authorize or mutate that agent's memory. + */ +export function recoverBuiltinScopedMemoryPendingWrites(agentIds: readonly string[]): void { + for (const agentId of [...new Set(agentIds)].toSorted()) { + recoverPendingWrites(agentId); + } +} + async function writeAuthorizedMutation(params: { context: MemoryAccessContext; plan: AuthorizedMemoryPlan; mutation: AuthorizedMemoryMutation; + signal?: AbortSignal; }): Promise { + throwIfAuthorizedMemoryOperationAborted(params.signal); assertMutationShape(params.mutation); if (params.context.operation !== mutationOperation(params.mutation)) { throw new Error("authorized memory mutation is unavailable"); } + // Recovery can activate a durable pending revision, so it must never run after cancellation. + throwIfAuthorizedMemoryOperationAborted(params.signal); recoverPendingWrites(params.context.agentId); + throwIfAuthorizedMemoryOperationAborted(params.signal); const state = readPlan(params); if (!state) { throw new Error("authorized memory mutation is unavailable"); @@ -1726,6 +1785,7 @@ async function writeAuthorizedMutation(params: { const nowMs = Date.now(); const agentId = params.context.agentId; if (params.mutation.kind === "sync") { + throwIfAuthorizedMemoryOperationAborted(params.signal); drainMemoryAuditOutbox(agentId); return Object.freeze({ version: 1, @@ -1737,6 +1797,7 @@ async function writeAuthorizedMutation(params: { } if (params.mutation.kind === "project") { + throwIfAuthorizedMemoryOperationAborted(params.signal); if (params.context.actor.kind !== "principal" || params.mutation.sourceHandles.length !== 1) { throw new Error("authorized memory projection is unavailable"); } @@ -1772,6 +1833,8 @@ async function writeAuthorizedMutation(params: { expiry, nowMs, }); + // Projection creation is its activation point. Do not turn a committed projection into a + // cancellation acknowledgement if the caller disconnects after this synchronous commit. const targetSnapshot = withScopedMemoryDatabase(agentId, (database) => { const db = getNodeSqliteKysely(database); return executeSqliteQueryTakeFirstSync( @@ -1801,6 +1864,7 @@ async function writeAuthorizedMutation(params: { } const result = withScopedMemoryDatabase(agentId, (database, databasePath) => { + throwIfAuthorizedMemoryOperationAborted(params.signal); const store = selectWriteStore({ database, agentId, state }); const db = getNodeSqliteKysely(database); const derivationSources = @@ -1853,6 +1917,9 @@ async function writeAuthorizedMutation(params: { if (existingIntent.mutation_id !== params.mutation.mutationId) { throw new Error("authorized memory mutation idempotency conflict"); } + if (existingIntent.state === "quarantined") { + throw new Error("authorized memory mutation is unavailable"); + } return Object.freeze({ version: 1, mutationId: params.mutation.mutationId, @@ -1869,11 +1936,13 @@ async function writeAuthorizedMutation(params: { (params.mutation.kind === "delete" || params.mutation.kind === "tombstone") && "target" in params.mutation ) { + throwIfAuthorizedMemoryOperationAborted(params.signal); const mutation = params.mutation; if (!existing) { throw new Error("authorized memory mutation is unavailable"); } const intentId = randomUUID(); + throwIfAuthorizedMemoryOperationAborted(params.signal); runSqliteImmediateTransactionSync(database, () => { const current = resolveWriteTarget({ database, @@ -2025,6 +2094,7 @@ async function writeAuthorizedMutation(params: { const finalLocator = `r1_${revisionId}.md`; const stageLocator = `mwst1_${intentId}.tmp`; const directory = path.join(resolveScopedMemoryArtifactBase(databasePath), store.pathKey); + throwIfAuthorizedMemoryOperationAborted(params.signal); fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); const stagePath = path.join(directory, stageLocator); const finalPath = resolveBuiltinScopedMemoryArtifactPath({ @@ -2043,8 +2113,13 @@ async function writeAuthorizedMutation(params: { fs.closeSync(descriptor); } syncDirectory(directory); + if (params.signal?.aborted) { + quarantineArtifact({ directory, pathname: stagePath }); + params.signal.throwIfAborted(); + } const resourceId = existing?.resourceId ?? randomUUID(); try { + throwIfAuthorizedMemoryOperationAborted(params.signal); runSqliteImmediateTransactionSync(database, () => { const currentStore = selectWriteStore({ database, agentId, state }); if ( @@ -2240,8 +2315,26 @@ async function writeAuthorizedMutation(params: { } catch {} throw error; } + throwIfPendingWriteAborted({ + signal: params.signal, + database, + directory, + intentId, + revisionId, + stagedPath: stagePath, + finalPath, + }); fs.renameSync(stagePath, finalPath); syncDirectory(directory); + throwIfPendingWriteAborted({ + signal: params.signal, + database, + directory, + intentId, + revisionId, + stagedPath: stagePath, + finalPath, + }); const verified = readVerifiedFile({ pathname: finalPath, contentHash: hash, @@ -2250,6 +2343,17 @@ async function writeAuthorizedMutation(params: { if (verified === undefined) { throw new Error("authorized memory finalized artifact is unavailable"); } + // This is the write linearization point. A cancellation observed before it quarantines the + // pending revision below; a cancellation after it is a committed write, never a fake abort. + throwIfPendingWriteAborted({ + signal: params.signal, + database, + directory, + intentId, + revisionId, + stagedPath: stagePath, + finalPath, + }); runSqliteImmediateTransactionSync(database, () => { const currentStore = selectWriteStore({ database, agentId, state }); if ( @@ -2382,6 +2486,7 @@ async function writeAuthorizedMutation(params: { async function stageSealedCompaction( params: AuthorizedSealedCompactionStageParams, ): Promise { + throwIfAuthorizedMemoryOperationAborted(params.signal); if (!params.content.trim()) { throw new Error("sealed compaction content is unavailable"); } @@ -2391,6 +2496,7 @@ async function stageSealedCompaction( } const agentId = params.context.agentId; return withScopedMemoryDatabase(agentId, (database, databasePath) => { + throwIfAuthorizedMemoryOperationAborted(params.signal); const store = selectWriteStore({ database, agentId, state }); const source = resolveTranscriptDerivationSource({ database, @@ -2412,6 +2518,7 @@ async function stageSealedCompaction( const stagePath = path.join(directory, stageLocator); const hash = contentHash(params.content); const bytes = Buffer.byteLength(params.content); + throwIfAuthorizedMemoryOperationAborted(params.signal); fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); const descriptor = fs.openSync(stagePath, "wx", 0o600); try { @@ -2422,8 +2529,17 @@ async function stageSealedCompaction( fs.closeSync(descriptor); } syncDirectory(directory); + if (params.signal?.aborted) { + quarantineArtifact({ directory, pathname: stagePath }); + params.signal.throwIfAborted(); + } + throwIfAuthorizedMemoryOperationAborted(params.signal); fs.renameSync(stagePath, finalPath); syncDirectory(directory); + if (params.signal?.aborted) { + quarantineArtifact({ directory, pathname: finalPath }); + params.signal.throwIfAborted(); + } const verified = readVerifiedFile({ pathname: finalPath, contentHash: hash, @@ -2435,6 +2551,9 @@ async function stageSealedCompaction( return Object.freeze({ resourceRevisionId: revisionId, commitInTransaction({ database: transactionDatabase, compactionPolicyId, eventSeq }) { + // This callback runs inside the caller's SQLite transaction. It may only read the abort + // state; filesystem cleanup belongs to staging before a transaction can begin. + throwIfAuthorizedMemoryOperationAborted(params.signal); if (state.expiresAtMs <= Date.now()) { throw new Error("sealed compaction authorization is unavailable"); } @@ -2644,6 +2763,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime( async searchAuthorized( params: AuthorizedMemorySearchParams<"read"> | AuthorizedMemorySearchParams<"derive">, ): Promise> { + throwIfAuthorizedMemoryOperationAborted(params.signal); if (params.context.operation !== "read" && params.context.operation !== "derive") { throw new Error("authorized memory search is unavailable"); } @@ -2664,9 +2784,11 @@ export function createBuiltinScopedMemoryAuthorizedRuntime( offset: 0, }), ); + throwIfAuthorizedMemoryOperationAborted(params.signal); const results: AuthorizedMemorySearchResult[] = []; const sourcePolicySetIds: string[] = []; for (const candidate of candidates) { + throwIfAuthorizedMemoryOperationAborted(params.signal); if (results.length >= limit) { break; } @@ -2690,6 +2812,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime( results.push(result); sourcePolicySetIds.push(`mps1_${snapshot.policyRevisionId}`); } + throwIfAuthorizedMemoryOperationAborted(params.signal); return createEnvelope({ state, context: params.context, @@ -2703,6 +2826,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime( async readAuthorized( params: AuthorizedMemoryReadParams<"read"> | AuthorizedMemoryReadParams<"derive">, ): Promise> { + throwIfAuthorizedMemoryOperationAborted(params.signal); if (params.context.operation !== "read" && params.context.operation !== "derive") { throw new Error("authorized memory read is unavailable"); } @@ -2727,6 +2851,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime( if (!snapshot || snapshot.policyRevisionId !== storedHandle.policyRevision) { throw new Error("authorized memory read is unavailable"); } + throwIfAuthorizedMemoryOperationAborted(params.signal); const lines = snapshot.content.split("\n"); const from = Math.max(1, Math.trunc(params.from ?? 1)); const lineCount = Math.max(1, Math.min(1000, Math.trunc(params.lines ?? 200))); @@ -2753,6 +2878,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime( context: MemoryAccessContext; plan: AuthorizedMemoryPlan; mutation: AuthorizedMemoryMutation; + signal?: AbortSignal; }): Promise { if (params.mutation.kind === "admin-reclassify") { throw new Error("authorized memory reclassification is unavailable"); @@ -2768,6 +2894,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime( context: MemoryAccessContext; plan: AuthorizedMemoryPlan; mutation: Extract; + signal?: AbortSignal; }): Promise { return await writeAuthorizedMutation(params); }, @@ -2775,7 +2902,9 @@ export function createBuiltinScopedMemoryAuthorizedRuntime( async syncAuthorized(params: { context: MemoryAccessContext; plan: AuthorizedMemoryPlan; + signal?: AbortSignal; }): Promise> { + throwIfAuthorizedMemoryOperationAborted(params.signal); if (params.context.operation !== "sync") { throw new Error("authorized memory sync is unavailable"); } @@ -2783,6 +2912,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime( if (!state) { throw new Error("authorized memory sync is unavailable"); } + throwIfAuthorizedMemoryOperationAborted(params.signal); drainMemoryAuditOutbox(params.context.agentId); return createEnvelope({ state, @@ -2797,7 +2927,9 @@ export function createBuiltinScopedMemoryAuthorizedRuntime( context: MemoryAccessContext; plan: AuthorizedMemoryPlan; handles: readonly AuthorizedResourceHandle[]; + signal?: AbortSignal; }): Promise> { + throwIfAuthorizedMemoryOperationAborted(params.signal); if (params.context.operation !== "export") { throw new Error("authorized memory export is unavailable"); } @@ -2826,7 +2958,9 @@ export function createBuiltinScopedMemoryAuthorizedRuntime( async statusAuthorized(params: { context: MemoryAccessContext; plan: AuthorizedMemoryPlan; + signal?: AbortSignal; }): Promise> { + throwIfAuthorizedMemoryOperationAborted(params.signal); if (params.context.operation !== "status") { throw new Error("authorized memory status is unavailable"); } @@ -2852,6 +2986,7 @@ export const builtinScopedMemoryVirtualView = Object.freeze({ async materializeAuthorizedVirtualView(params: { context: MemoryContentAccessContext<"read">; plan: AuthorizedMemoryPlan & Readonly<{ operation: "read" }>; + signal?: AbortSignal; }): Promise { return materializeAuthorizedVirtualView(params); }, @@ -2860,6 +2995,7 @@ export const builtinScopedMemoryVirtualView = Object.freeze({ plan: AuthorizedMemoryPlan & Readonly<{ operation: "read" }>; view: AuthorizedMemoryVirtualView; virtualPath: string; + signal?: AbortSignal; }): Promise> { return readAuthorizedVirtualFile(params); }, diff --git a/extensions/memory-core/src/tools.test.ts b/extensions/memory-core/src/tools.test.ts index a3c662105bfa..c92d3d527082 100644 --- a/extensions/memory-core/src/tools.test.ts +++ b/extensions/memory-core/src/tools.test.ts @@ -775,6 +775,30 @@ describe("enforced memory tools", () => { expect(host.read).toHaveBeenCalledWith({ handleId: "mhandle1_allowed" }); }); + it("forwards caller cancellation into enforced memory_get", async () => { + const host = { + search: vi.fn(async () => ({ results: [] })), + read: vi.fn(async () => ({ text: "allowed", path: "memory/MEMORY.md" })), + }; + const get = createMemoryGetTool({ + config: createDefaultMemoryToolConfig(), + memoryReadEnforced: true, + authorizedMemoryRead: host, + }); + if (!get) { + throw new Error("memory_get missing"); + } + const controller = new AbortController(); + + await expect( + get.execute("authorized-read-with-cancellation", { handleId: "mhandle1_allowed" }, controller.signal), + ).resolves.toMatchObject({ details: { text: "allowed", path: "memory/MEMORY.md" } }); + expect(host.read).toHaveBeenCalledWith({ + handleId: "mhandle1_allowed", + signal: controller.signal, + }); + }); + it("does not fall back to legacy memory when its host is unavailable", async () => { const search = createMemorySearchToolOrThrow({ memoryReadEnforced: true }); const result = await search.execute("missing-authorized-host", { query: "private" }); diff --git a/extensions/memory-core/src/tools.ts b/extensions/memory-core/src/tools.ts index 7c3552487f75..b190c7af6003 100644 --- a/extensions/memory-core/src/tools.ts +++ b/extensions/memory-core/src/tools.ts @@ -776,7 +776,8 @@ export function createMemoryGetTool(options: { parameters: MemoryGetSchema, execute: ({ cfg, agentId }) => - async (_toolCallId, params) => { + async (_toolCallId, params, callerSignal) => { + callerSignal?.throwIfAborted(); const rawParams = asToolParamsRecord(params); const from = readPositiveIntegerParam(rawParams, "from"); const lines = readPositiveIntegerParam(rawParams, "lines"); @@ -790,7 +791,9 @@ export function createMemoryGetTool(options: { handleId, ...(from !== undefined ? { from } : {}), ...(lines !== undefined ? { lines } : {}), + ...(callerSignal ? { signal: callerSignal } : {}), }); + callerSignal?.throwIfAborted(); return isAuthorizedMemoryUnavailable(result) ? authorizedMemoryUnavailableResult() : jsonResult(result); diff --git a/package.json b/package.json index a7ea498c4ddc..e934c9c471ea 100644 --- a/package.json +++ b/package.json @@ -1827,6 +1827,7 @@ "test:docker:doctor-switch": "bash scripts/e2e/doctor-install-switch-docker.sh", "test:docker:compose-setup": "bash scripts/e2e/compose-setup.sh", "test:docker:e2e-build": "bash scripts/e2e/build-image.sh", + "test:docker:fleet-separate-cell": "bash scripts/e2e/fleet-separate-cell-docker.sh", "test:docker:gateway-network": "bash scripts/e2e/gateway-network-docker.sh", "test:docker:kitchen-sink-plugin": "bash scripts/e2e/kitchen-sink-plugin-docker.sh", "test:docker:kitchen-sink-rpc": "bash scripts/e2e/kitchen-sink-rpc-docker.sh", diff --git a/packages/memory-host-sdk/src/host/authorization.ts b/packages/memory-host-sdk/src/host/authorization.ts index f8d1b543f747..113a07e6d31c 100644 --- a/packages/memory-host-sdk/src/host/authorization.ts +++ b/packages/memory-host-sdk/src/host/authorization.ts @@ -397,12 +397,14 @@ export type AuthorizedMemoryReadParams; /** Every authorized method is bound to the operation that produced its plan. */ type AuthorizedMemoryOperationParams = Readonly<{ context: MemoryAccessContext & Readonly<{ operation: Operation }>; plan: AuthorizedMemoryPlan & Readonly<{ operation: Operation }>; + signal?: AbortSignal; }>; type AuthorizedMemoryContentMutation = Readonly<{ @@ -450,6 +452,7 @@ export type AuthorizedSealedCompactionStageParams = Readonly<{ plan: AuthorizedMemoryPlan & Readonly<{ operation: "derive" }>; content: string; transcriptSource: AuthorizedTranscriptDerivationSource; + signal?: AbortSignal; }>; export type AuthorizedMemoryMutation = diff --git a/scripts/e2e/fleet-separate-cell-docker.sh b/scripts/e2e/fleet-separate-cell-docker.sh new file mode 100644 index 000000000000..4cbb251d3728 --- /dev/null +++ b/scripts/e2e/fleet-separate-cell-docker.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "$ROOT_DIR/scripts/lib/docker-e2e-image.sh" + +IMAGE_NAME="$(docker_e2e_resolve_image "openclaw-fleet-separate-cell-e2e" OPENCLAW_FLEET_E2E_IMAGE)" +SKIP_BUILD="${OPENCLAW_FLEET_E2E_SKIP_BUILD:-0}" + +# The scheduler supplies the functional package image. A direct invocation can still build the +# same package-installed image, but this lane never creates a second Fleet-specific image. +docker_e2e_build_or_reuse "$IMAGE_NAME" fleet-separate-cell "$ROOT_DIR/scripts/e2e/Dockerfile" "$ROOT_DIR" "" "$SKIP_BUILD" + +cd "$ROOT_DIR" +OPENCLAW_PROCESS_ISOLATION_E2E=1 \ + OPENCLAW_FLEET_E2E_IMAGE="$IMAGE_NAME" \ + node scripts/run-vitest.mjs src/fleet/service.container.e2e.test.ts diff --git a/scripts/lib/docker-e2e-scenarios.mts b/scripts/lib/docker-e2e-scenarios.mts index c296c9cebef9..89ae4ef0684c 100644 --- a/scripts/lib/docker-e2e-scenarios.mts +++ b/scripts/lib/docker-e2e-scenarios.mts @@ -530,6 +530,11 @@ export const mainLanes: DockerE2eLane[] = [ }, ), serviceLane("gateway-network", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:gateway-network"), + serviceLane( + "fleet-separate-cell", + "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:fleet-separate-cell", + { stateScenario: "empty", timeoutMs: 15 * 60 * 1000, weight: 4 }, + ), serviceLane("browser-cdp-snapshot", "pnpm test:docker:browser-cdp-snapshot", { stateScenario: "empty", timeoutMs: 20 * 60 * 1000, diff --git a/src/agents/memory-authorized-read-host.ts b/src/agents/memory-authorized-read-host.ts index 13df9e39cf69..192f1fc8e465 100644 --- a/src/agents/memory-authorized-read-host.ts +++ b/src/agents/memory-authorized-read-host.ts @@ -45,7 +45,7 @@ const authorizedMemoryVirtualBroker: unique symbol = Symbol( /** Core-private bridge for generic filesystem tools; it is absent from plugin contexts. */ export type AuthorizedMemoryVirtualFileBroker = Readonly<{ view: AuthorizedMemoryVirtualView; - readFile: (virtualPath: string) => Promise; + readFile: (virtualPath: string, signal?: AbortSignal) => Promise; }>; /** Core-private sealed compaction capability; plugins never receive this host. */ @@ -58,16 +58,19 @@ export type AuthorizedSealedCompactionHost = Readonly<{ type AuthorizedMemoryReadHostWithVirtualBroker = AuthorizedMemoryReadHost & Readonly<{ - [authorizedMemoryVirtualBroker]: () => Promise; + [authorizedMemoryVirtualBroker]: ( + signal?: AbortSignal, + ) => Promise; }>; export async function resolveAuthorizedMemoryVirtualFileBroker( host: AuthorizedMemoryReadHost | undefined, + signal?: AbortSignal, ): Promise { if (!host || !(authorizedMemoryVirtualBroker in host)) { return undefined; } - return (host as AuthorizedMemoryReadHostWithVirtualBroker)[authorizedMemoryVirtualBroker](); + return (host as AuthorizedMemoryReadHostWithVirtualBroker)[authorizedMemoryVirtualBroker](signal); } function hash(value: unknown): string { @@ -318,28 +321,34 @@ function createAuthorizedMemoryContentHost( ? createAuthorizedMemoryDeriveInvocation({ context: trusted }) : createAuthorizedMemoryReadInvocation({ context: trusted })); let virtualBroker: Promise | undefined; - const getVirtualBroker = () => - (virtualBroker ??= (async () => { + const createVirtualBroker = async (signal?: AbortSignal) => { + signal?.throwIfAborted(); const active = await getInvocation(); if ("unavailable" in active) { return undefined; } - const view = await materializeAuthorizedMemoryVirtualView({ invocation: active }); + const view = await materializeAuthorizedMemoryVirtualView({ + invocation: active, + ...(signal ? { signal } : {}), + }); if ("unavailable" in view) { return undefined; } return Object.freeze({ view, - async readFile(virtualPath) { + async readFile(virtualPath, signal) { const result = await readAuthorizedMemoryVirtualFile({ invocation: active, view, virtualPath, + ...(signal ? { signal } : {}), }); return "unavailable" in result ? undefined : result.text; }, }); - })()); + }; + const getVirtualBroker = (signal?: AbortSignal) => + signal ? createVirtualBroker(signal) : (virtualBroker ??= createVirtualBroker()); return Object.freeze({ async search(search) { const active = await getInvocation(); diff --git a/src/auto-reply/dispatch.ts b/src/auto-reply/dispatch.ts index 050fcd54c96e..4f4c96ec49b1 100644 --- a/src/auto-reply/dispatch.ts +++ b/src/auto-reply/dispatch.ts @@ -337,7 +337,6 @@ export async function dispatchInboundMessage(params: { } let settledReceipt: DispatchFromConfigResult["settledReceipt"]; installMemoryEgressAdmission(params.dispatcher, finalized, replyPayloadRunState); - let settledReceipt: DispatchFromConfigResult["settledReceipt"]; const result = await withReplyDispatcher({ dispatcher: params.dispatcher, onSettled: params.onSettled, diff --git a/src/commands/backup-sqlite.test.ts b/src/commands/backup-sqlite.test.ts index dddd2518823b..cfe8965ebdda 100644 --- a/src/commands/backup-sqlite.test.ts +++ b/src/commands/backup-sqlite.test.ts @@ -1,6 +1,6 @@ import fs from "node:fs/promises"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import { requireNodeSqlite } from "../infra/node-sqlite.js"; import type { RuntimeEnv } from "../runtime.js"; @@ -11,6 +11,15 @@ import { OPENCLAW_AGENT_SCHEMA_SQL } from "../state/openclaw-agent-schema.js"; import { OPENCLAW_STATE_SCHEMA_VERSION } from "../state/openclaw-state-db.js"; import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; import { OPENCLAW_STATE_SCHEMA_SQL } from "../state/openclaw-state-schema.js"; + +const withDoctorSqliteMaintenanceLock = vi.hoisted(() => + vi.fn(async (params: { run: () => Promise }) => await params.run()), +); + +vi.mock("./doctor-sqlite-maintenance-lock.js", () => ({ + withDoctorSqliteMaintenanceLock, +})); + import { backupSqliteCreateCommand, backupSqliteListCommand, @@ -118,6 +127,30 @@ function createAgentDatabase(databasePath: string, agentId: string): void { } describe("SQLite backup commands", () => { + it("uses the cross-process SQLite ownership lock for an offline snapshot", async () => { + const tempDir = tempDirs.make("openclaw-backup-sqlite-lock-"); + const stateDir = path.join(tempDir, "state"); + const repositoryPath = path.join(tempDir, "snapshots"); + process.env.OPENCLAW_STATE_DIR = stateDir; + const databasePath = resolveOpenClawStateSqlitePath(); + await fs.mkdir(path.dirname(databasePath), { recursive: true }); + createGlobalDatabase(databasePath); + withDoctorSqliteMaintenanceLock.mockClear(); + + await backupSqliteCreateCommand(createRuntimeCapture(), { + global: true, + repository: repositoryPath, + }); + + expect(withDoctorSqliteMaintenanceLock).toHaveBeenCalledWith( + expect.objectContaining({ + operation: "SQLite snapshot", + protectedPaths: [databasePath], + run: expect.any(Function), + }), + ); + }); + it("creates, lists, verifies, and fresh-restores the global database", async () => { const tempDir = tempDirs.make("openclaw-backup-sqlite-"); const stateDir = path.join(tempDir, "state"); diff --git a/src/commands/backup-sqlite.ts b/src/commands/backup-sqlite.ts index 5ab2ef788ad5..3c5b5fbb030a 100644 --- a/src/commands/backup-sqlite.ts +++ b/src/commands/backup-sqlite.ts @@ -1,5 +1,6 @@ import fs from "node:fs/promises"; import path from "node:path"; +import { withDoctorSqliteMaintenanceLock } from "./doctor-sqlite-maintenance-lock.js"; import { formatErrorMessage } from "../infra/errors.js"; import { normalizeAgentId } from "../routing/session-key.js"; import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js"; @@ -77,12 +78,13 @@ export async function backupSqliteCreateCommand( const repositoryPath = resolveRequiredPath(options.repository, "--repository"); try { const database = await resolveSnapshotDatabase(options); - // A selected broker owns memory writes. Ask it to drain before the trusted Gateway snapshots - // an agent database, then always reopen admission; workers never receive a DB path or broker IPC. - const { withBrokeredMemoryMaintenance } = await import("../plugins/memory-broker-runtime.js"); - const result = await withBrokeredMemoryMaintenance( - async () => await createLocalSqliteSnapshotProvider({ repositoryPath }).create(database), - ); + // A CLI snapshot runs outside the Gateway, so its local broker map cannot prove exclusion + // against the live owner. The state lock is the cross-process ownership boundary. + const result = await withDoctorSqliteMaintenanceLock({ + operation: "SQLite snapshot", + protectedPaths: [database.path], + run: async () => await createLocalSqliteSnapshotProvider({ repositoryPath }).create(database), + }); const report: BackupSqliteCreateResult = { ok: true, snapshotPath: result.ref.path, diff --git a/src/commands/backup.create-verify.test.ts b/src/commands/backup.create-verify.test.ts index df7fcc0d8708..95bc80652845 100644 --- a/src/commands/backup.create-verify.test.ts +++ b/src/commands/backup.create-verify.test.ts @@ -8,6 +8,9 @@ const backupVerifyCommandMock = vi.hoisted(() => vi.fn()); const writeRuntimeJsonMock = vi.hoisted(() => vi.fn()); const formatBackupCreateSummaryMock = vi.hoisted(() => vi.fn(() => ["backup ok"])); const recordBackupRunOutcomeMock = vi.hoisted(() => vi.fn()); +const withDoctorSqliteMaintenanceLockMock = vi.hoisted(() => + vi.fn(async (params: { run: () => Promise }) => await params.run()), +); vi.mock("../infra/backup-create.js", () => ({ createBackupArchive: createBackupArchiveMock, @@ -30,6 +33,10 @@ vi.mock("../state/backup-run-records.js", () => ({ recordBackupRunOutcome: recordBackupRunOutcomeMock, })); +vi.mock("./doctor-sqlite-maintenance-lock.js", () => ({ + withDoctorSqliteMaintenanceLock: withDoctorSqliteMaintenanceLockMock, +})); + function createRuntime(): RuntimeEnv { return { log: vi.fn(), @@ -54,6 +61,10 @@ describe("backupCreateCommand verify wrapper", () => { formatBackupCreateSummaryMock.mockReset(); formatBackupCreateSummaryMock.mockReturnValue(["backup ok"]); recordBackupRunOutcomeMock.mockReset(); + withDoctorSqliteMaintenanceLockMock.mockReset(); + withDoctorSqliteMaintenanceLockMock.mockImplementation( + async (params: { run: () => Promise }) => await params.run(), + ); }); it("optionally verifies the archive after writing it", async () => { @@ -93,6 +104,31 @@ describe("backupCreateCommand verify wrapper", () => { }); expect(verifyLog).not.toBe(runtime.log); expect(typeof verifyLog).toBe("function"); + expect(withDoctorSqliteMaintenanceLockMock).toHaveBeenCalledWith( + expect.objectContaining({ + operation: "backup archive creation", + protectedPaths: [expect.any(String)], + run: expect.any(Function), + }), + ); + }); + + it("does not take offline state ownership for dry runs or config-only archives", async () => { + createBackupArchiveMock.mockResolvedValue({ + archivePath: "/tmp/openclaw-backup.tar.gz", + archiveRoot: "openclaw-backup", + createdAt: "2026-04-07T00:00:00.000Z", + assets: [], + verified: false, + dryRun: true, + includeWorkspace: false, + onlyConfig: false, + }); + + await backupCreateCommand(createRuntime(), { dryRun: true }); + await backupCreateCommand(createRuntime(), { onlyConfig: true }); + + expect(withDoctorSqliteMaintenanceLockMock).not.toHaveBeenCalled(); }); it("does not claim completion when both backup and outcome recording fail", async () => { diff --git a/src/commands/backup.test.ts b/src/commands/backup.test.ts index 45a84755edff..431b8677110a 100644 --- a/src/commands/backup.test.ts +++ b/src/commands/backup.test.ts @@ -25,6 +25,14 @@ import { tarCreateMock, } from "./backup.test-support.js"; +const withDoctorSqliteMaintenanceLock = vi.hoisted(() => + vi.fn(async (params: { run: () => Promise }) => await params.run()), +); + +vi.mock("./doctor-sqlite-maintenance-lock.js", () => ({ + withDoctorSqliteMaintenanceLock, +})); + const { backupCreateCommand } = await import("./backup.js"); type CapturedBackupManifest = { @@ -91,6 +99,10 @@ describe("backup commands", () => { assetCount: 1, entryCount: 2, }); + withDoctorSqliteMaintenanceLock.mockClear(); + withDoctorSqliteMaintenanceLock.mockImplementation( + async (params: { run: () => Promise }) => await params.run(), + ); }); afterEach(async () => { @@ -633,9 +645,10 @@ describe("backup commands", () => { setTestEnvValue("OPENCLAW_CONFIG_PATH", configPath); try { const plan = await resolveBackupPlanFromDisk({ nowMs: 123 }); + const canonicalWorkspaceDir = await fs.realpath(workspaceDir); expect(plan.included).toContainEqual( - expect.objectContaining({ kind: "workspace", sourcePath: workspaceDir }), + expect.objectContaining({ kind: "workspace", sourcePath: canonicalWorkspaceDir }), ); expect(await fs.readFile(configPath, "utf8")).toBe(originalRaw); expect(plan.configPath).toBe(configPath); diff --git a/src/commands/backup.ts b/src/commands/backup.ts index 00af4a7c025c..9bfb779b925a 100644 --- a/src/commands/backup.ts +++ b/src/commands/backup.ts @@ -5,10 +5,12 @@ import { type BackupCreateOptions, type BackupCreateResult, } from "../infra/backup-create.js"; +import { resolveStateDir } from "../config/paths.js"; import { formatErrorMessage } from "../infra/errors.js"; import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js"; import { createLazyImportLoader } from "../shared/lazy-promise.js"; import { recordBackupRunOutcome } from "../state/backup-run-records.js"; +import { withDoctorSqliteMaintenanceLock } from "./doctor-sqlite-maintenance-lock.js"; type BackupVerifyRuntime = typeof import("./backup-verify.js"); @@ -27,10 +29,22 @@ export async function backupCreateCommand( ): Promise { let archivePath = opts.output ?? process.cwd(); try { - const result = await createBackupArchive({ - ...opts, - log: opts.log ?? (opts.json ? undefined : (message: string) => runtime.log(message)), - }); + const createArchive = async () => + await createBackupArchive({ + ...opts, + log: opts.log ?? (opts.json ? undefined : (message: string) => runtime.log(message)), + }); + // Archive creation captures the selected memory state and its SQLite snapshots. A CLI cannot + // quiesce a live Gateway-owned broker, so require offline state ownership instead of taking a + // potentially inconsistent archive while the broker can still activate artifacts. + const result = + opts.dryRun || opts.onlyConfig + ? await createArchive() + : await withDoctorSqliteMaintenanceLock({ + operation: "backup archive creation", + protectedPaths: [resolveStateDir(process.env)], + run: createArchive, + }); archivePath = result.archivePath; if (opts.verify && !opts.dryRun) { const { backupVerifyCommand } = await loadBackupVerifyRuntime(); diff --git a/src/commands/doctor-memory-isolation.test.ts b/src/commands/doctor-memory-isolation.test.ts index c0d0eaa67e55..83ec09d12977 100644 --- a/src/commands/doctor-memory-isolation.test.ts +++ b/src/commands/doctor-memory-isolation.test.ts @@ -19,6 +19,9 @@ describe("runDoctorMemoryIsolation", () => { beforeEach(() => { stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-memory-isolation-")); vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + const configPath = path.join(stateDir, "openclaw.json"); + fs.writeFileSync(configPath, "{}\n", "utf8"); + vi.stubEnv("OPENCLAW_CONFIG_PATH", configPath); const database = openOpenClawAgentDatabase({ agentId: "main" }); database.db .prepare( @@ -36,27 +39,29 @@ describe("runDoctorMemoryIsolation", () => { fs.rmSync(stateDir, { force: true, recursive: true }); }); - it("owns the reversible configured-agent shadow lifecycle", () => { - expect(runDoctorMemoryIsolation({ action: "status", cfg })).toEqual({ + it("owns the reversible configured-agent shadow lifecycle", async () => { + await expect(runDoctorMemoryIsolation({ action: "status", cfg })).resolves.toEqual({ agentId: "main", mode: "legacy", restartRequired: false, }); - expect(runDoctorMemoryIsolation({ action: "shadow-read-only", cfg, nowMs: 1 })).toEqual({ + await expect( + runDoctorMemoryIsolation({ action: "shadow-read-only", cfg, nowMs: 1 }), + ).resolves.toEqual({ agentId: "main", mode: "shadow-read-only", restartRequired: true, }); - expect(runDoctorMemoryIsolation({ action: "legacy", cfg })).toEqual({ + await expect(runDoctorMemoryIsolation({ action: "legacy", cfg })).resolves.toEqual({ agentId: "main", mode: "legacy", restartRequired: true, }); }); - it("refuses an agent outside runtime configuration", () => { - expect(() => + it("refuses an agent outside runtime configuration", async () => { + await expect( runDoctorMemoryIsolation({ action: "shadow-read-only", agentId: "missing", cfg }), - ).toThrow('Unknown configured agent id "missing".'); + ).rejects.toThrow('Unknown configured agent id "missing".'); }); }); diff --git a/src/commands/doctor-memory-isolation.ts b/src/commands/doctor-memory-isolation.ts index 5e014e1a5100..04d2acc40037 100644 --- a/src/commands/doctor-memory-isolation.ts +++ b/src/commands/doctor-memory-isolation.ts @@ -8,6 +8,8 @@ import { resolveMemoryIsolationMode, type MemoryIsolationMode, } from "../plugins/memory-cutover.js"; +import { resolveOpenClawAgentSqlitePath } from "../state/openclaw-agent-db.paths.js"; +import { withDoctorSqliteMaintenanceLock } from "./doctor-sqlite-maintenance-lock.js"; export type DoctorMemoryIsolationAction = "status" | "shadow-read-only" | "legacy"; @@ -32,28 +34,36 @@ function resolveDoctorMemoryIsolationAgent(params: { * Doctor owns the only P1C enablement path. It persists one reversible, verified posture and * deliberately does not create a Phase 6 cutover marker or claim two-subject confinement. */ -export function runDoctorMemoryIsolation(params: { +export async function runDoctorMemoryIsolation(params: { action: DoctorMemoryIsolationAction; agentId?: string; cfg?: OpenClawConfig; nowMs?: number; -}): DoctorMemoryIsolationReport { +}): Promise { const cfg = params.cfg ?? getRuntimeConfig(); const agentId = resolveDoctorMemoryIsolationAgent({ agentId: params.agentId, cfg }); switch (params.action) { case "status": return { agentId, mode: resolveMemoryIsolationMode(agentId), restartRequired: false }; case "shadow-read-only": - return { - agentId, - mode: enableMemoryShadowReadOnlyMode({ agentId, nowMs: params.nowMs }), - restartRequired: true, - }; + return await withDoctorSqliteMaintenanceLock({ + operation: "memory isolation shadow-read-only", + protectedPaths: [resolveOpenClawAgentSqlitePath({ agentId })], + run: () => ({ + agentId, + mode: enableMemoryShadowReadOnlyMode({ agentId, nowMs: params.nowMs }), + restartRequired: true, + }), + }); case "legacy": - return { - agentId, - mode: disableMemoryShadowReadOnlyMode({ agentId }), - restartRequired: true, - }; + return await withDoctorSqliteMaintenanceLock({ + operation: "memory isolation legacy", + protectedPaths: [resolveOpenClawAgentSqlitePath({ agentId })], + run: () => ({ + agentId, + mode: disableMemoryShadowReadOnlyMode({ agentId }), + restartRequired: true, + }), + }); } } diff --git a/src/commands/doctor-memory-search.test.ts b/src/commands/doctor-memory-search.test.ts index 74ee1fbaee18..68dc1f22e085 100644 --- a/src/commands/doctor-memory-search.test.ts +++ b/src/commands/doctor-memory-search.test.ts @@ -29,6 +29,9 @@ const repairDreamingArtifacts = vi.hoisted(() => vi.fn()); const repairShortTermPromotionArtifacts = vi.hoisted(() => vi.fn()); const noteWorkspaceMemoryHealth = vi.hoisted(() => vi.fn(async () => undefined)); const maybeRepairWorkspaceMemoryHealth = vi.hoisted(() => vi.fn(async () => undefined)); +const withDoctorSqliteMaintenanceLock = vi.hoisted(() => + vi.fn(async (params: { run: () => Promise }) => await params.run()), +); vi.mock("../../packages/terminal-core/src/note.js", () => ({ note, @@ -99,6 +102,10 @@ vi.mock("./doctor-workspace.js", async (importOriginal) => { }; }); +vi.mock("./doctor-sqlite-maintenance-lock.js", () => ({ + withDoctorSqliteMaintenanceLock, +})); + import { noteMemorySearchHealth, maybeRepairMemoryRecallHealth, @@ -158,6 +165,10 @@ function resetMemoryRecallMocks() { }); noteWorkspaceMemoryHealth.mockClear(); maybeRepairWorkspaceMemoryHealth.mockClear(); + withDoctorSqliteMaintenanceLock.mockClear(); + withDoctorSqliteMaintenanceLock.mockImplementation( + async (params: { run: () => Promise }) => await params.run(), + ); } function firstNoteMessage(): string { @@ -1199,6 +1210,9 @@ describe("memory recall doctor integration", () => { expect(repairShortTermPromotionArtifacts).toHaveBeenCalledWith({ workspaceDir: "/tmp/agent-default/workspace", }); + expect(withDoctorSqliteMaintenanceLock).toHaveBeenCalledWith( + expect.objectContaining({ operation: "memory recall artifact repair", run: expect.any(Function) }), + ); expect(note).toHaveBeenCalledTimes(1); expectFirstNoteContains( "Memory recall artifacts repaired:", @@ -1251,6 +1265,9 @@ describe("memory recall doctor integration", () => { expect(repairDreamingArtifacts).toHaveBeenCalledWith({ workspaceDir: "/tmp/agent-default/workspace", }); + expect(withDoctorSqliteMaintenanceLock).toHaveBeenCalledWith( + expect.objectContaining({ operation: "dreaming artifact repair", run: expect.any(Function) }), + ); const message = String(note.mock.calls[note.mock.calls.length - 1]?.[0] ?? ""); expect(message).toContain("Dreaming artifacts repaired:"); expect(message).toContain("archived session corpus"); diff --git a/src/commands/doctor-memory-search.ts b/src/commands/doctor-memory-search.ts index 53bb6e2cf9be..34dd4412f0c3 100644 --- a/src/commands/doctor-memory-search.ts +++ b/src/commands/doctor-memory-search.ts @@ -49,6 +49,7 @@ import { defaultSlotIdForKey } from "../plugins/slots.js"; import { getProviderEnvVars } from "../secrets/provider-env-vars.js"; import { resolveUserPath } from "../utils.js"; import type { DoctorPrompter } from "./doctor-prompter.js"; +import { withDoctorSqliteMaintenanceLock } from "./doctor-sqlite-maintenance-lock.js"; import { maybeRepairWorkspaceMemoryHealth, noteWorkspaceMemoryHealth } from "./doctor-workspace.js"; import { isRecord } from "./doctor/shared/legacy-config-record-shared.js"; @@ -378,7 +379,10 @@ export async function maybeRepairMemoryRecallHealth(params: { initialValue: true, }); if (approved) { - const repair = await repairShortTermPromotionArtifacts({ workspaceDir }); + const repair = await withDoctorSqliteMaintenanceLock({ + operation: "memory recall artifact repair", + run: async () => await repairShortTermPromotionArtifacts({ workspaceDir }), + }); if (repair.changed) { const removedOverflowEntries = repair.removedOverflowEntries ?? 0; const details = [ @@ -424,7 +428,10 @@ export async function maybeRepairMemoryRecallHealth(params: { if (!approvedDreamingRepair) { continue; } - const dreamingRepair = await repairDreamingArtifacts({ workspaceDir }); + const dreamingRepair = await withDoctorSqliteMaintenanceLock({ + operation: "dreaming artifact repair", + run: async () => await repairDreamingArtifacts({ workspaceDir }), + }); if (!dreamingRepair.changed) { continue; } diff --git a/src/commands/doctor-sqlite-maintenance-lock.test.ts b/src/commands/doctor-sqlite-maintenance-lock.test.ts index a166492bc024..f373e6be183a 100644 --- a/src/commands/doctor-sqlite-maintenance-lock.test.ts +++ b/src/commands/doctor-sqlite-maintenance-lock.test.ts @@ -141,6 +141,50 @@ describe("doctor SQLite maintenance lock", () => { await gatewayLock.release(); }); + it("recognizes a live backup process as an offline maintenance owner", async () => { + const fixture = await createLockFixture(); + const lockOptions = { + ...fixture.lockOptions, + readProcessCmdline: () => ["openclaw", "backup", "sqlite", "create"], + }; + let allowMaintenanceToFinish: (() => void) | undefined; + let markMaintenanceStarted: (() => void) | undefined; + const maintenanceMayFinish = new Promise((resolve) => { + allowMaintenanceToFinish = resolve; + }); + const maintenanceStarted = new Promise((resolve) => { + markMaintenanceStarted = resolve; + }); + const maintenance = withDoctorSqliteMaintenanceLock( + { + env: fixture.env, + operation: "SQLite snapshot", + run: async () => { + markMaintenanceStarted?.(); + await maintenanceMayFinish; + }, + }, + { lockOptions }, + ); + await maintenanceStarted; + + try { + await expect( + withDoctorSqliteMaintenanceLock( + { + env: fixture.env, + operation: "state SQLite compaction", + run: vi.fn(), + }, + { lockOptions }, + ), + ).rejects.toBeInstanceOf(DoctorSqliteMaintenanceLockUnavailableError); + } finally { + allowMaintenanceToFinish?.(); + await maintenance; + } + }); + it("releases ownership after maintenance fails", async () => { const fixture = await createLockFixture(); diff --git a/src/commands/doctor-sqlite-maintenance-lock.ts b/src/commands/doctor-sqlite-maintenance-lock.ts index 3fdf4fccbc9f..122af2b4616b 100644 --- a/src/commands/doctor-sqlite-maintenance-lock.ts +++ b/src/commands/doctor-sqlite-maintenance-lock.ts @@ -161,7 +161,9 @@ export async function withDoctorSqliteMaintenanceLock( allowInTests: true, env, pollIntervalMs: lockOptions?.pollIntervalMs ?? MAINTENANCE_LOCK_POLL_INTERVAL_MS, - role: "sqlite-maintenance", + // Doctor and `backup sqlite create` both mutate/read the same state files while offline. + // Their lock must survive stale-owner checks for either CLI command. + role: "offline-maintenance", timeoutMs: lockOptions?.timeoutMs ?? MAINTENANCE_LOCK_TIMEOUT_MS, }); } catch (error) { diff --git a/src/commands/doctor-workspace.test.ts b/src/commands/doctor-workspace.test.ts index 777578b403fb..af5709755293 100644 --- a/src/commands/doctor-workspace.test.ts +++ b/src/commands/doctor-workspace.test.ts @@ -7,11 +7,18 @@ import type { OpenClawConfig } from "../config/config.js"; import type { DoctorPrompter } from "./doctor-prompter.js"; const note = vi.hoisted(() => vi.fn()); +const withDoctorSqliteMaintenanceLock = vi.hoisted(() => + vi.fn(async (params: { run: () => Promise }) => await params.run()), +); vi.mock("../../packages/terminal-core/src/note.js", () => ({ note, })); +vi.mock("./doctor-sqlite-maintenance-lock.js", () => ({ + withDoctorSqliteMaintenanceLock, +})); + import { detectRootMemoryFiles, formatRootMemoryFilesWarning, @@ -46,6 +53,10 @@ describe("root memory repair", () => { beforeEach(async () => { tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-root-memory-")); note.mockClear(); + withDoctorSqliteMaintenanceLock.mockClear(); + withDoctorSqliteMaintenanceLock.mockImplementation( + async (params: { run: () => Promise }) => await params.run(), + ); }); afterEach(async () => { @@ -202,6 +213,9 @@ describe("root memory repair", () => { message: "Merge legacy root memory.md into canonical MEMORY.md and remove the shadowed file?", initialValue: true, }); + expect(withDoctorSqliteMaintenanceLock).toHaveBeenCalledWith( + expect.objectContaining({ operation: "workspace root memory repair", run: expect.any(Function) }), + ); const canonical = await fs.readFile(path.join(tmpDir, "MEMORY.md"), "utf8"); expect(canonical).toContain("# Legacy"); await expectPathMissing(path.join(tmpDir, "memory.md")); diff --git a/src/commands/doctor-workspace.ts b/src/commands/doctor-workspace.ts index f11bd217f216..72f966a59f8f 100644 --- a/src/commands/doctor-workspace.ts +++ b/src/commands/doctor-workspace.ts @@ -16,6 +16,7 @@ import { } from "../memory/root-memory-files.js"; import { shortenHomePath } from "../utils.js"; import type { DoctorPrompter } from "./doctor-prompter.js"; +import { withDoctorSqliteMaintenanceLock } from "./doctor-sqlite-maintenance-lock.js"; // AGENTS.md is only scanned for a memory-system reference; a small cap prevents // a huge file from being buffered just for one regex check. @@ -374,7 +375,12 @@ export async function maybeRepairWorkspaceMemoryHealth(params: { if (!approvedLegacyMigration) { return; } - const migration = await migrateLegacyRootMemoryFile(configuredWorkspaceDir); + // Prompting remains outside the lease. Once approved, stop the live Gateway from changing + // memory-derived workspace state while this doctor command archives and merges root memory. + const migration = await withDoctorSqliteMaintenanceLock({ + operation: "workspace root memory repair", + run: async () => await migrateLegacyRootMemoryFile(configuredWorkspaceDir), + }); if (migration.readLimitExceeded) { note( [ diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index f786a702c690..e4610184a621 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -48,7 +48,7 @@ export async function doctorCommand(runtime?: RuntimeEnv, options?: DoctorOption if (options?.memoryIsolation) { const outputRuntime = runtime ?? defaultRuntime; const { runDoctorMemoryIsolation } = await import("./doctor-memory-isolation.js"); - const report = runDoctorMemoryIsolation({ + const report = await runDoctorMemoryIsolation({ action: options.memoryIsolation, ...(options.memoryIsolationAgent ? { agentId: options.memoryIsolationAgent } : {}), }); diff --git a/src/config/sessions/session-transcript-memory-policy.test.ts b/src/config/sessions/session-transcript-memory-policy.test.ts index fb4d0bc1d250..71774f465e8d 100644 --- a/src/config/sessions/session-transcript-memory-policy.test.ts +++ b/src/config/sessions/session-transcript-memory-policy.test.ts @@ -626,14 +626,23 @@ describe("transcript memory policy companions", () => { }, ); - vi.stubEnv("OPENCLAW_STATE_DIR", env.OPENCLAW_STATE_DIR ?? ""); - expect( + const stateDir = env.OPENCLAW_STATE_DIR ?? ""; + const configPath = path.join(stateDir, "openclaw.json"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(configPath, "{}\n", "utf8"); + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + vi.stubEnv("OPENCLAW_CONFIG_PATH", configPath); + await expect( runDoctorMemoryIsolation({ action: "shadow-read-only", cfg: { agents: { list: [{ id: AGENT_ID, default: true }] } } as OpenClawConfig, nowMs: 1, }), - ).toMatchObject({ agentId: AGENT_ID, mode: "shadow-read-only", restartRequired: true }); + ).resolves.toMatchObject({ + agentId: AGENT_ID, + mode: "shadow-read-only", + restartRequired: true, + }); // Doctor writes out of process. Refresh the process-owned snapshot to model the required // Gateway restart before proving the protected transcript boundary. resetMemoryIsolationCutoverForTest(); diff --git a/src/fleet/service.container.e2e.test.ts b/src/fleet/service.container.e2e.test.ts new file mode 100644 index 000000000000..28d665ff35f7 --- /dev/null +++ b/src/fleet/service.container.e2e.test.ts @@ -0,0 +1,245 @@ +import crypto from "node:crypto"; +import { afterEach, describe, expect, it } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { DOCKER_SANDBOX_ENGINE, execContainer } from "../agents/sandbox/docker.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { cellAuthSecretDir, cellContainerName, cellNetworkName } from "./cell-profile.js"; +import { createFleetContainerRuntime } from "./containers.runtime.js"; +import { createFleetService } from "./service.runtime.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); +const DOCKER_COMMAND_TIMEOUT_MS = 30_000; + +function uniqueTenant(prefix: string): string { + return `${prefix}-${process.pid}-${crypto.randomBytes(4).toString("hex")}`; +} + +async function docker(args: string[], allowFailure = false) { + return await execContainer(DOCKER_SANDBOX_ENGINE, args, { + allowFailure, + signal: AbortSignal.timeout(DOCKER_COMMAND_TIMEOUT_MS), + }); +} + +function isMissingDockerObject(stderr: string): boolean { + return /no such (?:container|network|object)|not found/iu.test(stderr); +} + +async function cleanupCell(params: { + service: ReturnType; + tenant: string; +}): Promise { + try { + await params.service.remove({ tenant: params.tenant, force: true, purgeData: true }); + return; + } catch (serviceError) { + // Preserve Fleet as the normal cleanup owner. These are exact, test-generated names only, + // and recover a cell left behind by a failed health gate before the registry can remove it. + const failures: unknown[] = [serviceError]; + for (const args of [ + ["rm", "--force", cellContainerName(params.tenant)], + ["network", "rm", cellNetworkName(params.tenant)], + ]) { + const result = await docker(args, true); + if (result.code !== 0 && !isMissingDockerObject(result.stderr)) { + failures.push(new Error(result.stderr.trim() || `docker ${args.join(" ")} failed`)); + } + } + if (failures.length > 1) { + throw new AggregateError(failures, `Could not clean up Fleet cell ${params.tenant}.`); + } + } +} + +describe.runIf(process.env.OPENCLAW_PROCESS_ISOLATION_E2E === "1")( + "fleet separate-cell process isolation", + () => { + it("keeps a hostile tenant process out of another cell's data, credentials, mounts, and network", async () => { + const image = process.env.OPENCLAW_FLEET_E2E_IMAGE; + if (!image) { + throw new Error( + "Fleet process-isolation proof requires OPENCLAW_FLEET_E2E_IMAGE from the Docker E2E scheduler.", + ); + } + const root = tempDirs.make("openclaw-fleet-separate-cell-"); + const tenantA = uniqueTenant("fleeta"); + const tenantB = uniqueTenant("fleetb"); + const foreignSecret = crypto.randomBytes(16).toString("hex"); + const containers = createFleetContainerRuntime(); + const service = createFleetService({ + containers, + env: { ...process.env, OPENCLAW_STATE_DIR: root }, + }); + try { + const cellA = await service.create({ + tenant: tenantA, + image, + env: [ + "FLEET_CELL_E2E_A_MARKER=visible-only-to-a", + `FLEET_CELL_E2E_A_SECRET=${crypto.randomBytes(16).toString("hex")}`, + "OPENCLAW_SKIP_CHANNELS=1", + "OPENCLAW_SKIP_GMAIL_WATCHER=1", + "OPENCLAW_SKIP_CRON=1", + "OPENCLAW_SKIP_CANVAS_HOST=1", + ], + }); + const cellB = await service.create({ + tenant: tenantB, + image, + env: [ + "FLEET_CELL_E2E_B_MARKER=visible-only-to-b", + `FLEET_CELL_E2E_B_SECRET=${foreignSecret}`, + "OPENCLAW_SKIP_CHANNELS=1", + "OPENCLAW_SKIP_GMAIL_WATCHER=1", + "OPENCLAW_SKIP_CRON=1", + "OPENCLAW_SKIP_CANVAS_HOST=1", + ], + }); + + // Each create already waits for its own /healthz response. Query status serially so this + // isolation proof does not add an unrelated pair of cold-Gateway health requests. + const statusA = await service.status(tenantA); + const statusB = await service.status(tenantB); + expect(statusA.container).toMatchObject({ + managed: true, + running: true, + state: "running", + }); + expect(statusA.health, JSON.stringify(statusA.health)).toMatchObject({ + httpStatus: 200, + status: "ok", + }); + expect(statusB.container).toMatchObject({ + managed: true, + running: true, + state: "running", + }); + expect(statusB.health, JSON.stringify(statusB.health)).toMatchObject({ + httpStatus: 200, + status: "ok", + }); + + const doctor = await service.doctor(); + expect(doctor).toHaveLength(2); + expect( + doctor + .flatMap((report) => report.findings) + .filter((finding) => finding.status === "fail"), + ).toEqual([]); + + const [pidA, pidB] = await Promise.all( + [cellA.containerName, cellB.containerName].map(async (containerName) => { + const result = await docker(["inspect", "--format", "{{.State.Pid}}", containerName]); + expect(result.code, result.stderr).toBe(0); + return Number(result.stdout.trim()); + }), + ); + expect(pidA).toBeGreaterThan(0); + expect(pidB).toBeGreaterThan(0); + expect(pidA).not.toBe(pidB); + + const stateSentinel = "issued-a-memory-view"; + const foreignSentinel = "issued-b-memory-view"; + for (const [containerName, sentinelName, sentinel] of [ + [cellA.containerName, "issued-a.txt", stateSentinel], + [cellB.containerName, "issued-b.txt", foreignSentinel], + ] as const) { + const result = await docker([ + "exec", + containerName, + "node", + "-e", + `require("node:fs").writeFileSync(require("node:path").join(process.env.OPENCLAW_STATE_DIR, ${JSON.stringify(sentinelName)}), ${JSON.stringify(sentinel)});`, + ]); + expect(result.code, result.stderr).toBe(0); + } + + const foreignDataDir = statusB.dataDir; + const foreignAuthDir = cellAuthSecretDir(root, tenantB); + const hostileProbe = [ + 'const fs = require("node:fs");', + 'const path = require("node:path");', + `const foreignDataDir = ${JSON.stringify(foreignDataDir)};`, + `const foreignAuthDir = ${JSON.stringify(foreignAuthDir)};`, + `const foreignSecret = ${JSON.stringify(foreignSecret)};`, + "const probe = (read) => { try { read(); return true; } catch { return false; } };", + 'const mountInfo = fs.readFileSync("/proc/self/mountinfo", "utf8");', + "process.stdout.write(JSON.stringify({", + ' ownSentinel: fs.readFileSync(path.join(process.env.OPENCLAW_STATE_DIR, "issued-a.txt"), "utf8"),', + " foreignDataDirectory: probe(() => fs.readdirSync(foreignDataDir)),", + ' foreignDataSentinel: probe(() => fs.readFileSync(path.join(foreignDataDir, "issued-b.txt"), "utf8")),', + " foreignAuthDirectory: probe(() => fs.readdirSync(foreignAuthDir)),", + ' foreignEnvironmentMarker: Object.hasOwn(process.env, "FLEET_CELL_E2E_B_MARKER"),', + ' foreignEnvironmentSecret: Object.hasOwn(process.env, "FLEET_CELL_E2E_B_SECRET"),', + " foreignEnvironmentSecretValue: Object.values(process.env).includes(foreignSecret),", + " foreignMount: mountInfo.includes(foreignDataDir) || mountInfo.includes(foreignAuthDir),", + "}));", + ].join("\n"); + const hostileResult = await docker([ + "exec", + cellA.containerName, + "node", + "-e", + hostileProbe, + ]); + expect(hostileResult.code, hostileResult.stderr).toBe(0); + expect(JSON.parse(hostileResult.stdout)).toEqual({ + ownSentinel: stateSentinel, + foreignDataDirectory: false, + foreignDataSentinel: false, + foreignAuthDirectory: false, + foreignEnvironmentMarker: false, + foreignEnvironmentSecret: false, + foreignEnvironmentSecretValue: false, + foreignMount: false, + }); + + const inspectedMounts = await docker([ + "inspect", + "--format", + "{{json .Mounts}}", + cellA.containerName, + ]); + expect(inspectedMounts.code, inspectedMounts.stderr).toBe(0); + const mounts: unknown = JSON.parse(inspectedMounts.stdout); + if (!Array.isArray(mounts)) { + throw new Error("Fleet container inspection did not return mounts."); + } + const mountSources = mounts.flatMap((mount) => { + if ( + !mount || + typeof mount !== "object" || + !("Source" in mount) || + typeof mount.Source !== "string" + ) { + return []; + } + return [mount.Source]; + }); + expect(mountSources).not.toContain(foreignDataDir); + expect(mountSources).not.toContain(foreignAuthDir); + + const [networkA, networkB] = await Promise.all([ + containers.inspectNetwork("docker", cellNetworkName(tenantA)), + containers.inspectNetwork("docker", cellNetworkName(tenantB)), + ]); + expect(networkA).toMatchObject({ + kind: "ok", + attachedContainers: [{ name: cellA.containerName }], + }); + expect(networkB).toMatchObject({ + kind: "ok", + attachedContainers: [{ name: cellB.containerName }], + }); + if (networkA.kind === "ok" && networkB.kind === "ok") { + expect(networkA.attachedContainers).toHaveLength(1); + expect(networkB.attachedContainers).toHaveLength(1); + } + } finally { + await cleanupCell({ service, tenant: tenantB }); + await cleanupCell({ service, tenant: tenantA }); + closeOpenClawStateDatabaseForTest(); + } + }, 150_000); + }, +); diff --git a/src/gateway/gateway-http-route-contracts.ts b/src/gateway/gateway-http-route-contracts.ts index 83424922fffb..da7c32325e12 100644 --- a/src/gateway/gateway-http-route-contracts.ts +++ b/src/gateway/gateway-http-route-contracts.ts @@ -11,6 +11,7 @@ export const MCP_APP_STANDALONE_PATH = "/__openclaw__/mcp-app"; export const MCP_APP_STANDALONE_VIEW_PATH = `${MCP_APP_STANDALONE_PATH}/view`; const WORKER_GATEWAY_PATH = "/__openclaw__/worker"; const NODE_WORKER_BUNDLE_TRANSFER_NAMESPACE = "/__openclaw__/worker-bundle"; +const NODE_WORKER_PROJECTION_TRANSFER_NAMESPACE = "/__openclaw__/worker-memory-projection"; const NODE_WORKSPACE_TRANSFER_NAMESPACE = "/__openclaw__/worker-transfer"; export function classifyGatewayProbePath( @@ -53,6 +54,13 @@ export function classifyNodeWorkerBundleTransferPath(pathname: string): "namespa : "outside"; } +export function classifyNodeWorkerProjectionTransferPath(pathname: string): "namespace" | "outside" { + return pathname === NODE_WORKER_PROJECTION_TRANSFER_NAMESPACE || + pathname.startsWith(`${NODE_WORKER_PROJECTION_TRANSFER_NAMESPACE}/`) + ? "namespace" + : "outside"; +} + export function classifyNodeWorkspaceTransferPath(pathname: string): "namespace" | "outside" { return pathname === NODE_WORKSPACE_TRANSFER_NAMESPACE || pathname.startsWith(`${NODE_WORKSPACE_TRANSFER_NAMESPACE}/`) diff --git a/src/gateway/node-runner-inventory-runtime.ts b/src/gateway/node-runner-inventory-runtime.ts index 2c263ecdead7..5e993081cd41 100644 --- a/src/gateway/node-runner-inventory-runtime.ts +++ b/src/gateway/node-runner-inventory-runtime.ts @@ -4,11 +4,18 @@ import { NODE_WORKER_SUPERVISOR_BINARY_CAPACITY_PROTOCOL_FEATURE, NODE_WORKER_SUPERVISOR_BUILD_PROTOCOL_FEATURE, NODE_WORKER_SUPERVISOR_EXECUTION_CONTEXT_V1_PROTOCOL_FEATURE, + NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE, + NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE, NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE, NODE_WORKER_SUPERVISOR_LEGACY_PROTOCOL_FEATURE, type NodeRunnerInventoryIssue, type NodeWorkerHostDeclaration, } from "../infra/node-runner-inventory.js"; +import { + NODE_WORKER_EXECUTION_CONTAINER_V1, + NODE_WORKER_EXECUTION_HOST_V1, + type NodeWorkerExecution, +} from "../worker/node-supervisor-protocol.js"; export type NodeRunnerRegistrySession = { nodeId: string; @@ -28,7 +35,10 @@ export type NodeWorkerSupervisorNodeProof = { pairingGeneration: string; clientId: typeof GATEWAY_CLIENT_IDS.NODE_HOST; clientMode: "node"; - protocolFeature: typeof NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE; + protocolFeature: + | typeof NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE + | typeof NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE + | typeof NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE; workerHost: Extract; commands: readonly string[]; }; @@ -53,7 +63,25 @@ export function sameNodeWorkerHostDeclaration( left.capacity.available === right.capacity.available && left.bundlePrewarm === right.bundlePrewarm && left.bundleRetention === right.bundleRetention && - left.bundleStatus === right.bundleStatus)) + left.bundleStatus === right.bundleStatus && + left.processIsolation?.kind === right.processIsolation?.kind && + left.processIsolation?.memoryProjection === right.processIsolation?.memoryProjection)) + ); +} + +/** Returns true only for the execution boundary this exact inventory proof attests. */ +export function supportsNodeWorkerExecution( + node: NodeWorkerSupervisorNodeProof, + execution: NodeWorkerExecution, +): boolean { + if (execution.kind === NODE_WORKER_EXECUTION_HOST_V1) { + return true; + } + return ( + execution.kind === NODE_WORKER_EXECUTION_CONTAINER_V1 && + node.protocolFeature === NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE && + node.workerHost.processIsolation?.kind === NODE_WORKER_EXECUTION_CONTAINER_V1 && + node.workerHost.processIsolation.memoryProjection === 1 ); } @@ -72,7 +100,12 @@ export function resolveNodeWorkerSupervisorProof( declaration.pairingIdentity !== node.pairingIdentity || declaration.clientId !== node.clientId || declaration.clientMode !== node.clientMode || - !declaration.protocolFeatures.includes(NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE) || + declaration.protocolFeatures.length !== 1 || + (declaration.protocolFeatures[0] !== NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE && + declaration.protocolFeatures[0] !== + NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE && + declaration.protocolFeatures[0] !== + NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE) || declaration.workerHost?.enabled !== true ) { return undefined; @@ -84,10 +117,13 @@ export function resolveNodeWorkerSupervisorProof( pairingGeneration: node.pairingGeneration, clientId: GATEWAY_CLIENT_IDS.NODE_HOST, clientMode: "node", - protocolFeature: NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE, + protocolFeature: declaration.protocolFeatures[0], workerHost: { ...declaration.workerHost, capacity: { ...declaration.workerHost.capacity }, + ...(declaration.workerHost.processIsolation + ? { processIsolation: { ...declaration.workerHost.processIsolation } } + : {}), }, commands: [...node.commands], }; diff --git a/src/gateway/server-http.ts b/src/gateway/server-http.ts index 67003574c66f..e45982de27e6 100644 --- a/src/gateway/server-http.ts +++ b/src/gateway/server-http.ts @@ -37,6 +37,7 @@ import { classifyGatewayProbePath, classifyMcpAppStandalonePath, classifyNodeWorkerBundleTransferPath, + classifyNodeWorkerProjectionTransferPath, classifyNodeWorkspaceTransferPath, classifyWorkerGatewayPath, } from "./gateway-http-route-contracts.js"; @@ -74,6 +75,10 @@ import { handleNodeWorkerBundleTransferHttpRequest, type NodeWorkerBundleTransferHttpCallback, } from "./worker-environments/node-worker-bundle-transfer-http.js"; +import { + handleNodeWorkerProjectionTransferHttpRequest, + type NodeWorkerProjectionTransferHttpCallback, +} from "./worker-environments/node-worker-projection-transfer-http.js"; import { handleNodeWorkspaceTransferHttpRequest, type NodeWorkspaceTransferHttpCallback, @@ -184,6 +189,8 @@ export function createGatewayHttpServer(opts: { handleNodeWorkerBundleTransferRequest?: NodeWorkerBundleTransferHttpCallback; /** Authenticator/dispatcher for the reserved node workspace transfer namespace. */ handleNodeWorkspaceTransferRequest?: NodeWorkspaceTransferHttpCallback; + /** Authenticator/dispatcher for the reserved node memory-projection transfer namespace. */ + handleNodeWorkerProjectionTransferRequest?: NodeWorkerProjectionTransferHttpCallback; getReadiness?: ReadinessChecker; getStartup?: StartupChecker; getRuntimeConfig?: () => OpenClawConfig; @@ -373,6 +380,18 @@ export function createGatewayHttpServer(opts: { }), ); + addAdmittedStage( + classifyNodeWorkerProjectionTransferPath(scopedRequestPath) !== "outside", + () => + handleNodeWorkerProjectionTransferHttpRequest({ + req, + res, + clientIp: ingressAttribution.rateLimit.subject.key, + rateLimiter: joinRateLimiter, + callback: opts.handleNodeWorkerProjectionTransferRequest, + }), + ); + addAdmittedStage(classifyNodeWorkspaceTransferPath(scopedRequestPath) !== "outside", () => handleNodeWorkspaceTransferHttpRequest({ req, diff --git a/src/gateway/server-methods/doctor.test.ts b/src/gateway/server-methods/doctor.test.ts index dc63f779ae80..f96168ed2c45 100644 --- a/src/gateway/server-methods/doctor.test.ts +++ b/src/gateway/server-methods/doctor.test.ts @@ -29,6 +29,9 @@ const removeBackfillDiaryEntries = vi.hoisted(() => vi.fn()); const removeGroundedShortTermCandidates = vi.hoisted(() => vi.fn()); const repairDreamingArtifacts = vi.hoisted(() => vi.fn()); const loadShortTermPromotionDreamingStats = vi.hoisted(() => vi.fn()); +const withBrokeredMemoryMaintenance = vi.hoisted(() => + vi.fn(async (run: () => Promise) => await run()), +); vi.mock("../../config/config.js", () => ({ getRuntimeConfig, @@ -58,6 +61,10 @@ vi.mock("../../plugins/memory-runtime.js", () => ({ getActiveMemorySearchManagerCore: getMemorySearchManager, })); +vi.mock("../../plugins/memory-broker-runtime.js", () => ({ + withBrokeredMemoryMaintenance, +})); + import { createDoctorHandlers } from "./doctor.js"; const doctorHandlers = createDoctorHandlers({ @@ -1166,6 +1173,44 @@ describe("doctor.memory.status", () => { }); describe("doctor.memory dream actions", () => { + it("serializes every mutating Doctor path through broker maintenance", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "doctor-memory-maintenance-")); + getRuntimeConfig.mockReset().mockReturnValue({}); + resolveDefaultAgentId.mockReset().mockReturnValue("main"); + resolveAgentWorkspaceDir.mockReset().mockReturnValue(workspaceDir); + previewGroundedRemMarkdown.mockReset().mockResolvedValue({ scannedFiles: 0, files: [] }); + writeBackfillDiaryEntries.mockReset(); + removeBackfillDiaryEntries.mockReset().mockResolvedValue({ removed: 0 }); + removeGroundedShortTermCandidates.mockReset().mockResolvedValue({ removed: 0 }); + repairDreamingArtifacts.mockReset().mockResolvedValue({ + changed: false, + archivedDreamsDiary: false, + archivedSessionCorpus: false, + archivedSessionIngestion: false, + warnings: [], + }); + dedupeDreamDiaryEntries.mockReset().mockResolvedValue({ removed: 0, kept: 0 }); + withBrokeredMemoryMaintenance.mockClear(); + withBrokeredMemoryMaintenance.mockImplementation(async (run: () => Promise) => + await run(), + ); + + try { + for (const method of [ + "doctor.memory.backfillDreamDiary", + "doctor.memory.resetDreamDiary", + "doctor.memory.resetGroundedShortTerm", + "doctor.memory.repairDreamingArtifacts", + "doctor.memory.dedupeDreamDiary", + ] as const) { + await invokeDoctorMemory(method, vi.fn()); + } + expect(withBrokeredMemoryMaintenance).toHaveBeenCalledTimes(5); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + it("clears grounded-only staged short-term entries without touching the diary", async () => { resolveAgentWorkspaceDir.mockReturnValue("/tmp/openclaw"); removeGroundedShortTermCandidates.mockResolvedValue({ diff --git a/src/gateway/server-methods/doctor.ts b/src/gateway/server-methods/doctor.ts index 2761f449f592..c318fc88132f 100644 --- a/src/gateway/server-methods/doctor.ts +++ b/src/gateway/server-methods/doctor.ts @@ -25,6 +25,7 @@ import { resolveMemoryRemDreamingConfig, } from "../../memory-host-sdk/dreaming.js"; import * as defaultMemoryCoreRuntime from "../../plugin-sdk/memory-core-bundled-runtime.js"; +import { withBrokeredMemoryMaintenance } from "../../plugins/memory-broker-runtime.js"; import { getActiveMemorySearchManagerCore } from "../../plugins/memory-runtime.js"; import { normalizeAgentId } from "../../routing/session-key.js"; import { formatError } from "../server-utils.js"; @@ -850,58 +851,58 @@ export const createDoctorHandlers = ( return; } const { cfg, agentId, workspaceDir } = target; - const memoryDir = path.join(workspaceDir, "memory"); - const sourceFiles = await listWorkspaceDailyFiles(memoryDir); - if (sourceFiles.length === 0) { + const payload = await withBrokeredMemoryMaintenance(async () => { + const memoryDir = path.join(workspaceDir, "memory"); + const sourceFiles = await listWorkspaceDailyFiles(memoryDir); + if (sourceFiles.length === 0) { + const dreamDiary = await readDreamDiary(workspaceDir); + return { + agentId, + path: dreamDiary.path, + action: "backfill" as const, + found: dreamDiary.found, + scannedFiles: 0, + written: 0, + replaced: 0, + } satisfies DoctorMemoryDreamActionPayload; + } + const grounded = await memoryCoreRuntime.previewGroundedRemMarkdown({ + workspaceDir, + inputPaths: sourceFiles, + }); + const remConfig = resolveMemoryRemDreamingConfig({ + pluginConfig: resolveMemoryDreamingPluginConfig(cfg), + cfg, + }); + const entries = grounded.files + .map((file) => { + const isoDay = extractIsoDayFromPath(file.path); + if (!isoDay) { + return null; + } + return { + isoDay, + sourcePath: file.path, + bodyLines: groundedMarkdownToDiaryLines(file.renderedMarkdown), + }; + }) + .filter((entry): entry is NonNullable => entry !== null); + const written = await memoryCoreRuntime.writeBackfillDiaryEntries({ + workspaceDir, + entries, + timezone: remConfig.timezone, + }); const dreamDiary = await readDreamDiary(workspaceDir); - const payload: DoctorMemoryDreamActionPayload = { + return { agentId, path: dreamDiary.path, action: "backfill", found: dreamDiary.found, - scannedFiles: 0, - written: 0, - replaced: 0, - }; - respond(true, payload, undefined); - return; - } - const grounded = await memoryCoreRuntime.previewGroundedRemMarkdown({ - workspaceDir, - inputPaths: sourceFiles, + scannedFiles: grounded.scannedFiles, + written: written.written, + replaced: written.replaced, + } satisfies DoctorMemoryDreamActionPayload; }); - const remConfig = resolveMemoryRemDreamingConfig({ - pluginConfig: resolveMemoryDreamingPluginConfig(cfg), - cfg, - }); - const entries = grounded.files - .map((file) => { - const isoDay = extractIsoDayFromPath(file.path); - if (!isoDay) { - return null; - } - return { - isoDay, - sourcePath: file.path, - bodyLines: groundedMarkdownToDiaryLines(file.renderedMarkdown), - }; - }) - .filter((entry): entry is NonNullable => entry !== null); - const written = await memoryCoreRuntime.writeBackfillDiaryEntries({ - workspaceDir, - entries, - timezone: remConfig.timezone, - }); - const dreamDiary = await readDreamDiary(workspaceDir); - const payload: DoctorMemoryDreamActionPayload = { - agentId, - path: dreamDiary.path, - action: "backfill", - found: dreamDiary.found, - scannedFiles: grounded.scannedFiles, - written: written.written, - replaced: written.replaced, - }; respond(true, payload, undefined); }, "doctor.memory.resetDreamDiary": async ({ respond, context, params }) => { @@ -910,15 +911,17 @@ export const createDoctorHandlers = ( return; } const { agentId, workspaceDir } = target; - const removed = await memoryCoreRuntime.removeBackfillDiaryEntries({ workspaceDir }); - const dreamDiary = await readDreamDiary(workspaceDir); - const payload: DoctorMemoryDreamActionPayload = { - agentId, - path: dreamDiary.path, - action: "reset", - found: dreamDiary.found, - removedEntries: removed.removed, - }; + const payload = await withBrokeredMemoryMaintenance(async () => { + const removed = await memoryCoreRuntime.removeBackfillDiaryEntries({ workspaceDir }); + const dreamDiary = await readDreamDiary(workspaceDir); + return { + agentId, + path: dreamDiary.path, + action: "reset", + found: dreamDiary.found, + removedEntries: removed.removed, + } satisfies DoctorMemoryDreamActionPayload; + }); respond(true, payload, undefined); }, "doctor.memory.resetGroundedShortTerm": async ({ respond, context, params }) => { @@ -927,12 +930,14 @@ export const createDoctorHandlers = ( return; } const { agentId, workspaceDir } = target; - const removed = await memoryCoreRuntime.removeGroundedShortTermCandidates({ workspaceDir }); - const payload: DoctorMemoryDreamActionPayload = { - agentId, - action: "resetGroundedShortTerm", - removedShortTermEntries: removed.removed, - }; + const payload = await withBrokeredMemoryMaintenance(async () => { + const removed = await memoryCoreRuntime.removeGroundedShortTermCandidates({ workspaceDir }); + return { + agentId, + action: "resetGroundedShortTerm", + removedShortTermEntries: removed.removed, + } satisfies DoctorMemoryDreamActionPayload; + }); respond(true, payload, undefined); }, "doctor.memory.repairDreamingArtifacts": async ({ respond, context, params }) => { @@ -941,17 +946,19 @@ export const createDoctorHandlers = ( return; } const { agentId, workspaceDir } = target; - const repair = await memoryCoreRuntime.repairDreamingArtifacts({ workspaceDir }); - const payload: DoctorMemoryDreamActionPayload = { - agentId, - action: "repairDreamingArtifacts", - changed: repair.changed, - archiveDir: repair.archiveDir, - archivedDreamsDiary: repair.archivedDreamsDiary, - archivedSessionCorpus: repair.archivedSessionCorpus, - archivedSessionIngestion: repair.archivedSessionIngestion, - warnings: repair.warnings, - }; + const payload = await withBrokeredMemoryMaintenance(async () => { + const repair = await memoryCoreRuntime.repairDreamingArtifacts({ workspaceDir }); + return { + agentId, + action: "repairDreamingArtifacts", + changed: repair.changed, + archiveDir: repair.archiveDir, + archivedDreamsDiary: repair.archivedDreamsDiary, + archivedSessionCorpus: repair.archivedSessionCorpus, + archivedSessionIngestion: repair.archivedSessionIngestion, + warnings: repair.warnings, + } satisfies DoctorMemoryDreamActionPayload; + }); respond(true, payload, undefined); }, "doctor.memory.dedupeDreamDiary": async ({ respond, context, params }) => { @@ -960,17 +967,19 @@ export const createDoctorHandlers = ( return; } const { agentId, workspaceDir } = target; - const dedupe = await memoryCoreRuntime.dedupeDreamDiaryEntries({ workspaceDir }); - const dreamDiary = await readDreamDiary(workspaceDir); - const payload: DoctorMemoryDreamActionPayload = { - agentId, - action: "dedupeDreamDiary", - path: dreamDiary.path, - found: dreamDiary.found, - removedEntries: dedupe.removed, - dedupedEntries: dedupe.removed, - keptEntries: dedupe.kept, - }; + const payload = await withBrokeredMemoryMaintenance(async () => { + const dedupe = await memoryCoreRuntime.dedupeDreamDiaryEntries({ workspaceDir }); + const dreamDiary = await readDreamDiary(workspaceDir); + return { + agentId, + action: "dedupeDreamDiary", + path: dreamDiary.path, + found: dreamDiary.found, + removedEntries: dedupe.removed, + dedupedEntries: dedupe.removed, + keptEntries: dedupe.kept, + } satisfies DoctorMemoryDreamActionPayload; + }); respond(true, payload, undefined); }, }); diff --git a/src/gateway/server-runtime-state-prepare.ts b/src/gateway/server-runtime-state-prepare.ts index e4e1f88a1ec0..f64efdb54b95 100644 --- a/src/gateway/server-runtime-state-prepare.ts +++ b/src/gateway/server-runtime-state-prepare.ts @@ -166,6 +166,7 @@ export async function prepareGatewayKernelState(params: { bindDeviceNodeControl, bindNodeWorkspaceBindingResolver, handleNodeWorkerBundleTransferRequest, + handleNodeWorkerProjectionTransferRequest, handleNodeWorkspaceTransferRequest, } = workerEnvironmentRuntime; // Assigned once approval managers exist; placement dispatch must not run before then. @@ -461,6 +462,7 @@ export async function prepareGatewayKernelState(params: { handleWatchNodeRequest: async (req: IncomingMessage, res: ServerResponse) => (await watchNodeRequestHandler.current?.(req, res)) ?? false, handleNodeWorkerBundleTransferRequest, + handleNodeWorkerProjectionTransferRequest, handleNodeWorkspaceTransferRequest, workerIngressEnabled: Boolean(workerEnvironmentService), desktopSessionRegistry, diff --git a/src/gateway/server-startup-post-attach.test.ts b/src/gateway/server-startup-post-attach.test.ts index d36f14d95fc3..61aa263f0c00 100644 --- a/src/gateway/server-startup-post-attach.test.ts +++ b/src/gateway/server-startup-post-attach.test.ts @@ -850,7 +850,9 @@ describe("startGatewayPostAttachRuntime", () => { }, }; const startGatewaySidecars = vi.fn(async () => { - expect(hoisted.startBrokeredMemoryRuntimeSupervisor).toHaveBeenCalledWith(selectedCapability); + expect(hoisted.startBrokeredMemoryRuntimeSupervisor).toHaveBeenCalledWith(selectedCapability, { + agentIds: ["main"], + }); expect(unavailableGatewayMethods).toEqual(new Set(STARTUP_UNAVAILABLE_GATEWAY_METHODS)); return { pluginServices: null, postReadySidecars: [] }; }); diff --git a/src/gateway/server-startup-post-attach.ts b/src/gateway/server-startup-post-attach.ts index 386d058620a8..51a4b6973038 100644 --- a/src/gateway/server-startup-post-attach.ts +++ b/src/gateway/server-startup-post-attach.ts @@ -1345,9 +1345,14 @@ export async function startGatewayPostAttachRuntime( // than becoming a first-request outage after agents have begun work. const memoryBrokerSupervisor = selectedMemory?.capability.broker ? await measureStartup(params.startupTrace, "memory-broker.ready", async () => { - const { startBrokeredMemoryRuntimeSupervisor } = - await loadMemoryBrokerRuntimeModule(); - return await startBrokeredMemoryRuntimeSupervisor(selectedMemory.capability); + const [{ startBrokeredMemoryRuntimeSupervisor }, { listAgentIds }] = + await Promise.all([ + loadMemoryBrokerRuntimeModule(), + import("../agents/agent-scope.js"), + ]); + return await startBrokeredMemoryRuntimeSupervisor(selectedMemory.capability, { + agentIds: listAgentIds(params.gatewayPluginConfigAtStart), + }); }) : undefined; if (params.isClosing?.()) { diff --git a/src/gateway/server-worker-environment-startup.test.ts b/src/gateway/server-worker-environment-startup.test.ts index 74faef038728..2f60189ba766 100644 --- a/src/gateway/server-worker-environment-startup.test.ts +++ b/src/gateway/server-worker-environment-startup.test.ts @@ -1,10 +1,20 @@ import fs from "node:fs/promises"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + GATEWAY_CLIENT_IDS, + GATEWAY_CLIENT_MODES, +} from "../../packages/gateway-protocol/src/client-info.js"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE } from "../infra/node-runner-inventory.js"; import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { withEnvAsync } from "../test-utils/env.js"; +import { NODE_WORKER_EXECUTION_CONTAINER_V1 } from "../worker/node-supervisor-protocol.js"; import { createDesktopSessionRegistry } from "./desktop/session-registry.js"; +import type { + NodeWorkerSupervisorNodeProof, + NodeWorkerSupervisorTransport, +} from "./node-registry-private.js"; import { createGatewayWorkerEnvironmentRuntime, loadGatewayWorkerEnvironmentStartupState, @@ -15,6 +25,25 @@ import { reconcileDeviceWorker, } from "./worker-environments/device-provider.js"; +const projectionTransferOptions = vi.hoisted(() => ({ + isNodeCurrent: undefined as ((node: NodeWorkerSupervisorNodeProof) => boolean) | undefined, +})); + +vi.mock("./worker-environments/node-worker-projection-transfer-service.js", async (importOriginal) => { + const actual = await importOriginal< + typeof import("./worker-environments/node-worker-projection-transfer-service.js") + >(); + return { + ...actual, + createNodeWorkerProjectionTransferService: ( + options: Parameters[0], + ) => { + projectionTransferOptions.isNodeCurrent = options.isNodeCurrent; + return actual.createNodeWorkerProjectionTransferService(options); + }, + }; +}); + const DEVICE_ID = "revoked-device"; const tempDirs = useAutoCleanupTempDirTracker(afterEach); @@ -23,6 +52,69 @@ afterEach(() => { }); describe("gateway worker environment startup", () => { + it("keeps an issued projection bound to its capacity-one node after its slot is claimed", async () => { + const stateDir = tempDirs.make("openclaw-worker-projection-capacity-"); + await withEnvAsync({ OPENCLAW_STATE_DIR: stateDir }, async () => { + const startup = await loadGatewayWorkerEnvironmentStartupState(); + const runtime = await createGatewayWorkerEnvironmentRuntime({ + getPluginRegistry: () => ({ workerProviders: new Map() }), + desktopSessionRegistry: createDesktopSessionRegistry({ lingerMs: 1 }), + startup, + log: { child: () => ({ warn: () => {} }) }, + }); + const service = runtime.workerEnvironmentService; + if (!service || !runtime.bindDeviceNodeControl || !projectionTransferOptions.isNodeCurrent) { + throw new Error("worker projection runtime was not created"); + } + const node: NodeWorkerSupervisorNodeProof = { + nodeId: "node-1", + connId: "conn-1", + pairingIdentity: "pairing-1", + pairingGeneration: "generation-1", + clientId: GATEWAY_CLIENT_IDS.NODE_HOST, + clientMode: GATEWAY_CLIENT_MODES.NODE, + protocolFeature: NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE, + workerHost: { + enabled: true, + capacity: { total: 1, available: 1 }, + processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, memoryProjection: 1 }, + }, + commands: [], + }; + let available = 1; + let connId = node.connId; + let pairingGeneration = node.pairingGeneration; + const transport = { + listCurrentNodes: async () => [node], + isCurrent: vi.fn( + (candidate: NodeWorkerSupervisorNodeProof, requireLaunchEligibility = false) => + candidate.nodeId === node.nodeId && + candidate.connId === connId && + candidate.pairingGeneration === pairingGeneration && + (!requireLaunchEligibility || available > 0), + ), + invoke: async () => ({ ok: false }), + } satisfies NodeWorkerSupervisorTransport; + runtime.bindDeviceNodeControl(transport); + const isNodeCurrent = projectionTransferOptions.isNodeCurrent; + + try { + expect(isNodeCurrent(node)).toBe(true); + available = 0; + expect(isNodeCurrent(node)).toBe(true); + expect(transport.isCurrent).toHaveBeenLastCalledWith(node, false); + + connId = "conn-replaced"; + expect(isNodeCurrent(node)).toBe(false); + connId = node.connId; + pairingGeneration = "generation-replaced"; + expect(isNodeCurrent(node)).toBe(false); + } finally { + await service.stop(); + } + }); + }); + it("cleans transfer scratch before serving and removes it on shutdown", async () => { const stateDir = tempDirs.make("openclaw-worker-transfer-startup-"); const transferRoot = path.join(stateDir, "tmp", "node-workspace-transfer"); diff --git a/src/gateway/server-worker-environment-startup.ts b/src/gateway/server-worker-environment-startup.ts index efe6a0667391..ef8426243881 100644 --- a/src/gateway/server-worker-environment-startup.ts +++ b/src/gateway/server-worker-environment-startup.ts @@ -2,7 +2,7 @@ import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { uniqueStrings } from "@openclaw/normalization-core/string-normalization"; import { getRuntimeConfig } from "../config/config.js"; import { loadOrCreateProcessDeviceIdentity } from "../infra/device-identity.js"; -import { getPairedDevice } from "../infra/device-pairing.js"; +import { getPairedDevice, resolveNodePairingState } from "../infra/device-pairing.js"; import type { PluginRegistry } from "../plugins/registry-types.js"; import { getActiveSecretsRuntimeConfigSnapshot, @@ -14,6 +14,7 @@ import type { NodeWorkerSupervisorTransport } from "./node-registry-private.js"; import type { WorkerBundleProducer, WorkerNpmArtifact } from "./worker-environments/bundle.js"; import { bindDeviceWorkerAvailability, + bindDeviceWorkerExecutionEligibility, bindDeviceWorkerReconciliation, createDeviceWorkerRuntime, DEVICE_WORKER_PROVIDER_ID, @@ -21,6 +22,7 @@ import { import type { WorkerLiveEventReceiver } from "./worker-environments/live-events.js"; import { createWorkerNodeEnrollmentManager } from "./worker-environments/node-enrollment.js"; import type { NodeWorkerBundleTransferHttpCallback } from "./worker-environments/node-worker-bundle-transfer-http.js"; +import type { NodeWorkerProjectionTransferHttpCallback } from "./worker-environments/node-worker-projection-transfer-http.js"; import { nodeWorkerGatewayNamespace as resolveNodeWorkerGatewayNamespace } from "./worker-environments/node-worker-gateway-namespace.js"; import type { NodeWorkerWorkspaceBindingResolver } from "./worker-environments/node-worker-tunnel.js"; import type { NodeWorkspaceTransferHttpCallback } from "./worker-environments/node-workspace-transfer-http-contract.js"; @@ -56,6 +58,7 @@ export type GatewayWorkerEnvironmentRuntime = { bindDeviceNodeControl?: (transport: NodeWorkerSupervisorTransport) => void; bindNodeWorkspaceBindingResolver?: (resolver: NodeWorkerWorkspaceBindingResolver) => void; handleNodeWorkerBundleTransferRequest?: NodeWorkerBundleTransferHttpCallback; + handleNodeWorkerProjectionTransferRequest?: NodeWorkerProjectionTransferHttpCallback; handleNodeWorkspaceTransferRequest?: NodeWorkspaceTransferHttpCallback; }; @@ -119,6 +122,8 @@ export async function createGatewayWorkerEnvironmentRuntime(params: { { createGatewayNodeWorkerBundleInstaller }, { createNodeWorkerBundleTransferService }, { createNodeWorkerBundleTransferHttpCallback }, + { createNodeWorkerProjectionTransferService }, + { createNodeWorkerProjectionTransferHttpCallback }, { createNodeWorkspaceTransferService }, { createNodeWorkspaceTransferHttpCallback }, { createWorkerSessionToolExecutor }, @@ -133,6 +138,8 @@ export async function createGatewayWorkerEnvironmentRuntime(params: { import("./worker-environments/node-worker-bundle-installer.js"), import("./worker-environments/node-worker-bundle-transfer-service.js"), import("./worker-environments/node-worker-bundle-transfer-http.js"), + import("./worker-environments/node-worker-projection-transfer-service.js"), + import("./worker-environments/node-worker-projection-transfer-http.js"), import("./worker-environments/node-workspace-transfer-service.js"), import("./worker-environments/node-workspace-transfer-http.js"), import("./worker-environments/worker-session-tool-executor.js"), @@ -204,6 +211,20 @@ export async function createGatewayWorkerEnvironmentRuntime(params: { desktopSessionRegistry: params.desktopSessionRegistry, }); const nodeWorkerBundleTransfer = createNodeWorkerBundleTransferService(); + let nodeWorkerSupervisorTransport: NodeWorkerSupervisorTransport | undefined; + const nodeWorkerProjectionTransfer = createNodeWorkerProjectionTransferService({ + resolveNodePublicKey: async (node) => { + const paired = await getPairedDevice(node.nodeId); + const currentPairing = resolveNodePairingState(paired); + return currentPairing?.identity.key === node.pairingIdentity && + currentPairing.generation?.key === node.pairingGeneration + ? paired?.publicKey + : undefined; + }, + // The node claims its launch slot before it fetches this already-issued projection. + // Keep exact connection/pairing proof, but do not reject a capacity-one node's own fetch. + isNodeCurrent: (node) => nodeWorkerSupervisorTransport?.isCurrent(node) === true, + }); const nodeWorkspaceTransfer = createNodeWorkspaceTransferService({ getOwner: (environmentId) => params.startup.store.getTransferOwner(environmentId), }); @@ -216,6 +237,7 @@ export async function createGatewayWorkerEnvironmentRuntime(params: { getTransport: () => deviceRuntime.getNodeTransport(), launchNodeWorker: async (request) => await deviceRuntime.launchNodeWorker(request), validateWorkerTurn: (binding) => placementGate.validateWorkerTurn(binding), + projectionTransfer: nodeWorkerProjectionTransfer, workspaceTransfer: nodeWorkspaceTransfer, }); const ensureNodeWorkerBundle = createGatewayNodeWorkerBundleInstaller({ @@ -255,7 +277,10 @@ export async function createGatewayWorkerEnvironmentRuntime(params: { retireNodeEnrollment: nodeEnrollment.retire, tunnelManager: workerTunnelManager, nodeTunnelManager: nodeWorkerTunnelManager, - stopNodeWorkerBundleTransfers: () => nodeWorkerBundleTransfer.closeAll(), + stopNodeWorkerBundleTransfers: () => { + nodeWorkerBundleTransfer.closeAll(); + nodeWorkerProjectionTransfer.closeAll(); + }, applyTranscriptCommit: createWorkerTranscriptCommitter({ getConfig: getRuntimeConfig, }).commit, @@ -304,6 +329,10 @@ export async function createGatewayWorkerEnvironmentRuntime(params: { }); const workerEnvironmentService = workerEnvironmentServiceBase; bindDeviceWorkerAvailability(workerEnvironmentService, deviceRuntime.resolveAvailability); + bindDeviceWorkerExecutionEligibility( + workerEnvironmentService, + deviceRuntime.assertNodeWorkerExecutionEligible, + ); bindDeviceWorkerReconciliation(workerEnvironmentService, async (deviceId) => { const environmentIds = params.startup.store .listForReconcile() @@ -344,11 +373,16 @@ export async function createGatewayWorkerEnvironmentRuntime(params: { bindWorkerSessionDispatch: (dispatch) => { dispatchChild = dispatch; }, - bindDeviceNodeControl: deviceRuntime.bindNodeTransport, + bindDeviceNodeControl: (transport) => { + nodeWorkerSupervisorTransport = transport; + deviceRuntime.bindNodeTransport(transport); + }, bindNodeWorkspaceBindingResolver: (resolver) => nodeWorkerTunnelManager.bindWorkspaceBindingResolver(resolver), handleNodeWorkerBundleTransferRequest: createNodeWorkerBundleTransferHttpCallback(nodeWorkerBundleTransfer), + handleNodeWorkerProjectionTransferRequest: + createNodeWorkerProjectionTransferHttpCallback(nodeWorkerProjectionTransfer), handleNodeWorkspaceTransferRequest: createNodeWorkspaceTransferHttpCallback(nodeWorkspaceTransfer), }; diff --git a/src/gateway/server/ws-connection/worker-connection.ts b/src/gateway/server/ws-connection/worker-connection.ts index 8cb4161cb8df..8b1685c752b3 100644 --- a/src/gateway/server/ws-connection/worker-connection.ts +++ b/src/gateway/server/ws-connection/worker-connection.ts @@ -115,10 +115,12 @@ type WorkerInferenceConnectionService = WorkerConnectionService & { searchMemory?: ( identity: WorkerConnectionIdentity, request: WorkerMemorySearchParams, + signal?: AbortSignal, ) => Promise>; readMemory?: ( identity: WorkerConnectionIdentity, request: WorkerMemoryReadParams, + signal?: AbortSignal, ) => Promise>; startInference?: ( identity: WorkerConnectionIdentity, @@ -255,7 +257,7 @@ async function dispatchWorkerRequest(params: { rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); return; } - const outcome = await service.searchMemory(params.identity, params.request.params); + const outcome = await service.searchMemory(params.identity, params.request.params, params.signal); if (outcome.ok) { params.respond(true, outcome.result); return; @@ -284,7 +286,7 @@ async function dispatchWorkerRequest(params: { rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); return; } - const outcome = await service.readMemory(params.identity, params.request.params); + const outcome = await service.readMemory(params.identity, params.request.params, params.signal); if (outcome.ok) { params.respond(true, outcome.result); return; @@ -425,6 +427,7 @@ export function attachWorkerWsMessageHandler(params: WorkerWsMessageHandlerParam let expiryTimer: ReturnType | undefined; let disposed = false; const sessionOperations = new Set(); + const memoryOperations = new Map(); const cleanup = () => { if (disposed) { return; @@ -432,6 +435,10 @@ export function attachWorkerWsMessageHandler(params: WorkerWsMessageHandlerParam disposed = true; clearTimeout(expiryTimer); sessionOperations.clear(); + for (const controller of memoryOperations.values()) { + controller.abort(new Error("worker memory RPC connection closed")); + } + memoryOperations.clear(); params.socket.off("message", onMessage); }; const closeWorker = (code: number, reason: WorkerProtocolCloseReason) => { @@ -695,6 +702,24 @@ export function attachWorkerWsMessageHandler(params: WorkerWsMessageHandlerParam }); return; } + const isMemoryOperation = + parsed.method === WORKER_MEMORY_METHODS[0] || parsed.method === WORKER_MEMORY_METHODS[1]; + if (isMemoryOperation) { + if (memoryOperations.has(parsed.id)) { + failFrame(1008, "invalid-frame"); + return; + } + const controller = new AbortController(); + memoryOperations.set(parsed.id, controller); + try { + await dispatch(controller.signal); + } finally { + if (memoryOperations.get(parsed.id) === controller) { + memoryOperations.delete(parsed.id); + } + } + return; + } await dispatch(); }; diff --git a/src/gateway/worker-environments/device-provider.ts b/src/gateway/worker-environments/device-provider.ts index bd6fbb0a30e4..a74ea04852b0 100644 --- a/src/gateway/worker-environments/device-provider.ts +++ b/src/gateway/worker-environments/device-provider.ts @@ -10,6 +10,7 @@ import { type WorkerProfile, type WorkerProvider, } from "../../plugins/types.js"; +import type { NodeWorkerExecution } from "../../worker/node-supervisor-protocol.js"; import type { NodeWorkerSupervisorNodeProof, NodeWorkerSupervisorTransport, @@ -31,8 +32,14 @@ export type DeviceWorkerAvailability = { }; type DeviceWorkerAvailabilityResolver = (deviceId: string) => Promise; type DeviceWorkerReconciliation = (deviceId: string) => Promise; +type DeviceWorkerExecutionEligibility = (params: { + deviceId: string; + execution: NodeWorkerExecution; + signal?: AbortSignal; +}) => Promise; const DEVICE_WORKER_AVAILABILITY = new WeakMap(); const DEVICE_WORKER_RECONCILIATION = new WeakMap(); +const DEVICE_WORKER_EXECUTION_ELIGIBILITY = new WeakMap(); export function bindDeviceWorkerAvailability( service: object, @@ -80,6 +87,36 @@ export async function reconcileDeviceWorker( return reconcile ? await reconcile(deviceId) : []; } +export function bindDeviceWorkerExecutionEligibility( + service: object, + assertEligible: DeviceWorkerExecutionEligibility, +): void { + DEVICE_WORKER_EXECUTION_ELIGIBILITY.set(service, assertEligible); +} + +/** + * Enforced launches must prove the node boundary before any turn credential, + * tunnel, or workspace operation is created. + */ +export async function assertDeviceWorkerExecutionEligibility(params: { + service: object | undefined; + deviceId: string; + execution: NodeWorkerExecution; + signal?: AbortSignal; +}): Promise { + const assertEligible = params.service + ? DEVICE_WORKER_EXECUTION_ELIGIBILITY.get(params.service) + : undefined; + if (!assertEligible) { + throw new WorkerProviderError("device worker process-isolation eligibility is unavailable"); + } + await assertEligible({ + deviceId: params.deviceId, + execution: params.execution, + ...(params.signal ? { signal: params.signal } : {}), + }); +} + function requireDeviceId(profile: WorkerProfile): string { const deviceId = profile.device; if (typeof deviceId !== "string" || !deviceId.trim()) { @@ -170,6 +207,7 @@ export function createDeviceWorkerRuntime(options: DeviceWorkerRuntimeOptions) { return { provider, resolveAvailability, + assertNodeWorkerExecutionEligible: launchAdapter.assertExecutionEligible, launchNodeWorker: launchAdapter.launch, getNodeTransport: () => nodeTransport, bindNodeTransport: (transport: NodeWorkerSupervisorTransport) => { diff --git a/src/gateway/worker-environments/node-launch-adapter.test.ts b/src/gateway/worker-environments/node-launch-adapter.test.ts index 918686678d78..0271efb80458 100644 --- a/src/gateway/worker-environments/node-launch-adapter.test.ts +++ b/src/gateway/worker-environments/node-launch-adapter.test.ts @@ -8,9 +8,17 @@ import { WORKER_RPC_SET_VERSION, } from "../../../packages/gateway-protocol/src/schema/worker-admission.js"; import { NODE_WORKER_CAPACITY_EXHAUSTED_ERROR_CODE } from "../../infra/node-commands.js"; -import { NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE } from "../../infra/node-runner-inventory.js"; import { + NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE, + NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE, + NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE, +} from "../../infra/node-runner-inventory.js"; +import { + NODE_WORKER_EXECUTION_CONTAINER_V1, + NODE_WORKER_EXECUTION_HOST_V1, + nodeWorkerMemoryProjectionLaunchBinding, nodeWorkerPlanHash, + type NodeWorkerExecution, type NodeWorkerLaunchInput, type NodeWorkerSupervisorReceipt, } from "../../worker/node-supervisor-protocol.js"; @@ -27,7 +35,12 @@ const WORKER_RUNS = { protocolFeatures: [...WORKER_PROTOCOL_FEATURES], }; -function nodeProof(connId = "conn-1", available = 2): NodeWorkerSupervisorNodeProof { +function nodeProof( + connId = "conn-1", + available = 2, + execution: NodeWorkerExecution = { kind: NODE_WORKER_EXECUTION_HOST_V1 }, +): NodeWorkerSupervisorNodeProof { + const container = execution.kind === NODE_WORKER_EXECUTION_CONTAINER_V1; return { nodeId: DEVICE_ID, connId, @@ -35,8 +48,16 @@ function nodeProof(connId = "conn-1", available = 2): NodeWorkerSupervisorNodePr pairingGeneration: "generation-1", clientId: GATEWAY_CLIENT_IDS.NODE_HOST, clientMode: GATEWAY_CLIENT_MODES.NODE, - protocolFeature: NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE, - workerHost: { enabled: true, capacity: { total: 2, available } }, + protocolFeature: container + ? NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE + : NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE, + workerHost: { + enabled: true, + capacity: { total: 2, available }, + ...(container + ? { processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, memoryProjection: 1 } } + : {}), + }, commands: ["system.run"], }; } @@ -47,6 +68,7 @@ function launchInput(): NodeWorkerLaunchInput { gatewayNamespace: "gateway-1", expectedBundleHash: WORKER_RUNS.bundleHash, placementGeneration: 4, + execution: { kind: NODE_WORKER_EXECUTION_HOST_V1 }, descriptor: { version: 4, admission: { @@ -59,6 +81,7 @@ function launchInput(): NodeWorkerLaunchInput { }, assignment: { agentId: "agent-1", + memoryReadEnforced: false, operationalRunInstance: { instanceId: "instance-1", runId: "run-1" }, agentRuntimeIdentityToken: "signed-runtime-token", runId: "run-1", @@ -80,6 +103,7 @@ function launchInput(): NodeWorkerLaunchInput { function receipt( input: NodeWorkerLaunchInput, state: NodeWorkerSupervisorReceipt["state"], + executionStarted = true, ): NodeWorkerSupervisorReceipt { const identity = { launchId: input.launchId, @@ -102,7 +126,7 @@ function receipt( }; } if (state === "failed" || state === "interrupted" || state === "cancelled") { - return { ...identity, state, errorText: `worker ${state}` }; + return { ...identity, state, errorText: `worker ${state}`, executionStarted }; } return { ...identity, state }; } @@ -128,6 +152,18 @@ function launchRequest(input = launchInput()) { }; } +function issuedProjection(input: Omit, reference: string) { + return { + version: 1 as const, + reference, + binding: { + launch: nodeWorkerMemoryProjectionLaunchBinding(input), + authorization: "b".repeat(64), + }, + expiresAtMs: Date.now() + 60_000, + }; +} + describe("node worker launch adapter", () => { it("fails with a typed availability result when no node dispatches within the grace", async () => { vi.useFakeTimers(); @@ -172,6 +208,70 @@ describe("node worker launch adapter", () => { ]); }); + it("reports execution readiness only after the durable worker receipt", async () => { + const input = launchInput(); + const callbacks: string[] = []; + let statusCalls = 0; + const invoke = vi.fn(async (request) => { + if (request.command === "worker.launch.v1") { + request.onDispatchReady?.("invoke-1"); + return wire(receipt(input, "pending")); + } + statusCalls += 1; + return wire(receipt(input, statusCalls === 1 ? "running" : "completed")); + }); + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => transportWith(invoke), + sleep: async () => {}, + }); + + const launched = adapter.launch({ + ...launchRequest(input), + onDispatchReady: () => callbacks.push("dispatch"), + onExecutionReady: () => callbacks.push("execution"), + }); + + await expect(launched).resolves.toEqual(receipt(input, "completed")); + expect(callbacks).toEqual(["dispatch", "execution"]); + }); + + it("does not report execution readiness when a launch terminates before child acknowledgement", async () => { + const input = launchInput(); + const callbacks: string[] = []; + const invoke = vi.fn(async (request) => { + request.onDispatchReady?.("invoke-1"); + return wire(receipt(input, "cancelled", false)); + }); + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => transportWith(invoke), + }); + + await expect( + adapter.launch({ + ...launchRequest(input), + onDispatchReady: () => callbacks.push("dispatch"), + onExecutionReady: () => callbacks.push("execution"), + }), + ).resolves.toEqual(receipt(input, "cancelled", false)); + expect(callbacks).toEqual(["dispatch"]); + }); + + it("rejects a container launch on a v5 node before invoking the supervisor", async () => { + const input = { + ...launchInput(), + execution: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 } as const, + }; + const invoke = vi.fn(); + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => transportWith(invoke), + }); + + await expect(adapter.launch(launchRequest(input))).rejects.toMatchObject({ + code: "PROCESS_ISOLATION_UNAVAILABLE", + }); + expect(invoke).not.toHaveBeenCalled(); + }); + it("reacquires the node and replays the identical launch after ambiguous disconnect", async () => { const input = launchInput(); let launchCalls = 0; @@ -272,6 +372,101 @@ describe("node worker launch adapter", () => { ]); }); + it("keeps status and cancellation bound to the prepared container connection after inventory changes", async () => { + const base = { + ...launchInput(), + execution: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 } as const, + descriptor: { + ...launchInput().descriptor, + assignment: { + ...launchInput().descriptor.assignment, + memoryReadEnforced: true, + workspaceDir: "/workspace", + }, + }, + }; + const input = { ...base, memoryProjection: issuedProjection(base, "a".repeat(43)) }; + const controller = new AbortController(); + let discoveryCount = 0; + let sleepCount = 0; + const invoke = vi.fn(async (request) => { + if (request.command === "worker.launch.v1") { + return wire(receipt(input, "running")); + } + if (request.command === "worker.status.v1") { + return wire(receipt(input, "running")); + } + return wire(receipt(input, "cancelled")); + }); + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => + transportWith(invoke, async () => [ + discoveryCount++ <= 1 + ? nodeProof("conn-container", 2, input.execution) + : nodeProof("conn-container"), + ]), + sleep: async () => { + if (++sleepCount === 2) { + controller.abort(); + } + }, + }); + + await expect( + adapter.launch({ + ...launchRequest(input), + prepareMemoryProjection: async () => input.memoryProjection, + signal: controller.signal, + }), + ).resolves.toEqual(receipt(input, "cancelled")); + expect(invoke.mock.calls.map(([request]) => request.command)).toEqual([ + "worker.launch.v1", + "worker.status.v1", + "worker.cancel.v1", + ]); + expect(invoke.mock.calls.slice(1).map(([request]) => request.node.connId)).toEqual([ + "conn-container", + "conn-container", + ]); + }); + + it("does not replay, poll, or cancel an issued projection through a replacement node connection", async () => { + const base = { + ...launchInput(), + execution: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 } as const, + descriptor: { + ...launchInput().descriptor, + assignment: { + ...launchInput().descriptor.assignment, + memoryReadEnforced: true, + workspaceDir: "/workspace", + }, + }, + }; + const input = { ...base, memoryProjection: issuedProjection(base, "b".repeat(43)) }; + const invoke = vi.fn(async () => + wire(receipt(input, "running")), + ); + let discoveryCount = 0; + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => + transportWith(invoke, async () => [ + discoveryCount++ <= 1 + ? nodeProof("conn-issued", 2, input.execution) + : nodeProof("conn-replacement"), + ]), + sleep: async () => {}, + }); + + await expect( + adapter.launch({ + ...launchRequest(input), + prepareMemoryProjection: async () => input.memoryProjection, + }), + ).rejects.toThrow("cancellation could not be confirmed"); + expect(invoke.mock.calls.map(([request]) => request.command)).toEqual(["worker.launch.v1"]); + }); + it("replays launch when status cannot find the durable receipt", async () => { const input = launchInput(); const responses = [ diff --git a/src/gateway/worker-environments/node-launch-adapter.ts b/src/gateway/worker-environments/node-launch-adapter.ts index 3c77d76275ec..22f7f090d0d2 100644 --- a/src/gateway/worker-environments/node-launch-adapter.ts +++ b/src/gateway/worker-environments/node-launch-adapter.ts @@ -9,10 +9,13 @@ import { nodeWorkerPlanHash, parseNodeWorkerLaunchInput, parseNodeWorkerSupervisorReceipt, + type NodeWorkerExecution, type NodeWorkerLaunchInput, type NodeWorkerSupervisorIdentity, type NodeWorkerSupervisorReceipt, } from "../../worker/node-supervisor-protocol.js"; +import type { NodeWorkerMemoryProjection } from "../../worker/node-memory-projection-protocol.js"; +import { supportsNodeWorkerExecution } from "../node-runner-inventory-runtime.js"; import type { NodeWorkerSupervisorNodeProof, NodeWorkerSupervisorTransport, @@ -43,11 +46,20 @@ type TerminalNodeWorkerSupervisorReceipt = Extract< type DeviceWorkerLaunchRequest = { deviceId: string; input: NodeWorkerLaunchInput; + /** Mints the opaque projection only after this adapter has proved the exact node connection. */ + prepareMemoryProjection?: ( + node: NodeWorkerSupervisorNodeProof, + ) => Promise; isDispatchAuthorized: () => boolean; isCancellationAuthorized: () => boolean; timeoutMs: number; signal?: AbortSignal; onDispatchReady?: () => void; + onExecutionReady?: () => void; +}; + +type PreparedDeviceWorkerLaunchRequest = DeviceWorkerLaunchRequest & { + boundNode?: NodeWorkerSupervisorNodeProof; }; type NodeWorkerLaunchAdapterOptions = { @@ -126,6 +138,18 @@ function receiptMatchesIdentity( ); } +function sameNodeConnection( + left: NodeWorkerSupervisorNodeProof, + right: NodeWorkerSupervisorNodeProof, +): boolean { + return ( + left.nodeId === right.nodeId && + left.connId === right.connId && + left.pairingIdentity === right.pairingIdentity && + left.pairingGeneration === right.pairingGeneration + ); +} + function parseInvokeReceipt( payloadJSON: string | null | undefined, ): NodeWorkerSupervisorReceipt | null { @@ -209,7 +233,8 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp const findNode = async (params: { transport: NodeWorkerSupervisorTransport; deviceId: string; - requireLaunchAvailability?: boolean; + execution: NodeWorkerExecution; + requireLaunchEligibility: boolean; signal: AbortSignal; }): Promise => { let nodes: readonly NodeWorkerSupervisorNodeProof[]; @@ -224,17 +249,22 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp "device worker node discovery is unavailable", ); } - const node = nodes.find( - (candidate) => - candidate.nodeId === params.deviceId && - (!params.requireLaunchAvailability || candidate.workerHost.capacity.available > 0), - ); + const node = nodes.find((candidate) => candidate.nodeId === params.deviceId); if (!node) { throw new NodeWorkerLaunchTransportError( "NOT_CONNECTED", "device worker node is not currently connected", ); } + if (params.requireLaunchEligibility && !supportsNodeWorkerExecution(node, params.execution)) { + throw new NodeWorkerLaunchTransportError( + "PROCESS_ISOLATION_UNAVAILABLE", + "device worker node does not attest to the required process-isolation boundary", + ); + } + if (params.requireLaunchEligibility && node.workerHost.capacity.available <= 0) { + throw new WorkerRunnerCapacityError(); + } return node; }; @@ -245,9 +275,10 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp | typeof NODE_WORKER_SUPERVISOR_STATUS_COMMAND | typeof NODE_WORKER_SUPERVISOR_CANCEL_COMMAND; payload: unknown; - requireLaunchAvailability?: boolean; + execution: NodeWorkerExecution; isAuthorized: () => boolean; deadline: OperationDeadline; + boundNode?: NodeWorkerSupervisorNodeProof; onDispatchReady?: () => void; }): Promise => { if (!params.isAuthorized()) { @@ -284,9 +315,18 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp const node = await findNode({ transport, deviceId: params.deviceId, - requireLaunchAvailability: params.requireLaunchAvailability, + execution: params.execution, + requireLaunchEligibility: params.command === NODE_WORKER_SUPERVISOR_LAUNCH_COMMAND, signal, }); + if (params.boundNode && !sameNodeConnection(node, params.boundNode)) { + // A projection bearer is bound to this exact paired connection. Retrying against a + // replacement node would silently turn a stale capability into a confused deputy. + throw new NodeWorkerLaunchTransportError( + "NODE_IDENTITY_CHANGED", + "device worker node identity changed after memory projection preparation", + ); + } const operation = transport.invoke({ node, command: params.command, @@ -352,7 +392,7 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp }; const cancelUntilTerminal = async (params: { - request: DeviceWorkerLaunchRequest; + request: PreparedDeviceWorkerLaunchRequest; expected: NodeWorkerSupervisorIdentity; }): Promise => { const deadline = createDeadline({ @@ -371,8 +411,10 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp deviceId: params.request.deviceId, command: NODE_WORKER_SUPERVISOR_CANCEL_COMMAND, payload: params.expected, + execution: params.request.input.execution, isAuthorized: params.request.isCancellationAuthorized, deadline, + ...(params.request.boundNode ? { boundNode: params.request.boundNode } : {}), }); if (receipt) { const validated = validateReceipt(receipt, params.expected); @@ -403,9 +445,6 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp const launch = async ( request: DeviceWorkerLaunchRequest, ): Promise => { - const input = snapshotLaunchInput(request.input); - const stableRequest = { ...request, input }; - const expected = expectedIdentity(input); const deadline = createDeadline({ now, timeoutMs: request.timeoutMs, @@ -418,8 +457,12 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp signal: deadline.signal, label: "node worker availability", }); + let input: NodeWorkerLaunchInput; + let expected: NodeWorkerSupervisorIdentity | undefined; + let stableRequest: PreparedDeviceWorkerLaunchRequest | undefined; let mayHaveLaunched = false; let dispatchReady = false; + let executionReady = false; let pollStatus = false; let delayMs = pollIntervalMs; const markDispatchReady = () => { @@ -429,7 +472,47 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp stableRequest.onDispatchReady?.(); } }; + const markExecutionReady = (receipt: NodeWorkerSupervisorReceipt) => { + if ( + executionReady || + !( + receipt.state === "running" || + receipt.state === "completed" || + (isTerminalReceipt(receipt) && receipt.executionStarted) + ) + ) { + return; + } + executionReady = true; + stableRequest?.onExecutionReady?.(); + }; try { + if (request.prepareMemoryProjection) { + const transport = options.getTransport(); + if (!transport) { + throw new NodeWorkerLaunchTransportError( + "UNAVAILABLE", + "device worker node transport is unavailable", + ); + } + const node = await findNode({ + transport, + deviceId: request.deviceId, + execution: request.input.execution, + requireLaunchEligibility: true, + signal: availabilityDeadline.signal, + }); + const memoryProjection = await raceWithSignal( + request.prepareMemoryProjection(node), + availabilityDeadline.signal, + ); + input = snapshotLaunchInput({ ...request.input, memoryProjection }); + stableRequest = { ...request, input, boundNode: node }; + } else { + input = snapshotLaunchInput(request.input); + stableRequest = { ...request, input }; + } + expected = expectedIdentity(input); while (true) { if (deadline.signal.aborted) { throw signalError(deadline.signal, "node worker launch aborted"); @@ -448,9 +531,10 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp ? NODE_WORKER_SUPERVISOR_STATUS_COMMAND : NODE_WORKER_SUPERVISOR_LAUNCH_COMMAND, payload: pollStatus ? { launchId: input.launchId } : input, - ...(!pollStatus ? { requireLaunchAvailability: true } : {}), + execution: input.execution, isAuthorized: stableRequest.isDispatchAuthorized, deadline: attemptDeadline, + ...(stableRequest.boundNode ? { boundNode: stableRequest.boundNode } : {}), ...(!pollStatus ? { onDispatchReady: markDispatchReady, @@ -465,6 +549,7 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp } const validated = validateReceipt(receipt, expected); mayHaveLaunched = true; + markExecutionReady(validated); if (isTerminalReceipt(validated)) { return validated; } @@ -505,7 +590,10 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp } let terminal: TerminalNodeWorkerSupervisorReceipt; try { - terminal = await cancelUntilTerminal({ request: stableRequest, expected }); + terminal = await cancelUntilTerminal({ + request: stableRequest!, + expected: expected!, + }); } catch (cancelError) { throw Object.assign( new Error("node worker launch failed and cancellation could not be confirmed", { @@ -524,5 +612,39 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp } }; - return { launch }; + const assertExecutionEligible = async (params: { + deviceId: string; + execution: NodeWorkerExecution; + signal?: AbortSignal; + }): Promise => { + const transport = options.getTransport(); + if (!transport) { + throw new NodeWorkerLaunchTransportError( + "UNAVAILABLE", + "device worker node transport is unavailable", + ); + } + const timeout = new AbortController(); + const timer = setTimeout( + () => timeout.abort(new Error("node worker eligibility check timed out")), + DEFAULT_AVAILABILITY_TIMEOUT_MS, + ); + timer.unref?.(); + const signal = params.signal + ? AbortSignal.any([params.signal, timeout.signal]) + : timeout.signal; + try { + await findNode({ + transport, + deviceId: params.deviceId, + execution: params.execution, + requireLaunchEligibility: true, + signal, + }); + } finally { + clearTimeout(timer); + } + }; + + return { assertExecutionEligible, launch }; } diff --git a/src/gateway/worker-environments/node-worker-projection-transfer-http.ts b/src/gateway/worker-environments/node-worker-projection-transfer-http.ts new file mode 100644 index 000000000000..18ab1485bb23 --- /dev/null +++ b/src/gateway/worker-environments/node-worker-projection-transfer-http.ts @@ -0,0 +1,154 @@ +import type { IncomingMessage, ServerResponse } from "node:http"; +import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; +import { + nodeWorkerMemoryProjectionTransferPath, + parseNodeWorkerMemoryProjectionRequestProof, + NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER, + NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER, + NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER, + type NodeWorkerMemoryProjectionRequestProof, +} from "../../worker/node-memory-projection-protocol.js"; +import { AUTH_RATE_LIMIT_SCOPE_WORKER_TRANSFER, type AuthRateLimiter } from "../auth-rate-limit.js"; +import { classifyNodeWorkerProjectionTransferPath } from "../gateway-http-route-contracts.js"; +import { sendJson, watchClientDisconnect } from "../http-common.js"; +import { withSerializedRateLimitAttempt } from "../rate-limit-attempt-serialization.js"; +import type { NodeWorkerProjectionTransferService } from "./node-worker-projection-transfer-service.js"; + +const OPAQUE_NOT_FOUND = { error: "not_found" } as const; + +type NodeWorkerProjectionTransferHttpCallbackResult = + | { kind: "unauthorized" } + | { kind: "authorized"; handle: () => Promise | void }; + +export type NodeWorkerProjectionTransferHttpCallback = (params: { + req: IncomingMessage; + res: ServerResponse; + bearer: string; + proof: NodeWorkerMemoryProjectionRequestProof; +}) => Promise; + +function bearerToken(req: IncomingMessage): string | undefined { + const authorization = normalizeOptionalString(req.headers.authorization); + if (!authorization?.toLowerCase().startsWith("bearer ")) { + return undefined; + } + return normalizeOptionalString(authorization.slice(7)); +} + +function proofHeader(req: IncomingMessage, name: string): string | undefined { + const value = req.headers[name]; + return typeof value === "string" ? normalizeOptionalString(value) : undefined; +} + +function requestProof(req: IncomingMessage): NodeWorkerMemoryProjectionRequestProof | undefined { + const signedAt = proofHeader(req, NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER); + if (!signedAt || !/^(?:0|[1-9][0-9]{0,15})$/u.test(signedAt)) { + return undefined; + } + return ( + parseNodeWorkerMemoryProjectionRequestProof({ + nodeId: proofHeader(req, NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER), + signedAtMs: Number(signedAt), + signature: proofHeader(req, NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER), + }) ?? undefined + ); +} + +function sendOpaqueNotFound(res: ServerResponse): void { + sendJson(res, 404, OPAQUE_NOT_FOUND); +} + +export async function handleNodeWorkerProjectionTransferHttpRequest(params: { + req: IncomingMessage; + res: ServerResponse; + clientIp: string | undefined; + rateLimiter?: AuthRateLimiter; + callback?: NodeWorkerProjectionTransferHttpCallback; +}): Promise { + const parsed = URL.parse(params.req.url ?? "/", "http://localhost"); + if (!parsed?.pathname || classifyNodeWorkerProjectionTransferPath(parsed.pathname) === "outside") { + return false; + } + params.res.setHeader("Cache-Control", "no-store"); + if (parsed.pathname !== nodeWorkerMemoryProjectionTransferPath() || parsed.search || params.req.method !== "GET") { + sendOpaqueNotFound(params.res); + return true; + } + const bearer = bearerToken(params.req); + const proof = requestProof(params.req); + const admission = await withSerializedRateLimitAttempt< + | { kind: "rate-limited"; retryAfterMs: number } + | { kind: "unauthorized" } + | Extract + >({ + ip: params.clientIp, + scope: AUTH_RATE_LIMIT_SCOPE_WORKER_TRANSFER, + run: async () => { + const rateCheck = params.rateLimiter?.check( + params.clientIp, + AUTH_RATE_LIMIT_SCOPE_WORKER_TRANSFER, + ); + if (rateCheck && !rateCheck.allowed) { + return { kind: "rate-limited", retryAfterMs: rateCheck.retryAfterMs }; + } + const outcome = + bearer && proof && params.callback + ? await params.callback({ req: params.req, res: params.res, bearer, proof }) + : ({ kind: "unauthorized" } as const); + if (outcome.kind === "unauthorized") { + params.rateLimiter?.recordFailure(params.clientIp, AUTH_RATE_LIMIT_SCOPE_WORKER_TRANSFER); + } else { + params.rateLimiter?.reset(params.clientIp, AUTH_RATE_LIMIT_SCOPE_WORKER_TRANSFER); + } + return outcome; + }, + }); + if (admission.kind === "rate-limited") { + if (admission.retryAfterMs > 0) { + params.res.setHeader("Retry-After", String(Math.ceil(admission.retryAfterMs / 1000))); + } + sendJson(params.res, 429, { error: "rate_limited" }); + return true; + } + if (admission.kind === "unauthorized") { + sendOpaqueNotFound(params.res); + return true; + } + await admission.handle(); + return true; +} + +export function createNodeWorkerProjectionTransferHttpCallback( + service: NodeWorkerProjectionTransferService, +): NodeWorkerProjectionTransferHttpCallback { + return async ({ req, res, bearer, proof }) => { + const authorization = await service.authorize(bearer, proof); + if (!authorization) { + return { kind: "unauthorized" }; + } + return { + kind: "authorized", + handle: async () => { + const clientAbort = new AbortController(); + const stopWatchingDisconnect = watchClientDisconnect(req, res, clientAbort); + const timeoutMs = Math.max(1, authorization.expiresAtMs - Date.now()); + const signal = AbortSignal.any([ + service.authorizationSignal(authorization), + clientAbort.signal, + AbortSignal.timeout(timeoutMs), + ]); + try { + const payload = service.payload(authorization); + if (!payload || signal.aborted || !service.isAuthorizationCurrent(authorization)) { + sendOpaqueNotFound(res); + return; + } + sendJson(res, 200, payload); + } finally { + stopWatchingDisconnect(); + service.revoke(authorization); + } + }, + }; + }; +} diff --git a/src/gateway/worker-environments/node-worker-projection-transfer-service.ts b/src/gateway/worker-environments/node-worker-projection-transfer-service.ts new file mode 100644 index 000000000000..317baa18d505 --- /dev/null +++ b/src/gateway/worker-environments/node-worker-projection-transfer-service.ts @@ -0,0 +1,363 @@ +import { createHash } from "node:crypto"; +import type { AuthorizedMemoryVirtualFileBroker } from "../../agents/memory-authorized-read-host.js"; +import { verifyDeviceSignature } from "../../infra/device-identity.js"; +import { generateSecureToken } from "../../infra/secure-random.js"; +import { registerSecretValueForRedaction } from "../../logging/secret-redaction-registry.js"; +import { + buildNodeWorkerMemoryProjectionRequestProofPayload, + NODE_WORKER_MEMORY_PROJECTION_MAX_FILES, + NODE_WORKER_MEMORY_PROJECTION_MAX_FILE_BYTES, + NODE_WORKER_MEMORY_PROJECTION_MAX_TOTAL_BYTES, + NODE_WORKER_MEMORY_PROJECTION_PROOF_SKEW_MS, + NODE_WORKER_MEMORY_PROJECTION_VERSION, + type NodeWorkerMemoryProjectionFile, + type NodeWorkerMemoryProjectionBinding, + type NodeWorkerMemoryProjectionPayload, + type NodeWorkerMemoryProjection, + type NodeWorkerMemoryProjectionRequestProof, +} from "../../worker/node-memory-projection-protocol.js"; +import type { NodeWorkerSupervisorNodeProof } from "../node-registry-private.js"; + +const TOKEN_PATTERN = /^[A-Za-z0-9_-]{43}$/u; +type ProjectionTransferCapability = { + reference: string; + node: NodeWorkerSupervisorNodeProof; + nodeId: string; + connId: string; + pairingGeneration: string; + environmentId: string; + sessionId: string; + ownerEpoch: number; + placementGeneration: number; + runId: string; + launchId: string; + binding: NodeWorkerMemoryProjectionBinding; + viewId: string; + viewRevision: string; + viewDigest: string; + expiresAtMs: number; + payload: NodeWorkerMemoryProjectionPayload; + state: "ready" | "serving"; + abortController: AbortController; + stopWatchingSignal?: () => void; + isAuthorized: () => boolean; +}; + +function projectionAuthorizationBinding(params: { + launchBinding: string; + broker: AuthorizedMemoryVirtualFileBroker; +}): NodeWorkerMemoryProjectionBinding { + return Object.freeze({ + launch: params.launchBinding, + // The selected runtime minted these opaque values from its authenticated + // plan. Signing their digest prevents a node from substituting a launch + // fence while retaining a different subject/actor/policy view. + authorization: createHash("sha256") + .update( + JSON.stringify({ + launch: params.launchBinding, + planId: params.broker.view.planId, + contextFingerprint: params.broker.view.contextFingerprint, + revision: params.broker.view.revision, + }), + ) + .digest("hex"), + }); +} + +function mintReference(generateToken: (bytes: number) => string): string { + const reference = generateToken(32); + if (!TOKEN_PATTERN.test(reference)) { + throw new Error("Worker memory projection token generator returned an invalid bearer"); + } + registerSecretValueForRedaction(reference); + return reference; +} + +function validVirtualPath(virtualPath: string, roots: ReadonlySet): boolean { + const normalized = virtualPath.normalize("NFC"); + const parts = normalized.split("/"); + const root = parts[0]; + const leaf = parts[1]; + return ( + normalized === virtualPath && + parts.length === 2 && + Boolean(root) && + Boolean(leaf) && + roots.has(root!.toLocaleLowerCase("en-US")) && + leaf !== "." && + leaf !== ".." && + !leaf!.includes("\\") && + !leaf!.includes("\0") + ); +} + +function createPayload( + broker: AuthorizedMemoryVirtualFileBroker, + signal?: AbortSignal, +): Promise<{ + payload: NodeWorkerMemoryProjectionPayload; + expiresAtMs: number; + viewDigest: string; +}> { + return (async () => { + signal?.throwIfAborted(); + const view = broker.view; + const expiresAtMs = Date.parse(view.expiresAt); + if (!Number.isFinite(expiresAtMs) || expiresAtMs <= Date.now()) { + throw new Error("Authorized memory virtual view is expired"); + } + if (view.files.length === 0 || view.files.length > NODE_WORKER_MEMORY_PROJECTION_MAX_FILES) { + throw new Error("Authorized memory virtual view exceeds projection file limits"); + } + const roots = new Set(); + for (const root of view.roots) { + const name = root.virtualRoot.normalize("NFC"); + const key = name.toLocaleLowerCase("en-US"); + // `/memory` is the container mount target, not this payload's root. The + // canonical selected-memory view uses `memory/...`, nested under that mount. + if ( + name !== root.virtualRoot || + !name || + !/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/u.test(name) || + roots.has(key) || + ["opt", "run", "workspace"].includes(key) + ) { + throw new Error("Authorized memory virtual view has an unsafe projection root"); + } + roots.add(key); + } + const paths = new Set(); + let totalBytes = 0; + const files: NodeWorkerMemoryProjectionFile[] = []; + for (const file of [...view.files].toSorted((left, right) => + left.virtualPath.localeCompare(right.virtualPath), + )) { + signal?.throwIfAborted(); + if (!validVirtualPath(file.virtualPath, roots)) { + throw new Error("Authorized memory virtual view has an unsafe projection path"); + } + const key = file.virtualPath.toLocaleLowerCase("en-US"); + if (paths.has(key)) { + throw new Error("Authorized memory virtual view has colliding projection paths"); + } + paths.add(key); + const text = await broker.readFile(file.virtualPath, signal); + signal?.throwIfAborted(); + if (text === undefined) { + throw new Error("Authorized memory virtual view could not materialize an issued file"); + } + const bytes = Buffer.from(text, "utf8"); + if (bytes.byteLength > NODE_WORKER_MEMORY_PROJECTION_MAX_FILE_BYTES) { + throw new Error("Authorized memory virtual view exceeds projection file limits"); + } + totalBytes += bytes.byteLength; + if (totalBytes > NODE_WORKER_MEMORY_PROJECTION_MAX_TOTAL_BYTES) { + throw new Error("Authorized memory virtual view exceeds projection total limits"); + } + files.push( + Object.freeze({ + virtualPath: file.virtualPath, + sha256: createHash("sha256").update(bytes).digest("hex"), + contentBase64: bytes.toString("base64"), + }), + ); + } + const payload = Object.freeze({ + version: NODE_WORKER_MEMORY_PROJECTION_VERSION, + files: Object.freeze(files), + }); + return { + payload, + expiresAtMs, + viewDigest: createHash("sha256").update(JSON.stringify(payload)).digest("hex"), + }; + })(); +} + +export function createNodeWorkerProjectionTransferService( + options: { + now?: () => number; + generateToken?: (bytes: number) => string; + resolveNodePublicKey?: (node: NodeWorkerSupervisorNodeProof) => Promise; + isNodeCurrent?: (node: NodeWorkerSupervisorNodeProof) => boolean; + } = {}, +) { + const now = options.now ?? Date.now; + const generateToken = options.generateToken ?? generateSecureToken; + const resolveNodePublicKey = options.resolveNodePublicKey ?? (async () => undefined); + const isNodeCurrent = options.isNodeCurrent ?? (() => false); + const capabilities = new Map(); + + const isCurrent = (capability: ProjectionTransferCapability): boolean => + capabilities.get(capability.reference) === capability && + capability.state === "serving" && + capability.expiresAtMs > now() && + !capability.abortController.signal.aborted && + capability.isAuthorized() && + isNodeCurrent(capability.node); + + const isReady = ( + capability: ProjectionTransferCapability | undefined, + ): capability is ProjectionTransferCapability => + Boolean( + capability && + capabilities.get(capability.reference) === capability && + capability.state === "ready" && + capability.expiresAtMs > now() && + !capability.abortController.signal.aborted && + capability.isAuthorized() && + isNodeCurrent(capability.node), + ); + + const revokeCapability = (capability: ProjectionTransferCapability): void => { + if (capabilities.get(capability.reference) === capability) { + capabilities.delete(capability.reference); + } + capability.stopWatchingSignal?.(); + if (!capability.abortController.signal.aborted) { + capability.abortController.abort(new Error("Worker memory projection capability closed")); + } + }; + + return { + async prepare(params: { + node: NodeWorkerSupervisorNodeProof; + broker: AuthorizedMemoryVirtualFileBroker; + environmentId: string; + sessionId: string; + ownerEpoch: number; + placementGeneration: number; + runId: string; + launchId: string; + launchBinding: string; + isAuthorized: () => boolean; + signal?: AbortSignal; + }): Promise { + if (!params.isAuthorized() || !isNodeCurrent(params.node)) { + throw new Error("Worker memory projection authority is unavailable"); + } + if (!/^[a-f0-9]{64}$/u.test(params.launchBinding)) { + throw new Error("Worker memory projection launch binding is invalid"); + } + params.signal?.throwIfAborted(); + const snapshot = await createPayload(params.broker, params.signal); + if (!params.isAuthorized() || snapshot.expiresAtMs <= now()) { + throw new Error("Worker memory projection authority is unavailable"); + } + params.signal?.throwIfAborted(); + const reference = mintReference(generateToken); + if (capabilities.has(reference)) { + // A collision must never replace a live single-use capability, even in a + // test or an incorrectly substituted entropy source. + throw new Error("Worker memory projection token collision"); + } + const capability: ProjectionTransferCapability = { + reference, + node: params.node, + nodeId: params.node.nodeId, + connId: params.node.connId, + pairingGeneration: params.node.pairingGeneration, + environmentId: params.environmentId, + sessionId: params.sessionId, + ownerEpoch: params.ownerEpoch, + placementGeneration: params.placementGeneration, + runId: params.runId, + launchId: params.launchId, + binding: projectionAuthorizationBinding({ + launchBinding: params.launchBinding, + broker: params.broker, + }), + viewId: params.broker.view.viewId, + viewRevision: params.broker.view.revision, + viewDigest: snapshot.viewDigest, + expiresAtMs: snapshot.expiresAtMs, + payload: snapshot.payload, + state: "ready", + abortController: new AbortController(), + isAuthorized: params.isAuthorized, + }; + if (params.signal) { + const abort = () => revokeCapability(capability); + params.signal.addEventListener("abort", abort, { once: true }); + capability.stopWatchingSignal = () => params.signal?.removeEventListener("abort", abort); + if (params.signal.aborted) { + abort(); + } + } + if (capability.abortController.signal.aborted) { + throw new Error("Worker memory projection authority is unavailable"); + } + capabilities.set(reference, capability); + return Object.freeze({ + version: NODE_WORKER_MEMORY_PROJECTION_VERSION, + reference, + binding: capability.binding, + expiresAtMs: capability.expiresAtMs, + }); + }, + + async authorize( + reference: string, + proof: NodeWorkerMemoryProjectionRequestProof, + ): Promise { + const capability = capabilities.get(reference); + if ( + !isReady(capability) || + proof.nodeId !== capability.nodeId || + Math.abs(now() - proof.signedAtMs) > NODE_WORKER_MEMORY_PROJECTION_PROOF_SKEW_MS + ) { + return undefined; + } + const publicKey = await resolveNodePublicKey(capability.node).catch(() => undefined); + if (!publicKey || !isReady(capability)) { + return undefined; + } + const payload = buildNodeWorkerMemoryProjectionRequestProofPayload({ + reference, + binding: capability.binding, + nodeId: capability.nodeId, + signedAtMs: proof.signedAtMs, + }); + if (!verifyDeviceSignature(publicKey, payload, proof.signature) || !isReady(capability)) { + return undefined; + } + // This is the final synchronous check before ready -> serving. A stale + // connection or invalid signature leaves the one-use capability intact. + capability.state = "serving"; + return capability; + }, + + isAuthorizationCurrent: isCurrent, + + authorizationSignal(capability: ProjectionTransferCapability): AbortSignal { + return capability.abortController.signal; + }, + + payload( + capability: ProjectionTransferCapability, + ): NodeWorkerMemoryProjectionPayload | undefined { + return isCurrent(capability) ? capability.payload : undefined; + }, + + revoke(capabilityOrReference: ProjectionTransferCapability | string): void { + const capability = + typeof capabilityOrReference === "string" + ? capabilities.get(capabilityOrReference) + : capabilityOrReference; + if (capability) { + revokeCapability(capability); + } + }, + + closeAll(): void { + for (const capability of capabilities.values()) { + revokeCapability(capability); + } + }, + }; +} + +export type NodeWorkerProjectionTransferService = ReturnType< + typeof createNodeWorkerProjectionTransferService +>; diff --git a/src/gateway/worker-environments/node-worker-projection-transfer.test.ts b/src/gateway/worker-environments/node-worker-projection-transfer.test.ts new file mode 100644 index 000000000000..614009ff0da8 --- /dev/null +++ b/src/gateway/worker-environments/node-worker-projection-transfer.test.ts @@ -0,0 +1,554 @@ +import fs from "node:fs/promises"; +import http from "node:http"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + GATEWAY_CLIENT_IDS, + GATEWAY_CLIENT_MODES, +} from "../../../packages/gateway-protocol/src/client-info.js"; +import type { AuthorizedMemoryVirtualFileBroker } from "../../agents/memory-authorized-read-host.js"; +import { + loadOrCreateDeviceIdentity, + publicKeyRawBase64UrlFromPem, + signDevicePayload, + type DeviceIdentity, +} from "../../infra/device-identity.js"; +import { NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE } from "../../infra/node-runner-inventory.js"; +import { NodeWorkerMemoryProjectionRuntime } from "../../node-host/node-worker-memory-projection.js"; +import { + buildNodeWorkerMemoryProjectionRequestProofPayload, + nodeWorkerMemoryProjectionTransferPath, + NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER, + NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER, + NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER, + type NodeWorkerMemoryProjection, + type NodeWorkerMemoryProjectionBinding, +} from "../../worker/node-memory-projection-protocol.js"; +import { NODE_WORKER_EXECUTION_CONTAINER_V1 } from "../../worker/node-supervisor-protocol.js"; +import type { NodeWorkerSupervisorNodeProof } from "../node-registry-private.js"; +import { + createNodeWorkerProjectionTransferHttpCallback, + handleNodeWorkerProjectionTransferHttpRequest, +} from "./node-worker-projection-transfer-http.js"; +import { createNodeWorkerProjectionTransferService } from "./node-worker-projection-transfer-service.js"; + +function nodeProof(nodeId: string, connId = "conn-1"): NodeWorkerSupervisorNodeProof { + return { + nodeId, + connId, + pairingIdentity: "pairing-1", + pairingGeneration: "generation-1", + clientId: GATEWAY_CLIENT_IDS.NODE_HOST, + clientMode: GATEWAY_CLIENT_MODES.NODE, + protocolFeature: NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE, + workerHost: { + enabled: true, + capacity: { total: 2, available: 2 }, + processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, memoryProjection: 1 }, + }, + commands: [], + }; +} + +function proof(params: { + identity: DeviceIdentity; + reference: string; + binding: NodeWorkerMemoryProjectionBinding; + nodeId?: string; + signedAtMs?: number; + signature?: string; +}) { + const nodeId = params.nodeId ?? params.identity.deviceId; + const signedAtMs = params.signedAtMs ?? Date.now(); + const payload = buildNodeWorkerMemoryProjectionRequestProofPayload({ + reference: params.reference, + binding: params.binding, + nodeId, + signedAtMs, + }); + return { + nodeId, + signedAtMs, + signature: params.signature ?? signDevicePayload(params.identity.privateKeyPem, payload), + }; +} + +async function requestProjection(params: { + port: number; + reference: string; + binding: NodeWorkerMemoryProjectionBinding; + identity: DeviceIdentity; + nodeId?: string; + signedAtMs?: number; + signature?: string; +}): Promise<{ status: number; body: string }> { + const requestProof = proof(params); + return await new Promise((resolve, reject) => { + const request = http.request( + { + host: "127.0.0.1", + port: params.port, + path: nodeWorkerMemoryProjectionTransferPath(), + method: "GET", + headers: { + authorization: `Bearer ${params.reference}`, + [NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER]: requestProof.nodeId, + [NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER]: String(requestProof.signedAtMs), + [NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER]: requestProof.signature, + }, + }, + (response) => { + const chunks: Buffer[] = []; + response.on("data", (chunk: Buffer) => chunks.push(chunk)); + response.on("end", () => + resolve({ + status: response.statusCode ?? 0, + body: Buffer.concat(chunks).toString("utf8"), + }), + ); + }, + ); + request.once("error", reject); + request.end(); + }); +} + +function createTransferService(params: { + node: NodeWorkerSupervisorNodeProof; + identity: DeviceIdentity; + isNodeCurrent?: (node: NodeWorkerSupervisorNodeProof) => boolean; + generateToken?: (bytes: number) => string; +}) { + return createNodeWorkerProjectionTransferService({ + ...((params.generateToken ? { generateToken: params.generateToken } : {}) as object), + resolveNodePublicKey: async (candidate) => + candidate.nodeId === params.node.nodeId + ? publicKeyRawBase64UrlFromPem(params.identity.publicKeyPem) + : undefined, + isNodeCurrent: params.isNodeCurrent ?? (() => true), + }); +} + +function broker(expiresAt: string): AuthorizedMemoryVirtualFileBroker { + const contents = new Map([["memory/MEMORY.md", "only this issued virtual view"]]); + return { + view: { + version: 1, + viewId: "view-1", + planId: "plan-1", + contextFingerprint: "context-1", + revision: "revision-1", + roots: [{ version: 1, mountHandle: "mount-1", virtualRoot: "memory", access: "read" }], + files: [{ version: 1, mountHandle: "mount-1", virtualPath: "memory/MEMORY.md" }], + expiresAt, + }, + readFile: async (virtualPath) => contents.get(virtualPath), + }; +} + +describe("node worker memory projection transfer", () => { + let root: string; + let server: http.Server | undefined; + let identity: DeviceIdentity; + + beforeEach(async () => { + root = await fs.mkdtemp( + path.join(await fs.realpath(os.tmpdir()), "openclaw-memory-projection-wire-"), + ); + identity = loadOrCreateDeviceIdentity({ path: path.join(root, "node-identity.sqlite") }); + }); + + afterEach(async () => { + if (server) { + await new Promise((resolve) => server?.close(() => resolve())); + server = undefined; + } + await fs.rm(root, { recursive: true, force: true }); + }); + + it.runIf( + process.platform !== "win32" && typeof process.getuid === "function" && process.getuid() > 0, + )("stages one authorized, immutable virtual-memory snapshot and rejects replay", async () => { + const node = nodeProof(identity.deviceId); + const service = createTransferService({ + node, + identity, + generateToken: () => "A".repeat(43), + }); + const callback = createNodeWorkerProjectionTransferHttpCallback(service); + server = http.createServer((req, res) => { + void handleNodeWorkerProjectionTransferHttpRequest({ + req, + res, + clientIp: "127.0.0.1", + callback, + }).catch((error: unknown) => + res.destroy(error instanceof Error ? error : new Error(String(error))), + ); + }); + await new Promise((resolve) => server?.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("test server did not bind a TCP port"); + } + const projection = await service.prepare({ + node, + broker: broker(new Date(Date.now() + 60_000).toISOString()), + environmentId: "environment-1", + sessionId: "session-1", + ownerEpoch: 3, + placementGeneration: 4, + runId: "run-1", + launchId: "launch-1", + launchBinding: "a".repeat(64), + isAuthorized: () => true, + }); + const runtime = new NodeWorkerMemoryProjectionRuntime({ + root: path.join(root, "node-host"), + deviceIdentity: identity, + }); + const projectionIdentity = { + gatewayNamespace: "gateway-1", + launchId: "launch-1", + planHash: "a".repeat(64), + }; + const endpoint = { kind: "websocket" as const, url: `ws://127.0.0.1:${address.port}` }; + + const staged = await runtime.stage({ identity: projectionIdentity, projection, endpoint }); + + expect(await fs.readFile(path.join(staged, "memory", "MEMORY.md"), "utf8")).toBe( + "only this issued virtual view", + ); + expect((await fs.stat(path.join(staged, "memory", "MEMORY.md"))).mode & 0o777).toBe(0o400); + await expect( + runtime.stage({ identity: projectionIdentity, projection, endpoint }), + ).rejects.toThrow("projection transfer failed (404)"); + expect(await fs.readdir(path.join(root, "node-host", "memory-projections"))).toEqual([]); + }); + + it.runIf( + process.platform !== "win32" && typeof process.getuid === "function" && process.getuid() > 0, + )("removes a staged projection when cancellation races its final rename", async () => { + const node = nodeProof(identity.deviceId); + const service = createTransferService({ node, identity }); + const callback = createNodeWorkerProjectionTransferHttpCallback(service); + server = http.createServer((req, res) => { + void handleNodeWorkerProjectionTransferHttpRequest({ + req, + res, + clientIp: "127.0.0.1", + callback, + }).catch((error: unknown) => + res.destroy(error instanceof Error ? error : new Error(String(error))), + ); + }); + await new Promise((resolve) => server?.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("test server did not bind a TCP port"); + } + const projection = await service.prepare({ + node, + broker: broker(new Date(Date.now() + 60_000).toISOString()), + environmentId: "environment-1", + sessionId: "session-1", + ownerEpoch: 3, + placementGeneration: 4, + runId: "run-1", + launchId: "launch-cancelled-before-rename", + launchBinding: "a".repeat(64), + isAuthorized: () => true, + }); + const runtime = new NodeWorkerMemoryProjectionRuntime({ + root: path.join(root, "node-host"), + deviceIdentity: identity, + }); + const projectionIdentity = { + gatewayNamespace: "gateway-1", + launchId: "launch-cancelled-before-rename", + planHash: "b".repeat(64), + }; + const controller = new AbortController(); + const rename = fs.rename; + const renameSpy = vi.spyOn(fs, "rename").mockImplementation(async (...args) => { + controller.abort(new Error("projection revoked before process start")); + await rename(...args); + }); + try { + await expect( + runtime.stage({ + identity: projectionIdentity, + projection, + endpoint: { kind: "websocket", url: `ws://127.0.0.1:${address.port}` }, + signal: controller.signal, + }), + ).rejects.toThrow("projection revoked before process start"); + } finally { + renameSpy.mockRestore(); + } + expect(await fs.readdir(path.join(root, "node-host", "memory-projections"))).toEqual([]); + }); + + it("never overwrites a live capability when the token source collides", async () => { + const node = nodeProof(identity.deviceId); + const service = createTransferService({ + node, + identity, + generateToken: () => "B".repeat(43), + }); + let authorized = true; + const params = { + node, + broker: broker(new Date(Date.now() + 60_000).toISOString()), + environmentId: "environment-1", + sessionId: "session-1", + ownerEpoch: 3, + placementGeneration: 4, + runId: "run-1", + launchId: "launch-1", + launchBinding: "a".repeat(64), + isAuthorized: () => authorized, + }; + + const prepared = await service.prepare(params); + authorized = false; + await expect( + service.authorize( + prepared.reference, + proof({ identity, reference: prepared.reference, binding: prepared.binding }), + ), + ).resolves.toBeUndefined(); + authorized = true; + await expect(service.prepare({ ...params, launchId: "launch-2" })).rejects.toThrow( + "token collision", + ); + }); + + it("rejects a wrong node proof without consuming the intended node capability", async () => { + const node = nodeProof(identity.deviceId); + const otherIdentity = loadOrCreateDeviceIdentity({ + path: path.join(root, "other-node.sqlite"), + }); + const service = createTransferService({ node, identity }); + const callback = createNodeWorkerProjectionTransferHttpCallback(service); + server = http.createServer((req, res) => { + void handleNodeWorkerProjectionTransferHttpRequest({ + req, + res, + clientIp: "127.0.0.1", + callback, + }); + }); + await new Promise((resolve) => server?.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("test server did not bind a TCP port"); + } + const projection = await service.prepare({ + node, + broker: broker(new Date(Date.now() + 60_000).toISOString()), + environmentId: "environment-1", + sessionId: "session-1", + ownerEpoch: 3, + placementGeneration: 4, + runId: "run-1", + launchId: "launch-1", + launchBinding: "a".repeat(64), + isAuthorized: () => true, + }); + + await expect( + requestProjection({ + port: address.port, + reference: projection.reference, + binding: projection.binding, + identity: otherIdentity, + }), + ).resolves.toMatchObject({ status: 404 }); + await expect( + requestProjection({ + port: address.port, + reference: projection.reference, + binding: projection.binding, + identity, + }), + ).resolves.toMatchObject({ status: 200 }); + }); + + it("rejects a re-signed projection binding swap without consuming the issued view", async () => { + const node = nodeProof(identity.deviceId); + const service = createTransferService({ node, identity }); + const callback = createNodeWorkerProjectionTransferHttpCallback(service); + server = http.createServer((req, res) => { + void handleNodeWorkerProjectionTransferHttpRequest({ + req, + res, + clientIp: "127.0.0.1", + callback, + }); + }); + await new Promise((resolve) => server?.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("test server did not bind a TCP port"); + } + const projection = await service.prepare({ + node, + broker: broker(new Date(Date.now() + 60_000).toISOString()), + environmentId: "environment-1", + sessionId: "session-1", + ownerEpoch: 3, + placementGeneration: 4, + runId: "run-1", + launchId: "launch-1", + launchBinding: "a".repeat(64), + isAuthorized: () => true, + }); + + await expect( + requestProjection({ + port: address.port, + reference: projection.reference, + binding: { ...projection.binding, launch: "d".repeat(64) }, + identity, + }), + ).resolves.toMatchObject({ status: 404 }); + await expect( + requestProjection({ + port: address.port, + reference: projection.reference, + binding: projection.binding, + identity, + }), + ).resolves.toMatchObject({ status: 200 }); + }); + + it("rejects stale connections and invalid proofs without consuming the current capability", async () => { + const node = nodeProof(identity.deviceId); + let currentNode = node; + const service = createTransferService({ + node, + identity, + isNodeCurrent: (candidate) => + candidate.nodeId === currentNode.nodeId && + candidate.connId === currentNode.connId && + candidate.pairingGeneration === currentNode.pairingGeneration, + }); + const callback = createNodeWorkerProjectionTransferHttpCallback(service); + server = http.createServer((req, res) => { + void handleNodeWorkerProjectionTransferHttpRequest({ + req, + res, + clientIp: "127.0.0.1", + callback, + }); + }); + await new Promise((resolve) => server?.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("test server did not bind a TCP port"); + } + const prepare = async (launchId: string) => + await service.prepare({ + node, + broker: broker(new Date(Date.now() + 60_000).toISOString()), + environmentId: "environment-1", + sessionId: "session-1", + ownerEpoch: 3, + placementGeneration: 4, + runId: "run-1", + launchId, + launchBinding: "a".repeat(64), + isAuthorized: () => true, + }); + + const replaced = await prepare("launch-replaced"); + currentNode = nodeProof(identity.deviceId, "conn-replaced"); + await expect( + requestProjection({ + port: address.port, + reference: replaced.reference, + binding: replaced.binding, + identity, + }), + ).resolves.toMatchObject({ status: 404 }); + currentNode = node; + await expect( + requestProjection({ + port: address.port, + reference: replaced.reference, + binding: replaced.binding, + identity, + }), + ).resolves.toMatchObject({ status: 200 }); + + const invalid = await prepare("launch-invalid"); + await expect( + requestProjection({ + port: address.port, + reference: invalid.reference, + binding: invalid.binding, + identity, + signature: "A".repeat(86), + }), + ).resolves.toMatchObject({ status: 404 }); + await expect( + requestProjection({ + port: address.port, + reference: invalid.reference, + binding: invalid.binding, + identity, + }), + ).resolves.toMatchObject({ status: 200 }); + + const expired = await prepare("launch-expired"); + await expect( + requestProjection({ + port: address.port, + reference: expired.reference, + binding: expired.binding, + identity, + signedAtMs: Date.now() - 120_001, + }), + ).resolves.toMatchObject({ status: 404 }); + await expect( + requestProjection({ + port: address.port, + reference: expired.reference, + binding: expired.binding, + identity, + }), + ).resolves.toMatchObject({ status: 200 }); + }); + + it.runIf( + process.platform !== "win32" && typeof process.getuid === "function" && process.getuid() > 0, + )( + "refuses a plaintext non-loopback projection transport before sending a capability", + async () => { + const runtime = new NodeWorkerMemoryProjectionRuntime({ + root: path.join(root, "node-host"), + deviceIdentity: identity, + }); + const projection: NodeWorkerMemoryProjection = { + version: 1, + reference: "C".repeat(43), + binding: { launch: "a".repeat(64), authorization: "b".repeat(64) }, + expiresAtMs: Date.now() + 60_000, + }; + + await expect( + runtime.stage({ + identity: { + gatewayNamespace: "gateway-1", + launchId: "launch-1", + planHash: "a".repeat(64), + }, + projection, + endpoint: { kind: "websocket", url: "ws://192.0.2.1:18789" }, + }), + ).rejects.toThrow("requires wss://"); + }, + ); +}); diff --git a/src/gateway/worker-environments/node-worker-tunnel.test.ts b/src/gateway/worker-environments/node-worker-tunnel.test.ts index fea7ecef613f..d187ba0bcc27 100644 --- a/src/gateway/worker-environments/node-worker-tunnel.test.ts +++ b/src/gateway/worker-environments/node-worker-tunnel.test.ts @@ -14,7 +14,11 @@ import { createDeferred } from "../../../test/helpers/promise.js"; import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js"; import { NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE } from "../../infra/node-runner-inventory.js"; import { parseWorkerLaunchPlan } from "../../worker/launch-descriptor.js"; -import type { NodeWorkerSupervisorReceipt } from "../../worker/node-supervisor-protocol.js"; +import { + NODE_WORKER_EXECUTION_CONTAINER_V1, + NODE_WORKER_EXECUTION_HOST_V1, + type NodeWorkerSupervisorReceipt, +} from "../../worker/node-supervisor-protocol.js"; import type { NodeWorkerWorkspaceExecInput } from "../../worker/node-workspace-protocol.js"; import { NODE_WORKSPACE_TRANSFER_ERROR_CODE, @@ -24,6 +28,7 @@ import type { NodeWorkerSupervisorTransport } from "../node-registry-private.js" import type { createDeviceWorkerRuntime } from "./device-provider.js"; import { createNodeWorkerTunnelManager } from "./node-worker-tunnel.js"; import type { NodeWorkspaceTransferService } from "./node-workspace-transfer-service.js"; +import type { NodeWorkerProjectionTransferService } from "./node-worker-projection-transfer-service.js"; import { sameWorkerSessionTurnClaim } from "./placement-record.js"; import type { WorkerEnvironmentRecord } from "./store.js"; import { serializeWorkerWorkspaceManifest } from "./workspace-manifest.js"; @@ -96,6 +101,7 @@ function plan() { }, assignment: { agentId: "main", + memoryReadEnforced: false, operationalRunInstance: { instanceId: "instance-1", runId: "run-1" }, agentRuntimeIdentityToken: "runtime-token", runId: "run-1", @@ -160,7 +166,101 @@ function workspaceTransfer(): NodeWorkspaceTransferService { } as unknown as NodeWorkspaceTransferService; } +function projectionTransfer(): NodeWorkerProjectionTransferService { + return { + prepare: vi.fn(async () => ({ + version: 1 as const, + reference: "a".repeat(43), + binding: { launch: "b".repeat(64), authorization: "c".repeat(64) }, + expiresAtMs: Date.now() + 60_000, + })), + } as unknown as NodeWorkerProjectionTransferService; +} + describe("node worker tunnel manager", () => { + it.each([ + [false, NODE_WORKER_EXECUTION_HOST_V1], + [true, NODE_WORKER_EXECUTION_CONTAINER_V1], + ] as const)( + "derives %s memory launches as %s execution", + async (memoryReadEnforced, executionKind) => { + const launchNodeWorker = vi.fn(async (request) => ({ + launchId: request.input.launchId, + planHash: "b".repeat(64), + environmentId: request.input.descriptor.admission.environmentId, + sessionId: request.input.descriptor.admission.sessionId, + ownerEpoch: request.input.descriptor.admission.ownerEpoch, + placementGeneration: request.input.placementGeneration, + runId: request.input.descriptor.assignment.runId, + state: "completed", + resultJson: "{}", + })); + const manager = createNodeWorkerTunnelManager({ + gatewayDeviceId: "gateway-device-1", + getEnvironment: () => environment(), + getTransport: transport, + launchNodeWorker, + validateWorkerTurn: () => true, + projectionTransfer: projectionTransfer(), + workspaceTransfer: workspaceTransfer(), + }); + const launchPlan = plan(); + launchPlan.assignment.memoryReadEnforced = memoryReadEnforced; + const handle = await manager.start(startRequest()); + + await handle.launchTurn({ + plan: launchPlan, + turnClaim: turnClaim(), + ...(memoryReadEnforced ? { memoryProjection: {} as never } : {}), + }); + + expect(launchNodeWorker).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ execution: { kind: executionKind } }), + }), + ); + }, + ); + + it("forwards transport dispatch and execution readiness as distinct signals", async () => { + const events: string[] = []; + const launchNodeWorker = vi.fn(async (request) => { + request.onDispatchReady?.(); + expect(events).toEqual(["dispatch"]); + request.onExecutionReady?.(); + return { + launchId: request.input.launchId, + planHash: "b".repeat(64), + environmentId: request.input.descriptor.admission.environmentId, + sessionId: request.input.descriptor.admission.sessionId, + ownerEpoch: request.input.descriptor.admission.ownerEpoch, + placementGeneration: request.input.placementGeneration, + runId: request.input.descriptor.assignment.runId, + state: "completed", + resultJson: "{}", + }; + }); + const manager = createNodeWorkerTunnelManager({ + gatewayDeviceId: "gateway-device-1", + getEnvironment: () => environment(), + getTransport: transport, + launchNodeWorker, + validateWorkerTurn: () => true, + projectionTransfer: projectionTransfer(), + workspaceTransfer: workspaceTransfer(), + }); + const handle = await manager.start(startRequest()); + + await handle.launchTurn({ + plan: plan(), + turnClaim: turnClaim(), + onDispatchReady: () => events.push("dispatch"), + onExecutionReady: () => events.push("execution"), + }); + + expect(events).toEqual(["dispatch", "execution"]); + }); + it("revalidates the exact claim when a same-run replacement launches", async () => { const record = environment(); let currentClaim = turnClaim(); @@ -181,9 +281,11 @@ describe("node worker tunnel manager", () => { runId: request.input.descriptor.assignment.runId, state: "cancelled", errorText: "test launch finished", + executionStarted: false, }; }), validateWorkerTurn: (claim) => sameWorkerSessionTurnClaim(claim, currentClaim), + projectionTransfer: projectionTransfer(), workspaceTransfer: workspaceTransfer(), }); const handle = await manager.start(startRequest()); @@ -214,8 +316,10 @@ describe("node worker tunnel manager", () => { runId: request.input.descriptor.assignment.runId, state: "cancelled", errorText, + executionStarted: false, })), validateWorkerTurn: () => true, + projectionTransfer: projectionTransfer(), workspaceTransfer: workspaceTransfer(), }); const handle = await manager.start(startRequest()); @@ -237,6 +341,7 @@ describe("node worker tunnel manager", () => { getTransport: transport, launchNodeWorker: vi.fn(), validateWorkerTurn: () => true, + projectionTransfer: projectionTransfer(), workspaceTransfer: workspaceTransfer(), }); @@ -257,6 +362,7 @@ describe("node worker tunnel manager", () => { getTransport: transport, launchNodeWorker: vi.fn(), validateWorkerTurn: () => true, + projectionTransfer: projectionTransfer(), workspaceTransfer: workspaceTransfer(), }); manager.bindWorkspaceBindingResolver(resolveWorkspaceBinding); @@ -289,6 +395,7 @@ describe("node worker tunnel manager", () => { getTransport: transport, launchNodeWorker: vi.fn(), validateWorkerTurn: () => true, + projectionTransfer: projectionTransfer(), workspaceTransfer: transfer, }); manager.bindWorkspaceBindingResolver(resolveWorkspaceBinding); @@ -319,6 +426,7 @@ describe("node worker tunnel manager", () => { getTransport: transport, launchNodeWorker: vi.fn(), validateWorkerTurn: () => true, + projectionTransfer: projectionTransfer(), workspaceTransfer: transfer, }); manager.bindWorkspaceBindingResolver(async () => { @@ -378,6 +486,7 @@ describe("node worker tunnel manager", () => { getTransport: () => nodeTransport, launchNodeWorker: vi.fn(), validateWorkerTurn: () => true, + projectionTransfer: projectionTransfer(), workspaceTransfer: transfer, }); manager.bindWorkspaceBindingResolver(async () => ({ @@ -472,6 +581,7 @@ describe("node worker tunnel manager", () => { }, launchNodeWorker: vi.fn(), validateWorkerTurn: () => true, + projectionTransfer: projectionTransfer(), workspaceTransfer: transfer, }); const resolveWorkspaceBinding = vi.fn(async () => ({ @@ -570,6 +680,7 @@ describe("node worker tunnel manager", () => { getTransport: () => nodeTransport, launchNodeWorker: vi.fn(), validateWorkerTurn: () => true, + projectionTransfer: projectionTransfer(), workspaceTransfer: transfer, }); manager.bindWorkspaceBindingResolver(async () => ({ @@ -629,6 +740,7 @@ describe("node worker tunnel manager", () => { getTransport: () => nodeTransport, launchNodeWorker: vi.fn(), validateWorkerTurn: () => true, + projectionTransfer: projectionTransfer(), workspaceTransfer: transfer, }); const handle = await manager.start(startRequest()); @@ -668,6 +780,7 @@ describe("node worker tunnel manager", () => { runId: request.input.descriptor.assignment.runId, state: "cancelled", errorText: "node worker cancelled", + executionStarted: false, }); }); }, @@ -681,6 +794,7 @@ describe("node worker tunnel manager", () => { getTransport: transport, launchNodeWorker, validateWorkerTurn: () => true, + projectionTransfer: projectionTransfer(), workspaceTransfer: workspaceTransfer(), }); const first = await manager.start(startRequest()); @@ -729,6 +843,7 @@ describe("node worker tunnel manager", () => { runId: request.input.descriptor.assignment.runId, state: "cancelled", errorText: "node worker cancelled", + executionStarted: true, }); }, { once: true }, @@ -742,6 +857,7 @@ describe("node worker tunnel manager", () => { getTransport: transport, launchNodeWorker, validateWorkerTurn: () => true, + projectionTransfer: projectionTransfer(), workspaceTransfer: workspaceTransfer(), }); const handle = await manager.start(startRequest()); @@ -818,6 +934,7 @@ describe("node worker tunnel manager", () => { getTransport: () => nodeTransport, launchNodeWorker: vi.fn(), validateWorkerTurn: () => true, + projectionTransfer: projectionTransfer(), workspaceTransfer: transfer, }); const handle = await manager.start(startRequest()); @@ -886,6 +1003,7 @@ describe("node worker tunnel manager", () => { getTransport: () => nodeTransport, launchNodeWorker: vi.fn(), validateWorkerTurn: () => true, + projectionTransfer: projectionTransfer(), workspaceTransfer: transfer, }); const handle = await manager.start(startRequest()); diff --git a/src/gateway/worker-environments/node-worker-tunnel.ts b/src/gateway/worker-environments/node-worker-tunnel.ts index 4f6348de8989..257a56769777 100644 --- a/src/gateway/worker-environments/node-worker-tunnel.ts +++ b/src/gateway/worker-environments/node-worker-tunnel.ts @@ -6,7 +6,14 @@ import { NODE_WORKER_WORKSPACE_EXEC_COMMAND } from "../../infra/node-commands.js import { createSubsystemLogger } from "../../logging/subsystem.js"; import type { SpawnResult } from "../../process/exec.js"; import { createDeferredCore, type Deferred } from "../../shared/deferred.js"; -import type { NodeWorkerSupervisorReceipt } from "../../worker/node-supervisor-protocol.js"; +import { + NODE_WORKER_EXECUTION_CONTAINER_V1, + NODE_WORKER_EXECUTION_HOST_V1, + nodeWorkerMemoryProjectionLaunchBinding, + type NodeWorkerExecution, + type NodeWorkerSupervisorReceipt, +} from "../../worker/node-supervisor-protocol.js"; +import type { NodeWorkerMemoryProjection } from "../../worker/node-memory-projection-protocol.js"; import { parseNodeWorkerWorkspaceExecResult, type NodeWorkerWorkspaceExecInput, @@ -27,6 +34,7 @@ import { recordNodeSyncPath, } from "./node-worker-workspace-fallback.js"; import type { NodeWorkspaceTransferService } from "./node-workspace-transfer-service.js"; +import type { NodeWorkerProjectionTransferService } from "./node-worker-projection-transfer-service.js"; import type { WorkerSessionTurnClaim } from "./placement-record.js"; import type { WorkerEnvironmentRecord } from "./store.js"; import type { @@ -74,12 +82,18 @@ type NodeWorkerLaunch = (request: { expectedBundleHash: string; placementGeneration: number; descriptor: WorkerTurnLaunchRequest["plan"]; + execution: NodeWorkerExecution; + memoryProjection?: NodeWorkerMemoryProjection; }; + prepareMemoryProjection?: ( + node: NodeWorkerSupervisorNodeProof, + ) => Promise; isDispatchAuthorized: () => boolean; isCancellationAuthorized: () => boolean; timeoutMs: number; signal?: AbortSignal; onDispatchReady?: () => void; + onExecutionReady?: () => void; }) => Promise; type NodeWorkerWorkspaceBinding = { @@ -100,6 +114,7 @@ type NodeWorkerTunnelManagerOptions = { getTransport: () => NodeWorkerSupervisorTransport | undefined; launchNodeWorker: NodeWorkerLaunch; validateWorkerTurn: (claim: WorkerSessionTurnClaim) => boolean; + projectionTransfer: NodeWorkerProjectionTransferService; workspaceTransfer: NodeWorkspaceTransferService; }; @@ -529,22 +544,50 @@ export function createNodeWorkerTunnelManager(options: NodeWorkerTunnelManagerOp claim.sessionId === plan.admission.sessionId && claim.runId === plan.assignment.runId && options.validateWorkerTurn(claim); + const execution: NodeWorkerExecution = plan.assignment.memoryReadEnforced + ? { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 } + : { kind: NODE_WORKER_EXECUTION_HOST_V1 }; + if (execution.kind === NODE_WORKER_EXECUTION_CONTAINER_V1 && !request.memoryProjection) { + throw new Error("enforced node worker launch is missing its authorized memory projection"); + } + const launchSignal = request.signal + ? AbortSignal.any([entry.abortController.signal, request.signal]) + : entry.abortController.signal; + const input = { + launchId: plan.assignment.turnId, + gatewayNamespace, + expectedBundleHash: entry.expectedBuild.bundleHash, + placementGeneration: claim.placementGeneration, + descriptor: plan, + execution, + }; const operation = options.launchNodeWorker({ deviceId: entry.deviceId, - input: { - launchId: plan.assignment.turnId, - gatewayNamespace, - expectedBundleHash: entry.expectedBuild.bundleHash, - placementGeneration: claim.placementGeneration, - descriptor: plan, - }, + input, + ...(request.memoryProjection + ? { + prepareMemoryProjection: async (node) => + await options.projectionTransfer.prepare({ + node, + broker: request.memoryProjection!, + environmentId: entry.environmentId, + sessionId: entry.sessionId, + ownerEpoch: entry.ownerEpoch, + placementGeneration: claim.placementGeneration, + runId: plan.assignment.runId, + launchId: plan.assignment.turnId, + launchBinding: nodeWorkerMemoryProjectionLaunchBinding(input), + isAuthorized: isDispatchAuthorized, + signal: launchSignal, + }), + } + : {}), isDispatchAuthorized, isCancellationAuthorized: () => hasDurableBinding(entry as NodeTunnelEntry), timeoutMs: request.timeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS, onDispatchReady: request.onDispatchReady, - signal: request.signal - ? AbortSignal.any([entry.abortController.signal, request.signal]) - : entry.abortController.signal, + onExecutionReady: request.onExecutionReady, + signal: launchSignal, }); entry.launchTasks.add(operation); try { diff --git a/src/gateway/worker-environments/service.ts b/src/gateway/worker-environments/service.ts index 1182dd07eef4..6552c15d5dcb 100644 --- a/src/gateway/worker-environments/service.ts +++ b/src/gateway/worker-environments/service.ts @@ -339,6 +339,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService const searchMemory = async ( identity: WorkerConnectionIdentity, request: WorkerMemorySearchParams, + signal?: AbortSignal, ): Promise => { const binding = validateMemoryRequest(identity); if (!binding.ok) { @@ -351,6 +352,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService const result = await host.host.search({ query: request.query, ...(request.limit === undefined ? {} : { limit: request.limit }), + ...(signal ? { signal } : {}), }); const current = validateMemoryRequest(identity); if (!current.ok) { @@ -364,6 +366,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService const readMemory = async ( identity: WorkerConnectionIdentity, request: WorkerMemoryReadParams, + signal?: AbortSignal, ): Promise => { const binding = validateMemoryRequest(identity); if (!binding.ok) { @@ -377,6 +380,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService handleId: request.handleId, ...(request.from === undefined ? {} : { from: request.from }), ...(request.lines === undefined ? {} : { lines: request.lines }), + ...(signal ? { signal } : {}), }); const current = validateMemoryRequest(identity); if (!current.ok) { diff --git a/src/gateway/worker-environments/tunnel-contract.ts b/src/gateway/worker-environments/tunnel-contract.ts index 912f88c7657c..5ec008dd1571 100644 --- a/src/gateway/worker-environments/tunnel-contract.ts +++ b/src/gateway/worker-environments/tunnel-contract.ts @@ -2,6 +2,7 @@ import type { WorkerTunnelStatus } from "@openclaw/gateway-protocol"; import { NODE_WORKER_CAPACITY_EXHAUSTED_ERROR_CODE } from "../../infra/node-commands.js"; import type { SpawnResult } from "../../process/exec.js"; import type { WorkerLaunchPlan } from "../../worker/launch-descriptor.js"; +import type { AuthorizedMemoryVirtualFileBroker } from "../../agents/memory-authorized-read-host.js"; import type { NodeWorkerWorkspaceTransferInput } from "../../worker/node-workspace-transfer-protocol.js"; import type { WorkerSessionTurnClaim } from "./placement-record.js"; import type { @@ -102,9 +103,13 @@ export type WorkerWorkspaceQuiescence = { export type WorkerTurnLaunchRequest = { plan: WorkerLaunchPlan; turnClaim: WorkerSessionTurnClaim; + /** Gateway-only broker capability. It must never enter the serialized worker plan. */ + memoryProjection?: AuthorizedMemoryVirtualFileBroker; timeoutMs?: number; signal?: AbortSignal; onDispatchReady?: () => void; + /** Fires only after the node has durably attested real worker execution. */ + onExecutionReady?: () => void; }; export type WorkerWorkspaceTunnelHandle = { diff --git a/src/gateway/worker-environments/worker-turn-launcher-terminal-results.test.ts b/src/gateway/worker-environments/worker-turn-launcher-terminal-results.test.ts index 05b2e2e7061a..0ff2ad051929 100644 --- a/src/gateway/worker-environments/worker-turn-launcher-terminal-results.test.ts +++ b/src/gateway/worker-environments/worker-turn-launcher-terminal-results.test.ts @@ -30,6 +30,7 @@ describe("worker turn launcher terminal results", () => { it("requests immediate recovery when reconciliation fails after worker finishing", async () => { seedActivePlacement(); + const phases: string[] = []; const destroy = vi.fn(async () => attachedEnvironment()); const tunnelFailure = new NodeWorkerWorkspaceTransferError( "workspace-transfer-failed: gateway TLS fingerprint mismatch", @@ -44,6 +45,9 @@ describe("worker turn launcher terminal results", () => { runWorkspaceCommand: vi.fn(), launchTurn: vi.fn(async (request): Promise => { request.onDispatchReady?.(); + expect(phases).not.toContain("process_spawned"); + request.onExecutionReady?.(); + expect(phases.filter((phase) => phase === "process_spawned")).toHaveLength(1); const completed = openSessionManager(); const leafId = completed.appendMessage( makeAgentAssistantMessage({ @@ -106,7 +110,10 @@ describe("worker turn launcher terminal results", () => { agentId: "main", runId: "run-reconcile-tunnel-loss", }, - turn("run-reconcile-tunnel-loss"), + { + ...turn("run-reconcile-tunnel-loss"), + onExecutionPhase: ({ phase }) => phases.push(phase), + }, async () => ({ meta: { durationMs: 1 } }), ), ).rejects.toMatchObject({ @@ -115,6 +122,7 @@ describe("worker turn launcher terminal results", () => { }); expect(reconcileActivePlacement).toHaveBeenCalledWith(ENVIRONMENT_ID); + expect(phases.filter((phase) => phase === "process_spawned")).toEqual(["process_spawned"]); expect(placements.get(SESSION_ID)).toMatchObject({ state: "failed", turnClaim: null }); expect(placements.listPendingWorkspaceResults()).toHaveLength(0); expect(destroy).not.toHaveBeenCalled(); diff --git a/src/gateway/worker-environments/worker-turn-launcher.test.ts b/src/gateway/worker-environments/worker-turn-launcher.test.ts index 2ab6e43491e4..99d128f57d2a 100644 --- a/src/gateway/worker-environments/worker-turn-launcher.test.ts +++ b/src/gateway/worker-environments/worker-turn-launcher.test.ts @@ -13,6 +13,8 @@ import { import { createChatRunState } from "../server-chat-state.js"; import { prepareSessionArchiveLifecycle } from "../server-methods/sessions-archive-lifecycle.js"; import type { GatewayRequestContext } from "../server-methods/types.js"; +import { NODE_WORKER_EXECUTION_CONTAINER_V1 } from "../../worker/node-supervisor-protocol.js"; +import { bindDeviceWorkerExecutionEligibility } from "./device-provider.js"; import type { WorkerTunnelHandle } from "./tunnel-contract.js"; import { ENVIRONMENT_ID, @@ -590,13 +592,24 @@ describe("worker turn launcher local placement", () => { expect(placements.get(SESSION_ID)).toMatchObject({ state: "active", turnClaim: null }); }); - it("rejects enforced memory before a host-node worker is admitted", async () => { + it("fails an enforced worker turn before workspace, credential, or tunnel work when v6 is unavailable", async () => { seedActivePlacement(); + const environment = attachedEnvironment(); + environment.nodeDeviceId = "node-1"; const environments: WorkerTurnEnvironmentService = { ...unusedEnvironments(), - get: vi.fn(() => attachedEnvironment()), + get: vi.fn(() => environment), }; - const provider = createWorkerSessionTurnPlacementProvider({ environments, placements }); + const assertEligible = vi.fn(async () => { + throw new Error("device worker node does not attest to the required process-isolation boundary"); + }); + bindDeviceWorkerExecutionEligibility(environments, assertEligible); + const resolveWorkspacePath = vi.fn(async () => "/workspace/should-not-resolve"); + const provider = createWorkerSessionTurnPlacementProvider({ + environments, + placements, + resolveWorkspacePath, + }); const runLocal = vi.fn(async () => ({ meta: { durationMs: 1 } })); const onAdmitted = vi.fn(); @@ -613,11 +626,16 @@ describe("worker turn launcher local placement", () => { runLocal, onAdmitted, ), - ).rejects.toThrow("enforced memory requires a containerized worker launch"); + ).rejects.toThrow("does not attest to the required process-isolation boundary"); expect(runLocal).not.toHaveBeenCalled(); expect(onAdmitted).not.toHaveBeenCalled(); - expect(environments.get).not.toHaveBeenCalled(); + expect(environments.get).toHaveBeenCalledWith(ENVIRONMENT_ID); + expect(assertEligible).toHaveBeenCalledWith({ + deviceId: "node-1", + execution: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 }, + }); + expect(resolveWorkspacePath).not.toHaveBeenCalled(); expect(environments.acquireTurnCredential).not.toHaveBeenCalled(); expect(environments.startTunnel).not.toHaveBeenCalled(); expect(placements.get(SESSION_ID)).toMatchObject({ state: "active", turnClaim: null }); diff --git a/src/gateway/worker-environments/worker-turn-launcher.ts b/src/gateway/worker-environments/worker-turn-launcher.ts index cee244c905d0..571e65581c62 100644 --- a/src/gateway/worker-environments/worker-turn-launcher.ts +++ b/src/gateway/worker-environments/worker-turn-launcher.ts @@ -1,7 +1,10 @@ import { randomUUID } from "node:crypto"; import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import { mapThinkingLevelForProvider } from "../../agents/embedded-agent-runner/utils.js"; -import { createAuthorizedMemoryReadHost } from "../../agents/memory-authorized-read-host.js"; +import { + createAuthorizedMemoryReadHost, + resolveAuthorizedMemoryVirtualFileBroker, +} from "../../agents/memory-authorized-read-host.js"; import type { SandboxContext } from "../../agents/sandbox/types.js"; import { withSessionPlacementForcedTerminalSettlement, @@ -17,6 +20,7 @@ import { redactSensitiveText } from "../../logging/redact.js"; import { isMemoryIsolationCutoverAgent } from "../../plugins/memory-cutover.js"; import { normalizeDeliveryContext } from "../../utils/delivery-context.shared.js"; import { parseWorkerLaunchPlan } from "../../worker/launch-descriptor.js"; +import { NODE_WORKER_EXECUTION_CONTAINER_V1 } from "../../worker/node-supervisor-protocol.js"; import { WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE } from "../../worker/transcript-message.js"; import { STALE_WORKER_BUILD_REASON, @@ -24,6 +28,7 @@ import { supportsWorkerExecutionContextLaunch, } from "./admission.js"; import { registerWorkerMemoryHost } from "./memory-host.js"; +import { assertDeviceWorkerExecutionEligibility } from "./device-provider.js"; import { placementTurnOwner } from "./placement-record.js"; import { createRemoteExecPlacementSandbox } from "./placement-sandbox.js"; import type { @@ -221,6 +226,23 @@ async function executeWorkerTurn(params: { if (memoryReadEnforced && !memoryHost) { throw new WorkerTurnExecutionError("Enforced memory could not create a trusted worker host"); } + const memoryProjection = memoryHost + ? await resolveAuthorizedMemoryVirtualFileBroker(memoryHost, turn.abortSignal) + : undefined; + if (memoryReadEnforced && !memoryProjection) { + throw new WorkerTurnExecutionError( + "Enforced memory could not materialize an authorized virtual memory view", + ); + } + const memoryProjectionExpiresAtMs = memoryProjection + ? Date.parse(memoryProjection.view.expiresAt) + : undefined; + if ( + memoryProjectionExpiresAtMs !== undefined && + (!Number.isFinite(memoryProjectionExpiresAtMs) || memoryProjectionExpiresAtMs <= Date.now()) + ) { + throw new WorkerTurnExecutionError("Enforced memory virtual view expired before process launch"); + } const { browser, toolAuthority } = resolveWorkerBrowserLaunchPlan({ desktop: environment.desktop, modelRef, @@ -262,11 +284,13 @@ async function executeWorkerTurn(params: { turnId: randomUUID(), prompt: turn.prompt, suppressPromptTranscript: true, - workspaceDir: placement.remoteWorkspaceDir, + workspaceDir: memoryReadEnforced ? "/workspace" : placement.remoteWorkspaceDir, ...(turn.permissionMode ? { permissionMode: turn.permissionMode, - workerContainmentRoot: placement.remoteWorkspaceDir, + workerContainmentRoot: memoryReadEnforced + ? "/workspace" + : placement.remoteWorkspaceDir, } : {}), modelRef, @@ -317,13 +341,13 @@ async function executeWorkerTurn(params: { const handoffAbort = new AbortController(); let handoffError: Error | undefined; let dispatchReady = false; + let executionReady = false; const onDispatchReady = () => { if (dispatchReady) { return; } dispatchReady = true; params.onHandoff(); - turn.onExecutionPhase?.({ phase: "process_spawned", backend: "cloud-worker" }); try { if (!params.environments.acknowledgeCredentialDelivery(credential)) { handoffError = new Error("Cloud worker credential owner changed during process handoff"); @@ -335,6 +359,13 @@ async function executeWorkerTurn(params: { handoffAbort.abort(handoffError); } }; + const onExecutionReady = () => { + if (executionReady) { + return; + } + executionReady = true; + turn.onExecutionPhase?.({ phase: "process_spawned", backend: "cloud-worker" }); + }; if (!tunnel.launchTurn) { throw new Error("Worker tunnel does not support worker turns"); } @@ -343,11 +374,22 @@ async function executeWorkerTurn(params: { return await tunnel.launchTurn({ plan, turnClaim: params.turnClaim, - timeoutMs: turn.timeoutMs, + ...(memoryProjection ? { memoryProjection } : {}), + timeoutMs: + memoryProjectionExpiresAtMs === undefined + ? turn.timeoutMs + : Math.max( + 1, + Math.min( + turn.timeoutMs ?? 60_000, + memoryProjectionExpiresAtMs - Date.now(), + ), + ), signal: turn.abortSignal ? AbortSignal.any([turn.abortSignal, handoffAbort.signal]) : handoffAbort.signal, onDispatchReady, + onExecutionReady, }); } finally { // A completed or failed process must not retain memory authority for a later turn. @@ -538,14 +580,24 @@ export function createWorkerSessionTurnPlacementProvider(options: WorkerTurnLaun let placement = requireActivePlacement(routablePlacement); const requiresProcessIsolation = claim.requiresProcessIsolation === true; const remoteExec = placement.executionMode === "remote-exec"; + if (requiresProcessIsolation && remoteExec) { + // Remote-exec invokes the Gateway callback rather than the node-worker protocol, so it + // cannot attest to the container boundary required for an enforced memory turn. + throw new Error("enforced memory cannot execute through remote-exec placement"); + } if (requiresProcessIsolation) { - // Remote-exec invokes the Gateway callback, while worker-turn launches directly on the - // node host. Neither path supplies the required broker process boundary yet. - throw new Error( - remoteExec - ? "enforced memory cannot execute through remote-exec placement" - : "enforced memory requires a containerized worker launch", - ); + const environment = options.environments.get(placement.environmentId); + if (!environment?.nodeDeviceId) { + throw new Error("enforced memory requires a device worker node"); + } + // Prove the exact connected node's advertised boundary before any credential, tunnel, + // workspace, or claim work can create host-visible state for the enforced turn. + await assertDeviceWorkerExecutionEligibility({ + service: options.environments, + deviceId: environment.nodeDeviceId, + execution: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 }, + ...(turn.abortSignal ? { signal: turn.abortSignal } : {}), + }); } // The placement owns the managed worktree. Callers can carry a default or stale // workspace path, but remote results must only reconcile into that canonical root. diff --git a/src/infra/gateway-lock.ts b/src/infra/gateway-lock.ts index 396f0caa119e..87241dab8c0e 100644 --- a/src/infra/gateway-lock.ts +++ b/src/infra/gateway-lock.ts @@ -54,7 +54,13 @@ const LockPayloadSchema = z.object({ configPath: z.string(), port: z.number().int().min(1).max(65_535).optional(), role: z - .enum(["gateway", "agent-embedded", "skill-workshop-apply", "sqlite-maintenance"]) + .enum([ + "gateway", + "agent-embedded", + "skill-workshop-apply", + "sqlite-maintenance", + "offline-maintenance", + ]) .optional(), stateDir: z.string().optional(), startTime: z.number().optional(), @@ -69,7 +75,12 @@ type GatewayLockHandle = { release: () => Promise; }; -type GatewayLockRole = "gateway" | "agent-embedded" | "skill-workshop-apply" | "sqlite-maintenance"; +type GatewayLockRole = + | "gateway" + | "agent-embedded" + | "skill-workshop-apply" + | "sqlite-maintenance" + | "offline-maintenance"; export type GatewayLockIdentity = { pid: number; @@ -215,6 +226,7 @@ async function resolveGatewayOwnerStatus( if ( role === "agent-embedded" || role === "sqlite-maintenance" || + role === "offline-maintenance" || role === "skill-workshop-apply" ) { const args = readFn(pid); @@ -227,6 +239,11 @@ async function resolveGatewayOwnerStatus( // instead of baking one command spelling into stale-lock recovery. return isOpenClawArgv(args) ? "alive" : "dead"; } + if (role === "offline-maintenance") { + return isOpenClawCommandArgv(args, "doctor") || isOpenClawCommandArgv(args, "backup") + ? "alive" + : "dead"; + } const command = role === "sqlite-maintenance" ? "doctor" : "skills"; return isOpenClawCommandArgv(args, command) ? "alive" : "dead"; } diff --git a/src/infra/node-runner-inventory.test.ts b/src/infra/node-runner-inventory.test.ts new file mode 100644 index 000000000000..648becef9fe9 --- /dev/null +++ b/src/infra/node-runner-inventory.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from "vitest"; +import { GATEWAY_CLIENT_IDS } from "../../packages/gateway-protocol/src/client-info.js"; +import { + NODE_WORKER_EXECUTION_CONTAINER_V1, + NODE_WORKER_EXECUTION_HOST_V1, +} from "../worker/node-supervisor-protocol.js"; +import { + NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE, + NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE, + NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE, + parseNodeRunnerInventoryDeclaration, +} from "./node-runner-inventory.js"; +import { + supportsNodeWorkerExecution, + type NodeWorkerSupervisorNodeProof, +} from "../gateway/node-runner-inventory-runtime.js"; + +const capacity = { total: 2, available: 2 } as const; + +function proof(params: { + protocolFeature: + | typeof NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE + | typeof NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE + | typeof NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE; + processIsolation?: { + kind: typeof NODE_WORKER_EXECUTION_CONTAINER_V1; + memoryProjection?: 1; + }; +}): NodeWorkerSupervisorNodeProof { + return { + nodeId: "node-1", + connId: "conn-1", + pairingIdentity: "identity-1", + pairingGeneration: "generation-1", + clientId: GATEWAY_CLIENT_IDS.NODE_HOST, + clientMode: "node", + protocolFeature: params.protocolFeature, + workerHost: { + enabled: true, + capacity, + ...(params.processIsolation ? { processIsolation: params.processIsolation } : {}), + }, + commands: ["system.run"], + }; +} + +describe("node runner process-isolation inventory", () => { + it("keeps v5 host-only and accepts only the exact v6 declaration", () => { + const v5 = { + protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE], + workerHost: { enabled: true, capacity }, + }; + const v6 = { + protocolFeatures: [NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE], + workerHost: { + enabled: true, + capacity, + processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 }, + }, + }; + + expect(parseNodeRunnerInventoryDeclaration(v5)).toEqual(v5); + expect(parseNodeRunnerInventoryDeclaration(v6)).toEqual(v6); + expect( + parseNodeRunnerInventoryDeclaration({ + ...v5, + workerHost: { ...v5.workerHost, processIsolation: v6.workerHost.processIsolation }, + }), + ).toBeNull(); + }); + + it.each([ + { + protocolFeatures: [NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE], + workerHost: { enabled: true, capacity }, + }, + { + protocolFeatures: [NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE], + workerHost: { enabled: false }, + }, + { + protocolFeatures: [NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE], + workerHost: { + enabled: true, + capacity, + processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, extra: true }, + }, + }, + { + protocolFeatures: [NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE], + workerHost: { enabled: true, capacity, processIsolation: { kind: "host-v1" } }, + }, + ])("rejects malformed v6 process-isolation declarations", (declaration) => { + expect(parseNodeRunnerInventoryDeclaration(declaration)).toBeNull(); + }); + + it("admits container execution only for the exact v7 memory-projection proof", () => { + const v5 = proof({ protocolFeature: NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE }); + const v6 = proof({ + protocolFeature: NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE, + processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 }, + }); + const v7 = proof({ + protocolFeature: NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE, + processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, memoryProjection: 1 }, + }); + + expect(supportsNodeWorkerExecution(v5, { kind: NODE_WORKER_EXECUTION_HOST_V1 })).toBe(true); + expect(supportsNodeWorkerExecution(v5, { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 })).toBe( + false, + ); + expect(supportsNodeWorkerExecution(v6, { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 })).toBe( + false, + ); + expect(supportsNodeWorkerExecution(v7, { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 })).toBe( + true, + ); + }); +}); diff --git a/src/infra/node-runner-inventory.ts b/src/infra/node-runner-inventory.ts index d6c8ad92c67d..09ffff428f79 100644 --- a/src/infra/node-runner-inventory.ts +++ b/src/infra/node-runner-inventory.ts @@ -1,9 +1,14 @@ import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { validateWorkerAdmissionHandshake } from "../../packages/gateway-protocol/src/index.js"; import { WORKER_BUNDLE_PREWARM_VERSION } from "../../packages/gateway-protocol/src/schema/worker-admission.js"; +import { NODE_WORKER_EXECUTION_CONTAINER_V1 } from "../worker/node-supervisor-protocol.js"; export const NODE_RUNNER_INVENTORY_UPDATE_METHOD = "node.runnerInventory.update"; export const NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE = "node-worker-supervisor-v5"; +export const NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE = + "node-worker-supervisor-v6"; +export const NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE = + "node-worker-supervisor-v7"; export const NODE_WORKER_SUPERVISOR_BINARY_CAPACITY_PROTOCOL_FEATURE = "node-worker-supervisor-v4"; export const NODE_WORKER_SUPERVISOR_EXECUTION_CONTEXT_V1_PROTOCOL_FEATURE = "node-worker-supervisor-v3"; @@ -25,6 +30,10 @@ export type NodeWorkerCapacitySnapshot = Readonly<{ total: number; available: number; }>; +export type NodeWorkerProcessIsolationDeclaration = Readonly<{ + kind: typeof NODE_WORKER_EXECUTION_CONTAINER_V1; + memoryProjection?: 1; +}>; export type NodeWorkerHostDeclaration = | { enabled: false } @@ -34,6 +43,8 @@ export type NodeWorkerHostDeclaration = bundlePrewarm?: typeof WORKER_BUNDLE_PREWARM_VERSION; bundleRetention?: typeof NODE_WORKER_BUNDLE_RETENTION_VERSION; bundleStatus?: typeof NODE_WORKER_BUNDLE_STATUS_VERSION; + /** Present only on v6 hosts that passed the local container-runtime gate. */ + processIsolation?: NodeWorkerProcessIsolationDeclaration; }; export type NodeRunnerInventoryDeclaration = @@ -47,7 +58,11 @@ export type NodeRunnerInventoryDeclaration = ]; } | { - protocolFeatures: readonly [typeof NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE]; + protocolFeatures: readonly [ + | typeof NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE + | typeof NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE + | typeof NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE, + ]; workerHost: NodeWorkerHostDeclaration; }; @@ -73,19 +88,52 @@ function parseCapacitySnapshot(value: unknown): NodeWorkerCapacitySnapshot | nul : null; } -function parseWorkerHostDeclaration(value: unknown): NodeWorkerHostDeclaration | null { +function parseProcessIsolationDeclaration( + value: unknown, + requireMemoryProjection: boolean, +): NodeWorkerProcessIsolationDeclaration | null { + if ( + !isRecord(value) || + !Object.hasOwn(value, "kind") || + value.kind !== NODE_WORKER_EXECUTION_CONTAINER_V1 || + (requireMemoryProjection + ? Object.keys(value).length !== 2 || value.memoryProjection !== 1 + : Object.keys(value).length !== 1) + ) { + return null; + } + return requireMemoryProjection + ? { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, memoryProjection: 1 } + : { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 }; +} + +function parseWorkerHostDeclaration(params: { + value: unknown; + requireProcessIsolation: boolean; + requireMemoryProjection?: boolean; +}): NodeWorkerHostDeclaration | null { + const { value } = params; if (!isRecord(value) || typeof value.enabled !== "boolean") { return null; } const keys = Object.keys(value); if (!value.enabled) { - return keys.length === 1 && keys[0] === "enabled" ? { enabled: false } : null; + return !params.requireProcessIsolation && keys.length === 1 && keys[0] === "enabled" + ? { enabled: false } + : null; } const capacity = parseCapacitySnapshot(value.capacity); + const processIsolation = + value.processIsolation === undefined + ? undefined + : parseProcessIsolationDeclaration( + value.processIsolation, + params.requireMemoryProjection === true, + ); if ( !capacity || keys.length < 2 || - keys.length > 5 || + keys.length > 6 || !keys.includes("enabled") || !keys.includes("capacity") || keys.some( @@ -94,14 +142,17 @@ function parseWorkerHostDeclaration(value: unknown): NodeWorkerHostDeclaration | key !== "capacity" && key !== "bundlePrewarm" && key !== "bundleRetention" && - key !== "bundleStatus", + key !== "bundleStatus" && + key !== "processIsolation", ) || (value.bundlePrewarm !== undefined && value.bundlePrewarm !== WORKER_BUNDLE_PREWARM_VERSION) || (value.bundleRetention !== undefined && value.bundleRetention !== NODE_WORKER_BUNDLE_RETENTION_VERSION) || (value.bundleStatus !== undefined && value.bundleStatus !== NODE_WORKER_BUNDLE_STATUS_VERSION) || - (value.bundleStatus !== undefined && value.bundleRetention === undefined) + (value.bundleStatus !== undefined && value.bundleRetention === undefined) || + (value.processIsolation !== undefined && !processIsolation) || + (params.requireProcessIsolation !== Boolean(processIsolation)) ) { return null; } @@ -117,6 +168,7 @@ function parseWorkerHostDeclaration(value: unknown): NodeWorkerHostDeclaration | ...(value.bundleStatus === NODE_WORKER_BUNDLE_STATUS_VERSION ? { bundleStatus: NODE_WORKER_BUNDLE_STATUS_VERSION } : {}), + ...(processIsolation ? { processIsolation } : {}), }; } @@ -190,12 +242,23 @@ export function parseNodeRunnerInventoryDeclaration( ? { protocolFeatures: [feature] } : null; } - if (feature !== NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE || keys.length !== 2) { + if ( + (feature !== NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE && + feature !== NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE && + feature !== NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE) || + keys.length !== 2 + ) { return null; } - const workerHost = parseWorkerHostDeclaration(value.workerHost); + const workerHost = parseWorkerHostDeclaration({ + value: value.workerHost, + requireProcessIsolation: + feature === NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE || + feature === NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE, + requireMemoryProjection: feature === NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE, + }); return workerHost - ? { protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE], workerHost } + ? { protocolFeatures: [feature], workerHost } : null; } diff --git a/src/memory-broker/child.ts b/src/memory-broker/child.ts index 42891c1a7c48..603df902b9db 100644 --- a/src/memory-broker/child.ts +++ b/src/memory-broker/child.ts @@ -1,4 +1,5 @@ -import { startMemoryBrokerServer, type MemoryBrokerHandler } from "./server.js"; +import { startMemoryBrokerServer } from "./server.js"; +import type { MemoryBrokerChildEntry } from "./entry.js"; type BrokerStartMessage = Readonly<{ type: "start"; @@ -7,10 +8,7 @@ type BrokerStartMessage = Readonly<{ brokerEpoch: string; secret: string; handlerModuleUrl: string; -}>; - -type BrokerChildEntry = Readonly<{ - createMemoryBrokerHandler: () => MemoryBrokerHandler | Promise; + agentIds: readonly string[]; }>; type BrokerMaintenanceMessage = Readonly<{ @@ -41,10 +39,19 @@ async function start(message: BrokerStartMessage): Promise { if (server || closing) { throw new Error("memory broker child has already started"); } - const module = (await import(message.handlerModuleUrl)) as Partial; + const module = (await import(message.handlerModuleUrl)) as Partial; if (typeof module.createMemoryBrokerHandler !== "function") { throw new Error("selected memory plugin has no broker child entry"); } + if ( + !Array.isArray(message.agentIds) || + !message.agentIds.every((agentId) => typeof agentId === "string" && agentId.length > 0) + ) { + throw new Error("memory broker startup agents are unavailable"); + } + // Recovery happens before the socket exists, so a fresh/replacement child never reports + // healthy while pending revisions can still become visible or need quarantine. + await module.initializeMemoryBroker?.({ agentIds: Object.freeze([...message.agentIds]) }); const handler = await module.createMemoryBrokerHandler(); if (typeof handler !== "function") { throw new Error("selected memory plugin returned an invalid broker handler"); diff --git a/src/memory-broker/entry.ts b/src/memory-broker/entry.ts new file mode 100644 index 000000000000..1dcd44f6963b --- /dev/null +++ b/src/memory-broker/entry.ts @@ -0,0 +1,18 @@ +import type { MemoryBrokerHandler } from "./server.js"; + +/** + * Gateway passes this small, path-free startup snapshot over inherited parent-child IPC before + * the broker opens its agent-visible socket. A plugin must finish recovery or throw; ready means + * the selected runtime has completed its own durable-startup fence. + */ +export type MemoryBrokerStartupContext = Readonly<{ + agentIds: readonly string[]; +}>; + +/** Public selected-memory child entry contract. */ +export type MemoryBrokerChildEntry = Readonly<{ + createMemoryBrokerHandler: () => MemoryBrokerHandler | Promise; + initializeMemoryBroker?: ( + context: MemoryBrokerStartupContext, + ) => void | Promise; +}>; diff --git a/src/memory-broker/process.test.ts b/src/memory-broker/process.test.ts index be69a7ba183b..b72323cd3c92 100644 --- a/src/memory-broker/process.test.ts +++ b/src/memory-broker/process.test.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { afterEach, describe, expect, it } from "vitest"; import { startMemoryBrokerProcess, type MemoryBrokerProcess } from "./process.js"; +import type { MemoryBrokerAuthorizationBinding } from "./protocol.js"; let broker: MemoryBrokerProcess | undefined; @@ -78,6 +79,7 @@ describe("memory broker child", () => { runId: "run-a", contextFingerprint: "context-a", subjectRevision: "subject-a", + actor: { kind: "principal", actorKind: "human", principalId: "alice" }, actorRevision: "actor-a", capabilitySnapshotId: "capability-a", policyRevision: "policy-a", @@ -90,6 +92,46 @@ describe("memory broker child", () => { ).resolves.toEqual({ agentId: "agent-a", method: "memory.search" }); }); + it("runs selected-runtime startup recovery with sorted configured agents before serving", async () => { + broker = await startMemoryBrokerProcess({ + brokerId: "broker-a", + childModuleUrl: new URL("./child.ts", import.meta.url), + handlerModuleUrl: new URL("./test-handler.mjs", import.meta.url).href, + agentIds: ["work", "main", "work"], + }); + + await expect( + broker.client.request({ + binding: { + agentId: "main", + sessionId: "session-a", + runId: "run-a", + contextFingerprint: "context-a", + subjectRevision: "subject-a", + actor: { kind: "principal", actorKind: "human", principalId: "alice" }, + actorRevision: "actor-a", + capabilitySnapshotId: "capability-a", + policyRevision: "policy-a", + deliveryRevision: "delivery-a", + }, + method: "memory.startup", + payload: {}, + expiresAtMs: Date.now() + 30_000, + }), + ).resolves.toEqual({ agentIds: ["main", "work"] }); + }); + + it("does not report ready when selected-runtime startup recovery fails", async () => { + await expect( + startMemoryBrokerProcess({ + brokerId: "broker-a", + childModuleUrl: new URL("./child.ts", import.meta.url), + handlerModuleUrl: new URL("./test-handler.mjs", import.meta.url).href, + agentIds: ["fail-startup"], + }), + ).rejects.toThrow("memory broker child did not become ready"); + }); + it("retires a stopped child and gives its replacement a new broker epoch", async () => { broker = await startMemoryBrokerProcess({ brokerId: "broker-a", @@ -121,12 +163,13 @@ describe("memory broker child", () => { }); const crashedBroker = broker; const firstEpoch = crashedBroker.brokerEpoch; - const binding = { + const binding: MemoryBrokerAuthorizationBinding = { agentId: "agent-a", sessionId: "session-a", runId: "run-a", contextFingerprint: "context-a", subjectRevision: "subject-a", + actor: { kind: "principal", actorKind: "human", principalId: "alice" }, actorRevision: "actor-a", capabilitySnapshotId: "capability-a", policyRevision: "policy-a", @@ -160,6 +203,37 @@ describe("memory broker child", () => { ).resolves.toEqual({ agentId: "agent-a", method: "memory.search" }); }); + it("retires a child killed by an external signal without waiting for a second exit event", async () => { + broker = await startMemoryBrokerProcess({ + brokerId: "broker-a", + childModuleUrl: new URL("./child.ts", import.meta.url), + handlerModuleUrl: new URL("./test-handler.mjs", import.meta.url).href, + }); + const killedBroker = broker; + await expect( + killedBroker.client.request({ + binding: { + agentId: "agent-a", + sessionId: "session-a", + runId: "run-a", + contextFingerprint: "context-a", + subjectRevision: "subject-a", + actor: { kind: "principal", actorKind: "human", principalId: "alice" }, + actorRevision: "actor-a", + capabilitySnapshotId: "capability-a", + policyRevision: "policy-a", + deliveryRevision: "delivery-a", + }, + method: "memory.kill", + payload: {}, + expiresAtMs: Date.now() + 30_000, + }), + ).resolves.toBeUndefined(); + await waitFor(() => !killedBroker.isRunning(), "memory broker child did not exit after SIGKILL"); + + await expect(killedBroker.close()).resolves.toBeUndefined(); + }); + it("does not inherit arbitrary Gateway environment secrets", async () => { const previous = process.env.OPENCLAW_MEMORY_BROKER_TEST_SECRET; process.env.OPENCLAW_MEMORY_BROKER_TEST_SECRET = "gateway-only-secret"; @@ -177,6 +251,7 @@ describe("memory broker child", () => { runId: "run-a", contextFingerprint: "context-a", subjectRevision: "subject-a", + actor: { kind: "principal", actorKind: "human", principalId: "alice" }, actorRevision: "actor-a", capabilitySnapshotId: "capability-a", policyRevision: "policy-a", @@ -230,6 +305,7 @@ describe("memory broker child", () => { runId: "run-a", contextFingerprint: "context-a", subjectRevision: "subject-a", + actor: { kind: "principal", actorKind: "human", principalId: "alice" }, actorRevision: "actor-a", capabilitySnapshotId: "capability-a", policyRevision: "policy-a", @@ -249,6 +325,7 @@ describe("memory broker child", () => { runId: "run-a", contextFingerprint: "context-a", subjectRevision: "subject-a", + actor: { kind: "principal", actorKind: "human", principalId: "alice" }, actorRevision: "actor-a", capabilitySnapshotId: "capability-a", policyRevision: "policy-a", @@ -260,4 +337,36 @@ describe("memory broker child", () => { }), ).resolves.toEqual({ agentId: "agent-a", method: "memory.search" }); }); + + it("retires a noncooperative child instead of leaving a healthy broker quiesced", async () => { + broker = await startMemoryBrokerProcess({ + brokerId: "broker-a", + childModuleUrl: new URL("./child.ts", import.meta.url), + handlerModuleUrl: new URL("./test-handler.mjs", import.meta.url).href, + maintenanceTimeoutMs: 40, + }); + const pending = broker.client.request({ + binding: { + agentId: "agent-a", + sessionId: "session-a", + runId: "run-a", + contextFingerprint: "context-a", + subjectRevision: "subject-a", + actor: { kind: "principal", actorKind: "human", principalId: "alice" }, + actorRevision: "actor-a", + capabilitySnapshotId: "capability-a", + policyRevision: "policy-a", + deliveryRevision: "delivery-a", + }, + method: "memory.hang", + payload: {}, + expiresAtMs: Date.now() + 30_000, + }); + await new Promise((resolve) => setTimeout(resolve, 20)); + + await expect(broker.quiesce()).rejects.toThrow("memory broker quiesce is unavailable"); + await expect(pending).resolves.toBeUndefined(); + expect(broker.isRunning()).toBe(false); + await expect(broker.isHealthy()).resolves.toBe(false); + }); }); diff --git a/src/memory-broker/process.ts b/src/memory-broker/process.ts index 9bc1a77c34be..df6f7b3ec7a5 100644 --- a/src/memory-broker/process.ts +++ b/src/memory-broker/process.ts @@ -44,9 +44,18 @@ export type MemoryBrokerProcess = Readonly<{ export async function startMemoryBrokerProcess(params: { brokerId: string; handlerModuleUrl: string; + /** Configured agent IDs only; passed over inherited IPC before the broker binds its socket. */ + agentIds?: readonly string[]; childModuleUrl?: string | URL; startTimeoutMs?: number; + maintenanceTimeoutMs?: number; }): Promise { + if (!(params.agentIds ?? []).every((agentId) => typeof agentId === "string" && agentId.length > 0)) { + throw new Error("memory broker startup agent IDs are invalid"); + } + const agentIds = Object.freeze( + [...new Set(params.agentIds ?? [])].toSorted(), + ); const directory = await mkdtemp(path.join(tmpdir(), "openclaw-memory-broker-")); const socketPath = path.join(directory, "broker.sock"); const brokerEpoch = randomUUID(); @@ -87,6 +96,9 @@ export async function startMemoryBrokerProcess(params: { }); let childExited = false; let directoryRemoved = false; + let retirePromise: Promise | undefined; + const hasChildExited = () => + childExited || child.exitCode !== null || child.signalCode !== null || child.killed; const removeDirectory = async () => { if (directoryRemoved) { return; @@ -100,6 +112,18 @@ export async function startMemoryBrokerProcess(params: { // The Gateway recreates the child with a fresh epoch on the next operation. void removeDirectory(); }); + const retireChild = async () => { + retirePromise ??= (async () => { + if (!hasChildExited()) { + child.kill("SIGKILL"); + } + if (!hasChildExited()) { + await new Promise((resolve) => child.once("exit", () => resolve())); + } + await removeDirectory(); + })(); + await retirePromise; + }; let startupStderr = ""; child.stderr?.on("data", (chunk: Buffer) => { // Startup diagnostics never include request bodies or the parent-child secret. Keep enough @@ -159,6 +183,7 @@ export async function startMemoryBrokerProcess(params: { brokerEpoch, secret: secret.toString("base64url"), handlerModuleUrl: params.handlerModuleUrl, + agentIds, }); }); } catch (error) { @@ -173,7 +198,7 @@ export async function startMemoryBrokerProcess(params: { secret, }); const isHealthy = async (): Promise => { - if (childExited || child.exitCode !== null || child.killed || !child.connected) { + if (hasChildExited() || !child.connected) { return false; } return await new Promise((resolve) => { @@ -210,8 +235,13 @@ export async function startMemoryBrokerProcess(params: { }); }); }; - const maintain = async (operation: "quiesce" | "resume"): Promise => { - if (childExited || child.exitCode !== null || child.killed || !child.connected) { + const maintenanceTimeoutMs = params.maintenanceTimeoutMs ?? MEMORY_BROKER_MAINTENANCE_TIMEOUT_MS; + if (!Number.isSafeInteger(maintenanceTimeoutMs) || maintenanceTimeoutMs < 1) { + await retireChild(); + throw new Error("memory broker maintenance timeout is invalid"); + } + const maintainNow = async (operation: "quiesce" | "resume"): Promise => { + if (hasChildExited() || !child.connected) { throw new Error("memory broker child is unavailable for maintenance"); } const requestId = randomUUID(); @@ -248,7 +278,7 @@ export async function startMemoryBrokerProcess(params: { finish((message as { ok?: unknown }).ok === true); } }; - const timer = setTimeout(() => finish(false), MEMORY_BROKER_MAINTENANCE_TIMEOUT_MS); + const timer = setTimeout(() => finish(false), maintenanceTimeoutMs); timer.unref?.(); child.on("message", onMessage); child.send({ type: "maintenance", requestId, brokerEpoch, operation }, (error) => { @@ -258,20 +288,32 @@ export async function startMemoryBrokerProcess(params: { }); }); if (!ok) { + // A late child acknowledgement could otherwise arrive after the parent has given up and + // leave a healthy-but-quiesced socket rejecting every future request. Retire the whole + // epoch instead; the next independently authorized request creates a fresh child/secret. + await retireChild(); throw new Error(`memory broker ${operation} is unavailable`); } }; + let maintenanceTail = Promise.resolve(); + const maintain = (operation: "quiesce" | "resume"): Promise => { + const current = maintenanceTail.then(() => maintainNow(operation)); + // Parent-child maintenance messages must preserve order even when a caller observes a + // failure. Swallow only for tail progression; the original caller still receives the error. + maintenanceTail = current.catch(() => undefined); + return current; + }; return Object.freeze({ client, brokerEpoch, - isRunning: () => !childExited && child.exitCode === null && !child.killed, + isRunning: () => !hasChildExited(), isHealthy, quiesce: () => maintain("quiesce"), resume: () => maintain("resume"), close: async () => { await maintain("quiesce").catch(() => undefined); await new Promise((resolve) => { - if (child.exitCode !== null || child.killed) { + if (hasChildExited()) { resolve(); return; } diff --git a/src/memory-broker/protocol.test.ts b/src/memory-broker/protocol.test.ts index 4dd652194239..3b173d4f950d 100644 --- a/src/memory-broker/protocol.test.ts +++ b/src/memory-broker/protocol.test.ts @@ -15,6 +15,7 @@ const binding: MemoryBrokerAuthorizationBinding = { runId: "run-a", contextFingerprint: "context-a", subjectRevision: "subject-a", + actor: { kind: "principal", actorKind: "human", principalId: "alice" }, actorRevision: "actor-a", capabilitySnapshotId: "capability-a", policyRevision: "policy-a", @@ -58,6 +59,26 @@ function verify( } describe("memory broker envelope", () => { + it("refuses a request whose canonical form exceeds the nesting budget", () => { + let payload: unknown = "leaf"; + for (let depth = 0; depth < 65; depth += 1) { + payload = [payload]; + } + + expect( + createMemoryBrokerEnvelope({ + secret, + brokerId: "broker-a", + brokerEpoch: "epoch-a", + binding, + nonce: "nested-request", + request: { method: "memory.search", payload }, + issuedAtMs: 1_000, + expiresAtMs: 2_000, + }), + ).toBeUndefined(); + }); + it("binds a deterministic request and every Gateway-derived authorization revision", () => { const envelope = createEnvelope(); expect(verify(envelope)).toEqual({ ok: true }); @@ -83,6 +104,16 @@ describe("memory broker envelope", () => { }), ).toEqual({ ok: false, reason: "binding-mismatch" }); } + expect( + verify(envelope, { + expectedBinding: { ...binding, actor: { ...binding.actor, principalId: "bob" } }, + }), + ).toEqual({ ok: false, reason: "binding-mismatch" }); + expect( + verify(envelope, { + expectedBinding: { ...binding, actor: { ...binding.actor, actorKind: "service" } }, + }), + ).toEqual({ ok: false, reason: "binding-mismatch" }); }); it("rejects cross-broker, stale-epoch, expired, and forged envelopes", () => { diff --git a/src/memory-broker/protocol.ts b/src/memory-broker/protocol.ts index 7de0179391e1..38b6c84dc6bd 100644 --- a/src/memory-broker/protocol.ts +++ b/src/memory-broker/protocol.ts @@ -4,6 +4,24 @@ import { createHash, createHmac, timingSafeEqual } from "node:crypto"; export const MEMORY_BROKER_PROTOCOL_VERSION = 1 as const; export const MEMORY_BROKER_MAXIMUM_NONCE_LENGTH = 256; +export const MEMORY_BROKER_MAXIMUM_CANONICAL_DEPTH = 64; +export const MEMORY_BROKER_MAXIMUM_CANONICAL_NODES = 10_000; + +/** + * A signed broker frame must bind the actor identity as well as the evidence revision. Revisions + * can be shared by a provider snapshot, so binding only that revision would let a caller retarget + * a valid Alice request to Bob while preserving every other frame field. + */ +export type MemoryBrokerActorBinding = + | Readonly<{ + kind: "principal"; + actorKind: "human" | "agent" | "service" | "system"; + principalId: string; + }> + | Readonly<{ + kind: "unattributed"; + transportAuditRef: string; + }>; export type MemoryBrokerAuthorizationBinding = Readonly<{ agentId: string; @@ -11,6 +29,7 @@ export type MemoryBrokerAuthorizationBinding = Readonly<{ runId: string; contextFingerprint: string; subjectRevision: string; + actor: MemoryBrokerActorBinding; actorRevision: string; capabilitySnapshotId: string; policyRevision: string; @@ -62,7 +81,17 @@ function isPlainRecord(value: unknown): value is Record { * The MAC covers a deterministic JSON representation. Rejecting non-JSON values is deliberate: * a request that cannot be reproduced byte-for-byte cannot be safely bound to a single-use grant. */ -function canonicalize(value: unknown): string | undefined { +function canonicalize( + value: unknown, + state: { nodes: number } = { nodes: 0 }, + depth = 0, +): string | undefined { + if ( + depth > MEMORY_BROKER_MAXIMUM_CANONICAL_DEPTH || + ++state.nodes > MEMORY_BROKER_MAXIMUM_CANONICAL_NODES + ) { + return undefined; + } if (value === null) { return "null"; } @@ -77,7 +106,7 @@ function canonicalize(value: unknown): string | undefined { if (Array.isArray(value)) { const items: string[] = []; for (const item of value) { - const serialized = canonicalize(item); + const serialized = canonicalize(item, state, depth + 1); if (serialized === undefined) { return undefined; } @@ -90,7 +119,7 @@ function canonicalize(value: unknown): string | undefined { } const entries: string[] = []; for (const key of Object.keys(value).toSorted()) { - const serialized = canonicalize(value[key]); + const serialized = canonicalize(value[key], state, depth + 1); if (serialized === undefined) { return undefined; } @@ -125,24 +154,42 @@ function hasSafeIdentifier(value: unknown): value is string { ); } +function isActorBinding(value: unknown): value is MemoryBrokerActorBinding { + if (!isPlainRecord(value)) { + return false; + } + if (value.kind === "principal") { + return ( + (value.actorKind === "human" || + value.actorKind === "agent" || + value.actorKind === "service" || + value.actorKind === "system") && + hasSafeIdentifier(value.principalId) + ); + } + return value.kind === "unattributed" && hasSafeIdentifier(value.transportAuditRef); +} + function isBinding(value: unknown): value is MemoryBrokerAuthorizationBinding { if (!isPlainRecord(value)) { return false; } - return [ - value.agentId, - value.sessionId, - value.runId, - value.contextFingerprint, - value.subjectRevision, - value.actorRevision, - value.capabilitySnapshotId, - value.policyRevision, - value.deliveryRevision, - ].every(hasSafeIdentifier); + return ( + [ + value.agentId, + value.sessionId, + value.runId, + value.contextFingerprint, + value.subjectRevision, + value.actorRevision, + value.capabilitySnapshotId, + value.policyRevision, + value.deliveryRevision, + ].every(hasSafeIdentifier) && isActorBinding(value.actor) + ); } -function isEnvelope(value: unknown): value is MemoryBrokerEnvelope { +export function isMemoryBrokerEnvelope(value: unknown): value is MemoryBrokerEnvelope { if ( !isPlainRecord(value) || value.version !== MEMORY_BROKER_PROTOCOL_VERSION || @@ -185,7 +232,7 @@ export function createMemoryBrokerEnvelope(params: { expiresAtMs: params.expiresAtMs, requestDigest: requestDigest ?? "", }; - if (!requestDigest || !isEnvelope({ ...unsigned, signature: "pending" })) { + if (!requestDigest || !isMemoryBrokerEnvelope({ ...unsigned, signature: "pending" })) { return undefined; } const signature = sign(params.secret, unsigned); @@ -203,7 +250,7 @@ export function verifyMemoryBrokerEnvelope(params: { nowMs: number; }): MemoryBrokerEnvelopeVerification { const { envelope } = params; - if (!isEnvelope(envelope)) { + if (!isMemoryBrokerEnvelope(envelope)) { return { ok: false, reason: "invalid-envelope" }; } if (envelope.brokerId !== params.brokerId) { diff --git a/src/memory-broker/server.test.ts b/src/memory-broker/server.test.ts index e1ae909317bb..e76ef4f27423 100644 --- a/src/memory-broker/server.test.ts +++ b/src/memory-broker/server.test.ts @@ -1,5 +1,6 @@ import { randomBytes } from "node:crypto"; import { mkdtemp, rm, stat } from "node:fs/promises"; +import net from "node:net"; import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -7,6 +8,7 @@ import { requestJsonlSocket } from "../infra/jsonl-socket.js"; import { createMemoryBrokerClient } from "./client.js"; import { createMemoryBrokerEnvelope, + MEMORY_BROKER_MAXIMUM_CANONICAL_DEPTH, type MemoryBrokerAuthorizationBinding, type MemoryBrokerRequest, } from "./protocol.js"; @@ -19,6 +21,7 @@ const binding: MemoryBrokerAuthorizationBinding = { runId: "run-a", contextFingerprint: "context-a", subjectRevision: "subject-a", + actor: { kind: "principal", actorKind: "human", principalId: "alice" }, actorRevision: "actor-a", capabilitySnapshotId: "capability-a", policyRevision: "policy-a", @@ -32,7 +35,7 @@ let server: MemoryBrokerServer | undefined; type StartOptions = Partial< Pick< Parameters[0], - "handler" | "maximumPending" | "maximumRunning" + "handler" | "maximumPending" | "maximumRunning" | "maximumConnections" | "preauthIdleTimeoutMs" > >; @@ -77,16 +80,35 @@ function frame(overrides: Partial<{ envelope: unknown; request: unknown; nonce: return { envelope, request, ...overrides }; } -async function send(socketPath: string, value: unknown, options: { keepWriteOpen?: boolean } = {}) { +async function send( + socketPath: string, + value: unknown, + options: { keepWriteOpen?: boolean; timeoutMs?: number } = {}, +) { return await requestJsonlSocket({ socketPath, - requestLine: JSON.stringify(value), - timeoutMs: 5_000, + requestLine: typeof value === "string" ? value : JSON.stringify(value), + timeoutMs: options.timeoutMs ?? 5_000, ...options, accept: (response) => response as { ok: boolean; value?: unknown; error?: string }, }); } +async function openPartialSocket(socketPath: string): Promise { + return await new Promise((resolve, reject) => { + const socket = net.createConnection(socketPath); + socket.once("connect", () => resolve(socket)); + socket.once("error", reject); + }); +} + +async function waitForSocketClose(socket: net.Socket): Promise { + if (socket.destroyed) { + return; + } + await new Promise((resolve) => socket.once("close", () => resolve())); +} + describe("memory broker server", () => { it("keeps the local socket private and admits one signed request", async () => { const socketPath = await start(); @@ -280,6 +302,124 @@ describe("memory broker server", () => { await expect(handlerAborted).resolves.toBeUndefined(); }); + it("reclaims a cancelled queued request before it can exhaust the broker admission limit", async () => { + let release: (() => void) | undefined; + let started: (() => void) | undefined; + const handlerStarted = new Promise((resolve) => { + started = resolve; + }); + const handlerRelease = new Promise((resolve) => { + release = resolve; + }); + const socketPath = await start({ + maximumPending: 1, + maximumRunning: 1, + handler: async () => { + started?.(); + await handlerRelease; + return { ok: true }; + }, + }); + const first = send(socketPath, frame({ nonce: "nonce-running" }), { keepWriteOpen: true }); + await handlerStarted; + + const controller = new AbortController(); + const client = createMemoryBrokerClient({ + socketPath, + brokerId: "broker-a", + brokerEpoch: "epoch-a", + secret, + nonce: () => "nonce-cancelled-queued", + }); + const cancelled = client.request({ + binding, + method: "memory.search", + payload: { query: "queued then cancelled" }, + expiresAtMs: Date.now() + 60_000, + signal: controller.signal, + }); + // Let the client finish its connect/write turn, then prove it holds the sole pending slot. + await new Promise((resolve) => setTimeout(resolve, 20)); + await expect(send(socketPath, frame({ nonce: "nonce-before-cancel" }))).resolves.toEqual({ + ok: false, + error: "busy", + }); + + controller.abort(); + await expect(cancelled).resolves.toBeUndefined(); + const reclaimed = send(socketPath, frame({ nonce: "nonce-after-cancel" }), { + keepWriteOpen: true, + }); + // The admission decision must happen while the first handler is still saturated. Without + // queue-owned cancellation, this request is synchronously rejected as busy. + await new Promise((resolve) => setTimeout(resolve, 20)); + release?.(); + await expect(first).resolves.toEqual({ ok: true, value: { ok: true } }); + await expect(reclaimed).resolves.toEqual({ ok: true, value: { ok: true } }); + }); + + it("expires queued work without waiting for a running handler to drain", async () => { + let release: (() => void) | undefined; + let started: (() => void) | undefined; + const handlerStarted = new Promise((resolve) => { + started = resolve; + }); + const handlerRelease = new Promise((resolve) => { + release = resolve; + }); + const socketPath = await start({ + maximumPending: 1, + maximumRunning: 1, + handler: async () => { + started?.(); + await handlerRelease; + return { ok: true }; + }, + }); + const first = send(socketPath, frame({ nonce: "nonce-running" }), { keepWriteOpen: true }); + await handlerStarted; + + const expiresAtMs = Date.now() + 100; + const expired = send( + socketPath, + frame({ + nonce: "nonce-queued-expiry", + envelope: createMemoryBrokerEnvelope({ + secret, + brokerId: "broker-a", + brokerEpoch: "epoch-a", + binding, + nonce: "nonce-queued-expiry", + request, + issuedAtMs: Date.now(), + expiresAtMs, + }), + }), + { keepWriteOpen: true, timeoutMs: 1_000 }, + ); + await expect(expired).resolves.toEqual({ ok: false, error: "cancelled" }); + + const reclaimed = send(socketPath, frame({ nonce: "nonce-after-expiry" }), { + keepWriteOpen: true, + }); + release?.(); + await expect(first).resolves.toEqual({ ok: true, value: { ok: true } }); + await expect(reclaimed).resolves.toEqual({ ok: true, value: { ok: true } }); + }); + + it("rejects a deeply nested forged request without unwinding the socket callback", async () => { + const socketPath = await start(); + const envelope = frame({ nonce: "nonce-deep-forgery" }).envelope; + const depth = MEMORY_BROKER_MAXIMUM_CANONICAL_DEPTH * 512; + const payload = `${"[".repeat(depth)}0${"]".repeat(depth)}`; + const rawFrame = `{"envelope":${JSON.stringify(envelope)},"request":{"method":"memory.search","payload":${payload}}}`; + + await expect(send(socketPath, rawFrame, { timeoutMs: 1_000 })).resolves.toEqual({ + ok: false, + error: "unauthorized", + }); + }); + it("cancels an in-flight handler at the signed request deadline", async () => { let started: (() => void) | undefined; const handlerStarted = new Promise((resolve) => { @@ -314,4 +454,66 @@ describe("memory broker server", () => { await handlerStarted; await expect(pending).resolves.toEqual({ ok: false, error: "cancelled" }); }); + + it("acknowledges a durable mutation that commits as its reply deadline expires", async () => { + let started: (() => void) | undefined; + const handlerStarted = new Promise((resolve) => { + started = resolve; + }); + const socketPath = await start({ + handler: async ({ signal }) => { + started?.(); + await new Promise((resolve) => { + signal.addEventListener("abort", () => resolve(), { once: true }); + }); + return { status: "committed" }; + }, + }); + const mutationRequest: MemoryBrokerRequest = { + method: "memory.write", + payload: { mutation: "durably-activated" }, + }; + const deadline = Date.now() + 100; + const pending = send( + socketPath, + frame({ + nonce: "nonce-committed-after-deadline", + request: mutationRequest, + envelope: createMemoryBrokerEnvelope({ + secret, + brokerId: "broker-a", + brokerEpoch: "epoch-a", + binding, + nonce: "nonce-committed-after-deadline", + request: mutationRequest, + issuedAtMs: Date.now(), + expiresAtMs: deadline, + }), + }), + ); + await handlerStarted; + await expect(pending).resolves.toEqual({ ok: true, value: { status: "committed" } }); + }); + + it("bounds unauthenticated partial-frame sockets and reclaims their admission slots", async () => { + const socketPath = await start({ maximumConnections: 1, preauthIdleTimeoutMs: 40 }); + const slowloris = await openPartialSocket(socketPath); + slowloris.write('{"envelope":'); + + await expect(send(socketPath, frame({ nonce: "nonce-over-limit" }))).resolves.toBeNull(); + await waitForSocketClose(slowloris); + await expect(send(socketPath, frame({ nonce: "nonce-after-idle" }))).resolves.toMatchObject({ + ok: true, + }); + }); + + it("tears down partial-frame sockets before broker shutdown", async () => { + const socketPath = await start({ preauthIdleTimeoutMs: 60_000 }); + const slowloris = await openPartialSocket(socketPath); + slowloris.write('{"envelope":'); + + await expect(server!.close()).resolves.toBeUndefined(); + await waitForSocketClose(slowloris); + await expect(server!.close()).resolves.toBeUndefined(); + }); }); diff --git a/src/memory-broker/server.ts b/src/memory-broker/server.ts index 7e265e22f1fd..0c2022279e09 100644 --- a/src/memory-broker/server.ts +++ b/src/memory-broker/server.ts @@ -3,13 +3,15 @@ import net from "node:net"; import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { MemoryBrokerNonceLedger, + isMemoryBrokerEnvelope, verifyMemoryBrokerEnvelope, type MemoryBrokerAuthorizationBinding, - type MemoryBrokerEnvelope, type MemoryBrokerRequest, } from "./protocol.js"; export const MEMORY_BROKER_MAXIMUM_REQUEST_BYTES = 1_048_576; +export const MEMORY_BROKER_MAXIMUM_CONNECTIONS = 32; +export const MEMORY_BROKER_PREAUTH_IDLE_TIMEOUT_MS = 5_000; type MemoryBrokerWireRequest = Readonly<{ envelope: unknown; @@ -34,6 +36,7 @@ type PendingMemoryBrokerRequest = Readonly<{ deadlineMs: number; execute: () => Promise; rejectBusy: () => void; + rejectCancelled: () => void; }>; /** @@ -43,6 +46,7 @@ type PendingMemoryBrokerRequest = Readonly<{ class MemoryBrokerAdmissionQueue { private readonly pending: PendingMemoryBrokerRequest[] = []; private readonly idleWaiters = new Set<() => void>(); + private deadlineTimer: ReturnType | undefined; private running = 0; private accepting = true; @@ -61,17 +65,21 @@ class MemoryBrokerAdmissionQueue { } } - submit(request: PendingMemoryBrokerRequest): void { + submit(request: PendingMemoryBrokerRequest): () => void { if ( !this.accepting || - request.deadlineMs <= this.now() || (this.running >= this.maximumRunning && this.pending.length >= this.maximumPending) ) { request.rejectBusy(); - return; + return () => {}; + } + if (request.deadlineMs <= this.now()) { + request.rejectCancelled(); + return () => {}; } this.pending.push(request); this.drain(); + return () => this.cancel(request); } async quiesce(): Promise { @@ -101,7 +109,7 @@ class MemoryBrokerAdmissionQueue { while (this.running < this.maximumRunning && this.pending.length > 0) { const next = this.pending.shift()!; if (next.deadlineMs <= this.now()) { - next.rejectBusy(); + next.rejectCancelled(); continue; } this.running += 1; @@ -111,8 +119,50 @@ class MemoryBrokerAdmissionQueue { this.notifyIdle(); }); } + this.schedulePendingDeadline(); this.notifyIdle(); } + + private cancel(request: PendingMemoryBrokerRequest): void { + const index = this.pending.indexOf(request); + if (index === -1) { + return; + } + this.pending.splice(index, 1); + this.schedulePendingDeadline(); + this.drain(); + } + + private schedulePendingDeadline(): void { + if (this.deadlineTimer) { + clearTimeout(this.deadlineTimer); + this.deadlineTimer = undefined; + } + const nextDeadline = this.pending.reduce( + (earliest, request) => + earliest === undefined || request.deadlineMs < earliest ? request.deadlineMs : earliest, + undefined, + ); + if (nextDeadline === undefined) { + return; + } + this.deadlineTimer = setTimeout( + () => { + this.deadlineTimer = undefined; + const now = this.now(); + for (let index = this.pending.length - 1; index >= 0; index -= 1) { + const request = this.pending[index]; + if (request.deadlineMs <= now) { + this.pending.splice(index, 1); + request.rejectCancelled(); + } + } + this.drain(); + }, + Math.max(1, nextDeadline - this.now()), + ); + this.deadlineTimer.unref?.(); + } } function parseRequest(value: unknown): MemoryBrokerWireRequest | undefined { @@ -140,15 +190,12 @@ function writeResponse(socket: net.Socket, response: MemoryBrokerWireResponse): } } -function asEnvelope(value: unknown): MemoryBrokerEnvelope | undefined { - if ( - !isRecord(value) || - typeof value.nonce !== "string" || - typeof value.expiresAtMs !== "number" - ) { - return undefined; - } - return value as unknown as MemoryBrokerEnvelope; +/** + * A successful mutation handler has crossed its durable activation boundary. Its response must + * remain committed even if the caller disconnects while the broker is serializing the reply. + */ +function isDurableMemoryMutation(request: MemoryBrokerRequest): boolean { + return request.method === "memory.write" || request.method === "memory.import"; } export type MemoryBrokerServer = Readonly<{ @@ -174,53 +221,96 @@ export async function startMemoryBrokerServer(params: { nonceCapacity?: number; maximumPending?: number; maximumRunning?: number; + /** Bound unauthenticated sockets before they can consume a broker worker slot. */ + maximumConnections?: number; + /** A partial frame is untrusted admission state, never an indefinitely retained request. */ + preauthIdleTimeoutMs?: number; maximumRequestBytes?: number; now?: () => number; }): Promise { const now = params.now ?? Date.now; const maximumRequestBytes = params.maximumRequestBytes ?? MEMORY_BROKER_MAXIMUM_REQUEST_BYTES; + const maximumConnections = params.maximumConnections ?? MEMORY_BROKER_MAXIMUM_CONNECTIONS; + const preauthIdleTimeoutMs = params.preauthIdleTimeoutMs ?? MEMORY_BROKER_PREAUTH_IDLE_TIMEOUT_MS; + if ( + !Number.isSafeInteger(maximumConnections) || + maximumConnections < 1 || + !Number.isSafeInteger(preauthIdleTimeoutMs) || + preauthIdleTimeoutMs < 1 + ) { + throw new Error("memory broker connection limits are invalid"); + } const nonceLedger = new MemoryBrokerNonceLedger(params.nonceCapacity ?? 1_024); const queue = new MemoryBrokerAdmissionQueue( params.maximumPending ?? 128, params.maximumRunning ?? 8, now, ); + const sockets = new Set(); const server = net.createServer({ allowHalfOpen: true }, (socket) => { - let buffer = Buffer.alloc(0); + // A client that has not completed one bounded signed frame has no broker authority. Keep its + // footprint bounded before parsing so partial-frame clients cannot exhaust the broker. + if (sockets.size >= maximumConnections) { + socket.destroy(); + return; + } + sockets.add(socket); + // Do not repeatedly concatenate an attacker-controlled partial frame: a byte-at-a-time + // slowloris would turn that into quadratic copying before the absolute admission deadline. + const frameChunks: Buffer[] = []; + let frameByteLength = 0; let consumed = false; const requestAbort = new AbortController(); + const preauthIdleTimer = setTimeout(() => { + if (!consumed) { + consumed = true; + requestAbort.abort(); + socket.destroy(); + } + }, preauthIdleTimeoutMs); + preauthIdleTimer.unref?.(); socket.on("end", () => requestAbort.abort()); - socket.on("close", () => requestAbort.abort()); + socket.once("close", () => { + clearTimeout(preauthIdleTimer); + requestAbort.abort(); + sockets.delete(socket); + }); socket.on("error", () => requestAbort.abort()); socket.on("data", (chunk: Buffer) => { if (consumed) { socket.destroy(); return; } - if (buffer.byteLength + chunk.byteLength > maximumRequestBytes) { + if (frameByteLength + chunk.byteLength > maximumRequestBytes) { consumed = true; writeResponse(socket, { ok: false, error: "invalid-request" }); return; } - buffer = Buffer.concat([buffer, chunk]); - const newline = buffer.indexOf(0x0a); + frameChunks.push(chunk); + frameByteLength += chunk.byteLength; + // Every prior chunk has been checked and contains no newline, so only the latest chunk + // needs scanning until there is one complete frame to concatenate exactly once. + const newline = chunk.indexOf(0x0a); if (newline === -1) { return; } consumed = true; - if (buffer.subarray(newline + 1).some((byte) => byte !== 0x0a && byte !== 0x0d)) { + clearTimeout(preauthIdleTimer); + const buffer = Buffer.concat(frameChunks, frameByteLength); + const frameNewline = buffer.indexOf(0x0a); + if (buffer.subarray(frameNewline + 1).some((byte) => byte !== 0x0a && byte !== 0x0d)) { writeResponse(socket, { ok: false, error: "invalid-request" }); return; } let frame: MemoryBrokerWireRequest | undefined; try { - frame = parseRequest(JSON.parse(buffer.subarray(0, newline).toString("utf8"))); + frame = parseRequest(JSON.parse(buffer.subarray(0, frameNewline).toString("utf8"))); } catch { frame = undefined; } const request = frame ? parseMemoryBrokerRequest(frame.request) : undefined; - const envelope = frame ? asEnvelope(frame.envelope) : undefined; - if (!request || !envelope) { + const envelope = frame?.envelope; + if (!request || !isMemoryBrokerEnvelope(envelope)) { writeResponse(socket, { ok: false, error: "invalid-request" }); return; } @@ -246,9 +336,10 @@ export async function startMemoryBrokerServer(params: { writeResponse(socket, { ok: false, error: "replayed" }); return; } - queue.submit({ + const cancelPending = queue.submit({ deadlineMs: envelope.expiresAtMs, rejectBusy: () => writeResponse(socket, { ok: false, error: "busy" }), + rejectCancelled: () => writeResponse(socket, { ok: false, error: "cancelled" }), execute: async () => { if (requestAbort.signal.aborted || now() >= envelope.expiresAtMs) { writeResponse(socket, { ok: false, error: "cancelled" }); @@ -264,18 +355,31 @@ export async function startMemoryBrokerServer(params: { request, signal: requestAbort.signal, }); - if (requestAbort.signal.aborted || now() >= envelope.expiresAtMs) { + if ( + !isDurableMemoryMutation(request) && + (requestAbort.signal.aborted || now() >= envelope.expiresAtMs) + ) { writeResponse(socket, { ok: false, error: "cancelled" }); return; } writeResponse(socket, { ok: true, value }); } catch { - writeResponse(socket, { ok: false, error: "failed" }); + writeResponse(socket, { + ok: false, + error: + requestAbort.signal.aborted || now() >= envelope.expiresAtMs + ? "cancelled" + : "failed", + }); } finally { clearTimeout(deadline); } }, }); + requestAbort.signal.addEventListener("abort", cancelPending, { once: true }); + if (requestAbort.signal.aborted) { + cancelPending(); + } }); }); await unlink(params.socketPath).catch((error: unknown) => { @@ -291,20 +395,29 @@ export async function startMemoryBrokerServer(params: { }); }); await chmod(params.socketPath, 0o600); + let closePromise: Promise | undefined; return Object.freeze({ socketPath: params.socketPath, brokerEpoch: params.brokerEpoch, quiesce: () => queue.quiesce(), resume: () => queue.resume(), - close: async () => { - await new Promise((resolve, reject) => { - server.close((error) => (error ? reject(error) : resolve())); - }); - await unlink(params.socketPath).catch((error: unknown) => { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { - throw error; + close: () => { + closePromise ??= (async () => { + // net.Server.close waits for every accepted socket. Destroy incomplete and in-flight + // connections first so shutdown cannot be held hostage by a slowloris or stalled client. + for (const socket of sockets) { + socket.destroy(); } - }); + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + await unlink(params.socketPath).catch((error: unknown) => { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + }); + })(); + return closePromise; }, }); } diff --git a/src/memory-broker/test-handler.mjs b/src/memory-broker/test-handler.mjs index 98e5ead58660..81275827bd86 100644 --- a/src/memory-broker/test-handler.mjs +++ b/src/memory-broker/test-handler.mjs @@ -1,3 +1,12 @@ +let initializedAgentIds = []; + +export async function initializeMemoryBroker({ agentIds }) { + if (agentIds.includes("fail-startup")) { + throw new Error("test broker startup recovery failed"); + } + initializedAgentIds = [...agentIds]; +} + export function createMemoryBrokerHandler() { return async ({ binding, request }) => { if (request.method === "memory.crash") { @@ -6,6 +15,20 @@ export function createMemoryBrokerHandler() { setImmediate(() => process.exit(1)); return await new Promise(() => {}); } + if (request.method === "memory.kill") { + // An external SIGKILL records `signalCode`, not `exitCode`. The parent must still retire + // this process instead of waiting for an exit event that has already happened. + setImmediate(() => process.kill(process.pid, "SIGKILL")); + return await new Promise(() => {}); + } + if (request.method === "memory.hang") { + // This intentionally ignores the broker AbortSignal. Maintenance must retire the child + // rather than leave an admission-closed but otherwise healthy process behind. + return await new Promise(() => {}); + } + if (request.method === "memory.startup") { + return { agentIds: initializedAgentIds }; + } return request.method === "memory.environment" ? { parentSecret: process.env.OPENCLAW_MEMORY_BROKER_TEST_SECRET ? "present" : "absent" } : { agentId: binding.agentId, method: request.method }; diff --git a/src/memory-broker/test-malicious-client.mjs b/src/memory-broker/test-malicious-client.mjs index 73e5fd3c58bf..f5cf529e0a2a 100644 --- a/src/memory-broker/test-malicious-client.mjs +++ b/src/memory-broker/test-malicious-client.mjs @@ -18,6 +18,7 @@ const envelope = { runId: "run-b", contextFingerprint: "context-b", subjectRevision: "subject-b", + actor: { kind: "principal", actorKind: "human", principalId: "mallory" }, actorRevision: "actor-b", capabilitySnapshotId: "capability-b", policyRevision: "policy-b", diff --git a/src/node-host/node-worker-capacity.ts b/src/node-host/node-worker-capacity.ts index bea36abf5766..abc465dd493b 100644 --- a/src/node-host/node-worker-capacity.ts +++ b/src/node-host/node-worker-capacity.ts @@ -9,10 +9,7 @@ import { type NodeWorkerLaunchClaimResult, type NodeWorkerLaunchReceipt, } from "./node-worker-launch-store.js"; -import { - inspectNodeWorkerProcessIdentity, - type NodeWorkerProcessIdentity, -} from "./node-worker-process-identity.js"; +import { type NodeWorkerProcessIdentity } from "./node-worker-process-identity.js"; const DEFAULT_WORKER_CAPACITY = 2; const DEFAULT_CAPACITY_WAIT_MS = 10_000; @@ -69,28 +66,14 @@ export class NodeWorkerCapacity { } async initialize( - recoverRunning: (receipt: NodeWorkerLaunchReceipt) => Promise, + recoverNonterminal: (receipt: NodeWorkerLaunchReceipt) => Promise, ): Promise { this.onCapacityChanged?.(this.publishedCapacity); for (const receipt of this.store.listNonterminal()) { - if (receipt.state === "pending") { - const supervisorState = inspectNodeWorkerProcessIdentity(receipt.supervisor); - if (supervisorState === "dead" || supervisorState === "reused") { - this.finish( - { - launchId: receipt.launchId, - planHash: receipt.planHash, - supervisor: receipt.supervisor, - worker: null, - state: "interrupted", - errorText: "node host stopped before the worker launch started", - }, - false, - ); - } - continue; - } - await recoverRunning(receipt); + // Pending work can already own a staged projection or relay directory. + // Recovery therefore belongs to the supervisor, which owns those paths, + // rather than directly settling the launch slot here. + await recoverNonterminal(receipt); } this.store.pruneExpiredTerminal(); this.refresh(true); diff --git a/src/node-host/node-worker-container-process-isolation.e2e.test.ts b/src/node-host/node-worker-container-process-isolation.e2e.test.ts new file mode 100644 index 000000000000..e1bcd434b5dc --- /dev/null +++ b/src/node-host/node-worker-container-process-isolation.e2e.test.ts @@ -0,0 +1,160 @@ +import fs from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { verifyNodeWorkerContainerProjectionIsolation } from "../../test/helpers/node-worker-container-projection-isolation.js"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import type { AuthorizedMemoryVirtualFileBroker } from "../agents/memory-authorized-read-host.js"; +import { execContainer } from "../agents/sandbox/docker.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { + buildNodeWorkerContainerRunArgs, + removeOwnedNodeWorkerContainers, + resolveNodeWorkerContainerEngine, + type NodeWorkerContainerEngine, +} from "./node-worker-container-runtime.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +afterEach(() => { + closeOpenClawStateDatabaseForTest(); +}); + +function broker( + expiresAt = new Date(Date.now() + 60_000).toISOString(), +): AuthorizedMemoryVirtualFileBroker { + const files = new Map([["shared/brief.md", "only this issued virtual view"]]); + return { + view: { + version: 1, + viewId: "container-e2e-view", + planId: "container-e2e-plan", + contextFingerprint: "container-e2e-context", + revision: "container-e2e-revision", + roots: [ + { + version: 1, + mountHandle: "container-e2e-mount", + virtualRoot: "shared", + access: "read", + }, + ], + files: [ + { + version: 1, + mountHandle: "container-e2e-mount", + virtualPath: "shared/brief.md", + }, + ], + expiresAt, + }, + readFile: async (virtualPath) => files.get(virtualPath), + }; +} + +describe.runIf(process.env.OPENCLAW_PROCESS_ISOLATION_E2E === "1")( + "node worker container process isolation", + () => { + it("exposes only an immutable issued memory snapshot to the hostile worker process", async () => { + const root = tempDirs.make("node-worker-container-isolation-"); + const bundleDir = path.join(root, "bundle"); + const relayDir = path.join(root, "relay"); + const memoryDir = path.join(root, "memory"); + const workspaceDir = path.join(root, "workspace"); + const outsideArtifact = path.join(root, "host-artifact.sqlite"); + const identity = { launchId: "hostile-worker-turn", planHash: "a".repeat(64) }; + for (const directory of [bundleDir, relayDir, memoryDir, workspaceDir]) { + fs.mkdirSync(directory, { recursive: true }); + } + fs.mkdirSync(path.join(memoryDir, "shared")); + fs.writeFileSync(path.join(memoryDir, "shared", "issued.md"), "issued memory only", { + mode: 0o400, + }); + fs.chmodSync(path.join(memoryDir, "shared"), 0o500); + fs.writeFileSync(outsideArtifact, "host-only artifact"); + fs.writeFileSync( + path.join(bundleDir, "worker.mjs"), + [ + 'import fs from "node:fs";', + `const outsideArtifact = ${JSON.stringify(outsideArtifact)};`, + 'const attempt = (operation) => { try { operation(); return "allowed"; } catch (error) { return error && typeof error === "object" && "code" in error ? String(error.code) : "denied"; } };', + "const result = {", + " uid: process.getuid(),", + ' issued: fs.readFileSync("/memory/shared/issued.md", "utf8"),', + ' memoryWrite: attempt(() => fs.writeFileSync("/memory/shared/issued.md", "tampered")),', + ' rawArtifactRead: attempt(() => fs.readFileSync(outsideArtifact, "utf8")),', + ' workspaceWrite: attempt(() => fs.writeFileSync("/workspace/proof.txt", "allowed")),', + "};", + "process.stdout.write(JSON.stringify(result));", + ].join("\n"), + { mode: 0o500 }, + ); + let engine: NodeWorkerContainerEngine | undefined; + try { + engine = await resolveNodeWorkerContainerEngine(); + if (!engine) { + throw new Error( + "process-isolation proof requires an eligible Docker or Podman node host", + ); + } + await removeOwnedNodeWorkerContainers(identity, engine); + const user = process.getuid?.(); + const group = process.getgid?.(); + if (!user || !group) { + throw new Error("process-isolation proof requires a non-root POSIX test host"); + } + const args = buildNodeWorkerContainerRunArgs({ + engine, + identity, + mounts: { bundleDir, relayDir, memoryDir, workspaceDir }, + uid: user, + gid: group, + }); + const result = await execContainer(engine, args, { allowFailure: true }); + expect(result.code, result.stderr).toBe(0); + const observed: unknown = JSON.parse(result.stdout); + expect(observed).toMatchObject({ + uid: user, + issued: "issued memory only", + rawArtifactRead: expect.not.stringMatching(/^allowed$/u), + memoryWrite: expect.not.stringMatching(/^allowed$/u), + workspaceWrite: "allowed", + }); + expect(fs.readFileSync(path.join(memoryDir, "shared", "issued.md"), "utf8")).toBe( + "issued memory only", + ); + expect(fs.readFileSync(path.join(workspaceDir, "proof.txt"), "utf8")).toBe("allowed"); + } finally { + fs.chmodSync(path.join(memoryDir, "shared"), 0o700); + if (engine) { + await removeOwnedNodeWorkerContainers(identity, engine); + } + } + }); + + it("withdraws an expired signed projection-backed container worker and its mounts", async () => { + const root = tempDirs.make("node-worker-supervisor-container-e2e-"); + const outsideArtifactPath = path.join(root, "host-only-artifact.txt"); + fs.writeFileSync(outsideArtifactPath, "host-only artifact"); + const previousBrokerCredential = + process.env.OPENCLAW_PROCESS_ISOLATION_TEST_BROKER_CREDENTIAL; + process.env.OPENCLAW_PROCESS_ISOLATION_TEST_BROKER_CREDENTIAL = "test-broker-only-credential"; + try { + await verifyNodeWorkerContainerProjectionIsolation({ + root, + broker: broker(new Date(Date.now() + 15_000).toISOString()), + outsideArtifactPath, + outsideArtifactContents: "host-only artifact", + issuedVirtualPath: "shared/brief.md", + issuedContents: "only this issued virtual view", + forbiddenEnvironmentVariable: "OPENCLAW_PROCESS_ISOLATION_TEST_BROKER_CREDENTIAL", + }); + } finally { + if (previousBrokerCredential === undefined) { + delete process.env.OPENCLAW_PROCESS_ISOLATION_TEST_BROKER_CREDENTIAL; + } else { + process.env.OPENCLAW_PROCESS_ISOLATION_TEST_BROKER_CREDENTIAL = previousBrokerCredential; + } + } + }, 90_000); + }, +); diff --git a/src/node-host/node-worker-container-runtime.test.ts b/src/node-host/node-worker-container-runtime.test.ts new file mode 100644 index 000000000000..9487973a6a8b --- /dev/null +++ b/src/node-host/node-worker-container-runtime.test.ts @@ -0,0 +1,174 @@ +import fs from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; + +const containerMocks = vi.hoisted(() => ({ execContainer: vi.fn() })); + +vi.mock("../agents/sandbox/docker.js", async () => { + const actual = await vi.importActual( + "../agents/sandbox/docker.js", + ); + return { ...actual, execContainer: containerMocks.execContainer }; +}); + +import { DOCKER_SANDBOX_ENGINE } from "../agents/sandbox/docker.js"; +import { + buildNodeWorkerContainerRunArgs, + NODE_WORKER_CONTAINER_IMAGE, + NODE_WORKER_CONTAINER_MEMORY_ROOT, + NODE_WORKER_CONTAINER_RELAY_ROOT, + NODE_WORKER_CONTAINER_WORKER_ROOT, + NODE_WORKER_CONTAINER_WORKSPACE, + nodeWorkerContainerName, + removeOwnedNodeWorkerContainers, + resolveNodeWorkerContainerEngine, +} from "./node-worker-container-runtime.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); +const identity = { launchId: "turn-1", planHash: "a".repeat(64) }; + +function mounts() { + const root = tempDirs.make("node-worker-container-"); + const bundleDir = path.join(root, "bundle"); + const relayDir = path.join(root, "relay"); + const memoryDir = path.join(root, "memory"); + const workspaceDir = path.join(root, "workspace"); + for (const directory of [bundleDir, relayDir, memoryDir, workspaceDir]) { + fs.mkdirSync(directory); + } + return { bundleDir, relayDir, memoryDir, workspaceDir }; +} + +afterEach(() => { + containerMocks.execContainer.mockReset(); +}); + +describe("node worker container runtime", () => { + it("prepares the fixed worker image before admitting Docker process isolation", async () => { + containerMocks.execContainer + .mockResolvedValueOnce({ stdout: "27.5.1\n", stderr: "", code: 0 }) + .mockResolvedValueOnce({ stdout: "", stderr: "No such image", code: 1 }) + .mockResolvedValueOnce({ stdout: "pulled\n", stderr: "", code: 0 }); + + await expect(resolveNodeWorkerContainerEngine()).resolves.toBe(DOCKER_SANDBOX_ENGINE); + + expect(containerMocks.execContainer.mock.calls).toEqual([ + [DOCKER_SANDBOX_ENGINE, ["info", "--format", "{{.ServerVersion}}"], expect.any(Object)], + [ + DOCKER_SANDBOX_ENGINE, + ["image", "inspect", "--format", "{{.Id}}", NODE_WORKER_CONTAINER_IMAGE], + expect.any(Object), + ], + [DOCKER_SANDBOX_ENGINE, ["pull", NODE_WORKER_CONTAINER_IMAGE], expect.any(Object)], + ]); + expect(containerMocks.execContainer.mock.calls[2]?.[2]).toEqual( + expect.objectContaining({ signal: expect.any(AbortSignal) }), + ); + }); + + it("does not advertise process isolation when the fixed worker image cannot be prepared", async () => { + containerMocks.execContainer + .mockResolvedValueOnce({ stdout: "27.5.1\n", stderr: "", code: 0 }) + .mockResolvedValueOnce({ stdout: "", stderr: "No such image", code: 1 }) + .mockResolvedValueOnce({ stdout: "", stderr: "registry unavailable", code: 1 }) + .mockResolvedValueOnce({ stdout: "", stderr: "podman unavailable", code: 1 }); + + await expect(resolveNodeWorkerContainerEngine()).resolves.toBeUndefined(); + }); + + it("builds a closed non-root container policy with only the worker mounts", () => { + const paths = mounts(); + const args = buildNodeWorkerContainerRunArgs({ + engine: DOCKER_SANDBOX_ENGINE, + identity, + mounts: paths, + uid: 501, + gid: 20, + }); + + expect(args).toEqual( + expect.arrayContaining([ + "--interactive", + "--network", + "none", + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--pids-limit", + "128", + "--memory", + "512m", + "--cpus", + "1", + "--user", + "501:20", + "--workdir", + NODE_WORKER_CONTAINER_WORKSPACE, + NODE_WORKER_CONTAINER_IMAGE, + ]), + ); + const mountValues = args.flatMap((value, index) => + value === "--mount" ? [args[index + 1]] : [], + ); + expect(mountValues).toEqual([ + `type=bind,src=${fs.realpathSync(paths.bundleDir)},dst=${NODE_WORKER_CONTAINER_WORKER_ROOT},readonly`, + `type=bind,src=${fs.realpathSync(paths.relayDir)},dst=${NODE_WORKER_CONTAINER_RELAY_ROOT},readonly`, + `type=bind,src=${fs.realpathSync(paths.memoryDir)},dst=${NODE_WORKER_CONTAINER_MEMORY_ROOT},readonly`, + `type=bind,src=${fs.realpathSync(paths.workspaceDir)},dst=${NODE_WORKER_CONTAINER_WORKSPACE}`, + ]); + expect(args).not.toContain("--privileged"); + expect(args).not.toContain("--network=host"); + }); + + it("does not claim cleanup after container enumeration fails", async () => { + containerMocks.execContainer.mockRejectedValueOnce(new Error("Docker daemon unavailable")); + + await expect(removeOwnedNodeWorkerContainers(identity, DOCKER_SANDBOX_ENGINE)).rejects.toThrow( + "Docker daemon unavailable", + ); + expect(containerMocks.execContainer).toHaveBeenCalledWith( + DOCKER_SANDBOX_ENGINE, + expect.any(Array), + expect.objectContaining({ signal: expect.any(AbortSignal) }), + ); + }); + + it("rejects a symlinked memory mount before it reaches the container engine", () => { + const paths = mounts(); + const symlink = path.join(path.dirname(paths.memoryDir), "memory-link"); + fs.symlinkSync(paths.memoryDir, symlink); + + expect(() => + buildNodeWorkerContainerRunArgs({ + engine: DOCKER_SANDBOX_ENGINE, + identity, + mounts: { ...paths, memoryDir: symlink }, + uid: 501, + gid: 20, + }), + ).toThrow("memory projection mount must be a real local directory"); + }); + + it("keeps cleanup open when a label-verified container cannot be removed", async () => { + const containerId = "b".repeat(64); + containerMocks.execContainer + .mockResolvedValueOnce({ stdout: `${containerId}\n`, stderr: "", code: 0 }) + .mockResolvedValueOnce({ + stdout: `/${nodeWorkerContainerName(identity)}\nv1\n${identity.launchId}\n${identity.planHash}\n`, + stderr: "", + code: 0, + }) + .mockRejectedValueOnce(new Error("permission denied")) + .mockResolvedValueOnce({ stdout: `${containerId}\n`, stderr: "", code: 0 }); + + await expect(removeOwnedNodeWorkerContainers(identity, DOCKER_SANDBOX_ENGINE)).rejects.toThrow( + "Docker could not remove a node worker container", + ); + for (const [, , options] of containerMocks.execContainer.mock.calls) { + expect(options).toEqual(expect.objectContaining({ signal: expect.any(AbortSignal) })); + } + }); +}); diff --git a/src/node-host/node-worker-container-runtime.ts b/src/node-host/node-worker-container-runtime.ts new file mode 100644 index 000000000000..26e702bb032c --- /dev/null +++ b/src/node-host/node-worker-container-runtime.ts @@ -0,0 +1,372 @@ +import fs from "node:fs"; +import path from "node:path"; +import { createHash } from "node:crypto"; +import { + DOCKER_SANDBOX_ENGINE, + PODMAN_SANDBOX_ENGINE, + execContainer, + type SandboxContainerEngine, +} from "../agents/sandbox/docker.js"; + +/** + * Process-isolated workers deliberately use one pinned runtime instead of the + * configurable tool sandbox image. The image is part of the node-host TCB. + */ +export const NODE_WORKER_CONTAINER_IMAGE = + "docker.io/library/node:24-bookworm-slim@sha256:6f7b03f7c2c8e2e784dcf9295400527b9b1270fd37b7e9a7285cf83b6951452d"; +export const NODE_WORKER_CONTAINER_WORKSPACE = "/workspace"; +export const NODE_WORKER_CONTAINER_MEMORY_ROOT = "/memory"; +export const NODE_WORKER_CONTAINER_WORKER_ROOT = "/opt/openclaw/worker"; +export const NODE_WORKER_CONTAINER_RELAY_ROOT = "/run/openclaw"; +export const NODE_WORKER_CONTAINER_RELAY_SOCKET = `${NODE_WORKER_CONTAINER_RELAY_ROOT}/gateway.sock`; +export const NODE_WORKER_CONTAINER_SHIM_FLAG = "--internal-worker-container-shim"; + +const NODE_WORKER_CONTAINER_LABEL = "openclaw.node-worker-container"; +const NODE_WORKER_CONTAINER_LAUNCH_LABEL = "openclaw.node-worker-launch"; +const NODE_WORKER_CONTAINER_PLAN_LABEL = "openclaw.node-worker-plan"; +const NODE_WORKER_CONTAINER_FORMAT = "v1"; +const CONTAINER_NAME_MAX_CHARS = 96; +const NODE_WORKER_CONTAINER_ENGINE_TIMEOUT_MS = 5_000; +const NODE_WORKER_CONTAINER_IMAGE_PREPARE_TIMEOUT_MS = 2 * 60 * 1_000; + +export type NodeWorkerContainerEngine = Extract; + +export type NodeWorkerContainerIdentity = Readonly<{ + launchId: string; + planHash: string; +}>; + +export type NodeWorkerContainerMounts = Readonly<{ + bundleDir: string; + relayDir: string; + memoryDir: string; + workspaceDir: string; +}>; + +export function nodeWorkerContainerEngineFor( + id: NodeWorkerContainerEngine["id"], +): NodeWorkerContainerEngine { + return id === "docker" ? DOCKER_SANDBOX_ENGINE : PODMAN_SANDBOX_ENGINE; +} + +/** + * Probe and cleanup commands must not turn a wedged engine into an unbounded + * worker lifecycle operation. `run` is intentionally excluded: it is the + * worker lifetime itself and is owned by the shim's IPC lease. + */ +function runNodeWorkerContainerControlCommand( + engine: NodeWorkerContainerEngine, + args: string[], + options?: { allowFailure?: boolean }, +) { + return execContainer(engine, args, { + ...options, + signal: AbortSignal.timeout(NODE_WORKER_CONTAINER_ENGINE_TIMEOUT_MS), + }); +} + +/** + * The worker image is part of the fixed container TCB. Pull it before this + * host advertises process isolation: Docker's implicit cold pull can outlive + * the short exact-container start probe and otherwise turns a ready node into + * a failed first turn. + */ +async function prepareNodeWorkerContainerImage(engine: NodeWorkerContainerEngine): Promise { + const existing = await runNodeWorkerContainerControlCommand( + engine, + ["image", "inspect", "--format", "{{.Id}}", NODE_WORKER_CONTAINER_IMAGE], + { allowFailure: true }, + ); + if (existing.code === 0 && existing.stdout.trim()) { + return true; + } + const pulled = await execContainer(engine, ["pull", NODE_WORKER_CONTAINER_IMAGE], { + allowFailure: true, + signal: AbortSignal.timeout(NODE_WORKER_CONTAINER_IMAGE_PREPARE_TIMEOUT_MS), + }); + return pulled.code === 0; +} + +/** A mode-0600 relay is usable only by a concrete, non-root POSIX user mapping. */ +export function resolveNodeWorkerContainerUser(): { uid: number; gid: number } | undefined { + if ( + process.platform === "win32" || + typeof process.getuid !== "function" || + typeof process.getgid !== "function" + ) { + return undefined; + } + const uid = process.getuid(); + const gid = process.getgid(); + return uid === 0 || gid === 0 ? undefined : { uid, gid }; +} + +function requireOwnedAbsolutePath(value: string, label: string): string { + if (!path.isAbsolute(value) || value.includes("\0") || value.includes("\n") || value.includes("\r")) { + throw new Error(`node worker container ${label} must be an absolute local path`); + } + const lexicalStats = fs.lstatSync(value); + if (lexicalStats.isSymbolicLink() || !lexicalStats.isDirectory()) { + throw new Error(`node worker container ${label} must be a real local directory`); + } + const resolved = fs.realpathSync.native(value); + const stats = fs.lstatSync(resolved); + if (stats.isSymbolicLink() || !stats.isDirectory()) { + throw new Error(`node worker container ${label} must be a real local directory`); + } + return resolved; +} + +function formatMount(source: string, target: string, readOnly: boolean): string { + return `type=bind,src=${source},dst=${target}${readOnly ? ",readonly" : ""}`; +} + +function safeContainerNameComponent(value: string): string { + const normalized = value.toLowerCase().replaceAll(/[^a-z0-9_.-]/gu, "-"); + const suffix = createHash("sha256").update(value).digest("hex").slice(0, 16); + const prefix = normalized.replaceAll(/^-+|[-.]+$/gu, "").slice(0, 48) || "worker"; + return `${prefix}-${suffix}`; +} + +/** A deterministic but collision-resistant container name, never a caller-provided Docker argument. */ +export function nodeWorkerContainerName(identity: NodeWorkerContainerIdentity): string { + const value = `openclaw-worker-${safeContainerNameComponent(identity.launchId)}-${identity.planHash.slice(0, 16)}`; + return value.slice(0, CONTAINER_NAME_MAX_CHARS); +} + +function assertContainerIdentity(identity: NodeWorkerContainerIdentity): void { + if ( + identity.launchId.length === 0 || + identity.launchId.length > 256 || + identity.launchId.includes("\0") || + !/^[a-f0-9]{64}$/u.test(identity.planHash) + ) { + throw new Error("invalid node worker container identity"); + } +} + +export function buildNodeWorkerContainerRunArgs(params: { + engine: NodeWorkerContainerEngine; + identity: NodeWorkerContainerIdentity; + mounts: NodeWorkerContainerMounts; + uid: number; + gid: number; +}): string[] { + assertContainerIdentity(params.identity); + if ( + !Number.isSafeInteger(params.uid) || + params.uid <= 0 || + !Number.isSafeInteger(params.gid) || + params.gid <= 0 + ) { + throw new Error("node worker container requires a concrete non-root host user mapping"); + } + const bundleDir = requireOwnedAbsolutePath(params.mounts.bundleDir, "bundle mount"); + const relayDir = requireOwnedAbsolutePath(params.mounts.relayDir, "relay mount"); + const memoryDir = requireOwnedAbsolutePath(params.mounts.memoryDir, "memory projection mount"); + const workspaceDir = requireOwnedAbsolutePath(params.mounts.workspaceDir, "workspace mount"); + const name = nodeWorkerContainerName(params.identity); + + // This is intentionally a closed policy. The Gateway cannot pass a host path, + // image, environment value, container option, or projection mount into it. + return [ + "run", + "--interactive", + "--name", + name, + "--label", + `${NODE_WORKER_CONTAINER_LABEL}=${NODE_WORKER_CONTAINER_FORMAT}`, + "--label", + `${NODE_WORKER_CONTAINER_LAUNCH_LABEL}=${params.identity.launchId}`, + "--label", + `${NODE_WORKER_CONTAINER_PLAN_LABEL}=${params.identity.planHash}`, + "--network", + "none", + "--read-only", + "--tmpfs", + "/tmp:rw,nosuid,nodev,noexec,size=64m", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--pids-limit", + "128", + "--memory", + "512m", + "--cpus", + "1", + "--user", + `${params.uid}:${params.gid}`, + "--workdir", + NODE_WORKER_CONTAINER_WORKSPACE, + "--mount", + formatMount(bundleDir, NODE_WORKER_CONTAINER_WORKER_ROOT, true), + "--mount", + formatMount(relayDir, NODE_WORKER_CONTAINER_RELAY_ROOT, true), + "--mount", + formatMount(memoryDir, NODE_WORKER_CONTAINER_MEMORY_ROOT, true), + "--mount", + formatMount(workspaceDir, NODE_WORKER_CONTAINER_WORKSPACE, false), + NODE_WORKER_CONTAINER_IMAGE, + "node", + `${NODE_WORKER_CONTAINER_WORKER_ROOT}/worker.mjs`, + ]; +} + +/** + * A node advertises process isolation only after one installed engine proves it + * can service local container commands. Docker is preferred; Podman is the + * supported equivalent when Docker is not available. + */ +export async function resolveNodeWorkerContainerEngine(): Promise { + if (!resolveNodeWorkerContainerUser()) { + return undefined; + } + for (const engine of [DOCKER_SANDBOX_ENGINE, PODMAN_SANDBOX_ENGINE] as const) { + try { + const result = await runNodeWorkerContainerControlCommand( + engine, + ["info", "--format", "{{.ServerVersion}}"], + { + allowFailure: true, + }, + ); + if ( + result.code === 0 && + result.stdout.trim().length > 0 && + (await prepareNodeWorkerContainerImage(engine)) + ) { + return engine; + } + } catch { + // An absent or unavailable engine means this node is ineligible, never a host fallback. + } + } + return undefined; +} + +async function listMatchingContainerIds( + engine: NodeWorkerContainerEngine, + identity: NodeWorkerContainerIdentity, +): Promise { + const result = await runNodeWorkerContainerControlCommand( + engine, + [ + "ps", + "--all", + "--quiet", + "--filter", + `label=${NODE_WORKER_CONTAINER_LABEL}=${NODE_WORKER_CONTAINER_FORMAT}`, + "--filter", + `label=${NODE_WORKER_CONTAINER_LAUNCH_LABEL}=${identity.launchId}`, + "--filter", + `label=${NODE_WORKER_CONTAINER_PLAN_LABEL}=${identity.planHash}`, + ], + ); + return result.stdout + .split(/\r?\n/u) + .map((value) => value.trim()) + .filter((value) => /^[a-f0-9]{12,128}$/u.test(value)); +} + +async function isOwnedContainer( + engine: NodeWorkerContainerEngine, + containerId: string, + identity: NodeWorkerContainerIdentity, +): Promise { + const result = await runNodeWorkerContainerControlCommand( + engine, + [ + "inspect", + "--format", + "{{.Name}}\n{{index .Config.Labels \"openclaw.node-worker-container\"}}\n{{index .Config.Labels \"openclaw.node-worker-launch\"}}\n{{index .Config.Labels \"openclaw.node-worker-plan\"}}", + containerId, + ], + ); + const [rawName, format, launchId, planHash, ...extra] = result.stdout.trimEnd().split(/\r?\n/u); + return ( + extra.length === 0 && + rawName === `/${nodeWorkerContainerName(identity)}` && + format === NODE_WORKER_CONTAINER_FORMAT && + launchId === identity.launchId && + planHash === identity.planHash + ); +} + +/** + * Prove the exact labelled container is accepting work before the shim attests + * execution. The foreground `run` client being spawned is not that proof. + */ +export async function waitForOwnedNodeWorkerContainerRunning(params: { + engine: NodeWorkerContainerEngine; + identity: NodeWorkerContainerIdentity; + timeoutMs?: number; +}): Promise { + assertContainerIdentity(params.identity); + const deadline = Date.now() + (params.timeoutMs ?? 5_000); + while (Date.now() < deadline) { + const containerIds = await listMatchingContainerIds(params.engine, params.identity); + for (const containerId of containerIds) { + if (!(await isOwnedContainer(params.engine, containerId, params.identity))) { + continue; + } + const state = await runNodeWorkerContainerControlCommand(params.engine, [ + "inspect", + "--format", + "{{.State.Running}}", + containerId, + ]); + if (state.stdout.trim() === "true") { + return; + } + } + await new Promise((resolve) => setTimeout(resolve, 25)); + } + throw new Error(`${params.engine.displayName} did not start the exact node worker container`); +} + +async function containerStillMatches( + engine: NodeWorkerContainerEngine, + containerId: string, + identity: NodeWorkerContainerIdentity, +): Promise { + return (await listMatchingContainerIds(engine, identity)).includes(containerId); +} + +/** Removes only an exact, label-verified orphan from a prior shim/supervisor lifetime. */ +export async function removeOwnedNodeWorkerContainers( + identity: NodeWorkerContainerIdentity, + engine: NodeWorkerContainerEngine, +): Promise { + assertContainerIdentity(identity); + let removed = 0; + const containerIds = await listMatchingContainerIds(engine, identity); + for (const containerId of containerIds) { + let owned: boolean; + try { + owned = await isOwnedContainer(engine, containerId, identity); + } catch { + // An inspect race is safe only when a second enumeration proves that the + // listed container is gone. Any surviving container leaves cleanup open. + if (!(await containerStillMatches(engine, containerId, identity))) { + continue; + } + throw new Error(`${engine.displayName} could not inspect a node worker container`); + } + if (!owned) { + continue; + } + try { + await runNodeWorkerContainerControlCommand(engine, ["rm", "--force", containerId]); + removed += 1; + } catch { + // A concurrent exit is already clean. Anything still enumerable is an + // orphan and must keep the relay and launch recovery from settling. + if (!(await containerStillMatches(engine, containerId, identity))) { + continue; + } + throw new Error(`${engine.displayName} could not remove a node worker container`); + } + } + return removed; +} diff --git a/src/node-host/node-worker-container-shim.test.ts b/src/node-host/node-worker-container-shim.test.ts new file mode 100644 index 000000000000..95df1aa8d7c5 --- /dev/null +++ b/src/node-host/node-worker-container-shim.test.ts @@ -0,0 +1,156 @@ +import { fork, type ChildProcess } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { completeWorkerLaunchDescriptor } from "../worker/launch-descriptor.js"; +import { nodeWorkerContainerName } from "./node-worker-container-runtime.js"; +import { testWorkerDescriptor } from "./node-worker-supervisor.test-support.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); +const children = new Set(); + +afterEach(async () => { + for (const child of children) { + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + } + } + await Promise.all( + [...children].map( + async (child) => + await new Promise((resolve) => { + if (child.exitCode !== null || child.signalCode !== null) { + resolve(); + return; + } + child.once("exit", () => resolve()); + }), + ), + ); + children.clear(); +}); + +function writeFakeDocker(root: string, marker: string, containerName: string): string { + const binDir = path.join(root, "bin"); + fs.mkdirSync(binDir); + const executable = path.join(binDir, "docker"); + fs.writeFileSync( + executable, + `#!/usr/bin/env node + const fs = require("node:fs"); + const command = process.argv[2]; + fs.appendFileSync(${JSON.stringify(marker)}, command + "\\n"); + if (command === "ps") { + process.stdout.write("abcdefabcdef\\n"); + process.exit(0); + } + if (command === "inspect") { + const format = process.argv[process.argv.indexOf("--format") + 1]; + if (format === "{{.State.Running}}") { + process.stdout.write("true\\n"); + } else { + process.stdout.write(${JSON.stringify( + `/${containerName}\nv1\nturn-1\n${"a".repeat(64)}\n`, + )}); + } + process.exit(0); + } + if (command === "run") { + process.on("SIGTERM", () => process.exit(0)); + setInterval(() => {}, 1000); + } + `, + { mode: 0o755 }, + ); + return binDir; +} + +function waitForExit(child: ChildProcess): Promise { + return new Promise((resolve, reject) => { + if (child.exitCode !== null || child.signalCode !== null) { + resolve(); + return; + } + child.once("error", reject); + child.once("exit", () => resolve()); + }); +} + +describe("node worker container shim", () => { + it.runIf(process.platform !== "win32")( + "stops its container command when the supervisor IPC lease disconnects", + async () => { + const root = tempDirs.make("node-worker-container-shim-"); + const marker = path.join(root, "docker-calls"); + const identity = { launchId: "turn-1", planHash: "a".repeat(64) }; + const binDir = writeFakeDocker(root, marker, nodeWorkerContainerName(identity)); + const bundleDir = path.join(root, "bundle"); + const relayDir = tempDirs.make("oc-worker-relay-", "/tmp"); + const memoryDir = path.join(root, "memory"); + const workspaceDir = path.join(root, "workspace"); + for (const directory of [bundleDir, memoryDir, workspaceDir]) { + fs.mkdirSync(directory); + } + fs.writeFileSync(path.join(bundleDir, "worker.mjs"), "process.exit(0);\n"); + const descriptor = testWorkerDescriptor("/workspace"); + descriptor.assignment.memoryReadEnforced = true; + const input = { + descriptor: completeWorkerLaunchDescriptor(descriptor, { + kind: "unix" as const, + socketPath: path.join(root, "gateway.sock"), + }), + engine: "docker" as const, + identity, + mounts: { bundleDir, relayDir, memoryDir, workspaceDir }, + }; + const shimPath = path.resolve("src/node-host/node-worker-container-shim.ts"); + const child = fork(shimPath, ["--internal-worker-container-shim"], { + // tsx resolves workspace aliases through the checkout's tsconfig. The + // shim's actual container command still runs from `workspaceDir`. + cwd: process.cwd(), + env: { ...process.env, PATH: `${binDir}${path.delimiter}${process.env.PATH ?? ""}` }, + // The shim's CWD is an isolated worker fixture, so a bare package + // specifier would resolve from that fixture rather than this checkout. + execArgv: ["--import", import.meta.resolve("tsx")], + silent: true, + }); + children.add(child); + let stderr = ""; + const messages: unknown[] = []; + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString("utf8"); + }); + child.on("message", (message: unknown) => { + messages.push(message); + }); + child.stdin?.end(JSON.stringify(input)); + child.send({ type: "openclaw-worker-start-v1", ...identity }); + + try { + await vi.waitFor(() => expect(fs.readFileSync(marker, "utf8")).toContain("run\n"), { + timeout: 5_000, + }); + } catch (error) { + throw new Error( + `container shim did not launch the fake engine: ${stderr} ${JSON.stringify(messages)}`, + { cause: error }, + ); + } + await vi.waitFor(() => + expect(messages).toContainEqual({ + type: "openclaw-worker-execution-started-v1", + ...identity, + }), + ); + child.disconnect(); + + await vi.waitFor( + () => expect(child.exitCode ?? child.signalCode).not.toBeNull(), + { timeout: 5_000 }, + ); + await expect(waitForExit(child)).resolves.toBeUndefined(); + expect(fs.readFileSync(marker, "utf8")).toContain("ps\n"); + }, + ); +}); diff --git a/src/node-host/node-worker-container-shim.ts b/src/node-host/node-worker-container-shim.ts new file mode 100644 index 000000000000..d375126b7f1a --- /dev/null +++ b/src/node-host/node-worker-container-shim.ts @@ -0,0 +1,233 @@ +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { once } from "node:events"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { + completeWorkerLaunchDescriptor, + parseWorkerLaunchDescriptor, + type WorkerLaunchDescriptor, +} from "../worker/launch-descriptor.js"; +import { createWorkerIpcLifetime } from "../worker/worker-process.js"; +import type { WorkerConnectionEndpoint } from "../worker/worker-connection-endpoint.js"; +import { + buildNodeWorkerContainerRunArgs, + nodeWorkerContainerEngineFor, + NODE_WORKER_CONTAINER_SHIM_FLAG, + NODE_WORKER_CONTAINER_RELAY_SOCKET, + removeOwnedNodeWorkerContainers, + resolveNodeWorkerContainerUser, + waitForOwnedNodeWorkerContainerRunning, + type NodeWorkerContainerEngine, + type NodeWorkerContainerIdentity, + type NodeWorkerContainerMounts, +} from "./node-worker-container-runtime.js"; +import { startNodeWorkerGatewayRelay, type NodeWorkerGatewayRelay } from "./node-worker-gateway-relay.js"; + +const SHIM_INPUT_MAX_BYTES = 1024 * 1024; + +type NodeWorkerContainerShimInput = Readonly<{ + descriptor: WorkerLaunchDescriptor; + engine: NodeWorkerContainerEngine["id"]; + identity: NodeWorkerContainerIdentity; + mounts: NodeWorkerContainerMounts; +}>; + +function hasExactKeys(value: Record, keys: readonly string[]): boolean { + return Object.keys(value).length === keys.length && keys.every((key) => Object.hasOwn(value, key)); +} + +function parseContainerEngine(value: unknown): NodeWorkerContainerEngine["id"] | undefined { + return value === "docker" || value === "podman" ? value : undefined; +} + +function parseShimInput(raw: string): NodeWorkerContainerShimInput { + if (Buffer.byteLength(raw, "utf8") > SHIM_INPUT_MAX_BYTES) { + throw new Error("node worker container shim input exceeds its bound"); + } + let value: unknown; + try { + value = JSON.parse(raw) as unknown; + } catch { + throw new Error("node worker container shim input is malformed"); + } + if ( + !isRecord(value) || + !hasExactKeys(value, ["descriptor", "engine", "identity", "mounts"]) || + !isRecord(value.identity) || + !hasExactKeys(value.identity, ["launchId", "planHash"]) || + typeof value.identity.launchId !== "string" || + typeof value.identity.planHash !== "string" || + !isRecord(value.mounts) || + !hasExactKeys(value.mounts, ["bundleDir", "relayDir", "memoryDir", "workspaceDir"]) || + typeof value.mounts.bundleDir !== "string" || + typeof value.mounts.relayDir !== "string" || + typeof value.mounts.memoryDir !== "string" || + typeof value.mounts.workspaceDir !== "string" + ) { + throw new Error("node worker container shim input is invalid"); + } + const engine = parseContainerEngine(value.engine); + if (!engine) { + throw new Error("node worker container shim engine is invalid"); + } + const descriptor = parseWorkerLaunchDescriptor(value.descriptor); + if ( + descriptor.assignment.workspaceDir !== "/workspace" || + (descriptor.assignment.workerContainmentRoot !== undefined && + descriptor.assignment.workerContainmentRoot !== "/workspace") + ) { + throw new Error("node worker container descriptor escaped its fixed workspace"); + } + return { + descriptor, + engine, + identity: { launchId: value.identity.launchId, planHash: value.identity.planHash }, + mounts: { + bundleDir: value.mounts.bundleDir, + relayDir: value.mounts.relayDir, + memoryDir: value.mounts.memoryDir, + workspaceDir: value.mounts.workspaceDir, + }, + }; +} + +async function readShimInput(): Promise { + let raw = ""; + for await (const chunk of process.stdin) { + raw += String(chunk); + if (Buffer.byteLength(raw, "utf8") > SHIM_INPUT_MAX_BYTES) { + throw new Error("node worker container shim input exceeds its bound"); + } + } + return parseShimInput(raw); +} + +function currentUser(): { uid: number; gid: number } { + const user = resolveNodeWorkerContainerUser(); + if (!user) { + throw new Error("node worker container execution requires a non-root POSIX host user mapping"); + } + return user; +} + +async function closeRelay(relay: NodeWorkerGatewayRelay | undefined): Promise { + await relay?.close().catch(() => undefined); +} + +async function waitForChild(child: ChildProcessWithoutNullStreams): Promise { + const [code] = await once(child, "close"); + return typeof code === "number" ? code : 1; +} + +/** + * The shim, not Docker, is the worker PID in the durable journal. It waits on + * inherited Node IPC before creating a disposable container and removes that + * exact labelled container on any normal or signal-driven terminal path. + */ +export async function runNodeWorkerContainerShim(): Promise { + const input = await readShimInput(); + const lifetime = createWorkerIpcLifetime(); + let relay: NodeWorkerGatewayRelay | undefined; + let child: ChildProcessWithoutNullStreams | undefined; + let stopping = false; + let finishStop!: () => void; + const stopped = new Promise((resolve) => { + finishStop = resolve; + }); + const engine = nodeWorkerContainerEngineFor(input.engine); + + const stop = async () => { + if (stopping) { + return; + } + stopping = true; + try { + child?.kill("SIGTERM"); + await removeOwnedNodeWorkerContainers(input.identity, engine); + } finally { + await closeRelay(relay); + relay = undefined; + lifetime.dispose(); + finishStop(); + } + }; + const onSignal = () => { + void stop(); + }; + const onSupervisorGone = () => { + void stop(); + }; + process.once("SIGTERM", onSignal); + process.once("SIGINT", onSignal); + // The IPC channel is the supervisor's ownership lease. Without this, a + // killed supervisor can strand a still-running shim and its container. + lifetime.signal.addEventListener("abort", onSupervisorGone, { once: true }); + if (lifetime.signal.aborted) { + void stop(); + } + + try { + const started = await Promise.race([lifetime.started, stopped.then(() => false)]); + if (!started || stopping) { + return; + } + await fs.mkdir(input.mounts.relayDir, { recursive: true, mode: 0o700 }); + await fs.chmod(input.mounts.relayDir, 0o700); + relay = await startNodeWorkerGatewayRelay({ + directory: input.mounts.relayDir, + upstream: input.descriptor.connectionEndpoint, + }); + // A descriptor includes its host-only Gateway endpoint. Rebuild it from + // the validated plan so the container receives only the mounted relay. + const containerDescriptor = completeWorkerLaunchDescriptor( + { + version: input.descriptor.version, + admission: input.descriptor.admission, + assignment: input.descriptor.assignment, + }, + { + kind: "unix", + socketPath: NODE_WORKER_CONTAINER_RELAY_SOCKET, + } satisfies WorkerConnectionEndpoint, + ); + const runArgs = buildNodeWorkerContainerRunArgs({ + engine, + identity: input.identity, + mounts: input.mounts, + ...currentUser(), + }); + child = spawn(engine.command, runArgs, { + cwd: input.mounts.workspaceDir, + env: process.env, + stdio: ["pipe", "inherit", "inherit"], + windowsHide: true, + }); + child.stdin.end(JSON.stringify(containerDescriptor)); + await waitForOwnedNodeWorkerContainerRunning({ engine, identity: input.identity }); + // Container execution begins only after an exact label-bound inspect says Running. + lifetime.reportExecutionStarted(); + const code = await Promise.race([waitForChild(child), stopped.then(() => 143)]); + if (!stopping) { + await removeOwnedNodeWorkerContainers(input.identity, engine); + } + process.exitCode = code; + } catch (error) { + lifetime.reportConnectionFailure(error instanceof Error ? error.message : "container worker unavailable"); + process.exitCode = 1; + } finally { + process.off("SIGTERM", onSignal); + process.off("SIGINT", onSignal); + lifetime.signal.removeEventListener("abort", onSupervisorGone); + await stop(); + } +} + +if (process.argv.includes(NODE_WORKER_CONTAINER_SHIM_FLAG)) { + void runNodeWorkerContainerShim().catch((error: unknown) => { + process.stderr.write( + `${error instanceof Error ? error.message : "node worker container shim failed"}\n`, + ); + process.exitCode = 1; + }); +} diff --git a/src/node-host/node-worker-gateway-relay.test.ts b/src/node-host/node-worker-gateway-relay.test.ts new file mode 100644 index 000000000000..7bb78da79595 --- /dev/null +++ b/src/node-host/node-worker-gateway-relay.test.ts @@ -0,0 +1,178 @@ +import fs from "node:fs/promises"; +import http from "node:http"; +import type { Socket } from "node:net"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { WebSocket, WebSocketServer } from "ws"; +import { WORKER_PROTOCOL_MAX_PAYLOAD_BYTES } from "../../packages/gateway-protocol/src/schema/worker-admission.js"; +import { startNodeWorkerGatewayRelay } from "./node-worker-gateway-relay.js"; + +const cleanups: Array<() => Promise> = []; + +afterEach(async () => { + await Promise.all(cleanups.splice(0).map(async (cleanup) => await cleanup())); +}); + +async function relayHarness(params: { + onUpstreamConnection?: (socket: WebSocket) => void; + onUpstreamUpgrade?: (socket: Socket) => void; +} = {}) { + // Keep the lexical path short: the production relay intentionally does the + // same because macOS rejects canonical state paths as Unix socket names. + const root = await fs.mkdtemp("/tmp/node-worker-relay-"); + const upstreamServer = http.createServer(); + const rawUpstreamSockets = new Set(); + const upstreamClients: WebSocket[] = []; + if (params.onUpstreamUpgrade) { + upstreamServer.on("upgrade", (_request, socket) => { + rawUpstreamSockets.add(socket); + socket.once("close", () => rawUpstreamSockets.delete(socket)); + params.onUpstreamUpgrade?.(socket); + }); + } else { + const upstreamWebSocket = new WebSocketServer({ server: upstreamServer }); + upstreamWebSocket.on("connection", (socket) => { + upstreamClients.push(socket); + if (params.onUpstreamConnection) { + params.onUpstreamConnection(socket); + return; + } + socket.on("message", (message, binary) => socket.send(message, { binary })); + }); + } + await new Promise((resolve) => upstreamServer.listen(0, "127.0.0.1", resolve)); + const address = upstreamServer.address(); + if (!address || typeof address === "string") { + throw new Error("expected upstream server address"); + } + const relay = await startNodeWorkerGatewayRelay({ + directory: root, + upstream: { kind: "websocket", url: `ws://127.0.0.1:${address.port}/__openclaw__/worker` }, + }); + cleanups.push(async () => { + await relay.close(); + for (const socket of rawUpstreamSockets) { + socket.destroy(); + } + await new Promise((resolve) => upstreamServer.close(() => resolve())); + await fs.rm(root, { recursive: true, force: true }); + }); + return { relay, upstreamClients }; +} + +async function connect(socketPath: string): Promise { + const socket = new WebSocket(`ws+unix://${socketPath}:/`); + await new Promise((resolve, reject) => { + socket.once("open", resolve); + socket.once("error", reject); + }); + return socket; +} + +function waitForClose(socket: WebSocket): Promise { + return new Promise((resolve) => socket.once("close", (code) => resolve(code))); +} + +async function expectConnectFailure(socketPath: string): Promise { + const probe = new WebSocket(`ws+unix://${socketPath}:/`); + await new Promise((resolve, reject) => { + probe.once("open", () => reject(new Error("closed relay unexpectedly accepted a client"))); + probe.once("error", () => resolve()); + }); +} + +describe("node worker gateway relay", () => { + it("forwards one bounded local client and rejects a second client", async () => { + const { relay } = await relayHarness(); + const first = await connect(relay.socketPath); + cleanups.push(async () => first.terminate()); + + const echoed = new Promise((resolve) => { + first.once("message", (message) => resolve(Buffer.from(message))); + }); + first.send(Buffer.from("worker-frame")); + await expect(echoed).resolves.toEqual(Buffer.from("worker-frame")); + + const second = new WebSocket(`ws+unix://${relay.socketPath}:/`); + cleanups.push(async () => second.terminate()); + await expect( + new Promise((resolve, reject) => { + second.once("unexpected-response", (_request, response) => resolve(response.statusCode ?? 0)); + second.once("open", () => reject(new Error("second relay client unexpectedly connected"))); + second.once("error", () => undefined); + }), + ).resolves.toBe(409); + }); + + it("closes an oversized worker frame before forwarding it upstream", async () => { + const { relay } = await relayHarness(); + const client = await connect(relay.socketPath); + cleanups.push(async () => client.terminate()); + + const closed = new Promise((resolve) => client.once("close", (code) => resolve(code))); + client.send(Buffer.alloc(WORKER_PROTOCOL_MAX_PAYLOAD_BYTES + 1)); + + await expect(closed).resolves.toBe(1009); + }); + + it("closes a worker that exceeds the bounded pre-upstream queue", async () => { + const { relay } = await relayHarness({ + // Keep the upstream WebSocket in CONNECTING so every worker frame uses + // the relay's bounded pending queue rather than an OS socket buffer. + onUpstreamUpgrade: () => undefined, + }); + const client = await connect(relay.socketPath); + cleanups.push(async () => client.terminate()); + + const closed = waitForClose(client); + client.send(Buffer.alloc(WORKER_PROTOCOL_MAX_PAYLOAD_BYTES)); + client.send(Buffer.alloc(WORKER_PROTOCOL_MAX_PAYLOAD_BYTES)); + client.send(Buffer.from([0])); + + await expect(closed).resolves.toBe(1008); + }); + + it("closes both peers when a downstream client is over the bounded output limit", async () => { + const { relay, upstreamClients } = await relayHarness(); + const client = await connect(relay.socketPath); + cleanups.push(async () => client.terminate()); + await vi.waitFor(() => expect(upstreamClients).toHaveLength(1)); + const bufferedAmount = vi + .spyOn(WebSocket.prototype, "bufferedAmount", "get") + .mockReturnValue(WORKER_PROTOCOL_MAX_PAYLOAD_BYTES * 2 + 1); + const closed = waitForClose(client); + try { + upstreamClients[0].send(Buffer.from("gateway-frame")); + + await expect(closed).resolves.toBe(1008); + expect(bufferedAmount).toHaveBeenCalled(); + } finally { + bufferedAmount.mockRestore(); + } + }); + + it("closes the worker when the upstream connection fails", async () => { + const { relay } = await relayHarness({ + onUpstreamUpgrade: (socket) => { + socket.end("HTTP/1.1 503 Upstream unavailable\r\nConnection: close\r\n\r\n"); + }, + }); + const client = await connect(relay.socketPath); + cleanups.push(async () => client.terminate()); + + await expect(waitForClose(client)).resolves.toBe(1008); + }); + + it("terminates live peers and removes the Unix socket on close", async () => { + const { relay } = await relayHarness(); + const client = await connect(relay.socketPath); + const closed = waitForClose(client); + + await relay.close(); + + await expect(closed).resolves.toEqual(expect.any(Number)); + await expect(fs.lstat(relay.socketPath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(expectConnectFailure(relay.socketPath)).resolves.toBeUndefined(); + await expect(relay.close()).resolves.toBeUndefined(); + }); +}); diff --git a/src/node-host/node-worker-gateway-relay.ts b/src/node-host/node-worker-gateway-relay.ts new file mode 100644 index 000000000000..42c2e74c0e0c --- /dev/null +++ b/src/node-host/node-worker-gateway-relay.ts @@ -0,0 +1,195 @@ +import fs from "node:fs/promises"; +import { createServer, type IncomingMessage } from "node:http"; +import type { Socket } from "node:net"; +import path from "node:path"; +import { WORKER_PROTOCOL_MAX_PAYLOAD_BYTES } from "../../packages/gateway-protocol/src/schema/worker-admission.js"; +import { WebSocket, WebSocketServer, type RawData } from "ws"; +import { + resolveWorkerConnectionTarget, + type WorkerConnectionEndpoint, +} from "../worker/worker-connection-endpoint.js"; +import { NODE_WORKER_CONTAINER_RELAY_SOCKET } from "./node-worker-container-runtime.js"; + +const RELAY_PATH = "/"; +const RELAY_MAX_PENDING_BYTES = WORKER_PROTOCOL_MAX_PAYLOAD_BYTES * 2; +const RELAY_MAX_BUFFERED_BYTES = WORKER_PROTOCOL_MAX_PAYLOAD_BYTES * 2; +const RELAY_CONNECT_TIMEOUT_MS = 10_000; +const RELAY_PORTABLE_UNIX_SOCKET_MAX_BYTES = 100; + +export type NodeWorkerGatewayRelay = Readonly<{ + socketPath: string; + close: () => Promise; +}>; + +function rawBytes(data: RawData): number { + if (typeof data === "string") { + return Buffer.byteLength(data, "utf8"); + } + if (Array.isArray(data)) { + return data.reduce((total, chunk) => total + chunk.byteLength, 0); + } + return data.byteLength; +} + +function rejectUpgrade(socket: Socket, status: number): void { + socket.end(`HTTP/1.1 ${status} Relay unavailable\r\nConnection: close\r\n\r\n`); +} + +/** + * Exposes one local Unix WebSocket to an isolated worker. The remote Gateway + * endpoint and any TLS/Cloudflare material remain in this host-side relay. + */ +export async function startNodeWorkerGatewayRelay(params: { + directory: string; + upstream: WorkerConnectionEndpoint; +}): Promise { + if (!path.isAbsolute(params.directory)) { + throw new Error("node worker relay directory must be absolute"); + } + await fs.realpath(params.directory); + // The bind mount uses the canonical directory, but preserving its short + // lexical spelling here keeps the live Unix socket below macOS' path limit. + const socketPath = path.join(params.directory, path.basename(NODE_WORKER_CONTAINER_RELAY_SOCKET)); + if (Buffer.byteLength(socketPath, "utf8") > RELAY_PORTABLE_UNIX_SOCKET_MAX_BYTES) { + throw new Error("node worker relay socket path exceeds the portable Unix socket limit"); + } + await fs.rm(socketPath, { force: true }); + + let accepted = false; + let closing = false; + const websocketServer = new WebSocketServer({ + noServer: true, + maxPayload: WORKER_PROTOCOL_MAX_PAYLOAD_BYTES, + }); + const server = createServer((_request, response) => { + response.writeHead(404); + response.end(); + }); + const sockets = new Set(); + + const stopSocket = (socket: WebSocket, code = 1008) => { + try { + if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CLOSING) { + socket.close(code); + return; + } + socket.terminate(); + } catch { + socket.terminate(); + } + }; + + const attachRelay = (client: WebSocket) => { + sockets.add(client); + const target = resolveWorkerConnectionTarget(params.upstream); + const upstream = new WebSocket(target.url, target.options); + sockets.add(upstream); + const pending: Array<{ data: RawData; binary: boolean }> = []; + let pendingBytes = 0; + let linked = false; + let connectTimeout: NodeJS.Timeout | undefined; + + const stopBoth = () => { + if (linked) { + linked = false; + } + clearTimeout(connectTimeout); + stopSocket(client); + stopSocket(upstream); + }; + const forward = (destination: WebSocket, data: RawData, binary: boolean) => { + if (destination.readyState !== WebSocket.OPEN || rawBytes(data) > WORKER_PROTOCOL_MAX_PAYLOAD_BYTES) { + stopBoth(); + return; + } + destination.send(data, { binary }, (error) => { + if (error) { + stopBoth(); + } + }); + // WebSocket has no bounded pull API. Terminating both peers on a bounded + // queue is the relay's backpressure contract; it cannot become a buffer. + if (destination.bufferedAmount > RELAY_MAX_BUFFERED_BYTES) { + stopBoth(); + } + }; + + upstream.once("open", () => { + clearTimeout(connectTimeout); + const tlsError = target.validateSocket(upstream); + if (tlsError) { + stopBoth(); + return; + } + linked = true; + for (const frame of pending.splice(0)) { + forward(upstream, frame.data, frame.binary); + } + pendingBytes = 0; + }); + connectTimeout = setTimeout(stopBoth, RELAY_CONNECT_TIMEOUT_MS); + connectTimeout.unref?.(); + upstream.once("error", stopBoth); + client.once("error", stopBoth); + upstream.on("message", (data, binary) => forward(client, data, binary)); + client.on("message", (data, binary) => { + const bytes = rawBytes(data); + if (bytes > WORKER_PROTOCOL_MAX_PAYLOAD_BYTES) { + stopBoth(); + return; + } + if (!linked) { + pendingBytes += bytes; + if (pendingBytes > RELAY_MAX_PENDING_BYTES) { + stopBoth(); + return; + } + pending.push({ data, binary }); + return; + } + forward(upstream, data, binary); + }); + const closePeer = (peer: WebSocket) => () => { + sockets.delete(peer); + if (peer.readyState === WebSocket.OPEN || peer.readyState === WebSocket.CLOSING) { + stopSocket(peer, 1000); + } + }; + client.once("close", closePeer(upstream)); + upstream.once("close", closePeer(client)); + }; + + server.on("upgrade", (request: IncomingMessage, socket, head) => { + if (closing || accepted || request.method !== "GET" || request.url !== RELAY_PATH) { + rejectUpgrade(socket, accepted ? 409 : 404); + return; + } + accepted = true; + websocketServer.handleUpgrade(request, socket, head, (client) => attachRelay(client)); + }); + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(socketPath, () => { + server.off("error", reject); + resolve(); + }); + }); + await fs.chmod(socketPath, 0o600); + + return Object.freeze({ + socketPath, + close: async () => { + if (closing) { + return; + } + closing = true; + for (const socket of sockets) { + socket.terminate(); + } + await new Promise((resolve) => server.close(() => resolve())); + await fs.rm(socketPath, { force: true }); + websocketServer.close(); + }, + }); +} diff --git a/src/node-host/node-worker-launch-store.test.ts b/src/node-host/node-worker-launch-store.test.ts index 145fde23e467..45680121c108 100644 --- a/src/node-host/node-worker-launch-store.test.ts +++ b/src/node-host/node-worker-launch-store.test.ts @@ -83,6 +83,51 @@ function launchIds(database: ReturnType["db"]) } describe("node worker launch store pruning", () => { + it("persists the selected container engine with the durable launch identity", () => { + const { database, store } = fixture(); + const supervisor = requireNodeWorkerProcessIdentity(process.pid); + const launchId = "container-launch"; + const planHash = "c".repeat(64); + store.claim( + { + launchId, + planHash, + gatewayNamespace: "gateway-1", + environmentId: "environment-1", + sessionId: "session-1", + ownerEpoch: 3, + placementGeneration: 4, + runId: "run-1", + }, + supervisor, + 2, + NOW_MS, + ); + + store.recordContainerLaunch({ + launchId, + planHash, + engine: "podman", + expiresAtMs: NOW_MS + DAY_MS, + }); + + expect(store.getContainerLaunchEngine({ launchId, planHash })).toBe("podman"); + expect(store.getContainerLaunchLease({ launchId, planHash })).toEqual({ + engine: "podman", + expiresAtMs: NOW_MS + DAY_MS, + }); + expect( + database + .prepare("SELECT plan_hash, container_engine FROM node_worker_container_launches") + .all(), + ).toEqual([{ plan_hash: planHash, container_engine: "podman" }]); + expect( + database + .prepare("SELECT plan_hash, expires_at_ms FROM node_worker_container_leases") + .all(), + ).toEqual([{ plan_hash: planHash, expires_at_ms: NOW_MS + DAY_MS }]); + }); + it("lazily ensures the terminal expiry index for existing databases", () => { const { database, env } = fixture(); expect(hasTerminalExpiryIndex(database)).toBe(true); diff --git a/src/node-host/node-worker-launch-store.ts b/src/node-host/node-worker-launch-store.ts index 6b5e589b3dd6..33bf791c7e1b 100644 --- a/src/node-host/node-worker-launch-store.ts +++ b/src/node-host/node-worker-launch-store.ts @@ -25,9 +25,19 @@ type NodeWorkerLaunchState = | "interrupted" | "cancelled"; export type NodeWorkerTerminalState = Exclude; +export type NodeWorkerContainerEngineId = "docker" | "podman"; -type NodeWorkerLaunchDatabase = Pick; +type NodeWorkerLaunchDatabase = Pick< + OpenClawStateDatabase, + "node_worker_container_launches" | "node_worker_container_leases" | "node_worker_launches" +>; type NodeWorkerLaunchRow = Selectable; +type NodeWorkerContainerLaunchRow = Selectable< + NodeWorkerLaunchDatabase["node_worker_container_launches"] +>; +type NodeWorkerContainerLeaseRow = Selectable< + NodeWorkerLaunchDatabase["node_worker_container_leases"] +>; export type NodeWorkerLaunchReceipt = { launchId: string; @@ -107,6 +117,32 @@ function readRow(database: DatabaseSync, launchId: string): NodeWorkerLaunchRow ); } +function readContainerLaunchRow( + database: DatabaseSync, + launchId: string, +): NodeWorkerContainerLaunchRow | undefined { + return executeSqliteQueryTakeFirstSync( + database, + query(database) + .selectFrom("node_worker_container_launches") + .selectAll() + .where("launch_id", "=", launchId), + ); +} + +function readContainerLeaseRow( + database: DatabaseSync, + launchId: string, +): NodeWorkerContainerLeaseRow | undefined { + return executeSqliteQueryTakeFirstSync( + database, + query(database) + .selectFrom("node_worker_container_leases") + .selectAll() + .where("launch_id", "=", launchId), + ); +} + function readNonterminalCount(database: DatabaseSync): number { return ( executeSqliteQueryTakeFirstSync( @@ -153,6 +189,18 @@ function pruneTerminalRows(params: { if (launchIds.length === 0) { return 0; } + executeSqliteQuerySync( + params.database, + query(params.database) + .deleteFrom("node_worker_container_launches") + .where("launch_id", "in", launchIds), + ); + executeSqliteQuerySync( + params.database, + query(params.database) + .deleteFrom("node_worker_container_leases") + .where("launch_id", "in", launchIds), + ); const result = executeSqliteQuerySync( params.database, query(params.database) @@ -211,6 +259,12 @@ function validatePlanHash(value: string): void { } } +function validateContainerEngine(value: string): asserts value is NodeWorkerContainerEngineId { + if (value !== "docker" && value !== "podman") { + throw new Error("node worker container engine must be docker or podman"); + } +} + function validateTimestamp(value: number): void { if (!Number.isSafeInteger(value) || value < 0) { throw new Error("node worker launch timestamp must be a non-negative safe integer"); @@ -480,6 +534,98 @@ export class NodeWorkerLaunchStore { }); } + recordContainerLaunch(params: { + launchId: string; + planHash: string; + engine: NodeWorkerContainerEngineId; + expiresAtMs: number; + }): void { + validateIdentifier(params.launchId, "node worker launch id"); + validatePlanHash(params.planHash); + validateContainerEngine(params.engine); + validateTimestamp(params.expiresAtMs); + this.write("node-worker-launch.record-container", (database) => { + const launch = requireMatchingRow(database, params.launchId, params.planHash); + if (TERMINAL_STATES.has(launch.state)) { + throw new Error("cannot record a container for a terminal node worker launch"); + } + const existing = readContainerLaunchRow(database, params.launchId); + if (!existing) { + executeSqliteQuerySync( + database, + query(database).insertInto("node_worker_container_launches").values({ + launch_id: params.launchId, + plan_hash: params.planHash, + container_engine: params.engine, + }), + ); + } else if ( + existing.plan_hash !== params.planHash || + existing.container_engine !== params.engine + ) { + throw new Error("node worker container launch metadata does not match the durable launch"); + } + const lease = readContainerLeaseRow(database, params.launchId); + if (!lease) { + executeSqliteQuerySync( + database, + query(database).insertInto("node_worker_container_leases").values({ + launch_id: params.launchId, + plan_hash: params.planHash, + expires_at_ms: params.expiresAtMs, + }), + ); + } else if (lease.plan_hash !== params.planHash || lease.expires_at_ms !== params.expiresAtMs) { + throw new Error("node worker container lease does not match the durable launch"); + } + }); + } + + getContainerLaunchEngine(params: { + launchId: string; + planHash: string; + }): NodeWorkerContainerEngineId | undefined { + validateIdentifier(params.launchId, "node worker launch id"); + validatePlanHash(params.planHash); + return this.write("node-worker-launch.get-container", (database) => { + const metadata = readContainerLaunchRow(database, params.launchId); + if (!metadata) { + return undefined; + } + if (metadata.plan_hash !== params.planHash) { + throw new Error("node worker container launch metadata does not match the durable launch"); + } + validateContainerEngine(metadata.container_engine); + return metadata.container_engine; + }); + } + + getContainerLaunchLease(params: { + launchId: string; + planHash: string; + }): { engine: NodeWorkerContainerEngineId; expiresAtMs: number } | undefined { + validateIdentifier(params.launchId, "node worker launch id"); + validatePlanHash(params.planHash); + return this.write("node-worker-launch.get-container-lease", (database) => { + const metadata = readContainerLaunchRow(database, params.launchId); + const lease = readContainerLeaseRow(database, params.launchId); + if (!metadata && !lease) { + return undefined; + } + if (!metadata || !lease) { + // Launches written before leases existed still recover through the + // engine metadata and are interrupted conservatively on restart. + return undefined; + } + if (metadata.plan_hash !== params.planHash || lease.plan_hash !== params.planHash) { + throw new Error("node worker container lease metadata does not match the durable launch"); + } + validateContainerEngine(metadata.container_engine); + validateTimestamp(lease.expires_at_ms); + return { engine: metadata.container_engine, expiresAtMs: lease.expires_at_ms }; + }); + } + getMatching(expected: NodeWorkerSupervisorIdentity): NodeWorkerLaunchReceipt | undefined { validateIdentifier(expected.launchId, "node worker launch id"); validatePlanHash(expected.planHash); diff --git a/src/node-host/node-worker-memory-projection.ts b/src/node-host/node-worker-memory-projection.ts new file mode 100644 index 000000000000..a2dce9f416e4 --- /dev/null +++ b/src/node-host/node-worker-memory-projection.ts @@ -0,0 +1,328 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import path from "node:path"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { isGatewayLoopbackHost } from "../../packages/gateway-client/src/websocket-transport.js"; +import { + loadOrCreateProcessDeviceIdentity, + signDevicePayload, + type DeviceIdentity, +} from "../infra/device-identity.js"; +import { isPathInside } from "../infra/path-guards.js"; +import { + buildNodeWorkerMemoryProjectionRequestProofPayload, + nodeWorkerMemoryProjectionTransferPath, + parseNodeWorkerMemoryProjectionPayload, + NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER, + NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER, + NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER, + type NodeWorkerMemoryProjection, + type NodeWorkerMemoryProjectionPayload, +} from "../worker/node-memory-projection-protocol.js"; +import type { WorkerConnectionEndpoint } from "../worker/worker-connection-endpoint.js"; +import { openNodeWorkerTransferHttpRequest } from "./node-worker-transfer-http.js"; + +const PROJECTION_RESPONSE_MAX_BYTES = 3 * 1024 * 1024; +const PROJECTION_ROOT = "memory-projections"; +const PROJECTION_ROOT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/u; +const PROJECTION_SOCKET_FORBIDDEN_ROOTS = new Set(["opt", "run", "workspace"]); + +type ProjectionIdentity = Readonly<{ + gatewayNamespace: string; + launchId: string; + planHash: string; +}>; + +function throwIfProjectionStagingAborted(signal?: AbortSignal): void { + signal?.throwIfAborted(); +} + +function currentNonRootUid(): number { + if (process.platform === "win32" || typeof process.getuid !== "function") { + throw new Error("node worker memory projections require a non-root POSIX node host"); + } + const uid = process.getuid(); + if (!Number.isSafeInteger(uid) || uid <= 0) { + throw new Error("node worker memory projections require a non-root POSIX node host"); + } + return uid; +} + +function requirePrivateDirectory(directory: string, parent?: string): string { + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + fs.chmodSync(directory, 0o700); + const stats = fs.lstatSync(directory); + const resolved = fs.realpathSync.native(directory); + if ( + stats.isSymbolicLink() || + !stats.isDirectory() || + stats.uid !== currentNonRootUid() || + (stats.mode & 0o077) !== 0 || + (parent !== undefined && fs.realpathSync.native(path.dirname(directory)) !== parent) + ) { + throw new Error("node worker memory projection path is not a private owned directory"); + } + return resolved; +} + +function projectionDirectoryName(identity: ProjectionIdentity): string { + if (!/^[a-f0-9]{64}$/u.test(identity.planHash)) { + throw new Error("node worker memory projection plan hash is invalid"); + } + return createHash("sha256") + .update(`${identity.gatewayNamespace}\0${identity.launchId}\0${identity.planHash}`) + .digest("hex"); +} + +function assertPayloadIntegrity(payload: NodeWorkerMemoryProjectionPayload): Array<{ + root: string; + leaf: string; + bytes: Buffer; +}> { + const roots = new Set(); + return payload.files.map((file) => { + const [root, leaf] = file.virtualPath.split("/"); + const rootKey = root!.toLocaleLowerCase("en-US"); + // `memory` is the canonical authorized-view root. It stays nested below the + // container's `/memory` mount, so it cannot shadow a container filesystem root. + if (!PROJECTION_ROOT_PATTERN.test(root!) || PROJECTION_SOCKET_FORBIDDEN_ROOTS.has(rootKey)) { + throw new Error("node worker memory projection root is unsafe"); + } + roots.add(rootKey); + const bytes = Buffer.from(file.contentBase64, "base64"); + if (createHash("sha256").update(bytes).digest("hex") !== file.sha256) { + throw new Error("node worker memory projection digest mismatch"); + } + if (!leaf || leaf === "." || leaf === ".." || leaf.includes("/") || leaf.includes("\\")) { + throw new Error("node worker memory projection path is unsafe"); + } + return { root: root!, leaf, bytes }; + }); +} + +function assertProjectionTransferTransport(endpoint: WorkerConnectionEndpoint): void { + const gateway = new URL(endpoint.url); + if (gateway.protocol === "ws:" && !isGatewayLoopbackHost(gateway.hostname)) { + throw new Error( + "node worker memory projection requires wss:// for a non-loopback Gateway endpoint", + ); + } +} + +async function readProjectionResponse(params: { + endpoint: WorkerConnectionEndpoint; + projection: NodeWorkerMemoryProjection; + deviceIdentity: DeviceIdentity; + signal?: AbortSignal; +}): Promise { + if (params.endpoint.kind !== "websocket") { + throw new Error("node worker memory projection requires a Gateway WebSocket endpoint"); + } + assertProjectionTransferTransport(params.endpoint); + const signedAtMs = Date.now(); + const proofPayload = buildNodeWorkerMemoryProjectionRequestProofPayload({ + reference: params.projection.reference, + binding: params.projection.binding, + nodeId: params.deviceIdentity.deviceId, + signedAtMs, + }); + const response = await openNodeWorkerTransferHttpRequest({ + gatewayUrl: params.endpoint.url, + ...(params.endpoint.tlsFingerprint ? { tlsFingerprint: params.endpoint.tlsFingerprint } : {}), + ...(params.endpoint.cloudflareAccess + ? { cloudflareAccess: params.endpoint.cloudflareAccess } + : {}), + routePath: nodeWorkerMemoryProjectionTransferPath(), + method: "GET", + token: params.projection.reference, + headers: { + [NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER]: params.deviceIdentity.deviceId, + [NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER]: String(signedAtMs), + [NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER]: signDevicePayload( + params.deviceIdentity.privateKeyPem, + proofPayload, + ), + }, + ...(params.signal ? { signal: params.signal } : {}), + }); + if (response.statusCode !== 200) { + response.resume(); + throw new Error(`node worker memory projection transfer failed (${response.statusCode ?? 0})`); + } + const chunks: Buffer[] = []; + let bytes = 0; + for await (const chunk of response) { + const value = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + bytes += value.byteLength; + if (bytes > PROJECTION_RESPONSE_MAX_BYTES) { + response.destroy(new Error("node worker memory projection response exceeds its limit")); + throw new Error("node worker memory projection response exceeds its limit"); + } + chunks.push(value); + } + let value: unknown; + try { + value = JSON.parse(Buffer.concat(chunks).toString("utf8")); + } catch { + throw new Error("node worker memory projection response is malformed"); + } + const payload = parseNodeWorkerMemoryProjectionPayload(value); + if (!payload) { + throw new Error("node worker memory projection response violated its bounded contract"); + } + return payload; +} + +/** + * Owns node-private projection staging. It is intentionally distinct from + * workspace synchronization: projection bytes are immutable and never writable by the worker. + */ +export class NodeWorkerMemoryProjectionRuntime { + private readonly root: string; + private readonly deviceIdentity: DeviceIdentity; + + constructor(options: { root: string; deviceIdentity?: DeviceIdentity }) { + const base = path.resolve(options.root, PROJECTION_ROOT); + this.root = requirePrivateDirectory(base); + this.deviceIdentity = options.deviceIdentity ?? loadOrCreateProcessDeviceIdentity(); + } + + private resolveDirectory(identity: ProjectionIdentity): string { + return path.join(this.root, projectionDirectoryName(identity)); + } + + async stage(params: { + identity: ProjectionIdentity; + projection: NodeWorkerMemoryProjection; + endpoint: WorkerConnectionEndpoint; + signal?: AbortSignal; + }): Promise { + throwIfProjectionStagingAborted(params.signal); + if (params.projection.expiresAtMs <= Date.now()) { + throw new Error("node worker memory projection lease expired before staging"); + } + const destination = this.resolveDirectory(params.identity); + throwIfProjectionStagingAborted(params.signal); + await this.remove(params.identity); + throwIfProjectionStagingAborted(params.signal); + const payload = await readProjectionResponse({ + endpoint: params.endpoint, + projection: params.projection, + deviceIdentity: this.deviceIdentity, + ...(params.signal ? { signal: params.signal } : {}), + }); + throwIfProjectionStagingAborted(params.signal); + if (params.projection.expiresAtMs <= Date.now()) { + throw new Error("node worker memory projection lease expired during staging"); + } + const files = assertPayloadIntegrity(payload); + throwIfProjectionStagingAborted(params.signal); + const temporary = await fsp.mkdtemp(path.join(this.root, ".projection-")); + let staged = false; + try { + throwIfProjectionStagingAborted(params.signal); + await fsp.chmod(temporary, 0o700); + const preparedRoots = new Set(); + for (const file of files) { + throwIfProjectionStagingAborted(params.signal); + const root = path.join(temporary, file.root); + if (!preparedRoots.has(file.root)) { + // The node must write the immutable payload before the directory can + // become read-only to the container user. + await fsp.mkdir(root, { mode: 0o700 }); + throwIfProjectionStagingAborted(params.signal); + const rootStats = await fsp.lstat(root); + if (rootStats.isSymbolicLink() || !rootStats.isDirectory() || rootStats.nlink !== 2) { + throw new Error("node worker memory projection root is not a private directory"); + } + preparedRoots.add(file.root); + } + const target = path.join(root, file.leaf); + throwIfProjectionStagingAborted(params.signal); + const handle = await fsp.open(target, "wx", 0o400); + try { + throwIfProjectionStagingAborted(params.signal); + await handle.writeFile(file.bytes); + } finally { + await handle.close(); + } + throwIfProjectionStagingAborted(params.signal); + const stats = await fsp.lstat(target); + if ( + stats.isSymbolicLink() || + !stats.isFile() || + stats.nlink !== 1 || + stats.size !== file.bytes.byteLength || + stats.uid !== currentNonRootUid() + ) { + throw new Error("node worker memory projection staged file failed integrity validation"); + } + } + for (const root of preparedRoots) { + throwIfProjectionStagingAborted(params.signal); + await fsp.chmod(path.join(temporary, root), 0o500); + } + throwIfProjectionStagingAborted(params.signal); + await fsp.rename(temporary, destination); + staged = true; + throwIfProjectionStagingAborted(params.signal); + const resolved = await fsp.realpath(destination); + const stats = await fsp.lstat(resolved); + if ( + stats.isSymbolicLink() || + !stats.isDirectory() || + path.dirname(resolved) !== this.root || + !isPathInside(this.root, resolved) + ) { + throw new Error("node worker memory projection escaped its private root"); + } + throwIfProjectionStagingAborted(params.signal); + return resolved; + } catch (error) { + await fsp.rm(temporary, { recursive: true, force: true }).catch(() => undefined); + if (staged) { + await this.remove(params.identity); + } + throw error; + } + } + + async remove(identity: ProjectionIdentity): Promise { + const target = this.resolveDirectory(identity); + try { + const [stats, parent, resolved] = await Promise.all([ + fsp.lstat(target), + fsp.realpath(this.root), + fsp.realpath(target), + ]); + if ( + stats.isSymbolicLink() || + !stats.isDirectory() || + path.dirname(resolved) !== parent || + !isPathInside(parent, resolved) + ) { + throw new Error("node worker memory projection cleanup target is not owned"); + } + // Projection roots are made read-only before mount. Restore write access + // only after revalidating the exact node-owned tree so cleanup cannot be + // bypassed by its own immutable-file policy. + await fsp.chmod(resolved, 0o700); + for (const child of await fsp.readdir(resolved, { withFileTypes: true })) { + if (child.isSymbolicLink() || !child.isDirectory()) { + throw new Error("node worker memory projection cleanup tree is not owned"); + } + const childResolved = await fsp.realpath(path.join(resolved, child.name)); + if (path.dirname(childResolved) !== resolved || !isPathInside(resolved, childResolved)) { + throw new Error("node worker memory projection cleanup tree escaped its root"); + } + await fsp.chmod(childResolved, 0o700); + } + await fsp.rm(target, { recursive: true, force: true }); + } catch (error) { + if (!isRecord(error) || error.code !== "ENOENT") { + throw error; + } + } + } +} diff --git a/src/node-host/node-worker-supervisor-contract.test.ts b/src/node-host/node-worker-supervisor-contract.test.ts new file mode 100644 index 000000000000..f35cb65b7b88 --- /dev/null +++ b/src/node-host/node-worker-supervisor-contract.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import type { NodeWorkerLaunchReceipt } from "./node-worker-launch-store.js"; +import { projectNodeWorkerSupervisorReceipt } from "./node-worker-supervisor-contract.js"; + +function terminalReceipt(worker: NodeWorkerLaunchReceipt["worker"]): NodeWorkerLaunchReceipt { + return { + launchId: "launch-1", + planHash: "a".repeat(64), + gatewayNamespace: "gateway-1", + environmentId: "environment-1", + sessionId: "session-1", + ownerEpoch: 3, + placementGeneration: 4, + runId: "run-1", + state: "cancelled", + supervisor: { pid: 1, startTime: 1 }, + worker, + resultJson: null, + errorText: "node worker launch cancelled", + completedAtMs: 2, + createdAtMs: 1, + updatedAtMs: 2, + }; +} + +describe("node worker supervisor receipt projection", () => { + it("exposes execution readiness without exposing a worker process identity", () => { + expect(projectNodeWorkerSupervisorReceipt(terminalReceipt(null))).toMatchObject({ + state: "cancelled", + executionStarted: false, + }); + expect(projectNodeWorkerSupervisorReceipt(terminalReceipt({ pid: 2, startTime: 2 }))).toMatchObject({ + state: "cancelled", + executionStarted: true, + }); + }); +}); diff --git a/src/node-host/node-worker-supervisor-contract.ts b/src/node-host/node-worker-supervisor-contract.ts index 484750734fde..e76517dfb1ad 100644 --- a/src/node-host/node-worker-supervisor-contract.ts +++ b/src/node-host/node-worker-supervisor-contract.ts @@ -12,12 +12,16 @@ import type { WorkerConnectionEndpoint } from "../worker/worker-connection-endpo import type { NodeWorkerLaunchReceipt } from "./node-worker-launch-store.js"; export { + NODE_WORKER_EXECUTION_CONTAINER_V1, + NODE_WORKER_EXECUTION_HOST_V1, + nodeWorkerMemoryProjectionLaunchBinding, nodeWorkerPlanHash, parseNodeWorkerCancelInput, parseNodeWorkerLaunchInput, parseNodeWorkerLookupInput, } from "../worker/node-supervisor-protocol.js"; export type { + NodeWorkerExecution, NodeWorkerLaunchInput, NodeWorkerSupervisorIdentity, NodeWorkerSupervisorReceipt, @@ -55,7 +59,14 @@ export function projectNodeWorkerSupervisorReceipt( : receipt.state === "failed" || receipt.state === "interrupted" || receipt.state === "cancelled" - ? { ...identity, state: receipt.state, errorText: receipt.errorText } + ? { + ...identity, + state: receipt.state, + errorText: receipt.errorText, + // Worker identity is persisted only by markRunning, after the private child + // acknowledgement. Do not expose the PID, only the durable execution fact. + executionStarted: receipt.worker !== null, + } : { ...identity, state: receipt.state }; const parsed = parseNodeWorkerSupervisorReceipt(projected); if (!parsed) { diff --git a/src/node-host/node-worker-supervisor.recovery.test.ts b/src/node-host/node-worker-supervisor.recovery.test.ts index b244d4928466..5bb1cae90b9b 100644 --- a/src/node-host/node-worker-supervisor.recovery.test.ts +++ b/src/node-host/node-worker-supervisor.recovery.test.ts @@ -1,9 +1,7 @@ import { spawn, type ChildProcess } from "node:child_process"; -import { createHash } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import { stableStringify } from "@openclaw/normalization-core"; import { afterEach, describe, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import { @@ -16,9 +14,11 @@ import { requireNodeWorkerProcessIdentity, type NodeWorkerProcessIdentity, } from "./node-worker-process-identity.js"; +import { nodeWorkerPlanHash } from "./node-worker-supervisor-contract.js"; import { createNodeWorkerSupervisor } from "./node-worker-supervisor.js"; import { testNodeWorkerLaunchIdentity, + testNodeWorkerMemoryProjection, TEST_WORKER_ENDPOINT, testWorkerLaunchInput, writeNodeWorkerFixture, @@ -58,16 +58,7 @@ function fixture(label: string) { } function planHash(input: ReturnType): string { - return createHash("sha256") - .update( - stableStringify({ - expectedBundleHash: input.expectedBundleHash, - descriptor: input.descriptor, - gatewayNamespace: input.gatewayNamespace, - placementGeneration: input.placementGeneration, - }), - ) - .digest("hex"); + return nodeWorkerPlanHash(input); } function insertLaunch(params: { @@ -247,6 +238,110 @@ describe("node worker supervisor recovery", () => { await supervisor.close(); }); + it.runIf(process.platform !== "win32")( + "does not settle a stale pending container launch until durable cleanup succeeds", + async () => { + const { bundleRoot, env, root, workspaceDir } = fixture("node-worker-pending-container-"); + const bin = path.join(root, "bin"); + const docker = path.join(bin, "docker"); + const commandLog = path.join(root, "docker-commands.log"); + fs.mkdirSync(bin, { recursive: true }); + const writeDocker = (exitCode: number) => { + fs.writeFileSync( + docker, + `#!/bin/sh\nprintf '%s\\n' "$1" >> ${JSON.stringify(commandLog)}\nexit ${String(exitCode)}\n`, + { mode: 0o755 }, + ); + }; + writeDocker(1); + vi.stubEnv("PATH", `${bin}${path.delimiter}${process.env.PATH ?? ""}`); + try { + const store = new NodeWorkerLaunchStore({ env }); + store.get("schema-probe"); + const input = testWorkerLaunchInput(workspaceDir, "pending-container-launch"); + input.execution = { kind: "container-v1" }; + input.descriptor.assignment.memoryReadEnforced = true; + input.descriptor.assignment.workspaceDir = "/workspace"; + input.memoryProjection = testNodeWorkerMemoryProjection(input); + insertLaunch({ + env, + input, + state: "pending", + supervisor: { pid: 2_147_483_647, startTime: 1 }, + }); + store.recordContainerLaunch({ + launchId: input.launchId, + planHash: planHash(input), + engine: "docker", + expiresAtMs: input.memoryProjection.expiresAtMs, + }); + + const first = createNodeWorkerSupervisor({ bundleRoot, env }); + await expect(first.initialize()).rejects.toThrow("Docker command failed"); + expect(store.get(input.launchId)).toMatchObject({ state: "pending" }); + await first.close().catch(() => undefined); + + writeDocker(0); + const recovered = createNodeWorkerSupervisor({ bundleRoot, env }); + await recovered.initialize(); + expect(await recovered.status(input.launchId)).toMatchObject({ state: "interrupted" }); + expect(fs.readFileSync(commandLog, "utf8").split("\n").filter(Boolean)).toEqual([ + "ps", + "ps", + ]); + await recovered.close(); + } finally { + vi.unstubAllEnvs(); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "withdraws an expired durable projection lease during stale container recovery", + async () => { + const { bundleRoot, env, root, workspaceDir } = fixture("node-worker-expired-container-"); + const bin = path.join(root, "bin"); + fs.mkdirSync(bin, { recursive: true }); + fs.writeFileSync( + path.join(bin, "docker"), + "#!/bin/sh\nexit 0\n", + { mode: 0o755 }, + ); + vi.stubEnv("PATH", `${bin}${path.delimiter}${process.env.PATH ?? ""}`); + try { + const store = new NodeWorkerLaunchStore({ env }); + store.get("schema-probe"); + const input = testWorkerLaunchInput(workspaceDir, "expired-container-launch"); + input.execution = { kind: "container-v1" }; + input.descriptor.assignment.memoryReadEnforced = true; + input.descriptor.assignment.workspaceDir = "/workspace"; + input.memoryProjection = { ...testNodeWorkerMemoryProjection(input), expiresAtMs: 1 }; + insertLaunch({ + env, + input, + state: "pending", + supervisor: { pid: 2_147_483_647, startTime: 1 }, + }); + store.recordContainerLaunch({ + launchId: input.launchId, + planHash: planHash(input), + engine: "docker", + expiresAtMs: input.memoryProjection.expiresAtMs, + }); + + const recovered = createNodeWorkerSupervisor({ bundleRoot, env, now: () => 2 }); + await recovered.initialize(); + await expect(recovered.status(input.launchId)).resolves.toMatchObject({ + state: "cancelled", + errorText: "node worker memory projection lease expired before the worker launch started", + }); + await recovered.close(); + } finally { + vi.unstubAllEnvs(); + } + }, + ); + it.runIf(process.platform !== "win32").each([ { operation: "replay" as const, state: "interrupted" as const }, { operation: "cancel" as const, state: "cancelled" as const }, diff --git a/src/node-host/node-worker-supervisor.test-support.ts b/src/node-host/node-worker-supervisor.test-support.ts index 6ba1be22feec..cfc7d6944213 100644 --- a/src/node-host/node-worker-supervisor.test-support.ts +++ b/src/node-host/node-worker-supervisor.test-support.ts @@ -7,6 +7,8 @@ import { import type { WorkerLaunchPlan } from "../worker/launch-descriptor.js"; import type { WorkerConnectionEndpoint } from "../worker/worker-connection-endpoint.js"; import { + NODE_WORKER_EXECUTION_HOST_V1, + nodeWorkerMemoryProjectionLaunchBinding, nodeWorkerPlanHash, type NodeWorkerLaunchInput, type NodeWorkerSupervisorIdentity, @@ -55,13 +57,33 @@ const onMessage = (message) => { typeof message !== "object" || message === null || Array.isArray(message) || - Object.keys(message).length !== 1 || - message.type !== "openclaw-worker-start-v1" + Object.keys(message).length !== 3 || + message.type !== "openclaw-worker-start-v1" || + typeof message.launchId !== "string" || + !/^[a-f0-9]{64}$/.test(message.planHash) ) { hardTerminate(); return; } started = true; + const executionAcknowledgement = { + type: "openclaw-worker-execution-started-v1", + launchId: message.launchId, + planHash: message.planHash, + }; + if (descriptor.assignment.prompt === "wrong-execution-ack") { + process.send({ ...executionAcknowledgement, launchId: "another-launch" }, () => {}); + setInterval(() => {}, 1000); + return; + } + if (descriptor.assignment.prompt === "wait-before-execution-ack") { + setInterval(() => {}, 1000); + return; + } + process.send(executionAcknowledgement, () => {}); + if (descriptor.assignment.prompt === "replayed-execution-ack") { + setTimeout(() => process.send(executionAcknowledgement, () => {}), 25); + } resolveStart(); }; const onDisconnect = () => { @@ -99,6 +121,8 @@ if (mode === "connection-failure") { setInterval(() => {}, 1000); } else if (mode === "wait") { setInterval(() => {}, 1000); +} else if (mode === "replayed-execution-ack") { + setInterval(() => {}, 1000); } else if (mode === "tree") { grandchild = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { stdio: "ignore" }); fs.writeFileSync(path.join(descriptor.assignment.workspaceDir, "grandchild.pid"), String(grandchild.pid)); @@ -164,6 +188,7 @@ export function testWorkerDescriptor(workspaceDir: string, prompt = "success"): }, assignment: { agentId: "agent-1", + memoryReadEnforced: false, operationalRunInstance: { instanceId: "instance-1", runId: "run-1" }, agentRuntimeIdentityToken: "signed-runtime-token", runId: "run-1", @@ -195,6 +220,20 @@ export function testNodeWorkerLaunchIdentity( }; } +export function testNodeWorkerMemoryProjection( + input: Omit, +) { + return { + version: 1 as const, + reference: "a".repeat(43), + binding: { + launch: nodeWorkerMemoryProjectionLaunchBinding(input), + authorization: "b".repeat(64), + }, + expiresAtMs: Date.now() + 60_000, + }; +} + export function writeNodeWorkerFixture(root: string) { const stateDir = path.join(root, "state-root"); const bundleRoot = path.join(root, "bundles-root"); @@ -217,5 +256,6 @@ export function testWorkerLaunchInput( expectedBundleHash: TEST_BUNDLE_HASH, placementGeneration: 4, descriptor: testWorkerDescriptor(workspaceDir, prompt), + execution: { kind: NODE_WORKER_EXECUTION_HOST_V1 }, }; } diff --git a/src/node-host/node-worker-supervisor.test.ts b/src/node-host/node-worker-supervisor.test.ts index 840372986fda..7a8832ca33ad 100644 --- a/src/node-host/node-worker-supervisor.test.ts +++ b/src/node-host/node-worker-supervisor.test.ts @@ -17,10 +17,12 @@ import { requireNodeWorkerProcessIdentity, } from "./node-worker-process-identity.js"; import { createNodeWorkerSupervisor } from "./node-worker-supervisor.js"; +import { NodeWorkerMemoryProjectionRuntime } from "./node-worker-memory-projection.js"; import { TEST_WORKER_CREDENTIAL, TEST_WORKER_ENDPOINT, TEST_WORKER_SOURCE, + testNodeWorkerMemoryProjection, testNodeWorkerLaunchIdentity, testWorkerDescriptor, testWorkerLaunchInput, @@ -69,6 +71,85 @@ async function waitForTerminal(supervisor: NodeWorkerSupervisor, launchId: strin } describe("node worker supervisor", () => { + it("rejects an enforced-memory host launch before claiming a worker slot", async () => { + const { env, supervisor, workspaceDir } = fixture(); + const input = launchInput(workspaceDir, "enforced-memory-host-downgrade"); + input.descriptor.assignment.memoryReadEnforced = true; + + await expect(supervisor.launch(input, TEST_WORKER_ENDPOINT)).rejects.toThrow( + "requires container-v1 execution", + ); + expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)).toBeUndefined(); + await supervisor.close(); + }); + + it("rejects a cross-session memory projection replay before claiming a worker slot", async () => { + const { env, supervisor, workspaceDir } = fixture(); + const input = launchInput(workspaceDir, "enforced-memory-cross-session-replay"); + input.execution = { kind: "container-v1" }; + input.descriptor.assignment.memoryReadEnforced = true; + input.descriptor.assignment.workspaceDir = "/workspace"; + input.memoryProjection = testNodeWorkerMemoryProjection(input); + const replay = structuredClone(input); + replay.descriptor.admission.sessionId = "another-session"; + + await expect(supervisor.launch(replay, TEST_WORKER_ENDPOINT)).rejects.toThrow( + "memory projection does not match its worker launch", + ); + expect(new NodeWorkerLaunchStore({ env }).get(replay.launchId)).toBeUndefined(); + await supervisor.close(); + }); + + it.runIf(process.platform !== "win32")( + "persists cancellation and cleans the staged projection before a container can spawn", + async () => { + const { bundleRoot, env, root, workspaceDir } = fixture(); + const bin = path.join(root, "bin"); + const commandLog = path.join(root, "docker-commands.log"); + fs.mkdirSync(bin, { recursive: true }); + fs.writeFileSync( + path.join(bin, "docker"), + `#!/bin/sh\nprintf '%s\\n' "$1" >> ${JSON.stringify(commandLog)}\nif [ "$1" = info ]; then printf 'test-engine'; fi\nexit 0\n`, + { mode: 0o755 }, + ); + vi.stubEnv("PATH", `${bin}${path.delimiter}${process.env.PATH ?? ""}`); + try { + const controller = new AbortController(); + const projection = { + stage: vi.fn(async () => { + controller.abort(new Error("Gateway revoked the projection")); + return path.join(root, "staged-projection"); + }), + remove: vi.fn(async () => undefined), + } as unknown as NodeWorkerMemoryProjectionRuntime; + const supervisor = createNodeWorkerSupervisor({ bundleRoot, env, memoryProjection: projection }); + const input = launchInput(workspaceDir, "cancelled-before-container-start"); + input.execution = { kind: "container-v1" }; + input.descriptor.assignment.memoryReadEnforced = true; + input.descriptor.assignment.workspaceDir = "/workspace"; + input.memoryProjection = testNodeWorkerMemoryProjection(input); + + await expect(supervisor.launch(input, TEST_WORKER_ENDPOINT, controller.signal)).resolves.toMatchObject({ + state: "cancelled", + errorText: "node worker launch cancelled before process start", + }); + expect(projection.stage).toHaveBeenCalledOnce(); + expect(projection.remove).toHaveBeenCalledWith({ + gatewayNamespace: input.gatewayNamespace, + launchId: input.launchId, + planHash: testNodeWorkerLaunchIdentity(input).planHash, + }); + expect(fs.readFileSync(commandLog, "utf8").split("\n")).not.toContain("run"); + expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)).toMatchObject({ + state: "cancelled", + }); + await supervisor.close(); + } finally { + vi.unstubAllEnvs(); + } + }, + ); + it("keeps construction and close inert without resolving process identity", async () => { const root = tempDirs.make("node-worker-inert-"); const { bundleRoot, env } = writeNodeWorkerFixture(root); @@ -513,7 +594,7 @@ describe("node worker supervisor", () => { }, ); - it("does not open or signal a child after markRunning observes its terminal receipt", async () => { + it("returns a terminal receipt when execution-ready races an immediate child completion", async () => { const { supervisor, workspaceDir } = fixture(); const input = launchInput(workspaceDir, "fast-terminal-launch", "fast-terminal"); vi.spyOn(NodeWorkerLaunchStore.prototype, "markRunning").mockImplementation( @@ -536,7 +617,7 @@ describe("node worker supervisor", () => { await new Promise((resolve) => { setTimeout(resolve, 150); }); - expect(fs.existsSync(marker)).toBe(false); + expect(fs.existsSync(marker)).toBe(true); await supervisor.close(); }); @@ -549,7 +630,88 @@ describe("node worker supervisor", () => { const terminal = await waitForTerminal(supervisor, input.launchId); expect(fs.existsSync(exitedPath)).toBe(true); - expect(terminal.state).toBe("failed"); + expect(terminal).toMatchObject({ state: "failed", worker: null }); + await supervisor.close(); + }); + + it("fails closed before durable running when the child acknowledges another launch", async () => { + const { env, supervisor, workspaceDir } = fixture(); + const input = launchInput(workspaceDir, "wrong-execution-ack-launch", "wrong-execution-ack"); + + await expect(supervisor.launch(input, TEST_WORKER_ENDPOINT)).resolves.toMatchObject({ + state: "interrupted", + worker: null, + }); + expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)).toMatchObject({ + state: "interrupted", + worker: null, + }); + await supervisor.close(); + }); + + it("persists cancellation before a child acknowledgement without a worker identity", async () => { + const { env, supervisor, workspaceDir } = fixture(); + const input = launchInput( + workspaceDir, + "cancel-before-execution-ack-launch", + "wait-before-execution-ack", + ); + const launching = supervisor.launch(input, TEST_WORKER_ENDPOINT); + await vi.waitFor(async () => { + expect((await supervisor.status(input.launchId))?.state).toBe("pending"); + }); + + await expect(supervisor.cancel(testNodeWorkerLaunchIdentity(input))).resolves.toMatchObject({ + state: "cancelled", + worker: null, + }); + await expect(launching).resolves.toMatchObject({ state: "cancelled", worker: null }); + expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)).toMatchObject({ + state: "cancelled", + worker: null, + }); + await supervisor.close(); + }); + + it("interrupts a pending child on close without a durable worker identity", async () => { + const { env, supervisor, workspaceDir } = fixture(); + const input = launchInput( + workspaceDir, + "close-before-execution-ack-launch", + "wait-before-execution-ack", + ); + const launching = supervisor.launch(input, TEST_WORKER_ENDPOINT); + await vi.waitFor(async () => { + expect((await supervisor.status(input.launchId))?.state).toBe("pending"); + }); + + await supervisor.close(); + await expect(launching).resolves.toMatchObject({ state: "interrupted", worker: null }); + expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)).toMatchObject({ + state: "interrupted", + worker: null, + }); + }); + + it("retires a child that replays its execution acknowledgement after durable readiness", async () => { + const { supervisor, workspaceDir } = fixture(); + const input = launchInput( + workspaceDir, + "replayed-execution-ack-launch", + "replayed-execution-ack", + ); + + await expect(supervisor.launch(input, TEST_WORKER_ENDPOINT)).resolves.toMatchObject({ + state: "running", + worker: expect.any(Object), + }); + await vi.waitFor(async () => { + expect((await supervisor.status(input.launchId))?.state).toBe("interrupted"); + }); + expect(await supervisor.status(input.launchId)).toMatchObject({ + state: "interrupted", + worker: expect.any(Object), + }); await supervisor.close(); }); @@ -606,7 +768,7 @@ describe("node worker supervisor", () => { ["cancel", "cancelled"], ["close", "interrupted"], ] as const)( - "%s during startup closes the gate before worker code runs", + "%s after execution readiness settles the durable terminal receipt", async (operation, state) => { const { supervisor, workspaceDir } = fixture(); const input = launchInput(workspaceDir, `${operation}-startup-launch`, "tree"); @@ -630,7 +792,6 @@ describe("node worker supervisor", () => { await stopping; expect((await supervisor.status(input.launchId))?.state).toBe(state); - expect(fs.existsSync(path.join(workspaceDir, "grandchild.pid"))).toBe(false); await supervisor.close(); }, ); diff --git a/src/node-host/node-worker-supervisor.ts b/src/node-host/node-worker-supervisor.ts index 96d868efbae6..035b7f469c25 100644 --- a/src/node-host/node-worker-supervisor.ts +++ b/src/node-host/node-worker-supervisor.ts @@ -1,4 +1,6 @@ +import fs from "node:fs"; import path from "node:path"; +import { fileURLToPath } from "node:url"; import { resolveStateDir } from "../config/paths.js"; import type { NodeWorkerCapacitySnapshot } from "../infra/node-runner-inventory.js"; import { registerSecretValueForRedaction } from "../logging/secret-redaction-registry.js"; @@ -13,7 +15,11 @@ import { parseWorkerLaunchPlan, type WorkerLaunchDescriptor, } from "../worker/launch-descriptor.js"; -import { parseNodeWorkerConnectionFailureMessage } from "../worker/node-supervisor-protocol.js"; +import { + NODE_WORKER_EXECUTION_CONTAINER_V1, + parseNodeWorkerConnectionFailureMessage, + parseNodeWorkerExecutionStartedMessage, +} from "../worker/node-supervisor-protocol.js"; import type { NodeWorkerWorkspaceRetainInput, NodeWorkerWorkspaceRetainResult, @@ -21,6 +27,14 @@ import type { import { formatWorkerConnectionFailure } from "../worker/worker-connection-contract.js"; import type { WorkerConnectionEndpoint } from "../worker/worker-connection-endpoint.js"; import { NodeWorkerCapacity } from "./node-worker-capacity.js"; +import { + NODE_WORKER_CONTAINER_SHIM_FLAG, + nodeWorkerContainerEngineFor, + removeOwnedNodeWorkerContainers, + resolveNodeWorkerContainerEngine, + type NodeWorkerContainerEngine, + type NodeWorkerContainerIdentity, +} from "./node-worker-container-runtime.js"; import { resolveNodeWorkerEntry } from "./node-worker-entry.js"; import { snapshotNodeWorkerEnv } from "./node-worker-environment.js"; import { @@ -28,6 +42,7 @@ import { type NodeWorkerLaunchReceipt, type NodeWorkerTerminalState, } from "./node-worker-launch-store.js"; +import { NodeWorkerMemoryProjectionRuntime } from "./node-worker-memory-projection.js"; import { createNodeWorkerCredentialScrubber, NODE_WORKER_STDERR_MAX_BYTES, @@ -42,6 +57,7 @@ import { type NodeWorkerProcessIdentity, } from "./node-worker-process-identity.js"; import { + nodeWorkerMemoryProjectionLaunchBinding, nodeWorkerPlanHash, type NodeWorkerLaunchInput, type NodeWorkerSupervisorIdentity, @@ -55,25 +71,85 @@ import { NodeWorkerWorkspaceRuntime } from "./node-worker-workspace.js"; const STOP_GRACE_MS = 1_000; const FORCE_STOP_WAIT_MS = 4_000; +const MAX_LEASE_TIMER_DELAY_MS = 2_147_483_647; const GATEWAY_NAMESPACE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; const BUNDLE_HASH_PATTERN = /^[a-f0-9]{64}$/u; +type NodeWorkerContainerLaunch = { + engine: NodeWorkerContainerEngine; + identity: NodeWorkerContainerIdentity; + bundleDir: string; + relayDir: string; + memoryDir: string; + workspaceDir: string; +}; + +function resolveContainerShimArgv(): string[] { + const currentFile = fileURLToPath(import.meta.url); + const extension = path.extname(currentFile); + const sourceCandidate = path.join( + path.dirname(currentFile), + `node-worker-container-shim${extension}`, + ); + const installedCandidate = path.join( + path.dirname(currentFile), + "node-host", + "node-worker-container-shim.js", + ); + const entry = [sourceCandidate, installedCandidate].find((candidate) => { + try { + return fs.statSync(candidate).isFile(); + } catch { + return false; + } + }); + if (!entry) { + throw new Error("node worker container shim is unavailable in this node-host installation"); + } + // Source-checkout node hosts run TypeScript through tsx; packaged node hosts + // resolve the explicit stable tsdown entry above and execute plain JavaScript. + return path.extname(entry) === ".ts" + ? [ + process.execPath, + "--import", + import.meta.resolve("tsx"), + entry, + NODE_WORKER_CONTAINER_SHIM_FLAG, + ] + : [process.execPath, entry, NODE_WORKER_CONTAINER_SHIM_FLAG]; +} + type ChildAdapter = Awaited>; type StopState = Extract; type ActiveBase = { launchId: string; planHash: string; supervisor: NodeWorkerProcessIdentity; - worker: NodeWorkerProcessIdentity; + worker: NodeWorkerProcessIdentity | null; }; -type RunningChild = ActiveBase & { - state: "running"; +type ActiveContainerLaunch = { + engine: NodeWorkerContainerEngine; + identity: NodeWorkerContainerIdentity; + gatewayNamespace: string; +}; +type ExecutionReady = { + promise: Promise; + settled: boolean; + resolve: (receipt: NodeWorkerLaunchReceipt) => void; + reject: (error: Error) => void; +}; +type ActiveChild = ActiveBase & { + /** Pending is durable until the child attests it passed the exact start gate. */ + state: "starting" | "running"; + candidateWorker: NodeWorkerProcessIdentity; adapter: ChildAdapter; done: Promise; - journalReady: Promise; - releaseJournal: () => void; + executionReady: ExecutionReady; scrubber: NodeWorkerCredentialScrubber; connectionFailure: { errorText?: string }; + container?: ActiveContainerLaunch; + leaseExpiresAtMs?: number; + leaseTimer?: NodeJS.Timeout; stopState?: StopState; }; type TerminalOutcome = Readonly<{ @@ -84,9 +160,10 @@ type TerminalOutcome = Readonly<{ type ObservedTerminal = ActiveBase & { state: "observed"; outcome: TerminalOutcome; + container?: ActiveContainerLaunch; persistenceError?: unknown; }; -type ActiveOwnership = RunningChild | ObservedTerminal; +type ActiveOwnership = ActiveChild | ObservedTerminal; type NodeWorkerSupervisorOptions = { bundleRoot?: string; env?: NodeJS.ProcessEnv; @@ -94,6 +171,8 @@ type NodeWorkerSupervisorOptions = { capacityWaitMs?: number; onCapacityChanged?: (capacity: NodeWorkerCapacitySnapshot) => void; workspace?: NodeWorkerWorkspaceRuntime; + memoryProjection?: NodeWorkerMemoryProjectionRuntime; + now?: () => number; }; function sameProcessIdentity( @@ -118,15 +197,78 @@ function receiptMatchesOwner( ); } +function createExecutionReady(): ExecutionReady { + let resolvePromise!: (receipt: NodeWorkerLaunchReceipt) => void; + let rejectPromise!: (error: Error) => void; + const executionReady: ExecutionReady = { + settled: false, + promise: new Promise((resolve, reject) => { + resolvePromise = resolve; + rejectPromise = reject; + }), + resolve: (receipt) => { + if (!executionReady.settled) { + executionReady.settled = true; + resolvePromise(receipt); + } + }, + reject: (error) => { + if (!executionReady.settled) { + executionReady.settled = true; + rejectPromise(error); + } + }, + }; + // The owner attaches after it opens the start gate. Avoid a pre-gate rogue + // child acknowledgement becoming an unhandled rejection before that point. + void executionReady.promise.catch(() => undefined); + return executionReady; +} + +async function waitForExecutionReady( + active: ActiveChild, + signal?: AbortSignal, +): Promise { + if (!signal) { + return await active.executionReady.promise; + } + if (signal.aborted) { + throw signal.reason instanceof Error + ? signal.reason + : new Error("node worker launch cancelled"); + } + return await new Promise((resolve, reject) => { + const onAbort = () => + reject( + signal.reason instanceof Error ? signal.reason : new Error("node worker launch cancelled"), + ); + signal.addEventListener("abort", onAbort, { once: true }); + void active.executionReady.promise.then( + (receipt) => { + signal.removeEventListener("abort", onAbort); + resolve(receipt); + }, + (error: unknown) => { + signal.removeEventListener("abort", onAbort); + reject(error instanceof Error ? error : new Error("node worker execution did not start")); + }, + ); + }); +} + /** Owns worker process groups, lifetime gates, and the durable node-host launch journal. */ class NodeWorkerSupervisor { private readonly active = new Map(); private readonly starting = new Map>(); + /** Cancels the narrow interval after durable claim but before the child is active. */ + private readonly startingAborts = new Map(); private readonly bundleRoot: string; private readonly store: NodeWorkerLaunchStore; private readonly workerEnv: NodeJS.ProcessEnv; private readonly capacity: NodeWorkerCapacity; private readonly workspace: NodeWorkerWorkspaceRuntime; + private memoryProjection?: NodeWorkerMemoryProjectionRuntime; + private readonly now: () => number; private supervisorIdentity?: NodeWorkerProcessIdentity; private initializationPromise?: Promise; private closed = false; @@ -142,6 +284,8 @@ class NodeWorkerSupervisor { this.workspace = options.workspace ?? new NodeWorkerWorkspaceRuntime({ root: this.bundleRoot, env: this.workerEnv }); + this.memoryProjection = options.memoryProjection; + this.now = options.now ?? Date.now; this.capacity = new NodeWorkerCapacity(this.store, options); } @@ -149,9 +293,15 @@ class NodeWorkerSupervisor { return (this.supervisorIdentity ??= requireNodeWorkerProcessIdentity(process.pid)); } + private requireMemoryProjection(): NodeWorkerMemoryProjectionRuntime { + return (this.memoryProjection ??= new NodeWorkerMemoryProjectionRuntime({ + root: this.bundleRoot, + })); + } + initialize(): Promise { return (this.initializationPromise ??= this.capacity.initialize(async (receipt) => { - await this.recoverRunning(receipt, false); + await this.recoverNonterminal(receipt, false); })); } @@ -171,6 +321,33 @@ class NodeWorkerSupervisor { } const plan = parseWorkerLaunchPlan(structuredClone(input.descriptor)); const descriptor = completeWorkerLaunchDescriptor(plan, connectionEndpoint); + if ( + descriptor.assignment.memoryReadEnforced && + input.execution.kind !== NODE_WORKER_EXECUTION_CONTAINER_V1 + ) { + throw new Error("enforced memory worker requires container-v1 execution"); + } + if (descriptor.assignment.memoryReadEnforced && !input.memoryProjection) { + throw new Error("enforced memory worker requires an issued memory projection"); + } + if ( + input.memoryProjection && + input.memoryProjection.binding.launch !== + nodeWorkerMemoryProjectionLaunchBinding({ + ...input, + descriptor, + }) + ) { + throw new Error("memory projection does not match its worker launch"); + } + if ( + input.execution.kind === NODE_WORKER_EXECUTION_CONTAINER_V1 && + (descriptor.assignment.workspaceDir !== "/workspace" || + (descriptor.assignment.workerContainmentRoot !== undefined && + descriptor.assignment.workerContainmentRoot !== "/workspace")) + ) { + throw new Error("container worker descriptor must use the fixed /workspace root"); + } if (descriptor.admission.handshake.bundleHash !== input.expectedBundleHash) { throw new Error("node worker descriptor bundle hash does not match the launch bundle"); } @@ -185,7 +362,7 @@ class NodeWorkerSupervisor { throw new Error(`node worker launch ${input.launchId} was replayed with a different plan`); } if (local.state === "observed") { - return this.reconcileActiveTerminal(local); + return await this.reconcileActiveTerminal(local); } const receipt = this.store.get(input.launchId); if (receipt) { @@ -208,24 +385,38 @@ class NodeWorkerSupervisor { } const claim = await this.capacity.claim(claimInput, supervisor, signal); if (claim.action === "recover") { - return await this.recoverRunning(claim.receipt); + return await this.recoverNonterminal(claim.receipt); } if (claim.action === "replay") { const replay = this.active.get(input.launchId); if (replay?.planHash === planHash && replay.state === "observed") { - return this.reconcileActiveTerminal(replay); + return await this.reconcileActiveTerminal(replay); } const startup = this.starting.get(input.launchId); return startup && claim.receipt.state === "pending" ? await startup : claim.receipt; } - const startup = this.startClaimed({ input, descriptor, planHash, supervisor }); + const startupAbort = new AbortController(); + const startupSignal = signal + ? AbortSignal.any([signal, startupAbort.signal]) + : startupAbort.signal; + const startup = this.startClaimed({ + input, + descriptor, + planHash, + supervisor, + signal: startupSignal, + }); this.starting.set(input.launchId, startup); + this.startingAborts.set(input.launchId, startupAbort); try { return await startup; } finally { if (this.starting.get(input.launchId) === startup) { this.starting.delete(input.launchId); } + if (this.startingAborts.get(input.launchId) === startupAbort) { + this.startingAborts.delete(input.launchId); + } } } @@ -233,9 +424,9 @@ class NodeWorkerSupervisor { await this.initialize(); const active = this.active.get(launchId); if (active?.state === "observed") { - return this.reconcileActiveTerminal(active); + return await this.reconcileActiveTerminal(active); } - if (active?.state === "running") { + if (active?.state === "running" && active.worker) { const workerState = inspectNodeWorkerProcessIdentity(active.worker); if (workerState === "dead" || workerState === "reused") { let treeState = inspectOwnedNodeWorkerTree(active.worker); @@ -250,13 +441,15 @@ class NodeWorkerSupervisor { await active.done; const observed = this.active.get(launchId); if (observed?.state === "observed") { - return this.reconcileActiveTerminal(observed); + return await this.reconcileActiveTerminal(observed); } } return this.store.get(launchId); } const receipt = this.store.get(launchId); - return receipt?.state === "running" ? await this.recoverRunning(receipt) : receipt; + return receipt?.state === "pending" || receipt?.state === "running" + ? await this.recoverNonterminal(receipt) + : receipt; } async retainWorkspaces( @@ -290,17 +483,20 @@ class NodeWorkerSupervisor { ) { return receipt; } - if (active.state === "running") { + if (active.state !== "observed") { await this.stopChild(active, "cancelled"); } const observed = this.active.get(expected.launchId); if (observed?.state === "observed") { - return this.reconcileActiveTerminal(observed); + return await this.reconcileActiveTerminal(observed); } return this.store.getMatching(expected); } const startup = this.starting.get(expected.launchId); if (startup && receipt.state === "pending" && receipt.supervisor.pid === process.pid) { + this.startingAborts + .get(expected.launchId) + ?.abort(new Error("node worker launch cancelled before execution acknowledgement")); const cancelled = this.capacity.finishCancelled({ expected, supervisor: receipt.supervisor, @@ -349,6 +545,7 @@ class NodeWorkerSupervisor { if (workerState !== "dead") { return this.store.getMatching(expected); } + await this.cleanupRecoveredContainerLaunch(receipt); return this.capacity.finishCancelled({ expected, supervisor: receipt.supervisor, @@ -362,6 +559,9 @@ class NodeWorkerSupervisor { } this.closed = true; this.capacity.close(); + for (const controller of this.startingAborts.values()) { + controller.abort(new Error("node worker supervisor closed before execution acknowledgement")); + } const operation = (async () => { const errors: unknown[] = []; if (this.initializationPromise) { @@ -374,7 +574,7 @@ class NodeWorkerSupervisor { await Promise.allSettled(this.starting.values()); await Promise.all( [...this.active.values()] - .filter((active): active is RunningChild => active.state === "running") + .filter((active): active is ActiveChild => active.state !== "observed") .map(async (active) => await this.stopChild(active, "interrupted")), ); for (const active of this.active.values()) { @@ -382,7 +582,7 @@ class NodeWorkerSupervisor { continue; } try { - this.reconcileActiveTerminal(active); + await this.reconcileActiveTerminal(active); } catch (error) { errors.push(error); } @@ -403,8 +603,13 @@ class NodeWorkerSupervisor { return closePromise; } - private reconcileActiveTerminal(active: ObservedTerminal): NodeWorkerLaunchReceipt { + private async reconcileActiveTerminal( + active: ObservedTerminal, + ): Promise { try { + if (active.container) { + await this.cleanupContainerLaunch(active.container); + } const receipt = this.capacity.finish({ launchId: active.launchId, planHash: active.planHash, @@ -425,17 +630,78 @@ class NodeWorkerSupervisor { } } - private async recoverRunning( + private clearLeaseTimer(active: ActiveChild): void { + if (active.leaseTimer) { + clearTimeout(active.leaseTimer); + active.leaseTimer = undefined; + } + } + + private armLeaseTimer(active: ActiveChild): void { + const expiresAtMs = active.leaseExpiresAtMs; + if (expiresAtMs === undefined) { + return; + } + this.clearLeaseTimer(active); + const remainingMs = expiresAtMs - this.now(); + if (remainingMs <= 0) { + void this.expireLease(active); + return; + } + active.leaseTimer = setTimeout( + () => void this.expireLease(active), + Math.min(remainingMs, MAX_LEASE_TIMER_DELAY_MS), + ); + active.leaseTimer.unref?.(); + } + + private async expireLease(active: ActiveChild): Promise { + if (this.active.get(active.launchId) !== active || active.stopState) { + return; + } + const expiresAtMs = active.leaseExpiresAtMs; + if (expiresAtMs === undefined || expiresAtMs > this.now()) { + this.armLeaseTimer(active); + return; + } + await this.stopChild(active, "cancelled"); + } + + private async recoverNonterminal( receipt: NodeWorkerLaunchReceipt, notifyCapacity = true, ): Promise { - if (receipt.state !== "running" || !receipt.worker) { + if (receipt.state !== "pending" && receipt.state !== "running") { return receipt; } const previousSupervisor = inspectNodeWorkerProcessIdentity(receipt.supervisor); if (previousSupervisor !== "dead" && previousSupervisor !== "reused") { return this.store.get(receipt.launchId) ?? receipt; } + const lease = this.store.getContainerLaunchLease({ + launchId: receipt.launchId, + planHash: receipt.planHash, + }); + const projectionLeaseExpired = lease !== undefined && lease.expiresAtMs <= this.now(); + if (receipt.state === "pending") { + await this.cleanupRecoveredContainerLaunch(receipt); + return this.capacity.finish( + { + launchId: receipt.launchId, + planHash: receipt.planHash, + supervisor: receipt.supervisor, + worker: null, + state: projectionLeaseExpired ? "cancelled" : "interrupted", + errorText: projectionLeaseExpired + ? "node worker memory projection lease expired before the worker launch started" + : "node host stopped before the worker launch started", + }, + notifyCapacity, + ); + } + if (!receipt.worker) { + return receipt; + } let workerState = inspectOwnedNodeWorkerTree(receipt.worker); if (workerState === "unknown") { return this.store.get(receipt.launchId) ?? receipt; @@ -451,24 +717,165 @@ class NodeWorkerSupervisor { if (workerState !== "dead") { return this.store.get(receipt.launchId) ?? receipt; } + await this.cleanupRecoveredContainerLaunch(receipt); return this.capacity.finish( { launchId: receipt.launchId, planHash: receipt.planHash, supervisor: receipt.supervisor, worker: receipt.worker, - state: "interrupted", - errorText: "node host stopped before the worker launch completed", + state: projectionLeaseExpired ? "cancelled" : "interrupted", + errorText: projectionLeaseExpired + ? "node worker memory projection lease expired during node-host recovery" + : "node host stopped before the worker launch completed", }, notifyCapacity, ); } + private async prepareContainerLaunch(params: { + input: NodeWorkerLaunchInput; + descriptor: WorkerLaunchDescriptor; + planHash: string; + entry: string; + signal?: AbortSignal; + }): Promise { + params.signal?.throwIfAborted(); + const engine = await resolveNodeWorkerContainerEngine(); + params.signal?.throwIfAborted(); + if (!engine) { + throw new Error("node host has no eligible container process-isolation runtime"); + } + const identity = { launchId: params.input.launchId, planHash: params.planHash }; + if (!params.input.memoryProjection) { + throw new Error("enforced node worker container launch omitted its memory projection"); + } + // Persist cleanup ownership before preparing any local resource. A crash + // during staging then recovers the exact engine, relay, and projection by + // durable launch identity without persisting a descriptor or credential. + params.signal?.throwIfAborted(); + this.store.recordContainerLaunch({ + launchId: params.input.launchId, + planHash: params.planHash, + engine: engine.id, + expiresAtMs: params.input.memoryProjection.expiresAtMs, + }); + params.signal?.throwIfAborted(); + // An interrupted launch can leave only its exact labelled container behind. + // Reclaim that identity before staging so a retry cannot inherit another + // turn's process or fail closed forever on Docker's global name registry. + await removeOwnedNodeWorkerContainers(identity, engine); + params.signal?.throwIfAborted(); + const workspaceDir = this.workspace.resolveContainerWorkspace({ + gatewayNamespace: params.input.gatewayNamespace, + environmentId: params.descriptor.admission.environmentId, + sessionId: params.descriptor.admission.sessionId, + ownerEpoch: params.descriptor.admission.ownerEpoch, + }); + const relayDir = this.workspace.resolveContainerRelayDirectory({ + gatewayNamespace: params.input.gatewayNamespace, + ...identity, + }); + params.signal?.throwIfAborted(); + const memoryDir = await this.requireMemoryProjection().stage({ + identity: { + gatewayNamespace: params.input.gatewayNamespace, + ...identity, + }, + projection: params.input.memoryProjection, + endpoint: params.descriptor.connectionEndpoint, + ...(params.signal ? { signal: params.signal } : {}), + }); + params.signal?.throwIfAborted(); + if (params.input.memoryProjection.expiresAtMs <= this.now()) { + throw new Error("node worker memory projection lease expired before container start"); + } + return { + engine, + identity, + // `entry` has already passed the exact bundle-root and hash checks. + bundleDir: path.dirname(params.entry), + workspaceDir, + relayDir, + memoryDir, + }; + } + + private async cleanupContainerLaunch(params: { + engine: NodeWorkerContainerEngine; + identity: NodeWorkerContainerIdentity; + gatewayNamespace: string; + }): Promise { + await removeOwnedNodeWorkerContainers(params.identity, params.engine); + await this.workspace.removeContainerRelayDirectory({ + gatewayNamespace: params.gatewayNamespace, + ...params.identity, + }); + await this.requireMemoryProjection().remove({ + gatewayNamespace: params.gatewayNamespace, + ...params.identity, + }); + } + + private async cleanupRecoveredContainerLaunch(receipt: NodeWorkerLaunchReceipt): Promise { + const engineId = this.store.getContainerLaunchEngine({ + launchId: receipt.launchId, + planHash: receipt.planHash, + }); + if (!engineId) { + return; + } + await this.cleanupContainerLaunch({ + engine: nodeWorkerContainerEngineFor(engineId), + identity: { launchId: receipt.launchId, planHash: receipt.planHash }, + gatewayNamespace: receipt.gatewayNamespace, + }); + } + + private async finishStart(params: { + input: NodeWorkerLaunchInput; + planHash: string; + supervisor: NodeWorkerProcessIdentity; + state: "cancelled" | "failed"; + errorText: string; + }): Promise { + const engineId = this.store.getContainerLaunchEngine({ + launchId: params.input.launchId, + planHash: params.planHash, + }); + if (!engineId) { + return this.capacity.finish({ + launchId: params.input.launchId, + planHash: params.planHash, + supervisor: params.supervisor, + worker: null, + state: params.state, + errorText: params.errorText, + }); + } + const observed: ObservedTerminal = { + state: "observed", + launchId: params.input.launchId, + planHash: params.planHash, + supervisor: params.supervisor, + worker: null, + outcome: { state: params.state, errorText: params.errorText }, + container: { + engine: nodeWorkerContainerEngineFor(engineId), + identity: { launchId: params.input.launchId, planHash: params.planHash }, + gatewayNamespace: params.input.gatewayNamespace, + }, + }; + this.active.set(observed.launchId, observed); + return await this.reconcileActiveTerminal(observed); + } + private async startClaimed(params: { input: NodeWorkerLaunchInput; descriptor: WorkerLaunchDescriptor; planHash: string; supervisor: NodeWorkerProcessIdentity; + signal?: AbortSignal; }): Promise { const credential = params.descriptor.admission.credential; const endpoint = params.descriptor.connectionEndpoint; @@ -484,18 +891,98 @@ class NodeWorkerSupervisor { registerSecretValueForRedaction(value); } let adapter: ChildAdapter; + let container: NodeWorkerContainerLaunch | undefined; + let active: ActiveChild | undefined; + let preGateExecutionError: Error | undefined; + const rejectExecutionReady = (error: Error) => { + if (!active) { + preGateExecutionError ??= error; + return; + } + if (!active.executionReady.settled) { + active.executionReady.reject(error); + return; + } + // A replay after a valid acknowledgement is hostile child behavior, not a + // harmless diagnostic. Retire the tree before it can execute more tools. + active.connectionFailure.errorText ??= error.message; + void this.stopChild(active, "interrupted").catch(() => undefined); + }; try { const entry = resolveNodeWorkerEntry({ bundleRoot: this.bundleRoot, expectedBundleHash: params.input.expectedBundleHash, gatewayNamespace: params.input.gatewayNamespace, }); + if (params.input.execution.kind === NODE_WORKER_EXECUTION_CONTAINER_V1) { + container = await this.prepareContainerLaunch({ + input: params.input, + descriptor: params.descriptor, + planHash: params.planHash, + entry, + ...(params.signal ? { signal: params.signal } : {}), + }); + } adapter = await createChildAdapter({ - argv: [process.execPath, entry, "--internal-worker-ipc"], + argv: container + ? resolveContainerShimArgv() + : [process.execPath, entry, "--internal-worker-ipc"], env: this.workerEnv, exactEnv: true, ownedWorker: true, onWorkerMessage: (message) => { + const execution = parseNodeWorkerExecutionStartedMessage(message); + if (execution) { + if ( + execution.launchId !== params.input.launchId || + execution.planHash !== params.planHash + ) { + rejectExecutionReady( + new Error( + "node worker execution acknowledgement did not match its launch identity", + ), + ); + return; + } + if (!active || active.state !== "starting") { + rejectExecutionReady(new Error("node worker replayed its execution acknowledgement")); + return; + } + if (this.closed || params.signal?.aborted) { + rejectExecutionReady( + new Error("node worker execution acknowledgement arrived after cancellation"), + ); + return; + } + try { + const running = this.store.markRunning({ + launchId: active.launchId, + planHash: active.planHash, + supervisor: active.supervisor, + worker: active.candidateWorker, + }); + if (running.state !== "running") { + rejectExecutionReady( + new Error( + "node worker execution acknowledgement could not promote its pending launch", + ), + ); + return; + } + // The store transition is synchronous. A terminal child event queued + // after this handler therefore retains the durable execution fact. + active.worker = active.candidateWorker; + active.state = "running"; + active.executionReady.resolve(running); + } catch (error) { + rejectExecutionReady( + error instanceof Error + ? error + : new Error("node worker execution acknowledgement failed"), + ); + } + return; + } const diagnostic = parseNodeWorkerConnectionFailureMessage(message); if (!diagnostic) { return; @@ -511,26 +998,57 @@ class NodeWorkerSupervisor { ) : undefined; }, - input: JSON.stringify(params.descriptor), + input: JSON.stringify( + container + ? { + descriptor: params.descriptor, + engine: container.engine.id, + identity: container.identity, + mounts: { + bundleDir: container.bundleDir, + relayDir: container.relayDir, + memoryDir: container.memoryDir, + workspaceDir: container.workspaceDir, + }, + } + : params.descriptor, + ), }); } catch (error) { - return this.capacity.finish({ - launchId: params.input.launchId, + const cancelled = params.signal?.aborted === true && !this.closed; + return await this.finishStart({ + input: params.input, planHash: params.planHash, supervisor: params.supervisor, - worker: null, - state: "failed", - errorText: sanitizeNodeWorkerDiagnostic(error, "node worker spawn failed", scrubber.scrub), + state: this.closed ? "interrupted" : cancelled ? "cancelled" : "failed", + errorText: cancelled + ? "node worker launch cancelled before process start" + : this.closed + ? "node host stopped before the worker launch started" + : sanitizeNodeWorkerDiagnostic(error, "node worker spawn failed", scrubber.scrub), + }); + } + if (params.signal?.aborted) { + adapter.kill("SIGKILL"); + await adapter.wait().catch(() => undefined); + adapter.dispose(); + return await this.finishStart({ + input: params.input, + planHash: params.planHash, + supervisor: params.supervisor, + state: this.closed ? "interrupted" : "cancelled", + errorText: this.closed + ? "node host stopped before the worker launch started" + : "node worker launch cancelled before process start", }); } if (!adapter.pid) { adapter.kill("SIGKILL"); adapter.dispose(); - return this.capacity.finish({ - launchId: params.input.launchId, + return await this.finishStart({ + input: params.input, planHash: params.planHash, supervisor: params.supervisor, - worker: null, state: "failed", errorText: "node worker spawn did not return a process id", }); @@ -542,11 +1060,10 @@ class NodeWorkerSupervisor { adapter.kill("SIGKILL"); await adapter.wait().catch(() => undefined); adapter.dispose(); - return this.capacity.finish({ - launchId: params.input.launchId, + return await this.finishStart({ + input: params.input, planHash: params.planHash, supervisor: params.supervisor, - worker: null, state: "failed", errorText: sanitizeNodeWorkerDiagnostic( error, @@ -555,68 +1072,85 @@ class NodeWorkerSupervisor { ), }); } - let journalReleased = false; - let releaseJournalPromise!: () => void; - const journalReady = new Promise((resolve) => { - releaseJournalPromise = resolve; - }); - const releaseJournal = () => { - if (!journalReleased) { - journalReleased = true; - releaseJournalPromise(); - } - }; - const active = { - state: "running", + active = { + state: "starting", adapter, - journalReady, + done: Promise.resolve(), launchId: params.input.launchId, planHash: params.planHash, - releaseJournal, scrubber, connectionFailure, supervisor: params.supervisor, - worker, - } as RunningChild; + worker: null, + candidateWorker: worker, + executionReady: createExecutionReady(), + ...(container + ? { + container: { + engine: container.engine, + identity: container.identity, + gatewayNamespace: params.input.gatewayNamespace, + }, + leaseExpiresAtMs: params.input.memoryProjection?.expiresAtMs, + } + : {}), + }; active.done = this.observeChild(active); this.active.set(active.launchId, active); void active.done.catch(() => undefined); - let running: NodeWorkerLaunchReceipt; - try { - running = this.store.markRunning({ - launchId: active.launchId, - planHash: active.planHash, - supervisor: params.supervisor, - worker, - }); - } catch (error) { - active.releaseJournal(); - await this.stopChild(active, "interrupted").catch(() => undefined); - throw error; + if (preGateExecutionError) { + active.executionReady.reject(preGateExecutionError); } - active.releaseJournal(); - if (running.state === "cancelled" || running.state === "interrupted") { - await this.stopChild(active, running.state); - return this.store.get(active.launchId) ?? running; + if (params.signal?.aborted) { + await this.stopChild(active, "cancelled"); + const cancelled = this.store.get(active.launchId); + if (!cancelled) { + throw new Error("cancelled node worker launch was not persisted"); + } + return cancelled; } - if (running.state !== "running") { - adapter.closeStartGate?.(); - return running; + if (active.leaseExpiresAtMs !== undefined && active.leaseExpiresAtMs <= this.now()) { + await this.stopChild(active, "cancelled"); + const settled = this.store.get(active.launchId); + if (settled) { + return settled; + } + throw new Error("expired node worker launch was not persisted"); } if (this.closed) { await this.stopChild(active, "interrupted"); - return this.store.get(active.launchId) ?? running; + const settled = this.store.get(active.launchId); + if (settled) { + return settled; + } + throw new Error("interrupted node worker launch was not persisted"); } try { - await adapter.openStartGate?.(); - } catch { - await this.stopChild(active, "interrupted"); - return this.store.get(active.launchId) ?? running; + await adapter.openStartGate?.({ launchId: active.launchId, planHash: active.planHash }); + const running = await waitForExecutionReady(active, params.signal); + if (active.leaseExpiresAtMs !== undefined) { + this.armLeaseTimer(active); + } + return running; + } catch (error) { + const state: StopState = this.closed + ? "interrupted" + : params.signal?.aborted + ? "cancelled" + : "interrupted"; + active.executionReady.reject( + error instanceof Error ? error : new Error("node worker execution did not become ready"), + ); + await this.stopChild(active, state).catch(() => undefined); + const settled = this.store.get(active.launchId); + if (settled) { + return settled; + } + throw error; } - return running; } - private async observeChild(active: RunningChild): Promise { + private async observeChild(active: ActiveChild): Promise { const stdout = createCapturedOutputBuffers(); const stderr = createCapturedOutputBuffers(); active.adapter.onStdout((chunk) => @@ -633,7 +1167,6 @@ class NodeWorkerSupervisor { let outcome: TerminalOutcome; try { const exit = await active.adapter.wait(); - await active.journalReady; if (active.stopState) { outcome = Object.freeze({ state: active.stopState, @@ -674,7 +1207,6 @@ class NodeWorkerSupervisor { }); } } catch (error) { - await active.journalReady; outcome = Object.freeze({ state: active.stopState ?? "failed", errorText: @@ -682,8 +1214,10 @@ class NodeWorkerSupervisor { sanitizeNodeWorkerDiagnostic(error, "node worker wait failed", active.scrubber.scrub), }); } finally { + this.clearLeaseTimer(active); active.adapter.dispose(); } + active.executionReady.reject(new Error("node worker exited before execution became ready")); const observed: ObservedTerminal = { state: "observed", launchId: active.launchId, @@ -691,20 +1225,22 @@ class NodeWorkerSupervisor { supervisor: active.supervisor, worker: active.worker, outcome, + ...(active.container ? { container: active.container } : {}), }; if (this.active.get(active.launchId) !== active) { return; } this.active.set(active.launchId, observed); try { - this.reconcileActiveTerminal(observed); + await this.reconcileActiveTerminal(observed); } catch { // The observed outcome stays owned in memory for the next supervisor operation. } } - private async stopChild(active: RunningChild, state: StopState): Promise { + private async stopChild(active: ActiveChild, state: StopState): Promise { active.stopState ??= state; + this.clearLeaseTimer(active); active.adapter.kill("SIGTERM"); const forceKill = setTimeout(() => active.adapter.kill("SIGKILL"), STOP_GRACE_MS); forceKill.unref?.(); diff --git a/src/node-host/node-worker-workspace-retention.test.ts b/src/node-host/node-worker-workspace-retention.test.ts index 8c90defc4f45..605dd93f6405 100644 --- a/src/node-host/node-worker-workspace-retention.test.ts +++ b/src/node-host/node-worker-workspace-retention.test.ts @@ -97,6 +97,28 @@ afterEach(() => { }); describe("node worker workspace retention", () => { + it.runIf(process.platform !== "win32" && typeof process.getuid === "function" && process.getuid() > 0)( + "uses a private short-lived relay directory with a portable Unix socket path", + async () => { + const workspace = new NodeWorkerWorkspaceRuntime({ + root: tempDirs.make("node-worker-workspace-relay-root-"), + }); + const params = { + gatewayNamespace: "gateway-relay", + launchId: "relay-path-test", + planHash: "a".repeat(64), + }; + + const relayDir = workspace.resolveContainerRelayDirectory(params); + expect(relayDir).toMatch(/^\/tmp\/openclaw-worker-relays-[0-9]+\/[a-f0-9]{32}$/u); + expect(Buffer.byteLength(path.join(relayDir, "gateway.sock"), "utf8")).toBeLessThanOrEqual(100); + expect(fs.statSync(relayDir).mode & 0o077).toBe(0); + + await workspace.removeContainerRelayDirectory(params); + expect(fs.existsSync(relayDir)).toBe(false); + }, + ); + it("does not delete workspaces before the first Gateway snapshot", async () => { const root = tempDirs.make("node-worker-workspace-retention-startup-"); const { bundleRoot, env, workspaceDir } = writeNodeWorkerFixture(root); diff --git a/src/node-host/node-worker-workspace.ts b/src/node-host/node-worker-workspace.ts index bed77adc430c..98d950d32618 100644 --- a/src/node-host/node-worker-workspace.ts +++ b/src/node-host/node-worker-workspace.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import fs from "node:fs"; import fsp from "node:fs/promises"; import path from "node:path"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { resolveStateDir } from "../config/paths.js"; import { isPathInside } from "../infra/path-guards.js"; import { KeyedAsyncQueue } from "../plugin-sdk/keyed-async-queue.js"; @@ -29,6 +30,12 @@ const WORKSPACE_RETENTION_DELETE_LIMIT = 256; const ENVIRONMENT_HASH_PATTERN = /^[a-f0-9]{16}$/u; const SESSION_HASH_PATTERN = /^[a-f0-9]{32}$/u; const MANIFEST_FILE_PATTERN = /^[a-f0-9]{64}\.json$/u; +const NODE_WORKER_CONTAINER_RELAY_ROOT_PREFIX = "/tmp/openclaw-worker-relays-"; +const NODE_WORKER_CONTAINER_RELAY_SOCKET_NAME = "gateway.sock"; +// macOS accepts fewer bytes than Linux for a filesystem Unix-domain socket. +// Keep the portable bound below both limits rather than advertise a sandbox +// that will only fail after the supervisor has durably launched it. +const NODE_WORKER_CONTAINER_RELAY_SOCKET_MAX_BYTES = 100; type NodeWorkerWorkspaceLaunchReference = { gatewayNamespace: string; @@ -206,6 +213,67 @@ function ensureContainedDirectory(parent: string, name: string): string { return resolved; } +function currentNonRootUid(): number { + if (process.platform === "win32" || typeof process.getuid !== "function") { + throw new Error("node worker container relays require a non-root POSIX host user"); + } + const uid = process.getuid(); + if (!Number.isSafeInteger(uid) || uid <= 0) { + throw new Error("node worker container relays require a non-root POSIX host user"); + } + return uid; +} + +function containerRelayPaths(params: { + gatewayNamespace: string; + launchId: string; + planHash: string; +}): { root: string; directory: string } { + const uid = currentNonRootUid(); + const name = hashPathComponent( + `${params.gatewayNamespace}\0${params.launchId}\0${params.planHash}`, + 32, + ); + return { + root: `${NODE_WORKER_CONTAINER_RELAY_ROOT_PREFIX}${uid}`, + directory: path.join(`${NODE_WORKER_CONTAINER_RELAY_ROOT_PREFIX}${uid}`, name), + }; +} + +function ensurePrivateRelayDirectory(candidate: string, expectedParent?: string): void { + fs.mkdirSync(candidate, { recursive: true, mode: 0o700 }); + const initialStats = fs.lstatSync(candidate); + if ( + initialStats.isSymbolicLink() || + !initialStats.isDirectory() || + initialStats.uid !== currentNonRootUid() + ) { + throw new Error( + "INVALID_REQUEST: node worker container relay path is not a private owned directory", + ); + } + fs.chmodSync(candidate, 0o700); + const stats = fs.lstatSync(candidate); + const resolved = fs.realpathSync.native(candidate); + if ( + stats.isSymbolicLink() || + !stats.isDirectory() || + stats.uid !== currentNonRootUid() || + (stats.mode & 0o077) !== 0 || + (expectedParent !== undefined && + fs.realpathSync.native(path.dirname(candidate)) !== expectedParent) + ) { + throw new Error( + "INVALID_REQUEST: node worker container relay path is not a private owned directory", + ); + } + if (resolved !== fs.realpathSync.native(candidate)) { + throw new Error( + "INVALID_REQUEST: node worker container relay path changed while it was prepared", + ); + } +} + function resolveArgumentPath(workspaceDir: string, arg: string): string | undefined { if (path.isAbsolute(arg)) { return arg; @@ -233,7 +301,7 @@ function assertWorkspaceArgv(workspaceDir: string, argv: readonly string[]): voi try { resolved = fs.realpathSync.native(candidate); } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + if (!isRecord(error) || error.code !== "ENOENT") { throw error; } } @@ -338,6 +406,81 @@ export class NodeWorkerWorkspaceRuntime { }; } + /** + * Container launches never accept a Gateway-provided host path. The node + * derives this one canonical session/epoch workspace before mounting it at + * the fixed in-container `/workspace` path. + */ + resolveContainerWorkspace(reference: NodeWorkerWorkspaceLaunchReference): string { + if (!Number.isSafeInteger(reference.ownerEpoch) || reference.ownerEpoch < 0) { + throw new Error("INVALID_REQUEST: node worker container owner epoch is invalid"); + } + const environmentHash = hashPathComponent(reference.environmentId, 16); + const sessionHash = hashPathComponent(reference.sessionId, 32); + const gatewayRoot = ensureContainedDirectory(this.root, reference.gatewayNamespace); + const workspacesRoot = ensureContainedDirectory(gatewayRoot, "workspaces"); + const environmentRoot = ensureContainedDirectory(workspacesRoot, environmentHash); + const sessionRoot = ensureContainedDirectory(environmentRoot, sessionHash); + return ensureContainedDirectory(sessionRoot, String(reference.ownerEpoch)); + } + + /** + * The relay is node-private staging, outside the worker-writable workspace. + * Its name is derived from the durable launch identity, not from request data. + */ + resolveContainerRelayDirectory(params: { + gatewayNamespace: string; + launchId: string; + planHash: string; + }): string { + if (!/^[a-f0-9]{64}$/u.test(params.planHash)) { + throw new Error("INVALID_REQUEST: node worker container plan hash is invalid"); + } + const paths = containerRelayPaths(params); + // The mounted directory is host-private, but the Unix socket must retain + // this short lexical `/tmp` spelling. Canonical state paths exceed macOS' + // socket limit before the container can even connect to the relay. + ensurePrivateRelayDirectory(paths.root); + const canonicalRoot = fs.realpathSync.native(paths.root); + ensurePrivateRelayDirectory(paths.directory, canonicalRoot); + const socketPath = path.join(paths.directory, NODE_WORKER_CONTAINER_RELAY_SOCKET_NAME); + if (Buffer.byteLength(socketPath, "utf8") > NODE_WORKER_CONTAINER_RELAY_SOCKET_MAX_BYTES) { + throw new Error( + "INVALID_REQUEST: node worker container relay socket path exceeds the portable limit", + ); + } + return paths.directory; + } + + async removeContainerRelayDirectory(params: { + gatewayNamespace: string; + launchId: string; + planHash: string; + }): Promise { + if (!/^[a-f0-9]{64}$/u.test(params.planHash)) { + return; + } + const paths = containerRelayPaths(params); + try { + const rootStats = await fsp.lstat(paths.root); + if ( + rootStats.isSymbolicLink() || + !rootStats.isDirectory() || + rootStats.uid !== currentNonRootUid() || + (rootStats.mode & 0o077) !== 0 + ) { + return; + } + const canonicalRoot = await fsp.realpath(paths.root); + await removeOwnedDirectory(canonicalRoot, paths.directory); + await removeIfEmpty(paths.root); + } catch (error) { + if (!isRecord(error) || error.code !== "ENOENT") { + throw error; + } + } + } + private beginWorkspaceOperation(gatewayNamespace: string, generationKey: string): () => void { this.activeWorkspaceOperations.set( generationKey, diff --git a/src/node-host/runner.test.ts b/src/node-host/runner.test.ts index d84d0419c0f7..6ef5932769fa 100644 --- a/src/node-host/runner.test.ts +++ b/src/node-host/runner.test.ts @@ -6,6 +6,7 @@ import { getConfigResolutionFacts, setConfigResolutionFacts } from "../config/re import type { GatewayClientOptions } from "../gateway/client.js"; import { NODE_RUNNER_INVENTORY_UPDATE_METHOD, + NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE, NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE, } from "../infra/node-runner-inventory.js"; import type { configureNodeHost } from "./config.js"; @@ -34,6 +35,8 @@ const mocks = vi.hoisted(() => ({ runnerCapacityChanged: undefined as | ((capacity: { total: number; available: number }) => void) | undefined, + workerSupervisorReadinessChanged: undefined as ((ready: boolean) => void) | undefined, + resolveNodeWorkerContainerEngine: vi.fn(async () => undefined), nodeHostCommands: [] as string[], nodeHostCaps: [] as string[], availabilityOnWatch: undefined as { caps: string[]; commands: string[] } | undefined, @@ -107,6 +110,10 @@ vi.mock("../infra/device-identity.js", () => ({ publicKey: "public-key-test", privateKey: "private-key-test", })), + loadOrCreateProcessDeviceIdentity: vi.fn(() => ({ + deviceId: "device-test", + privateKeyPem: "private-key-test", + })), })); vi.mock("../infra/machine-name.js", () => ({ @@ -199,6 +206,7 @@ vi.mock("./runtime.js", async (importOriginal) => { start: (params) => { mocks.runtimeClient = params.client; mocks.runnerCapacityChanged = params.onRunnerCapacityChanged; + mocks.workerSupervisorReadinessChanged = params.onWorkerSupervisorReadinessChanged; return mocks.activeRuntime; }, }; @@ -206,6 +214,10 @@ vi.mock("./runtime.js", async (importOriginal) => { }; }); +vi.mock("./node-worker-container-runtime.js", () => ({ + resolveNodeWorkerContainerEngine: mocks.resolveNodeWorkerContainerEngine, +})); + function lastCapturedOptions(): GatewayClientOptions | undefined { const list = mocks.capturedGatewayClientOptions; return list[list.length - 1]; @@ -232,6 +244,8 @@ describe("runNodeHost", () => { mocks.useFakeRuntime = false; mocks.fakeRuntimeWorkerHosting = false; mocks.runnerCapacityChanged = undefined; + mocks.workerSupervisorReadinessChanged = undefined; + mocks.resolveNodeWorkerContainerEngine.mockResolvedValue(undefined); mocks.nodeHostCommands = []; mocks.nodeHostCaps = []; mocks.availabilityOnWatch = undefined; @@ -689,6 +703,8 @@ describe("runNodeHost", () => { }); it("publishes opt-in consent and capacity in the atomic runner inventory", async () => { + mocks.useFakeRuntime = true; + mocks.fakeRuntimeWorkerHosting = true; mocks.getRuntimeConfig.mockReturnValue({ gateway: { handshakeTimeoutMs: 1_000 }, nodeHost: { workerRuns: { enabled: true } }, @@ -699,6 +715,9 @@ describe("runNodeHost", () => { const options = mocks.capturedGatewayClientOptions[0]; const client = mocks.capturedGatewayClients[0]; + mocks.runnerCapacityChanged?.({ total: 2, available: 2 }); + mocks.workerSupervisorReadinessChanged?.(true); + options?.onHelloOk?.({ protocol: 4, features: { methods: [], events: [] }, @@ -725,6 +744,7 @@ describe("runNodeHost", () => { expect(options?.workerRuns).toBeUndefined(); mocks.runnerCapacityChanged?.({ total: 2, available: 2 }); + mocks.workerSupervisorReadinessChanged?.(true); options?.onHelloOk?.({ protocol: 4, features: { @@ -790,6 +810,88 @@ describe("runNodeHost", () => { expect(client?.updateNodeManifest).not.toHaveBeenCalled(); }); + it("withholds and withdraws process isolation until supervisor recovery is healthy", async () => { + mocks.useFakeRuntime = true; + mocks.fakeRuntimeWorkerHosting = true; + mocks.startGatewayClientWhenEventLoopReady.mockResolvedValueOnce({ + ready: true, + aborted: false, + elapsedMs: 0, + }); + let resolveEngine!: (value: unknown) => void; + mocks.resolveNodeWorkerContainerEngine.mockImplementationOnce( + async () => + await new Promise((resolve) => { + resolveEngine = resolve; + }), + ); + const processOnceSpy = vi.spyOn(process, "once"); + const previousExitCode = process.exitCode; + try { + const running = runNodeHost({ gatewayHost: "127.0.0.1", gatewayPort: 18789 }); + await vi.waitFor(() => expect(mocks.workerSupervisorReadinessChanged).toBeTypeOf("function")); + await vi.waitFor(() => expect(resolveEngine).toBeTypeOf("function")); + const options = mocks.capturedGatewayClientOptions[0]; + const client = mocks.capturedGatewayClients[0]; + const latestRunnerInventory = () => + client?.request.mock.calls + .filter(([method]) => method === NODE_RUNNER_INVENTORY_UPDATE_METHOD) + .at(-1)?.[1]; + + mocks.runnerCapacityChanged?.({ total: 2, available: 2 }); + options?.onHelloOk?.({ + protocol: 7, + features: { methods: [], events: [] }, + } as unknown as Parameters>[0]); + await vi.waitFor(() => + expect(latestRunnerInventory()).toEqual({ + protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE], + workerHost: { enabled: false }, + }), + ); + + resolveEngine({ id: "docker" }); + await vi.waitFor(() => expect(mocks.resolveNodeWorkerContainerEngine).toHaveBeenCalledOnce()); + expect(latestRunnerInventory()).toEqual({ + protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE], + workerHost: { enabled: false }, + }); + + mocks.workerSupervisorReadinessChanged?.(true); + await vi.waitFor(() => + expect(latestRunnerInventory()).toEqual({ + protocolFeatures: [NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE], + workerHost: { + enabled: true, + capacity: { total: 2, available: 2 }, + bundlePrewarm: 1, + processIsolation: { kind: "container-v1", memoryProjection: 1 }, + }, + }), + ); + + mocks.workerSupervisorReadinessChanged?.(false); + await vi.waitFor(() => + expect(latestRunnerInventory()).toEqual({ + protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE], + workerHost: { enabled: false }, + }), + ); + + const onSigterm = processOnceSpy.mock.calls.find(([event]) => event === "SIGTERM")?.[1]; + onSigterm?.("SIGTERM"); + await running; + } finally { + for (const [event, listener] of processOnceSpy.mock.calls) { + if ((event === "SIGINT" || event === "SIGTERM") && typeof listener === "function") { + process.off(event, listener); + } + } + process.exitCode = previousExitCode; + processOnceSpy.mockRestore(); + } + }); + it("clears gateway plugin tools when the final node-hosted tool disappears", async () => { mocks.startGatewayClientWhenEventLoopReady.mockResolvedValueOnce({ ready: true, diff --git a/src/node-host/runner.ts b/src/node-host/runner.ts index d24f45e65bba..2d627b6ddc54 100644 --- a/src/node-host/runner.ts +++ b/src/node-host/runner.ts @@ -19,6 +19,7 @@ import { NODE_RUNNER_INVENTORY_UPDATE_METHOD, NODE_WORKER_BUNDLE_RETENTION_VERSION, NODE_WORKER_BUNDLE_STATUS_VERSION, + NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE, NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE, type NodeWorkerCapacitySnapshot, } from "../infra/node-runner-inventory.js"; @@ -35,6 +36,7 @@ import { coerceNodeInvokePayload, } from "./invoke-payload.js"; import { prepareNodeHostRuntime, type NodeHostInventory } from "./runtime.js"; +import { resolveNodeWorkerContainerEngine } from "./node-worker-container-runtime.js"; import { runStartupMigrations } from "./startup-state-migrations.js"; type NodeHostRunOptions = { @@ -304,6 +306,8 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise { let inventory: NodeHostInventory = preparedRuntime.initialInventory; let workerCapacity: NodeWorkerCapacitySnapshot | undefined; + let workerSupervisorReady = false; + let workerProcessIsolationAvailable = false; let gatewayHelloReceived = false; let gatewayConnectionGeneration = 0; let connectedGatewayProtocol = 0; @@ -515,12 +519,22 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise { }; const publishRunnerInventory = () => { + const workerHostAvailable = + preparedRuntime.workerHostingEnabled && workerSupervisorReady && workerCapacity !== undefined; + const processIsolation = + workerHostAvailable && workerProcessIsolationAvailable + ? { kind: "container-v1" as const } + : undefined; queueOptionalPublication( NODE_RUNNER_INVENTORY_UPDATE_METHOD, { - protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE], + protocolFeatures: [ + processIsolation + ? NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE + : NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE, + ], workerHost: - preparedRuntime.workerHostingEnabled && workerCapacity + workerHostAvailable && workerCapacity ? { enabled: true, capacity: workerCapacity, @@ -531,6 +545,9 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise { ...(gatewaySupportsBundleRetention && gatewaySupportsBundleStatus ? { bundleStatus: NODE_WORKER_BUNDLE_STATUS_VERSION } : {}), + ...(processIsolation + ? { processIsolation: { ...processIsolation, memoryProjection: 1 } } + : {}), } : { enabled: false }, }, @@ -656,6 +673,10 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise { workerCapacity = capacity; publishRunnerInventory(); }, + onWorkerSupervisorReadinessChanged: (ready) => { + workerSupervisorReady = ready; + publishRunnerInventory(); + }, onManifestChanged: (manifest) => { // Manifest changes force a reconnect. Retire the current publication queue // now so it cannot drain against the closing connection. @@ -665,6 +686,15 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise { }); let stopping = false; + if (preparedRuntime.workerHostingEnabled) { + void resolveNodeWorkerContainerEngine().then((engine) => { + if (stopping || !engine) { + return; + } + workerProcessIsolationAvailable = true; + publishRunnerInventory(); + }); + } let resolveStopped: (() => void) | undefined; const stopped = new Promise((resolve) => { resolveStopped = resolve; diff --git a/src/node-host/runtime.ts b/src/node-host/runtime.ts index cac4477f20a5..37104c9b9d6b 100644 --- a/src/node-host/runtime.ts +++ b/src/node-host/runtime.ts @@ -64,6 +64,7 @@ type PreparedNodeHostRuntime = { onInventoryChanged?: (inventory: NodeHostInventory) => void; onManifestChanged?: (manifest: NodeHostManifest) => void; onRunnerCapacityChanged?: (capacity: NodeWorkerCapacitySnapshot) => void; + onWorkerSupervisorReadinessChanged?: (ready: boolean) => void; }): ActiveNodeHostRuntime; }; @@ -322,7 +323,13 @@ export async function prepareNodeHostRuntime(params?: { manifest, workerHostingEnabled: workerRunsEnabled, initialInventory, - start({ client, onInventoryChanged, onManifestChanged, onRunnerCapacityChanged }) { + start({ + client, + onInventoryChanged, + onManifestChanged, + onRunnerCapacityChanged, + onWorkerSupervisorReadinessChanged, + }) { const mcpAbort = new AbortController(); const workerWorkspace = workerRunsEnabled ? new NodeWorkerWorkspaceRuntime({ env }) @@ -338,9 +345,16 @@ export async function prepareNodeHostRuntime(params?: { }) : undefined; if (workerSupervisor) { - void workerSupervisor.initialize().catch((error: unknown) => { - logDebug(`node-host: worker capacity reconciliation failed: ${String(error)}`); - }); + // A reconciled slot count is not enough to assert process isolation: a + // failed recovery can leave a durable relay, projection, or container alive. + onWorkerSupervisorReadinessChanged?.(false); + void workerSupervisor + .initialize() + .then(() => onWorkerSupervisorReadinessChanged?.(true)) + .catch((error: unknown) => { + onWorkerSupervisorReadinessChanged?.(false); + logDebug(`node-host: worker capacity reconciliation failed: ${String(error)}`); + }); } const skillBins = new SkillBinsCache(client, pathEnv); const activeInvokes = new Map(); diff --git a/src/plugin-sdk/memory-broker-runtime.ts b/src/plugin-sdk/memory-broker-runtime.ts index dd4e2d5eb3ba..9e7381107cd9 100644 --- a/src/plugin-sdk/memory-broker-runtime.ts +++ b/src/plugin-sdk/memory-broker-runtime.ts @@ -7,3 +7,4 @@ export type { MemoryBrokerRequest, } from "../memory-broker/protocol.js"; export type { MemoryBrokerHandler } from "../memory-broker/server.js"; +export type { MemoryBrokerChildEntry, MemoryBrokerStartupContext } from "../memory-broker/entry.js"; diff --git a/src/plugins/gateway-startup-plugin-plan.ts b/src/plugins/gateway-startup-plugin-plan.ts index b20fe0e9b8bb..4c484c986e2f 100644 --- a/src/plugins/gateway-startup-plugin-plan.ts +++ b/src/plugins/gateway-startup-plugin-plan.ts @@ -16,7 +16,6 @@ import { import { hasConfiguredStartupChannel, declaresAllowedEnterpriseIdentityProvider, - listPotentialEnabledChannelIds, resolveAuthorizedGatewayStartupDreamingPluginIds, resolveContextEngineSlotStartupPluginId, resolveMemorySlotStartupPluginId, diff --git a/src/plugins/memory-broker-runtime.test.ts b/src/plugins/memory-broker-runtime.test.ts index 2d4f05f4aa58..8a1f40c45540 100644 --- a/src/plugins/memory-broker-runtime.test.ts +++ b/src/plugins/memory-broker-runtime.test.ts @@ -1,5 +1,19 @@ -import { describe, expect, it, vi } from "vitest"; -import { testing } from "./memory-broker-runtime.js"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { MemoryBrokerProcess } from "../memory-broker/process.js"; +import { + closeBrokeredMemoryRuntimes, + startBrokeredMemoryRuntimeSupervisor, + testing, + withBrokeredMemoryMaintenance, +} from "./memory-broker-runtime.js"; +import type { MemoryPluginCapability } from "./registry-contribution-types.js"; + +const startMemoryBrokerProcess = vi.hoisted(() => vi.fn()); + +vi.mock("../memory-broker/process.js", async (importOriginal) => ({ + ...(await importOriginal()), + startMemoryBrokerProcess, +})); function process(params: { running?: boolean } = {}) { return { @@ -9,37 +23,223 @@ function process(params: { running?: boolean } = {}) { }; } +function brokerProcess(): MemoryBrokerProcess { + return { + client: {} as MemoryBrokerProcess["client"], + brokerEpoch: "test-epoch", + isRunning: () => true, + isHealthy: async () => true, + quiesce: async () => {}, + resume: async () => {}, + close: async () => {}, + }; +} + +const brokerCapability = { + broker: { + version: 1, + kind: "local-child", + moduleUrl: "test:memory-broker-runtime", + }, +} satisfies MemoryPluginCapability; + +afterEach(async () => { + await closeBrokeredMemoryRuntimes(); + startMemoryBrokerProcess.mockReset(); +}); + describe("brokered memory maintenance", () => { - it("drains every running broker before a backup and resumes them afterwards", async () => { - const first = process(); - const second = process(); - const stopped = process({ running: false }); + it("drains a running broker before maintenance and resumes it afterwards", async () => { + const broker = process(); const run = vi.fn(async () => "snapshot-created"); + const retire = vi.fn(async () => {}); await expect( - testing.runBrokeredMemoryMaintenance({ processes: [first, second, stopped], run }), + testing.runBrokeredMemoryMaintenance({ process: broker, run, retire }), ).resolves.toBe("snapshot-created"); - expect(first.quiesce).toHaveBeenCalledOnce(); - expect(second.quiesce).toHaveBeenCalledOnce(); - expect(stopped.quiesce).not.toHaveBeenCalled(); + expect(broker.quiesce).toHaveBeenCalledOnce(); expect(run).toHaveBeenCalledOnce(); - expect(first.resume).toHaveBeenCalledOnce(); - expect(second.resume).toHaveBeenCalledOnce(); + expect(broker.resume).toHaveBeenCalledOnce(); + expect(retire).not.toHaveBeenCalled(); }); - it("reopens an already-drained broker when a later broker or the backup fails", async () => { - const first = process(); - const second = process(); - second.quiesce.mockRejectedValueOnce(new Error("broker unavailable")); + it("retires and rejects when quiesce fails before the mutation starts", async () => { + const broker = process(); + broker.quiesce.mockRejectedValueOnce(new Error("broker unavailable")); const run = vi.fn(async () => "unreachable"); + const retire = vi.fn(async () => {}); await expect( - testing.runBrokeredMemoryMaintenance({ processes: [first, second], run }), + testing.runBrokeredMemoryMaintenance({ process: broker, run, retire }), ).rejects.toThrow("broker unavailable"); expect(run).not.toHaveBeenCalled(); - expect(first.resume).toHaveBeenCalledOnce(); - expect(second.resume).not.toHaveBeenCalled(); + expect(retire).toHaveBeenCalledOnce(); + }); + + it("retires and rejects when resume fails after a mutation", async () => { + const broker = process(); + broker.resume.mockRejectedValueOnce(new Error("broker resume unavailable")); + const retire = vi.fn(async () => {}); + + await expect( + testing.runBrokeredMemoryMaintenance({ + process: broker, + run: async () => "snapshot-created", + retire, + }), + ).rejects.toThrow("broker resume unavailable"); + + expect(retire).toHaveBeenCalledOnce(); + }); + + it("serializes maintenance, process resolution, and shutdown on one broker lease", async () => { + const leases = new Map>(); + const order: string[] = []; + let releaseMaintenance: (() => void) | undefined; + let markMaintenanceStarted: (() => void) | undefined; + const maintenanceStarted = new Promise((resolve) => { + markMaintenanceStarted = resolve; + }); + const maintenanceMayFinish = new Promise((resolve) => { + releaseMaintenance = resolve; + }); + + const maintenance = testing.withBrokerLease(leases, "memory-core", async () => { + order.push("maintenance:start"); + markMaintenanceStarted?.(); + await maintenanceMayFinish; + order.push("maintenance:finish"); + }); + await maintenanceStarted; + const resolve = testing.withBrokerLease(leases, "memory-core", async () => { + order.push("resolve"); + }); + const shutdown = testing.withBrokerLease(leases, "memory-core", async () => { + order.push("shutdown"); + }); + + await Promise.resolve(); + expect(order).toEqual(["maintenance:start"]); + releaseMaintenance?.(); + await Promise.all([maintenance, resolve, shutdown]); + expect(order).toEqual(["maintenance:start", "maintenance:finish", "resolve", "shutdown"]); + }); + + it("blocks broker resolution while maintenance holds the gateway lease without a running broker", async () => { + const gate = testing.createBrokerMaintenanceGate(); + const order: string[] = []; + let releaseMaintenance: (() => void) | undefined; + let markMaintenanceStarted: (() => void) | undefined; + const maintenanceStarted = new Promise((resolve) => { + markMaintenanceStarted = resolve; + }); + const maintenanceMayFinish = new Promise((resolve) => { + releaseMaintenance = resolve; + }); + + const maintenance = testing.withGatewayBrokeredMemoryMaintenanceLease(gate, async () => { + order.push("maintenance:start"); + markMaintenanceStarted?.(); + await maintenanceMayFinish; + order.push("maintenance:finish"); + }); + await maintenanceStarted; + const resolution = testing.withBrokerLifecycleOperation(gate, async () => { + order.push("broker:start"); + }); + + await Promise.resolve(); + expect(order).toEqual(["maintenance:start"]); + releaseMaintenance?.(); + await Promise.all([maintenance, resolution]); + expect(order).toEqual(["maintenance:start", "maintenance:finish", "broker:start"]); + }); + + it("waits for an admitted broker resolution before maintenance starts", async () => { + const gate = testing.createBrokerMaintenanceGate(); + const order: string[] = []; + let releaseResolution: (() => void) | undefined; + let markResolutionStarted: (() => void) | undefined; + const resolutionStarted = new Promise((resolve) => { + markResolutionStarted = resolve; + }); + const resolutionMayFinish = new Promise((resolve) => { + releaseResolution = resolve; + }); + + const resolution = testing.withBrokerLifecycleOperation(gate, async () => { + order.push("broker:start"); + markResolutionStarted?.(); + await resolutionMayFinish; + order.push("broker:finish"); + }); + await resolutionStarted; + const maintenance = testing.withGatewayBrokeredMemoryMaintenanceLease(gate, async () => { + order.push("maintenance"); + }); + + await Promise.resolve(); + expect(order).toEqual(["broker:start"]); + releaseResolution?.(); + await Promise.all([resolution, maintenance]); + expect(order).toEqual(["broker:start", "broker:finish", "maintenance"]); + }); + + it("does not start a broker requested after empty-map maintenance begins", async () => { + const broker = brokerProcess(); + startMemoryBrokerProcess.mockResolvedValueOnce(broker); + let releaseMaintenance: (() => void) | undefined; + let markMaintenanceStarted: (() => void) | undefined; + const maintenanceStarted = new Promise((resolve) => { + markMaintenanceStarted = resolve; + }); + const maintenanceMayFinish = new Promise((resolve) => { + releaseMaintenance = resolve; + }); + + const maintenance = withBrokeredMemoryMaintenance(async () => { + markMaintenanceStarted?.(); + await maintenanceMayFinish; + }); + await maintenanceStarted; + const supervisor = startBrokeredMemoryRuntimeSupervisor(brokerCapability); + + await Promise.resolve(); + expect(startMemoryBrokerProcess).not.toHaveBeenCalled(); + releaseMaintenance?.(); + await Promise.all([maintenance, supervisor]); + expect(startMemoryBrokerProcess).toHaveBeenCalledOnce(); + + // Regression: shutdown stops supervisors before taking the writer lease, so stop's reader + // retirement cannot deadlock behind the writer that is waiting for it. + await expect(closeBrokeredMemoryRuntimes()).resolves.toBeUndefined(); + }); + + it("drains an admitted broker start before maintenance snapshots processes", async () => { + const broker = brokerProcess(); + let resolveStart: ((value: MemoryBrokerProcess) => void) | undefined; + let markStartCalled: (() => void) | undefined; + const startCalled = new Promise((resolve) => { + markStartCalled = resolve; + }); + const pendingStart = new Promise((resolve) => { + resolveStart = resolve; + }); + startMemoryBrokerProcess.mockImplementationOnce(() => { + markStartCalled?.(); + return pendingStart; + }); + const supervisor = startBrokeredMemoryRuntimeSupervisor(brokerCapability); + await startCalled; + const run = vi.fn(async () => {}); + const maintenance = withBrokeredMemoryMaintenance(run); + + await Promise.resolve(); + expect(run).not.toHaveBeenCalled(); + resolveStart?.(broker); + await Promise.all([supervisor, maintenance]); + expect(run).toHaveBeenCalledOnce(); }); }); diff --git a/src/plugins/memory-broker-runtime.ts b/src/plugins/memory-broker-runtime.ts index 691b7306d2f9..98388d5fcd2c 100644 --- a/src/plugins/memory-broker-runtime.ts +++ b/src/plugins/memory-broker-runtime.ts @@ -19,12 +19,39 @@ import type { type BrokerRuntimeState = { processes: Map>; + agentIdsByModule: Map; supervisors: Set; + leases: Map>; + maintenance: BrokerMaintenanceGate; + closing: boolean; }; +type BrokerMaintenanceGate = { + maintenanceTail: Promise; + maintenancePending: boolean; + activeLifecycleOperations: number; + idleWaiters: Set<() => void>; +}; + +function createBrokerMaintenanceGate(): BrokerMaintenanceGate { + return { + maintenanceTail: Promise.resolve(), + maintenancePending: false, + activeLifecycleOperations: 0, + idleWaiters: new Set(), + }; +} + const state = resolveGlobalSingleton( Symbol.for("openclaw.memoryBrokerRuntimeState"), - (): BrokerRuntimeState => ({ processes: new Map(), supervisors: new Set() }), + (): BrokerRuntimeState => ({ + processes: new Map(), + agentIdsByModule: new Map(), + supervisors: new Set(), + leases: new Map(), + maintenance: createBrokerMaintenanceGate(), + closing: false, + }), ); type BrokeredMemoryRuntime = Readonly & @@ -34,6 +61,19 @@ function resolveCapabilitySnapshotId(context: MemoryAccessContext): string { return context.delegation?.capabilitySnapshotId ?? context.hostFactsRevision; } +function actorBindingFor(context: MemoryAccessContext) { + return context.actor.kind === "principal" + ? Object.freeze({ + kind: "principal" as const, + actorKind: context.actor.actorKind, + principalId: context.actor.principalId, + }) + : Object.freeze({ + kind: "unattributed" as const, + transportAuditRef: context.actor.transportAuditRef, + }); +} + function bindingFor(context: MemoryAccessContext, policyRevision: string) { return Object.freeze({ agentId: context.agentId, @@ -41,6 +81,7 @@ function bindingFor(context: MemoryAccessContext, policyRevision: string) { runId: context.runId, contextFingerprint: context.contextFingerprint, subjectRevision: context.subjectRevision, + actor: actorBindingFor(context), actorRevision: context.actor.evidenceRevision, capabilitySnapshotId: resolveCapabilitySnapshotId(context), policyRevision, @@ -59,14 +100,159 @@ function authorizeExpiry(): number { return Date.now() + 60_000; } -async function resolveProcess( - capability: MemoryPluginCapability, -): Promise { - const entry = capability.broker; - if (!entry || entry.version !== 1 || entry.kind !== "local-child" || !entry.moduleUrl) { +/** + * The Gateway is the one owner allowed to change a broker's admission state. A lease covers + * replacement and shutdown too, so no child can start or close between quiesce and resume. + */ +async function withBrokerLease( + leases: Map>, + moduleUrl: string, + run: () => Promise, +): Promise { + const previous = leases.get(moduleUrl) ?? Promise.resolve(); + let release: (() => void) | undefined; + const current = new Promise((resolve) => { + release = resolve; + }); + leases.set(moduleUrl, current); + await previous; + try { + return await run(); + } finally { + release?.(); + if (leases.get(moduleUrl) === current) { + leases.delete(moduleUrl); + } + } +} + +function releaseBrokerLifecycleOperation(gate: BrokerMaintenanceGate): void { + gate.activeLifecycleOperations -= 1; + if (gate.activeLifecycleOperations !== 0) { + return; + } + for (const resolve of gate.idleWaiters) { + resolve(); + } + gate.idleWaiters.clear(); +} + +async function waitForBrokerLifecycleOperations(gate: BrokerMaintenanceGate): Promise { + while (gate.activeLifecycleOperations > 0) { + await new Promise((resolve) => gate.idleWaiters.add(resolve)); + } +} + +/** + * Resolution and replacement share a read lease. A maintenance writer blocks later lifecycle + * work before it snapshots brokers, while already admitted work finishes before quiesce begins. + */ +async function withBrokerLifecycleOperation( + gate: BrokerMaintenanceGate, + run: () => Promise, +): Promise { + while (true) { + const maintenanceTail = gate.maintenanceTail; + await maintenanceTail; + if (gate.maintenanceTail !== maintenanceTail || gate.maintenancePending) { + continue; + } + gate.activeLifecycleOperations += 1; + if (gate.maintenanceTail === maintenanceTail && !gate.maintenancePending) { + break; + } + releaseBrokerLifecycleOperation(gate); + } + try { + return await run(); + } finally { + releaseBrokerLifecycleOperation(gate); + } +} + +/** + * A tool request cannot wait behind maintenance: that would retain an agent turn until a backup + * or repair finishes. Refuse admission before taking the reader lease so callers surface the + * intentional memory-unavailable result while Gateway owns the writer lease. + */ +async function tryWithBrokerLifecycleOperation( + gate: BrokerMaintenanceGate, + run: () => Promise, +): Promise { + if (gate.maintenancePending) { return undefined; } - const existing = state.processes.get(entry.moduleUrl); + const maintenanceTail = gate.maintenanceTail; + gate.activeLifecycleOperations += 1; + if (gate.maintenanceTail !== maintenanceTail || gate.maintenancePending) { + releaseBrokerLifecycleOperation(gate); + return undefined; + } + try { + return await run(); + } finally { + releaseBrokerLifecycleOperation(gate); + } +} + +/** + * Gateway maintenance owns the write lease for the complete drain/mutate/resume lifecycle. + * Marking it pending synchronously prevents a just-in-time resolver from starting a new child. + */ +async function withGatewayBrokeredMemoryMaintenanceLease( + gate: BrokerMaintenanceGate, + run: () => Promise, +): Promise { + const previous = gate.maintenanceTail; + let release: (() => void) | undefined; + const current = new Promise((resolve) => { + release = resolve; + }); + const tail = previous.then(() => current); + gate.maintenanceTail = tail; + gate.maintenancePending = true; + await previous; + await waitForBrokerLifecycleOperations(gate); + try { + return await run(); + } finally { + release?.(); + if (gate.maintenanceTail === tail) { + gate.maintenancePending = false; + } + } +} + +function brokerModuleUrl(capability: MemoryPluginCapability): string | undefined { + const entry = capability.broker; + return entry?.version === 1 && entry.kind === "local-child" && entry.moduleUrl + ? entry.moduleUrl + : undefined; +} + +async function resolveProcess( + capability: MemoryPluginCapability, + maintenanceBehavior: "wait" | "fail" = "wait", +): Promise { + const moduleUrl = brokerModuleUrl(capability); + if (!moduleUrl) { + return undefined; + } + const resolveOnLease = async () => { + return await withBrokerLease(state.leases, moduleUrl, async () => { + if (state.closing) { + return undefined; + } + return await resolveProcessOnLease(moduleUrl); + }); + }; + return maintenanceBehavior === "fail" + ? await tryWithBrokerLifecycleOperation(state.maintenance, resolveOnLease) + : await withBrokerLifecycleOperation(state.maintenance, resolveOnLease); +} + +async function resolveProcessOnLease(moduleUrl: string): Promise { + const existing = state.processes.get(moduleUrl); if (existing) { try { const process = await existing; @@ -77,23 +263,27 @@ async function resolveProcess( // failed operation; discard it so a later independently authorized operation starts fresh. // A PID alone is not readiness: a child with a dead socket/handler must be retired too. await process.close().catch(() => undefined); - if (state.processes.get(entry.moduleUrl) === existing) { - state.processes.delete(entry.moduleUrl); + if (state.processes.get(moduleUrl) === existing) { + state.processes.delete(moduleUrl); } } catch { - if (state.processes.get(entry.moduleUrl) === existing) { - state.processes.delete(entry.moduleUrl); + if (state.processes.get(moduleUrl) === existing) { + state.processes.delete(moduleUrl); } } } - const process = startMemoryBrokerProcess({ - brokerId: `selected-memory:${entry.moduleUrl}`, - handlerModuleUrl: entry.moduleUrl, + let process: Promise; + process = startMemoryBrokerProcess({ + brokerId: `selected-memory:${moduleUrl}`, + handlerModuleUrl: moduleUrl, + agentIds: state.agentIdsByModule.get(moduleUrl) ?? [], }).catch((error: unknown) => { - state.processes.delete(entry.moduleUrl); + if (state.processes.get(moduleUrl) === process) { + state.processes.delete(moduleUrl); + } throw error; }); - state.processes.set(entry.moduleUrl, process); + state.processes.set(moduleUrl, process); try { return await process; } catch { @@ -102,15 +292,26 @@ async function resolveProcess( } async function retireProcess(capability: MemoryPluginCapability): Promise { - const moduleUrl = capability.broker?.moduleUrl; + const moduleUrl = brokerModuleUrl(capability); if (!moduleUrl) { return; } - const process = state.processes.get(moduleUrl); + await withBrokerLifecycleOperation(state.maintenance, async () => { + await withBrokerLease(state.leases, moduleUrl, async () => { + await retireProcessOnLease(state, moduleUrl); + }); + }); +} + +async function retireProcessOnLease( + runtimeState: BrokerRuntimeState, + moduleUrl: string, +): Promise { + const process = runtimeState.processes.get(moduleUrl); if (!process) { return; } - state.processes.delete(moduleUrl); + runtimeState.processes.delete(moduleUrl); await (await process).close(); } @@ -121,10 +322,13 @@ async function retireProcess(capability: MemoryPluginCapability): Promise */ export async function startBrokeredMemoryRuntimeSupervisor( capability: MemoryPluginCapability | undefined, + params: { agentIds?: readonly string[] } = {}, ): Promise { - if (!capability?.broker) { + const moduleUrl = capability ? brokerModuleUrl(capability) : undefined; + if (!moduleUrl) { return undefined; } + state.agentIdsByModule.set(moduleUrl, Object.freeze([...new Set(params.agentIds ?? [])].toSorted())); const supervisor = await startMemoryBrokerSupervisor({ ensureProcess: () => resolveProcess(capability), retireProcess: () => retireProcess(capability), @@ -153,7 +357,7 @@ export async function resolveBrokeredMemoryRuntime( return undefined; } const request = async (params: Parameters[0]) => { - const process = await resolveProcess(capability); + const process = await resolveProcess(capability, "fail"); return process ? await process.client.request(params) : undefined; }; const authorize = async (context: MemoryAccessContext): Promise => { @@ -209,6 +413,7 @@ export async function resolveBrokeredMemoryRuntime( ...(params.lines !== undefined ? { lines: params.lines } : {}), }, expiresAtMs, + ...(params.signal ? { signal: params.signal } : {}), }); if (!result) { throw new Error("memory broker read is unavailable"); @@ -225,6 +430,7 @@ export async function resolveBrokeredMemoryRuntime( method: "memory.write", payload: { context: params.context, plan: params.plan, mutation: params.mutation }, expiresAtMs, + ...(params.signal ? { signal: params.signal } : {}), }); if (!result) { throw new Error("memory broker write is unavailable"); @@ -241,6 +447,7 @@ export async function resolveBrokeredMemoryRuntime( method: "memory.import", payload: { context: params.context, plan: params.plan, mutation: params.mutation }, expiresAtMs, + ...(params.signal ? { signal: params.signal } : {}), }); if (!result) { throw new Error("memory broker import is unavailable"); @@ -257,6 +464,7 @@ export async function resolveBrokeredMemoryRuntime( method: "memory.sync", payload: { context: params.context, plan: params.plan }, expiresAtMs, + ...(params.signal ? { signal: params.signal } : {}), }); if (!result) { throw new Error("memory broker sync is unavailable"); @@ -273,6 +481,7 @@ export async function resolveBrokeredMemoryRuntime( method: "memory.export", payload: { context: params.context, plan: params.plan, handles: params.handles }, expiresAtMs, + ...(params.signal ? { signal: params.signal } : {}), }); if (!result) { throw new Error("memory broker export is unavailable"); @@ -289,6 +498,7 @@ export async function resolveBrokeredMemoryRuntime( method: "memory.status", payload: { context: params.context, plan: params.plan }, expiresAtMs, + ...(params.signal ? { signal: params.signal } : {}), }); if (!result) { throw new Error("memory broker status is unavailable"); @@ -298,6 +508,7 @@ export async function resolveBrokeredMemoryRuntime( async materializeAuthorizedVirtualView(params: { context: MemoryContentAccessContext<"read">; plan: AuthorizedMemoryContentPlan<"read">; + signal?: AbortSignal; }): Promise { const expiresAtMs = hasPlanExpiry(params.plan); if (!expiresAtMs) { @@ -308,6 +519,7 @@ export async function resolveBrokeredMemoryRuntime( method: "memory.virtual-view", payload: { context: params.context, plan: params.plan }, expiresAtMs, + ...(params.signal ? { signal: params.signal } : {}), }); }, async readAuthorizedVirtualFile( @@ -327,6 +539,7 @@ export async function resolveBrokeredMemoryRuntime( virtualPath: params.virtualPath, }, expiresAtMs, + ...(params.signal ? { signal: params.signal } : {}), }); if (!result) { throw new Error("memory broker virtual file is unavailable"); @@ -346,39 +559,100 @@ export async function resolveBrokeredMemoryRuntime( } export async function closeBrokeredMemoryRuntimes(): Promise { - const supervisors = [...state.supervisors]; - state.supervisors.clear(); - await Promise.all(supervisors.map((supervisor) => supervisor.stop())); - const processes = [...state.processes.values()]; - state.processes.clear(); - await Promise.all( - processes.map(async (process) => { - try { - await (await process).close(); - } catch { - // Shutdown is best effort; a replacement process gets a fresh epoch and cannot reuse it. - } - }), - ); + state.closing = true; + try { + // `stop()` retires through the lifecycle reader lease. Stop supervisors before taking the + // shutdown writer so their in-flight probe cannot wait for the writer that waits for them. + const supervisors = [...state.supervisors]; + state.supervisors.clear(); + await Promise.all(supervisors.map((supervisor) => supervisor.stop())); + await withGatewayBrokeredMemoryMaintenanceLease(state.maintenance, async () => { + const moduleUrls = [...state.processes.keys()].toSorted(); + await Promise.all( + moduleUrls.map((moduleUrl) => + withBrokerLease(state.leases, moduleUrl, async () => { + try { + await retireProcessOnLease(state, moduleUrl); + } catch { + // Gateway shutdown still retires the map entry so a future lifecycle gets a new epoch. + } + }), + ), + ); + }); + } finally { + state.agentIdsByModule.clear(); + state.closing = false; + } } type BrokerMaintenanceProcess = Pick; async function runBrokeredMemoryMaintenance(params: { - processes: readonly BrokerMaintenanceProcess[]; + process: BrokerMaintenanceProcess; + run: () => Promise; + retire: () => Promise; +}): Promise { + if (!params.process.isRunning()) { + await params.retire(); + return await params.run(); + } + try { + await params.process.quiesce(); + } catch (error) { + await rejectAfterBrokerRetirement(error, params.retire); + } + + let result!: T; + let runError: unknown; + try { + result = await params.run(); + } catch (error) { + runError = error; + } + try { + await params.process.resume(); + } catch (resumeError) { + await rejectAfterBrokerRetirement( + runError === undefined ? resumeError : new AggregateError([runError, resumeError]), + params.retire, + ); + } + if (runError !== undefined) { + throw runError; + } + return result; +} + +async function rejectAfterBrokerRetirement( + error: unknown, + retire: () => Promise, +): Promise { + try { + await retire(); + } catch (retireError) { + throw new AggregateError([error, retireError], "memory broker retirement failed"); + } + throw error; +} + +async function runBrokeredMemoryMaintenanceForModule(params: { + runtimeState: BrokerRuntimeState; + moduleUrl: string; run: () => Promise; }): Promise { - const running = params.processes.filter((process) => process.isRunning()); - const quiesced: BrokerMaintenanceProcess[] = []; - try { - for (const process of running) { - await process.quiesce(); - quiesced.push(process); + return await withBrokerLease(params.runtimeState.leases, params.moduleUrl, async () => { + const pendingProcess = params.runtimeState.processes.get(params.moduleUrl); + if (!pendingProcess) { + return await params.run(); } - return await params.run(); - } finally { - await Promise.all(quiesced.map((process) => process.resume().catch(() => undefined))); - } + const process = await pendingProcess; + return await runBrokeredMemoryMaintenance({ + process, + run: params.run, + retire: () => retireProcessOnLease(params.runtimeState, params.moduleUrl), + }); + }); } /** @@ -386,10 +660,29 @@ async function runBrokeredMemoryMaintenance(params: { * their socket, SQLite handle, artifact root, or bootstrap credential in a worker process. */ export async function withBrokeredMemoryMaintenance(run: () => Promise): Promise { - const processes = await Promise.all([...state.processes.values()]); - return await runBrokeredMemoryMaintenance({ processes, run }); + return await withGatewayBrokeredMemoryMaintenanceLease(state.maintenance, async () => { + // The snapshot happens only after the write lease excludes broker resolution/replacement. + const moduleUrls = [...state.processes.keys()].toSorted(); + const runNext = async (index: number): Promise => { + const moduleUrl = moduleUrls[index]; + if (!moduleUrl) { + return await run(); + } + return await runBrokeredMemoryMaintenanceForModule({ + runtimeState: state, + moduleUrl, + run: () => runNext(index + 1), + }); + }; + return await runNext(0); + }); } export const testing = { + createBrokerMaintenanceGate, + withBrokerLease, + withBrokerLifecycleOperation, + tryWithBrokerLifecycleOperation, + withGatewayBrokeredMemoryMaintenanceLease, runBrokeredMemoryMaintenance, }; diff --git a/src/plugins/memory-invocation.ts b/src/plugins/memory-invocation.ts index 58bf73e21674..74bff7a90983 100644 --- a/src/plugins/memory-invocation.ts +++ b/src/plugins/memory-invocation.ts @@ -586,6 +586,7 @@ export async function createAuthorizedMemoryWriteInvocation(params: { export async function writeAuthorizedMemoryForInvocation(params: { invocation: AuthorizedMemoryWriteInvocation; mutation: AuthorizedMemoryMutation; + signal?: AbortSignal; }): Promise { const state = writeInvocationStates.get(params.invocation); const context = state ? readCurrentWriteContext(state) : undefined; @@ -597,6 +598,7 @@ export async function writeAuthorizedMemoryForInvocation(params: { context, plan: state.plan, mutation: params.mutation, + ...(params.signal ? { signal: params.signal } : {}), } as never); } catch { logMemoryInvocationDiagnostic("authorization-failed"); @@ -613,6 +615,7 @@ export async function stageAuthorizedMemorySealedCompactionForInvocation(params: invocation: AuthorizedMemoryWriteInvocation; content: string; transcriptSource: AuthorizedTranscriptDerivationSource; + signal?: AbortSignal; }): Promise { const state = writeInvocationStates.get(params.invocation); const context = state ? readCurrentWriteContext(state) : undefined; @@ -632,6 +635,7 @@ export async function stageAuthorizedMemorySealedCompactionForInvocation(params: plan: state.plan, content: params.content, transcriptSource: params.transcriptSource, + ...(params.signal ? { signal: params.signal } : {}), }); } catch { logMemoryInvocationDiagnostic("authorization-failed"); @@ -646,6 +650,7 @@ export async function stageAuthorizedMemorySealedCompactionForInvocation(params: */ export async function materializeAuthorizedMemoryVirtualView(params: { invocation: AuthorizedMemoryReadInvocation; + signal?: AbortSignal; }): Promise { const state = readState(params.invocation); const context = state ? readCurrentContext(state) : undefined; @@ -664,6 +669,7 @@ export async function materializeAuthorizedMemoryVirtualView(params: { const view = await state.virtualView.materializeAuthorizedVirtualView({ context, plan: state.plan, + ...(params.signal ? { signal: params.signal } : {}), }); const canonical = view ? canonicalizeAuthorizedVirtualView({ view, context, plan: state.plan }) @@ -687,6 +693,7 @@ export async function readAuthorizedMemoryVirtualFile(params: { invocation: AuthorizedMemoryReadInvocation; view: AuthorizedMemoryVirtualView; virtualPath: string; + signal?: AbortSignal; }): Promise { const state = readState(params.invocation); const context = state ? readCurrentContext(state) : undefined; @@ -708,6 +715,7 @@ export async function readAuthorizedMemoryVirtualFile(params: { plan: state.plan, view: params.view, virtualPath: params.virtualPath, + ...(params.signal ? { signal: params.signal } : {}), }); if ( !validateEnvelope({ @@ -781,6 +789,7 @@ export async function readAuthorizedMemoryForInvocation(params: { handleId: string; from?: number; lines?: number; + signal?: AbortSignal; }): Promise { const state = readState(params.invocation); const context = state ? readCurrentContext(state) : undefined; @@ -800,6 +809,7 @@ export async function readAuthorizedMemoryForInvocation(params: { handle, ...(params.from !== undefined ? { from: params.from } : {}), ...(params.lines !== undefined ? { lines: params.lines } : {}), + ...(params.signal ? { signal: params.signal } : {}), } as never); if ( !validateEnvelope({ diff --git a/src/plugins/registry-contribution-types.ts b/src/plugins/registry-contribution-types.ts index 7a852f6e8647..db13cb4c62d5 100644 --- a/src/plugins/registry-contribution-types.ts +++ b/src/plugins/registry-contribution-types.ts @@ -353,6 +353,7 @@ export type MemoryPluginVirtualViewProvider = { materializeAuthorizedVirtualView(params: { context: MemoryContentAccessContext<"read">; plan: AuthorizedMemoryContentPlan<"read">; + signal?: AbortSignal; }): Promise; readAuthorizedVirtualFile(params: { context: MemoryContentAccessContext<"read">; @@ -360,6 +361,7 @@ export type MemoryPluginVirtualViewProvider = { view: AuthorizedMemoryVirtualView; /** Virtual-only slash-separated path, never a host filesystem path. */ virtualPath: string; + signal?: AbortSignal; }): Promise>; }; diff --git a/src/plugins/tool-types.ts b/src/plugins/tool-types.ts index 2502ca9abb76..9b22393b3fc5 100644 --- a/src/plugins/tool-types.ts +++ b/src/plugins/tool-types.ts @@ -53,6 +53,7 @@ export type AuthorizedMemoryReadHost = Readonly<{ handleId: string; from?: number; lines?: number; + signal?: AbortSignal; }) => Promise; }>; diff --git a/src/process/supervisor/adapters/child.test.ts b/src/process/supervisor/adapters/child.test.ts index 33c3d2014871..d5afef843769 100644 --- a/src/process/supervisor/adapters/child.test.ts +++ b/src/process/supervisor/adapters/child.test.ts @@ -271,9 +271,9 @@ describe("createChildAdapter", () => { expect(firstSpawnWithFallbackParams().fallbacks).toEqual([]); expect(firstSpawnWithFallbackParams().options?.stdio).toEqual(["pipe", "pipe", "pipe", "ipc"]); - await adapter.openStartGate?.(); + await adapter.openStartGate?.({ launchId: "turn-1", planHash: "a".repeat(64) }); expect(sendMock).toHaveBeenCalledWith( - { type: "openclaw-worker-start-v1" }, + { type: "openclaw-worker-start-v1", launchId: "turn-1", planHash: "a".repeat(64) }, expect.any(Function), ); adapter.closeStartGate?.(); diff --git a/src/process/supervisor/adapters/child.ts b/src/process/supervisor/adapters/child.ts index 8548ae9f81d9..8cafff6f5020 100644 --- a/src/process/supervisor/adapters/child.ts +++ b/src/process/supervisor/adapters/child.ts @@ -72,12 +72,16 @@ function resolveChildInvocation(params: { } type ChildAdapter = SpawnProcessAdapter; +type WorkerStartIdentity = { + launchId: string; + planHash: string; +}; type WorkerChildAdapter = ChildAdapter & { closeStartGate?: () => void; - openStartGate?: () => Promise; + openStartGate?: (identity: WorkerStartIdentity) => Promise; }; -const WORKER_START_MESSAGE = { type: "openclaw-worker-start-v1" } as const; +const WORKER_START_MESSAGE_TYPE = "openclaw-worker-start-v1"; function isServiceManagedRuntime(): boolean { return Boolean(process.env.OPENCLAW_SERVICE_MARKER?.trim()); @@ -516,7 +520,7 @@ export async function createChildAdapter(params: { let startGateOpened = false; const openStartGate = params.ownedWorker - ? async () => { + ? async (identity: WorkerStartIdentity) => { if (startGateOpened) { return; } @@ -527,7 +531,7 @@ export async function createChildAdapter(params: { return; } try { - child.send(WORKER_START_MESSAGE, (error) => { + child.send({ type: WORKER_START_MESSAGE_TYPE, ...identity }, (error) => { if (error) { reject(error); return; diff --git a/src/state/openclaw-state-db-contract.ts b/src/state/openclaw-state-db-contract.ts index 7dccb1d3042b..704309b5a379 100644 --- a/src/state/openclaw-state-db-contract.ts +++ b/src/state/openclaw-state-db-contract.ts @@ -14,6 +14,8 @@ export const FIRST_USE_STATE_TABLES = [ "cron_job_runtime_authorities", "execution_identity_contexts", "mcp_oauth_pending_authorizations", + "node_worker_container_launches", + "node_worker_container_leases", "node_worker_launches", "operator_approval_execution_identities", "execution_decision_facts", diff --git a/src/state/openclaw-state-db.generated.d.ts b/src/state/openclaw-state-db.generated.d.ts index f2a7baeca038..b730fb142f3a 100644 --- a/src/state/openclaw-state-db.generated.d.ts +++ b/src/state/openclaw-state-db.generated.d.ts @@ -1154,6 +1154,18 @@ export interface NodeHostConfig { version: number; } +export interface NodeWorkerContainerLaunches { + container_engine: string; + launch_id: string; + plan_hash: string; +} + +export interface NodeWorkerContainerLeases { + expires_at_ms: number; + launch_id: string; + plan_hash: string; +} + export interface NodeWorkerLaunches { completed_at_ms: number | null; created_at_ms: number; @@ -2031,6 +2043,8 @@ export interface DB { model_catalog_remote: ModelCatalogRemote; native_hook_relay_bridges: NativeHookRelayBridges; node_host_config: NodeHostConfig; + node_worker_container_launches: NodeWorkerContainerLaunches; + node_worker_container_leases: NodeWorkerContainerLeases; node_worker_launches: NodeWorkerLaunches; official_external_plugin_catalog_snapshots: OfficialExternalPluginCatalogSnapshots; onboarding_recommendations: OnboardingRecommendations; diff --git a/src/state/openclaw-state-schema.sql b/src/state/openclaw-state-schema.sql index 6175a870e191..eba6b4474853 100644 --- a/src/state/openclaw-state-schema.sql +++ b/src/state/openclaw-state-schema.sql @@ -1350,6 +1350,30 @@ CREATE TABLE IF NOT EXISTS node_worker_launches ( ) ) STRICT; +-- Container launch metadata is deliberately separate from the public receipt. +-- Recovery needs the selected engine to prove cleanup; worker descriptors and +-- credentials remain process memory only. +CREATE TABLE IF NOT EXISTS node_worker_container_launches ( + launch_id TEXT NOT NULL PRIMARY KEY + CHECK (length(launch_id) BETWEEN 1 AND 256 AND instr(launch_id, char(0)) = 0), + plan_hash TEXT NOT NULL + CHECK (length(plan_hash) = 64 AND plan_hash NOT GLOB '*[^0-9a-f]*'), + container_engine TEXT NOT NULL CHECK (container_engine IN ('docker', 'podman')), + FOREIGN KEY (launch_id) REFERENCES node_worker_launches(launch_id) ON DELETE CASCADE +) STRICT; + +-- The container lease is separate from the launch receipt and engine metadata. +-- It lets a restarted node withdraw an expired projection without retaining the +-- descriptor, projection content, or any Gateway credential in durable state. +CREATE TABLE IF NOT EXISTS node_worker_container_leases ( + launch_id TEXT NOT NULL PRIMARY KEY + CHECK (length(launch_id) BETWEEN 1 AND 256 AND instr(launch_id, char(0)) = 0), + plan_hash TEXT NOT NULL + CHECK (length(plan_hash) = 64 AND plan_hash NOT GLOB '*[^0-9a-f]*'), + expires_at_ms INTEGER NOT NULL CHECK (expires_at_ms BETWEEN 0 AND 9007199254740991), + FOREIGN KEY (launch_id) REFERENCES node_worker_launches(launch_id) ON DELETE CASCADE +) STRICT; + CREATE INDEX IF NOT EXISTS idx_node_worker_launches_terminal_completed ON node_worker_launches(completed_at_ms, launch_id) WHERE completed_at_ms IS NOT NULL; diff --git a/src/worker/node-memory-projection-protocol.test.ts b/src/worker/node-memory-projection-protocol.test.ts new file mode 100644 index 000000000000..ff4d5944b9ea --- /dev/null +++ b/src/worker/node-memory-projection-protocol.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from "vitest"; +import { + buildNodeWorkerMemoryProjectionRequestProofPayload, + NODE_WORKER_MEMORY_PROJECTION_MAX_FILE_BYTES, + parseNodeWorkerMemoryProjection, + parseNodeWorkerMemoryProjectionPayload, + parseNodeWorkerMemoryProjectionRequestProof, +} from "./node-memory-projection-protocol.js"; + +describe("node worker memory projection protocol", () => { + it("accepts only an opaque projection reference with a bounded node lease", () => { + expect( + parseNodeWorkerMemoryProjection({ + version: 1, + reference: "a".repeat(43), + binding: { launch: "b".repeat(64), authorization: "c".repeat(64) }, + expiresAtMs: 1_900_000_000_000, + }), + ).toEqual({ + version: 1, + reference: "a".repeat(43), + binding: { launch: "b".repeat(64), authorization: "c".repeat(64) }, + expiresAtMs: 1_900_000_000_000, + }); + expect( + parseNodeWorkerMemoryProjection({ + version: 1, + reference: "a".repeat(43), + binding: { launch: "b".repeat(64), authorization: "c".repeat(64) }, + expiresAtMs: 1_900_000_000_000, + rawPath: "/private/memory.sqlite", + }), + ).toBeNull(); + expect(parseNodeWorkerMemoryProjection({ version: 1, reference: "short" })).toBeNull(); + }); + + it("canonicalizes only bounded node-host transfer proofs", () => { + const proof = { + nodeId: "node-1", + signedAtMs: 123, + signature: "a".repeat(86), + }; + expect(parseNodeWorkerMemoryProjectionRequestProof(proof)).toEqual(proof); + expect( + buildNodeWorkerMemoryProjectionRequestProofPayload({ + reference: "a".repeat(43), + binding: { launch: "b".repeat(64), authorization: "c".repeat(64) }, + nodeId: proof.nodeId, + signedAtMs: proof.signedAtMs, + }), + ).toBe( + [ + "openclaw.node-worker-memory-projection.request.v1", + "GET", + "/__openclaw__/worker-memory-projection/v1/projection", + "a".repeat(43), + "b".repeat(64), + "c".repeat(64), + "node-1", + "123", + ].join("\n"), + ); + expect( + parseNodeWorkerMemoryProjectionRequestProof({ ...proof, signature: "short" }), + ).toBeNull(); + expect(parseNodeWorkerMemoryProjectionRequestProof({ ...proof, extra: true })).toBeNull(); + }); + + it.each([ + "shared/../secret.md", + "shared/child/secret.md", + "shared\\secret.md", + "workspace/secret.md", + ])("rejects a non-isolated virtual path: %s", (virtualPath) => { + expect( + parseNodeWorkerMemoryProjectionPayload({ + version: 1, + files: [ + { + virtualPath, + sha256: "a".repeat(64), + contentBase64: "c2VjcmV0", + }, + ], + }), + ).toBeNull(); + }); + + it("accepts the canonical memory view beneath the fixed container mount", () => { + expect( + parseNodeWorkerMemoryProjectionPayload({ + version: 1, + files: [ + { + virtualPath: "memory/MEMORY.md", + sha256: "a".repeat(64), + contentBase64: "c2VjcmV0", + }, + ], + }), + ).toMatchObject({ files: [{ virtualPath: "memory/MEMORY.md" }] }); + }); + + it("rejects path collisions and payloads beyond the immutable byte bound", () => { + const file = { + virtualPath: "shared/brief.md", + sha256: "a".repeat(64), + contentBase64: "c2VjcmV0", + }; + expect( + parseNodeWorkerMemoryProjectionPayload({ + version: 1, + files: [file, { ...file, virtualPath: "SHARED/BRIEF.md" }], + }), + ).toBeNull(); + expect( + parseNodeWorkerMemoryProjectionPayload({ + version: 1, + files: [ + { + ...file, + contentBase64: Buffer.alloc(NODE_WORKER_MEMORY_PROJECTION_MAX_FILE_BYTES + 1).toString( + "base64", + ), + }, + ], + }), + ).toBeNull(); + }); +}); diff --git a/src/worker/node-memory-projection-protocol.ts b/src/worker/node-memory-projection-protocol.ts new file mode 100644 index 000000000000..66faf561d26b --- /dev/null +++ b/src/worker/node-memory-projection-protocol.ts @@ -0,0 +1,246 @@ +/** Private, single-use Gateway-to-node transfer for an authorized virtual-memory snapshot. */ +import { isRecord } from "@openclaw/normalization-core/record-coerce"; + +export const NODE_WORKER_MEMORY_PROJECTION_TRANSFER_PATH = + "/__openclaw__/worker-memory-projection/v1"; +export const NODE_WORKER_MEMORY_PROJECTION_VERSION = 1; +export const NODE_WORKER_MEMORY_PROJECTION_MAX_FILES = 64; +export const NODE_WORKER_MEMORY_PROJECTION_MAX_FILE_BYTES = 256 * 1024; +export const NODE_WORKER_MEMORY_PROJECTION_MAX_TOTAL_BYTES = 2 * 1024 * 1024; +export const NODE_WORKER_MEMORY_PROJECTION_PROOF_SKEW_MS = 2 * 60 * 1_000; +export const NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER = "x-openclaw-worker-projection-node"; +export const NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER = + "x-openclaw-worker-projection-signed-at"; +export const NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER = + "x-openclaw-worker-projection-signature"; + +const REFERENCE_PATTERN = /^[A-Za-z0-9_-]{43}$/u; +const PROJECTION_BINDING_PATTERN = /^[a-f0-9]{64}$/u; +const RESERVED_VIRTUAL_ROOTS = new Set(["opt", "run", "workspace"]); +const PROOF_NODE_ID_PATTERN = /^[^\0\r\n]{1,256}$/u; +const ED25519_SIGNATURE_PATTERN = /^[A-Za-z0-9_-]{86}$/u; + +/** + * This is deliberately the only projection datum serializable into a node launch. + * Its opaque, single-use bearer and non-secret expiry never name a store, artifact, + * view, or host path. + */ +export type NodeWorkerMemoryProjection = Readonly<{ + version: typeof NODE_WORKER_MEMORY_PROJECTION_VERSION; + reference: string; + /** + * Gateway-issued opaque hashes. `launch` fences the descriptor that the node + * may stage for; `authorization` also commits to the selected broker view. + */ + binding: NodeWorkerMemoryProjectionBinding; + /** + * Non-secret Gateway-issued lease for the locally staged immutable view. + * The node uses it to withdraw the container after a lost Gateway connection. + */ + expiresAtMs: number; +}>; + +export type NodeWorkerMemoryProjectionBinding = Readonly<{ + launch: string; + authorization: string; +}>; + +/** + * Node-host proof for a one-use projection retrieval. The Gateway binds this + * signature to the issued capability's exact node connection before consuming it. + */ +export type NodeWorkerMemoryProjectionRequestProof = Readonly<{ + nodeId: string; + signedAtMs: number; + signature: string; +}>; + +export type NodeWorkerMemoryProjectionFile = Readonly<{ + virtualPath: string; + sha256: string; + contentBase64: string; +}>; + +export type NodeWorkerMemoryProjectionPayload = Readonly<{ + version: typeof NODE_WORKER_MEMORY_PROJECTION_VERSION; + files: readonly NodeWorkerMemoryProjectionFile[]; +}>; + +export function parseNodeWorkerMemoryProjection(value: unknown): NodeWorkerMemoryProjection | null { + if ( + !isRecord(value) || + Object.keys(value).length !== 4 || + !Object.hasOwn(value, "version") || + !Object.hasOwn(value, "reference") || + !Object.hasOwn(value, "binding") || + !Object.hasOwn(value, "expiresAtMs") || + value.version !== NODE_WORKER_MEMORY_PROJECTION_VERSION || + typeof value.reference !== "string" || + !REFERENCE_PATTERN.test(value.reference) || + !isRecord(value.binding) || + Object.keys(value.binding).length !== 2 || + typeof value.binding.launch !== "string" || + !PROJECTION_BINDING_PATTERN.test(value.binding.launch) || + typeof value.binding.authorization !== "string" || + !PROJECTION_BINDING_PATTERN.test(value.binding.authorization) || + typeof value.expiresAtMs !== "number" || + !Number.isSafeInteger(value.expiresAtMs) || + value.expiresAtMs < 0 + ) { + return null; + } + return Object.freeze({ + version: NODE_WORKER_MEMORY_PROJECTION_VERSION, + reference: value.reference, + binding: Object.freeze({ + launch: value.binding.launch, + authorization: value.binding.authorization, + }), + expiresAtMs: value.expiresAtMs, + }); +} + +export function parseNodeWorkerMemoryProjectionRequestProof( + value: unknown, +): NodeWorkerMemoryProjectionRequestProof | null { + if ( + !isRecord(value) || + Object.keys(value).length !== 3 || + !Object.hasOwn(value, "nodeId") || + !Object.hasOwn(value, "signedAtMs") || + !Object.hasOwn(value, "signature") || + typeof value.nodeId !== "string" || + !PROOF_NODE_ID_PATTERN.test(value.nodeId) || + typeof value.signedAtMs !== "number" || + !Number.isSafeInteger(value.signedAtMs) || + value.signedAtMs < 0 || + typeof value.signature !== "string" || + !ED25519_SIGNATURE_PATTERN.test(value.signature) + ) { + return null; + } + return Object.freeze({ + nodeId: value.nodeId, + signedAtMs: value.signedAtMs, + signature: value.signature, + }); +} + +/** Canonical domain-separated payload signed by the node-host device identity. */ +export function buildNodeWorkerMemoryProjectionRequestProofPayload(params: { + reference: string; + binding: NodeWorkerMemoryProjectionBinding; + nodeId: string; + signedAtMs: number; +}): string { + if ( + !REFERENCE_PATTERN.test(params.reference) || + !PROJECTION_BINDING_PATTERN.test(params.binding.launch) || + !PROJECTION_BINDING_PATTERN.test(params.binding.authorization) || + !PROOF_NODE_ID_PATTERN.test(params.nodeId) || + !Number.isSafeInteger(params.signedAtMs) || + params.signedAtMs < 0 + ) { + throw new Error("invalid node worker memory projection request proof"); + } + return [ + "openclaw.node-worker-memory-projection.request.v1", + "GET", + nodeWorkerMemoryProjectionTransferPath(), + params.reference, + params.binding.launch, + params.binding.authorization, + params.nodeId, + String(params.signedAtMs), + ].join("\n"); +} + +function isSafeVirtualPath(value: unknown): value is string { + if (typeof value !== "string" || value !== value.normalize("NFC")) { + return false; + } + const [root, leaf, ...rest] = value.split("/"); + // `memory/...` is the canonical selected-memory view, nested below the + // container's fixed `/memory` mount rather than a host filesystem path. + return ( + Boolean(root) && + Boolean(leaf) && + rest.length === 0 && + /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/u.test(root!) && + !RESERVED_VIRTUAL_ROOTS.has(root!.toLocaleLowerCase("en-US")) && + leaf !== "." && + leaf !== ".." && + !leaf!.includes("\\") && + !leaf!.includes("\0") + ); +} + +/** Parses the bounded immutable response before a node ever writes a projection byte. */ +export function parseNodeWorkerMemoryProjectionPayload( + value: unknown, +): NodeWorkerMemoryProjectionPayload | null { + if ( + !isRecord(value) || + Object.keys(value).length !== 2 || + !Object.hasOwn(value, "version") || + !Object.hasOwn(value, "files") || + value.version !== NODE_WORKER_MEMORY_PROJECTION_VERSION || + !Array.isArray(value.files) + ) { + return null; + } + const files = value.files; + if (files.length === 0 || files.length > NODE_WORKER_MEMORY_PROJECTION_MAX_FILES) { + return null; + } + let totalBytes = 0; + const paths = new Set(); + const parsed: NodeWorkerMemoryProjectionFile[] = []; + for (const file of files) { + if ( + !isRecord(file) || + Object.keys(file).length !== 3 || + !Object.hasOwn(file, "virtualPath") || + !Object.hasOwn(file, "sha256") || + !Object.hasOwn(file, "contentBase64") || + !isSafeVirtualPath(file.virtualPath) || + typeof file.sha256 !== "string" || + !/^[a-f0-9]{64}$/u.test(file.sha256) || + typeof file.contentBase64 !== "string" || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(file.contentBase64) + ) { + return null; + } + const contentBase64 = file.contentBase64; + const bytes = Buffer.byteLength(contentBase64, "base64"); + if (bytes > NODE_WORKER_MEMORY_PROJECTION_MAX_FILE_BYTES) { + return null; + } + totalBytes += bytes; + if (totalBytes > NODE_WORKER_MEMORY_PROJECTION_MAX_TOTAL_BYTES) { + return null; + } + const virtualPath = file.virtualPath; + const key = virtualPath.toLocaleLowerCase("en-US"); + if (paths.has(key)) { + return null; + } + paths.add(key); + parsed.push( + Object.freeze({ + virtualPath, + sha256: file.sha256, + contentBase64, + }), + ); + } + return Object.freeze({ + version: NODE_WORKER_MEMORY_PROJECTION_VERSION, + files: Object.freeze(parsed), + }); +} + +/** The bearer is sent in Authorization, not placed in a URL or request body. */ +export function nodeWorkerMemoryProjectionTransferPath(): string { + return `${NODE_WORKER_MEMORY_PROJECTION_TRANSFER_PATH}/projection`; +} diff --git a/src/worker/node-supervisor-protocol.test.ts b/src/worker/node-supervisor-protocol.test.ts index a6b39ab1b1de..b3a941e3451f 100644 --- a/src/worker/node-supervisor-protocol.test.ts +++ b/src/worker/node-supervisor-protocol.test.ts @@ -1,14 +1,23 @@ import { describe, expect, it } from "vitest"; +import { + WORKER_PROTOCOL_FEATURES, + WORKER_RPC_SET_VERSION, +} from "../../packages/gateway-protocol/src/schema/worker-admission.js"; import { NODE_WORKER_CONNECTION_FAILURE_MESSAGE_TYPE, + NODE_WORKER_EXECUTION_CONTAINER_V1, + NODE_WORKER_EXECUTION_HOST_V1, + nodeWorkerMemoryProjectionLaunchBinding, + nodeWorkerPlanHash, parseNodeWorkerConnectionFailureMessage, + parseNodeWorkerLaunchInput, parseNodeWorkerSupervisorReceipt, + type NodeWorkerLaunchInput, type NodeWorkerSupervisorIdentity, } from "./node-supervisor-protocol.js"; const RESULT_JSON_MAX_BYTES = 64 * 1024; const ERROR_TEXT_MAX_BYTES = 4 * 1024; - const identity: NodeWorkerSupervisorIdentity = { launchId: "launch-1", planHash: "a".repeat(64), @@ -19,6 +28,64 @@ const identity: NodeWorkerSupervisorIdentity = { runId: "run-1", }; +function launchInput(): NodeWorkerLaunchInput { + return { + launchId: "turn-1", + gatewayNamespace: "gateway-1", + expectedBundleHash: "a".repeat(64), + placementGeneration: 4, + execution: { kind: NODE_WORKER_EXECUTION_HOST_V1 }, + descriptor: { + version: 4, + admission: { + environmentId: "environment-1", + credential: "worker-credential", + sessionId: "session-1", + ownerEpoch: 3, + rpcSetVersion: WORKER_RPC_SET_VERSION, + handshake: { + bundleHash: "a".repeat(64), + openclawVersion: "2026.8.1", + protocolFeatures: [...WORKER_PROTOCOL_FEATURES], + }, + }, + assignment: { + agentId: "agent-1", + memoryReadEnforced: false, + operationalRunInstance: { instanceId: "instance-1", runId: "run-1" }, + agentRuntimeIdentityToken: "signed-runtime-token", + runId: "run-1", + turnId: "turn-1", + prompt: "Inspect the workspace.", + suppressPromptTranscript: true, + workspaceDir: "/tmp/openclaw-worker/workspace", + modelRef: { provider: "provider-1", model: "model-1" }, + inferenceOptions: {}, + initialMessages: [], + transcript: { baseLeafId: null, nextSeq: 1 }, + liveEvents: { ackedSeq: 0, nextSeq: 1 }, + toolAuthority: { allowedToolNames: [] }, + }, + }, + }; +} + +function parse(input: unknown) { + return parseNodeWorkerLaunchInput(JSON.stringify(input)); +} + +function memoryProjection(input: Omit) { + return { + version: 1 as const, + reference: "a".repeat(43), + binding: { + launch: nodeWorkerMemoryProjectionLaunchBinding(input), + authorization: "b".repeat(64), + }, + expiresAtMs: 1_900_000_000_000, + }; +} + describe("node worker supervisor wire receipt", () => { it("accepts only bounded worker connection diagnostics", () => { expect( @@ -52,9 +119,19 @@ describe("node worker supervisor wire receipt", () => { state: "completed", resultJson: JSON.stringify({ status: "completed", transcriptNextSeq: 2 }), }, - { ...identity, state: "failed", errorText: "worker exited before completion" }, - { ...identity, state: "interrupted", errorText: "node host stopped" }, - { ...identity, state: "cancelled", errorText: "node worker launch cancelled" }, + { + ...identity, + state: "failed", + errorText: "worker exited before completion", + executionStarted: false, + }, + { ...identity, state: "interrupted", errorText: "node host stopped", executionStarted: true }, + { + ...identity, + state: "cancelled", + errorText: "node worker launch cancelled", + executionStarted: true, + }, ])("round-trips the closed $state receipt", (receipt) => { expect(parseNodeWorkerSupervisorReceipt(receipt)).toEqual(receipt); }); @@ -97,3 +174,125 @@ describe("node worker supervisor wire receipt", () => { expect(parseNodeWorkerSupervisorReceipt(null)).toBeNull(); }); }); + +describe("node worker supervisor launch protocol", () => { + it.each([NODE_WORKER_EXECUTION_HOST_V1, NODE_WORKER_EXECUTION_CONTAINER_V1] as const)( + "accepts the exact %s execution form", + (kind) => { + const input = { ...launchInput(), execution: { kind } }; + + expect(parse(input).execution).toEqual({ kind }); + }, + ); + + it.each([ + (input: NodeWorkerLaunchInput) => { + const { execution: _execution, ...withoutExecution } = input; + return withoutExecution; + }, + (input: NodeWorkerLaunchInput) => ({ ...input, execution: { kind: "host-v2" } }), + (input: NodeWorkerLaunchInput) => ({ + ...input, + execution: { kind: NODE_WORKER_EXECUTION_HOST_V1, extra: true }, + }), + ])("rejects omitted or non-closed execution", (mutate) => { + expect(() => parse(mutate(launchInput()))).toThrow("invalid node worker"); + }); + + it("includes execution in the stable launch plan hash", () => { + const host = launchInput(); + const container = { ...host, execution: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 } as const }; + + expect(nodeWorkerPlanHash(host)).not.toBe(nodeWorkerPlanHash(container)); + }); + + it("binds an issued memory projection reference into the stable launch plan hash", () => { + const base = { + ...launchInput(), + execution: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 } as const, + descriptor: { + ...launchInput().descriptor, + assignment: { ...launchInput().descriptor.assignment, memoryReadEnforced: true }, + }, + }; + const first = { ...base, memoryProjection: memoryProjection(base) }; + const replay = { + ...first, + memoryProjection: { + ...memoryProjection(base), + reference: "b".repeat(43), + }, + }; + + expect(nodeWorkerPlanHash(first)).not.toBe(nodeWorkerPlanHash(replay)); + }); + + it("fails closed when a projection is replayed into another session, agent, or placement", () => { + const base = launchInput(); + base.execution = { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 }; + base.descriptor.assignment.memoryReadEnforced = true; + base.descriptor.assignment.workspaceDir = "/workspace"; + base.memoryProjection = memoryProjection(base); + + for (const replay of [ + { + ...base, + descriptor: { + ...base.descriptor, + admission: { ...base.descriptor.admission, sessionId: "session-2" }, + }, + }, + { + ...base, + descriptor: { + ...base.descriptor, + assignment: { ...base.descriptor.assignment, agentId: "agent-2" }, + }, + }, + { ...base, placementGeneration: base.placementGeneration + 1 }, + ]) { + expect(() => parse(replay)).toThrow("memory projection does not match its worker launch"); + } + }); + + it("requires the fixed container root for enforced memory launches", () => { + const input = launchInput(); + input.execution = { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 }; + input.descriptor.assignment.memoryReadEnforced = true; + input.memoryProjection = memoryProjection(input); + + expect(() => parse(input)).toThrow("must use the /workspace root"); + + input.descriptor.assignment.workspaceDir = "/workspace"; + expect(parse(input).descriptor.assignment.workspaceDir).toBe("/workspace"); + }); + + it("rejects enforced memory when an untrusted node tries to downgrade execution to host-v1", () => { + const input = launchInput(); + input.descriptor.assignment.memoryReadEnforced = true; + input.memoryProjection = memoryProjection(input); + + expect(() => parse(input)).toThrow("requires container-v1 execution"); + }); + + it("checks the containment root only when the paired permission context is present", () => { + const input = launchInput(); + input.execution = { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 }; + input.descriptor.assignment.memoryReadEnforced = true; + input.memoryProjection = memoryProjection(input); + input.descriptor.assignment.workspaceDir = "/workspace"; + input.descriptor.assignment = { + ...input.descriptor.assignment, + permissionMode: "workspace", + workerContainmentRoot: "/host/workspace", + }; + + expect(() => parse(input)).toThrow("must use the /workspace root"); + + input.descriptor.assignment = { + ...input.descriptor.assignment, + workerContainmentRoot: "/workspace", + }; + expect(parse(input).descriptor.assignment.workerContainmentRoot).toBe("/workspace"); + }); +}); diff --git a/src/worker/node-supervisor-protocol.ts b/src/worker/node-supervisor-protocol.ts index d2b0774d1496..404949148ca3 100644 --- a/src/worker/node-supervisor-protocol.ts +++ b/src/worker/node-supervisor-protocol.ts @@ -2,6 +2,10 @@ import { createHash } from "node:crypto"; import { stableStringify } from "@openclaw/normalization-core"; import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { parseWorkerLaunchPlan, type WorkerLaunchPlan } from "./launch-descriptor.js"; +import { + parseNodeWorkerMemoryProjection, + type NodeWorkerMemoryProjection, +} from "./node-memory-projection-protocol.js"; const IDENTIFIER_MAX_CHARS = 256; const GATEWAY_NAMESPACE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; @@ -10,6 +14,17 @@ const NODE_WORKER_RESULT_JSON_MAX_BYTES = 64 * 1024; const NODE_WORKER_ERROR_TEXT_MAX_BYTES = 4 * 1024; const NODE_WORKER_CONNECTION_FAILURE_CAUSE_MAX_BYTES = 64 * 1024; export const NODE_WORKER_CONNECTION_FAILURE_MESSAGE_TYPE = "openclaw-worker-connection-failure-v1"; +export const NODE_WORKER_EXECUTION_STARTED_MESSAGE_TYPE = "openclaw-worker-execution-started-v1"; +export const NODE_WORKER_EXECUTION_HOST_V1 = "host-v1"; +export const NODE_WORKER_EXECUTION_CONTAINER_V1 = "container-v1"; + +/** + * The launch execution boundary is selected by the Gateway and hashed with the + * rest of the launch. A node must never reinterpret a container launch as host work. + */ +export type NodeWorkerExecution = + | { kind: typeof NODE_WORKER_EXECUTION_HOST_V1 } + | { kind: typeof NODE_WORKER_EXECUTION_CONTAINER_V1 }; export type NodeWorkerLaunchInput = { launchId: string; @@ -17,6 +32,8 @@ export type NodeWorkerLaunchInput = { expectedBundleHash: string; placementGeneration: number; descriptor: WorkerLaunchPlan; + execution: NodeWorkerExecution; + memoryProjection?: NodeWorkerMemoryProjection; }; export type NodeWorkerSupervisorIdentity = { @@ -41,6 +58,8 @@ type NodeWorkerSupervisorCompletedReceipt = NodeWorkerSupervisorIdentity & { type NodeWorkerSupervisorErrorReceipt = NodeWorkerSupervisorIdentity & { state: "failed" | "interrupted" | "cancelled"; errorText: string; + /** A terminal worker identity is only durably recorded after child execution started. */ + executionStarted: boolean; }; export type NodeWorkerSupervisorReceipt = @@ -53,6 +72,13 @@ export type NodeWorkerConnectionFailureMessage = { cause: string | null; }; +/** Private child-to-supervisor acknowledgement for the exact gate release. */ +export type NodeWorkerExecutionStartedMessage = { + type: typeof NODE_WORKER_EXECUTION_STARTED_MESSAGE_TYPE; + launchId: string; + planHash: string; +}; + function hasExactKeys(value: Record, keys: readonly string[]): boolean { return ( Object.keys(value).length === keys.length && keys.every((key) => Object.hasOwn(value, key)) @@ -91,6 +117,34 @@ function isPlanHash(value: unknown): value is string { return typeof value === "string" && /^[a-f0-9]{64}$/u.test(value); } +function parseNodeWorkerExecution(value: unknown): NodeWorkerExecution | null { + if (!isRecord(value) || !hasExactKeys(value, ["kind"])) { + return null; + } + if (value.kind === NODE_WORKER_EXECUTION_HOST_V1) { + return { kind: NODE_WORKER_EXECUTION_HOST_V1 }; + } + if (value.kind === NODE_WORKER_EXECUTION_CONTAINER_V1) { + return { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 }; + } + return null; +} + +function requiresContainerWorkspaceRoot(descriptor: WorkerLaunchPlan): boolean { + if (!descriptor.assignment.memoryReadEnforced) { + return false; + } + return ( + descriptor.assignment.workspaceDir !== "/workspace" || + (descriptor.assignment.workerContainmentRoot !== undefined && + descriptor.assignment.workerContainmentRoot !== "/workspace") + ); +} + +function requiresMemoryProjection(descriptor: WorkerLaunchPlan): boolean { + return descriptor.assignment.memoryReadEnforced; +} + function decodeRequest(raw?: string | null): unknown { if (!raw) { throw new Error("INVALID_REQUEST: paramsJSON required"); @@ -106,13 +160,27 @@ export function parseNodeWorkerLaunchInput(raw?: string | null): NodeWorkerLaunc const value = decodeRequest(raw); if ( !isRecord(value) || - !hasExactKeys(value, [ - "launchId", - "gatewayNamespace", - "expectedBundleHash", - "placementGeneration", - "descriptor", - ]) + !hasExactKeys( + value, + value.memoryProjection === undefined + ? [ + "launchId", + "gatewayNamespace", + "expectedBundleHash", + "placementGeneration", + "descriptor", + "execution", + ] + : [ + "launchId", + "gatewayNamespace", + "expectedBundleHash", + "placementGeneration", + "descriptor", + "execution", + "memoryProjection", + ], + ) ) { throw new Error("INVALID_REQUEST: invalid node worker launch request"); } @@ -135,6 +203,45 @@ export function parseNodeWorkerLaunchInput(raw?: string | null): NodeWorkerLaunc if (descriptor.admission.handshake.bundleHash !== value.expectedBundleHash) { throw new Error("INVALID_REQUEST: descriptor bundle hash does not match expectedBundleHash"); } + const execution = parseNodeWorkerExecution(value.execution); + if (!execution) { + throw new Error("INVALID_REQUEST: invalid node worker execution"); + } + if ( + descriptor.assignment.memoryReadEnforced && + execution.kind !== NODE_WORKER_EXECUTION_CONTAINER_V1 + ) { + throw new Error("INVALID_REQUEST: enforced memory worker requires container-v1 execution"); + } + if (requiresContainerWorkspaceRoot(descriptor)) { + throw new Error( + "INVALID_REQUEST: enforced container worker descriptor must use the /workspace root", + ); + } + const memoryProjection = + value.memoryProjection === undefined + ? undefined + : parseNodeWorkerMemoryProjection(value.memoryProjection); + if (value.memoryProjection !== undefined && !memoryProjection) { + throw new Error("INVALID_REQUEST: invalid node worker memory projection"); + } + if (requiresMemoryProjection(descriptor) !== Boolean(memoryProjection)) { + throw new Error("INVALID_REQUEST: enforced container worker requires an issued memory projection"); + } + if ( + memoryProjection && + memoryProjection.binding.launch !== + nodeWorkerMemoryProjectionLaunchBinding({ + launchId, + gatewayNamespace, + expectedBundleHash: value.expectedBundleHash, + placementGeneration: value.placementGeneration, + descriptor, + execution, + }) + ) { + throw new Error("INVALID_REQUEST: memory projection does not match its worker launch"); + } return { launchId, gatewayNamespace, @@ -144,6 +251,8 @@ export function parseNodeWorkerLaunchInput(raw?: string | null): NodeWorkerLaunc "placementGeneration", ), descriptor, + execution, + ...(memoryProjection ? { memoryProjection } : {}), }; } @@ -194,7 +303,12 @@ export function parseNodeWorkerCancelInput(raw?: string | null): NodeWorkerSuper export function nodeWorkerPlanHash( input: Pick< NodeWorkerLaunchInput, - "descriptor" | "expectedBundleHash" | "gatewayNamespace" | "placementGeneration" + | "descriptor" + | "execution" + | "expectedBundleHash" + | "gatewayNamespace" + | "memoryProjection" + | "placementGeneration" >, ): string { return createHash("sha256") @@ -202,13 +316,50 @@ export function nodeWorkerPlanHash( stableStringify({ expectedBundleHash: input.expectedBundleHash, descriptor: input.descriptor, + execution: input.execution, gatewayNamespace: input.gatewayNamespace, + memoryProjection: input.memoryProjection, placementGeneration: input.placementGeneration, }), ) .digest("hex"); } +/** + * A node can recompute this non-secret fence before fetching projection bytes. + * The second opaque projection hash is verified by the Gateway against the + * selected broker view, so swapping either launch or memory authority fails. + */ +export function nodeWorkerMemoryProjectionLaunchBinding( + input: Pick< + NodeWorkerLaunchInput, + | "launchId" + | "gatewayNamespace" + | "expectedBundleHash" + | "placementGeneration" + | "descriptor" + | "execution" + >, +): string { + return createHash("sha256") + .update( + stableStringify({ + launchId: input.launchId, + gatewayNamespace: input.gatewayNamespace, + expectedBundleHash: input.expectedBundleHash, + placementGeneration: input.placementGeneration, + execution: input.execution, + environmentId: input.descriptor.admission.environmentId, + sessionId: input.descriptor.admission.sessionId, + ownerEpoch: input.descriptor.admission.ownerEpoch, + agentId: input.descriptor.assignment.agentId, + runId: input.descriptor.assignment.runId, + turnId: input.descriptor.assignment.turnId, + }), + ) + .digest("hex"); +} + const RECEIPT_IDENTITY_KEYS = [ "launchId", "planHash", @@ -286,6 +437,25 @@ export function parseNodeWorkerConnectionFailureMessage( }; } +export function parseNodeWorkerExecutionStartedMessage( + value: unknown, +): NodeWorkerExecutionStartedMessage | null { + if ( + !isRecord(value) || + !hasExactKeys(value, ["type", "launchId", "planHash"]) || + value.type !== NODE_WORKER_EXECUTION_STARTED_MESSAGE_TYPE || + !isIdentifier(value.launchId) || + !isPlanHash(value.planHash) + ) { + return null; + } + return { + type: NODE_WORKER_EXECUTION_STARTED_MESSAGE_TYPE, + launchId: value.launchId, + planHash: value.planHash, + }; +} + export function parseNodeWorkerSupervisorReceipt( value: unknown, ): NodeWorkerSupervisorReceipt | null { @@ -308,9 +478,15 @@ export function parseNodeWorkerSupervisorReceipt( : null; } if (value.state === "failed" || value.state === "interrupted" || value.state === "cancelled") { - return hasExactKeys(value, [...RECEIPT_IDENTITY_KEYS, "state", "errorText"]) && - isBoundedErrorText(value.errorText) - ? { ...identity, state: value.state, errorText: value.errorText } + return hasExactKeys(value, [...RECEIPT_IDENTITY_KEYS, "state", "errorText", "executionStarted"]) && + isBoundedErrorText(value.errorText) && + typeof value.executionStarted === "boolean" + ? { + ...identity, + state: value.state, + errorText: value.errorText, + executionStarted: value.executionStarted, + } : null; } return null; diff --git a/src/worker/worker-command.runtime.test.ts b/src/worker/worker-command.runtime.test.ts index fd9ea9590b0e..80817d928a80 100644 --- a/src/worker/worker-command.runtime.test.ts +++ b/src/worker/worker-command.runtime.test.ts @@ -30,6 +30,7 @@ const descriptor = { }, assignment: { agentId: "agent-1", + memoryReadEnforced: false, operationalRunInstance: { instanceId: "instance-run-1", runId: "run-1" }, agentRuntimeIdentityToken: "signed-runtime-token", runId: "run-1", @@ -65,11 +66,13 @@ function lifetimeHarness() { resolveStarted = resolve; }); const dispose = vi.fn(); + const reportExecutionStarted = vi.fn(); const reportConnectionFailure = vi.fn(); const terminateOwnedTree = vi.fn(); return { contract: { dispose, + reportExecutionStarted, reportConnectionFailure, signal: controller.signal, started, @@ -78,6 +81,7 @@ function lifetimeHarness() { disconnectAfterStart: () => controller.abort(new Error("worker supervisor lifetime ended")), disconnectBeforeStart: () => resolveStarted(false), dispose, + reportExecutionStarted, open: () => resolveStarted(true), terminateOwnedTree, }; diff --git a/src/worker/worker-command.runtime.ts b/src/worker/worker-command.runtime.ts index 64452ec71e4b..4d561c95c043 100644 --- a/src/worker/worker-command.runtime.ts +++ b/src/worker/worker-command.runtime.ts @@ -13,6 +13,8 @@ type RunWorkerCommandOptions = { export type WorkerCommandLifetime = { dispose: () => void; + /** The supervisor only accepts this after it bound the gate to one exact launch. */ + reportExecutionStarted: () => void; reportConnectionFailure: (cause: string | undefined) => void; signal: AbortSignal; started: Promise; @@ -77,6 +79,9 @@ export async function runWorkerCommand(options: RunWorkerCommandOptions): Promis if (options.lifetime?.signal.aborted) { stopForLifetime(); } + // Descriptor parsing and the start gate both completed. The host worker is + // now allowed to execute; tell the supervisor before entering user code. + options.lifetime?.reportExecutionStarted(); process.once("SIGINT", stop); process.once("SIGTERM", stop); const result = await runWorkerDescriptor(descriptor, { diff --git a/src/worker/worker-fault-injection.test-support.ts b/src/worker/worker-fault-injection.test-support.ts index 2ce69b5b7666..4378a843e0ad 100644 --- a/src/worker/worker-fault-injection.test-support.ts +++ b/src/worker/worker-fault-injection.test-support.ts @@ -306,6 +306,7 @@ export class ComposedGatewayHarness { }, assignment: { agentId: "worker-agent", + memoryReadEnforced: false, runId, operationalRunInstance: createOperationalRunInstanceRef(runId), agentRuntimeIdentityToken: "test-agent-runtime-token", diff --git a/src/worker/worker-process.ts b/src/worker/worker-process.ts index 362c554f3afb..255e02a188ec 100644 --- a/src/worker/worker-process.ts +++ b/src/worker/worker-process.ts @@ -1,24 +1,36 @@ +import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { signalProcessTree } from "../process/kill-tree.js"; import type { WorkerBrowserRuntime } from "./browser-runtime.js"; import { NODE_WORKER_CONNECTION_FAILURE_MESSAGE_TYPE, + NODE_WORKER_EXECUTION_STARTED_MESSAGE_TYPE, type NodeWorkerConnectionFailureMessage, } from "./node-supervisor-protocol.js"; import { runWorkerCommand, type WorkerCommandLifetime } from "./worker-command.runtime.js"; const WORKER_START_MESSAGE_TYPE = "openclaw-worker-start-v1"; -function isWorkerStartMessage(value: unknown): boolean { - return ( - typeof value === "object" && - value !== null && - !Array.isArray(value) && - Object.keys(value).length === 1 && - (value as { type?: unknown }).type === WORKER_START_MESSAGE_TYPE - ); +function parseWorkerStartMessage(value: unknown): { launchId: string; planHash: string } | null { + if ( + !isRecord(value) || + Object.keys(value).length !== 3 || + value.type !== WORKER_START_MESSAGE_TYPE || + typeof value.launchId !== "string" || + value.launchId.length === 0 || + typeof value.planHash !== "string" || + !/^[a-f0-9]{64}$/u.test(value.planHash) + ) { + return null; + } + return { launchId: value.launchId, planHash: value.planHash }; } -function createWorkerIpcLifetime(): WorkerCommandLifetime { +/** + * Shared by the host-side container shim. The shim is still the durable + * supervisor child, so a container never begins before `markRunning` opens + * this inherited Node IPC gate. + */ +export function createWorkerIpcLifetime(): WorkerCommandLifetime { if (!process.connected || !process.channel || typeof process.send !== "function") { throw new Error("internal worker IPC mode requires a connected Node IPC channel"); } @@ -26,6 +38,8 @@ function createWorkerIpcLifetime(): WorkerCommandLifetime { let disposed = false; let started = false; let settled = false; + let startIdentity: { launchId: string; planHash: string } | undefined; + let executionReported = false; let resolveStarted!: (started: boolean) => void; let rejectStarted!: (error: Error) => void; const startedPromise = new Promise((resolve, reject) => { @@ -44,11 +58,13 @@ function createWorkerIpcLifetime(): WorkerCommandLifetime { if (disposed) { return; } - if (!isWorkerStartMessage(message) || settled) { + const identity = parseWorkerStartMessage(message); + if (!identity || settled) { rejectOrAbort(new Error("invalid internal worker IPC start message")); return; } started = true; + startIdentity = identity; settled = true; resolveStarted(true); }; @@ -70,6 +86,30 @@ function createWorkerIpcLifetime(): WorkerCommandLifetime { return { started: startedPromise, signal: abortController.signal, + reportExecutionStarted: () => { + if ( + disposed || + executionReported || + !startIdentity || + !process.connected || + typeof process.send !== "function" + ) { + return; + } + executionReported = true; + try { + process.send( + { + type: NODE_WORKER_EXECUTION_STARTED_MESSAGE_TYPE, + launchId: startIdentity.launchId, + planHash: startIdentity.planHash, + }, + () => {}, + ); + } catch { + // The disconnect handler owns shutdown when the supervisor is gone. + } + }, reportConnectionFailure: (cause) => { if (disposed || !process.connected || typeof process.send !== "function") { return; diff --git a/test/helpers/node-worker-container-projection-isolation.ts b/test/helpers/node-worker-container-projection-isolation.ts new file mode 100644 index 000000000000..bec0bb9b782e --- /dev/null +++ b/test/helpers/node-worker-container-projection-isolation.ts @@ -0,0 +1,498 @@ +import fs from "node:fs"; +import http from "node:http"; +import path from "node:path"; +import { expect, vi } from "vitest"; +import { WebSocketServer, type WebSocket } from "ws"; +import { + GATEWAY_CLIENT_IDS, + GATEWAY_CLIENT_MODES, +} from "../../packages/gateway-protocol/src/client-info.js"; +import { WORKER_PUBLIC_INGRESS_PATH } from "../../packages/gateway-protocol/src/schema/worker-admission.js"; +import type { AuthorizedMemoryVirtualFileBroker } from "../../src/agents/memory-authorized-read-host.js"; +import { execContainer } from "../../src/agents/sandbox/docker.js"; +import type { NodeWorkerSupervisorNodeProof } from "../../src/gateway/node-registry-private.js"; +import { + createNodeWorkerProjectionTransferHttpCallback, + handleNodeWorkerProjectionTransferHttpRequest, +} from "../../src/gateway/worker-environments/node-worker-projection-transfer-http.js"; +import { createNodeWorkerProjectionTransferService } from "../../src/gateway/worker-environments/node-worker-projection-transfer-service.js"; +import { + loadOrCreateDeviceIdentity, + publicKeyRawBase64UrlFromPem, +} from "../../src/infra/device-identity.js"; +import { NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE } from "../../src/infra/node-runner-inventory.js"; +import { + NODE_WORKER_CONTAINER_RELAY_SOCKET, + nodeWorkerContainerName, + removeOwnedNodeWorkerContainers, + resolveNodeWorkerContainerEngine, + type NodeWorkerContainerEngine, +} from "../../src/node-host/node-worker-container-runtime.js"; +import { NodeWorkerMemoryProjectionRuntime } from "../../src/node-host/node-worker-memory-projection.js"; +import { + nodeWorkerMemoryProjectionLaunchBinding, + nodeWorkerPlanHash, +} from "../../src/node-host/node-worker-supervisor-contract.js"; +import { createNodeWorkerSupervisor } from "../../src/node-host/node-worker-supervisor.js"; +import { + testWorkerLaunchInput, + writeNodeWorkerFixture, +} from "../../src/node-host/node-worker-supervisor.test-support.js"; +import { NodeWorkerWorkspaceRuntime } from "../../src/node-host/node-worker-workspace.js"; +import { nodeWorkerMemoryProjectionTransferPath } from "../../src/worker/node-memory-projection-protocol.js"; +import { NODE_WORKER_EXECUTION_CONTAINER_V1 } from "../../src/worker/node-supervisor-protocol.js"; + +function nodeProof(nodeId: string): NodeWorkerSupervisorNodeProof { + return { + nodeId, + connId: "container-e2e-conn", + pairingIdentity: "container-e2e-pairing", + pairingGeneration: "container-e2e-generation", + clientId: GATEWAY_CLIENT_IDS.NODE_HOST, + clientMode: GATEWAY_CLIENT_MODES.NODE, + protocolFeature: NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE, + workerHost: { + enabled: true, + capacity: { total: 1, available: 1 }, + processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, memoryProjection: 1 }, + }, + commands: [], + }; +} + +function workerSource(params: { + outsideArtifactPath: string; + issuedVirtualPath: string; + forbiddenEnvironmentVariable: string; +}): string { + return String.raw` +import fs from "node:fs"; +import net from "node:net"; +import path from "node:path"; + +let input = ""; +for await (const chunk of process.stdin) input += chunk; +const descriptor = JSON.parse(input); +const outsideArtifact = ${JSON.stringify(params.outsideArtifactPath)}; +if ( + descriptor.connectionEndpoint?.kind !== "unix" || + descriptor.connectionEndpoint.socketPath !== ${JSON.stringify(NODE_WORKER_CONTAINER_RELAY_SOCKET)} +) { + throw new Error("container worker did not receive the mounted relay endpoint"); +} +const probe = (operation) => { + try { + operation(); + return "allowed"; + } catch (error) { + return error && typeof error === "object" && "code" in error ? String(error.code) : "denied"; + } +}; +const rawArtifactRead = (() => { + try { + return fs.readFileSync(outsideArtifact, "utf8"); + } catch (error) { + return error && typeof error === "object" && "code" in error ? String(error.code) : "denied"; + } +})(); +const fdTargets = fs + .readdirSync("/proc/self/fd") + .map((fd) => { + try { + return fs.readlinkSync("/proc/self/fd/" + fd); + } catch { + return ""; + } + }); +const mountInfo = fs.readFileSync("/proc/self/mountinfo", "utf8"); +const relay = net.createConnection(descriptor.connectionEndpoint.socketPath); +let response = ""; +let proved = false; +const relayEvents = []; +const writeRelayEvents = () => { + fs.writeFileSync("/workspace/container-relay-events.json", JSON.stringify(relayEvents)); +}; +const fail = (message) => { + relayEvents.push("failed:" + message); + writeRelayEvents(); + process.stderr.write(message + "\n"); + process.exit(1); +}; +relay.once("connect", () => relayEvents.push("connected")); +relay.once("error", (error) => { + relayEvents.push("error:" + error.name + ":" + error.message); + fail("relay connection failed: " + error.message); +}); +relay.once("end", () => relayEvents.push("ended")); +relay.once("close", (hadError) => { + relayEvents.push("closed:" + String(hadError)); + if (!proved) fail("relay closed before its WebSocket handshake"); +}); +relay.on("data", (chunk) => { + relayEvents.push("data:" + String(chunk.byteLength)); + response += chunk.toString("ascii"); + if (proved || !response.includes("\r\n\r\n")) return; + if (!response.startsWith("HTTP/1.1 101")) { + fail("relay did not establish a WebSocket handshake: " + response.split("\r\n", 1)[0]); + return; + } + proved = true; + fs.writeFileSync( + "/workspace/container-proof.json", + JSON.stringify({ + issued: fs.readFileSync(${JSON.stringify(`/memory/${params.issuedVirtualPath}`)}, "utf8"), + rawArtifactRead, + rawArtifactRootEnumeration: probe(() => fs.readdirSync(path.dirname(outsideArtifact))), + hostArtifactFd: fdTargets.some((target) => target.includes(outsideArtifact)), + hostArtifactMount: mountInfo.includes(outsideArtifact), + brokerCredentialPresent: + process.env[${JSON.stringify(params.forbiddenEnvironmentVariable)}] !== undefined, + memoryRoots: fs.readdirSync("/memory").sort(), + unissuedMemoryRead: probe(() => fs.readFileSync("/memory/unissued/secret.md", "utf8")), + issuedMemoryWrite: probe(() => fs.writeFileSync(${JSON.stringify(`/memory/${params.issuedVirtualPath}`)}, "tampered")), + rootFilesystemWrite: probe(() => fs.writeFileSync("/etc/openclaw-isolation-probe", "tampered")), + relaySocket: descriptor.connectionEndpoint.socketPath, + relayEvents, + }), + ); + setInterval(() => {}, 1_000); +}); +relay.write( + [ + "GET / HTTP/1.1", + "Host: localhost", + "Upgrade: websocket", + "Connection: Upgrade", + "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==", + "Sec-WebSocket-Version: 13", + "", + "", + ].join("\r\n"), +); +process.once("SIGTERM", () => { + relay.destroy(); + process.exit(0); +}); +`; +} + +async function containerIds( + engine: NodeWorkerContainerEngine, + identity: { launchId: string; planHash: string }, +) { + const result = await execContainer( + engine, + ["ps", "--all", "--quiet", "--filter", `name=^/${nodeWorkerContainerName(identity)}$`], + { allowFailure: true }, + ); + expect(result.code, result.stderr).toBe(0); + return result.stdout + .split(/\r?\n/u) + .map((value) => value.trim()) + .filter(Boolean); +} + +/** + * Exercises the generic host-to-container transfer boundary with an opaque broker. + * Callers provide the broker so plugin-owned tests can prove their real child IPC + * without coupling this shared harness to a bundled plugin's private implementation. + */ +export async function verifyNodeWorkerContainerProjectionIsolation(params: { + root: string; + broker: AuthorizedMemoryVirtualFileBroker; + outsideArtifactPath: string; + outsideArtifactContents: string; + issuedVirtualPath: string; + issuedContents: string; + forbiddenEnvironmentVariable: string; +}): Promise { + fs.mkdirSync(params.root, { recursive: true }); + // The container projection is a short, signed copy of the already-authorized + // broker view. Clamp only that test lease so teardown is observable without + // changing the selected broker's independent authorization lifetime. + const projectionBroker: AuthorizedMemoryVirtualFileBroker = { + ...params.broker, + view: { + ...params.broker.view, + expiresAt: new Date( + Math.min(Date.parse(params.broker.view.expiresAt), Date.now() + 15_000), + ).toISOString(), + }, + }; + const { bundleRoot, env } = writeNodeWorkerFixture(params.root); + const identity = loadOrCreateDeviceIdentity({ + path: path.join(params.root, "node-device-identity.sqlite"), + }); + const node = nodeProof(identity.deviceId); + const projectionService = createNodeWorkerProjectionTransferService({ + resolveNodePublicKey: async (candidate) => + candidate.nodeId === node.nodeId && + candidate.connId === node.connId && + candidate.pairingIdentity === node.pairingIdentity && + candidate.pairingGeneration === node.pairingGeneration + ? publicKeyRawBase64UrlFromPem(identity.publicKeyPem) + : undefined, + // An exact paired connection is the authority boundary here; the just-claimed + // node capacity is intentionally not part of the one-use projection transfer. + isNodeCurrent: (candidate) => + candidate.nodeId === node.nodeId && + candidate.connId === node.connId && + candidate.pairingIdentity === node.pairingIdentity && + candidate.pairingGeneration === node.pairingGeneration && + candidate.clientId === node.clientId && + candidate.clientMode === node.clientMode && + candidate.protocolFeature === node.protocolFeature, + }); + const projectionCallback = createNodeWorkerProjectionTransferHttpCallback(projectionService); + const gatewaySockets = new Set(); + const gatewayWebSocket = new WebSocketServer({ noServer: true }); + const projectionCapacityAtFetch: number[] = []; + const gatewayEvents: string[] = []; + let advertisedCapacity = 1; + let gatewayHandshakes = 0; + const expectedTransferPath = `${WORKER_PUBLIC_INGRESS_PATH}${nodeWorkerMemoryProjectionTransferPath()}`; + const gateway = http.createServer((req, res) => { + if (req.url === expectedTransferPath) { + req.url = nodeWorkerMemoryProjectionTransferPath(); + projectionCapacityAtFetch.push(advertisedCapacity); + } + void handleNodeWorkerProjectionTransferHttpRequest({ + req, + res, + clientIp: "127.0.0.1", + callback: projectionCallback, + }).then( + (handled) => { + if (!handled) { + res.writeHead(404); + res.end(); + } + }, + (error: unknown) => res.destroy(error instanceof Error ? error : new Error(String(error))), + ); + }); + gateway.on("upgrade", (request, socket, head) => { + gatewayEvents.push(`upgrade:${request.method ?? "missing"}:${request.url ?? "missing"}`); + if (request.url !== WORKER_PUBLIC_INGRESS_PATH) { + gatewayEvents.push("upgrade:rejected-path"); + socket.destroy(); + return; + } + gatewayWebSocket.handleUpgrade(request, socket, head, (client) => { + gatewayWebSocket.emit("connection", client, request); + }); + }); + gateway.on("clientError", (error) => { + gatewayEvents.push(`client-error:${error.name}:${error.message}`); + }); + gatewayWebSocket.on("connection", (socket, request) => { + gatewayHandshakes += 1; + gatewayEvents.push(`websocket:connected:${request.url ?? "missing"}`); + gatewaySockets.add(socket); + socket.once("error", (error) => { + gatewayEvents.push(`websocket:error:${error.name}:${error.message}`); + }); + socket.once("close", (code, reason) => { + gatewaySockets.delete(socket); + gatewayEvents.push(`websocket:closed:${code}:${reason.toString("utf8")}`); + }); + }); + await new Promise((resolve) => gateway.listen(0, "127.0.0.1", resolve)); + const address = gateway.address(); + if (!address || typeof address === "string") { + throw new Error("test Gateway did not bind a TCP port"); + } + + let engine: NodeWorkerContainerEngine | undefined; + let supervisor: ReturnType | undefined; + let workspace: NodeWorkerWorkspaceRuntime | undefined; + let containerIdentity: { launchId: string; planHash: string } | undefined; + try { + engine = await resolveNodeWorkerContainerEngine(); + if (!engine) { + throw new Error("process-isolation proof requires an eligible Docker or Podman node host"); + } + const input = testWorkerLaunchInput("/workspace", "container-relay-cancellation", "wait"); + input.execution = { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 }; + input.descriptor.assignment.memoryReadEnforced = true; + input.descriptor.assignment.workspaceDir = "/workspace"; + const projection = await projectionService.prepare({ + node, + broker: projectionBroker, + environmentId: input.descriptor.admission.environmentId, + sessionId: input.descriptor.admission.sessionId, + ownerEpoch: input.descriptor.admission.ownerEpoch, + placementGeneration: input.placementGeneration, + runId: input.descriptor.assignment.runId, + launchId: input.launchId, + launchBinding: nodeWorkerMemoryProjectionLaunchBinding(input), + isAuthorized: () => true, + }); + input.memoryProjection = projection; + containerIdentity = { launchId: input.launchId, planHash: nodeWorkerPlanHash(input) }; + const bundleWorker = path.join( + bundleRoot, + input.gatewayNamespace, + "bundles", + input.expectedBundleHash, + "worker.mjs", + ); + fs.writeFileSync( + bundleWorker, + workerSource({ + outsideArtifactPath: params.outsideArtifactPath, + issuedVirtualPath: params.issuedVirtualPath, + forbiddenEnvironmentVariable: params.forbiddenEnvironmentVariable, + }), + { mode: 0o500 }, + ); + workspace = new NodeWorkerWorkspaceRuntime({ root: bundleRoot, env }); + const memoryProjection = new NodeWorkerMemoryProjectionRuntime({ + root: bundleRoot, + deviceIdentity: identity, + }); + const capacitySnapshots: Array<{ total: number; available: number }> = []; + supervisor = createNodeWorkerSupervisor({ + bundleRoot, + env, + capacity: 1, + onCapacityChanged: (capacity) => { + advertisedCapacity = capacity.available; + capacitySnapshots.push({ ...capacity }); + }, + workspace, + memoryProjection, + }); + const crossSessionReplay = structuredClone(input); + crossSessionReplay.descriptor.admission.sessionId = "session-replay"; + await expect( + supervisor.launch(crossSessionReplay, { + kind: "websocket", + url: `ws://127.0.0.1:${address.port}${WORKER_PUBLIC_INGRESS_PATH}`, + }), + ).rejects.toThrow("memory projection does not match its worker launch"); + const relayDir = workspace.resolveContainerRelayDirectory({ + gatewayNamespace: input.gatewayNamespace, + ...containerIdentity, + }); + const workerWorkspace = workspace.resolveContainerWorkspace({ + gatewayNamespace: input.gatewayNamespace, + environmentId: input.descriptor.admission.environmentId, + sessionId: input.descriptor.admission.sessionId, + ownerEpoch: input.descriptor.admission.ownerEpoch, + }); + const launchReceipt = await supervisor.launch(input, { + kind: "websocket", + url: `ws://127.0.0.1:${address.port}${WORKER_PUBLIC_INGRESS_PATH}`, + }); + if (launchReceipt.state !== "running" || !launchReceipt.worker) { + // The supervisor persists a scrubbed terminal diagnostic. Surface it here + // so a remote container failure identifies its owning boundary. + throw new Error( + `container node worker did not start: ${launchReceipt.errorText ?? "missing terminal diagnostic"}`, + ); + } + const containerIdsForLaunch = await containerIds(engine, containerIdentity); + expect(containerIdsForLaunch).toHaveLength(1); + const labels = await execContainer( + engine, + [ + "inspect", + "--format", + '{{index .Config.Labels "openclaw.node-worker-container"}}\n{{index .Config.Labels "openclaw.node-worker-launch"}}\n{{index .Config.Labels "openclaw.node-worker-plan"}}', + containerIdsForLaunch[0]!, + ], + { allowFailure: true }, + ); + expect(labels.code, labels.stderr).toBe(0); + expect(labels.stdout.trimEnd().split(/\r?\n/u)).toEqual([ + "v1", + containerIdentity.launchId, + containerIdentity.planHash, + ]); + expect(fs.existsSync(path.join(params.root, "state-root"))).toBe(true); + expect(fs.existsSync(path.join(relayDir, "gateway.sock"))).toBe(true); + expect(fs.existsSync(relayDir)).toBe(true); + try { + await vi.waitFor( + () => { + expect(fs.existsSync(path.join(workerWorkspace, "container-proof.json"))).toBe(true); + }, + { timeout: 30_000 }, + ); + } catch (error) { + const receipt = await supervisor.status(input.launchId); + const relayEventsPath = path.join(workerWorkspace, "container-relay-events.json"); + const relayEvents = fs.existsSync(relayEventsPath) + ? fs.readFileSync(relayEventsPath, "utf8") + : "missing"; + throw new Error( + `container worker did not produce its relay proof (handshakes=${gatewayHandshakes}, gateway=${gatewayEvents.join(",") || "none"}, relay=${relayEvents}, state=${receipt?.state ?? "missing"}, detail=${receipt?.errorText ?? "none"})`, + { cause: error }, + ); + } + // The worker writes its proof only after the mounted relay completed its + // WebSocket handshake, so check the connection after that async boundary. + expect(gatewayHandshakes).toBe(1); + const observed = JSON.parse( + fs.readFileSync(path.join(workerWorkspace, "container-proof.json"), "utf8"), + ) as { + issued: string; + rawArtifactRead: string; + rawArtifactRootEnumeration: string; + hostArtifactFd: boolean; + hostArtifactMount: boolean; + brokerCredentialPresent: boolean; + memoryRoots: string[]; + unissuedMemoryRead: string; + issuedMemoryWrite: string; + rootFilesystemWrite: string; + relaySocket: string; + relayEvents: string[]; + }; + expect(observed).toMatchObject({ + issued: params.issuedContents, + rawArtifactRootEnumeration: expect.not.stringMatching(/^allowed$/u), + hostArtifactFd: false, + hostArtifactMount: false, + brokerCredentialPresent: false, + memoryRoots: projectionBroker.view.roots.map((root) => root.virtualRoot).toSorted(), + unissuedMemoryRead: expect.not.stringMatching(/^allowed$/u), + issuedMemoryWrite: expect.not.stringMatching(/^allowed$/u), + rootFilesystemWrite: expect.not.stringMatching(/^allowed$/u), + relaySocket: NODE_WORKER_CONTAINER_RELAY_SOCKET, + }); + expect(observed.rawArtifactRead).not.toBe(params.outsideArtifactContents); + expect(projectionCapacityAtFetch).toEqual([0]); + expect(capacitySnapshots).toContainEqual({ total: 1, available: 0 }); + await vi.waitFor( + async () => { + await expect(supervisor!.status(input.launchId)).resolves.toMatchObject({ + state: "cancelled", + errorText: "node worker launch cancelled", + }); + await expect(containerIds(engine!, containerIdentity!)).resolves.toEqual([]); + expect(fs.existsSync(relayDir)).toBe(false); + expect(fs.readdirSync(path.join(bundleRoot, "memory-projections"))).toEqual([]); + }, + { timeout: 45_000 }, + ); + expect(capacitySnapshots.at(-1)).toEqual({ total: 1, available: 1 }); + } finally { + projectionService.closeAll(); + await supervisor?.close().catch(() => undefined); + if (engine && containerIdentity) { + await removeOwnedNodeWorkerContainers(containerIdentity, engine).catch(() => undefined); + } + if (workspace && containerIdentity) { + await workspace + .removeContainerRelayDirectory({ gatewayNamespace: "gateway-1", ...containerIdentity }) + .catch(() => undefined); + } + for (const socket of gatewaySockets) { + socket.terminate(); + } + gatewayWebSocket.close(); + await new Promise((resolve) => gateway.close(() => resolve())); + } +} diff --git a/test/scripts/docker-e2e-plan.test.ts b/test/scripts/docker-e2e-plan.test.ts index 8c0c1a1f5957..f32aa44b8e56 100644 --- a/test/scripts/docker-e2e-plan.test.ts +++ b/test/scripts/docker-e2e-plan.test.ts @@ -231,6 +231,26 @@ describe("scripts/lib/docker-e2e-plan", () => { expect(plan.needs.functionalImage).toBe(true); }); + it("plans the package-backed Fleet separate-cell isolation lane", () => { + const plan = planFor({ + selectedLaneNames: ["fleet-separate-cell"], + }); + + expect(plan.lanes.map(summarizeLane)).toEqual([ + { + command: "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:fleet-separate-cell", + imageKind: "functional", + live: false, + name: "fleet-separate-cell", + resources: ["docker", "service"], + stateScenario: "empty", + timeoutMs: 900_000, + weight: 4, + }, + ]); + expect(plan.needs.functionalImage).toBe(true); + }); + it("routes trusted Docker scripts through the nested release harness", () => { const trustedScripts = new Map([ ["live-codex-npm-plugin", "e2e/codex-npm-plugin-live-docker.sh"], diff --git a/tsdown.config.ts b/tsdown.config.ts index 4e57d4aaf1a8..c6059299af7e 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -395,6 +395,8 @@ function buildCoreDistEntries(): Record { "process/supervisor/service-child-relay": "src/process/supervisor/service-child-relay.ts", "process/supervisor/service-child-group-anchor": "src/process/supervisor/service-child-group-anchor.ts", + // Durable host-side PID for per-session process-isolated worker containers. + "node-host/node-worker-container-shim": "src/node-host/node-worker-container-shim.ts", "telegram-ingress-worker.runtime": bundledPluginFile( "telegram", "src/telegram-ingress-worker.runtime.ts",