feat(memory): isolate selected runtime behind broker

This commit is contained in:
Galin Iliev
2026-08-19 07:01:45 -07:00
parent 04ac2f5365
commit 32ec272e75
111 changed files with 9920 additions and 522 deletions
+2 -3
View File
@@ -3479,11 +3479,10 @@ src/node-host/invoke-payload.ts 5
src/node-host/invoke.ts 7
src/node-host/mcp.ts 3
src/node-host/node-worker-bundle-installer.ts 1
src/node-host/node-worker-supervisor.ts 1
src/node-host/node-worker-transfer-client.ts 3
src/node-host/node-worker-transfer-http.ts 2
src/node-host/node-worker-tree-control.ts 2
src/node-host/node-worker-workspace.ts 7
src/node-host/node-worker-workspace.ts 6
src/node-host/plugin-node-host.ts 1
src/node-host/pty-command.ts 11
src/node-host/runtime.ts 2
@@ -3979,7 +3978,7 @@ src/worker/node-workspace-protocol.ts 3
src/worker/worker-command.runtime.ts 1
src/worker/worker-connection-admission.ts 2
src/worker/worker-connection-frames.ts 4
src/worker/worker-process.ts 2
src/worker/worker-process.ts 1
src/worker/worker-session-tools.ts 4
src/worker/workspace-rsync-receiver.ts 1
ui/src/api/gateway.ts 1
@@ -1,28 +1,47 @@
import type { MemoryBrokerAuthorizationBinding } from "openclaw/plugin-sdk/memory-broker-runtime";
import { describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
authorize: vi.fn(),
export: vi.fn(),
import: vi.fn(),
read: vi.fn(),
recoverPendingWrites: vi.fn(),
search: vi.fn(),
status: vi.fn(),
sync: vi.fn(),
virtualFile: vi.fn(),
virtualView: vi.fn(),
write: vi.fn(),
}));
vi.mock("./scoped-memory-runtime.js", () => ({
builtinScopedMemoryAuthorizedRuntime: {
authorize: mocks.authorize,
exportAuthorized: mocks.export,
importAuthorized: mocks.import,
readAuthorized: mocks.read,
searchAuthorized: mocks.search,
statusAuthorized: mocks.status,
syncAuthorized: mocks.sync,
writeAuthorized: mocks.write,
},
builtinScopedMemoryVirtualView: {},
builtinScopedMemoryVirtualView: {
materializeAuthorizedVirtualView: mocks.virtualView,
readAuthorizedVirtualFile: mocks.virtualFile,
},
recoverBuiltinScopedMemoryPendingWrites: mocks.recoverPendingWrites,
}));
const { createMemoryBrokerHandler } = await import("./broker-entry.js");
const { createMemoryBrokerHandler, initializeMemoryBroker } = await import("./broker-entry.js");
const binding = {
const binding: MemoryBrokerAuthorizationBinding = {
agentId: "agent-a",
sessionId: "session-a",
runId: "run-a",
contextFingerprint: "context-a",
subjectRevision: "subject-a",
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
actorRevision: "actor-a",
// Without a delegation, Gateway uses hostFactsRevision for the capability snapshot.
capabilitySnapshotId: "policy-a",
@@ -36,12 +55,25 @@ const context = {
runId: binding.runId,
contextFingerprint: binding.contextFingerprint,
subjectRevision: binding.subjectRevision,
actor: { evidenceRevision: binding.actorRevision },
actor: {
kind: binding.actor.kind,
actorKind: binding.actor.actorKind,
principalId: binding.actor.principalId,
evidenceRevision: binding.actorRevision,
},
delivery: { deliveryRevision: binding.deliveryRevision },
hostFactsRevision: binding.policyRevision,
};
describe("memory-core broker entry", () => {
it("recovers only the configured agent set before the broker exposes its socket", () => {
mocks.recoverPendingWrites.mockClear();
initializeMemoryBroker({ agentIds: ["main", "work"] });
expect(mocks.recoverPendingWrites).toHaveBeenCalledWith(["main", "work"]);
});
it("rejects a client-provided context that does not exactly match the Gateway binding", async () => {
const handler = createMemoryBrokerHandler();
@@ -58,8 +90,27 @@ describe("memory-core broker entry", () => {
expect(mocks.authorize).not.toHaveBeenCalled();
});
it("rejects an actor principal substitution that keeps the same evidence revision", async () => {
const handler = createMemoryBrokerHandler();
await expect(
handler({
binding,
request: {
method: "memory.authorize",
payload: {
context: { ...context, actor: { ...context.actor, principalId: "bob" } },
},
},
signal: new AbortController().signal,
}),
).rejects.toThrow("memory broker binding is unavailable");
expect(mocks.authorize).not.toHaveBeenCalled();
});
it("dispatches a bound operation only after its plan policy revision matches", async () => {
const handler = createMemoryBrokerHandler();
const controller = new AbortController();
const plan = { memoryPolicyRevision: binding.policyRevision };
mocks.status.mockResolvedValue({ version: 1, value: { backend: "builtin" } });
@@ -70,10 +121,10 @@ describe("memory-core broker entry", () => {
method: "memory.status",
payload: { context, plan },
},
signal: new AbortController().signal,
signal: controller.signal,
}),
).resolves.toEqual({ version: 1, value: { backend: "builtin" } });
expect(mocks.status).toHaveBeenCalledWith({ context, plan });
expect(mocks.status).toHaveBeenCalledWith({ context, plan, signal: controller.signal });
});
it("forwards the broker cancellation signal into content search", async () => {
@@ -101,4 +152,77 @@ describe("memory-core broker entry", () => {
signal: controller.signal,
});
});
it("forwards the broker cancellation signal into every authorized operation", async () => {
const handler = createMemoryBrokerHandler();
const controller = new AbortController();
const plan = { memoryPolicyRevision: binding.policyRevision };
const readContext = { ...context, operation: "read" as const };
const writeContext = { ...context, operation: "append" as const };
mocks.read.mockResolvedValue({ version: 1, value: { text: "ok" } });
mocks.virtualView.mockResolvedValue({ version: 1, files: [] });
mocks.virtualFile.mockResolvedValue({ version: 1, value: { text: "ok" } });
mocks.write.mockResolvedValue({ status: "committed" });
mocks.import.mockResolvedValue({ status: "committed" });
mocks.sync.mockResolvedValue({ version: 1, value: { status: "completed" } });
mocks.export.mockResolvedValue({ version: 1, value: {} });
await Promise.all([
handler({
binding,
request: { method: "memory.read", payload: { context: readContext, plan, handle: {} } },
signal: controller.signal,
}),
handler({
binding,
request: { method: "memory.virtual-view", payload: { context: readContext, plan } },
signal: controller.signal,
}),
handler({
binding,
request: {
method: "memory.virtual-file",
payload: { context: readContext, plan, view: {}, virtualPath: "private/1.md" },
},
signal: controller.signal,
}),
handler({
binding,
request: { method: "memory.write", payload: { context: writeContext, plan, mutation: {} } },
signal: controller.signal,
}),
handler({
binding,
request: {
method: "memory.import",
payload: { context: writeContext, plan, mutation: { kind: "import" } },
},
signal: controller.signal,
}),
handler({
binding,
request: { method: "memory.sync", payload: { context: writeContext, plan } },
signal: controller.signal,
}),
handler({
binding,
request: { method: "memory.export", payload: { context: writeContext, plan, handles: [] } },
signal: controller.signal,
}),
]);
for (const operation of [
mocks.read,
mocks.virtualView,
mocks.virtualFile,
mocks.write,
mocks.import,
mocks.sync,
mocks.export,
]) {
expect(operation).toHaveBeenCalledWith(
expect.objectContaining({ signal: controller.signal }),
);
}
});
});
@@ -9,10 +9,12 @@ import type {
import type {
MemoryBrokerAuthorizationBinding,
MemoryBrokerHandler,
MemoryBrokerStartupContext,
} from "openclaw/plugin-sdk/memory-broker-runtime";
import {
builtinScopedMemoryAuthorizedRuntime,
builtinScopedMemoryVirtualView,
recoverBuiltinScopedMemoryPendingWrites,
} from "./scoped-memory-runtime.js";
type BrokerPayload = Readonly<{
@@ -37,6 +39,27 @@ function asPayload(value: unknown): BrokerPayload {
return value as BrokerPayload;
}
function hasBoundActor(params: {
binding: MemoryBrokerAuthorizationBinding["actor"];
context: MemoryAccessContext;
}): boolean {
const { binding, context } = params;
if (context.actor.kind !== binding.kind) {
return false;
}
if (binding.kind === "principal") {
return (
context.actor.kind === "principal" &&
context.actor.actorKind === binding.actorKind &&
context.actor.principalId === binding.principalId
);
}
return (
context.actor.kind === "unattributed" &&
context.actor.transportAuditRef === binding.transportAuditRef
);
}
function assertBoundContext(params: {
binding: MemoryBrokerAuthorizationBinding;
context: MemoryAccessContext;
@@ -52,6 +75,7 @@ function assertBoundContext(params: {
context.runId !== binding.runId ||
context.contextFingerprint !== binding.contextFingerprint ||
context.subjectRevision !== binding.subjectRevision ||
!hasBoundActor({ binding: binding.actor, context }) ||
context.actor.evidenceRevision !== binding.actorRevision ||
capabilitySnapshotId !== binding.capabilitySnapshotId ||
policyRevision !== binding.policyRevision ||
@@ -75,6 +99,11 @@ function readPlan(payload: BrokerPayload): AuthorizedMemoryPlan {
return payload.plan;
}
/** Complete selected-memory recovery before the broker child exposes its authenticated socket. */
export function initializeMemoryBroker(context: MemoryBrokerStartupContext): void {
recoverBuiltinScopedMemoryPendingWrites(context.agentIds);
}
/**
* This entry is loaded only by the Gateway-owned broker child. It owns the selected runtime's
* process-local plan and handle maps, so Gateway and workers can retain only opaque DTOs.
@@ -115,6 +144,7 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler {
handle: payload.handle,
...(Number.isSafeInteger(payload.from) ? { from: payload.from } : {}),
...(Number.isSafeInteger(payload.lines) ? { lines: payload.lines } : {}),
signal,
});
}
case "memory.virtual-view": {
@@ -125,6 +155,7 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler {
return await builtinScopedMemoryVirtualView.materializeAuthorizedVirtualView({
context,
plan: readPlan(payload) as never,
signal,
});
}
case "memory.virtual-file": {
@@ -141,6 +172,7 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler {
plan: readPlan(payload) as never,
view: payload.view,
virtualPath: payload.virtualPath,
signal,
});
}
case "memory.write":
@@ -151,6 +183,7 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler {
context: payload.context,
plan: readPlan(payload),
mutation: payload.mutation,
signal,
} as never);
case "memory.import":
if (!payload.mutation || payload.mutation.kind !== "import") {
@@ -160,11 +193,13 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler {
context: payload.context,
plan: readPlan(payload),
mutation: payload.mutation,
signal,
} as never);
case "memory.sync":
return await builtinScopedMemoryAuthorizedRuntime.syncAuthorized({
context: payload.context,
plan: readPlan(payload),
signal,
} as never);
case "memory.export":
if (!Array.isArray(payload.handles)) {
@@ -174,11 +209,13 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler {
context: payload.context,
plan: readPlan(payload),
handles: payload.handles,
signal,
} as never);
case "memory.status":
return await builtinScopedMemoryAuthorizedRuntime.statusAuthorized({
context: payload.context,
plan: readPlan(payload),
signal,
} as never);
default:
throw new Error("memory broker operation is unavailable");
@@ -1,12 +1,17 @@
import { spawnSync } from "node:child_process";
import { createHash, randomUUID } from "node:crypto";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/core";
import type {
AuthorizedMemoryPlan,
MemoryAccessContext,
MemoryContentAccessContext,
} from "openclaw/plugin-sdk/memory-authorization";
import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api";
import { closeOpenClawAgentDatabasesForTest } from "openclaw/plugin-sdk/sqlite-runtime-testing";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
@@ -21,7 +26,7 @@ import {
consumeAdmittedChannelMemoryIdentityFromContext,
createChannelMemoryIdentityAdmission,
} from "../../../../src/channels/message-access/memory-identity-admission.js";
import { appendSqliteTranscriptMessage } from "../../../../src/config/sessions/session-accessor.sqlite-transcript-write.js";
import { appendTranscriptMessage } from "../../../../src/config/sessions/session-accessor.sqlite-transcript-write.js";
import { readAuthorizedTranscriptDerivation } from "../../../../src/config/sessions/session-transcript-memory-policy.js";
import { withOwnedSessionTranscriptWrites } from "../../../../src/config/sessions/transcript-write-context.js";
import {
@@ -29,7 +34,12 @@ import {
resetAgentRunRegistryForTest,
} from "../../../../src/infra/agent-run-registry.js";
import { startMemoryBrokerProcess } from "../../../../src/memory-broker/process.js";
import { requestJsonlSocket } from "../../../../src/infra/jsonl-socket.js";
import { admitMemoryAuthorizationReadRuntime } from "../../../../src/plugins/memory-authorization-runtime.js";
import {
closeBrokeredMemoryRuntimes,
withBrokeredMemoryMaintenance,
} from "../../../../src/plugins/memory-broker-runtime.js";
import { resetMemoryIsolationCutoverForTest } from "../../../../src/plugins/memory-cutover.js";
import {
persistMemoryRunExposureBeforeContentInDatabase,
@@ -56,6 +66,8 @@ import {
openOpenClawStateDatabase,
} from "../../../../src/state/openclaw-state-db.js";
import { ensureProfileForEmail } from "../../../../src/state/user-profiles.js";
import { verifyNodeWorkerContainerProjectionIsolation } from "../../../../test/helpers/node-worker-container-projection-isolation.js";
import memoryCorePlugin from "../../index.js";
import { MEMORY_CORE_AUTHORIZATION_CAPABILITIES } from "../authorization.js";
import { createMemoryRuntime } from "../runtime-provider.js";
import { resolveScopedMemoryArtifactBase, withScopedMemoryDatabase } from "./scoped-memory-db.js";
@@ -63,6 +75,7 @@ import { builtinScopedMemoryConformanceAdapter } from "./scoped-memory-policy.js
import {
createBuiltinScopedMemoryResource,
readBuiltinScopedMemoryRevisionSnapshot,
resolveBuiltinScopedMemoryArtifactPath,
setBuiltinScopedMemoryRevisionLifecycle,
} from "./scoped-memory-resources.js";
import {
@@ -86,6 +99,62 @@ vi.mock("../../../../src/auto-reply/reply/dispatch-from-config.js", () => ({
const { dispatchInboundMessage } = await import("../../../../src/auto-reply/dispatch.js");
const dockerAvailable =
spawnSync("docker", ["info"], { stdio: "ignore", timeout: 3_000 }).status === 0;
function constrainedSandboxConfig(params: {
image: string;
prefix: string;
workspaceRoot: string;
}): OpenClawConfig {
return {
agents: {
defaults: {
skipBootstrap: true,
sandbox: {
mode: "all",
backend: "docker",
scope: "session",
workspaceAccess: "none",
workspaceRoot: params.workspaceRoot,
docker: { image: params.image, containerPrefix: params.prefix },
browser: { enabled: false },
prune: { idleHours: 0, maxAgeDays: 0 },
},
},
},
};
}
function resolveSelectedMemoryBrokerChildPid(): number {
const processList = spawnSync("ps", ["-axo", "pid=,ppid=,command="], { encoding: "utf8" });
if (processList.status !== 0) {
throw new Error("fixture failed to inspect the selected memory broker child");
}
const child = processList.stdout
.split("\n")
.map((line) => /^\s*(\d+)\s+(\d+)\s+(.*)$/u.exec(line))
.find(
(match) =>
match?.[2] === String(process.pid) &&
/(?:^|[\\/])memory-broker[\\/]child\.(?:ts|js)(?:\s|$)/u.test(match[3]),
);
if (!child?.[1]) {
throw new Error("fixture failed to locate the selected memory broker child");
}
return Number(child[1]);
}
function resolveNewMemoryBrokerSocketPath(existingDirectories: ReadonlySet<string>): string {
const directory = fs
.readdirSync(os.tmpdir())
.find((name) => name.startsWith("openclaw-memory-broker-") && !existingDirectories.has(name));
if (!directory) {
throw new Error("fixture failed to locate the selected memory broker socket");
}
return path.join(os.tmpdir(), directory, "broker.sock");
}
describe("builtin scoped authorized runtime", () => {
let stateDir = "";
@@ -94,7 +163,8 @@ describe("builtin scoped authorized runtime", () => {
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
});
afterEach(() => {
afterEach(async () => {
await closeBrokeredMemoryRuntimes();
dispatchReplyFromConfig.mockReset();
resetBuiltinScopedMemoryAuthorizedRuntimeForTest();
resetMemoryIsolationCutoverForTest();
@@ -129,11 +199,45 @@ describe("builtin scoped authorized runtime", () => {
authorizationConformance: builtinScopedMemoryConformanceAdapter,
virtualView: builtinScopedMemoryVirtualView,
runtime: { ...createMemoryRuntime(), ...builtinScopedMemoryAuthorizedRuntime },
broker: {
version: 1,
kind: "local-child",
moduleUrl: new URL("./broker-entry.ts", import.meta.url).href,
},
},
});
setActivePluginRegistry(registry);
}
function installDefaultMemoryCoreSelectedRuntime() {
const registry = createEmptyPluginRegistry();
registry.plugins.push({ id: "memory-core", memorySlotSelected: true } as never);
const runtime = {
llm: { acquireLocalService: async () => undefined },
state: {
openKeyedStore: () => ({
lookup: () => undefined,
register: () => undefined,
delete: () => undefined,
list: () => [],
}),
},
} as unknown as OpenClawPluginApi["runtime"];
memoryCorePlugin.register(
createTestPluginApi({
runtime,
registerMemoryCapability(capability) {
registry.memoryCapabilities.push({ pluginId: "memory-core", capability });
},
}),
);
if (registry.memoryCapabilities.length !== 1) {
throw new Error("fixture failed to register the default memory-core capability");
}
setActivePluginRegistry(registry);
return registry.memoryCapabilities[0]!.capability;
}
function createSession(params: {
sessionKey: string;
sessionId: string;
@@ -543,18 +647,61 @@ describe("builtin scoped authorized runtime", () => {
it("keeps the selected runtime plan state in a real broker child across IPC", async () => {
createPrivateResource("alice", "BROKER_CHILD_SERIALIZED_PLAN_ONLY");
const startupRecovered = createWriteRecoveryFixture({
principalId: "alice",
content: "BROKER_STARTUP_RECOVERY_SENTINEL",
placement: "stage",
});
const context = createContext("alice");
const broker = await startMemoryBrokerProcess({
brokerId: "memory-core-test-broker",
handlerModuleUrl: new URL("./broker-entry.ts", import.meta.url).href,
agentIds: ["main"],
});
try {
// The child reached ready only after its plugin startup hook recovered the staged revision.
// This assertion occurs before the first broker request, so authorize cannot hide deferred
// recovery as request-time repair.
withScopedMemoryDatabase("main", (database) => {
expect(
database
.prepare(
`SELECT revision.lifecycle_state, intent.state
FROM memory_resource_revisions AS revision
JOIN memory_write_intents AS intent ON intent.pending_revision_id = revision.revision_id
WHERE revision.revision_id = ?`,
)
.get(startupRecovered.revisionId),
).toEqual({ lifecycle_state: "active", state: "active" });
expect(
database
.prepare("SELECT text FROM memory_scoped_chunks WHERE revision_id = ?")
.all(startupRecovered.revisionId),
).toEqual([{ text: "BROKER_STARTUP_RECOVERY_SENTINEL" }]);
});
expect(fs.existsSync(path.join(startupRecovered.directory, startupRecovered.stageLocator))).toBe(
false,
);
expect(fs.existsSync(path.join(startupRecovered.directory, startupRecovered.finalLocator))).toBe(
true,
);
const authorizationBinding = {
agentId: context.agentId,
sessionId: context.sessionId,
runId: context.runId,
contextFingerprint: context.contextFingerprint,
subjectRevision: context.subjectRevision,
actor:
context.actor.kind === "principal"
? {
kind: "principal" as const,
actorKind: context.actor.actorKind,
principalId: context.actor.principalId,
}
: {
kind: "unattributed" as const,
transportAuditRef: context.actor.transportAuditRef,
},
actorRevision: context.actor.evidenceRevision,
capabilitySnapshotId: context.delegation?.capabilitySnapshotId ?? context.hostFactsRevision,
policyRevision: context.hostFactsRevision,
@@ -591,6 +738,65 @@ describe("builtin scoped authorized runtime", () => {
}),
).resolves.toBeUndefined();
await expect(
broker.client.request({
binding: { ...authorizationBinding, policyRevision: plan.memoryPolicyRevision },
method: "memory.search",
payload: {
context: { ...context, sessionId: "replayed-session" },
plan,
query: "BROKER_CHILD_SERIALIZED_PLAN_ONLY",
limit: 10,
},
expiresAtMs: Date.parse(plan.expiresAt),
}),
).resolves.toBeUndefined();
await expect(
broker.client.request({
binding: { ...authorizationBinding, policyRevision: plan.memoryPolicyRevision },
method: "memory.search",
payload: {
context: { ...context, agentId: "replayed-agent" },
plan,
query: "BROKER_CHILD_SERIALIZED_PLAN_ONLY",
limit: 10,
},
expiresAtMs: Date.parse(plan.expiresAt),
}),
).resolves.toBeUndefined();
await expect(
broker.client.request({
binding: { ...authorizationBinding, policyRevision: plan.memoryPolicyRevision },
method: "memory.search",
payload: {
context: {
...context,
actor: { ...context.actor, principalId: "mallory" },
},
plan,
query: "BROKER_CHILD_SERIALIZED_PLAN_ONLY",
limit: 10,
},
expiresAtMs: Date.parse(plan.expiresAt),
}),
).resolves.toBeUndefined();
await expect(
broker.client.request({
binding: { ...authorizationBinding, policyRevision: plan.memoryPolicyRevision },
method: "memory.search",
payload: {
context: { ...context, hostFactsRevision: "revoked-capability-snapshot" },
plan,
query: "BROKER_CHILD_SERIALIZED_PLAN_ONLY",
limit: 10,
},
expiresAtMs: Date.parse(plan.expiresAt),
}),
).resolves.toBeUndefined();
await expect(
broker.client.request({
binding: { ...authorizationBinding, policyRevision: "stale-policy-revision" },
@@ -1230,7 +1436,7 @@ describe("builtin scoped authorized runtime", () => {
withTranscriptWrite: async (run) => await run(),
},
async () => {
await appendSqliteTranscriptMessage(
await appendTranscriptMessage(
{
agentId: context.agentId,
sessionId: context.sessionId,
@@ -1360,6 +1566,484 @@ describe("builtin scoped authorized runtime", () => {
});
});
it("returns intentional unavailability while maintenance quiesces the real selected broker", async () => {
const session = { sessionKey: "agent:main:direct:alice", sessionId: "alice-session" };
const alicePrincipalId = createVerifiedDirectSession({ name: "alice", ...session });
createPrivateResource(alicePrincipalId, "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL");
markCutOver();
installBuiltinSelectedRuntime();
const host = createAuthorizedMemoryReadHost({
agentId: "main",
...session,
deliveryContext: { channel: "telegram", accountId: "default", to: "alice" },
});
if (!host) {
throw new Error("fixture failed to build an authorized memory host");
}
await expect(
host.search({ query: "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL", limit: 1 }),
).resolves.toMatchObject({
results: [{ snippet: "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL" }],
});
let enterMaintenance: (() => void) | undefined;
let releaseMaintenance: (() => void) | undefined;
const maintenanceEntered = new Promise<void>((resolve) => {
enterMaintenance = resolve;
});
const maintenanceRelease = new Promise<void>((resolve) => {
releaseMaintenance = resolve;
});
const maintenance = withBrokeredMemoryMaintenance(async () => {
enterMaintenance?.();
await maintenanceRelease;
});
await maintenanceEntered;
await expect(
host.search({ query: "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL", limit: 1 }),
).resolves.toEqual({
disabled: true,
unavailable: true,
error: "memory unavailable",
});
releaseMaintenance?.();
await maintenance;
await expect(
host.search({ query: "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL", limit: 1 }),
).resolves.toMatchObject({
results: [{ snippet: "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL" }],
});
});
it.runIf(process.platform !== "win32")(
"restarts the default selected broker only after Gateway reauthorizes and rejects a pre-crash envelope",
async () => {
const session = { sessionKey: "agent:main:direct:alice", sessionId: "alice-session" };
const alicePrincipalId = createVerifiedDirectSession({ name: "alice", ...session });
createPrivateResource(alicePrincipalId, "SELECTED_BROKER_CRASH_RESTART_SENTINEL");
markCutOver();
installDefaultMemoryCoreSelectedRuntime();
const brokerDirectoriesBefore = new Set(
fs.readdirSync(os.tmpdir()).filter((name) => name.startsWith("openclaw-memory-broker-")),
);
const createHost = () => {
const host = createAuthorizedMemoryReadHost({
agentId: "main",
...session,
deliveryContext: { channel: "telegram", accountId: "default", to: "alice" },
});
if (!host) {
throw new Error("fixture failed to build an authorized memory host");
}
return host;
};
const host = createHost();
const writes = vi.spyOn(net.Socket.prototype, "write");
let staleRequestLine: string | undefined;
try {
await expect(
host.search({ query: "SELECTED_BROKER_CRASH_RESTART_SENTINEL", limit: 1 }),
).resolves.toMatchObject({
results: [{ snippet: "SELECTED_BROKER_CRASH_RESTART_SENTINEL" }],
});
staleRequestLine = writes.mock.calls
.map(([chunk]) => (typeof chunk === "string" ? chunk.trim() : undefined))
.find((line) => {
if (!line) {
return false;
}
try {
const frame = JSON.parse(line) as {
envelope?: unknown;
request?: { method?: unknown };
};
return frame.envelope !== undefined && frame.request?.method === "memory.authorize";
} catch {
return false;
}
});
} finally {
writes.mockRestore();
}
if (!staleRequestLine) {
throw new Error("fixture failed to capture a signed selected-broker authorization frame");
}
const firstSocketPath = resolveNewMemoryBrokerSocketPath(brokerDirectoriesBefore);
expect(fs.existsSync(firstSocketPath)).toBe(true);
process.kill(resolveSelectedMemoryBrokerChildPid(), "SIGKILL");
await vi.waitFor(() => expect(fs.existsSync(firstSocketPath)).toBe(false), {
timeout: 2_000,
interval: 20,
});
await vi.waitFor(
async () => {
await expect(
createHost().search({ query: "SELECTED_BROKER_CRASH_RESTART_SENTINEL", limit: 1 }),
).resolves.toMatchObject({
results: [{ snippet: "SELECTED_BROKER_CRASH_RESTART_SENTINEL" }],
});
},
{ timeout: 2_000, interval: 20 },
);
const replacementSocketPath = resolveNewMemoryBrokerSocketPath(
new Set([...brokerDirectoriesBefore, path.basename(path.dirname(firstSocketPath))]),
);
expect(replacementSocketPath).not.toBe(firstSocketPath);
await expect(
requestJsonlSocket({
socketPath: replacementSocketPath,
requestLine: staleRequestLine,
timeoutMs: 1_000,
keepWriteOpen: true,
accept: (response) =>
response && typeof response === "object" && !Array.isArray(response)
? (response as { ok: boolean; error?: string })
: undefined,
}),
).resolves.toEqual({ ok: false, error: "unauthorized" });
},
);
it.runIf(process.platform !== "win32")(
"rotates the selected broker epoch across controlled Gateway shutdown before reauthorization",
async () => {
const session = { sessionKey: "agent:main:direct:alice", sessionId: "alice-session" };
const alicePrincipalId = createVerifiedDirectSession({ name: "alice", ...session });
createPrivateResource(alicePrincipalId, "SELECTED_BROKER_CONTROLLED_RESTART_SENTINEL");
markCutOver();
installDefaultMemoryCoreSelectedRuntime();
const brokerDirectoriesBefore = new Set(
fs.readdirSync(os.tmpdir()).filter((name) => name.startsWith("openclaw-memory-broker-")),
);
const createHost = () => {
const host = createAuthorizedMemoryReadHost({
agentId: "main",
...session,
deliveryContext: { channel: "telegram", accountId: "default", to: "alice" },
});
if (!host) {
throw new Error("fixture failed to build an authorized memory host");
}
return host;
};
const writes = vi.spyOn(net.Socket.prototype, "write");
let preShutdownRequestLine: string | undefined;
try {
await expect(
createHost().search({ query: "SELECTED_BROKER_CONTROLLED_RESTART_SENTINEL", limit: 1 }),
).resolves.toMatchObject({
results: [{ snippet: "SELECTED_BROKER_CONTROLLED_RESTART_SENTINEL" }],
});
preShutdownRequestLine = writes.mock.calls
.map(([chunk]) => (typeof chunk === "string" ? chunk.trim() : undefined))
.find((line) => {
if (!line) {
return false;
}
try {
const frame = JSON.parse(line) as {
envelope?: unknown;
request?: { method?: unknown };
};
return frame.envelope !== undefined && frame.request?.method === "memory.authorize";
} catch {
return false;
}
});
} finally {
writes.mockRestore();
}
if (!preShutdownRequestLine) {
throw new Error("fixture failed to capture a signed pre-shutdown authorization frame");
}
const firstSocketPath = resolveNewMemoryBrokerSocketPath(brokerDirectoriesBefore);
await closeBrokeredMemoryRuntimes();
await vi.waitFor(() => expect(fs.existsSync(firstSocketPath)).toBe(false), {
timeout: 2_000,
interval: 20,
});
await expect(
createHost().search({ query: "SELECTED_BROKER_CONTROLLED_RESTART_SENTINEL", limit: 1 }),
).resolves.toMatchObject({
results: [{ snippet: "SELECTED_BROKER_CONTROLLED_RESTART_SENTINEL" }],
});
const replacementSocketPath = resolveNewMemoryBrokerSocketPath(
new Set([...brokerDirectoriesBefore, path.basename(path.dirname(firstSocketPath))]),
);
await expect(
requestJsonlSocket({
socketPath: replacementSocketPath,
requestLine: preShutdownRequestLine,
timeoutMs: 1_000,
keepWriteOpen: true,
accept: (response) =>
response && typeof response === "object" && !Array.isArray(response)
? (response as { ok: boolean; error?: string })
: undefined,
}),
).resolves.toEqual({ ok: false, error: "unauthorized" });
},
);
it.runIf(dockerAvailable && process.env.OPENCLAW_PROCESS_ISOLATION_E2E === "1")(
"keeps real memory-core artifacts behind its broker when a malicious model-facing tool runs in a constrained process",
async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-memory-core-container-e2e-"));
const workspaceDir = path.join(root, "workspace");
const image = process.env.OPENCLAW_SANDBOX_TEST_IMAGE ?? "openclaw-sandbox:bookworm-slim";
const session = { sessionKey: "agent:main:direct:alice", sessionId: "alice-session" };
const bobSession = { sessionKey: "agent:main:direct:bob", sessionId: "bob-session" };
const disposeProjections: Array<() => Promise<void>> = [];
const runtimeIds: string[] = [];
try {
fs.mkdirSync(workspaceDir, { recursive: true });
vi.stubEnv("OPENCLAW_MEMORY_BROKER_TEST_SECRET", "gateway-only-test-secret");
const alicePrincipalId = createVerifiedDirectSession({ name: "alice", ...session });
const aliceResource = createPrivateResource(
alicePrincipalId,
"ALICE_REAL_BROKER_ARTIFACT_SENTINEL",
);
const bobPrincipalId = createVerifiedDirectSession({
name: "bob",
...bobSession,
});
const bobResource = createPrivateResource(
bobPrincipalId,
"BOB_REAL_BROKER_ARTIFACT_SENTINEL",
);
markCutOver();
const capability = installDefaultMemoryCoreSelectedRuntime();
expect(capability.broker).toMatchObject({ version: 1, kind: "local-child" });
const brokerDirectoriesBefore = new Set(
fs.readdirSync(os.tmpdir()).filter((name) => name.startsWith("openclaw-memory-broker-")),
);
const host = createAuthorizedMemoryReadHost({
agentId: "main",
...session,
deliveryContext: { channel: "telegram", accountId: "default", to: "alice" },
});
if (!host) {
throw new Error("fixture failed to construct the selected memory-core host");
}
const indexed = await host.search({ query: "REAL_BROKER_ARTIFACT_SENTINEL", limit: 10 });
expect(indexed).toMatchObject({
results: [{ path: "memory/MEMORY.md", snippet: "ALICE_REAL_BROKER_ARTIFACT_SENTINEL" }],
});
expect(JSON.stringify(indexed)).not.toContain("BOB_REAL_BROKER_ARTIFACT_SENTINEL");
const broker = await resolveAuthorizedMemoryVirtualFileBroker(host);
const virtualFile = broker?.view.files[0];
if (!broker || !virtualFile) {
throw new Error("fixture failed to materialize a broker-backed virtual view");
}
await expect(broker.readFile(virtualFile.virtualPath)).resolves.toBe(
"ALICE_REAL_BROKER_ARTIFACT_SENTINEL",
);
const brokerDirectory = fs
.readdirSync(os.tmpdir())
.find(
(name) =>
name.startsWith("openclaw-memory-broker-") && !brokerDirectoriesBefore.has(name),
);
if (!brokerDirectory) {
throw new Error("fixture failed to locate the selected memory-core broker socket");
}
const brokerSocketPath = path.join(os.tmpdir(), brokerDirectory, "broker.sock");
expect(fs.existsSync(brokerSocketPath)).toBe(true);
const artifactPathFor = (revision: { revisionId: string; artifactLocator: string }) => {
let artifactPath: string | undefined;
withScopedMemoryDatabase("main", (database, databasePath) => {
const row = database
.prepare(
`SELECT root.path_key
FROM memory_resource_revisions AS revision
JOIN memory_resources AS resource ON resource.resource_id = revision.resource_id
JOIN memory_stores AS store ON store.store_id = resource.store_id
JOIN memory_storage_roots AS root ON root.storage_root_id = store.storage_root_id
WHERE revision.revision_id = ?`,
)
.get(revision.revisionId) as { path_key?: string } | undefined;
if (row?.path_key) {
artifactPath = resolveBuiltinScopedMemoryArtifactPath({
databasePath,
pathKey: row.path_key,
artifactLocator: revision.artifactLocator,
});
}
});
if (!artifactPath) {
throw new Error("fixture failed to locate a scoped memory artifact");
}
return artifactPath;
};
const aliceArtifactPath = artifactPathFor(aliceResource);
const bobArtifactPath = artifactPathFor(bobResource);
const agentDatabasePath = path.join(
stateDir,
"agents",
"main",
"agent",
"openclaw-agent.sqlite",
);
expect(fs.existsSync(aliceArtifactPath)).toBe(true);
expect(fs.existsSync(bobArtifactPath)).toBe(true);
expect(fs.existsSync(agentDatabasePath)).toBe(true);
const [{ resolveSandboxContext }, { stageAuthorizedVirtualProjectionMountPlan }] =
await Promise.all([
import("../../../../src/agents/sandbox/context.js"),
import("../../../../src/agents/sandbox/authorized-virtual-projection-staging.js"),
]);
const sandbox = await resolveSandboxContext({
agentId: "main",
config: constrainedSandboxConfig({
image,
prefix: `oc-qa-memory-core-${process.pid}-`,
workspaceRoot: path.join(root, "sandboxes"),
}),
sessionKey: session.sessionKey,
workspaceDir,
prepareAuthorizedVirtualProjectionMountPlan: async ({ agentWorkspaceDir }) => {
const staged = await stageAuthorizedVirtualProjectionMountPlan({
agentWorkspaceDir,
broker,
});
return staged;
},
});
if (!sandbox?.backend) {
throw new Error("fixture failed to start the constrained model-tool process");
}
runtimeIds.push(sandbox.runtimeId);
if (sandbox.disposeAuthorizedVirtualProjectionMountPlan) {
disposeProjections.push(sandbox.disposeAuthorizedVirtualProjectionMountPlan);
}
const result = await sandbox.backend.runShellCommand({
script: [
'test "$(cat "$1")" = "$2"',
'test "$(find /memory -type f -print | sort)" = "$1"',
'test ! -e "$3"',
'test ! -e "$4"',
'test ! -e "$5"',
'test ! -e "$6"',
'test ! -e "$7"',
'test -z "${OPENCLAW_MEMORY_BROKER_TEST_SECRET:-}"',
'! find /proc -path "*/fd/*" -lname "$3" -print -quit 2>/dev/null | grep -q .',
'! find /proc -path "*/fd/*" -lname "$5" -print -quit 2>/dev/null | grep -q .',
'! printf denied > "$1"',
].join(" && "),
args: [
`/memory/${virtualFile.virtualPath}`,
"ALICE_REAL_BROKER_ARTIFACT_SENTINEL",
aliceArtifactPath,
bobArtifactPath,
agentDatabasePath,
stateDir,
brokerSocketPath,
],
});
expect(result.code, result.stderr.toString()).toBe(0);
await verifyNodeWorkerContainerProjectionIsolation({
root: path.join(root, "node-worker-projection"),
broker,
outsideArtifactPath: aliceArtifactPath,
outsideArtifactContents: "ALICE_REAL_BROKER_ARTIFACT_SENTINEL",
issuedVirtualPath: virtualFile.virtualPath,
issuedContents: "ALICE_REAL_BROKER_ARTIFACT_SENTINEL",
forbiddenEnvironmentVariable: "OPENCLAW_MEMORY_BROKER_TEST_SECRET",
});
const bobHost = createAuthorizedMemoryReadHost({
agentId: "main",
...bobSession,
deliveryContext: { channel: "telegram", accountId: "default", to: "bob" },
});
const bobBroker = await resolveAuthorizedMemoryVirtualFileBroker(bobHost);
const bobVirtualFile = bobBroker?.view.files[0];
if (!bobHost || !bobBroker || !bobVirtualFile) {
throw new Error("fixture failed to materialize Bob's selected memory-core virtual view");
}
await expect(
bobHost.search({ query: "REAL_BROKER_ARTIFACT_SENTINEL", limit: 10 }),
).resolves.toMatchObject({
results: [{ path: "memory/MEMORY.md", snippet: "BOB_REAL_BROKER_ARTIFACT_SENTINEL" }],
});
await expect(bobBroker.readFile(bobVirtualFile.virtualPath)).resolves.toBe(
"BOB_REAL_BROKER_ARTIFACT_SENTINEL",
);
const bobSandbox = await resolveSandboxContext({
agentId: "main",
config: constrainedSandboxConfig({
image,
prefix: `oc-qa-memory-core-${process.pid}-`,
workspaceRoot: path.join(root, "sandboxes"),
}),
sessionKey: bobSession.sessionKey,
workspaceDir,
prepareAuthorizedVirtualProjectionMountPlan: async ({ agentWorkspaceDir }) => {
return await stageAuthorizedVirtualProjectionMountPlan({
agentWorkspaceDir,
broker: bobBroker,
});
},
});
if (!bobSandbox?.backend) {
throw new Error("fixture failed to start Bob's constrained model-tool process");
}
runtimeIds.push(bobSandbox.runtimeId);
if (bobSandbox.disposeAuthorizedVirtualProjectionMountPlan) {
disposeProjections.push(bobSandbox.disposeAuthorizedVirtualProjectionMountPlan);
}
expect(bobSandbox.runtimeId).not.toBe(sandbox.runtimeId);
const bobResult = await bobSandbox.backend.runShellCommand({
script: [
'test "$(cat "$1")" = "$2"',
'test "$(find /memory -type f -print | sort)" = "$1"',
'! grep -R -F "$3" /memory',
'! printf denied > "$1"',
].join(" && "),
args: [
`/memory/${bobVirtualFile.virtualPath}`,
"BOB_REAL_BROKER_ARTIFACT_SENTINEL",
"ALICE_REAL_BROKER_ARTIFACT_SENTINEL",
],
});
expect(bobResult.code, bobResult.stderr.toString()).toBe(0);
} finally {
if (runtimeIds.length > 0) {
const [{ execDocker }, { removeSandboxContainer }] = await Promise.all([
import("../../../../src/agents/sandbox/docker.js"),
import("../../../../src/agents/sandbox/manage.js"),
]);
for (const runtimeId of runtimeIds) {
await removeSandboxContainer(runtimeId);
await execDocker(["rm", "-f", runtimeId], { allowFailure: true });
}
}
await Promise.all(disposeProjections.map((disposeProjection) => disposeProjection()));
fs.rmSync(root, { recursive: true, force: true });
}
},
120_000,
);
it("invalidates materialized virtual views after binding revocation and plan expiry", async () => {
const session = { sessionKey: "agent:main:direct:alice", sessionId: "alice-session" };
const alicePrincipalId = createVerifiedDirectSession({ name: "alice", ...session });
@@ -2017,6 +2701,131 @@ describe("builtin scoped authorized runtime", () => {
});
});
it("quarantines cancelled writes before activation so recovery cannot publish them", async () => {
const principalId = "cancelled-writer";
const store = createBuiltinScopedMemoryStore({
agentId: "main",
scopeKind: "user",
audienceKind: "user",
audienceId: principalId,
authorityKind: "user",
authorityOwnerId: principalId,
defaultCapabilities: ["append"],
actor: { kind: "human", id: principalId },
reason: "cancellation fixture",
});
const context = {
...createContext(principalId),
operation: "append" as const,
} satisfies MemoryAccessContext;
const plan = await builtinScopedMemoryAuthorizedRuntime.authorize(context);
const preAborted = new AbortController();
preAborted.abort();
await expect(
builtinScopedMemoryAuthorizedRuntime.writeAuthorized({
context,
plan,
mutation: {
version: 1,
kind: "remember",
mutationId: "cancelled-before-stage",
idempotencyKey: "cancelled-before-stage-request",
content: "CANCELLED_BEFORE_STAGE_SENTINEL",
contentType: "markdown",
},
signal: preAborted.signal,
}),
).rejects.toThrow();
withScopedMemoryDatabase("main", (database) => {
expect(
database
.prepare("SELECT count(*) AS count FROM memory_resources WHERE store_id = ?")
.get(store.storeId),
).toEqual({ count: 0 });
expect(
database
.prepare("SELECT count(*) AS count FROM memory_write_intents WHERE store_id = ?")
.get(store.storeId),
).toEqual({ count: 0 });
});
// This test-only signal becomes aborted at the exact synchronous fence after final rename.
// It proves a late pre-activation disconnect leaves durable intent/artifact state quarantined.
let abortedReads = 0;
const abortAfterFinalRename = {
get aborted() {
abortedReads += 1;
return abortedReads > 2;
},
throwIfAborted() {
if (abortedReads > 2) {
throw new Error("authorized write cancelled after final rename");
}
},
} as AbortSignal;
await expect(
builtinScopedMemoryAuthorizedRuntime.writeAuthorized({
context,
plan,
mutation: {
version: 1,
kind: "remember",
mutationId: "cancelled-after-rename",
idempotencyKey: "cancelled-after-rename-request",
content: "CANCELLED_AFTER_RENAME_SENTINEL",
contentType: "markdown",
},
signal: abortAfterFinalRename,
}),
).rejects.toThrow("cancelled after final rename");
let revisionId = "";
let directory = "";
withScopedMemoryDatabase("main", (database, databasePath) => {
const intent = database
.prepare(
`SELECT intent.pending_revision_id, revision.lifecycle_state, intent.state, root.path_key
FROM memory_write_intents AS intent
JOIN memory_resource_revisions AS revision ON revision.revision_id = intent.pending_revision_id
JOIN memory_stores AS store ON store.store_id = intent.store_id
JOIN memory_storage_roots AS root ON root.storage_root_id = store.storage_root_id
WHERE intent.idempotency_key = ?`,
)
.get("cancelled-after-rename-request") as {
pending_revision_id: string;
lifecycle_state: string;
path_key: string;
state: string;
};
revisionId = intent.pending_revision_id;
directory = path.join(resolveScopedMemoryArtifactBase(databasePath), intent.path_key);
expect(intent).toMatchObject({ lifecycle_state: "quarantined", state: "quarantined" });
expect(
database
.prepare("SELECT * FROM memory_scoped_chunks WHERE revision_id = ?")
.all(revisionId),
).toEqual([]);
});
expect(fs.readdirSync(path.join(path.dirname(directory), ".quarantine"))).not.toEqual([]);
// A later authorization runs recovery. It must retain the tombstone rather than promote it.
await builtinScopedMemoryAuthorizedRuntime.authorize(context);
withScopedMemoryDatabase("main", (database) => {
expect(
database
.prepare(
`SELECT revision.lifecycle_state, intent.state
FROM memory_resource_revisions AS revision
JOIN memory_write_intents AS intent ON intent.pending_revision_id = revision.revision_id
WHERE revision.revision_id = ?`,
)
.get(revisionId),
).toEqual({ lifecycle_state: "quarantined", state: "quarantined" });
});
});
it("recovers each interrupted write boundary without exposing a pending revision", async () => {
const stageOnly = createWriteRecoveryFixture({
principalId: "recovery-stage-only",
@@ -466,7 +466,9 @@ function readPlan(params: {
function materializeAuthorizedVirtualView(params: {
context: MemoryContentAccessContext<"read">;
plan: AuthorizedMemoryPlan & Readonly<{ operation: "read" }>;
signal?: AbortSignal;
}): AuthorizedMemoryVirtualView | undefined {
throwIfAuthorizedMemoryOperationAborted(params.signal);
pruneExpiredVirtualViews();
const state = readPlan(params);
if (!state || state.stores.length !== state.plan.mounts.length) {
@@ -483,6 +485,7 @@ function materializeAuthorizedVirtualView(params: {
);
const revisionByVirtualPath = new Map<string, string>();
const files = state.stores.flatMap((store, index) => {
throwIfAuthorizedMemoryOperationAborted(params.signal);
const root = roots[index]!;
const rows = withScopedMemoryDatabase(
params.context.agentId,
@@ -504,6 +507,7 @@ function materializeAuthorizedVirtualView(params: {
}>,
);
return rows.flatMap((row, ordinal) => {
throwIfAuthorizedMemoryOperationAborted(params.signal);
const virtualPath = `${root.virtualRoot}/${ordinal + 1}.md`;
revisionByVirtualPath.set(virtualPath, row.revision_id);
return [
@@ -542,7 +546,9 @@ function readAuthorizedVirtualFile(params: {
plan: AuthorizedMemoryPlan & Readonly<{ operation: "read" }>;
view: AuthorizedMemoryVirtualView;
virtualPath: string;
signal?: AbortSignal;
}): AuthorizedMemoryResultEnvelope<MemoryReadResult> {
throwIfAuthorizedMemoryOperationAborted(params.signal);
pruneExpiredVirtualViews();
const state = readPlan(params);
const allocation = virtualViews.get(params.view.viewId);
@@ -571,6 +577,7 @@ function readAuthorizedVirtualFile(params: {
if (!snapshot) {
throw new Error("authorized memory virtual view is unavailable");
}
throwIfAuthorizedMemoryOperationAborted(params.signal);
return createEnvelope({
state,
context: params.context,
@@ -779,6 +786,14 @@ function syncDirectory(directory: string): void {
}
}
/**
* Broker cancellation is an authorization fence: check it before any recovery,
* filesystem staging, or durable write so cancelled work cannot be recovered as a new revision.
*/
function throwIfAuthorizedMemoryOperationAborted(signal?: AbortSignal): void {
signal?.throwIfAborted();
}
function readVerifiedFile(params: {
pathname: string;
contentHash: string;
@@ -1360,6 +1375,34 @@ function quarantineWriteIntent(params: {
});
}
/**
* A cancelled write may already have durable pending rows and staged/final bytes. Quarantine both
* before surfacing the abort so recovery never promotes the cancelled revision on a later request.
*/
function throwIfPendingWriteAborted(params: {
signal?: AbortSignal;
database: Parameters<typeof getNodeSqliteKysely<ScopedMemoryDatabase>>[0];
directory: string;
intentId: string;
revisionId: string;
stagedPath: string;
finalPath: string;
}): void {
if (!params.signal?.aborted) {
return;
}
quarantineWriteIntent({
database: params.database,
intentId: params.intentId,
revisionId: params.revisionId,
nowMs: Date.now(),
reasonCode: "authorized-write-cancelled-before-activation",
});
quarantineArtifact({ directory: params.directory, pathname: params.stagedPath });
quarantineArtifact({ directory: params.directory, pathname: params.finalPath });
params.signal.throwIfAborted();
}
function indexRecoveredRevision(params: {
database: Parameters<typeof getNodeSqliteKysely<ScopedMemoryDatabase>>[0];
intentId: string;
@@ -1709,16 +1752,32 @@ function recoverPendingWrites(agentId: string): void {
});
}
/**
* The selected broker calls this before accepting its first socket request. Recovery is explicit
* at the broker lifecycle boundary so a replaced child cannot defer pending-write repair until a
* later caller happens to authorize or mutate that agent's memory.
*/
export function recoverBuiltinScopedMemoryPendingWrites(agentIds: readonly string[]): void {
for (const agentId of [...new Set(agentIds)].toSorted()) {
recoverPendingWrites(agentId);
}
}
async function writeAuthorizedMutation(params: {
context: MemoryAccessContext;
plan: AuthorizedMemoryPlan;
mutation: AuthorizedMemoryMutation;
signal?: AbortSignal;
}): Promise<MemoryWriteResult> {
throwIfAuthorizedMemoryOperationAborted(params.signal);
assertMutationShape(params.mutation);
if (params.context.operation !== mutationOperation(params.mutation)) {
throw new Error("authorized memory mutation is unavailable");
}
// Recovery can activate a durable pending revision, so it must never run after cancellation.
throwIfAuthorizedMemoryOperationAborted(params.signal);
recoverPendingWrites(params.context.agentId);
throwIfAuthorizedMemoryOperationAborted(params.signal);
const state = readPlan(params);
if (!state) {
throw new Error("authorized memory mutation is unavailable");
@@ -1726,6 +1785,7 @@ async function writeAuthorizedMutation(params: {
const nowMs = Date.now();
const agentId = params.context.agentId;
if (params.mutation.kind === "sync") {
throwIfAuthorizedMemoryOperationAborted(params.signal);
drainMemoryAuditOutbox(agentId);
return Object.freeze({
version: 1,
@@ -1737,6 +1797,7 @@ async function writeAuthorizedMutation(params: {
}
if (params.mutation.kind === "project") {
throwIfAuthorizedMemoryOperationAborted(params.signal);
if (params.context.actor.kind !== "principal" || params.mutation.sourceHandles.length !== 1) {
throw new Error("authorized memory projection is unavailable");
}
@@ -1772,6 +1833,8 @@ async function writeAuthorizedMutation(params: {
expiry,
nowMs,
});
// Projection creation is its activation point. Do not turn a committed projection into a
// cancellation acknowledgement if the caller disconnects after this synchronous commit.
const targetSnapshot = withScopedMemoryDatabase(agentId, (database) => {
const db = getNodeSqliteKysely<ScopedMemoryDatabase>(database);
return executeSqliteQueryTakeFirstSync(
@@ -1801,6 +1864,7 @@ async function writeAuthorizedMutation(params: {
}
const result = withScopedMemoryDatabase(agentId, (database, databasePath) => {
throwIfAuthorizedMemoryOperationAborted(params.signal);
const store = selectWriteStore({ database, agentId, state });
const db = getNodeSqliteKysely<ScopedMemoryDatabase>(database);
const derivationSources =
@@ -1853,6 +1917,9 @@ async function writeAuthorizedMutation(params: {
if (existingIntent.mutation_id !== params.mutation.mutationId) {
throw new Error("authorized memory mutation idempotency conflict");
}
if (existingIntent.state === "quarantined") {
throw new Error("authorized memory mutation is unavailable");
}
return Object.freeze({
version: 1,
mutationId: params.mutation.mutationId,
@@ -1869,11 +1936,13 @@ async function writeAuthorizedMutation(params: {
(params.mutation.kind === "delete" || params.mutation.kind === "tombstone") &&
"target" in params.mutation
) {
throwIfAuthorizedMemoryOperationAborted(params.signal);
const mutation = params.mutation;
if (!existing) {
throw new Error("authorized memory mutation is unavailable");
}
const intentId = randomUUID();
throwIfAuthorizedMemoryOperationAborted(params.signal);
runSqliteImmediateTransactionSync(database, () => {
const current = resolveWriteTarget({
database,
@@ -2025,6 +2094,7 @@ async function writeAuthorizedMutation(params: {
const finalLocator = `r1_${revisionId}.md`;
const stageLocator = `mwst1_${intentId}.tmp`;
const directory = path.join(resolveScopedMemoryArtifactBase(databasePath), store.pathKey);
throwIfAuthorizedMemoryOperationAborted(params.signal);
fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
const stagePath = path.join(directory, stageLocator);
const finalPath = resolveBuiltinScopedMemoryArtifactPath({
@@ -2043,8 +2113,13 @@ async function writeAuthorizedMutation(params: {
fs.closeSync(descriptor);
}
syncDirectory(directory);
if (params.signal?.aborted) {
quarantineArtifact({ directory, pathname: stagePath });
params.signal.throwIfAborted();
}
const resourceId = existing?.resourceId ?? randomUUID();
try {
throwIfAuthorizedMemoryOperationAborted(params.signal);
runSqliteImmediateTransactionSync(database, () => {
const currentStore = selectWriteStore({ database, agentId, state });
if (
@@ -2240,8 +2315,26 @@ async function writeAuthorizedMutation(params: {
} catch {}
throw error;
}
throwIfPendingWriteAborted({
signal: params.signal,
database,
directory,
intentId,
revisionId,
stagedPath: stagePath,
finalPath,
});
fs.renameSync(stagePath, finalPath);
syncDirectory(directory);
throwIfPendingWriteAborted({
signal: params.signal,
database,
directory,
intentId,
revisionId,
stagedPath: stagePath,
finalPath,
});
const verified = readVerifiedFile({
pathname: finalPath,
contentHash: hash,
@@ -2250,6 +2343,17 @@ async function writeAuthorizedMutation(params: {
if (verified === undefined) {
throw new Error("authorized memory finalized artifact is unavailable");
}
// This is the write linearization point. A cancellation observed before it quarantines the
// pending revision below; a cancellation after it is a committed write, never a fake abort.
throwIfPendingWriteAborted({
signal: params.signal,
database,
directory,
intentId,
revisionId,
stagedPath: stagePath,
finalPath,
});
runSqliteImmediateTransactionSync(database, () => {
const currentStore = selectWriteStore({ database, agentId, state });
if (
@@ -2382,6 +2486,7 @@ async function writeAuthorizedMutation(params: {
async function stageSealedCompaction(
params: AuthorizedSealedCompactionStageParams,
): Promise<AuthorizedSealedCompactionArtifact> {
throwIfAuthorizedMemoryOperationAborted(params.signal);
if (!params.content.trim()) {
throw new Error("sealed compaction content is unavailable");
}
@@ -2391,6 +2496,7 @@ async function stageSealedCompaction(
}
const agentId = params.context.agentId;
return withScopedMemoryDatabase(agentId, (database, databasePath) => {
throwIfAuthorizedMemoryOperationAborted(params.signal);
const store = selectWriteStore({ database, agentId, state });
const source = resolveTranscriptDerivationSource({
database,
@@ -2412,6 +2518,7 @@ async function stageSealedCompaction(
const stagePath = path.join(directory, stageLocator);
const hash = contentHash(params.content);
const bytes = Buffer.byteLength(params.content);
throwIfAuthorizedMemoryOperationAborted(params.signal);
fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
const descriptor = fs.openSync(stagePath, "wx", 0o600);
try {
@@ -2422,8 +2529,17 @@ async function stageSealedCompaction(
fs.closeSync(descriptor);
}
syncDirectory(directory);
if (params.signal?.aborted) {
quarantineArtifact({ directory, pathname: stagePath });
params.signal.throwIfAborted();
}
throwIfAuthorizedMemoryOperationAborted(params.signal);
fs.renameSync(stagePath, finalPath);
syncDirectory(directory);
if (params.signal?.aborted) {
quarantineArtifact({ directory, pathname: finalPath });
params.signal.throwIfAborted();
}
const verified = readVerifiedFile({
pathname: finalPath,
contentHash: hash,
@@ -2435,6 +2551,9 @@ async function stageSealedCompaction(
return Object.freeze({
resourceRevisionId: revisionId,
commitInTransaction({ database: transactionDatabase, compactionPolicyId, eventSeq }) {
// This callback runs inside the caller's SQLite transaction. It may only read the abort
// state; filesystem cleanup belongs to staging before a transaction can begin.
throwIfAuthorizedMemoryOperationAborted(params.signal);
if (state.expiresAtMs <= Date.now()) {
throw new Error("sealed compaction authorization is unavailable");
}
@@ -2644,6 +2763,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
async searchAuthorized(
params: AuthorizedMemorySearchParams<"read"> | AuthorizedMemorySearchParams<"derive">,
): Promise<AuthorizedMemoryResultEnvelope<readonly AuthorizedMemorySearchResult[]>> {
throwIfAuthorizedMemoryOperationAborted(params.signal);
if (params.context.operation !== "read" && params.context.operation !== "derive") {
throw new Error("authorized memory search is unavailable");
}
@@ -2664,9 +2784,11 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
offset: 0,
}),
);
throwIfAuthorizedMemoryOperationAborted(params.signal);
const results: AuthorizedMemorySearchResult[] = [];
const sourcePolicySetIds: string[] = [];
for (const candidate of candidates) {
throwIfAuthorizedMemoryOperationAborted(params.signal);
if (results.length >= limit) {
break;
}
@@ -2690,6 +2812,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
results.push(result);
sourcePolicySetIds.push(`mps1_${snapshot.policyRevisionId}`);
}
throwIfAuthorizedMemoryOperationAborted(params.signal);
return createEnvelope({
state,
context: params.context,
@@ -2703,6 +2826,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
async readAuthorized(
params: AuthorizedMemoryReadParams<"read"> | AuthorizedMemoryReadParams<"derive">,
): Promise<AuthorizedMemoryResultEnvelope<MemoryReadResult>> {
throwIfAuthorizedMemoryOperationAborted(params.signal);
if (params.context.operation !== "read" && params.context.operation !== "derive") {
throw new Error("authorized memory read is unavailable");
}
@@ -2727,6 +2851,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
if (!snapshot || snapshot.policyRevisionId !== storedHandle.policyRevision) {
throw new Error("authorized memory read is unavailable");
}
throwIfAuthorizedMemoryOperationAborted(params.signal);
const lines = snapshot.content.split("\n");
const from = Math.max(1, Math.trunc(params.from ?? 1));
const lineCount = Math.max(1, Math.min(1000, Math.trunc(params.lines ?? 200)));
@@ -2753,6 +2878,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
context: MemoryAccessContext;
plan: AuthorizedMemoryPlan;
mutation: AuthorizedMemoryMutation;
signal?: AbortSignal;
}): Promise<MemoryWriteResult> {
if (params.mutation.kind === "admin-reclassify") {
throw new Error("authorized memory reclassification is unavailable");
@@ -2768,6 +2894,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
context: MemoryAccessContext;
plan: AuthorizedMemoryPlan;
mutation: Extract<AuthorizedMemoryMutation, { kind: "import" }>;
signal?: AbortSignal;
}): Promise<MemoryWriteResult> {
return await writeAuthorizedMutation(params);
},
@@ -2775,7 +2902,9 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
async syncAuthorized(params: {
context: MemoryAccessContext;
plan: AuthorizedMemoryPlan;
signal?: AbortSignal;
}): Promise<AuthorizedMemoryResultEnvelope<MemorySyncResult>> {
throwIfAuthorizedMemoryOperationAborted(params.signal);
if (params.context.operation !== "sync") {
throw new Error("authorized memory sync is unavailable");
}
@@ -2783,6 +2912,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
if (!state) {
throw new Error("authorized memory sync is unavailable");
}
throwIfAuthorizedMemoryOperationAborted(params.signal);
drainMemoryAuditOutbox(params.context.agentId);
return createEnvelope({
state,
@@ -2797,7 +2927,9 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
context: MemoryAccessContext;
plan: AuthorizedMemoryPlan;
handles: readonly AuthorizedResourceHandle[];
signal?: AbortSignal;
}): Promise<AuthorizedMemoryResultEnvelope<MemoryExportResult>> {
throwIfAuthorizedMemoryOperationAborted(params.signal);
if (params.context.operation !== "export") {
throw new Error("authorized memory export is unavailable");
}
@@ -2826,7 +2958,9 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
async statusAuthorized(params: {
context: MemoryAccessContext;
plan: AuthorizedMemoryPlan;
signal?: AbortSignal;
}): Promise<AuthorizedMemoryResultEnvelope<AuthorizedMemoryStatus>> {
throwIfAuthorizedMemoryOperationAborted(params.signal);
if (params.context.operation !== "status") {
throw new Error("authorized memory status is unavailable");
}
@@ -2852,6 +2986,7 @@ export const builtinScopedMemoryVirtualView = Object.freeze({
async materializeAuthorizedVirtualView(params: {
context: MemoryContentAccessContext<"read">;
plan: AuthorizedMemoryPlan & Readonly<{ operation: "read" }>;
signal?: AbortSignal;
}): Promise<AuthorizedMemoryVirtualView | undefined> {
return materializeAuthorizedVirtualView(params);
},
@@ -2860,6 +2995,7 @@ export const builtinScopedMemoryVirtualView = Object.freeze({
plan: AuthorizedMemoryPlan & Readonly<{ operation: "read" }>;
view: AuthorizedMemoryVirtualView;
virtualPath: string;
signal?: AbortSignal;
}): Promise<AuthorizedMemoryResultEnvelope<MemoryReadResult>> {
return readAuthorizedVirtualFile(params);
},
+24
View File
@@ -775,6 +775,30 @@ describe("enforced memory tools", () => {
expect(host.read).toHaveBeenCalledWith({ handleId: "mhandle1_allowed" });
});
it("forwards caller cancellation into enforced memory_get", async () => {
const host = {
search: vi.fn(async () => ({ results: [] })),
read: vi.fn(async () => ({ text: "allowed", path: "memory/MEMORY.md" })),
};
const get = createMemoryGetTool({
config: createDefaultMemoryToolConfig(),
memoryReadEnforced: true,
authorizedMemoryRead: host,
});
if (!get) {
throw new Error("memory_get missing");
}
const controller = new AbortController();
await expect(
get.execute("authorized-read-with-cancellation", { handleId: "mhandle1_allowed" }, controller.signal),
).resolves.toMatchObject({ details: { text: "allowed", path: "memory/MEMORY.md" } });
expect(host.read).toHaveBeenCalledWith({
handleId: "mhandle1_allowed",
signal: controller.signal,
});
});
it("does not fall back to legacy memory when its host is unavailable", async () => {
const search = createMemorySearchToolOrThrow({ memoryReadEnforced: true });
const result = await search.execute("missing-authorized-host", { query: "private" });
+4 -1
View File
@@ -776,7 +776,8 @@ export function createMemoryGetTool(options: {
parameters: MemoryGetSchema,
execute:
({ cfg, agentId }) =>
async (_toolCallId, params) => {
async (_toolCallId, params, callerSignal) => {
callerSignal?.throwIfAborted();
const rawParams = asToolParamsRecord(params);
const from = readPositiveIntegerParam(rawParams, "from");
const lines = readPositiveIntegerParam(rawParams, "lines");
@@ -790,7 +791,9 @@ export function createMemoryGetTool(options: {
handleId,
...(from !== undefined ? { from } : {}),
...(lines !== undefined ? { lines } : {}),
...(callerSignal ? { signal: callerSignal } : {}),
});
callerSignal?.throwIfAborted();
return isAuthorizedMemoryUnavailable(result)
? authorizedMemoryUnavailableResult()
: jsonResult(result);
+1
View File
@@ -1827,6 +1827,7 @@
"test:docker:doctor-switch": "bash scripts/e2e/doctor-install-switch-docker.sh",
"test:docker:compose-setup": "bash scripts/e2e/compose-setup.sh",
"test:docker:e2e-build": "bash scripts/e2e/build-image.sh",
"test:docker:fleet-separate-cell": "bash scripts/e2e/fleet-separate-cell-docker.sh",
"test:docker:gateway-network": "bash scripts/e2e/gateway-network-docker.sh",
"test:docker:kitchen-sink-plugin": "bash scripts/e2e/kitchen-sink-plugin-docker.sh",
"test:docker:kitchen-sink-rpc": "bash scripts/e2e/kitchen-sink-rpc-docker.sh",
@@ -397,12 +397,14 @@ export type AuthorizedMemoryReadParams<Operation extends MemoryContentAccessOper
handle: AuthorizedResourceHandle;
from?: number;
lines?: number;
signal?: AbortSignal;
}>;
/** Every authorized method is bound to the operation that produced its plan. */
type AuthorizedMemoryOperationParams<Operation extends MemoryOperation> = Readonly<{
context: MemoryAccessContext & Readonly<{ operation: Operation }>;
plan: AuthorizedMemoryPlan & Readonly<{ operation: Operation }>;
signal?: AbortSignal;
}>;
type AuthorizedMemoryContentMutation = Readonly<{
@@ -450,6 +452,7 @@ export type AuthorizedSealedCompactionStageParams = Readonly<{
plan: AuthorizedMemoryPlan & Readonly<{ operation: "derive" }>;
content: string;
transcriptSource: AuthorizedTranscriptDerivationSource;
signal?: AbortSignal;
}>;
export type AuthorizedMemoryMutation =
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
source "$ROOT_DIR/scripts/lib/docker-e2e-image.sh"
IMAGE_NAME="$(docker_e2e_resolve_image "openclaw-fleet-separate-cell-e2e" OPENCLAW_FLEET_E2E_IMAGE)"
SKIP_BUILD="${OPENCLAW_FLEET_E2E_SKIP_BUILD:-0}"
# The scheduler supplies the functional package image. A direct invocation can still build the
# same package-installed image, but this lane never creates a second Fleet-specific image.
docker_e2e_build_or_reuse "$IMAGE_NAME" fleet-separate-cell "$ROOT_DIR/scripts/e2e/Dockerfile" "$ROOT_DIR" "" "$SKIP_BUILD"
cd "$ROOT_DIR"
OPENCLAW_PROCESS_ISOLATION_E2E=1 \
OPENCLAW_FLEET_E2E_IMAGE="$IMAGE_NAME" \
node scripts/run-vitest.mjs src/fleet/service.container.e2e.test.ts
+5
View File
@@ -530,6 +530,11 @@ export const mainLanes: DockerE2eLane[] = [
},
),
serviceLane("gateway-network", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:gateway-network"),
serviceLane(
"fleet-separate-cell",
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:fleet-separate-cell",
{ stateScenario: "empty", timeoutMs: 15 * 60 * 1000, weight: 4 },
),
serviceLane("browser-cdp-snapshot", "pnpm test:docker:browser-cdp-snapshot", {
stateScenario: "empty",
timeoutMs: 20 * 60 * 1000,
+17 -8
View File
@@ -45,7 +45,7 @@ const authorizedMemoryVirtualBroker: unique symbol = Symbol(
/** Core-private bridge for generic filesystem tools; it is absent from plugin contexts. */
export type AuthorizedMemoryVirtualFileBroker = Readonly<{
view: AuthorizedMemoryVirtualView;
readFile: (virtualPath: string) => Promise<string | undefined>;
readFile: (virtualPath: string, signal?: AbortSignal) => Promise<string | undefined>;
}>;
/** Core-private sealed compaction capability; plugins never receive this host. */
@@ -58,16 +58,19 @@ export type AuthorizedSealedCompactionHost = Readonly<{
type AuthorizedMemoryReadHostWithVirtualBroker = AuthorizedMemoryReadHost &
Readonly<{
[authorizedMemoryVirtualBroker]: () => Promise<AuthorizedMemoryVirtualFileBroker | undefined>;
[authorizedMemoryVirtualBroker]: (
signal?: AbortSignal,
) => Promise<AuthorizedMemoryVirtualFileBroker | undefined>;
}>;
export async function resolveAuthorizedMemoryVirtualFileBroker(
host: AuthorizedMemoryReadHost | undefined,
signal?: AbortSignal,
): Promise<AuthorizedMemoryVirtualFileBroker | undefined> {
if (!host || !(authorizedMemoryVirtualBroker in host)) {
return undefined;
}
return (host as AuthorizedMemoryReadHostWithVirtualBroker)[authorizedMemoryVirtualBroker]();
return (host as AuthorizedMemoryReadHostWithVirtualBroker)[authorizedMemoryVirtualBroker](signal);
}
function hash(value: unknown): string {
@@ -318,28 +321,34 @@ function createAuthorizedMemoryContentHost(
? createAuthorizedMemoryDeriveInvocation({ context: trusted })
: createAuthorizedMemoryReadInvocation({ context: trusted }));
let virtualBroker: Promise<AuthorizedMemoryVirtualFileBroker | undefined> | undefined;
const getVirtualBroker = () =>
(virtualBroker ??= (async () => {
const createVirtualBroker = async (signal?: AbortSignal) => {
signal?.throwIfAborted();
const active = await getInvocation();
if ("unavailable" in active) {
return undefined;
}
const view = await materializeAuthorizedMemoryVirtualView({ invocation: active });
const view = await materializeAuthorizedMemoryVirtualView({
invocation: active,
...(signal ? { signal } : {}),
});
if ("unavailable" in view) {
return undefined;
}
return Object.freeze({
view,
async readFile(virtualPath) {
async readFile(virtualPath, signal) {
const result = await readAuthorizedMemoryVirtualFile({
invocation: active,
view,
virtualPath,
...(signal ? { signal } : {}),
});
return "unavailable" in result ? undefined : result.text;
},
});
})());
};
const getVirtualBroker = (signal?: AbortSignal) =>
signal ? createVirtualBroker(signal) : (virtualBroker ??= createVirtualBroker());
return Object.freeze({
async search(search) {
const active = await getInvocation();
-1
View File
@@ -337,7 +337,6 @@ export async function dispatchInboundMessage(params: {
}
let settledReceipt: DispatchFromConfigResult["settledReceipt"];
installMemoryEgressAdmission(params.dispatcher, finalized, replyPayloadRunState);
let settledReceipt: DispatchFromConfigResult["settledReceipt"];
const result = await withReplyDispatcher({
dispatcher: params.dispatcher,
onSettled: params.onSettled,
+34 -1
View File
@@ -1,6 +1,6 @@
import fs from "node:fs/promises";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { requireNodeSqlite } from "../infra/node-sqlite.js";
import type { RuntimeEnv } from "../runtime.js";
@@ -11,6 +11,15 @@ import { OPENCLAW_AGENT_SCHEMA_SQL } from "../state/openclaw-agent-schema.js";
import { OPENCLAW_STATE_SCHEMA_VERSION } from "../state/openclaw-state-db.js";
import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js";
import { OPENCLAW_STATE_SCHEMA_SQL } from "../state/openclaw-state-schema.js";
const withDoctorSqliteMaintenanceLock = vi.hoisted(() =>
vi.fn(async (params: { run: () => Promise<unknown> }) => await params.run()),
);
vi.mock("./doctor-sqlite-maintenance-lock.js", () => ({
withDoctorSqliteMaintenanceLock,
}));
import {
backupSqliteCreateCommand,
backupSqliteListCommand,
@@ -118,6 +127,30 @@ function createAgentDatabase(databasePath: string, agentId: string): void {
}
describe("SQLite backup commands", () => {
it("uses the cross-process SQLite ownership lock for an offline snapshot", async () => {
const tempDir = tempDirs.make("openclaw-backup-sqlite-lock-");
const stateDir = path.join(tempDir, "state");
const repositoryPath = path.join(tempDir, "snapshots");
process.env.OPENCLAW_STATE_DIR = stateDir;
const databasePath = resolveOpenClawStateSqlitePath();
await fs.mkdir(path.dirname(databasePath), { recursive: true });
createGlobalDatabase(databasePath);
withDoctorSqliteMaintenanceLock.mockClear();
await backupSqliteCreateCommand(createRuntimeCapture(), {
global: true,
repository: repositoryPath,
});
expect(withDoctorSqliteMaintenanceLock).toHaveBeenCalledWith(
expect.objectContaining({
operation: "SQLite snapshot",
protectedPaths: [databasePath],
run: expect.any(Function),
}),
);
});
it("creates, lists, verifies, and fresh-restores the global database", async () => {
const tempDir = tempDirs.make("openclaw-backup-sqlite-");
const stateDir = path.join(tempDir, "state");
+8 -6
View File
@@ -1,5 +1,6 @@
import fs from "node:fs/promises";
import path from "node:path";
import { withDoctorSqliteMaintenanceLock } from "./doctor-sqlite-maintenance-lock.js";
import { formatErrorMessage } from "../infra/errors.js";
import { normalizeAgentId } from "../routing/session-key.js";
import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js";
@@ -77,12 +78,13 @@ export async function backupSqliteCreateCommand(
const repositoryPath = resolveRequiredPath(options.repository, "--repository");
try {
const database = await resolveSnapshotDatabase(options);
// A selected broker owns memory writes. Ask it to drain before the trusted Gateway snapshots
// an agent database, then always reopen admission; workers never receive a DB path or broker IPC.
const { withBrokeredMemoryMaintenance } = await import("../plugins/memory-broker-runtime.js");
const result = await withBrokeredMemoryMaintenance(
async () => await createLocalSqliteSnapshotProvider({ repositoryPath }).create(database),
);
// A CLI snapshot runs outside the Gateway, so its local broker map cannot prove exclusion
// against the live owner. The state lock is the cross-process ownership boundary.
const result = await withDoctorSqliteMaintenanceLock({
operation: "SQLite snapshot",
protectedPaths: [database.path],
run: async () => await createLocalSqliteSnapshotProvider({ repositoryPath }).create(database),
});
const report: BackupSqliteCreateResult = {
ok: true,
snapshotPath: result.ref.path,
+36
View File
@@ -8,6 +8,9 @@ const backupVerifyCommandMock = vi.hoisted(() => vi.fn());
const writeRuntimeJsonMock = vi.hoisted(() => vi.fn());
const formatBackupCreateSummaryMock = vi.hoisted(() => vi.fn(() => ["backup ok"]));
const recordBackupRunOutcomeMock = vi.hoisted(() => vi.fn());
const withDoctorSqliteMaintenanceLockMock = vi.hoisted(() =>
vi.fn(async (params: { run: () => Promise<unknown> }) => await params.run()),
);
vi.mock("../infra/backup-create.js", () => ({
createBackupArchive: createBackupArchiveMock,
@@ -30,6 +33,10 @@ vi.mock("../state/backup-run-records.js", () => ({
recordBackupRunOutcome: recordBackupRunOutcomeMock,
}));
vi.mock("./doctor-sqlite-maintenance-lock.js", () => ({
withDoctorSqliteMaintenanceLock: withDoctorSqliteMaintenanceLockMock,
}));
function createRuntime(): RuntimeEnv {
return {
log: vi.fn(),
@@ -54,6 +61,10 @@ describe("backupCreateCommand verify wrapper", () => {
formatBackupCreateSummaryMock.mockReset();
formatBackupCreateSummaryMock.mockReturnValue(["backup ok"]);
recordBackupRunOutcomeMock.mockReset();
withDoctorSqliteMaintenanceLockMock.mockReset();
withDoctorSqliteMaintenanceLockMock.mockImplementation(
async (params: { run: () => Promise<unknown> }) => await params.run(),
);
});
it("optionally verifies the archive after writing it", async () => {
@@ -93,6 +104,31 @@ describe("backupCreateCommand verify wrapper", () => {
});
expect(verifyLog).not.toBe(runtime.log);
expect(typeof verifyLog).toBe("function");
expect(withDoctorSqliteMaintenanceLockMock).toHaveBeenCalledWith(
expect.objectContaining({
operation: "backup archive creation",
protectedPaths: [expect.any(String)],
run: expect.any(Function),
}),
);
});
it("does not take offline state ownership for dry runs or config-only archives", async () => {
createBackupArchiveMock.mockResolvedValue({
archivePath: "/tmp/openclaw-backup.tar.gz",
archiveRoot: "openclaw-backup",
createdAt: "2026-04-07T00:00:00.000Z",
assets: [],
verified: false,
dryRun: true,
includeWorkspace: false,
onlyConfig: false,
});
await backupCreateCommand(createRuntime(), { dryRun: true });
await backupCreateCommand(createRuntime(), { onlyConfig: true });
expect(withDoctorSqliteMaintenanceLockMock).not.toHaveBeenCalled();
});
it("does not claim completion when both backup and outcome recording fail", async () => {
+14 -1
View File
@@ -25,6 +25,14 @@ import {
tarCreateMock,
} from "./backup.test-support.js";
const withDoctorSqliteMaintenanceLock = vi.hoisted(() =>
vi.fn(async (params: { run: () => Promise<unknown> }) => await params.run()),
);
vi.mock("./doctor-sqlite-maintenance-lock.js", () => ({
withDoctorSqliteMaintenanceLock,
}));
const { backupCreateCommand } = await import("./backup.js");
type CapturedBackupManifest = {
@@ -91,6 +99,10 @@ describe("backup commands", () => {
assetCount: 1,
entryCount: 2,
});
withDoctorSqliteMaintenanceLock.mockClear();
withDoctorSqliteMaintenanceLock.mockImplementation(
async (params: { run: () => Promise<unknown> }) => await params.run(),
);
});
afterEach(async () => {
@@ -633,9 +645,10 @@ describe("backup commands", () => {
setTestEnvValue("OPENCLAW_CONFIG_PATH", configPath);
try {
const plan = await resolveBackupPlanFromDisk({ nowMs: 123 });
const canonicalWorkspaceDir = await fs.realpath(workspaceDir);
expect(plan.included).toContainEqual(
expect.objectContaining({ kind: "workspace", sourcePath: workspaceDir }),
expect.objectContaining({ kind: "workspace", sourcePath: canonicalWorkspaceDir }),
);
expect(await fs.readFile(configPath, "utf8")).toBe(originalRaw);
expect(plan.configPath).toBe(configPath);
+18 -4
View File
@@ -5,10 +5,12 @@ import {
type BackupCreateOptions,
type BackupCreateResult,
} from "../infra/backup-create.js";
import { resolveStateDir } from "../config/paths.js";
import { formatErrorMessage } from "../infra/errors.js";
import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js";
import { createLazyImportLoader } from "../shared/lazy-promise.js";
import { recordBackupRunOutcome } from "../state/backup-run-records.js";
import { withDoctorSqliteMaintenanceLock } from "./doctor-sqlite-maintenance-lock.js";
type BackupVerifyRuntime = typeof import("./backup-verify.js");
@@ -27,10 +29,22 @@ export async function backupCreateCommand(
): Promise<BackupCreateResult> {
let archivePath = opts.output ?? process.cwd();
try {
const result = await createBackupArchive({
...opts,
log: opts.log ?? (opts.json ? undefined : (message: string) => runtime.log(message)),
});
const createArchive = async () =>
await createBackupArchive({
...opts,
log: opts.log ?? (opts.json ? undefined : (message: string) => runtime.log(message)),
});
// Archive creation captures the selected memory state and its SQLite snapshots. A CLI cannot
// quiesce a live Gateway-owned broker, so require offline state ownership instead of taking a
// potentially inconsistent archive while the broker can still activate artifacts.
const result =
opts.dryRun || opts.onlyConfig
? await createArchive()
: await withDoctorSqliteMaintenanceLock({
operation: "backup archive creation",
protectedPaths: [resolveStateDir(process.env)],
run: createArchive,
});
archivePath = result.archivePath;
if (opts.verify && !opts.dryRun) {
const { backupVerifyCommand } = await loadBackupVerifyRuntime();
+12 -7
View File
@@ -19,6 +19,9 @@ describe("runDoctorMemoryIsolation", () => {
beforeEach(() => {
stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-memory-isolation-"));
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
const configPath = path.join(stateDir, "openclaw.json");
fs.writeFileSync(configPath, "{}\n", "utf8");
vi.stubEnv("OPENCLAW_CONFIG_PATH", configPath);
const database = openOpenClawAgentDatabase({ agentId: "main" });
database.db
.prepare(
@@ -36,27 +39,29 @@ describe("runDoctorMemoryIsolation", () => {
fs.rmSync(stateDir, { force: true, recursive: true });
});
it("owns the reversible configured-agent shadow lifecycle", () => {
expect(runDoctorMemoryIsolation({ action: "status", cfg })).toEqual({
it("owns the reversible configured-agent shadow lifecycle", async () => {
await expect(runDoctorMemoryIsolation({ action: "status", cfg })).resolves.toEqual({
agentId: "main",
mode: "legacy",
restartRequired: false,
});
expect(runDoctorMemoryIsolation({ action: "shadow-read-only", cfg, nowMs: 1 })).toEqual({
await expect(
runDoctorMemoryIsolation({ action: "shadow-read-only", cfg, nowMs: 1 }),
).resolves.toEqual({
agentId: "main",
mode: "shadow-read-only",
restartRequired: true,
});
expect(runDoctorMemoryIsolation({ action: "legacy", cfg })).toEqual({
await expect(runDoctorMemoryIsolation({ action: "legacy", cfg })).resolves.toEqual({
agentId: "main",
mode: "legacy",
restartRequired: true,
});
});
it("refuses an agent outside runtime configuration", () => {
expect(() =>
it("refuses an agent outside runtime configuration", async () => {
await expect(
runDoctorMemoryIsolation({ action: "shadow-read-only", agentId: "missing", cfg }),
).toThrow('Unknown configured agent id "missing".');
).rejects.toThrow('Unknown configured agent id "missing".');
});
});
+22 -12
View File
@@ -8,6 +8,8 @@ import {
resolveMemoryIsolationMode,
type MemoryIsolationMode,
} from "../plugins/memory-cutover.js";
import { resolveOpenClawAgentSqlitePath } from "../state/openclaw-agent-db.paths.js";
import { withDoctorSqliteMaintenanceLock } from "./doctor-sqlite-maintenance-lock.js";
export type DoctorMemoryIsolationAction = "status" | "shadow-read-only" | "legacy";
@@ -32,28 +34,36 @@ function resolveDoctorMemoryIsolationAgent(params: {
* Doctor owns the only P1C enablement path. It persists one reversible, verified posture and
* deliberately does not create a Phase 6 cutover marker or claim two-subject confinement.
*/
export function runDoctorMemoryIsolation(params: {
export async function runDoctorMemoryIsolation(params: {
action: DoctorMemoryIsolationAction;
agentId?: string;
cfg?: OpenClawConfig;
nowMs?: number;
}): DoctorMemoryIsolationReport {
}): Promise<DoctorMemoryIsolationReport> {
const cfg = params.cfg ?? getRuntimeConfig();
const agentId = resolveDoctorMemoryIsolationAgent({ agentId: params.agentId, cfg });
switch (params.action) {
case "status":
return { agentId, mode: resolveMemoryIsolationMode(agentId), restartRequired: false };
case "shadow-read-only":
return {
agentId,
mode: enableMemoryShadowReadOnlyMode({ agentId, nowMs: params.nowMs }),
restartRequired: true,
};
return await withDoctorSqliteMaintenanceLock({
operation: "memory isolation shadow-read-only",
protectedPaths: [resolveOpenClawAgentSqlitePath({ agentId })],
run: () => ({
agentId,
mode: enableMemoryShadowReadOnlyMode({ agentId, nowMs: params.nowMs }),
restartRequired: true,
}),
});
case "legacy":
return {
agentId,
mode: disableMemoryShadowReadOnlyMode({ agentId }),
restartRequired: true,
};
return await withDoctorSqliteMaintenanceLock({
operation: "memory isolation legacy",
protectedPaths: [resolveOpenClawAgentSqlitePath({ agentId })],
run: () => ({
agentId,
mode: disableMemoryShadowReadOnlyMode({ agentId }),
restartRequired: true,
}),
});
}
}
+17
View File
@@ -29,6 +29,9 @@ const repairDreamingArtifacts = vi.hoisted(() => vi.fn());
const repairShortTermPromotionArtifacts = vi.hoisted(() => vi.fn());
const noteWorkspaceMemoryHealth = vi.hoisted(() => vi.fn(async () => undefined));
const maybeRepairWorkspaceMemoryHealth = vi.hoisted(() => vi.fn(async () => undefined));
const withDoctorSqliteMaintenanceLock = vi.hoisted(() =>
vi.fn(async (params: { run: () => Promise<unknown> }) => await params.run()),
);
vi.mock("../../packages/terminal-core/src/note.js", () => ({
note,
@@ -99,6 +102,10 @@ vi.mock("./doctor-workspace.js", async (importOriginal) => {
};
});
vi.mock("./doctor-sqlite-maintenance-lock.js", () => ({
withDoctorSqliteMaintenanceLock,
}));
import {
noteMemorySearchHealth,
maybeRepairMemoryRecallHealth,
@@ -158,6 +165,10 @@ function resetMemoryRecallMocks() {
});
noteWorkspaceMemoryHealth.mockClear();
maybeRepairWorkspaceMemoryHealth.mockClear();
withDoctorSqliteMaintenanceLock.mockClear();
withDoctorSqliteMaintenanceLock.mockImplementation(
async (params: { run: () => Promise<unknown> }) => await params.run(),
);
}
function firstNoteMessage(): string {
@@ -1199,6 +1210,9 @@ describe("memory recall doctor integration", () => {
expect(repairShortTermPromotionArtifacts).toHaveBeenCalledWith({
workspaceDir: "/tmp/agent-default/workspace",
});
expect(withDoctorSqliteMaintenanceLock).toHaveBeenCalledWith(
expect.objectContaining({ operation: "memory recall artifact repair", run: expect.any(Function) }),
);
expect(note).toHaveBeenCalledTimes(1);
expectFirstNoteContains(
"Memory recall artifacts repaired:",
@@ -1251,6 +1265,9 @@ describe("memory recall doctor integration", () => {
expect(repairDreamingArtifacts).toHaveBeenCalledWith({
workspaceDir: "/tmp/agent-default/workspace",
});
expect(withDoctorSqliteMaintenanceLock).toHaveBeenCalledWith(
expect.objectContaining({ operation: "dreaming artifact repair", run: expect.any(Function) }),
);
const message = String(note.mock.calls[note.mock.calls.length - 1]?.[0] ?? "");
expect(message).toContain("Dreaming artifacts repaired:");
expect(message).toContain("archived session corpus");
+9 -2
View File
@@ -49,6 +49,7 @@ import { defaultSlotIdForKey } from "../plugins/slots.js";
import { getProviderEnvVars } from "../secrets/provider-env-vars.js";
import { resolveUserPath } from "../utils.js";
import type { DoctorPrompter } from "./doctor-prompter.js";
import { withDoctorSqliteMaintenanceLock } from "./doctor-sqlite-maintenance-lock.js";
import { maybeRepairWorkspaceMemoryHealth, noteWorkspaceMemoryHealth } from "./doctor-workspace.js";
import { isRecord } from "./doctor/shared/legacy-config-record-shared.js";
@@ -378,7 +379,10 @@ export async function maybeRepairMemoryRecallHealth(params: {
initialValue: true,
});
if (approved) {
const repair = await repairShortTermPromotionArtifacts({ workspaceDir });
const repair = await withDoctorSqliteMaintenanceLock({
operation: "memory recall artifact repair",
run: async () => await repairShortTermPromotionArtifacts({ workspaceDir }),
});
if (repair.changed) {
const removedOverflowEntries = repair.removedOverflowEntries ?? 0;
const details = [
@@ -424,7 +428,10 @@ export async function maybeRepairMemoryRecallHealth(params: {
if (!approvedDreamingRepair) {
continue;
}
const dreamingRepair = await repairDreamingArtifacts({ workspaceDir });
const dreamingRepair = await withDoctorSqliteMaintenanceLock({
operation: "dreaming artifact repair",
run: async () => await repairDreamingArtifacts({ workspaceDir }),
});
if (!dreamingRepair.changed) {
continue;
}
@@ -141,6 +141,50 @@ describe("doctor SQLite maintenance lock", () => {
await gatewayLock.release();
});
it("recognizes a live backup process as an offline maintenance owner", async () => {
const fixture = await createLockFixture();
const lockOptions = {
...fixture.lockOptions,
readProcessCmdline: () => ["openclaw", "backup", "sqlite", "create"],
};
let allowMaintenanceToFinish: (() => void) | undefined;
let markMaintenanceStarted: (() => void) | undefined;
const maintenanceMayFinish = new Promise<void>((resolve) => {
allowMaintenanceToFinish = resolve;
});
const maintenanceStarted = new Promise<void>((resolve) => {
markMaintenanceStarted = resolve;
});
const maintenance = withDoctorSqliteMaintenanceLock(
{
env: fixture.env,
operation: "SQLite snapshot",
run: async () => {
markMaintenanceStarted?.();
await maintenanceMayFinish;
},
},
{ lockOptions },
);
await maintenanceStarted;
try {
await expect(
withDoctorSqliteMaintenanceLock(
{
env: fixture.env,
operation: "state SQLite compaction",
run: vi.fn(),
},
{ lockOptions },
),
).rejects.toBeInstanceOf(DoctorSqliteMaintenanceLockUnavailableError);
} finally {
allowMaintenanceToFinish?.();
await maintenance;
}
});
it("releases ownership after maintenance fails", async () => {
const fixture = await createLockFixture();
@@ -161,7 +161,9 @@ export async function withDoctorSqliteMaintenanceLock<T>(
allowInTests: true,
env,
pollIntervalMs: lockOptions?.pollIntervalMs ?? MAINTENANCE_LOCK_POLL_INTERVAL_MS,
role: "sqlite-maintenance",
// Doctor and `backup sqlite create` both mutate/read the same state files while offline.
// Their lock must survive stale-owner checks for either CLI command.
role: "offline-maintenance",
timeoutMs: lockOptions?.timeoutMs ?? MAINTENANCE_LOCK_TIMEOUT_MS,
});
} catch (error) {
+14
View File
@@ -7,11 +7,18 @@ import type { OpenClawConfig } from "../config/config.js";
import type { DoctorPrompter } from "./doctor-prompter.js";
const note = vi.hoisted(() => vi.fn());
const withDoctorSqliteMaintenanceLock = vi.hoisted(() =>
vi.fn(async (params: { run: () => Promise<unknown> }) => await params.run()),
);
vi.mock("../../packages/terminal-core/src/note.js", () => ({
note,
}));
vi.mock("./doctor-sqlite-maintenance-lock.js", () => ({
withDoctorSqliteMaintenanceLock,
}));
import {
detectRootMemoryFiles,
formatRootMemoryFilesWarning,
@@ -46,6 +53,10 @@ describe("root memory repair", () => {
beforeEach(async () => {
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-root-memory-"));
note.mockClear();
withDoctorSqliteMaintenanceLock.mockClear();
withDoctorSqliteMaintenanceLock.mockImplementation(
async (params: { run: () => Promise<unknown> }) => await params.run(),
);
});
afterEach(async () => {
@@ -202,6 +213,9 @@ describe("root memory repair", () => {
message: "Merge legacy root memory.md into canonical MEMORY.md and remove the shadowed file?",
initialValue: true,
});
expect(withDoctorSqliteMaintenanceLock).toHaveBeenCalledWith(
expect.objectContaining({ operation: "workspace root memory repair", run: expect.any(Function) }),
);
const canonical = await fs.readFile(path.join(tmpDir, "MEMORY.md"), "utf8");
expect(canonical).toContain("# Legacy");
await expectPathMissing(path.join(tmpDir, "memory.md"));
+7 -1
View File
@@ -16,6 +16,7 @@ import {
} from "../memory/root-memory-files.js";
import { shortenHomePath } from "../utils.js";
import type { DoctorPrompter } from "./doctor-prompter.js";
import { withDoctorSqliteMaintenanceLock } from "./doctor-sqlite-maintenance-lock.js";
// AGENTS.md is only scanned for a memory-system reference; a small cap prevents
// a huge file from being buffered just for one regex check.
@@ -374,7 +375,12 @@ export async function maybeRepairWorkspaceMemoryHealth(params: {
if (!approvedLegacyMigration) {
return;
}
const migration = await migrateLegacyRootMemoryFile(configuredWorkspaceDir);
// Prompting remains outside the lease. Once approved, stop the live Gateway from changing
// memory-derived workspace state while this doctor command archives and merges root memory.
const migration = await withDoctorSqliteMaintenanceLock({
operation: "workspace root memory repair",
run: async () => await migrateLegacyRootMemoryFile(configuredWorkspaceDir),
});
if (migration.readLimitExceeded) {
note(
[
+1 -1
View File
@@ -48,7 +48,7 @@ export async function doctorCommand(runtime?: RuntimeEnv, options?: DoctorOption
if (options?.memoryIsolation) {
const outputRuntime = runtime ?? defaultRuntime;
const { runDoctorMemoryIsolation } = await import("./doctor-memory-isolation.js");
const report = runDoctorMemoryIsolation({
const report = await runDoctorMemoryIsolation({
action: options.memoryIsolation,
...(options.memoryIsolationAgent ? { agentId: options.memoryIsolationAgent } : {}),
});
@@ -626,14 +626,23 @@ describe("transcript memory policy companions", () => {
},
);
vi.stubEnv("OPENCLAW_STATE_DIR", env.OPENCLAW_STATE_DIR ?? "");
expect(
const stateDir = env.OPENCLAW_STATE_DIR ?? "";
const configPath = path.join(stateDir, "openclaw.json");
fs.mkdirSync(stateDir, { recursive: true });
fs.writeFileSync(configPath, "{}\n", "utf8");
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
vi.stubEnv("OPENCLAW_CONFIG_PATH", configPath);
await expect(
runDoctorMemoryIsolation({
action: "shadow-read-only",
cfg: { agents: { list: [{ id: AGENT_ID, default: true }] } } as OpenClawConfig,
nowMs: 1,
}),
).toMatchObject({ agentId: AGENT_ID, mode: "shadow-read-only", restartRequired: true });
).resolves.toMatchObject({
agentId: AGENT_ID,
mode: "shadow-read-only",
restartRequired: true,
});
// Doctor writes out of process. Refresh the process-owned snapshot to model the required
// Gateway restart before proving the protected transcript boundary.
resetMemoryIsolationCutoverForTest();
+245
View File
@@ -0,0 +1,245 @@
import crypto from "node:crypto";
import { afterEach, describe, expect, it } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { DOCKER_SANDBOX_ENGINE, execContainer } from "../agents/sandbox/docker.js";
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
import { cellAuthSecretDir, cellContainerName, cellNetworkName } from "./cell-profile.js";
import { createFleetContainerRuntime } from "./containers.runtime.js";
import { createFleetService } from "./service.runtime.js";
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
const DOCKER_COMMAND_TIMEOUT_MS = 30_000;
function uniqueTenant(prefix: string): string {
return `${prefix}-${process.pid}-${crypto.randomBytes(4).toString("hex")}`;
}
async function docker(args: string[], allowFailure = false) {
return await execContainer(DOCKER_SANDBOX_ENGINE, args, {
allowFailure,
signal: AbortSignal.timeout(DOCKER_COMMAND_TIMEOUT_MS),
});
}
function isMissingDockerObject(stderr: string): boolean {
return /no such (?:container|network|object)|not found/iu.test(stderr);
}
async function cleanupCell(params: {
service: ReturnType<typeof createFleetService>;
tenant: string;
}): Promise<void> {
try {
await params.service.remove({ tenant: params.tenant, force: true, purgeData: true });
return;
} catch (serviceError) {
// Preserve Fleet as the normal cleanup owner. These are exact, test-generated names only,
// and recover a cell left behind by a failed health gate before the registry can remove it.
const failures: unknown[] = [serviceError];
for (const args of [
["rm", "--force", cellContainerName(params.tenant)],
["network", "rm", cellNetworkName(params.tenant)],
]) {
const result = await docker(args, true);
if (result.code !== 0 && !isMissingDockerObject(result.stderr)) {
failures.push(new Error(result.stderr.trim() || `docker ${args.join(" ")} failed`));
}
}
if (failures.length > 1) {
throw new AggregateError(failures, `Could not clean up Fleet cell ${params.tenant}.`);
}
}
}
describe.runIf(process.env.OPENCLAW_PROCESS_ISOLATION_E2E === "1")(
"fleet separate-cell process isolation",
() => {
it("keeps a hostile tenant process out of another cell's data, credentials, mounts, and network", async () => {
const image = process.env.OPENCLAW_FLEET_E2E_IMAGE;
if (!image) {
throw new Error(
"Fleet process-isolation proof requires OPENCLAW_FLEET_E2E_IMAGE from the Docker E2E scheduler.",
);
}
const root = tempDirs.make("openclaw-fleet-separate-cell-");
const tenantA = uniqueTenant("fleeta");
const tenantB = uniqueTenant("fleetb");
const foreignSecret = crypto.randomBytes(16).toString("hex");
const containers = createFleetContainerRuntime();
const service = createFleetService({
containers,
env: { ...process.env, OPENCLAW_STATE_DIR: root },
});
try {
const cellA = await service.create({
tenant: tenantA,
image,
env: [
"FLEET_CELL_E2E_A_MARKER=visible-only-to-a",
`FLEET_CELL_E2E_A_SECRET=${crypto.randomBytes(16).toString("hex")}`,
"OPENCLAW_SKIP_CHANNELS=1",
"OPENCLAW_SKIP_GMAIL_WATCHER=1",
"OPENCLAW_SKIP_CRON=1",
"OPENCLAW_SKIP_CANVAS_HOST=1",
],
});
const cellB = await service.create({
tenant: tenantB,
image,
env: [
"FLEET_CELL_E2E_B_MARKER=visible-only-to-b",
`FLEET_CELL_E2E_B_SECRET=${foreignSecret}`,
"OPENCLAW_SKIP_CHANNELS=1",
"OPENCLAW_SKIP_GMAIL_WATCHER=1",
"OPENCLAW_SKIP_CRON=1",
"OPENCLAW_SKIP_CANVAS_HOST=1",
],
});
// Each create already waits for its own /healthz response. Query status serially so this
// isolation proof does not add an unrelated pair of cold-Gateway health requests.
const statusA = await service.status(tenantA);
const statusB = await service.status(tenantB);
expect(statusA.container).toMatchObject({
managed: true,
running: true,
state: "running",
});
expect(statusA.health, JSON.stringify(statusA.health)).toMatchObject({
httpStatus: 200,
status: "ok",
});
expect(statusB.container).toMatchObject({
managed: true,
running: true,
state: "running",
});
expect(statusB.health, JSON.stringify(statusB.health)).toMatchObject({
httpStatus: 200,
status: "ok",
});
const doctor = await service.doctor();
expect(doctor).toHaveLength(2);
expect(
doctor
.flatMap((report) => report.findings)
.filter((finding) => finding.status === "fail"),
).toEqual([]);
const [pidA, pidB] = await Promise.all(
[cellA.containerName, cellB.containerName].map(async (containerName) => {
const result = await docker(["inspect", "--format", "{{.State.Pid}}", containerName]);
expect(result.code, result.stderr).toBe(0);
return Number(result.stdout.trim());
}),
);
expect(pidA).toBeGreaterThan(0);
expect(pidB).toBeGreaterThan(0);
expect(pidA).not.toBe(pidB);
const stateSentinel = "issued-a-memory-view";
const foreignSentinel = "issued-b-memory-view";
for (const [containerName, sentinelName, sentinel] of [
[cellA.containerName, "issued-a.txt", stateSentinel],
[cellB.containerName, "issued-b.txt", foreignSentinel],
] as const) {
const result = await docker([
"exec",
containerName,
"node",
"-e",
`require("node:fs").writeFileSync(require("node:path").join(process.env.OPENCLAW_STATE_DIR, ${JSON.stringify(sentinelName)}), ${JSON.stringify(sentinel)});`,
]);
expect(result.code, result.stderr).toBe(0);
}
const foreignDataDir = statusB.dataDir;
const foreignAuthDir = cellAuthSecretDir(root, tenantB);
const hostileProbe = [
'const fs = require("node:fs");',
'const path = require("node:path");',
`const foreignDataDir = ${JSON.stringify(foreignDataDir)};`,
`const foreignAuthDir = ${JSON.stringify(foreignAuthDir)};`,
`const foreignSecret = ${JSON.stringify(foreignSecret)};`,
"const probe = (read) => { try { read(); return true; } catch { return false; } };",
'const mountInfo = fs.readFileSync("/proc/self/mountinfo", "utf8");',
"process.stdout.write(JSON.stringify({",
' ownSentinel: fs.readFileSync(path.join(process.env.OPENCLAW_STATE_DIR, "issued-a.txt"), "utf8"),',
" foreignDataDirectory: probe(() => fs.readdirSync(foreignDataDir)),",
' foreignDataSentinel: probe(() => fs.readFileSync(path.join(foreignDataDir, "issued-b.txt"), "utf8")),',
" foreignAuthDirectory: probe(() => fs.readdirSync(foreignAuthDir)),",
' foreignEnvironmentMarker: Object.hasOwn(process.env, "FLEET_CELL_E2E_B_MARKER"),',
' foreignEnvironmentSecret: Object.hasOwn(process.env, "FLEET_CELL_E2E_B_SECRET"),',
" foreignEnvironmentSecretValue: Object.values(process.env).includes(foreignSecret),",
" foreignMount: mountInfo.includes(foreignDataDir) || mountInfo.includes(foreignAuthDir),",
"}));",
].join("\n");
const hostileResult = await docker([
"exec",
cellA.containerName,
"node",
"-e",
hostileProbe,
]);
expect(hostileResult.code, hostileResult.stderr).toBe(0);
expect(JSON.parse(hostileResult.stdout)).toEqual({
ownSentinel: stateSentinel,
foreignDataDirectory: false,
foreignDataSentinel: false,
foreignAuthDirectory: false,
foreignEnvironmentMarker: false,
foreignEnvironmentSecret: false,
foreignEnvironmentSecretValue: false,
foreignMount: false,
});
const inspectedMounts = await docker([
"inspect",
"--format",
"{{json .Mounts}}",
cellA.containerName,
]);
expect(inspectedMounts.code, inspectedMounts.stderr).toBe(0);
const mounts: unknown = JSON.parse(inspectedMounts.stdout);
if (!Array.isArray(mounts)) {
throw new Error("Fleet container inspection did not return mounts.");
}
const mountSources = mounts.flatMap((mount) => {
if (
!mount ||
typeof mount !== "object" ||
!("Source" in mount) ||
typeof mount.Source !== "string"
) {
return [];
}
return [mount.Source];
});
expect(mountSources).not.toContain(foreignDataDir);
expect(mountSources).not.toContain(foreignAuthDir);
const [networkA, networkB] = await Promise.all([
containers.inspectNetwork("docker", cellNetworkName(tenantA)),
containers.inspectNetwork("docker", cellNetworkName(tenantB)),
]);
expect(networkA).toMatchObject({
kind: "ok",
attachedContainers: [{ name: cellA.containerName }],
});
expect(networkB).toMatchObject({
kind: "ok",
attachedContainers: [{ name: cellB.containerName }],
});
if (networkA.kind === "ok" && networkB.kind === "ok") {
expect(networkA.attachedContainers).toHaveLength(1);
expect(networkB.attachedContainers).toHaveLength(1);
}
} finally {
await cleanupCell({ service, tenant: tenantB });
await cleanupCell({ service, tenant: tenantA });
closeOpenClawStateDatabaseForTest();
}
}, 150_000);
},
);
@@ -11,6 +11,7 @@ export const MCP_APP_STANDALONE_PATH = "/__openclaw__/mcp-app";
export const MCP_APP_STANDALONE_VIEW_PATH = `${MCP_APP_STANDALONE_PATH}/view`;
const WORKER_GATEWAY_PATH = "/__openclaw__/worker";
const NODE_WORKER_BUNDLE_TRANSFER_NAMESPACE = "/__openclaw__/worker-bundle";
const NODE_WORKER_PROJECTION_TRANSFER_NAMESPACE = "/__openclaw__/worker-memory-projection";
const NODE_WORKSPACE_TRANSFER_NAMESPACE = "/__openclaw__/worker-transfer";
export function classifyGatewayProbePath(
@@ -53,6 +54,13 @@ export function classifyNodeWorkerBundleTransferPath(pathname: string): "namespa
: "outside";
}
export function classifyNodeWorkerProjectionTransferPath(pathname: string): "namespace" | "outside" {
return pathname === NODE_WORKER_PROJECTION_TRANSFER_NAMESPACE ||
pathname.startsWith(`${NODE_WORKER_PROJECTION_TRANSFER_NAMESPACE}/`)
? "namespace"
: "outside";
}
export function classifyNodeWorkspaceTransferPath(pathname: string): "namespace" | "outside" {
return pathname === NODE_WORKSPACE_TRANSFER_NAMESPACE ||
pathname.startsWith(`${NODE_WORKSPACE_TRANSFER_NAMESPACE}/`)
+40 -4
View File
@@ -4,11 +4,18 @@ import {
NODE_WORKER_SUPERVISOR_BINARY_CAPACITY_PROTOCOL_FEATURE,
NODE_WORKER_SUPERVISOR_BUILD_PROTOCOL_FEATURE,
NODE_WORKER_SUPERVISOR_EXECUTION_CONTEXT_V1_PROTOCOL_FEATURE,
NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE,
NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
NODE_WORKER_SUPERVISOR_LEGACY_PROTOCOL_FEATURE,
type NodeRunnerInventoryIssue,
type NodeWorkerHostDeclaration,
} from "../infra/node-runner-inventory.js";
import {
NODE_WORKER_EXECUTION_CONTAINER_V1,
NODE_WORKER_EXECUTION_HOST_V1,
type NodeWorkerExecution,
} from "../worker/node-supervisor-protocol.js";
export type NodeRunnerRegistrySession = {
nodeId: string;
@@ -28,7 +35,10 @@ export type NodeWorkerSupervisorNodeProof = {
pairingGeneration: string;
clientId: typeof GATEWAY_CLIENT_IDS.NODE_HOST;
clientMode: "node";
protocolFeature: typeof NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE;
protocolFeature:
| typeof NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE
| typeof NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE
| typeof NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE;
workerHost: Extract<NodeWorkerHostDeclaration, { enabled: true }>;
commands: readonly string[];
};
@@ -53,7 +63,25 @@ export function sameNodeWorkerHostDeclaration(
left.capacity.available === right.capacity.available &&
left.bundlePrewarm === right.bundlePrewarm &&
left.bundleRetention === right.bundleRetention &&
left.bundleStatus === right.bundleStatus))
left.bundleStatus === right.bundleStatus &&
left.processIsolation?.kind === right.processIsolation?.kind &&
left.processIsolation?.memoryProjection === right.processIsolation?.memoryProjection))
);
}
/** Returns true only for the execution boundary this exact inventory proof attests. */
export function supportsNodeWorkerExecution(
node: NodeWorkerSupervisorNodeProof,
execution: NodeWorkerExecution,
): boolean {
if (execution.kind === NODE_WORKER_EXECUTION_HOST_V1) {
return true;
}
return (
execution.kind === NODE_WORKER_EXECUTION_CONTAINER_V1 &&
node.protocolFeature === NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE &&
node.workerHost.processIsolation?.kind === NODE_WORKER_EXECUTION_CONTAINER_V1 &&
node.workerHost.processIsolation.memoryProjection === 1
);
}
@@ -72,7 +100,12 @@ export function resolveNodeWorkerSupervisorProof(
declaration.pairingIdentity !== node.pairingIdentity ||
declaration.clientId !== node.clientId ||
declaration.clientMode !== node.clientMode ||
!declaration.protocolFeatures.includes(NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE) ||
declaration.protocolFeatures.length !== 1 ||
(declaration.protocolFeatures[0] !== NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE &&
declaration.protocolFeatures[0] !==
NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE &&
declaration.protocolFeatures[0] !==
NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE) ||
declaration.workerHost?.enabled !== true
) {
return undefined;
@@ -84,10 +117,13 @@ export function resolveNodeWorkerSupervisorProof(
pairingGeneration: node.pairingGeneration,
clientId: GATEWAY_CLIENT_IDS.NODE_HOST,
clientMode: "node",
protocolFeature: NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
protocolFeature: declaration.protocolFeatures[0],
workerHost: {
...declaration.workerHost,
capacity: { ...declaration.workerHost.capacity },
...(declaration.workerHost.processIsolation
? { processIsolation: { ...declaration.workerHost.processIsolation } }
: {}),
},
commands: [...node.commands],
};
+19
View File
@@ -37,6 +37,7 @@ import {
classifyGatewayProbePath,
classifyMcpAppStandalonePath,
classifyNodeWorkerBundleTransferPath,
classifyNodeWorkerProjectionTransferPath,
classifyNodeWorkspaceTransferPath,
classifyWorkerGatewayPath,
} from "./gateway-http-route-contracts.js";
@@ -74,6 +75,10 @@ import {
handleNodeWorkerBundleTransferHttpRequest,
type NodeWorkerBundleTransferHttpCallback,
} from "./worker-environments/node-worker-bundle-transfer-http.js";
import {
handleNodeWorkerProjectionTransferHttpRequest,
type NodeWorkerProjectionTransferHttpCallback,
} from "./worker-environments/node-worker-projection-transfer-http.js";
import {
handleNodeWorkspaceTransferHttpRequest,
type NodeWorkspaceTransferHttpCallback,
@@ -184,6 +189,8 @@ export function createGatewayHttpServer(opts: {
handleNodeWorkerBundleTransferRequest?: NodeWorkerBundleTransferHttpCallback;
/** Authenticator/dispatcher for the reserved node workspace transfer namespace. */
handleNodeWorkspaceTransferRequest?: NodeWorkspaceTransferHttpCallback;
/** Authenticator/dispatcher for the reserved node memory-projection transfer namespace. */
handleNodeWorkerProjectionTransferRequest?: NodeWorkerProjectionTransferHttpCallback;
getReadiness?: ReadinessChecker;
getStartup?: StartupChecker;
getRuntimeConfig?: () => OpenClawConfig;
@@ -373,6 +380,18 @@ export function createGatewayHttpServer(opts: {
}),
);
addAdmittedStage(
classifyNodeWorkerProjectionTransferPath(scopedRequestPath) !== "outside",
() =>
handleNodeWorkerProjectionTransferHttpRequest({
req,
res,
clientIp: ingressAttribution.rateLimit.subject.key,
rateLimiter: joinRateLimiter,
callback: opts.handleNodeWorkerProjectionTransferRequest,
}),
);
addAdmittedStage(classifyNodeWorkspaceTransferPath(scopedRequestPath) !== "outside", () =>
handleNodeWorkspaceTransferHttpRequest({
req,
+45
View File
@@ -29,6 +29,9 @@ const removeBackfillDiaryEntries = vi.hoisted(() => vi.fn());
const removeGroundedShortTermCandidates = vi.hoisted(() => vi.fn());
const repairDreamingArtifacts = vi.hoisted(() => vi.fn());
const loadShortTermPromotionDreamingStats = vi.hoisted(() => vi.fn());
const withBrokeredMemoryMaintenance = vi.hoisted(() =>
vi.fn(async (run: () => Promise<unknown>) => await run()),
);
vi.mock("../../config/config.js", () => ({
getRuntimeConfig,
@@ -58,6 +61,10 @@ vi.mock("../../plugins/memory-runtime.js", () => ({
getActiveMemorySearchManagerCore: getMemorySearchManager,
}));
vi.mock("../../plugins/memory-broker-runtime.js", () => ({
withBrokeredMemoryMaintenance,
}));
import { createDoctorHandlers } from "./doctor.js";
const doctorHandlers = createDoctorHandlers({
@@ -1166,6 +1173,44 @@ describe("doctor.memory.status", () => {
});
describe("doctor.memory dream actions", () => {
it("serializes every mutating Doctor path through broker maintenance", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "doctor-memory-maintenance-"));
getRuntimeConfig.mockReset().mockReturnValue({});
resolveDefaultAgentId.mockReset().mockReturnValue("main");
resolveAgentWorkspaceDir.mockReset().mockReturnValue(workspaceDir);
previewGroundedRemMarkdown.mockReset().mockResolvedValue({ scannedFiles: 0, files: [] });
writeBackfillDiaryEntries.mockReset();
removeBackfillDiaryEntries.mockReset().mockResolvedValue({ removed: 0 });
removeGroundedShortTermCandidates.mockReset().mockResolvedValue({ removed: 0 });
repairDreamingArtifacts.mockReset().mockResolvedValue({
changed: false,
archivedDreamsDiary: false,
archivedSessionCorpus: false,
archivedSessionIngestion: false,
warnings: [],
});
dedupeDreamDiaryEntries.mockReset().mockResolvedValue({ removed: 0, kept: 0 });
withBrokeredMemoryMaintenance.mockClear();
withBrokeredMemoryMaintenance.mockImplementation(async (run: () => Promise<unknown>) =>
await run(),
);
try {
for (const method of [
"doctor.memory.backfillDreamDiary",
"doctor.memory.resetDreamDiary",
"doctor.memory.resetGroundedShortTerm",
"doctor.memory.repairDreamingArtifacts",
"doctor.memory.dedupeDreamDiary",
] as const) {
await invokeDoctorMemory(method, vi.fn());
}
expect(withBrokeredMemoryMaintenance).toHaveBeenCalledTimes(5);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("clears grounded-only staged short-term entries without touching the diary", async () => {
resolveAgentWorkspaceDir.mockReturnValue("/tmp/openclaw");
removeGroundedShortTermCandidates.mockResolvedValue({
+92 -83
View File
@@ -25,6 +25,7 @@ import {
resolveMemoryRemDreamingConfig,
} from "../../memory-host-sdk/dreaming.js";
import * as defaultMemoryCoreRuntime from "../../plugin-sdk/memory-core-bundled-runtime.js";
import { withBrokeredMemoryMaintenance } from "../../plugins/memory-broker-runtime.js";
import { getActiveMemorySearchManagerCore } from "../../plugins/memory-runtime.js";
import { normalizeAgentId } from "../../routing/session-key.js";
import { formatError } from "../server-utils.js";
@@ -850,58 +851,58 @@ export const createDoctorHandlers = (
return;
}
const { cfg, agentId, workspaceDir } = target;
const memoryDir = path.join(workspaceDir, "memory");
const sourceFiles = await listWorkspaceDailyFiles(memoryDir);
if (sourceFiles.length === 0) {
const payload = await withBrokeredMemoryMaintenance(async () => {
const memoryDir = path.join(workspaceDir, "memory");
const sourceFiles = await listWorkspaceDailyFiles(memoryDir);
if (sourceFiles.length === 0) {
const dreamDiary = await readDreamDiary(workspaceDir);
return {
agentId,
path: dreamDiary.path,
action: "backfill" as const,
found: dreamDiary.found,
scannedFiles: 0,
written: 0,
replaced: 0,
} satisfies DoctorMemoryDreamActionPayload;
}
const grounded = await memoryCoreRuntime.previewGroundedRemMarkdown({
workspaceDir,
inputPaths: sourceFiles,
});
const remConfig = resolveMemoryRemDreamingConfig({
pluginConfig: resolveMemoryDreamingPluginConfig(cfg),
cfg,
});
const entries = grounded.files
.map((file) => {
const isoDay = extractIsoDayFromPath(file.path);
if (!isoDay) {
return null;
}
return {
isoDay,
sourcePath: file.path,
bodyLines: groundedMarkdownToDiaryLines(file.renderedMarkdown),
};
})
.filter((entry): entry is NonNullable<typeof entry> => entry !== null);
const written = await memoryCoreRuntime.writeBackfillDiaryEntries({
workspaceDir,
entries,
timezone: remConfig.timezone,
});
const dreamDiary = await readDreamDiary(workspaceDir);
const payload: DoctorMemoryDreamActionPayload = {
return {
agentId,
path: dreamDiary.path,
action: "backfill",
found: dreamDiary.found,
scannedFiles: 0,
written: 0,
replaced: 0,
};
respond(true, payload, undefined);
return;
}
const grounded = await memoryCoreRuntime.previewGroundedRemMarkdown({
workspaceDir,
inputPaths: sourceFiles,
scannedFiles: grounded.scannedFiles,
written: written.written,
replaced: written.replaced,
} satisfies DoctorMemoryDreamActionPayload;
});
const remConfig = resolveMemoryRemDreamingConfig({
pluginConfig: resolveMemoryDreamingPluginConfig(cfg),
cfg,
});
const entries = grounded.files
.map((file) => {
const isoDay = extractIsoDayFromPath(file.path);
if (!isoDay) {
return null;
}
return {
isoDay,
sourcePath: file.path,
bodyLines: groundedMarkdownToDiaryLines(file.renderedMarkdown),
};
})
.filter((entry): entry is NonNullable<typeof entry> => entry !== null);
const written = await memoryCoreRuntime.writeBackfillDiaryEntries({
workspaceDir,
entries,
timezone: remConfig.timezone,
});
const dreamDiary = await readDreamDiary(workspaceDir);
const payload: DoctorMemoryDreamActionPayload = {
agentId,
path: dreamDiary.path,
action: "backfill",
found: dreamDiary.found,
scannedFiles: grounded.scannedFiles,
written: written.written,
replaced: written.replaced,
};
respond(true, payload, undefined);
},
"doctor.memory.resetDreamDiary": async ({ respond, context, params }) => {
@@ -910,15 +911,17 @@ export const createDoctorHandlers = (
return;
}
const { agentId, workspaceDir } = target;
const removed = await memoryCoreRuntime.removeBackfillDiaryEntries({ workspaceDir });
const dreamDiary = await readDreamDiary(workspaceDir);
const payload: DoctorMemoryDreamActionPayload = {
agentId,
path: dreamDiary.path,
action: "reset",
found: dreamDiary.found,
removedEntries: removed.removed,
};
const payload = await withBrokeredMemoryMaintenance(async () => {
const removed = await memoryCoreRuntime.removeBackfillDiaryEntries({ workspaceDir });
const dreamDiary = await readDreamDiary(workspaceDir);
return {
agentId,
path: dreamDiary.path,
action: "reset",
found: dreamDiary.found,
removedEntries: removed.removed,
} satisfies DoctorMemoryDreamActionPayload;
});
respond(true, payload, undefined);
},
"doctor.memory.resetGroundedShortTerm": async ({ respond, context, params }) => {
@@ -927,12 +930,14 @@ export const createDoctorHandlers = (
return;
}
const { agentId, workspaceDir } = target;
const removed = await memoryCoreRuntime.removeGroundedShortTermCandidates({ workspaceDir });
const payload: DoctorMemoryDreamActionPayload = {
agentId,
action: "resetGroundedShortTerm",
removedShortTermEntries: removed.removed,
};
const payload = await withBrokeredMemoryMaintenance(async () => {
const removed = await memoryCoreRuntime.removeGroundedShortTermCandidates({ workspaceDir });
return {
agentId,
action: "resetGroundedShortTerm",
removedShortTermEntries: removed.removed,
} satisfies DoctorMemoryDreamActionPayload;
});
respond(true, payload, undefined);
},
"doctor.memory.repairDreamingArtifacts": async ({ respond, context, params }) => {
@@ -941,17 +946,19 @@ export const createDoctorHandlers = (
return;
}
const { agentId, workspaceDir } = target;
const repair = await memoryCoreRuntime.repairDreamingArtifacts({ workspaceDir });
const payload: DoctorMemoryDreamActionPayload = {
agentId,
action: "repairDreamingArtifacts",
changed: repair.changed,
archiveDir: repair.archiveDir,
archivedDreamsDiary: repair.archivedDreamsDiary,
archivedSessionCorpus: repair.archivedSessionCorpus,
archivedSessionIngestion: repair.archivedSessionIngestion,
warnings: repair.warnings,
};
const payload = await withBrokeredMemoryMaintenance(async () => {
const repair = await memoryCoreRuntime.repairDreamingArtifacts({ workspaceDir });
return {
agentId,
action: "repairDreamingArtifacts",
changed: repair.changed,
archiveDir: repair.archiveDir,
archivedDreamsDiary: repair.archivedDreamsDiary,
archivedSessionCorpus: repair.archivedSessionCorpus,
archivedSessionIngestion: repair.archivedSessionIngestion,
warnings: repair.warnings,
} satisfies DoctorMemoryDreamActionPayload;
});
respond(true, payload, undefined);
},
"doctor.memory.dedupeDreamDiary": async ({ respond, context, params }) => {
@@ -960,17 +967,19 @@ export const createDoctorHandlers = (
return;
}
const { agentId, workspaceDir } = target;
const dedupe = await memoryCoreRuntime.dedupeDreamDiaryEntries({ workspaceDir });
const dreamDiary = await readDreamDiary(workspaceDir);
const payload: DoctorMemoryDreamActionPayload = {
agentId,
action: "dedupeDreamDiary",
path: dreamDiary.path,
found: dreamDiary.found,
removedEntries: dedupe.removed,
dedupedEntries: dedupe.removed,
keptEntries: dedupe.kept,
};
const payload = await withBrokeredMemoryMaintenance(async () => {
const dedupe = await memoryCoreRuntime.dedupeDreamDiaryEntries({ workspaceDir });
const dreamDiary = await readDreamDiary(workspaceDir);
return {
agentId,
action: "dedupeDreamDiary",
path: dreamDiary.path,
found: dreamDiary.found,
removedEntries: dedupe.removed,
dedupedEntries: dedupe.removed,
keptEntries: dedupe.kept,
} satisfies DoctorMemoryDreamActionPayload;
});
respond(true, payload, undefined);
},
});
@@ -166,6 +166,7 @@ export async function prepareGatewayKernelState(params: {
bindDeviceNodeControl,
bindNodeWorkspaceBindingResolver,
handleNodeWorkerBundleTransferRequest,
handleNodeWorkerProjectionTransferRequest,
handleNodeWorkspaceTransferRequest,
} = workerEnvironmentRuntime;
// Assigned once approval managers exist; placement dispatch must not run before then.
@@ -461,6 +462,7 @@ export async function prepareGatewayKernelState(params: {
handleWatchNodeRequest: async (req: IncomingMessage, res: ServerResponse) =>
(await watchNodeRequestHandler.current?.(req, res)) ?? false,
handleNodeWorkerBundleTransferRequest,
handleNodeWorkerProjectionTransferRequest,
handleNodeWorkspaceTransferRequest,
workerIngressEnabled: Boolean(workerEnvironmentService),
desktopSessionRegistry,
@@ -850,7 +850,9 @@ describe("startGatewayPostAttachRuntime", () => {
},
};
const startGatewaySidecars = vi.fn(async () => {
expect(hoisted.startBrokeredMemoryRuntimeSupervisor).toHaveBeenCalledWith(selectedCapability);
expect(hoisted.startBrokeredMemoryRuntimeSupervisor).toHaveBeenCalledWith(selectedCapability, {
agentIds: ["main"],
});
expect(unavailableGatewayMethods).toEqual(new Set(STARTUP_UNAVAILABLE_GATEWAY_METHODS));
return { pluginServices: null, postReadySidecars: [] };
});
+8 -3
View File
@@ -1345,9 +1345,14 @@ export async function startGatewayPostAttachRuntime(
// than becoming a first-request outage after agents have begun work.
const memoryBrokerSupervisor = selectedMemory?.capability.broker
? await measureStartup(params.startupTrace, "memory-broker.ready", async () => {
const { startBrokeredMemoryRuntimeSupervisor } =
await loadMemoryBrokerRuntimeModule();
return await startBrokeredMemoryRuntimeSupervisor(selectedMemory.capability);
const [{ startBrokeredMemoryRuntimeSupervisor }, { listAgentIds }] =
await Promise.all([
loadMemoryBrokerRuntimeModule(),
import("../agents/agent-scope.js"),
]);
return await startBrokeredMemoryRuntimeSupervisor(selectedMemory.capability, {
agentIds: listAgentIds(params.gatewayPluginConfigAtStart),
});
})
: undefined;
if (params.isClosing?.()) {
@@ -1,10 +1,20 @@
import fs from "node:fs/promises";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
GATEWAY_CLIENT_IDS,
GATEWAY_CLIENT_MODES,
} from "../../packages/gateway-protocol/src/client-info.js";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE } from "../infra/node-runner-inventory.js";
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
import { withEnvAsync } from "../test-utils/env.js";
import { NODE_WORKER_EXECUTION_CONTAINER_V1 } from "../worker/node-supervisor-protocol.js";
import { createDesktopSessionRegistry } from "./desktop/session-registry.js";
import type {
NodeWorkerSupervisorNodeProof,
NodeWorkerSupervisorTransport,
} from "./node-registry-private.js";
import {
createGatewayWorkerEnvironmentRuntime,
loadGatewayWorkerEnvironmentStartupState,
@@ -15,6 +25,25 @@ import {
reconcileDeviceWorker,
} from "./worker-environments/device-provider.js";
const projectionTransferOptions = vi.hoisted(() => ({
isNodeCurrent: undefined as ((node: NodeWorkerSupervisorNodeProof) => boolean) | undefined,
}));
vi.mock("./worker-environments/node-worker-projection-transfer-service.js", async (importOriginal) => {
const actual = await importOriginal<
typeof import("./worker-environments/node-worker-projection-transfer-service.js")
>();
return {
...actual,
createNodeWorkerProjectionTransferService: (
options: Parameters<typeof actual.createNodeWorkerProjectionTransferService>[0],
) => {
projectionTransferOptions.isNodeCurrent = options.isNodeCurrent;
return actual.createNodeWorkerProjectionTransferService(options);
},
};
});
const DEVICE_ID = "revoked-device";
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
@@ -23,6 +52,69 @@ afterEach(() => {
});
describe("gateway worker environment startup", () => {
it("keeps an issued projection bound to its capacity-one node after its slot is claimed", async () => {
const stateDir = tempDirs.make("openclaw-worker-projection-capacity-");
await withEnvAsync({ OPENCLAW_STATE_DIR: stateDir }, async () => {
const startup = await loadGatewayWorkerEnvironmentStartupState();
const runtime = await createGatewayWorkerEnvironmentRuntime({
getPluginRegistry: () => ({ workerProviders: new Map() }),
desktopSessionRegistry: createDesktopSessionRegistry({ lingerMs: 1 }),
startup,
log: { child: () => ({ warn: () => {} }) },
});
const service = runtime.workerEnvironmentService;
if (!service || !runtime.bindDeviceNodeControl || !projectionTransferOptions.isNodeCurrent) {
throw new Error("worker projection runtime was not created");
}
const node: NodeWorkerSupervisorNodeProof = {
nodeId: "node-1",
connId: "conn-1",
pairingIdentity: "pairing-1",
pairingGeneration: "generation-1",
clientId: GATEWAY_CLIENT_IDS.NODE_HOST,
clientMode: GATEWAY_CLIENT_MODES.NODE,
protocolFeature: NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
workerHost: {
enabled: true,
capacity: { total: 1, available: 1 },
processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, memoryProjection: 1 },
},
commands: [],
};
let available = 1;
let connId = node.connId;
let pairingGeneration = node.pairingGeneration;
const transport = {
listCurrentNodes: async () => [node],
isCurrent: vi.fn(
(candidate: NodeWorkerSupervisorNodeProof, requireLaunchEligibility = false) =>
candidate.nodeId === node.nodeId &&
candidate.connId === connId &&
candidate.pairingGeneration === pairingGeneration &&
(!requireLaunchEligibility || available > 0),
),
invoke: async () => ({ ok: false }),
} satisfies NodeWorkerSupervisorTransport;
runtime.bindDeviceNodeControl(transport);
const isNodeCurrent = projectionTransferOptions.isNodeCurrent;
try {
expect(isNodeCurrent(node)).toBe(true);
available = 0;
expect(isNodeCurrent(node)).toBe(true);
expect(transport.isCurrent).toHaveBeenLastCalledWith(node, false);
connId = "conn-replaced";
expect(isNodeCurrent(node)).toBe(false);
connId = node.connId;
pairingGeneration = "generation-replaced";
expect(isNodeCurrent(node)).toBe(false);
} finally {
await service.stop();
}
});
});
it("cleans transfer scratch before serving and removes it on shutdown", async () => {
const stateDir = tempDirs.make("openclaw-worker-transfer-startup-");
const transferRoot = path.join(stateDir, "tmp", "node-workspace-transfer");
@@ -2,7 +2,7 @@ import { isRecord } from "@openclaw/normalization-core/record-coerce";
import { uniqueStrings } from "@openclaw/normalization-core/string-normalization";
import { getRuntimeConfig } from "../config/config.js";
import { loadOrCreateProcessDeviceIdentity } from "../infra/device-identity.js";
import { getPairedDevice } from "../infra/device-pairing.js";
import { getPairedDevice, resolveNodePairingState } from "../infra/device-pairing.js";
import type { PluginRegistry } from "../plugins/registry-types.js";
import {
getActiveSecretsRuntimeConfigSnapshot,
@@ -14,6 +14,7 @@ import type { NodeWorkerSupervisorTransport } from "./node-registry-private.js";
import type { WorkerBundleProducer, WorkerNpmArtifact } from "./worker-environments/bundle.js";
import {
bindDeviceWorkerAvailability,
bindDeviceWorkerExecutionEligibility,
bindDeviceWorkerReconciliation,
createDeviceWorkerRuntime,
DEVICE_WORKER_PROVIDER_ID,
@@ -21,6 +22,7 @@ import {
import type { WorkerLiveEventReceiver } from "./worker-environments/live-events.js";
import { createWorkerNodeEnrollmentManager } from "./worker-environments/node-enrollment.js";
import type { NodeWorkerBundleTransferHttpCallback } from "./worker-environments/node-worker-bundle-transfer-http.js";
import type { NodeWorkerProjectionTransferHttpCallback } from "./worker-environments/node-worker-projection-transfer-http.js";
import { nodeWorkerGatewayNamespace as resolveNodeWorkerGatewayNamespace } from "./worker-environments/node-worker-gateway-namespace.js";
import type { NodeWorkerWorkspaceBindingResolver } from "./worker-environments/node-worker-tunnel.js";
import type { NodeWorkspaceTransferHttpCallback } from "./worker-environments/node-workspace-transfer-http-contract.js";
@@ -56,6 +58,7 @@ export type GatewayWorkerEnvironmentRuntime = {
bindDeviceNodeControl?: (transport: NodeWorkerSupervisorTransport) => void;
bindNodeWorkspaceBindingResolver?: (resolver: NodeWorkerWorkspaceBindingResolver) => void;
handleNodeWorkerBundleTransferRequest?: NodeWorkerBundleTransferHttpCallback;
handleNodeWorkerProjectionTransferRequest?: NodeWorkerProjectionTransferHttpCallback;
handleNodeWorkspaceTransferRequest?: NodeWorkspaceTransferHttpCallback;
};
@@ -119,6 +122,8 @@ export async function createGatewayWorkerEnvironmentRuntime(params: {
{ createGatewayNodeWorkerBundleInstaller },
{ createNodeWorkerBundleTransferService },
{ createNodeWorkerBundleTransferHttpCallback },
{ createNodeWorkerProjectionTransferService },
{ createNodeWorkerProjectionTransferHttpCallback },
{ createNodeWorkspaceTransferService },
{ createNodeWorkspaceTransferHttpCallback },
{ createWorkerSessionToolExecutor },
@@ -133,6 +138,8 @@ export async function createGatewayWorkerEnvironmentRuntime(params: {
import("./worker-environments/node-worker-bundle-installer.js"),
import("./worker-environments/node-worker-bundle-transfer-service.js"),
import("./worker-environments/node-worker-bundle-transfer-http.js"),
import("./worker-environments/node-worker-projection-transfer-service.js"),
import("./worker-environments/node-worker-projection-transfer-http.js"),
import("./worker-environments/node-workspace-transfer-service.js"),
import("./worker-environments/node-workspace-transfer-http.js"),
import("./worker-environments/worker-session-tool-executor.js"),
@@ -204,6 +211,20 @@ export async function createGatewayWorkerEnvironmentRuntime(params: {
desktopSessionRegistry: params.desktopSessionRegistry,
});
const nodeWorkerBundleTransfer = createNodeWorkerBundleTransferService();
let nodeWorkerSupervisorTransport: NodeWorkerSupervisorTransport | undefined;
const nodeWorkerProjectionTransfer = createNodeWorkerProjectionTransferService({
resolveNodePublicKey: async (node) => {
const paired = await getPairedDevice(node.nodeId);
const currentPairing = resolveNodePairingState(paired);
return currentPairing?.identity.key === node.pairingIdentity &&
currentPairing.generation?.key === node.pairingGeneration
? paired?.publicKey
: undefined;
},
// The node claims its launch slot before it fetches this already-issued projection.
// Keep exact connection/pairing proof, but do not reject a capacity-one node's own fetch.
isNodeCurrent: (node) => nodeWorkerSupervisorTransport?.isCurrent(node) === true,
});
const nodeWorkspaceTransfer = createNodeWorkspaceTransferService({
getOwner: (environmentId) => params.startup.store.getTransferOwner(environmentId),
});
@@ -216,6 +237,7 @@ export async function createGatewayWorkerEnvironmentRuntime(params: {
getTransport: () => deviceRuntime.getNodeTransport(),
launchNodeWorker: async (request) => await deviceRuntime.launchNodeWorker(request),
validateWorkerTurn: (binding) => placementGate.validateWorkerTurn(binding),
projectionTransfer: nodeWorkerProjectionTransfer,
workspaceTransfer: nodeWorkspaceTransfer,
});
const ensureNodeWorkerBundle = createGatewayNodeWorkerBundleInstaller({
@@ -255,7 +277,10 @@ export async function createGatewayWorkerEnvironmentRuntime(params: {
retireNodeEnrollment: nodeEnrollment.retire,
tunnelManager: workerTunnelManager,
nodeTunnelManager: nodeWorkerTunnelManager,
stopNodeWorkerBundleTransfers: () => nodeWorkerBundleTransfer.closeAll(),
stopNodeWorkerBundleTransfers: () => {
nodeWorkerBundleTransfer.closeAll();
nodeWorkerProjectionTransfer.closeAll();
},
applyTranscriptCommit: createWorkerTranscriptCommitter({
getConfig: getRuntimeConfig,
}).commit,
@@ -304,6 +329,10 @@ export async function createGatewayWorkerEnvironmentRuntime(params: {
});
const workerEnvironmentService = workerEnvironmentServiceBase;
bindDeviceWorkerAvailability(workerEnvironmentService, deviceRuntime.resolveAvailability);
bindDeviceWorkerExecutionEligibility(
workerEnvironmentService,
deviceRuntime.assertNodeWorkerExecutionEligible,
);
bindDeviceWorkerReconciliation(workerEnvironmentService, async (deviceId) => {
const environmentIds = params.startup.store
.listForReconcile()
@@ -344,11 +373,16 @@ export async function createGatewayWorkerEnvironmentRuntime(params: {
bindWorkerSessionDispatch: (dispatch) => {
dispatchChild = dispatch;
},
bindDeviceNodeControl: deviceRuntime.bindNodeTransport,
bindDeviceNodeControl: (transport) => {
nodeWorkerSupervisorTransport = transport;
deviceRuntime.bindNodeTransport(transport);
},
bindNodeWorkspaceBindingResolver: (resolver) =>
nodeWorkerTunnelManager.bindWorkspaceBindingResolver(resolver),
handleNodeWorkerBundleTransferRequest:
createNodeWorkerBundleTransferHttpCallback(nodeWorkerBundleTransfer),
handleNodeWorkerProjectionTransferRequest:
createNodeWorkerProjectionTransferHttpCallback(nodeWorkerProjectionTransfer),
handleNodeWorkspaceTransferRequest:
createNodeWorkspaceTransferHttpCallback(nodeWorkspaceTransfer),
};
@@ -115,10 +115,12 @@ type WorkerInferenceConnectionService = WorkerConnectionService & {
searchMemory?: (
identity: WorkerConnectionIdentity,
request: WorkerMemorySearchParams,
signal?: AbortSignal,
) => Promise<WorkerServiceResult<unknown, { reason: string }>>;
readMemory?: (
identity: WorkerConnectionIdentity,
request: WorkerMemoryReadParams,
signal?: AbortSignal,
) => Promise<WorkerServiceResult<unknown, { reason: string }>>;
startInference?: (
identity: WorkerConnectionIdentity,
@@ -255,7 +257,7 @@ async function dispatchWorkerRequest(params: {
rejectWorkerRequest({ ...params, reason: "method-not-allowed" });
return;
}
const outcome = await service.searchMemory(params.identity, params.request.params);
const outcome = await service.searchMemory(params.identity, params.request.params, params.signal);
if (outcome.ok) {
params.respond(true, outcome.result);
return;
@@ -284,7 +286,7 @@ async function dispatchWorkerRequest(params: {
rejectWorkerRequest({ ...params, reason: "method-not-allowed" });
return;
}
const outcome = await service.readMemory(params.identity, params.request.params);
const outcome = await service.readMemory(params.identity, params.request.params, params.signal);
if (outcome.ok) {
params.respond(true, outcome.result);
return;
@@ -425,6 +427,7 @@ export function attachWorkerWsMessageHandler(params: WorkerWsMessageHandlerParam
let expiryTimer: ReturnType<typeof setTimeout> | undefined;
let disposed = false;
const sessionOperations = new Set<string>();
const memoryOperations = new Map<string, AbortController>();
const cleanup = () => {
if (disposed) {
return;
@@ -432,6 +435,10 @@ export function attachWorkerWsMessageHandler(params: WorkerWsMessageHandlerParam
disposed = true;
clearTimeout(expiryTimer);
sessionOperations.clear();
for (const controller of memoryOperations.values()) {
controller.abort(new Error("worker memory RPC connection closed"));
}
memoryOperations.clear();
params.socket.off("message", onMessage);
};
const closeWorker = (code: number, reason: WorkerProtocolCloseReason) => {
@@ -695,6 +702,24 @@ export function attachWorkerWsMessageHandler(params: WorkerWsMessageHandlerParam
});
return;
}
const isMemoryOperation =
parsed.method === WORKER_MEMORY_METHODS[0] || parsed.method === WORKER_MEMORY_METHODS[1];
if (isMemoryOperation) {
if (memoryOperations.has(parsed.id)) {
failFrame(1008, "invalid-frame");
return;
}
const controller = new AbortController();
memoryOperations.set(parsed.id, controller);
try {
await dispatch(controller.signal);
} finally {
if (memoryOperations.get(parsed.id) === controller) {
memoryOperations.delete(parsed.id);
}
}
return;
}
await dispatch();
};
@@ -10,6 +10,7 @@ import {
type WorkerProfile,
type WorkerProvider,
} from "../../plugins/types.js";
import type { NodeWorkerExecution } from "../../worker/node-supervisor-protocol.js";
import type {
NodeWorkerSupervisorNodeProof,
NodeWorkerSupervisorTransport,
@@ -31,8 +32,14 @@ export type DeviceWorkerAvailability = {
};
type DeviceWorkerAvailabilityResolver = (deviceId: string) => Promise<DeviceWorkerAvailability>;
type DeviceWorkerReconciliation = (deviceId: string) => Promise<readonly string[]>;
type DeviceWorkerExecutionEligibility = (params: {
deviceId: string;
execution: NodeWorkerExecution;
signal?: AbortSignal;
}) => Promise<void>;
const DEVICE_WORKER_AVAILABILITY = new WeakMap<object, DeviceWorkerAvailabilityResolver>();
const DEVICE_WORKER_RECONCILIATION = new WeakMap<object, DeviceWorkerReconciliation>();
const DEVICE_WORKER_EXECUTION_ELIGIBILITY = new WeakMap<object, DeviceWorkerExecutionEligibility>();
export function bindDeviceWorkerAvailability(
service: object,
@@ -80,6 +87,36 @@ export async function reconcileDeviceWorker(
return reconcile ? await reconcile(deviceId) : [];
}
export function bindDeviceWorkerExecutionEligibility(
service: object,
assertEligible: DeviceWorkerExecutionEligibility,
): void {
DEVICE_WORKER_EXECUTION_ELIGIBILITY.set(service, assertEligible);
}
/**
* Enforced launches must prove the node boundary before any turn credential,
* tunnel, or workspace operation is created.
*/
export async function assertDeviceWorkerExecutionEligibility(params: {
service: object | undefined;
deviceId: string;
execution: NodeWorkerExecution;
signal?: AbortSignal;
}): Promise<void> {
const assertEligible = params.service
? DEVICE_WORKER_EXECUTION_ELIGIBILITY.get(params.service)
: undefined;
if (!assertEligible) {
throw new WorkerProviderError("device worker process-isolation eligibility is unavailable");
}
await assertEligible({
deviceId: params.deviceId,
execution: params.execution,
...(params.signal ? { signal: params.signal } : {}),
});
}
function requireDeviceId(profile: WorkerProfile): string {
const deviceId = profile.device;
if (typeof deviceId !== "string" || !deviceId.trim()) {
@@ -170,6 +207,7 @@ export function createDeviceWorkerRuntime(options: DeviceWorkerRuntimeOptions) {
return {
provider,
resolveAvailability,
assertNodeWorkerExecutionEligible: launchAdapter.assertExecutionEligible,
launchNodeWorker: launchAdapter.launch,
getNodeTransport: () => nodeTransport,
bindNodeTransport: (transport: NodeWorkerSupervisorTransport) => {
@@ -8,9 +8,17 @@ import {
WORKER_RPC_SET_VERSION,
} from "../../../packages/gateway-protocol/src/schema/worker-admission.js";
import { NODE_WORKER_CAPACITY_EXHAUSTED_ERROR_CODE } from "../../infra/node-commands.js";
import { NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE } from "../../infra/node-runner-inventory.js";
import {
NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE,
NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
} from "../../infra/node-runner-inventory.js";
import {
NODE_WORKER_EXECUTION_CONTAINER_V1,
NODE_WORKER_EXECUTION_HOST_V1,
nodeWorkerMemoryProjectionLaunchBinding,
nodeWorkerPlanHash,
type NodeWorkerExecution,
type NodeWorkerLaunchInput,
type NodeWorkerSupervisorReceipt,
} from "../../worker/node-supervisor-protocol.js";
@@ -27,7 +35,12 @@ const WORKER_RUNS = {
protocolFeatures: [...WORKER_PROTOCOL_FEATURES],
};
function nodeProof(connId = "conn-1", available = 2): NodeWorkerSupervisorNodeProof {
function nodeProof(
connId = "conn-1",
available = 2,
execution: NodeWorkerExecution = { kind: NODE_WORKER_EXECUTION_HOST_V1 },
): NodeWorkerSupervisorNodeProof {
const container = execution.kind === NODE_WORKER_EXECUTION_CONTAINER_V1;
return {
nodeId: DEVICE_ID,
connId,
@@ -35,8 +48,16 @@ function nodeProof(connId = "conn-1", available = 2): NodeWorkerSupervisorNodePr
pairingGeneration: "generation-1",
clientId: GATEWAY_CLIENT_IDS.NODE_HOST,
clientMode: GATEWAY_CLIENT_MODES.NODE,
protocolFeature: NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
workerHost: { enabled: true, capacity: { total: 2, available } },
protocolFeature: container
? NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE
: NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
workerHost: {
enabled: true,
capacity: { total: 2, available },
...(container
? { processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, memoryProjection: 1 } }
: {}),
},
commands: ["system.run"],
};
}
@@ -47,6 +68,7 @@ function launchInput(): NodeWorkerLaunchInput {
gatewayNamespace: "gateway-1",
expectedBundleHash: WORKER_RUNS.bundleHash,
placementGeneration: 4,
execution: { kind: NODE_WORKER_EXECUTION_HOST_V1 },
descriptor: {
version: 4,
admission: {
@@ -59,6 +81,7 @@ function launchInput(): NodeWorkerLaunchInput {
},
assignment: {
agentId: "agent-1",
memoryReadEnforced: false,
operationalRunInstance: { instanceId: "instance-1", runId: "run-1" },
agentRuntimeIdentityToken: "signed-runtime-token",
runId: "run-1",
@@ -80,6 +103,7 @@ function launchInput(): NodeWorkerLaunchInput {
function receipt(
input: NodeWorkerLaunchInput,
state: NodeWorkerSupervisorReceipt["state"],
executionStarted = true,
): NodeWorkerSupervisorReceipt {
const identity = {
launchId: input.launchId,
@@ -102,7 +126,7 @@ function receipt(
};
}
if (state === "failed" || state === "interrupted" || state === "cancelled") {
return { ...identity, state, errorText: `worker ${state}` };
return { ...identity, state, errorText: `worker ${state}`, executionStarted };
}
return { ...identity, state };
}
@@ -128,6 +152,18 @@ function launchRequest(input = launchInput()) {
};
}
function issuedProjection(input: Omit<NodeWorkerLaunchInput, "memoryProjection">, reference: string) {
return {
version: 1 as const,
reference,
binding: {
launch: nodeWorkerMemoryProjectionLaunchBinding(input),
authorization: "b".repeat(64),
},
expiresAtMs: Date.now() + 60_000,
};
}
describe("node worker launch adapter", () => {
it("fails with a typed availability result when no node dispatches within the grace", async () => {
vi.useFakeTimers();
@@ -172,6 +208,70 @@ describe("node worker launch adapter", () => {
]);
});
it("reports execution readiness only after the durable worker receipt", async () => {
const input = launchInput();
const callbacks: string[] = [];
let statusCalls = 0;
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async (request) => {
if (request.command === "worker.launch.v1") {
request.onDispatchReady?.("invoke-1");
return wire(receipt(input, "pending"));
}
statusCalls += 1;
return wire(receipt(input, statusCalls === 1 ? "running" : "completed"));
});
const adapter = createNodeWorkerLaunchAdapter({
getTransport: () => transportWith(invoke),
sleep: async () => {},
});
const launched = adapter.launch({
...launchRequest(input),
onDispatchReady: () => callbacks.push("dispatch"),
onExecutionReady: () => callbacks.push("execution"),
});
await expect(launched).resolves.toEqual(receipt(input, "completed"));
expect(callbacks).toEqual(["dispatch", "execution"]);
});
it("does not report execution readiness when a launch terminates before child acknowledgement", async () => {
const input = launchInput();
const callbacks: string[] = [];
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async (request) => {
request.onDispatchReady?.("invoke-1");
return wire(receipt(input, "cancelled", false));
});
const adapter = createNodeWorkerLaunchAdapter({
getTransport: () => transportWith(invoke),
});
await expect(
adapter.launch({
...launchRequest(input),
onDispatchReady: () => callbacks.push("dispatch"),
onExecutionReady: () => callbacks.push("execution"),
}),
).resolves.toEqual(receipt(input, "cancelled", false));
expect(callbacks).toEqual(["dispatch"]);
});
it("rejects a container launch on a v5 node before invoking the supervisor", async () => {
const input = {
...launchInput(),
execution: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 } as const,
};
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>();
const adapter = createNodeWorkerLaunchAdapter({
getTransport: () => transportWith(invoke),
});
await expect(adapter.launch(launchRequest(input))).rejects.toMatchObject({
code: "PROCESS_ISOLATION_UNAVAILABLE",
});
expect(invoke).not.toHaveBeenCalled();
});
it("reacquires the node and replays the identical launch after ambiguous disconnect", async () => {
const input = launchInput();
let launchCalls = 0;
@@ -272,6 +372,101 @@ describe("node worker launch adapter", () => {
]);
});
it("keeps status and cancellation bound to the prepared container connection after inventory changes", async () => {
const base = {
...launchInput(),
execution: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 } as const,
descriptor: {
...launchInput().descriptor,
assignment: {
...launchInput().descriptor.assignment,
memoryReadEnforced: true,
workspaceDir: "/workspace",
},
},
};
const input = { ...base, memoryProjection: issuedProjection(base, "a".repeat(43)) };
const controller = new AbortController();
let discoveryCount = 0;
let sleepCount = 0;
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async (request) => {
if (request.command === "worker.launch.v1") {
return wire(receipt(input, "running"));
}
if (request.command === "worker.status.v1") {
return wire(receipt(input, "running"));
}
return wire(receipt(input, "cancelled"));
});
const adapter = createNodeWorkerLaunchAdapter({
getTransport: () =>
transportWith(invoke, async () => [
discoveryCount++ <= 1
? nodeProof("conn-container", 2, input.execution)
: nodeProof("conn-container"),
]),
sleep: async () => {
if (++sleepCount === 2) {
controller.abort();
}
},
});
await expect(
adapter.launch({
...launchRequest(input),
prepareMemoryProjection: async () => input.memoryProjection,
signal: controller.signal,
}),
).resolves.toEqual(receipt(input, "cancelled"));
expect(invoke.mock.calls.map(([request]) => request.command)).toEqual([
"worker.launch.v1",
"worker.status.v1",
"worker.cancel.v1",
]);
expect(invoke.mock.calls.slice(1).map(([request]) => request.node.connId)).toEqual([
"conn-container",
"conn-container",
]);
});
it("does not replay, poll, or cancel an issued projection through a replacement node connection", async () => {
const base = {
...launchInput(),
execution: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 } as const,
descriptor: {
...launchInput().descriptor,
assignment: {
...launchInput().descriptor.assignment,
memoryReadEnforced: true,
workspaceDir: "/workspace",
},
},
};
const input = { ...base, memoryProjection: issuedProjection(base, "b".repeat(43)) };
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async () =>
wire(receipt(input, "running")),
);
let discoveryCount = 0;
const adapter = createNodeWorkerLaunchAdapter({
getTransport: () =>
transportWith(invoke, async () => [
discoveryCount++ <= 1
? nodeProof("conn-issued", 2, input.execution)
: nodeProof("conn-replacement"),
]),
sleep: async () => {},
});
await expect(
adapter.launch({
...launchRequest(input),
prepareMemoryProjection: async () => input.memoryProjection,
}),
).rejects.toThrow("cancellation could not be confirmed");
expect(invoke.mock.calls.map(([request]) => request.command)).toEqual(["worker.launch.v1"]);
});
it("replays launch when status cannot find the durable receipt", async () => {
const input = launchInput();
const responses = [
@@ -9,10 +9,13 @@ import {
nodeWorkerPlanHash,
parseNodeWorkerLaunchInput,
parseNodeWorkerSupervisorReceipt,
type NodeWorkerExecution,
type NodeWorkerLaunchInput,
type NodeWorkerSupervisorIdentity,
type NodeWorkerSupervisorReceipt,
} from "../../worker/node-supervisor-protocol.js";
import type { NodeWorkerMemoryProjection } from "../../worker/node-memory-projection-protocol.js";
import { supportsNodeWorkerExecution } from "../node-runner-inventory-runtime.js";
import type {
NodeWorkerSupervisorNodeProof,
NodeWorkerSupervisorTransport,
@@ -43,11 +46,20 @@ type TerminalNodeWorkerSupervisorReceipt = Extract<
type DeviceWorkerLaunchRequest = {
deviceId: string;
input: NodeWorkerLaunchInput;
/** Mints the opaque projection only after this adapter has proved the exact node connection. */
prepareMemoryProjection?: (
node: NodeWorkerSupervisorNodeProof,
) => Promise<NodeWorkerMemoryProjection>;
isDispatchAuthorized: () => boolean;
isCancellationAuthorized: () => boolean;
timeoutMs: number;
signal?: AbortSignal;
onDispatchReady?: () => void;
onExecutionReady?: () => void;
};
type PreparedDeviceWorkerLaunchRequest = DeviceWorkerLaunchRequest & {
boundNode?: NodeWorkerSupervisorNodeProof;
};
type NodeWorkerLaunchAdapterOptions = {
@@ -126,6 +138,18 @@ function receiptMatchesIdentity(
);
}
function sameNodeConnection(
left: NodeWorkerSupervisorNodeProof,
right: NodeWorkerSupervisorNodeProof,
): boolean {
return (
left.nodeId === right.nodeId &&
left.connId === right.connId &&
left.pairingIdentity === right.pairingIdentity &&
left.pairingGeneration === right.pairingGeneration
);
}
function parseInvokeReceipt(
payloadJSON: string | null | undefined,
): NodeWorkerSupervisorReceipt | null {
@@ -209,7 +233,8 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
const findNode = async (params: {
transport: NodeWorkerSupervisorTransport;
deviceId: string;
requireLaunchAvailability?: boolean;
execution: NodeWorkerExecution;
requireLaunchEligibility: boolean;
signal: AbortSignal;
}): Promise<NodeWorkerSupervisorNodeProof> => {
let nodes: readonly NodeWorkerSupervisorNodeProof[];
@@ -224,17 +249,22 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
"device worker node discovery is unavailable",
);
}
const node = nodes.find(
(candidate) =>
candidate.nodeId === params.deviceId &&
(!params.requireLaunchAvailability || candidate.workerHost.capacity.available > 0),
);
const node = nodes.find((candidate) => candidate.nodeId === params.deviceId);
if (!node) {
throw new NodeWorkerLaunchTransportError(
"NOT_CONNECTED",
"device worker node is not currently connected",
);
}
if (params.requireLaunchEligibility && !supportsNodeWorkerExecution(node, params.execution)) {
throw new NodeWorkerLaunchTransportError(
"PROCESS_ISOLATION_UNAVAILABLE",
"device worker node does not attest to the required process-isolation boundary",
);
}
if (params.requireLaunchEligibility && node.workerHost.capacity.available <= 0) {
throw new WorkerRunnerCapacityError();
}
return node;
};
@@ -245,9 +275,10 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
| typeof NODE_WORKER_SUPERVISOR_STATUS_COMMAND
| typeof NODE_WORKER_SUPERVISOR_CANCEL_COMMAND;
payload: unknown;
requireLaunchAvailability?: boolean;
execution: NodeWorkerExecution;
isAuthorized: () => boolean;
deadline: OperationDeadline;
boundNode?: NodeWorkerSupervisorNodeProof;
onDispatchReady?: () => void;
}): Promise<NodeWorkerSupervisorReceipt | null> => {
if (!params.isAuthorized()) {
@@ -284,9 +315,18 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
const node = await findNode({
transport,
deviceId: params.deviceId,
requireLaunchAvailability: params.requireLaunchAvailability,
execution: params.execution,
requireLaunchEligibility: params.command === NODE_WORKER_SUPERVISOR_LAUNCH_COMMAND,
signal,
});
if (params.boundNode && !sameNodeConnection(node, params.boundNode)) {
// A projection bearer is bound to this exact paired connection. Retrying against a
// replacement node would silently turn a stale capability into a confused deputy.
throw new NodeWorkerLaunchTransportError(
"NODE_IDENTITY_CHANGED",
"device worker node identity changed after memory projection preparation",
);
}
const operation = transport.invoke({
node,
command: params.command,
@@ -352,7 +392,7 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
};
const cancelUntilTerminal = async (params: {
request: DeviceWorkerLaunchRequest;
request: PreparedDeviceWorkerLaunchRequest;
expected: NodeWorkerSupervisorIdentity;
}): Promise<TerminalNodeWorkerSupervisorReceipt> => {
const deadline = createDeadline({
@@ -371,8 +411,10 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
deviceId: params.request.deviceId,
command: NODE_WORKER_SUPERVISOR_CANCEL_COMMAND,
payload: params.expected,
execution: params.request.input.execution,
isAuthorized: params.request.isCancellationAuthorized,
deadline,
...(params.request.boundNode ? { boundNode: params.request.boundNode } : {}),
});
if (receipt) {
const validated = validateReceipt(receipt, params.expected);
@@ -403,9 +445,6 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
const launch = async (
request: DeviceWorkerLaunchRequest,
): Promise<TerminalNodeWorkerSupervisorReceipt> => {
const input = snapshotLaunchInput(request.input);
const stableRequest = { ...request, input };
const expected = expectedIdentity(input);
const deadline = createDeadline({
now,
timeoutMs: request.timeoutMs,
@@ -418,8 +457,12 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
signal: deadline.signal,
label: "node worker availability",
});
let input: NodeWorkerLaunchInput;
let expected: NodeWorkerSupervisorIdentity | undefined;
let stableRequest: PreparedDeviceWorkerLaunchRequest | undefined;
let mayHaveLaunched = false;
let dispatchReady = false;
let executionReady = false;
let pollStatus = false;
let delayMs = pollIntervalMs;
const markDispatchReady = () => {
@@ -429,7 +472,47 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
stableRequest.onDispatchReady?.();
}
};
const markExecutionReady = (receipt: NodeWorkerSupervisorReceipt) => {
if (
executionReady ||
!(
receipt.state === "running" ||
receipt.state === "completed" ||
(isTerminalReceipt(receipt) && receipt.executionStarted)
)
) {
return;
}
executionReady = true;
stableRequest?.onExecutionReady?.();
};
try {
if (request.prepareMemoryProjection) {
const transport = options.getTransport();
if (!transport) {
throw new NodeWorkerLaunchTransportError(
"UNAVAILABLE",
"device worker node transport is unavailable",
);
}
const node = await findNode({
transport,
deviceId: request.deviceId,
execution: request.input.execution,
requireLaunchEligibility: true,
signal: availabilityDeadline.signal,
});
const memoryProjection = await raceWithSignal(
request.prepareMemoryProjection(node),
availabilityDeadline.signal,
);
input = snapshotLaunchInput({ ...request.input, memoryProjection });
stableRequest = { ...request, input, boundNode: node };
} else {
input = snapshotLaunchInput(request.input);
stableRequest = { ...request, input };
}
expected = expectedIdentity(input);
while (true) {
if (deadline.signal.aborted) {
throw signalError(deadline.signal, "node worker launch aborted");
@@ -448,9 +531,10 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
? NODE_WORKER_SUPERVISOR_STATUS_COMMAND
: NODE_WORKER_SUPERVISOR_LAUNCH_COMMAND,
payload: pollStatus ? { launchId: input.launchId } : input,
...(!pollStatus ? { requireLaunchAvailability: true } : {}),
execution: input.execution,
isAuthorized: stableRequest.isDispatchAuthorized,
deadline: attemptDeadline,
...(stableRequest.boundNode ? { boundNode: stableRequest.boundNode } : {}),
...(!pollStatus
? {
onDispatchReady: markDispatchReady,
@@ -465,6 +549,7 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
}
const validated = validateReceipt(receipt, expected);
mayHaveLaunched = true;
markExecutionReady(validated);
if (isTerminalReceipt(validated)) {
return validated;
}
@@ -505,7 +590,10 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
}
let terminal: TerminalNodeWorkerSupervisorReceipt;
try {
terminal = await cancelUntilTerminal({ request: stableRequest, expected });
terminal = await cancelUntilTerminal({
request: stableRequest!,
expected: expected!,
});
} catch (cancelError) {
throw Object.assign(
new Error("node worker launch failed and cancellation could not be confirmed", {
@@ -524,5 +612,39 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
}
};
return { launch };
const assertExecutionEligible = async (params: {
deviceId: string;
execution: NodeWorkerExecution;
signal?: AbortSignal;
}): Promise<void> => {
const transport = options.getTransport();
if (!transport) {
throw new NodeWorkerLaunchTransportError(
"UNAVAILABLE",
"device worker node transport is unavailable",
);
}
const timeout = new AbortController();
const timer = setTimeout(
() => timeout.abort(new Error("node worker eligibility check timed out")),
DEFAULT_AVAILABILITY_TIMEOUT_MS,
);
timer.unref?.();
const signal = params.signal
? AbortSignal.any([params.signal, timeout.signal])
: timeout.signal;
try {
await findNode({
transport,
deviceId: params.deviceId,
execution: params.execution,
requireLaunchEligibility: true,
signal,
});
} finally {
clearTimeout(timer);
}
};
return { assertExecutionEligible, launch };
}
@@ -0,0 +1,154 @@
import type { IncomingMessage, ServerResponse } from "node:http";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import {
nodeWorkerMemoryProjectionTransferPath,
parseNodeWorkerMemoryProjectionRequestProof,
NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER,
NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER,
NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER,
type NodeWorkerMemoryProjectionRequestProof,
} from "../../worker/node-memory-projection-protocol.js";
import { AUTH_RATE_LIMIT_SCOPE_WORKER_TRANSFER, type AuthRateLimiter } from "../auth-rate-limit.js";
import { classifyNodeWorkerProjectionTransferPath } from "../gateway-http-route-contracts.js";
import { sendJson, watchClientDisconnect } from "../http-common.js";
import { withSerializedRateLimitAttempt } from "../rate-limit-attempt-serialization.js";
import type { NodeWorkerProjectionTransferService } from "./node-worker-projection-transfer-service.js";
const OPAQUE_NOT_FOUND = { error: "not_found" } as const;
type NodeWorkerProjectionTransferHttpCallbackResult =
| { kind: "unauthorized" }
| { kind: "authorized"; handle: () => Promise<void> | void };
export type NodeWorkerProjectionTransferHttpCallback = (params: {
req: IncomingMessage;
res: ServerResponse;
bearer: string;
proof: NodeWorkerMemoryProjectionRequestProof;
}) => Promise<NodeWorkerProjectionTransferHttpCallbackResult>;
function bearerToken(req: IncomingMessage): string | undefined {
const authorization = normalizeOptionalString(req.headers.authorization);
if (!authorization?.toLowerCase().startsWith("bearer ")) {
return undefined;
}
return normalizeOptionalString(authorization.slice(7));
}
function proofHeader(req: IncomingMessage, name: string): string | undefined {
const value = req.headers[name];
return typeof value === "string" ? normalizeOptionalString(value) : undefined;
}
function requestProof(req: IncomingMessage): NodeWorkerMemoryProjectionRequestProof | undefined {
const signedAt = proofHeader(req, NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER);
if (!signedAt || !/^(?:0|[1-9][0-9]{0,15})$/u.test(signedAt)) {
return undefined;
}
return (
parseNodeWorkerMemoryProjectionRequestProof({
nodeId: proofHeader(req, NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER),
signedAtMs: Number(signedAt),
signature: proofHeader(req, NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER),
}) ?? undefined
);
}
function sendOpaqueNotFound(res: ServerResponse): void {
sendJson(res, 404, OPAQUE_NOT_FOUND);
}
export async function handleNodeWorkerProjectionTransferHttpRequest(params: {
req: IncomingMessage;
res: ServerResponse;
clientIp: string | undefined;
rateLimiter?: AuthRateLimiter;
callback?: NodeWorkerProjectionTransferHttpCallback;
}): Promise<boolean> {
const parsed = URL.parse(params.req.url ?? "/", "http://localhost");
if (!parsed?.pathname || classifyNodeWorkerProjectionTransferPath(parsed.pathname) === "outside") {
return false;
}
params.res.setHeader("Cache-Control", "no-store");
if (parsed.pathname !== nodeWorkerMemoryProjectionTransferPath() || parsed.search || params.req.method !== "GET") {
sendOpaqueNotFound(params.res);
return true;
}
const bearer = bearerToken(params.req);
const proof = requestProof(params.req);
const admission = await withSerializedRateLimitAttempt<
| { kind: "rate-limited"; retryAfterMs: number }
| { kind: "unauthorized" }
| Extract<NodeWorkerProjectionTransferHttpCallbackResult, { kind: "authorized" }>
>({
ip: params.clientIp,
scope: AUTH_RATE_LIMIT_SCOPE_WORKER_TRANSFER,
run: async () => {
const rateCheck = params.rateLimiter?.check(
params.clientIp,
AUTH_RATE_LIMIT_SCOPE_WORKER_TRANSFER,
);
if (rateCheck && !rateCheck.allowed) {
return { kind: "rate-limited", retryAfterMs: rateCheck.retryAfterMs };
}
const outcome =
bearer && proof && params.callback
? await params.callback({ req: params.req, res: params.res, bearer, proof })
: ({ kind: "unauthorized" } as const);
if (outcome.kind === "unauthorized") {
params.rateLimiter?.recordFailure(params.clientIp, AUTH_RATE_LIMIT_SCOPE_WORKER_TRANSFER);
} else {
params.rateLimiter?.reset(params.clientIp, AUTH_RATE_LIMIT_SCOPE_WORKER_TRANSFER);
}
return outcome;
},
});
if (admission.kind === "rate-limited") {
if (admission.retryAfterMs > 0) {
params.res.setHeader("Retry-After", String(Math.ceil(admission.retryAfterMs / 1000)));
}
sendJson(params.res, 429, { error: "rate_limited" });
return true;
}
if (admission.kind === "unauthorized") {
sendOpaqueNotFound(params.res);
return true;
}
await admission.handle();
return true;
}
export function createNodeWorkerProjectionTransferHttpCallback(
service: NodeWorkerProjectionTransferService,
): NodeWorkerProjectionTransferHttpCallback {
return async ({ req, res, bearer, proof }) => {
const authorization = await service.authorize(bearer, proof);
if (!authorization) {
return { kind: "unauthorized" };
}
return {
kind: "authorized",
handle: async () => {
const clientAbort = new AbortController();
const stopWatchingDisconnect = watchClientDisconnect(req, res, clientAbort);
const timeoutMs = Math.max(1, authorization.expiresAtMs - Date.now());
const signal = AbortSignal.any([
service.authorizationSignal(authorization),
clientAbort.signal,
AbortSignal.timeout(timeoutMs),
]);
try {
const payload = service.payload(authorization);
if (!payload || signal.aborted || !service.isAuthorizationCurrent(authorization)) {
sendOpaqueNotFound(res);
return;
}
sendJson(res, 200, payload);
} finally {
stopWatchingDisconnect();
service.revoke(authorization);
}
},
};
};
}
@@ -0,0 +1,363 @@
import { createHash } from "node:crypto";
import type { AuthorizedMemoryVirtualFileBroker } from "../../agents/memory-authorized-read-host.js";
import { verifyDeviceSignature } from "../../infra/device-identity.js";
import { generateSecureToken } from "../../infra/secure-random.js";
import { registerSecretValueForRedaction } from "../../logging/secret-redaction-registry.js";
import {
buildNodeWorkerMemoryProjectionRequestProofPayload,
NODE_WORKER_MEMORY_PROJECTION_MAX_FILES,
NODE_WORKER_MEMORY_PROJECTION_MAX_FILE_BYTES,
NODE_WORKER_MEMORY_PROJECTION_MAX_TOTAL_BYTES,
NODE_WORKER_MEMORY_PROJECTION_PROOF_SKEW_MS,
NODE_WORKER_MEMORY_PROJECTION_VERSION,
type NodeWorkerMemoryProjectionFile,
type NodeWorkerMemoryProjectionBinding,
type NodeWorkerMemoryProjectionPayload,
type NodeWorkerMemoryProjection,
type NodeWorkerMemoryProjectionRequestProof,
} from "../../worker/node-memory-projection-protocol.js";
import type { NodeWorkerSupervisorNodeProof } from "../node-registry-private.js";
const TOKEN_PATTERN = /^[A-Za-z0-9_-]{43}$/u;
type ProjectionTransferCapability = {
reference: string;
node: NodeWorkerSupervisorNodeProof;
nodeId: string;
connId: string;
pairingGeneration: string;
environmentId: string;
sessionId: string;
ownerEpoch: number;
placementGeneration: number;
runId: string;
launchId: string;
binding: NodeWorkerMemoryProjectionBinding;
viewId: string;
viewRevision: string;
viewDigest: string;
expiresAtMs: number;
payload: NodeWorkerMemoryProjectionPayload;
state: "ready" | "serving";
abortController: AbortController;
stopWatchingSignal?: () => void;
isAuthorized: () => boolean;
};
function projectionAuthorizationBinding(params: {
launchBinding: string;
broker: AuthorizedMemoryVirtualFileBroker;
}): NodeWorkerMemoryProjectionBinding {
return Object.freeze({
launch: params.launchBinding,
// The selected runtime minted these opaque values from its authenticated
// plan. Signing their digest prevents a node from substituting a launch
// fence while retaining a different subject/actor/policy view.
authorization: createHash("sha256")
.update(
JSON.stringify({
launch: params.launchBinding,
planId: params.broker.view.planId,
contextFingerprint: params.broker.view.contextFingerprint,
revision: params.broker.view.revision,
}),
)
.digest("hex"),
});
}
function mintReference(generateToken: (bytes: number) => string): string {
const reference = generateToken(32);
if (!TOKEN_PATTERN.test(reference)) {
throw new Error("Worker memory projection token generator returned an invalid bearer");
}
registerSecretValueForRedaction(reference);
return reference;
}
function validVirtualPath(virtualPath: string, roots: ReadonlySet<string>): boolean {
const normalized = virtualPath.normalize("NFC");
const parts = normalized.split("/");
const root = parts[0];
const leaf = parts[1];
return (
normalized === virtualPath &&
parts.length === 2 &&
Boolean(root) &&
Boolean(leaf) &&
roots.has(root!.toLocaleLowerCase("en-US")) &&
leaf !== "." &&
leaf !== ".." &&
!leaf!.includes("\\") &&
!leaf!.includes("\0")
);
}
function createPayload(
broker: AuthorizedMemoryVirtualFileBroker,
signal?: AbortSignal,
): Promise<{
payload: NodeWorkerMemoryProjectionPayload;
expiresAtMs: number;
viewDigest: string;
}> {
return (async () => {
signal?.throwIfAborted();
const view = broker.view;
const expiresAtMs = Date.parse(view.expiresAt);
if (!Number.isFinite(expiresAtMs) || expiresAtMs <= Date.now()) {
throw new Error("Authorized memory virtual view is expired");
}
if (view.files.length === 0 || view.files.length > NODE_WORKER_MEMORY_PROJECTION_MAX_FILES) {
throw new Error("Authorized memory virtual view exceeds projection file limits");
}
const roots = new Set<string>();
for (const root of view.roots) {
const name = root.virtualRoot.normalize("NFC");
const key = name.toLocaleLowerCase("en-US");
// `/memory` is the container mount target, not this payload's root. The
// canonical selected-memory view uses `memory/...`, nested under that mount.
if (
name !== root.virtualRoot ||
!name ||
!/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/u.test(name) ||
roots.has(key) ||
["opt", "run", "workspace"].includes(key)
) {
throw new Error("Authorized memory virtual view has an unsafe projection root");
}
roots.add(key);
}
const paths = new Set<string>();
let totalBytes = 0;
const files: NodeWorkerMemoryProjectionFile[] = [];
for (const file of [...view.files].toSorted((left, right) =>
left.virtualPath.localeCompare(right.virtualPath),
)) {
signal?.throwIfAborted();
if (!validVirtualPath(file.virtualPath, roots)) {
throw new Error("Authorized memory virtual view has an unsafe projection path");
}
const key = file.virtualPath.toLocaleLowerCase("en-US");
if (paths.has(key)) {
throw new Error("Authorized memory virtual view has colliding projection paths");
}
paths.add(key);
const text = await broker.readFile(file.virtualPath, signal);
signal?.throwIfAborted();
if (text === undefined) {
throw new Error("Authorized memory virtual view could not materialize an issued file");
}
const bytes = Buffer.from(text, "utf8");
if (bytes.byteLength > NODE_WORKER_MEMORY_PROJECTION_MAX_FILE_BYTES) {
throw new Error("Authorized memory virtual view exceeds projection file limits");
}
totalBytes += bytes.byteLength;
if (totalBytes > NODE_WORKER_MEMORY_PROJECTION_MAX_TOTAL_BYTES) {
throw new Error("Authorized memory virtual view exceeds projection total limits");
}
files.push(
Object.freeze({
virtualPath: file.virtualPath,
sha256: createHash("sha256").update(bytes).digest("hex"),
contentBase64: bytes.toString("base64"),
}),
);
}
const payload = Object.freeze({
version: NODE_WORKER_MEMORY_PROJECTION_VERSION,
files: Object.freeze(files),
});
return {
payload,
expiresAtMs,
viewDigest: createHash("sha256").update(JSON.stringify(payload)).digest("hex"),
};
})();
}
export function createNodeWorkerProjectionTransferService(
options: {
now?: () => number;
generateToken?: (bytes: number) => string;
resolveNodePublicKey?: (node: NodeWorkerSupervisorNodeProof) => Promise<string | undefined>;
isNodeCurrent?: (node: NodeWorkerSupervisorNodeProof) => boolean;
} = {},
) {
const now = options.now ?? Date.now;
const generateToken = options.generateToken ?? generateSecureToken;
const resolveNodePublicKey = options.resolveNodePublicKey ?? (async () => undefined);
const isNodeCurrent = options.isNodeCurrent ?? (() => false);
const capabilities = new Map<string, ProjectionTransferCapability>();
const isCurrent = (capability: ProjectionTransferCapability): boolean =>
capabilities.get(capability.reference) === capability &&
capability.state === "serving" &&
capability.expiresAtMs > now() &&
!capability.abortController.signal.aborted &&
capability.isAuthorized() &&
isNodeCurrent(capability.node);
const isReady = (
capability: ProjectionTransferCapability | undefined,
): capability is ProjectionTransferCapability =>
Boolean(
capability &&
capabilities.get(capability.reference) === capability &&
capability.state === "ready" &&
capability.expiresAtMs > now() &&
!capability.abortController.signal.aborted &&
capability.isAuthorized() &&
isNodeCurrent(capability.node),
);
const revokeCapability = (capability: ProjectionTransferCapability): void => {
if (capabilities.get(capability.reference) === capability) {
capabilities.delete(capability.reference);
}
capability.stopWatchingSignal?.();
if (!capability.abortController.signal.aborted) {
capability.abortController.abort(new Error("Worker memory projection capability closed"));
}
};
return {
async prepare(params: {
node: NodeWorkerSupervisorNodeProof;
broker: AuthorizedMemoryVirtualFileBroker;
environmentId: string;
sessionId: string;
ownerEpoch: number;
placementGeneration: number;
runId: string;
launchId: string;
launchBinding: string;
isAuthorized: () => boolean;
signal?: AbortSignal;
}): Promise<NodeWorkerMemoryProjection> {
if (!params.isAuthorized() || !isNodeCurrent(params.node)) {
throw new Error("Worker memory projection authority is unavailable");
}
if (!/^[a-f0-9]{64}$/u.test(params.launchBinding)) {
throw new Error("Worker memory projection launch binding is invalid");
}
params.signal?.throwIfAborted();
const snapshot = await createPayload(params.broker, params.signal);
if (!params.isAuthorized() || snapshot.expiresAtMs <= now()) {
throw new Error("Worker memory projection authority is unavailable");
}
params.signal?.throwIfAborted();
const reference = mintReference(generateToken);
if (capabilities.has(reference)) {
// A collision must never replace a live single-use capability, even in a
// test or an incorrectly substituted entropy source.
throw new Error("Worker memory projection token collision");
}
const capability: ProjectionTransferCapability = {
reference,
node: params.node,
nodeId: params.node.nodeId,
connId: params.node.connId,
pairingGeneration: params.node.pairingGeneration,
environmentId: params.environmentId,
sessionId: params.sessionId,
ownerEpoch: params.ownerEpoch,
placementGeneration: params.placementGeneration,
runId: params.runId,
launchId: params.launchId,
binding: projectionAuthorizationBinding({
launchBinding: params.launchBinding,
broker: params.broker,
}),
viewId: params.broker.view.viewId,
viewRevision: params.broker.view.revision,
viewDigest: snapshot.viewDigest,
expiresAtMs: snapshot.expiresAtMs,
payload: snapshot.payload,
state: "ready",
abortController: new AbortController(),
isAuthorized: params.isAuthorized,
};
if (params.signal) {
const abort = () => revokeCapability(capability);
params.signal.addEventListener("abort", abort, { once: true });
capability.stopWatchingSignal = () => params.signal?.removeEventListener("abort", abort);
if (params.signal.aborted) {
abort();
}
}
if (capability.abortController.signal.aborted) {
throw new Error("Worker memory projection authority is unavailable");
}
capabilities.set(reference, capability);
return Object.freeze({
version: NODE_WORKER_MEMORY_PROJECTION_VERSION,
reference,
binding: capability.binding,
expiresAtMs: capability.expiresAtMs,
});
},
async authorize(
reference: string,
proof: NodeWorkerMemoryProjectionRequestProof,
): Promise<ProjectionTransferCapability | undefined> {
const capability = capabilities.get(reference);
if (
!isReady(capability) ||
proof.nodeId !== capability.nodeId ||
Math.abs(now() - proof.signedAtMs) > NODE_WORKER_MEMORY_PROJECTION_PROOF_SKEW_MS
) {
return undefined;
}
const publicKey = await resolveNodePublicKey(capability.node).catch(() => undefined);
if (!publicKey || !isReady(capability)) {
return undefined;
}
const payload = buildNodeWorkerMemoryProjectionRequestProofPayload({
reference,
binding: capability.binding,
nodeId: capability.nodeId,
signedAtMs: proof.signedAtMs,
});
if (!verifyDeviceSignature(publicKey, payload, proof.signature) || !isReady(capability)) {
return undefined;
}
// This is the final synchronous check before ready -> serving. A stale
// connection or invalid signature leaves the one-use capability intact.
capability.state = "serving";
return capability;
},
isAuthorizationCurrent: isCurrent,
authorizationSignal(capability: ProjectionTransferCapability): AbortSignal {
return capability.abortController.signal;
},
payload(
capability: ProjectionTransferCapability,
): NodeWorkerMemoryProjectionPayload | undefined {
return isCurrent(capability) ? capability.payload : undefined;
},
revoke(capabilityOrReference: ProjectionTransferCapability | string): void {
const capability =
typeof capabilityOrReference === "string"
? capabilities.get(capabilityOrReference)
: capabilityOrReference;
if (capability) {
revokeCapability(capability);
}
},
closeAll(): void {
for (const capability of capabilities.values()) {
revokeCapability(capability);
}
},
};
}
export type NodeWorkerProjectionTransferService = ReturnType<
typeof createNodeWorkerProjectionTransferService
>;
@@ -0,0 +1,554 @@
import fs from "node:fs/promises";
import http from "node:http";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
GATEWAY_CLIENT_IDS,
GATEWAY_CLIENT_MODES,
} from "../../../packages/gateway-protocol/src/client-info.js";
import type { AuthorizedMemoryVirtualFileBroker } from "../../agents/memory-authorized-read-host.js";
import {
loadOrCreateDeviceIdentity,
publicKeyRawBase64UrlFromPem,
signDevicePayload,
type DeviceIdentity,
} from "../../infra/device-identity.js";
import { NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE } from "../../infra/node-runner-inventory.js";
import { NodeWorkerMemoryProjectionRuntime } from "../../node-host/node-worker-memory-projection.js";
import {
buildNodeWorkerMemoryProjectionRequestProofPayload,
nodeWorkerMemoryProjectionTransferPath,
NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER,
NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER,
NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER,
type NodeWorkerMemoryProjection,
type NodeWorkerMemoryProjectionBinding,
} from "../../worker/node-memory-projection-protocol.js";
import { NODE_WORKER_EXECUTION_CONTAINER_V1 } from "../../worker/node-supervisor-protocol.js";
import type { NodeWorkerSupervisorNodeProof } from "../node-registry-private.js";
import {
createNodeWorkerProjectionTransferHttpCallback,
handleNodeWorkerProjectionTransferHttpRequest,
} from "./node-worker-projection-transfer-http.js";
import { createNodeWorkerProjectionTransferService } from "./node-worker-projection-transfer-service.js";
function nodeProof(nodeId: string, connId = "conn-1"): NodeWorkerSupervisorNodeProof {
return {
nodeId,
connId,
pairingIdentity: "pairing-1",
pairingGeneration: "generation-1",
clientId: GATEWAY_CLIENT_IDS.NODE_HOST,
clientMode: GATEWAY_CLIENT_MODES.NODE,
protocolFeature: NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
workerHost: {
enabled: true,
capacity: { total: 2, available: 2 },
processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, memoryProjection: 1 },
},
commands: [],
};
}
function proof(params: {
identity: DeviceIdentity;
reference: string;
binding: NodeWorkerMemoryProjectionBinding;
nodeId?: string;
signedAtMs?: number;
signature?: string;
}) {
const nodeId = params.nodeId ?? params.identity.deviceId;
const signedAtMs = params.signedAtMs ?? Date.now();
const payload = buildNodeWorkerMemoryProjectionRequestProofPayload({
reference: params.reference,
binding: params.binding,
nodeId,
signedAtMs,
});
return {
nodeId,
signedAtMs,
signature: params.signature ?? signDevicePayload(params.identity.privateKeyPem, payload),
};
}
async function requestProjection(params: {
port: number;
reference: string;
binding: NodeWorkerMemoryProjectionBinding;
identity: DeviceIdentity;
nodeId?: string;
signedAtMs?: number;
signature?: string;
}): Promise<{ status: number; body: string }> {
const requestProof = proof(params);
return await new Promise((resolve, reject) => {
const request = http.request(
{
host: "127.0.0.1",
port: params.port,
path: nodeWorkerMemoryProjectionTransferPath(),
method: "GET",
headers: {
authorization: `Bearer ${params.reference}`,
[NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER]: requestProof.nodeId,
[NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER]: String(requestProof.signedAtMs),
[NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER]: requestProof.signature,
},
},
(response) => {
const chunks: Buffer[] = [];
response.on("data", (chunk: Buffer) => chunks.push(chunk));
response.on("end", () =>
resolve({
status: response.statusCode ?? 0,
body: Buffer.concat(chunks).toString("utf8"),
}),
);
},
);
request.once("error", reject);
request.end();
});
}
function createTransferService(params: {
node: NodeWorkerSupervisorNodeProof;
identity: DeviceIdentity;
isNodeCurrent?: (node: NodeWorkerSupervisorNodeProof) => boolean;
generateToken?: (bytes: number) => string;
}) {
return createNodeWorkerProjectionTransferService({
...((params.generateToken ? { generateToken: params.generateToken } : {}) as object),
resolveNodePublicKey: async (candidate) =>
candidate.nodeId === params.node.nodeId
? publicKeyRawBase64UrlFromPem(params.identity.publicKeyPem)
: undefined,
isNodeCurrent: params.isNodeCurrent ?? (() => true),
});
}
function broker(expiresAt: string): AuthorizedMemoryVirtualFileBroker {
const contents = new Map([["memory/MEMORY.md", "only this issued virtual view"]]);
return {
view: {
version: 1,
viewId: "view-1",
planId: "plan-1",
contextFingerprint: "context-1",
revision: "revision-1",
roots: [{ version: 1, mountHandle: "mount-1", virtualRoot: "memory", access: "read" }],
files: [{ version: 1, mountHandle: "mount-1", virtualPath: "memory/MEMORY.md" }],
expiresAt,
},
readFile: async (virtualPath) => contents.get(virtualPath),
};
}
describe("node worker memory projection transfer", () => {
let root: string;
let server: http.Server | undefined;
let identity: DeviceIdentity;
beforeEach(async () => {
root = await fs.mkdtemp(
path.join(await fs.realpath(os.tmpdir()), "openclaw-memory-projection-wire-"),
);
identity = loadOrCreateDeviceIdentity({ path: path.join(root, "node-identity.sqlite") });
});
afterEach(async () => {
if (server) {
await new Promise<void>((resolve) => server?.close(() => resolve()));
server = undefined;
}
await fs.rm(root, { recursive: true, force: true });
});
it.runIf(
process.platform !== "win32" && typeof process.getuid === "function" && process.getuid() > 0,
)("stages one authorized, immutable virtual-memory snapshot and rejects replay", async () => {
const node = nodeProof(identity.deviceId);
const service = createTransferService({
node,
identity,
generateToken: () => "A".repeat(43),
});
const callback = createNodeWorkerProjectionTransferHttpCallback(service);
server = http.createServer((req, res) => {
void handleNodeWorkerProjectionTransferHttpRequest({
req,
res,
clientIp: "127.0.0.1",
callback,
}).catch((error: unknown) =>
res.destroy(error instanceof Error ? error : new Error(String(error))),
);
});
await new Promise<void>((resolve) => server?.listen(0, "127.0.0.1", resolve));
const address = server.address();
if (!address || typeof address === "string") {
throw new Error("test server did not bind a TCP port");
}
const projection = await service.prepare({
node,
broker: broker(new Date(Date.now() + 60_000).toISOString()),
environmentId: "environment-1",
sessionId: "session-1",
ownerEpoch: 3,
placementGeneration: 4,
runId: "run-1",
launchId: "launch-1",
launchBinding: "a".repeat(64),
isAuthorized: () => true,
});
const runtime = new NodeWorkerMemoryProjectionRuntime({
root: path.join(root, "node-host"),
deviceIdentity: identity,
});
const projectionIdentity = {
gatewayNamespace: "gateway-1",
launchId: "launch-1",
planHash: "a".repeat(64),
};
const endpoint = { kind: "websocket" as const, url: `ws://127.0.0.1:${address.port}` };
const staged = await runtime.stage({ identity: projectionIdentity, projection, endpoint });
expect(await fs.readFile(path.join(staged, "memory", "MEMORY.md"), "utf8")).toBe(
"only this issued virtual view",
);
expect((await fs.stat(path.join(staged, "memory", "MEMORY.md"))).mode & 0o777).toBe(0o400);
await expect(
runtime.stage({ identity: projectionIdentity, projection, endpoint }),
).rejects.toThrow("projection transfer failed (404)");
expect(await fs.readdir(path.join(root, "node-host", "memory-projections"))).toEqual([]);
});
it.runIf(
process.platform !== "win32" && typeof process.getuid === "function" && process.getuid() > 0,
)("removes a staged projection when cancellation races its final rename", async () => {
const node = nodeProof(identity.deviceId);
const service = createTransferService({ node, identity });
const callback = createNodeWorkerProjectionTransferHttpCallback(service);
server = http.createServer((req, res) => {
void handleNodeWorkerProjectionTransferHttpRequest({
req,
res,
clientIp: "127.0.0.1",
callback,
}).catch((error: unknown) =>
res.destroy(error instanceof Error ? error : new Error(String(error))),
);
});
await new Promise<void>((resolve) => server?.listen(0, "127.0.0.1", resolve));
const address = server.address();
if (!address || typeof address === "string") {
throw new Error("test server did not bind a TCP port");
}
const projection = await service.prepare({
node,
broker: broker(new Date(Date.now() + 60_000).toISOString()),
environmentId: "environment-1",
sessionId: "session-1",
ownerEpoch: 3,
placementGeneration: 4,
runId: "run-1",
launchId: "launch-cancelled-before-rename",
launchBinding: "a".repeat(64),
isAuthorized: () => true,
});
const runtime = new NodeWorkerMemoryProjectionRuntime({
root: path.join(root, "node-host"),
deviceIdentity: identity,
});
const projectionIdentity = {
gatewayNamespace: "gateway-1",
launchId: "launch-cancelled-before-rename",
planHash: "b".repeat(64),
};
const controller = new AbortController();
const rename = fs.rename;
const renameSpy = vi.spyOn(fs, "rename").mockImplementation(async (...args) => {
controller.abort(new Error("projection revoked before process start"));
await rename(...args);
});
try {
await expect(
runtime.stage({
identity: projectionIdentity,
projection,
endpoint: { kind: "websocket", url: `ws://127.0.0.1:${address.port}` },
signal: controller.signal,
}),
).rejects.toThrow("projection revoked before process start");
} finally {
renameSpy.mockRestore();
}
expect(await fs.readdir(path.join(root, "node-host", "memory-projections"))).toEqual([]);
});
it("never overwrites a live capability when the token source collides", async () => {
const node = nodeProof(identity.deviceId);
const service = createTransferService({
node,
identity,
generateToken: () => "B".repeat(43),
});
let authorized = true;
const params = {
node,
broker: broker(new Date(Date.now() + 60_000).toISOString()),
environmentId: "environment-1",
sessionId: "session-1",
ownerEpoch: 3,
placementGeneration: 4,
runId: "run-1",
launchId: "launch-1",
launchBinding: "a".repeat(64),
isAuthorized: () => authorized,
};
const prepared = await service.prepare(params);
authorized = false;
await expect(
service.authorize(
prepared.reference,
proof({ identity, reference: prepared.reference, binding: prepared.binding }),
),
).resolves.toBeUndefined();
authorized = true;
await expect(service.prepare({ ...params, launchId: "launch-2" })).rejects.toThrow(
"token collision",
);
});
it("rejects a wrong node proof without consuming the intended node capability", async () => {
const node = nodeProof(identity.deviceId);
const otherIdentity = loadOrCreateDeviceIdentity({
path: path.join(root, "other-node.sqlite"),
});
const service = createTransferService({ node, identity });
const callback = createNodeWorkerProjectionTransferHttpCallback(service);
server = http.createServer((req, res) => {
void handleNodeWorkerProjectionTransferHttpRequest({
req,
res,
clientIp: "127.0.0.1",
callback,
});
});
await new Promise<void>((resolve) => server?.listen(0, "127.0.0.1", resolve));
const address = server.address();
if (!address || typeof address === "string") {
throw new Error("test server did not bind a TCP port");
}
const projection = await service.prepare({
node,
broker: broker(new Date(Date.now() + 60_000).toISOString()),
environmentId: "environment-1",
sessionId: "session-1",
ownerEpoch: 3,
placementGeneration: 4,
runId: "run-1",
launchId: "launch-1",
launchBinding: "a".repeat(64),
isAuthorized: () => true,
});
await expect(
requestProjection({
port: address.port,
reference: projection.reference,
binding: projection.binding,
identity: otherIdentity,
}),
).resolves.toMatchObject({ status: 404 });
await expect(
requestProjection({
port: address.port,
reference: projection.reference,
binding: projection.binding,
identity,
}),
).resolves.toMatchObject({ status: 200 });
});
it("rejects a re-signed projection binding swap without consuming the issued view", async () => {
const node = nodeProof(identity.deviceId);
const service = createTransferService({ node, identity });
const callback = createNodeWorkerProjectionTransferHttpCallback(service);
server = http.createServer((req, res) => {
void handleNodeWorkerProjectionTransferHttpRequest({
req,
res,
clientIp: "127.0.0.1",
callback,
});
});
await new Promise<void>((resolve) => server?.listen(0, "127.0.0.1", resolve));
const address = server.address();
if (!address || typeof address === "string") {
throw new Error("test server did not bind a TCP port");
}
const projection = await service.prepare({
node,
broker: broker(new Date(Date.now() + 60_000).toISOString()),
environmentId: "environment-1",
sessionId: "session-1",
ownerEpoch: 3,
placementGeneration: 4,
runId: "run-1",
launchId: "launch-1",
launchBinding: "a".repeat(64),
isAuthorized: () => true,
});
await expect(
requestProjection({
port: address.port,
reference: projection.reference,
binding: { ...projection.binding, launch: "d".repeat(64) },
identity,
}),
).resolves.toMatchObject({ status: 404 });
await expect(
requestProjection({
port: address.port,
reference: projection.reference,
binding: projection.binding,
identity,
}),
).resolves.toMatchObject({ status: 200 });
});
it("rejects stale connections and invalid proofs without consuming the current capability", async () => {
const node = nodeProof(identity.deviceId);
let currentNode = node;
const service = createTransferService({
node,
identity,
isNodeCurrent: (candidate) =>
candidate.nodeId === currentNode.nodeId &&
candidate.connId === currentNode.connId &&
candidate.pairingGeneration === currentNode.pairingGeneration,
});
const callback = createNodeWorkerProjectionTransferHttpCallback(service);
server = http.createServer((req, res) => {
void handleNodeWorkerProjectionTransferHttpRequest({
req,
res,
clientIp: "127.0.0.1",
callback,
});
});
await new Promise<void>((resolve) => server?.listen(0, "127.0.0.1", resolve));
const address = server.address();
if (!address || typeof address === "string") {
throw new Error("test server did not bind a TCP port");
}
const prepare = async (launchId: string) =>
await service.prepare({
node,
broker: broker(new Date(Date.now() + 60_000).toISOString()),
environmentId: "environment-1",
sessionId: "session-1",
ownerEpoch: 3,
placementGeneration: 4,
runId: "run-1",
launchId,
launchBinding: "a".repeat(64),
isAuthorized: () => true,
});
const replaced = await prepare("launch-replaced");
currentNode = nodeProof(identity.deviceId, "conn-replaced");
await expect(
requestProjection({
port: address.port,
reference: replaced.reference,
binding: replaced.binding,
identity,
}),
).resolves.toMatchObject({ status: 404 });
currentNode = node;
await expect(
requestProjection({
port: address.port,
reference: replaced.reference,
binding: replaced.binding,
identity,
}),
).resolves.toMatchObject({ status: 200 });
const invalid = await prepare("launch-invalid");
await expect(
requestProjection({
port: address.port,
reference: invalid.reference,
binding: invalid.binding,
identity,
signature: "A".repeat(86),
}),
).resolves.toMatchObject({ status: 404 });
await expect(
requestProjection({
port: address.port,
reference: invalid.reference,
binding: invalid.binding,
identity,
}),
).resolves.toMatchObject({ status: 200 });
const expired = await prepare("launch-expired");
await expect(
requestProjection({
port: address.port,
reference: expired.reference,
binding: expired.binding,
identity,
signedAtMs: Date.now() - 120_001,
}),
).resolves.toMatchObject({ status: 404 });
await expect(
requestProjection({
port: address.port,
reference: expired.reference,
binding: expired.binding,
identity,
}),
).resolves.toMatchObject({ status: 200 });
});
it.runIf(
process.platform !== "win32" && typeof process.getuid === "function" && process.getuid() > 0,
)(
"refuses a plaintext non-loopback projection transport before sending a capability",
async () => {
const runtime = new NodeWorkerMemoryProjectionRuntime({
root: path.join(root, "node-host"),
deviceIdentity: identity,
});
const projection: NodeWorkerMemoryProjection = {
version: 1,
reference: "C".repeat(43),
binding: { launch: "a".repeat(64), authorization: "b".repeat(64) },
expiresAtMs: Date.now() + 60_000,
};
await expect(
runtime.stage({
identity: {
gatewayNamespace: "gateway-1",
launchId: "launch-1",
planHash: "a".repeat(64),
},
projection,
endpoint: { kind: "websocket", url: "ws://192.0.2.1:18789" },
}),
).rejects.toThrow("requires wss://");
},
);
});
@@ -14,7 +14,11 @@ import { createDeferred } from "../../../test/helpers/promise.js";
import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js";
import { NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE } from "../../infra/node-runner-inventory.js";
import { parseWorkerLaunchPlan } from "../../worker/launch-descriptor.js";
import type { NodeWorkerSupervisorReceipt } from "../../worker/node-supervisor-protocol.js";
import {
NODE_WORKER_EXECUTION_CONTAINER_V1,
NODE_WORKER_EXECUTION_HOST_V1,
type NodeWorkerSupervisorReceipt,
} from "../../worker/node-supervisor-protocol.js";
import type { NodeWorkerWorkspaceExecInput } from "../../worker/node-workspace-protocol.js";
import {
NODE_WORKSPACE_TRANSFER_ERROR_CODE,
@@ -24,6 +28,7 @@ import type { NodeWorkerSupervisorTransport } from "../node-registry-private.js"
import type { createDeviceWorkerRuntime } from "./device-provider.js";
import { createNodeWorkerTunnelManager } from "./node-worker-tunnel.js";
import type { NodeWorkspaceTransferService } from "./node-workspace-transfer-service.js";
import type { NodeWorkerProjectionTransferService } from "./node-worker-projection-transfer-service.js";
import { sameWorkerSessionTurnClaim } from "./placement-record.js";
import type { WorkerEnvironmentRecord } from "./store.js";
import { serializeWorkerWorkspaceManifest } from "./workspace-manifest.js";
@@ -96,6 +101,7 @@ function plan() {
},
assignment: {
agentId: "main",
memoryReadEnforced: false,
operationalRunInstance: { instanceId: "instance-1", runId: "run-1" },
agentRuntimeIdentityToken: "runtime-token",
runId: "run-1",
@@ -160,7 +166,101 @@ function workspaceTransfer(): NodeWorkspaceTransferService {
} as unknown as NodeWorkspaceTransferService;
}
function projectionTransfer(): NodeWorkerProjectionTransferService {
return {
prepare: vi.fn(async () => ({
version: 1 as const,
reference: "a".repeat(43),
binding: { launch: "b".repeat(64), authorization: "c".repeat(64) },
expiresAtMs: Date.now() + 60_000,
})),
} as unknown as NodeWorkerProjectionTransferService;
}
describe("node worker tunnel manager", () => {
it.each([
[false, NODE_WORKER_EXECUTION_HOST_V1],
[true, NODE_WORKER_EXECUTION_CONTAINER_V1],
] as const)(
"derives %s memory launches as %s execution",
async (memoryReadEnforced, executionKind) => {
const launchNodeWorker = vi.fn<NodeWorkerLaunch>(async (request) => ({
launchId: request.input.launchId,
planHash: "b".repeat(64),
environmentId: request.input.descriptor.admission.environmentId,
sessionId: request.input.descriptor.admission.sessionId,
ownerEpoch: request.input.descriptor.admission.ownerEpoch,
placementGeneration: request.input.placementGeneration,
runId: request.input.descriptor.assignment.runId,
state: "completed",
resultJson: "{}",
}));
const manager = createNodeWorkerTunnelManager({
gatewayDeviceId: "gateway-device-1",
getEnvironment: () => environment(),
getTransport: transport,
launchNodeWorker,
validateWorkerTurn: () => true,
projectionTransfer: projectionTransfer(),
workspaceTransfer: workspaceTransfer(),
});
const launchPlan = plan();
launchPlan.assignment.memoryReadEnforced = memoryReadEnforced;
const handle = await manager.start(startRequest());
await handle.launchTurn({
plan: launchPlan,
turnClaim: turnClaim(),
...(memoryReadEnforced ? { memoryProjection: {} as never } : {}),
});
expect(launchNodeWorker).toHaveBeenCalledWith(
expect.objectContaining({
input: expect.objectContaining({ execution: { kind: executionKind } }),
}),
);
},
);
it("forwards transport dispatch and execution readiness as distinct signals", async () => {
const events: string[] = [];
const launchNodeWorker = vi.fn<NodeWorkerLaunch>(async (request) => {
request.onDispatchReady?.();
expect(events).toEqual(["dispatch"]);
request.onExecutionReady?.();
return {
launchId: request.input.launchId,
planHash: "b".repeat(64),
environmentId: request.input.descriptor.admission.environmentId,
sessionId: request.input.descriptor.admission.sessionId,
ownerEpoch: request.input.descriptor.admission.ownerEpoch,
placementGeneration: request.input.placementGeneration,
runId: request.input.descriptor.assignment.runId,
state: "completed",
resultJson: "{}",
};
});
const manager = createNodeWorkerTunnelManager({
gatewayDeviceId: "gateway-device-1",
getEnvironment: () => environment(),
getTransport: transport,
launchNodeWorker,
validateWorkerTurn: () => true,
projectionTransfer: projectionTransfer(),
workspaceTransfer: workspaceTransfer(),
});
const handle = await manager.start(startRequest());
await handle.launchTurn({
plan: plan(),
turnClaim: turnClaim(),
onDispatchReady: () => events.push("dispatch"),
onExecutionReady: () => events.push("execution"),
});
expect(events).toEqual(["dispatch", "execution"]);
});
it("revalidates the exact claim when a same-run replacement launches", async () => {
const record = environment();
let currentClaim = turnClaim();
@@ -181,9 +281,11 @@ describe("node worker tunnel manager", () => {
runId: request.input.descriptor.assignment.runId,
state: "cancelled",
errorText: "test launch finished",
executionStarted: false,
};
}),
validateWorkerTurn: (claim) => sameWorkerSessionTurnClaim(claim, currentClaim),
projectionTransfer: projectionTransfer(),
workspaceTransfer: workspaceTransfer(),
});
const handle = await manager.start(startRequest());
@@ -214,8 +316,10 @@ describe("node worker tunnel manager", () => {
runId: request.input.descriptor.assignment.runId,
state: "cancelled",
errorText,
executionStarted: false,
})),
validateWorkerTurn: () => true,
projectionTransfer: projectionTransfer(),
workspaceTransfer: workspaceTransfer(),
});
const handle = await manager.start(startRequest());
@@ -237,6 +341,7 @@ describe("node worker tunnel manager", () => {
getTransport: transport,
launchNodeWorker: vi.fn(),
validateWorkerTurn: () => true,
projectionTransfer: projectionTransfer(),
workspaceTransfer: workspaceTransfer(),
});
@@ -257,6 +362,7 @@ describe("node worker tunnel manager", () => {
getTransport: transport,
launchNodeWorker: vi.fn(),
validateWorkerTurn: () => true,
projectionTransfer: projectionTransfer(),
workspaceTransfer: workspaceTransfer(),
});
manager.bindWorkspaceBindingResolver(resolveWorkspaceBinding);
@@ -289,6 +395,7 @@ describe("node worker tunnel manager", () => {
getTransport: transport,
launchNodeWorker: vi.fn(),
validateWorkerTurn: () => true,
projectionTransfer: projectionTransfer(),
workspaceTransfer: transfer,
});
manager.bindWorkspaceBindingResolver(resolveWorkspaceBinding);
@@ -319,6 +426,7 @@ describe("node worker tunnel manager", () => {
getTransport: transport,
launchNodeWorker: vi.fn(),
validateWorkerTurn: () => true,
projectionTransfer: projectionTransfer(),
workspaceTransfer: transfer,
});
manager.bindWorkspaceBindingResolver(async () => {
@@ -378,6 +486,7 @@ describe("node worker tunnel manager", () => {
getTransport: () => nodeTransport,
launchNodeWorker: vi.fn(),
validateWorkerTurn: () => true,
projectionTransfer: projectionTransfer(),
workspaceTransfer: transfer,
});
manager.bindWorkspaceBindingResolver(async () => ({
@@ -472,6 +581,7 @@ describe("node worker tunnel manager", () => {
},
launchNodeWorker: vi.fn(),
validateWorkerTurn: () => true,
projectionTransfer: projectionTransfer(),
workspaceTransfer: transfer,
});
const resolveWorkspaceBinding = vi.fn(async () => ({
@@ -570,6 +680,7 @@ describe("node worker tunnel manager", () => {
getTransport: () => nodeTransport,
launchNodeWorker: vi.fn(),
validateWorkerTurn: () => true,
projectionTransfer: projectionTransfer(),
workspaceTransfer: transfer,
});
manager.bindWorkspaceBindingResolver(async () => ({
@@ -629,6 +740,7 @@ describe("node worker tunnel manager", () => {
getTransport: () => nodeTransport,
launchNodeWorker: vi.fn(),
validateWorkerTurn: () => true,
projectionTransfer: projectionTransfer(),
workspaceTransfer: transfer,
});
const handle = await manager.start(startRequest());
@@ -668,6 +780,7 @@ describe("node worker tunnel manager", () => {
runId: request.input.descriptor.assignment.runId,
state: "cancelled",
errorText: "node worker cancelled",
executionStarted: false,
});
});
},
@@ -681,6 +794,7 @@ describe("node worker tunnel manager", () => {
getTransport: transport,
launchNodeWorker,
validateWorkerTurn: () => true,
projectionTransfer: projectionTransfer(),
workspaceTransfer: workspaceTransfer(),
});
const first = await manager.start(startRequest());
@@ -729,6 +843,7 @@ describe("node worker tunnel manager", () => {
runId: request.input.descriptor.assignment.runId,
state: "cancelled",
errorText: "node worker cancelled",
executionStarted: true,
});
},
{ once: true },
@@ -742,6 +857,7 @@ describe("node worker tunnel manager", () => {
getTransport: transport,
launchNodeWorker,
validateWorkerTurn: () => true,
projectionTransfer: projectionTransfer(),
workspaceTransfer: workspaceTransfer(),
});
const handle = await manager.start(startRequest());
@@ -818,6 +934,7 @@ describe("node worker tunnel manager", () => {
getTransport: () => nodeTransport,
launchNodeWorker: vi.fn(),
validateWorkerTurn: () => true,
projectionTransfer: projectionTransfer(),
workspaceTransfer: transfer,
});
const handle = await manager.start(startRequest());
@@ -886,6 +1003,7 @@ describe("node worker tunnel manager", () => {
getTransport: () => nodeTransport,
launchNodeWorker: vi.fn(),
validateWorkerTurn: () => true,
projectionTransfer: projectionTransfer(),
workspaceTransfer: transfer,
});
const handle = await manager.start(startRequest());
@@ -6,7 +6,14 @@ import { NODE_WORKER_WORKSPACE_EXEC_COMMAND } from "../../infra/node-commands.js
import { createSubsystemLogger } from "../../logging/subsystem.js";
import type { SpawnResult } from "../../process/exec.js";
import { createDeferredCore, type Deferred } from "../../shared/deferred.js";
import type { NodeWorkerSupervisorReceipt } from "../../worker/node-supervisor-protocol.js";
import {
NODE_WORKER_EXECUTION_CONTAINER_V1,
NODE_WORKER_EXECUTION_HOST_V1,
nodeWorkerMemoryProjectionLaunchBinding,
type NodeWorkerExecution,
type NodeWorkerSupervisorReceipt,
} from "../../worker/node-supervisor-protocol.js";
import type { NodeWorkerMemoryProjection } from "../../worker/node-memory-projection-protocol.js";
import {
parseNodeWorkerWorkspaceExecResult,
type NodeWorkerWorkspaceExecInput,
@@ -27,6 +34,7 @@ import {
recordNodeSyncPath,
} from "./node-worker-workspace-fallback.js";
import type { NodeWorkspaceTransferService } from "./node-workspace-transfer-service.js";
import type { NodeWorkerProjectionTransferService } from "./node-worker-projection-transfer-service.js";
import type { WorkerSessionTurnClaim } from "./placement-record.js";
import type { WorkerEnvironmentRecord } from "./store.js";
import type {
@@ -74,12 +82,18 @@ type NodeWorkerLaunch = (request: {
expectedBundleHash: string;
placementGeneration: number;
descriptor: WorkerTurnLaunchRequest["plan"];
execution: NodeWorkerExecution;
memoryProjection?: NodeWorkerMemoryProjection;
};
prepareMemoryProjection?: (
node: NodeWorkerSupervisorNodeProof,
) => Promise<NodeWorkerMemoryProjection>;
isDispatchAuthorized: () => boolean;
isCancellationAuthorized: () => boolean;
timeoutMs: number;
signal?: AbortSignal;
onDispatchReady?: () => void;
onExecutionReady?: () => void;
}) => Promise<TerminalNodeWorkerSupervisorReceipt>;
type NodeWorkerWorkspaceBinding = {
@@ -100,6 +114,7 @@ type NodeWorkerTunnelManagerOptions = {
getTransport: () => NodeWorkerSupervisorTransport | undefined;
launchNodeWorker: NodeWorkerLaunch;
validateWorkerTurn: (claim: WorkerSessionTurnClaim) => boolean;
projectionTransfer: NodeWorkerProjectionTransferService;
workspaceTransfer: NodeWorkspaceTransferService;
};
@@ -529,22 +544,50 @@ export function createNodeWorkerTunnelManager(options: NodeWorkerTunnelManagerOp
claim.sessionId === plan.admission.sessionId &&
claim.runId === plan.assignment.runId &&
options.validateWorkerTurn(claim);
const execution: NodeWorkerExecution = plan.assignment.memoryReadEnforced
? { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 }
: { kind: NODE_WORKER_EXECUTION_HOST_V1 };
if (execution.kind === NODE_WORKER_EXECUTION_CONTAINER_V1 && !request.memoryProjection) {
throw new Error("enforced node worker launch is missing its authorized memory projection");
}
const launchSignal = request.signal
? AbortSignal.any([entry.abortController.signal, request.signal])
: entry.abortController.signal;
const input = {
launchId: plan.assignment.turnId,
gatewayNamespace,
expectedBundleHash: entry.expectedBuild.bundleHash,
placementGeneration: claim.placementGeneration,
descriptor: plan,
execution,
};
const operation = options.launchNodeWorker({
deviceId: entry.deviceId,
input: {
launchId: plan.assignment.turnId,
gatewayNamespace,
expectedBundleHash: entry.expectedBuild.bundleHash,
placementGeneration: claim.placementGeneration,
descriptor: plan,
},
input,
...(request.memoryProjection
? {
prepareMemoryProjection: async (node) =>
await options.projectionTransfer.prepare({
node,
broker: request.memoryProjection!,
environmentId: entry.environmentId,
sessionId: entry.sessionId,
ownerEpoch: entry.ownerEpoch,
placementGeneration: claim.placementGeneration,
runId: plan.assignment.runId,
launchId: plan.assignment.turnId,
launchBinding: nodeWorkerMemoryProjectionLaunchBinding(input),
isAuthorized: isDispatchAuthorized,
signal: launchSignal,
}),
}
: {}),
isDispatchAuthorized,
isCancellationAuthorized: () => hasDurableBinding(entry as NodeTunnelEntry),
timeoutMs: request.timeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS,
onDispatchReady: request.onDispatchReady,
signal: request.signal
? AbortSignal.any([entry.abortController.signal, request.signal])
: entry.abortController.signal,
onExecutionReady: request.onExecutionReady,
signal: launchSignal,
});
entry.launchTasks.add(operation);
try {
@@ -339,6 +339,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
const searchMemory = async (
identity: WorkerConnectionIdentity,
request: WorkerMemorySearchParams,
signal?: AbortSignal,
): Promise<WorkerMemoryServiceResult> => {
const binding = validateMemoryRequest(identity);
if (!binding.ok) {
@@ -351,6 +352,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
const result = await host.host.search({
query: request.query,
...(request.limit === undefined ? {} : { limit: request.limit }),
...(signal ? { signal } : {}),
});
const current = validateMemoryRequest(identity);
if (!current.ok) {
@@ -364,6 +366,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
const readMemory = async (
identity: WorkerConnectionIdentity,
request: WorkerMemoryReadParams,
signal?: AbortSignal,
): Promise<WorkerMemoryServiceResult> => {
const binding = validateMemoryRequest(identity);
if (!binding.ok) {
@@ -377,6 +380,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
handleId: request.handleId,
...(request.from === undefined ? {} : { from: request.from }),
...(request.lines === undefined ? {} : { lines: request.lines }),
...(signal ? { signal } : {}),
});
const current = validateMemoryRequest(identity);
if (!current.ok) {
@@ -2,6 +2,7 @@ import type { WorkerTunnelStatus } from "@openclaw/gateway-protocol";
import { NODE_WORKER_CAPACITY_EXHAUSTED_ERROR_CODE } from "../../infra/node-commands.js";
import type { SpawnResult } from "../../process/exec.js";
import type { WorkerLaunchPlan } from "../../worker/launch-descriptor.js";
import type { AuthorizedMemoryVirtualFileBroker } from "../../agents/memory-authorized-read-host.js";
import type { NodeWorkerWorkspaceTransferInput } from "../../worker/node-workspace-transfer-protocol.js";
import type { WorkerSessionTurnClaim } from "./placement-record.js";
import type {
@@ -102,9 +103,13 @@ export type WorkerWorkspaceQuiescence = {
export type WorkerTurnLaunchRequest = {
plan: WorkerLaunchPlan;
turnClaim: WorkerSessionTurnClaim;
/** Gateway-only broker capability. It must never enter the serialized worker plan. */
memoryProjection?: AuthorizedMemoryVirtualFileBroker;
timeoutMs?: number;
signal?: AbortSignal;
onDispatchReady?: () => void;
/** Fires only after the node has durably attested real worker execution. */
onExecutionReady?: () => void;
};
export type WorkerWorkspaceTunnelHandle = {
@@ -30,6 +30,7 @@ describe("worker turn launcher terminal results", () => {
it("requests immediate recovery when reconciliation fails after worker finishing", async () => {
seedActivePlacement();
const phases: string[] = [];
const destroy = vi.fn(async () => attachedEnvironment());
const tunnelFailure = new NodeWorkerWorkspaceTransferError(
"workspace-transfer-failed: gateway TLS fingerprint mismatch",
@@ -44,6 +45,9 @@ describe("worker turn launcher terminal results", () => {
runWorkspaceCommand: vi.fn(),
launchTurn: vi.fn(async (request): Promise<SpawnResult> => {
request.onDispatchReady?.();
expect(phases).not.toContain("process_spawned");
request.onExecutionReady?.();
expect(phases.filter((phase) => phase === "process_spawned")).toHaveLength(1);
const completed = openSessionManager();
const leafId = completed.appendMessage(
makeAgentAssistantMessage({
@@ -106,7 +110,10 @@ describe("worker turn launcher terminal results", () => {
agentId: "main",
runId: "run-reconcile-tunnel-loss",
},
turn("run-reconcile-tunnel-loss"),
{
...turn("run-reconcile-tunnel-loss"),
onExecutionPhase: ({ phase }) => phases.push(phase),
},
async () => ({ meta: { durationMs: 1 } }),
),
).rejects.toMatchObject({
@@ -115,6 +122,7 @@ describe("worker turn launcher terminal results", () => {
});
expect(reconcileActivePlacement).toHaveBeenCalledWith(ENVIRONMENT_ID);
expect(phases.filter((phase) => phase === "process_spawned")).toEqual(["process_spawned"]);
expect(placements.get(SESSION_ID)).toMatchObject({ state: "failed", turnClaim: null });
expect(placements.listPendingWorkspaceResults()).toHaveLength(0);
expect(destroy).not.toHaveBeenCalled();
@@ -13,6 +13,8 @@ import {
import { createChatRunState } from "../server-chat-state.js";
import { prepareSessionArchiveLifecycle } from "../server-methods/sessions-archive-lifecycle.js";
import type { GatewayRequestContext } from "../server-methods/types.js";
import { NODE_WORKER_EXECUTION_CONTAINER_V1 } from "../../worker/node-supervisor-protocol.js";
import { bindDeviceWorkerExecutionEligibility } from "./device-provider.js";
import type { WorkerTunnelHandle } from "./tunnel-contract.js";
import {
ENVIRONMENT_ID,
@@ -590,13 +592,24 @@ describe("worker turn launcher local placement", () => {
expect(placements.get(SESSION_ID)).toMatchObject({ state: "active", turnClaim: null });
});
it("rejects enforced memory before a host-node worker is admitted", async () => {
it("fails an enforced worker turn before workspace, credential, or tunnel work when v6 is unavailable", async () => {
seedActivePlacement();
const environment = attachedEnvironment();
environment.nodeDeviceId = "node-1";
const environments: WorkerTurnEnvironmentService = {
...unusedEnvironments(),
get: vi.fn(() => attachedEnvironment()),
get: vi.fn(() => environment),
};
const provider = createWorkerSessionTurnPlacementProvider({ environments, placements });
const assertEligible = vi.fn(async () => {
throw new Error("device worker node does not attest to the required process-isolation boundary");
});
bindDeviceWorkerExecutionEligibility(environments, assertEligible);
const resolveWorkspacePath = vi.fn(async () => "/workspace/should-not-resolve");
const provider = createWorkerSessionTurnPlacementProvider({
environments,
placements,
resolveWorkspacePath,
});
const runLocal = vi.fn(async () => ({ meta: { durationMs: 1 } }));
const onAdmitted = vi.fn();
@@ -613,11 +626,16 @@ describe("worker turn launcher local placement", () => {
runLocal,
onAdmitted,
),
).rejects.toThrow("enforced memory requires a containerized worker launch");
).rejects.toThrow("does not attest to the required process-isolation boundary");
expect(runLocal).not.toHaveBeenCalled();
expect(onAdmitted).not.toHaveBeenCalled();
expect(environments.get).not.toHaveBeenCalled();
expect(environments.get).toHaveBeenCalledWith(ENVIRONMENT_ID);
expect(assertEligible).toHaveBeenCalledWith({
deviceId: "node-1",
execution: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 },
});
expect(resolveWorkspacePath).not.toHaveBeenCalled();
expect(environments.acquireTurnCredential).not.toHaveBeenCalled();
expect(environments.startTunnel).not.toHaveBeenCalled();
expect(placements.get(SESSION_ID)).toMatchObject({ state: "active", turnClaim: null });
@@ -1,7 +1,10 @@
import { randomUUID } from "node:crypto";
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
import { mapThinkingLevelForProvider } from "../../agents/embedded-agent-runner/utils.js";
import { createAuthorizedMemoryReadHost } from "../../agents/memory-authorized-read-host.js";
import {
createAuthorizedMemoryReadHost,
resolveAuthorizedMemoryVirtualFileBroker,
} from "../../agents/memory-authorized-read-host.js";
import type { SandboxContext } from "../../agents/sandbox/types.js";
import {
withSessionPlacementForcedTerminalSettlement,
@@ -17,6 +20,7 @@ import { redactSensitiveText } from "../../logging/redact.js";
import { isMemoryIsolationCutoverAgent } from "../../plugins/memory-cutover.js";
import { normalizeDeliveryContext } from "../../utils/delivery-context.shared.js";
import { parseWorkerLaunchPlan } from "../../worker/launch-descriptor.js";
import { NODE_WORKER_EXECUTION_CONTAINER_V1 } from "../../worker/node-supervisor-protocol.js";
import { WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE } from "../../worker/transcript-message.js";
import {
STALE_WORKER_BUILD_REASON,
@@ -24,6 +28,7 @@ import {
supportsWorkerExecutionContextLaunch,
} from "./admission.js";
import { registerWorkerMemoryHost } from "./memory-host.js";
import { assertDeviceWorkerExecutionEligibility } from "./device-provider.js";
import { placementTurnOwner } from "./placement-record.js";
import { createRemoteExecPlacementSandbox } from "./placement-sandbox.js";
import type {
@@ -221,6 +226,23 @@ async function executeWorkerTurn(params: {
if (memoryReadEnforced && !memoryHost) {
throw new WorkerTurnExecutionError("Enforced memory could not create a trusted worker host");
}
const memoryProjection = memoryHost
? await resolveAuthorizedMemoryVirtualFileBroker(memoryHost, turn.abortSignal)
: undefined;
if (memoryReadEnforced && !memoryProjection) {
throw new WorkerTurnExecutionError(
"Enforced memory could not materialize an authorized virtual memory view",
);
}
const memoryProjectionExpiresAtMs = memoryProjection
? Date.parse(memoryProjection.view.expiresAt)
: undefined;
if (
memoryProjectionExpiresAtMs !== undefined &&
(!Number.isFinite(memoryProjectionExpiresAtMs) || memoryProjectionExpiresAtMs <= Date.now())
) {
throw new WorkerTurnExecutionError("Enforced memory virtual view expired before process launch");
}
const { browser, toolAuthority } = resolveWorkerBrowserLaunchPlan({
desktop: environment.desktop,
modelRef,
@@ -262,11 +284,13 @@ async function executeWorkerTurn(params: {
turnId: randomUUID(),
prompt: turn.prompt,
suppressPromptTranscript: true,
workspaceDir: placement.remoteWorkspaceDir,
workspaceDir: memoryReadEnforced ? "/workspace" : placement.remoteWorkspaceDir,
...(turn.permissionMode
? {
permissionMode: turn.permissionMode,
workerContainmentRoot: placement.remoteWorkspaceDir,
workerContainmentRoot: memoryReadEnforced
? "/workspace"
: placement.remoteWorkspaceDir,
}
: {}),
modelRef,
@@ -317,13 +341,13 @@ async function executeWorkerTurn(params: {
const handoffAbort = new AbortController();
let handoffError: Error | undefined;
let dispatchReady = false;
let executionReady = false;
const onDispatchReady = () => {
if (dispatchReady) {
return;
}
dispatchReady = true;
params.onHandoff();
turn.onExecutionPhase?.({ phase: "process_spawned", backend: "cloud-worker" });
try {
if (!params.environments.acknowledgeCredentialDelivery(credential)) {
handoffError = new Error("Cloud worker credential owner changed during process handoff");
@@ -335,6 +359,13 @@ async function executeWorkerTurn(params: {
handoffAbort.abort(handoffError);
}
};
const onExecutionReady = () => {
if (executionReady) {
return;
}
executionReady = true;
turn.onExecutionPhase?.({ phase: "process_spawned", backend: "cloud-worker" });
};
if (!tunnel.launchTurn) {
throw new Error("Worker tunnel does not support worker turns");
}
@@ -343,11 +374,22 @@ async function executeWorkerTurn(params: {
return await tunnel.launchTurn({
plan,
turnClaim: params.turnClaim,
timeoutMs: turn.timeoutMs,
...(memoryProjection ? { memoryProjection } : {}),
timeoutMs:
memoryProjectionExpiresAtMs === undefined
? turn.timeoutMs
: Math.max(
1,
Math.min(
turn.timeoutMs ?? 60_000,
memoryProjectionExpiresAtMs - Date.now(),
),
),
signal: turn.abortSignal
? AbortSignal.any([turn.abortSignal, handoffAbort.signal])
: handoffAbort.signal,
onDispatchReady,
onExecutionReady,
});
} finally {
// A completed or failed process must not retain memory authority for a later turn.
@@ -538,14 +580,24 @@ export function createWorkerSessionTurnPlacementProvider(options: WorkerTurnLaun
let placement = requireActivePlacement(routablePlacement);
const requiresProcessIsolation = claim.requiresProcessIsolation === true;
const remoteExec = placement.executionMode === "remote-exec";
if (requiresProcessIsolation && remoteExec) {
// Remote-exec invokes the Gateway callback rather than the node-worker protocol, so it
// cannot attest to the container boundary required for an enforced memory turn.
throw new Error("enforced memory cannot execute through remote-exec placement");
}
if (requiresProcessIsolation) {
// Remote-exec invokes the Gateway callback, while worker-turn launches directly on the
// node host. Neither path supplies the required broker process boundary yet.
throw new Error(
remoteExec
? "enforced memory cannot execute through remote-exec placement"
: "enforced memory requires a containerized worker launch",
);
const environment = options.environments.get(placement.environmentId);
if (!environment?.nodeDeviceId) {
throw new Error("enforced memory requires a device worker node");
}
// Prove the exact connected node's advertised boundary before any credential, tunnel,
// workspace, or claim work can create host-visible state for the enforced turn.
await assertDeviceWorkerExecutionEligibility({
service: options.environments,
deviceId: environment.nodeDeviceId,
execution: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 },
...(turn.abortSignal ? { signal: turn.abortSignal } : {}),
});
}
// The placement owns the managed worktree. Callers can carry a default or stale
// workspace path, but remote results must only reconcile into that canonical root.
+19 -2
View File
@@ -54,7 +54,13 @@ const LockPayloadSchema = z.object({
configPath: z.string(),
port: z.number().int().min(1).max(65_535).optional(),
role: z
.enum(["gateway", "agent-embedded", "skill-workshop-apply", "sqlite-maintenance"])
.enum([
"gateway",
"agent-embedded",
"skill-workshop-apply",
"sqlite-maintenance",
"offline-maintenance",
])
.optional(),
stateDir: z.string().optional(),
startTime: z.number().optional(),
@@ -69,7 +75,12 @@ type GatewayLockHandle = {
release: () => Promise<void>;
};
type GatewayLockRole = "gateway" | "agent-embedded" | "skill-workshop-apply" | "sqlite-maintenance";
type GatewayLockRole =
| "gateway"
| "agent-embedded"
| "skill-workshop-apply"
| "sqlite-maintenance"
| "offline-maintenance";
export type GatewayLockIdentity = {
pid: number;
@@ -215,6 +226,7 @@ async function resolveGatewayOwnerStatus(
if (
role === "agent-embedded" ||
role === "sqlite-maintenance" ||
role === "offline-maintenance" ||
role === "skill-workshop-apply"
) {
const args = readFn(pid);
@@ -227,6 +239,11 @@ async function resolveGatewayOwnerStatus(
// instead of baking one command spelling into stale-lock recovery.
return isOpenClawArgv(args) ? "alive" : "dead";
}
if (role === "offline-maintenance") {
return isOpenClawCommandArgv(args, "doctor") || isOpenClawCommandArgv(args, "backup")
? "alive"
: "dead";
}
const command = role === "sqlite-maintenance" ? "doctor" : "skills";
return isOpenClawCommandArgv(args, command) ? "alive" : "dead";
}
+119
View File
@@ -0,0 +1,119 @@
import { describe, expect, it } from "vitest";
import { GATEWAY_CLIENT_IDS } from "../../packages/gateway-protocol/src/client-info.js";
import {
NODE_WORKER_EXECUTION_CONTAINER_V1,
NODE_WORKER_EXECUTION_HOST_V1,
} from "../worker/node-supervisor-protocol.js";
import {
NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE,
NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
parseNodeRunnerInventoryDeclaration,
} from "./node-runner-inventory.js";
import {
supportsNodeWorkerExecution,
type NodeWorkerSupervisorNodeProof,
} from "../gateway/node-runner-inventory-runtime.js";
const capacity = { total: 2, available: 2 } as const;
function proof(params: {
protocolFeature:
| typeof NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE
| typeof NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE
| typeof NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE;
processIsolation?: {
kind: typeof NODE_WORKER_EXECUTION_CONTAINER_V1;
memoryProjection?: 1;
};
}): NodeWorkerSupervisorNodeProof {
return {
nodeId: "node-1",
connId: "conn-1",
pairingIdentity: "identity-1",
pairingGeneration: "generation-1",
clientId: GATEWAY_CLIENT_IDS.NODE_HOST,
clientMode: "node",
protocolFeature: params.protocolFeature,
workerHost: {
enabled: true,
capacity,
...(params.processIsolation ? { processIsolation: params.processIsolation } : {}),
},
commands: ["system.run"],
};
}
describe("node runner process-isolation inventory", () => {
it("keeps v5 host-only and accepts only the exact v6 declaration", () => {
const v5 = {
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE],
workerHost: { enabled: true, capacity },
};
const v6 = {
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE],
workerHost: {
enabled: true,
capacity,
processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 },
},
};
expect(parseNodeRunnerInventoryDeclaration(v5)).toEqual(v5);
expect(parseNodeRunnerInventoryDeclaration(v6)).toEqual(v6);
expect(
parseNodeRunnerInventoryDeclaration({
...v5,
workerHost: { ...v5.workerHost, processIsolation: v6.workerHost.processIsolation },
}),
).toBeNull();
});
it.each([
{
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE],
workerHost: { enabled: true, capacity },
},
{
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE],
workerHost: { enabled: false },
},
{
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE],
workerHost: {
enabled: true,
capacity,
processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, extra: true },
},
},
{
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE],
workerHost: { enabled: true, capacity, processIsolation: { kind: "host-v1" } },
},
])("rejects malformed v6 process-isolation declarations", (declaration) => {
expect(parseNodeRunnerInventoryDeclaration(declaration)).toBeNull();
});
it("admits container execution only for the exact v7 memory-projection proof", () => {
const v5 = proof({ protocolFeature: NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE });
const v6 = proof({
protocolFeature: NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE,
processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 },
});
const v7 = proof({
protocolFeature: NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, memoryProjection: 1 },
});
expect(supportsNodeWorkerExecution(v5, { kind: NODE_WORKER_EXECUTION_HOST_V1 })).toBe(true);
expect(supportsNodeWorkerExecution(v5, { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 })).toBe(
false,
);
expect(supportsNodeWorkerExecution(v6, { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 })).toBe(
false,
);
expect(supportsNodeWorkerExecution(v7, { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 })).toBe(
true,
);
});
});
+72 -9
View File
@@ -1,9 +1,14 @@
import { isRecord } from "@openclaw/normalization-core/record-coerce";
import { validateWorkerAdmissionHandshake } from "../../packages/gateway-protocol/src/index.js";
import { WORKER_BUNDLE_PREWARM_VERSION } from "../../packages/gateway-protocol/src/schema/worker-admission.js";
import { NODE_WORKER_EXECUTION_CONTAINER_V1 } from "../worker/node-supervisor-protocol.js";
export const NODE_RUNNER_INVENTORY_UPDATE_METHOD = "node.runnerInventory.update";
export const NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE = "node-worker-supervisor-v5";
export const NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE =
"node-worker-supervisor-v6";
export const NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE =
"node-worker-supervisor-v7";
export const NODE_WORKER_SUPERVISOR_BINARY_CAPACITY_PROTOCOL_FEATURE = "node-worker-supervisor-v4";
export const NODE_WORKER_SUPERVISOR_EXECUTION_CONTEXT_V1_PROTOCOL_FEATURE =
"node-worker-supervisor-v3";
@@ -25,6 +30,10 @@ export type NodeWorkerCapacitySnapshot = Readonly<{
total: number;
available: number;
}>;
export type NodeWorkerProcessIsolationDeclaration = Readonly<{
kind: typeof NODE_WORKER_EXECUTION_CONTAINER_V1;
memoryProjection?: 1;
}>;
export type NodeWorkerHostDeclaration =
| { enabled: false }
@@ -34,6 +43,8 @@ export type NodeWorkerHostDeclaration =
bundlePrewarm?: typeof WORKER_BUNDLE_PREWARM_VERSION;
bundleRetention?: typeof NODE_WORKER_BUNDLE_RETENTION_VERSION;
bundleStatus?: typeof NODE_WORKER_BUNDLE_STATUS_VERSION;
/** Present only on v6 hosts that passed the local container-runtime gate. */
processIsolation?: NodeWorkerProcessIsolationDeclaration;
};
export type NodeRunnerInventoryDeclaration =
@@ -47,7 +58,11 @@ export type NodeRunnerInventoryDeclaration =
];
}
| {
protocolFeatures: readonly [typeof NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE];
protocolFeatures: readonly [
| typeof NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE
| typeof NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE
| typeof NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
];
workerHost: NodeWorkerHostDeclaration;
};
@@ -73,19 +88,52 @@ function parseCapacitySnapshot(value: unknown): NodeWorkerCapacitySnapshot | nul
: null;
}
function parseWorkerHostDeclaration(value: unknown): NodeWorkerHostDeclaration | null {
function parseProcessIsolationDeclaration(
value: unknown,
requireMemoryProjection: boolean,
): NodeWorkerProcessIsolationDeclaration | null {
if (
!isRecord(value) ||
!Object.hasOwn(value, "kind") ||
value.kind !== NODE_WORKER_EXECUTION_CONTAINER_V1 ||
(requireMemoryProjection
? Object.keys(value).length !== 2 || value.memoryProjection !== 1
: Object.keys(value).length !== 1)
) {
return null;
}
return requireMemoryProjection
? { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, memoryProjection: 1 }
: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 };
}
function parseWorkerHostDeclaration(params: {
value: unknown;
requireProcessIsolation: boolean;
requireMemoryProjection?: boolean;
}): NodeWorkerHostDeclaration | null {
const { value } = params;
if (!isRecord(value) || typeof value.enabled !== "boolean") {
return null;
}
const keys = Object.keys(value);
if (!value.enabled) {
return keys.length === 1 && keys[0] === "enabled" ? { enabled: false } : null;
return !params.requireProcessIsolation && keys.length === 1 && keys[0] === "enabled"
? { enabled: false }
: null;
}
const capacity = parseCapacitySnapshot(value.capacity);
const processIsolation =
value.processIsolation === undefined
? undefined
: parseProcessIsolationDeclaration(
value.processIsolation,
params.requireMemoryProjection === true,
);
if (
!capacity ||
keys.length < 2 ||
keys.length > 5 ||
keys.length > 6 ||
!keys.includes("enabled") ||
!keys.includes("capacity") ||
keys.some(
@@ -94,14 +142,17 @@ function parseWorkerHostDeclaration(value: unknown): NodeWorkerHostDeclaration |
key !== "capacity" &&
key !== "bundlePrewarm" &&
key !== "bundleRetention" &&
key !== "bundleStatus",
key !== "bundleStatus" &&
key !== "processIsolation",
) ||
(value.bundlePrewarm !== undefined && value.bundlePrewarm !== WORKER_BUNDLE_PREWARM_VERSION) ||
(value.bundleRetention !== undefined &&
value.bundleRetention !== NODE_WORKER_BUNDLE_RETENTION_VERSION) ||
(value.bundleStatus !== undefined &&
value.bundleStatus !== NODE_WORKER_BUNDLE_STATUS_VERSION) ||
(value.bundleStatus !== undefined && value.bundleRetention === undefined)
(value.bundleStatus !== undefined && value.bundleRetention === undefined) ||
(value.processIsolation !== undefined && !processIsolation) ||
(params.requireProcessIsolation !== Boolean(processIsolation))
) {
return null;
}
@@ -117,6 +168,7 @@ function parseWorkerHostDeclaration(value: unknown): NodeWorkerHostDeclaration |
...(value.bundleStatus === NODE_WORKER_BUNDLE_STATUS_VERSION
? { bundleStatus: NODE_WORKER_BUNDLE_STATUS_VERSION }
: {}),
...(processIsolation ? { processIsolation } : {}),
};
}
@@ -190,12 +242,23 @@ export function parseNodeRunnerInventoryDeclaration(
? { protocolFeatures: [feature] }
: null;
}
if (feature !== NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE || keys.length !== 2) {
if (
(feature !== NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE &&
feature !== NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE &&
feature !== NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE) ||
keys.length !== 2
) {
return null;
}
const workerHost = parseWorkerHostDeclaration(value.workerHost);
const workerHost = parseWorkerHostDeclaration({
value: value.workerHost,
requireProcessIsolation:
feature === NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE ||
feature === NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
requireMemoryProjection: feature === NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
});
return workerHost
? { protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE], workerHost }
? { protocolFeatures: [feature], workerHost }
: null;
}
+13 -6
View File
@@ -1,4 +1,5 @@
import { startMemoryBrokerServer, type MemoryBrokerHandler } from "./server.js";
import { startMemoryBrokerServer } from "./server.js";
import type { MemoryBrokerChildEntry } from "./entry.js";
type BrokerStartMessage = Readonly<{
type: "start";
@@ -7,10 +8,7 @@ type BrokerStartMessage = Readonly<{
brokerEpoch: string;
secret: string;
handlerModuleUrl: string;
}>;
type BrokerChildEntry = Readonly<{
createMemoryBrokerHandler: () => MemoryBrokerHandler | Promise<MemoryBrokerHandler>;
agentIds: readonly string[];
}>;
type BrokerMaintenanceMessage = Readonly<{
@@ -41,10 +39,19 @@ async function start(message: BrokerStartMessage): Promise<void> {
if (server || closing) {
throw new Error("memory broker child has already started");
}
const module = (await import(message.handlerModuleUrl)) as Partial<BrokerChildEntry>;
const module = (await import(message.handlerModuleUrl)) as Partial<MemoryBrokerChildEntry>;
if (typeof module.createMemoryBrokerHandler !== "function") {
throw new Error("selected memory plugin has no broker child entry");
}
if (
!Array.isArray(message.agentIds) ||
!message.agentIds.every((agentId) => typeof agentId === "string" && agentId.length > 0)
) {
throw new Error("memory broker startup agents are unavailable");
}
// Recovery happens before the socket exists, so a fresh/replacement child never reports
// healthy while pending revisions can still become visible or need quarantine.
await module.initializeMemoryBroker?.({ agentIds: Object.freeze([...message.agentIds]) });
const handler = await module.createMemoryBrokerHandler();
if (typeof handler !== "function") {
throw new Error("selected memory plugin returned an invalid broker handler");
+18
View File
@@ -0,0 +1,18 @@
import type { MemoryBrokerHandler } from "./server.js";
/**
* Gateway passes this small, path-free startup snapshot over inherited parent-child IPC before
* the broker opens its agent-visible socket. A plugin must finish recovery or throw; ready means
* the selected runtime has completed its own durable-startup fence.
*/
export type MemoryBrokerStartupContext = Readonly<{
agentIds: readonly string[];
}>;
/** Public selected-memory child entry contract. */
export type MemoryBrokerChildEntry = Readonly<{
createMemoryBrokerHandler: () => MemoryBrokerHandler | Promise<MemoryBrokerHandler>;
initializeMemoryBroker?: (
context: MemoryBrokerStartupContext,
) => void | Promise<void>;
}>;
+110 -1
View File
@@ -5,6 +5,7 @@ import path from "node:path";
import { fileURLToPath } from "node:url";
import { afterEach, describe, expect, it } from "vitest";
import { startMemoryBrokerProcess, type MemoryBrokerProcess } from "./process.js";
import type { MemoryBrokerAuthorizationBinding } from "./protocol.js";
let broker: MemoryBrokerProcess | undefined;
@@ -78,6 +79,7 @@ describe("memory broker child", () => {
runId: "run-a",
contextFingerprint: "context-a",
subjectRevision: "subject-a",
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
actorRevision: "actor-a",
capabilitySnapshotId: "capability-a",
policyRevision: "policy-a",
@@ -90,6 +92,46 @@ describe("memory broker child", () => {
).resolves.toEqual({ agentId: "agent-a", method: "memory.search" });
});
it("runs selected-runtime startup recovery with sorted configured agents before serving", async () => {
broker = await startMemoryBrokerProcess({
brokerId: "broker-a",
childModuleUrl: new URL("./child.ts", import.meta.url),
handlerModuleUrl: new URL("./test-handler.mjs", import.meta.url).href,
agentIds: ["work", "main", "work"],
});
await expect(
broker.client.request({
binding: {
agentId: "main",
sessionId: "session-a",
runId: "run-a",
contextFingerprint: "context-a",
subjectRevision: "subject-a",
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
actorRevision: "actor-a",
capabilitySnapshotId: "capability-a",
policyRevision: "policy-a",
deliveryRevision: "delivery-a",
},
method: "memory.startup",
payload: {},
expiresAtMs: Date.now() + 30_000,
}),
).resolves.toEqual({ agentIds: ["main", "work"] });
});
it("does not report ready when selected-runtime startup recovery fails", async () => {
await expect(
startMemoryBrokerProcess({
brokerId: "broker-a",
childModuleUrl: new URL("./child.ts", import.meta.url),
handlerModuleUrl: new URL("./test-handler.mjs", import.meta.url).href,
agentIds: ["fail-startup"],
}),
).rejects.toThrow("memory broker child did not become ready");
});
it("retires a stopped child and gives its replacement a new broker epoch", async () => {
broker = await startMemoryBrokerProcess({
brokerId: "broker-a",
@@ -121,12 +163,13 @@ describe("memory broker child", () => {
});
const crashedBroker = broker;
const firstEpoch = crashedBroker.brokerEpoch;
const binding = {
const binding: MemoryBrokerAuthorizationBinding = {
agentId: "agent-a",
sessionId: "session-a",
runId: "run-a",
contextFingerprint: "context-a",
subjectRevision: "subject-a",
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
actorRevision: "actor-a",
capabilitySnapshotId: "capability-a",
policyRevision: "policy-a",
@@ -160,6 +203,37 @@ describe("memory broker child", () => {
).resolves.toEqual({ agentId: "agent-a", method: "memory.search" });
});
it("retires a child killed by an external signal without waiting for a second exit event", async () => {
broker = await startMemoryBrokerProcess({
brokerId: "broker-a",
childModuleUrl: new URL("./child.ts", import.meta.url),
handlerModuleUrl: new URL("./test-handler.mjs", import.meta.url).href,
});
const killedBroker = broker;
await expect(
killedBroker.client.request({
binding: {
agentId: "agent-a",
sessionId: "session-a",
runId: "run-a",
contextFingerprint: "context-a",
subjectRevision: "subject-a",
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
actorRevision: "actor-a",
capabilitySnapshotId: "capability-a",
policyRevision: "policy-a",
deliveryRevision: "delivery-a",
},
method: "memory.kill",
payload: {},
expiresAtMs: Date.now() + 30_000,
}),
).resolves.toBeUndefined();
await waitFor(() => !killedBroker.isRunning(), "memory broker child did not exit after SIGKILL");
await expect(killedBroker.close()).resolves.toBeUndefined();
});
it("does not inherit arbitrary Gateway environment secrets", async () => {
const previous = process.env.OPENCLAW_MEMORY_BROKER_TEST_SECRET;
process.env.OPENCLAW_MEMORY_BROKER_TEST_SECRET = "gateway-only-secret";
@@ -177,6 +251,7 @@ describe("memory broker child", () => {
runId: "run-a",
contextFingerprint: "context-a",
subjectRevision: "subject-a",
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
actorRevision: "actor-a",
capabilitySnapshotId: "capability-a",
policyRevision: "policy-a",
@@ -230,6 +305,7 @@ describe("memory broker child", () => {
runId: "run-a",
contextFingerprint: "context-a",
subjectRevision: "subject-a",
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
actorRevision: "actor-a",
capabilitySnapshotId: "capability-a",
policyRevision: "policy-a",
@@ -249,6 +325,7 @@ describe("memory broker child", () => {
runId: "run-a",
contextFingerprint: "context-a",
subjectRevision: "subject-a",
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
actorRevision: "actor-a",
capabilitySnapshotId: "capability-a",
policyRevision: "policy-a",
@@ -260,4 +337,36 @@ describe("memory broker child", () => {
}),
).resolves.toEqual({ agentId: "agent-a", method: "memory.search" });
});
it("retires a noncooperative child instead of leaving a healthy broker quiesced", async () => {
broker = await startMemoryBrokerProcess({
brokerId: "broker-a",
childModuleUrl: new URL("./child.ts", import.meta.url),
handlerModuleUrl: new URL("./test-handler.mjs", import.meta.url).href,
maintenanceTimeoutMs: 40,
});
const pending = broker.client.request({
binding: {
agentId: "agent-a",
sessionId: "session-a",
runId: "run-a",
contextFingerprint: "context-a",
subjectRevision: "subject-a",
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
actorRevision: "actor-a",
capabilitySnapshotId: "capability-a",
policyRevision: "policy-a",
deliveryRevision: "delivery-a",
},
method: "memory.hang",
payload: {},
expiresAtMs: Date.now() + 30_000,
});
await new Promise((resolve) => setTimeout(resolve, 20));
await expect(broker.quiesce()).rejects.toThrow("memory broker quiesce is unavailable");
await expect(pending).resolves.toBeUndefined();
expect(broker.isRunning()).toBe(false);
await expect(broker.isHealthy()).resolves.toBe(false);
});
});
+48 -6
View File
@@ -44,9 +44,18 @@ export type MemoryBrokerProcess = Readonly<{
export async function startMemoryBrokerProcess(params: {
brokerId: string;
handlerModuleUrl: string;
/** Configured agent IDs only; passed over inherited IPC before the broker binds its socket. */
agentIds?: readonly string[];
childModuleUrl?: string | URL;
startTimeoutMs?: number;
maintenanceTimeoutMs?: number;
}): Promise<MemoryBrokerProcess> {
if (!(params.agentIds ?? []).every((agentId) => typeof agentId === "string" && agentId.length > 0)) {
throw new Error("memory broker startup agent IDs are invalid");
}
const agentIds = Object.freeze(
[...new Set(params.agentIds ?? [])].toSorted(),
);
const directory = await mkdtemp(path.join(tmpdir(), "openclaw-memory-broker-"));
const socketPath = path.join(directory, "broker.sock");
const brokerEpoch = randomUUID();
@@ -87,6 +96,9 @@ export async function startMemoryBrokerProcess(params: {
});
let childExited = false;
let directoryRemoved = false;
let retirePromise: Promise<void> | undefined;
const hasChildExited = () =>
childExited || child.exitCode !== null || child.signalCode !== null || child.killed;
const removeDirectory = async () => {
if (directoryRemoved) {
return;
@@ -100,6 +112,18 @@ export async function startMemoryBrokerProcess(params: {
// The Gateway recreates the child with a fresh epoch on the next operation.
void removeDirectory();
});
const retireChild = async () => {
retirePromise ??= (async () => {
if (!hasChildExited()) {
child.kill("SIGKILL");
}
if (!hasChildExited()) {
await new Promise<void>((resolve) => child.once("exit", () => resolve()));
}
await removeDirectory();
})();
await retirePromise;
};
let startupStderr = "";
child.stderr?.on("data", (chunk: Buffer) => {
// Startup diagnostics never include request bodies or the parent-child secret. Keep enough
@@ -159,6 +183,7 @@ export async function startMemoryBrokerProcess(params: {
brokerEpoch,
secret: secret.toString("base64url"),
handlerModuleUrl: params.handlerModuleUrl,
agentIds,
});
});
} catch (error) {
@@ -173,7 +198,7 @@ export async function startMemoryBrokerProcess(params: {
secret,
});
const isHealthy = async (): Promise<boolean> => {
if (childExited || child.exitCode !== null || child.killed || !child.connected) {
if (hasChildExited() || !child.connected) {
return false;
}
return await new Promise<boolean>((resolve) => {
@@ -210,8 +235,13 @@ export async function startMemoryBrokerProcess(params: {
});
});
};
const maintain = async (operation: "quiesce" | "resume"): Promise<void> => {
if (childExited || child.exitCode !== null || child.killed || !child.connected) {
const maintenanceTimeoutMs = params.maintenanceTimeoutMs ?? MEMORY_BROKER_MAINTENANCE_TIMEOUT_MS;
if (!Number.isSafeInteger(maintenanceTimeoutMs) || maintenanceTimeoutMs < 1) {
await retireChild();
throw new Error("memory broker maintenance timeout is invalid");
}
const maintainNow = async (operation: "quiesce" | "resume"): Promise<void> => {
if (hasChildExited() || !child.connected) {
throw new Error("memory broker child is unavailable for maintenance");
}
const requestId = randomUUID();
@@ -248,7 +278,7 @@ export async function startMemoryBrokerProcess(params: {
finish((message as { ok?: unknown }).ok === true);
}
};
const timer = setTimeout(() => finish(false), MEMORY_BROKER_MAINTENANCE_TIMEOUT_MS);
const timer = setTimeout(() => finish(false), maintenanceTimeoutMs);
timer.unref?.();
child.on("message", onMessage);
child.send({ type: "maintenance", requestId, brokerEpoch, operation }, (error) => {
@@ -258,20 +288,32 @@ export async function startMemoryBrokerProcess(params: {
});
});
if (!ok) {
// A late child acknowledgement could otherwise arrive after the parent has given up and
// leave a healthy-but-quiesced socket rejecting every future request. Retire the whole
// epoch instead; the next independently authorized request creates a fresh child/secret.
await retireChild();
throw new Error(`memory broker ${operation} is unavailable`);
}
};
let maintenanceTail = Promise.resolve();
const maintain = (operation: "quiesce" | "resume"): Promise<void> => {
const current = maintenanceTail.then(() => maintainNow(operation));
// Parent-child maintenance messages must preserve order even when a caller observes a
// failure. Swallow only for tail progression; the original caller still receives the error.
maintenanceTail = current.catch(() => undefined);
return current;
};
return Object.freeze({
client,
brokerEpoch,
isRunning: () => !childExited && child.exitCode === null && !child.killed,
isRunning: () => !hasChildExited(),
isHealthy,
quiesce: () => maintain("quiesce"),
resume: () => maintain("resume"),
close: async () => {
await maintain("quiesce").catch(() => undefined);
await new Promise<void>((resolve) => {
if (child.exitCode !== null || child.killed) {
if (hasChildExited()) {
resolve();
return;
}
+31
View File
@@ -15,6 +15,7 @@ const binding: MemoryBrokerAuthorizationBinding = {
runId: "run-a",
contextFingerprint: "context-a",
subjectRevision: "subject-a",
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
actorRevision: "actor-a",
capabilitySnapshotId: "capability-a",
policyRevision: "policy-a",
@@ -58,6 +59,26 @@ function verify(
}
describe("memory broker envelope", () => {
it("refuses a request whose canonical form exceeds the nesting budget", () => {
let payload: unknown = "leaf";
for (let depth = 0; depth < 65; depth += 1) {
payload = [payload];
}
expect(
createMemoryBrokerEnvelope({
secret,
brokerId: "broker-a",
brokerEpoch: "epoch-a",
binding,
nonce: "nested-request",
request: { method: "memory.search", payload },
issuedAtMs: 1_000,
expiresAtMs: 2_000,
}),
).toBeUndefined();
});
it("binds a deterministic request and every Gateway-derived authorization revision", () => {
const envelope = createEnvelope();
expect(verify(envelope)).toEqual({ ok: true });
@@ -83,6 +104,16 @@ describe("memory broker envelope", () => {
}),
).toEqual({ ok: false, reason: "binding-mismatch" });
}
expect(
verify(envelope, {
expectedBinding: { ...binding, actor: { ...binding.actor, principalId: "bob" } },
}),
).toEqual({ ok: false, reason: "binding-mismatch" });
expect(
verify(envelope, {
expectedBinding: { ...binding, actor: { ...binding.actor, actorKind: "service" } },
}),
).toEqual({ ok: false, reason: "binding-mismatch" });
});
it("rejects cross-broker, stale-epoch, expired, and forged envelopes", () => {
+64 -17
View File
@@ -4,6 +4,24 @@ import { createHash, createHmac, timingSafeEqual } from "node:crypto";
export const MEMORY_BROKER_PROTOCOL_VERSION = 1 as const;
export const MEMORY_BROKER_MAXIMUM_NONCE_LENGTH = 256;
export const MEMORY_BROKER_MAXIMUM_CANONICAL_DEPTH = 64;
export const MEMORY_BROKER_MAXIMUM_CANONICAL_NODES = 10_000;
/**
* A signed broker frame must bind the actor identity as well as the evidence revision. Revisions
* can be shared by a provider snapshot, so binding only that revision would let a caller retarget
* a valid Alice request to Bob while preserving every other frame field.
*/
export type MemoryBrokerActorBinding =
| Readonly<{
kind: "principal";
actorKind: "human" | "agent" | "service" | "system";
principalId: string;
}>
| Readonly<{
kind: "unattributed";
transportAuditRef: string;
}>;
export type MemoryBrokerAuthorizationBinding = Readonly<{
agentId: string;
@@ -11,6 +29,7 @@ export type MemoryBrokerAuthorizationBinding = Readonly<{
runId: string;
contextFingerprint: string;
subjectRevision: string;
actor: MemoryBrokerActorBinding;
actorRevision: string;
capabilitySnapshotId: string;
policyRevision: string;
@@ -62,7 +81,17 @@ function isPlainRecord(value: unknown): value is Record<string, unknown> {
* The MAC covers a deterministic JSON representation. Rejecting non-JSON values is deliberate:
* a request that cannot be reproduced byte-for-byte cannot be safely bound to a single-use grant.
*/
function canonicalize(value: unknown): string | undefined {
function canonicalize(
value: unknown,
state: { nodes: number } = { nodes: 0 },
depth = 0,
): string | undefined {
if (
depth > MEMORY_BROKER_MAXIMUM_CANONICAL_DEPTH ||
++state.nodes > MEMORY_BROKER_MAXIMUM_CANONICAL_NODES
) {
return undefined;
}
if (value === null) {
return "null";
}
@@ -77,7 +106,7 @@ function canonicalize(value: unknown): string | undefined {
if (Array.isArray(value)) {
const items: string[] = [];
for (const item of value) {
const serialized = canonicalize(item);
const serialized = canonicalize(item, state, depth + 1);
if (serialized === undefined) {
return undefined;
}
@@ -90,7 +119,7 @@ function canonicalize(value: unknown): string | undefined {
}
const entries: string[] = [];
for (const key of Object.keys(value).toSorted()) {
const serialized = canonicalize(value[key]);
const serialized = canonicalize(value[key], state, depth + 1);
if (serialized === undefined) {
return undefined;
}
@@ -125,24 +154,42 @@ function hasSafeIdentifier(value: unknown): value is string {
);
}
function isActorBinding(value: unknown): value is MemoryBrokerActorBinding {
if (!isPlainRecord(value)) {
return false;
}
if (value.kind === "principal") {
return (
(value.actorKind === "human" ||
value.actorKind === "agent" ||
value.actorKind === "service" ||
value.actorKind === "system") &&
hasSafeIdentifier(value.principalId)
);
}
return value.kind === "unattributed" && hasSafeIdentifier(value.transportAuditRef);
}
function isBinding(value: unknown): value is MemoryBrokerAuthorizationBinding {
if (!isPlainRecord(value)) {
return false;
}
return [
value.agentId,
value.sessionId,
value.runId,
value.contextFingerprint,
value.subjectRevision,
value.actorRevision,
value.capabilitySnapshotId,
value.policyRevision,
value.deliveryRevision,
].every(hasSafeIdentifier);
return (
[
value.agentId,
value.sessionId,
value.runId,
value.contextFingerprint,
value.subjectRevision,
value.actorRevision,
value.capabilitySnapshotId,
value.policyRevision,
value.deliveryRevision,
].every(hasSafeIdentifier) && isActorBinding(value.actor)
);
}
function isEnvelope(value: unknown): value is MemoryBrokerEnvelope {
export function isMemoryBrokerEnvelope(value: unknown): value is MemoryBrokerEnvelope {
if (
!isPlainRecord(value) ||
value.version !== MEMORY_BROKER_PROTOCOL_VERSION ||
@@ -185,7 +232,7 @@ export function createMemoryBrokerEnvelope(params: {
expiresAtMs: params.expiresAtMs,
requestDigest: requestDigest ?? "",
};
if (!requestDigest || !isEnvelope({ ...unsigned, signature: "pending" })) {
if (!requestDigest || !isMemoryBrokerEnvelope({ ...unsigned, signature: "pending" })) {
return undefined;
}
const signature = sign(params.secret, unsigned);
@@ -203,7 +250,7 @@ export function verifyMemoryBrokerEnvelope(params: {
nowMs: number;
}): MemoryBrokerEnvelopeVerification {
const { envelope } = params;
if (!isEnvelope(envelope)) {
if (!isMemoryBrokerEnvelope(envelope)) {
return { ok: false, reason: "invalid-envelope" };
}
if (envelope.brokerId !== params.brokerId) {
+206 -4
View File
@@ -1,5 +1,6 @@
import { randomBytes } from "node:crypto";
import { mkdtemp, rm, stat } from "node:fs/promises";
import net from "node:net";
import { tmpdir } from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
@@ -7,6 +8,7 @@ import { requestJsonlSocket } from "../infra/jsonl-socket.js";
import { createMemoryBrokerClient } from "./client.js";
import {
createMemoryBrokerEnvelope,
MEMORY_BROKER_MAXIMUM_CANONICAL_DEPTH,
type MemoryBrokerAuthorizationBinding,
type MemoryBrokerRequest,
} from "./protocol.js";
@@ -19,6 +21,7 @@ const binding: MemoryBrokerAuthorizationBinding = {
runId: "run-a",
contextFingerprint: "context-a",
subjectRevision: "subject-a",
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
actorRevision: "actor-a",
capabilitySnapshotId: "capability-a",
policyRevision: "policy-a",
@@ -32,7 +35,7 @@ let server: MemoryBrokerServer | undefined;
type StartOptions = Partial<
Pick<
Parameters<typeof startMemoryBrokerServer>[0],
"handler" | "maximumPending" | "maximumRunning"
"handler" | "maximumPending" | "maximumRunning" | "maximumConnections" | "preauthIdleTimeoutMs"
>
>;
@@ -77,16 +80,35 @@ function frame(overrides: Partial<{ envelope: unknown; request: unknown; nonce:
return { envelope, request, ...overrides };
}
async function send(socketPath: string, value: unknown, options: { keepWriteOpen?: boolean } = {}) {
async function send(
socketPath: string,
value: unknown,
options: { keepWriteOpen?: boolean; timeoutMs?: number } = {},
) {
return await requestJsonlSocket({
socketPath,
requestLine: JSON.stringify(value),
timeoutMs: 5_000,
requestLine: typeof value === "string" ? value : JSON.stringify(value),
timeoutMs: options.timeoutMs ?? 5_000,
...options,
accept: (response) => response as { ok: boolean; value?: unknown; error?: string },
});
}
async function openPartialSocket(socketPath: string): Promise<net.Socket> {
return await new Promise<net.Socket>((resolve, reject) => {
const socket = net.createConnection(socketPath);
socket.once("connect", () => resolve(socket));
socket.once("error", reject);
});
}
async function waitForSocketClose(socket: net.Socket): Promise<void> {
if (socket.destroyed) {
return;
}
await new Promise<void>((resolve) => socket.once("close", () => resolve()));
}
describe("memory broker server", () => {
it("keeps the local socket private and admits one signed request", async () => {
const socketPath = await start();
@@ -280,6 +302,124 @@ describe("memory broker server", () => {
await expect(handlerAborted).resolves.toBeUndefined();
});
it("reclaims a cancelled queued request before it can exhaust the broker admission limit", async () => {
let release: (() => void) | undefined;
let started: (() => void) | undefined;
const handlerStarted = new Promise<void>((resolve) => {
started = resolve;
});
const handlerRelease = new Promise<void>((resolve) => {
release = resolve;
});
const socketPath = await start({
maximumPending: 1,
maximumRunning: 1,
handler: async () => {
started?.();
await handlerRelease;
return { ok: true };
},
});
const first = send(socketPath, frame({ nonce: "nonce-running" }), { keepWriteOpen: true });
await handlerStarted;
const controller = new AbortController();
const client = createMemoryBrokerClient({
socketPath,
brokerId: "broker-a",
brokerEpoch: "epoch-a",
secret,
nonce: () => "nonce-cancelled-queued",
});
const cancelled = client.request({
binding,
method: "memory.search",
payload: { query: "queued then cancelled" },
expiresAtMs: Date.now() + 60_000,
signal: controller.signal,
});
// Let the client finish its connect/write turn, then prove it holds the sole pending slot.
await new Promise<void>((resolve) => setTimeout(resolve, 20));
await expect(send(socketPath, frame({ nonce: "nonce-before-cancel" }))).resolves.toEqual({
ok: false,
error: "busy",
});
controller.abort();
await expect(cancelled).resolves.toBeUndefined();
const reclaimed = send(socketPath, frame({ nonce: "nonce-after-cancel" }), {
keepWriteOpen: true,
});
// The admission decision must happen while the first handler is still saturated. Without
// queue-owned cancellation, this request is synchronously rejected as busy.
await new Promise<void>((resolve) => setTimeout(resolve, 20));
release?.();
await expect(first).resolves.toEqual({ ok: true, value: { ok: true } });
await expect(reclaimed).resolves.toEqual({ ok: true, value: { ok: true } });
});
it("expires queued work without waiting for a running handler to drain", async () => {
let release: (() => void) | undefined;
let started: (() => void) | undefined;
const handlerStarted = new Promise<void>((resolve) => {
started = resolve;
});
const handlerRelease = new Promise<void>((resolve) => {
release = resolve;
});
const socketPath = await start({
maximumPending: 1,
maximumRunning: 1,
handler: async () => {
started?.();
await handlerRelease;
return { ok: true };
},
});
const first = send(socketPath, frame({ nonce: "nonce-running" }), { keepWriteOpen: true });
await handlerStarted;
const expiresAtMs = Date.now() + 100;
const expired = send(
socketPath,
frame({
nonce: "nonce-queued-expiry",
envelope: createMemoryBrokerEnvelope({
secret,
brokerId: "broker-a",
brokerEpoch: "epoch-a",
binding,
nonce: "nonce-queued-expiry",
request,
issuedAtMs: Date.now(),
expiresAtMs,
}),
}),
{ keepWriteOpen: true, timeoutMs: 1_000 },
);
await expect(expired).resolves.toEqual({ ok: false, error: "cancelled" });
const reclaimed = send(socketPath, frame({ nonce: "nonce-after-expiry" }), {
keepWriteOpen: true,
});
release?.();
await expect(first).resolves.toEqual({ ok: true, value: { ok: true } });
await expect(reclaimed).resolves.toEqual({ ok: true, value: { ok: true } });
});
it("rejects a deeply nested forged request without unwinding the socket callback", async () => {
const socketPath = await start();
const envelope = frame({ nonce: "nonce-deep-forgery" }).envelope;
const depth = MEMORY_BROKER_MAXIMUM_CANONICAL_DEPTH * 512;
const payload = `${"[".repeat(depth)}0${"]".repeat(depth)}`;
const rawFrame = `{"envelope":${JSON.stringify(envelope)},"request":{"method":"memory.search","payload":${payload}}}`;
await expect(send(socketPath, rawFrame, { timeoutMs: 1_000 })).resolves.toEqual({
ok: false,
error: "unauthorized",
});
});
it("cancels an in-flight handler at the signed request deadline", async () => {
let started: (() => void) | undefined;
const handlerStarted = new Promise<void>((resolve) => {
@@ -314,4 +454,66 @@ describe("memory broker server", () => {
await handlerStarted;
await expect(pending).resolves.toEqual({ ok: false, error: "cancelled" });
});
it("acknowledges a durable mutation that commits as its reply deadline expires", async () => {
let started: (() => void) | undefined;
const handlerStarted = new Promise<void>((resolve) => {
started = resolve;
});
const socketPath = await start({
handler: async ({ signal }) => {
started?.();
await new Promise<void>((resolve) => {
signal.addEventListener("abort", () => resolve(), { once: true });
});
return { status: "committed" };
},
});
const mutationRequest: MemoryBrokerRequest = {
method: "memory.write",
payload: { mutation: "durably-activated" },
};
const deadline = Date.now() + 100;
const pending = send(
socketPath,
frame({
nonce: "nonce-committed-after-deadline",
request: mutationRequest,
envelope: createMemoryBrokerEnvelope({
secret,
brokerId: "broker-a",
brokerEpoch: "epoch-a",
binding,
nonce: "nonce-committed-after-deadline",
request: mutationRequest,
issuedAtMs: Date.now(),
expiresAtMs: deadline,
}),
}),
);
await handlerStarted;
await expect(pending).resolves.toEqual({ ok: true, value: { status: "committed" } });
});
it("bounds unauthenticated partial-frame sockets and reclaims their admission slots", async () => {
const socketPath = await start({ maximumConnections: 1, preauthIdleTimeoutMs: 40 });
const slowloris = await openPartialSocket(socketPath);
slowloris.write('{"envelope":');
await expect(send(socketPath, frame({ nonce: "nonce-over-limit" }))).resolves.toBeNull();
await waitForSocketClose(slowloris);
await expect(send(socketPath, frame({ nonce: "nonce-after-idle" }))).resolves.toMatchObject({
ok: true,
});
});
it("tears down partial-frame sockets before broker shutdown", async () => {
const socketPath = await start({ preauthIdleTimeoutMs: 60_000 });
const slowloris = await openPartialSocket(socketPath);
slowloris.write('{"envelope":');
await expect(server!.close()).resolves.toBeUndefined();
await waitForSocketClose(slowloris);
await expect(server!.close()).resolves.toBeUndefined();
});
});
+147 -34
View File
@@ -3,13 +3,15 @@ import net from "node:net";
import { isRecord } from "@openclaw/normalization-core/record-coerce";
import {
MemoryBrokerNonceLedger,
isMemoryBrokerEnvelope,
verifyMemoryBrokerEnvelope,
type MemoryBrokerAuthorizationBinding,
type MemoryBrokerEnvelope,
type MemoryBrokerRequest,
} from "./protocol.js";
export const MEMORY_BROKER_MAXIMUM_REQUEST_BYTES = 1_048_576;
export const MEMORY_BROKER_MAXIMUM_CONNECTIONS = 32;
export const MEMORY_BROKER_PREAUTH_IDLE_TIMEOUT_MS = 5_000;
type MemoryBrokerWireRequest = Readonly<{
envelope: unknown;
@@ -34,6 +36,7 @@ type PendingMemoryBrokerRequest = Readonly<{
deadlineMs: number;
execute: () => Promise<void>;
rejectBusy: () => void;
rejectCancelled: () => void;
}>;
/**
@@ -43,6 +46,7 @@ type PendingMemoryBrokerRequest = Readonly<{
class MemoryBrokerAdmissionQueue {
private readonly pending: PendingMemoryBrokerRequest[] = [];
private readonly idleWaiters = new Set<() => void>();
private deadlineTimer: ReturnType<typeof setTimeout> | undefined;
private running = 0;
private accepting = true;
@@ -61,17 +65,21 @@ class MemoryBrokerAdmissionQueue {
}
}
submit(request: PendingMemoryBrokerRequest): void {
submit(request: PendingMemoryBrokerRequest): () => void {
if (
!this.accepting ||
request.deadlineMs <= this.now() ||
(this.running >= this.maximumRunning && this.pending.length >= this.maximumPending)
) {
request.rejectBusy();
return;
return () => {};
}
if (request.deadlineMs <= this.now()) {
request.rejectCancelled();
return () => {};
}
this.pending.push(request);
this.drain();
return () => this.cancel(request);
}
async quiesce(): Promise<void> {
@@ -101,7 +109,7 @@ class MemoryBrokerAdmissionQueue {
while (this.running < this.maximumRunning && this.pending.length > 0) {
const next = this.pending.shift()!;
if (next.deadlineMs <= this.now()) {
next.rejectBusy();
next.rejectCancelled();
continue;
}
this.running += 1;
@@ -111,8 +119,50 @@ class MemoryBrokerAdmissionQueue {
this.notifyIdle();
});
}
this.schedulePendingDeadline();
this.notifyIdle();
}
private cancel(request: PendingMemoryBrokerRequest): void {
const index = this.pending.indexOf(request);
if (index === -1) {
return;
}
this.pending.splice(index, 1);
this.schedulePendingDeadline();
this.drain();
}
private schedulePendingDeadline(): void {
if (this.deadlineTimer) {
clearTimeout(this.deadlineTimer);
this.deadlineTimer = undefined;
}
const nextDeadline = this.pending.reduce<number | undefined>(
(earliest, request) =>
earliest === undefined || request.deadlineMs < earliest ? request.deadlineMs : earliest,
undefined,
);
if (nextDeadline === undefined) {
return;
}
this.deadlineTimer = setTimeout(
() => {
this.deadlineTimer = undefined;
const now = this.now();
for (let index = this.pending.length - 1; index >= 0; index -= 1) {
const request = this.pending[index];
if (request.deadlineMs <= now) {
this.pending.splice(index, 1);
request.rejectCancelled();
}
}
this.drain();
},
Math.max(1, nextDeadline - this.now()),
);
this.deadlineTimer.unref?.();
}
}
function parseRequest(value: unknown): MemoryBrokerWireRequest | undefined {
@@ -140,15 +190,12 @@ function writeResponse(socket: net.Socket, response: MemoryBrokerWireResponse):
}
}
function asEnvelope(value: unknown): MemoryBrokerEnvelope | undefined {
if (
!isRecord(value) ||
typeof value.nonce !== "string" ||
typeof value.expiresAtMs !== "number"
) {
return undefined;
}
return value as unknown as MemoryBrokerEnvelope;
/**
* A successful mutation handler has crossed its durable activation boundary. Its response must
* remain committed even if the caller disconnects while the broker is serializing the reply.
*/
function isDurableMemoryMutation(request: MemoryBrokerRequest): boolean {
return request.method === "memory.write" || request.method === "memory.import";
}
export type MemoryBrokerServer = Readonly<{
@@ -174,53 +221,96 @@ export async function startMemoryBrokerServer(params: {
nonceCapacity?: number;
maximumPending?: number;
maximumRunning?: number;
/** Bound unauthenticated sockets before they can consume a broker worker slot. */
maximumConnections?: number;
/** A partial frame is untrusted admission state, never an indefinitely retained request. */
preauthIdleTimeoutMs?: number;
maximumRequestBytes?: number;
now?: () => number;
}): Promise<MemoryBrokerServer> {
const now = params.now ?? Date.now;
const maximumRequestBytes = params.maximumRequestBytes ?? MEMORY_BROKER_MAXIMUM_REQUEST_BYTES;
const maximumConnections = params.maximumConnections ?? MEMORY_BROKER_MAXIMUM_CONNECTIONS;
const preauthIdleTimeoutMs = params.preauthIdleTimeoutMs ?? MEMORY_BROKER_PREAUTH_IDLE_TIMEOUT_MS;
if (
!Number.isSafeInteger(maximumConnections) ||
maximumConnections < 1 ||
!Number.isSafeInteger(preauthIdleTimeoutMs) ||
preauthIdleTimeoutMs < 1
) {
throw new Error("memory broker connection limits are invalid");
}
const nonceLedger = new MemoryBrokerNonceLedger(params.nonceCapacity ?? 1_024);
const queue = new MemoryBrokerAdmissionQueue(
params.maximumPending ?? 128,
params.maximumRunning ?? 8,
now,
);
const sockets = new Set<net.Socket>();
const server = net.createServer({ allowHalfOpen: true }, (socket) => {
let buffer = Buffer.alloc(0);
// A client that has not completed one bounded signed frame has no broker authority. Keep its
// footprint bounded before parsing so partial-frame clients cannot exhaust the broker.
if (sockets.size >= maximumConnections) {
socket.destroy();
return;
}
sockets.add(socket);
// Do not repeatedly concatenate an attacker-controlled partial frame: a byte-at-a-time
// slowloris would turn that into quadratic copying before the absolute admission deadline.
const frameChunks: Buffer[] = [];
let frameByteLength = 0;
let consumed = false;
const requestAbort = new AbortController();
const preauthIdleTimer = setTimeout(() => {
if (!consumed) {
consumed = true;
requestAbort.abort();
socket.destroy();
}
}, preauthIdleTimeoutMs);
preauthIdleTimer.unref?.();
socket.on("end", () => requestAbort.abort());
socket.on("close", () => requestAbort.abort());
socket.once("close", () => {
clearTimeout(preauthIdleTimer);
requestAbort.abort();
sockets.delete(socket);
});
socket.on("error", () => requestAbort.abort());
socket.on("data", (chunk: Buffer) => {
if (consumed) {
socket.destroy();
return;
}
if (buffer.byteLength + chunk.byteLength > maximumRequestBytes) {
if (frameByteLength + chunk.byteLength > maximumRequestBytes) {
consumed = true;
writeResponse(socket, { ok: false, error: "invalid-request" });
return;
}
buffer = Buffer.concat([buffer, chunk]);
const newline = buffer.indexOf(0x0a);
frameChunks.push(chunk);
frameByteLength += chunk.byteLength;
// Every prior chunk has been checked and contains no newline, so only the latest chunk
// needs scanning until there is one complete frame to concatenate exactly once.
const newline = chunk.indexOf(0x0a);
if (newline === -1) {
return;
}
consumed = true;
if (buffer.subarray(newline + 1).some((byte) => byte !== 0x0a && byte !== 0x0d)) {
clearTimeout(preauthIdleTimer);
const buffer = Buffer.concat(frameChunks, frameByteLength);
const frameNewline = buffer.indexOf(0x0a);
if (buffer.subarray(frameNewline + 1).some((byte) => byte !== 0x0a && byte !== 0x0d)) {
writeResponse(socket, { ok: false, error: "invalid-request" });
return;
}
let frame: MemoryBrokerWireRequest | undefined;
try {
frame = parseRequest(JSON.parse(buffer.subarray(0, newline).toString("utf8")));
frame = parseRequest(JSON.parse(buffer.subarray(0, frameNewline).toString("utf8")));
} catch {
frame = undefined;
}
const request = frame ? parseMemoryBrokerRequest(frame.request) : undefined;
const envelope = frame ? asEnvelope(frame.envelope) : undefined;
if (!request || !envelope) {
const envelope = frame?.envelope;
if (!request || !isMemoryBrokerEnvelope(envelope)) {
writeResponse(socket, { ok: false, error: "invalid-request" });
return;
}
@@ -246,9 +336,10 @@ export async function startMemoryBrokerServer(params: {
writeResponse(socket, { ok: false, error: "replayed" });
return;
}
queue.submit({
const cancelPending = queue.submit({
deadlineMs: envelope.expiresAtMs,
rejectBusy: () => writeResponse(socket, { ok: false, error: "busy" }),
rejectCancelled: () => writeResponse(socket, { ok: false, error: "cancelled" }),
execute: async () => {
if (requestAbort.signal.aborted || now() >= envelope.expiresAtMs) {
writeResponse(socket, { ok: false, error: "cancelled" });
@@ -264,18 +355,31 @@ export async function startMemoryBrokerServer(params: {
request,
signal: requestAbort.signal,
});
if (requestAbort.signal.aborted || now() >= envelope.expiresAtMs) {
if (
!isDurableMemoryMutation(request) &&
(requestAbort.signal.aborted || now() >= envelope.expiresAtMs)
) {
writeResponse(socket, { ok: false, error: "cancelled" });
return;
}
writeResponse(socket, { ok: true, value });
} catch {
writeResponse(socket, { ok: false, error: "failed" });
writeResponse(socket, {
ok: false,
error:
requestAbort.signal.aborted || now() >= envelope.expiresAtMs
? "cancelled"
: "failed",
});
} finally {
clearTimeout(deadline);
}
},
});
requestAbort.signal.addEventListener("abort", cancelPending, { once: true });
if (requestAbort.signal.aborted) {
cancelPending();
}
});
});
await unlink(params.socketPath).catch((error: unknown) => {
@@ -291,20 +395,29 @@ export async function startMemoryBrokerServer(params: {
});
});
await chmod(params.socketPath, 0o600);
let closePromise: Promise<void> | undefined;
return Object.freeze({
socketPath: params.socketPath,
brokerEpoch: params.brokerEpoch,
quiesce: () => queue.quiesce(),
resume: () => queue.resume(),
close: async () => {
await new Promise<void>((resolve, reject) => {
server.close((error) => (error ? reject(error) : resolve()));
});
await unlink(params.socketPath).catch((error: unknown) => {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
throw error;
close: () => {
closePromise ??= (async () => {
// net.Server.close waits for every accepted socket. Destroy incomplete and in-flight
// connections first so shutdown cannot be held hostage by a slowloris or stalled client.
for (const socket of sockets) {
socket.destroy();
}
});
await new Promise<void>((resolve, reject) => {
server.close((error) => (error ? reject(error) : resolve()));
});
await unlink(params.socketPath).catch((error: unknown) => {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
throw error;
}
});
})();
return closePromise;
},
});
}
+23
View File
@@ -1,3 +1,12 @@
let initializedAgentIds = [];
export async function initializeMemoryBroker({ agentIds }) {
if (agentIds.includes("fail-startup")) {
throw new Error("test broker startup recovery failed");
}
initializedAgentIds = [...agentIds];
}
export function createMemoryBrokerHandler() {
return async ({ binding, request }) => {
if (request.method === "memory.crash") {
@@ -6,6 +15,20 @@ export function createMemoryBrokerHandler() {
setImmediate(() => process.exit(1));
return await new Promise(() => {});
}
if (request.method === "memory.kill") {
// An external SIGKILL records `signalCode`, not `exitCode`. The parent must still retire
// this process instead of waiting for an exit event that has already happened.
setImmediate(() => process.kill(process.pid, "SIGKILL"));
return await new Promise(() => {});
}
if (request.method === "memory.hang") {
// This intentionally ignores the broker AbortSignal. Maintenance must retire the child
// rather than leave an admission-closed but otherwise healthy process behind.
return await new Promise(() => {});
}
if (request.method === "memory.startup") {
return { agentIds: initializedAgentIds };
}
return request.method === "memory.environment"
? { parentSecret: process.env.OPENCLAW_MEMORY_BROKER_TEST_SECRET ? "present" : "absent" }
: { agentId: binding.agentId, method: request.method };
@@ -18,6 +18,7 @@ const envelope = {
runId: "run-b",
contextFingerprint: "context-b",
subjectRevision: "subject-b",
actor: { kind: "principal", actorKind: "human", principalId: "mallory" },
actorRevision: "actor-b",
capabilitySnapshotId: "capability-b",
policyRevision: "policy-b",
+6 -23
View File
@@ -9,10 +9,7 @@ import {
type NodeWorkerLaunchClaimResult,
type NodeWorkerLaunchReceipt,
} from "./node-worker-launch-store.js";
import {
inspectNodeWorkerProcessIdentity,
type NodeWorkerProcessIdentity,
} from "./node-worker-process-identity.js";
import { type NodeWorkerProcessIdentity } from "./node-worker-process-identity.js";
const DEFAULT_WORKER_CAPACITY = 2;
const DEFAULT_CAPACITY_WAIT_MS = 10_000;
@@ -69,28 +66,14 @@ export class NodeWorkerCapacity {
}
async initialize(
recoverRunning: (receipt: NodeWorkerLaunchReceipt) => Promise<void>,
recoverNonterminal: (receipt: NodeWorkerLaunchReceipt) => Promise<void>,
): Promise<void> {
this.onCapacityChanged?.(this.publishedCapacity);
for (const receipt of this.store.listNonterminal()) {
if (receipt.state === "pending") {
const supervisorState = inspectNodeWorkerProcessIdentity(receipt.supervisor);
if (supervisorState === "dead" || supervisorState === "reused") {
this.finish(
{
launchId: receipt.launchId,
planHash: receipt.planHash,
supervisor: receipt.supervisor,
worker: null,
state: "interrupted",
errorText: "node host stopped before the worker launch started",
},
false,
);
}
continue;
}
await recoverRunning(receipt);
// Pending work can already own a staged projection or relay directory.
// Recovery therefore belongs to the supervisor, which owns those paths,
// rather than directly settling the launch slot here.
await recoverNonterminal(receipt);
}
this.store.pruneExpiredTerminal();
this.refresh(true);
@@ -0,0 +1,160 @@
import fs from "node:fs";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { verifyNodeWorkerContainerProjectionIsolation } from "../../test/helpers/node-worker-container-projection-isolation.js";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import type { AuthorizedMemoryVirtualFileBroker } from "../agents/memory-authorized-read-host.js";
import { execContainer } from "../agents/sandbox/docker.js";
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
import {
buildNodeWorkerContainerRunArgs,
removeOwnedNodeWorkerContainers,
resolveNodeWorkerContainerEngine,
type NodeWorkerContainerEngine,
} from "./node-worker-container-runtime.js";
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
afterEach(() => {
closeOpenClawStateDatabaseForTest();
});
function broker(
expiresAt = new Date(Date.now() + 60_000).toISOString(),
): AuthorizedMemoryVirtualFileBroker {
const files = new Map([["shared/brief.md", "only this issued virtual view"]]);
return {
view: {
version: 1,
viewId: "container-e2e-view",
planId: "container-e2e-plan",
contextFingerprint: "container-e2e-context",
revision: "container-e2e-revision",
roots: [
{
version: 1,
mountHandle: "container-e2e-mount",
virtualRoot: "shared",
access: "read",
},
],
files: [
{
version: 1,
mountHandle: "container-e2e-mount",
virtualPath: "shared/brief.md",
},
],
expiresAt,
},
readFile: async (virtualPath) => files.get(virtualPath),
};
}
describe.runIf(process.env.OPENCLAW_PROCESS_ISOLATION_E2E === "1")(
"node worker container process isolation",
() => {
it("exposes only an immutable issued memory snapshot to the hostile worker process", async () => {
const root = tempDirs.make("node-worker-container-isolation-");
const bundleDir = path.join(root, "bundle");
const relayDir = path.join(root, "relay");
const memoryDir = path.join(root, "memory");
const workspaceDir = path.join(root, "workspace");
const outsideArtifact = path.join(root, "host-artifact.sqlite");
const identity = { launchId: "hostile-worker-turn", planHash: "a".repeat(64) };
for (const directory of [bundleDir, relayDir, memoryDir, workspaceDir]) {
fs.mkdirSync(directory, { recursive: true });
}
fs.mkdirSync(path.join(memoryDir, "shared"));
fs.writeFileSync(path.join(memoryDir, "shared", "issued.md"), "issued memory only", {
mode: 0o400,
});
fs.chmodSync(path.join(memoryDir, "shared"), 0o500);
fs.writeFileSync(outsideArtifact, "host-only artifact");
fs.writeFileSync(
path.join(bundleDir, "worker.mjs"),
[
'import fs from "node:fs";',
`const outsideArtifact = ${JSON.stringify(outsideArtifact)};`,
'const attempt = (operation) => { try { operation(); return "allowed"; } catch (error) { return error && typeof error === "object" && "code" in error ? String(error.code) : "denied"; } };',
"const result = {",
" uid: process.getuid(),",
' issued: fs.readFileSync("/memory/shared/issued.md", "utf8"),',
' memoryWrite: attempt(() => fs.writeFileSync("/memory/shared/issued.md", "tampered")),',
' rawArtifactRead: attempt(() => fs.readFileSync(outsideArtifact, "utf8")),',
' workspaceWrite: attempt(() => fs.writeFileSync("/workspace/proof.txt", "allowed")),',
"};",
"process.stdout.write(JSON.stringify(result));",
].join("\n"),
{ mode: 0o500 },
);
let engine: NodeWorkerContainerEngine | undefined;
try {
engine = await resolveNodeWorkerContainerEngine();
if (!engine) {
throw new Error(
"process-isolation proof requires an eligible Docker or Podman node host",
);
}
await removeOwnedNodeWorkerContainers(identity, engine);
const user = process.getuid?.();
const group = process.getgid?.();
if (!user || !group) {
throw new Error("process-isolation proof requires a non-root POSIX test host");
}
const args = buildNodeWorkerContainerRunArgs({
engine,
identity,
mounts: { bundleDir, relayDir, memoryDir, workspaceDir },
uid: user,
gid: group,
});
const result = await execContainer(engine, args, { allowFailure: true });
expect(result.code, result.stderr).toBe(0);
const observed: unknown = JSON.parse(result.stdout);
expect(observed).toMatchObject({
uid: user,
issued: "issued memory only",
rawArtifactRead: expect.not.stringMatching(/^allowed$/u),
memoryWrite: expect.not.stringMatching(/^allowed$/u),
workspaceWrite: "allowed",
});
expect(fs.readFileSync(path.join(memoryDir, "shared", "issued.md"), "utf8")).toBe(
"issued memory only",
);
expect(fs.readFileSync(path.join(workspaceDir, "proof.txt"), "utf8")).toBe("allowed");
} finally {
fs.chmodSync(path.join(memoryDir, "shared"), 0o700);
if (engine) {
await removeOwnedNodeWorkerContainers(identity, engine);
}
}
});
it("withdraws an expired signed projection-backed container worker and its mounts", async () => {
const root = tempDirs.make("node-worker-supervisor-container-e2e-");
const outsideArtifactPath = path.join(root, "host-only-artifact.txt");
fs.writeFileSync(outsideArtifactPath, "host-only artifact");
const previousBrokerCredential =
process.env.OPENCLAW_PROCESS_ISOLATION_TEST_BROKER_CREDENTIAL;
process.env.OPENCLAW_PROCESS_ISOLATION_TEST_BROKER_CREDENTIAL = "test-broker-only-credential";
try {
await verifyNodeWorkerContainerProjectionIsolation({
root,
broker: broker(new Date(Date.now() + 15_000).toISOString()),
outsideArtifactPath,
outsideArtifactContents: "host-only artifact",
issuedVirtualPath: "shared/brief.md",
issuedContents: "only this issued virtual view",
forbiddenEnvironmentVariable: "OPENCLAW_PROCESS_ISOLATION_TEST_BROKER_CREDENTIAL",
});
} finally {
if (previousBrokerCredential === undefined) {
delete process.env.OPENCLAW_PROCESS_ISOLATION_TEST_BROKER_CREDENTIAL;
} else {
process.env.OPENCLAW_PROCESS_ISOLATION_TEST_BROKER_CREDENTIAL = previousBrokerCredential;
}
}
}, 90_000);
},
);
@@ -0,0 +1,174 @@
import fs from "node:fs";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
const containerMocks = vi.hoisted(() => ({ execContainer: vi.fn() }));
vi.mock("../agents/sandbox/docker.js", async () => {
const actual = await vi.importActual<typeof import("../agents/sandbox/docker.js")>(
"../agents/sandbox/docker.js",
);
return { ...actual, execContainer: containerMocks.execContainer };
});
import { DOCKER_SANDBOX_ENGINE } from "../agents/sandbox/docker.js";
import {
buildNodeWorkerContainerRunArgs,
NODE_WORKER_CONTAINER_IMAGE,
NODE_WORKER_CONTAINER_MEMORY_ROOT,
NODE_WORKER_CONTAINER_RELAY_ROOT,
NODE_WORKER_CONTAINER_WORKER_ROOT,
NODE_WORKER_CONTAINER_WORKSPACE,
nodeWorkerContainerName,
removeOwnedNodeWorkerContainers,
resolveNodeWorkerContainerEngine,
} from "./node-worker-container-runtime.js";
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
const identity = { launchId: "turn-1", planHash: "a".repeat(64) };
function mounts() {
const root = tempDirs.make("node-worker-container-");
const bundleDir = path.join(root, "bundle");
const relayDir = path.join(root, "relay");
const memoryDir = path.join(root, "memory");
const workspaceDir = path.join(root, "workspace");
for (const directory of [bundleDir, relayDir, memoryDir, workspaceDir]) {
fs.mkdirSync(directory);
}
return { bundleDir, relayDir, memoryDir, workspaceDir };
}
afterEach(() => {
containerMocks.execContainer.mockReset();
});
describe("node worker container runtime", () => {
it("prepares the fixed worker image before admitting Docker process isolation", async () => {
containerMocks.execContainer
.mockResolvedValueOnce({ stdout: "27.5.1\n", stderr: "", code: 0 })
.mockResolvedValueOnce({ stdout: "", stderr: "No such image", code: 1 })
.mockResolvedValueOnce({ stdout: "pulled\n", stderr: "", code: 0 });
await expect(resolveNodeWorkerContainerEngine()).resolves.toBe(DOCKER_SANDBOX_ENGINE);
expect(containerMocks.execContainer.mock.calls).toEqual([
[DOCKER_SANDBOX_ENGINE, ["info", "--format", "{{.ServerVersion}}"], expect.any(Object)],
[
DOCKER_SANDBOX_ENGINE,
["image", "inspect", "--format", "{{.Id}}", NODE_WORKER_CONTAINER_IMAGE],
expect.any(Object),
],
[DOCKER_SANDBOX_ENGINE, ["pull", NODE_WORKER_CONTAINER_IMAGE], expect.any(Object)],
]);
expect(containerMocks.execContainer.mock.calls[2]?.[2]).toEqual(
expect.objectContaining({ signal: expect.any(AbortSignal) }),
);
});
it("does not advertise process isolation when the fixed worker image cannot be prepared", async () => {
containerMocks.execContainer
.mockResolvedValueOnce({ stdout: "27.5.1\n", stderr: "", code: 0 })
.mockResolvedValueOnce({ stdout: "", stderr: "No such image", code: 1 })
.mockResolvedValueOnce({ stdout: "", stderr: "registry unavailable", code: 1 })
.mockResolvedValueOnce({ stdout: "", stderr: "podman unavailable", code: 1 });
await expect(resolveNodeWorkerContainerEngine()).resolves.toBeUndefined();
});
it("builds a closed non-root container policy with only the worker mounts", () => {
const paths = mounts();
const args = buildNodeWorkerContainerRunArgs({
engine: DOCKER_SANDBOX_ENGINE,
identity,
mounts: paths,
uid: 501,
gid: 20,
});
expect(args).toEqual(
expect.arrayContaining([
"--interactive",
"--network",
"none",
"--read-only",
"--cap-drop",
"ALL",
"--security-opt",
"no-new-privileges",
"--pids-limit",
"128",
"--memory",
"512m",
"--cpus",
"1",
"--user",
"501:20",
"--workdir",
NODE_WORKER_CONTAINER_WORKSPACE,
NODE_WORKER_CONTAINER_IMAGE,
]),
);
const mountValues = args.flatMap((value, index) =>
value === "--mount" ? [args[index + 1]] : [],
);
expect(mountValues).toEqual([
`type=bind,src=${fs.realpathSync(paths.bundleDir)},dst=${NODE_WORKER_CONTAINER_WORKER_ROOT},readonly`,
`type=bind,src=${fs.realpathSync(paths.relayDir)},dst=${NODE_WORKER_CONTAINER_RELAY_ROOT},readonly`,
`type=bind,src=${fs.realpathSync(paths.memoryDir)},dst=${NODE_WORKER_CONTAINER_MEMORY_ROOT},readonly`,
`type=bind,src=${fs.realpathSync(paths.workspaceDir)},dst=${NODE_WORKER_CONTAINER_WORKSPACE}`,
]);
expect(args).not.toContain("--privileged");
expect(args).not.toContain("--network=host");
});
it("does not claim cleanup after container enumeration fails", async () => {
containerMocks.execContainer.mockRejectedValueOnce(new Error("Docker daemon unavailable"));
await expect(removeOwnedNodeWorkerContainers(identity, DOCKER_SANDBOX_ENGINE)).rejects.toThrow(
"Docker daemon unavailable",
);
expect(containerMocks.execContainer).toHaveBeenCalledWith(
DOCKER_SANDBOX_ENGINE,
expect.any(Array),
expect.objectContaining({ signal: expect.any(AbortSignal) }),
);
});
it("rejects a symlinked memory mount before it reaches the container engine", () => {
const paths = mounts();
const symlink = path.join(path.dirname(paths.memoryDir), "memory-link");
fs.symlinkSync(paths.memoryDir, symlink);
expect(() =>
buildNodeWorkerContainerRunArgs({
engine: DOCKER_SANDBOX_ENGINE,
identity,
mounts: { ...paths, memoryDir: symlink },
uid: 501,
gid: 20,
}),
).toThrow("memory projection mount must be a real local directory");
});
it("keeps cleanup open when a label-verified container cannot be removed", async () => {
const containerId = "b".repeat(64);
containerMocks.execContainer
.mockResolvedValueOnce({ stdout: `${containerId}\n`, stderr: "", code: 0 })
.mockResolvedValueOnce({
stdout: `/${nodeWorkerContainerName(identity)}\nv1\n${identity.launchId}\n${identity.planHash}\n`,
stderr: "",
code: 0,
})
.mockRejectedValueOnce(new Error("permission denied"))
.mockResolvedValueOnce({ stdout: `${containerId}\n`, stderr: "", code: 0 });
await expect(removeOwnedNodeWorkerContainers(identity, DOCKER_SANDBOX_ENGINE)).rejects.toThrow(
"Docker could not remove a node worker container",
);
for (const [, , options] of containerMocks.execContainer.mock.calls) {
expect(options).toEqual(expect.objectContaining({ signal: expect.any(AbortSignal) }));
}
});
});
@@ -0,0 +1,372 @@
import fs from "node:fs";
import path from "node:path";
import { createHash } from "node:crypto";
import {
DOCKER_SANDBOX_ENGINE,
PODMAN_SANDBOX_ENGINE,
execContainer,
type SandboxContainerEngine,
} from "../agents/sandbox/docker.js";
/**
* Process-isolated workers deliberately use one pinned runtime instead of the
* configurable tool sandbox image. The image is part of the node-host TCB.
*/
export const NODE_WORKER_CONTAINER_IMAGE =
"docker.io/library/node:24-bookworm-slim@sha256:6f7b03f7c2c8e2e784dcf9295400527b9b1270fd37b7e9a7285cf83b6951452d";
export const NODE_WORKER_CONTAINER_WORKSPACE = "/workspace";
export const NODE_WORKER_CONTAINER_MEMORY_ROOT = "/memory";
export const NODE_WORKER_CONTAINER_WORKER_ROOT = "/opt/openclaw/worker";
export const NODE_WORKER_CONTAINER_RELAY_ROOT = "/run/openclaw";
export const NODE_WORKER_CONTAINER_RELAY_SOCKET = `${NODE_WORKER_CONTAINER_RELAY_ROOT}/gateway.sock`;
export const NODE_WORKER_CONTAINER_SHIM_FLAG = "--internal-worker-container-shim";
const NODE_WORKER_CONTAINER_LABEL = "openclaw.node-worker-container";
const NODE_WORKER_CONTAINER_LAUNCH_LABEL = "openclaw.node-worker-launch";
const NODE_WORKER_CONTAINER_PLAN_LABEL = "openclaw.node-worker-plan";
const NODE_WORKER_CONTAINER_FORMAT = "v1";
const CONTAINER_NAME_MAX_CHARS = 96;
const NODE_WORKER_CONTAINER_ENGINE_TIMEOUT_MS = 5_000;
const NODE_WORKER_CONTAINER_IMAGE_PREPARE_TIMEOUT_MS = 2 * 60 * 1_000;
export type NodeWorkerContainerEngine = Extract<SandboxContainerEngine, { id: "docker" | "podman" }>;
export type NodeWorkerContainerIdentity = Readonly<{
launchId: string;
planHash: string;
}>;
export type NodeWorkerContainerMounts = Readonly<{
bundleDir: string;
relayDir: string;
memoryDir: string;
workspaceDir: string;
}>;
export function nodeWorkerContainerEngineFor(
id: NodeWorkerContainerEngine["id"],
): NodeWorkerContainerEngine {
return id === "docker" ? DOCKER_SANDBOX_ENGINE : PODMAN_SANDBOX_ENGINE;
}
/**
* Probe and cleanup commands must not turn a wedged engine into an unbounded
* worker lifecycle operation. `run` is intentionally excluded: it is the
* worker lifetime itself and is owned by the shim's IPC lease.
*/
function runNodeWorkerContainerControlCommand(
engine: NodeWorkerContainerEngine,
args: string[],
options?: { allowFailure?: boolean },
) {
return execContainer(engine, args, {
...options,
signal: AbortSignal.timeout(NODE_WORKER_CONTAINER_ENGINE_TIMEOUT_MS),
});
}
/**
* The worker image is part of the fixed container TCB. Pull it before this
* host advertises process isolation: Docker's implicit cold pull can outlive
* the short exact-container start probe and otherwise turns a ready node into
* a failed first turn.
*/
async function prepareNodeWorkerContainerImage(engine: NodeWorkerContainerEngine): Promise<boolean> {
const existing = await runNodeWorkerContainerControlCommand(
engine,
["image", "inspect", "--format", "{{.Id}}", NODE_WORKER_CONTAINER_IMAGE],
{ allowFailure: true },
);
if (existing.code === 0 && existing.stdout.trim()) {
return true;
}
const pulled = await execContainer(engine, ["pull", NODE_WORKER_CONTAINER_IMAGE], {
allowFailure: true,
signal: AbortSignal.timeout(NODE_WORKER_CONTAINER_IMAGE_PREPARE_TIMEOUT_MS),
});
return pulled.code === 0;
}
/** A mode-0600 relay is usable only by a concrete, non-root POSIX user mapping. */
export function resolveNodeWorkerContainerUser(): { uid: number; gid: number } | undefined {
if (
process.platform === "win32" ||
typeof process.getuid !== "function" ||
typeof process.getgid !== "function"
) {
return undefined;
}
const uid = process.getuid();
const gid = process.getgid();
return uid === 0 || gid === 0 ? undefined : { uid, gid };
}
function requireOwnedAbsolutePath(value: string, label: string): string {
if (!path.isAbsolute(value) || value.includes("\0") || value.includes("\n") || value.includes("\r")) {
throw new Error(`node worker container ${label} must be an absolute local path`);
}
const lexicalStats = fs.lstatSync(value);
if (lexicalStats.isSymbolicLink() || !lexicalStats.isDirectory()) {
throw new Error(`node worker container ${label} must be a real local directory`);
}
const resolved = fs.realpathSync.native(value);
const stats = fs.lstatSync(resolved);
if (stats.isSymbolicLink() || !stats.isDirectory()) {
throw new Error(`node worker container ${label} must be a real local directory`);
}
return resolved;
}
function formatMount(source: string, target: string, readOnly: boolean): string {
return `type=bind,src=${source},dst=${target}${readOnly ? ",readonly" : ""}`;
}
function safeContainerNameComponent(value: string): string {
const normalized = value.toLowerCase().replaceAll(/[^a-z0-9_.-]/gu, "-");
const suffix = createHash("sha256").update(value).digest("hex").slice(0, 16);
const prefix = normalized.replaceAll(/^-+|[-.]+$/gu, "").slice(0, 48) || "worker";
return `${prefix}-${suffix}`;
}
/** A deterministic but collision-resistant container name, never a caller-provided Docker argument. */
export function nodeWorkerContainerName(identity: NodeWorkerContainerIdentity): string {
const value = `openclaw-worker-${safeContainerNameComponent(identity.launchId)}-${identity.planHash.slice(0, 16)}`;
return value.slice(0, CONTAINER_NAME_MAX_CHARS);
}
function assertContainerIdentity(identity: NodeWorkerContainerIdentity): void {
if (
identity.launchId.length === 0 ||
identity.launchId.length > 256 ||
identity.launchId.includes("\0") ||
!/^[a-f0-9]{64}$/u.test(identity.planHash)
) {
throw new Error("invalid node worker container identity");
}
}
export function buildNodeWorkerContainerRunArgs(params: {
engine: NodeWorkerContainerEngine;
identity: NodeWorkerContainerIdentity;
mounts: NodeWorkerContainerMounts;
uid: number;
gid: number;
}): string[] {
assertContainerIdentity(params.identity);
if (
!Number.isSafeInteger(params.uid) ||
params.uid <= 0 ||
!Number.isSafeInteger(params.gid) ||
params.gid <= 0
) {
throw new Error("node worker container requires a concrete non-root host user mapping");
}
const bundleDir = requireOwnedAbsolutePath(params.mounts.bundleDir, "bundle mount");
const relayDir = requireOwnedAbsolutePath(params.mounts.relayDir, "relay mount");
const memoryDir = requireOwnedAbsolutePath(params.mounts.memoryDir, "memory projection mount");
const workspaceDir = requireOwnedAbsolutePath(params.mounts.workspaceDir, "workspace mount");
const name = nodeWorkerContainerName(params.identity);
// This is intentionally a closed policy. The Gateway cannot pass a host path,
// image, environment value, container option, or projection mount into it.
return [
"run",
"--interactive",
"--name",
name,
"--label",
`${NODE_WORKER_CONTAINER_LABEL}=${NODE_WORKER_CONTAINER_FORMAT}`,
"--label",
`${NODE_WORKER_CONTAINER_LAUNCH_LABEL}=${params.identity.launchId}`,
"--label",
`${NODE_WORKER_CONTAINER_PLAN_LABEL}=${params.identity.planHash}`,
"--network",
"none",
"--read-only",
"--tmpfs",
"/tmp:rw,nosuid,nodev,noexec,size=64m",
"--cap-drop",
"ALL",
"--security-opt",
"no-new-privileges",
"--pids-limit",
"128",
"--memory",
"512m",
"--cpus",
"1",
"--user",
`${params.uid}:${params.gid}`,
"--workdir",
NODE_WORKER_CONTAINER_WORKSPACE,
"--mount",
formatMount(bundleDir, NODE_WORKER_CONTAINER_WORKER_ROOT, true),
"--mount",
formatMount(relayDir, NODE_WORKER_CONTAINER_RELAY_ROOT, true),
"--mount",
formatMount(memoryDir, NODE_WORKER_CONTAINER_MEMORY_ROOT, true),
"--mount",
formatMount(workspaceDir, NODE_WORKER_CONTAINER_WORKSPACE, false),
NODE_WORKER_CONTAINER_IMAGE,
"node",
`${NODE_WORKER_CONTAINER_WORKER_ROOT}/worker.mjs`,
];
}
/**
* A node advertises process isolation only after one installed engine proves it
* can service local container commands. Docker is preferred; Podman is the
* supported equivalent when Docker is not available.
*/
export async function resolveNodeWorkerContainerEngine(): Promise<NodeWorkerContainerEngine | undefined> {
if (!resolveNodeWorkerContainerUser()) {
return undefined;
}
for (const engine of [DOCKER_SANDBOX_ENGINE, PODMAN_SANDBOX_ENGINE] as const) {
try {
const result = await runNodeWorkerContainerControlCommand(
engine,
["info", "--format", "{{.ServerVersion}}"],
{
allowFailure: true,
},
);
if (
result.code === 0 &&
result.stdout.trim().length > 0 &&
(await prepareNodeWorkerContainerImage(engine))
) {
return engine;
}
} catch {
// An absent or unavailable engine means this node is ineligible, never a host fallback.
}
}
return undefined;
}
async function listMatchingContainerIds(
engine: NodeWorkerContainerEngine,
identity: NodeWorkerContainerIdentity,
): Promise<string[]> {
const result = await runNodeWorkerContainerControlCommand(
engine,
[
"ps",
"--all",
"--quiet",
"--filter",
`label=${NODE_WORKER_CONTAINER_LABEL}=${NODE_WORKER_CONTAINER_FORMAT}`,
"--filter",
`label=${NODE_WORKER_CONTAINER_LAUNCH_LABEL}=${identity.launchId}`,
"--filter",
`label=${NODE_WORKER_CONTAINER_PLAN_LABEL}=${identity.planHash}`,
],
);
return result.stdout
.split(/\r?\n/u)
.map((value) => value.trim())
.filter((value) => /^[a-f0-9]{12,128}$/u.test(value));
}
async function isOwnedContainer(
engine: NodeWorkerContainerEngine,
containerId: string,
identity: NodeWorkerContainerIdentity,
): Promise<boolean> {
const result = await runNodeWorkerContainerControlCommand(
engine,
[
"inspect",
"--format",
"{{.Name}}\n{{index .Config.Labels \"openclaw.node-worker-container\"}}\n{{index .Config.Labels \"openclaw.node-worker-launch\"}}\n{{index .Config.Labels \"openclaw.node-worker-plan\"}}",
containerId,
],
);
const [rawName, format, launchId, planHash, ...extra] = result.stdout.trimEnd().split(/\r?\n/u);
return (
extra.length === 0 &&
rawName === `/${nodeWorkerContainerName(identity)}` &&
format === NODE_WORKER_CONTAINER_FORMAT &&
launchId === identity.launchId &&
planHash === identity.planHash
);
}
/**
* Prove the exact labelled container is accepting work before the shim attests
* execution. The foreground `run` client being spawned is not that proof.
*/
export async function waitForOwnedNodeWorkerContainerRunning(params: {
engine: NodeWorkerContainerEngine;
identity: NodeWorkerContainerIdentity;
timeoutMs?: number;
}): Promise<void> {
assertContainerIdentity(params.identity);
const deadline = Date.now() + (params.timeoutMs ?? 5_000);
while (Date.now() < deadline) {
const containerIds = await listMatchingContainerIds(params.engine, params.identity);
for (const containerId of containerIds) {
if (!(await isOwnedContainer(params.engine, containerId, params.identity))) {
continue;
}
const state = await runNodeWorkerContainerControlCommand(params.engine, [
"inspect",
"--format",
"{{.State.Running}}",
containerId,
]);
if (state.stdout.trim() === "true") {
return;
}
}
await new Promise<void>((resolve) => setTimeout(resolve, 25));
}
throw new Error(`${params.engine.displayName} did not start the exact node worker container`);
}
async function containerStillMatches(
engine: NodeWorkerContainerEngine,
containerId: string,
identity: NodeWorkerContainerIdentity,
): Promise<boolean> {
return (await listMatchingContainerIds(engine, identity)).includes(containerId);
}
/** Removes only an exact, label-verified orphan from a prior shim/supervisor lifetime. */
export async function removeOwnedNodeWorkerContainers(
identity: NodeWorkerContainerIdentity,
engine: NodeWorkerContainerEngine,
): Promise<number> {
assertContainerIdentity(identity);
let removed = 0;
const containerIds = await listMatchingContainerIds(engine, identity);
for (const containerId of containerIds) {
let owned: boolean;
try {
owned = await isOwnedContainer(engine, containerId, identity);
} catch {
// An inspect race is safe only when a second enumeration proves that the
// listed container is gone. Any surviving container leaves cleanup open.
if (!(await containerStillMatches(engine, containerId, identity))) {
continue;
}
throw new Error(`${engine.displayName} could not inspect a node worker container`);
}
if (!owned) {
continue;
}
try {
await runNodeWorkerContainerControlCommand(engine, ["rm", "--force", containerId]);
removed += 1;
} catch {
// A concurrent exit is already clean. Anything still enumerable is an
// orphan and must keep the relay and launch recovery from settling.
if (!(await containerStillMatches(engine, containerId, identity))) {
continue;
}
throw new Error(`${engine.displayName} could not remove a node worker container`);
}
}
return removed;
}
@@ -0,0 +1,156 @@
import { fork, type ChildProcess } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { completeWorkerLaunchDescriptor } from "../worker/launch-descriptor.js";
import { nodeWorkerContainerName } from "./node-worker-container-runtime.js";
import { testWorkerDescriptor } from "./node-worker-supervisor.test-support.js";
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
const children = new Set<ChildProcess>();
afterEach(async () => {
for (const child of children) {
if (child.exitCode === null && child.signalCode === null) {
child.kill("SIGKILL");
}
}
await Promise.all(
[...children].map(
async (child) =>
await new Promise<void>((resolve) => {
if (child.exitCode !== null || child.signalCode !== null) {
resolve();
return;
}
child.once("exit", () => resolve());
}),
),
);
children.clear();
});
function writeFakeDocker(root: string, marker: string, containerName: string): string {
const binDir = path.join(root, "bin");
fs.mkdirSync(binDir);
const executable = path.join(binDir, "docker");
fs.writeFileSync(
executable,
`#!/usr/bin/env node
const fs = require("node:fs");
const command = process.argv[2];
fs.appendFileSync(${JSON.stringify(marker)}, command + "\\n");
if (command === "ps") {
process.stdout.write("abcdefabcdef\\n");
process.exit(0);
}
if (command === "inspect") {
const format = process.argv[process.argv.indexOf("--format") + 1];
if (format === "{{.State.Running}}") {
process.stdout.write("true\\n");
} else {
process.stdout.write(${JSON.stringify(
`/${containerName}\nv1\nturn-1\n${"a".repeat(64)}\n`,
)});
}
process.exit(0);
}
if (command === "run") {
process.on("SIGTERM", () => process.exit(0));
setInterval(() => {}, 1000);
}
`,
{ mode: 0o755 },
);
return binDir;
}
function waitForExit(child: ChildProcess): Promise<void> {
return new Promise<void>((resolve, reject) => {
if (child.exitCode !== null || child.signalCode !== null) {
resolve();
return;
}
child.once("error", reject);
child.once("exit", () => resolve());
});
}
describe("node worker container shim", () => {
it.runIf(process.platform !== "win32")(
"stops its container command when the supervisor IPC lease disconnects",
async () => {
const root = tempDirs.make("node-worker-container-shim-");
const marker = path.join(root, "docker-calls");
const identity = { launchId: "turn-1", planHash: "a".repeat(64) };
const binDir = writeFakeDocker(root, marker, nodeWorkerContainerName(identity));
const bundleDir = path.join(root, "bundle");
const relayDir = tempDirs.make("oc-worker-relay-", "/tmp");
const memoryDir = path.join(root, "memory");
const workspaceDir = path.join(root, "workspace");
for (const directory of [bundleDir, memoryDir, workspaceDir]) {
fs.mkdirSync(directory);
}
fs.writeFileSync(path.join(bundleDir, "worker.mjs"), "process.exit(0);\n");
const descriptor = testWorkerDescriptor("/workspace");
descriptor.assignment.memoryReadEnforced = true;
const input = {
descriptor: completeWorkerLaunchDescriptor(descriptor, {
kind: "unix" as const,
socketPath: path.join(root, "gateway.sock"),
}),
engine: "docker" as const,
identity,
mounts: { bundleDir, relayDir, memoryDir, workspaceDir },
};
const shimPath = path.resolve("src/node-host/node-worker-container-shim.ts");
const child = fork(shimPath, ["--internal-worker-container-shim"], {
// tsx resolves workspace aliases through the checkout's tsconfig. The
// shim's actual container command still runs from `workspaceDir`.
cwd: process.cwd(),
env: { ...process.env, PATH: `${binDir}${path.delimiter}${process.env.PATH ?? ""}` },
// The shim's CWD is an isolated worker fixture, so a bare package
// specifier would resolve from that fixture rather than this checkout.
execArgv: ["--import", import.meta.resolve("tsx")],
silent: true,
});
children.add(child);
let stderr = "";
const messages: unknown[] = [];
child.stderr?.on("data", (chunk: Buffer) => {
stderr += chunk.toString("utf8");
});
child.on("message", (message: unknown) => {
messages.push(message);
});
child.stdin?.end(JSON.stringify(input));
child.send({ type: "openclaw-worker-start-v1", ...identity });
try {
await vi.waitFor(() => expect(fs.readFileSync(marker, "utf8")).toContain("run\n"), {
timeout: 5_000,
});
} catch (error) {
throw new Error(
`container shim did not launch the fake engine: ${stderr} ${JSON.stringify(messages)}`,
{ cause: error },
);
}
await vi.waitFor(() =>
expect(messages).toContainEqual({
type: "openclaw-worker-execution-started-v1",
...identity,
}),
);
child.disconnect();
await vi.waitFor(
() => expect(child.exitCode ?? child.signalCode).not.toBeNull(),
{ timeout: 5_000 },
);
await expect(waitForExit(child)).resolves.toBeUndefined();
expect(fs.readFileSync(marker, "utf8")).toContain("ps\n");
},
);
});
+233
View File
@@ -0,0 +1,233 @@
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
import fs from "node:fs/promises";
import path from "node:path";
import { once } from "node:events";
import { isRecord } from "@openclaw/normalization-core/record-coerce";
import {
completeWorkerLaunchDescriptor,
parseWorkerLaunchDescriptor,
type WorkerLaunchDescriptor,
} from "../worker/launch-descriptor.js";
import { createWorkerIpcLifetime } from "../worker/worker-process.js";
import type { WorkerConnectionEndpoint } from "../worker/worker-connection-endpoint.js";
import {
buildNodeWorkerContainerRunArgs,
nodeWorkerContainerEngineFor,
NODE_WORKER_CONTAINER_SHIM_FLAG,
NODE_WORKER_CONTAINER_RELAY_SOCKET,
removeOwnedNodeWorkerContainers,
resolveNodeWorkerContainerUser,
waitForOwnedNodeWorkerContainerRunning,
type NodeWorkerContainerEngine,
type NodeWorkerContainerIdentity,
type NodeWorkerContainerMounts,
} from "./node-worker-container-runtime.js";
import { startNodeWorkerGatewayRelay, type NodeWorkerGatewayRelay } from "./node-worker-gateway-relay.js";
const SHIM_INPUT_MAX_BYTES = 1024 * 1024;
type NodeWorkerContainerShimInput = Readonly<{
descriptor: WorkerLaunchDescriptor;
engine: NodeWorkerContainerEngine["id"];
identity: NodeWorkerContainerIdentity;
mounts: NodeWorkerContainerMounts;
}>;
function hasExactKeys(value: Record<string, unknown>, keys: readonly string[]): boolean {
return Object.keys(value).length === keys.length && keys.every((key) => Object.hasOwn(value, key));
}
function parseContainerEngine(value: unknown): NodeWorkerContainerEngine["id"] | undefined {
return value === "docker" || value === "podman" ? value : undefined;
}
function parseShimInput(raw: string): NodeWorkerContainerShimInput {
if (Buffer.byteLength(raw, "utf8") > SHIM_INPUT_MAX_BYTES) {
throw new Error("node worker container shim input exceeds its bound");
}
let value: unknown;
try {
value = JSON.parse(raw) as unknown;
} catch {
throw new Error("node worker container shim input is malformed");
}
if (
!isRecord(value) ||
!hasExactKeys(value, ["descriptor", "engine", "identity", "mounts"]) ||
!isRecord(value.identity) ||
!hasExactKeys(value.identity, ["launchId", "planHash"]) ||
typeof value.identity.launchId !== "string" ||
typeof value.identity.planHash !== "string" ||
!isRecord(value.mounts) ||
!hasExactKeys(value.mounts, ["bundleDir", "relayDir", "memoryDir", "workspaceDir"]) ||
typeof value.mounts.bundleDir !== "string" ||
typeof value.mounts.relayDir !== "string" ||
typeof value.mounts.memoryDir !== "string" ||
typeof value.mounts.workspaceDir !== "string"
) {
throw new Error("node worker container shim input is invalid");
}
const engine = parseContainerEngine(value.engine);
if (!engine) {
throw new Error("node worker container shim engine is invalid");
}
const descriptor = parseWorkerLaunchDescriptor(value.descriptor);
if (
descriptor.assignment.workspaceDir !== "/workspace" ||
(descriptor.assignment.workerContainmentRoot !== undefined &&
descriptor.assignment.workerContainmentRoot !== "/workspace")
) {
throw new Error("node worker container descriptor escaped its fixed workspace");
}
return {
descriptor,
engine,
identity: { launchId: value.identity.launchId, planHash: value.identity.planHash },
mounts: {
bundleDir: value.mounts.bundleDir,
relayDir: value.mounts.relayDir,
memoryDir: value.mounts.memoryDir,
workspaceDir: value.mounts.workspaceDir,
},
};
}
async function readShimInput(): Promise<NodeWorkerContainerShimInput> {
let raw = "";
for await (const chunk of process.stdin) {
raw += String(chunk);
if (Buffer.byteLength(raw, "utf8") > SHIM_INPUT_MAX_BYTES) {
throw new Error("node worker container shim input exceeds its bound");
}
}
return parseShimInput(raw);
}
function currentUser(): { uid: number; gid: number } {
const user = resolveNodeWorkerContainerUser();
if (!user) {
throw new Error("node worker container execution requires a non-root POSIX host user mapping");
}
return user;
}
async function closeRelay(relay: NodeWorkerGatewayRelay | undefined): Promise<void> {
await relay?.close().catch(() => undefined);
}
async function waitForChild(child: ChildProcessWithoutNullStreams): Promise<number> {
const [code] = await once(child, "close");
return typeof code === "number" ? code : 1;
}
/**
* The shim, not Docker, is the worker PID in the durable journal. It waits on
* inherited Node IPC before creating a disposable container and removes that
* exact labelled container on any normal or signal-driven terminal path.
*/
export async function runNodeWorkerContainerShim(): Promise<void> {
const input = await readShimInput();
const lifetime = createWorkerIpcLifetime();
let relay: NodeWorkerGatewayRelay | undefined;
let child: ChildProcessWithoutNullStreams | undefined;
let stopping = false;
let finishStop!: () => void;
const stopped = new Promise<void>((resolve) => {
finishStop = resolve;
});
const engine = nodeWorkerContainerEngineFor(input.engine);
const stop = async () => {
if (stopping) {
return;
}
stopping = true;
try {
child?.kill("SIGTERM");
await removeOwnedNodeWorkerContainers(input.identity, engine);
} finally {
await closeRelay(relay);
relay = undefined;
lifetime.dispose();
finishStop();
}
};
const onSignal = () => {
void stop();
};
const onSupervisorGone = () => {
void stop();
};
process.once("SIGTERM", onSignal);
process.once("SIGINT", onSignal);
// The IPC channel is the supervisor's ownership lease. Without this, a
// killed supervisor can strand a still-running shim and its container.
lifetime.signal.addEventListener("abort", onSupervisorGone, { once: true });
if (lifetime.signal.aborted) {
void stop();
}
try {
const started = await Promise.race([lifetime.started, stopped.then(() => false)]);
if (!started || stopping) {
return;
}
await fs.mkdir(input.mounts.relayDir, { recursive: true, mode: 0o700 });
await fs.chmod(input.mounts.relayDir, 0o700);
relay = await startNodeWorkerGatewayRelay({
directory: input.mounts.relayDir,
upstream: input.descriptor.connectionEndpoint,
});
// A descriptor includes its host-only Gateway endpoint. Rebuild it from
// the validated plan so the container receives only the mounted relay.
const containerDescriptor = completeWorkerLaunchDescriptor(
{
version: input.descriptor.version,
admission: input.descriptor.admission,
assignment: input.descriptor.assignment,
},
{
kind: "unix",
socketPath: NODE_WORKER_CONTAINER_RELAY_SOCKET,
} satisfies WorkerConnectionEndpoint,
);
const runArgs = buildNodeWorkerContainerRunArgs({
engine,
identity: input.identity,
mounts: input.mounts,
...currentUser(),
});
child = spawn(engine.command, runArgs, {
cwd: input.mounts.workspaceDir,
env: process.env,
stdio: ["pipe", "inherit", "inherit"],
windowsHide: true,
});
child.stdin.end(JSON.stringify(containerDescriptor));
await waitForOwnedNodeWorkerContainerRunning({ engine, identity: input.identity });
// Container execution begins only after an exact label-bound inspect says Running.
lifetime.reportExecutionStarted();
const code = await Promise.race([waitForChild(child), stopped.then(() => 143)]);
if (!stopping) {
await removeOwnedNodeWorkerContainers(input.identity, engine);
}
process.exitCode = code;
} catch (error) {
lifetime.reportConnectionFailure(error instanceof Error ? error.message : "container worker unavailable");
process.exitCode = 1;
} finally {
process.off("SIGTERM", onSignal);
process.off("SIGINT", onSignal);
lifetime.signal.removeEventListener("abort", onSupervisorGone);
await stop();
}
}
if (process.argv.includes(NODE_WORKER_CONTAINER_SHIM_FLAG)) {
void runNodeWorkerContainerShim().catch((error: unknown) => {
process.stderr.write(
`${error instanceof Error ? error.message : "node worker container shim failed"}\n`,
);
process.exitCode = 1;
});
}
@@ -0,0 +1,178 @@
import fs from "node:fs/promises";
import http from "node:http";
import type { Socket } from "node:net";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { WebSocket, WebSocketServer } from "ws";
import { WORKER_PROTOCOL_MAX_PAYLOAD_BYTES } from "../../packages/gateway-protocol/src/schema/worker-admission.js";
import { startNodeWorkerGatewayRelay } from "./node-worker-gateway-relay.js";
const cleanups: Array<() => Promise<void>> = [];
afterEach(async () => {
await Promise.all(cleanups.splice(0).map(async (cleanup) => await cleanup()));
});
async function relayHarness(params: {
onUpstreamConnection?: (socket: WebSocket) => void;
onUpstreamUpgrade?: (socket: Socket) => void;
} = {}) {
// Keep the lexical path short: the production relay intentionally does the
// same because macOS rejects canonical state paths as Unix socket names.
const root = await fs.mkdtemp("/tmp/node-worker-relay-");
const upstreamServer = http.createServer();
const rawUpstreamSockets = new Set<Socket>();
const upstreamClients: WebSocket[] = [];
if (params.onUpstreamUpgrade) {
upstreamServer.on("upgrade", (_request, socket) => {
rawUpstreamSockets.add(socket);
socket.once("close", () => rawUpstreamSockets.delete(socket));
params.onUpstreamUpgrade?.(socket);
});
} else {
const upstreamWebSocket = new WebSocketServer({ server: upstreamServer });
upstreamWebSocket.on("connection", (socket) => {
upstreamClients.push(socket);
if (params.onUpstreamConnection) {
params.onUpstreamConnection(socket);
return;
}
socket.on("message", (message, binary) => socket.send(message, { binary }));
});
}
await new Promise<void>((resolve) => upstreamServer.listen(0, "127.0.0.1", resolve));
const address = upstreamServer.address();
if (!address || typeof address === "string") {
throw new Error("expected upstream server address");
}
const relay = await startNodeWorkerGatewayRelay({
directory: root,
upstream: { kind: "websocket", url: `ws://127.0.0.1:${address.port}/__openclaw__/worker` },
});
cleanups.push(async () => {
await relay.close();
for (const socket of rawUpstreamSockets) {
socket.destroy();
}
await new Promise<void>((resolve) => upstreamServer.close(() => resolve()));
await fs.rm(root, { recursive: true, force: true });
});
return { relay, upstreamClients };
}
async function connect(socketPath: string): Promise<WebSocket> {
const socket = new WebSocket(`ws+unix://${socketPath}:/`);
await new Promise<void>((resolve, reject) => {
socket.once("open", resolve);
socket.once("error", reject);
});
return socket;
}
function waitForClose(socket: WebSocket): Promise<number> {
return new Promise<number>((resolve) => socket.once("close", (code) => resolve(code)));
}
async function expectConnectFailure(socketPath: string): Promise<void> {
const probe = new WebSocket(`ws+unix://${socketPath}:/`);
await new Promise<void>((resolve, reject) => {
probe.once("open", () => reject(new Error("closed relay unexpectedly accepted a client")));
probe.once("error", () => resolve());
});
}
describe("node worker gateway relay", () => {
it("forwards one bounded local client and rejects a second client", async () => {
const { relay } = await relayHarness();
const first = await connect(relay.socketPath);
cleanups.push(async () => first.terminate());
const echoed = new Promise<Buffer>((resolve) => {
first.once("message", (message) => resolve(Buffer.from(message)));
});
first.send(Buffer.from("worker-frame"));
await expect(echoed).resolves.toEqual(Buffer.from("worker-frame"));
const second = new WebSocket(`ws+unix://${relay.socketPath}:/`);
cleanups.push(async () => second.terminate());
await expect(
new Promise<number>((resolve, reject) => {
second.once("unexpected-response", (_request, response) => resolve(response.statusCode ?? 0));
second.once("open", () => reject(new Error("second relay client unexpectedly connected")));
second.once("error", () => undefined);
}),
).resolves.toBe(409);
});
it("closes an oversized worker frame before forwarding it upstream", async () => {
const { relay } = await relayHarness();
const client = await connect(relay.socketPath);
cleanups.push(async () => client.terminate());
const closed = new Promise<number>((resolve) => client.once("close", (code) => resolve(code)));
client.send(Buffer.alloc(WORKER_PROTOCOL_MAX_PAYLOAD_BYTES + 1));
await expect(closed).resolves.toBe(1009);
});
it("closes a worker that exceeds the bounded pre-upstream queue", async () => {
const { relay } = await relayHarness({
// Keep the upstream WebSocket in CONNECTING so every worker frame uses
// the relay's bounded pending queue rather than an OS socket buffer.
onUpstreamUpgrade: () => undefined,
});
const client = await connect(relay.socketPath);
cleanups.push(async () => client.terminate());
const closed = waitForClose(client);
client.send(Buffer.alloc(WORKER_PROTOCOL_MAX_PAYLOAD_BYTES));
client.send(Buffer.alloc(WORKER_PROTOCOL_MAX_PAYLOAD_BYTES));
client.send(Buffer.from([0]));
await expect(closed).resolves.toBe(1008);
});
it("closes both peers when a downstream client is over the bounded output limit", async () => {
const { relay, upstreamClients } = await relayHarness();
const client = await connect(relay.socketPath);
cleanups.push(async () => client.terminate());
await vi.waitFor(() => expect(upstreamClients).toHaveLength(1));
const bufferedAmount = vi
.spyOn(WebSocket.prototype, "bufferedAmount", "get")
.mockReturnValue(WORKER_PROTOCOL_MAX_PAYLOAD_BYTES * 2 + 1);
const closed = waitForClose(client);
try {
upstreamClients[0].send(Buffer.from("gateway-frame"));
await expect(closed).resolves.toBe(1008);
expect(bufferedAmount).toHaveBeenCalled();
} finally {
bufferedAmount.mockRestore();
}
});
it("closes the worker when the upstream connection fails", async () => {
const { relay } = await relayHarness({
onUpstreamUpgrade: (socket) => {
socket.end("HTTP/1.1 503 Upstream unavailable\r\nConnection: close\r\n\r\n");
},
});
const client = await connect(relay.socketPath);
cleanups.push(async () => client.terminate());
await expect(waitForClose(client)).resolves.toBe(1008);
});
it("terminates live peers and removes the Unix socket on close", async () => {
const { relay } = await relayHarness();
const client = await connect(relay.socketPath);
const closed = waitForClose(client);
await relay.close();
await expect(closed).resolves.toEqual(expect.any(Number));
await expect(fs.lstat(relay.socketPath)).rejects.toMatchObject({ code: "ENOENT" });
await expect(expectConnectFailure(relay.socketPath)).resolves.toBeUndefined();
await expect(relay.close()).resolves.toBeUndefined();
});
});
+195
View File
@@ -0,0 +1,195 @@
import fs from "node:fs/promises";
import { createServer, type IncomingMessage } from "node:http";
import type { Socket } from "node:net";
import path from "node:path";
import { WORKER_PROTOCOL_MAX_PAYLOAD_BYTES } from "../../packages/gateway-protocol/src/schema/worker-admission.js";
import { WebSocket, WebSocketServer, type RawData } from "ws";
import {
resolveWorkerConnectionTarget,
type WorkerConnectionEndpoint,
} from "../worker/worker-connection-endpoint.js";
import { NODE_WORKER_CONTAINER_RELAY_SOCKET } from "./node-worker-container-runtime.js";
const RELAY_PATH = "/";
const RELAY_MAX_PENDING_BYTES = WORKER_PROTOCOL_MAX_PAYLOAD_BYTES * 2;
const RELAY_MAX_BUFFERED_BYTES = WORKER_PROTOCOL_MAX_PAYLOAD_BYTES * 2;
const RELAY_CONNECT_TIMEOUT_MS = 10_000;
const RELAY_PORTABLE_UNIX_SOCKET_MAX_BYTES = 100;
export type NodeWorkerGatewayRelay = Readonly<{
socketPath: string;
close: () => Promise<void>;
}>;
function rawBytes(data: RawData): number {
if (typeof data === "string") {
return Buffer.byteLength(data, "utf8");
}
if (Array.isArray(data)) {
return data.reduce((total, chunk) => total + chunk.byteLength, 0);
}
return data.byteLength;
}
function rejectUpgrade(socket: Socket, status: number): void {
socket.end(`HTTP/1.1 ${status} Relay unavailable\r\nConnection: close\r\n\r\n`);
}
/**
* Exposes one local Unix WebSocket to an isolated worker. The remote Gateway
* endpoint and any TLS/Cloudflare material remain in this host-side relay.
*/
export async function startNodeWorkerGatewayRelay(params: {
directory: string;
upstream: WorkerConnectionEndpoint;
}): Promise<NodeWorkerGatewayRelay> {
if (!path.isAbsolute(params.directory)) {
throw new Error("node worker relay directory must be absolute");
}
await fs.realpath(params.directory);
// The bind mount uses the canonical directory, but preserving its short
// lexical spelling here keeps the live Unix socket below macOS' path limit.
const socketPath = path.join(params.directory, path.basename(NODE_WORKER_CONTAINER_RELAY_SOCKET));
if (Buffer.byteLength(socketPath, "utf8") > RELAY_PORTABLE_UNIX_SOCKET_MAX_BYTES) {
throw new Error("node worker relay socket path exceeds the portable Unix socket limit");
}
await fs.rm(socketPath, { force: true });
let accepted = false;
let closing = false;
const websocketServer = new WebSocketServer({
noServer: true,
maxPayload: WORKER_PROTOCOL_MAX_PAYLOAD_BYTES,
});
const server = createServer((_request, response) => {
response.writeHead(404);
response.end();
});
const sockets = new Set<WebSocket>();
const stopSocket = (socket: WebSocket, code = 1008) => {
try {
if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CLOSING) {
socket.close(code);
return;
}
socket.terminate();
} catch {
socket.terminate();
}
};
const attachRelay = (client: WebSocket) => {
sockets.add(client);
const target = resolveWorkerConnectionTarget(params.upstream);
const upstream = new WebSocket(target.url, target.options);
sockets.add(upstream);
const pending: Array<{ data: RawData; binary: boolean }> = [];
let pendingBytes = 0;
let linked = false;
let connectTimeout: NodeJS.Timeout | undefined;
const stopBoth = () => {
if (linked) {
linked = false;
}
clearTimeout(connectTimeout);
stopSocket(client);
stopSocket(upstream);
};
const forward = (destination: WebSocket, data: RawData, binary: boolean) => {
if (destination.readyState !== WebSocket.OPEN || rawBytes(data) > WORKER_PROTOCOL_MAX_PAYLOAD_BYTES) {
stopBoth();
return;
}
destination.send(data, { binary }, (error) => {
if (error) {
stopBoth();
}
});
// WebSocket has no bounded pull API. Terminating both peers on a bounded
// queue is the relay's backpressure contract; it cannot become a buffer.
if (destination.bufferedAmount > RELAY_MAX_BUFFERED_BYTES) {
stopBoth();
}
};
upstream.once("open", () => {
clearTimeout(connectTimeout);
const tlsError = target.validateSocket(upstream);
if (tlsError) {
stopBoth();
return;
}
linked = true;
for (const frame of pending.splice(0)) {
forward(upstream, frame.data, frame.binary);
}
pendingBytes = 0;
});
connectTimeout = setTimeout(stopBoth, RELAY_CONNECT_TIMEOUT_MS);
connectTimeout.unref?.();
upstream.once("error", stopBoth);
client.once("error", stopBoth);
upstream.on("message", (data, binary) => forward(client, data, binary));
client.on("message", (data, binary) => {
const bytes = rawBytes(data);
if (bytes > WORKER_PROTOCOL_MAX_PAYLOAD_BYTES) {
stopBoth();
return;
}
if (!linked) {
pendingBytes += bytes;
if (pendingBytes > RELAY_MAX_PENDING_BYTES) {
stopBoth();
return;
}
pending.push({ data, binary });
return;
}
forward(upstream, data, binary);
});
const closePeer = (peer: WebSocket) => () => {
sockets.delete(peer);
if (peer.readyState === WebSocket.OPEN || peer.readyState === WebSocket.CLOSING) {
stopSocket(peer, 1000);
}
};
client.once("close", closePeer(upstream));
upstream.once("close", closePeer(client));
};
server.on("upgrade", (request: IncomingMessage, socket, head) => {
if (closing || accepted || request.method !== "GET" || request.url !== RELAY_PATH) {
rejectUpgrade(socket, accepted ? 409 : 404);
return;
}
accepted = true;
websocketServer.handleUpgrade(request, socket, head, (client) => attachRelay(client));
});
await new Promise<void>((resolve, reject) => {
server.once("error", reject);
server.listen(socketPath, () => {
server.off("error", reject);
resolve();
});
});
await fs.chmod(socketPath, 0o600);
return Object.freeze({
socketPath,
close: async () => {
if (closing) {
return;
}
closing = true;
for (const socket of sockets) {
socket.terminate();
}
await new Promise<void>((resolve) => server.close(() => resolve()));
await fs.rm(socketPath, { force: true });
websocketServer.close();
},
});
}
@@ -83,6 +83,51 @@ function launchIds(database: ReturnType<typeof openOpenClawStateDatabase>["db"])
}
describe("node worker launch store pruning", () => {
it("persists the selected container engine with the durable launch identity", () => {
const { database, store } = fixture();
const supervisor = requireNodeWorkerProcessIdentity(process.pid);
const launchId = "container-launch";
const planHash = "c".repeat(64);
store.claim(
{
launchId,
planHash,
gatewayNamespace: "gateway-1",
environmentId: "environment-1",
sessionId: "session-1",
ownerEpoch: 3,
placementGeneration: 4,
runId: "run-1",
},
supervisor,
2,
NOW_MS,
);
store.recordContainerLaunch({
launchId,
planHash,
engine: "podman",
expiresAtMs: NOW_MS + DAY_MS,
});
expect(store.getContainerLaunchEngine({ launchId, planHash })).toBe("podman");
expect(store.getContainerLaunchLease({ launchId, planHash })).toEqual({
engine: "podman",
expiresAtMs: NOW_MS + DAY_MS,
});
expect(
database
.prepare("SELECT plan_hash, container_engine FROM node_worker_container_launches")
.all(),
).toEqual([{ plan_hash: planHash, container_engine: "podman" }]);
expect(
database
.prepare("SELECT plan_hash, expires_at_ms FROM node_worker_container_leases")
.all(),
).toEqual([{ plan_hash: planHash, expires_at_ms: NOW_MS + DAY_MS }]);
});
it("lazily ensures the terminal expiry index for existing databases", () => {
const { database, env } = fixture();
expect(hasTerminalExpiryIndex(database)).toBe(true);
+147 -1
View File
@@ -25,9 +25,19 @@ type NodeWorkerLaunchState =
| "interrupted"
| "cancelled";
export type NodeWorkerTerminalState = Exclude<NodeWorkerLaunchState, "pending" | "running">;
export type NodeWorkerContainerEngineId = "docker" | "podman";
type NodeWorkerLaunchDatabase = Pick<OpenClawStateDatabase, "node_worker_launches">;
type NodeWorkerLaunchDatabase = Pick<
OpenClawStateDatabase,
"node_worker_container_launches" | "node_worker_container_leases" | "node_worker_launches"
>;
type NodeWorkerLaunchRow = Selectable<NodeWorkerLaunchDatabase["node_worker_launches"]>;
type NodeWorkerContainerLaunchRow = Selectable<
NodeWorkerLaunchDatabase["node_worker_container_launches"]
>;
type NodeWorkerContainerLeaseRow = Selectable<
NodeWorkerLaunchDatabase["node_worker_container_leases"]
>;
export type NodeWorkerLaunchReceipt = {
launchId: string;
@@ -107,6 +117,32 @@ function readRow(database: DatabaseSync, launchId: string): NodeWorkerLaunchRow
);
}
function readContainerLaunchRow(
database: DatabaseSync,
launchId: string,
): NodeWorkerContainerLaunchRow | undefined {
return executeSqliteQueryTakeFirstSync(
database,
query(database)
.selectFrom("node_worker_container_launches")
.selectAll()
.where("launch_id", "=", launchId),
);
}
function readContainerLeaseRow(
database: DatabaseSync,
launchId: string,
): NodeWorkerContainerLeaseRow | undefined {
return executeSqliteQueryTakeFirstSync(
database,
query(database)
.selectFrom("node_worker_container_leases")
.selectAll()
.where("launch_id", "=", launchId),
);
}
function readNonterminalCount(database: DatabaseSync): number {
return (
executeSqliteQueryTakeFirstSync(
@@ -153,6 +189,18 @@ function pruneTerminalRows(params: {
if (launchIds.length === 0) {
return 0;
}
executeSqliteQuerySync(
params.database,
query(params.database)
.deleteFrom("node_worker_container_launches")
.where("launch_id", "in", launchIds),
);
executeSqliteQuerySync(
params.database,
query(params.database)
.deleteFrom("node_worker_container_leases")
.where("launch_id", "in", launchIds),
);
const result = executeSqliteQuerySync(
params.database,
query(params.database)
@@ -211,6 +259,12 @@ function validatePlanHash(value: string): void {
}
}
function validateContainerEngine(value: string): asserts value is NodeWorkerContainerEngineId {
if (value !== "docker" && value !== "podman") {
throw new Error("node worker container engine must be docker or podman");
}
}
function validateTimestamp(value: number): void {
if (!Number.isSafeInteger(value) || value < 0) {
throw new Error("node worker launch timestamp must be a non-negative safe integer");
@@ -480,6 +534,98 @@ export class NodeWorkerLaunchStore {
});
}
recordContainerLaunch(params: {
launchId: string;
planHash: string;
engine: NodeWorkerContainerEngineId;
expiresAtMs: number;
}): void {
validateIdentifier(params.launchId, "node worker launch id");
validatePlanHash(params.planHash);
validateContainerEngine(params.engine);
validateTimestamp(params.expiresAtMs);
this.write("node-worker-launch.record-container", (database) => {
const launch = requireMatchingRow(database, params.launchId, params.planHash);
if (TERMINAL_STATES.has(launch.state)) {
throw new Error("cannot record a container for a terminal node worker launch");
}
const existing = readContainerLaunchRow(database, params.launchId);
if (!existing) {
executeSqliteQuerySync(
database,
query(database).insertInto("node_worker_container_launches").values({
launch_id: params.launchId,
plan_hash: params.planHash,
container_engine: params.engine,
}),
);
} else if (
existing.plan_hash !== params.planHash ||
existing.container_engine !== params.engine
) {
throw new Error("node worker container launch metadata does not match the durable launch");
}
const lease = readContainerLeaseRow(database, params.launchId);
if (!lease) {
executeSqliteQuerySync(
database,
query(database).insertInto("node_worker_container_leases").values({
launch_id: params.launchId,
plan_hash: params.planHash,
expires_at_ms: params.expiresAtMs,
}),
);
} else if (lease.plan_hash !== params.planHash || lease.expires_at_ms !== params.expiresAtMs) {
throw new Error("node worker container lease does not match the durable launch");
}
});
}
getContainerLaunchEngine(params: {
launchId: string;
planHash: string;
}): NodeWorkerContainerEngineId | undefined {
validateIdentifier(params.launchId, "node worker launch id");
validatePlanHash(params.planHash);
return this.write("node-worker-launch.get-container", (database) => {
const metadata = readContainerLaunchRow(database, params.launchId);
if (!metadata) {
return undefined;
}
if (metadata.plan_hash !== params.planHash) {
throw new Error("node worker container launch metadata does not match the durable launch");
}
validateContainerEngine(metadata.container_engine);
return metadata.container_engine;
});
}
getContainerLaunchLease(params: {
launchId: string;
planHash: string;
}): { engine: NodeWorkerContainerEngineId; expiresAtMs: number } | undefined {
validateIdentifier(params.launchId, "node worker launch id");
validatePlanHash(params.planHash);
return this.write("node-worker-launch.get-container-lease", (database) => {
const metadata = readContainerLaunchRow(database, params.launchId);
const lease = readContainerLeaseRow(database, params.launchId);
if (!metadata && !lease) {
return undefined;
}
if (!metadata || !lease) {
// Launches written before leases existed still recover through the
// engine metadata and are interrupted conservatively on restart.
return undefined;
}
if (metadata.plan_hash !== params.planHash || lease.plan_hash !== params.planHash) {
throw new Error("node worker container lease metadata does not match the durable launch");
}
validateContainerEngine(metadata.container_engine);
validateTimestamp(lease.expires_at_ms);
return { engine: metadata.container_engine, expiresAtMs: lease.expires_at_ms };
});
}
getMatching(expected: NodeWorkerSupervisorIdentity): NodeWorkerLaunchReceipt | undefined {
validateIdentifier(expected.launchId, "node worker launch id");
validatePlanHash(expected.planHash);
@@ -0,0 +1,328 @@
import { createHash } from "node:crypto";
import fs from "node:fs";
import fsp from "node:fs/promises";
import path from "node:path";
import { isRecord } from "@openclaw/normalization-core/record-coerce";
import { isGatewayLoopbackHost } from "../../packages/gateway-client/src/websocket-transport.js";
import {
loadOrCreateProcessDeviceIdentity,
signDevicePayload,
type DeviceIdentity,
} from "../infra/device-identity.js";
import { isPathInside } from "../infra/path-guards.js";
import {
buildNodeWorkerMemoryProjectionRequestProofPayload,
nodeWorkerMemoryProjectionTransferPath,
parseNodeWorkerMemoryProjectionPayload,
NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER,
NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER,
NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER,
type NodeWorkerMemoryProjection,
type NodeWorkerMemoryProjectionPayload,
} from "../worker/node-memory-projection-protocol.js";
import type { WorkerConnectionEndpoint } from "../worker/worker-connection-endpoint.js";
import { openNodeWorkerTransferHttpRequest } from "./node-worker-transfer-http.js";
const PROJECTION_RESPONSE_MAX_BYTES = 3 * 1024 * 1024;
const PROJECTION_ROOT = "memory-projections";
const PROJECTION_ROOT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/u;
const PROJECTION_SOCKET_FORBIDDEN_ROOTS = new Set(["opt", "run", "workspace"]);
type ProjectionIdentity = Readonly<{
gatewayNamespace: string;
launchId: string;
planHash: string;
}>;
function throwIfProjectionStagingAborted(signal?: AbortSignal): void {
signal?.throwIfAborted();
}
function currentNonRootUid(): number {
if (process.platform === "win32" || typeof process.getuid !== "function") {
throw new Error("node worker memory projections require a non-root POSIX node host");
}
const uid = process.getuid();
if (!Number.isSafeInteger(uid) || uid <= 0) {
throw new Error("node worker memory projections require a non-root POSIX node host");
}
return uid;
}
function requirePrivateDirectory(directory: string, parent?: string): string {
fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
fs.chmodSync(directory, 0o700);
const stats = fs.lstatSync(directory);
const resolved = fs.realpathSync.native(directory);
if (
stats.isSymbolicLink() ||
!stats.isDirectory() ||
stats.uid !== currentNonRootUid() ||
(stats.mode & 0o077) !== 0 ||
(parent !== undefined && fs.realpathSync.native(path.dirname(directory)) !== parent)
) {
throw new Error("node worker memory projection path is not a private owned directory");
}
return resolved;
}
function projectionDirectoryName(identity: ProjectionIdentity): string {
if (!/^[a-f0-9]{64}$/u.test(identity.planHash)) {
throw new Error("node worker memory projection plan hash is invalid");
}
return createHash("sha256")
.update(`${identity.gatewayNamespace}\0${identity.launchId}\0${identity.planHash}`)
.digest("hex");
}
function assertPayloadIntegrity(payload: NodeWorkerMemoryProjectionPayload): Array<{
root: string;
leaf: string;
bytes: Buffer;
}> {
const roots = new Set<string>();
return payload.files.map((file) => {
const [root, leaf] = file.virtualPath.split("/");
const rootKey = root!.toLocaleLowerCase("en-US");
// `memory` is the canonical authorized-view root. It stays nested below the
// container's `/memory` mount, so it cannot shadow a container filesystem root.
if (!PROJECTION_ROOT_PATTERN.test(root!) || PROJECTION_SOCKET_FORBIDDEN_ROOTS.has(rootKey)) {
throw new Error("node worker memory projection root is unsafe");
}
roots.add(rootKey);
const bytes = Buffer.from(file.contentBase64, "base64");
if (createHash("sha256").update(bytes).digest("hex") !== file.sha256) {
throw new Error("node worker memory projection digest mismatch");
}
if (!leaf || leaf === "." || leaf === ".." || leaf.includes("/") || leaf.includes("\\")) {
throw new Error("node worker memory projection path is unsafe");
}
return { root: root!, leaf, bytes };
});
}
function assertProjectionTransferTransport(endpoint: WorkerConnectionEndpoint): void {
const gateway = new URL(endpoint.url);
if (gateway.protocol === "ws:" && !isGatewayLoopbackHost(gateway.hostname)) {
throw new Error(
"node worker memory projection requires wss:// for a non-loopback Gateway endpoint",
);
}
}
async function readProjectionResponse(params: {
endpoint: WorkerConnectionEndpoint;
projection: NodeWorkerMemoryProjection;
deviceIdentity: DeviceIdentity;
signal?: AbortSignal;
}): Promise<NodeWorkerMemoryProjectionPayload> {
if (params.endpoint.kind !== "websocket") {
throw new Error("node worker memory projection requires a Gateway WebSocket endpoint");
}
assertProjectionTransferTransport(params.endpoint);
const signedAtMs = Date.now();
const proofPayload = buildNodeWorkerMemoryProjectionRequestProofPayload({
reference: params.projection.reference,
binding: params.projection.binding,
nodeId: params.deviceIdentity.deviceId,
signedAtMs,
});
const response = await openNodeWorkerTransferHttpRequest({
gatewayUrl: params.endpoint.url,
...(params.endpoint.tlsFingerprint ? { tlsFingerprint: params.endpoint.tlsFingerprint } : {}),
...(params.endpoint.cloudflareAccess
? { cloudflareAccess: params.endpoint.cloudflareAccess }
: {}),
routePath: nodeWorkerMemoryProjectionTransferPath(),
method: "GET",
token: params.projection.reference,
headers: {
[NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER]: params.deviceIdentity.deviceId,
[NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER]: String(signedAtMs),
[NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER]: signDevicePayload(
params.deviceIdentity.privateKeyPem,
proofPayload,
),
},
...(params.signal ? { signal: params.signal } : {}),
});
if (response.statusCode !== 200) {
response.resume();
throw new Error(`node worker memory projection transfer failed (${response.statusCode ?? 0})`);
}
const chunks: Buffer[] = [];
let bytes = 0;
for await (const chunk of response) {
const value = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
bytes += value.byteLength;
if (bytes > PROJECTION_RESPONSE_MAX_BYTES) {
response.destroy(new Error("node worker memory projection response exceeds its limit"));
throw new Error("node worker memory projection response exceeds its limit");
}
chunks.push(value);
}
let value: unknown;
try {
value = JSON.parse(Buffer.concat(chunks).toString("utf8"));
} catch {
throw new Error("node worker memory projection response is malformed");
}
const payload = parseNodeWorkerMemoryProjectionPayload(value);
if (!payload) {
throw new Error("node worker memory projection response violated its bounded contract");
}
return payload;
}
/**
* Owns node-private projection staging. It is intentionally distinct from
* workspace synchronization: projection bytes are immutable and never writable by the worker.
*/
export class NodeWorkerMemoryProjectionRuntime {
private readonly root: string;
private readonly deviceIdentity: DeviceIdentity;
constructor(options: { root: string; deviceIdentity?: DeviceIdentity }) {
const base = path.resolve(options.root, PROJECTION_ROOT);
this.root = requirePrivateDirectory(base);
this.deviceIdentity = options.deviceIdentity ?? loadOrCreateProcessDeviceIdentity();
}
private resolveDirectory(identity: ProjectionIdentity): string {
return path.join(this.root, projectionDirectoryName(identity));
}
async stage(params: {
identity: ProjectionIdentity;
projection: NodeWorkerMemoryProjection;
endpoint: WorkerConnectionEndpoint;
signal?: AbortSignal;
}): Promise<string> {
throwIfProjectionStagingAborted(params.signal);
if (params.projection.expiresAtMs <= Date.now()) {
throw new Error("node worker memory projection lease expired before staging");
}
const destination = this.resolveDirectory(params.identity);
throwIfProjectionStagingAborted(params.signal);
await this.remove(params.identity);
throwIfProjectionStagingAborted(params.signal);
const payload = await readProjectionResponse({
endpoint: params.endpoint,
projection: params.projection,
deviceIdentity: this.deviceIdentity,
...(params.signal ? { signal: params.signal } : {}),
});
throwIfProjectionStagingAborted(params.signal);
if (params.projection.expiresAtMs <= Date.now()) {
throw new Error("node worker memory projection lease expired during staging");
}
const files = assertPayloadIntegrity(payload);
throwIfProjectionStagingAborted(params.signal);
const temporary = await fsp.mkdtemp(path.join(this.root, ".projection-"));
let staged = false;
try {
throwIfProjectionStagingAborted(params.signal);
await fsp.chmod(temporary, 0o700);
const preparedRoots = new Set<string>();
for (const file of files) {
throwIfProjectionStagingAborted(params.signal);
const root = path.join(temporary, file.root);
if (!preparedRoots.has(file.root)) {
// The node must write the immutable payload before the directory can
// become read-only to the container user.
await fsp.mkdir(root, { mode: 0o700 });
throwIfProjectionStagingAborted(params.signal);
const rootStats = await fsp.lstat(root);
if (rootStats.isSymbolicLink() || !rootStats.isDirectory() || rootStats.nlink !== 2) {
throw new Error("node worker memory projection root is not a private directory");
}
preparedRoots.add(file.root);
}
const target = path.join(root, file.leaf);
throwIfProjectionStagingAborted(params.signal);
const handle = await fsp.open(target, "wx", 0o400);
try {
throwIfProjectionStagingAborted(params.signal);
await handle.writeFile(file.bytes);
} finally {
await handle.close();
}
throwIfProjectionStagingAborted(params.signal);
const stats = await fsp.lstat(target);
if (
stats.isSymbolicLink() ||
!stats.isFile() ||
stats.nlink !== 1 ||
stats.size !== file.bytes.byteLength ||
stats.uid !== currentNonRootUid()
) {
throw new Error("node worker memory projection staged file failed integrity validation");
}
}
for (const root of preparedRoots) {
throwIfProjectionStagingAborted(params.signal);
await fsp.chmod(path.join(temporary, root), 0o500);
}
throwIfProjectionStagingAborted(params.signal);
await fsp.rename(temporary, destination);
staged = true;
throwIfProjectionStagingAborted(params.signal);
const resolved = await fsp.realpath(destination);
const stats = await fsp.lstat(resolved);
if (
stats.isSymbolicLink() ||
!stats.isDirectory() ||
path.dirname(resolved) !== this.root ||
!isPathInside(this.root, resolved)
) {
throw new Error("node worker memory projection escaped its private root");
}
throwIfProjectionStagingAborted(params.signal);
return resolved;
} catch (error) {
await fsp.rm(temporary, { recursive: true, force: true }).catch(() => undefined);
if (staged) {
await this.remove(params.identity);
}
throw error;
}
}
async remove(identity: ProjectionIdentity): Promise<void> {
const target = this.resolveDirectory(identity);
try {
const [stats, parent, resolved] = await Promise.all([
fsp.lstat(target),
fsp.realpath(this.root),
fsp.realpath(target),
]);
if (
stats.isSymbolicLink() ||
!stats.isDirectory() ||
path.dirname(resolved) !== parent ||
!isPathInside(parent, resolved)
) {
throw new Error("node worker memory projection cleanup target is not owned");
}
// Projection roots are made read-only before mount. Restore write access
// only after revalidating the exact node-owned tree so cleanup cannot be
// bypassed by its own immutable-file policy.
await fsp.chmod(resolved, 0o700);
for (const child of await fsp.readdir(resolved, { withFileTypes: true })) {
if (child.isSymbolicLink() || !child.isDirectory()) {
throw new Error("node worker memory projection cleanup tree is not owned");
}
const childResolved = await fsp.realpath(path.join(resolved, child.name));
if (path.dirname(childResolved) !== resolved || !isPathInside(resolved, childResolved)) {
throw new Error("node worker memory projection cleanup tree escaped its root");
}
await fsp.chmod(childResolved, 0o700);
}
await fsp.rm(target, { recursive: true, force: true });
} catch (error) {
if (!isRecord(error) || error.code !== "ENOENT") {
throw error;
}
}
}
}
@@ -0,0 +1,37 @@
import { describe, expect, it } from "vitest";
import type { NodeWorkerLaunchReceipt } from "./node-worker-launch-store.js";
import { projectNodeWorkerSupervisorReceipt } from "./node-worker-supervisor-contract.js";
function terminalReceipt(worker: NodeWorkerLaunchReceipt["worker"]): NodeWorkerLaunchReceipt {
return {
launchId: "launch-1",
planHash: "a".repeat(64),
gatewayNamespace: "gateway-1",
environmentId: "environment-1",
sessionId: "session-1",
ownerEpoch: 3,
placementGeneration: 4,
runId: "run-1",
state: "cancelled",
supervisor: { pid: 1, startTime: 1 },
worker,
resultJson: null,
errorText: "node worker launch cancelled",
completedAtMs: 2,
createdAtMs: 1,
updatedAtMs: 2,
};
}
describe("node worker supervisor receipt projection", () => {
it("exposes execution readiness without exposing a worker process identity", () => {
expect(projectNodeWorkerSupervisorReceipt(terminalReceipt(null))).toMatchObject({
state: "cancelled",
executionStarted: false,
});
expect(projectNodeWorkerSupervisorReceipt(terminalReceipt({ pid: 2, startTime: 2 }))).toMatchObject({
state: "cancelled",
executionStarted: true,
});
});
});
@@ -12,12 +12,16 @@ import type { WorkerConnectionEndpoint } from "../worker/worker-connection-endpo
import type { NodeWorkerLaunchReceipt } from "./node-worker-launch-store.js";
export {
NODE_WORKER_EXECUTION_CONTAINER_V1,
NODE_WORKER_EXECUTION_HOST_V1,
nodeWorkerMemoryProjectionLaunchBinding,
nodeWorkerPlanHash,
parseNodeWorkerCancelInput,
parseNodeWorkerLaunchInput,
parseNodeWorkerLookupInput,
} from "../worker/node-supervisor-protocol.js";
export type {
NodeWorkerExecution,
NodeWorkerLaunchInput,
NodeWorkerSupervisorIdentity,
NodeWorkerSupervisorReceipt,
@@ -55,7 +59,14 @@ export function projectNodeWorkerSupervisorReceipt(
: receipt.state === "failed" ||
receipt.state === "interrupted" ||
receipt.state === "cancelled"
? { ...identity, state: receipt.state, errorText: receipt.errorText }
? {
...identity,
state: receipt.state,
errorText: receipt.errorText,
// Worker identity is persisted only by markRunning, after the private child
// acknowledgement. Do not expose the PID, only the durable execution fact.
executionStarted: receipt.worker !== null,
}
: { ...identity, state: receipt.state };
const parsed = parseNodeWorkerSupervisorReceipt(projected);
if (!parsed) {
@@ -1,9 +1,7 @@
import { spawn, type ChildProcess } from "node:child_process";
import { createHash } from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { stableStringify } from "@openclaw/normalization-core";
import { afterEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import {
@@ -16,9 +14,11 @@ import {
requireNodeWorkerProcessIdentity,
type NodeWorkerProcessIdentity,
} from "./node-worker-process-identity.js";
import { nodeWorkerPlanHash } from "./node-worker-supervisor-contract.js";
import { createNodeWorkerSupervisor } from "./node-worker-supervisor.js";
import {
testNodeWorkerLaunchIdentity,
testNodeWorkerMemoryProjection,
TEST_WORKER_ENDPOINT,
testWorkerLaunchInput,
writeNodeWorkerFixture,
@@ -58,16 +58,7 @@ function fixture(label: string) {
}
function planHash(input: ReturnType<typeof testWorkerLaunchInput>): string {
return createHash("sha256")
.update(
stableStringify({
expectedBundleHash: input.expectedBundleHash,
descriptor: input.descriptor,
gatewayNamespace: input.gatewayNamespace,
placementGeneration: input.placementGeneration,
}),
)
.digest("hex");
return nodeWorkerPlanHash(input);
}
function insertLaunch(params: {
@@ -247,6 +238,110 @@ describe("node worker supervisor recovery", () => {
await supervisor.close();
});
it.runIf(process.platform !== "win32")(
"does not settle a stale pending container launch until durable cleanup succeeds",
async () => {
const { bundleRoot, env, root, workspaceDir } = fixture("node-worker-pending-container-");
const bin = path.join(root, "bin");
const docker = path.join(bin, "docker");
const commandLog = path.join(root, "docker-commands.log");
fs.mkdirSync(bin, { recursive: true });
const writeDocker = (exitCode: number) => {
fs.writeFileSync(
docker,
`#!/bin/sh\nprintf '%s\\n' "$1" >> ${JSON.stringify(commandLog)}\nexit ${String(exitCode)}\n`,
{ mode: 0o755 },
);
};
writeDocker(1);
vi.stubEnv("PATH", `${bin}${path.delimiter}${process.env.PATH ?? ""}`);
try {
const store = new NodeWorkerLaunchStore({ env });
store.get("schema-probe");
const input = testWorkerLaunchInput(workspaceDir, "pending-container-launch");
input.execution = { kind: "container-v1" };
input.descriptor.assignment.memoryReadEnforced = true;
input.descriptor.assignment.workspaceDir = "/workspace";
input.memoryProjection = testNodeWorkerMemoryProjection(input);
insertLaunch({
env,
input,
state: "pending",
supervisor: { pid: 2_147_483_647, startTime: 1 },
});
store.recordContainerLaunch({
launchId: input.launchId,
planHash: planHash(input),
engine: "docker",
expiresAtMs: input.memoryProjection.expiresAtMs,
});
const first = createNodeWorkerSupervisor({ bundleRoot, env });
await expect(first.initialize()).rejects.toThrow("Docker command failed");
expect(store.get(input.launchId)).toMatchObject({ state: "pending" });
await first.close().catch(() => undefined);
writeDocker(0);
const recovered = createNodeWorkerSupervisor({ bundleRoot, env });
await recovered.initialize();
expect(await recovered.status(input.launchId)).toMatchObject({ state: "interrupted" });
expect(fs.readFileSync(commandLog, "utf8").split("\n").filter(Boolean)).toEqual([
"ps",
"ps",
]);
await recovered.close();
} finally {
vi.unstubAllEnvs();
}
},
);
it.runIf(process.platform !== "win32")(
"withdraws an expired durable projection lease during stale container recovery",
async () => {
const { bundleRoot, env, root, workspaceDir } = fixture("node-worker-expired-container-");
const bin = path.join(root, "bin");
fs.mkdirSync(bin, { recursive: true });
fs.writeFileSync(
path.join(bin, "docker"),
"#!/bin/sh\nexit 0\n",
{ mode: 0o755 },
);
vi.stubEnv("PATH", `${bin}${path.delimiter}${process.env.PATH ?? ""}`);
try {
const store = new NodeWorkerLaunchStore({ env });
store.get("schema-probe");
const input = testWorkerLaunchInput(workspaceDir, "expired-container-launch");
input.execution = { kind: "container-v1" };
input.descriptor.assignment.memoryReadEnforced = true;
input.descriptor.assignment.workspaceDir = "/workspace";
input.memoryProjection = { ...testNodeWorkerMemoryProjection(input), expiresAtMs: 1 };
insertLaunch({
env,
input,
state: "pending",
supervisor: { pid: 2_147_483_647, startTime: 1 },
});
store.recordContainerLaunch({
launchId: input.launchId,
planHash: planHash(input),
engine: "docker",
expiresAtMs: input.memoryProjection.expiresAtMs,
});
const recovered = createNodeWorkerSupervisor({ bundleRoot, env, now: () => 2 });
await recovered.initialize();
await expect(recovered.status(input.launchId)).resolves.toMatchObject({
state: "cancelled",
errorText: "node worker memory projection lease expired before the worker launch started",
});
await recovered.close();
} finally {
vi.unstubAllEnvs();
}
},
);
it.runIf(process.platform !== "win32").each([
{ operation: "replay" as const, state: "interrupted" as const },
{ operation: "cancel" as const, state: "cancelled" as const },
@@ -7,6 +7,8 @@ import {
import type { WorkerLaunchPlan } from "../worker/launch-descriptor.js";
import type { WorkerConnectionEndpoint } from "../worker/worker-connection-endpoint.js";
import {
NODE_WORKER_EXECUTION_HOST_V1,
nodeWorkerMemoryProjectionLaunchBinding,
nodeWorkerPlanHash,
type NodeWorkerLaunchInput,
type NodeWorkerSupervisorIdentity,
@@ -55,13 +57,33 @@ const onMessage = (message) => {
typeof message !== "object" ||
message === null ||
Array.isArray(message) ||
Object.keys(message).length !== 1 ||
message.type !== "openclaw-worker-start-v1"
Object.keys(message).length !== 3 ||
message.type !== "openclaw-worker-start-v1" ||
typeof message.launchId !== "string" ||
!/^[a-f0-9]{64}$/.test(message.planHash)
) {
hardTerminate();
return;
}
started = true;
const executionAcknowledgement = {
type: "openclaw-worker-execution-started-v1",
launchId: message.launchId,
planHash: message.planHash,
};
if (descriptor.assignment.prompt === "wrong-execution-ack") {
process.send({ ...executionAcknowledgement, launchId: "another-launch" }, () => {});
setInterval(() => {}, 1000);
return;
}
if (descriptor.assignment.prompt === "wait-before-execution-ack") {
setInterval(() => {}, 1000);
return;
}
process.send(executionAcknowledgement, () => {});
if (descriptor.assignment.prompt === "replayed-execution-ack") {
setTimeout(() => process.send(executionAcknowledgement, () => {}), 25);
}
resolveStart();
};
const onDisconnect = () => {
@@ -99,6 +121,8 @@ if (mode === "connection-failure") {
setInterval(() => {}, 1000);
} else if (mode === "wait") {
setInterval(() => {}, 1000);
} else if (mode === "replayed-execution-ack") {
setInterval(() => {}, 1000);
} else if (mode === "tree") {
grandchild = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { stdio: "ignore" });
fs.writeFileSync(path.join(descriptor.assignment.workspaceDir, "grandchild.pid"), String(grandchild.pid));
@@ -164,6 +188,7 @@ export function testWorkerDescriptor(workspaceDir: string, prompt = "success"):
},
assignment: {
agentId: "agent-1",
memoryReadEnforced: false,
operationalRunInstance: { instanceId: "instance-1", runId: "run-1" },
agentRuntimeIdentityToken: "signed-runtime-token",
runId: "run-1",
@@ -195,6 +220,20 @@ export function testNodeWorkerLaunchIdentity(
};
}
export function testNodeWorkerMemoryProjection(
input: Omit<NodeWorkerLaunchInput, "memoryProjection">,
) {
return {
version: 1 as const,
reference: "a".repeat(43),
binding: {
launch: nodeWorkerMemoryProjectionLaunchBinding(input),
authorization: "b".repeat(64),
},
expiresAtMs: Date.now() + 60_000,
};
}
export function writeNodeWorkerFixture(root: string) {
const stateDir = path.join(root, "state-root");
const bundleRoot = path.join(root, "bundles-root");
@@ -217,5 +256,6 @@ export function testWorkerLaunchInput(
expectedBundleHash: TEST_BUNDLE_HASH,
placementGeneration: 4,
descriptor: testWorkerDescriptor(workspaceDir, prompt),
execution: { kind: NODE_WORKER_EXECUTION_HOST_V1 },
};
}
+166 -5
View File
@@ -17,10 +17,12 @@ import {
requireNodeWorkerProcessIdentity,
} from "./node-worker-process-identity.js";
import { createNodeWorkerSupervisor } from "./node-worker-supervisor.js";
import { NodeWorkerMemoryProjectionRuntime } from "./node-worker-memory-projection.js";
import {
TEST_WORKER_CREDENTIAL,
TEST_WORKER_ENDPOINT,
TEST_WORKER_SOURCE,
testNodeWorkerMemoryProjection,
testNodeWorkerLaunchIdentity,
testWorkerDescriptor,
testWorkerLaunchInput,
@@ -69,6 +71,85 @@ async function waitForTerminal(supervisor: NodeWorkerSupervisor, launchId: strin
}
describe("node worker supervisor", () => {
it("rejects an enforced-memory host launch before claiming a worker slot", async () => {
const { env, supervisor, workspaceDir } = fixture();
const input = launchInput(workspaceDir, "enforced-memory-host-downgrade");
input.descriptor.assignment.memoryReadEnforced = true;
await expect(supervisor.launch(input, TEST_WORKER_ENDPOINT)).rejects.toThrow(
"requires container-v1 execution",
);
expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)).toBeUndefined();
await supervisor.close();
});
it("rejects a cross-session memory projection replay before claiming a worker slot", async () => {
const { env, supervisor, workspaceDir } = fixture();
const input = launchInput(workspaceDir, "enforced-memory-cross-session-replay");
input.execution = { kind: "container-v1" };
input.descriptor.assignment.memoryReadEnforced = true;
input.descriptor.assignment.workspaceDir = "/workspace";
input.memoryProjection = testNodeWorkerMemoryProjection(input);
const replay = structuredClone(input);
replay.descriptor.admission.sessionId = "another-session";
await expect(supervisor.launch(replay, TEST_WORKER_ENDPOINT)).rejects.toThrow(
"memory projection does not match its worker launch",
);
expect(new NodeWorkerLaunchStore({ env }).get(replay.launchId)).toBeUndefined();
await supervisor.close();
});
it.runIf(process.platform !== "win32")(
"persists cancellation and cleans the staged projection before a container can spawn",
async () => {
const { bundleRoot, env, root, workspaceDir } = fixture();
const bin = path.join(root, "bin");
const commandLog = path.join(root, "docker-commands.log");
fs.mkdirSync(bin, { recursive: true });
fs.writeFileSync(
path.join(bin, "docker"),
`#!/bin/sh\nprintf '%s\\n' "$1" >> ${JSON.stringify(commandLog)}\nif [ "$1" = info ]; then printf 'test-engine'; fi\nexit 0\n`,
{ mode: 0o755 },
);
vi.stubEnv("PATH", `${bin}${path.delimiter}${process.env.PATH ?? ""}`);
try {
const controller = new AbortController();
const projection = {
stage: vi.fn(async () => {
controller.abort(new Error("Gateway revoked the projection"));
return path.join(root, "staged-projection");
}),
remove: vi.fn(async () => undefined),
} as unknown as NodeWorkerMemoryProjectionRuntime;
const supervisor = createNodeWorkerSupervisor({ bundleRoot, env, memoryProjection: projection });
const input = launchInput(workspaceDir, "cancelled-before-container-start");
input.execution = { kind: "container-v1" };
input.descriptor.assignment.memoryReadEnforced = true;
input.descriptor.assignment.workspaceDir = "/workspace";
input.memoryProjection = testNodeWorkerMemoryProjection(input);
await expect(supervisor.launch(input, TEST_WORKER_ENDPOINT, controller.signal)).resolves.toMatchObject({
state: "cancelled",
errorText: "node worker launch cancelled before process start",
});
expect(projection.stage).toHaveBeenCalledOnce();
expect(projection.remove).toHaveBeenCalledWith({
gatewayNamespace: input.gatewayNamespace,
launchId: input.launchId,
planHash: testNodeWorkerLaunchIdentity(input).planHash,
});
expect(fs.readFileSync(commandLog, "utf8").split("\n")).not.toContain("run");
expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)).toMatchObject({
state: "cancelled",
});
await supervisor.close();
} finally {
vi.unstubAllEnvs();
}
},
);
it("keeps construction and close inert without resolving process identity", async () => {
const root = tempDirs.make("node-worker-inert-");
const { bundleRoot, env } = writeNodeWorkerFixture(root);
@@ -513,7 +594,7 @@ describe("node worker supervisor", () => {
},
);
it("does not open or signal a child after markRunning observes its terminal receipt", async () => {
it("returns a terminal receipt when execution-ready races an immediate child completion", async () => {
const { supervisor, workspaceDir } = fixture();
const input = launchInput(workspaceDir, "fast-terminal-launch", "fast-terminal");
vi.spyOn(NodeWorkerLaunchStore.prototype, "markRunning").mockImplementation(
@@ -536,7 +617,7 @@ describe("node worker supervisor", () => {
await new Promise((resolve) => {
setTimeout(resolve, 150);
});
expect(fs.existsSync(marker)).toBe(false);
expect(fs.existsSync(marker)).toBe(true);
await supervisor.close();
});
@@ -549,7 +630,88 @@ describe("node worker supervisor", () => {
const terminal = await waitForTerminal(supervisor, input.launchId);
expect(fs.existsSync(exitedPath)).toBe(true);
expect(terminal.state).toBe("failed");
expect(terminal).toMatchObject({ state: "failed", worker: null });
await supervisor.close();
});
it("fails closed before durable running when the child acknowledges another launch", async () => {
const { env, supervisor, workspaceDir } = fixture();
const input = launchInput(workspaceDir, "wrong-execution-ack-launch", "wrong-execution-ack");
await expect(supervisor.launch(input, TEST_WORKER_ENDPOINT)).resolves.toMatchObject({
state: "interrupted",
worker: null,
});
expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)).toMatchObject({
state: "interrupted",
worker: null,
});
await supervisor.close();
});
it("persists cancellation before a child acknowledgement without a worker identity", async () => {
const { env, supervisor, workspaceDir } = fixture();
const input = launchInput(
workspaceDir,
"cancel-before-execution-ack-launch",
"wait-before-execution-ack",
);
const launching = supervisor.launch(input, TEST_WORKER_ENDPOINT);
await vi.waitFor(async () => {
expect((await supervisor.status(input.launchId))?.state).toBe("pending");
});
await expect(supervisor.cancel(testNodeWorkerLaunchIdentity(input))).resolves.toMatchObject({
state: "cancelled",
worker: null,
});
await expect(launching).resolves.toMatchObject({ state: "cancelled", worker: null });
expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)).toMatchObject({
state: "cancelled",
worker: null,
});
await supervisor.close();
});
it("interrupts a pending child on close without a durable worker identity", async () => {
const { env, supervisor, workspaceDir } = fixture();
const input = launchInput(
workspaceDir,
"close-before-execution-ack-launch",
"wait-before-execution-ack",
);
const launching = supervisor.launch(input, TEST_WORKER_ENDPOINT);
await vi.waitFor(async () => {
expect((await supervisor.status(input.launchId))?.state).toBe("pending");
});
await supervisor.close();
await expect(launching).resolves.toMatchObject({ state: "interrupted", worker: null });
expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)).toMatchObject({
state: "interrupted",
worker: null,
});
});
it("retires a child that replays its execution acknowledgement after durable readiness", async () => {
const { supervisor, workspaceDir } = fixture();
const input = launchInput(
workspaceDir,
"replayed-execution-ack-launch",
"replayed-execution-ack",
);
await expect(supervisor.launch(input, TEST_WORKER_ENDPOINT)).resolves.toMatchObject({
state: "running",
worker: expect.any(Object),
});
await vi.waitFor(async () => {
expect((await supervisor.status(input.launchId))?.state).toBe("interrupted");
});
expect(await supervisor.status(input.launchId)).toMatchObject({
state: "interrupted",
worker: expect.any(Object),
});
await supervisor.close();
});
@@ -606,7 +768,7 @@ describe("node worker supervisor", () => {
["cancel", "cancelled"],
["close", "interrupted"],
] as const)(
"%s during startup closes the gate before worker code runs",
"%s after execution readiness settles the durable terminal receipt",
async (operation, state) => {
const { supervisor, workspaceDir } = fixture();
const input = launchInput(workspaceDir, `${operation}-startup-launch`, "tree");
@@ -630,7 +792,6 @@ describe("node worker supervisor", () => {
await stopping;
expect((await supervisor.status(input.launchId))?.state).toBe(state);
expect(fs.existsSync(path.join(workspaceDir, "grandchild.pid"))).toBe(false);
await supervisor.close();
},
);
File diff suppressed because it is too large Load Diff
@@ -97,6 +97,28 @@ afterEach(() => {
});
describe("node worker workspace retention", () => {
it.runIf(process.platform !== "win32" && typeof process.getuid === "function" && process.getuid() > 0)(
"uses a private short-lived relay directory with a portable Unix socket path",
async () => {
const workspace = new NodeWorkerWorkspaceRuntime({
root: tempDirs.make("node-worker-workspace-relay-root-"),
});
const params = {
gatewayNamespace: "gateway-relay",
launchId: "relay-path-test",
planHash: "a".repeat(64),
};
const relayDir = workspace.resolveContainerRelayDirectory(params);
expect(relayDir).toMatch(/^\/tmp\/openclaw-worker-relays-[0-9]+\/[a-f0-9]{32}$/u);
expect(Buffer.byteLength(path.join(relayDir, "gateway.sock"), "utf8")).toBeLessThanOrEqual(100);
expect(fs.statSync(relayDir).mode & 0o077).toBe(0);
await workspace.removeContainerRelayDirectory(params);
expect(fs.existsSync(relayDir)).toBe(false);
},
);
it("does not delete workspaces before the first Gateway snapshot", async () => {
const root = tempDirs.make("node-worker-workspace-retention-startup-");
const { bundleRoot, env, workspaceDir } = writeNodeWorkerFixture(root);
+144 -1
View File
@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
import fs from "node:fs";
import fsp from "node:fs/promises";
import path from "node:path";
import { isRecord } from "@openclaw/normalization-core/record-coerce";
import { resolveStateDir } from "../config/paths.js";
import { isPathInside } from "../infra/path-guards.js";
import { KeyedAsyncQueue } from "../plugin-sdk/keyed-async-queue.js";
@@ -29,6 +30,12 @@ const WORKSPACE_RETENTION_DELETE_LIMIT = 256;
const ENVIRONMENT_HASH_PATTERN = /^[a-f0-9]{16}$/u;
const SESSION_HASH_PATTERN = /^[a-f0-9]{32}$/u;
const MANIFEST_FILE_PATTERN = /^[a-f0-9]{64}\.json$/u;
const NODE_WORKER_CONTAINER_RELAY_ROOT_PREFIX = "/tmp/openclaw-worker-relays-";
const NODE_WORKER_CONTAINER_RELAY_SOCKET_NAME = "gateway.sock";
// macOS accepts fewer bytes than Linux for a filesystem Unix-domain socket.
// Keep the portable bound below both limits rather than advertise a sandbox
// that will only fail after the supervisor has durably launched it.
const NODE_WORKER_CONTAINER_RELAY_SOCKET_MAX_BYTES = 100;
type NodeWorkerWorkspaceLaunchReference = {
gatewayNamespace: string;
@@ -206,6 +213,67 @@ function ensureContainedDirectory(parent: string, name: string): string {
return resolved;
}
function currentNonRootUid(): number {
if (process.platform === "win32" || typeof process.getuid !== "function") {
throw new Error("node worker container relays require a non-root POSIX host user");
}
const uid = process.getuid();
if (!Number.isSafeInteger(uid) || uid <= 0) {
throw new Error("node worker container relays require a non-root POSIX host user");
}
return uid;
}
function containerRelayPaths(params: {
gatewayNamespace: string;
launchId: string;
planHash: string;
}): { root: string; directory: string } {
const uid = currentNonRootUid();
const name = hashPathComponent(
`${params.gatewayNamespace}\0${params.launchId}\0${params.planHash}`,
32,
);
return {
root: `${NODE_WORKER_CONTAINER_RELAY_ROOT_PREFIX}${uid}`,
directory: path.join(`${NODE_WORKER_CONTAINER_RELAY_ROOT_PREFIX}${uid}`, name),
};
}
function ensurePrivateRelayDirectory(candidate: string, expectedParent?: string): void {
fs.mkdirSync(candidate, { recursive: true, mode: 0o700 });
const initialStats = fs.lstatSync(candidate);
if (
initialStats.isSymbolicLink() ||
!initialStats.isDirectory() ||
initialStats.uid !== currentNonRootUid()
) {
throw new Error(
"INVALID_REQUEST: node worker container relay path is not a private owned directory",
);
}
fs.chmodSync(candidate, 0o700);
const stats = fs.lstatSync(candidate);
const resolved = fs.realpathSync.native(candidate);
if (
stats.isSymbolicLink() ||
!stats.isDirectory() ||
stats.uid !== currentNonRootUid() ||
(stats.mode & 0o077) !== 0 ||
(expectedParent !== undefined &&
fs.realpathSync.native(path.dirname(candidate)) !== expectedParent)
) {
throw new Error(
"INVALID_REQUEST: node worker container relay path is not a private owned directory",
);
}
if (resolved !== fs.realpathSync.native(candidate)) {
throw new Error(
"INVALID_REQUEST: node worker container relay path changed while it was prepared",
);
}
}
function resolveArgumentPath(workspaceDir: string, arg: string): string | undefined {
if (path.isAbsolute(arg)) {
return arg;
@@ -233,7 +301,7 @@ function assertWorkspaceArgv(workspaceDir: string, argv: readonly string[]): voi
try {
resolved = fs.realpathSync.native(candidate);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
if (!isRecord(error) || error.code !== "ENOENT") {
throw error;
}
}
@@ -338,6 +406,81 @@ export class NodeWorkerWorkspaceRuntime {
};
}
/**
* Container launches never accept a Gateway-provided host path. The node
* derives this one canonical session/epoch workspace before mounting it at
* the fixed in-container `/workspace` path.
*/
resolveContainerWorkspace(reference: NodeWorkerWorkspaceLaunchReference): string {
if (!Number.isSafeInteger(reference.ownerEpoch) || reference.ownerEpoch < 0) {
throw new Error("INVALID_REQUEST: node worker container owner epoch is invalid");
}
const environmentHash = hashPathComponent(reference.environmentId, 16);
const sessionHash = hashPathComponent(reference.sessionId, 32);
const gatewayRoot = ensureContainedDirectory(this.root, reference.gatewayNamespace);
const workspacesRoot = ensureContainedDirectory(gatewayRoot, "workspaces");
const environmentRoot = ensureContainedDirectory(workspacesRoot, environmentHash);
const sessionRoot = ensureContainedDirectory(environmentRoot, sessionHash);
return ensureContainedDirectory(sessionRoot, String(reference.ownerEpoch));
}
/**
* The relay is node-private staging, outside the worker-writable workspace.
* Its name is derived from the durable launch identity, not from request data.
*/
resolveContainerRelayDirectory(params: {
gatewayNamespace: string;
launchId: string;
planHash: string;
}): string {
if (!/^[a-f0-9]{64}$/u.test(params.planHash)) {
throw new Error("INVALID_REQUEST: node worker container plan hash is invalid");
}
const paths = containerRelayPaths(params);
// The mounted directory is host-private, but the Unix socket must retain
// this short lexical `/tmp` spelling. Canonical state paths exceed macOS'
// socket limit before the container can even connect to the relay.
ensurePrivateRelayDirectory(paths.root);
const canonicalRoot = fs.realpathSync.native(paths.root);
ensurePrivateRelayDirectory(paths.directory, canonicalRoot);
const socketPath = path.join(paths.directory, NODE_WORKER_CONTAINER_RELAY_SOCKET_NAME);
if (Buffer.byteLength(socketPath, "utf8") > NODE_WORKER_CONTAINER_RELAY_SOCKET_MAX_BYTES) {
throw new Error(
"INVALID_REQUEST: node worker container relay socket path exceeds the portable limit",
);
}
return paths.directory;
}
async removeContainerRelayDirectory(params: {
gatewayNamespace: string;
launchId: string;
planHash: string;
}): Promise<void> {
if (!/^[a-f0-9]{64}$/u.test(params.planHash)) {
return;
}
const paths = containerRelayPaths(params);
try {
const rootStats = await fsp.lstat(paths.root);
if (
rootStats.isSymbolicLink() ||
!rootStats.isDirectory() ||
rootStats.uid !== currentNonRootUid() ||
(rootStats.mode & 0o077) !== 0
) {
return;
}
const canonicalRoot = await fsp.realpath(paths.root);
await removeOwnedDirectory(canonicalRoot, paths.directory);
await removeIfEmpty(paths.root);
} catch (error) {
if (!isRecord(error) || error.code !== "ENOENT") {
throw error;
}
}
}
private beginWorkspaceOperation(gatewayNamespace: string, generationKey: string): () => void {
this.activeWorkspaceOperations.set(
generationKey,
+102
View File
@@ -6,6 +6,7 @@ import { getConfigResolutionFacts, setConfigResolutionFacts } from "../config/re
import type { GatewayClientOptions } from "../gateway/client.js";
import {
NODE_RUNNER_INVENTORY_UPDATE_METHOD,
NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
} from "../infra/node-runner-inventory.js";
import type { configureNodeHost } from "./config.js";
@@ -34,6 +35,8 @@ const mocks = vi.hoisted(() => ({
runnerCapacityChanged: undefined as
| ((capacity: { total: number; available: number }) => void)
| undefined,
workerSupervisorReadinessChanged: undefined as ((ready: boolean) => void) | undefined,
resolveNodeWorkerContainerEngine: vi.fn(async () => undefined),
nodeHostCommands: [] as string[],
nodeHostCaps: [] as string[],
availabilityOnWatch: undefined as { caps: string[]; commands: string[] } | undefined,
@@ -107,6 +110,10 @@ vi.mock("../infra/device-identity.js", () => ({
publicKey: "public-key-test",
privateKey: "private-key-test",
})),
loadOrCreateProcessDeviceIdentity: vi.fn(() => ({
deviceId: "device-test",
privateKeyPem: "private-key-test",
})),
}));
vi.mock("../infra/machine-name.js", () => ({
@@ -199,6 +206,7 @@ vi.mock("./runtime.js", async (importOriginal) => {
start: (params) => {
mocks.runtimeClient = params.client;
mocks.runnerCapacityChanged = params.onRunnerCapacityChanged;
mocks.workerSupervisorReadinessChanged = params.onWorkerSupervisorReadinessChanged;
return mocks.activeRuntime;
},
};
@@ -206,6 +214,10 @@ vi.mock("./runtime.js", async (importOriginal) => {
};
});
vi.mock("./node-worker-container-runtime.js", () => ({
resolveNodeWorkerContainerEngine: mocks.resolveNodeWorkerContainerEngine,
}));
function lastCapturedOptions(): GatewayClientOptions | undefined {
const list = mocks.capturedGatewayClientOptions;
return list[list.length - 1];
@@ -232,6 +244,8 @@ describe("runNodeHost", () => {
mocks.useFakeRuntime = false;
mocks.fakeRuntimeWorkerHosting = false;
mocks.runnerCapacityChanged = undefined;
mocks.workerSupervisorReadinessChanged = undefined;
mocks.resolveNodeWorkerContainerEngine.mockResolvedValue(undefined);
mocks.nodeHostCommands = [];
mocks.nodeHostCaps = [];
mocks.availabilityOnWatch = undefined;
@@ -689,6 +703,8 @@ describe("runNodeHost", () => {
});
it("publishes opt-in consent and capacity in the atomic runner inventory", async () => {
mocks.useFakeRuntime = true;
mocks.fakeRuntimeWorkerHosting = true;
mocks.getRuntimeConfig.mockReturnValue({
gateway: { handshakeTimeoutMs: 1_000 },
nodeHost: { workerRuns: { enabled: true } },
@@ -699,6 +715,9 @@ describe("runNodeHost", () => {
const options = mocks.capturedGatewayClientOptions[0];
const client = mocks.capturedGatewayClients[0];
mocks.runnerCapacityChanged?.({ total: 2, available: 2 });
mocks.workerSupervisorReadinessChanged?.(true);
options?.onHelloOk?.({
protocol: 4,
features: { methods: [], events: [] },
@@ -725,6 +744,7 @@ describe("runNodeHost", () => {
expect(options?.workerRuns).toBeUndefined();
mocks.runnerCapacityChanged?.({ total: 2, available: 2 });
mocks.workerSupervisorReadinessChanged?.(true);
options?.onHelloOk?.({
protocol: 4,
features: {
@@ -790,6 +810,88 @@ describe("runNodeHost", () => {
expect(client?.updateNodeManifest).not.toHaveBeenCalled();
});
it("withholds and withdraws process isolation until supervisor recovery is healthy", async () => {
mocks.useFakeRuntime = true;
mocks.fakeRuntimeWorkerHosting = true;
mocks.startGatewayClientWhenEventLoopReady.mockResolvedValueOnce({
ready: true,
aborted: false,
elapsedMs: 0,
});
let resolveEngine!: (value: unknown) => void;
mocks.resolveNodeWorkerContainerEngine.mockImplementationOnce(
async () =>
await new Promise((resolve) => {
resolveEngine = resolve;
}),
);
const processOnceSpy = vi.spyOn(process, "once");
const previousExitCode = process.exitCode;
try {
const running = runNodeHost({ gatewayHost: "127.0.0.1", gatewayPort: 18789 });
await vi.waitFor(() => expect(mocks.workerSupervisorReadinessChanged).toBeTypeOf("function"));
await vi.waitFor(() => expect(resolveEngine).toBeTypeOf("function"));
const options = mocks.capturedGatewayClientOptions[0];
const client = mocks.capturedGatewayClients[0];
const latestRunnerInventory = () =>
client?.request.mock.calls
.filter(([method]) => method === NODE_RUNNER_INVENTORY_UPDATE_METHOD)
.at(-1)?.[1];
mocks.runnerCapacityChanged?.({ total: 2, available: 2 });
options?.onHelloOk?.({
protocol: 7,
features: { methods: [], events: [] },
} as unknown as Parameters<NonNullable<GatewayClientOptions["onHelloOk"]>>[0]);
await vi.waitFor(() =>
expect(latestRunnerInventory()).toEqual({
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE],
workerHost: { enabled: false },
}),
);
resolveEngine({ id: "docker" });
await vi.waitFor(() => expect(mocks.resolveNodeWorkerContainerEngine).toHaveBeenCalledOnce());
expect(latestRunnerInventory()).toEqual({
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE],
workerHost: { enabled: false },
});
mocks.workerSupervisorReadinessChanged?.(true);
await vi.waitFor(() =>
expect(latestRunnerInventory()).toEqual({
protocolFeatures: [NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE],
workerHost: {
enabled: true,
capacity: { total: 2, available: 2 },
bundlePrewarm: 1,
processIsolation: { kind: "container-v1", memoryProjection: 1 },
},
}),
);
mocks.workerSupervisorReadinessChanged?.(false);
await vi.waitFor(() =>
expect(latestRunnerInventory()).toEqual({
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE],
workerHost: { enabled: false },
}),
);
const onSigterm = processOnceSpy.mock.calls.find(([event]) => event === "SIGTERM")?.[1];
onSigterm?.("SIGTERM");
await running;
} finally {
for (const [event, listener] of processOnceSpy.mock.calls) {
if ((event === "SIGINT" || event === "SIGTERM") && typeof listener === "function") {
process.off(event, listener);
}
}
process.exitCode = previousExitCode;
processOnceSpy.mockRestore();
}
});
it("clears gateway plugin tools when the final node-hosted tool disappears", async () => {
mocks.startGatewayClientWhenEventLoopReady.mockResolvedValueOnce({
ready: true,
+32 -2
View File
@@ -19,6 +19,7 @@ import {
NODE_RUNNER_INVENTORY_UPDATE_METHOD,
NODE_WORKER_BUNDLE_RETENTION_VERSION,
NODE_WORKER_BUNDLE_STATUS_VERSION,
NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
type NodeWorkerCapacitySnapshot,
} from "../infra/node-runner-inventory.js";
@@ -35,6 +36,7 @@ import {
coerceNodeInvokePayload,
} from "./invoke-payload.js";
import { prepareNodeHostRuntime, type NodeHostInventory } from "./runtime.js";
import { resolveNodeWorkerContainerEngine } from "./node-worker-container-runtime.js";
import { runStartupMigrations } from "./startup-state-migrations.js";
type NodeHostRunOptions = {
@@ -304,6 +306,8 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise<void> {
let inventory: NodeHostInventory = preparedRuntime.initialInventory;
let workerCapacity: NodeWorkerCapacitySnapshot | undefined;
let workerSupervisorReady = false;
let workerProcessIsolationAvailable = false;
let gatewayHelloReceived = false;
let gatewayConnectionGeneration = 0;
let connectedGatewayProtocol = 0;
@@ -515,12 +519,22 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise<void> {
};
const publishRunnerInventory = () => {
const workerHostAvailable =
preparedRuntime.workerHostingEnabled && workerSupervisorReady && workerCapacity !== undefined;
const processIsolation =
workerHostAvailable && workerProcessIsolationAvailable
? { kind: "container-v1" as const }
: undefined;
queueOptionalPublication(
NODE_RUNNER_INVENTORY_UPDATE_METHOD,
{
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE],
protocolFeatures: [
processIsolation
? NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE
: NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
],
workerHost:
preparedRuntime.workerHostingEnabled && workerCapacity
workerHostAvailable && workerCapacity
? {
enabled: true,
capacity: workerCapacity,
@@ -531,6 +545,9 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise<void> {
...(gatewaySupportsBundleRetention && gatewaySupportsBundleStatus
? { bundleStatus: NODE_WORKER_BUNDLE_STATUS_VERSION }
: {}),
...(processIsolation
? { processIsolation: { ...processIsolation, memoryProjection: 1 } }
: {}),
}
: { enabled: false },
},
@@ -656,6 +673,10 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise<void> {
workerCapacity = capacity;
publishRunnerInventory();
},
onWorkerSupervisorReadinessChanged: (ready) => {
workerSupervisorReady = ready;
publishRunnerInventory();
},
onManifestChanged: (manifest) => {
// Manifest changes force a reconnect. Retire the current publication queue
// now so it cannot drain against the closing connection.
@@ -665,6 +686,15 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise<void> {
});
let stopping = false;
if (preparedRuntime.workerHostingEnabled) {
void resolveNodeWorkerContainerEngine().then((engine) => {
if (stopping || !engine) {
return;
}
workerProcessIsolationAvailable = true;
publishRunnerInventory();
});
}
let resolveStopped: (() => void) | undefined;
const stopped = new Promise<void>((resolve) => {
resolveStopped = resolve;
+18 -4
View File
@@ -64,6 +64,7 @@ type PreparedNodeHostRuntime = {
onInventoryChanged?: (inventory: NodeHostInventory) => void;
onManifestChanged?: (manifest: NodeHostManifest) => void;
onRunnerCapacityChanged?: (capacity: NodeWorkerCapacitySnapshot) => void;
onWorkerSupervisorReadinessChanged?: (ready: boolean) => void;
}): ActiveNodeHostRuntime;
};
@@ -322,7 +323,13 @@ export async function prepareNodeHostRuntime(params?: {
manifest,
workerHostingEnabled: workerRunsEnabled,
initialInventory,
start({ client, onInventoryChanged, onManifestChanged, onRunnerCapacityChanged }) {
start({
client,
onInventoryChanged,
onManifestChanged,
onRunnerCapacityChanged,
onWorkerSupervisorReadinessChanged,
}) {
const mcpAbort = new AbortController();
const workerWorkspace = workerRunsEnabled
? new NodeWorkerWorkspaceRuntime({ env })
@@ -338,9 +345,16 @@ export async function prepareNodeHostRuntime(params?: {
})
: undefined;
if (workerSupervisor) {
void workerSupervisor.initialize().catch((error: unknown) => {
logDebug(`node-host: worker capacity reconciliation failed: ${String(error)}`);
});
// A reconciled slot count is not enough to assert process isolation: a
// failed recovery can leave a durable relay, projection, or container alive.
onWorkerSupervisorReadinessChanged?.(false);
void workerSupervisor
.initialize()
.then(() => onWorkerSupervisorReadinessChanged?.(true))
.catch((error: unknown) => {
onWorkerSupervisorReadinessChanged?.(false);
logDebug(`node-host: worker capacity reconciliation failed: ${String(error)}`);
});
}
const skillBins = new SkillBinsCache(client, pathEnv);
const activeInvokes = new Map<string, ActiveNodeInvoke>();
+1
View File
@@ -7,3 +7,4 @@ export type {
MemoryBrokerRequest,
} from "../memory-broker/protocol.js";
export type { MemoryBrokerHandler } from "../memory-broker/server.js";
export type { MemoryBrokerChildEntry, MemoryBrokerStartupContext } from "../memory-broker/entry.js";
@@ -16,7 +16,6 @@ import {
import {
hasConfiguredStartupChannel,
declaresAllowedEnterpriseIdentityProvider,
listPotentialEnabledChannelIds,
resolveAuthorizedGatewayStartupDreamingPluginIds,
resolveContextEngineSlotStartupPluginId,
resolveMemorySlotStartupPluginId,
+219 -19
View File
@@ -1,5 +1,19 @@
import { describe, expect, it, vi } from "vitest";
import { testing } from "./memory-broker-runtime.js";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { MemoryBrokerProcess } from "../memory-broker/process.js";
import {
closeBrokeredMemoryRuntimes,
startBrokeredMemoryRuntimeSupervisor,
testing,
withBrokeredMemoryMaintenance,
} from "./memory-broker-runtime.js";
import type { MemoryPluginCapability } from "./registry-contribution-types.js";
const startMemoryBrokerProcess = vi.hoisted(() => vi.fn());
vi.mock("../memory-broker/process.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../memory-broker/process.js")>()),
startMemoryBrokerProcess,
}));
function process(params: { running?: boolean } = {}) {
return {
@@ -9,37 +23,223 @@ function process(params: { running?: boolean } = {}) {
};
}
function brokerProcess(): MemoryBrokerProcess {
return {
client: {} as MemoryBrokerProcess["client"],
brokerEpoch: "test-epoch",
isRunning: () => true,
isHealthy: async () => true,
quiesce: async () => {},
resume: async () => {},
close: async () => {},
};
}
const brokerCapability = {
broker: {
version: 1,
kind: "local-child",
moduleUrl: "test:memory-broker-runtime",
},
} satisfies MemoryPluginCapability;
afterEach(async () => {
await closeBrokeredMemoryRuntimes();
startMemoryBrokerProcess.mockReset();
});
describe("brokered memory maintenance", () => {
it("drains every running broker before a backup and resumes them afterwards", async () => {
const first = process();
const second = process();
const stopped = process({ running: false });
it("drains a running broker before maintenance and resumes it afterwards", async () => {
const broker = process();
const run = vi.fn(async () => "snapshot-created");
const retire = vi.fn(async () => {});
await expect(
testing.runBrokeredMemoryMaintenance({ processes: [first, second, stopped], run }),
testing.runBrokeredMemoryMaintenance({ process: broker, run, retire }),
).resolves.toBe("snapshot-created");
expect(first.quiesce).toHaveBeenCalledOnce();
expect(second.quiesce).toHaveBeenCalledOnce();
expect(stopped.quiesce).not.toHaveBeenCalled();
expect(broker.quiesce).toHaveBeenCalledOnce();
expect(run).toHaveBeenCalledOnce();
expect(first.resume).toHaveBeenCalledOnce();
expect(second.resume).toHaveBeenCalledOnce();
expect(broker.resume).toHaveBeenCalledOnce();
expect(retire).not.toHaveBeenCalled();
});
it("reopens an already-drained broker when a later broker or the backup fails", async () => {
const first = process();
const second = process();
second.quiesce.mockRejectedValueOnce(new Error("broker unavailable"));
it("retires and rejects when quiesce fails before the mutation starts", async () => {
const broker = process();
broker.quiesce.mockRejectedValueOnce(new Error("broker unavailable"));
const run = vi.fn(async () => "unreachable");
const retire = vi.fn(async () => {});
await expect(
testing.runBrokeredMemoryMaintenance({ processes: [first, second], run }),
testing.runBrokeredMemoryMaintenance({ process: broker, run, retire }),
).rejects.toThrow("broker unavailable");
expect(run).not.toHaveBeenCalled();
expect(first.resume).toHaveBeenCalledOnce();
expect(second.resume).not.toHaveBeenCalled();
expect(retire).toHaveBeenCalledOnce();
});
it("retires and rejects when resume fails after a mutation", async () => {
const broker = process();
broker.resume.mockRejectedValueOnce(new Error("broker resume unavailable"));
const retire = vi.fn(async () => {});
await expect(
testing.runBrokeredMemoryMaintenance({
process: broker,
run: async () => "snapshot-created",
retire,
}),
).rejects.toThrow("broker resume unavailable");
expect(retire).toHaveBeenCalledOnce();
});
it("serializes maintenance, process resolution, and shutdown on one broker lease", async () => {
const leases = new Map<string, Promise<void>>();
const order: string[] = [];
let releaseMaintenance: (() => void) | undefined;
let markMaintenanceStarted: (() => void) | undefined;
const maintenanceStarted = new Promise<void>((resolve) => {
markMaintenanceStarted = resolve;
});
const maintenanceMayFinish = new Promise<void>((resolve) => {
releaseMaintenance = resolve;
});
const maintenance = testing.withBrokerLease(leases, "memory-core", async () => {
order.push("maintenance:start");
markMaintenanceStarted?.();
await maintenanceMayFinish;
order.push("maintenance:finish");
});
await maintenanceStarted;
const resolve = testing.withBrokerLease(leases, "memory-core", async () => {
order.push("resolve");
});
const shutdown = testing.withBrokerLease(leases, "memory-core", async () => {
order.push("shutdown");
});
await Promise.resolve();
expect(order).toEqual(["maintenance:start"]);
releaseMaintenance?.();
await Promise.all([maintenance, resolve, shutdown]);
expect(order).toEqual(["maintenance:start", "maintenance:finish", "resolve", "shutdown"]);
});
it("blocks broker resolution while maintenance holds the gateway lease without a running broker", async () => {
const gate = testing.createBrokerMaintenanceGate();
const order: string[] = [];
let releaseMaintenance: (() => void) | undefined;
let markMaintenanceStarted: (() => void) | undefined;
const maintenanceStarted = new Promise<void>((resolve) => {
markMaintenanceStarted = resolve;
});
const maintenanceMayFinish = new Promise<void>((resolve) => {
releaseMaintenance = resolve;
});
const maintenance = testing.withGatewayBrokeredMemoryMaintenanceLease(gate, async () => {
order.push("maintenance:start");
markMaintenanceStarted?.();
await maintenanceMayFinish;
order.push("maintenance:finish");
});
await maintenanceStarted;
const resolution = testing.withBrokerLifecycleOperation(gate, async () => {
order.push("broker:start");
});
await Promise.resolve();
expect(order).toEqual(["maintenance:start"]);
releaseMaintenance?.();
await Promise.all([maintenance, resolution]);
expect(order).toEqual(["maintenance:start", "maintenance:finish", "broker:start"]);
});
it("waits for an admitted broker resolution before maintenance starts", async () => {
const gate = testing.createBrokerMaintenanceGate();
const order: string[] = [];
let releaseResolution: (() => void) | undefined;
let markResolutionStarted: (() => void) | undefined;
const resolutionStarted = new Promise<void>((resolve) => {
markResolutionStarted = resolve;
});
const resolutionMayFinish = new Promise<void>((resolve) => {
releaseResolution = resolve;
});
const resolution = testing.withBrokerLifecycleOperation(gate, async () => {
order.push("broker:start");
markResolutionStarted?.();
await resolutionMayFinish;
order.push("broker:finish");
});
await resolutionStarted;
const maintenance = testing.withGatewayBrokeredMemoryMaintenanceLease(gate, async () => {
order.push("maintenance");
});
await Promise.resolve();
expect(order).toEqual(["broker:start"]);
releaseResolution?.();
await Promise.all([resolution, maintenance]);
expect(order).toEqual(["broker:start", "broker:finish", "maintenance"]);
});
it("does not start a broker requested after empty-map maintenance begins", async () => {
const broker = brokerProcess();
startMemoryBrokerProcess.mockResolvedValueOnce(broker);
let releaseMaintenance: (() => void) | undefined;
let markMaintenanceStarted: (() => void) | undefined;
const maintenanceStarted = new Promise<void>((resolve) => {
markMaintenanceStarted = resolve;
});
const maintenanceMayFinish = new Promise<void>((resolve) => {
releaseMaintenance = resolve;
});
const maintenance = withBrokeredMemoryMaintenance(async () => {
markMaintenanceStarted?.();
await maintenanceMayFinish;
});
await maintenanceStarted;
const supervisor = startBrokeredMemoryRuntimeSupervisor(brokerCapability);
await Promise.resolve();
expect(startMemoryBrokerProcess).not.toHaveBeenCalled();
releaseMaintenance?.();
await Promise.all([maintenance, supervisor]);
expect(startMemoryBrokerProcess).toHaveBeenCalledOnce();
// Regression: shutdown stops supervisors before taking the writer lease, so stop's reader
// retirement cannot deadlock behind the writer that is waiting for it.
await expect(closeBrokeredMemoryRuntimes()).resolves.toBeUndefined();
});
it("drains an admitted broker start before maintenance snapshots processes", async () => {
const broker = brokerProcess();
let resolveStart: ((value: MemoryBrokerProcess) => void) | undefined;
let markStartCalled: (() => void) | undefined;
const startCalled = new Promise<void>((resolve) => {
markStartCalled = resolve;
});
const pendingStart = new Promise<MemoryBrokerProcess>((resolve) => {
resolveStart = resolve;
});
startMemoryBrokerProcess.mockImplementationOnce(() => {
markStartCalled?.();
return pendingStart;
});
const supervisor = startBrokeredMemoryRuntimeSupervisor(brokerCapability);
await startCalled;
const run = vi.fn(async () => {});
const maintenance = withBrokeredMemoryMaintenance(run);
await Promise.resolve();
expect(run).not.toHaveBeenCalled();
resolveStart?.(broker);
await Promise.all([supervisor, maintenance]);
expect(run).toHaveBeenCalledOnce();
});
});
+341 -48
View File
@@ -19,12 +19,39 @@ import type {
type BrokerRuntimeState = {
processes: Map<string, Promise<MemoryBrokerProcess>>;
agentIdsByModule: Map<string, readonly string[]>;
supervisors: Set<MemoryBrokerSupervisor>;
leases: Map<string, Promise<void>>;
maintenance: BrokerMaintenanceGate;
closing: boolean;
};
type BrokerMaintenanceGate = {
maintenanceTail: Promise<void>;
maintenancePending: boolean;
activeLifecycleOperations: number;
idleWaiters: Set<() => void>;
};
function createBrokerMaintenanceGate(): BrokerMaintenanceGate {
return {
maintenanceTail: Promise.resolve(),
maintenancePending: false,
activeLifecycleOperations: 0,
idleWaiters: new Set(),
};
}
const state = resolveGlobalSingleton(
Symbol.for("openclaw.memoryBrokerRuntimeState"),
(): BrokerRuntimeState => ({ processes: new Map(), supervisors: new Set() }),
(): BrokerRuntimeState => ({
processes: new Map(),
agentIdsByModule: new Map(),
supervisors: new Set(),
leases: new Map(),
maintenance: createBrokerMaintenanceGate(),
closing: false,
}),
);
type BrokeredMemoryRuntime = Readonly<AuthorizedMemoryRuntime> &
@@ -34,6 +61,19 @@ function resolveCapabilitySnapshotId(context: MemoryAccessContext): string {
return context.delegation?.capabilitySnapshotId ?? context.hostFactsRevision;
}
function actorBindingFor(context: MemoryAccessContext) {
return context.actor.kind === "principal"
? Object.freeze({
kind: "principal" as const,
actorKind: context.actor.actorKind,
principalId: context.actor.principalId,
})
: Object.freeze({
kind: "unattributed" as const,
transportAuditRef: context.actor.transportAuditRef,
});
}
function bindingFor(context: MemoryAccessContext, policyRevision: string) {
return Object.freeze({
agentId: context.agentId,
@@ -41,6 +81,7 @@ function bindingFor(context: MemoryAccessContext, policyRevision: string) {
runId: context.runId,
contextFingerprint: context.contextFingerprint,
subjectRevision: context.subjectRevision,
actor: actorBindingFor(context),
actorRevision: context.actor.evidenceRevision,
capabilitySnapshotId: resolveCapabilitySnapshotId(context),
policyRevision,
@@ -59,14 +100,159 @@ function authorizeExpiry(): number {
return Date.now() + 60_000;
}
async function resolveProcess(
capability: MemoryPluginCapability,
): Promise<MemoryBrokerProcess | undefined> {
const entry = capability.broker;
if (!entry || entry.version !== 1 || entry.kind !== "local-child" || !entry.moduleUrl) {
/**
* The Gateway is the one owner allowed to change a broker's admission state. A lease covers
* replacement and shutdown too, so no child can start or close between quiesce and resume.
*/
async function withBrokerLease<T>(
leases: Map<string, Promise<void>>,
moduleUrl: string,
run: () => Promise<T>,
): Promise<T> {
const previous = leases.get(moduleUrl) ?? Promise.resolve();
let release: (() => void) | undefined;
const current = new Promise<void>((resolve) => {
release = resolve;
});
leases.set(moduleUrl, current);
await previous;
try {
return await run();
} finally {
release?.();
if (leases.get(moduleUrl) === current) {
leases.delete(moduleUrl);
}
}
}
function releaseBrokerLifecycleOperation(gate: BrokerMaintenanceGate): void {
gate.activeLifecycleOperations -= 1;
if (gate.activeLifecycleOperations !== 0) {
return;
}
for (const resolve of gate.idleWaiters) {
resolve();
}
gate.idleWaiters.clear();
}
async function waitForBrokerLifecycleOperations(gate: BrokerMaintenanceGate): Promise<void> {
while (gate.activeLifecycleOperations > 0) {
await new Promise<void>((resolve) => gate.idleWaiters.add(resolve));
}
}
/**
* Resolution and replacement share a read lease. A maintenance writer blocks later lifecycle
* work before it snapshots brokers, while already admitted work finishes before quiesce begins.
*/
async function withBrokerLifecycleOperation<T>(
gate: BrokerMaintenanceGate,
run: () => Promise<T>,
): Promise<T> {
while (true) {
const maintenanceTail = gate.maintenanceTail;
await maintenanceTail;
if (gate.maintenanceTail !== maintenanceTail || gate.maintenancePending) {
continue;
}
gate.activeLifecycleOperations += 1;
if (gate.maintenanceTail === maintenanceTail && !gate.maintenancePending) {
break;
}
releaseBrokerLifecycleOperation(gate);
}
try {
return await run();
} finally {
releaseBrokerLifecycleOperation(gate);
}
}
/**
* A tool request cannot wait behind maintenance: that would retain an agent turn until a backup
* or repair finishes. Refuse admission before taking the reader lease so callers surface the
* intentional memory-unavailable result while Gateway owns the writer lease.
*/
async function tryWithBrokerLifecycleOperation<T>(
gate: BrokerMaintenanceGate,
run: () => Promise<T>,
): Promise<T | undefined> {
if (gate.maintenancePending) {
return undefined;
}
const existing = state.processes.get(entry.moduleUrl);
const maintenanceTail = gate.maintenanceTail;
gate.activeLifecycleOperations += 1;
if (gate.maintenanceTail !== maintenanceTail || gate.maintenancePending) {
releaseBrokerLifecycleOperation(gate);
return undefined;
}
try {
return await run();
} finally {
releaseBrokerLifecycleOperation(gate);
}
}
/**
* Gateway maintenance owns the write lease for the complete drain/mutate/resume lifecycle.
* Marking it pending synchronously prevents a just-in-time resolver from starting a new child.
*/
async function withGatewayBrokeredMemoryMaintenanceLease<T>(
gate: BrokerMaintenanceGate,
run: () => Promise<T>,
): Promise<T> {
const previous = gate.maintenanceTail;
let release: (() => void) | undefined;
const current = new Promise<void>((resolve) => {
release = resolve;
});
const tail = previous.then(() => current);
gate.maintenanceTail = tail;
gate.maintenancePending = true;
await previous;
await waitForBrokerLifecycleOperations(gate);
try {
return await run();
} finally {
release?.();
if (gate.maintenanceTail === tail) {
gate.maintenancePending = false;
}
}
}
function brokerModuleUrl(capability: MemoryPluginCapability): string | undefined {
const entry = capability.broker;
return entry?.version === 1 && entry.kind === "local-child" && entry.moduleUrl
? entry.moduleUrl
: undefined;
}
async function resolveProcess(
capability: MemoryPluginCapability,
maintenanceBehavior: "wait" | "fail" = "wait",
): Promise<MemoryBrokerProcess | undefined> {
const moduleUrl = brokerModuleUrl(capability);
if (!moduleUrl) {
return undefined;
}
const resolveOnLease = async () => {
return await withBrokerLease(state.leases, moduleUrl, async () => {
if (state.closing) {
return undefined;
}
return await resolveProcessOnLease(moduleUrl);
});
};
return maintenanceBehavior === "fail"
? await tryWithBrokerLifecycleOperation(state.maintenance, resolveOnLease)
: await withBrokerLifecycleOperation(state.maintenance, resolveOnLease);
}
async function resolveProcessOnLease(moduleUrl: string): Promise<MemoryBrokerProcess | undefined> {
const existing = state.processes.get(moduleUrl);
if (existing) {
try {
const process = await existing;
@@ -77,23 +263,27 @@ async function resolveProcess(
// failed operation; discard it so a later independently authorized operation starts fresh.
// A PID alone is not readiness: a child with a dead socket/handler must be retired too.
await process.close().catch(() => undefined);
if (state.processes.get(entry.moduleUrl) === existing) {
state.processes.delete(entry.moduleUrl);
if (state.processes.get(moduleUrl) === existing) {
state.processes.delete(moduleUrl);
}
} catch {
if (state.processes.get(entry.moduleUrl) === existing) {
state.processes.delete(entry.moduleUrl);
if (state.processes.get(moduleUrl) === existing) {
state.processes.delete(moduleUrl);
}
}
}
const process = startMemoryBrokerProcess({
brokerId: `selected-memory:${entry.moduleUrl}`,
handlerModuleUrl: entry.moduleUrl,
let process: Promise<MemoryBrokerProcess>;
process = startMemoryBrokerProcess({
brokerId: `selected-memory:${moduleUrl}`,
handlerModuleUrl: moduleUrl,
agentIds: state.agentIdsByModule.get(moduleUrl) ?? [],
}).catch((error: unknown) => {
state.processes.delete(entry.moduleUrl);
if (state.processes.get(moduleUrl) === process) {
state.processes.delete(moduleUrl);
}
throw error;
});
state.processes.set(entry.moduleUrl, process);
state.processes.set(moduleUrl, process);
try {
return await process;
} catch {
@@ -102,15 +292,26 @@ async function resolveProcess(
}
async function retireProcess(capability: MemoryPluginCapability): Promise<void> {
const moduleUrl = capability.broker?.moduleUrl;
const moduleUrl = brokerModuleUrl(capability);
if (!moduleUrl) {
return;
}
const process = state.processes.get(moduleUrl);
await withBrokerLifecycleOperation(state.maintenance, async () => {
await withBrokerLease(state.leases, moduleUrl, async () => {
await retireProcessOnLease(state, moduleUrl);
});
});
}
async function retireProcessOnLease(
runtimeState: BrokerRuntimeState,
moduleUrl: string,
): Promise<void> {
const process = runtimeState.processes.get(moduleUrl);
if (!process) {
return;
}
state.processes.delete(moduleUrl);
runtimeState.processes.delete(moduleUrl);
await (await process).close();
}
@@ -121,10 +322,13 @@ async function retireProcess(capability: MemoryPluginCapability): Promise<void>
*/
export async function startBrokeredMemoryRuntimeSupervisor(
capability: MemoryPluginCapability | undefined,
params: { agentIds?: readonly string[] } = {},
): Promise<MemoryBrokerSupervisor | undefined> {
if (!capability?.broker) {
const moduleUrl = capability ? brokerModuleUrl(capability) : undefined;
if (!moduleUrl) {
return undefined;
}
state.agentIdsByModule.set(moduleUrl, Object.freeze([...new Set(params.agentIds ?? [])].toSorted()));
const supervisor = await startMemoryBrokerSupervisor({
ensureProcess: () => resolveProcess(capability),
retireProcess: () => retireProcess(capability),
@@ -153,7 +357,7 @@ export async function resolveBrokeredMemoryRuntime(
return undefined;
}
const request = async <T>(params: Parameters<MemoryBrokerProcess["client"]["request"]>[0]) => {
const process = await resolveProcess(capability);
const process = await resolveProcess(capability, "fail");
return process ? await process.client.request<T>(params) : undefined;
};
const authorize = async (context: MemoryAccessContext): Promise<AuthorizedMemoryPlan> => {
@@ -209,6 +413,7 @@ export async function resolveBrokeredMemoryRuntime(
...(params.lines !== undefined ? { lines: params.lines } : {}),
},
expiresAtMs,
...(params.signal ? { signal: params.signal } : {}),
});
if (!result) {
throw new Error("memory broker read is unavailable");
@@ -225,6 +430,7 @@ export async function resolveBrokeredMemoryRuntime(
method: "memory.write",
payload: { context: params.context, plan: params.plan, mutation: params.mutation },
expiresAtMs,
...(params.signal ? { signal: params.signal } : {}),
});
if (!result) {
throw new Error("memory broker write is unavailable");
@@ -241,6 +447,7 @@ export async function resolveBrokeredMemoryRuntime(
method: "memory.import",
payload: { context: params.context, plan: params.plan, mutation: params.mutation },
expiresAtMs,
...(params.signal ? { signal: params.signal } : {}),
});
if (!result) {
throw new Error("memory broker import is unavailable");
@@ -257,6 +464,7 @@ export async function resolveBrokeredMemoryRuntime(
method: "memory.sync",
payload: { context: params.context, plan: params.plan },
expiresAtMs,
...(params.signal ? { signal: params.signal } : {}),
});
if (!result) {
throw new Error("memory broker sync is unavailable");
@@ -273,6 +481,7 @@ export async function resolveBrokeredMemoryRuntime(
method: "memory.export",
payload: { context: params.context, plan: params.plan, handles: params.handles },
expiresAtMs,
...(params.signal ? { signal: params.signal } : {}),
});
if (!result) {
throw new Error("memory broker export is unavailable");
@@ -289,6 +498,7 @@ export async function resolveBrokeredMemoryRuntime(
method: "memory.status",
payload: { context: params.context, plan: params.plan },
expiresAtMs,
...(params.signal ? { signal: params.signal } : {}),
});
if (!result) {
throw new Error("memory broker status is unavailable");
@@ -298,6 +508,7 @@ export async function resolveBrokeredMemoryRuntime(
async materializeAuthorizedVirtualView(params: {
context: MemoryContentAccessContext<"read">;
plan: AuthorizedMemoryContentPlan<"read">;
signal?: AbortSignal;
}): Promise<AuthorizedMemoryVirtualView | undefined> {
const expiresAtMs = hasPlanExpiry(params.plan);
if (!expiresAtMs) {
@@ -308,6 +519,7 @@ export async function resolveBrokeredMemoryRuntime(
method: "memory.virtual-view",
payload: { context: params.context, plan: params.plan },
expiresAtMs,
...(params.signal ? { signal: params.signal } : {}),
});
},
async readAuthorizedVirtualFile(
@@ -327,6 +539,7 @@ export async function resolveBrokeredMemoryRuntime(
virtualPath: params.virtualPath,
},
expiresAtMs,
...(params.signal ? { signal: params.signal } : {}),
});
if (!result) {
throw new Error("memory broker virtual file is unavailable");
@@ -346,39 +559,100 @@ export async function resolveBrokeredMemoryRuntime(
}
export async function closeBrokeredMemoryRuntimes(): Promise<void> {
const supervisors = [...state.supervisors];
state.supervisors.clear();
await Promise.all(supervisors.map((supervisor) => supervisor.stop()));
const processes = [...state.processes.values()];
state.processes.clear();
await Promise.all(
processes.map(async (process) => {
try {
await (await process).close();
} catch {
// Shutdown is best effort; a replacement process gets a fresh epoch and cannot reuse it.
}
}),
);
state.closing = true;
try {
// `stop()` retires through the lifecycle reader lease. Stop supervisors before taking the
// shutdown writer so their in-flight probe cannot wait for the writer that waits for them.
const supervisors = [...state.supervisors];
state.supervisors.clear();
await Promise.all(supervisors.map((supervisor) => supervisor.stop()));
await withGatewayBrokeredMemoryMaintenanceLease(state.maintenance, async () => {
const moduleUrls = [...state.processes.keys()].toSorted();
await Promise.all(
moduleUrls.map((moduleUrl) =>
withBrokerLease(state.leases, moduleUrl, async () => {
try {
await retireProcessOnLease(state, moduleUrl);
} catch {
// Gateway shutdown still retires the map entry so a future lifecycle gets a new epoch.
}
}),
),
);
});
} finally {
state.agentIdsByModule.clear();
state.closing = false;
}
}
type BrokerMaintenanceProcess = Pick<MemoryBrokerProcess, "isRunning" | "quiesce" | "resume">;
async function runBrokeredMemoryMaintenance<T>(params: {
processes: readonly BrokerMaintenanceProcess[];
process: BrokerMaintenanceProcess;
run: () => Promise<T>;
retire: () => Promise<void>;
}): Promise<T> {
if (!params.process.isRunning()) {
await params.retire();
return await params.run();
}
try {
await params.process.quiesce();
} catch (error) {
await rejectAfterBrokerRetirement(error, params.retire);
}
let result!: T;
let runError: unknown;
try {
result = await params.run();
} catch (error) {
runError = error;
}
try {
await params.process.resume();
} catch (resumeError) {
await rejectAfterBrokerRetirement(
runError === undefined ? resumeError : new AggregateError([runError, resumeError]),
params.retire,
);
}
if (runError !== undefined) {
throw runError;
}
return result;
}
async function rejectAfterBrokerRetirement(
error: unknown,
retire: () => Promise<void>,
): Promise<never> {
try {
await retire();
} catch (retireError) {
throw new AggregateError([error, retireError], "memory broker retirement failed");
}
throw error;
}
async function runBrokeredMemoryMaintenanceForModule<T>(params: {
runtimeState: BrokerRuntimeState;
moduleUrl: string;
run: () => Promise<T>;
}): Promise<T> {
const running = params.processes.filter((process) => process.isRunning());
const quiesced: BrokerMaintenanceProcess[] = [];
try {
for (const process of running) {
await process.quiesce();
quiesced.push(process);
return await withBrokerLease(params.runtimeState.leases, params.moduleUrl, async () => {
const pendingProcess = params.runtimeState.processes.get(params.moduleUrl);
if (!pendingProcess) {
return await params.run();
}
return await params.run();
} finally {
await Promise.all(quiesced.map((process) => process.resume().catch(() => undefined)));
}
const process = await pendingProcess;
return await runBrokeredMemoryMaintenance({
process,
run: params.run,
retire: () => retireProcessOnLease(params.runtimeState, params.moduleUrl),
});
});
}
/**
@@ -386,10 +660,29 @@ async function runBrokeredMemoryMaintenance<T>(params: {
* their socket, SQLite handle, artifact root, or bootstrap credential in a worker process.
*/
export async function withBrokeredMemoryMaintenance<T>(run: () => Promise<T>): Promise<T> {
const processes = await Promise.all([...state.processes.values()]);
return await runBrokeredMemoryMaintenance({ processes, run });
return await withGatewayBrokeredMemoryMaintenanceLease(state.maintenance, async () => {
// The snapshot happens only after the write lease excludes broker resolution/replacement.
const moduleUrls = [...state.processes.keys()].toSorted();
const runNext = async (index: number): Promise<T> => {
const moduleUrl = moduleUrls[index];
if (!moduleUrl) {
return await run();
}
return await runBrokeredMemoryMaintenanceForModule({
runtimeState: state,
moduleUrl,
run: () => runNext(index + 1),
});
};
return await runNext(0);
});
}
export const testing = {
createBrokerMaintenanceGate,
withBrokerLease,
withBrokerLifecycleOperation,
tryWithBrokerLifecycleOperation,
withGatewayBrokeredMemoryMaintenanceLease,
runBrokeredMemoryMaintenance,
};
+10
View File
@@ -586,6 +586,7 @@ export async function createAuthorizedMemoryWriteInvocation(params: {
export async function writeAuthorizedMemoryForInvocation(params: {
invocation: AuthorizedMemoryWriteInvocation;
mutation: AuthorizedMemoryMutation;
signal?: AbortSignal;
}): Promise<MemoryWriteResult | MemoryInvocationUnavailable> {
const state = writeInvocationStates.get(params.invocation);
const context = state ? readCurrentWriteContext(state) : undefined;
@@ -597,6 +598,7 @@ export async function writeAuthorizedMemoryForInvocation(params: {
context,
plan: state.plan,
mutation: params.mutation,
...(params.signal ? { signal: params.signal } : {}),
} as never);
} catch {
logMemoryInvocationDiagnostic("authorization-failed");
@@ -613,6 +615,7 @@ export async function stageAuthorizedMemorySealedCompactionForInvocation(params:
invocation: AuthorizedMemoryWriteInvocation;
content: string;
transcriptSource: AuthorizedTranscriptDerivationSource;
signal?: AbortSignal;
}): Promise<AuthorizedSealedCompactionArtifact | MemoryInvocationUnavailable> {
const state = writeInvocationStates.get(params.invocation);
const context = state ? readCurrentWriteContext(state) : undefined;
@@ -632,6 +635,7 @@ export async function stageAuthorizedMemorySealedCompactionForInvocation(params:
plan: state.plan,
content: params.content,
transcriptSource: params.transcriptSource,
...(params.signal ? { signal: params.signal } : {}),
});
} catch {
logMemoryInvocationDiagnostic("authorization-failed");
@@ -646,6 +650,7 @@ export async function stageAuthorizedMemorySealedCompactionForInvocation(params:
*/
export async function materializeAuthorizedMemoryVirtualView(params: {
invocation: AuthorizedMemoryReadInvocation;
signal?: AbortSignal;
}): Promise<AuthorizedMemoryVirtualView | MemoryInvocationUnavailable> {
const state = readState(params.invocation);
const context = state ? readCurrentContext(state) : undefined;
@@ -664,6 +669,7 @@ export async function materializeAuthorizedMemoryVirtualView(params: {
const view = await state.virtualView.materializeAuthorizedVirtualView({
context,
plan: state.plan,
...(params.signal ? { signal: params.signal } : {}),
});
const canonical = view
? canonicalizeAuthorizedVirtualView({ view, context, plan: state.plan })
@@ -687,6 +693,7 @@ export async function readAuthorizedMemoryVirtualFile(params: {
invocation: AuthorizedMemoryReadInvocation;
view: AuthorizedMemoryVirtualView;
virtualPath: string;
signal?: AbortSignal;
}): Promise<MemoryReadResult | MemoryInvocationUnavailable> {
const state = readState(params.invocation);
const context = state ? readCurrentContext(state) : undefined;
@@ -708,6 +715,7 @@ export async function readAuthorizedMemoryVirtualFile(params: {
plan: state.plan,
view: params.view,
virtualPath: params.virtualPath,
...(params.signal ? { signal: params.signal } : {}),
});
if (
!validateEnvelope({
@@ -781,6 +789,7 @@ export async function readAuthorizedMemoryForInvocation(params: {
handleId: string;
from?: number;
lines?: number;
signal?: AbortSignal;
}): Promise<MemoryReadResult | MemoryInvocationUnavailable> {
const state = readState(params.invocation);
const context = state ? readCurrentContext(state) : undefined;
@@ -800,6 +809,7 @@ export async function readAuthorizedMemoryForInvocation(params: {
handle,
...(params.from !== undefined ? { from: params.from } : {}),
...(params.lines !== undefined ? { lines: params.lines } : {}),
...(params.signal ? { signal: params.signal } : {}),
} as never);
if (
!validateEnvelope({
@@ -353,6 +353,7 @@ export type MemoryPluginVirtualViewProvider = {
materializeAuthorizedVirtualView(params: {
context: MemoryContentAccessContext<"read">;
plan: AuthorizedMemoryContentPlan<"read">;
signal?: AbortSignal;
}): Promise<AuthorizedMemoryVirtualView | undefined>;
readAuthorizedVirtualFile(params: {
context: MemoryContentAccessContext<"read">;
@@ -360,6 +361,7 @@ export type MemoryPluginVirtualViewProvider = {
view: AuthorizedMemoryVirtualView;
/** Virtual-only slash-separated path, never a host filesystem path. */
virtualPath: string;
signal?: AbortSignal;
}): Promise<AuthorizedMemoryResultEnvelope<MemoryReadResult>>;
};
+1
View File
@@ -53,6 +53,7 @@ export type AuthorizedMemoryReadHost = Readonly<{
handleId: string;
from?: number;
lines?: number;
signal?: AbortSignal;
}) => Promise<MemoryReadResult | AuthorizedMemoryReadUnavailable>;
}>;
@@ -271,9 +271,9 @@ describe("createChildAdapter", () => {
expect(firstSpawnWithFallbackParams().fallbacks).toEqual([]);
expect(firstSpawnWithFallbackParams().options?.stdio).toEqual(["pipe", "pipe", "pipe", "ipc"]);
await adapter.openStartGate?.();
await adapter.openStartGate?.({ launchId: "turn-1", planHash: "a".repeat(64) });
expect(sendMock).toHaveBeenCalledWith(
{ type: "openclaw-worker-start-v1" },
{ type: "openclaw-worker-start-v1", launchId: "turn-1", planHash: "a".repeat(64) },
expect.any(Function),
);
adapter.closeStartGate?.();
+8 -4
View File
@@ -72,12 +72,16 @@ function resolveChildInvocation(params: {
}
type ChildAdapter = SpawnProcessAdapter<NodeJS.Signals | null>;
type WorkerStartIdentity = {
launchId: string;
planHash: string;
};
type WorkerChildAdapter = ChildAdapter & {
closeStartGate?: () => void;
openStartGate?: () => Promise<void>;
openStartGate?: (identity: WorkerStartIdentity) => Promise<void>;
};
const WORKER_START_MESSAGE = { type: "openclaw-worker-start-v1" } as const;
const WORKER_START_MESSAGE_TYPE = "openclaw-worker-start-v1";
function isServiceManagedRuntime(): boolean {
return Boolean(process.env.OPENCLAW_SERVICE_MARKER?.trim());
@@ -516,7 +520,7 @@ export async function createChildAdapter(params: {
let startGateOpened = false;
const openStartGate = params.ownedWorker
? async () => {
? async (identity: WorkerStartIdentity) => {
if (startGateOpened) {
return;
}
@@ -527,7 +531,7 @@ export async function createChildAdapter(params: {
return;
}
try {
child.send(WORKER_START_MESSAGE, (error) => {
child.send({ type: WORKER_START_MESSAGE_TYPE, ...identity }, (error) => {
if (error) {
reject(error);
return;
+2
View File
@@ -14,6 +14,8 @@ export const FIRST_USE_STATE_TABLES = [
"cron_job_runtime_authorities",
"execution_identity_contexts",
"mcp_oauth_pending_authorizations",
"node_worker_container_launches",
"node_worker_container_leases",
"node_worker_launches",
"operator_approval_execution_identities",
"execution_decision_facts",
+14
View File
@@ -1154,6 +1154,18 @@ export interface NodeHostConfig {
version: number;
}
export interface NodeWorkerContainerLaunches {
container_engine: string;
launch_id: string;
plan_hash: string;
}
export interface NodeWorkerContainerLeases {
expires_at_ms: number;
launch_id: string;
plan_hash: string;
}
export interface NodeWorkerLaunches {
completed_at_ms: number | null;
created_at_ms: number;
@@ -2031,6 +2043,8 @@ export interface DB {
model_catalog_remote: ModelCatalogRemote;
native_hook_relay_bridges: NativeHookRelayBridges;
node_host_config: NodeHostConfig;
node_worker_container_launches: NodeWorkerContainerLaunches;
node_worker_container_leases: NodeWorkerContainerLeases;
node_worker_launches: NodeWorkerLaunches;
official_external_plugin_catalog_snapshots: OfficialExternalPluginCatalogSnapshots;
onboarding_recommendations: OnboardingRecommendations;
+24
View File
@@ -1350,6 +1350,30 @@ CREATE TABLE IF NOT EXISTS node_worker_launches (
)
) STRICT;
-- Container launch metadata is deliberately separate from the public receipt.
-- Recovery needs the selected engine to prove cleanup; worker descriptors and
-- credentials remain process memory only.
CREATE TABLE IF NOT EXISTS node_worker_container_launches (
launch_id TEXT NOT NULL PRIMARY KEY
CHECK (length(launch_id) BETWEEN 1 AND 256 AND instr(launch_id, char(0)) = 0),
plan_hash TEXT NOT NULL
CHECK (length(plan_hash) = 64 AND plan_hash NOT GLOB '*[^0-9a-f]*'),
container_engine TEXT NOT NULL CHECK (container_engine IN ('docker', 'podman')),
FOREIGN KEY (launch_id) REFERENCES node_worker_launches(launch_id) ON DELETE CASCADE
) STRICT;
-- The container lease is separate from the launch receipt and engine metadata.
-- It lets a restarted node withdraw an expired projection without retaining the
-- descriptor, projection content, or any Gateway credential in durable state.
CREATE TABLE IF NOT EXISTS node_worker_container_leases (
launch_id TEXT NOT NULL PRIMARY KEY
CHECK (length(launch_id) BETWEEN 1 AND 256 AND instr(launch_id, char(0)) = 0),
plan_hash TEXT NOT NULL
CHECK (length(plan_hash) = 64 AND plan_hash NOT GLOB '*[^0-9a-f]*'),
expires_at_ms INTEGER NOT NULL CHECK (expires_at_ms BETWEEN 0 AND 9007199254740991),
FOREIGN KEY (launch_id) REFERENCES node_worker_launches(launch_id) ON DELETE CASCADE
) STRICT;
CREATE INDEX IF NOT EXISTS idx_node_worker_launches_terminal_completed
ON node_worker_launches(completed_at_ms, launch_id)
WHERE completed_at_ms IS NOT NULL;

Some files were not shown because too many files have changed in this diff Show More