mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-26 12:26:38 -06:00
feat(memory): isolate selected runtime behind broker
This commit is contained in:
@@ -3479,11 +3479,10 @@ src/node-host/invoke-payload.ts 5
|
||||
src/node-host/invoke.ts 7
|
||||
src/node-host/mcp.ts 3
|
||||
src/node-host/node-worker-bundle-installer.ts 1
|
||||
src/node-host/node-worker-supervisor.ts 1
|
||||
src/node-host/node-worker-transfer-client.ts 3
|
||||
src/node-host/node-worker-transfer-http.ts 2
|
||||
src/node-host/node-worker-tree-control.ts 2
|
||||
src/node-host/node-worker-workspace.ts 7
|
||||
src/node-host/node-worker-workspace.ts 6
|
||||
src/node-host/plugin-node-host.ts 1
|
||||
src/node-host/pty-command.ts 11
|
||||
src/node-host/runtime.ts 2
|
||||
@@ -3979,7 +3978,7 @@ src/worker/node-workspace-protocol.ts 3
|
||||
src/worker/worker-command.runtime.ts 1
|
||||
src/worker/worker-connection-admission.ts 2
|
||||
src/worker/worker-connection-frames.ts 4
|
||||
src/worker/worker-process.ts 2
|
||||
src/worker/worker-process.ts 1
|
||||
src/worker/worker-session-tools.ts 4
|
||||
src/worker/workspace-rsync-receiver.ts 1
|
||||
ui/src/api/gateway.ts 1
|
||||
|
||||
@@ -1,28 +1,47 @@
|
||||
import type { MemoryBrokerAuthorizationBinding } from "openclaw/plugin-sdk/memory-broker-runtime";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
authorize: vi.fn(),
|
||||
export: vi.fn(),
|
||||
import: vi.fn(),
|
||||
read: vi.fn(),
|
||||
recoverPendingWrites: vi.fn(),
|
||||
search: vi.fn(),
|
||||
status: vi.fn(),
|
||||
sync: vi.fn(),
|
||||
virtualFile: vi.fn(),
|
||||
virtualView: vi.fn(),
|
||||
write: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./scoped-memory-runtime.js", () => ({
|
||||
builtinScopedMemoryAuthorizedRuntime: {
|
||||
authorize: mocks.authorize,
|
||||
exportAuthorized: mocks.export,
|
||||
importAuthorized: mocks.import,
|
||||
readAuthorized: mocks.read,
|
||||
searchAuthorized: mocks.search,
|
||||
statusAuthorized: mocks.status,
|
||||
syncAuthorized: mocks.sync,
|
||||
writeAuthorized: mocks.write,
|
||||
},
|
||||
builtinScopedMemoryVirtualView: {},
|
||||
builtinScopedMemoryVirtualView: {
|
||||
materializeAuthorizedVirtualView: mocks.virtualView,
|
||||
readAuthorizedVirtualFile: mocks.virtualFile,
|
||||
},
|
||||
recoverBuiltinScopedMemoryPendingWrites: mocks.recoverPendingWrites,
|
||||
}));
|
||||
|
||||
const { createMemoryBrokerHandler } = await import("./broker-entry.js");
|
||||
const { createMemoryBrokerHandler, initializeMemoryBroker } = await import("./broker-entry.js");
|
||||
|
||||
const binding = {
|
||||
const binding: MemoryBrokerAuthorizationBinding = {
|
||||
agentId: "agent-a",
|
||||
sessionId: "session-a",
|
||||
runId: "run-a",
|
||||
contextFingerprint: "context-a",
|
||||
subjectRevision: "subject-a",
|
||||
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
|
||||
actorRevision: "actor-a",
|
||||
// Without a delegation, Gateway uses hostFactsRevision for the capability snapshot.
|
||||
capabilitySnapshotId: "policy-a",
|
||||
@@ -36,12 +55,25 @@ const context = {
|
||||
runId: binding.runId,
|
||||
contextFingerprint: binding.contextFingerprint,
|
||||
subjectRevision: binding.subjectRevision,
|
||||
actor: { evidenceRevision: binding.actorRevision },
|
||||
actor: {
|
||||
kind: binding.actor.kind,
|
||||
actorKind: binding.actor.actorKind,
|
||||
principalId: binding.actor.principalId,
|
||||
evidenceRevision: binding.actorRevision,
|
||||
},
|
||||
delivery: { deliveryRevision: binding.deliveryRevision },
|
||||
hostFactsRevision: binding.policyRevision,
|
||||
};
|
||||
|
||||
describe("memory-core broker entry", () => {
|
||||
it("recovers only the configured agent set before the broker exposes its socket", () => {
|
||||
mocks.recoverPendingWrites.mockClear();
|
||||
|
||||
initializeMemoryBroker({ agentIds: ["main", "work"] });
|
||||
|
||||
expect(mocks.recoverPendingWrites).toHaveBeenCalledWith(["main", "work"]);
|
||||
});
|
||||
|
||||
it("rejects a client-provided context that does not exactly match the Gateway binding", async () => {
|
||||
const handler = createMemoryBrokerHandler();
|
||||
|
||||
@@ -58,8 +90,27 @@ describe("memory-core broker entry", () => {
|
||||
expect(mocks.authorize).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects an actor principal substitution that keeps the same evidence revision", async () => {
|
||||
const handler = createMemoryBrokerHandler();
|
||||
|
||||
await expect(
|
||||
handler({
|
||||
binding,
|
||||
request: {
|
||||
method: "memory.authorize",
|
||||
payload: {
|
||||
context: { ...context, actor: { ...context.actor, principalId: "bob" } },
|
||||
},
|
||||
},
|
||||
signal: new AbortController().signal,
|
||||
}),
|
||||
).rejects.toThrow("memory broker binding is unavailable");
|
||||
expect(mocks.authorize).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("dispatches a bound operation only after its plan policy revision matches", async () => {
|
||||
const handler = createMemoryBrokerHandler();
|
||||
const controller = new AbortController();
|
||||
const plan = { memoryPolicyRevision: binding.policyRevision };
|
||||
mocks.status.mockResolvedValue({ version: 1, value: { backend: "builtin" } });
|
||||
|
||||
@@ -70,10 +121,10 @@ describe("memory-core broker entry", () => {
|
||||
method: "memory.status",
|
||||
payload: { context, plan },
|
||||
},
|
||||
signal: new AbortController().signal,
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).resolves.toEqual({ version: 1, value: { backend: "builtin" } });
|
||||
expect(mocks.status).toHaveBeenCalledWith({ context, plan });
|
||||
expect(mocks.status).toHaveBeenCalledWith({ context, plan, signal: controller.signal });
|
||||
});
|
||||
|
||||
it("forwards the broker cancellation signal into content search", async () => {
|
||||
@@ -101,4 +152,77 @@ describe("memory-core broker entry", () => {
|
||||
signal: controller.signal,
|
||||
});
|
||||
});
|
||||
|
||||
it("forwards the broker cancellation signal into every authorized operation", async () => {
|
||||
const handler = createMemoryBrokerHandler();
|
||||
const controller = new AbortController();
|
||||
const plan = { memoryPolicyRevision: binding.policyRevision };
|
||||
const readContext = { ...context, operation: "read" as const };
|
||||
const writeContext = { ...context, operation: "append" as const };
|
||||
mocks.read.mockResolvedValue({ version: 1, value: { text: "ok" } });
|
||||
mocks.virtualView.mockResolvedValue({ version: 1, files: [] });
|
||||
mocks.virtualFile.mockResolvedValue({ version: 1, value: { text: "ok" } });
|
||||
mocks.write.mockResolvedValue({ status: "committed" });
|
||||
mocks.import.mockResolvedValue({ status: "committed" });
|
||||
mocks.sync.mockResolvedValue({ version: 1, value: { status: "completed" } });
|
||||
mocks.export.mockResolvedValue({ version: 1, value: {} });
|
||||
|
||||
await Promise.all([
|
||||
handler({
|
||||
binding,
|
||||
request: { method: "memory.read", payload: { context: readContext, plan, handle: {} } },
|
||||
signal: controller.signal,
|
||||
}),
|
||||
handler({
|
||||
binding,
|
||||
request: { method: "memory.virtual-view", payload: { context: readContext, plan } },
|
||||
signal: controller.signal,
|
||||
}),
|
||||
handler({
|
||||
binding,
|
||||
request: {
|
||||
method: "memory.virtual-file",
|
||||
payload: { context: readContext, plan, view: {}, virtualPath: "private/1.md" },
|
||||
},
|
||||
signal: controller.signal,
|
||||
}),
|
||||
handler({
|
||||
binding,
|
||||
request: { method: "memory.write", payload: { context: writeContext, plan, mutation: {} } },
|
||||
signal: controller.signal,
|
||||
}),
|
||||
handler({
|
||||
binding,
|
||||
request: {
|
||||
method: "memory.import",
|
||||
payload: { context: writeContext, plan, mutation: { kind: "import" } },
|
||||
},
|
||||
signal: controller.signal,
|
||||
}),
|
||||
handler({
|
||||
binding,
|
||||
request: { method: "memory.sync", payload: { context: writeContext, plan } },
|
||||
signal: controller.signal,
|
||||
}),
|
||||
handler({
|
||||
binding,
|
||||
request: { method: "memory.export", payload: { context: writeContext, plan, handles: [] } },
|
||||
signal: controller.signal,
|
||||
}),
|
||||
]);
|
||||
|
||||
for (const operation of [
|
||||
mocks.read,
|
||||
mocks.virtualView,
|
||||
mocks.virtualFile,
|
||||
mocks.write,
|
||||
mocks.import,
|
||||
mocks.sync,
|
||||
mocks.export,
|
||||
]) {
|
||||
expect(operation).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ signal: controller.signal }),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,10 +9,12 @@ import type {
|
||||
import type {
|
||||
MemoryBrokerAuthorizationBinding,
|
||||
MemoryBrokerHandler,
|
||||
MemoryBrokerStartupContext,
|
||||
} from "openclaw/plugin-sdk/memory-broker-runtime";
|
||||
import {
|
||||
builtinScopedMemoryAuthorizedRuntime,
|
||||
builtinScopedMemoryVirtualView,
|
||||
recoverBuiltinScopedMemoryPendingWrites,
|
||||
} from "./scoped-memory-runtime.js";
|
||||
|
||||
type BrokerPayload = Readonly<{
|
||||
@@ -37,6 +39,27 @@ function asPayload(value: unknown): BrokerPayload {
|
||||
return value as BrokerPayload;
|
||||
}
|
||||
|
||||
function hasBoundActor(params: {
|
||||
binding: MemoryBrokerAuthorizationBinding["actor"];
|
||||
context: MemoryAccessContext;
|
||||
}): boolean {
|
||||
const { binding, context } = params;
|
||||
if (context.actor.kind !== binding.kind) {
|
||||
return false;
|
||||
}
|
||||
if (binding.kind === "principal") {
|
||||
return (
|
||||
context.actor.kind === "principal" &&
|
||||
context.actor.actorKind === binding.actorKind &&
|
||||
context.actor.principalId === binding.principalId
|
||||
);
|
||||
}
|
||||
return (
|
||||
context.actor.kind === "unattributed" &&
|
||||
context.actor.transportAuditRef === binding.transportAuditRef
|
||||
);
|
||||
}
|
||||
|
||||
function assertBoundContext(params: {
|
||||
binding: MemoryBrokerAuthorizationBinding;
|
||||
context: MemoryAccessContext;
|
||||
@@ -52,6 +75,7 @@ function assertBoundContext(params: {
|
||||
context.runId !== binding.runId ||
|
||||
context.contextFingerprint !== binding.contextFingerprint ||
|
||||
context.subjectRevision !== binding.subjectRevision ||
|
||||
!hasBoundActor({ binding: binding.actor, context }) ||
|
||||
context.actor.evidenceRevision !== binding.actorRevision ||
|
||||
capabilitySnapshotId !== binding.capabilitySnapshotId ||
|
||||
policyRevision !== binding.policyRevision ||
|
||||
@@ -75,6 +99,11 @@ function readPlan(payload: BrokerPayload): AuthorizedMemoryPlan {
|
||||
return payload.plan;
|
||||
}
|
||||
|
||||
/** Complete selected-memory recovery before the broker child exposes its authenticated socket. */
|
||||
export function initializeMemoryBroker(context: MemoryBrokerStartupContext): void {
|
||||
recoverBuiltinScopedMemoryPendingWrites(context.agentIds);
|
||||
}
|
||||
|
||||
/**
|
||||
* This entry is loaded only by the Gateway-owned broker child. It owns the selected runtime's
|
||||
* process-local plan and handle maps, so Gateway and workers can retain only opaque DTOs.
|
||||
@@ -115,6 +144,7 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler {
|
||||
handle: payload.handle,
|
||||
...(Number.isSafeInteger(payload.from) ? { from: payload.from } : {}),
|
||||
...(Number.isSafeInteger(payload.lines) ? { lines: payload.lines } : {}),
|
||||
signal,
|
||||
});
|
||||
}
|
||||
case "memory.virtual-view": {
|
||||
@@ -125,6 +155,7 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler {
|
||||
return await builtinScopedMemoryVirtualView.materializeAuthorizedVirtualView({
|
||||
context,
|
||||
plan: readPlan(payload) as never,
|
||||
signal,
|
||||
});
|
||||
}
|
||||
case "memory.virtual-file": {
|
||||
@@ -141,6 +172,7 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler {
|
||||
plan: readPlan(payload) as never,
|
||||
view: payload.view,
|
||||
virtualPath: payload.virtualPath,
|
||||
signal,
|
||||
});
|
||||
}
|
||||
case "memory.write":
|
||||
@@ -151,6 +183,7 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler {
|
||||
context: payload.context,
|
||||
plan: readPlan(payload),
|
||||
mutation: payload.mutation,
|
||||
signal,
|
||||
} as never);
|
||||
case "memory.import":
|
||||
if (!payload.mutation || payload.mutation.kind !== "import") {
|
||||
@@ -160,11 +193,13 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler {
|
||||
context: payload.context,
|
||||
plan: readPlan(payload),
|
||||
mutation: payload.mutation,
|
||||
signal,
|
||||
} as never);
|
||||
case "memory.sync":
|
||||
return await builtinScopedMemoryAuthorizedRuntime.syncAuthorized({
|
||||
context: payload.context,
|
||||
plan: readPlan(payload),
|
||||
signal,
|
||||
} as never);
|
||||
case "memory.export":
|
||||
if (!Array.isArray(payload.handles)) {
|
||||
@@ -174,11 +209,13 @@ export function createMemoryBrokerHandler(): MemoryBrokerHandler {
|
||||
context: payload.context,
|
||||
plan: readPlan(payload),
|
||||
handles: payload.handles,
|
||||
signal,
|
||||
} as never);
|
||||
case "memory.status":
|
||||
return await builtinScopedMemoryAuthorizedRuntime.statusAuthorized({
|
||||
context: payload.context,
|
||||
plan: readPlan(payload),
|
||||
signal,
|
||||
} as never);
|
||||
default:
|
||||
throw new Error("memory broker operation is unavailable");
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import net from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/core";
|
||||
import type {
|
||||
AuthorizedMemoryPlan,
|
||||
MemoryAccessContext,
|
||||
MemoryContentAccessContext,
|
||||
} from "openclaw/plugin-sdk/memory-authorization";
|
||||
import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api";
|
||||
import { closeOpenClawAgentDatabasesForTest } from "openclaw/plugin-sdk/sqlite-runtime-testing";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
@@ -21,7 +26,7 @@ import {
|
||||
consumeAdmittedChannelMemoryIdentityFromContext,
|
||||
createChannelMemoryIdentityAdmission,
|
||||
} from "../../../../src/channels/message-access/memory-identity-admission.js";
|
||||
import { appendSqliteTranscriptMessage } from "../../../../src/config/sessions/session-accessor.sqlite-transcript-write.js";
|
||||
import { appendTranscriptMessage } from "../../../../src/config/sessions/session-accessor.sqlite-transcript-write.js";
|
||||
import { readAuthorizedTranscriptDerivation } from "../../../../src/config/sessions/session-transcript-memory-policy.js";
|
||||
import { withOwnedSessionTranscriptWrites } from "../../../../src/config/sessions/transcript-write-context.js";
|
||||
import {
|
||||
@@ -29,7 +34,12 @@ import {
|
||||
resetAgentRunRegistryForTest,
|
||||
} from "../../../../src/infra/agent-run-registry.js";
|
||||
import { startMemoryBrokerProcess } from "../../../../src/memory-broker/process.js";
|
||||
import { requestJsonlSocket } from "../../../../src/infra/jsonl-socket.js";
|
||||
import { admitMemoryAuthorizationReadRuntime } from "../../../../src/plugins/memory-authorization-runtime.js";
|
||||
import {
|
||||
closeBrokeredMemoryRuntimes,
|
||||
withBrokeredMemoryMaintenance,
|
||||
} from "../../../../src/plugins/memory-broker-runtime.js";
|
||||
import { resetMemoryIsolationCutoverForTest } from "../../../../src/plugins/memory-cutover.js";
|
||||
import {
|
||||
persistMemoryRunExposureBeforeContentInDatabase,
|
||||
@@ -56,6 +66,8 @@ import {
|
||||
openOpenClawStateDatabase,
|
||||
} from "../../../../src/state/openclaw-state-db.js";
|
||||
import { ensureProfileForEmail } from "../../../../src/state/user-profiles.js";
|
||||
import { verifyNodeWorkerContainerProjectionIsolation } from "../../../../test/helpers/node-worker-container-projection-isolation.js";
|
||||
import memoryCorePlugin from "../../index.js";
|
||||
import { MEMORY_CORE_AUTHORIZATION_CAPABILITIES } from "../authorization.js";
|
||||
import { createMemoryRuntime } from "../runtime-provider.js";
|
||||
import { resolveScopedMemoryArtifactBase, withScopedMemoryDatabase } from "./scoped-memory-db.js";
|
||||
@@ -63,6 +75,7 @@ import { builtinScopedMemoryConformanceAdapter } from "./scoped-memory-policy.js
|
||||
import {
|
||||
createBuiltinScopedMemoryResource,
|
||||
readBuiltinScopedMemoryRevisionSnapshot,
|
||||
resolveBuiltinScopedMemoryArtifactPath,
|
||||
setBuiltinScopedMemoryRevisionLifecycle,
|
||||
} from "./scoped-memory-resources.js";
|
||||
import {
|
||||
@@ -86,6 +99,62 @@ vi.mock("../../../../src/auto-reply/reply/dispatch-from-config.js", () => ({
|
||||
|
||||
const { dispatchInboundMessage } = await import("../../../../src/auto-reply/dispatch.js");
|
||||
|
||||
const dockerAvailable =
|
||||
spawnSync("docker", ["info"], { stdio: "ignore", timeout: 3_000 }).status === 0;
|
||||
|
||||
function constrainedSandboxConfig(params: {
|
||||
image: string;
|
||||
prefix: string;
|
||||
workspaceRoot: string;
|
||||
}): OpenClawConfig {
|
||||
return {
|
||||
agents: {
|
||||
defaults: {
|
||||
skipBootstrap: true,
|
||||
sandbox: {
|
||||
mode: "all",
|
||||
backend: "docker",
|
||||
scope: "session",
|
||||
workspaceAccess: "none",
|
||||
workspaceRoot: params.workspaceRoot,
|
||||
docker: { image: params.image, containerPrefix: params.prefix },
|
||||
browser: { enabled: false },
|
||||
prune: { idleHours: 0, maxAgeDays: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function resolveSelectedMemoryBrokerChildPid(): number {
|
||||
const processList = spawnSync("ps", ["-axo", "pid=,ppid=,command="], { encoding: "utf8" });
|
||||
if (processList.status !== 0) {
|
||||
throw new Error("fixture failed to inspect the selected memory broker child");
|
||||
}
|
||||
const child = processList.stdout
|
||||
.split("\n")
|
||||
.map((line) => /^\s*(\d+)\s+(\d+)\s+(.*)$/u.exec(line))
|
||||
.find(
|
||||
(match) =>
|
||||
match?.[2] === String(process.pid) &&
|
||||
/(?:^|[\\/])memory-broker[\\/]child\.(?:ts|js)(?:\s|$)/u.test(match[3]),
|
||||
);
|
||||
if (!child?.[1]) {
|
||||
throw new Error("fixture failed to locate the selected memory broker child");
|
||||
}
|
||||
return Number(child[1]);
|
||||
}
|
||||
|
||||
function resolveNewMemoryBrokerSocketPath(existingDirectories: ReadonlySet<string>): string {
|
||||
const directory = fs
|
||||
.readdirSync(os.tmpdir())
|
||||
.find((name) => name.startsWith("openclaw-memory-broker-") && !existingDirectories.has(name));
|
||||
if (!directory) {
|
||||
throw new Error("fixture failed to locate the selected memory broker socket");
|
||||
}
|
||||
return path.join(os.tmpdir(), directory, "broker.sock");
|
||||
}
|
||||
|
||||
describe("builtin scoped authorized runtime", () => {
|
||||
let stateDir = "";
|
||||
|
||||
@@ -94,7 +163,8 @@ describe("builtin scoped authorized runtime", () => {
|
||||
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
afterEach(async () => {
|
||||
await closeBrokeredMemoryRuntimes();
|
||||
dispatchReplyFromConfig.mockReset();
|
||||
resetBuiltinScopedMemoryAuthorizedRuntimeForTest();
|
||||
resetMemoryIsolationCutoverForTest();
|
||||
@@ -129,11 +199,45 @@ describe("builtin scoped authorized runtime", () => {
|
||||
authorizationConformance: builtinScopedMemoryConformanceAdapter,
|
||||
virtualView: builtinScopedMemoryVirtualView,
|
||||
runtime: { ...createMemoryRuntime(), ...builtinScopedMemoryAuthorizedRuntime },
|
||||
broker: {
|
||||
version: 1,
|
||||
kind: "local-child",
|
||||
moduleUrl: new URL("./broker-entry.ts", import.meta.url).href,
|
||||
},
|
||||
},
|
||||
});
|
||||
setActivePluginRegistry(registry);
|
||||
}
|
||||
|
||||
function installDefaultMemoryCoreSelectedRuntime() {
|
||||
const registry = createEmptyPluginRegistry();
|
||||
registry.plugins.push({ id: "memory-core", memorySlotSelected: true } as never);
|
||||
const runtime = {
|
||||
llm: { acquireLocalService: async () => undefined },
|
||||
state: {
|
||||
openKeyedStore: () => ({
|
||||
lookup: () => undefined,
|
||||
register: () => undefined,
|
||||
delete: () => undefined,
|
||||
list: () => [],
|
||||
}),
|
||||
},
|
||||
} as unknown as OpenClawPluginApi["runtime"];
|
||||
memoryCorePlugin.register(
|
||||
createTestPluginApi({
|
||||
runtime,
|
||||
registerMemoryCapability(capability) {
|
||||
registry.memoryCapabilities.push({ pluginId: "memory-core", capability });
|
||||
},
|
||||
}),
|
||||
);
|
||||
if (registry.memoryCapabilities.length !== 1) {
|
||||
throw new Error("fixture failed to register the default memory-core capability");
|
||||
}
|
||||
setActivePluginRegistry(registry);
|
||||
return registry.memoryCapabilities[0]!.capability;
|
||||
}
|
||||
|
||||
function createSession(params: {
|
||||
sessionKey: string;
|
||||
sessionId: string;
|
||||
@@ -543,18 +647,61 @@ describe("builtin scoped authorized runtime", () => {
|
||||
|
||||
it("keeps the selected runtime plan state in a real broker child across IPC", async () => {
|
||||
createPrivateResource("alice", "BROKER_CHILD_SERIALIZED_PLAN_ONLY");
|
||||
const startupRecovered = createWriteRecoveryFixture({
|
||||
principalId: "alice",
|
||||
content: "BROKER_STARTUP_RECOVERY_SENTINEL",
|
||||
placement: "stage",
|
||||
});
|
||||
const context = createContext("alice");
|
||||
const broker = await startMemoryBrokerProcess({
|
||||
brokerId: "memory-core-test-broker",
|
||||
handlerModuleUrl: new URL("./broker-entry.ts", import.meta.url).href,
|
||||
agentIds: ["main"],
|
||||
});
|
||||
try {
|
||||
// The child reached ready only after its plugin startup hook recovered the staged revision.
|
||||
// This assertion occurs before the first broker request, so authorize cannot hide deferred
|
||||
// recovery as request-time repair.
|
||||
withScopedMemoryDatabase("main", (database) => {
|
||||
expect(
|
||||
database
|
||||
.prepare(
|
||||
`SELECT revision.lifecycle_state, intent.state
|
||||
FROM memory_resource_revisions AS revision
|
||||
JOIN memory_write_intents AS intent ON intent.pending_revision_id = revision.revision_id
|
||||
WHERE revision.revision_id = ?`,
|
||||
)
|
||||
.get(startupRecovered.revisionId),
|
||||
).toEqual({ lifecycle_state: "active", state: "active" });
|
||||
expect(
|
||||
database
|
||||
.prepare("SELECT text FROM memory_scoped_chunks WHERE revision_id = ?")
|
||||
.all(startupRecovered.revisionId),
|
||||
).toEqual([{ text: "BROKER_STARTUP_RECOVERY_SENTINEL" }]);
|
||||
});
|
||||
expect(fs.existsSync(path.join(startupRecovered.directory, startupRecovered.stageLocator))).toBe(
|
||||
false,
|
||||
);
|
||||
expect(fs.existsSync(path.join(startupRecovered.directory, startupRecovered.finalLocator))).toBe(
|
||||
true,
|
||||
);
|
||||
const authorizationBinding = {
|
||||
agentId: context.agentId,
|
||||
sessionId: context.sessionId,
|
||||
runId: context.runId,
|
||||
contextFingerprint: context.contextFingerprint,
|
||||
subjectRevision: context.subjectRevision,
|
||||
actor:
|
||||
context.actor.kind === "principal"
|
||||
? {
|
||||
kind: "principal" as const,
|
||||
actorKind: context.actor.actorKind,
|
||||
principalId: context.actor.principalId,
|
||||
}
|
||||
: {
|
||||
kind: "unattributed" as const,
|
||||
transportAuditRef: context.actor.transportAuditRef,
|
||||
},
|
||||
actorRevision: context.actor.evidenceRevision,
|
||||
capabilitySnapshotId: context.delegation?.capabilitySnapshotId ?? context.hostFactsRevision,
|
||||
policyRevision: context.hostFactsRevision,
|
||||
@@ -591,6 +738,65 @@ describe("builtin scoped authorized runtime", () => {
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
await expect(
|
||||
broker.client.request({
|
||||
binding: { ...authorizationBinding, policyRevision: plan.memoryPolicyRevision },
|
||||
method: "memory.search",
|
||||
payload: {
|
||||
context: { ...context, sessionId: "replayed-session" },
|
||||
plan,
|
||||
query: "BROKER_CHILD_SERIALIZED_PLAN_ONLY",
|
||||
limit: 10,
|
||||
},
|
||||
expiresAtMs: Date.parse(plan.expiresAt),
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
await expect(
|
||||
broker.client.request({
|
||||
binding: { ...authorizationBinding, policyRevision: plan.memoryPolicyRevision },
|
||||
method: "memory.search",
|
||||
payload: {
|
||||
context: { ...context, agentId: "replayed-agent" },
|
||||
plan,
|
||||
query: "BROKER_CHILD_SERIALIZED_PLAN_ONLY",
|
||||
limit: 10,
|
||||
},
|
||||
expiresAtMs: Date.parse(plan.expiresAt),
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
await expect(
|
||||
broker.client.request({
|
||||
binding: { ...authorizationBinding, policyRevision: plan.memoryPolicyRevision },
|
||||
method: "memory.search",
|
||||
payload: {
|
||||
context: {
|
||||
...context,
|
||||
actor: { ...context.actor, principalId: "mallory" },
|
||||
},
|
||||
plan,
|
||||
query: "BROKER_CHILD_SERIALIZED_PLAN_ONLY",
|
||||
limit: 10,
|
||||
},
|
||||
expiresAtMs: Date.parse(plan.expiresAt),
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
await expect(
|
||||
broker.client.request({
|
||||
binding: { ...authorizationBinding, policyRevision: plan.memoryPolicyRevision },
|
||||
method: "memory.search",
|
||||
payload: {
|
||||
context: { ...context, hostFactsRevision: "revoked-capability-snapshot" },
|
||||
plan,
|
||||
query: "BROKER_CHILD_SERIALIZED_PLAN_ONLY",
|
||||
limit: 10,
|
||||
},
|
||||
expiresAtMs: Date.parse(plan.expiresAt),
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
await expect(
|
||||
broker.client.request({
|
||||
binding: { ...authorizationBinding, policyRevision: "stale-policy-revision" },
|
||||
@@ -1230,7 +1436,7 @@ describe("builtin scoped authorized runtime", () => {
|
||||
withTranscriptWrite: async (run) => await run(),
|
||||
},
|
||||
async () => {
|
||||
await appendSqliteTranscriptMessage(
|
||||
await appendTranscriptMessage(
|
||||
{
|
||||
agentId: context.agentId,
|
||||
sessionId: context.sessionId,
|
||||
@@ -1360,6 +1566,484 @@ describe("builtin scoped authorized runtime", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("returns intentional unavailability while maintenance quiesces the real selected broker", async () => {
|
||||
const session = { sessionKey: "agent:main:direct:alice", sessionId: "alice-session" };
|
||||
const alicePrincipalId = createVerifiedDirectSession({ name: "alice", ...session });
|
||||
createPrivateResource(alicePrincipalId, "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL");
|
||||
markCutOver();
|
||||
installBuiltinSelectedRuntime();
|
||||
|
||||
const host = createAuthorizedMemoryReadHost({
|
||||
agentId: "main",
|
||||
...session,
|
||||
deliveryContext: { channel: "telegram", accountId: "default", to: "alice" },
|
||||
});
|
||||
if (!host) {
|
||||
throw new Error("fixture failed to build an authorized memory host");
|
||||
}
|
||||
|
||||
await expect(
|
||||
host.search({ query: "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL", limit: 1 }),
|
||||
).resolves.toMatchObject({
|
||||
results: [{ snippet: "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL" }],
|
||||
});
|
||||
|
||||
let enterMaintenance: (() => void) | undefined;
|
||||
let releaseMaintenance: (() => void) | undefined;
|
||||
const maintenanceEntered = new Promise<void>((resolve) => {
|
||||
enterMaintenance = resolve;
|
||||
});
|
||||
const maintenanceRelease = new Promise<void>((resolve) => {
|
||||
releaseMaintenance = resolve;
|
||||
});
|
||||
const maintenance = withBrokeredMemoryMaintenance(async () => {
|
||||
enterMaintenance?.();
|
||||
await maintenanceRelease;
|
||||
});
|
||||
await maintenanceEntered;
|
||||
|
||||
await expect(
|
||||
host.search({ query: "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL", limit: 1 }),
|
||||
).resolves.toEqual({
|
||||
disabled: true,
|
||||
unavailable: true,
|
||||
error: "memory unavailable",
|
||||
});
|
||||
|
||||
releaseMaintenance?.();
|
||||
await maintenance;
|
||||
await expect(
|
||||
host.search({ query: "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL", limit: 1 }),
|
||||
).resolves.toMatchObject({
|
||||
results: [{ snippet: "MAINTENANCE_REAL_SELECTED_BROKER_SENTINEL" }],
|
||||
});
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"restarts the default selected broker only after Gateway reauthorizes and rejects a pre-crash envelope",
|
||||
async () => {
|
||||
const session = { sessionKey: "agent:main:direct:alice", sessionId: "alice-session" };
|
||||
const alicePrincipalId = createVerifiedDirectSession({ name: "alice", ...session });
|
||||
createPrivateResource(alicePrincipalId, "SELECTED_BROKER_CRASH_RESTART_SENTINEL");
|
||||
markCutOver();
|
||||
installDefaultMemoryCoreSelectedRuntime();
|
||||
|
||||
const brokerDirectoriesBefore = new Set(
|
||||
fs.readdirSync(os.tmpdir()).filter((name) => name.startsWith("openclaw-memory-broker-")),
|
||||
);
|
||||
const createHost = () => {
|
||||
const host = createAuthorizedMemoryReadHost({
|
||||
agentId: "main",
|
||||
...session,
|
||||
deliveryContext: { channel: "telegram", accountId: "default", to: "alice" },
|
||||
});
|
||||
if (!host) {
|
||||
throw new Error("fixture failed to build an authorized memory host");
|
||||
}
|
||||
return host;
|
||||
};
|
||||
const host = createHost();
|
||||
|
||||
const writes = vi.spyOn(net.Socket.prototype, "write");
|
||||
let staleRequestLine: string | undefined;
|
||||
try {
|
||||
await expect(
|
||||
host.search({ query: "SELECTED_BROKER_CRASH_RESTART_SENTINEL", limit: 1 }),
|
||||
).resolves.toMatchObject({
|
||||
results: [{ snippet: "SELECTED_BROKER_CRASH_RESTART_SENTINEL" }],
|
||||
});
|
||||
staleRequestLine = writes.mock.calls
|
||||
.map(([chunk]) => (typeof chunk === "string" ? chunk.trim() : undefined))
|
||||
.find((line) => {
|
||||
if (!line) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
const frame = JSON.parse(line) as {
|
||||
envelope?: unknown;
|
||||
request?: { method?: unknown };
|
||||
};
|
||||
return frame.envelope !== undefined && frame.request?.method === "memory.authorize";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
writes.mockRestore();
|
||||
}
|
||||
if (!staleRequestLine) {
|
||||
throw new Error("fixture failed to capture a signed selected-broker authorization frame");
|
||||
}
|
||||
|
||||
const firstSocketPath = resolveNewMemoryBrokerSocketPath(brokerDirectoriesBefore);
|
||||
expect(fs.existsSync(firstSocketPath)).toBe(true);
|
||||
process.kill(resolveSelectedMemoryBrokerChildPid(), "SIGKILL");
|
||||
await vi.waitFor(() => expect(fs.existsSync(firstSocketPath)).toBe(false), {
|
||||
timeout: 2_000,
|
||||
interval: 20,
|
||||
});
|
||||
|
||||
await vi.waitFor(
|
||||
async () => {
|
||||
await expect(
|
||||
createHost().search({ query: "SELECTED_BROKER_CRASH_RESTART_SENTINEL", limit: 1 }),
|
||||
).resolves.toMatchObject({
|
||||
results: [{ snippet: "SELECTED_BROKER_CRASH_RESTART_SENTINEL" }],
|
||||
});
|
||||
},
|
||||
{ timeout: 2_000, interval: 20 },
|
||||
);
|
||||
|
||||
const replacementSocketPath = resolveNewMemoryBrokerSocketPath(
|
||||
new Set([...brokerDirectoriesBefore, path.basename(path.dirname(firstSocketPath))]),
|
||||
);
|
||||
expect(replacementSocketPath).not.toBe(firstSocketPath);
|
||||
await expect(
|
||||
requestJsonlSocket({
|
||||
socketPath: replacementSocketPath,
|
||||
requestLine: staleRequestLine,
|
||||
timeoutMs: 1_000,
|
||||
keepWriteOpen: true,
|
||||
accept: (response) =>
|
||||
response && typeof response === "object" && !Array.isArray(response)
|
||||
? (response as { ok: boolean; error?: string })
|
||||
: undefined,
|
||||
}),
|
||||
).resolves.toEqual({ ok: false, error: "unauthorized" });
|
||||
},
|
||||
);
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"rotates the selected broker epoch across controlled Gateway shutdown before reauthorization",
|
||||
async () => {
|
||||
const session = { sessionKey: "agent:main:direct:alice", sessionId: "alice-session" };
|
||||
const alicePrincipalId = createVerifiedDirectSession({ name: "alice", ...session });
|
||||
createPrivateResource(alicePrincipalId, "SELECTED_BROKER_CONTROLLED_RESTART_SENTINEL");
|
||||
markCutOver();
|
||||
installDefaultMemoryCoreSelectedRuntime();
|
||||
|
||||
const brokerDirectoriesBefore = new Set(
|
||||
fs.readdirSync(os.tmpdir()).filter((name) => name.startsWith("openclaw-memory-broker-")),
|
||||
);
|
||||
const createHost = () => {
|
||||
const host = createAuthorizedMemoryReadHost({
|
||||
agentId: "main",
|
||||
...session,
|
||||
deliveryContext: { channel: "telegram", accountId: "default", to: "alice" },
|
||||
});
|
||||
if (!host) {
|
||||
throw new Error("fixture failed to build an authorized memory host");
|
||||
}
|
||||
return host;
|
||||
};
|
||||
|
||||
const writes = vi.spyOn(net.Socket.prototype, "write");
|
||||
let preShutdownRequestLine: string | undefined;
|
||||
try {
|
||||
await expect(
|
||||
createHost().search({ query: "SELECTED_BROKER_CONTROLLED_RESTART_SENTINEL", limit: 1 }),
|
||||
).resolves.toMatchObject({
|
||||
results: [{ snippet: "SELECTED_BROKER_CONTROLLED_RESTART_SENTINEL" }],
|
||||
});
|
||||
preShutdownRequestLine = writes.mock.calls
|
||||
.map(([chunk]) => (typeof chunk === "string" ? chunk.trim() : undefined))
|
||||
.find((line) => {
|
||||
if (!line) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
const frame = JSON.parse(line) as {
|
||||
envelope?: unknown;
|
||||
request?: { method?: unknown };
|
||||
};
|
||||
return frame.envelope !== undefined && frame.request?.method === "memory.authorize";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
writes.mockRestore();
|
||||
}
|
||||
if (!preShutdownRequestLine) {
|
||||
throw new Error("fixture failed to capture a signed pre-shutdown authorization frame");
|
||||
}
|
||||
|
||||
const firstSocketPath = resolveNewMemoryBrokerSocketPath(brokerDirectoriesBefore);
|
||||
await closeBrokeredMemoryRuntimes();
|
||||
await vi.waitFor(() => expect(fs.existsSync(firstSocketPath)).toBe(false), {
|
||||
timeout: 2_000,
|
||||
interval: 20,
|
||||
});
|
||||
|
||||
await expect(
|
||||
createHost().search({ query: "SELECTED_BROKER_CONTROLLED_RESTART_SENTINEL", limit: 1 }),
|
||||
).resolves.toMatchObject({
|
||||
results: [{ snippet: "SELECTED_BROKER_CONTROLLED_RESTART_SENTINEL" }],
|
||||
});
|
||||
const replacementSocketPath = resolveNewMemoryBrokerSocketPath(
|
||||
new Set([...brokerDirectoriesBefore, path.basename(path.dirname(firstSocketPath))]),
|
||||
);
|
||||
await expect(
|
||||
requestJsonlSocket({
|
||||
socketPath: replacementSocketPath,
|
||||
requestLine: preShutdownRequestLine,
|
||||
timeoutMs: 1_000,
|
||||
keepWriteOpen: true,
|
||||
accept: (response) =>
|
||||
response && typeof response === "object" && !Array.isArray(response)
|
||||
? (response as { ok: boolean; error?: string })
|
||||
: undefined,
|
||||
}),
|
||||
).resolves.toEqual({ ok: false, error: "unauthorized" });
|
||||
},
|
||||
);
|
||||
|
||||
it.runIf(dockerAvailable && process.env.OPENCLAW_PROCESS_ISOLATION_E2E === "1")(
|
||||
"keeps real memory-core artifacts behind its broker when a malicious model-facing tool runs in a constrained process",
|
||||
async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-memory-core-container-e2e-"));
|
||||
const workspaceDir = path.join(root, "workspace");
|
||||
const image = process.env.OPENCLAW_SANDBOX_TEST_IMAGE ?? "openclaw-sandbox:bookworm-slim";
|
||||
const session = { sessionKey: "agent:main:direct:alice", sessionId: "alice-session" };
|
||||
const bobSession = { sessionKey: "agent:main:direct:bob", sessionId: "bob-session" };
|
||||
const disposeProjections: Array<() => Promise<void>> = [];
|
||||
const runtimeIds: string[] = [];
|
||||
try {
|
||||
fs.mkdirSync(workspaceDir, { recursive: true });
|
||||
vi.stubEnv("OPENCLAW_MEMORY_BROKER_TEST_SECRET", "gateway-only-test-secret");
|
||||
const alicePrincipalId = createVerifiedDirectSession({ name: "alice", ...session });
|
||||
const aliceResource = createPrivateResource(
|
||||
alicePrincipalId,
|
||||
"ALICE_REAL_BROKER_ARTIFACT_SENTINEL",
|
||||
);
|
||||
const bobPrincipalId = createVerifiedDirectSession({
|
||||
name: "bob",
|
||||
...bobSession,
|
||||
});
|
||||
const bobResource = createPrivateResource(
|
||||
bobPrincipalId,
|
||||
"BOB_REAL_BROKER_ARTIFACT_SENTINEL",
|
||||
);
|
||||
markCutOver();
|
||||
const capability = installDefaultMemoryCoreSelectedRuntime();
|
||||
expect(capability.broker).toMatchObject({ version: 1, kind: "local-child" });
|
||||
|
||||
const brokerDirectoriesBefore = new Set(
|
||||
fs.readdirSync(os.tmpdir()).filter((name) => name.startsWith("openclaw-memory-broker-")),
|
||||
);
|
||||
const host = createAuthorizedMemoryReadHost({
|
||||
agentId: "main",
|
||||
...session,
|
||||
deliveryContext: { channel: "telegram", accountId: "default", to: "alice" },
|
||||
});
|
||||
if (!host) {
|
||||
throw new Error("fixture failed to construct the selected memory-core host");
|
||||
}
|
||||
const indexed = await host.search({ query: "REAL_BROKER_ARTIFACT_SENTINEL", limit: 10 });
|
||||
expect(indexed).toMatchObject({
|
||||
results: [{ path: "memory/MEMORY.md", snippet: "ALICE_REAL_BROKER_ARTIFACT_SENTINEL" }],
|
||||
});
|
||||
expect(JSON.stringify(indexed)).not.toContain("BOB_REAL_BROKER_ARTIFACT_SENTINEL");
|
||||
const broker = await resolveAuthorizedMemoryVirtualFileBroker(host);
|
||||
const virtualFile = broker?.view.files[0];
|
||||
if (!broker || !virtualFile) {
|
||||
throw new Error("fixture failed to materialize a broker-backed virtual view");
|
||||
}
|
||||
await expect(broker.readFile(virtualFile.virtualPath)).resolves.toBe(
|
||||
"ALICE_REAL_BROKER_ARTIFACT_SENTINEL",
|
||||
);
|
||||
|
||||
const brokerDirectory = fs
|
||||
.readdirSync(os.tmpdir())
|
||||
.find(
|
||||
(name) =>
|
||||
name.startsWith("openclaw-memory-broker-") && !brokerDirectoriesBefore.has(name),
|
||||
);
|
||||
if (!brokerDirectory) {
|
||||
throw new Error("fixture failed to locate the selected memory-core broker socket");
|
||||
}
|
||||
const brokerSocketPath = path.join(os.tmpdir(), brokerDirectory, "broker.sock");
|
||||
expect(fs.existsSync(brokerSocketPath)).toBe(true);
|
||||
|
||||
const artifactPathFor = (revision: { revisionId: string; artifactLocator: string }) => {
|
||||
let artifactPath: string | undefined;
|
||||
withScopedMemoryDatabase("main", (database, databasePath) => {
|
||||
const row = database
|
||||
.prepare(
|
||||
`SELECT root.path_key
|
||||
FROM memory_resource_revisions AS revision
|
||||
JOIN memory_resources AS resource ON resource.resource_id = revision.resource_id
|
||||
JOIN memory_stores AS store ON store.store_id = resource.store_id
|
||||
JOIN memory_storage_roots AS root ON root.storage_root_id = store.storage_root_id
|
||||
WHERE revision.revision_id = ?`,
|
||||
)
|
||||
.get(revision.revisionId) as { path_key?: string } | undefined;
|
||||
if (row?.path_key) {
|
||||
artifactPath = resolveBuiltinScopedMemoryArtifactPath({
|
||||
databasePath,
|
||||
pathKey: row.path_key,
|
||||
artifactLocator: revision.artifactLocator,
|
||||
});
|
||||
}
|
||||
});
|
||||
if (!artifactPath) {
|
||||
throw new Error("fixture failed to locate a scoped memory artifact");
|
||||
}
|
||||
return artifactPath;
|
||||
};
|
||||
const aliceArtifactPath = artifactPathFor(aliceResource);
|
||||
const bobArtifactPath = artifactPathFor(bobResource);
|
||||
const agentDatabasePath = path.join(
|
||||
stateDir,
|
||||
"agents",
|
||||
"main",
|
||||
"agent",
|
||||
"openclaw-agent.sqlite",
|
||||
);
|
||||
expect(fs.existsSync(aliceArtifactPath)).toBe(true);
|
||||
expect(fs.existsSync(bobArtifactPath)).toBe(true);
|
||||
expect(fs.existsSync(agentDatabasePath)).toBe(true);
|
||||
|
||||
const [{ resolveSandboxContext }, { stageAuthorizedVirtualProjectionMountPlan }] =
|
||||
await Promise.all([
|
||||
import("../../../../src/agents/sandbox/context.js"),
|
||||
import("../../../../src/agents/sandbox/authorized-virtual-projection-staging.js"),
|
||||
]);
|
||||
const sandbox = await resolveSandboxContext({
|
||||
agentId: "main",
|
||||
config: constrainedSandboxConfig({
|
||||
image,
|
||||
prefix: `oc-qa-memory-core-${process.pid}-`,
|
||||
workspaceRoot: path.join(root, "sandboxes"),
|
||||
}),
|
||||
sessionKey: session.sessionKey,
|
||||
workspaceDir,
|
||||
prepareAuthorizedVirtualProjectionMountPlan: async ({ agentWorkspaceDir }) => {
|
||||
const staged = await stageAuthorizedVirtualProjectionMountPlan({
|
||||
agentWorkspaceDir,
|
||||
broker,
|
||||
});
|
||||
return staged;
|
||||
},
|
||||
});
|
||||
if (!sandbox?.backend) {
|
||||
throw new Error("fixture failed to start the constrained model-tool process");
|
||||
}
|
||||
runtimeIds.push(sandbox.runtimeId);
|
||||
if (sandbox.disposeAuthorizedVirtualProjectionMountPlan) {
|
||||
disposeProjections.push(sandbox.disposeAuthorizedVirtualProjectionMountPlan);
|
||||
}
|
||||
|
||||
const result = await sandbox.backend.runShellCommand({
|
||||
script: [
|
||||
'test "$(cat "$1")" = "$2"',
|
||||
'test "$(find /memory -type f -print | sort)" = "$1"',
|
||||
'test ! -e "$3"',
|
||||
'test ! -e "$4"',
|
||||
'test ! -e "$5"',
|
||||
'test ! -e "$6"',
|
||||
'test ! -e "$7"',
|
||||
'test -z "${OPENCLAW_MEMORY_BROKER_TEST_SECRET:-}"',
|
||||
'! find /proc -path "*/fd/*" -lname "$3" -print -quit 2>/dev/null | grep -q .',
|
||||
'! find /proc -path "*/fd/*" -lname "$5" -print -quit 2>/dev/null | grep -q .',
|
||||
'! printf denied > "$1"',
|
||||
].join(" && "),
|
||||
args: [
|
||||
`/memory/${virtualFile.virtualPath}`,
|
||||
"ALICE_REAL_BROKER_ARTIFACT_SENTINEL",
|
||||
aliceArtifactPath,
|
||||
bobArtifactPath,
|
||||
agentDatabasePath,
|
||||
stateDir,
|
||||
brokerSocketPath,
|
||||
],
|
||||
});
|
||||
expect(result.code, result.stderr.toString()).toBe(0);
|
||||
|
||||
await verifyNodeWorkerContainerProjectionIsolation({
|
||||
root: path.join(root, "node-worker-projection"),
|
||||
broker,
|
||||
outsideArtifactPath: aliceArtifactPath,
|
||||
outsideArtifactContents: "ALICE_REAL_BROKER_ARTIFACT_SENTINEL",
|
||||
issuedVirtualPath: virtualFile.virtualPath,
|
||||
issuedContents: "ALICE_REAL_BROKER_ARTIFACT_SENTINEL",
|
||||
forbiddenEnvironmentVariable: "OPENCLAW_MEMORY_BROKER_TEST_SECRET",
|
||||
});
|
||||
|
||||
const bobHost = createAuthorizedMemoryReadHost({
|
||||
agentId: "main",
|
||||
...bobSession,
|
||||
deliveryContext: { channel: "telegram", accountId: "default", to: "bob" },
|
||||
});
|
||||
const bobBroker = await resolveAuthorizedMemoryVirtualFileBroker(bobHost);
|
||||
const bobVirtualFile = bobBroker?.view.files[0];
|
||||
if (!bobHost || !bobBroker || !bobVirtualFile) {
|
||||
throw new Error("fixture failed to materialize Bob's selected memory-core virtual view");
|
||||
}
|
||||
await expect(
|
||||
bobHost.search({ query: "REAL_BROKER_ARTIFACT_SENTINEL", limit: 10 }),
|
||||
).resolves.toMatchObject({
|
||||
results: [{ path: "memory/MEMORY.md", snippet: "BOB_REAL_BROKER_ARTIFACT_SENTINEL" }],
|
||||
});
|
||||
await expect(bobBroker.readFile(bobVirtualFile.virtualPath)).resolves.toBe(
|
||||
"BOB_REAL_BROKER_ARTIFACT_SENTINEL",
|
||||
);
|
||||
const bobSandbox = await resolveSandboxContext({
|
||||
agentId: "main",
|
||||
config: constrainedSandboxConfig({
|
||||
image,
|
||||
prefix: `oc-qa-memory-core-${process.pid}-`,
|
||||
workspaceRoot: path.join(root, "sandboxes"),
|
||||
}),
|
||||
sessionKey: bobSession.sessionKey,
|
||||
workspaceDir,
|
||||
prepareAuthorizedVirtualProjectionMountPlan: async ({ agentWorkspaceDir }) => {
|
||||
return await stageAuthorizedVirtualProjectionMountPlan({
|
||||
agentWorkspaceDir,
|
||||
broker: bobBroker,
|
||||
});
|
||||
},
|
||||
});
|
||||
if (!bobSandbox?.backend) {
|
||||
throw new Error("fixture failed to start Bob's constrained model-tool process");
|
||||
}
|
||||
runtimeIds.push(bobSandbox.runtimeId);
|
||||
if (bobSandbox.disposeAuthorizedVirtualProjectionMountPlan) {
|
||||
disposeProjections.push(bobSandbox.disposeAuthorizedVirtualProjectionMountPlan);
|
||||
}
|
||||
expect(bobSandbox.runtimeId).not.toBe(sandbox.runtimeId);
|
||||
const bobResult = await bobSandbox.backend.runShellCommand({
|
||||
script: [
|
||||
'test "$(cat "$1")" = "$2"',
|
||||
'test "$(find /memory -type f -print | sort)" = "$1"',
|
||||
'! grep -R -F "$3" /memory',
|
||||
'! printf denied > "$1"',
|
||||
].join(" && "),
|
||||
args: [
|
||||
`/memory/${bobVirtualFile.virtualPath}`,
|
||||
"BOB_REAL_BROKER_ARTIFACT_SENTINEL",
|
||||
"ALICE_REAL_BROKER_ARTIFACT_SENTINEL",
|
||||
],
|
||||
});
|
||||
expect(bobResult.code, bobResult.stderr.toString()).toBe(0);
|
||||
} finally {
|
||||
if (runtimeIds.length > 0) {
|
||||
const [{ execDocker }, { removeSandboxContainer }] = await Promise.all([
|
||||
import("../../../../src/agents/sandbox/docker.js"),
|
||||
import("../../../../src/agents/sandbox/manage.js"),
|
||||
]);
|
||||
for (const runtimeId of runtimeIds) {
|
||||
await removeSandboxContainer(runtimeId);
|
||||
await execDocker(["rm", "-f", runtimeId], { allowFailure: true });
|
||||
}
|
||||
}
|
||||
await Promise.all(disposeProjections.map((disposeProjection) => disposeProjection()));
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
120_000,
|
||||
);
|
||||
|
||||
it("invalidates materialized virtual views after binding revocation and plan expiry", async () => {
|
||||
const session = { sessionKey: "agent:main:direct:alice", sessionId: "alice-session" };
|
||||
const alicePrincipalId = createVerifiedDirectSession({ name: "alice", ...session });
|
||||
@@ -2017,6 +2701,131 @@ describe("builtin scoped authorized runtime", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("quarantines cancelled writes before activation so recovery cannot publish them", async () => {
|
||||
const principalId = "cancelled-writer";
|
||||
const store = createBuiltinScopedMemoryStore({
|
||||
agentId: "main",
|
||||
scopeKind: "user",
|
||||
audienceKind: "user",
|
||||
audienceId: principalId,
|
||||
authorityKind: "user",
|
||||
authorityOwnerId: principalId,
|
||||
defaultCapabilities: ["append"],
|
||||
actor: { kind: "human", id: principalId },
|
||||
reason: "cancellation fixture",
|
||||
});
|
||||
const context = {
|
||||
...createContext(principalId),
|
||||
operation: "append" as const,
|
||||
} satisfies MemoryAccessContext;
|
||||
const plan = await builtinScopedMemoryAuthorizedRuntime.authorize(context);
|
||||
const preAborted = new AbortController();
|
||||
preAborted.abort();
|
||||
|
||||
await expect(
|
||||
builtinScopedMemoryAuthorizedRuntime.writeAuthorized({
|
||||
context,
|
||||
plan,
|
||||
mutation: {
|
||||
version: 1,
|
||||
kind: "remember",
|
||||
mutationId: "cancelled-before-stage",
|
||||
idempotencyKey: "cancelled-before-stage-request",
|
||||
content: "CANCELLED_BEFORE_STAGE_SENTINEL",
|
||||
contentType: "markdown",
|
||||
},
|
||||
signal: preAborted.signal,
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
|
||||
withScopedMemoryDatabase("main", (database) => {
|
||||
expect(
|
||||
database
|
||||
.prepare("SELECT count(*) AS count FROM memory_resources WHERE store_id = ?")
|
||||
.get(store.storeId),
|
||||
).toEqual({ count: 0 });
|
||||
expect(
|
||||
database
|
||||
.prepare("SELECT count(*) AS count FROM memory_write_intents WHERE store_id = ?")
|
||||
.get(store.storeId),
|
||||
).toEqual({ count: 0 });
|
||||
});
|
||||
|
||||
// This test-only signal becomes aborted at the exact synchronous fence after final rename.
|
||||
// It proves a late pre-activation disconnect leaves durable intent/artifact state quarantined.
|
||||
let abortedReads = 0;
|
||||
const abortAfterFinalRename = {
|
||||
get aborted() {
|
||||
abortedReads += 1;
|
||||
return abortedReads > 2;
|
||||
},
|
||||
throwIfAborted() {
|
||||
if (abortedReads > 2) {
|
||||
throw new Error("authorized write cancelled after final rename");
|
||||
}
|
||||
},
|
||||
} as AbortSignal;
|
||||
await expect(
|
||||
builtinScopedMemoryAuthorizedRuntime.writeAuthorized({
|
||||
context,
|
||||
plan,
|
||||
mutation: {
|
||||
version: 1,
|
||||
kind: "remember",
|
||||
mutationId: "cancelled-after-rename",
|
||||
idempotencyKey: "cancelled-after-rename-request",
|
||||
content: "CANCELLED_AFTER_RENAME_SENTINEL",
|
||||
contentType: "markdown",
|
||||
},
|
||||
signal: abortAfterFinalRename,
|
||||
}),
|
||||
).rejects.toThrow("cancelled after final rename");
|
||||
|
||||
let revisionId = "";
|
||||
let directory = "";
|
||||
withScopedMemoryDatabase("main", (database, databasePath) => {
|
||||
const intent = database
|
||||
.prepare(
|
||||
`SELECT intent.pending_revision_id, revision.lifecycle_state, intent.state, root.path_key
|
||||
FROM memory_write_intents AS intent
|
||||
JOIN memory_resource_revisions AS revision ON revision.revision_id = intent.pending_revision_id
|
||||
JOIN memory_stores AS store ON store.store_id = intent.store_id
|
||||
JOIN memory_storage_roots AS root ON root.storage_root_id = store.storage_root_id
|
||||
WHERE intent.idempotency_key = ?`,
|
||||
)
|
||||
.get("cancelled-after-rename-request") as {
|
||||
pending_revision_id: string;
|
||||
lifecycle_state: string;
|
||||
path_key: string;
|
||||
state: string;
|
||||
};
|
||||
revisionId = intent.pending_revision_id;
|
||||
directory = path.join(resolveScopedMemoryArtifactBase(databasePath), intent.path_key);
|
||||
expect(intent).toMatchObject({ lifecycle_state: "quarantined", state: "quarantined" });
|
||||
expect(
|
||||
database
|
||||
.prepare("SELECT * FROM memory_scoped_chunks WHERE revision_id = ?")
|
||||
.all(revisionId),
|
||||
).toEqual([]);
|
||||
});
|
||||
expect(fs.readdirSync(path.join(path.dirname(directory), ".quarantine"))).not.toEqual([]);
|
||||
|
||||
// A later authorization runs recovery. It must retain the tombstone rather than promote it.
|
||||
await builtinScopedMemoryAuthorizedRuntime.authorize(context);
|
||||
withScopedMemoryDatabase("main", (database) => {
|
||||
expect(
|
||||
database
|
||||
.prepare(
|
||||
`SELECT revision.lifecycle_state, intent.state
|
||||
FROM memory_resource_revisions AS revision
|
||||
JOIN memory_write_intents AS intent ON intent.pending_revision_id = revision.revision_id
|
||||
WHERE revision.revision_id = ?`,
|
||||
)
|
||||
.get(revisionId),
|
||||
).toEqual({ lifecycle_state: "quarantined", state: "quarantined" });
|
||||
});
|
||||
});
|
||||
|
||||
it("recovers each interrupted write boundary without exposing a pending revision", async () => {
|
||||
const stageOnly = createWriteRecoveryFixture({
|
||||
principalId: "recovery-stage-only",
|
||||
|
||||
@@ -466,7 +466,9 @@ function readPlan(params: {
|
||||
function materializeAuthorizedVirtualView(params: {
|
||||
context: MemoryContentAccessContext<"read">;
|
||||
plan: AuthorizedMemoryPlan & Readonly<{ operation: "read" }>;
|
||||
signal?: AbortSignal;
|
||||
}): AuthorizedMemoryVirtualView | undefined {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
pruneExpiredVirtualViews();
|
||||
const state = readPlan(params);
|
||||
if (!state || state.stores.length !== state.plan.mounts.length) {
|
||||
@@ -483,6 +485,7 @@ function materializeAuthorizedVirtualView(params: {
|
||||
);
|
||||
const revisionByVirtualPath = new Map<string, string>();
|
||||
const files = state.stores.flatMap((store, index) => {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
const root = roots[index]!;
|
||||
const rows = withScopedMemoryDatabase(
|
||||
params.context.agentId,
|
||||
@@ -504,6 +507,7 @@ function materializeAuthorizedVirtualView(params: {
|
||||
}>,
|
||||
);
|
||||
return rows.flatMap((row, ordinal) => {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
const virtualPath = `${root.virtualRoot}/${ordinal + 1}.md`;
|
||||
revisionByVirtualPath.set(virtualPath, row.revision_id);
|
||||
return [
|
||||
@@ -542,7 +546,9 @@ function readAuthorizedVirtualFile(params: {
|
||||
plan: AuthorizedMemoryPlan & Readonly<{ operation: "read" }>;
|
||||
view: AuthorizedMemoryVirtualView;
|
||||
virtualPath: string;
|
||||
signal?: AbortSignal;
|
||||
}): AuthorizedMemoryResultEnvelope<MemoryReadResult> {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
pruneExpiredVirtualViews();
|
||||
const state = readPlan(params);
|
||||
const allocation = virtualViews.get(params.view.viewId);
|
||||
@@ -571,6 +577,7 @@ function readAuthorizedVirtualFile(params: {
|
||||
if (!snapshot) {
|
||||
throw new Error("authorized memory virtual view is unavailable");
|
||||
}
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
return createEnvelope({
|
||||
state,
|
||||
context: params.context,
|
||||
@@ -779,6 +786,14 @@ function syncDirectory(directory: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Broker cancellation is an authorization fence: check it before any recovery,
|
||||
* filesystem staging, or durable write so cancelled work cannot be recovered as a new revision.
|
||||
*/
|
||||
function throwIfAuthorizedMemoryOperationAborted(signal?: AbortSignal): void {
|
||||
signal?.throwIfAborted();
|
||||
}
|
||||
|
||||
function readVerifiedFile(params: {
|
||||
pathname: string;
|
||||
contentHash: string;
|
||||
@@ -1360,6 +1375,34 @@ function quarantineWriteIntent(params: {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* A cancelled write may already have durable pending rows and staged/final bytes. Quarantine both
|
||||
* before surfacing the abort so recovery never promotes the cancelled revision on a later request.
|
||||
*/
|
||||
function throwIfPendingWriteAborted(params: {
|
||||
signal?: AbortSignal;
|
||||
database: Parameters<typeof getNodeSqliteKysely<ScopedMemoryDatabase>>[0];
|
||||
directory: string;
|
||||
intentId: string;
|
||||
revisionId: string;
|
||||
stagedPath: string;
|
||||
finalPath: string;
|
||||
}): void {
|
||||
if (!params.signal?.aborted) {
|
||||
return;
|
||||
}
|
||||
quarantineWriteIntent({
|
||||
database: params.database,
|
||||
intentId: params.intentId,
|
||||
revisionId: params.revisionId,
|
||||
nowMs: Date.now(),
|
||||
reasonCode: "authorized-write-cancelled-before-activation",
|
||||
});
|
||||
quarantineArtifact({ directory: params.directory, pathname: params.stagedPath });
|
||||
quarantineArtifact({ directory: params.directory, pathname: params.finalPath });
|
||||
params.signal.throwIfAborted();
|
||||
}
|
||||
|
||||
function indexRecoveredRevision(params: {
|
||||
database: Parameters<typeof getNodeSqliteKysely<ScopedMemoryDatabase>>[0];
|
||||
intentId: string;
|
||||
@@ -1709,16 +1752,32 @@ function recoverPendingWrites(agentId: string): void {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The selected broker calls this before accepting its first socket request. Recovery is explicit
|
||||
* at the broker lifecycle boundary so a replaced child cannot defer pending-write repair until a
|
||||
* later caller happens to authorize or mutate that agent's memory.
|
||||
*/
|
||||
export function recoverBuiltinScopedMemoryPendingWrites(agentIds: readonly string[]): void {
|
||||
for (const agentId of [...new Set(agentIds)].toSorted()) {
|
||||
recoverPendingWrites(agentId);
|
||||
}
|
||||
}
|
||||
|
||||
async function writeAuthorizedMutation(params: {
|
||||
context: MemoryAccessContext;
|
||||
plan: AuthorizedMemoryPlan;
|
||||
mutation: AuthorizedMemoryMutation;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<MemoryWriteResult> {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
assertMutationShape(params.mutation);
|
||||
if (params.context.operation !== mutationOperation(params.mutation)) {
|
||||
throw new Error("authorized memory mutation is unavailable");
|
||||
}
|
||||
// Recovery can activate a durable pending revision, so it must never run after cancellation.
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
recoverPendingWrites(params.context.agentId);
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
const state = readPlan(params);
|
||||
if (!state) {
|
||||
throw new Error("authorized memory mutation is unavailable");
|
||||
@@ -1726,6 +1785,7 @@ async function writeAuthorizedMutation(params: {
|
||||
const nowMs = Date.now();
|
||||
const agentId = params.context.agentId;
|
||||
if (params.mutation.kind === "sync") {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
drainMemoryAuditOutbox(agentId);
|
||||
return Object.freeze({
|
||||
version: 1,
|
||||
@@ -1737,6 +1797,7 @@ async function writeAuthorizedMutation(params: {
|
||||
}
|
||||
|
||||
if (params.mutation.kind === "project") {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
if (params.context.actor.kind !== "principal" || params.mutation.sourceHandles.length !== 1) {
|
||||
throw new Error("authorized memory projection is unavailable");
|
||||
}
|
||||
@@ -1772,6 +1833,8 @@ async function writeAuthorizedMutation(params: {
|
||||
expiry,
|
||||
nowMs,
|
||||
});
|
||||
// Projection creation is its activation point. Do not turn a committed projection into a
|
||||
// cancellation acknowledgement if the caller disconnects after this synchronous commit.
|
||||
const targetSnapshot = withScopedMemoryDatabase(agentId, (database) => {
|
||||
const db = getNodeSqliteKysely<ScopedMemoryDatabase>(database);
|
||||
return executeSqliteQueryTakeFirstSync(
|
||||
@@ -1801,6 +1864,7 @@ async function writeAuthorizedMutation(params: {
|
||||
}
|
||||
|
||||
const result = withScopedMemoryDatabase(agentId, (database, databasePath) => {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
const store = selectWriteStore({ database, agentId, state });
|
||||
const db = getNodeSqliteKysely<ScopedMemoryDatabase>(database);
|
||||
const derivationSources =
|
||||
@@ -1853,6 +1917,9 @@ async function writeAuthorizedMutation(params: {
|
||||
if (existingIntent.mutation_id !== params.mutation.mutationId) {
|
||||
throw new Error("authorized memory mutation idempotency conflict");
|
||||
}
|
||||
if (existingIntent.state === "quarantined") {
|
||||
throw new Error("authorized memory mutation is unavailable");
|
||||
}
|
||||
return Object.freeze({
|
||||
version: 1,
|
||||
mutationId: params.mutation.mutationId,
|
||||
@@ -1869,11 +1936,13 @@ async function writeAuthorizedMutation(params: {
|
||||
(params.mutation.kind === "delete" || params.mutation.kind === "tombstone") &&
|
||||
"target" in params.mutation
|
||||
) {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
const mutation = params.mutation;
|
||||
if (!existing) {
|
||||
throw new Error("authorized memory mutation is unavailable");
|
||||
}
|
||||
const intentId = randomUUID();
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
runSqliteImmediateTransactionSync(database, () => {
|
||||
const current = resolveWriteTarget({
|
||||
database,
|
||||
@@ -2025,6 +2094,7 @@ async function writeAuthorizedMutation(params: {
|
||||
const finalLocator = `r1_${revisionId}.md`;
|
||||
const stageLocator = `mwst1_${intentId}.tmp`;
|
||||
const directory = path.join(resolveScopedMemoryArtifactBase(databasePath), store.pathKey);
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
|
||||
const stagePath = path.join(directory, stageLocator);
|
||||
const finalPath = resolveBuiltinScopedMemoryArtifactPath({
|
||||
@@ -2043,8 +2113,13 @@ async function writeAuthorizedMutation(params: {
|
||||
fs.closeSync(descriptor);
|
||||
}
|
||||
syncDirectory(directory);
|
||||
if (params.signal?.aborted) {
|
||||
quarantineArtifact({ directory, pathname: stagePath });
|
||||
params.signal.throwIfAborted();
|
||||
}
|
||||
const resourceId = existing?.resourceId ?? randomUUID();
|
||||
try {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
runSqliteImmediateTransactionSync(database, () => {
|
||||
const currentStore = selectWriteStore({ database, agentId, state });
|
||||
if (
|
||||
@@ -2240,8 +2315,26 @@ async function writeAuthorizedMutation(params: {
|
||||
} catch {}
|
||||
throw error;
|
||||
}
|
||||
throwIfPendingWriteAborted({
|
||||
signal: params.signal,
|
||||
database,
|
||||
directory,
|
||||
intentId,
|
||||
revisionId,
|
||||
stagedPath: stagePath,
|
||||
finalPath,
|
||||
});
|
||||
fs.renameSync(stagePath, finalPath);
|
||||
syncDirectory(directory);
|
||||
throwIfPendingWriteAborted({
|
||||
signal: params.signal,
|
||||
database,
|
||||
directory,
|
||||
intentId,
|
||||
revisionId,
|
||||
stagedPath: stagePath,
|
||||
finalPath,
|
||||
});
|
||||
const verified = readVerifiedFile({
|
||||
pathname: finalPath,
|
||||
contentHash: hash,
|
||||
@@ -2250,6 +2343,17 @@ async function writeAuthorizedMutation(params: {
|
||||
if (verified === undefined) {
|
||||
throw new Error("authorized memory finalized artifact is unavailable");
|
||||
}
|
||||
// This is the write linearization point. A cancellation observed before it quarantines the
|
||||
// pending revision below; a cancellation after it is a committed write, never a fake abort.
|
||||
throwIfPendingWriteAborted({
|
||||
signal: params.signal,
|
||||
database,
|
||||
directory,
|
||||
intentId,
|
||||
revisionId,
|
||||
stagedPath: stagePath,
|
||||
finalPath,
|
||||
});
|
||||
runSqliteImmediateTransactionSync(database, () => {
|
||||
const currentStore = selectWriteStore({ database, agentId, state });
|
||||
if (
|
||||
@@ -2382,6 +2486,7 @@ async function writeAuthorizedMutation(params: {
|
||||
async function stageSealedCompaction(
|
||||
params: AuthorizedSealedCompactionStageParams,
|
||||
): Promise<AuthorizedSealedCompactionArtifact> {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
if (!params.content.trim()) {
|
||||
throw new Error("sealed compaction content is unavailable");
|
||||
}
|
||||
@@ -2391,6 +2496,7 @@ async function stageSealedCompaction(
|
||||
}
|
||||
const agentId = params.context.agentId;
|
||||
return withScopedMemoryDatabase(agentId, (database, databasePath) => {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
const store = selectWriteStore({ database, agentId, state });
|
||||
const source = resolveTranscriptDerivationSource({
|
||||
database,
|
||||
@@ -2412,6 +2518,7 @@ async function stageSealedCompaction(
|
||||
const stagePath = path.join(directory, stageLocator);
|
||||
const hash = contentHash(params.content);
|
||||
const bytes = Buffer.byteLength(params.content);
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
|
||||
const descriptor = fs.openSync(stagePath, "wx", 0o600);
|
||||
try {
|
||||
@@ -2422,8 +2529,17 @@ async function stageSealedCompaction(
|
||||
fs.closeSync(descriptor);
|
||||
}
|
||||
syncDirectory(directory);
|
||||
if (params.signal?.aborted) {
|
||||
quarantineArtifact({ directory, pathname: stagePath });
|
||||
params.signal.throwIfAborted();
|
||||
}
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
fs.renameSync(stagePath, finalPath);
|
||||
syncDirectory(directory);
|
||||
if (params.signal?.aborted) {
|
||||
quarantineArtifact({ directory, pathname: finalPath });
|
||||
params.signal.throwIfAborted();
|
||||
}
|
||||
const verified = readVerifiedFile({
|
||||
pathname: finalPath,
|
||||
contentHash: hash,
|
||||
@@ -2435,6 +2551,9 @@ async function stageSealedCompaction(
|
||||
return Object.freeze({
|
||||
resourceRevisionId: revisionId,
|
||||
commitInTransaction({ database: transactionDatabase, compactionPolicyId, eventSeq }) {
|
||||
// This callback runs inside the caller's SQLite transaction. It may only read the abort
|
||||
// state; filesystem cleanup belongs to staging before a transaction can begin.
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
if (state.expiresAtMs <= Date.now()) {
|
||||
throw new Error("sealed compaction authorization is unavailable");
|
||||
}
|
||||
@@ -2644,6 +2763,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
|
||||
async searchAuthorized(
|
||||
params: AuthorizedMemorySearchParams<"read"> | AuthorizedMemorySearchParams<"derive">,
|
||||
): Promise<AuthorizedMemoryResultEnvelope<readonly AuthorizedMemorySearchResult[]>> {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
if (params.context.operation !== "read" && params.context.operation !== "derive") {
|
||||
throw new Error("authorized memory search is unavailable");
|
||||
}
|
||||
@@ -2664,9 +2784,11 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
|
||||
offset: 0,
|
||||
}),
|
||||
);
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
const results: AuthorizedMemorySearchResult[] = [];
|
||||
const sourcePolicySetIds: string[] = [];
|
||||
for (const candidate of candidates) {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
if (results.length >= limit) {
|
||||
break;
|
||||
}
|
||||
@@ -2690,6 +2812,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
|
||||
results.push(result);
|
||||
sourcePolicySetIds.push(`mps1_${snapshot.policyRevisionId}`);
|
||||
}
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
return createEnvelope({
|
||||
state,
|
||||
context: params.context,
|
||||
@@ -2703,6 +2826,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
|
||||
async readAuthorized(
|
||||
params: AuthorizedMemoryReadParams<"read"> | AuthorizedMemoryReadParams<"derive">,
|
||||
): Promise<AuthorizedMemoryResultEnvelope<MemoryReadResult>> {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
if (params.context.operation !== "read" && params.context.operation !== "derive") {
|
||||
throw new Error("authorized memory read is unavailable");
|
||||
}
|
||||
@@ -2727,6 +2851,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
|
||||
if (!snapshot || snapshot.policyRevisionId !== storedHandle.policyRevision) {
|
||||
throw new Error("authorized memory read is unavailable");
|
||||
}
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
const lines = snapshot.content.split("\n");
|
||||
const from = Math.max(1, Math.trunc(params.from ?? 1));
|
||||
const lineCount = Math.max(1, Math.min(1000, Math.trunc(params.lines ?? 200)));
|
||||
@@ -2753,6 +2878,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
|
||||
context: MemoryAccessContext;
|
||||
plan: AuthorizedMemoryPlan;
|
||||
mutation: AuthorizedMemoryMutation;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<MemoryWriteResult> {
|
||||
if (params.mutation.kind === "admin-reclassify") {
|
||||
throw new Error("authorized memory reclassification is unavailable");
|
||||
@@ -2768,6 +2894,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
|
||||
context: MemoryAccessContext;
|
||||
plan: AuthorizedMemoryPlan;
|
||||
mutation: Extract<AuthorizedMemoryMutation, { kind: "import" }>;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<MemoryWriteResult> {
|
||||
return await writeAuthorizedMutation(params);
|
||||
},
|
||||
@@ -2775,7 +2902,9 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
|
||||
async syncAuthorized(params: {
|
||||
context: MemoryAccessContext;
|
||||
plan: AuthorizedMemoryPlan;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<AuthorizedMemoryResultEnvelope<MemorySyncResult>> {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
if (params.context.operation !== "sync") {
|
||||
throw new Error("authorized memory sync is unavailable");
|
||||
}
|
||||
@@ -2783,6 +2912,7 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
|
||||
if (!state) {
|
||||
throw new Error("authorized memory sync is unavailable");
|
||||
}
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
drainMemoryAuditOutbox(params.context.agentId);
|
||||
return createEnvelope({
|
||||
state,
|
||||
@@ -2797,7 +2927,9 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
|
||||
context: MemoryAccessContext;
|
||||
plan: AuthorizedMemoryPlan;
|
||||
handles: readonly AuthorizedResourceHandle[];
|
||||
signal?: AbortSignal;
|
||||
}): Promise<AuthorizedMemoryResultEnvelope<MemoryExportResult>> {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
if (params.context.operation !== "export") {
|
||||
throw new Error("authorized memory export is unavailable");
|
||||
}
|
||||
@@ -2826,7 +2958,9 @@ export function createBuiltinScopedMemoryAuthorizedRuntime(
|
||||
async statusAuthorized(params: {
|
||||
context: MemoryAccessContext;
|
||||
plan: AuthorizedMemoryPlan;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<AuthorizedMemoryResultEnvelope<AuthorizedMemoryStatus>> {
|
||||
throwIfAuthorizedMemoryOperationAborted(params.signal);
|
||||
if (params.context.operation !== "status") {
|
||||
throw new Error("authorized memory status is unavailable");
|
||||
}
|
||||
@@ -2852,6 +2986,7 @@ export const builtinScopedMemoryVirtualView = Object.freeze({
|
||||
async materializeAuthorizedVirtualView(params: {
|
||||
context: MemoryContentAccessContext<"read">;
|
||||
plan: AuthorizedMemoryPlan & Readonly<{ operation: "read" }>;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<AuthorizedMemoryVirtualView | undefined> {
|
||||
return materializeAuthorizedVirtualView(params);
|
||||
},
|
||||
@@ -2860,6 +2995,7 @@ export const builtinScopedMemoryVirtualView = Object.freeze({
|
||||
plan: AuthorizedMemoryPlan & Readonly<{ operation: "read" }>;
|
||||
view: AuthorizedMemoryVirtualView;
|
||||
virtualPath: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<AuthorizedMemoryResultEnvelope<MemoryReadResult>> {
|
||||
return readAuthorizedVirtualFile(params);
|
||||
},
|
||||
|
||||
@@ -775,6 +775,30 @@ describe("enforced memory tools", () => {
|
||||
expect(host.read).toHaveBeenCalledWith({ handleId: "mhandle1_allowed" });
|
||||
});
|
||||
|
||||
it("forwards caller cancellation into enforced memory_get", async () => {
|
||||
const host = {
|
||||
search: vi.fn(async () => ({ results: [] })),
|
||||
read: vi.fn(async () => ({ text: "allowed", path: "memory/MEMORY.md" })),
|
||||
};
|
||||
const get = createMemoryGetTool({
|
||||
config: createDefaultMemoryToolConfig(),
|
||||
memoryReadEnforced: true,
|
||||
authorizedMemoryRead: host,
|
||||
});
|
||||
if (!get) {
|
||||
throw new Error("memory_get missing");
|
||||
}
|
||||
const controller = new AbortController();
|
||||
|
||||
await expect(
|
||||
get.execute("authorized-read-with-cancellation", { handleId: "mhandle1_allowed" }, controller.signal),
|
||||
).resolves.toMatchObject({ details: { text: "allowed", path: "memory/MEMORY.md" } });
|
||||
expect(host.read).toHaveBeenCalledWith({
|
||||
handleId: "mhandle1_allowed",
|
||||
signal: controller.signal,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not fall back to legacy memory when its host is unavailable", async () => {
|
||||
const search = createMemorySearchToolOrThrow({ memoryReadEnforced: true });
|
||||
const result = await search.execute("missing-authorized-host", { query: "private" });
|
||||
|
||||
@@ -776,7 +776,8 @@ export function createMemoryGetTool(options: {
|
||||
parameters: MemoryGetSchema,
|
||||
execute:
|
||||
({ cfg, agentId }) =>
|
||||
async (_toolCallId, params) => {
|
||||
async (_toolCallId, params, callerSignal) => {
|
||||
callerSignal?.throwIfAborted();
|
||||
const rawParams = asToolParamsRecord(params);
|
||||
const from = readPositiveIntegerParam(rawParams, "from");
|
||||
const lines = readPositiveIntegerParam(rawParams, "lines");
|
||||
@@ -790,7 +791,9 @@ export function createMemoryGetTool(options: {
|
||||
handleId,
|
||||
...(from !== undefined ? { from } : {}),
|
||||
...(lines !== undefined ? { lines } : {}),
|
||||
...(callerSignal ? { signal: callerSignal } : {}),
|
||||
});
|
||||
callerSignal?.throwIfAborted();
|
||||
return isAuthorizedMemoryUnavailable(result)
|
||||
? authorizedMemoryUnavailableResult()
|
||||
: jsonResult(result);
|
||||
|
||||
@@ -1827,6 +1827,7 @@
|
||||
"test:docker:doctor-switch": "bash scripts/e2e/doctor-install-switch-docker.sh",
|
||||
"test:docker:compose-setup": "bash scripts/e2e/compose-setup.sh",
|
||||
"test:docker:e2e-build": "bash scripts/e2e/build-image.sh",
|
||||
"test:docker:fleet-separate-cell": "bash scripts/e2e/fleet-separate-cell-docker.sh",
|
||||
"test:docker:gateway-network": "bash scripts/e2e/gateway-network-docker.sh",
|
||||
"test:docker:kitchen-sink-plugin": "bash scripts/e2e/kitchen-sink-plugin-docker.sh",
|
||||
"test:docker:kitchen-sink-rpc": "bash scripts/e2e/kitchen-sink-rpc-docker.sh",
|
||||
|
||||
@@ -397,12 +397,14 @@ export type AuthorizedMemoryReadParams<Operation extends MemoryContentAccessOper
|
||||
handle: AuthorizedResourceHandle;
|
||||
from?: number;
|
||||
lines?: number;
|
||||
signal?: AbortSignal;
|
||||
}>;
|
||||
|
||||
/** Every authorized method is bound to the operation that produced its plan. */
|
||||
type AuthorizedMemoryOperationParams<Operation extends MemoryOperation> = Readonly<{
|
||||
context: MemoryAccessContext & Readonly<{ operation: Operation }>;
|
||||
plan: AuthorizedMemoryPlan & Readonly<{ operation: Operation }>;
|
||||
signal?: AbortSignal;
|
||||
}>;
|
||||
|
||||
type AuthorizedMemoryContentMutation = Readonly<{
|
||||
@@ -450,6 +452,7 @@ export type AuthorizedSealedCompactionStageParams = Readonly<{
|
||||
plan: AuthorizedMemoryPlan & Readonly<{ operation: "derive" }>;
|
||||
content: string;
|
||||
transcriptSource: AuthorizedTranscriptDerivationSource;
|
||||
signal?: AbortSignal;
|
||||
}>;
|
||||
|
||||
export type AuthorizedMemoryMutation =
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
source "$ROOT_DIR/scripts/lib/docker-e2e-image.sh"
|
||||
|
||||
IMAGE_NAME="$(docker_e2e_resolve_image "openclaw-fleet-separate-cell-e2e" OPENCLAW_FLEET_E2E_IMAGE)"
|
||||
SKIP_BUILD="${OPENCLAW_FLEET_E2E_SKIP_BUILD:-0}"
|
||||
|
||||
# The scheduler supplies the functional package image. A direct invocation can still build the
|
||||
# same package-installed image, but this lane never creates a second Fleet-specific image.
|
||||
docker_e2e_build_or_reuse "$IMAGE_NAME" fleet-separate-cell "$ROOT_DIR/scripts/e2e/Dockerfile" "$ROOT_DIR" "" "$SKIP_BUILD"
|
||||
|
||||
cd "$ROOT_DIR"
|
||||
OPENCLAW_PROCESS_ISOLATION_E2E=1 \
|
||||
OPENCLAW_FLEET_E2E_IMAGE="$IMAGE_NAME" \
|
||||
node scripts/run-vitest.mjs src/fleet/service.container.e2e.test.ts
|
||||
@@ -530,6 +530,11 @@ export const mainLanes: DockerE2eLane[] = [
|
||||
},
|
||||
),
|
||||
serviceLane("gateway-network", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:gateway-network"),
|
||||
serviceLane(
|
||||
"fleet-separate-cell",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:fleet-separate-cell",
|
||||
{ stateScenario: "empty", timeoutMs: 15 * 60 * 1000, weight: 4 },
|
||||
),
|
||||
serviceLane("browser-cdp-snapshot", "pnpm test:docker:browser-cdp-snapshot", {
|
||||
stateScenario: "empty",
|
||||
timeoutMs: 20 * 60 * 1000,
|
||||
|
||||
@@ -45,7 +45,7 @@ const authorizedMemoryVirtualBroker: unique symbol = Symbol(
|
||||
/** Core-private bridge for generic filesystem tools; it is absent from plugin contexts. */
|
||||
export type AuthorizedMemoryVirtualFileBroker = Readonly<{
|
||||
view: AuthorizedMemoryVirtualView;
|
||||
readFile: (virtualPath: string) => Promise<string | undefined>;
|
||||
readFile: (virtualPath: string, signal?: AbortSignal) => Promise<string | undefined>;
|
||||
}>;
|
||||
|
||||
/** Core-private sealed compaction capability; plugins never receive this host. */
|
||||
@@ -58,16 +58,19 @@ export type AuthorizedSealedCompactionHost = Readonly<{
|
||||
|
||||
type AuthorizedMemoryReadHostWithVirtualBroker = AuthorizedMemoryReadHost &
|
||||
Readonly<{
|
||||
[authorizedMemoryVirtualBroker]: () => Promise<AuthorizedMemoryVirtualFileBroker | undefined>;
|
||||
[authorizedMemoryVirtualBroker]: (
|
||||
signal?: AbortSignal,
|
||||
) => Promise<AuthorizedMemoryVirtualFileBroker | undefined>;
|
||||
}>;
|
||||
|
||||
export async function resolveAuthorizedMemoryVirtualFileBroker(
|
||||
host: AuthorizedMemoryReadHost | undefined,
|
||||
signal?: AbortSignal,
|
||||
): Promise<AuthorizedMemoryVirtualFileBroker | undefined> {
|
||||
if (!host || !(authorizedMemoryVirtualBroker in host)) {
|
||||
return undefined;
|
||||
}
|
||||
return (host as AuthorizedMemoryReadHostWithVirtualBroker)[authorizedMemoryVirtualBroker]();
|
||||
return (host as AuthorizedMemoryReadHostWithVirtualBroker)[authorizedMemoryVirtualBroker](signal);
|
||||
}
|
||||
|
||||
function hash(value: unknown): string {
|
||||
@@ -318,28 +321,34 @@ function createAuthorizedMemoryContentHost(
|
||||
? createAuthorizedMemoryDeriveInvocation({ context: trusted })
|
||||
: createAuthorizedMemoryReadInvocation({ context: trusted }));
|
||||
let virtualBroker: Promise<AuthorizedMemoryVirtualFileBroker | undefined> | undefined;
|
||||
const getVirtualBroker = () =>
|
||||
(virtualBroker ??= (async () => {
|
||||
const createVirtualBroker = async (signal?: AbortSignal) => {
|
||||
signal?.throwIfAborted();
|
||||
const active = await getInvocation();
|
||||
if ("unavailable" in active) {
|
||||
return undefined;
|
||||
}
|
||||
const view = await materializeAuthorizedMemoryVirtualView({ invocation: active });
|
||||
const view = await materializeAuthorizedMemoryVirtualView({
|
||||
invocation: active,
|
||||
...(signal ? { signal } : {}),
|
||||
});
|
||||
if ("unavailable" in view) {
|
||||
return undefined;
|
||||
}
|
||||
return Object.freeze({
|
||||
view,
|
||||
async readFile(virtualPath) {
|
||||
async readFile(virtualPath, signal) {
|
||||
const result = await readAuthorizedMemoryVirtualFile({
|
||||
invocation: active,
|
||||
view,
|
||||
virtualPath,
|
||||
...(signal ? { signal } : {}),
|
||||
});
|
||||
return "unavailable" in result ? undefined : result.text;
|
||||
},
|
||||
});
|
||||
})());
|
||||
};
|
||||
const getVirtualBroker = (signal?: AbortSignal) =>
|
||||
signal ? createVirtualBroker(signal) : (virtualBroker ??= createVirtualBroker());
|
||||
return Object.freeze({
|
||||
async search(search) {
|
||||
const active = await getInvocation();
|
||||
|
||||
@@ -337,7 +337,6 @@ export async function dispatchInboundMessage(params: {
|
||||
}
|
||||
let settledReceipt: DispatchFromConfigResult["settledReceipt"];
|
||||
installMemoryEgressAdmission(params.dispatcher, finalized, replyPayloadRunState);
|
||||
let settledReceipt: DispatchFromConfigResult["settledReceipt"];
|
||||
const result = await withReplyDispatcher({
|
||||
dispatcher: params.dispatcher,
|
||||
onSettled: params.onSettled,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import { requireNodeSqlite } from "../infra/node-sqlite.js";
|
||||
import type { RuntimeEnv } from "../runtime.js";
|
||||
@@ -11,6 +11,15 @@ import { OPENCLAW_AGENT_SCHEMA_SQL } from "../state/openclaw-agent-schema.js";
|
||||
import { OPENCLAW_STATE_SCHEMA_VERSION } from "../state/openclaw-state-db.js";
|
||||
import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js";
|
||||
import { OPENCLAW_STATE_SCHEMA_SQL } from "../state/openclaw-state-schema.js";
|
||||
|
||||
const withDoctorSqliteMaintenanceLock = vi.hoisted(() =>
|
||||
vi.fn(async (params: { run: () => Promise<unknown> }) => await params.run()),
|
||||
);
|
||||
|
||||
vi.mock("./doctor-sqlite-maintenance-lock.js", () => ({
|
||||
withDoctorSqliteMaintenanceLock,
|
||||
}));
|
||||
|
||||
import {
|
||||
backupSqliteCreateCommand,
|
||||
backupSqliteListCommand,
|
||||
@@ -118,6 +127,30 @@ function createAgentDatabase(databasePath: string, agentId: string): void {
|
||||
}
|
||||
|
||||
describe("SQLite backup commands", () => {
|
||||
it("uses the cross-process SQLite ownership lock for an offline snapshot", async () => {
|
||||
const tempDir = tempDirs.make("openclaw-backup-sqlite-lock-");
|
||||
const stateDir = path.join(tempDir, "state");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
process.env.OPENCLAW_STATE_DIR = stateDir;
|
||||
const databasePath = resolveOpenClawStateSqlitePath();
|
||||
await fs.mkdir(path.dirname(databasePath), { recursive: true });
|
||||
createGlobalDatabase(databasePath);
|
||||
withDoctorSqliteMaintenanceLock.mockClear();
|
||||
|
||||
await backupSqliteCreateCommand(createRuntimeCapture(), {
|
||||
global: true,
|
||||
repository: repositoryPath,
|
||||
});
|
||||
|
||||
expect(withDoctorSqliteMaintenanceLock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
operation: "SQLite snapshot",
|
||||
protectedPaths: [databasePath],
|
||||
run: expect.any(Function),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("creates, lists, verifies, and fresh-restores the global database", async () => {
|
||||
const tempDir = tempDirs.make("openclaw-backup-sqlite-");
|
||||
const stateDir = path.join(tempDir, "state");
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { withDoctorSqliteMaintenanceLock } from "./doctor-sqlite-maintenance-lock.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import { normalizeAgentId } from "../routing/session-key.js";
|
||||
import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js";
|
||||
@@ -77,12 +78,13 @@ export async function backupSqliteCreateCommand(
|
||||
const repositoryPath = resolveRequiredPath(options.repository, "--repository");
|
||||
try {
|
||||
const database = await resolveSnapshotDatabase(options);
|
||||
// A selected broker owns memory writes. Ask it to drain before the trusted Gateway snapshots
|
||||
// an agent database, then always reopen admission; workers never receive a DB path or broker IPC.
|
||||
const { withBrokeredMemoryMaintenance } = await import("../plugins/memory-broker-runtime.js");
|
||||
const result = await withBrokeredMemoryMaintenance(
|
||||
async () => await createLocalSqliteSnapshotProvider({ repositoryPath }).create(database),
|
||||
);
|
||||
// A CLI snapshot runs outside the Gateway, so its local broker map cannot prove exclusion
|
||||
// against the live owner. The state lock is the cross-process ownership boundary.
|
||||
const result = await withDoctorSqliteMaintenanceLock({
|
||||
operation: "SQLite snapshot",
|
||||
protectedPaths: [database.path],
|
||||
run: async () => await createLocalSqliteSnapshotProvider({ repositoryPath }).create(database),
|
||||
});
|
||||
const report: BackupSqliteCreateResult = {
|
||||
ok: true,
|
||||
snapshotPath: result.ref.path,
|
||||
|
||||
@@ -8,6 +8,9 @@ const backupVerifyCommandMock = vi.hoisted(() => vi.fn());
|
||||
const writeRuntimeJsonMock = vi.hoisted(() => vi.fn());
|
||||
const formatBackupCreateSummaryMock = vi.hoisted(() => vi.fn(() => ["backup ok"]));
|
||||
const recordBackupRunOutcomeMock = vi.hoisted(() => vi.fn());
|
||||
const withDoctorSqliteMaintenanceLockMock = vi.hoisted(() =>
|
||||
vi.fn(async (params: { run: () => Promise<unknown> }) => await params.run()),
|
||||
);
|
||||
|
||||
vi.mock("../infra/backup-create.js", () => ({
|
||||
createBackupArchive: createBackupArchiveMock,
|
||||
@@ -30,6 +33,10 @@ vi.mock("../state/backup-run-records.js", () => ({
|
||||
recordBackupRunOutcome: recordBackupRunOutcomeMock,
|
||||
}));
|
||||
|
||||
vi.mock("./doctor-sqlite-maintenance-lock.js", () => ({
|
||||
withDoctorSqliteMaintenanceLock: withDoctorSqliteMaintenanceLockMock,
|
||||
}));
|
||||
|
||||
function createRuntime(): RuntimeEnv {
|
||||
return {
|
||||
log: vi.fn(),
|
||||
@@ -54,6 +61,10 @@ describe("backupCreateCommand verify wrapper", () => {
|
||||
formatBackupCreateSummaryMock.mockReset();
|
||||
formatBackupCreateSummaryMock.mockReturnValue(["backup ok"]);
|
||||
recordBackupRunOutcomeMock.mockReset();
|
||||
withDoctorSqliteMaintenanceLockMock.mockReset();
|
||||
withDoctorSqliteMaintenanceLockMock.mockImplementation(
|
||||
async (params: { run: () => Promise<unknown> }) => await params.run(),
|
||||
);
|
||||
});
|
||||
|
||||
it("optionally verifies the archive after writing it", async () => {
|
||||
@@ -93,6 +104,31 @@ describe("backupCreateCommand verify wrapper", () => {
|
||||
});
|
||||
expect(verifyLog).not.toBe(runtime.log);
|
||||
expect(typeof verifyLog).toBe("function");
|
||||
expect(withDoctorSqliteMaintenanceLockMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
operation: "backup archive creation",
|
||||
protectedPaths: [expect.any(String)],
|
||||
run: expect.any(Function),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not take offline state ownership for dry runs or config-only archives", async () => {
|
||||
createBackupArchiveMock.mockResolvedValue({
|
||||
archivePath: "/tmp/openclaw-backup.tar.gz",
|
||||
archiveRoot: "openclaw-backup",
|
||||
createdAt: "2026-04-07T00:00:00.000Z",
|
||||
assets: [],
|
||||
verified: false,
|
||||
dryRun: true,
|
||||
includeWorkspace: false,
|
||||
onlyConfig: false,
|
||||
});
|
||||
|
||||
await backupCreateCommand(createRuntime(), { dryRun: true });
|
||||
await backupCreateCommand(createRuntime(), { onlyConfig: true });
|
||||
|
||||
expect(withDoctorSqliteMaintenanceLockMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not claim completion when both backup and outcome recording fail", async () => {
|
||||
|
||||
@@ -25,6 +25,14 @@ import {
|
||||
tarCreateMock,
|
||||
} from "./backup.test-support.js";
|
||||
|
||||
const withDoctorSqliteMaintenanceLock = vi.hoisted(() =>
|
||||
vi.fn(async (params: { run: () => Promise<unknown> }) => await params.run()),
|
||||
);
|
||||
|
||||
vi.mock("./doctor-sqlite-maintenance-lock.js", () => ({
|
||||
withDoctorSqliteMaintenanceLock,
|
||||
}));
|
||||
|
||||
const { backupCreateCommand } = await import("./backup.js");
|
||||
|
||||
type CapturedBackupManifest = {
|
||||
@@ -91,6 +99,10 @@ describe("backup commands", () => {
|
||||
assetCount: 1,
|
||||
entryCount: 2,
|
||||
});
|
||||
withDoctorSqliteMaintenanceLock.mockClear();
|
||||
withDoctorSqliteMaintenanceLock.mockImplementation(
|
||||
async (params: { run: () => Promise<unknown> }) => await params.run(),
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
@@ -633,9 +645,10 @@ describe("backup commands", () => {
|
||||
setTestEnvValue("OPENCLAW_CONFIG_PATH", configPath);
|
||||
try {
|
||||
const plan = await resolveBackupPlanFromDisk({ nowMs: 123 });
|
||||
const canonicalWorkspaceDir = await fs.realpath(workspaceDir);
|
||||
|
||||
expect(plan.included).toContainEqual(
|
||||
expect.objectContaining({ kind: "workspace", sourcePath: workspaceDir }),
|
||||
expect.objectContaining({ kind: "workspace", sourcePath: canonicalWorkspaceDir }),
|
||||
);
|
||||
expect(await fs.readFile(configPath, "utf8")).toBe(originalRaw);
|
||||
expect(plan.configPath).toBe(configPath);
|
||||
|
||||
+18
-4
@@ -5,10 +5,12 @@ import {
|
||||
type BackupCreateOptions,
|
||||
type BackupCreateResult,
|
||||
} from "../infra/backup-create.js";
|
||||
import { resolveStateDir } from "../config/paths.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js";
|
||||
import { createLazyImportLoader } from "../shared/lazy-promise.js";
|
||||
import { recordBackupRunOutcome } from "../state/backup-run-records.js";
|
||||
import { withDoctorSqliteMaintenanceLock } from "./doctor-sqlite-maintenance-lock.js";
|
||||
|
||||
type BackupVerifyRuntime = typeof import("./backup-verify.js");
|
||||
|
||||
@@ -27,10 +29,22 @@ export async function backupCreateCommand(
|
||||
): Promise<BackupCreateResult> {
|
||||
let archivePath = opts.output ?? process.cwd();
|
||||
try {
|
||||
const result = await createBackupArchive({
|
||||
...opts,
|
||||
log: opts.log ?? (opts.json ? undefined : (message: string) => runtime.log(message)),
|
||||
});
|
||||
const createArchive = async () =>
|
||||
await createBackupArchive({
|
||||
...opts,
|
||||
log: opts.log ?? (opts.json ? undefined : (message: string) => runtime.log(message)),
|
||||
});
|
||||
// Archive creation captures the selected memory state and its SQLite snapshots. A CLI cannot
|
||||
// quiesce a live Gateway-owned broker, so require offline state ownership instead of taking a
|
||||
// potentially inconsistent archive while the broker can still activate artifacts.
|
||||
const result =
|
||||
opts.dryRun || opts.onlyConfig
|
||||
? await createArchive()
|
||||
: await withDoctorSqliteMaintenanceLock({
|
||||
operation: "backup archive creation",
|
||||
protectedPaths: [resolveStateDir(process.env)],
|
||||
run: createArchive,
|
||||
});
|
||||
archivePath = result.archivePath;
|
||||
if (opts.verify && !opts.dryRun) {
|
||||
const { backupVerifyCommand } = await loadBackupVerifyRuntime();
|
||||
|
||||
@@ -19,6 +19,9 @@ describe("runDoctorMemoryIsolation", () => {
|
||||
beforeEach(() => {
|
||||
stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-memory-isolation-"));
|
||||
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
|
||||
const configPath = path.join(stateDir, "openclaw.json");
|
||||
fs.writeFileSync(configPath, "{}\n", "utf8");
|
||||
vi.stubEnv("OPENCLAW_CONFIG_PATH", configPath);
|
||||
const database = openOpenClawAgentDatabase({ agentId: "main" });
|
||||
database.db
|
||||
.prepare(
|
||||
@@ -36,27 +39,29 @@ describe("runDoctorMemoryIsolation", () => {
|
||||
fs.rmSync(stateDir, { force: true, recursive: true });
|
||||
});
|
||||
|
||||
it("owns the reversible configured-agent shadow lifecycle", () => {
|
||||
expect(runDoctorMemoryIsolation({ action: "status", cfg })).toEqual({
|
||||
it("owns the reversible configured-agent shadow lifecycle", async () => {
|
||||
await expect(runDoctorMemoryIsolation({ action: "status", cfg })).resolves.toEqual({
|
||||
agentId: "main",
|
||||
mode: "legacy",
|
||||
restartRequired: false,
|
||||
});
|
||||
expect(runDoctorMemoryIsolation({ action: "shadow-read-only", cfg, nowMs: 1 })).toEqual({
|
||||
await expect(
|
||||
runDoctorMemoryIsolation({ action: "shadow-read-only", cfg, nowMs: 1 }),
|
||||
).resolves.toEqual({
|
||||
agentId: "main",
|
||||
mode: "shadow-read-only",
|
||||
restartRequired: true,
|
||||
});
|
||||
expect(runDoctorMemoryIsolation({ action: "legacy", cfg })).toEqual({
|
||||
await expect(runDoctorMemoryIsolation({ action: "legacy", cfg })).resolves.toEqual({
|
||||
agentId: "main",
|
||||
mode: "legacy",
|
||||
restartRequired: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses an agent outside runtime configuration", () => {
|
||||
expect(() =>
|
||||
it("refuses an agent outside runtime configuration", async () => {
|
||||
await expect(
|
||||
runDoctorMemoryIsolation({ action: "shadow-read-only", agentId: "missing", cfg }),
|
||||
).toThrow('Unknown configured agent id "missing".');
|
||||
).rejects.toThrow('Unknown configured agent id "missing".');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -8,6 +8,8 @@ import {
|
||||
resolveMemoryIsolationMode,
|
||||
type MemoryIsolationMode,
|
||||
} from "../plugins/memory-cutover.js";
|
||||
import { resolveOpenClawAgentSqlitePath } from "../state/openclaw-agent-db.paths.js";
|
||||
import { withDoctorSqliteMaintenanceLock } from "./doctor-sqlite-maintenance-lock.js";
|
||||
|
||||
export type DoctorMemoryIsolationAction = "status" | "shadow-read-only" | "legacy";
|
||||
|
||||
@@ -32,28 +34,36 @@ function resolveDoctorMemoryIsolationAgent(params: {
|
||||
* Doctor owns the only P1C enablement path. It persists one reversible, verified posture and
|
||||
* deliberately does not create a Phase 6 cutover marker or claim two-subject confinement.
|
||||
*/
|
||||
export function runDoctorMemoryIsolation(params: {
|
||||
export async function runDoctorMemoryIsolation(params: {
|
||||
action: DoctorMemoryIsolationAction;
|
||||
agentId?: string;
|
||||
cfg?: OpenClawConfig;
|
||||
nowMs?: number;
|
||||
}): DoctorMemoryIsolationReport {
|
||||
}): Promise<DoctorMemoryIsolationReport> {
|
||||
const cfg = params.cfg ?? getRuntimeConfig();
|
||||
const agentId = resolveDoctorMemoryIsolationAgent({ agentId: params.agentId, cfg });
|
||||
switch (params.action) {
|
||||
case "status":
|
||||
return { agentId, mode: resolveMemoryIsolationMode(agentId), restartRequired: false };
|
||||
case "shadow-read-only":
|
||||
return {
|
||||
agentId,
|
||||
mode: enableMemoryShadowReadOnlyMode({ agentId, nowMs: params.nowMs }),
|
||||
restartRequired: true,
|
||||
};
|
||||
return await withDoctorSqliteMaintenanceLock({
|
||||
operation: "memory isolation shadow-read-only",
|
||||
protectedPaths: [resolveOpenClawAgentSqlitePath({ agentId })],
|
||||
run: () => ({
|
||||
agentId,
|
||||
mode: enableMemoryShadowReadOnlyMode({ agentId, nowMs: params.nowMs }),
|
||||
restartRequired: true,
|
||||
}),
|
||||
});
|
||||
case "legacy":
|
||||
return {
|
||||
agentId,
|
||||
mode: disableMemoryShadowReadOnlyMode({ agentId }),
|
||||
restartRequired: true,
|
||||
};
|
||||
return await withDoctorSqliteMaintenanceLock({
|
||||
operation: "memory isolation legacy",
|
||||
protectedPaths: [resolveOpenClawAgentSqlitePath({ agentId })],
|
||||
run: () => ({
|
||||
agentId,
|
||||
mode: disableMemoryShadowReadOnlyMode({ agentId }),
|
||||
restartRequired: true,
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,6 +29,9 @@ const repairDreamingArtifacts = vi.hoisted(() => vi.fn());
|
||||
const repairShortTermPromotionArtifacts = vi.hoisted(() => vi.fn());
|
||||
const noteWorkspaceMemoryHealth = vi.hoisted(() => vi.fn(async () => undefined));
|
||||
const maybeRepairWorkspaceMemoryHealth = vi.hoisted(() => vi.fn(async () => undefined));
|
||||
const withDoctorSqliteMaintenanceLock = vi.hoisted(() =>
|
||||
vi.fn(async (params: { run: () => Promise<unknown> }) => await params.run()),
|
||||
);
|
||||
|
||||
vi.mock("../../packages/terminal-core/src/note.js", () => ({
|
||||
note,
|
||||
@@ -99,6 +102,10 @@ vi.mock("./doctor-workspace.js", async (importOriginal) => {
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("./doctor-sqlite-maintenance-lock.js", () => ({
|
||||
withDoctorSqliteMaintenanceLock,
|
||||
}));
|
||||
|
||||
import {
|
||||
noteMemorySearchHealth,
|
||||
maybeRepairMemoryRecallHealth,
|
||||
@@ -158,6 +165,10 @@ function resetMemoryRecallMocks() {
|
||||
});
|
||||
noteWorkspaceMemoryHealth.mockClear();
|
||||
maybeRepairWorkspaceMemoryHealth.mockClear();
|
||||
withDoctorSqliteMaintenanceLock.mockClear();
|
||||
withDoctorSqliteMaintenanceLock.mockImplementation(
|
||||
async (params: { run: () => Promise<unknown> }) => await params.run(),
|
||||
);
|
||||
}
|
||||
|
||||
function firstNoteMessage(): string {
|
||||
@@ -1199,6 +1210,9 @@ describe("memory recall doctor integration", () => {
|
||||
expect(repairShortTermPromotionArtifacts).toHaveBeenCalledWith({
|
||||
workspaceDir: "/tmp/agent-default/workspace",
|
||||
});
|
||||
expect(withDoctorSqliteMaintenanceLock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ operation: "memory recall artifact repair", run: expect.any(Function) }),
|
||||
);
|
||||
expect(note).toHaveBeenCalledTimes(1);
|
||||
expectFirstNoteContains(
|
||||
"Memory recall artifacts repaired:",
|
||||
@@ -1251,6 +1265,9 @@ describe("memory recall doctor integration", () => {
|
||||
expect(repairDreamingArtifacts).toHaveBeenCalledWith({
|
||||
workspaceDir: "/tmp/agent-default/workspace",
|
||||
});
|
||||
expect(withDoctorSqliteMaintenanceLock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ operation: "dreaming artifact repair", run: expect.any(Function) }),
|
||||
);
|
||||
const message = String(note.mock.calls[note.mock.calls.length - 1]?.[0] ?? "");
|
||||
expect(message).toContain("Dreaming artifacts repaired:");
|
||||
expect(message).toContain("archived session corpus");
|
||||
|
||||
@@ -49,6 +49,7 @@ import { defaultSlotIdForKey } from "../plugins/slots.js";
|
||||
import { getProviderEnvVars } from "../secrets/provider-env-vars.js";
|
||||
import { resolveUserPath } from "../utils.js";
|
||||
import type { DoctorPrompter } from "./doctor-prompter.js";
|
||||
import { withDoctorSqliteMaintenanceLock } from "./doctor-sqlite-maintenance-lock.js";
|
||||
import { maybeRepairWorkspaceMemoryHealth, noteWorkspaceMemoryHealth } from "./doctor-workspace.js";
|
||||
import { isRecord } from "./doctor/shared/legacy-config-record-shared.js";
|
||||
|
||||
@@ -378,7 +379,10 @@ export async function maybeRepairMemoryRecallHealth(params: {
|
||||
initialValue: true,
|
||||
});
|
||||
if (approved) {
|
||||
const repair = await repairShortTermPromotionArtifacts({ workspaceDir });
|
||||
const repair = await withDoctorSqliteMaintenanceLock({
|
||||
operation: "memory recall artifact repair",
|
||||
run: async () => await repairShortTermPromotionArtifacts({ workspaceDir }),
|
||||
});
|
||||
if (repair.changed) {
|
||||
const removedOverflowEntries = repair.removedOverflowEntries ?? 0;
|
||||
const details = [
|
||||
@@ -424,7 +428,10 @@ export async function maybeRepairMemoryRecallHealth(params: {
|
||||
if (!approvedDreamingRepair) {
|
||||
continue;
|
||||
}
|
||||
const dreamingRepair = await repairDreamingArtifacts({ workspaceDir });
|
||||
const dreamingRepair = await withDoctorSqliteMaintenanceLock({
|
||||
operation: "dreaming artifact repair",
|
||||
run: async () => await repairDreamingArtifacts({ workspaceDir }),
|
||||
});
|
||||
if (!dreamingRepair.changed) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -141,6 +141,50 @@ describe("doctor SQLite maintenance lock", () => {
|
||||
await gatewayLock.release();
|
||||
});
|
||||
|
||||
it("recognizes a live backup process as an offline maintenance owner", async () => {
|
||||
const fixture = await createLockFixture();
|
||||
const lockOptions = {
|
||||
...fixture.lockOptions,
|
||||
readProcessCmdline: () => ["openclaw", "backup", "sqlite", "create"],
|
||||
};
|
||||
let allowMaintenanceToFinish: (() => void) | undefined;
|
||||
let markMaintenanceStarted: (() => void) | undefined;
|
||||
const maintenanceMayFinish = new Promise<void>((resolve) => {
|
||||
allowMaintenanceToFinish = resolve;
|
||||
});
|
||||
const maintenanceStarted = new Promise<void>((resolve) => {
|
||||
markMaintenanceStarted = resolve;
|
||||
});
|
||||
const maintenance = withDoctorSqliteMaintenanceLock(
|
||||
{
|
||||
env: fixture.env,
|
||||
operation: "SQLite snapshot",
|
||||
run: async () => {
|
||||
markMaintenanceStarted?.();
|
||||
await maintenanceMayFinish;
|
||||
},
|
||||
},
|
||||
{ lockOptions },
|
||||
);
|
||||
await maintenanceStarted;
|
||||
|
||||
try {
|
||||
await expect(
|
||||
withDoctorSqliteMaintenanceLock(
|
||||
{
|
||||
env: fixture.env,
|
||||
operation: "state SQLite compaction",
|
||||
run: vi.fn(),
|
||||
},
|
||||
{ lockOptions },
|
||||
),
|
||||
).rejects.toBeInstanceOf(DoctorSqliteMaintenanceLockUnavailableError);
|
||||
} finally {
|
||||
allowMaintenanceToFinish?.();
|
||||
await maintenance;
|
||||
}
|
||||
});
|
||||
|
||||
it("releases ownership after maintenance fails", async () => {
|
||||
const fixture = await createLockFixture();
|
||||
|
||||
|
||||
@@ -161,7 +161,9 @@ export async function withDoctorSqliteMaintenanceLock<T>(
|
||||
allowInTests: true,
|
||||
env,
|
||||
pollIntervalMs: lockOptions?.pollIntervalMs ?? MAINTENANCE_LOCK_POLL_INTERVAL_MS,
|
||||
role: "sqlite-maintenance",
|
||||
// Doctor and `backup sqlite create` both mutate/read the same state files while offline.
|
||||
// Their lock must survive stale-owner checks for either CLI command.
|
||||
role: "offline-maintenance",
|
||||
timeoutMs: lockOptions?.timeoutMs ?? MAINTENANCE_LOCK_TIMEOUT_MS,
|
||||
});
|
||||
} catch (error) {
|
||||
|
||||
@@ -7,11 +7,18 @@ import type { OpenClawConfig } from "../config/config.js";
|
||||
import type { DoctorPrompter } from "./doctor-prompter.js";
|
||||
|
||||
const note = vi.hoisted(() => vi.fn());
|
||||
const withDoctorSqliteMaintenanceLock = vi.hoisted(() =>
|
||||
vi.fn(async (params: { run: () => Promise<unknown> }) => await params.run()),
|
||||
);
|
||||
|
||||
vi.mock("../../packages/terminal-core/src/note.js", () => ({
|
||||
note,
|
||||
}));
|
||||
|
||||
vi.mock("./doctor-sqlite-maintenance-lock.js", () => ({
|
||||
withDoctorSqliteMaintenanceLock,
|
||||
}));
|
||||
|
||||
import {
|
||||
detectRootMemoryFiles,
|
||||
formatRootMemoryFilesWarning,
|
||||
@@ -46,6 +53,10 @@ describe("root memory repair", () => {
|
||||
beforeEach(async () => {
|
||||
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-root-memory-"));
|
||||
note.mockClear();
|
||||
withDoctorSqliteMaintenanceLock.mockClear();
|
||||
withDoctorSqliteMaintenanceLock.mockImplementation(
|
||||
async (params: { run: () => Promise<unknown> }) => await params.run(),
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
@@ -202,6 +213,9 @@ describe("root memory repair", () => {
|
||||
message: "Merge legacy root memory.md into canonical MEMORY.md and remove the shadowed file?",
|
||||
initialValue: true,
|
||||
});
|
||||
expect(withDoctorSqliteMaintenanceLock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ operation: "workspace root memory repair", run: expect.any(Function) }),
|
||||
);
|
||||
const canonical = await fs.readFile(path.join(tmpDir, "MEMORY.md"), "utf8");
|
||||
expect(canonical).toContain("# Legacy");
|
||||
await expectPathMissing(path.join(tmpDir, "memory.md"));
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
} from "../memory/root-memory-files.js";
|
||||
import { shortenHomePath } from "../utils.js";
|
||||
import type { DoctorPrompter } from "./doctor-prompter.js";
|
||||
import { withDoctorSqliteMaintenanceLock } from "./doctor-sqlite-maintenance-lock.js";
|
||||
|
||||
// AGENTS.md is only scanned for a memory-system reference; a small cap prevents
|
||||
// a huge file from being buffered just for one regex check.
|
||||
@@ -374,7 +375,12 @@ export async function maybeRepairWorkspaceMemoryHealth(params: {
|
||||
if (!approvedLegacyMigration) {
|
||||
return;
|
||||
}
|
||||
const migration = await migrateLegacyRootMemoryFile(configuredWorkspaceDir);
|
||||
// Prompting remains outside the lease. Once approved, stop the live Gateway from changing
|
||||
// memory-derived workspace state while this doctor command archives and merges root memory.
|
||||
const migration = await withDoctorSqliteMaintenanceLock({
|
||||
operation: "workspace root memory repair",
|
||||
run: async () => await migrateLegacyRootMemoryFile(configuredWorkspaceDir),
|
||||
});
|
||||
if (migration.readLimitExceeded) {
|
||||
note(
|
||||
[
|
||||
|
||||
@@ -48,7 +48,7 @@ export async function doctorCommand(runtime?: RuntimeEnv, options?: DoctorOption
|
||||
if (options?.memoryIsolation) {
|
||||
const outputRuntime = runtime ?? defaultRuntime;
|
||||
const { runDoctorMemoryIsolation } = await import("./doctor-memory-isolation.js");
|
||||
const report = runDoctorMemoryIsolation({
|
||||
const report = await runDoctorMemoryIsolation({
|
||||
action: options.memoryIsolation,
|
||||
...(options.memoryIsolationAgent ? { agentId: options.memoryIsolationAgent } : {}),
|
||||
});
|
||||
|
||||
@@ -626,14 +626,23 @@ describe("transcript memory policy companions", () => {
|
||||
},
|
||||
);
|
||||
|
||||
vi.stubEnv("OPENCLAW_STATE_DIR", env.OPENCLAW_STATE_DIR ?? "");
|
||||
expect(
|
||||
const stateDir = env.OPENCLAW_STATE_DIR ?? "";
|
||||
const configPath = path.join(stateDir, "openclaw.json");
|
||||
fs.mkdirSync(stateDir, { recursive: true });
|
||||
fs.writeFileSync(configPath, "{}\n", "utf8");
|
||||
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
|
||||
vi.stubEnv("OPENCLAW_CONFIG_PATH", configPath);
|
||||
await expect(
|
||||
runDoctorMemoryIsolation({
|
||||
action: "shadow-read-only",
|
||||
cfg: { agents: { list: [{ id: AGENT_ID, default: true }] } } as OpenClawConfig,
|
||||
nowMs: 1,
|
||||
}),
|
||||
).toMatchObject({ agentId: AGENT_ID, mode: "shadow-read-only", restartRequired: true });
|
||||
).resolves.toMatchObject({
|
||||
agentId: AGENT_ID,
|
||||
mode: "shadow-read-only",
|
||||
restartRequired: true,
|
||||
});
|
||||
// Doctor writes out of process. Refresh the process-owned snapshot to model the required
|
||||
// Gateway restart before proving the protected transcript boundary.
|
||||
resetMemoryIsolationCutoverForTest();
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
import crypto from "node:crypto";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import { DOCKER_SANDBOX_ENGINE, execContainer } from "../agents/sandbox/docker.js";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
|
||||
import { cellAuthSecretDir, cellContainerName, cellNetworkName } from "./cell-profile.js";
|
||||
import { createFleetContainerRuntime } from "./containers.runtime.js";
|
||||
import { createFleetService } from "./service.runtime.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
const DOCKER_COMMAND_TIMEOUT_MS = 30_000;
|
||||
|
||||
function uniqueTenant(prefix: string): string {
|
||||
return `${prefix}-${process.pid}-${crypto.randomBytes(4).toString("hex")}`;
|
||||
}
|
||||
|
||||
async function docker(args: string[], allowFailure = false) {
|
||||
return await execContainer(DOCKER_SANDBOX_ENGINE, args, {
|
||||
allowFailure,
|
||||
signal: AbortSignal.timeout(DOCKER_COMMAND_TIMEOUT_MS),
|
||||
});
|
||||
}
|
||||
|
||||
function isMissingDockerObject(stderr: string): boolean {
|
||||
return /no such (?:container|network|object)|not found/iu.test(stderr);
|
||||
}
|
||||
|
||||
async function cleanupCell(params: {
|
||||
service: ReturnType<typeof createFleetService>;
|
||||
tenant: string;
|
||||
}): Promise<void> {
|
||||
try {
|
||||
await params.service.remove({ tenant: params.tenant, force: true, purgeData: true });
|
||||
return;
|
||||
} catch (serviceError) {
|
||||
// Preserve Fleet as the normal cleanup owner. These are exact, test-generated names only,
|
||||
// and recover a cell left behind by a failed health gate before the registry can remove it.
|
||||
const failures: unknown[] = [serviceError];
|
||||
for (const args of [
|
||||
["rm", "--force", cellContainerName(params.tenant)],
|
||||
["network", "rm", cellNetworkName(params.tenant)],
|
||||
]) {
|
||||
const result = await docker(args, true);
|
||||
if (result.code !== 0 && !isMissingDockerObject(result.stderr)) {
|
||||
failures.push(new Error(result.stderr.trim() || `docker ${args.join(" ")} failed`));
|
||||
}
|
||||
}
|
||||
if (failures.length > 1) {
|
||||
throw new AggregateError(failures, `Could not clean up Fleet cell ${params.tenant}.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe.runIf(process.env.OPENCLAW_PROCESS_ISOLATION_E2E === "1")(
|
||||
"fleet separate-cell process isolation",
|
||||
() => {
|
||||
it("keeps a hostile tenant process out of another cell's data, credentials, mounts, and network", async () => {
|
||||
const image = process.env.OPENCLAW_FLEET_E2E_IMAGE;
|
||||
if (!image) {
|
||||
throw new Error(
|
||||
"Fleet process-isolation proof requires OPENCLAW_FLEET_E2E_IMAGE from the Docker E2E scheduler.",
|
||||
);
|
||||
}
|
||||
const root = tempDirs.make("openclaw-fleet-separate-cell-");
|
||||
const tenantA = uniqueTenant("fleeta");
|
||||
const tenantB = uniqueTenant("fleetb");
|
||||
const foreignSecret = crypto.randomBytes(16).toString("hex");
|
||||
const containers = createFleetContainerRuntime();
|
||||
const service = createFleetService({
|
||||
containers,
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: root },
|
||||
});
|
||||
try {
|
||||
const cellA = await service.create({
|
||||
tenant: tenantA,
|
||||
image,
|
||||
env: [
|
||||
"FLEET_CELL_E2E_A_MARKER=visible-only-to-a",
|
||||
`FLEET_CELL_E2E_A_SECRET=${crypto.randomBytes(16).toString("hex")}`,
|
||||
"OPENCLAW_SKIP_CHANNELS=1",
|
||||
"OPENCLAW_SKIP_GMAIL_WATCHER=1",
|
||||
"OPENCLAW_SKIP_CRON=1",
|
||||
"OPENCLAW_SKIP_CANVAS_HOST=1",
|
||||
],
|
||||
});
|
||||
const cellB = await service.create({
|
||||
tenant: tenantB,
|
||||
image,
|
||||
env: [
|
||||
"FLEET_CELL_E2E_B_MARKER=visible-only-to-b",
|
||||
`FLEET_CELL_E2E_B_SECRET=${foreignSecret}`,
|
||||
"OPENCLAW_SKIP_CHANNELS=1",
|
||||
"OPENCLAW_SKIP_GMAIL_WATCHER=1",
|
||||
"OPENCLAW_SKIP_CRON=1",
|
||||
"OPENCLAW_SKIP_CANVAS_HOST=1",
|
||||
],
|
||||
});
|
||||
|
||||
// Each create already waits for its own /healthz response. Query status serially so this
|
||||
// isolation proof does not add an unrelated pair of cold-Gateway health requests.
|
||||
const statusA = await service.status(tenantA);
|
||||
const statusB = await service.status(tenantB);
|
||||
expect(statusA.container).toMatchObject({
|
||||
managed: true,
|
||||
running: true,
|
||||
state: "running",
|
||||
});
|
||||
expect(statusA.health, JSON.stringify(statusA.health)).toMatchObject({
|
||||
httpStatus: 200,
|
||||
status: "ok",
|
||||
});
|
||||
expect(statusB.container).toMatchObject({
|
||||
managed: true,
|
||||
running: true,
|
||||
state: "running",
|
||||
});
|
||||
expect(statusB.health, JSON.stringify(statusB.health)).toMatchObject({
|
||||
httpStatus: 200,
|
||||
status: "ok",
|
||||
});
|
||||
|
||||
const doctor = await service.doctor();
|
||||
expect(doctor).toHaveLength(2);
|
||||
expect(
|
||||
doctor
|
||||
.flatMap((report) => report.findings)
|
||||
.filter((finding) => finding.status === "fail"),
|
||||
).toEqual([]);
|
||||
|
||||
const [pidA, pidB] = await Promise.all(
|
||||
[cellA.containerName, cellB.containerName].map(async (containerName) => {
|
||||
const result = await docker(["inspect", "--format", "{{.State.Pid}}", containerName]);
|
||||
expect(result.code, result.stderr).toBe(0);
|
||||
return Number(result.stdout.trim());
|
||||
}),
|
||||
);
|
||||
expect(pidA).toBeGreaterThan(0);
|
||||
expect(pidB).toBeGreaterThan(0);
|
||||
expect(pidA).not.toBe(pidB);
|
||||
|
||||
const stateSentinel = "issued-a-memory-view";
|
||||
const foreignSentinel = "issued-b-memory-view";
|
||||
for (const [containerName, sentinelName, sentinel] of [
|
||||
[cellA.containerName, "issued-a.txt", stateSentinel],
|
||||
[cellB.containerName, "issued-b.txt", foreignSentinel],
|
||||
] as const) {
|
||||
const result = await docker([
|
||||
"exec",
|
||||
containerName,
|
||||
"node",
|
||||
"-e",
|
||||
`require("node:fs").writeFileSync(require("node:path").join(process.env.OPENCLAW_STATE_DIR, ${JSON.stringify(sentinelName)}), ${JSON.stringify(sentinel)});`,
|
||||
]);
|
||||
expect(result.code, result.stderr).toBe(0);
|
||||
}
|
||||
|
||||
const foreignDataDir = statusB.dataDir;
|
||||
const foreignAuthDir = cellAuthSecretDir(root, tenantB);
|
||||
const hostileProbe = [
|
||||
'const fs = require("node:fs");',
|
||||
'const path = require("node:path");',
|
||||
`const foreignDataDir = ${JSON.stringify(foreignDataDir)};`,
|
||||
`const foreignAuthDir = ${JSON.stringify(foreignAuthDir)};`,
|
||||
`const foreignSecret = ${JSON.stringify(foreignSecret)};`,
|
||||
"const probe = (read) => { try { read(); return true; } catch { return false; } };",
|
||||
'const mountInfo = fs.readFileSync("/proc/self/mountinfo", "utf8");',
|
||||
"process.stdout.write(JSON.stringify({",
|
||||
' ownSentinel: fs.readFileSync(path.join(process.env.OPENCLAW_STATE_DIR, "issued-a.txt"), "utf8"),',
|
||||
" foreignDataDirectory: probe(() => fs.readdirSync(foreignDataDir)),",
|
||||
' foreignDataSentinel: probe(() => fs.readFileSync(path.join(foreignDataDir, "issued-b.txt"), "utf8")),',
|
||||
" foreignAuthDirectory: probe(() => fs.readdirSync(foreignAuthDir)),",
|
||||
' foreignEnvironmentMarker: Object.hasOwn(process.env, "FLEET_CELL_E2E_B_MARKER"),',
|
||||
' foreignEnvironmentSecret: Object.hasOwn(process.env, "FLEET_CELL_E2E_B_SECRET"),',
|
||||
" foreignEnvironmentSecretValue: Object.values(process.env).includes(foreignSecret),",
|
||||
" foreignMount: mountInfo.includes(foreignDataDir) || mountInfo.includes(foreignAuthDir),",
|
||||
"}));",
|
||||
].join("\n");
|
||||
const hostileResult = await docker([
|
||||
"exec",
|
||||
cellA.containerName,
|
||||
"node",
|
||||
"-e",
|
||||
hostileProbe,
|
||||
]);
|
||||
expect(hostileResult.code, hostileResult.stderr).toBe(0);
|
||||
expect(JSON.parse(hostileResult.stdout)).toEqual({
|
||||
ownSentinel: stateSentinel,
|
||||
foreignDataDirectory: false,
|
||||
foreignDataSentinel: false,
|
||||
foreignAuthDirectory: false,
|
||||
foreignEnvironmentMarker: false,
|
||||
foreignEnvironmentSecret: false,
|
||||
foreignEnvironmentSecretValue: false,
|
||||
foreignMount: false,
|
||||
});
|
||||
|
||||
const inspectedMounts = await docker([
|
||||
"inspect",
|
||||
"--format",
|
||||
"{{json .Mounts}}",
|
||||
cellA.containerName,
|
||||
]);
|
||||
expect(inspectedMounts.code, inspectedMounts.stderr).toBe(0);
|
||||
const mounts: unknown = JSON.parse(inspectedMounts.stdout);
|
||||
if (!Array.isArray(mounts)) {
|
||||
throw new Error("Fleet container inspection did not return mounts.");
|
||||
}
|
||||
const mountSources = mounts.flatMap((mount) => {
|
||||
if (
|
||||
!mount ||
|
||||
typeof mount !== "object" ||
|
||||
!("Source" in mount) ||
|
||||
typeof mount.Source !== "string"
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
return [mount.Source];
|
||||
});
|
||||
expect(mountSources).not.toContain(foreignDataDir);
|
||||
expect(mountSources).not.toContain(foreignAuthDir);
|
||||
|
||||
const [networkA, networkB] = await Promise.all([
|
||||
containers.inspectNetwork("docker", cellNetworkName(tenantA)),
|
||||
containers.inspectNetwork("docker", cellNetworkName(tenantB)),
|
||||
]);
|
||||
expect(networkA).toMatchObject({
|
||||
kind: "ok",
|
||||
attachedContainers: [{ name: cellA.containerName }],
|
||||
});
|
||||
expect(networkB).toMatchObject({
|
||||
kind: "ok",
|
||||
attachedContainers: [{ name: cellB.containerName }],
|
||||
});
|
||||
if (networkA.kind === "ok" && networkB.kind === "ok") {
|
||||
expect(networkA.attachedContainers).toHaveLength(1);
|
||||
expect(networkB.attachedContainers).toHaveLength(1);
|
||||
}
|
||||
} finally {
|
||||
await cleanupCell({ service, tenant: tenantB });
|
||||
await cleanupCell({ service, tenant: tenantA });
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
}
|
||||
}, 150_000);
|
||||
},
|
||||
);
|
||||
@@ -11,6 +11,7 @@ export const MCP_APP_STANDALONE_PATH = "/__openclaw__/mcp-app";
|
||||
export const MCP_APP_STANDALONE_VIEW_PATH = `${MCP_APP_STANDALONE_PATH}/view`;
|
||||
const WORKER_GATEWAY_PATH = "/__openclaw__/worker";
|
||||
const NODE_WORKER_BUNDLE_TRANSFER_NAMESPACE = "/__openclaw__/worker-bundle";
|
||||
const NODE_WORKER_PROJECTION_TRANSFER_NAMESPACE = "/__openclaw__/worker-memory-projection";
|
||||
const NODE_WORKSPACE_TRANSFER_NAMESPACE = "/__openclaw__/worker-transfer";
|
||||
|
||||
export function classifyGatewayProbePath(
|
||||
@@ -53,6 +54,13 @@ export function classifyNodeWorkerBundleTransferPath(pathname: string): "namespa
|
||||
: "outside";
|
||||
}
|
||||
|
||||
export function classifyNodeWorkerProjectionTransferPath(pathname: string): "namespace" | "outside" {
|
||||
return pathname === NODE_WORKER_PROJECTION_TRANSFER_NAMESPACE ||
|
||||
pathname.startsWith(`${NODE_WORKER_PROJECTION_TRANSFER_NAMESPACE}/`)
|
||||
? "namespace"
|
||||
: "outside";
|
||||
}
|
||||
|
||||
export function classifyNodeWorkspaceTransferPath(pathname: string): "namespace" | "outside" {
|
||||
return pathname === NODE_WORKSPACE_TRANSFER_NAMESPACE ||
|
||||
pathname.startsWith(`${NODE_WORKSPACE_TRANSFER_NAMESPACE}/`)
|
||||
|
||||
@@ -4,11 +4,18 @@ import {
|
||||
NODE_WORKER_SUPERVISOR_BINARY_CAPACITY_PROTOCOL_FEATURE,
|
||||
NODE_WORKER_SUPERVISOR_BUILD_PROTOCOL_FEATURE,
|
||||
NODE_WORKER_SUPERVISOR_EXECUTION_CONTEXT_V1_PROTOCOL_FEATURE,
|
||||
NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
|
||||
NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE,
|
||||
NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
|
||||
NODE_WORKER_SUPERVISOR_LEGACY_PROTOCOL_FEATURE,
|
||||
type NodeRunnerInventoryIssue,
|
||||
type NodeWorkerHostDeclaration,
|
||||
} from "../infra/node-runner-inventory.js";
|
||||
import {
|
||||
NODE_WORKER_EXECUTION_CONTAINER_V1,
|
||||
NODE_WORKER_EXECUTION_HOST_V1,
|
||||
type NodeWorkerExecution,
|
||||
} from "../worker/node-supervisor-protocol.js";
|
||||
|
||||
export type NodeRunnerRegistrySession = {
|
||||
nodeId: string;
|
||||
@@ -28,7 +35,10 @@ export type NodeWorkerSupervisorNodeProof = {
|
||||
pairingGeneration: string;
|
||||
clientId: typeof GATEWAY_CLIENT_IDS.NODE_HOST;
|
||||
clientMode: "node";
|
||||
protocolFeature: typeof NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE;
|
||||
protocolFeature:
|
||||
| typeof NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE
|
||||
| typeof NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE
|
||||
| typeof NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE;
|
||||
workerHost: Extract<NodeWorkerHostDeclaration, { enabled: true }>;
|
||||
commands: readonly string[];
|
||||
};
|
||||
@@ -53,7 +63,25 @@ export function sameNodeWorkerHostDeclaration(
|
||||
left.capacity.available === right.capacity.available &&
|
||||
left.bundlePrewarm === right.bundlePrewarm &&
|
||||
left.bundleRetention === right.bundleRetention &&
|
||||
left.bundleStatus === right.bundleStatus))
|
||||
left.bundleStatus === right.bundleStatus &&
|
||||
left.processIsolation?.kind === right.processIsolation?.kind &&
|
||||
left.processIsolation?.memoryProjection === right.processIsolation?.memoryProjection))
|
||||
);
|
||||
}
|
||||
|
||||
/** Returns true only for the execution boundary this exact inventory proof attests. */
|
||||
export function supportsNodeWorkerExecution(
|
||||
node: NodeWorkerSupervisorNodeProof,
|
||||
execution: NodeWorkerExecution,
|
||||
): boolean {
|
||||
if (execution.kind === NODE_WORKER_EXECUTION_HOST_V1) {
|
||||
return true;
|
||||
}
|
||||
return (
|
||||
execution.kind === NODE_WORKER_EXECUTION_CONTAINER_V1 &&
|
||||
node.protocolFeature === NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE &&
|
||||
node.workerHost.processIsolation?.kind === NODE_WORKER_EXECUTION_CONTAINER_V1 &&
|
||||
node.workerHost.processIsolation.memoryProjection === 1
|
||||
);
|
||||
}
|
||||
|
||||
@@ -72,7 +100,12 @@ export function resolveNodeWorkerSupervisorProof(
|
||||
declaration.pairingIdentity !== node.pairingIdentity ||
|
||||
declaration.clientId !== node.clientId ||
|
||||
declaration.clientMode !== node.clientMode ||
|
||||
!declaration.protocolFeatures.includes(NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE) ||
|
||||
declaration.protocolFeatures.length !== 1 ||
|
||||
(declaration.protocolFeatures[0] !== NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE &&
|
||||
declaration.protocolFeatures[0] !==
|
||||
NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE &&
|
||||
declaration.protocolFeatures[0] !==
|
||||
NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE) ||
|
||||
declaration.workerHost?.enabled !== true
|
||||
) {
|
||||
return undefined;
|
||||
@@ -84,10 +117,13 @@ export function resolveNodeWorkerSupervisorProof(
|
||||
pairingGeneration: node.pairingGeneration,
|
||||
clientId: GATEWAY_CLIENT_IDS.NODE_HOST,
|
||||
clientMode: "node",
|
||||
protocolFeature: NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
|
||||
protocolFeature: declaration.protocolFeatures[0],
|
||||
workerHost: {
|
||||
...declaration.workerHost,
|
||||
capacity: { ...declaration.workerHost.capacity },
|
||||
...(declaration.workerHost.processIsolation
|
||||
? { processIsolation: { ...declaration.workerHost.processIsolation } }
|
||||
: {}),
|
||||
},
|
||||
commands: [...node.commands],
|
||||
};
|
||||
|
||||
@@ -37,6 +37,7 @@ import {
|
||||
classifyGatewayProbePath,
|
||||
classifyMcpAppStandalonePath,
|
||||
classifyNodeWorkerBundleTransferPath,
|
||||
classifyNodeWorkerProjectionTransferPath,
|
||||
classifyNodeWorkspaceTransferPath,
|
||||
classifyWorkerGatewayPath,
|
||||
} from "./gateway-http-route-contracts.js";
|
||||
@@ -74,6 +75,10 @@ import {
|
||||
handleNodeWorkerBundleTransferHttpRequest,
|
||||
type NodeWorkerBundleTransferHttpCallback,
|
||||
} from "./worker-environments/node-worker-bundle-transfer-http.js";
|
||||
import {
|
||||
handleNodeWorkerProjectionTransferHttpRequest,
|
||||
type NodeWorkerProjectionTransferHttpCallback,
|
||||
} from "./worker-environments/node-worker-projection-transfer-http.js";
|
||||
import {
|
||||
handleNodeWorkspaceTransferHttpRequest,
|
||||
type NodeWorkspaceTransferHttpCallback,
|
||||
@@ -184,6 +189,8 @@ export function createGatewayHttpServer(opts: {
|
||||
handleNodeWorkerBundleTransferRequest?: NodeWorkerBundleTransferHttpCallback;
|
||||
/** Authenticator/dispatcher for the reserved node workspace transfer namespace. */
|
||||
handleNodeWorkspaceTransferRequest?: NodeWorkspaceTransferHttpCallback;
|
||||
/** Authenticator/dispatcher for the reserved node memory-projection transfer namespace. */
|
||||
handleNodeWorkerProjectionTransferRequest?: NodeWorkerProjectionTransferHttpCallback;
|
||||
getReadiness?: ReadinessChecker;
|
||||
getStartup?: StartupChecker;
|
||||
getRuntimeConfig?: () => OpenClawConfig;
|
||||
@@ -373,6 +380,18 @@ export function createGatewayHttpServer(opts: {
|
||||
}),
|
||||
);
|
||||
|
||||
addAdmittedStage(
|
||||
classifyNodeWorkerProjectionTransferPath(scopedRequestPath) !== "outside",
|
||||
() =>
|
||||
handleNodeWorkerProjectionTransferHttpRequest({
|
||||
req,
|
||||
res,
|
||||
clientIp: ingressAttribution.rateLimit.subject.key,
|
||||
rateLimiter: joinRateLimiter,
|
||||
callback: opts.handleNodeWorkerProjectionTransferRequest,
|
||||
}),
|
||||
);
|
||||
|
||||
addAdmittedStage(classifyNodeWorkspaceTransferPath(scopedRequestPath) !== "outside", () =>
|
||||
handleNodeWorkspaceTransferHttpRequest({
|
||||
req,
|
||||
|
||||
@@ -29,6 +29,9 @@ const removeBackfillDiaryEntries = vi.hoisted(() => vi.fn());
|
||||
const removeGroundedShortTermCandidates = vi.hoisted(() => vi.fn());
|
||||
const repairDreamingArtifacts = vi.hoisted(() => vi.fn());
|
||||
const loadShortTermPromotionDreamingStats = vi.hoisted(() => vi.fn());
|
||||
const withBrokeredMemoryMaintenance = vi.hoisted(() =>
|
||||
vi.fn(async (run: () => Promise<unknown>) => await run()),
|
||||
);
|
||||
|
||||
vi.mock("../../config/config.js", () => ({
|
||||
getRuntimeConfig,
|
||||
@@ -58,6 +61,10 @@ vi.mock("../../plugins/memory-runtime.js", () => ({
|
||||
getActiveMemorySearchManagerCore: getMemorySearchManager,
|
||||
}));
|
||||
|
||||
vi.mock("../../plugins/memory-broker-runtime.js", () => ({
|
||||
withBrokeredMemoryMaintenance,
|
||||
}));
|
||||
|
||||
import { createDoctorHandlers } from "./doctor.js";
|
||||
|
||||
const doctorHandlers = createDoctorHandlers({
|
||||
@@ -1166,6 +1173,44 @@ describe("doctor.memory.status", () => {
|
||||
});
|
||||
|
||||
describe("doctor.memory dream actions", () => {
|
||||
it("serializes every mutating Doctor path through broker maintenance", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "doctor-memory-maintenance-"));
|
||||
getRuntimeConfig.mockReset().mockReturnValue({});
|
||||
resolveDefaultAgentId.mockReset().mockReturnValue("main");
|
||||
resolveAgentWorkspaceDir.mockReset().mockReturnValue(workspaceDir);
|
||||
previewGroundedRemMarkdown.mockReset().mockResolvedValue({ scannedFiles: 0, files: [] });
|
||||
writeBackfillDiaryEntries.mockReset();
|
||||
removeBackfillDiaryEntries.mockReset().mockResolvedValue({ removed: 0 });
|
||||
removeGroundedShortTermCandidates.mockReset().mockResolvedValue({ removed: 0 });
|
||||
repairDreamingArtifacts.mockReset().mockResolvedValue({
|
||||
changed: false,
|
||||
archivedDreamsDiary: false,
|
||||
archivedSessionCorpus: false,
|
||||
archivedSessionIngestion: false,
|
||||
warnings: [],
|
||||
});
|
||||
dedupeDreamDiaryEntries.mockReset().mockResolvedValue({ removed: 0, kept: 0 });
|
||||
withBrokeredMemoryMaintenance.mockClear();
|
||||
withBrokeredMemoryMaintenance.mockImplementation(async (run: () => Promise<unknown>) =>
|
||||
await run(),
|
||||
);
|
||||
|
||||
try {
|
||||
for (const method of [
|
||||
"doctor.memory.backfillDreamDiary",
|
||||
"doctor.memory.resetDreamDiary",
|
||||
"doctor.memory.resetGroundedShortTerm",
|
||||
"doctor.memory.repairDreamingArtifacts",
|
||||
"doctor.memory.dedupeDreamDiary",
|
||||
] as const) {
|
||||
await invokeDoctorMemory(method, vi.fn());
|
||||
}
|
||||
expect(withBrokeredMemoryMaintenance).toHaveBeenCalledTimes(5);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("clears grounded-only staged short-term entries without touching the diary", async () => {
|
||||
resolveAgentWorkspaceDir.mockReturnValue("/tmp/openclaw");
|
||||
removeGroundedShortTermCandidates.mockResolvedValue({
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
resolveMemoryRemDreamingConfig,
|
||||
} from "../../memory-host-sdk/dreaming.js";
|
||||
import * as defaultMemoryCoreRuntime from "../../plugin-sdk/memory-core-bundled-runtime.js";
|
||||
import { withBrokeredMemoryMaintenance } from "../../plugins/memory-broker-runtime.js";
|
||||
import { getActiveMemorySearchManagerCore } from "../../plugins/memory-runtime.js";
|
||||
import { normalizeAgentId } from "../../routing/session-key.js";
|
||||
import { formatError } from "../server-utils.js";
|
||||
@@ -850,58 +851,58 @@ export const createDoctorHandlers = (
|
||||
return;
|
||||
}
|
||||
const { cfg, agentId, workspaceDir } = target;
|
||||
const memoryDir = path.join(workspaceDir, "memory");
|
||||
const sourceFiles = await listWorkspaceDailyFiles(memoryDir);
|
||||
if (sourceFiles.length === 0) {
|
||||
const payload = await withBrokeredMemoryMaintenance(async () => {
|
||||
const memoryDir = path.join(workspaceDir, "memory");
|
||||
const sourceFiles = await listWorkspaceDailyFiles(memoryDir);
|
||||
if (sourceFiles.length === 0) {
|
||||
const dreamDiary = await readDreamDiary(workspaceDir);
|
||||
return {
|
||||
agentId,
|
||||
path: dreamDiary.path,
|
||||
action: "backfill" as const,
|
||||
found: dreamDiary.found,
|
||||
scannedFiles: 0,
|
||||
written: 0,
|
||||
replaced: 0,
|
||||
} satisfies DoctorMemoryDreamActionPayload;
|
||||
}
|
||||
const grounded = await memoryCoreRuntime.previewGroundedRemMarkdown({
|
||||
workspaceDir,
|
||||
inputPaths: sourceFiles,
|
||||
});
|
||||
const remConfig = resolveMemoryRemDreamingConfig({
|
||||
pluginConfig: resolveMemoryDreamingPluginConfig(cfg),
|
||||
cfg,
|
||||
});
|
||||
const entries = grounded.files
|
||||
.map((file) => {
|
||||
const isoDay = extractIsoDayFromPath(file.path);
|
||||
if (!isoDay) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
isoDay,
|
||||
sourcePath: file.path,
|
||||
bodyLines: groundedMarkdownToDiaryLines(file.renderedMarkdown),
|
||||
};
|
||||
})
|
||||
.filter((entry): entry is NonNullable<typeof entry> => entry !== null);
|
||||
const written = await memoryCoreRuntime.writeBackfillDiaryEntries({
|
||||
workspaceDir,
|
||||
entries,
|
||||
timezone: remConfig.timezone,
|
||||
});
|
||||
const dreamDiary = await readDreamDiary(workspaceDir);
|
||||
const payload: DoctorMemoryDreamActionPayload = {
|
||||
return {
|
||||
agentId,
|
||||
path: dreamDiary.path,
|
||||
action: "backfill",
|
||||
found: dreamDiary.found,
|
||||
scannedFiles: 0,
|
||||
written: 0,
|
||||
replaced: 0,
|
||||
};
|
||||
respond(true, payload, undefined);
|
||||
return;
|
||||
}
|
||||
const grounded = await memoryCoreRuntime.previewGroundedRemMarkdown({
|
||||
workspaceDir,
|
||||
inputPaths: sourceFiles,
|
||||
scannedFiles: grounded.scannedFiles,
|
||||
written: written.written,
|
||||
replaced: written.replaced,
|
||||
} satisfies DoctorMemoryDreamActionPayload;
|
||||
});
|
||||
const remConfig = resolveMemoryRemDreamingConfig({
|
||||
pluginConfig: resolveMemoryDreamingPluginConfig(cfg),
|
||||
cfg,
|
||||
});
|
||||
const entries = grounded.files
|
||||
.map((file) => {
|
||||
const isoDay = extractIsoDayFromPath(file.path);
|
||||
if (!isoDay) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
isoDay,
|
||||
sourcePath: file.path,
|
||||
bodyLines: groundedMarkdownToDiaryLines(file.renderedMarkdown),
|
||||
};
|
||||
})
|
||||
.filter((entry): entry is NonNullable<typeof entry> => entry !== null);
|
||||
const written = await memoryCoreRuntime.writeBackfillDiaryEntries({
|
||||
workspaceDir,
|
||||
entries,
|
||||
timezone: remConfig.timezone,
|
||||
});
|
||||
const dreamDiary = await readDreamDiary(workspaceDir);
|
||||
const payload: DoctorMemoryDreamActionPayload = {
|
||||
agentId,
|
||||
path: dreamDiary.path,
|
||||
action: "backfill",
|
||||
found: dreamDiary.found,
|
||||
scannedFiles: grounded.scannedFiles,
|
||||
written: written.written,
|
||||
replaced: written.replaced,
|
||||
};
|
||||
respond(true, payload, undefined);
|
||||
},
|
||||
"doctor.memory.resetDreamDiary": async ({ respond, context, params }) => {
|
||||
@@ -910,15 +911,17 @@ export const createDoctorHandlers = (
|
||||
return;
|
||||
}
|
||||
const { agentId, workspaceDir } = target;
|
||||
const removed = await memoryCoreRuntime.removeBackfillDiaryEntries({ workspaceDir });
|
||||
const dreamDiary = await readDreamDiary(workspaceDir);
|
||||
const payload: DoctorMemoryDreamActionPayload = {
|
||||
agentId,
|
||||
path: dreamDiary.path,
|
||||
action: "reset",
|
||||
found: dreamDiary.found,
|
||||
removedEntries: removed.removed,
|
||||
};
|
||||
const payload = await withBrokeredMemoryMaintenance(async () => {
|
||||
const removed = await memoryCoreRuntime.removeBackfillDiaryEntries({ workspaceDir });
|
||||
const dreamDiary = await readDreamDiary(workspaceDir);
|
||||
return {
|
||||
agentId,
|
||||
path: dreamDiary.path,
|
||||
action: "reset",
|
||||
found: dreamDiary.found,
|
||||
removedEntries: removed.removed,
|
||||
} satisfies DoctorMemoryDreamActionPayload;
|
||||
});
|
||||
respond(true, payload, undefined);
|
||||
},
|
||||
"doctor.memory.resetGroundedShortTerm": async ({ respond, context, params }) => {
|
||||
@@ -927,12 +930,14 @@ export const createDoctorHandlers = (
|
||||
return;
|
||||
}
|
||||
const { agentId, workspaceDir } = target;
|
||||
const removed = await memoryCoreRuntime.removeGroundedShortTermCandidates({ workspaceDir });
|
||||
const payload: DoctorMemoryDreamActionPayload = {
|
||||
agentId,
|
||||
action: "resetGroundedShortTerm",
|
||||
removedShortTermEntries: removed.removed,
|
||||
};
|
||||
const payload = await withBrokeredMemoryMaintenance(async () => {
|
||||
const removed = await memoryCoreRuntime.removeGroundedShortTermCandidates({ workspaceDir });
|
||||
return {
|
||||
agentId,
|
||||
action: "resetGroundedShortTerm",
|
||||
removedShortTermEntries: removed.removed,
|
||||
} satisfies DoctorMemoryDreamActionPayload;
|
||||
});
|
||||
respond(true, payload, undefined);
|
||||
},
|
||||
"doctor.memory.repairDreamingArtifacts": async ({ respond, context, params }) => {
|
||||
@@ -941,17 +946,19 @@ export const createDoctorHandlers = (
|
||||
return;
|
||||
}
|
||||
const { agentId, workspaceDir } = target;
|
||||
const repair = await memoryCoreRuntime.repairDreamingArtifacts({ workspaceDir });
|
||||
const payload: DoctorMemoryDreamActionPayload = {
|
||||
agentId,
|
||||
action: "repairDreamingArtifacts",
|
||||
changed: repair.changed,
|
||||
archiveDir: repair.archiveDir,
|
||||
archivedDreamsDiary: repair.archivedDreamsDiary,
|
||||
archivedSessionCorpus: repair.archivedSessionCorpus,
|
||||
archivedSessionIngestion: repair.archivedSessionIngestion,
|
||||
warnings: repair.warnings,
|
||||
};
|
||||
const payload = await withBrokeredMemoryMaintenance(async () => {
|
||||
const repair = await memoryCoreRuntime.repairDreamingArtifacts({ workspaceDir });
|
||||
return {
|
||||
agentId,
|
||||
action: "repairDreamingArtifacts",
|
||||
changed: repair.changed,
|
||||
archiveDir: repair.archiveDir,
|
||||
archivedDreamsDiary: repair.archivedDreamsDiary,
|
||||
archivedSessionCorpus: repair.archivedSessionCorpus,
|
||||
archivedSessionIngestion: repair.archivedSessionIngestion,
|
||||
warnings: repair.warnings,
|
||||
} satisfies DoctorMemoryDreamActionPayload;
|
||||
});
|
||||
respond(true, payload, undefined);
|
||||
},
|
||||
"doctor.memory.dedupeDreamDiary": async ({ respond, context, params }) => {
|
||||
@@ -960,17 +967,19 @@ export const createDoctorHandlers = (
|
||||
return;
|
||||
}
|
||||
const { agentId, workspaceDir } = target;
|
||||
const dedupe = await memoryCoreRuntime.dedupeDreamDiaryEntries({ workspaceDir });
|
||||
const dreamDiary = await readDreamDiary(workspaceDir);
|
||||
const payload: DoctorMemoryDreamActionPayload = {
|
||||
agentId,
|
||||
action: "dedupeDreamDiary",
|
||||
path: dreamDiary.path,
|
||||
found: dreamDiary.found,
|
||||
removedEntries: dedupe.removed,
|
||||
dedupedEntries: dedupe.removed,
|
||||
keptEntries: dedupe.kept,
|
||||
};
|
||||
const payload = await withBrokeredMemoryMaintenance(async () => {
|
||||
const dedupe = await memoryCoreRuntime.dedupeDreamDiaryEntries({ workspaceDir });
|
||||
const dreamDiary = await readDreamDiary(workspaceDir);
|
||||
return {
|
||||
agentId,
|
||||
action: "dedupeDreamDiary",
|
||||
path: dreamDiary.path,
|
||||
found: dreamDiary.found,
|
||||
removedEntries: dedupe.removed,
|
||||
dedupedEntries: dedupe.removed,
|
||||
keptEntries: dedupe.kept,
|
||||
} satisfies DoctorMemoryDreamActionPayload;
|
||||
});
|
||||
respond(true, payload, undefined);
|
||||
},
|
||||
});
|
||||
|
||||
@@ -166,6 +166,7 @@ export async function prepareGatewayKernelState(params: {
|
||||
bindDeviceNodeControl,
|
||||
bindNodeWorkspaceBindingResolver,
|
||||
handleNodeWorkerBundleTransferRequest,
|
||||
handleNodeWorkerProjectionTransferRequest,
|
||||
handleNodeWorkspaceTransferRequest,
|
||||
} = workerEnvironmentRuntime;
|
||||
// Assigned once approval managers exist; placement dispatch must not run before then.
|
||||
@@ -461,6 +462,7 @@ export async function prepareGatewayKernelState(params: {
|
||||
handleWatchNodeRequest: async (req: IncomingMessage, res: ServerResponse) =>
|
||||
(await watchNodeRequestHandler.current?.(req, res)) ?? false,
|
||||
handleNodeWorkerBundleTransferRequest,
|
||||
handleNodeWorkerProjectionTransferRequest,
|
||||
handleNodeWorkspaceTransferRequest,
|
||||
workerIngressEnabled: Boolean(workerEnvironmentService),
|
||||
desktopSessionRegistry,
|
||||
|
||||
@@ -850,7 +850,9 @@ describe("startGatewayPostAttachRuntime", () => {
|
||||
},
|
||||
};
|
||||
const startGatewaySidecars = vi.fn(async () => {
|
||||
expect(hoisted.startBrokeredMemoryRuntimeSupervisor).toHaveBeenCalledWith(selectedCapability);
|
||||
expect(hoisted.startBrokeredMemoryRuntimeSupervisor).toHaveBeenCalledWith(selectedCapability, {
|
||||
agentIds: ["main"],
|
||||
});
|
||||
expect(unavailableGatewayMethods).toEqual(new Set(STARTUP_UNAVAILABLE_GATEWAY_METHODS));
|
||||
return { pluginServices: null, postReadySidecars: [] };
|
||||
});
|
||||
|
||||
@@ -1345,9 +1345,14 @@ export async function startGatewayPostAttachRuntime(
|
||||
// than becoming a first-request outage after agents have begun work.
|
||||
const memoryBrokerSupervisor = selectedMemory?.capability.broker
|
||||
? await measureStartup(params.startupTrace, "memory-broker.ready", async () => {
|
||||
const { startBrokeredMemoryRuntimeSupervisor } =
|
||||
await loadMemoryBrokerRuntimeModule();
|
||||
return await startBrokeredMemoryRuntimeSupervisor(selectedMemory.capability);
|
||||
const [{ startBrokeredMemoryRuntimeSupervisor }, { listAgentIds }] =
|
||||
await Promise.all([
|
||||
loadMemoryBrokerRuntimeModule(),
|
||||
import("../agents/agent-scope.js"),
|
||||
]);
|
||||
return await startBrokeredMemoryRuntimeSupervisor(selectedMemory.capability, {
|
||||
agentIds: listAgentIds(params.gatewayPluginConfigAtStart),
|
||||
});
|
||||
})
|
||||
: undefined;
|
||||
if (params.isClosing?.()) {
|
||||
|
||||
@@ -1,10 +1,20 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
GATEWAY_CLIENT_IDS,
|
||||
GATEWAY_CLIENT_MODES,
|
||||
} from "../../packages/gateway-protocol/src/client-info.js";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import { NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE } from "../infra/node-runner-inventory.js";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
|
||||
import { withEnvAsync } from "../test-utils/env.js";
|
||||
import { NODE_WORKER_EXECUTION_CONTAINER_V1 } from "../worker/node-supervisor-protocol.js";
|
||||
import { createDesktopSessionRegistry } from "./desktop/session-registry.js";
|
||||
import type {
|
||||
NodeWorkerSupervisorNodeProof,
|
||||
NodeWorkerSupervisorTransport,
|
||||
} from "./node-registry-private.js";
|
||||
import {
|
||||
createGatewayWorkerEnvironmentRuntime,
|
||||
loadGatewayWorkerEnvironmentStartupState,
|
||||
@@ -15,6 +25,25 @@ import {
|
||||
reconcileDeviceWorker,
|
||||
} from "./worker-environments/device-provider.js";
|
||||
|
||||
const projectionTransferOptions = vi.hoisted(() => ({
|
||||
isNodeCurrent: undefined as ((node: NodeWorkerSupervisorNodeProof) => boolean) | undefined,
|
||||
}));
|
||||
|
||||
vi.mock("./worker-environments/node-worker-projection-transfer-service.js", async (importOriginal) => {
|
||||
const actual = await importOriginal<
|
||||
typeof import("./worker-environments/node-worker-projection-transfer-service.js")
|
||||
>();
|
||||
return {
|
||||
...actual,
|
||||
createNodeWorkerProjectionTransferService: (
|
||||
options: Parameters<typeof actual.createNodeWorkerProjectionTransferService>[0],
|
||||
) => {
|
||||
projectionTransferOptions.isNodeCurrent = options.isNodeCurrent;
|
||||
return actual.createNodeWorkerProjectionTransferService(options);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const DEVICE_ID = "revoked-device";
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
@@ -23,6 +52,69 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("gateway worker environment startup", () => {
|
||||
it("keeps an issued projection bound to its capacity-one node after its slot is claimed", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-worker-projection-capacity-");
|
||||
await withEnvAsync({ OPENCLAW_STATE_DIR: stateDir }, async () => {
|
||||
const startup = await loadGatewayWorkerEnvironmentStartupState();
|
||||
const runtime = await createGatewayWorkerEnvironmentRuntime({
|
||||
getPluginRegistry: () => ({ workerProviders: new Map() }),
|
||||
desktopSessionRegistry: createDesktopSessionRegistry({ lingerMs: 1 }),
|
||||
startup,
|
||||
log: { child: () => ({ warn: () => {} }) },
|
||||
});
|
||||
const service = runtime.workerEnvironmentService;
|
||||
if (!service || !runtime.bindDeviceNodeControl || !projectionTransferOptions.isNodeCurrent) {
|
||||
throw new Error("worker projection runtime was not created");
|
||||
}
|
||||
const node: NodeWorkerSupervisorNodeProof = {
|
||||
nodeId: "node-1",
|
||||
connId: "conn-1",
|
||||
pairingIdentity: "pairing-1",
|
||||
pairingGeneration: "generation-1",
|
||||
clientId: GATEWAY_CLIENT_IDS.NODE_HOST,
|
||||
clientMode: GATEWAY_CLIENT_MODES.NODE,
|
||||
protocolFeature: NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
|
||||
workerHost: {
|
||||
enabled: true,
|
||||
capacity: { total: 1, available: 1 },
|
||||
processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, memoryProjection: 1 },
|
||||
},
|
||||
commands: [],
|
||||
};
|
||||
let available = 1;
|
||||
let connId = node.connId;
|
||||
let pairingGeneration = node.pairingGeneration;
|
||||
const transport = {
|
||||
listCurrentNodes: async () => [node],
|
||||
isCurrent: vi.fn(
|
||||
(candidate: NodeWorkerSupervisorNodeProof, requireLaunchEligibility = false) =>
|
||||
candidate.nodeId === node.nodeId &&
|
||||
candidate.connId === connId &&
|
||||
candidate.pairingGeneration === pairingGeneration &&
|
||||
(!requireLaunchEligibility || available > 0),
|
||||
),
|
||||
invoke: async () => ({ ok: false }),
|
||||
} satisfies NodeWorkerSupervisorTransport;
|
||||
runtime.bindDeviceNodeControl(transport);
|
||||
const isNodeCurrent = projectionTransferOptions.isNodeCurrent;
|
||||
|
||||
try {
|
||||
expect(isNodeCurrent(node)).toBe(true);
|
||||
available = 0;
|
||||
expect(isNodeCurrent(node)).toBe(true);
|
||||
expect(transport.isCurrent).toHaveBeenLastCalledWith(node, false);
|
||||
|
||||
connId = "conn-replaced";
|
||||
expect(isNodeCurrent(node)).toBe(false);
|
||||
connId = node.connId;
|
||||
pairingGeneration = "generation-replaced";
|
||||
expect(isNodeCurrent(node)).toBe(false);
|
||||
} finally {
|
||||
await service.stop();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("cleans transfer scratch before serving and removes it on shutdown", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-worker-transfer-startup-");
|
||||
const transferRoot = path.join(stateDir, "tmp", "node-workspace-transfer");
|
||||
|
||||
@@ -2,7 +2,7 @@ import { isRecord } from "@openclaw/normalization-core/record-coerce";
|
||||
import { uniqueStrings } from "@openclaw/normalization-core/string-normalization";
|
||||
import { getRuntimeConfig } from "../config/config.js";
|
||||
import { loadOrCreateProcessDeviceIdentity } from "../infra/device-identity.js";
|
||||
import { getPairedDevice } from "../infra/device-pairing.js";
|
||||
import { getPairedDevice, resolveNodePairingState } from "../infra/device-pairing.js";
|
||||
import type { PluginRegistry } from "../plugins/registry-types.js";
|
||||
import {
|
||||
getActiveSecretsRuntimeConfigSnapshot,
|
||||
@@ -14,6 +14,7 @@ import type { NodeWorkerSupervisorTransport } from "./node-registry-private.js";
|
||||
import type { WorkerBundleProducer, WorkerNpmArtifact } from "./worker-environments/bundle.js";
|
||||
import {
|
||||
bindDeviceWorkerAvailability,
|
||||
bindDeviceWorkerExecutionEligibility,
|
||||
bindDeviceWorkerReconciliation,
|
||||
createDeviceWorkerRuntime,
|
||||
DEVICE_WORKER_PROVIDER_ID,
|
||||
@@ -21,6 +22,7 @@ import {
|
||||
import type { WorkerLiveEventReceiver } from "./worker-environments/live-events.js";
|
||||
import { createWorkerNodeEnrollmentManager } from "./worker-environments/node-enrollment.js";
|
||||
import type { NodeWorkerBundleTransferHttpCallback } from "./worker-environments/node-worker-bundle-transfer-http.js";
|
||||
import type { NodeWorkerProjectionTransferHttpCallback } from "./worker-environments/node-worker-projection-transfer-http.js";
|
||||
import { nodeWorkerGatewayNamespace as resolveNodeWorkerGatewayNamespace } from "./worker-environments/node-worker-gateway-namespace.js";
|
||||
import type { NodeWorkerWorkspaceBindingResolver } from "./worker-environments/node-worker-tunnel.js";
|
||||
import type { NodeWorkspaceTransferHttpCallback } from "./worker-environments/node-workspace-transfer-http-contract.js";
|
||||
@@ -56,6 +58,7 @@ export type GatewayWorkerEnvironmentRuntime = {
|
||||
bindDeviceNodeControl?: (transport: NodeWorkerSupervisorTransport) => void;
|
||||
bindNodeWorkspaceBindingResolver?: (resolver: NodeWorkerWorkspaceBindingResolver) => void;
|
||||
handleNodeWorkerBundleTransferRequest?: NodeWorkerBundleTransferHttpCallback;
|
||||
handleNodeWorkerProjectionTransferRequest?: NodeWorkerProjectionTransferHttpCallback;
|
||||
handleNodeWorkspaceTransferRequest?: NodeWorkspaceTransferHttpCallback;
|
||||
};
|
||||
|
||||
@@ -119,6 +122,8 @@ export async function createGatewayWorkerEnvironmentRuntime(params: {
|
||||
{ createGatewayNodeWorkerBundleInstaller },
|
||||
{ createNodeWorkerBundleTransferService },
|
||||
{ createNodeWorkerBundleTransferHttpCallback },
|
||||
{ createNodeWorkerProjectionTransferService },
|
||||
{ createNodeWorkerProjectionTransferHttpCallback },
|
||||
{ createNodeWorkspaceTransferService },
|
||||
{ createNodeWorkspaceTransferHttpCallback },
|
||||
{ createWorkerSessionToolExecutor },
|
||||
@@ -133,6 +138,8 @@ export async function createGatewayWorkerEnvironmentRuntime(params: {
|
||||
import("./worker-environments/node-worker-bundle-installer.js"),
|
||||
import("./worker-environments/node-worker-bundle-transfer-service.js"),
|
||||
import("./worker-environments/node-worker-bundle-transfer-http.js"),
|
||||
import("./worker-environments/node-worker-projection-transfer-service.js"),
|
||||
import("./worker-environments/node-worker-projection-transfer-http.js"),
|
||||
import("./worker-environments/node-workspace-transfer-service.js"),
|
||||
import("./worker-environments/node-workspace-transfer-http.js"),
|
||||
import("./worker-environments/worker-session-tool-executor.js"),
|
||||
@@ -204,6 +211,20 @@ export async function createGatewayWorkerEnvironmentRuntime(params: {
|
||||
desktopSessionRegistry: params.desktopSessionRegistry,
|
||||
});
|
||||
const nodeWorkerBundleTransfer = createNodeWorkerBundleTransferService();
|
||||
let nodeWorkerSupervisorTransport: NodeWorkerSupervisorTransport | undefined;
|
||||
const nodeWorkerProjectionTransfer = createNodeWorkerProjectionTransferService({
|
||||
resolveNodePublicKey: async (node) => {
|
||||
const paired = await getPairedDevice(node.nodeId);
|
||||
const currentPairing = resolveNodePairingState(paired);
|
||||
return currentPairing?.identity.key === node.pairingIdentity &&
|
||||
currentPairing.generation?.key === node.pairingGeneration
|
||||
? paired?.publicKey
|
||||
: undefined;
|
||||
},
|
||||
// The node claims its launch slot before it fetches this already-issued projection.
|
||||
// Keep exact connection/pairing proof, but do not reject a capacity-one node's own fetch.
|
||||
isNodeCurrent: (node) => nodeWorkerSupervisorTransport?.isCurrent(node) === true,
|
||||
});
|
||||
const nodeWorkspaceTransfer = createNodeWorkspaceTransferService({
|
||||
getOwner: (environmentId) => params.startup.store.getTransferOwner(environmentId),
|
||||
});
|
||||
@@ -216,6 +237,7 @@ export async function createGatewayWorkerEnvironmentRuntime(params: {
|
||||
getTransport: () => deviceRuntime.getNodeTransport(),
|
||||
launchNodeWorker: async (request) => await deviceRuntime.launchNodeWorker(request),
|
||||
validateWorkerTurn: (binding) => placementGate.validateWorkerTurn(binding),
|
||||
projectionTransfer: nodeWorkerProjectionTransfer,
|
||||
workspaceTransfer: nodeWorkspaceTransfer,
|
||||
});
|
||||
const ensureNodeWorkerBundle = createGatewayNodeWorkerBundleInstaller({
|
||||
@@ -255,7 +277,10 @@ export async function createGatewayWorkerEnvironmentRuntime(params: {
|
||||
retireNodeEnrollment: nodeEnrollment.retire,
|
||||
tunnelManager: workerTunnelManager,
|
||||
nodeTunnelManager: nodeWorkerTunnelManager,
|
||||
stopNodeWorkerBundleTransfers: () => nodeWorkerBundleTransfer.closeAll(),
|
||||
stopNodeWorkerBundleTransfers: () => {
|
||||
nodeWorkerBundleTransfer.closeAll();
|
||||
nodeWorkerProjectionTransfer.closeAll();
|
||||
},
|
||||
applyTranscriptCommit: createWorkerTranscriptCommitter({
|
||||
getConfig: getRuntimeConfig,
|
||||
}).commit,
|
||||
@@ -304,6 +329,10 @@ export async function createGatewayWorkerEnvironmentRuntime(params: {
|
||||
});
|
||||
const workerEnvironmentService = workerEnvironmentServiceBase;
|
||||
bindDeviceWorkerAvailability(workerEnvironmentService, deviceRuntime.resolveAvailability);
|
||||
bindDeviceWorkerExecutionEligibility(
|
||||
workerEnvironmentService,
|
||||
deviceRuntime.assertNodeWorkerExecutionEligible,
|
||||
);
|
||||
bindDeviceWorkerReconciliation(workerEnvironmentService, async (deviceId) => {
|
||||
const environmentIds = params.startup.store
|
||||
.listForReconcile()
|
||||
@@ -344,11 +373,16 @@ export async function createGatewayWorkerEnvironmentRuntime(params: {
|
||||
bindWorkerSessionDispatch: (dispatch) => {
|
||||
dispatchChild = dispatch;
|
||||
},
|
||||
bindDeviceNodeControl: deviceRuntime.bindNodeTransport,
|
||||
bindDeviceNodeControl: (transport) => {
|
||||
nodeWorkerSupervisorTransport = transport;
|
||||
deviceRuntime.bindNodeTransport(transport);
|
||||
},
|
||||
bindNodeWorkspaceBindingResolver: (resolver) =>
|
||||
nodeWorkerTunnelManager.bindWorkspaceBindingResolver(resolver),
|
||||
handleNodeWorkerBundleTransferRequest:
|
||||
createNodeWorkerBundleTransferHttpCallback(nodeWorkerBundleTransfer),
|
||||
handleNodeWorkerProjectionTransferRequest:
|
||||
createNodeWorkerProjectionTransferHttpCallback(nodeWorkerProjectionTransfer),
|
||||
handleNodeWorkspaceTransferRequest:
|
||||
createNodeWorkspaceTransferHttpCallback(nodeWorkspaceTransfer),
|
||||
};
|
||||
|
||||
@@ -115,10 +115,12 @@ type WorkerInferenceConnectionService = WorkerConnectionService & {
|
||||
searchMemory?: (
|
||||
identity: WorkerConnectionIdentity,
|
||||
request: WorkerMemorySearchParams,
|
||||
signal?: AbortSignal,
|
||||
) => Promise<WorkerServiceResult<unknown, { reason: string }>>;
|
||||
readMemory?: (
|
||||
identity: WorkerConnectionIdentity,
|
||||
request: WorkerMemoryReadParams,
|
||||
signal?: AbortSignal,
|
||||
) => Promise<WorkerServiceResult<unknown, { reason: string }>>;
|
||||
startInference?: (
|
||||
identity: WorkerConnectionIdentity,
|
||||
@@ -255,7 +257,7 @@ async function dispatchWorkerRequest(params: {
|
||||
rejectWorkerRequest({ ...params, reason: "method-not-allowed" });
|
||||
return;
|
||||
}
|
||||
const outcome = await service.searchMemory(params.identity, params.request.params);
|
||||
const outcome = await service.searchMemory(params.identity, params.request.params, params.signal);
|
||||
if (outcome.ok) {
|
||||
params.respond(true, outcome.result);
|
||||
return;
|
||||
@@ -284,7 +286,7 @@ async function dispatchWorkerRequest(params: {
|
||||
rejectWorkerRequest({ ...params, reason: "method-not-allowed" });
|
||||
return;
|
||||
}
|
||||
const outcome = await service.readMemory(params.identity, params.request.params);
|
||||
const outcome = await service.readMemory(params.identity, params.request.params, params.signal);
|
||||
if (outcome.ok) {
|
||||
params.respond(true, outcome.result);
|
||||
return;
|
||||
@@ -425,6 +427,7 @@ export function attachWorkerWsMessageHandler(params: WorkerWsMessageHandlerParam
|
||||
let expiryTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let disposed = false;
|
||||
const sessionOperations = new Set<string>();
|
||||
const memoryOperations = new Map<string, AbortController>();
|
||||
const cleanup = () => {
|
||||
if (disposed) {
|
||||
return;
|
||||
@@ -432,6 +435,10 @@ export function attachWorkerWsMessageHandler(params: WorkerWsMessageHandlerParam
|
||||
disposed = true;
|
||||
clearTimeout(expiryTimer);
|
||||
sessionOperations.clear();
|
||||
for (const controller of memoryOperations.values()) {
|
||||
controller.abort(new Error("worker memory RPC connection closed"));
|
||||
}
|
||||
memoryOperations.clear();
|
||||
params.socket.off("message", onMessage);
|
||||
};
|
||||
const closeWorker = (code: number, reason: WorkerProtocolCloseReason) => {
|
||||
@@ -695,6 +702,24 @@ export function attachWorkerWsMessageHandler(params: WorkerWsMessageHandlerParam
|
||||
});
|
||||
return;
|
||||
}
|
||||
const isMemoryOperation =
|
||||
parsed.method === WORKER_MEMORY_METHODS[0] || parsed.method === WORKER_MEMORY_METHODS[1];
|
||||
if (isMemoryOperation) {
|
||||
if (memoryOperations.has(parsed.id)) {
|
||||
failFrame(1008, "invalid-frame");
|
||||
return;
|
||||
}
|
||||
const controller = new AbortController();
|
||||
memoryOperations.set(parsed.id, controller);
|
||||
try {
|
||||
await dispatch(controller.signal);
|
||||
} finally {
|
||||
if (memoryOperations.get(parsed.id) === controller) {
|
||||
memoryOperations.delete(parsed.id);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
await dispatch();
|
||||
};
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
type WorkerProfile,
|
||||
type WorkerProvider,
|
||||
} from "../../plugins/types.js";
|
||||
import type { NodeWorkerExecution } from "../../worker/node-supervisor-protocol.js";
|
||||
import type {
|
||||
NodeWorkerSupervisorNodeProof,
|
||||
NodeWorkerSupervisorTransport,
|
||||
@@ -31,8 +32,14 @@ export type DeviceWorkerAvailability = {
|
||||
};
|
||||
type DeviceWorkerAvailabilityResolver = (deviceId: string) => Promise<DeviceWorkerAvailability>;
|
||||
type DeviceWorkerReconciliation = (deviceId: string) => Promise<readonly string[]>;
|
||||
type DeviceWorkerExecutionEligibility = (params: {
|
||||
deviceId: string;
|
||||
execution: NodeWorkerExecution;
|
||||
signal?: AbortSignal;
|
||||
}) => Promise<void>;
|
||||
const DEVICE_WORKER_AVAILABILITY = new WeakMap<object, DeviceWorkerAvailabilityResolver>();
|
||||
const DEVICE_WORKER_RECONCILIATION = new WeakMap<object, DeviceWorkerReconciliation>();
|
||||
const DEVICE_WORKER_EXECUTION_ELIGIBILITY = new WeakMap<object, DeviceWorkerExecutionEligibility>();
|
||||
|
||||
export function bindDeviceWorkerAvailability(
|
||||
service: object,
|
||||
@@ -80,6 +87,36 @@ export async function reconcileDeviceWorker(
|
||||
return reconcile ? await reconcile(deviceId) : [];
|
||||
}
|
||||
|
||||
export function bindDeviceWorkerExecutionEligibility(
|
||||
service: object,
|
||||
assertEligible: DeviceWorkerExecutionEligibility,
|
||||
): void {
|
||||
DEVICE_WORKER_EXECUTION_ELIGIBILITY.set(service, assertEligible);
|
||||
}
|
||||
|
||||
/**
|
||||
* Enforced launches must prove the node boundary before any turn credential,
|
||||
* tunnel, or workspace operation is created.
|
||||
*/
|
||||
export async function assertDeviceWorkerExecutionEligibility(params: {
|
||||
service: object | undefined;
|
||||
deviceId: string;
|
||||
execution: NodeWorkerExecution;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<void> {
|
||||
const assertEligible = params.service
|
||||
? DEVICE_WORKER_EXECUTION_ELIGIBILITY.get(params.service)
|
||||
: undefined;
|
||||
if (!assertEligible) {
|
||||
throw new WorkerProviderError("device worker process-isolation eligibility is unavailable");
|
||||
}
|
||||
await assertEligible({
|
||||
deviceId: params.deviceId,
|
||||
execution: params.execution,
|
||||
...(params.signal ? { signal: params.signal } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
function requireDeviceId(profile: WorkerProfile): string {
|
||||
const deviceId = profile.device;
|
||||
if (typeof deviceId !== "string" || !deviceId.trim()) {
|
||||
@@ -170,6 +207,7 @@ export function createDeviceWorkerRuntime(options: DeviceWorkerRuntimeOptions) {
|
||||
return {
|
||||
provider,
|
||||
resolveAvailability,
|
||||
assertNodeWorkerExecutionEligible: launchAdapter.assertExecutionEligible,
|
||||
launchNodeWorker: launchAdapter.launch,
|
||||
getNodeTransport: () => nodeTransport,
|
||||
bindNodeTransport: (transport: NodeWorkerSupervisorTransport) => {
|
||||
|
||||
@@ -8,9 +8,17 @@ import {
|
||||
WORKER_RPC_SET_VERSION,
|
||||
} from "../../../packages/gateway-protocol/src/schema/worker-admission.js";
|
||||
import { NODE_WORKER_CAPACITY_EXHAUSTED_ERROR_CODE } from "../../infra/node-commands.js";
|
||||
import { NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE } from "../../infra/node-runner-inventory.js";
|
||||
import {
|
||||
NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
|
||||
NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE,
|
||||
NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
|
||||
} from "../../infra/node-runner-inventory.js";
|
||||
import {
|
||||
NODE_WORKER_EXECUTION_CONTAINER_V1,
|
||||
NODE_WORKER_EXECUTION_HOST_V1,
|
||||
nodeWorkerMemoryProjectionLaunchBinding,
|
||||
nodeWorkerPlanHash,
|
||||
type NodeWorkerExecution,
|
||||
type NodeWorkerLaunchInput,
|
||||
type NodeWorkerSupervisorReceipt,
|
||||
} from "../../worker/node-supervisor-protocol.js";
|
||||
@@ -27,7 +35,12 @@ const WORKER_RUNS = {
|
||||
protocolFeatures: [...WORKER_PROTOCOL_FEATURES],
|
||||
};
|
||||
|
||||
function nodeProof(connId = "conn-1", available = 2): NodeWorkerSupervisorNodeProof {
|
||||
function nodeProof(
|
||||
connId = "conn-1",
|
||||
available = 2,
|
||||
execution: NodeWorkerExecution = { kind: NODE_WORKER_EXECUTION_HOST_V1 },
|
||||
): NodeWorkerSupervisorNodeProof {
|
||||
const container = execution.kind === NODE_WORKER_EXECUTION_CONTAINER_V1;
|
||||
return {
|
||||
nodeId: DEVICE_ID,
|
||||
connId,
|
||||
@@ -35,8 +48,16 @@ function nodeProof(connId = "conn-1", available = 2): NodeWorkerSupervisorNodePr
|
||||
pairingGeneration: "generation-1",
|
||||
clientId: GATEWAY_CLIENT_IDS.NODE_HOST,
|
||||
clientMode: GATEWAY_CLIENT_MODES.NODE,
|
||||
protocolFeature: NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
|
||||
workerHost: { enabled: true, capacity: { total: 2, available } },
|
||||
protocolFeature: container
|
||||
? NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE
|
||||
: NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
|
||||
workerHost: {
|
||||
enabled: true,
|
||||
capacity: { total: 2, available },
|
||||
...(container
|
||||
? { processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, memoryProjection: 1 } }
|
||||
: {}),
|
||||
},
|
||||
commands: ["system.run"],
|
||||
};
|
||||
}
|
||||
@@ -47,6 +68,7 @@ function launchInput(): NodeWorkerLaunchInput {
|
||||
gatewayNamespace: "gateway-1",
|
||||
expectedBundleHash: WORKER_RUNS.bundleHash,
|
||||
placementGeneration: 4,
|
||||
execution: { kind: NODE_WORKER_EXECUTION_HOST_V1 },
|
||||
descriptor: {
|
||||
version: 4,
|
||||
admission: {
|
||||
@@ -59,6 +81,7 @@ function launchInput(): NodeWorkerLaunchInput {
|
||||
},
|
||||
assignment: {
|
||||
agentId: "agent-1",
|
||||
memoryReadEnforced: false,
|
||||
operationalRunInstance: { instanceId: "instance-1", runId: "run-1" },
|
||||
agentRuntimeIdentityToken: "signed-runtime-token",
|
||||
runId: "run-1",
|
||||
@@ -80,6 +103,7 @@ function launchInput(): NodeWorkerLaunchInput {
|
||||
function receipt(
|
||||
input: NodeWorkerLaunchInput,
|
||||
state: NodeWorkerSupervisorReceipt["state"],
|
||||
executionStarted = true,
|
||||
): NodeWorkerSupervisorReceipt {
|
||||
const identity = {
|
||||
launchId: input.launchId,
|
||||
@@ -102,7 +126,7 @@ function receipt(
|
||||
};
|
||||
}
|
||||
if (state === "failed" || state === "interrupted" || state === "cancelled") {
|
||||
return { ...identity, state, errorText: `worker ${state}` };
|
||||
return { ...identity, state, errorText: `worker ${state}`, executionStarted };
|
||||
}
|
||||
return { ...identity, state };
|
||||
}
|
||||
@@ -128,6 +152,18 @@ function launchRequest(input = launchInput()) {
|
||||
};
|
||||
}
|
||||
|
||||
function issuedProjection(input: Omit<NodeWorkerLaunchInput, "memoryProjection">, reference: string) {
|
||||
return {
|
||||
version: 1 as const,
|
||||
reference,
|
||||
binding: {
|
||||
launch: nodeWorkerMemoryProjectionLaunchBinding(input),
|
||||
authorization: "b".repeat(64),
|
||||
},
|
||||
expiresAtMs: Date.now() + 60_000,
|
||||
};
|
||||
}
|
||||
|
||||
describe("node worker launch adapter", () => {
|
||||
it("fails with a typed availability result when no node dispatches within the grace", async () => {
|
||||
vi.useFakeTimers();
|
||||
@@ -172,6 +208,70 @@ describe("node worker launch adapter", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("reports execution readiness only after the durable worker receipt", async () => {
|
||||
const input = launchInput();
|
||||
const callbacks: string[] = [];
|
||||
let statusCalls = 0;
|
||||
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async (request) => {
|
||||
if (request.command === "worker.launch.v1") {
|
||||
request.onDispatchReady?.("invoke-1");
|
||||
return wire(receipt(input, "pending"));
|
||||
}
|
||||
statusCalls += 1;
|
||||
return wire(receipt(input, statusCalls === 1 ? "running" : "completed"));
|
||||
});
|
||||
const adapter = createNodeWorkerLaunchAdapter({
|
||||
getTransport: () => transportWith(invoke),
|
||||
sleep: async () => {},
|
||||
});
|
||||
|
||||
const launched = adapter.launch({
|
||||
...launchRequest(input),
|
||||
onDispatchReady: () => callbacks.push("dispatch"),
|
||||
onExecutionReady: () => callbacks.push("execution"),
|
||||
});
|
||||
|
||||
await expect(launched).resolves.toEqual(receipt(input, "completed"));
|
||||
expect(callbacks).toEqual(["dispatch", "execution"]);
|
||||
});
|
||||
|
||||
it("does not report execution readiness when a launch terminates before child acknowledgement", async () => {
|
||||
const input = launchInput();
|
||||
const callbacks: string[] = [];
|
||||
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async (request) => {
|
||||
request.onDispatchReady?.("invoke-1");
|
||||
return wire(receipt(input, "cancelled", false));
|
||||
});
|
||||
const adapter = createNodeWorkerLaunchAdapter({
|
||||
getTransport: () => transportWith(invoke),
|
||||
});
|
||||
|
||||
await expect(
|
||||
adapter.launch({
|
||||
...launchRequest(input),
|
||||
onDispatchReady: () => callbacks.push("dispatch"),
|
||||
onExecutionReady: () => callbacks.push("execution"),
|
||||
}),
|
||||
).resolves.toEqual(receipt(input, "cancelled", false));
|
||||
expect(callbacks).toEqual(["dispatch"]);
|
||||
});
|
||||
|
||||
it("rejects a container launch on a v5 node before invoking the supervisor", async () => {
|
||||
const input = {
|
||||
...launchInput(),
|
||||
execution: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 } as const,
|
||||
};
|
||||
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>();
|
||||
const adapter = createNodeWorkerLaunchAdapter({
|
||||
getTransport: () => transportWith(invoke),
|
||||
});
|
||||
|
||||
await expect(adapter.launch(launchRequest(input))).rejects.toMatchObject({
|
||||
code: "PROCESS_ISOLATION_UNAVAILABLE",
|
||||
});
|
||||
expect(invoke).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reacquires the node and replays the identical launch after ambiguous disconnect", async () => {
|
||||
const input = launchInput();
|
||||
let launchCalls = 0;
|
||||
@@ -272,6 +372,101 @@ describe("node worker launch adapter", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps status and cancellation bound to the prepared container connection after inventory changes", async () => {
|
||||
const base = {
|
||||
...launchInput(),
|
||||
execution: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 } as const,
|
||||
descriptor: {
|
||||
...launchInput().descriptor,
|
||||
assignment: {
|
||||
...launchInput().descriptor.assignment,
|
||||
memoryReadEnforced: true,
|
||||
workspaceDir: "/workspace",
|
||||
},
|
||||
},
|
||||
};
|
||||
const input = { ...base, memoryProjection: issuedProjection(base, "a".repeat(43)) };
|
||||
const controller = new AbortController();
|
||||
let discoveryCount = 0;
|
||||
let sleepCount = 0;
|
||||
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async (request) => {
|
||||
if (request.command === "worker.launch.v1") {
|
||||
return wire(receipt(input, "running"));
|
||||
}
|
||||
if (request.command === "worker.status.v1") {
|
||||
return wire(receipt(input, "running"));
|
||||
}
|
||||
return wire(receipt(input, "cancelled"));
|
||||
});
|
||||
const adapter = createNodeWorkerLaunchAdapter({
|
||||
getTransport: () =>
|
||||
transportWith(invoke, async () => [
|
||||
discoveryCount++ <= 1
|
||||
? nodeProof("conn-container", 2, input.execution)
|
||||
: nodeProof("conn-container"),
|
||||
]),
|
||||
sleep: async () => {
|
||||
if (++sleepCount === 2) {
|
||||
controller.abort();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
adapter.launch({
|
||||
...launchRequest(input),
|
||||
prepareMemoryProjection: async () => input.memoryProjection,
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).resolves.toEqual(receipt(input, "cancelled"));
|
||||
expect(invoke.mock.calls.map(([request]) => request.command)).toEqual([
|
||||
"worker.launch.v1",
|
||||
"worker.status.v1",
|
||||
"worker.cancel.v1",
|
||||
]);
|
||||
expect(invoke.mock.calls.slice(1).map(([request]) => request.node.connId)).toEqual([
|
||||
"conn-container",
|
||||
"conn-container",
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not replay, poll, or cancel an issued projection through a replacement node connection", async () => {
|
||||
const base = {
|
||||
...launchInput(),
|
||||
execution: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 } as const,
|
||||
descriptor: {
|
||||
...launchInput().descriptor,
|
||||
assignment: {
|
||||
...launchInput().descriptor.assignment,
|
||||
memoryReadEnforced: true,
|
||||
workspaceDir: "/workspace",
|
||||
},
|
||||
},
|
||||
};
|
||||
const input = { ...base, memoryProjection: issuedProjection(base, "b".repeat(43)) };
|
||||
const invoke = vi.fn<NodeWorkerSupervisorTransport["invoke"]>(async () =>
|
||||
wire(receipt(input, "running")),
|
||||
);
|
||||
let discoveryCount = 0;
|
||||
const adapter = createNodeWorkerLaunchAdapter({
|
||||
getTransport: () =>
|
||||
transportWith(invoke, async () => [
|
||||
discoveryCount++ <= 1
|
||||
? nodeProof("conn-issued", 2, input.execution)
|
||||
: nodeProof("conn-replacement"),
|
||||
]),
|
||||
sleep: async () => {},
|
||||
});
|
||||
|
||||
await expect(
|
||||
adapter.launch({
|
||||
...launchRequest(input),
|
||||
prepareMemoryProjection: async () => input.memoryProjection,
|
||||
}),
|
||||
).rejects.toThrow("cancellation could not be confirmed");
|
||||
expect(invoke.mock.calls.map(([request]) => request.command)).toEqual(["worker.launch.v1"]);
|
||||
});
|
||||
|
||||
it("replays launch when status cannot find the durable receipt", async () => {
|
||||
const input = launchInput();
|
||||
const responses = [
|
||||
|
||||
@@ -9,10 +9,13 @@ import {
|
||||
nodeWorkerPlanHash,
|
||||
parseNodeWorkerLaunchInput,
|
||||
parseNodeWorkerSupervisorReceipt,
|
||||
type NodeWorkerExecution,
|
||||
type NodeWorkerLaunchInput,
|
||||
type NodeWorkerSupervisorIdentity,
|
||||
type NodeWorkerSupervisorReceipt,
|
||||
} from "../../worker/node-supervisor-protocol.js";
|
||||
import type { NodeWorkerMemoryProjection } from "../../worker/node-memory-projection-protocol.js";
|
||||
import { supportsNodeWorkerExecution } from "../node-runner-inventory-runtime.js";
|
||||
import type {
|
||||
NodeWorkerSupervisorNodeProof,
|
||||
NodeWorkerSupervisorTransport,
|
||||
@@ -43,11 +46,20 @@ type TerminalNodeWorkerSupervisorReceipt = Extract<
|
||||
type DeviceWorkerLaunchRequest = {
|
||||
deviceId: string;
|
||||
input: NodeWorkerLaunchInput;
|
||||
/** Mints the opaque projection only after this adapter has proved the exact node connection. */
|
||||
prepareMemoryProjection?: (
|
||||
node: NodeWorkerSupervisorNodeProof,
|
||||
) => Promise<NodeWorkerMemoryProjection>;
|
||||
isDispatchAuthorized: () => boolean;
|
||||
isCancellationAuthorized: () => boolean;
|
||||
timeoutMs: number;
|
||||
signal?: AbortSignal;
|
||||
onDispatchReady?: () => void;
|
||||
onExecutionReady?: () => void;
|
||||
};
|
||||
|
||||
type PreparedDeviceWorkerLaunchRequest = DeviceWorkerLaunchRequest & {
|
||||
boundNode?: NodeWorkerSupervisorNodeProof;
|
||||
};
|
||||
|
||||
type NodeWorkerLaunchAdapterOptions = {
|
||||
@@ -126,6 +138,18 @@ function receiptMatchesIdentity(
|
||||
);
|
||||
}
|
||||
|
||||
function sameNodeConnection(
|
||||
left: NodeWorkerSupervisorNodeProof,
|
||||
right: NodeWorkerSupervisorNodeProof,
|
||||
): boolean {
|
||||
return (
|
||||
left.nodeId === right.nodeId &&
|
||||
left.connId === right.connId &&
|
||||
left.pairingIdentity === right.pairingIdentity &&
|
||||
left.pairingGeneration === right.pairingGeneration
|
||||
);
|
||||
}
|
||||
|
||||
function parseInvokeReceipt(
|
||||
payloadJSON: string | null | undefined,
|
||||
): NodeWorkerSupervisorReceipt | null {
|
||||
@@ -209,7 +233,8 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
|
||||
const findNode = async (params: {
|
||||
transport: NodeWorkerSupervisorTransport;
|
||||
deviceId: string;
|
||||
requireLaunchAvailability?: boolean;
|
||||
execution: NodeWorkerExecution;
|
||||
requireLaunchEligibility: boolean;
|
||||
signal: AbortSignal;
|
||||
}): Promise<NodeWorkerSupervisorNodeProof> => {
|
||||
let nodes: readonly NodeWorkerSupervisorNodeProof[];
|
||||
@@ -224,17 +249,22 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
|
||||
"device worker node discovery is unavailable",
|
||||
);
|
||||
}
|
||||
const node = nodes.find(
|
||||
(candidate) =>
|
||||
candidate.nodeId === params.deviceId &&
|
||||
(!params.requireLaunchAvailability || candidate.workerHost.capacity.available > 0),
|
||||
);
|
||||
const node = nodes.find((candidate) => candidate.nodeId === params.deviceId);
|
||||
if (!node) {
|
||||
throw new NodeWorkerLaunchTransportError(
|
||||
"NOT_CONNECTED",
|
||||
"device worker node is not currently connected",
|
||||
);
|
||||
}
|
||||
if (params.requireLaunchEligibility && !supportsNodeWorkerExecution(node, params.execution)) {
|
||||
throw new NodeWorkerLaunchTransportError(
|
||||
"PROCESS_ISOLATION_UNAVAILABLE",
|
||||
"device worker node does not attest to the required process-isolation boundary",
|
||||
);
|
||||
}
|
||||
if (params.requireLaunchEligibility && node.workerHost.capacity.available <= 0) {
|
||||
throw new WorkerRunnerCapacityError();
|
||||
}
|
||||
return node;
|
||||
};
|
||||
|
||||
@@ -245,9 +275,10 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
|
||||
| typeof NODE_WORKER_SUPERVISOR_STATUS_COMMAND
|
||||
| typeof NODE_WORKER_SUPERVISOR_CANCEL_COMMAND;
|
||||
payload: unknown;
|
||||
requireLaunchAvailability?: boolean;
|
||||
execution: NodeWorkerExecution;
|
||||
isAuthorized: () => boolean;
|
||||
deadline: OperationDeadline;
|
||||
boundNode?: NodeWorkerSupervisorNodeProof;
|
||||
onDispatchReady?: () => void;
|
||||
}): Promise<NodeWorkerSupervisorReceipt | null> => {
|
||||
if (!params.isAuthorized()) {
|
||||
@@ -284,9 +315,18 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
|
||||
const node = await findNode({
|
||||
transport,
|
||||
deviceId: params.deviceId,
|
||||
requireLaunchAvailability: params.requireLaunchAvailability,
|
||||
execution: params.execution,
|
||||
requireLaunchEligibility: params.command === NODE_WORKER_SUPERVISOR_LAUNCH_COMMAND,
|
||||
signal,
|
||||
});
|
||||
if (params.boundNode && !sameNodeConnection(node, params.boundNode)) {
|
||||
// A projection bearer is bound to this exact paired connection. Retrying against a
|
||||
// replacement node would silently turn a stale capability into a confused deputy.
|
||||
throw new NodeWorkerLaunchTransportError(
|
||||
"NODE_IDENTITY_CHANGED",
|
||||
"device worker node identity changed after memory projection preparation",
|
||||
);
|
||||
}
|
||||
const operation = transport.invoke({
|
||||
node,
|
||||
command: params.command,
|
||||
@@ -352,7 +392,7 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
|
||||
};
|
||||
|
||||
const cancelUntilTerminal = async (params: {
|
||||
request: DeviceWorkerLaunchRequest;
|
||||
request: PreparedDeviceWorkerLaunchRequest;
|
||||
expected: NodeWorkerSupervisorIdentity;
|
||||
}): Promise<TerminalNodeWorkerSupervisorReceipt> => {
|
||||
const deadline = createDeadline({
|
||||
@@ -371,8 +411,10 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
|
||||
deviceId: params.request.deviceId,
|
||||
command: NODE_WORKER_SUPERVISOR_CANCEL_COMMAND,
|
||||
payload: params.expected,
|
||||
execution: params.request.input.execution,
|
||||
isAuthorized: params.request.isCancellationAuthorized,
|
||||
deadline,
|
||||
...(params.request.boundNode ? { boundNode: params.request.boundNode } : {}),
|
||||
});
|
||||
if (receipt) {
|
||||
const validated = validateReceipt(receipt, params.expected);
|
||||
@@ -403,9 +445,6 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
|
||||
const launch = async (
|
||||
request: DeviceWorkerLaunchRequest,
|
||||
): Promise<TerminalNodeWorkerSupervisorReceipt> => {
|
||||
const input = snapshotLaunchInput(request.input);
|
||||
const stableRequest = { ...request, input };
|
||||
const expected = expectedIdentity(input);
|
||||
const deadline = createDeadline({
|
||||
now,
|
||||
timeoutMs: request.timeoutMs,
|
||||
@@ -418,8 +457,12 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
|
||||
signal: deadline.signal,
|
||||
label: "node worker availability",
|
||||
});
|
||||
let input: NodeWorkerLaunchInput;
|
||||
let expected: NodeWorkerSupervisorIdentity | undefined;
|
||||
let stableRequest: PreparedDeviceWorkerLaunchRequest | undefined;
|
||||
let mayHaveLaunched = false;
|
||||
let dispatchReady = false;
|
||||
let executionReady = false;
|
||||
let pollStatus = false;
|
||||
let delayMs = pollIntervalMs;
|
||||
const markDispatchReady = () => {
|
||||
@@ -429,7 +472,47 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
|
||||
stableRequest.onDispatchReady?.();
|
||||
}
|
||||
};
|
||||
const markExecutionReady = (receipt: NodeWorkerSupervisorReceipt) => {
|
||||
if (
|
||||
executionReady ||
|
||||
!(
|
||||
receipt.state === "running" ||
|
||||
receipt.state === "completed" ||
|
||||
(isTerminalReceipt(receipt) && receipt.executionStarted)
|
||||
)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
executionReady = true;
|
||||
stableRequest?.onExecutionReady?.();
|
||||
};
|
||||
try {
|
||||
if (request.prepareMemoryProjection) {
|
||||
const transport = options.getTransport();
|
||||
if (!transport) {
|
||||
throw new NodeWorkerLaunchTransportError(
|
||||
"UNAVAILABLE",
|
||||
"device worker node transport is unavailable",
|
||||
);
|
||||
}
|
||||
const node = await findNode({
|
||||
transport,
|
||||
deviceId: request.deviceId,
|
||||
execution: request.input.execution,
|
||||
requireLaunchEligibility: true,
|
||||
signal: availabilityDeadline.signal,
|
||||
});
|
||||
const memoryProjection = await raceWithSignal(
|
||||
request.prepareMemoryProjection(node),
|
||||
availabilityDeadline.signal,
|
||||
);
|
||||
input = snapshotLaunchInput({ ...request.input, memoryProjection });
|
||||
stableRequest = { ...request, input, boundNode: node };
|
||||
} else {
|
||||
input = snapshotLaunchInput(request.input);
|
||||
stableRequest = { ...request, input };
|
||||
}
|
||||
expected = expectedIdentity(input);
|
||||
while (true) {
|
||||
if (deadline.signal.aborted) {
|
||||
throw signalError(deadline.signal, "node worker launch aborted");
|
||||
@@ -448,9 +531,10 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
|
||||
? NODE_WORKER_SUPERVISOR_STATUS_COMMAND
|
||||
: NODE_WORKER_SUPERVISOR_LAUNCH_COMMAND,
|
||||
payload: pollStatus ? { launchId: input.launchId } : input,
|
||||
...(!pollStatus ? { requireLaunchAvailability: true } : {}),
|
||||
execution: input.execution,
|
||||
isAuthorized: stableRequest.isDispatchAuthorized,
|
||||
deadline: attemptDeadline,
|
||||
...(stableRequest.boundNode ? { boundNode: stableRequest.boundNode } : {}),
|
||||
...(!pollStatus
|
||||
? {
|
||||
onDispatchReady: markDispatchReady,
|
||||
@@ -465,6 +549,7 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
|
||||
}
|
||||
const validated = validateReceipt(receipt, expected);
|
||||
mayHaveLaunched = true;
|
||||
markExecutionReady(validated);
|
||||
if (isTerminalReceipt(validated)) {
|
||||
return validated;
|
||||
}
|
||||
@@ -505,7 +590,10 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
|
||||
}
|
||||
let terminal: TerminalNodeWorkerSupervisorReceipt;
|
||||
try {
|
||||
terminal = await cancelUntilTerminal({ request: stableRequest, expected });
|
||||
terminal = await cancelUntilTerminal({
|
||||
request: stableRequest!,
|
||||
expected: expected!,
|
||||
});
|
||||
} catch (cancelError) {
|
||||
throw Object.assign(
|
||||
new Error("node worker launch failed and cancellation could not be confirmed", {
|
||||
@@ -524,5 +612,39 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp
|
||||
}
|
||||
};
|
||||
|
||||
return { launch };
|
||||
const assertExecutionEligible = async (params: {
|
||||
deviceId: string;
|
||||
execution: NodeWorkerExecution;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<void> => {
|
||||
const transport = options.getTransport();
|
||||
if (!transport) {
|
||||
throw new NodeWorkerLaunchTransportError(
|
||||
"UNAVAILABLE",
|
||||
"device worker node transport is unavailable",
|
||||
);
|
||||
}
|
||||
const timeout = new AbortController();
|
||||
const timer = setTimeout(
|
||||
() => timeout.abort(new Error("node worker eligibility check timed out")),
|
||||
DEFAULT_AVAILABILITY_TIMEOUT_MS,
|
||||
);
|
||||
timer.unref?.();
|
||||
const signal = params.signal
|
||||
? AbortSignal.any([params.signal, timeout.signal])
|
||||
: timeout.signal;
|
||||
try {
|
||||
await findNode({
|
||||
transport,
|
||||
deviceId: params.deviceId,
|
||||
execution: params.execution,
|
||||
requireLaunchEligibility: true,
|
||||
signal,
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
};
|
||||
|
||||
return { assertExecutionEligible, launch };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
import type { IncomingMessage, ServerResponse } from "node:http";
|
||||
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
|
||||
import {
|
||||
nodeWorkerMemoryProjectionTransferPath,
|
||||
parseNodeWorkerMemoryProjectionRequestProof,
|
||||
NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER,
|
||||
NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER,
|
||||
NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER,
|
||||
type NodeWorkerMemoryProjectionRequestProof,
|
||||
} from "../../worker/node-memory-projection-protocol.js";
|
||||
import { AUTH_RATE_LIMIT_SCOPE_WORKER_TRANSFER, type AuthRateLimiter } from "../auth-rate-limit.js";
|
||||
import { classifyNodeWorkerProjectionTransferPath } from "../gateway-http-route-contracts.js";
|
||||
import { sendJson, watchClientDisconnect } from "../http-common.js";
|
||||
import { withSerializedRateLimitAttempt } from "../rate-limit-attempt-serialization.js";
|
||||
import type { NodeWorkerProjectionTransferService } from "./node-worker-projection-transfer-service.js";
|
||||
|
||||
const OPAQUE_NOT_FOUND = { error: "not_found" } as const;
|
||||
|
||||
type NodeWorkerProjectionTransferHttpCallbackResult =
|
||||
| { kind: "unauthorized" }
|
||||
| { kind: "authorized"; handle: () => Promise<void> | void };
|
||||
|
||||
export type NodeWorkerProjectionTransferHttpCallback = (params: {
|
||||
req: IncomingMessage;
|
||||
res: ServerResponse;
|
||||
bearer: string;
|
||||
proof: NodeWorkerMemoryProjectionRequestProof;
|
||||
}) => Promise<NodeWorkerProjectionTransferHttpCallbackResult>;
|
||||
|
||||
function bearerToken(req: IncomingMessage): string | undefined {
|
||||
const authorization = normalizeOptionalString(req.headers.authorization);
|
||||
if (!authorization?.toLowerCase().startsWith("bearer ")) {
|
||||
return undefined;
|
||||
}
|
||||
return normalizeOptionalString(authorization.slice(7));
|
||||
}
|
||||
|
||||
function proofHeader(req: IncomingMessage, name: string): string | undefined {
|
||||
const value = req.headers[name];
|
||||
return typeof value === "string" ? normalizeOptionalString(value) : undefined;
|
||||
}
|
||||
|
||||
function requestProof(req: IncomingMessage): NodeWorkerMemoryProjectionRequestProof | undefined {
|
||||
const signedAt = proofHeader(req, NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER);
|
||||
if (!signedAt || !/^(?:0|[1-9][0-9]{0,15})$/u.test(signedAt)) {
|
||||
return undefined;
|
||||
}
|
||||
return (
|
||||
parseNodeWorkerMemoryProjectionRequestProof({
|
||||
nodeId: proofHeader(req, NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER),
|
||||
signedAtMs: Number(signedAt),
|
||||
signature: proofHeader(req, NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER),
|
||||
}) ?? undefined
|
||||
);
|
||||
}
|
||||
|
||||
function sendOpaqueNotFound(res: ServerResponse): void {
|
||||
sendJson(res, 404, OPAQUE_NOT_FOUND);
|
||||
}
|
||||
|
||||
export async function handleNodeWorkerProjectionTransferHttpRequest(params: {
|
||||
req: IncomingMessage;
|
||||
res: ServerResponse;
|
||||
clientIp: string | undefined;
|
||||
rateLimiter?: AuthRateLimiter;
|
||||
callback?: NodeWorkerProjectionTransferHttpCallback;
|
||||
}): Promise<boolean> {
|
||||
const parsed = URL.parse(params.req.url ?? "/", "http://localhost");
|
||||
if (!parsed?.pathname || classifyNodeWorkerProjectionTransferPath(parsed.pathname) === "outside") {
|
||||
return false;
|
||||
}
|
||||
params.res.setHeader("Cache-Control", "no-store");
|
||||
if (parsed.pathname !== nodeWorkerMemoryProjectionTransferPath() || parsed.search || params.req.method !== "GET") {
|
||||
sendOpaqueNotFound(params.res);
|
||||
return true;
|
||||
}
|
||||
const bearer = bearerToken(params.req);
|
||||
const proof = requestProof(params.req);
|
||||
const admission = await withSerializedRateLimitAttempt<
|
||||
| { kind: "rate-limited"; retryAfterMs: number }
|
||||
| { kind: "unauthorized" }
|
||||
| Extract<NodeWorkerProjectionTransferHttpCallbackResult, { kind: "authorized" }>
|
||||
>({
|
||||
ip: params.clientIp,
|
||||
scope: AUTH_RATE_LIMIT_SCOPE_WORKER_TRANSFER,
|
||||
run: async () => {
|
||||
const rateCheck = params.rateLimiter?.check(
|
||||
params.clientIp,
|
||||
AUTH_RATE_LIMIT_SCOPE_WORKER_TRANSFER,
|
||||
);
|
||||
if (rateCheck && !rateCheck.allowed) {
|
||||
return { kind: "rate-limited", retryAfterMs: rateCheck.retryAfterMs };
|
||||
}
|
||||
const outcome =
|
||||
bearer && proof && params.callback
|
||||
? await params.callback({ req: params.req, res: params.res, bearer, proof })
|
||||
: ({ kind: "unauthorized" } as const);
|
||||
if (outcome.kind === "unauthorized") {
|
||||
params.rateLimiter?.recordFailure(params.clientIp, AUTH_RATE_LIMIT_SCOPE_WORKER_TRANSFER);
|
||||
} else {
|
||||
params.rateLimiter?.reset(params.clientIp, AUTH_RATE_LIMIT_SCOPE_WORKER_TRANSFER);
|
||||
}
|
||||
return outcome;
|
||||
},
|
||||
});
|
||||
if (admission.kind === "rate-limited") {
|
||||
if (admission.retryAfterMs > 0) {
|
||||
params.res.setHeader("Retry-After", String(Math.ceil(admission.retryAfterMs / 1000)));
|
||||
}
|
||||
sendJson(params.res, 429, { error: "rate_limited" });
|
||||
return true;
|
||||
}
|
||||
if (admission.kind === "unauthorized") {
|
||||
sendOpaqueNotFound(params.res);
|
||||
return true;
|
||||
}
|
||||
await admission.handle();
|
||||
return true;
|
||||
}
|
||||
|
||||
export function createNodeWorkerProjectionTransferHttpCallback(
|
||||
service: NodeWorkerProjectionTransferService,
|
||||
): NodeWorkerProjectionTransferHttpCallback {
|
||||
return async ({ req, res, bearer, proof }) => {
|
||||
const authorization = await service.authorize(bearer, proof);
|
||||
if (!authorization) {
|
||||
return { kind: "unauthorized" };
|
||||
}
|
||||
return {
|
||||
kind: "authorized",
|
||||
handle: async () => {
|
||||
const clientAbort = new AbortController();
|
||||
const stopWatchingDisconnect = watchClientDisconnect(req, res, clientAbort);
|
||||
const timeoutMs = Math.max(1, authorization.expiresAtMs - Date.now());
|
||||
const signal = AbortSignal.any([
|
||||
service.authorizationSignal(authorization),
|
||||
clientAbort.signal,
|
||||
AbortSignal.timeout(timeoutMs),
|
||||
]);
|
||||
try {
|
||||
const payload = service.payload(authorization);
|
||||
if (!payload || signal.aborted || !service.isAuthorizationCurrent(authorization)) {
|
||||
sendOpaqueNotFound(res);
|
||||
return;
|
||||
}
|
||||
sendJson(res, 200, payload);
|
||||
} finally {
|
||||
stopWatchingDisconnect();
|
||||
service.revoke(authorization);
|
||||
}
|
||||
},
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,363 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import type { AuthorizedMemoryVirtualFileBroker } from "../../agents/memory-authorized-read-host.js";
|
||||
import { verifyDeviceSignature } from "../../infra/device-identity.js";
|
||||
import { generateSecureToken } from "../../infra/secure-random.js";
|
||||
import { registerSecretValueForRedaction } from "../../logging/secret-redaction-registry.js";
|
||||
import {
|
||||
buildNodeWorkerMemoryProjectionRequestProofPayload,
|
||||
NODE_WORKER_MEMORY_PROJECTION_MAX_FILES,
|
||||
NODE_WORKER_MEMORY_PROJECTION_MAX_FILE_BYTES,
|
||||
NODE_WORKER_MEMORY_PROJECTION_MAX_TOTAL_BYTES,
|
||||
NODE_WORKER_MEMORY_PROJECTION_PROOF_SKEW_MS,
|
||||
NODE_WORKER_MEMORY_PROJECTION_VERSION,
|
||||
type NodeWorkerMemoryProjectionFile,
|
||||
type NodeWorkerMemoryProjectionBinding,
|
||||
type NodeWorkerMemoryProjectionPayload,
|
||||
type NodeWorkerMemoryProjection,
|
||||
type NodeWorkerMemoryProjectionRequestProof,
|
||||
} from "../../worker/node-memory-projection-protocol.js";
|
||||
import type { NodeWorkerSupervisorNodeProof } from "../node-registry-private.js";
|
||||
|
||||
const TOKEN_PATTERN = /^[A-Za-z0-9_-]{43}$/u;
|
||||
type ProjectionTransferCapability = {
|
||||
reference: string;
|
||||
node: NodeWorkerSupervisorNodeProof;
|
||||
nodeId: string;
|
||||
connId: string;
|
||||
pairingGeneration: string;
|
||||
environmentId: string;
|
||||
sessionId: string;
|
||||
ownerEpoch: number;
|
||||
placementGeneration: number;
|
||||
runId: string;
|
||||
launchId: string;
|
||||
binding: NodeWorkerMemoryProjectionBinding;
|
||||
viewId: string;
|
||||
viewRevision: string;
|
||||
viewDigest: string;
|
||||
expiresAtMs: number;
|
||||
payload: NodeWorkerMemoryProjectionPayload;
|
||||
state: "ready" | "serving";
|
||||
abortController: AbortController;
|
||||
stopWatchingSignal?: () => void;
|
||||
isAuthorized: () => boolean;
|
||||
};
|
||||
|
||||
function projectionAuthorizationBinding(params: {
|
||||
launchBinding: string;
|
||||
broker: AuthorizedMemoryVirtualFileBroker;
|
||||
}): NodeWorkerMemoryProjectionBinding {
|
||||
return Object.freeze({
|
||||
launch: params.launchBinding,
|
||||
// The selected runtime minted these opaque values from its authenticated
|
||||
// plan. Signing their digest prevents a node from substituting a launch
|
||||
// fence while retaining a different subject/actor/policy view.
|
||||
authorization: createHash("sha256")
|
||||
.update(
|
||||
JSON.stringify({
|
||||
launch: params.launchBinding,
|
||||
planId: params.broker.view.planId,
|
||||
contextFingerprint: params.broker.view.contextFingerprint,
|
||||
revision: params.broker.view.revision,
|
||||
}),
|
||||
)
|
||||
.digest("hex"),
|
||||
});
|
||||
}
|
||||
|
||||
function mintReference(generateToken: (bytes: number) => string): string {
|
||||
const reference = generateToken(32);
|
||||
if (!TOKEN_PATTERN.test(reference)) {
|
||||
throw new Error("Worker memory projection token generator returned an invalid bearer");
|
||||
}
|
||||
registerSecretValueForRedaction(reference);
|
||||
return reference;
|
||||
}
|
||||
|
||||
function validVirtualPath(virtualPath: string, roots: ReadonlySet<string>): boolean {
|
||||
const normalized = virtualPath.normalize("NFC");
|
||||
const parts = normalized.split("/");
|
||||
const root = parts[0];
|
||||
const leaf = parts[1];
|
||||
return (
|
||||
normalized === virtualPath &&
|
||||
parts.length === 2 &&
|
||||
Boolean(root) &&
|
||||
Boolean(leaf) &&
|
||||
roots.has(root!.toLocaleLowerCase("en-US")) &&
|
||||
leaf !== "." &&
|
||||
leaf !== ".." &&
|
||||
!leaf!.includes("\\") &&
|
||||
!leaf!.includes("\0")
|
||||
);
|
||||
}
|
||||
|
||||
function createPayload(
|
||||
broker: AuthorizedMemoryVirtualFileBroker,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{
|
||||
payload: NodeWorkerMemoryProjectionPayload;
|
||||
expiresAtMs: number;
|
||||
viewDigest: string;
|
||||
}> {
|
||||
return (async () => {
|
||||
signal?.throwIfAborted();
|
||||
const view = broker.view;
|
||||
const expiresAtMs = Date.parse(view.expiresAt);
|
||||
if (!Number.isFinite(expiresAtMs) || expiresAtMs <= Date.now()) {
|
||||
throw new Error("Authorized memory virtual view is expired");
|
||||
}
|
||||
if (view.files.length === 0 || view.files.length > NODE_WORKER_MEMORY_PROJECTION_MAX_FILES) {
|
||||
throw new Error("Authorized memory virtual view exceeds projection file limits");
|
||||
}
|
||||
const roots = new Set<string>();
|
||||
for (const root of view.roots) {
|
||||
const name = root.virtualRoot.normalize("NFC");
|
||||
const key = name.toLocaleLowerCase("en-US");
|
||||
// `/memory` is the container mount target, not this payload's root. The
|
||||
// canonical selected-memory view uses `memory/...`, nested under that mount.
|
||||
if (
|
||||
name !== root.virtualRoot ||
|
||||
!name ||
|
||||
!/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/u.test(name) ||
|
||||
roots.has(key) ||
|
||||
["opt", "run", "workspace"].includes(key)
|
||||
) {
|
||||
throw new Error("Authorized memory virtual view has an unsafe projection root");
|
||||
}
|
||||
roots.add(key);
|
||||
}
|
||||
const paths = new Set<string>();
|
||||
let totalBytes = 0;
|
||||
const files: NodeWorkerMemoryProjectionFile[] = [];
|
||||
for (const file of [...view.files].toSorted((left, right) =>
|
||||
left.virtualPath.localeCompare(right.virtualPath),
|
||||
)) {
|
||||
signal?.throwIfAborted();
|
||||
if (!validVirtualPath(file.virtualPath, roots)) {
|
||||
throw new Error("Authorized memory virtual view has an unsafe projection path");
|
||||
}
|
||||
const key = file.virtualPath.toLocaleLowerCase("en-US");
|
||||
if (paths.has(key)) {
|
||||
throw new Error("Authorized memory virtual view has colliding projection paths");
|
||||
}
|
||||
paths.add(key);
|
||||
const text = await broker.readFile(file.virtualPath, signal);
|
||||
signal?.throwIfAborted();
|
||||
if (text === undefined) {
|
||||
throw new Error("Authorized memory virtual view could not materialize an issued file");
|
||||
}
|
||||
const bytes = Buffer.from(text, "utf8");
|
||||
if (bytes.byteLength > NODE_WORKER_MEMORY_PROJECTION_MAX_FILE_BYTES) {
|
||||
throw new Error("Authorized memory virtual view exceeds projection file limits");
|
||||
}
|
||||
totalBytes += bytes.byteLength;
|
||||
if (totalBytes > NODE_WORKER_MEMORY_PROJECTION_MAX_TOTAL_BYTES) {
|
||||
throw new Error("Authorized memory virtual view exceeds projection total limits");
|
||||
}
|
||||
files.push(
|
||||
Object.freeze({
|
||||
virtualPath: file.virtualPath,
|
||||
sha256: createHash("sha256").update(bytes).digest("hex"),
|
||||
contentBase64: bytes.toString("base64"),
|
||||
}),
|
||||
);
|
||||
}
|
||||
const payload = Object.freeze({
|
||||
version: NODE_WORKER_MEMORY_PROJECTION_VERSION,
|
||||
files: Object.freeze(files),
|
||||
});
|
||||
return {
|
||||
payload,
|
||||
expiresAtMs,
|
||||
viewDigest: createHash("sha256").update(JSON.stringify(payload)).digest("hex"),
|
||||
};
|
||||
})();
|
||||
}
|
||||
|
||||
export function createNodeWorkerProjectionTransferService(
|
||||
options: {
|
||||
now?: () => number;
|
||||
generateToken?: (bytes: number) => string;
|
||||
resolveNodePublicKey?: (node: NodeWorkerSupervisorNodeProof) => Promise<string | undefined>;
|
||||
isNodeCurrent?: (node: NodeWorkerSupervisorNodeProof) => boolean;
|
||||
} = {},
|
||||
) {
|
||||
const now = options.now ?? Date.now;
|
||||
const generateToken = options.generateToken ?? generateSecureToken;
|
||||
const resolveNodePublicKey = options.resolveNodePublicKey ?? (async () => undefined);
|
||||
const isNodeCurrent = options.isNodeCurrent ?? (() => false);
|
||||
const capabilities = new Map<string, ProjectionTransferCapability>();
|
||||
|
||||
const isCurrent = (capability: ProjectionTransferCapability): boolean =>
|
||||
capabilities.get(capability.reference) === capability &&
|
||||
capability.state === "serving" &&
|
||||
capability.expiresAtMs > now() &&
|
||||
!capability.abortController.signal.aborted &&
|
||||
capability.isAuthorized() &&
|
||||
isNodeCurrent(capability.node);
|
||||
|
||||
const isReady = (
|
||||
capability: ProjectionTransferCapability | undefined,
|
||||
): capability is ProjectionTransferCapability =>
|
||||
Boolean(
|
||||
capability &&
|
||||
capabilities.get(capability.reference) === capability &&
|
||||
capability.state === "ready" &&
|
||||
capability.expiresAtMs > now() &&
|
||||
!capability.abortController.signal.aborted &&
|
||||
capability.isAuthorized() &&
|
||||
isNodeCurrent(capability.node),
|
||||
);
|
||||
|
||||
const revokeCapability = (capability: ProjectionTransferCapability): void => {
|
||||
if (capabilities.get(capability.reference) === capability) {
|
||||
capabilities.delete(capability.reference);
|
||||
}
|
||||
capability.stopWatchingSignal?.();
|
||||
if (!capability.abortController.signal.aborted) {
|
||||
capability.abortController.abort(new Error("Worker memory projection capability closed"));
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
async prepare(params: {
|
||||
node: NodeWorkerSupervisorNodeProof;
|
||||
broker: AuthorizedMemoryVirtualFileBroker;
|
||||
environmentId: string;
|
||||
sessionId: string;
|
||||
ownerEpoch: number;
|
||||
placementGeneration: number;
|
||||
runId: string;
|
||||
launchId: string;
|
||||
launchBinding: string;
|
||||
isAuthorized: () => boolean;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<NodeWorkerMemoryProjection> {
|
||||
if (!params.isAuthorized() || !isNodeCurrent(params.node)) {
|
||||
throw new Error("Worker memory projection authority is unavailable");
|
||||
}
|
||||
if (!/^[a-f0-9]{64}$/u.test(params.launchBinding)) {
|
||||
throw new Error("Worker memory projection launch binding is invalid");
|
||||
}
|
||||
params.signal?.throwIfAborted();
|
||||
const snapshot = await createPayload(params.broker, params.signal);
|
||||
if (!params.isAuthorized() || snapshot.expiresAtMs <= now()) {
|
||||
throw new Error("Worker memory projection authority is unavailable");
|
||||
}
|
||||
params.signal?.throwIfAborted();
|
||||
const reference = mintReference(generateToken);
|
||||
if (capabilities.has(reference)) {
|
||||
// A collision must never replace a live single-use capability, even in a
|
||||
// test or an incorrectly substituted entropy source.
|
||||
throw new Error("Worker memory projection token collision");
|
||||
}
|
||||
const capability: ProjectionTransferCapability = {
|
||||
reference,
|
||||
node: params.node,
|
||||
nodeId: params.node.nodeId,
|
||||
connId: params.node.connId,
|
||||
pairingGeneration: params.node.pairingGeneration,
|
||||
environmentId: params.environmentId,
|
||||
sessionId: params.sessionId,
|
||||
ownerEpoch: params.ownerEpoch,
|
||||
placementGeneration: params.placementGeneration,
|
||||
runId: params.runId,
|
||||
launchId: params.launchId,
|
||||
binding: projectionAuthorizationBinding({
|
||||
launchBinding: params.launchBinding,
|
||||
broker: params.broker,
|
||||
}),
|
||||
viewId: params.broker.view.viewId,
|
||||
viewRevision: params.broker.view.revision,
|
||||
viewDigest: snapshot.viewDigest,
|
||||
expiresAtMs: snapshot.expiresAtMs,
|
||||
payload: snapshot.payload,
|
||||
state: "ready",
|
||||
abortController: new AbortController(),
|
||||
isAuthorized: params.isAuthorized,
|
||||
};
|
||||
if (params.signal) {
|
||||
const abort = () => revokeCapability(capability);
|
||||
params.signal.addEventListener("abort", abort, { once: true });
|
||||
capability.stopWatchingSignal = () => params.signal?.removeEventListener("abort", abort);
|
||||
if (params.signal.aborted) {
|
||||
abort();
|
||||
}
|
||||
}
|
||||
if (capability.abortController.signal.aborted) {
|
||||
throw new Error("Worker memory projection authority is unavailable");
|
||||
}
|
||||
capabilities.set(reference, capability);
|
||||
return Object.freeze({
|
||||
version: NODE_WORKER_MEMORY_PROJECTION_VERSION,
|
||||
reference,
|
||||
binding: capability.binding,
|
||||
expiresAtMs: capability.expiresAtMs,
|
||||
});
|
||||
},
|
||||
|
||||
async authorize(
|
||||
reference: string,
|
||||
proof: NodeWorkerMemoryProjectionRequestProof,
|
||||
): Promise<ProjectionTransferCapability | undefined> {
|
||||
const capability = capabilities.get(reference);
|
||||
if (
|
||||
!isReady(capability) ||
|
||||
proof.nodeId !== capability.nodeId ||
|
||||
Math.abs(now() - proof.signedAtMs) > NODE_WORKER_MEMORY_PROJECTION_PROOF_SKEW_MS
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
const publicKey = await resolveNodePublicKey(capability.node).catch(() => undefined);
|
||||
if (!publicKey || !isReady(capability)) {
|
||||
return undefined;
|
||||
}
|
||||
const payload = buildNodeWorkerMemoryProjectionRequestProofPayload({
|
||||
reference,
|
||||
binding: capability.binding,
|
||||
nodeId: capability.nodeId,
|
||||
signedAtMs: proof.signedAtMs,
|
||||
});
|
||||
if (!verifyDeviceSignature(publicKey, payload, proof.signature) || !isReady(capability)) {
|
||||
return undefined;
|
||||
}
|
||||
// This is the final synchronous check before ready -> serving. A stale
|
||||
// connection or invalid signature leaves the one-use capability intact.
|
||||
capability.state = "serving";
|
||||
return capability;
|
||||
},
|
||||
|
||||
isAuthorizationCurrent: isCurrent,
|
||||
|
||||
authorizationSignal(capability: ProjectionTransferCapability): AbortSignal {
|
||||
return capability.abortController.signal;
|
||||
},
|
||||
|
||||
payload(
|
||||
capability: ProjectionTransferCapability,
|
||||
): NodeWorkerMemoryProjectionPayload | undefined {
|
||||
return isCurrent(capability) ? capability.payload : undefined;
|
||||
},
|
||||
|
||||
revoke(capabilityOrReference: ProjectionTransferCapability | string): void {
|
||||
const capability =
|
||||
typeof capabilityOrReference === "string"
|
||||
? capabilities.get(capabilityOrReference)
|
||||
: capabilityOrReference;
|
||||
if (capability) {
|
||||
revokeCapability(capability);
|
||||
}
|
||||
},
|
||||
|
||||
closeAll(): void {
|
||||
for (const capability of capabilities.values()) {
|
||||
revokeCapability(capability);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type NodeWorkerProjectionTransferService = ReturnType<
|
||||
typeof createNodeWorkerProjectionTransferService
|
||||
>;
|
||||
@@ -0,0 +1,554 @@
|
||||
import fs from "node:fs/promises";
|
||||
import http from "node:http";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
GATEWAY_CLIENT_IDS,
|
||||
GATEWAY_CLIENT_MODES,
|
||||
} from "../../../packages/gateway-protocol/src/client-info.js";
|
||||
import type { AuthorizedMemoryVirtualFileBroker } from "../../agents/memory-authorized-read-host.js";
|
||||
import {
|
||||
loadOrCreateDeviceIdentity,
|
||||
publicKeyRawBase64UrlFromPem,
|
||||
signDevicePayload,
|
||||
type DeviceIdentity,
|
||||
} from "../../infra/device-identity.js";
|
||||
import { NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE } from "../../infra/node-runner-inventory.js";
|
||||
import { NodeWorkerMemoryProjectionRuntime } from "../../node-host/node-worker-memory-projection.js";
|
||||
import {
|
||||
buildNodeWorkerMemoryProjectionRequestProofPayload,
|
||||
nodeWorkerMemoryProjectionTransferPath,
|
||||
NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER,
|
||||
NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER,
|
||||
NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER,
|
||||
type NodeWorkerMemoryProjection,
|
||||
type NodeWorkerMemoryProjectionBinding,
|
||||
} from "../../worker/node-memory-projection-protocol.js";
|
||||
import { NODE_WORKER_EXECUTION_CONTAINER_V1 } from "../../worker/node-supervisor-protocol.js";
|
||||
import type { NodeWorkerSupervisorNodeProof } from "../node-registry-private.js";
|
||||
import {
|
||||
createNodeWorkerProjectionTransferHttpCallback,
|
||||
handleNodeWorkerProjectionTransferHttpRequest,
|
||||
} from "./node-worker-projection-transfer-http.js";
|
||||
import { createNodeWorkerProjectionTransferService } from "./node-worker-projection-transfer-service.js";
|
||||
|
||||
function nodeProof(nodeId: string, connId = "conn-1"): NodeWorkerSupervisorNodeProof {
|
||||
return {
|
||||
nodeId,
|
||||
connId,
|
||||
pairingIdentity: "pairing-1",
|
||||
pairingGeneration: "generation-1",
|
||||
clientId: GATEWAY_CLIENT_IDS.NODE_HOST,
|
||||
clientMode: GATEWAY_CLIENT_MODES.NODE,
|
||||
protocolFeature: NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
|
||||
workerHost: {
|
||||
enabled: true,
|
||||
capacity: { total: 2, available: 2 },
|
||||
processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, memoryProjection: 1 },
|
||||
},
|
||||
commands: [],
|
||||
};
|
||||
}
|
||||
|
||||
function proof(params: {
|
||||
identity: DeviceIdentity;
|
||||
reference: string;
|
||||
binding: NodeWorkerMemoryProjectionBinding;
|
||||
nodeId?: string;
|
||||
signedAtMs?: number;
|
||||
signature?: string;
|
||||
}) {
|
||||
const nodeId = params.nodeId ?? params.identity.deviceId;
|
||||
const signedAtMs = params.signedAtMs ?? Date.now();
|
||||
const payload = buildNodeWorkerMemoryProjectionRequestProofPayload({
|
||||
reference: params.reference,
|
||||
binding: params.binding,
|
||||
nodeId,
|
||||
signedAtMs,
|
||||
});
|
||||
return {
|
||||
nodeId,
|
||||
signedAtMs,
|
||||
signature: params.signature ?? signDevicePayload(params.identity.privateKeyPem, payload),
|
||||
};
|
||||
}
|
||||
|
||||
async function requestProjection(params: {
|
||||
port: number;
|
||||
reference: string;
|
||||
binding: NodeWorkerMemoryProjectionBinding;
|
||||
identity: DeviceIdentity;
|
||||
nodeId?: string;
|
||||
signedAtMs?: number;
|
||||
signature?: string;
|
||||
}): Promise<{ status: number; body: string }> {
|
||||
const requestProof = proof(params);
|
||||
return await new Promise((resolve, reject) => {
|
||||
const request = http.request(
|
||||
{
|
||||
host: "127.0.0.1",
|
||||
port: params.port,
|
||||
path: nodeWorkerMemoryProjectionTransferPath(),
|
||||
method: "GET",
|
||||
headers: {
|
||||
authorization: `Bearer ${params.reference}`,
|
||||
[NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER]: requestProof.nodeId,
|
||||
[NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER]: String(requestProof.signedAtMs),
|
||||
[NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER]: requestProof.signature,
|
||||
},
|
||||
},
|
||||
(response) => {
|
||||
const chunks: Buffer[] = [];
|
||||
response.on("data", (chunk: Buffer) => chunks.push(chunk));
|
||||
response.on("end", () =>
|
||||
resolve({
|
||||
status: response.statusCode ?? 0,
|
||||
body: Buffer.concat(chunks).toString("utf8"),
|
||||
}),
|
||||
);
|
||||
},
|
||||
);
|
||||
request.once("error", reject);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
function createTransferService(params: {
|
||||
node: NodeWorkerSupervisorNodeProof;
|
||||
identity: DeviceIdentity;
|
||||
isNodeCurrent?: (node: NodeWorkerSupervisorNodeProof) => boolean;
|
||||
generateToken?: (bytes: number) => string;
|
||||
}) {
|
||||
return createNodeWorkerProjectionTransferService({
|
||||
...((params.generateToken ? { generateToken: params.generateToken } : {}) as object),
|
||||
resolveNodePublicKey: async (candidate) =>
|
||||
candidate.nodeId === params.node.nodeId
|
||||
? publicKeyRawBase64UrlFromPem(params.identity.publicKeyPem)
|
||||
: undefined,
|
||||
isNodeCurrent: params.isNodeCurrent ?? (() => true),
|
||||
});
|
||||
}
|
||||
|
||||
function broker(expiresAt: string): AuthorizedMemoryVirtualFileBroker {
|
||||
const contents = new Map([["memory/MEMORY.md", "only this issued virtual view"]]);
|
||||
return {
|
||||
view: {
|
||||
version: 1,
|
||||
viewId: "view-1",
|
||||
planId: "plan-1",
|
||||
contextFingerprint: "context-1",
|
||||
revision: "revision-1",
|
||||
roots: [{ version: 1, mountHandle: "mount-1", virtualRoot: "memory", access: "read" }],
|
||||
files: [{ version: 1, mountHandle: "mount-1", virtualPath: "memory/MEMORY.md" }],
|
||||
expiresAt,
|
||||
},
|
||||
readFile: async (virtualPath) => contents.get(virtualPath),
|
||||
};
|
||||
}
|
||||
|
||||
describe("node worker memory projection transfer", () => {
|
||||
let root: string;
|
||||
let server: http.Server | undefined;
|
||||
let identity: DeviceIdentity;
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await fs.mkdtemp(
|
||||
path.join(await fs.realpath(os.tmpdir()), "openclaw-memory-projection-wire-"),
|
||||
);
|
||||
identity = loadOrCreateDeviceIdentity({ path: path.join(root, "node-identity.sqlite") });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (server) {
|
||||
await new Promise<void>((resolve) => server?.close(() => resolve()));
|
||||
server = undefined;
|
||||
}
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it.runIf(
|
||||
process.platform !== "win32" && typeof process.getuid === "function" && process.getuid() > 0,
|
||||
)("stages one authorized, immutable virtual-memory snapshot and rejects replay", async () => {
|
||||
const node = nodeProof(identity.deviceId);
|
||||
const service = createTransferService({
|
||||
node,
|
||||
identity,
|
||||
generateToken: () => "A".repeat(43),
|
||||
});
|
||||
const callback = createNodeWorkerProjectionTransferHttpCallback(service);
|
||||
server = http.createServer((req, res) => {
|
||||
void handleNodeWorkerProjectionTransferHttpRequest({
|
||||
req,
|
||||
res,
|
||||
clientIp: "127.0.0.1",
|
||||
callback,
|
||||
}).catch((error: unknown) =>
|
||||
res.destroy(error instanceof Error ? error : new Error(String(error))),
|
||||
);
|
||||
});
|
||||
await new Promise<void>((resolve) => server?.listen(0, "127.0.0.1", resolve));
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("test server did not bind a TCP port");
|
||||
}
|
||||
const projection = await service.prepare({
|
||||
node,
|
||||
broker: broker(new Date(Date.now() + 60_000).toISOString()),
|
||||
environmentId: "environment-1",
|
||||
sessionId: "session-1",
|
||||
ownerEpoch: 3,
|
||||
placementGeneration: 4,
|
||||
runId: "run-1",
|
||||
launchId: "launch-1",
|
||||
launchBinding: "a".repeat(64),
|
||||
isAuthorized: () => true,
|
||||
});
|
||||
const runtime = new NodeWorkerMemoryProjectionRuntime({
|
||||
root: path.join(root, "node-host"),
|
||||
deviceIdentity: identity,
|
||||
});
|
||||
const projectionIdentity = {
|
||||
gatewayNamespace: "gateway-1",
|
||||
launchId: "launch-1",
|
||||
planHash: "a".repeat(64),
|
||||
};
|
||||
const endpoint = { kind: "websocket" as const, url: `ws://127.0.0.1:${address.port}` };
|
||||
|
||||
const staged = await runtime.stage({ identity: projectionIdentity, projection, endpoint });
|
||||
|
||||
expect(await fs.readFile(path.join(staged, "memory", "MEMORY.md"), "utf8")).toBe(
|
||||
"only this issued virtual view",
|
||||
);
|
||||
expect((await fs.stat(path.join(staged, "memory", "MEMORY.md"))).mode & 0o777).toBe(0o400);
|
||||
await expect(
|
||||
runtime.stage({ identity: projectionIdentity, projection, endpoint }),
|
||||
).rejects.toThrow("projection transfer failed (404)");
|
||||
expect(await fs.readdir(path.join(root, "node-host", "memory-projections"))).toEqual([]);
|
||||
});
|
||||
|
||||
it.runIf(
|
||||
process.platform !== "win32" && typeof process.getuid === "function" && process.getuid() > 0,
|
||||
)("removes a staged projection when cancellation races its final rename", async () => {
|
||||
const node = nodeProof(identity.deviceId);
|
||||
const service = createTransferService({ node, identity });
|
||||
const callback = createNodeWorkerProjectionTransferHttpCallback(service);
|
||||
server = http.createServer((req, res) => {
|
||||
void handleNodeWorkerProjectionTransferHttpRequest({
|
||||
req,
|
||||
res,
|
||||
clientIp: "127.0.0.1",
|
||||
callback,
|
||||
}).catch((error: unknown) =>
|
||||
res.destroy(error instanceof Error ? error : new Error(String(error))),
|
||||
);
|
||||
});
|
||||
await new Promise<void>((resolve) => server?.listen(0, "127.0.0.1", resolve));
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("test server did not bind a TCP port");
|
||||
}
|
||||
const projection = await service.prepare({
|
||||
node,
|
||||
broker: broker(new Date(Date.now() + 60_000).toISOString()),
|
||||
environmentId: "environment-1",
|
||||
sessionId: "session-1",
|
||||
ownerEpoch: 3,
|
||||
placementGeneration: 4,
|
||||
runId: "run-1",
|
||||
launchId: "launch-cancelled-before-rename",
|
||||
launchBinding: "a".repeat(64),
|
||||
isAuthorized: () => true,
|
||||
});
|
||||
const runtime = new NodeWorkerMemoryProjectionRuntime({
|
||||
root: path.join(root, "node-host"),
|
||||
deviceIdentity: identity,
|
||||
});
|
||||
const projectionIdentity = {
|
||||
gatewayNamespace: "gateway-1",
|
||||
launchId: "launch-cancelled-before-rename",
|
||||
planHash: "b".repeat(64),
|
||||
};
|
||||
const controller = new AbortController();
|
||||
const rename = fs.rename;
|
||||
const renameSpy = vi.spyOn(fs, "rename").mockImplementation(async (...args) => {
|
||||
controller.abort(new Error("projection revoked before process start"));
|
||||
await rename(...args);
|
||||
});
|
||||
try {
|
||||
await expect(
|
||||
runtime.stage({
|
||||
identity: projectionIdentity,
|
||||
projection,
|
||||
endpoint: { kind: "websocket", url: `ws://127.0.0.1:${address.port}` },
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toThrow("projection revoked before process start");
|
||||
} finally {
|
||||
renameSpy.mockRestore();
|
||||
}
|
||||
expect(await fs.readdir(path.join(root, "node-host", "memory-projections"))).toEqual([]);
|
||||
});
|
||||
|
||||
it("never overwrites a live capability when the token source collides", async () => {
|
||||
const node = nodeProof(identity.deviceId);
|
||||
const service = createTransferService({
|
||||
node,
|
||||
identity,
|
||||
generateToken: () => "B".repeat(43),
|
||||
});
|
||||
let authorized = true;
|
||||
const params = {
|
||||
node,
|
||||
broker: broker(new Date(Date.now() + 60_000).toISOString()),
|
||||
environmentId: "environment-1",
|
||||
sessionId: "session-1",
|
||||
ownerEpoch: 3,
|
||||
placementGeneration: 4,
|
||||
runId: "run-1",
|
||||
launchId: "launch-1",
|
||||
launchBinding: "a".repeat(64),
|
||||
isAuthorized: () => authorized,
|
||||
};
|
||||
|
||||
const prepared = await service.prepare(params);
|
||||
authorized = false;
|
||||
await expect(
|
||||
service.authorize(
|
||||
prepared.reference,
|
||||
proof({ identity, reference: prepared.reference, binding: prepared.binding }),
|
||||
),
|
||||
).resolves.toBeUndefined();
|
||||
authorized = true;
|
||||
await expect(service.prepare({ ...params, launchId: "launch-2" })).rejects.toThrow(
|
||||
"token collision",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a wrong node proof without consuming the intended node capability", async () => {
|
||||
const node = nodeProof(identity.deviceId);
|
||||
const otherIdentity = loadOrCreateDeviceIdentity({
|
||||
path: path.join(root, "other-node.sqlite"),
|
||||
});
|
||||
const service = createTransferService({ node, identity });
|
||||
const callback = createNodeWorkerProjectionTransferHttpCallback(service);
|
||||
server = http.createServer((req, res) => {
|
||||
void handleNodeWorkerProjectionTransferHttpRequest({
|
||||
req,
|
||||
res,
|
||||
clientIp: "127.0.0.1",
|
||||
callback,
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve) => server?.listen(0, "127.0.0.1", resolve));
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("test server did not bind a TCP port");
|
||||
}
|
||||
const projection = await service.prepare({
|
||||
node,
|
||||
broker: broker(new Date(Date.now() + 60_000).toISOString()),
|
||||
environmentId: "environment-1",
|
||||
sessionId: "session-1",
|
||||
ownerEpoch: 3,
|
||||
placementGeneration: 4,
|
||||
runId: "run-1",
|
||||
launchId: "launch-1",
|
||||
launchBinding: "a".repeat(64),
|
||||
isAuthorized: () => true,
|
||||
});
|
||||
|
||||
await expect(
|
||||
requestProjection({
|
||||
port: address.port,
|
||||
reference: projection.reference,
|
||||
binding: projection.binding,
|
||||
identity: otherIdentity,
|
||||
}),
|
||||
).resolves.toMatchObject({ status: 404 });
|
||||
await expect(
|
||||
requestProjection({
|
||||
port: address.port,
|
||||
reference: projection.reference,
|
||||
binding: projection.binding,
|
||||
identity,
|
||||
}),
|
||||
).resolves.toMatchObject({ status: 200 });
|
||||
});
|
||||
|
||||
it("rejects a re-signed projection binding swap without consuming the issued view", async () => {
|
||||
const node = nodeProof(identity.deviceId);
|
||||
const service = createTransferService({ node, identity });
|
||||
const callback = createNodeWorkerProjectionTransferHttpCallback(service);
|
||||
server = http.createServer((req, res) => {
|
||||
void handleNodeWorkerProjectionTransferHttpRequest({
|
||||
req,
|
||||
res,
|
||||
clientIp: "127.0.0.1",
|
||||
callback,
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve) => server?.listen(0, "127.0.0.1", resolve));
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("test server did not bind a TCP port");
|
||||
}
|
||||
const projection = await service.prepare({
|
||||
node,
|
||||
broker: broker(new Date(Date.now() + 60_000).toISOString()),
|
||||
environmentId: "environment-1",
|
||||
sessionId: "session-1",
|
||||
ownerEpoch: 3,
|
||||
placementGeneration: 4,
|
||||
runId: "run-1",
|
||||
launchId: "launch-1",
|
||||
launchBinding: "a".repeat(64),
|
||||
isAuthorized: () => true,
|
||||
});
|
||||
|
||||
await expect(
|
||||
requestProjection({
|
||||
port: address.port,
|
||||
reference: projection.reference,
|
||||
binding: { ...projection.binding, launch: "d".repeat(64) },
|
||||
identity,
|
||||
}),
|
||||
).resolves.toMatchObject({ status: 404 });
|
||||
await expect(
|
||||
requestProjection({
|
||||
port: address.port,
|
||||
reference: projection.reference,
|
||||
binding: projection.binding,
|
||||
identity,
|
||||
}),
|
||||
).resolves.toMatchObject({ status: 200 });
|
||||
});
|
||||
|
||||
it("rejects stale connections and invalid proofs without consuming the current capability", async () => {
|
||||
const node = nodeProof(identity.deviceId);
|
||||
let currentNode = node;
|
||||
const service = createTransferService({
|
||||
node,
|
||||
identity,
|
||||
isNodeCurrent: (candidate) =>
|
||||
candidate.nodeId === currentNode.nodeId &&
|
||||
candidate.connId === currentNode.connId &&
|
||||
candidate.pairingGeneration === currentNode.pairingGeneration,
|
||||
});
|
||||
const callback = createNodeWorkerProjectionTransferHttpCallback(service);
|
||||
server = http.createServer((req, res) => {
|
||||
void handleNodeWorkerProjectionTransferHttpRequest({
|
||||
req,
|
||||
res,
|
||||
clientIp: "127.0.0.1",
|
||||
callback,
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve) => server?.listen(0, "127.0.0.1", resolve));
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("test server did not bind a TCP port");
|
||||
}
|
||||
const prepare = async (launchId: string) =>
|
||||
await service.prepare({
|
||||
node,
|
||||
broker: broker(new Date(Date.now() + 60_000).toISOString()),
|
||||
environmentId: "environment-1",
|
||||
sessionId: "session-1",
|
||||
ownerEpoch: 3,
|
||||
placementGeneration: 4,
|
||||
runId: "run-1",
|
||||
launchId,
|
||||
launchBinding: "a".repeat(64),
|
||||
isAuthorized: () => true,
|
||||
});
|
||||
|
||||
const replaced = await prepare("launch-replaced");
|
||||
currentNode = nodeProof(identity.deviceId, "conn-replaced");
|
||||
await expect(
|
||||
requestProjection({
|
||||
port: address.port,
|
||||
reference: replaced.reference,
|
||||
binding: replaced.binding,
|
||||
identity,
|
||||
}),
|
||||
).resolves.toMatchObject({ status: 404 });
|
||||
currentNode = node;
|
||||
await expect(
|
||||
requestProjection({
|
||||
port: address.port,
|
||||
reference: replaced.reference,
|
||||
binding: replaced.binding,
|
||||
identity,
|
||||
}),
|
||||
).resolves.toMatchObject({ status: 200 });
|
||||
|
||||
const invalid = await prepare("launch-invalid");
|
||||
await expect(
|
||||
requestProjection({
|
||||
port: address.port,
|
||||
reference: invalid.reference,
|
||||
binding: invalid.binding,
|
||||
identity,
|
||||
signature: "A".repeat(86),
|
||||
}),
|
||||
).resolves.toMatchObject({ status: 404 });
|
||||
await expect(
|
||||
requestProjection({
|
||||
port: address.port,
|
||||
reference: invalid.reference,
|
||||
binding: invalid.binding,
|
||||
identity,
|
||||
}),
|
||||
).resolves.toMatchObject({ status: 200 });
|
||||
|
||||
const expired = await prepare("launch-expired");
|
||||
await expect(
|
||||
requestProjection({
|
||||
port: address.port,
|
||||
reference: expired.reference,
|
||||
binding: expired.binding,
|
||||
identity,
|
||||
signedAtMs: Date.now() - 120_001,
|
||||
}),
|
||||
).resolves.toMatchObject({ status: 404 });
|
||||
await expect(
|
||||
requestProjection({
|
||||
port: address.port,
|
||||
reference: expired.reference,
|
||||
binding: expired.binding,
|
||||
identity,
|
||||
}),
|
||||
).resolves.toMatchObject({ status: 200 });
|
||||
});
|
||||
|
||||
it.runIf(
|
||||
process.platform !== "win32" && typeof process.getuid === "function" && process.getuid() > 0,
|
||||
)(
|
||||
"refuses a plaintext non-loopback projection transport before sending a capability",
|
||||
async () => {
|
||||
const runtime = new NodeWorkerMemoryProjectionRuntime({
|
||||
root: path.join(root, "node-host"),
|
||||
deviceIdentity: identity,
|
||||
});
|
||||
const projection: NodeWorkerMemoryProjection = {
|
||||
version: 1,
|
||||
reference: "C".repeat(43),
|
||||
binding: { launch: "a".repeat(64), authorization: "b".repeat(64) },
|
||||
expiresAtMs: Date.now() + 60_000,
|
||||
};
|
||||
|
||||
await expect(
|
||||
runtime.stage({
|
||||
identity: {
|
||||
gatewayNamespace: "gateway-1",
|
||||
launchId: "launch-1",
|
||||
planHash: "a".repeat(64),
|
||||
},
|
||||
projection,
|
||||
endpoint: { kind: "websocket", url: "ws://192.0.2.1:18789" },
|
||||
}),
|
||||
).rejects.toThrow("requires wss://");
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -14,7 +14,11 @@ import { createDeferred } from "../../../test/helpers/promise.js";
|
||||
import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js";
|
||||
import { NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE } from "../../infra/node-runner-inventory.js";
|
||||
import { parseWorkerLaunchPlan } from "../../worker/launch-descriptor.js";
|
||||
import type { NodeWorkerSupervisorReceipt } from "../../worker/node-supervisor-protocol.js";
|
||||
import {
|
||||
NODE_WORKER_EXECUTION_CONTAINER_V1,
|
||||
NODE_WORKER_EXECUTION_HOST_V1,
|
||||
type NodeWorkerSupervisorReceipt,
|
||||
} from "../../worker/node-supervisor-protocol.js";
|
||||
import type { NodeWorkerWorkspaceExecInput } from "../../worker/node-workspace-protocol.js";
|
||||
import {
|
||||
NODE_WORKSPACE_TRANSFER_ERROR_CODE,
|
||||
@@ -24,6 +28,7 @@ import type { NodeWorkerSupervisorTransport } from "../node-registry-private.js"
|
||||
import type { createDeviceWorkerRuntime } from "./device-provider.js";
|
||||
import { createNodeWorkerTunnelManager } from "./node-worker-tunnel.js";
|
||||
import type { NodeWorkspaceTransferService } from "./node-workspace-transfer-service.js";
|
||||
import type { NodeWorkerProjectionTransferService } from "./node-worker-projection-transfer-service.js";
|
||||
import { sameWorkerSessionTurnClaim } from "./placement-record.js";
|
||||
import type { WorkerEnvironmentRecord } from "./store.js";
|
||||
import { serializeWorkerWorkspaceManifest } from "./workspace-manifest.js";
|
||||
@@ -96,6 +101,7 @@ function plan() {
|
||||
},
|
||||
assignment: {
|
||||
agentId: "main",
|
||||
memoryReadEnforced: false,
|
||||
operationalRunInstance: { instanceId: "instance-1", runId: "run-1" },
|
||||
agentRuntimeIdentityToken: "runtime-token",
|
||||
runId: "run-1",
|
||||
@@ -160,7 +166,101 @@ function workspaceTransfer(): NodeWorkspaceTransferService {
|
||||
} as unknown as NodeWorkspaceTransferService;
|
||||
}
|
||||
|
||||
function projectionTransfer(): NodeWorkerProjectionTransferService {
|
||||
return {
|
||||
prepare: vi.fn(async () => ({
|
||||
version: 1 as const,
|
||||
reference: "a".repeat(43),
|
||||
binding: { launch: "b".repeat(64), authorization: "c".repeat(64) },
|
||||
expiresAtMs: Date.now() + 60_000,
|
||||
})),
|
||||
} as unknown as NodeWorkerProjectionTransferService;
|
||||
}
|
||||
|
||||
describe("node worker tunnel manager", () => {
|
||||
it.each([
|
||||
[false, NODE_WORKER_EXECUTION_HOST_V1],
|
||||
[true, NODE_WORKER_EXECUTION_CONTAINER_V1],
|
||||
] as const)(
|
||||
"derives %s memory launches as %s execution",
|
||||
async (memoryReadEnforced, executionKind) => {
|
||||
const launchNodeWorker = vi.fn<NodeWorkerLaunch>(async (request) => ({
|
||||
launchId: request.input.launchId,
|
||||
planHash: "b".repeat(64),
|
||||
environmentId: request.input.descriptor.admission.environmentId,
|
||||
sessionId: request.input.descriptor.admission.sessionId,
|
||||
ownerEpoch: request.input.descriptor.admission.ownerEpoch,
|
||||
placementGeneration: request.input.placementGeneration,
|
||||
runId: request.input.descriptor.assignment.runId,
|
||||
state: "completed",
|
||||
resultJson: "{}",
|
||||
}));
|
||||
const manager = createNodeWorkerTunnelManager({
|
||||
gatewayDeviceId: "gateway-device-1",
|
||||
getEnvironment: () => environment(),
|
||||
getTransport: transport,
|
||||
launchNodeWorker,
|
||||
validateWorkerTurn: () => true,
|
||||
projectionTransfer: projectionTransfer(),
|
||||
workspaceTransfer: workspaceTransfer(),
|
||||
});
|
||||
const launchPlan = plan();
|
||||
launchPlan.assignment.memoryReadEnforced = memoryReadEnforced;
|
||||
const handle = await manager.start(startRequest());
|
||||
|
||||
await handle.launchTurn({
|
||||
plan: launchPlan,
|
||||
turnClaim: turnClaim(),
|
||||
...(memoryReadEnforced ? { memoryProjection: {} as never } : {}),
|
||||
});
|
||||
|
||||
expect(launchNodeWorker).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
input: expect.objectContaining({ execution: { kind: executionKind } }),
|
||||
}),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("forwards transport dispatch and execution readiness as distinct signals", async () => {
|
||||
const events: string[] = [];
|
||||
const launchNodeWorker = vi.fn<NodeWorkerLaunch>(async (request) => {
|
||||
request.onDispatchReady?.();
|
||||
expect(events).toEqual(["dispatch"]);
|
||||
request.onExecutionReady?.();
|
||||
return {
|
||||
launchId: request.input.launchId,
|
||||
planHash: "b".repeat(64),
|
||||
environmentId: request.input.descriptor.admission.environmentId,
|
||||
sessionId: request.input.descriptor.admission.sessionId,
|
||||
ownerEpoch: request.input.descriptor.admission.ownerEpoch,
|
||||
placementGeneration: request.input.placementGeneration,
|
||||
runId: request.input.descriptor.assignment.runId,
|
||||
state: "completed",
|
||||
resultJson: "{}",
|
||||
};
|
||||
});
|
||||
const manager = createNodeWorkerTunnelManager({
|
||||
gatewayDeviceId: "gateway-device-1",
|
||||
getEnvironment: () => environment(),
|
||||
getTransport: transport,
|
||||
launchNodeWorker,
|
||||
validateWorkerTurn: () => true,
|
||||
projectionTransfer: projectionTransfer(),
|
||||
workspaceTransfer: workspaceTransfer(),
|
||||
});
|
||||
const handle = await manager.start(startRequest());
|
||||
|
||||
await handle.launchTurn({
|
||||
plan: plan(),
|
||||
turnClaim: turnClaim(),
|
||||
onDispatchReady: () => events.push("dispatch"),
|
||||
onExecutionReady: () => events.push("execution"),
|
||||
});
|
||||
|
||||
expect(events).toEqual(["dispatch", "execution"]);
|
||||
});
|
||||
|
||||
it("revalidates the exact claim when a same-run replacement launches", async () => {
|
||||
const record = environment();
|
||||
let currentClaim = turnClaim();
|
||||
@@ -181,9 +281,11 @@ describe("node worker tunnel manager", () => {
|
||||
runId: request.input.descriptor.assignment.runId,
|
||||
state: "cancelled",
|
||||
errorText: "test launch finished",
|
||||
executionStarted: false,
|
||||
};
|
||||
}),
|
||||
validateWorkerTurn: (claim) => sameWorkerSessionTurnClaim(claim, currentClaim),
|
||||
projectionTransfer: projectionTransfer(),
|
||||
workspaceTransfer: workspaceTransfer(),
|
||||
});
|
||||
const handle = await manager.start(startRequest());
|
||||
@@ -214,8 +316,10 @@ describe("node worker tunnel manager", () => {
|
||||
runId: request.input.descriptor.assignment.runId,
|
||||
state: "cancelled",
|
||||
errorText,
|
||||
executionStarted: false,
|
||||
})),
|
||||
validateWorkerTurn: () => true,
|
||||
projectionTransfer: projectionTransfer(),
|
||||
workspaceTransfer: workspaceTransfer(),
|
||||
});
|
||||
const handle = await manager.start(startRequest());
|
||||
@@ -237,6 +341,7 @@ describe("node worker tunnel manager", () => {
|
||||
getTransport: transport,
|
||||
launchNodeWorker: vi.fn(),
|
||||
validateWorkerTurn: () => true,
|
||||
projectionTransfer: projectionTransfer(),
|
||||
workspaceTransfer: workspaceTransfer(),
|
||||
});
|
||||
|
||||
@@ -257,6 +362,7 @@ describe("node worker tunnel manager", () => {
|
||||
getTransport: transport,
|
||||
launchNodeWorker: vi.fn(),
|
||||
validateWorkerTurn: () => true,
|
||||
projectionTransfer: projectionTransfer(),
|
||||
workspaceTransfer: workspaceTransfer(),
|
||||
});
|
||||
manager.bindWorkspaceBindingResolver(resolveWorkspaceBinding);
|
||||
@@ -289,6 +395,7 @@ describe("node worker tunnel manager", () => {
|
||||
getTransport: transport,
|
||||
launchNodeWorker: vi.fn(),
|
||||
validateWorkerTurn: () => true,
|
||||
projectionTransfer: projectionTransfer(),
|
||||
workspaceTransfer: transfer,
|
||||
});
|
||||
manager.bindWorkspaceBindingResolver(resolveWorkspaceBinding);
|
||||
@@ -319,6 +426,7 @@ describe("node worker tunnel manager", () => {
|
||||
getTransport: transport,
|
||||
launchNodeWorker: vi.fn(),
|
||||
validateWorkerTurn: () => true,
|
||||
projectionTransfer: projectionTransfer(),
|
||||
workspaceTransfer: transfer,
|
||||
});
|
||||
manager.bindWorkspaceBindingResolver(async () => {
|
||||
@@ -378,6 +486,7 @@ describe("node worker tunnel manager", () => {
|
||||
getTransport: () => nodeTransport,
|
||||
launchNodeWorker: vi.fn(),
|
||||
validateWorkerTurn: () => true,
|
||||
projectionTransfer: projectionTransfer(),
|
||||
workspaceTransfer: transfer,
|
||||
});
|
||||
manager.bindWorkspaceBindingResolver(async () => ({
|
||||
@@ -472,6 +581,7 @@ describe("node worker tunnel manager", () => {
|
||||
},
|
||||
launchNodeWorker: vi.fn(),
|
||||
validateWorkerTurn: () => true,
|
||||
projectionTransfer: projectionTransfer(),
|
||||
workspaceTransfer: transfer,
|
||||
});
|
||||
const resolveWorkspaceBinding = vi.fn(async () => ({
|
||||
@@ -570,6 +680,7 @@ describe("node worker tunnel manager", () => {
|
||||
getTransport: () => nodeTransport,
|
||||
launchNodeWorker: vi.fn(),
|
||||
validateWorkerTurn: () => true,
|
||||
projectionTransfer: projectionTransfer(),
|
||||
workspaceTransfer: transfer,
|
||||
});
|
||||
manager.bindWorkspaceBindingResolver(async () => ({
|
||||
@@ -629,6 +740,7 @@ describe("node worker tunnel manager", () => {
|
||||
getTransport: () => nodeTransport,
|
||||
launchNodeWorker: vi.fn(),
|
||||
validateWorkerTurn: () => true,
|
||||
projectionTransfer: projectionTransfer(),
|
||||
workspaceTransfer: transfer,
|
||||
});
|
||||
const handle = await manager.start(startRequest());
|
||||
@@ -668,6 +780,7 @@ describe("node worker tunnel manager", () => {
|
||||
runId: request.input.descriptor.assignment.runId,
|
||||
state: "cancelled",
|
||||
errorText: "node worker cancelled",
|
||||
executionStarted: false,
|
||||
});
|
||||
});
|
||||
},
|
||||
@@ -681,6 +794,7 @@ describe("node worker tunnel manager", () => {
|
||||
getTransport: transport,
|
||||
launchNodeWorker,
|
||||
validateWorkerTurn: () => true,
|
||||
projectionTransfer: projectionTransfer(),
|
||||
workspaceTransfer: workspaceTransfer(),
|
||||
});
|
||||
const first = await manager.start(startRequest());
|
||||
@@ -729,6 +843,7 @@ describe("node worker tunnel manager", () => {
|
||||
runId: request.input.descriptor.assignment.runId,
|
||||
state: "cancelled",
|
||||
errorText: "node worker cancelled",
|
||||
executionStarted: true,
|
||||
});
|
||||
},
|
||||
{ once: true },
|
||||
@@ -742,6 +857,7 @@ describe("node worker tunnel manager", () => {
|
||||
getTransport: transport,
|
||||
launchNodeWorker,
|
||||
validateWorkerTurn: () => true,
|
||||
projectionTransfer: projectionTransfer(),
|
||||
workspaceTransfer: workspaceTransfer(),
|
||||
});
|
||||
const handle = await manager.start(startRequest());
|
||||
@@ -818,6 +934,7 @@ describe("node worker tunnel manager", () => {
|
||||
getTransport: () => nodeTransport,
|
||||
launchNodeWorker: vi.fn(),
|
||||
validateWorkerTurn: () => true,
|
||||
projectionTransfer: projectionTransfer(),
|
||||
workspaceTransfer: transfer,
|
||||
});
|
||||
const handle = await manager.start(startRequest());
|
||||
@@ -886,6 +1003,7 @@ describe("node worker tunnel manager", () => {
|
||||
getTransport: () => nodeTransport,
|
||||
launchNodeWorker: vi.fn(),
|
||||
validateWorkerTurn: () => true,
|
||||
projectionTransfer: projectionTransfer(),
|
||||
workspaceTransfer: transfer,
|
||||
});
|
||||
const handle = await manager.start(startRequest());
|
||||
|
||||
@@ -6,7 +6,14 @@ import { NODE_WORKER_WORKSPACE_EXEC_COMMAND } from "../../infra/node-commands.js
|
||||
import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
import type { SpawnResult } from "../../process/exec.js";
|
||||
import { createDeferredCore, type Deferred } from "../../shared/deferred.js";
|
||||
import type { NodeWorkerSupervisorReceipt } from "../../worker/node-supervisor-protocol.js";
|
||||
import {
|
||||
NODE_WORKER_EXECUTION_CONTAINER_V1,
|
||||
NODE_WORKER_EXECUTION_HOST_V1,
|
||||
nodeWorkerMemoryProjectionLaunchBinding,
|
||||
type NodeWorkerExecution,
|
||||
type NodeWorkerSupervisorReceipt,
|
||||
} from "../../worker/node-supervisor-protocol.js";
|
||||
import type { NodeWorkerMemoryProjection } from "../../worker/node-memory-projection-protocol.js";
|
||||
import {
|
||||
parseNodeWorkerWorkspaceExecResult,
|
||||
type NodeWorkerWorkspaceExecInput,
|
||||
@@ -27,6 +34,7 @@ import {
|
||||
recordNodeSyncPath,
|
||||
} from "./node-worker-workspace-fallback.js";
|
||||
import type { NodeWorkspaceTransferService } from "./node-workspace-transfer-service.js";
|
||||
import type { NodeWorkerProjectionTransferService } from "./node-worker-projection-transfer-service.js";
|
||||
import type { WorkerSessionTurnClaim } from "./placement-record.js";
|
||||
import type { WorkerEnvironmentRecord } from "./store.js";
|
||||
import type {
|
||||
@@ -74,12 +82,18 @@ type NodeWorkerLaunch = (request: {
|
||||
expectedBundleHash: string;
|
||||
placementGeneration: number;
|
||||
descriptor: WorkerTurnLaunchRequest["plan"];
|
||||
execution: NodeWorkerExecution;
|
||||
memoryProjection?: NodeWorkerMemoryProjection;
|
||||
};
|
||||
prepareMemoryProjection?: (
|
||||
node: NodeWorkerSupervisorNodeProof,
|
||||
) => Promise<NodeWorkerMemoryProjection>;
|
||||
isDispatchAuthorized: () => boolean;
|
||||
isCancellationAuthorized: () => boolean;
|
||||
timeoutMs: number;
|
||||
signal?: AbortSignal;
|
||||
onDispatchReady?: () => void;
|
||||
onExecutionReady?: () => void;
|
||||
}) => Promise<TerminalNodeWorkerSupervisorReceipt>;
|
||||
|
||||
type NodeWorkerWorkspaceBinding = {
|
||||
@@ -100,6 +114,7 @@ type NodeWorkerTunnelManagerOptions = {
|
||||
getTransport: () => NodeWorkerSupervisorTransport | undefined;
|
||||
launchNodeWorker: NodeWorkerLaunch;
|
||||
validateWorkerTurn: (claim: WorkerSessionTurnClaim) => boolean;
|
||||
projectionTransfer: NodeWorkerProjectionTransferService;
|
||||
workspaceTransfer: NodeWorkspaceTransferService;
|
||||
};
|
||||
|
||||
@@ -529,22 +544,50 @@ export function createNodeWorkerTunnelManager(options: NodeWorkerTunnelManagerOp
|
||||
claim.sessionId === plan.admission.sessionId &&
|
||||
claim.runId === plan.assignment.runId &&
|
||||
options.validateWorkerTurn(claim);
|
||||
const execution: NodeWorkerExecution = plan.assignment.memoryReadEnforced
|
||||
? { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 }
|
||||
: { kind: NODE_WORKER_EXECUTION_HOST_V1 };
|
||||
if (execution.kind === NODE_WORKER_EXECUTION_CONTAINER_V1 && !request.memoryProjection) {
|
||||
throw new Error("enforced node worker launch is missing its authorized memory projection");
|
||||
}
|
||||
const launchSignal = request.signal
|
||||
? AbortSignal.any([entry.abortController.signal, request.signal])
|
||||
: entry.abortController.signal;
|
||||
const input = {
|
||||
launchId: plan.assignment.turnId,
|
||||
gatewayNamespace,
|
||||
expectedBundleHash: entry.expectedBuild.bundleHash,
|
||||
placementGeneration: claim.placementGeneration,
|
||||
descriptor: plan,
|
||||
execution,
|
||||
};
|
||||
const operation = options.launchNodeWorker({
|
||||
deviceId: entry.deviceId,
|
||||
input: {
|
||||
launchId: plan.assignment.turnId,
|
||||
gatewayNamespace,
|
||||
expectedBundleHash: entry.expectedBuild.bundleHash,
|
||||
placementGeneration: claim.placementGeneration,
|
||||
descriptor: plan,
|
||||
},
|
||||
input,
|
||||
...(request.memoryProjection
|
||||
? {
|
||||
prepareMemoryProjection: async (node) =>
|
||||
await options.projectionTransfer.prepare({
|
||||
node,
|
||||
broker: request.memoryProjection!,
|
||||
environmentId: entry.environmentId,
|
||||
sessionId: entry.sessionId,
|
||||
ownerEpoch: entry.ownerEpoch,
|
||||
placementGeneration: claim.placementGeneration,
|
||||
runId: plan.assignment.runId,
|
||||
launchId: plan.assignment.turnId,
|
||||
launchBinding: nodeWorkerMemoryProjectionLaunchBinding(input),
|
||||
isAuthorized: isDispatchAuthorized,
|
||||
signal: launchSignal,
|
||||
}),
|
||||
}
|
||||
: {}),
|
||||
isDispatchAuthorized,
|
||||
isCancellationAuthorized: () => hasDurableBinding(entry as NodeTunnelEntry),
|
||||
timeoutMs: request.timeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS,
|
||||
onDispatchReady: request.onDispatchReady,
|
||||
signal: request.signal
|
||||
? AbortSignal.any([entry.abortController.signal, request.signal])
|
||||
: entry.abortController.signal,
|
||||
onExecutionReady: request.onExecutionReady,
|
||||
signal: launchSignal,
|
||||
});
|
||||
entry.launchTasks.add(operation);
|
||||
try {
|
||||
|
||||
@@ -339,6 +339,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
|
||||
const searchMemory = async (
|
||||
identity: WorkerConnectionIdentity,
|
||||
request: WorkerMemorySearchParams,
|
||||
signal?: AbortSignal,
|
||||
): Promise<WorkerMemoryServiceResult> => {
|
||||
const binding = validateMemoryRequest(identity);
|
||||
if (!binding.ok) {
|
||||
@@ -351,6 +352,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
|
||||
const result = await host.host.search({
|
||||
query: request.query,
|
||||
...(request.limit === undefined ? {} : { limit: request.limit }),
|
||||
...(signal ? { signal } : {}),
|
||||
});
|
||||
const current = validateMemoryRequest(identity);
|
||||
if (!current.ok) {
|
||||
@@ -364,6 +366,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
|
||||
const readMemory = async (
|
||||
identity: WorkerConnectionIdentity,
|
||||
request: WorkerMemoryReadParams,
|
||||
signal?: AbortSignal,
|
||||
): Promise<WorkerMemoryServiceResult> => {
|
||||
const binding = validateMemoryRequest(identity);
|
||||
if (!binding.ok) {
|
||||
@@ -377,6 +380,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
|
||||
handleId: request.handleId,
|
||||
...(request.from === undefined ? {} : { from: request.from }),
|
||||
...(request.lines === undefined ? {} : { lines: request.lines }),
|
||||
...(signal ? { signal } : {}),
|
||||
});
|
||||
const current = validateMemoryRequest(identity);
|
||||
if (!current.ok) {
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { WorkerTunnelStatus } from "@openclaw/gateway-protocol";
|
||||
import { NODE_WORKER_CAPACITY_EXHAUSTED_ERROR_CODE } from "../../infra/node-commands.js";
|
||||
import type { SpawnResult } from "../../process/exec.js";
|
||||
import type { WorkerLaunchPlan } from "../../worker/launch-descriptor.js";
|
||||
import type { AuthorizedMemoryVirtualFileBroker } from "../../agents/memory-authorized-read-host.js";
|
||||
import type { NodeWorkerWorkspaceTransferInput } from "../../worker/node-workspace-transfer-protocol.js";
|
||||
import type { WorkerSessionTurnClaim } from "./placement-record.js";
|
||||
import type {
|
||||
@@ -102,9 +103,13 @@ export type WorkerWorkspaceQuiescence = {
|
||||
export type WorkerTurnLaunchRequest = {
|
||||
plan: WorkerLaunchPlan;
|
||||
turnClaim: WorkerSessionTurnClaim;
|
||||
/** Gateway-only broker capability. It must never enter the serialized worker plan. */
|
||||
memoryProjection?: AuthorizedMemoryVirtualFileBroker;
|
||||
timeoutMs?: number;
|
||||
signal?: AbortSignal;
|
||||
onDispatchReady?: () => void;
|
||||
/** Fires only after the node has durably attested real worker execution. */
|
||||
onExecutionReady?: () => void;
|
||||
};
|
||||
|
||||
export type WorkerWorkspaceTunnelHandle = {
|
||||
|
||||
@@ -30,6 +30,7 @@ describe("worker turn launcher terminal results", () => {
|
||||
|
||||
it("requests immediate recovery when reconciliation fails after worker finishing", async () => {
|
||||
seedActivePlacement();
|
||||
const phases: string[] = [];
|
||||
const destroy = vi.fn(async () => attachedEnvironment());
|
||||
const tunnelFailure = new NodeWorkerWorkspaceTransferError(
|
||||
"workspace-transfer-failed: gateway TLS fingerprint mismatch",
|
||||
@@ -44,6 +45,9 @@ describe("worker turn launcher terminal results", () => {
|
||||
runWorkspaceCommand: vi.fn(),
|
||||
launchTurn: vi.fn(async (request): Promise<SpawnResult> => {
|
||||
request.onDispatchReady?.();
|
||||
expect(phases).not.toContain("process_spawned");
|
||||
request.onExecutionReady?.();
|
||||
expect(phases.filter((phase) => phase === "process_spawned")).toHaveLength(1);
|
||||
const completed = openSessionManager();
|
||||
const leafId = completed.appendMessage(
|
||||
makeAgentAssistantMessage({
|
||||
@@ -106,7 +110,10 @@ describe("worker turn launcher terminal results", () => {
|
||||
agentId: "main",
|
||||
runId: "run-reconcile-tunnel-loss",
|
||||
},
|
||||
turn("run-reconcile-tunnel-loss"),
|
||||
{
|
||||
...turn("run-reconcile-tunnel-loss"),
|
||||
onExecutionPhase: ({ phase }) => phases.push(phase),
|
||||
},
|
||||
async () => ({ meta: { durationMs: 1 } }),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
@@ -115,6 +122,7 @@ describe("worker turn launcher terminal results", () => {
|
||||
});
|
||||
|
||||
expect(reconcileActivePlacement).toHaveBeenCalledWith(ENVIRONMENT_ID);
|
||||
expect(phases.filter((phase) => phase === "process_spawned")).toEqual(["process_spawned"]);
|
||||
expect(placements.get(SESSION_ID)).toMatchObject({ state: "failed", turnClaim: null });
|
||||
expect(placements.listPendingWorkspaceResults()).toHaveLength(0);
|
||||
expect(destroy).not.toHaveBeenCalled();
|
||||
|
||||
@@ -13,6 +13,8 @@ import {
|
||||
import { createChatRunState } from "../server-chat-state.js";
|
||||
import { prepareSessionArchiveLifecycle } from "../server-methods/sessions-archive-lifecycle.js";
|
||||
import type { GatewayRequestContext } from "../server-methods/types.js";
|
||||
import { NODE_WORKER_EXECUTION_CONTAINER_V1 } from "../../worker/node-supervisor-protocol.js";
|
||||
import { bindDeviceWorkerExecutionEligibility } from "./device-provider.js";
|
||||
import type { WorkerTunnelHandle } from "./tunnel-contract.js";
|
||||
import {
|
||||
ENVIRONMENT_ID,
|
||||
@@ -590,13 +592,24 @@ describe("worker turn launcher local placement", () => {
|
||||
expect(placements.get(SESSION_ID)).toMatchObject({ state: "active", turnClaim: null });
|
||||
});
|
||||
|
||||
it("rejects enforced memory before a host-node worker is admitted", async () => {
|
||||
it("fails an enforced worker turn before workspace, credential, or tunnel work when v6 is unavailable", async () => {
|
||||
seedActivePlacement();
|
||||
const environment = attachedEnvironment();
|
||||
environment.nodeDeviceId = "node-1";
|
||||
const environments: WorkerTurnEnvironmentService = {
|
||||
...unusedEnvironments(),
|
||||
get: vi.fn(() => attachedEnvironment()),
|
||||
get: vi.fn(() => environment),
|
||||
};
|
||||
const provider = createWorkerSessionTurnPlacementProvider({ environments, placements });
|
||||
const assertEligible = vi.fn(async () => {
|
||||
throw new Error("device worker node does not attest to the required process-isolation boundary");
|
||||
});
|
||||
bindDeviceWorkerExecutionEligibility(environments, assertEligible);
|
||||
const resolveWorkspacePath = vi.fn(async () => "/workspace/should-not-resolve");
|
||||
const provider = createWorkerSessionTurnPlacementProvider({
|
||||
environments,
|
||||
placements,
|
||||
resolveWorkspacePath,
|
||||
});
|
||||
const runLocal = vi.fn(async () => ({ meta: { durationMs: 1 } }));
|
||||
const onAdmitted = vi.fn();
|
||||
|
||||
@@ -613,11 +626,16 @@ describe("worker turn launcher local placement", () => {
|
||||
runLocal,
|
||||
onAdmitted,
|
||||
),
|
||||
).rejects.toThrow("enforced memory requires a containerized worker launch");
|
||||
).rejects.toThrow("does not attest to the required process-isolation boundary");
|
||||
|
||||
expect(runLocal).not.toHaveBeenCalled();
|
||||
expect(onAdmitted).not.toHaveBeenCalled();
|
||||
expect(environments.get).not.toHaveBeenCalled();
|
||||
expect(environments.get).toHaveBeenCalledWith(ENVIRONMENT_ID);
|
||||
expect(assertEligible).toHaveBeenCalledWith({
|
||||
deviceId: "node-1",
|
||||
execution: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 },
|
||||
});
|
||||
expect(resolveWorkspacePath).not.toHaveBeenCalled();
|
||||
expect(environments.acquireTurnCredential).not.toHaveBeenCalled();
|
||||
expect(environments.startTunnel).not.toHaveBeenCalled();
|
||||
expect(placements.get(SESSION_ID)).toMatchObject({ state: "active", turnClaim: null });
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
|
||||
import { mapThinkingLevelForProvider } from "../../agents/embedded-agent-runner/utils.js";
|
||||
import { createAuthorizedMemoryReadHost } from "../../agents/memory-authorized-read-host.js";
|
||||
import {
|
||||
createAuthorizedMemoryReadHost,
|
||||
resolveAuthorizedMemoryVirtualFileBroker,
|
||||
} from "../../agents/memory-authorized-read-host.js";
|
||||
import type { SandboxContext } from "../../agents/sandbox/types.js";
|
||||
import {
|
||||
withSessionPlacementForcedTerminalSettlement,
|
||||
@@ -17,6 +20,7 @@ import { redactSensitiveText } from "../../logging/redact.js";
|
||||
import { isMemoryIsolationCutoverAgent } from "../../plugins/memory-cutover.js";
|
||||
import { normalizeDeliveryContext } from "../../utils/delivery-context.shared.js";
|
||||
import { parseWorkerLaunchPlan } from "../../worker/launch-descriptor.js";
|
||||
import { NODE_WORKER_EXECUTION_CONTAINER_V1 } from "../../worker/node-supervisor-protocol.js";
|
||||
import { WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE } from "../../worker/transcript-message.js";
|
||||
import {
|
||||
STALE_WORKER_BUILD_REASON,
|
||||
@@ -24,6 +28,7 @@ import {
|
||||
supportsWorkerExecutionContextLaunch,
|
||||
} from "./admission.js";
|
||||
import { registerWorkerMemoryHost } from "./memory-host.js";
|
||||
import { assertDeviceWorkerExecutionEligibility } from "./device-provider.js";
|
||||
import { placementTurnOwner } from "./placement-record.js";
|
||||
import { createRemoteExecPlacementSandbox } from "./placement-sandbox.js";
|
||||
import type {
|
||||
@@ -221,6 +226,23 @@ async function executeWorkerTurn(params: {
|
||||
if (memoryReadEnforced && !memoryHost) {
|
||||
throw new WorkerTurnExecutionError("Enforced memory could not create a trusted worker host");
|
||||
}
|
||||
const memoryProjection = memoryHost
|
||||
? await resolveAuthorizedMemoryVirtualFileBroker(memoryHost, turn.abortSignal)
|
||||
: undefined;
|
||||
if (memoryReadEnforced && !memoryProjection) {
|
||||
throw new WorkerTurnExecutionError(
|
||||
"Enforced memory could not materialize an authorized virtual memory view",
|
||||
);
|
||||
}
|
||||
const memoryProjectionExpiresAtMs = memoryProjection
|
||||
? Date.parse(memoryProjection.view.expiresAt)
|
||||
: undefined;
|
||||
if (
|
||||
memoryProjectionExpiresAtMs !== undefined &&
|
||||
(!Number.isFinite(memoryProjectionExpiresAtMs) || memoryProjectionExpiresAtMs <= Date.now())
|
||||
) {
|
||||
throw new WorkerTurnExecutionError("Enforced memory virtual view expired before process launch");
|
||||
}
|
||||
const { browser, toolAuthority } = resolveWorkerBrowserLaunchPlan({
|
||||
desktop: environment.desktop,
|
||||
modelRef,
|
||||
@@ -262,11 +284,13 @@ async function executeWorkerTurn(params: {
|
||||
turnId: randomUUID(),
|
||||
prompt: turn.prompt,
|
||||
suppressPromptTranscript: true,
|
||||
workspaceDir: placement.remoteWorkspaceDir,
|
||||
workspaceDir: memoryReadEnforced ? "/workspace" : placement.remoteWorkspaceDir,
|
||||
...(turn.permissionMode
|
||||
? {
|
||||
permissionMode: turn.permissionMode,
|
||||
workerContainmentRoot: placement.remoteWorkspaceDir,
|
||||
workerContainmentRoot: memoryReadEnforced
|
||||
? "/workspace"
|
||||
: placement.remoteWorkspaceDir,
|
||||
}
|
||||
: {}),
|
||||
modelRef,
|
||||
@@ -317,13 +341,13 @@ async function executeWorkerTurn(params: {
|
||||
const handoffAbort = new AbortController();
|
||||
let handoffError: Error | undefined;
|
||||
let dispatchReady = false;
|
||||
let executionReady = false;
|
||||
const onDispatchReady = () => {
|
||||
if (dispatchReady) {
|
||||
return;
|
||||
}
|
||||
dispatchReady = true;
|
||||
params.onHandoff();
|
||||
turn.onExecutionPhase?.({ phase: "process_spawned", backend: "cloud-worker" });
|
||||
try {
|
||||
if (!params.environments.acknowledgeCredentialDelivery(credential)) {
|
||||
handoffError = new Error("Cloud worker credential owner changed during process handoff");
|
||||
@@ -335,6 +359,13 @@ async function executeWorkerTurn(params: {
|
||||
handoffAbort.abort(handoffError);
|
||||
}
|
||||
};
|
||||
const onExecutionReady = () => {
|
||||
if (executionReady) {
|
||||
return;
|
||||
}
|
||||
executionReady = true;
|
||||
turn.onExecutionPhase?.({ phase: "process_spawned", backend: "cloud-worker" });
|
||||
};
|
||||
if (!tunnel.launchTurn) {
|
||||
throw new Error("Worker tunnel does not support worker turns");
|
||||
}
|
||||
@@ -343,11 +374,22 @@ async function executeWorkerTurn(params: {
|
||||
return await tunnel.launchTurn({
|
||||
plan,
|
||||
turnClaim: params.turnClaim,
|
||||
timeoutMs: turn.timeoutMs,
|
||||
...(memoryProjection ? { memoryProjection } : {}),
|
||||
timeoutMs:
|
||||
memoryProjectionExpiresAtMs === undefined
|
||||
? turn.timeoutMs
|
||||
: Math.max(
|
||||
1,
|
||||
Math.min(
|
||||
turn.timeoutMs ?? 60_000,
|
||||
memoryProjectionExpiresAtMs - Date.now(),
|
||||
),
|
||||
),
|
||||
signal: turn.abortSignal
|
||||
? AbortSignal.any([turn.abortSignal, handoffAbort.signal])
|
||||
: handoffAbort.signal,
|
||||
onDispatchReady,
|
||||
onExecutionReady,
|
||||
});
|
||||
} finally {
|
||||
// A completed or failed process must not retain memory authority for a later turn.
|
||||
@@ -538,14 +580,24 @@ export function createWorkerSessionTurnPlacementProvider(options: WorkerTurnLaun
|
||||
let placement = requireActivePlacement(routablePlacement);
|
||||
const requiresProcessIsolation = claim.requiresProcessIsolation === true;
|
||||
const remoteExec = placement.executionMode === "remote-exec";
|
||||
if (requiresProcessIsolation && remoteExec) {
|
||||
// Remote-exec invokes the Gateway callback rather than the node-worker protocol, so it
|
||||
// cannot attest to the container boundary required for an enforced memory turn.
|
||||
throw new Error("enforced memory cannot execute through remote-exec placement");
|
||||
}
|
||||
if (requiresProcessIsolation) {
|
||||
// Remote-exec invokes the Gateway callback, while worker-turn launches directly on the
|
||||
// node host. Neither path supplies the required broker process boundary yet.
|
||||
throw new Error(
|
||||
remoteExec
|
||||
? "enforced memory cannot execute through remote-exec placement"
|
||||
: "enforced memory requires a containerized worker launch",
|
||||
);
|
||||
const environment = options.environments.get(placement.environmentId);
|
||||
if (!environment?.nodeDeviceId) {
|
||||
throw new Error("enforced memory requires a device worker node");
|
||||
}
|
||||
// Prove the exact connected node's advertised boundary before any credential, tunnel,
|
||||
// workspace, or claim work can create host-visible state for the enforced turn.
|
||||
await assertDeviceWorkerExecutionEligibility({
|
||||
service: options.environments,
|
||||
deviceId: environment.nodeDeviceId,
|
||||
execution: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 },
|
||||
...(turn.abortSignal ? { signal: turn.abortSignal } : {}),
|
||||
});
|
||||
}
|
||||
// The placement owns the managed worktree. Callers can carry a default or stale
|
||||
// workspace path, but remote results must only reconcile into that canonical root.
|
||||
|
||||
@@ -54,7 +54,13 @@ const LockPayloadSchema = z.object({
|
||||
configPath: z.string(),
|
||||
port: z.number().int().min(1).max(65_535).optional(),
|
||||
role: z
|
||||
.enum(["gateway", "agent-embedded", "skill-workshop-apply", "sqlite-maintenance"])
|
||||
.enum([
|
||||
"gateway",
|
||||
"agent-embedded",
|
||||
"skill-workshop-apply",
|
||||
"sqlite-maintenance",
|
||||
"offline-maintenance",
|
||||
])
|
||||
.optional(),
|
||||
stateDir: z.string().optional(),
|
||||
startTime: z.number().optional(),
|
||||
@@ -69,7 +75,12 @@ type GatewayLockHandle = {
|
||||
release: () => Promise<void>;
|
||||
};
|
||||
|
||||
type GatewayLockRole = "gateway" | "agent-embedded" | "skill-workshop-apply" | "sqlite-maintenance";
|
||||
type GatewayLockRole =
|
||||
| "gateway"
|
||||
| "agent-embedded"
|
||||
| "skill-workshop-apply"
|
||||
| "sqlite-maintenance"
|
||||
| "offline-maintenance";
|
||||
|
||||
export type GatewayLockIdentity = {
|
||||
pid: number;
|
||||
@@ -215,6 +226,7 @@ async function resolveGatewayOwnerStatus(
|
||||
if (
|
||||
role === "agent-embedded" ||
|
||||
role === "sqlite-maintenance" ||
|
||||
role === "offline-maintenance" ||
|
||||
role === "skill-workshop-apply"
|
||||
) {
|
||||
const args = readFn(pid);
|
||||
@@ -227,6 +239,11 @@ async function resolveGatewayOwnerStatus(
|
||||
// instead of baking one command spelling into stale-lock recovery.
|
||||
return isOpenClawArgv(args) ? "alive" : "dead";
|
||||
}
|
||||
if (role === "offline-maintenance") {
|
||||
return isOpenClawCommandArgv(args, "doctor") || isOpenClawCommandArgv(args, "backup")
|
||||
? "alive"
|
||||
: "dead";
|
||||
}
|
||||
const command = role === "sqlite-maintenance" ? "doctor" : "skills";
|
||||
return isOpenClawCommandArgv(args, command) ? "alive" : "dead";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { GATEWAY_CLIENT_IDS } from "../../packages/gateway-protocol/src/client-info.js";
|
||||
import {
|
||||
NODE_WORKER_EXECUTION_CONTAINER_V1,
|
||||
NODE_WORKER_EXECUTION_HOST_V1,
|
||||
} from "../worker/node-supervisor-protocol.js";
|
||||
import {
|
||||
NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE,
|
||||
NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
|
||||
NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
|
||||
parseNodeRunnerInventoryDeclaration,
|
||||
} from "./node-runner-inventory.js";
|
||||
import {
|
||||
supportsNodeWorkerExecution,
|
||||
type NodeWorkerSupervisorNodeProof,
|
||||
} from "../gateway/node-runner-inventory-runtime.js";
|
||||
|
||||
const capacity = { total: 2, available: 2 } as const;
|
||||
|
||||
function proof(params: {
|
||||
protocolFeature:
|
||||
| typeof NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE
|
||||
| typeof NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE
|
||||
| typeof NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE;
|
||||
processIsolation?: {
|
||||
kind: typeof NODE_WORKER_EXECUTION_CONTAINER_V1;
|
||||
memoryProjection?: 1;
|
||||
};
|
||||
}): NodeWorkerSupervisorNodeProof {
|
||||
return {
|
||||
nodeId: "node-1",
|
||||
connId: "conn-1",
|
||||
pairingIdentity: "identity-1",
|
||||
pairingGeneration: "generation-1",
|
||||
clientId: GATEWAY_CLIENT_IDS.NODE_HOST,
|
||||
clientMode: "node",
|
||||
protocolFeature: params.protocolFeature,
|
||||
workerHost: {
|
||||
enabled: true,
|
||||
capacity,
|
||||
...(params.processIsolation ? { processIsolation: params.processIsolation } : {}),
|
||||
},
|
||||
commands: ["system.run"],
|
||||
};
|
||||
}
|
||||
|
||||
describe("node runner process-isolation inventory", () => {
|
||||
it("keeps v5 host-only and accepts only the exact v6 declaration", () => {
|
||||
const v5 = {
|
||||
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE],
|
||||
workerHost: { enabled: true, capacity },
|
||||
};
|
||||
const v6 = {
|
||||
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE],
|
||||
workerHost: {
|
||||
enabled: true,
|
||||
capacity,
|
||||
processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 },
|
||||
},
|
||||
};
|
||||
|
||||
expect(parseNodeRunnerInventoryDeclaration(v5)).toEqual(v5);
|
||||
expect(parseNodeRunnerInventoryDeclaration(v6)).toEqual(v6);
|
||||
expect(
|
||||
parseNodeRunnerInventoryDeclaration({
|
||||
...v5,
|
||||
workerHost: { ...v5.workerHost, processIsolation: v6.workerHost.processIsolation },
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE],
|
||||
workerHost: { enabled: true, capacity },
|
||||
},
|
||||
{
|
||||
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE],
|
||||
workerHost: { enabled: false },
|
||||
},
|
||||
{
|
||||
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE],
|
||||
workerHost: {
|
||||
enabled: true,
|
||||
capacity,
|
||||
processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, extra: true },
|
||||
},
|
||||
},
|
||||
{
|
||||
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE],
|
||||
workerHost: { enabled: true, capacity, processIsolation: { kind: "host-v1" } },
|
||||
},
|
||||
])("rejects malformed v6 process-isolation declarations", (declaration) => {
|
||||
expect(parseNodeRunnerInventoryDeclaration(declaration)).toBeNull();
|
||||
});
|
||||
|
||||
it("admits container execution only for the exact v7 memory-projection proof", () => {
|
||||
const v5 = proof({ protocolFeature: NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE });
|
||||
const v6 = proof({
|
||||
protocolFeature: NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE,
|
||||
processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 },
|
||||
});
|
||||
const v7 = proof({
|
||||
protocolFeature: NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
|
||||
processIsolation: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, memoryProjection: 1 },
|
||||
});
|
||||
|
||||
expect(supportsNodeWorkerExecution(v5, { kind: NODE_WORKER_EXECUTION_HOST_V1 })).toBe(true);
|
||||
expect(supportsNodeWorkerExecution(v5, { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 })).toBe(
|
||||
false,
|
||||
);
|
||||
expect(supportsNodeWorkerExecution(v6, { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 })).toBe(
|
||||
false,
|
||||
);
|
||||
expect(supportsNodeWorkerExecution(v7, { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 })).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,9 +1,14 @@
|
||||
import { isRecord } from "@openclaw/normalization-core/record-coerce";
|
||||
import { validateWorkerAdmissionHandshake } from "../../packages/gateway-protocol/src/index.js";
|
||||
import { WORKER_BUNDLE_PREWARM_VERSION } from "../../packages/gateway-protocol/src/schema/worker-admission.js";
|
||||
import { NODE_WORKER_EXECUTION_CONTAINER_V1 } from "../worker/node-supervisor-protocol.js";
|
||||
|
||||
export const NODE_RUNNER_INVENTORY_UPDATE_METHOD = "node.runnerInventory.update";
|
||||
export const NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE = "node-worker-supervisor-v5";
|
||||
export const NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE =
|
||||
"node-worker-supervisor-v6";
|
||||
export const NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE =
|
||||
"node-worker-supervisor-v7";
|
||||
export const NODE_WORKER_SUPERVISOR_BINARY_CAPACITY_PROTOCOL_FEATURE = "node-worker-supervisor-v4";
|
||||
export const NODE_WORKER_SUPERVISOR_EXECUTION_CONTEXT_V1_PROTOCOL_FEATURE =
|
||||
"node-worker-supervisor-v3";
|
||||
@@ -25,6 +30,10 @@ export type NodeWorkerCapacitySnapshot = Readonly<{
|
||||
total: number;
|
||||
available: number;
|
||||
}>;
|
||||
export type NodeWorkerProcessIsolationDeclaration = Readonly<{
|
||||
kind: typeof NODE_WORKER_EXECUTION_CONTAINER_V1;
|
||||
memoryProjection?: 1;
|
||||
}>;
|
||||
|
||||
export type NodeWorkerHostDeclaration =
|
||||
| { enabled: false }
|
||||
@@ -34,6 +43,8 @@ export type NodeWorkerHostDeclaration =
|
||||
bundlePrewarm?: typeof WORKER_BUNDLE_PREWARM_VERSION;
|
||||
bundleRetention?: typeof NODE_WORKER_BUNDLE_RETENTION_VERSION;
|
||||
bundleStatus?: typeof NODE_WORKER_BUNDLE_STATUS_VERSION;
|
||||
/** Present only on v6 hosts that passed the local container-runtime gate. */
|
||||
processIsolation?: NodeWorkerProcessIsolationDeclaration;
|
||||
};
|
||||
|
||||
export type NodeRunnerInventoryDeclaration =
|
||||
@@ -47,7 +58,11 @@ export type NodeRunnerInventoryDeclaration =
|
||||
];
|
||||
}
|
||||
| {
|
||||
protocolFeatures: readonly [typeof NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE];
|
||||
protocolFeatures: readonly [
|
||||
| typeof NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE
|
||||
| typeof NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE
|
||||
| typeof NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
|
||||
];
|
||||
workerHost: NodeWorkerHostDeclaration;
|
||||
};
|
||||
|
||||
@@ -73,19 +88,52 @@ function parseCapacitySnapshot(value: unknown): NodeWorkerCapacitySnapshot | nul
|
||||
: null;
|
||||
}
|
||||
|
||||
function parseWorkerHostDeclaration(value: unknown): NodeWorkerHostDeclaration | null {
|
||||
function parseProcessIsolationDeclaration(
|
||||
value: unknown,
|
||||
requireMemoryProjection: boolean,
|
||||
): NodeWorkerProcessIsolationDeclaration | null {
|
||||
if (
|
||||
!isRecord(value) ||
|
||||
!Object.hasOwn(value, "kind") ||
|
||||
value.kind !== NODE_WORKER_EXECUTION_CONTAINER_V1 ||
|
||||
(requireMemoryProjection
|
||||
? Object.keys(value).length !== 2 || value.memoryProjection !== 1
|
||||
: Object.keys(value).length !== 1)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return requireMemoryProjection
|
||||
? { kind: NODE_WORKER_EXECUTION_CONTAINER_V1, memoryProjection: 1 }
|
||||
: { kind: NODE_WORKER_EXECUTION_CONTAINER_V1 };
|
||||
}
|
||||
|
||||
function parseWorkerHostDeclaration(params: {
|
||||
value: unknown;
|
||||
requireProcessIsolation: boolean;
|
||||
requireMemoryProjection?: boolean;
|
||||
}): NodeWorkerHostDeclaration | null {
|
||||
const { value } = params;
|
||||
if (!isRecord(value) || typeof value.enabled !== "boolean") {
|
||||
return null;
|
||||
}
|
||||
const keys = Object.keys(value);
|
||||
if (!value.enabled) {
|
||||
return keys.length === 1 && keys[0] === "enabled" ? { enabled: false } : null;
|
||||
return !params.requireProcessIsolation && keys.length === 1 && keys[0] === "enabled"
|
||||
? { enabled: false }
|
||||
: null;
|
||||
}
|
||||
const capacity = parseCapacitySnapshot(value.capacity);
|
||||
const processIsolation =
|
||||
value.processIsolation === undefined
|
||||
? undefined
|
||||
: parseProcessIsolationDeclaration(
|
||||
value.processIsolation,
|
||||
params.requireMemoryProjection === true,
|
||||
);
|
||||
if (
|
||||
!capacity ||
|
||||
keys.length < 2 ||
|
||||
keys.length > 5 ||
|
||||
keys.length > 6 ||
|
||||
!keys.includes("enabled") ||
|
||||
!keys.includes("capacity") ||
|
||||
keys.some(
|
||||
@@ -94,14 +142,17 @@ function parseWorkerHostDeclaration(value: unknown): NodeWorkerHostDeclaration |
|
||||
key !== "capacity" &&
|
||||
key !== "bundlePrewarm" &&
|
||||
key !== "bundleRetention" &&
|
||||
key !== "bundleStatus",
|
||||
key !== "bundleStatus" &&
|
||||
key !== "processIsolation",
|
||||
) ||
|
||||
(value.bundlePrewarm !== undefined && value.bundlePrewarm !== WORKER_BUNDLE_PREWARM_VERSION) ||
|
||||
(value.bundleRetention !== undefined &&
|
||||
value.bundleRetention !== NODE_WORKER_BUNDLE_RETENTION_VERSION) ||
|
||||
(value.bundleStatus !== undefined &&
|
||||
value.bundleStatus !== NODE_WORKER_BUNDLE_STATUS_VERSION) ||
|
||||
(value.bundleStatus !== undefined && value.bundleRetention === undefined)
|
||||
(value.bundleStatus !== undefined && value.bundleRetention === undefined) ||
|
||||
(value.processIsolation !== undefined && !processIsolation) ||
|
||||
(params.requireProcessIsolation !== Boolean(processIsolation))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
@@ -117,6 +168,7 @@ function parseWorkerHostDeclaration(value: unknown): NodeWorkerHostDeclaration |
|
||||
...(value.bundleStatus === NODE_WORKER_BUNDLE_STATUS_VERSION
|
||||
? { bundleStatus: NODE_WORKER_BUNDLE_STATUS_VERSION }
|
||||
: {}),
|
||||
...(processIsolation ? { processIsolation } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -190,12 +242,23 @@ export function parseNodeRunnerInventoryDeclaration(
|
||||
? { protocolFeatures: [feature] }
|
||||
: null;
|
||||
}
|
||||
if (feature !== NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE || keys.length !== 2) {
|
||||
if (
|
||||
(feature !== NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE &&
|
||||
feature !== NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE &&
|
||||
feature !== NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE) ||
|
||||
keys.length !== 2
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const workerHost = parseWorkerHostDeclaration(value.workerHost);
|
||||
const workerHost = parseWorkerHostDeclaration({
|
||||
value: value.workerHost,
|
||||
requireProcessIsolation:
|
||||
feature === NODE_WORKER_SUPERVISOR_PROCESS_ISOLATION_PROTOCOL_FEATURE ||
|
||||
feature === NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
|
||||
requireMemoryProjection: feature === NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
|
||||
});
|
||||
return workerHost
|
||||
? { protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE], workerHost }
|
||||
? { protocolFeatures: [feature], workerHost }
|
||||
: null;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { startMemoryBrokerServer, type MemoryBrokerHandler } from "./server.js";
|
||||
import { startMemoryBrokerServer } from "./server.js";
|
||||
import type { MemoryBrokerChildEntry } from "./entry.js";
|
||||
|
||||
type BrokerStartMessage = Readonly<{
|
||||
type: "start";
|
||||
@@ -7,10 +8,7 @@ type BrokerStartMessage = Readonly<{
|
||||
brokerEpoch: string;
|
||||
secret: string;
|
||||
handlerModuleUrl: string;
|
||||
}>;
|
||||
|
||||
type BrokerChildEntry = Readonly<{
|
||||
createMemoryBrokerHandler: () => MemoryBrokerHandler | Promise<MemoryBrokerHandler>;
|
||||
agentIds: readonly string[];
|
||||
}>;
|
||||
|
||||
type BrokerMaintenanceMessage = Readonly<{
|
||||
@@ -41,10 +39,19 @@ async function start(message: BrokerStartMessage): Promise<void> {
|
||||
if (server || closing) {
|
||||
throw new Error("memory broker child has already started");
|
||||
}
|
||||
const module = (await import(message.handlerModuleUrl)) as Partial<BrokerChildEntry>;
|
||||
const module = (await import(message.handlerModuleUrl)) as Partial<MemoryBrokerChildEntry>;
|
||||
if (typeof module.createMemoryBrokerHandler !== "function") {
|
||||
throw new Error("selected memory plugin has no broker child entry");
|
||||
}
|
||||
if (
|
||||
!Array.isArray(message.agentIds) ||
|
||||
!message.agentIds.every((agentId) => typeof agentId === "string" && agentId.length > 0)
|
||||
) {
|
||||
throw new Error("memory broker startup agents are unavailable");
|
||||
}
|
||||
// Recovery happens before the socket exists, so a fresh/replacement child never reports
|
||||
// healthy while pending revisions can still become visible or need quarantine.
|
||||
await module.initializeMemoryBroker?.({ agentIds: Object.freeze([...message.agentIds]) });
|
||||
const handler = await module.createMemoryBrokerHandler();
|
||||
if (typeof handler !== "function") {
|
||||
throw new Error("selected memory plugin returned an invalid broker handler");
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { MemoryBrokerHandler } from "./server.js";
|
||||
|
||||
/**
|
||||
* Gateway passes this small, path-free startup snapshot over inherited parent-child IPC before
|
||||
* the broker opens its agent-visible socket. A plugin must finish recovery or throw; ready means
|
||||
* the selected runtime has completed its own durable-startup fence.
|
||||
*/
|
||||
export type MemoryBrokerStartupContext = Readonly<{
|
||||
agentIds: readonly string[];
|
||||
}>;
|
||||
|
||||
/** Public selected-memory child entry contract. */
|
||||
export type MemoryBrokerChildEntry = Readonly<{
|
||||
createMemoryBrokerHandler: () => MemoryBrokerHandler | Promise<MemoryBrokerHandler>;
|
||||
initializeMemoryBroker?: (
|
||||
context: MemoryBrokerStartupContext,
|
||||
) => void | Promise<void>;
|
||||
}>;
|
||||
@@ -5,6 +5,7 @@ import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { startMemoryBrokerProcess, type MemoryBrokerProcess } from "./process.js";
|
||||
import type { MemoryBrokerAuthorizationBinding } from "./protocol.js";
|
||||
|
||||
let broker: MemoryBrokerProcess | undefined;
|
||||
|
||||
@@ -78,6 +79,7 @@ describe("memory broker child", () => {
|
||||
runId: "run-a",
|
||||
contextFingerprint: "context-a",
|
||||
subjectRevision: "subject-a",
|
||||
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
|
||||
actorRevision: "actor-a",
|
||||
capabilitySnapshotId: "capability-a",
|
||||
policyRevision: "policy-a",
|
||||
@@ -90,6 +92,46 @@ describe("memory broker child", () => {
|
||||
).resolves.toEqual({ agentId: "agent-a", method: "memory.search" });
|
||||
});
|
||||
|
||||
it("runs selected-runtime startup recovery with sorted configured agents before serving", async () => {
|
||||
broker = await startMemoryBrokerProcess({
|
||||
brokerId: "broker-a",
|
||||
childModuleUrl: new URL("./child.ts", import.meta.url),
|
||||
handlerModuleUrl: new URL("./test-handler.mjs", import.meta.url).href,
|
||||
agentIds: ["work", "main", "work"],
|
||||
});
|
||||
|
||||
await expect(
|
||||
broker.client.request({
|
||||
binding: {
|
||||
agentId: "main",
|
||||
sessionId: "session-a",
|
||||
runId: "run-a",
|
||||
contextFingerprint: "context-a",
|
||||
subjectRevision: "subject-a",
|
||||
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
|
||||
actorRevision: "actor-a",
|
||||
capabilitySnapshotId: "capability-a",
|
||||
policyRevision: "policy-a",
|
||||
deliveryRevision: "delivery-a",
|
||||
},
|
||||
method: "memory.startup",
|
||||
payload: {},
|
||||
expiresAtMs: Date.now() + 30_000,
|
||||
}),
|
||||
).resolves.toEqual({ agentIds: ["main", "work"] });
|
||||
});
|
||||
|
||||
it("does not report ready when selected-runtime startup recovery fails", async () => {
|
||||
await expect(
|
||||
startMemoryBrokerProcess({
|
||||
brokerId: "broker-a",
|
||||
childModuleUrl: new URL("./child.ts", import.meta.url),
|
||||
handlerModuleUrl: new URL("./test-handler.mjs", import.meta.url).href,
|
||||
agentIds: ["fail-startup"],
|
||||
}),
|
||||
).rejects.toThrow("memory broker child did not become ready");
|
||||
});
|
||||
|
||||
it("retires a stopped child and gives its replacement a new broker epoch", async () => {
|
||||
broker = await startMemoryBrokerProcess({
|
||||
brokerId: "broker-a",
|
||||
@@ -121,12 +163,13 @@ describe("memory broker child", () => {
|
||||
});
|
||||
const crashedBroker = broker;
|
||||
const firstEpoch = crashedBroker.brokerEpoch;
|
||||
const binding = {
|
||||
const binding: MemoryBrokerAuthorizationBinding = {
|
||||
agentId: "agent-a",
|
||||
sessionId: "session-a",
|
||||
runId: "run-a",
|
||||
contextFingerprint: "context-a",
|
||||
subjectRevision: "subject-a",
|
||||
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
|
||||
actorRevision: "actor-a",
|
||||
capabilitySnapshotId: "capability-a",
|
||||
policyRevision: "policy-a",
|
||||
@@ -160,6 +203,37 @@ describe("memory broker child", () => {
|
||||
).resolves.toEqual({ agentId: "agent-a", method: "memory.search" });
|
||||
});
|
||||
|
||||
it("retires a child killed by an external signal without waiting for a second exit event", async () => {
|
||||
broker = await startMemoryBrokerProcess({
|
||||
brokerId: "broker-a",
|
||||
childModuleUrl: new URL("./child.ts", import.meta.url),
|
||||
handlerModuleUrl: new URL("./test-handler.mjs", import.meta.url).href,
|
||||
});
|
||||
const killedBroker = broker;
|
||||
await expect(
|
||||
killedBroker.client.request({
|
||||
binding: {
|
||||
agentId: "agent-a",
|
||||
sessionId: "session-a",
|
||||
runId: "run-a",
|
||||
contextFingerprint: "context-a",
|
||||
subjectRevision: "subject-a",
|
||||
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
|
||||
actorRevision: "actor-a",
|
||||
capabilitySnapshotId: "capability-a",
|
||||
policyRevision: "policy-a",
|
||||
deliveryRevision: "delivery-a",
|
||||
},
|
||||
method: "memory.kill",
|
||||
payload: {},
|
||||
expiresAtMs: Date.now() + 30_000,
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
await waitFor(() => !killedBroker.isRunning(), "memory broker child did not exit after SIGKILL");
|
||||
|
||||
await expect(killedBroker.close()).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not inherit arbitrary Gateway environment secrets", async () => {
|
||||
const previous = process.env.OPENCLAW_MEMORY_BROKER_TEST_SECRET;
|
||||
process.env.OPENCLAW_MEMORY_BROKER_TEST_SECRET = "gateway-only-secret";
|
||||
@@ -177,6 +251,7 @@ describe("memory broker child", () => {
|
||||
runId: "run-a",
|
||||
contextFingerprint: "context-a",
|
||||
subjectRevision: "subject-a",
|
||||
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
|
||||
actorRevision: "actor-a",
|
||||
capabilitySnapshotId: "capability-a",
|
||||
policyRevision: "policy-a",
|
||||
@@ -230,6 +305,7 @@ describe("memory broker child", () => {
|
||||
runId: "run-a",
|
||||
contextFingerprint: "context-a",
|
||||
subjectRevision: "subject-a",
|
||||
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
|
||||
actorRevision: "actor-a",
|
||||
capabilitySnapshotId: "capability-a",
|
||||
policyRevision: "policy-a",
|
||||
@@ -249,6 +325,7 @@ describe("memory broker child", () => {
|
||||
runId: "run-a",
|
||||
contextFingerprint: "context-a",
|
||||
subjectRevision: "subject-a",
|
||||
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
|
||||
actorRevision: "actor-a",
|
||||
capabilitySnapshotId: "capability-a",
|
||||
policyRevision: "policy-a",
|
||||
@@ -260,4 +337,36 @@ describe("memory broker child", () => {
|
||||
}),
|
||||
).resolves.toEqual({ agentId: "agent-a", method: "memory.search" });
|
||||
});
|
||||
|
||||
it("retires a noncooperative child instead of leaving a healthy broker quiesced", async () => {
|
||||
broker = await startMemoryBrokerProcess({
|
||||
brokerId: "broker-a",
|
||||
childModuleUrl: new URL("./child.ts", import.meta.url),
|
||||
handlerModuleUrl: new URL("./test-handler.mjs", import.meta.url).href,
|
||||
maintenanceTimeoutMs: 40,
|
||||
});
|
||||
const pending = broker.client.request({
|
||||
binding: {
|
||||
agentId: "agent-a",
|
||||
sessionId: "session-a",
|
||||
runId: "run-a",
|
||||
contextFingerprint: "context-a",
|
||||
subjectRevision: "subject-a",
|
||||
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
|
||||
actorRevision: "actor-a",
|
||||
capabilitySnapshotId: "capability-a",
|
||||
policyRevision: "policy-a",
|
||||
deliveryRevision: "delivery-a",
|
||||
},
|
||||
method: "memory.hang",
|
||||
payload: {},
|
||||
expiresAtMs: Date.now() + 30_000,
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
|
||||
await expect(broker.quiesce()).rejects.toThrow("memory broker quiesce is unavailable");
|
||||
await expect(pending).resolves.toBeUndefined();
|
||||
expect(broker.isRunning()).toBe(false);
|
||||
await expect(broker.isHealthy()).resolves.toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -44,9 +44,18 @@ export type MemoryBrokerProcess = Readonly<{
|
||||
export async function startMemoryBrokerProcess(params: {
|
||||
brokerId: string;
|
||||
handlerModuleUrl: string;
|
||||
/** Configured agent IDs only; passed over inherited IPC before the broker binds its socket. */
|
||||
agentIds?: readonly string[];
|
||||
childModuleUrl?: string | URL;
|
||||
startTimeoutMs?: number;
|
||||
maintenanceTimeoutMs?: number;
|
||||
}): Promise<MemoryBrokerProcess> {
|
||||
if (!(params.agentIds ?? []).every((agentId) => typeof agentId === "string" && agentId.length > 0)) {
|
||||
throw new Error("memory broker startup agent IDs are invalid");
|
||||
}
|
||||
const agentIds = Object.freeze(
|
||||
[...new Set(params.agentIds ?? [])].toSorted(),
|
||||
);
|
||||
const directory = await mkdtemp(path.join(tmpdir(), "openclaw-memory-broker-"));
|
||||
const socketPath = path.join(directory, "broker.sock");
|
||||
const brokerEpoch = randomUUID();
|
||||
@@ -87,6 +96,9 @@ export async function startMemoryBrokerProcess(params: {
|
||||
});
|
||||
let childExited = false;
|
||||
let directoryRemoved = false;
|
||||
let retirePromise: Promise<void> | undefined;
|
||||
const hasChildExited = () =>
|
||||
childExited || child.exitCode !== null || child.signalCode !== null || child.killed;
|
||||
const removeDirectory = async () => {
|
||||
if (directoryRemoved) {
|
||||
return;
|
||||
@@ -100,6 +112,18 @@ export async function startMemoryBrokerProcess(params: {
|
||||
// The Gateway recreates the child with a fresh epoch on the next operation.
|
||||
void removeDirectory();
|
||||
});
|
||||
const retireChild = async () => {
|
||||
retirePromise ??= (async () => {
|
||||
if (!hasChildExited()) {
|
||||
child.kill("SIGKILL");
|
||||
}
|
||||
if (!hasChildExited()) {
|
||||
await new Promise<void>((resolve) => child.once("exit", () => resolve()));
|
||||
}
|
||||
await removeDirectory();
|
||||
})();
|
||||
await retirePromise;
|
||||
};
|
||||
let startupStderr = "";
|
||||
child.stderr?.on("data", (chunk: Buffer) => {
|
||||
// Startup diagnostics never include request bodies or the parent-child secret. Keep enough
|
||||
@@ -159,6 +183,7 @@ export async function startMemoryBrokerProcess(params: {
|
||||
brokerEpoch,
|
||||
secret: secret.toString("base64url"),
|
||||
handlerModuleUrl: params.handlerModuleUrl,
|
||||
agentIds,
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
@@ -173,7 +198,7 @@ export async function startMemoryBrokerProcess(params: {
|
||||
secret,
|
||||
});
|
||||
const isHealthy = async (): Promise<boolean> => {
|
||||
if (childExited || child.exitCode !== null || child.killed || !child.connected) {
|
||||
if (hasChildExited() || !child.connected) {
|
||||
return false;
|
||||
}
|
||||
return await new Promise<boolean>((resolve) => {
|
||||
@@ -210,8 +235,13 @@ export async function startMemoryBrokerProcess(params: {
|
||||
});
|
||||
});
|
||||
};
|
||||
const maintain = async (operation: "quiesce" | "resume"): Promise<void> => {
|
||||
if (childExited || child.exitCode !== null || child.killed || !child.connected) {
|
||||
const maintenanceTimeoutMs = params.maintenanceTimeoutMs ?? MEMORY_BROKER_MAINTENANCE_TIMEOUT_MS;
|
||||
if (!Number.isSafeInteger(maintenanceTimeoutMs) || maintenanceTimeoutMs < 1) {
|
||||
await retireChild();
|
||||
throw new Error("memory broker maintenance timeout is invalid");
|
||||
}
|
||||
const maintainNow = async (operation: "quiesce" | "resume"): Promise<void> => {
|
||||
if (hasChildExited() || !child.connected) {
|
||||
throw new Error("memory broker child is unavailable for maintenance");
|
||||
}
|
||||
const requestId = randomUUID();
|
||||
@@ -248,7 +278,7 @@ export async function startMemoryBrokerProcess(params: {
|
||||
finish((message as { ok?: unknown }).ok === true);
|
||||
}
|
||||
};
|
||||
const timer = setTimeout(() => finish(false), MEMORY_BROKER_MAINTENANCE_TIMEOUT_MS);
|
||||
const timer = setTimeout(() => finish(false), maintenanceTimeoutMs);
|
||||
timer.unref?.();
|
||||
child.on("message", onMessage);
|
||||
child.send({ type: "maintenance", requestId, brokerEpoch, operation }, (error) => {
|
||||
@@ -258,20 +288,32 @@ export async function startMemoryBrokerProcess(params: {
|
||||
});
|
||||
});
|
||||
if (!ok) {
|
||||
// A late child acknowledgement could otherwise arrive after the parent has given up and
|
||||
// leave a healthy-but-quiesced socket rejecting every future request. Retire the whole
|
||||
// epoch instead; the next independently authorized request creates a fresh child/secret.
|
||||
await retireChild();
|
||||
throw new Error(`memory broker ${operation} is unavailable`);
|
||||
}
|
||||
};
|
||||
let maintenanceTail = Promise.resolve();
|
||||
const maintain = (operation: "quiesce" | "resume"): Promise<void> => {
|
||||
const current = maintenanceTail.then(() => maintainNow(operation));
|
||||
// Parent-child maintenance messages must preserve order even when a caller observes a
|
||||
// failure. Swallow only for tail progression; the original caller still receives the error.
|
||||
maintenanceTail = current.catch(() => undefined);
|
||||
return current;
|
||||
};
|
||||
return Object.freeze({
|
||||
client,
|
||||
brokerEpoch,
|
||||
isRunning: () => !childExited && child.exitCode === null && !child.killed,
|
||||
isRunning: () => !hasChildExited(),
|
||||
isHealthy,
|
||||
quiesce: () => maintain("quiesce"),
|
||||
resume: () => maintain("resume"),
|
||||
close: async () => {
|
||||
await maintain("quiesce").catch(() => undefined);
|
||||
await new Promise<void>((resolve) => {
|
||||
if (child.exitCode !== null || child.killed) {
|
||||
if (hasChildExited()) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ const binding: MemoryBrokerAuthorizationBinding = {
|
||||
runId: "run-a",
|
||||
contextFingerprint: "context-a",
|
||||
subjectRevision: "subject-a",
|
||||
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
|
||||
actorRevision: "actor-a",
|
||||
capabilitySnapshotId: "capability-a",
|
||||
policyRevision: "policy-a",
|
||||
@@ -58,6 +59,26 @@ function verify(
|
||||
}
|
||||
|
||||
describe("memory broker envelope", () => {
|
||||
it("refuses a request whose canonical form exceeds the nesting budget", () => {
|
||||
let payload: unknown = "leaf";
|
||||
for (let depth = 0; depth < 65; depth += 1) {
|
||||
payload = [payload];
|
||||
}
|
||||
|
||||
expect(
|
||||
createMemoryBrokerEnvelope({
|
||||
secret,
|
||||
brokerId: "broker-a",
|
||||
brokerEpoch: "epoch-a",
|
||||
binding,
|
||||
nonce: "nested-request",
|
||||
request: { method: "memory.search", payload },
|
||||
issuedAtMs: 1_000,
|
||||
expiresAtMs: 2_000,
|
||||
}),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("binds a deterministic request and every Gateway-derived authorization revision", () => {
|
||||
const envelope = createEnvelope();
|
||||
expect(verify(envelope)).toEqual({ ok: true });
|
||||
@@ -83,6 +104,16 @@ describe("memory broker envelope", () => {
|
||||
}),
|
||||
).toEqual({ ok: false, reason: "binding-mismatch" });
|
||||
}
|
||||
expect(
|
||||
verify(envelope, {
|
||||
expectedBinding: { ...binding, actor: { ...binding.actor, principalId: "bob" } },
|
||||
}),
|
||||
).toEqual({ ok: false, reason: "binding-mismatch" });
|
||||
expect(
|
||||
verify(envelope, {
|
||||
expectedBinding: { ...binding, actor: { ...binding.actor, actorKind: "service" } },
|
||||
}),
|
||||
).toEqual({ ok: false, reason: "binding-mismatch" });
|
||||
});
|
||||
|
||||
it("rejects cross-broker, stale-epoch, expired, and forged envelopes", () => {
|
||||
|
||||
@@ -4,6 +4,24 @@ import { createHash, createHmac, timingSafeEqual } from "node:crypto";
|
||||
export const MEMORY_BROKER_PROTOCOL_VERSION = 1 as const;
|
||||
|
||||
export const MEMORY_BROKER_MAXIMUM_NONCE_LENGTH = 256;
|
||||
export const MEMORY_BROKER_MAXIMUM_CANONICAL_DEPTH = 64;
|
||||
export const MEMORY_BROKER_MAXIMUM_CANONICAL_NODES = 10_000;
|
||||
|
||||
/**
|
||||
* A signed broker frame must bind the actor identity as well as the evidence revision. Revisions
|
||||
* can be shared by a provider snapshot, so binding only that revision would let a caller retarget
|
||||
* a valid Alice request to Bob while preserving every other frame field.
|
||||
*/
|
||||
export type MemoryBrokerActorBinding =
|
||||
| Readonly<{
|
||||
kind: "principal";
|
||||
actorKind: "human" | "agent" | "service" | "system";
|
||||
principalId: string;
|
||||
}>
|
||||
| Readonly<{
|
||||
kind: "unattributed";
|
||||
transportAuditRef: string;
|
||||
}>;
|
||||
|
||||
export type MemoryBrokerAuthorizationBinding = Readonly<{
|
||||
agentId: string;
|
||||
@@ -11,6 +29,7 @@ export type MemoryBrokerAuthorizationBinding = Readonly<{
|
||||
runId: string;
|
||||
contextFingerprint: string;
|
||||
subjectRevision: string;
|
||||
actor: MemoryBrokerActorBinding;
|
||||
actorRevision: string;
|
||||
capabilitySnapshotId: string;
|
||||
policyRevision: string;
|
||||
@@ -62,7 +81,17 @@ function isPlainRecord(value: unknown): value is Record<string, unknown> {
|
||||
* The MAC covers a deterministic JSON representation. Rejecting non-JSON values is deliberate:
|
||||
* a request that cannot be reproduced byte-for-byte cannot be safely bound to a single-use grant.
|
||||
*/
|
||||
function canonicalize(value: unknown): string | undefined {
|
||||
function canonicalize(
|
||||
value: unknown,
|
||||
state: { nodes: number } = { nodes: 0 },
|
||||
depth = 0,
|
||||
): string | undefined {
|
||||
if (
|
||||
depth > MEMORY_BROKER_MAXIMUM_CANONICAL_DEPTH ||
|
||||
++state.nodes > MEMORY_BROKER_MAXIMUM_CANONICAL_NODES
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
if (value === null) {
|
||||
return "null";
|
||||
}
|
||||
@@ -77,7 +106,7 @@ function canonicalize(value: unknown): string | undefined {
|
||||
if (Array.isArray(value)) {
|
||||
const items: string[] = [];
|
||||
for (const item of value) {
|
||||
const serialized = canonicalize(item);
|
||||
const serialized = canonicalize(item, state, depth + 1);
|
||||
if (serialized === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
@@ -90,7 +119,7 @@ function canonicalize(value: unknown): string | undefined {
|
||||
}
|
||||
const entries: string[] = [];
|
||||
for (const key of Object.keys(value).toSorted()) {
|
||||
const serialized = canonicalize(value[key]);
|
||||
const serialized = canonicalize(value[key], state, depth + 1);
|
||||
if (serialized === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
@@ -125,24 +154,42 @@ function hasSafeIdentifier(value: unknown): value is string {
|
||||
);
|
||||
}
|
||||
|
||||
function isActorBinding(value: unknown): value is MemoryBrokerActorBinding {
|
||||
if (!isPlainRecord(value)) {
|
||||
return false;
|
||||
}
|
||||
if (value.kind === "principal") {
|
||||
return (
|
||||
(value.actorKind === "human" ||
|
||||
value.actorKind === "agent" ||
|
||||
value.actorKind === "service" ||
|
||||
value.actorKind === "system") &&
|
||||
hasSafeIdentifier(value.principalId)
|
||||
);
|
||||
}
|
||||
return value.kind === "unattributed" && hasSafeIdentifier(value.transportAuditRef);
|
||||
}
|
||||
|
||||
function isBinding(value: unknown): value is MemoryBrokerAuthorizationBinding {
|
||||
if (!isPlainRecord(value)) {
|
||||
return false;
|
||||
}
|
||||
return [
|
||||
value.agentId,
|
||||
value.sessionId,
|
||||
value.runId,
|
||||
value.contextFingerprint,
|
||||
value.subjectRevision,
|
||||
value.actorRevision,
|
||||
value.capabilitySnapshotId,
|
||||
value.policyRevision,
|
||||
value.deliveryRevision,
|
||||
].every(hasSafeIdentifier);
|
||||
return (
|
||||
[
|
||||
value.agentId,
|
||||
value.sessionId,
|
||||
value.runId,
|
||||
value.contextFingerprint,
|
||||
value.subjectRevision,
|
||||
value.actorRevision,
|
||||
value.capabilitySnapshotId,
|
||||
value.policyRevision,
|
||||
value.deliveryRevision,
|
||||
].every(hasSafeIdentifier) && isActorBinding(value.actor)
|
||||
);
|
||||
}
|
||||
|
||||
function isEnvelope(value: unknown): value is MemoryBrokerEnvelope {
|
||||
export function isMemoryBrokerEnvelope(value: unknown): value is MemoryBrokerEnvelope {
|
||||
if (
|
||||
!isPlainRecord(value) ||
|
||||
value.version !== MEMORY_BROKER_PROTOCOL_VERSION ||
|
||||
@@ -185,7 +232,7 @@ export function createMemoryBrokerEnvelope(params: {
|
||||
expiresAtMs: params.expiresAtMs,
|
||||
requestDigest: requestDigest ?? "",
|
||||
};
|
||||
if (!requestDigest || !isEnvelope({ ...unsigned, signature: "pending" })) {
|
||||
if (!requestDigest || !isMemoryBrokerEnvelope({ ...unsigned, signature: "pending" })) {
|
||||
return undefined;
|
||||
}
|
||||
const signature = sign(params.secret, unsigned);
|
||||
@@ -203,7 +250,7 @@ export function verifyMemoryBrokerEnvelope(params: {
|
||||
nowMs: number;
|
||||
}): MemoryBrokerEnvelopeVerification {
|
||||
const { envelope } = params;
|
||||
if (!isEnvelope(envelope)) {
|
||||
if (!isMemoryBrokerEnvelope(envelope)) {
|
||||
return { ok: false, reason: "invalid-envelope" };
|
||||
}
|
||||
if (envelope.brokerId !== params.brokerId) {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { mkdtemp, rm, stat } from "node:fs/promises";
|
||||
import net from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
@@ -7,6 +8,7 @@ import { requestJsonlSocket } from "../infra/jsonl-socket.js";
|
||||
import { createMemoryBrokerClient } from "./client.js";
|
||||
import {
|
||||
createMemoryBrokerEnvelope,
|
||||
MEMORY_BROKER_MAXIMUM_CANONICAL_DEPTH,
|
||||
type MemoryBrokerAuthorizationBinding,
|
||||
type MemoryBrokerRequest,
|
||||
} from "./protocol.js";
|
||||
@@ -19,6 +21,7 @@ const binding: MemoryBrokerAuthorizationBinding = {
|
||||
runId: "run-a",
|
||||
contextFingerprint: "context-a",
|
||||
subjectRevision: "subject-a",
|
||||
actor: { kind: "principal", actorKind: "human", principalId: "alice" },
|
||||
actorRevision: "actor-a",
|
||||
capabilitySnapshotId: "capability-a",
|
||||
policyRevision: "policy-a",
|
||||
@@ -32,7 +35,7 @@ let server: MemoryBrokerServer | undefined;
|
||||
type StartOptions = Partial<
|
||||
Pick<
|
||||
Parameters<typeof startMemoryBrokerServer>[0],
|
||||
"handler" | "maximumPending" | "maximumRunning"
|
||||
"handler" | "maximumPending" | "maximumRunning" | "maximumConnections" | "preauthIdleTimeoutMs"
|
||||
>
|
||||
>;
|
||||
|
||||
@@ -77,16 +80,35 @@ function frame(overrides: Partial<{ envelope: unknown; request: unknown; nonce:
|
||||
return { envelope, request, ...overrides };
|
||||
}
|
||||
|
||||
async function send(socketPath: string, value: unknown, options: { keepWriteOpen?: boolean } = {}) {
|
||||
async function send(
|
||||
socketPath: string,
|
||||
value: unknown,
|
||||
options: { keepWriteOpen?: boolean; timeoutMs?: number } = {},
|
||||
) {
|
||||
return await requestJsonlSocket({
|
||||
socketPath,
|
||||
requestLine: JSON.stringify(value),
|
||||
timeoutMs: 5_000,
|
||||
requestLine: typeof value === "string" ? value : JSON.stringify(value),
|
||||
timeoutMs: options.timeoutMs ?? 5_000,
|
||||
...options,
|
||||
accept: (response) => response as { ok: boolean; value?: unknown; error?: string },
|
||||
});
|
||||
}
|
||||
|
||||
async function openPartialSocket(socketPath: string): Promise<net.Socket> {
|
||||
return await new Promise<net.Socket>((resolve, reject) => {
|
||||
const socket = net.createConnection(socketPath);
|
||||
socket.once("connect", () => resolve(socket));
|
||||
socket.once("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForSocketClose(socket: net.Socket): Promise<void> {
|
||||
if (socket.destroyed) {
|
||||
return;
|
||||
}
|
||||
await new Promise<void>((resolve) => socket.once("close", () => resolve()));
|
||||
}
|
||||
|
||||
describe("memory broker server", () => {
|
||||
it("keeps the local socket private and admits one signed request", async () => {
|
||||
const socketPath = await start();
|
||||
@@ -280,6 +302,124 @@ describe("memory broker server", () => {
|
||||
await expect(handlerAborted).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("reclaims a cancelled queued request before it can exhaust the broker admission limit", async () => {
|
||||
let release: (() => void) | undefined;
|
||||
let started: (() => void) | undefined;
|
||||
const handlerStarted = new Promise<void>((resolve) => {
|
||||
started = resolve;
|
||||
});
|
||||
const handlerRelease = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const socketPath = await start({
|
||||
maximumPending: 1,
|
||||
maximumRunning: 1,
|
||||
handler: async () => {
|
||||
started?.();
|
||||
await handlerRelease;
|
||||
return { ok: true };
|
||||
},
|
||||
});
|
||||
const first = send(socketPath, frame({ nonce: "nonce-running" }), { keepWriteOpen: true });
|
||||
await handlerStarted;
|
||||
|
||||
const controller = new AbortController();
|
||||
const client = createMemoryBrokerClient({
|
||||
socketPath,
|
||||
brokerId: "broker-a",
|
||||
brokerEpoch: "epoch-a",
|
||||
secret,
|
||||
nonce: () => "nonce-cancelled-queued",
|
||||
});
|
||||
const cancelled = client.request({
|
||||
binding,
|
||||
method: "memory.search",
|
||||
payload: { query: "queued then cancelled" },
|
||||
expiresAtMs: Date.now() + 60_000,
|
||||
signal: controller.signal,
|
||||
});
|
||||
// Let the client finish its connect/write turn, then prove it holds the sole pending slot.
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 20));
|
||||
await expect(send(socketPath, frame({ nonce: "nonce-before-cancel" }))).resolves.toEqual({
|
||||
ok: false,
|
||||
error: "busy",
|
||||
});
|
||||
|
||||
controller.abort();
|
||||
await expect(cancelled).resolves.toBeUndefined();
|
||||
const reclaimed = send(socketPath, frame({ nonce: "nonce-after-cancel" }), {
|
||||
keepWriteOpen: true,
|
||||
});
|
||||
// The admission decision must happen while the first handler is still saturated. Without
|
||||
// queue-owned cancellation, this request is synchronously rejected as busy.
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 20));
|
||||
release?.();
|
||||
await expect(first).resolves.toEqual({ ok: true, value: { ok: true } });
|
||||
await expect(reclaimed).resolves.toEqual({ ok: true, value: { ok: true } });
|
||||
});
|
||||
|
||||
it("expires queued work without waiting for a running handler to drain", async () => {
|
||||
let release: (() => void) | undefined;
|
||||
let started: (() => void) | undefined;
|
||||
const handlerStarted = new Promise<void>((resolve) => {
|
||||
started = resolve;
|
||||
});
|
||||
const handlerRelease = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const socketPath = await start({
|
||||
maximumPending: 1,
|
||||
maximumRunning: 1,
|
||||
handler: async () => {
|
||||
started?.();
|
||||
await handlerRelease;
|
||||
return { ok: true };
|
||||
},
|
||||
});
|
||||
const first = send(socketPath, frame({ nonce: "nonce-running" }), { keepWriteOpen: true });
|
||||
await handlerStarted;
|
||||
|
||||
const expiresAtMs = Date.now() + 100;
|
||||
const expired = send(
|
||||
socketPath,
|
||||
frame({
|
||||
nonce: "nonce-queued-expiry",
|
||||
envelope: createMemoryBrokerEnvelope({
|
||||
secret,
|
||||
brokerId: "broker-a",
|
||||
brokerEpoch: "epoch-a",
|
||||
binding,
|
||||
nonce: "nonce-queued-expiry",
|
||||
request,
|
||||
issuedAtMs: Date.now(),
|
||||
expiresAtMs,
|
||||
}),
|
||||
}),
|
||||
{ keepWriteOpen: true, timeoutMs: 1_000 },
|
||||
);
|
||||
await expect(expired).resolves.toEqual({ ok: false, error: "cancelled" });
|
||||
|
||||
const reclaimed = send(socketPath, frame({ nonce: "nonce-after-expiry" }), {
|
||||
keepWriteOpen: true,
|
||||
});
|
||||
release?.();
|
||||
await expect(first).resolves.toEqual({ ok: true, value: { ok: true } });
|
||||
await expect(reclaimed).resolves.toEqual({ ok: true, value: { ok: true } });
|
||||
});
|
||||
|
||||
it("rejects a deeply nested forged request without unwinding the socket callback", async () => {
|
||||
const socketPath = await start();
|
||||
const envelope = frame({ nonce: "nonce-deep-forgery" }).envelope;
|
||||
const depth = MEMORY_BROKER_MAXIMUM_CANONICAL_DEPTH * 512;
|
||||
const payload = `${"[".repeat(depth)}0${"]".repeat(depth)}`;
|
||||
const rawFrame = `{"envelope":${JSON.stringify(envelope)},"request":{"method":"memory.search","payload":${payload}}}`;
|
||||
|
||||
await expect(send(socketPath, rawFrame, { timeoutMs: 1_000 })).resolves.toEqual({
|
||||
ok: false,
|
||||
error: "unauthorized",
|
||||
});
|
||||
});
|
||||
|
||||
it("cancels an in-flight handler at the signed request deadline", async () => {
|
||||
let started: (() => void) | undefined;
|
||||
const handlerStarted = new Promise<void>((resolve) => {
|
||||
@@ -314,4 +454,66 @@ describe("memory broker server", () => {
|
||||
await handlerStarted;
|
||||
await expect(pending).resolves.toEqual({ ok: false, error: "cancelled" });
|
||||
});
|
||||
|
||||
it("acknowledges a durable mutation that commits as its reply deadline expires", async () => {
|
||||
let started: (() => void) | undefined;
|
||||
const handlerStarted = new Promise<void>((resolve) => {
|
||||
started = resolve;
|
||||
});
|
||||
const socketPath = await start({
|
||||
handler: async ({ signal }) => {
|
||||
started?.();
|
||||
await new Promise<void>((resolve) => {
|
||||
signal.addEventListener("abort", () => resolve(), { once: true });
|
||||
});
|
||||
return { status: "committed" };
|
||||
},
|
||||
});
|
||||
const mutationRequest: MemoryBrokerRequest = {
|
||||
method: "memory.write",
|
||||
payload: { mutation: "durably-activated" },
|
||||
};
|
||||
const deadline = Date.now() + 100;
|
||||
const pending = send(
|
||||
socketPath,
|
||||
frame({
|
||||
nonce: "nonce-committed-after-deadline",
|
||||
request: mutationRequest,
|
||||
envelope: createMemoryBrokerEnvelope({
|
||||
secret,
|
||||
brokerId: "broker-a",
|
||||
brokerEpoch: "epoch-a",
|
||||
binding,
|
||||
nonce: "nonce-committed-after-deadline",
|
||||
request: mutationRequest,
|
||||
issuedAtMs: Date.now(),
|
||||
expiresAtMs: deadline,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
await handlerStarted;
|
||||
await expect(pending).resolves.toEqual({ ok: true, value: { status: "committed" } });
|
||||
});
|
||||
|
||||
it("bounds unauthenticated partial-frame sockets and reclaims their admission slots", async () => {
|
||||
const socketPath = await start({ maximumConnections: 1, preauthIdleTimeoutMs: 40 });
|
||||
const slowloris = await openPartialSocket(socketPath);
|
||||
slowloris.write('{"envelope":');
|
||||
|
||||
await expect(send(socketPath, frame({ nonce: "nonce-over-limit" }))).resolves.toBeNull();
|
||||
await waitForSocketClose(slowloris);
|
||||
await expect(send(socketPath, frame({ nonce: "nonce-after-idle" }))).resolves.toMatchObject({
|
||||
ok: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("tears down partial-frame sockets before broker shutdown", async () => {
|
||||
const socketPath = await start({ preauthIdleTimeoutMs: 60_000 });
|
||||
const slowloris = await openPartialSocket(socketPath);
|
||||
slowloris.write('{"envelope":');
|
||||
|
||||
await expect(server!.close()).resolves.toBeUndefined();
|
||||
await waitForSocketClose(slowloris);
|
||||
await expect(server!.close()).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
+147
-34
@@ -3,13 +3,15 @@ import net from "node:net";
|
||||
import { isRecord } from "@openclaw/normalization-core/record-coerce";
|
||||
import {
|
||||
MemoryBrokerNonceLedger,
|
||||
isMemoryBrokerEnvelope,
|
||||
verifyMemoryBrokerEnvelope,
|
||||
type MemoryBrokerAuthorizationBinding,
|
||||
type MemoryBrokerEnvelope,
|
||||
type MemoryBrokerRequest,
|
||||
} from "./protocol.js";
|
||||
|
||||
export const MEMORY_BROKER_MAXIMUM_REQUEST_BYTES = 1_048_576;
|
||||
export const MEMORY_BROKER_MAXIMUM_CONNECTIONS = 32;
|
||||
export const MEMORY_BROKER_PREAUTH_IDLE_TIMEOUT_MS = 5_000;
|
||||
|
||||
type MemoryBrokerWireRequest = Readonly<{
|
||||
envelope: unknown;
|
||||
@@ -34,6 +36,7 @@ type PendingMemoryBrokerRequest = Readonly<{
|
||||
deadlineMs: number;
|
||||
execute: () => Promise<void>;
|
||||
rejectBusy: () => void;
|
||||
rejectCancelled: () => void;
|
||||
}>;
|
||||
|
||||
/**
|
||||
@@ -43,6 +46,7 @@ type PendingMemoryBrokerRequest = Readonly<{
|
||||
class MemoryBrokerAdmissionQueue {
|
||||
private readonly pending: PendingMemoryBrokerRequest[] = [];
|
||||
private readonly idleWaiters = new Set<() => void>();
|
||||
private deadlineTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
private running = 0;
|
||||
private accepting = true;
|
||||
|
||||
@@ -61,17 +65,21 @@ class MemoryBrokerAdmissionQueue {
|
||||
}
|
||||
}
|
||||
|
||||
submit(request: PendingMemoryBrokerRequest): void {
|
||||
submit(request: PendingMemoryBrokerRequest): () => void {
|
||||
if (
|
||||
!this.accepting ||
|
||||
request.deadlineMs <= this.now() ||
|
||||
(this.running >= this.maximumRunning && this.pending.length >= this.maximumPending)
|
||||
) {
|
||||
request.rejectBusy();
|
||||
return;
|
||||
return () => {};
|
||||
}
|
||||
if (request.deadlineMs <= this.now()) {
|
||||
request.rejectCancelled();
|
||||
return () => {};
|
||||
}
|
||||
this.pending.push(request);
|
||||
this.drain();
|
||||
return () => this.cancel(request);
|
||||
}
|
||||
|
||||
async quiesce(): Promise<void> {
|
||||
@@ -101,7 +109,7 @@ class MemoryBrokerAdmissionQueue {
|
||||
while (this.running < this.maximumRunning && this.pending.length > 0) {
|
||||
const next = this.pending.shift()!;
|
||||
if (next.deadlineMs <= this.now()) {
|
||||
next.rejectBusy();
|
||||
next.rejectCancelled();
|
||||
continue;
|
||||
}
|
||||
this.running += 1;
|
||||
@@ -111,8 +119,50 @@ class MemoryBrokerAdmissionQueue {
|
||||
this.notifyIdle();
|
||||
});
|
||||
}
|
||||
this.schedulePendingDeadline();
|
||||
this.notifyIdle();
|
||||
}
|
||||
|
||||
private cancel(request: PendingMemoryBrokerRequest): void {
|
||||
const index = this.pending.indexOf(request);
|
||||
if (index === -1) {
|
||||
return;
|
||||
}
|
||||
this.pending.splice(index, 1);
|
||||
this.schedulePendingDeadline();
|
||||
this.drain();
|
||||
}
|
||||
|
||||
private schedulePendingDeadline(): void {
|
||||
if (this.deadlineTimer) {
|
||||
clearTimeout(this.deadlineTimer);
|
||||
this.deadlineTimer = undefined;
|
||||
}
|
||||
const nextDeadline = this.pending.reduce<number | undefined>(
|
||||
(earliest, request) =>
|
||||
earliest === undefined || request.deadlineMs < earliest ? request.deadlineMs : earliest,
|
||||
undefined,
|
||||
);
|
||||
if (nextDeadline === undefined) {
|
||||
return;
|
||||
}
|
||||
this.deadlineTimer = setTimeout(
|
||||
() => {
|
||||
this.deadlineTimer = undefined;
|
||||
const now = this.now();
|
||||
for (let index = this.pending.length - 1; index >= 0; index -= 1) {
|
||||
const request = this.pending[index];
|
||||
if (request.deadlineMs <= now) {
|
||||
this.pending.splice(index, 1);
|
||||
request.rejectCancelled();
|
||||
}
|
||||
}
|
||||
this.drain();
|
||||
},
|
||||
Math.max(1, nextDeadline - this.now()),
|
||||
);
|
||||
this.deadlineTimer.unref?.();
|
||||
}
|
||||
}
|
||||
|
||||
function parseRequest(value: unknown): MemoryBrokerWireRequest | undefined {
|
||||
@@ -140,15 +190,12 @@ function writeResponse(socket: net.Socket, response: MemoryBrokerWireResponse):
|
||||
}
|
||||
}
|
||||
|
||||
function asEnvelope(value: unknown): MemoryBrokerEnvelope | undefined {
|
||||
if (
|
||||
!isRecord(value) ||
|
||||
typeof value.nonce !== "string" ||
|
||||
typeof value.expiresAtMs !== "number"
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return value as unknown as MemoryBrokerEnvelope;
|
||||
/**
|
||||
* A successful mutation handler has crossed its durable activation boundary. Its response must
|
||||
* remain committed even if the caller disconnects while the broker is serializing the reply.
|
||||
*/
|
||||
function isDurableMemoryMutation(request: MemoryBrokerRequest): boolean {
|
||||
return request.method === "memory.write" || request.method === "memory.import";
|
||||
}
|
||||
|
||||
export type MemoryBrokerServer = Readonly<{
|
||||
@@ -174,53 +221,96 @@ export async function startMemoryBrokerServer(params: {
|
||||
nonceCapacity?: number;
|
||||
maximumPending?: number;
|
||||
maximumRunning?: number;
|
||||
/** Bound unauthenticated sockets before they can consume a broker worker slot. */
|
||||
maximumConnections?: number;
|
||||
/** A partial frame is untrusted admission state, never an indefinitely retained request. */
|
||||
preauthIdleTimeoutMs?: number;
|
||||
maximumRequestBytes?: number;
|
||||
now?: () => number;
|
||||
}): Promise<MemoryBrokerServer> {
|
||||
const now = params.now ?? Date.now;
|
||||
const maximumRequestBytes = params.maximumRequestBytes ?? MEMORY_BROKER_MAXIMUM_REQUEST_BYTES;
|
||||
const maximumConnections = params.maximumConnections ?? MEMORY_BROKER_MAXIMUM_CONNECTIONS;
|
||||
const preauthIdleTimeoutMs = params.preauthIdleTimeoutMs ?? MEMORY_BROKER_PREAUTH_IDLE_TIMEOUT_MS;
|
||||
if (
|
||||
!Number.isSafeInteger(maximumConnections) ||
|
||||
maximumConnections < 1 ||
|
||||
!Number.isSafeInteger(preauthIdleTimeoutMs) ||
|
||||
preauthIdleTimeoutMs < 1
|
||||
) {
|
||||
throw new Error("memory broker connection limits are invalid");
|
||||
}
|
||||
const nonceLedger = new MemoryBrokerNonceLedger(params.nonceCapacity ?? 1_024);
|
||||
const queue = new MemoryBrokerAdmissionQueue(
|
||||
params.maximumPending ?? 128,
|
||||
params.maximumRunning ?? 8,
|
||||
now,
|
||||
);
|
||||
const sockets = new Set<net.Socket>();
|
||||
const server = net.createServer({ allowHalfOpen: true }, (socket) => {
|
||||
let buffer = Buffer.alloc(0);
|
||||
// A client that has not completed one bounded signed frame has no broker authority. Keep its
|
||||
// footprint bounded before parsing so partial-frame clients cannot exhaust the broker.
|
||||
if (sockets.size >= maximumConnections) {
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
sockets.add(socket);
|
||||
// Do not repeatedly concatenate an attacker-controlled partial frame: a byte-at-a-time
|
||||
// slowloris would turn that into quadratic copying before the absolute admission deadline.
|
||||
const frameChunks: Buffer[] = [];
|
||||
let frameByteLength = 0;
|
||||
let consumed = false;
|
||||
const requestAbort = new AbortController();
|
||||
const preauthIdleTimer = setTimeout(() => {
|
||||
if (!consumed) {
|
||||
consumed = true;
|
||||
requestAbort.abort();
|
||||
socket.destroy();
|
||||
}
|
||||
}, preauthIdleTimeoutMs);
|
||||
preauthIdleTimer.unref?.();
|
||||
socket.on("end", () => requestAbort.abort());
|
||||
socket.on("close", () => requestAbort.abort());
|
||||
socket.once("close", () => {
|
||||
clearTimeout(preauthIdleTimer);
|
||||
requestAbort.abort();
|
||||
sockets.delete(socket);
|
||||
});
|
||||
socket.on("error", () => requestAbort.abort());
|
||||
socket.on("data", (chunk: Buffer) => {
|
||||
if (consumed) {
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
if (buffer.byteLength + chunk.byteLength > maximumRequestBytes) {
|
||||
if (frameByteLength + chunk.byteLength > maximumRequestBytes) {
|
||||
consumed = true;
|
||||
writeResponse(socket, { ok: false, error: "invalid-request" });
|
||||
return;
|
||||
}
|
||||
buffer = Buffer.concat([buffer, chunk]);
|
||||
const newline = buffer.indexOf(0x0a);
|
||||
frameChunks.push(chunk);
|
||||
frameByteLength += chunk.byteLength;
|
||||
// Every prior chunk has been checked and contains no newline, so only the latest chunk
|
||||
// needs scanning until there is one complete frame to concatenate exactly once.
|
||||
const newline = chunk.indexOf(0x0a);
|
||||
if (newline === -1) {
|
||||
return;
|
||||
}
|
||||
consumed = true;
|
||||
if (buffer.subarray(newline + 1).some((byte) => byte !== 0x0a && byte !== 0x0d)) {
|
||||
clearTimeout(preauthIdleTimer);
|
||||
const buffer = Buffer.concat(frameChunks, frameByteLength);
|
||||
const frameNewline = buffer.indexOf(0x0a);
|
||||
if (buffer.subarray(frameNewline + 1).some((byte) => byte !== 0x0a && byte !== 0x0d)) {
|
||||
writeResponse(socket, { ok: false, error: "invalid-request" });
|
||||
return;
|
||||
}
|
||||
let frame: MemoryBrokerWireRequest | undefined;
|
||||
try {
|
||||
frame = parseRequest(JSON.parse(buffer.subarray(0, newline).toString("utf8")));
|
||||
frame = parseRequest(JSON.parse(buffer.subarray(0, frameNewline).toString("utf8")));
|
||||
} catch {
|
||||
frame = undefined;
|
||||
}
|
||||
const request = frame ? parseMemoryBrokerRequest(frame.request) : undefined;
|
||||
const envelope = frame ? asEnvelope(frame.envelope) : undefined;
|
||||
if (!request || !envelope) {
|
||||
const envelope = frame?.envelope;
|
||||
if (!request || !isMemoryBrokerEnvelope(envelope)) {
|
||||
writeResponse(socket, { ok: false, error: "invalid-request" });
|
||||
return;
|
||||
}
|
||||
@@ -246,9 +336,10 @@ export async function startMemoryBrokerServer(params: {
|
||||
writeResponse(socket, { ok: false, error: "replayed" });
|
||||
return;
|
||||
}
|
||||
queue.submit({
|
||||
const cancelPending = queue.submit({
|
||||
deadlineMs: envelope.expiresAtMs,
|
||||
rejectBusy: () => writeResponse(socket, { ok: false, error: "busy" }),
|
||||
rejectCancelled: () => writeResponse(socket, { ok: false, error: "cancelled" }),
|
||||
execute: async () => {
|
||||
if (requestAbort.signal.aborted || now() >= envelope.expiresAtMs) {
|
||||
writeResponse(socket, { ok: false, error: "cancelled" });
|
||||
@@ -264,18 +355,31 @@ export async function startMemoryBrokerServer(params: {
|
||||
request,
|
||||
signal: requestAbort.signal,
|
||||
});
|
||||
if (requestAbort.signal.aborted || now() >= envelope.expiresAtMs) {
|
||||
if (
|
||||
!isDurableMemoryMutation(request) &&
|
||||
(requestAbort.signal.aborted || now() >= envelope.expiresAtMs)
|
||||
) {
|
||||
writeResponse(socket, { ok: false, error: "cancelled" });
|
||||
return;
|
||||
}
|
||||
writeResponse(socket, { ok: true, value });
|
||||
} catch {
|
||||
writeResponse(socket, { ok: false, error: "failed" });
|
||||
writeResponse(socket, {
|
||||
ok: false,
|
||||
error:
|
||||
requestAbort.signal.aborted || now() >= envelope.expiresAtMs
|
||||
? "cancelled"
|
||||
: "failed",
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(deadline);
|
||||
}
|
||||
},
|
||||
});
|
||||
requestAbort.signal.addEventListener("abort", cancelPending, { once: true });
|
||||
if (requestAbort.signal.aborted) {
|
||||
cancelPending();
|
||||
}
|
||||
});
|
||||
});
|
||||
await unlink(params.socketPath).catch((error: unknown) => {
|
||||
@@ -291,20 +395,29 @@ export async function startMemoryBrokerServer(params: {
|
||||
});
|
||||
});
|
||||
await chmod(params.socketPath, 0o600);
|
||||
let closePromise: Promise<void> | undefined;
|
||||
return Object.freeze({
|
||||
socketPath: params.socketPath,
|
||||
brokerEpoch: params.brokerEpoch,
|
||||
quiesce: () => queue.quiesce(),
|
||||
resume: () => queue.resume(),
|
||||
close: async () => {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
await unlink(params.socketPath).catch((error: unknown) => {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
throw error;
|
||||
close: () => {
|
||||
closePromise ??= (async () => {
|
||||
// net.Server.close waits for every accepted socket. Destroy incomplete and in-flight
|
||||
// connections first so shutdown cannot be held hostage by a slowloris or stalled client.
|
||||
for (const socket of sockets) {
|
||||
socket.destroy();
|
||||
}
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
await unlink(params.socketPath).catch((error: unknown) => {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
})();
|
||||
return closePromise;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,3 +1,12 @@
|
||||
let initializedAgentIds = [];
|
||||
|
||||
export async function initializeMemoryBroker({ agentIds }) {
|
||||
if (agentIds.includes("fail-startup")) {
|
||||
throw new Error("test broker startup recovery failed");
|
||||
}
|
||||
initializedAgentIds = [...agentIds];
|
||||
}
|
||||
|
||||
export function createMemoryBrokerHandler() {
|
||||
return async ({ binding, request }) => {
|
||||
if (request.method === "memory.crash") {
|
||||
@@ -6,6 +15,20 @@ export function createMemoryBrokerHandler() {
|
||||
setImmediate(() => process.exit(1));
|
||||
return await new Promise(() => {});
|
||||
}
|
||||
if (request.method === "memory.kill") {
|
||||
// An external SIGKILL records `signalCode`, not `exitCode`. The parent must still retire
|
||||
// this process instead of waiting for an exit event that has already happened.
|
||||
setImmediate(() => process.kill(process.pid, "SIGKILL"));
|
||||
return await new Promise(() => {});
|
||||
}
|
||||
if (request.method === "memory.hang") {
|
||||
// This intentionally ignores the broker AbortSignal. Maintenance must retire the child
|
||||
// rather than leave an admission-closed but otherwise healthy process behind.
|
||||
return await new Promise(() => {});
|
||||
}
|
||||
if (request.method === "memory.startup") {
|
||||
return { agentIds: initializedAgentIds };
|
||||
}
|
||||
return request.method === "memory.environment"
|
||||
? { parentSecret: process.env.OPENCLAW_MEMORY_BROKER_TEST_SECRET ? "present" : "absent" }
|
||||
: { agentId: binding.agentId, method: request.method };
|
||||
|
||||
@@ -18,6 +18,7 @@ const envelope = {
|
||||
runId: "run-b",
|
||||
contextFingerprint: "context-b",
|
||||
subjectRevision: "subject-b",
|
||||
actor: { kind: "principal", actorKind: "human", principalId: "mallory" },
|
||||
actorRevision: "actor-b",
|
||||
capabilitySnapshotId: "capability-b",
|
||||
policyRevision: "policy-b",
|
||||
|
||||
@@ -9,10 +9,7 @@ import {
|
||||
type NodeWorkerLaunchClaimResult,
|
||||
type NodeWorkerLaunchReceipt,
|
||||
} from "./node-worker-launch-store.js";
|
||||
import {
|
||||
inspectNodeWorkerProcessIdentity,
|
||||
type NodeWorkerProcessIdentity,
|
||||
} from "./node-worker-process-identity.js";
|
||||
import { type NodeWorkerProcessIdentity } from "./node-worker-process-identity.js";
|
||||
|
||||
const DEFAULT_WORKER_CAPACITY = 2;
|
||||
const DEFAULT_CAPACITY_WAIT_MS = 10_000;
|
||||
@@ -69,28 +66,14 @@ export class NodeWorkerCapacity {
|
||||
}
|
||||
|
||||
async initialize(
|
||||
recoverRunning: (receipt: NodeWorkerLaunchReceipt) => Promise<void>,
|
||||
recoverNonterminal: (receipt: NodeWorkerLaunchReceipt) => Promise<void>,
|
||||
): Promise<void> {
|
||||
this.onCapacityChanged?.(this.publishedCapacity);
|
||||
for (const receipt of this.store.listNonterminal()) {
|
||||
if (receipt.state === "pending") {
|
||||
const supervisorState = inspectNodeWorkerProcessIdentity(receipt.supervisor);
|
||||
if (supervisorState === "dead" || supervisorState === "reused") {
|
||||
this.finish(
|
||||
{
|
||||
launchId: receipt.launchId,
|
||||
planHash: receipt.planHash,
|
||||
supervisor: receipt.supervisor,
|
||||
worker: null,
|
||||
state: "interrupted",
|
||||
errorText: "node host stopped before the worker launch started",
|
||||
},
|
||||
false,
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
await recoverRunning(receipt);
|
||||
// Pending work can already own a staged projection or relay directory.
|
||||
// Recovery therefore belongs to the supervisor, which owns those paths,
|
||||
// rather than directly settling the launch slot here.
|
||||
await recoverNonterminal(receipt);
|
||||
}
|
||||
this.store.pruneExpiredTerminal();
|
||||
this.refresh(true);
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { verifyNodeWorkerContainerProjectionIsolation } from "../../test/helpers/node-worker-container-projection-isolation.js";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import type { AuthorizedMemoryVirtualFileBroker } from "../agents/memory-authorized-read-host.js";
|
||||
import { execContainer } from "../agents/sandbox/docker.js";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
|
||||
import {
|
||||
buildNodeWorkerContainerRunArgs,
|
||||
removeOwnedNodeWorkerContainers,
|
||||
resolveNodeWorkerContainerEngine,
|
||||
type NodeWorkerContainerEngine,
|
||||
} from "./node-worker-container-runtime.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
afterEach(() => {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
});
|
||||
|
||||
function broker(
|
||||
expiresAt = new Date(Date.now() + 60_000).toISOString(),
|
||||
): AuthorizedMemoryVirtualFileBroker {
|
||||
const files = new Map([["shared/brief.md", "only this issued virtual view"]]);
|
||||
return {
|
||||
view: {
|
||||
version: 1,
|
||||
viewId: "container-e2e-view",
|
||||
planId: "container-e2e-plan",
|
||||
contextFingerprint: "container-e2e-context",
|
||||
revision: "container-e2e-revision",
|
||||
roots: [
|
||||
{
|
||||
version: 1,
|
||||
mountHandle: "container-e2e-mount",
|
||||
virtualRoot: "shared",
|
||||
access: "read",
|
||||
},
|
||||
],
|
||||
files: [
|
||||
{
|
||||
version: 1,
|
||||
mountHandle: "container-e2e-mount",
|
||||
virtualPath: "shared/brief.md",
|
||||
},
|
||||
],
|
||||
expiresAt,
|
||||
},
|
||||
readFile: async (virtualPath) => files.get(virtualPath),
|
||||
};
|
||||
}
|
||||
|
||||
describe.runIf(process.env.OPENCLAW_PROCESS_ISOLATION_E2E === "1")(
|
||||
"node worker container process isolation",
|
||||
() => {
|
||||
it("exposes only an immutable issued memory snapshot to the hostile worker process", async () => {
|
||||
const root = tempDirs.make("node-worker-container-isolation-");
|
||||
const bundleDir = path.join(root, "bundle");
|
||||
const relayDir = path.join(root, "relay");
|
||||
const memoryDir = path.join(root, "memory");
|
||||
const workspaceDir = path.join(root, "workspace");
|
||||
const outsideArtifact = path.join(root, "host-artifact.sqlite");
|
||||
const identity = { launchId: "hostile-worker-turn", planHash: "a".repeat(64) };
|
||||
for (const directory of [bundleDir, relayDir, memoryDir, workspaceDir]) {
|
||||
fs.mkdirSync(directory, { recursive: true });
|
||||
}
|
||||
fs.mkdirSync(path.join(memoryDir, "shared"));
|
||||
fs.writeFileSync(path.join(memoryDir, "shared", "issued.md"), "issued memory only", {
|
||||
mode: 0o400,
|
||||
});
|
||||
fs.chmodSync(path.join(memoryDir, "shared"), 0o500);
|
||||
fs.writeFileSync(outsideArtifact, "host-only artifact");
|
||||
fs.writeFileSync(
|
||||
path.join(bundleDir, "worker.mjs"),
|
||||
[
|
||||
'import fs from "node:fs";',
|
||||
`const outsideArtifact = ${JSON.stringify(outsideArtifact)};`,
|
||||
'const attempt = (operation) => { try { operation(); return "allowed"; } catch (error) { return error && typeof error === "object" && "code" in error ? String(error.code) : "denied"; } };',
|
||||
"const result = {",
|
||||
" uid: process.getuid(),",
|
||||
' issued: fs.readFileSync("/memory/shared/issued.md", "utf8"),',
|
||||
' memoryWrite: attempt(() => fs.writeFileSync("/memory/shared/issued.md", "tampered")),',
|
||||
' rawArtifactRead: attempt(() => fs.readFileSync(outsideArtifact, "utf8")),',
|
||||
' workspaceWrite: attempt(() => fs.writeFileSync("/workspace/proof.txt", "allowed")),',
|
||||
"};",
|
||||
"process.stdout.write(JSON.stringify(result));",
|
||||
].join("\n"),
|
||||
{ mode: 0o500 },
|
||||
);
|
||||
let engine: NodeWorkerContainerEngine | undefined;
|
||||
try {
|
||||
engine = await resolveNodeWorkerContainerEngine();
|
||||
if (!engine) {
|
||||
throw new Error(
|
||||
"process-isolation proof requires an eligible Docker or Podman node host",
|
||||
);
|
||||
}
|
||||
await removeOwnedNodeWorkerContainers(identity, engine);
|
||||
const user = process.getuid?.();
|
||||
const group = process.getgid?.();
|
||||
if (!user || !group) {
|
||||
throw new Error("process-isolation proof requires a non-root POSIX test host");
|
||||
}
|
||||
const args = buildNodeWorkerContainerRunArgs({
|
||||
engine,
|
||||
identity,
|
||||
mounts: { bundleDir, relayDir, memoryDir, workspaceDir },
|
||||
uid: user,
|
||||
gid: group,
|
||||
});
|
||||
const result = await execContainer(engine, args, { allowFailure: true });
|
||||
expect(result.code, result.stderr).toBe(0);
|
||||
const observed: unknown = JSON.parse(result.stdout);
|
||||
expect(observed).toMatchObject({
|
||||
uid: user,
|
||||
issued: "issued memory only",
|
||||
rawArtifactRead: expect.not.stringMatching(/^allowed$/u),
|
||||
memoryWrite: expect.not.stringMatching(/^allowed$/u),
|
||||
workspaceWrite: "allowed",
|
||||
});
|
||||
expect(fs.readFileSync(path.join(memoryDir, "shared", "issued.md"), "utf8")).toBe(
|
||||
"issued memory only",
|
||||
);
|
||||
expect(fs.readFileSync(path.join(workspaceDir, "proof.txt"), "utf8")).toBe("allowed");
|
||||
} finally {
|
||||
fs.chmodSync(path.join(memoryDir, "shared"), 0o700);
|
||||
if (engine) {
|
||||
await removeOwnedNodeWorkerContainers(identity, engine);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("withdraws an expired signed projection-backed container worker and its mounts", async () => {
|
||||
const root = tempDirs.make("node-worker-supervisor-container-e2e-");
|
||||
const outsideArtifactPath = path.join(root, "host-only-artifact.txt");
|
||||
fs.writeFileSync(outsideArtifactPath, "host-only artifact");
|
||||
const previousBrokerCredential =
|
||||
process.env.OPENCLAW_PROCESS_ISOLATION_TEST_BROKER_CREDENTIAL;
|
||||
process.env.OPENCLAW_PROCESS_ISOLATION_TEST_BROKER_CREDENTIAL = "test-broker-only-credential";
|
||||
try {
|
||||
await verifyNodeWorkerContainerProjectionIsolation({
|
||||
root,
|
||||
broker: broker(new Date(Date.now() + 15_000).toISOString()),
|
||||
outsideArtifactPath,
|
||||
outsideArtifactContents: "host-only artifact",
|
||||
issuedVirtualPath: "shared/brief.md",
|
||||
issuedContents: "only this issued virtual view",
|
||||
forbiddenEnvironmentVariable: "OPENCLAW_PROCESS_ISOLATION_TEST_BROKER_CREDENTIAL",
|
||||
});
|
||||
} finally {
|
||||
if (previousBrokerCredential === undefined) {
|
||||
delete process.env.OPENCLAW_PROCESS_ISOLATION_TEST_BROKER_CREDENTIAL;
|
||||
} else {
|
||||
process.env.OPENCLAW_PROCESS_ISOLATION_TEST_BROKER_CREDENTIAL = previousBrokerCredential;
|
||||
}
|
||||
}
|
||||
}, 90_000);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,174 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
|
||||
const containerMocks = vi.hoisted(() => ({ execContainer: vi.fn() }));
|
||||
|
||||
vi.mock("../agents/sandbox/docker.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("../agents/sandbox/docker.js")>(
|
||||
"../agents/sandbox/docker.js",
|
||||
);
|
||||
return { ...actual, execContainer: containerMocks.execContainer };
|
||||
});
|
||||
|
||||
import { DOCKER_SANDBOX_ENGINE } from "../agents/sandbox/docker.js";
|
||||
import {
|
||||
buildNodeWorkerContainerRunArgs,
|
||||
NODE_WORKER_CONTAINER_IMAGE,
|
||||
NODE_WORKER_CONTAINER_MEMORY_ROOT,
|
||||
NODE_WORKER_CONTAINER_RELAY_ROOT,
|
||||
NODE_WORKER_CONTAINER_WORKER_ROOT,
|
||||
NODE_WORKER_CONTAINER_WORKSPACE,
|
||||
nodeWorkerContainerName,
|
||||
removeOwnedNodeWorkerContainers,
|
||||
resolveNodeWorkerContainerEngine,
|
||||
} from "./node-worker-container-runtime.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
const identity = { launchId: "turn-1", planHash: "a".repeat(64) };
|
||||
|
||||
function mounts() {
|
||||
const root = tempDirs.make("node-worker-container-");
|
||||
const bundleDir = path.join(root, "bundle");
|
||||
const relayDir = path.join(root, "relay");
|
||||
const memoryDir = path.join(root, "memory");
|
||||
const workspaceDir = path.join(root, "workspace");
|
||||
for (const directory of [bundleDir, relayDir, memoryDir, workspaceDir]) {
|
||||
fs.mkdirSync(directory);
|
||||
}
|
||||
return { bundleDir, relayDir, memoryDir, workspaceDir };
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
containerMocks.execContainer.mockReset();
|
||||
});
|
||||
|
||||
describe("node worker container runtime", () => {
|
||||
it("prepares the fixed worker image before admitting Docker process isolation", async () => {
|
||||
containerMocks.execContainer
|
||||
.mockResolvedValueOnce({ stdout: "27.5.1\n", stderr: "", code: 0 })
|
||||
.mockResolvedValueOnce({ stdout: "", stderr: "No such image", code: 1 })
|
||||
.mockResolvedValueOnce({ stdout: "pulled\n", stderr: "", code: 0 });
|
||||
|
||||
await expect(resolveNodeWorkerContainerEngine()).resolves.toBe(DOCKER_SANDBOX_ENGINE);
|
||||
|
||||
expect(containerMocks.execContainer.mock.calls).toEqual([
|
||||
[DOCKER_SANDBOX_ENGINE, ["info", "--format", "{{.ServerVersion}}"], expect.any(Object)],
|
||||
[
|
||||
DOCKER_SANDBOX_ENGINE,
|
||||
["image", "inspect", "--format", "{{.Id}}", NODE_WORKER_CONTAINER_IMAGE],
|
||||
expect.any(Object),
|
||||
],
|
||||
[DOCKER_SANDBOX_ENGINE, ["pull", NODE_WORKER_CONTAINER_IMAGE], expect.any(Object)],
|
||||
]);
|
||||
expect(containerMocks.execContainer.mock.calls[2]?.[2]).toEqual(
|
||||
expect.objectContaining({ signal: expect.any(AbortSignal) }),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not advertise process isolation when the fixed worker image cannot be prepared", async () => {
|
||||
containerMocks.execContainer
|
||||
.mockResolvedValueOnce({ stdout: "27.5.1\n", stderr: "", code: 0 })
|
||||
.mockResolvedValueOnce({ stdout: "", stderr: "No such image", code: 1 })
|
||||
.mockResolvedValueOnce({ stdout: "", stderr: "registry unavailable", code: 1 })
|
||||
.mockResolvedValueOnce({ stdout: "", stderr: "podman unavailable", code: 1 });
|
||||
|
||||
await expect(resolveNodeWorkerContainerEngine()).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("builds a closed non-root container policy with only the worker mounts", () => {
|
||||
const paths = mounts();
|
||||
const args = buildNodeWorkerContainerRunArgs({
|
||||
engine: DOCKER_SANDBOX_ENGINE,
|
||||
identity,
|
||||
mounts: paths,
|
||||
uid: 501,
|
||||
gid: 20,
|
||||
});
|
||||
|
||||
expect(args).toEqual(
|
||||
expect.arrayContaining([
|
||||
"--interactive",
|
||||
"--network",
|
||||
"none",
|
||||
"--read-only",
|
||||
"--cap-drop",
|
||||
"ALL",
|
||||
"--security-opt",
|
||||
"no-new-privileges",
|
||||
"--pids-limit",
|
||||
"128",
|
||||
"--memory",
|
||||
"512m",
|
||||
"--cpus",
|
||||
"1",
|
||||
"--user",
|
||||
"501:20",
|
||||
"--workdir",
|
||||
NODE_WORKER_CONTAINER_WORKSPACE,
|
||||
NODE_WORKER_CONTAINER_IMAGE,
|
||||
]),
|
||||
);
|
||||
const mountValues = args.flatMap((value, index) =>
|
||||
value === "--mount" ? [args[index + 1]] : [],
|
||||
);
|
||||
expect(mountValues).toEqual([
|
||||
`type=bind,src=${fs.realpathSync(paths.bundleDir)},dst=${NODE_WORKER_CONTAINER_WORKER_ROOT},readonly`,
|
||||
`type=bind,src=${fs.realpathSync(paths.relayDir)},dst=${NODE_WORKER_CONTAINER_RELAY_ROOT},readonly`,
|
||||
`type=bind,src=${fs.realpathSync(paths.memoryDir)},dst=${NODE_WORKER_CONTAINER_MEMORY_ROOT},readonly`,
|
||||
`type=bind,src=${fs.realpathSync(paths.workspaceDir)},dst=${NODE_WORKER_CONTAINER_WORKSPACE}`,
|
||||
]);
|
||||
expect(args).not.toContain("--privileged");
|
||||
expect(args).not.toContain("--network=host");
|
||||
});
|
||||
|
||||
it("does not claim cleanup after container enumeration fails", async () => {
|
||||
containerMocks.execContainer.mockRejectedValueOnce(new Error("Docker daemon unavailable"));
|
||||
|
||||
await expect(removeOwnedNodeWorkerContainers(identity, DOCKER_SANDBOX_ENGINE)).rejects.toThrow(
|
||||
"Docker daemon unavailable",
|
||||
);
|
||||
expect(containerMocks.execContainer).toHaveBeenCalledWith(
|
||||
DOCKER_SANDBOX_ENGINE,
|
||||
expect.any(Array),
|
||||
expect.objectContaining({ signal: expect.any(AbortSignal) }),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a symlinked memory mount before it reaches the container engine", () => {
|
||||
const paths = mounts();
|
||||
const symlink = path.join(path.dirname(paths.memoryDir), "memory-link");
|
||||
fs.symlinkSync(paths.memoryDir, symlink);
|
||||
|
||||
expect(() =>
|
||||
buildNodeWorkerContainerRunArgs({
|
||||
engine: DOCKER_SANDBOX_ENGINE,
|
||||
identity,
|
||||
mounts: { ...paths, memoryDir: symlink },
|
||||
uid: 501,
|
||||
gid: 20,
|
||||
}),
|
||||
).toThrow("memory projection mount must be a real local directory");
|
||||
});
|
||||
|
||||
it("keeps cleanup open when a label-verified container cannot be removed", async () => {
|
||||
const containerId = "b".repeat(64);
|
||||
containerMocks.execContainer
|
||||
.mockResolvedValueOnce({ stdout: `${containerId}\n`, stderr: "", code: 0 })
|
||||
.mockResolvedValueOnce({
|
||||
stdout: `/${nodeWorkerContainerName(identity)}\nv1\n${identity.launchId}\n${identity.planHash}\n`,
|
||||
stderr: "",
|
||||
code: 0,
|
||||
})
|
||||
.mockRejectedValueOnce(new Error("permission denied"))
|
||||
.mockResolvedValueOnce({ stdout: `${containerId}\n`, stderr: "", code: 0 });
|
||||
|
||||
await expect(removeOwnedNodeWorkerContainers(identity, DOCKER_SANDBOX_ENGINE)).rejects.toThrow(
|
||||
"Docker could not remove a node worker container",
|
||||
);
|
||||
for (const [, , options] of containerMocks.execContainer.mock.calls) {
|
||||
expect(options).toEqual(expect.objectContaining({ signal: expect.any(AbortSignal) }));
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,372 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
DOCKER_SANDBOX_ENGINE,
|
||||
PODMAN_SANDBOX_ENGINE,
|
||||
execContainer,
|
||||
type SandboxContainerEngine,
|
||||
} from "../agents/sandbox/docker.js";
|
||||
|
||||
/**
|
||||
* Process-isolated workers deliberately use one pinned runtime instead of the
|
||||
* configurable tool sandbox image. The image is part of the node-host TCB.
|
||||
*/
|
||||
export const NODE_WORKER_CONTAINER_IMAGE =
|
||||
"docker.io/library/node:24-bookworm-slim@sha256:6f7b03f7c2c8e2e784dcf9295400527b9b1270fd37b7e9a7285cf83b6951452d";
|
||||
export const NODE_WORKER_CONTAINER_WORKSPACE = "/workspace";
|
||||
export const NODE_WORKER_CONTAINER_MEMORY_ROOT = "/memory";
|
||||
export const NODE_WORKER_CONTAINER_WORKER_ROOT = "/opt/openclaw/worker";
|
||||
export const NODE_WORKER_CONTAINER_RELAY_ROOT = "/run/openclaw";
|
||||
export const NODE_WORKER_CONTAINER_RELAY_SOCKET = `${NODE_WORKER_CONTAINER_RELAY_ROOT}/gateway.sock`;
|
||||
export const NODE_WORKER_CONTAINER_SHIM_FLAG = "--internal-worker-container-shim";
|
||||
|
||||
const NODE_WORKER_CONTAINER_LABEL = "openclaw.node-worker-container";
|
||||
const NODE_WORKER_CONTAINER_LAUNCH_LABEL = "openclaw.node-worker-launch";
|
||||
const NODE_WORKER_CONTAINER_PLAN_LABEL = "openclaw.node-worker-plan";
|
||||
const NODE_WORKER_CONTAINER_FORMAT = "v1";
|
||||
const CONTAINER_NAME_MAX_CHARS = 96;
|
||||
const NODE_WORKER_CONTAINER_ENGINE_TIMEOUT_MS = 5_000;
|
||||
const NODE_WORKER_CONTAINER_IMAGE_PREPARE_TIMEOUT_MS = 2 * 60 * 1_000;
|
||||
|
||||
export type NodeWorkerContainerEngine = Extract<SandboxContainerEngine, { id: "docker" | "podman" }>;
|
||||
|
||||
export type NodeWorkerContainerIdentity = Readonly<{
|
||||
launchId: string;
|
||||
planHash: string;
|
||||
}>;
|
||||
|
||||
export type NodeWorkerContainerMounts = Readonly<{
|
||||
bundleDir: string;
|
||||
relayDir: string;
|
||||
memoryDir: string;
|
||||
workspaceDir: string;
|
||||
}>;
|
||||
|
||||
export function nodeWorkerContainerEngineFor(
|
||||
id: NodeWorkerContainerEngine["id"],
|
||||
): NodeWorkerContainerEngine {
|
||||
return id === "docker" ? DOCKER_SANDBOX_ENGINE : PODMAN_SANDBOX_ENGINE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe and cleanup commands must not turn a wedged engine into an unbounded
|
||||
* worker lifecycle operation. `run` is intentionally excluded: it is the
|
||||
* worker lifetime itself and is owned by the shim's IPC lease.
|
||||
*/
|
||||
function runNodeWorkerContainerControlCommand(
|
||||
engine: NodeWorkerContainerEngine,
|
||||
args: string[],
|
||||
options?: { allowFailure?: boolean },
|
||||
) {
|
||||
return execContainer(engine, args, {
|
||||
...options,
|
||||
signal: AbortSignal.timeout(NODE_WORKER_CONTAINER_ENGINE_TIMEOUT_MS),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The worker image is part of the fixed container TCB. Pull it before this
|
||||
* host advertises process isolation: Docker's implicit cold pull can outlive
|
||||
* the short exact-container start probe and otherwise turns a ready node into
|
||||
* a failed first turn.
|
||||
*/
|
||||
async function prepareNodeWorkerContainerImage(engine: NodeWorkerContainerEngine): Promise<boolean> {
|
||||
const existing = await runNodeWorkerContainerControlCommand(
|
||||
engine,
|
||||
["image", "inspect", "--format", "{{.Id}}", NODE_WORKER_CONTAINER_IMAGE],
|
||||
{ allowFailure: true },
|
||||
);
|
||||
if (existing.code === 0 && existing.stdout.trim()) {
|
||||
return true;
|
||||
}
|
||||
const pulled = await execContainer(engine, ["pull", NODE_WORKER_CONTAINER_IMAGE], {
|
||||
allowFailure: true,
|
||||
signal: AbortSignal.timeout(NODE_WORKER_CONTAINER_IMAGE_PREPARE_TIMEOUT_MS),
|
||||
});
|
||||
return pulled.code === 0;
|
||||
}
|
||||
|
||||
/** A mode-0600 relay is usable only by a concrete, non-root POSIX user mapping. */
|
||||
export function resolveNodeWorkerContainerUser(): { uid: number; gid: number } | undefined {
|
||||
if (
|
||||
process.platform === "win32" ||
|
||||
typeof process.getuid !== "function" ||
|
||||
typeof process.getgid !== "function"
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
const uid = process.getuid();
|
||||
const gid = process.getgid();
|
||||
return uid === 0 || gid === 0 ? undefined : { uid, gid };
|
||||
}
|
||||
|
||||
function requireOwnedAbsolutePath(value: string, label: string): string {
|
||||
if (!path.isAbsolute(value) || value.includes("\0") || value.includes("\n") || value.includes("\r")) {
|
||||
throw new Error(`node worker container ${label} must be an absolute local path`);
|
||||
}
|
||||
const lexicalStats = fs.lstatSync(value);
|
||||
if (lexicalStats.isSymbolicLink() || !lexicalStats.isDirectory()) {
|
||||
throw new Error(`node worker container ${label} must be a real local directory`);
|
||||
}
|
||||
const resolved = fs.realpathSync.native(value);
|
||||
const stats = fs.lstatSync(resolved);
|
||||
if (stats.isSymbolicLink() || !stats.isDirectory()) {
|
||||
throw new Error(`node worker container ${label} must be a real local directory`);
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
function formatMount(source: string, target: string, readOnly: boolean): string {
|
||||
return `type=bind,src=${source},dst=${target}${readOnly ? ",readonly" : ""}`;
|
||||
}
|
||||
|
||||
function safeContainerNameComponent(value: string): string {
|
||||
const normalized = value.toLowerCase().replaceAll(/[^a-z0-9_.-]/gu, "-");
|
||||
const suffix = createHash("sha256").update(value).digest("hex").slice(0, 16);
|
||||
const prefix = normalized.replaceAll(/^-+|[-.]+$/gu, "").slice(0, 48) || "worker";
|
||||
return `${prefix}-${suffix}`;
|
||||
}
|
||||
|
||||
/** A deterministic but collision-resistant container name, never a caller-provided Docker argument. */
|
||||
export function nodeWorkerContainerName(identity: NodeWorkerContainerIdentity): string {
|
||||
const value = `openclaw-worker-${safeContainerNameComponent(identity.launchId)}-${identity.planHash.slice(0, 16)}`;
|
||||
return value.slice(0, CONTAINER_NAME_MAX_CHARS);
|
||||
}
|
||||
|
||||
function assertContainerIdentity(identity: NodeWorkerContainerIdentity): void {
|
||||
if (
|
||||
identity.launchId.length === 0 ||
|
||||
identity.launchId.length > 256 ||
|
||||
identity.launchId.includes("\0") ||
|
||||
!/^[a-f0-9]{64}$/u.test(identity.planHash)
|
||||
) {
|
||||
throw new Error("invalid node worker container identity");
|
||||
}
|
||||
}
|
||||
|
||||
export function buildNodeWorkerContainerRunArgs(params: {
|
||||
engine: NodeWorkerContainerEngine;
|
||||
identity: NodeWorkerContainerIdentity;
|
||||
mounts: NodeWorkerContainerMounts;
|
||||
uid: number;
|
||||
gid: number;
|
||||
}): string[] {
|
||||
assertContainerIdentity(params.identity);
|
||||
if (
|
||||
!Number.isSafeInteger(params.uid) ||
|
||||
params.uid <= 0 ||
|
||||
!Number.isSafeInteger(params.gid) ||
|
||||
params.gid <= 0
|
||||
) {
|
||||
throw new Error("node worker container requires a concrete non-root host user mapping");
|
||||
}
|
||||
const bundleDir = requireOwnedAbsolutePath(params.mounts.bundleDir, "bundle mount");
|
||||
const relayDir = requireOwnedAbsolutePath(params.mounts.relayDir, "relay mount");
|
||||
const memoryDir = requireOwnedAbsolutePath(params.mounts.memoryDir, "memory projection mount");
|
||||
const workspaceDir = requireOwnedAbsolutePath(params.mounts.workspaceDir, "workspace mount");
|
||||
const name = nodeWorkerContainerName(params.identity);
|
||||
|
||||
// This is intentionally a closed policy. The Gateway cannot pass a host path,
|
||||
// image, environment value, container option, or projection mount into it.
|
||||
return [
|
||||
"run",
|
||||
"--interactive",
|
||||
"--name",
|
||||
name,
|
||||
"--label",
|
||||
`${NODE_WORKER_CONTAINER_LABEL}=${NODE_WORKER_CONTAINER_FORMAT}`,
|
||||
"--label",
|
||||
`${NODE_WORKER_CONTAINER_LAUNCH_LABEL}=${params.identity.launchId}`,
|
||||
"--label",
|
||||
`${NODE_WORKER_CONTAINER_PLAN_LABEL}=${params.identity.planHash}`,
|
||||
"--network",
|
||||
"none",
|
||||
"--read-only",
|
||||
"--tmpfs",
|
||||
"/tmp:rw,nosuid,nodev,noexec,size=64m",
|
||||
"--cap-drop",
|
||||
"ALL",
|
||||
"--security-opt",
|
||||
"no-new-privileges",
|
||||
"--pids-limit",
|
||||
"128",
|
||||
"--memory",
|
||||
"512m",
|
||||
"--cpus",
|
||||
"1",
|
||||
"--user",
|
||||
`${params.uid}:${params.gid}`,
|
||||
"--workdir",
|
||||
NODE_WORKER_CONTAINER_WORKSPACE,
|
||||
"--mount",
|
||||
formatMount(bundleDir, NODE_WORKER_CONTAINER_WORKER_ROOT, true),
|
||||
"--mount",
|
||||
formatMount(relayDir, NODE_WORKER_CONTAINER_RELAY_ROOT, true),
|
||||
"--mount",
|
||||
formatMount(memoryDir, NODE_WORKER_CONTAINER_MEMORY_ROOT, true),
|
||||
"--mount",
|
||||
formatMount(workspaceDir, NODE_WORKER_CONTAINER_WORKSPACE, false),
|
||||
NODE_WORKER_CONTAINER_IMAGE,
|
||||
"node",
|
||||
`${NODE_WORKER_CONTAINER_WORKER_ROOT}/worker.mjs`,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* A node advertises process isolation only after one installed engine proves it
|
||||
* can service local container commands. Docker is preferred; Podman is the
|
||||
* supported equivalent when Docker is not available.
|
||||
*/
|
||||
export async function resolveNodeWorkerContainerEngine(): Promise<NodeWorkerContainerEngine | undefined> {
|
||||
if (!resolveNodeWorkerContainerUser()) {
|
||||
return undefined;
|
||||
}
|
||||
for (const engine of [DOCKER_SANDBOX_ENGINE, PODMAN_SANDBOX_ENGINE] as const) {
|
||||
try {
|
||||
const result = await runNodeWorkerContainerControlCommand(
|
||||
engine,
|
||||
["info", "--format", "{{.ServerVersion}}"],
|
||||
{
|
||||
allowFailure: true,
|
||||
},
|
||||
);
|
||||
if (
|
||||
result.code === 0 &&
|
||||
result.stdout.trim().length > 0 &&
|
||||
(await prepareNodeWorkerContainerImage(engine))
|
||||
) {
|
||||
return engine;
|
||||
}
|
||||
} catch {
|
||||
// An absent or unavailable engine means this node is ineligible, never a host fallback.
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async function listMatchingContainerIds(
|
||||
engine: NodeWorkerContainerEngine,
|
||||
identity: NodeWorkerContainerIdentity,
|
||||
): Promise<string[]> {
|
||||
const result = await runNodeWorkerContainerControlCommand(
|
||||
engine,
|
||||
[
|
||||
"ps",
|
||||
"--all",
|
||||
"--quiet",
|
||||
"--filter",
|
||||
`label=${NODE_WORKER_CONTAINER_LABEL}=${NODE_WORKER_CONTAINER_FORMAT}`,
|
||||
"--filter",
|
||||
`label=${NODE_WORKER_CONTAINER_LAUNCH_LABEL}=${identity.launchId}`,
|
||||
"--filter",
|
||||
`label=${NODE_WORKER_CONTAINER_PLAN_LABEL}=${identity.planHash}`,
|
||||
],
|
||||
);
|
||||
return result.stdout
|
||||
.split(/\r?\n/u)
|
||||
.map((value) => value.trim())
|
||||
.filter((value) => /^[a-f0-9]{12,128}$/u.test(value));
|
||||
}
|
||||
|
||||
async function isOwnedContainer(
|
||||
engine: NodeWorkerContainerEngine,
|
||||
containerId: string,
|
||||
identity: NodeWorkerContainerIdentity,
|
||||
): Promise<boolean> {
|
||||
const result = await runNodeWorkerContainerControlCommand(
|
||||
engine,
|
||||
[
|
||||
"inspect",
|
||||
"--format",
|
||||
"{{.Name}}\n{{index .Config.Labels \"openclaw.node-worker-container\"}}\n{{index .Config.Labels \"openclaw.node-worker-launch\"}}\n{{index .Config.Labels \"openclaw.node-worker-plan\"}}",
|
||||
containerId,
|
||||
],
|
||||
);
|
||||
const [rawName, format, launchId, planHash, ...extra] = result.stdout.trimEnd().split(/\r?\n/u);
|
||||
return (
|
||||
extra.length === 0 &&
|
||||
rawName === `/${nodeWorkerContainerName(identity)}` &&
|
||||
format === NODE_WORKER_CONTAINER_FORMAT &&
|
||||
launchId === identity.launchId &&
|
||||
planHash === identity.planHash
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Prove the exact labelled container is accepting work before the shim attests
|
||||
* execution. The foreground `run` client being spawned is not that proof.
|
||||
*/
|
||||
export async function waitForOwnedNodeWorkerContainerRunning(params: {
|
||||
engine: NodeWorkerContainerEngine;
|
||||
identity: NodeWorkerContainerIdentity;
|
||||
timeoutMs?: number;
|
||||
}): Promise<void> {
|
||||
assertContainerIdentity(params.identity);
|
||||
const deadline = Date.now() + (params.timeoutMs ?? 5_000);
|
||||
while (Date.now() < deadline) {
|
||||
const containerIds = await listMatchingContainerIds(params.engine, params.identity);
|
||||
for (const containerId of containerIds) {
|
||||
if (!(await isOwnedContainer(params.engine, containerId, params.identity))) {
|
||||
continue;
|
||||
}
|
||||
const state = await runNodeWorkerContainerControlCommand(params.engine, [
|
||||
"inspect",
|
||||
"--format",
|
||||
"{{.State.Running}}",
|
||||
containerId,
|
||||
]);
|
||||
if (state.stdout.trim() === "true") {
|
||||
return;
|
||||
}
|
||||
}
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 25));
|
||||
}
|
||||
throw new Error(`${params.engine.displayName} did not start the exact node worker container`);
|
||||
}
|
||||
|
||||
async function containerStillMatches(
|
||||
engine: NodeWorkerContainerEngine,
|
||||
containerId: string,
|
||||
identity: NodeWorkerContainerIdentity,
|
||||
): Promise<boolean> {
|
||||
return (await listMatchingContainerIds(engine, identity)).includes(containerId);
|
||||
}
|
||||
|
||||
/** Removes only an exact, label-verified orphan from a prior shim/supervisor lifetime. */
|
||||
export async function removeOwnedNodeWorkerContainers(
|
||||
identity: NodeWorkerContainerIdentity,
|
||||
engine: NodeWorkerContainerEngine,
|
||||
): Promise<number> {
|
||||
assertContainerIdentity(identity);
|
||||
let removed = 0;
|
||||
const containerIds = await listMatchingContainerIds(engine, identity);
|
||||
for (const containerId of containerIds) {
|
||||
let owned: boolean;
|
||||
try {
|
||||
owned = await isOwnedContainer(engine, containerId, identity);
|
||||
} catch {
|
||||
// An inspect race is safe only when a second enumeration proves that the
|
||||
// listed container is gone. Any surviving container leaves cleanup open.
|
||||
if (!(await containerStillMatches(engine, containerId, identity))) {
|
||||
continue;
|
||||
}
|
||||
throw new Error(`${engine.displayName} could not inspect a node worker container`);
|
||||
}
|
||||
if (!owned) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await runNodeWorkerContainerControlCommand(engine, ["rm", "--force", containerId]);
|
||||
removed += 1;
|
||||
} catch {
|
||||
// A concurrent exit is already clean. Anything still enumerable is an
|
||||
// orphan and must keep the relay and launch recovery from settling.
|
||||
if (!(await containerStillMatches(engine, containerId, identity))) {
|
||||
continue;
|
||||
}
|
||||
throw new Error(`${engine.displayName} could not remove a node worker container`);
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
import { fork, type ChildProcess } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import { completeWorkerLaunchDescriptor } from "../worker/launch-descriptor.js";
|
||||
import { nodeWorkerContainerName } from "./node-worker-container-runtime.js";
|
||||
import { testWorkerDescriptor } from "./node-worker-supervisor.test-support.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
const children = new Set<ChildProcess>();
|
||||
|
||||
afterEach(async () => {
|
||||
for (const child of children) {
|
||||
if (child.exitCode === null && child.signalCode === null) {
|
||||
child.kill("SIGKILL");
|
||||
}
|
||||
}
|
||||
await Promise.all(
|
||||
[...children].map(
|
||||
async (child) =>
|
||||
await new Promise<void>((resolve) => {
|
||||
if (child.exitCode !== null || child.signalCode !== null) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
child.once("exit", () => resolve());
|
||||
}),
|
||||
),
|
||||
);
|
||||
children.clear();
|
||||
});
|
||||
|
||||
function writeFakeDocker(root: string, marker: string, containerName: string): string {
|
||||
const binDir = path.join(root, "bin");
|
||||
fs.mkdirSync(binDir);
|
||||
const executable = path.join(binDir, "docker");
|
||||
fs.writeFileSync(
|
||||
executable,
|
||||
`#!/usr/bin/env node
|
||||
const fs = require("node:fs");
|
||||
const command = process.argv[2];
|
||||
fs.appendFileSync(${JSON.stringify(marker)}, command + "\\n");
|
||||
if (command === "ps") {
|
||||
process.stdout.write("abcdefabcdef\\n");
|
||||
process.exit(0);
|
||||
}
|
||||
if (command === "inspect") {
|
||||
const format = process.argv[process.argv.indexOf("--format") + 1];
|
||||
if (format === "{{.State.Running}}") {
|
||||
process.stdout.write("true\\n");
|
||||
} else {
|
||||
process.stdout.write(${JSON.stringify(
|
||||
`/${containerName}\nv1\nturn-1\n${"a".repeat(64)}\n`,
|
||||
)});
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
if (command === "run") {
|
||||
process.on("SIGTERM", () => process.exit(0));
|
||||
setInterval(() => {}, 1000);
|
||||
}
|
||||
`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
return binDir;
|
||||
}
|
||||
|
||||
function waitForExit(child: ChildProcess): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
if (child.exitCode !== null || child.signalCode !== null) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
child.once("error", reject);
|
||||
child.once("exit", () => resolve());
|
||||
});
|
||||
}
|
||||
|
||||
describe("node worker container shim", () => {
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"stops its container command when the supervisor IPC lease disconnects",
|
||||
async () => {
|
||||
const root = tempDirs.make("node-worker-container-shim-");
|
||||
const marker = path.join(root, "docker-calls");
|
||||
const identity = { launchId: "turn-1", planHash: "a".repeat(64) };
|
||||
const binDir = writeFakeDocker(root, marker, nodeWorkerContainerName(identity));
|
||||
const bundleDir = path.join(root, "bundle");
|
||||
const relayDir = tempDirs.make("oc-worker-relay-", "/tmp");
|
||||
const memoryDir = path.join(root, "memory");
|
||||
const workspaceDir = path.join(root, "workspace");
|
||||
for (const directory of [bundleDir, memoryDir, workspaceDir]) {
|
||||
fs.mkdirSync(directory);
|
||||
}
|
||||
fs.writeFileSync(path.join(bundleDir, "worker.mjs"), "process.exit(0);\n");
|
||||
const descriptor = testWorkerDescriptor("/workspace");
|
||||
descriptor.assignment.memoryReadEnforced = true;
|
||||
const input = {
|
||||
descriptor: completeWorkerLaunchDescriptor(descriptor, {
|
||||
kind: "unix" as const,
|
||||
socketPath: path.join(root, "gateway.sock"),
|
||||
}),
|
||||
engine: "docker" as const,
|
||||
identity,
|
||||
mounts: { bundleDir, relayDir, memoryDir, workspaceDir },
|
||||
};
|
||||
const shimPath = path.resolve("src/node-host/node-worker-container-shim.ts");
|
||||
const child = fork(shimPath, ["--internal-worker-container-shim"], {
|
||||
// tsx resolves workspace aliases through the checkout's tsconfig. The
|
||||
// shim's actual container command still runs from `workspaceDir`.
|
||||
cwd: process.cwd(),
|
||||
env: { ...process.env, PATH: `${binDir}${path.delimiter}${process.env.PATH ?? ""}` },
|
||||
// The shim's CWD is an isolated worker fixture, so a bare package
|
||||
// specifier would resolve from that fixture rather than this checkout.
|
||||
execArgv: ["--import", import.meta.resolve("tsx")],
|
||||
silent: true,
|
||||
});
|
||||
children.add(child);
|
||||
let stderr = "";
|
||||
const messages: unknown[] = [];
|
||||
child.stderr?.on("data", (chunk: Buffer) => {
|
||||
stderr += chunk.toString("utf8");
|
||||
});
|
||||
child.on("message", (message: unknown) => {
|
||||
messages.push(message);
|
||||
});
|
||||
child.stdin?.end(JSON.stringify(input));
|
||||
child.send({ type: "openclaw-worker-start-v1", ...identity });
|
||||
|
||||
try {
|
||||
await vi.waitFor(() => expect(fs.readFileSync(marker, "utf8")).toContain("run\n"), {
|
||||
timeout: 5_000,
|
||||
});
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`container shim did not launch the fake engine: ${stderr} ${JSON.stringify(messages)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
await vi.waitFor(() =>
|
||||
expect(messages).toContainEqual({
|
||||
type: "openclaw-worker-execution-started-v1",
|
||||
...identity,
|
||||
}),
|
||||
);
|
||||
child.disconnect();
|
||||
|
||||
await vi.waitFor(
|
||||
() => expect(child.exitCode ?? child.signalCode).not.toBeNull(),
|
||||
{ timeout: 5_000 },
|
||||
);
|
||||
await expect(waitForExit(child)).resolves.toBeUndefined();
|
||||
expect(fs.readFileSync(marker, "utf8")).toContain("ps\n");
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,233 @@
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { once } from "node:events";
|
||||
import { isRecord } from "@openclaw/normalization-core/record-coerce";
|
||||
import {
|
||||
completeWorkerLaunchDescriptor,
|
||||
parseWorkerLaunchDescriptor,
|
||||
type WorkerLaunchDescriptor,
|
||||
} from "../worker/launch-descriptor.js";
|
||||
import { createWorkerIpcLifetime } from "../worker/worker-process.js";
|
||||
import type { WorkerConnectionEndpoint } from "../worker/worker-connection-endpoint.js";
|
||||
import {
|
||||
buildNodeWorkerContainerRunArgs,
|
||||
nodeWorkerContainerEngineFor,
|
||||
NODE_WORKER_CONTAINER_SHIM_FLAG,
|
||||
NODE_WORKER_CONTAINER_RELAY_SOCKET,
|
||||
removeOwnedNodeWorkerContainers,
|
||||
resolveNodeWorkerContainerUser,
|
||||
waitForOwnedNodeWorkerContainerRunning,
|
||||
type NodeWorkerContainerEngine,
|
||||
type NodeWorkerContainerIdentity,
|
||||
type NodeWorkerContainerMounts,
|
||||
} from "./node-worker-container-runtime.js";
|
||||
import { startNodeWorkerGatewayRelay, type NodeWorkerGatewayRelay } from "./node-worker-gateway-relay.js";
|
||||
|
||||
const SHIM_INPUT_MAX_BYTES = 1024 * 1024;
|
||||
|
||||
type NodeWorkerContainerShimInput = Readonly<{
|
||||
descriptor: WorkerLaunchDescriptor;
|
||||
engine: NodeWorkerContainerEngine["id"];
|
||||
identity: NodeWorkerContainerIdentity;
|
||||
mounts: NodeWorkerContainerMounts;
|
||||
}>;
|
||||
|
||||
function hasExactKeys(value: Record<string, unknown>, keys: readonly string[]): boolean {
|
||||
return Object.keys(value).length === keys.length && keys.every((key) => Object.hasOwn(value, key));
|
||||
}
|
||||
|
||||
function parseContainerEngine(value: unknown): NodeWorkerContainerEngine["id"] | undefined {
|
||||
return value === "docker" || value === "podman" ? value : undefined;
|
||||
}
|
||||
|
||||
function parseShimInput(raw: string): NodeWorkerContainerShimInput {
|
||||
if (Buffer.byteLength(raw, "utf8") > SHIM_INPUT_MAX_BYTES) {
|
||||
throw new Error("node worker container shim input exceeds its bound");
|
||||
}
|
||||
let value: unknown;
|
||||
try {
|
||||
value = JSON.parse(raw) as unknown;
|
||||
} catch {
|
||||
throw new Error("node worker container shim input is malformed");
|
||||
}
|
||||
if (
|
||||
!isRecord(value) ||
|
||||
!hasExactKeys(value, ["descriptor", "engine", "identity", "mounts"]) ||
|
||||
!isRecord(value.identity) ||
|
||||
!hasExactKeys(value.identity, ["launchId", "planHash"]) ||
|
||||
typeof value.identity.launchId !== "string" ||
|
||||
typeof value.identity.planHash !== "string" ||
|
||||
!isRecord(value.mounts) ||
|
||||
!hasExactKeys(value.mounts, ["bundleDir", "relayDir", "memoryDir", "workspaceDir"]) ||
|
||||
typeof value.mounts.bundleDir !== "string" ||
|
||||
typeof value.mounts.relayDir !== "string" ||
|
||||
typeof value.mounts.memoryDir !== "string" ||
|
||||
typeof value.mounts.workspaceDir !== "string"
|
||||
) {
|
||||
throw new Error("node worker container shim input is invalid");
|
||||
}
|
||||
const engine = parseContainerEngine(value.engine);
|
||||
if (!engine) {
|
||||
throw new Error("node worker container shim engine is invalid");
|
||||
}
|
||||
const descriptor = parseWorkerLaunchDescriptor(value.descriptor);
|
||||
if (
|
||||
descriptor.assignment.workspaceDir !== "/workspace" ||
|
||||
(descriptor.assignment.workerContainmentRoot !== undefined &&
|
||||
descriptor.assignment.workerContainmentRoot !== "/workspace")
|
||||
) {
|
||||
throw new Error("node worker container descriptor escaped its fixed workspace");
|
||||
}
|
||||
return {
|
||||
descriptor,
|
||||
engine,
|
||||
identity: { launchId: value.identity.launchId, planHash: value.identity.planHash },
|
||||
mounts: {
|
||||
bundleDir: value.mounts.bundleDir,
|
||||
relayDir: value.mounts.relayDir,
|
||||
memoryDir: value.mounts.memoryDir,
|
||||
workspaceDir: value.mounts.workspaceDir,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function readShimInput(): Promise<NodeWorkerContainerShimInput> {
|
||||
let raw = "";
|
||||
for await (const chunk of process.stdin) {
|
||||
raw += String(chunk);
|
||||
if (Buffer.byteLength(raw, "utf8") > SHIM_INPUT_MAX_BYTES) {
|
||||
throw new Error("node worker container shim input exceeds its bound");
|
||||
}
|
||||
}
|
||||
return parseShimInput(raw);
|
||||
}
|
||||
|
||||
function currentUser(): { uid: number; gid: number } {
|
||||
const user = resolveNodeWorkerContainerUser();
|
||||
if (!user) {
|
||||
throw new Error("node worker container execution requires a non-root POSIX host user mapping");
|
||||
}
|
||||
return user;
|
||||
}
|
||||
|
||||
async function closeRelay(relay: NodeWorkerGatewayRelay | undefined): Promise<void> {
|
||||
await relay?.close().catch(() => undefined);
|
||||
}
|
||||
|
||||
async function waitForChild(child: ChildProcessWithoutNullStreams): Promise<number> {
|
||||
const [code] = await once(child, "close");
|
||||
return typeof code === "number" ? code : 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* The shim, not Docker, is the worker PID in the durable journal. It waits on
|
||||
* inherited Node IPC before creating a disposable container and removes that
|
||||
* exact labelled container on any normal or signal-driven terminal path.
|
||||
*/
|
||||
export async function runNodeWorkerContainerShim(): Promise<void> {
|
||||
const input = await readShimInput();
|
||||
const lifetime = createWorkerIpcLifetime();
|
||||
let relay: NodeWorkerGatewayRelay | undefined;
|
||||
let child: ChildProcessWithoutNullStreams | undefined;
|
||||
let stopping = false;
|
||||
let finishStop!: () => void;
|
||||
const stopped = new Promise<void>((resolve) => {
|
||||
finishStop = resolve;
|
||||
});
|
||||
const engine = nodeWorkerContainerEngineFor(input.engine);
|
||||
|
||||
const stop = async () => {
|
||||
if (stopping) {
|
||||
return;
|
||||
}
|
||||
stopping = true;
|
||||
try {
|
||||
child?.kill("SIGTERM");
|
||||
await removeOwnedNodeWorkerContainers(input.identity, engine);
|
||||
} finally {
|
||||
await closeRelay(relay);
|
||||
relay = undefined;
|
||||
lifetime.dispose();
|
||||
finishStop();
|
||||
}
|
||||
};
|
||||
const onSignal = () => {
|
||||
void stop();
|
||||
};
|
||||
const onSupervisorGone = () => {
|
||||
void stop();
|
||||
};
|
||||
process.once("SIGTERM", onSignal);
|
||||
process.once("SIGINT", onSignal);
|
||||
// The IPC channel is the supervisor's ownership lease. Without this, a
|
||||
// killed supervisor can strand a still-running shim and its container.
|
||||
lifetime.signal.addEventListener("abort", onSupervisorGone, { once: true });
|
||||
if (lifetime.signal.aborted) {
|
||||
void stop();
|
||||
}
|
||||
|
||||
try {
|
||||
const started = await Promise.race([lifetime.started, stopped.then(() => false)]);
|
||||
if (!started || stopping) {
|
||||
return;
|
||||
}
|
||||
await fs.mkdir(input.mounts.relayDir, { recursive: true, mode: 0o700 });
|
||||
await fs.chmod(input.mounts.relayDir, 0o700);
|
||||
relay = await startNodeWorkerGatewayRelay({
|
||||
directory: input.mounts.relayDir,
|
||||
upstream: input.descriptor.connectionEndpoint,
|
||||
});
|
||||
// A descriptor includes its host-only Gateway endpoint. Rebuild it from
|
||||
// the validated plan so the container receives only the mounted relay.
|
||||
const containerDescriptor = completeWorkerLaunchDescriptor(
|
||||
{
|
||||
version: input.descriptor.version,
|
||||
admission: input.descriptor.admission,
|
||||
assignment: input.descriptor.assignment,
|
||||
},
|
||||
{
|
||||
kind: "unix",
|
||||
socketPath: NODE_WORKER_CONTAINER_RELAY_SOCKET,
|
||||
} satisfies WorkerConnectionEndpoint,
|
||||
);
|
||||
const runArgs = buildNodeWorkerContainerRunArgs({
|
||||
engine,
|
||||
identity: input.identity,
|
||||
mounts: input.mounts,
|
||||
...currentUser(),
|
||||
});
|
||||
child = spawn(engine.command, runArgs, {
|
||||
cwd: input.mounts.workspaceDir,
|
||||
env: process.env,
|
||||
stdio: ["pipe", "inherit", "inherit"],
|
||||
windowsHide: true,
|
||||
});
|
||||
child.stdin.end(JSON.stringify(containerDescriptor));
|
||||
await waitForOwnedNodeWorkerContainerRunning({ engine, identity: input.identity });
|
||||
// Container execution begins only after an exact label-bound inspect says Running.
|
||||
lifetime.reportExecutionStarted();
|
||||
const code = await Promise.race([waitForChild(child), stopped.then(() => 143)]);
|
||||
if (!stopping) {
|
||||
await removeOwnedNodeWorkerContainers(input.identity, engine);
|
||||
}
|
||||
process.exitCode = code;
|
||||
} catch (error) {
|
||||
lifetime.reportConnectionFailure(error instanceof Error ? error.message : "container worker unavailable");
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
process.off("SIGTERM", onSignal);
|
||||
process.off("SIGINT", onSignal);
|
||||
lifetime.signal.removeEventListener("abort", onSupervisorGone);
|
||||
await stop();
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv.includes(NODE_WORKER_CONTAINER_SHIM_FLAG)) {
|
||||
void runNodeWorkerContainerShim().catch((error: unknown) => {
|
||||
process.stderr.write(
|
||||
`${error instanceof Error ? error.message : "node worker container shim failed"}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
import fs from "node:fs/promises";
|
||||
import http from "node:http";
|
||||
import type { Socket } from "node:net";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { WebSocket, WebSocketServer } from "ws";
|
||||
import { WORKER_PROTOCOL_MAX_PAYLOAD_BYTES } from "../../packages/gateway-protocol/src/schema/worker-admission.js";
|
||||
import { startNodeWorkerGatewayRelay } from "./node-worker-gateway-relay.js";
|
||||
|
||||
const cleanups: Array<() => Promise<void>> = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(cleanups.splice(0).map(async (cleanup) => await cleanup()));
|
||||
});
|
||||
|
||||
async function relayHarness(params: {
|
||||
onUpstreamConnection?: (socket: WebSocket) => void;
|
||||
onUpstreamUpgrade?: (socket: Socket) => void;
|
||||
} = {}) {
|
||||
// Keep the lexical path short: the production relay intentionally does the
|
||||
// same because macOS rejects canonical state paths as Unix socket names.
|
||||
const root = await fs.mkdtemp("/tmp/node-worker-relay-");
|
||||
const upstreamServer = http.createServer();
|
||||
const rawUpstreamSockets = new Set<Socket>();
|
||||
const upstreamClients: WebSocket[] = [];
|
||||
if (params.onUpstreamUpgrade) {
|
||||
upstreamServer.on("upgrade", (_request, socket) => {
|
||||
rawUpstreamSockets.add(socket);
|
||||
socket.once("close", () => rawUpstreamSockets.delete(socket));
|
||||
params.onUpstreamUpgrade?.(socket);
|
||||
});
|
||||
} else {
|
||||
const upstreamWebSocket = new WebSocketServer({ server: upstreamServer });
|
||||
upstreamWebSocket.on("connection", (socket) => {
|
||||
upstreamClients.push(socket);
|
||||
if (params.onUpstreamConnection) {
|
||||
params.onUpstreamConnection(socket);
|
||||
return;
|
||||
}
|
||||
socket.on("message", (message, binary) => socket.send(message, { binary }));
|
||||
});
|
||||
}
|
||||
await new Promise<void>((resolve) => upstreamServer.listen(0, "127.0.0.1", resolve));
|
||||
const address = upstreamServer.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("expected upstream server address");
|
||||
}
|
||||
const relay = await startNodeWorkerGatewayRelay({
|
||||
directory: root,
|
||||
upstream: { kind: "websocket", url: `ws://127.0.0.1:${address.port}/__openclaw__/worker` },
|
||||
});
|
||||
cleanups.push(async () => {
|
||||
await relay.close();
|
||||
for (const socket of rawUpstreamSockets) {
|
||||
socket.destroy();
|
||||
}
|
||||
await new Promise<void>((resolve) => upstreamServer.close(() => resolve()));
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
});
|
||||
return { relay, upstreamClients };
|
||||
}
|
||||
|
||||
async function connect(socketPath: string): Promise<WebSocket> {
|
||||
const socket = new WebSocket(`ws+unix://${socketPath}:/`);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
socket.once("open", resolve);
|
||||
socket.once("error", reject);
|
||||
});
|
||||
return socket;
|
||||
}
|
||||
|
||||
function waitForClose(socket: WebSocket): Promise<number> {
|
||||
return new Promise<number>((resolve) => socket.once("close", (code) => resolve(code)));
|
||||
}
|
||||
|
||||
async function expectConnectFailure(socketPath: string): Promise<void> {
|
||||
const probe = new WebSocket(`ws+unix://${socketPath}:/`);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
probe.once("open", () => reject(new Error("closed relay unexpectedly accepted a client")));
|
||||
probe.once("error", () => resolve());
|
||||
});
|
||||
}
|
||||
|
||||
describe("node worker gateway relay", () => {
|
||||
it("forwards one bounded local client and rejects a second client", async () => {
|
||||
const { relay } = await relayHarness();
|
||||
const first = await connect(relay.socketPath);
|
||||
cleanups.push(async () => first.terminate());
|
||||
|
||||
const echoed = new Promise<Buffer>((resolve) => {
|
||||
first.once("message", (message) => resolve(Buffer.from(message)));
|
||||
});
|
||||
first.send(Buffer.from("worker-frame"));
|
||||
await expect(echoed).resolves.toEqual(Buffer.from("worker-frame"));
|
||||
|
||||
const second = new WebSocket(`ws+unix://${relay.socketPath}:/`);
|
||||
cleanups.push(async () => second.terminate());
|
||||
await expect(
|
||||
new Promise<number>((resolve, reject) => {
|
||||
second.once("unexpected-response", (_request, response) => resolve(response.statusCode ?? 0));
|
||||
second.once("open", () => reject(new Error("second relay client unexpectedly connected")));
|
||||
second.once("error", () => undefined);
|
||||
}),
|
||||
).resolves.toBe(409);
|
||||
});
|
||||
|
||||
it("closes an oversized worker frame before forwarding it upstream", async () => {
|
||||
const { relay } = await relayHarness();
|
||||
const client = await connect(relay.socketPath);
|
||||
cleanups.push(async () => client.terminate());
|
||||
|
||||
const closed = new Promise<number>((resolve) => client.once("close", (code) => resolve(code)));
|
||||
client.send(Buffer.alloc(WORKER_PROTOCOL_MAX_PAYLOAD_BYTES + 1));
|
||||
|
||||
await expect(closed).resolves.toBe(1009);
|
||||
});
|
||||
|
||||
it("closes a worker that exceeds the bounded pre-upstream queue", async () => {
|
||||
const { relay } = await relayHarness({
|
||||
// Keep the upstream WebSocket in CONNECTING so every worker frame uses
|
||||
// the relay's bounded pending queue rather than an OS socket buffer.
|
||||
onUpstreamUpgrade: () => undefined,
|
||||
});
|
||||
const client = await connect(relay.socketPath);
|
||||
cleanups.push(async () => client.terminate());
|
||||
|
||||
const closed = waitForClose(client);
|
||||
client.send(Buffer.alloc(WORKER_PROTOCOL_MAX_PAYLOAD_BYTES));
|
||||
client.send(Buffer.alloc(WORKER_PROTOCOL_MAX_PAYLOAD_BYTES));
|
||||
client.send(Buffer.from([0]));
|
||||
|
||||
await expect(closed).resolves.toBe(1008);
|
||||
});
|
||||
|
||||
it("closes both peers when a downstream client is over the bounded output limit", async () => {
|
||||
const { relay, upstreamClients } = await relayHarness();
|
||||
const client = await connect(relay.socketPath);
|
||||
cleanups.push(async () => client.terminate());
|
||||
await vi.waitFor(() => expect(upstreamClients).toHaveLength(1));
|
||||
const bufferedAmount = vi
|
||||
.spyOn(WebSocket.prototype, "bufferedAmount", "get")
|
||||
.mockReturnValue(WORKER_PROTOCOL_MAX_PAYLOAD_BYTES * 2 + 1);
|
||||
const closed = waitForClose(client);
|
||||
try {
|
||||
upstreamClients[0].send(Buffer.from("gateway-frame"));
|
||||
|
||||
await expect(closed).resolves.toBe(1008);
|
||||
expect(bufferedAmount).toHaveBeenCalled();
|
||||
} finally {
|
||||
bufferedAmount.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("closes the worker when the upstream connection fails", async () => {
|
||||
const { relay } = await relayHarness({
|
||||
onUpstreamUpgrade: (socket) => {
|
||||
socket.end("HTTP/1.1 503 Upstream unavailable\r\nConnection: close\r\n\r\n");
|
||||
},
|
||||
});
|
||||
const client = await connect(relay.socketPath);
|
||||
cleanups.push(async () => client.terminate());
|
||||
|
||||
await expect(waitForClose(client)).resolves.toBe(1008);
|
||||
});
|
||||
|
||||
it("terminates live peers and removes the Unix socket on close", async () => {
|
||||
const { relay } = await relayHarness();
|
||||
const client = await connect(relay.socketPath);
|
||||
const closed = waitForClose(client);
|
||||
|
||||
await relay.close();
|
||||
|
||||
await expect(closed).resolves.toEqual(expect.any(Number));
|
||||
await expect(fs.lstat(relay.socketPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(expectConnectFailure(relay.socketPath)).resolves.toBeUndefined();
|
||||
await expect(relay.close()).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,195 @@
|
||||
import fs from "node:fs/promises";
|
||||
import { createServer, type IncomingMessage } from "node:http";
|
||||
import type { Socket } from "node:net";
|
||||
import path from "node:path";
|
||||
import { WORKER_PROTOCOL_MAX_PAYLOAD_BYTES } from "../../packages/gateway-protocol/src/schema/worker-admission.js";
|
||||
import { WebSocket, WebSocketServer, type RawData } from "ws";
|
||||
import {
|
||||
resolveWorkerConnectionTarget,
|
||||
type WorkerConnectionEndpoint,
|
||||
} from "../worker/worker-connection-endpoint.js";
|
||||
import { NODE_WORKER_CONTAINER_RELAY_SOCKET } from "./node-worker-container-runtime.js";
|
||||
|
||||
const RELAY_PATH = "/";
|
||||
const RELAY_MAX_PENDING_BYTES = WORKER_PROTOCOL_MAX_PAYLOAD_BYTES * 2;
|
||||
const RELAY_MAX_BUFFERED_BYTES = WORKER_PROTOCOL_MAX_PAYLOAD_BYTES * 2;
|
||||
const RELAY_CONNECT_TIMEOUT_MS = 10_000;
|
||||
const RELAY_PORTABLE_UNIX_SOCKET_MAX_BYTES = 100;
|
||||
|
||||
export type NodeWorkerGatewayRelay = Readonly<{
|
||||
socketPath: string;
|
||||
close: () => Promise<void>;
|
||||
}>;
|
||||
|
||||
function rawBytes(data: RawData): number {
|
||||
if (typeof data === "string") {
|
||||
return Buffer.byteLength(data, "utf8");
|
||||
}
|
||||
if (Array.isArray(data)) {
|
||||
return data.reduce((total, chunk) => total + chunk.byteLength, 0);
|
||||
}
|
||||
return data.byteLength;
|
||||
}
|
||||
|
||||
function rejectUpgrade(socket: Socket, status: number): void {
|
||||
socket.end(`HTTP/1.1 ${status} Relay unavailable\r\nConnection: close\r\n\r\n`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Exposes one local Unix WebSocket to an isolated worker. The remote Gateway
|
||||
* endpoint and any TLS/Cloudflare material remain in this host-side relay.
|
||||
*/
|
||||
export async function startNodeWorkerGatewayRelay(params: {
|
||||
directory: string;
|
||||
upstream: WorkerConnectionEndpoint;
|
||||
}): Promise<NodeWorkerGatewayRelay> {
|
||||
if (!path.isAbsolute(params.directory)) {
|
||||
throw new Error("node worker relay directory must be absolute");
|
||||
}
|
||||
await fs.realpath(params.directory);
|
||||
// The bind mount uses the canonical directory, but preserving its short
|
||||
// lexical spelling here keeps the live Unix socket below macOS' path limit.
|
||||
const socketPath = path.join(params.directory, path.basename(NODE_WORKER_CONTAINER_RELAY_SOCKET));
|
||||
if (Buffer.byteLength(socketPath, "utf8") > RELAY_PORTABLE_UNIX_SOCKET_MAX_BYTES) {
|
||||
throw new Error("node worker relay socket path exceeds the portable Unix socket limit");
|
||||
}
|
||||
await fs.rm(socketPath, { force: true });
|
||||
|
||||
let accepted = false;
|
||||
let closing = false;
|
||||
const websocketServer = new WebSocketServer({
|
||||
noServer: true,
|
||||
maxPayload: WORKER_PROTOCOL_MAX_PAYLOAD_BYTES,
|
||||
});
|
||||
const server = createServer((_request, response) => {
|
||||
response.writeHead(404);
|
||||
response.end();
|
||||
});
|
||||
const sockets = new Set<WebSocket>();
|
||||
|
||||
const stopSocket = (socket: WebSocket, code = 1008) => {
|
||||
try {
|
||||
if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CLOSING) {
|
||||
socket.close(code);
|
||||
return;
|
||||
}
|
||||
socket.terminate();
|
||||
} catch {
|
||||
socket.terminate();
|
||||
}
|
||||
};
|
||||
|
||||
const attachRelay = (client: WebSocket) => {
|
||||
sockets.add(client);
|
||||
const target = resolveWorkerConnectionTarget(params.upstream);
|
||||
const upstream = new WebSocket(target.url, target.options);
|
||||
sockets.add(upstream);
|
||||
const pending: Array<{ data: RawData; binary: boolean }> = [];
|
||||
let pendingBytes = 0;
|
||||
let linked = false;
|
||||
let connectTimeout: NodeJS.Timeout | undefined;
|
||||
|
||||
const stopBoth = () => {
|
||||
if (linked) {
|
||||
linked = false;
|
||||
}
|
||||
clearTimeout(connectTimeout);
|
||||
stopSocket(client);
|
||||
stopSocket(upstream);
|
||||
};
|
||||
const forward = (destination: WebSocket, data: RawData, binary: boolean) => {
|
||||
if (destination.readyState !== WebSocket.OPEN || rawBytes(data) > WORKER_PROTOCOL_MAX_PAYLOAD_BYTES) {
|
||||
stopBoth();
|
||||
return;
|
||||
}
|
||||
destination.send(data, { binary }, (error) => {
|
||||
if (error) {
|
||||
stopBoth();
|
||||
}
|
||||
});
|
||||
// WebSocket has no bounded pull API. Terminating both peers on a bounded
|
||||
// queue is the relay's backpressure contract; it cannot become a buffer.
|
||||
if (destination.bufferedAmount > RELAY_MAX_BUFFERED_BYTES) {
|
||||
stopBoth();
|
||||
}
|
||||
};
|
||||
|
||||
upstream.once("open", () => {
|
||||
clearTimeout(connectTimeout);
|
||||
const tlsError = target.validateSocket(upstream);
|
||||
if (tlsError) {
|
||||
stopBoth();
|
||||
return;
|
||||
}
|
||||
linked = true;
|
||||
for (const frame of pending.splice(0)) {
|
||||
forward(upstream, frame.data, frame.binary);
|
||||
}
|
||||
pendingBytes = 0;
|
||||
});
|
||||
connectTimeout = setTimeout(stopBoth, RELAY_CONNECT_TIMEOUT_MS);
|
||||
connectTimeout.unref?.();
|
||||
upstream.once("error", stopBoth);
|
||||
client.once("error", stopBoth);
|
||||
upstream.on("message", (data, binary) => forward(client, data, binary));
|
||||
client.on("message", (data, binary) => {
|
||||
const bytes = rawBytes(data);
|
||||
if (bytes > WORKER_PROTOCOL_MAX_PAYLOAD_BYTES) {
|
||||
stopBoth();
|
||||
return;
|
||||
}
|
||||
if (!linked) {
|
||||
pendingBytes += bytes;
|
||||
if (pendingBytes > RELAY_MAX_PENDING_BYTES) {
|
||||
stopBoth();
|
||||
return;
|
||||
}
|
||||
pending.push({ data, binary });
|
||||
return;
|
||||
}
|
||||
forward(upstream, data, binary);
|
||||
});
|
||||
const closePeer = (peer: WebSocket) => () => {
|
||||
sockets.delete(peer);
|
||||
if (peer.readyState === WebSocket.OPEN || peer.readyState === WebSocket.CLOSING) {
|
||||
stopSocket(peer, 1000);
|
||||
}
|
||||
};
|
||||
client.once("close", closePeer(upstream));
|
||||
upstream.once("close", closePeer(client));
|
||||
};
|
||||
|
||||
server.on("upgrade", (request: IncomingMessage, socket, head) => {
|
||||
if (closing || accepted || request.method !== "GET" || request.url !== RELAY_PATH) {
|
||||
rejectUpgrade(socket, accepted ? 409 : 404);
|
||||
return;
|
||||
}
|
||||
accepted = true;
|
||||
websocketServer.handleUpgrade(request, socket, head, (client) => attachRelay(client));
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(socketPath, () => {
|
||||
server.off("error", reject);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
await fs.chmod(socketPath, 0o600);
|
||||
|
||||
return Object.freeze({
|
||||
socketPath,
|
||||
close: async () => {
|
||||
if (closing) {
|
||||
return;
|
||||
}
|
||||
closing = true;
|
||||
for (const socket of sockets) {
|
||||
socket.terminate();
|
||||
}
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
await fs.rm(socketPath, { force: true });
|
||||
websocketServer.close();
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -83,6 +83,51 @@ function launchIds(database: ReturnType<typeof openOpenClawStateDatabase>["db"])
|
||||
}
|
||||
|
||||
describe("node worker launch store pruning", () => {
|
||||
it("persists the selected container engine with the durable launch identity", () => {
|
||||
const { database, store } = fixture();
|
||||
const supervisor = requireNodeWorkerProcessIdentity(process.pid);
|
||||
const launchId = "container-launch";
|
||||
const planHash = "c".repeat(64);
|
||||
store.claim(
|
||||
{
|
||||
launchId,
|
||||
planHash,
|
||||
gatewayNamespace: "gateway-1",
|
||||
environmentId: "environment-1",
|
||||
sessionId: "session-1",
|
||||
ownerEpoch: 3,
|
||||
placementGeneration: 4,
|
||||
runId: "run-1",
|
||||
},
|
||||
supervisor,
|
||||
2,
|
||||
NOW_MS,
|
||||
);
|
||||
|
||||
store.recordContainerLaunch({
|
||||
launchId,
|
||||
planHash,
|
||||
engine: "podman",
|
||||
expiresAtMs: NOW_MS + DAY_MS,
|
||||
});
|
||||
|
||||
expect(store.getContainerLaunchEngine({ launchId, planHash })).toBe("podman");
|
||||
expect(store.getContainerLaunchLease({ launchId, planHash })).toEqual({
|
||||
engine: "podman",
|
||||
expiresAtMs: NOW_MS + DAY_MS,
|
||||
});
|
||||
expect(
|
||||
database
|
||||
.prepare("SELECT plan_hash, container_engine FROM node_worker_container_launches")
|
||||
.all(),
|
||||
).toEqual([{ plan_hash: planHash, container_engine: "podman" }]);
|
||||
expect(
|
||||
database
|
||||
.prepare("SELECT plan_hash, expires_at_ms FROM node_worker_container_leases")
|
||||
.all(),
|
||||
).toEqual([{ plan_hash: planHash, expires_at_ms: NOW_MS + DAY_MS }]);
|
||||
});
|
||||
|
||||
it("lazily ensures the terminal expiry index for existing databases", () => {
|
||||
const { database, env } = fixture();
|
||||
expect(hasTerminalExpiryIndex(database)).toBe(true);
|
||||
|
||||
@@ -25,9 +25,19 @@ type NodeWorkerLaunchState =
|
||||
| "interrupted"
|
||||
| "cancelled";
|
||||
export type NodeWorkerTerminalState = Exclude<NodeWorkerLaunchState, "pending" | "running">;
|
||||
export type NodeWorkerContainerEngineId = "docker" | "podman";
|
||||
|
||||
type NodeWorkerLaunchDatabase = Pick<OpenClawStateDatabase, "node_worker_launches">;
|
||||
type NodeWorkerLaunchDatabase = Pick<
|
||||
OpenClawStateDatabase,
|
||||
"node_worker_container_launches" | "node_worker_container_leases" | "node_worker_launches"
|
||||
>;
|
||||
type NodeWorkerLaunchRow = Selectable<NodeWorkerLaunchDatabase["node_worker_launches"]>;
|
||||
type NodeWorkerContainerLaunchRow = Selectable<
|
||||
NodeWorkerLaunchDatabase["node_worker_container_launches"]
|
||||
>;
|
||||
type NodeWorkerContainerLeaseRow = Selectable<
|
||||
NodeWorkerLaunchDatabase["node_worker_container_leases"]
|
||||
>;
|
||||
|
||||
export type NodeWorkerLaunchReceipt = {
|
||||
launchId: string;
|
||||
@@ -107,6 +117,32 @@ function readRow(database: DatabaseSync, launchId: string): NodeWorkerLaunchRow
|
||||
);
|
||||
}
|
||||
|
||||
function readContainerLaunchRow(
|
||||
database: DatabaseSync,
|
||||
launchId: string,
|
||||
): NodeWorkerContainerLaunchRow | undefined {
|
||||
return executeSqliteQueryTakeFirstSync(
|
||||
database,
|
||||
query(database)
|
||||
.selectFrom("node_worker_container_launches")
|
||||
.selectAll()
|
||||
.where("launch_id", "=", launchId),
|
||||
);
|
||||
}
|
||||
|
||||
function readContainerLeaseRow(
|
||||
database: DatabaseSync,
|
||||
launchId: string,
|
||||
): NodeWorkerContainerLeaseRow | undefined {
|
||||
return executeSqliteQueryTakeFirstSync(
|
||||
database,
|
||||
query(database)
|
||||
.selectFrom("node_worker_container_leases")
|
||||
.selectAll()
|
||||
.where("launch_id", "=", launchId),
|
||||
);
|
||||
}
|
||||
|
||||
function readNonterminalCount(database: DatabaseSync): number {
|
||||
return (
|
||||
executeSqliteQueryTakeFirstSync(
|
||||
@@ -153,6 +189,18 @@ function pruneTerminalRows(params: {
|
||||
if (launchIds.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
executeSqliteQuerySync(
|
||||
params.database,
|
||||
query(params.database)
|
||||
.deleteFrom("node_worker_container_launches")
|
||||
.where("launch_id", "in", launchIds),
|
||||
);
|
||||
executeSqliteQuerySync(
|
||||
params.database,
|
||||
query(params.database)
|
||||
.deleteFrom("node_worker_container_leases")
|
||||
.where("launch_id", "in", launchIds),
|
||||
);
|
||||
const result = executeSqliteQuerySync(
|
||||
params.database,
|
||||
query(params.database)
|
||||
@@ -211,6 +259,12 @@ function validatePlanHash(value: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
function validateContainerEngine(value: string): asserts value is NodeWorkerContainerEngineId {
|
||||
if (value !== "docker" && value !== "podman") {
|
||||
throw new Error("node worker container engine must be docker or podman");
|
||||
}
|
||||
}
|
||||
|
||||
function validateTimestamp(value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new Error("node worker launch timestamp must be a non-negative safe integer");
|
||||
@@ -480,6 +534,98 @@ export class NodeWorkerLaunchStore {
|
||||
});
|
||||
}
|
||||
|
||||
recordContainerLaunch(params: {
|
||||
launchId: string;
|
||||
planHash: string;
|
||||
engine: NodeWorkerContainerEngineId;
|
||||
expiresAtMs: number;
|
||||
}): void {
|
||||
validateIdentifier(params.launchId, "node worker launch id");
|
||||
validatePlanHash(params.planHash);
|
||||
validateContainerEngine(params.engine);
|
||||
validateTimestamp(params.expiresAtMs);
|
||||
this.write("node-worker-launch.record-container", (database) => {
|
||||
const launch = requireMatchingRow(database, params.launchId, params.planHash);
|
||||
if (TERMINAL_STATES.has(launch.state)) {
|
||||
throw new Error("cannot record a container for a terminal node worker launch");
|
||||
}
|
||||
const existing = readContainerLaunchRow(database, params.launchId);
|
||||
if (!existing) {
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
query(database).insertInto("node_worker_container_launches").values({
|
||||
launch_id: params.launchId,
|
||||
plan_hash: params.planHash,
|
||||
container_engine: params.engine,
|
||||
}),
|
||||
);
|
||||
} else if (
|
||||
existing.plan_hash !== params.planHash ||
|
||||
existing.container_engine !== params.engine
|
||||
) {
|
||||
throw new Error("node worker container launch metadata does not match the durable launch");
|
||||
}
|
||||
const lease = readContainerLeaseRow(database, params.launchId);
|
||||
if (!lease) {
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
query(database).insertInto("node_worker_container_leases").values({
|
||||
launch_id: params.launchId,
|
||||
plan_hash: params.planHash,
|
||||
expires_at_ms: params.expiresAtMs,
|
||||
}),
|
||||
);
|
||||
} else if (lease.plan_hash !== params.planHash || lease.expires_at_ms !== params.expiresAtMs) {
|
||||
throw new Error("node worker container lease does not match the durable launch");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
getContainerLaunchEngine(params: {
|
||||
launchId: string;
|
||||
planHash: string;
|
||||
}): NodeWorkerContainerEngineId | undefined {
|
||||
validateIdentifier(params.launchId, "node worker launch id");
|
||||
validatePlanHash(params.planHash);
|
||||
return this.write("node-worker-launch.get-container", (database) => {
|
||||
const metadata = readContainerLaunchRow(database, params.launchId);
|
||||
if (!metadata) {
|
||||
return undefined;
|
||||
}
|
||||
if (metadata.plan_hash !== params.planHash) {
|
||||
throw new Error("node worker container launch metadata does not match the durable launch");
|
||||
}
|
||||
validateContainerEngine(metadata.container_engine);
|
||||
return metadata.container_engine;
|
||||
});
|
||||
}
|
||||
|
||||
getContainerLaunchLease(params: {
|
||||
launchId: string;
|
||||
planHash: string;
|
||||
}): { engine: NodeWorkerContainerEngineId; expiresAtMs: number } | undefined {
|
||||
validateIdentifier(params.launchId, "node worker launch id");
|
||||
validatePlanHash(params.planHash);
|
||||
return this.write("node-worker-launch.get-container-lease", (database) => {
|
||||
const metadata = readContainerLaunchRow(database, params.launchId);
|
||||
const lease = readContainerLeaseRow(database, params.launchId);
|
||||
if (!metadata && !lease) {
|
||||
return undefined;
|
||||
}
|
||||
if (!metadata || !lease) {
|
||||
// Launches written before leases existed still recover through the
|
||||
// engine metadata and are interrupted conservatively on restart.
|
||||
return undefined;
|
||||
}
|
||||
if (metadata.plan_hash !== params.planHash || lease.plan_hash !== params.planHash) {
|
||||
throw new Error("node worker container lease metadata does not match the durable launch");
|
||||
}
|
||||
validateContainerEngine(metadata.container_engine);
|
||||
validateTimestamp(lease.expires_at_ms);
|
||||
return { engine: metadata.container_engine, expiresAtMs: lease.expires_at_ms };
|
||||
});
|
||||
}
|
||||
|
||||
getMatching(expected: NodeWorkerSupervisorIdentity): NodeWorkerLaunchReceipt | undefined {
|
||||
validateIdentifier(expected.launchId, "node worker launch id");
|
||||
validatePlanHash(expected.planHash);
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import fsp from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { isRecord } from "@openclaw/normalization-core/record-coerce";
|
||||
import { isGatewayLoopbackHost } from "../../packages/gateway-client/src/websocket-transport.js";
|
||||
import {
|
||||
loadOrCreateProcessDeviceIdentity,
|
||||
signDevicePayload,
|
||||
type DeviceIdentity,
|
||||
} from "../infra/device-identity.js";
|
||||
import { isPathInside } from "../infra/path-guards.js";
|
||||
import {
|
||||
buildNodeWorkerMemoryProjectionRequestProofPayload,
|
||||
nodeWorkerMemoryProjectionTransferPath,
|
||||
parseNodeWorkerMemoryProjectionPayload,
|
||||
NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER,
|
||||
NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER,
|
||||
NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER,
|
||||
type NodeWorkerMemoryProjection,
|
||||
type NodeWorkerMemoryProjectionPayload,
|
||||
} from "../worker/node-memory-projection-protocol.js";
|
||||
import type { WorkerConnectionEndpoint } from "../worker/worker-connection-endpoint.js";
|
||||
import { openNodeWorkerTransferHttpRequest } from "./node-worker-transfer-http.js";
|
||||
|
||||
const PROJECTION_RESPONSE_MAX_BYTES = 3 * 1024 * 1024;
|
||||
const PROJECTION_ROOT = "memory-projections";
|
||||
const PROJECTION_ROOT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/u;
|
||||
const PROJECTION_SOCKET_FORBIDDEN_ROOTS = new Set(["opt", "run", "workspace"]);
|
||||
|
||||
type ProjectionIdentity = Readonly<{
|
||||
gatewayNamespace: string;
|
||||
launchId: string;
|
||||
planHash: string;
|
||||
}>;
|
||||
|
||||
function throwIfProjectionStagingAborted(signal?: AbortSignal): void {
|
||||
signal?.throwIfAborted();
|
||||
}
|
||||
|
||||
function currentNonRootUid(): number {
|
||||
if (process.platform === "win32" || typeof process.getuid !== "function") {
|
||||
throw new Error("node worker memory projections require a non-root POSIX node host");
|
||||
}
|
||||
const uid = process.getuid();
|
||||
if (!Number.isSafeInteger(uid) || uid <= 0) {
|
||||
throw new Error("node worker memory projections require a non-root POSIX node host");
|
||||
}
|
||||
return uid;
|
||||
}
|
||||
|
||||
function requirePrivateDirectory(directory: string, parent?: string): string {
|
||||
fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
|
||||
fs.chmodSync(directory, 0o700);
|
||||
const stats = fs.lstatSync(directory);
|
||||
const resolved = fs.realpathSync.native(directory);
|
||||
if (
|
||||
stats.isSymbolicLink() ||
|
||||
!stats.isDirectory() ||
|
||||
stats.uid !== currentNonRootUid() ||
|
||||
(stats.mode & 0o077) !== 0 ||
|
||||
(parent !== undefined && fs.realpathSync.native(path.dirname(directory)) !== parent)
|
||||
) {
|
||||
throw new Error("node worker memory projection path is not a private owned directory");
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
function projectionDirectoryName(identity: ProjectionIdentity): string {
|
||||
if (!/^[a-f0-9]{64}$/u.test(identity.planHash)) {
|
||||
throw new Error("node worker memory projection plan hash is invalid");
|
||||
}
|
||||
return createHash("sha256")
|
||||
.update(`${identity.gatewayNamespace}\0${identity.launchId}\0${identity.planHash}`)
|
||||
.digest("hex");
|
||||
}
|
||||
|
||||
function assertPayloadIntegrity(payload: NodeWorkerMemoryProjectionPayload): Array<{
|
||||
root: string;
|
||||
leaf: string;
|
||||
bytes: Buffer;
|
||||
}> {
|
||||
const roots = new Set<string>();
|
||||
return payload.files.map((file) => {
|
||||
const [root, leaf] = file.virtualPath.split("/");
|
||||
const rootKey = root!.toLocaleLowerCase("en-US");
|
||||
// `memory` is the canonical authorized-view root. It stays nested below the
|
||||
// container's `/memory` mount, so it cannot shadow a container filesystem root.
|
||||
if (!PROJECTION_ROOT_PATTERN.test(root!) || PROJECTION_SOCKET_FORBIDDEN_ROOTS.has(rootKey)) {
|
||||
throw new Error("node worker memory projection root is unsafe");
|
||||
}
|
||||
roots.add(rootKey);
|
||||
const bytes = Buffer.from(file.contentBase64, "base64");
|
||||
if (createHash("sha256").update(bytes).digest("hex") !== file.sha256) {
|
||||
throw new Error("node worker memory projection digest mismatch");
|
||||
}
|
||||
if (!leaf || leaf === "." || leaf === ".." || leaf.includes("/") || leaf.includes("\\")) {
|
||||
throw new Error("node worker memory projection path is unsafe");
|
||||
}
|
||||
return { root: root!, leaf, bytes };
|
||||
});
|
||||
}
|
||||
|
||||
function assertProjectionTransferTransport(endpoint: WorkerConnectionEndpoint): void {
|
||||
const gateway = new URL(endpoint.url);
|
||||
if (gateway.protocol === "ws:" && !isGatewayLoopbackHost(gateway.hostname)) {
|
||||
throw new Error(
|
||||
"node worker memory projection requires wss:// for a non-loopback Gateway endpoint",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function readProjectionResponse(params: {
|
||||
endpoint: WorkerConnectionEndpoint;
|
||||
projection: NodeWorkerMemoryProjection;
|
||||
deviceIdentity: DeviceIdentity;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<NodeWorkerMemoryProjectionPayload> {
|
||||
if (params.endpoint.kind !== "websocket") {
|
||||
throw new Error("node worker memory projection requires a Gateway WebSocket endpoint");
|
||||
}
|
||||
assertProjectionTransferTransport(params.endpoint);
|
||||
const signedAtMs = Date.now();
|
||||
const proofPayload = buildNodeWorkerMemoryProjectionRequestProofPayload({
|
||||
reference: params.projection.reference,
|
||||
binding: params.projection.binding,
|
||||
nodeId: params.deviceIdentity.deviceId,
|
||||
signedAtMs,
|
||||
});
|
||||
const response = await openNodeWorkerTransferHttpRequest({
|
||||
gatewayUrl: params.endpoint.url,
|
||||
...(params.endpoint.tlsFingerprint ? { tlsFingerprint: params.endpoint.tlsFingerprint } : {}),
|
||||
...(params.endpoint.cloudflareAccess
|
||||
? { cloudflareAccess: params.endpoint.cloudflareAccess }
|
||||
: {}),
|
||||
routePath: nodeWorkerMemoryProjectionTransferPath(),
|
||||
method: "GET",
|
||||
token: params.projection.reference,
|
||||
headers: {
|
||||
[NODE_WORKER_MEMORY_PROJECTION_PROOF_NODE_HEADER]: params.deviceIdentity.deviceId,
|
||||
[NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNED_AT_HEADER]: String(signedAtMs),
|
||||
[NODE_WORKER_MEMORY_PROJECTION_PROOF_SIGNATURE_HEADER]: signDevicePayload(
|
||||
params.deviceIdentity.privateKeyPem,
|
||||
proofPayload,
|
||||
),
|
||||
},
|
||||
...(params.signal ? { signal: params.signal } : {}),
|
||||
});
|
||||
if (response.statusCode !== 200) {
|
||||
response.resume();
|
||||
throw new Error(`node worker memory projection transfer failed (${response.statusCode ?? 0})`);
|
||||
}
|
||||
const chunks: Buffer[] = [];
|
||||
let bytes = 0;
|
||||
for await (const chunk of response) {
|
||||
const value = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
||||
bytes += value.byteLength;
|
||||
if (bytes > PROJECTION_RESPONSE_MAX_BYTES) {
|
||||
response.destroy(new Error("node worker memory projection response exceeds its limit"));
|
||||
throw new Error("node worker memory projection response exceeds its limit");
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
let value: unknown;
|
||||
try {
|
||||
value = JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
||||
} catch {
|
||||
throw new Error("node worker memory projection response is malformed");
|
||||
}
|
||||
const payload = parseNodeWorkerMemoryProjectionPayload(value);
|
||||
if (!payload) {
|
||||
throw new Error("node worker memory projection response violated its bounded contract");
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns node-private projection staging. It is intentionally distinct from
|
||||
* workspace synchronization: projection bytes are immutable and never writable by the worker.
|
||||
*/
|
||||
export class NodeWorkerMemoryProjectionRuntime {
|
||||
private readonly root: string;
|
||||
private readonly deviceIdentity: DeviceIdentity;
|
||||
|
||||
constructor(options: { root: string; deviceIdentity?: DeviceIdentity }) {
|
||||
const base = path.resolve(options.root, PROJECTION_ROOT);
|
||||
this.root = requirePrivateDirectory(base);
|
||||
this.deviceIdentity = options.deviceIdentity ?? loadOrCreateProcessDeviceIdentity();
|
||||
}
|
||||
|
||||
private resolveDirectory(identity: ProjectionIdentity): string {
|
||||
return path.join(this.root, projectionDirectoryName(identity));
|
||||
}
|
||||
|
||||
async stage(params: {
|
||||
identity: ProjectionIdentity;
|
||||
projection: NodeWorkerMemoryProjection;
|
||||
endpoint: WorkerConnectionEndpoint;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<string> {
|
||||
throwIfProjectionStagingAborted(params.signal);
|
||||
if (params.projection.expiresAtMs <= Date.now()) {
|
||||
throw new Error("node worker memory projection lease expired before staging");
|
||||
}
|
||||
const destination = this.resolveDirectory(params.identity);
|
||||
throwIfProjectionStagingAborted(params.signal);
|
||||
await this.remove(params.identity);
|
||||
throwIfProjectionStagingAborted(params.signal);
|
||||
const payload = await readProjectionResponse({
|
||||
endpoint: params.endpoint,
|
||||
projection: params.projection,
|
||||
deviceIdentity: this.deviceIdentity,
|
||||
...(params.signal ? { signal: params.signal } : {}),
|
||||
});
|
||||
throwIfProjectionStagingAborted(params.signal);
|
||||
if (params.projection.expiresAtMs <= Date.now()) {
|
||||
throw new Error("node worker memory projection lease expired during staging");
|
||||
}
|
||||
const files = assertPayloadIntegrity(payload);
|
||||
throwIfProjectionStagingAborted(params.signal);
|
||||
const temporary = await fsp.mkdtemp(path.join(this.root, ".projection-"));
|
||||
let staged = false;
|
||||
try {
|
||||
throwIfProjectionStagingAborted(params.signal);
|
||||
await fsp.chmod(temporary, 0o700);
|
||||
const preparedRoots = new Set<string>();
|
||||
for (const file of files) {
|
||||
throwIfProjectionStagingAborted(params.signal);
|
||||
const root = path.join(temporary, file.root);
|
||||
if (!preparedRoots.has(file.root)) {
|
||||
// The node must write the immutable payload before the directory can
|
||||
// become read-only to the container user.
|
||||
await fsp.mkdir(root, { mode: 0o700 });
|
||||
throwIfProjectionStagingAborted(params.signal);
|
||||
const rootStats = await fsp.lstat(root);
|
||||
if (rootStats.isSymbolicLink() || !rootStats.isDirectory() || rootStats.nlink !== 2) {
|
||||
throw new Error("node worker memory projection root is not a private directory");
|
||||
}
|
||||
preparedRoots.add(file.root);
|
||||
}
|
||||
const target = path.join(root, file.leaf);
|
||||
throwIfProjectionStagingAborted(params.signal);
|
||||
const handle = await fsp.open(target, "wx", 0o400);
|
||||
try {
|
||||
throwIfProjectionStagingAborted(params.signal);
|
||||
await handle.writeFile(file.bytes);
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
throwIfProjectionStagingAborted(params.signal);
|
||||
const stats = await fsp.lstat(target);
|
||||
if (
|
||||
stats.isSymbolicLink() ||
|
||||
!stats.isFile() ||
|
||||
stats.nlink !== 1 ||
|
||||
stats.size !== file.bytes.byteLength ||
|
||||
stats.uid !== currentNonRootUid()
|
||||
) {
|
||||
throw new Error("node worker memory projection staged file failed integrity validation");
|
||||
}
|
||||
}
|
||||
for (const root of preparedRoots) {
|
||||
throwIfProjectionStagingAborted(params.signal);
|
||||
await fsp.chmod(path.join(temporary, root), 0o500);
|
||||
}
|
||||
throwIfProjectionStagingAborted(params.signal);
|
||||
await fsp.rename(temporary, destination);
|
||||
staged = true;
|
||||
throwIfProjectionStagingAborted(params.signal);
|
||||
const resolved = await fsp.realpath(destination);
|
||||
const stats = await fsp.lstat(resolved);
|
||||
if (
|
||||
stats.isSymbolicLink() ||
|
||||
!stats.isDirectory() ||
|
||||
path.dirname(resolved) !== this.root ||
|
||||
!isPathInside(this.root, resolved)
|
||||
) {
|
||||
throw new Error("node worker memory projection escaped its private root");
|
||||
}
|
||||
throwIfProjectionStagingAborted(params.signal);
|
||||
return resolved;
|
||||
} catch (error) {
|
||||
await fsp.rm(temporary, { recursive: true, force: true }).catch(() => undefined);
|
||||
if (staged) {
|
||||
await this.remove(params.identity);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async remove(identity: ProjectionIdentity): Promise<void> {
|
||||
const target = this.resolveDirectory(identity);
|
||||
try {
|
||||
const [stats, parent, resolved] = await Promise.all([
|
||||
fsp.lstat(target),
|
||||
fsp.realpath(this.root),
|
||||
fsp.realpath(target),
|
||||
]);
|
||||
if (
|
||||
stats.isSymbolicLink() ||
|
||||
!stats.isDirectory() ||
|
||||
path.dirname(resolved) !== parent ||
|
||||
!isPathInside(parent, resolved)
|
||||
) {
|
||||
throw new Error("node worker memory projection cleanup target is not owned");
|
||||
}
|
||||
// Projection roots are made read-only before mount. Restore write access
|
||||
// only after revalidating the exact node-owned tree so cleanup cannot be
|
||||
// bypassed by its own immutable-file policy.
|
||||
await fsp.chmod(resolved, 0o700);
|
||||
for (const child of await fsp.readdir(resolved, { withFileTypes: true })) {
|
||||
if (child.isSymbolicLink() || !child.isDirectory()) {
|
||||
throw new Error("node worker memory projection cleanup tree is not owned");
|
||||
}
|
||||
const childResolved = await fsp.realpath(path.join(resolved, child.name));
|
||||
if (path.dirname(childResolved) !== resolved || !isPathInside(resolved, childResolved)) {
|
||||
throw new Error("node worker memory projection cleanup tree escaped its root");
|
||||
}
|
||||
await fsp.chmod(childResolved, 0o700);
|
||||
}
|
||||
await fsp.rm(target, { recursive: true, force: true });
|
||||
} catch (error) {
|
||||
if (!isRecord(error) || error.code !== "ENOENT") {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { NodeWorkerLaunchReceipt } from "./node-worker-launch-store.js";
|
||||
import { projectNodeWorkerSupervisorReceipt } from "./node-worker-supervisor-contract.js";
|
||||
|
||||
function terminalReceipt(worker: NodeWorkerLaunchReceipt["worker"]): NodeWorkerLaunchReceipt {
|
||||
return {
|
||||
launchId: "launch-1",
|
||||
planHash: "a".repeat(64),
|
||||
gatewayNamespace: "gateway-1",
|
||||
environmentId: "environment-1",
|
||||
sessionId: "session-1",
|
||||
ownerEpoch: 3,
|
||||
placementGeneration: 4,
|
||||
runId: "run-1",
|
||||
state: "cancelled",
|
||||
supervisor: { pid: 1, startTime: 1 },
|
||||
worker,
|
||||
resultJson: null,
|
||||
errorText: "node worker launch cancelled",
|
||||
completedAtMs: 2,
|
||||
createdAtMs: 1,
|
||||
updatedAtMs: 2,
|
||||
};
|
||||
}
|
||||
|
||||
describe("node worker supervisor receipt projection", () => {
|
||||
it("exposes execution readiness without exposing a worker process identity", () => {
|
||||
expect(projectNodeWorkerSupervisorReceipt(terminalReceipt(null))).toMatchObject({
|
||||
state: "cancelled",
|
||||
executionStarted: false,
|
||||
});
|
||||
expect(projectNodeWorkerSupervisorReceipt(terminalReceipt({ pid: 2, startTime: 2 }))).toMatchObject({
|
||||
state: "cancelled",
|
||||
executionStarted: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -12,12 +12,16 @@ import type { WorkerConnectionEndpoint } from "../worker/worker-connection-endpo
|
||||
import type { NodeWorkerLaunchReceipt } from "./node-worker-launch-store.js";
|
||||
|
||||
export {
|
||||
NODE_WORKER_EXECUTION_CONTAINER_V1,
|
||||
NODE_WORKER_EXECUTION_HOST_V1,
|
||||
nodeWorkerMemoryProjectionLaunchBinding,
|
||||
nodeWorkerPlanHash,
|
||||
parseNodeWorkerCancelInput,
|
||||
parseNodeWorkerLaunchInput,
|
||||
parseNodeWorkerLookupInput,
|
||||
} from "../worker/node-supervisor-protocol.js";
|
||||
export type {
|
||||
NodeWorkerExecution,
|
||||
NodeWorkerLaunchInput,
|
||||
NodeWorkerSupervisorIdentity,
|
||||
NodeWorkerSupervisorReceipt,
|
||||
@@ -55,7 +59,14 @@ export function projectNodeWorkerSupervisorReceipt(
|
||||
: receipt.state === "failed" ||
|
||||
receipt.state === "interrupted" ||
|
||||
receipt.state === "cancelled"
|
||||
? { ...identity, state: receipt.state, errorText: receipt.errorText }
|
||||
? {
|
||||
...identity,
|
||||
state: receipt.state,
|
||||
errorText: receipt.errorText,
|
||||
// Worker identity is persisted only by markRunning, after the private child
|
||||
// acknowledgement. Do not expose the PID, only the durable execution fact.
|
||||
executionStarted: receipt.worker !== null,
|
||||
}
|
||||
: { ...identity, state: receipt.state };
|
||||
const parsed = parseNodeWorkerSupervisorReceipt(projected);
|
||||
if (!parsed) {
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { stableStringify } from "@openclaw/normalization-core";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import {
|
||||
@@ -16,9 +14,11 @@ import {
|
||||
requireNodeWorkerProcessIdentity,
|
||||
type NodeWorkerProcessIdentity,
|
||||
} from "./node-worker-process-identity.js";
|
||||
import { nodeWorkerPlanHash } from "./node-worker-supervisor-contract.js";
|
||||
import { createNodeWorkerSupervisor } from "./node-worker-supervisor.js";
|
||||
import {
|
||||
testNodeWorkerLaunchIdentity,
|
||||
testNodeWorkerMemoryProjection,
|
||||
TEST_WORKER_ENDPOINT,
|
||||
testWorkerLaunchInput,
|
||||
writeNodeWorkerFixture,
|
||||
@@ -58,16 +58,7 @@ function fixture(label: string) {
|
||||
}
|
||||
|
||||
function planHash(input: ReturnType<typeof testWorkerLaunchInput>): string {
|
||||
return createHash("sha256")
|
||||
.update(
|
||||
stableStringify({
|
||||
expectedBundleHash: input.expectedBundleHash,
|
||||
descriptor: input.descriptor,
|
||||
gatewayNamespace: input.gatewayNamespace,
|
||||
placementGeneration: input.placementGeneration,
|
||||
}),
|
||||
)
|
||||
.digest("hex");
|
||||
return nodeWorkerPlanHash(input);
|
||||
}
|
||||
|
||||
function insertLaunch(params: {
|
||||
@@ -247,6 +238,110 @@ describe("node worker supervisor recovery", () => {
|
||||
await supervisor.close();
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"does not settle a stale pending container launch until durable cleanup succeeds",
|
||||
async () => {
|
||||
const { bundleRoot, env, root, workspaceDir } = fixture("node-worker-pending-container-");
|
||||
const bin = path.join(root, "bin");
|
||||
const docker = path.join(bin, "docker");
|
||||
const commandLog = path.join(root, "docker-commands.log");
|
||||
fs.mkdirSync(bin, { recursive: true });
|
||||
const writeDocker = (exitCode: number) => {
|
||||
fs.writeFileSync(
|
||||
docker,
|
||||
`#!/bin/sh\nprintf '%s\\n' "$1" >> ${JSON.stringify(commandLog)}\nexit ${String(exitCode)}\n`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
};
|
||||
writeDocker(1);
|
||||
vi.stubEnv("PATH", `${bin}${path.delimiter}${process.env.PATH ?? ""}`);
|
||||
try {
|
||||
const store = new NodeWorkerLaunchStore({ env });
|
||||
store.get("schema-probe");
|
||||
const input = testWorkerLaunchInput(workspaceDir, "pending-container-launch");
|
||||
input.execution = { kind: "container-v1" };
|
||||
input.descriptor.assignment.memoryReadEnforced = true;
|
||||
input.descriptor.assignment.workspaceDir = "/workspace";
|
||||
input.memoryProjection = testNodeWorkerMemoryProjection(input);
|
||||
insertLaunch({
|
||||
env,
|
||||
input,
|
||||
state: "pending",
|
||||
supervisor: { pid: 2_147_483_647, startTime: 1 },
|
||||
});
|
||||
store.recordContainerLaunch({
|
||||
launchId: input.launchId,
|
||||
planHash: planHash(input),
|
||||
engine: "docker",
|
||||
expiresAtMs: input.memoryProjection.expiresAtMs,
|
||||
});
|
||||
|
||||
const first = createNodeWorkerSupervisor({ bundleRoot, env });
|
||||
await expect(first.initialize()).rejects.toThrow("Docker command failed");
|
||||
expect(store.get(input.launchId)).toMatchObject({ state: "pending" });
|
||||
await first.close().catch(() => undefined);
|
||||
|
||||
writeDocker(0);
|
||||
const recovered = createNodeWorkerSupervisor({ bundleRoot, env });
|
||||
await recovered.initialize();
|
||||
expect(await recovered.status(input.launchId)).toMatchObject({ state: "interrupted" });
|
||||
expect(fs.readFileSync(commandLog, "utf8").split("\n").filter(Boolean)).toEqual([
|
||||
"ps",
|
||||
"ps",
|
||||
]);
|
||||
await recovered.close();
|
||||
} finally {
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"withdraws an expired durable projection lease during stale container recovery",
|
||||
async () => {
|
||||
const { bundleRoot, env, root, workspaceDir } = fixture("node-worker-expired-container-");
|
||||
const bin = path.join(root, "bin");
|
||||
fs.mkdirSync(bin, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(bin, "docker"),
|
||||
"#!/bin/sh\nexit 0\n",
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
vi.stubEnv("PATH", `${bin}${path.delimiter}${process.env.PATH ?? ""}`);
|
||||
try {
|
||||
const store = new NodeWorkerLaunchStore({ env });
|
||||
store.get("schema-probe");
|
||||
const input = testWorkerLaunchInput(workspaceDir, "expired-container-launch");
|
||||
input.execution = { kind: "container-v1" };
|
||||
input.descriptor.assignment.memoryReadEnforced = true;
|
||||
input.descriptor.assignment.workspaceDir = "/workspace";
|
||||
input.memoryProjection = { ...testNodeWorkerMemoryProjection(input), expiresAtMs: 1 };
|
||||
insertLaunch({
|
||||
env,
|
||||
input,
|
||||
state: "pending",
|
||||
supervisor: { pid: 2_147_483_647, startTime: 1 },
|
||||
});
|
||||
store.recordContainerLaunch({
|
||||
launchId: input.launchId,
|
||||
planHash: planHash(input),
|
||||
engine: "docker",
|
||||
expiresAtMs: input.memoryProjection.expiresAtMs,
|
||||
});
|
||||
|
||||
const recovered = createNodeWorkerSupervisor({ bundleRoot, env, now: () => 2 });
|
||||
await recovered.initialize();
|
||||
await expect(recovered.status(input.launchId)).resolves.toMatchObject({
|
||||
state: "cancelled",
|
||||
errorText: "node worker memory projection lease expired before the worker launch started",
|
||||
});
|
||||
await recovered.close();
|
||||
} finally {
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.runIf(process.platform !== "win32").each([
|
||||
{ operation: "replay" as const, state: "interrupted" as const },
|
||||
{ operation: "cancel" as const, state: "cancelled" as const },
|
||||
|
||||
@@ -7,6 +7,8 @@ import {
|
||||
import type { WorkerLaunchPlan } from "../worker/launch-descriptor.js";
|
||||
import type { WorkerConnectionEndpoint } from "../worker/worker-connection-endpoint.js";
|
||||
import {
|
||||
NODE_WORKER_EXECUTION_HOST_V1,
|
||||
nodeWorkerMemoryProjectionLaunchBinding,
|
||||
nodeWorkerPlanHash,
|
||||
type NodeWorkerLaunchInput,
|
||||
type NodeWorkerSupervisorIdentity,
|
||||
@@ -55,13 +57,33 @@ const onMessage = (message) => {
|
||||
typeof message !== "object" ||
|
||||
message === null ||
|
||||
Array.isArray(message) ||
|
||||
Object.keys(message).length !== 1 ||
|
||||
message.type !== "openclaw-worker-start-v1"
|
||||
Object.keys(message).length !== 3 ||
|
||||
message.type !== "openclaw-worker-start-v1" ||
|
||||
typeof message.launchId !== "string" ||
|
||||
!/^[a-f0-9]{64}$/.test(message.planHash)
|
||||
) {
|
||||
hardTerminate();
|
||||
return;
|
||||
}
|
||||
started = true;
|
||||
const executionAcknowledgement = {
|
||||
type: "openclaw-worker-execution-started-v1",
|
||||
launchId: message.launchId,
|
||||
planHash: message.planHash,
|
||||
};
|
||||
if (descriptor.assignment.prompt === "wrong-execution-ack") {
|
||||
process.send({ ...executionAcknowledgement, launchId: "another-launch" }, () => {});
|
||||
setInterval(() => {}, 1000);
|
||||
return;
|
||||
}
|
||||
if (descriptor.assignment.prompt === "wait-before-execution-ack") {
|
||||
setInterval(() => {}, 1000);
|
||||
return;
|
||||
}
|
||||
process.send(executionAcknowledgement, () => {});
|
||||
if (descriptor.assignment.prompt === "replayed-execution-ack") {
|
||||
setTimeout(() => process.send(executionAcknowledgement, () => {}), 25);
|
||||
}
|
||||
resolveStart();
|
||||
};
|
||||
const onDisconnect = () => {
|
||||
@@ -99,6 +121,8 @@ if (mode === "connection-failure") {
|
||||
setInterval(() => {}, 1000);
|
||||
} else if (mode === "wait") {
|
||||
setInterval(() => {}, 1000);
|
||||
} else if (mode === "replayed-execution-ack") {
|
||||
setInterval(() => {}, 1000);
|
||||
} else if (mode === "tree") {
|
||||
grandchild = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { stdio: "ignore" });
|
||||
fs.writeFileSync(path.join(descriptor.assignment.workspaceDir, "grandchild.pid"), String(grandchild.pid));
|
||||
@@ -164,6 +188,7 @@ export function testWorkerDescriptor(workspaceDir: string, prompt = "success"):
|
||||
},
|
||||
assignment: {
|
||||
agentId: "agent-1",
|
||||
memoryReadEnforced: false,
|
||||
operationalRunInstance: { instanceId: "instance-1", runId: "run-1" },
|
||||
agentRuntimeIdentityToken: "signed-runtime-token",
|
||||
runId: "run-1",
|
||||
@@ -195,6 +220,20 @@ export function testNodeWorkerLaunchIdentity(
|
||||
};
|
||||
}
|
||||
|
||||
export function testNodeWorkerMemoryProjection(
|
||||
input: Omit<NodeWorkerLaunchInput, "memoryProjection">,
|
||||
) {
|
||||
return {
|
||||
version: 1 as const,
|
||||
reference: "a".repeat(43),
|
||||
binding: {
|
||||
launch: nodeWorkerMemoryProjectionLaunchBinding(input),
|
||||
authorization: "b".repeat(64),
|
||||
},
|
||||
expiresAtMs: Date.now() + 60_000,
|
||||
};
|
||||
}
|
||||
|
||||
export function writeNodeWorkerFixture(root: string) {
|
||||
const stateDir = path.join(root, "state-root");
|
||||
const bundleRoot = path.join(root, "bundles-root");
|
||||
@@ -217,5 +256,6 @@ export function testWorkerLaunchInput(
|
||||
expectedBundleHash: TEST_BUNDLE_HASH,
|
||||
placementGeneration: 4,
|
||||
descriptor: testWorkerDescriptor(workspaceDir, prompt),
|
||||
execution: { kind: NODE_WORKER_EXECUTION_HOST_V1 },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -17,10 +17,12 @@ import {
|
||||
requireNodeWorkerProcessIdentity,
|
||||
} from "./node-worker-process-identity.js";
|
||||
import { createNodeWorkerSupervisor } from "./node-worker-supervisor.js";
|
||||
import { NodeWorkerMemoryProjectionRuntime } from "./node-worker-memory-projection.js";
|
||||
import {
|
||||
TEST_WORKER_CREDENTIAL,
|
||||
TEST_WORKER_ENDPOINT,
|
||||
TEST_WORKER_SOURCE,
|
||||
testNodeWorkerMemoryProjection,
|
||||
testNodeWorkerLaunchIdentity,
|
||||
testWorkerDescriptor,
|
||||
testWorkerLaunchInput,
|
||||
@@ -69,6 +71,85 @@ async function waitForTerminal(supervisor: NodeWorkerSupervisor, launchId: strin
|
||||
}
|
||||
|
||||
describe("node worker supervisor", () => {
|
||||
it("rejects an enforced-memory host launch before claiming a worker slot", async () => {
|
||||
const { env, supervisor, workspaceDir } = fixture();
|
||||
const input = launchInput(workspaceDir, "enforced-memory-host-downgrade");
|
||||
input.descriptor.assignment.memoryReadEnforced = true;
|
||||
|
||||
await expect(supervisor.launch(input, TEST_WORKER_ENDPOINT)).rejects.toThrow(
|
||||
"requires container-v1 execution",
|
||||
);
|
||||
expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)).toBeUndefined();
|
||||
await supervisor.close();
|
||||
});
|
||||
|
||||
it("rejects a cross-session memory projection replay before claiming a worker slot", async () => {
|
||||
const { env, supervisor, workspaceDir } = fixture();
|
||||
const input = launchInput(workspaceDir, "enforced-memory-cross-session-replay");
|
||||
input.execution = { kind: "container-v1" };
|
||||
input.descriptor.assignment.memoryReadEnforced = true;
|
||||
input.descriptor.assignment.workspaceDir = "/workspace";
|
||||
input.memoryProjection = testNodeWorkerMemoryProjection(input);
|
||||
const replay = structuredClone(input);
|
||||
replay.descriptor.admission.sessionId = "another-session";
|
||||
|
||||
await expect(supervisor.launch(replay, TEST_WORKER_ENDPOINT)).rejects.toThrow(
|
||||
"memory projection does not match its worker launch",
|
||||
);
|
||||
expect(new NodeWorkerLaunchStore({ env }).get(replay.launchId)).toBeUndefined();
|
||||
await supervisor.close();
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"persists cancellation and cleans the staged projection before a container can spawn",
|
||||
async () => {
|
||||
const { bundleRoot, env, root, workspaceDir } = fixture();
|
||||
const bin = path.join(root, "bin");
|
||||
const commandLog = path.join(root, "docker-commands.log");
|
||||
fs.mkdirSync(bin, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(bin, "docker"),
|
||||
`#!/bin/sh\nprintf '%s\\n' "$1" >> ${JSON.stringify(commandLog)}\nif [ "$1" = info ]; then printf 'test-engine'; fi\nexit 0\n`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
vi.stubEnv("PATH", `${bin}${path.delimiter}${process.env.PATH ?? ""}`);
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const projection = {
|
||||
stage: vi.fn(async () => {
|
||||
controller.abort(new Error("Gateway revoked the projection"));
|
||||
return path.join(root, "staged-projection");
|
||||
}),
|
||||
remove: vi.fn(async () => undefined),
|
||||
} as unknown as NodeWorkerMemoryProjectionRuntime;
|
||||
const supervisor = createNodeWorkerSupervisor({ bundleRoot, env, memoryProjection: projection });
|
||||
const input = launchInput(workspaceDir, "cancelled-before-container-start");
|
||||
input.execution = { kind: "container-v1" };
|
||||
input.descriptor.assignment.memoryReadEnforced = true;
|
||||
input.descriptor.assignment.workspaceDir = "/workspace";
|
||||
input.memoryProjection = testNodeWorkerMemoryProjection(input);
|
||||
|
||||
await expect(supervisor.launch(input, TEST_WORKER_ENDPOINT, controller.signal)).resolves.toMatchObject({
|
||||
state: "cancelled",
|
||||
errorText: "node worker launch cancelled before process start",
|
||||
});
|
||||
expect(projection.stage).toHaveBeenCalledOnce();
|
||||
expect(projection.remove).toHaveBeenCalledWith({
|
||||
gatewayNamespace: input.gatewayNamespace,
|
||||
launchId: input.launchId,
|
||||
planHash: testNodeWorkerLaunchIdentity(input).planHash,
|
||||
});
|
||||
expect(fs.readFileSync(commandLog, "utf8").split("\n")).not.toContain("run");
|
||||
expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)).toMatchObject({
|
||||
state: "cancelled",
|
||||
});
|
||||
await supervisor.close();
|
||||
} finally {
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps construction and close inert without resolving process identity", async () => {
|
||||
const root = tempDirs.make("node-worker-inert-");
|
||||
const { bundleRoot, env } = writeNodeWorkerFixture(root);
|
||||
@@ -513,7 +594,7 @@ describe("node worker supervisor", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("does not open or signal a child after markRunning observes its terminal receipt", async () => {
|
||||
it("returns a terminal receipt when execution-ready races an immediate child completion", async () => {
|
||||
const { supervisor, workspaceDir } = fixture();
|
||||
const input = launchInput(workspaceDir, "fast-terminal-launch", "fast-terminal");
|
||||
vi.spyOn(NodeWorkerLaunchStore.prototype, "markRunning").mockImplementation(
|
||||
@@ -536,7 +617,7 @@ describe("node worker supervisor", () => {
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, 150);
|
||||
});
|
||||
expect(fs.existsSync(marker)).toBe(false);
|
||||
expect(fs.existsSync(marker)).toBe(true);
|
||||
await supervisor.close();
|
||||
});
|
||||
|
||||
@@ -549,7 +630,88 @@ describe("node worker supervisor", () => {
|
||||
const terminal = await waitForTerminal(supervisor, input.launchId);
|
||||
|
||||
expect(fs.existsSync(exitedPath)).toBe(true);
|
||||
expect(terminal.state).toBe("failed");
|
||||
expect(terminal).toMatchObject({ state: "failed", worker: null });
|
||||
await supervisor.close();
|
||||
});
|
||||
|
||||
it("fails closed before durable running when the child acknowledges another launch", async () => {
|
||||
const { env, supervisor, workspaceDir } = fixture();
|
||||
const input = launchInput(workspaceDir, "wrong-execution-ack-launch", "wrong-execution-ack");
|
||||
|
||||
await expect(supervisor.launch(input, TEST_WORKER_ENDPOINT)).resolves.toMatchObject({
|
||||
state: "interrupted",
|
||||
worker: null,
|
||||
});
|
||||
expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)).toMatchObject({
|
||||
state: "interrupted",
|
||||
worker: null,
|
||||
});
|
||||
await supervisor.close();
|
||||
});
|
||||
|
||||
it("persists cancellation before a child acknowledgement without a worker identity", async () => {
|
||||
const { env, supervisor, workspaceDir } = fixture();
|
||||
const input = launchInput(
|
||||
workspaceDir,
|
||||
"cancel-before-execution-ack-launch",
|
||||
"wait-before-execution-ack",
|
||||
);
|
||||
const launching = supervisor.launch(input, TEST_WORKER_ENDPOINT);
|
||||
await vi.waitFor(async () => {
|
||||
expect((await supervisor.status(input.launchId))?.state).toBe("pending");
|
||||
});
|
||||
|
||||
await expect(supervisor.cancel(testNodeWorkerLaunchIdentity(input))).resolves.toMatchObject({
|
||||
state: "cancelled",
|
||||
worker: null,
|
||||
});
|
||||
await expect(launching).resolves.toMatchObject({ state: "cancelled", worker: null });
|
||||
expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)).toMatchObject({
|
||||
state: "cancelled",
|
||||
worker: null,
|
||||
});
|
||||
await supervisor.close();
|
||||
});
|
||||
|
||||
it("interrupts a pending child on close without a durable worker identity", async () => {
|
||||
const { env, supervisor, workspaceDir } = fixture();
|
||||
const input = launchInput(
|
||||
workspaceDir,
|
||||
"close-before-execution-ack-launch",
|
||||
"wait-before-execution-ack",
|
||||
);
|
||||
const launching = supervisor.launch(input, TEST_WORKER_ENDPOINT);
|
||||
await vi.waitFor(async () => {
|
||||
expect((await supervisor.status(input.launchId))?.state).toBe("pending");
|
||||
});
|
||||
|
||||
await supervisor.close();
|
||||
await expect(launching).resolves.toMatchObject({ state: "interrupted", worker: null });
|
||||
expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)).toMatchObject({
|
||||
state: "interrupted",
|
||||
worker: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("retires a child that replays its execution acknowledgement after durable readiness", async () => {
|
||||
const { supervisor, workspaceDir } = fixture();
|
||||
const input = launchInput(
|
||||
workspaceDir,
|
||||
"replayed-execution-ack-launch",
|
||||
"replayed-execution-ack",
|
||||
);
|
||||
|
||||
await expect(supervisor.launch(input, TEST_WORKER_ENDPOINT)).resolves.toMatchObject({
|
||||
state: "running",
|
||||
worker: expect.any(Object),
|
||||
});
|
||||
await vi.waitFor(async () => {
|
||||
expect((await supervisor.status(input.launchId))?.state).toBe("interrupted");
|
||||
});
|
||||
expect(await supervisor.status(input.launchId)).toMatchObject({
|
||||
state: "interrupted",
|
||||
worker: expect.any(Object),
|
||||
});
|
||||
await supervisor.close();
|
||||
});
|
||||
|
||||
@@ -606,7 +768,7 @@ describe("node worker supervisor", () => {
|
||||
["cancel", "cancelled"],
|
||||
["close", "interrupted"],
|
||||
] as const)(
|
||||
"%s during startup closes the gate before worker code runs",
|
||||
"%s after execution readiness settles the durable terminal receipt",
|
||||
async (operation, state) => {
|
||||
const { supervisor, workspaceDir } = fixture();
|
||||
const input = launchInput(workspaceDir, `${operation}-startup-launch`, "tree");
|
||||
@@ -630,7 +792,6 @@ describe("node worker supervisor", () => {
|
||||
await stopping;
|
||||
|
||||
expect((await supervisor.status(input.launchId))?.state).toBe(state);
|
||||
expect(fs.existsSync(path.join(workspaceDir, "grandchild.pid"))).toBe(false);
|
||||
await supervisor.close();
|
||||
},
|
||||
);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -97,6 +97,28 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("node worker workspace retention", () => {
|
||||
it.runIf(process.platform !== "win32" && typeof process.getuid === "function" && process.getuid() > 0)(
|
||||
"uses a private short-lived relay directory with a portable Unix socket path",
|
||||
async () => {
|
||||
const workspace = new NodeWorkerWorkspaceRuntime({
|
||||
root: tempDirs.make("node-worker-workspace-relay-root-"),
|
||||
});
|
||||
const params = {
|
||||
gatewayNamespace: "gateway-relay",
|
||||
launchId: "relay-path-test",
|
||||
planHash: "a".repeat(64),
|
||||
};
|
||||
|
||||
const relayDir = workspace.resolveContainerRelayDirectory(params);
|
||||
expect(relayDir).toMatch(/^\/tmp\/openclaw-worker-relays-[0-9]+\/[a-f0-9]{32}$/u);
|
||||
expect(Buffer.byteLength(path.join(relayDir, "gateway.sock"), "utf8")).toBeLessThanOrEqual(100);
|
||||
expect(fs.statSync(relayDir).mode & 0o077).toBe(0);
|
||||
|
||||
await workspace.removeContainerRelayDirectory(params);
|
||||
expect(fs.existsSync(relayDir)).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it("does not delete workspaces before the first Gateway snapshot", async () => {
|
||||
const root = tempDirs.make("node-worker-workspace-retention-startup-");
|
||||
const { bundleRoot, env, workspaceDir } = writeNodeWorkerFixture(root);
|
||||
|
||||
@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import fsp from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { isRecord } from "@openclaw/normalization-core/record-coerce";
|
||||
import { resolveStateDir } from "../config/paths.js";
|
||||
import { isPathInside } from "../infra/path-guards.js";
|
||||
import { KeyedAsyncQueue } from "../plugin-sdk/keyed-async-queue.js";
|
||||
@@ -29,6 +30,12 @@ const WORKSPACE_RETENTION_DELETE_LIMIT = 256;
|
||||
const ENVIRONMENT_HASH_PATTERN = /^[a-f0-9]{16}$/u;
|
||||
const SESSION_HASH_PATTERN = /^[a-f0-9]{32}$/u;
|
||||
const MANIFEST_FILE_PATTERN = /^[a-f0-9]{64}\.json$/u;
|
||||
const NODE_WORKER_CONTAINER_RELAY_ROOT_PREFIX = "/tmp/openclaw-worker-relays-";
|
||||
const NODE_WORKER_CONTAINER_RELAY_SOCKET_NAME = "gateway.sock";
|
||||
// macOS accepts fewer bytes than Linux for a filesystem Unix-domain socket.
|
||||
// Keep the portable bound below both limits rather than advertise a sandbox
|
||||
// that will only fail after the supervisor has durably launched it.
|
||||
const NODE_WORKER_CONTAINER_RELAY_SOCKET_MAX_BYTES = 100;
|
||||
|
||||
type NodeWorkerWorkspaceLaunchReference = {
|
||||
gatewayNamespace: string;
|
||||
@@ -206,6 +213,67 @@ function ensureContainedDirectory(parent: string, name: string): string {
|
||||
return resolved;
|
||||
}
|
||||
|
||||
function currentNonRootUid(): number {
|
||||
if (process.platform === "win32" || typeof process.getuid !== "function") {
|
||||
throw new Error("node worker container relays require a non-root POSIX host user");
|
||||
}
|
||||
const uid = process.getuid();
|
||||
if (!Number.isSafeInteger(uid) || uid <= 0) {
|
||||
throw new Error("node worker container relays require a non-root POSIX host user");
|
||||
}
|
||||
return uid;
|
||||
}
|
||||
|
||||
function containerRelayPaths(params: {
|
||||
gatewayNamespace: string;
|
||||
launchId: string;
|
||||
planHash: string;
|
||||
}): { root: string; directory: string } {
|
||||
const uid = currentNonRootUid();
|
||||
const name = hashPathComponent(
|
||||
`${params.gatewayNamespace}\0${params.launchId}\0${params.planHash}`,
|
||||
32,
|
||||
);
|
||||
return {
|
||||
root: `${NODE_WORKER_CONTAINER_RELAY_ROOT_PREFIX}${uid}`,
|
||||
directory: path.join(`${NODE_WORKER_CONTAINER_RELAY_ROOT_PREFIX}${uid}`, name),
|
||||
};
|
||||
}
|
||||
|
||||
function ensurePrivateRelayDirectory(candidate: string, expectedParent?: string): void {
|
||||
fs.mkdirSync(candidate, { recursive: true, mode: 0o700 });
|
||||
const initialStats = fs.lstatSync(candidate);
|
||||
if (
|
||||
initialStats.isSymbolicLink() ||
|
||||
!initialStats.isDirectory() ||
|
||||
initialStats.uid !== currentNonRootUid()
|
||||
) {
|
||||
throw new Error(
|
||||
"INVALID_REQUEST: node worker container relay path is not a private owned directory",
|
||||
);
|
||||
}
|
||||
fs.chmodSync(candidate, 0o700);
|
||||
const stats = fs.lstatSync(candidate);
|
||||
const resolved = fs.realpathSync.native(candidate);
|
||||
if (
|
||||
stats.isSymbolicLink() ||
|
||||
!stats.isDirectory() ||
|
||||
stats.uid !== currentNonRootUid() ||
|
||||
(stats.mode & 0o077) !== 0 ||
|
||||
(expectedParent !== undefined &&
|
||||
fs.realpathSync.native(path.dirname(candidate)) !== expectedParent)
|
||||
) {
|
||||
throw new Error(
|
||||
"INVALID_REQUEST: node worker container relay path is not a private owned directory",
|
||||
);
|
||||
}
|
||||
if (resolved !== fs.realpathSync.native(candidate)) {
|
||||
throw new Error(
|
||||
"INVALID_REQUEST: node worker container relay path changed while it was prepared",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function resolveArgumentPath(workspaceDir: string, arg: string): string | undefined {
|
||||
if (path.isAbsolute(arg)) {
|
||||
return arg;
|
||||
@@ -233,7 +301,7 @@ function assertWorkspaceArgv(workspaceDir: string, argv: readonly string[]): voi
|
||||
try {
|
||||
resolved = fs.realpathSync.native(candidate);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
if (!isRecord(error) || error.code !== "ENOENT") {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -338,6 +406,81 @@ export class NodeWorkerWorkspaceRuntime {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Container launches never accept a Gateway-provided host path. The node
|
||||
* derives this one canonical session/epoch workspace before mounting it at
|
||||
* the fixed in-container `/workspace` path.
|
||||
*/
|
||||
resolveContainerWorkspace(reference: NodeWorkerWorkspaceLaunchReference): string {
|
||||
if (!Number.isSafeInteger(reference.ownerEpoch) || reference.ownerEpoch < 0) {
|
||||
throw new Error("INVALID_REQUEST: node worker container owner epoch is invalid");
|
||||
}
|
||||
const environmentHash = hashPathComponent(reference.environmentId, 16);
|
||||
const sessionHash = hashPathComponent(reference.sessionId, 32);
|
||||
const gatewayRoot = ensureContainedDirectory(this.root, reference.gatewayNamespace);
|
||||
const workspacesRoot = ensureContainedDirectory(gatewayRoot, "workspaces");
|
||||
const environmentRoot = ensureContainedDirectory(workspacesRoot, environmentHash);
|
||||
const sessionRoot = ensureContainedDirectory(environmentRoot, sessionHash);
|
||||
return ensureContainedDirectory(sessionRoot, String(reference.ownerEpoch));
|
||||
}
|
||||
|
||||
/**
|
||||
* The relay is node-private staging, outside the worker-writable workspace.
|
||||
* Its name is derived from the durable launch identity, not from request data.
|
||||
*/
|
||||
resolveContainerRelayDirectory(params: {
|
||||
gatewayNamespace: string;
|
||||
launchId: string;
|
||||
planHash: string;
|
||||
}): string {
|
||||
if (!/^[a-f0-9]{64}$/u.test(params.planHash)) {
|
||||
throw new Error("INVALID_REQUEST: node worker container plan hash is invalid");
|
||||
}
|
||||
const paths = containerRelayPaths(params);
|
||||
// The mounted directory is host-private, but the Unix socket must retain
|
||||
// this short lexical `/tmp` spelling. Canonical state paths exceed macOS'
|
||||
// socket limit before the container can even connect to the relay.
|
||||
ensurePrivateRelayDirectory(paths.root);
|
||||
const canonicalRoot = fs.realpathSync.native(paths.root);
|
||||
ensurePrivateRelayDirectory(paths.directory, canonicalRoot);
|
||||
const socketPath = path.join(paths.directory, NODE_WORKER_CONTAINER_RELAY_SOCKET_NAME);
|
||||
if (Buffer.byteLength(socketPath, "utf8") > NODE_WORKER_CONTAINER_RELAY_SOCKET_MAX_BYTES) {
|
||||
throw new Error(
|
||||
"INVALID_REQUEST: node worker container relay socket path exceeds the portable limit",
|
||||
);
|
||||
}
|
||||
return paths.directory;
|
||||
}
|
||||
|
||||
async removeContainerRelayDirectory(params: {
|
||||
gatewayNamespace: string;
|
||||
launchId: string;
|
||||
planHash: string;
|
||||
}): Promise<void> {
|
||||
if (!/^[a-f0-9]{64}$/u.test(params.planHash)) {
|
||||
return;
|
||||
}
|
||||
const paths = containerRelayPaths(params);
|
||||
try {
|
||||
const rootStats = await fsp.lstat(paths.root);
|
||||
if (
|
||||
rootStats.isSymbolicLink() ||
|
||||
!rootStats.isDirectory() ||
|
||||
rootStats.uid !== currentNonRootUid() ||
|
||||
(rootStats.mode & 0o077) !== 0
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const canonicalRoot = await fsp.realpath(paths.root);
|
||||
await removeOwnedDirectory(canonicalRoot, paths.directory);
|
||||
await removeIfEmpty(paths.root);
|
||||
} catch (error) {
|
||||
if (!isRecord(error) || error.code !== "ENOENT") {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private beginWorkspaceOperation(gatewayNamespace: string, generationKey: string): () => void {
|
||||
this.activeWorkspaceOperations.set(
|
||||
generationKey,
|
||||
|
||||
@@ -6,6 +6,7 @@ import { getConfigResolutionFacts, setConfigResolutionFacts } from "../config/re
|
||||
import type { GatewayClientOptions } from "../gateway/client.js";
|
||||
import {
|
||||
NODE_RUNNER_INVENTORY_UPDATE_METHOD,
|
||||
NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
|
||||
NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
|
||||
} from "../infra/node-runner-inventory.js";
|
||||
import type { configureNodeHost } from "./config.js";
|
||||
@@ -34,6 +35,8 @@ const mocks = vi.hoisted(() => ({
|
||||
runnerCapacityChanged: undefined as
|
||||
| ((capacity: { total: number; available: number }) => void)
|
||||
| undefined,
|
||||
workerSupervisorReadinessChanged: undefined as ((ready: boolean) => void) | undefined,
|
||||
resolveNodeWorkerContainerEngine: vi.fn(async () => undefined),
|
||||
nodeHostCommands: [] as string[],
|
||||
nodeHostCaps: [] as string[],
|
||||
availabilityOnWatch: undefined as { caps: string[]; commands: string[] } | undefined,
|
||||
@@ -107,6 +110,10 @@ vi.mock("../infra/device-identity.js", () => ({
|
||||
publicKey: "public-key-test",
|
||||
privateKey: "private-key-test",
|
||||
})),
|
||||
loadOrCreateProcessDeviceIdentity: vi.fn(() => ({
|
||||
deviceId: "device-test",
|
||||
privateKeyPem: "private-key-test",
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("../infra/machine-name.js", () => ({
|
||||
@@ -199,6 +206,7 @@ vi.mock("./runtime.js", async (importOriginal) => {
|
||||
start: (params) => {
|
||||
mocks.runtimeClient = params.client;
|
||||
mocks.runnerCapacityChanged = params.onRunnerCapacityChanged;
|
||||
mocks.workerSupervisorReadinessChanged = params.onWorkerSupervisorReadinessChanged;
|
||||
return mocks.activeRuntime;
|
||||
},
|
||||
};
|
||||
@@ -206,6 +214,10 @@ vi.mock("./runtime.js", async (importOriginal) => {
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("./node-worker-container-runtime.js", () => ({
|
||||
resolveNodeWorkerContainerEngine: mocks.resolveNodeWorkerContainerEngine,
|
||||
}));
|
||||
|
||||
function lastCapturedOptions(): GatewayClientOptions | undefined {
|
||||
const list = mocks.capturedGatewayClientOptions;
|
||||
return list[list.length - 1];
|
||||
@@ -232,6 +244,8 @@ describe("runNodeHost", () => {
|
||||
mocks.useFakeRuntime = false;
|
||||
mocks.fakeRuntimeWorkerHosting = false;
|
||||
mocks.runnerCapacityChanged = undefined;
|
||||
mocks.workerSupervisorReadinessChanged = undefined;
|
||||
mocks.resolveNodeWorkerContainerEngine.mockResolvedValue(undefined);
|
||||
mocks.nodeHostCommands = [];
|
||||
mocks.nodeHostCaps = [];
|
||||
mocks.availabilityOnWatch = undefined;
|
||||
@@ -689,6 +703,8 @@ describe("runNodeHost", () => {
|
||||
});
|
||||
|
||||
it("publishes opt-in consent and capacity in the atomic runner inventory", async () => {
|
||||
mocks.useFakeRuntime = true;
|
||||
mocks.fakeRuntimeWorkerHosting = true;
|
||||
mocks.getRuntimeConfig.mockReturnValue({
|
||||
gateway: { handshakeTimeoutMs: 1_000 },
|
||||
nodeHost: { workerRuns: { enabled: true } },
|
||||
@@ -699,6 +715,9 @@ describe("runNodeHost", () => {
|
||||
const options = mocks.capturedGatewayClientOptions[0];
|
||||
const client = mocks.capturedGatewayClients[0];
|
||||
|
||||
mocks.runnerCapacityChanged?.({ total: 2, available: 2 });
|
||||
mocks.workerSupervisorReadinessChanged?.(true);
|
||||
|
||||
options?.onHelloOk?.({
|
||||
protocol: 4,
|
||||
features: { methods: [], events: [] },
|
||||
@@ -725,6 +744,7 @@ describe("runNodeHost", () => {
|
||||
expect(options?.workerRuns).toBeUndefined();
|
||||
|
||||
mocks.runnerCapacityChanged?.({ total: 2, available: 2 });
|
||||
mocks.workerSupervisorReadinessChanged?.(true);
|
||||
options?.onHelloOk?.({
|
||||
protocol: 4,
|
||||
features: {
|
||||
@@ -790,6 +810,88 @@ describe("runNodeHost", () => {
|
||||
expect(client?.updateNodeManifest).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("withholds and withdraws process isolation until supervisor recovery is healthy", async () => {
|
||||
mocks.useFakeRuntime = true;
|
||||
mocks.fakeRuntimeWorkerHosting = true;
|
||||
mocks.startGatewayClientWhenEventLoopReady.mockResolvedValueOnce({
|
||||
ready: true,
|
||||
aborted: false,
|
||||
elapsedMs: 0,
|
||||
});
|
||||
let resolveEngine!: (value: unknown) => void;
|
||||
mocks.resolveNodeWorkerContainerEngine.mockImplementationOnce(
|
||||
async () =>
|
||||
await new Promise((resolve) => {
|
||||
resolveEngine = resolve;
|
||||
}),
|
||||
);
|
||||
const processOnceSpy = vi.spyOn(process, "once");
|
||||
const previousExitCode = process.exitCode;
|
||||
try {
|
||||
const running = runNodeHost({ gatewayHost: "127.0.0.1", gatewayPort: 18789 });
|
||||
await vi.waitFor(() => expect(mocks.workerSupervisorReadinessChanged).toBeTypeOf("function"));
|
||||
await vi.waitFor(() => expect(resolveEngine).toBeTypeOf("function"));
|
||||
const options = mocks.capturedGatewayClientOptions[0];
|
||||
const client = mocks.capturedGatewayClients[0];
|
||||
const latestRunnerInventory = () =>
|
||||
client?.request.mock.calls
|
||||
.filter(([method]) => method === NODE_RUNNER_INVENTORY_UPDATE_METHOD)
|
||||
.at(-1)?.[1];
|
||||
|
||||
mocks.runnerCapacityChanged?.({ total: 2, available: 2 });
|
||||
options?.onHelloOk?.({
|
||||
protocol: 7,
|
||||
features: { methods: [], events: [] },
|
||||
} as unknown as Parameters<NonNullable<GatewayClientOptions["onHelloOk"]>>[0]);
|
||||
await vi.waitFor(() =>
|
||||
expect(latestRunnerInventory()).toEqual({
|
||||
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE],
|
||||
workerHost: { enabled: false },
|
||||
}),
|
||||
);
|
||||
|
||||
resolveEngine({ id: "docker" });
|
||||
await vi.waitFor(() => expect(mocks.resolveNodeWorkerContainerEngine).toHaveBeenCalledOnce());
|
||||
expect(latestRunnerInventory()).toEqual({
|
||||
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE],
|
||||
workerHost: { enabled: false },
|
||||
});
|
||||
|
||||
mocks.workerSupervisorReadinessChanged?.(true);
|
||||
await vi.waitFor(() =>
|
||||
expect(latestRunnerInventory()).toEqual({
|
||||
protocolFeatures: [NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE],
|
||||
workerHost: {
|
||||
enabled: true,
|
||||
capacity: { total: 2, available: 2 },
|
||||
bundlePrewarm: 1,
|
||||
processIsolation: { kind: "container-v1", memoryProjection: 1 },
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
mocks.workerSupervisorReadinessChanged?.(false);
|
||||
await vi.waitFor(() =>
|
||||
expect(latestRunnerInventory()).toEqual({
|
||||
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE],
|
||||
workerHost: { enabled: false },
|
||||
}),
|
||||
);
|
||||
|
||||
const onSigterm = processOnceSpy.mock.calls.find(([event]) => event === "SIGTERM")?.[1];
|
||||
onSigterm?.("SIGTERM");
|
||||
await running;
|
||||
} finally {
|
||||
for (const [event, listener] of processOnceSpy.mock.calls) {
|
||||
if ((event === "SIGINT" || event === "SIGTERM") && typeof listener === "function") {
|
||||
process.off(event, listener);
|
||||
}
|
||||
}
|
||||
process.exitCode = previousExitCode;
|
||||
processOnceSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("clears gateway plugin tools when the final node-hosted tool disappears", async () => {
|
||||
mocks.startGatewayClientWhenEventLoopReady.mockResolvedValueOnce({
|
||||
ready: true,
|
||||
|
||||
+32
-2
@@ -19,6 +19,7 @@ import {
|
||||
NODE_RUNNER_INVENTORY_UPDATE_METHOD,
|
||||
NODE_WORKER_BUNDLE_RETENTION_VERSION,
|
||||
NODE_WORKER_BUNDLE_STATUS_VERSION,
|
||||
NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE,
|
||||
NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
|
||||
type NodeWorkerCapacitySnapshot,
|
||||
} from "../infra/node-runner-inventory.js";
|
||||
@@ -35,6 +36,7 @@ import {
|
||||
coerceNodeInvokePayload,
|
||||
} from "./invoke-payload.js";
|
||||
import { prepareNodeHostRuntime, type NodeHostInventory } from "./runtime.js";
|
||||
import { resolveNodeWorkerContainerEngine } from "./node-worker-container-runtime.js";
|
||||
import { runStartupMigrations } from "./startup-state-migrations.js";
|
||||
|
||||
type NodeHostRunOptions = {
|
||||
@@ -304,6 +306,8 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise<void> {
|
||||
|
||||
let inventory: NodeHostInventory = preparedRuntime.initialInventory;
|
||||
let workerCapacity: NodeWorkerCapacitySnapshot | undefined;
|
||||
let workerSupervisorReady = false;
|
||||
let workerProcessIsolationAvailable = false;
|
||||
let gatewayHelloReceived = false;
|
||||
let gatewayConnectionGeneration = 0;
|
||||
let connectedGatewayProtocol = 0;
|
||||
@@ -515,12 +519,22 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise<void> {
|
||||
};
|
||||
|
||||
const publishRunnerInventory = () => {
|
||||
const workerHostAvailable =
|
||||
preparedRuntime.workerHostingEnabled && workerSupervisorReady && workerCapacity !== undefined;
|
||||
const processIsolation =
|
||||
workerHostAvailable && workerProcessIsolationAvailable
|
||||
? { kind: "container-v1" as const }
|
||||
: undefined;
|
||||
queueOptionalPublication(
|
||||
NODE_RUNNER_INVENTORY_UPDATE_METHOD,
|
||||
{
|
||||
protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE],
|
||||
protocolFeatures: [
|
||||
processIsolation
|
||||
? NODE_WORKER_SUPERVISOR_MEMORY_PROJECTION_PROTOCOL_FEATURE
|
||||
: NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE,
|
||||
],
|
||||
workerHost:
|
||||
preparedRuntime.workerHostingEnabled && workerCapacity
|
||||
workerHostAvailable && workerCapacity
|
||||
? {
|
||||
enabled: true,
|
||||
capacity: workerCapacity,
|
||||
@@ -531,6 +545,9 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise<void> {
|
||||
...(gatewaySupportsBundleRetention && gatewaySupportsBundleStatus
|
||||
? { bundleStatus: NODE_WORKER_BUNDLE_STATUS_VERSION }
|
||||
: {}),
|
||||
...(processIsolation
|
||||
? { processIsolation: { ...processIsolation, memoryProjection: 1 } }
|
||||
: {}),
|
||||
}
|
||||
: { enabled: false },
|
||||
},
|
||||
@@ -656,6 +673,10 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise<void> {
|
||||
workerCapacity = capacity;
|
||||
publishRunnerInventory();
|
||||
},
|
||||
onWorkerSupervisorReadinessChanged: (ready) => {
|
||||
workerSupervisorReady = ready;
|
||||
publishRunnerInventory();
|
||||
},
|
||||
onManifestChanged: (manifest) => {
|
||||
// Manifest changes force a reconnect. Retire the current publication queue
|
||||
// now so it cannot drain against the closing connection.
|
||||
@@ -665,6 +686,15 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise<void> {
|
||||
});
|
||||
|
||||
let stopping = false;
|
||||
if (preparedRuntime.workerHostingEnabled) {
|
||||
void resolveNodeWorkerContainerEngine().then((engine) => {
|
||||
if (stopping || !engine) {
|
||||
return;
|
||||
}
|
||||
workerProcessIsolationAvailable = true;
|
||||
publishRunnerInventory();
|
||||
});
|
||||
}
|
||||
let resolveStopped: (() => void) | undefined;
|
||||
const stopped = new Promise<void>((resolve) => {
|
||||
resolveStopped = resolve;
|
||||
|
||||
@@ -64,6 +64,7 @@ type PreparedNodeHostRuntime = {
|
||||
onInventoryChanged?: (inventory: NodeHostInventory) => void;
|
||||
onManifestChanged?: (manifest: NodeHostManifest) => void;
|
||||
onRunnerCapacityChanged?: (capacity: NodeWorkerCapacitySnapshot) => void;
|
||||
onWorkerSupervisorReadinessChanged?: (ready: boolean) => void;
|
||||
}): ActiveNodeHostRuntime;
|
||||
};
|
||||
|
||||
@@ -322,7 +323,13 @@ export async function prepareNodeHostRuntime(params?: {
|
||||
manifest,
|
||||
workerHostingEnabled: workerRunsEnabled,
|
||||
initialInventory,
|
||||
start({ client, onInventoryChanged, onManifestChanged, onRunnerCapacityChanged }) {
|
||||
start({
|
||||
client,
|
||||
onInventoryChanged,
|
||||
onManifestChanged,
|
||||
onRunnerCapacityChanged,
|
||||
onWorkerSupervisorReadinessChanged,
|
||||
}) {
|
||||
const mcpAbort = new AbortController();
|
||||
const workerWorkspace = workerRunsEnabled
|
||||
? new NodeWorkerWorkspaceRuntime({ env })
|
||||
@@ -338,9 +345,16 @@ export async function prepareNodeHostRuntime(params?: {
|
||||
})
|
||||
: undefined;
|
||||
if (workerSupervisor) {
|
||||
void workerSupervisor.initialize().catch((error: unknown) => {
|
||||
logDebug(`node-host: worker capacity reconciliation failed: ${String(error)}`);
|
||||
});
|
||||
// A reconciled slot count is not enough to assert process isolation: a
|
||||
// failed recovery can leave a durable relay, projection, or container alive.
|
||||
onWorkerSupervisorReadinessChanged?.(false);
|
||||
void workerSupervisor
|
||||
.initialize()
|
||||
.then(() => onWorkerSupervisorReadinessChanged?.(true))
|
||||
.catch((error: unknown) => {
|
||||
onWorkerSupervisorReadinessChanged?.(false);
|
||||
logDebug(`node-host: worker capacity reconciliation failed: ${String(error)}`);
|
||||
});
|
||||
}
|
||||
const skillBins = new SkillBinsCache(client, pathEnv);
|
||||
const activeInvokes = new Map<string, ActiveNodeInvoke>();
|
||||
|
||||
@@ -7,3 +7,4 @@ export type {
|
||||
MemoryBrokerRequest,
|
||||
} from "../memory-broker/protocol.js";
|
||||
export type { MemoryBrokerHandler } from "../memory-broker/server.js";
|
||||
export type { MemoryBrokerChildEntry, MemoryBrokerStartupContext } from "../memory-broker/entry.js";
|
||||
|
||||
@@ -16,7 +16,6 @@ import {
|
||||
import {
|
||||
hasConfiguredStartupChannel,
|
||||
declaresAllowedEnterpriseIdentityProvider,
|
||||
listPotentialEnabledChannelIds,
|
||||
resolveAuthorizedGatewayStartupDreamingPluginIds,
|
||||
resolveContextEngineSlotStartupPluginId,
|
||||
resolveMemorySlotStartupPluginId,
|
||||
|
||||
@@ -1,5 +1,19 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { testing } from "./memory-broker-runtime.js";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { MemoryBrokerProcess } from "../memory-broker/process.js";
|
||||
import {
|
||||
closeBrokeredMemoryRuntimes,
|
||||
startBrokeredMemoryRuntimeSupervisor,
|
||||
testing,
|
||||
withBrokeredMemoryMaintenance,
|
||||
} from "./memory-broker-runtime.js";
|
||||
import type { MemoryPluginCapability } from "./registry-contribution-types.js";
|
||||
|
||||
const startMemoryBrokerProcess = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("../memory-broker/process.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../memory-broker/process.js")>()),
|
||||
startMemoryBrokerProcess,
|
||||
}));
|
||||
|
||||
function process(params: { running?: boolean } = {}) {
|
||||
return {
|
||||
@@ -9,37 +23,223 @@ function process(params: { running?: boolean } = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
function brokerProcess(): MemoryBrokerProcess {
|
||||
return {
|
||||
client: {} as MemoryBrokerProcess["client"],
|
||||
brokerEpoch: "test-epoch",
|
||||
isRunning: () => true,
|
||||
isHealthy: async () => true,
|
||||
quiesce: async () => {},
|
||||
resume: async () => {},
|
||||
close: async () => {},
|
||||
};
|
||||
}
|
||||
|
||||
const brokerCapability = {
|
||||
broker: {
|
||||
version: 1,
|
||||
kind: "local-child",
|
||||
moduleUrl: "test:memory-broker-runtime",
|
||||
},
|
||||
} satisfies MemoryPluginCapability;
|
||||
|
||||
afterEach(async () => {
|
||||
await closeBrokeredMemoryRuntimes();
|
||||
startMemoryBrokerProcess.mockReset();
|
||||
});
|
||||
|
||||
describe("brokered memory maintenance", () => {
|
||||
it("drains every running broker before a backup and resumes them afterwards", async () => {
|
||||
const first = process();
|
||||
const second = process();
|
||||
const stopped = process({ running: false });
|
||||
it("drains a running broker before maintenance and resumes it afterwards", async () => {
|
||||
const broker = process();
|
||||
const run = vi.fn(async () => "snapshot-created");
|
||||
const retire = vi.fn(async () => {});
|
||||
|
||||
await expect(
|
||||
testing.runBrokeredMemoryMaintenance({ processes: [first, second, stopped], run }),
|
||||
testing.runBrokeredMemoryMaintenance({ process: broker, run, retire }),
|
||||
).resolves.toBe("snapshot-created");
|
||||
|
||||
expect(first.quiesce).toHaveBeenCalledOnce();
|
||||
expect(second.quiesce).toHaveBeenCalledOnce();
|
||||
expect(stopped.quiesce).not.toHaveBeenCalled();
|
||||
expect(broker.quiesce).toHaveBeenCalledOnce();
|
||||
expect(run).toHaveBeenCalledOnce();
|
||||
expect(first.resume).toHaveBeenCalledOnce();
|
||||
expect(second.resume).toHaveBeenCalledOnce();
|
||||
expect(broker.resume).toHaveBeenCalledOnce();
|
||||
expect(retire).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reopens an already-drained broker when a later broker or the backup fails", async () => {
|
||||
const first = process();
|
||||
const second = process();
|
||||
second.quiesce.mockRejectedValueOnce(new Error("broker unavailable"));
|
||||
it("retires and rejects when quiesce fails before the mutation starts", async () => {
|
||||
const broker = process();
|
||||
broker.quiesce.mockRejectedValueOnce(new Error("broker unavailable"));
|
||||
const run = vi.fn(async () => "unreachable");
|
||||
const retire = vi.fn(async () => {});
|
||||
|
||||
await expect(
|
||||
testing.runBrokeredMemoryMaintenance({ processes: [first, second], run }),
|
||||
testing.runBrokeredMemoryMaintenance({ process: broker, run, retire }),
|
||||
).rejects.toThrow("broker unavailable");
|
||||
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
expect(first.resume).toHaveBeenCalledOnce();
|
||||
expect(second.resume).not.toHaveBeenCalled();
|
||||
expect(retire).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("retires and rejects when resume fails after a mutation", async () => {
|
||||
const broker = process();
|
||||
broker.resume.mockRejectedValueOnce(new Error("broker resume unavailable"));
|
||||
const retire = vi.fn(async () => {});
|
||||
|
||||
await expect(
|
||||
testing.runBrokeredMemoryMaintenance({
|
||||
process: broker,
|
||||
run: async () => "snapshot-created",
|
||||
retire,
|
||||
}),
|
||||
).rejects.toThrow("broker resume unavailable");
|
||||
|
||||
expect(retire).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("serializes maintenance, process resolution, and shutdown on one broker lease", async () => {
|
||||
const leases = new Map<string, Promise<void>>();
|
||||
const order: string[] = [];
|
||||
let releaseMaintenance: (() => void) | undefined;
|
||||
let markMaintenanceStarted: (() => void) | undefined;
|
||||
const maintenanceStarted = new Promise<void>((resolve) => {
|
||||
markMaintenanceStarted = resolve;
|
||||
});
|
||||
const maintenanceMayFinish = new Promise<void>((resolve) => {
|
||||
releaseMaintenance = resolve;
|
||||
});
|
||||
|
||||
const maintenance = testing.withBrokerLease(leases, "memory-core", async () => {
|
||||
order.push("maintenance:start");
|
||||
markMaintenanceStarted?.();
|
||||
await maintenanceMayFinish;
|
||||
order.push("maintenance:finish");
|
||||
});
|
||||
await maintenanceStarted;
|
||||
const resolve = testing.withBrokerLease(leases, "memory-core", async () => {
|
||||
order.push("resolve");
|
||||
});
|
||||
const shutdown = testing.withBrokerLease(leases, "memory-core", async () => {
|
||||
order.push("shutdown");
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
expect(order).toEqual(["maintenance:start"]);
|
||||
releaseMaintenance?.();
|
||||
await Promise.all([maintenance, resolve, shutdown]);
|
||||
expect(order).toEqual(["maintenance:start", "maintenance:finish", "resolve", "shutdown"]);
|
||||
});
|
||||
|
||||
it("blocks broker resolution while maintenance holds the gateway lease without a running broker", async () => {
|
||||
const gate = testing.createBrokerMaintenanceGate();
|
||||
const order: string[] = [];
|
||||
let releaseMaintenance: (() => void) | undefined;
|
||||
let markMaintenanceStarted: (() => void) | undefined;
|
||||
const maintenanceStarted = new Promise<void>((resolve) => {
|
||||
markMaintenanceStarted = resolve;
|
||||
});
|
||||
const maintenanceMayFinish = new Promise<void>((resolve) => {
|
||||
releaseMaintenance = resolve;
|
||||
});
|
||||
|
||||
const maintenance = testing.withGatewayBrokeredMemoryMaintenanceLease(gate, async () => {
|
||||
order.push("maintenance:start");
|
||||
markMaintenanceStarted?.();
|
||||
await maintenanceMayFinish;
|
||||
order.push("maintenance:finish");
|
||||
});
|
||||
await maintenanceStarted;
|
||||
const resolution = testing.withBrokerLifecycleOperation(gate, async () => {
|
||||
order.push("broker:start");
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
expect(order).toEqual(["maintenance:start"]);
|
||||
releaseMaintenance?.();
|
||||
await Promise.all([maintenance, resolution]);
|
||||
expect(order).toEqual(["maintenance:start", "maintenance:finish", "broker:start"]);
|
||||
});
|
||||
|
||||
it("waits for an admitted broker resolution before maintenance starts", async () => {
|
||||
const gate = testing.createBrokerMaintenanceGate();
|
||||
const order: string[] = [];
|
||||
let releaseResolution: (() => void) | undefined;
|
||||
let markResolutionStarted: (() => void) | undefined;
|
||||
const resolutionStarted = new Promise<void>((resolve) => {
|
||||
markResolutionStarted = resolve;
|
||||
});
|
||||
const resolutionMayFinish = new Promise<void>((resolve) => {
|
||||
releaseResolution = resolve;
|
||||
});
|
||||
|
||||
const resolution = testing.withBrokerLifecycleOperation(gate, async () => {
|
||||
order.push("broker:start");
|
||||
markResolutionStarted?.();
|
||||
await resolutionMayFinish;
|
||||
order.push("broker:finish");
|
||||
});
|
||||
await resolutionStarted;
|
||||
const maintenance = testing.withGatewayBrokeredMemoryMaintenanceLease(gate, async () => {
|
||||
order.push("maintenance");
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
expect(order).toEqual(["broker:start"]);
|
||||
releaseResolution?.();
|
||||
await Promise.all([resolution, maintenance]);
|
||||
expect(order).toEqual(["broker:start", "broker:finish", "maintenance"]);
|
||||
});
|
||||
|
||||
it("does not start a broker requested after empty-map maintenance begins", async () => {
|
||||
const broker = brokerProcess();
|
||||
startMemoryBrokerProcess.mockResolvedValueOnce(broker);
|
||||
let releaseMaintenance: (() => void) | undefined;
|
||||
let markMaintenanceStarted: (() => void) | undefined;
|
||||
const maintenanceStarted = new Promise<void>((resolve) => {
|
||||
markMaintenanceStarted = resolve;
|
||||
});
|
||||
const maintenanceMayFinish = new Promise<void>((resolve) => {
|
||||
releaseMaintenance = resolve;
|
||||
});
|
||||
|
||||
const maintenance = withBrokeredMemoryMaintenance(async () => {
|
||||
markMaintenanceStarted?.();
|
||||
await maintenanceMayFinish;
|
||||
});
|
||||
await maintenanceStarted;
|
||||
const supervisor = startBrokeredMemoryRuntimeSupervisor(brokerCapability);
|
||||
|
||||
await Promise.resolve();
|
||||
expect(startMemoryBrokerProcess).not.toHaveBeenCalled();
|
||||
releaseMaintenance?.();
|
||||
await Promise.all([maintenance, supervisor]);
|
||||
expect(startMemoryBrokerProcess).toHaveBeenCalledOnce();
|
||||
|
||||
// Regression: shutdown stops supervisors before taking the writer lease, so stop's reader
|
||||
// retirement cannot deadlock behind the writer that is waiting for it.
|
||||
await expect(closeBrokeredMemoryRuntimes()).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("drains an admitted broker start before maintenance snapshots processes", async () => {
|
||||
const broker = brokerProcess();
|
||||
let resolveStart: ((value: MemoryBrokerProcess) => void) | undefined;
|
||||
let markStartCalled: (() => void) | undefined;
|
||||
const startCalled = new Promise<void>((resolve) => {
|
||||
markStartCalled = resolve;
|
||||
});
|
||||
const pendingStart = new Promise<MemoryBrokerProcess>((resolve) => {
|
||||
resolveStart = resolve;
|
||||
});
|
||||
startMemoryBrokerProcess.mockImplementationOnce(() => {
|
||||
markStartCalled?.();
|
||||
return pendingStart;
|
||||
});
|
||||
const supervisor = startBrokeredMemoryRuntimeSupervisor(brokerCapability);
|
||||
await startCalled;
|
||||
const run = vi.fn(async () => {});
|
||||
const maintenance = withBrokeredMemoryMaintenance(run);
|
||||
|
||||
await Promise.resolve();
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
resolveStart?.(broker);
|
||||
await Promise.all([supervisor, maintenance]);
|
||||
expect(run).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -19,12 +19,39 @@ import type {
|
||||
|
||||
type BrokerRuntimeState = {
|
||||
processes: Map<string, Promise<MemoryBrokerProcess>>;
|
||||
agentIdsByModule: Map<string, readonly string[]>;
|
||||
supervisors: Set<MemoryBrokerSupervisor>;
|
||||
leases: Map<string, Promise<void>>;
|
||||
maintenance: BrokerMaintenanceGate;
|
||||
closing: boolean;
|
||||
};
|
||||
|
||||
type BrokerMaintenanceGate = {
|
||||
maintenanceTail: Promise<void>;
|
||||
maintenancePending: boolean;
|
||||
activeLifecycleOperations: number;
|
||||
idleWaiters: Set<() => void>;
|
||||
};
|
||||
|
||||
function createBrokerMaintenanceGate(): BrokerMaintenanceGate {
|
||||
return {
|
||||
maintenanceTail: Promise.resolve(),
|
||||
maintenancePending: false,
|
||||
activeLifecycleOperations: 0,
|
||||
idleWaiters: new Set(),
|
||||
};
|
||||
}
|
||||
|
||||
const state = resolveGlobalSingleton(
|
||||
Symbol.for("openclaw.memoryBrokerRuntimeState"),
|
||||
(): BrokerRuntimeState => ({ processes: new Map(), supervisors: new Set() }),
|
||||
(): BrokerRuntimeState => ({
|
||||
processes: new Map(),
|
||||
agentIdsByModule: new Map(),
|
||||
supervisors: new Set(),
|
||||
leases: new Map(),
|
||||
maintenance: createBrokerMaintenanceGate(),
|
||||
closing: false,
|
||||
}),
|
||||
);
|
||||
|
||||
type BrokeredMemoryRuntime = Readonly<AuthorizedMemoryRuntime> &
|
||||
@@ -34,6 +61,19 @@ function resolveCapabilitySnapshotId(context: MemoryAccessContext): string {
|
||||
return context.delegation?.capabilitySnapshotId ?? context.hostFactsRevision;
|
||||
}
|
||||
|
||||
function actorBindingFor(context: MemoryAccessContext) {
|
||||
return context.actor.kind === "principal"
|
||||
? Object.freeze({
|
||||
kind: "principal" as const,
|
||||
actorKind: context.actor.actorKind,
|
||||
principalId: context.actor.principalId,
|
||||
})
|
||||
: Object.freeze({
|
||||
kind: "unattributed" as const,
|
||||
transportAuditRef: context.actor.transportAuditRef,
|
||||
});
|
||||
}
|
||||
|
||||
function bindingFor(context: MemoryAccessContext, policyRevision: string) {
|
||||
return Object.freeze({
|
||||
agentId: context.agentId,
|
||||
@@ -41,6 +81,7 @@ function bindingFor(context: MemoryAccessContext, policyRevision: string) {
|
||||
runId: context.runId,
|
||||
contextFingerprint: context.contextFingerprint,
|
||||
subjectRevision: context.subjectRevision,
|
||||
actor: actorBindingFor(context),
|
||||
actorRevision: context.actor.evidenceRevision,
|
||||
capabilitySnapshotId: resolveCapabilitySnapshotId(context),
|
||||
policyRevision,
|
||||
@@ -59,14 +100,159 @@ function authorizeExpiry(): number {
|
||||
return Date.now() + 60_000;
|
||||
}
|
||||
|
||||
async function resolveProcess(
|
||||
capability: MemoryPluginCapability,
|
||||
): Promise<MemoryBrokerProcess | undefined> {
|
||||
const entry = capability.broker;
|
||||
if (!entry || entry.version !== 1 || entry.kind !== "local-child" || !entry.moduleUrl) {
|
||||
/**
|
||||
* The Gateway is the one owner allowed to change a broker's admission state. A lease covers
|
||||
* replacement and shutdown too, so no child can start or close between quiesce and resume.
|
||||
*/
|
||||
async function withBrokerLease<T>(
|
||||
leases: Map<string, Promise<void>>,
|
||||
moduleUrl: string,
|
||||
run: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
const previous = leases.get(moduleUrl) ?? Promise.resolve();
|
||||
let release: (() => void) | undefined;
|
||||
const current = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
leases.set(moduleUrl, current);
|
||||
await previous;
|
||||
try {
|
||||
return await run();
|
||||
} finally {
|
||||
release?.();
|
||||
if (leases.get(moduleUrl) === current) {
|
||||
leases.delete(moduleUrl);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function releaseBrokerLifecycleOperation(gate: BrokerMaintenanceGate): void {
|
||||
gate.activeLifecycleOperations -= 1;
|
||||
if (gate.activeLifecycleOperations !== 0) {
|
||||
return;
|
||||
}
|
||||
for (const resolve of gate.idleWaiters) {
|
||||
resolve();
|
||||
}
|
||||
gate.idleWaiters.clear();
|
||||
}
|
||||
|
||||
async function waitForBrokerLifecycleOperations(gate: BrokerMaintenanceGate): Promise<void> {
|
||||
while (gate.activeLifecycleOperations > 0) {
|
||||
await new Promise<void>((resolve) => gate.idleWaiters.add(resolve));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolution and replacement share a read lease. A maintenance writer blocks later lifecycle
|
||||
* work before it snapshots brokers, while already admitted work finishes before quiesce begins.
|
||||
*/
|
||||
async function withBrokerLifecycleOperation<T>(
|
||||
gate: BrokerMaintenanceGate,
|
||||
run: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
while (true) {
|
||||
const maintenanceTail = gate.maintenanceTail;
|
||||
await maintenanceTail;
|
||||
if (gate.maintenanceTail !== maintenanceTail || gate.maintenancePending) {
|
||||
continue;
|
||||
}
|
||||
gate.activeLifecycleOperations += 1;
|
||||
if (gate.maintenanceTail === maintenanceTail && !gate.maintenancePending) {
|
||||
break;
|
||||
}
|
||||
releaseBrokerLifecycleOperation(gate);
|
||||
}
|
||||
try {
|
||||
return await run();
|
||||
} finally {
|
||||
releaseBrokerLifecycleOperation(gate);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A tool request cannot wait behind maintenance: that would retain an agent turn until a backup
|
||||
* or repair finishes. Refuse admission before taking the reader lease so callers surface the
|
||||
* intentional memory-unavailable result while Gateway owns the writer lease.
|
||||
*/
|
||||
async function tryWithBrokerLifecycleOperation<T>(
|
||||
gate: BrokerMaintenanceGate,
|
||||
run: () => Promise<T>,
|
||||
): Promise<T | undefined> {
|
||||
if (gate.maintenancePending) {
|
||||
return undefined;
|
||||
}
|
||||
const existing = state.processes.get(entry.moduleUrl);
|
||||
const maintenanceTail = gate.maintenanceTail;
|
||||
gate.activeLifecycleOperations += 1;
|
||||
if (gate.maintenanceTail !== maintenanceTail || gate.maintenancePending) {
|
||||
releaseBrokerLifecycleOperation(gate);
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
return await run();
|
||||
} finally {
|
||||
releaseBrokerLifecycleOperation(gate);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gateway maintenance owns the write lease for the complete drain/mutate/resume lifecycle.
|
||||
* Marking it pending synchronously prevents a just-in-time resolver from starting a new child.
|
||||
*/
|
||||
async function withGatewayBrokeredMemoryMaintenanceLease<T>(
|
||||
gate: BrokerMaintenanceGate,
|
||||
run: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
const previous = gate.maintenanceTail;
|
||||
let release: (() => void) | undefined;
|
||||
const current = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const tail = previous.then(() => current);
|
||||
gate.maintenanceTail = tail;
|
||||
gate.maintenancePending = true;
|
||||
await previous;
|
||||
await waitForBrokerLifecycleOperations(gate);
|
||||
try {
|
||||
return await run();
|
||||
} finally {
|
||||
release?.();
|
||||
if (gate.maintenanceTail === tail) {
|
||||
gate.maintenancePending = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function brokerModuleUrl(capability: MemoryPluginCapability): string | undefined {
|
||||
const entry = capability.broker;
|
||||
return entry?.version === 1 && entry.kind === "local-child" && entry.moduleUrl
|
||||
? entry.moduleUrl
|
||||
: undefined;
|
||||
}
|
||||
|
||||
async function resolveProcess(
|
||||
capability: MemoryPluginCapability,
|
||||
maintenanceBehavior: "wait" | "fail" = "wait",
|
||||
): Promise<MemoryBrokerProcess | undefined> {
|
||||
const moduleUrl = brokerModuleUrl(capability);
|
||||
if (!moduleUrl) {
|
||||
return undefined;
|
||||
}
|
||||
const resolveOnLease = async () => {
|
||||
return await withBrokerLease(state.leases, moduleUrl, async () => {
|
||||
if (state.closing) {
|
||||
return undefined;
|
||||
}
|
||||
return await resolveProcessOnLease(moduleUrl);
|
||||
});
|
||||
};
|
||||
return maintenanceBehavior === "fail"
|
||||
? await tryWithBrokerLifecycleOperation(state.maintenance, resolveOnLease)
|
||||
: await withBrokerLifecycleOperation(state.maintenance, resolveOnLease);
|
||||
}
|
||||
|
||||
async function resolveProcessOnLease(moduleUrl: string): Promise<MemoryBrokerProcess | undefined> {
|
||||
const existing = state.processes.get(moduleUrl);
|
||||
if (existing) {
|
||||
try {
|
||||
const process = await existing;
|
||||
@@ -77,23 +263,27 @@ async function resolveProcess(
|
||||
// failed operation; discard it so a later independently authorized operation starts fresh.
|
||||
// A PID alone is not readiness: a child with a dead socket/handler must be retired too.
|
||||
await process.close().catch(() => undefined);
|
||||
if (state.processes.get(entry.moduleUrl) === existing) {
|
||||
state.processes.delete(entry.moduleUrl);
|
||||
if (state.processes.get(moduleUrl) === existing) {
|
||||
state.processes.delete(moduleUrl);
|
||||
}
|
||||
} catch {
|
||||
if (state.processes.get(entry.moduleUrl) === existing) {
|
||||
state.processes.delete(entry.moduleUrl);
|
||||
if (state.processes.get(moduleUrl) === existing) {
|
||||
state.processes.delete(moduleUrl);
|
||||
}
|
||||
}
|
||||
}
|
||||
const process = startMemoryBrokerProcess({
|
||||
brokerId: `selected-memory:${entry.moduleUrl}`,
|
||||
handlerModuleUrl: entry.moduleUrl,
|
||||
let process: Promise<MemoryBrokerProcess>;
|
||||
process = startMemoryBrokerProcess({
|
||||
brokerId: `selected-memory:${moduleUrl}`,
|
||||
handlerModuleUrl: moduleUrl,
|
||||
agentIds: state.agentIdsByModule.get(moduleUrl) ?? [],
|
||||
}).catch((error: unknown) => {
|
||||
state.processes.delete(entry.moduleUrl);
|
||||
if (state.processes.get(moduleUrl) === process) {
|
||||
state.processes.delete(moduleUrl);
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
state.processes.set(entry.moduleUrl, process);
|
||||
state.processes.set(moduleUrl, process);
|
||||
try {
|
||||
return await process;
|
||||
} catch {
|
||||
@@ -102,15 +292,26 @@ async function resolveProcess(
|
||||
}
|
||||
|
||||
async function retireProcess(capability: MemoryPluginCapability): Promise<void> {
|
||||
const moduleUrl = capability.broker?.moduleUrl;
|
||||
const moduleUrl = brokerModuleUrl(capability);
|
||||
if (!moduleUrl) {
|
||||
return;
|
||||
}
|
||||
const process = state.processes.get(moduleUrl);
|
||||
await withBrokerLifecycleOperation(state.maintenance, async () => {
|
||||
await withBrokerLease(state.leases, moduleUrl, async () => {
|
||||
await retireProcessOnLease(state, moduleUrl);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function retireProcessOnLease(
|
||||
runtimeState: BrokerRuntimeState,
|
||||
moduleUrl: string,
|
||||
): Promise<void> {
|
||||
const process = runtimeState.processes.get(moduleUrl);
|
||||
if (!process) {
|
||||
return;
|
||||
}
|
||||
state.processes.delete(moduleUrl);
|
||||
runtimeState.processes.delete(moduleUrl);
|
||||
await (await process).close();
|
||||
}
|
||||
|
||||
@@ -121,10 +322,13 @@ async function retireProcess(capability: MemoryPluginCapability): Promise<void>
|
||||
*/
|
||||
export async function startBrokeredMemoryRuntimeSupervisor(
|
||||
capability: MemoryPluginCapability | undefined,
|
||||
params: { agentIds?: readonly string[] } = {},
|
||||
): Promise<MemoryBrokerSupervisor | undefined> {
|
||||
if (!capability?.broker) {
|
||||
const moduleUrl = capability ? brokerModuleUrl(capability) : undefined;
|
||||
if (!moduleUrl) {
|
||||
return undefined;
|
||||
}
|
||||
state.agentIdsByModule.set(moduleUrl, Object.freeze([...new Set(params.agentIds ?? [])].toSorted()));
|
||||
const supervisor = await startMemoryBrokerSupervisor({
|
||||
ensureProcess: () => resolveProcess(capability),
|
||||
retireProcess: () => retireProcess(capability),
|
||||
@@ -153,7 +357,7 @@ export async function resolveBrokeredMemoryRuntime(
|
||||
return undefined;
|
||||
}
|
||||
const request = async <T>(params: Parameters<MemoryBrokerProcess["client"]["request"]>[0]) => {
|
||||
const process = await resolveProcess(capability);
|
||||
const process = await resolveProcess(capability, "fail");
|
||||
return process ? await process.client.request<T>(params) : undefined;
|
||||
};
|
||||
const authorize = async (context: MemoryAccessContext): Promise<AuthorizedMemoryPlan> => {
|
||||
@@ -209,6 +413,7 @@ export async function resolveBrokeredMemoryRuntime(
|
||||
...(params.lines !== undefined ? { lines: params.lines } : {}),
|
||||
},
|
||||
expiresAtMs,
|
||||
...(params.signal ? { signal: params.signal } : {}),
|
||||
});
|
||||
if (!result) {
|
||||
throw new Error("memory broker read is unavailable");
|
||||
@@ -225,6 +430,7 @@ export async function resolveBrokeredMemoryRuntime(
|
||||
method: "memory.write",
|
||||
payload: { context: params.context, plan: params.plan, mutation: params.mutation },
|
||||
expiresAtMs,
|
||||
...(params.signal ? { signal: params.signal } : {}),
|
||||
});
|
||||
if (!result) {
|
||||
throw new Error("memory broker write is unavailable");
|
||||
@@ -241,6 +447,7 @@ export async function resolveBrokeredMemoryRuntime(
|
||||
method: "memory.import",
|
||||
payload: { context: params.context, plan: params.plan, mutation: params.mutation },
|
||||
expiresAtMs,
|
||||
...(params.signal ? { signal: params.signal } : {}),
|
||||
});
|
||||
if (!result) {
|
||||
throw new Error("memory broker import is unavailable");
|
||||
@@ -257,6 +464,7 @@ export async function resolveBrokeredMemoryRuntime(
|
||||
method: "memory.sync",
|
||||
payload: { context: params.context, plan: params.plan },
|
||||
expiresAtMs,
|
||||
...(params.signal ? { signal: params.signal } : {}),
|
||||
});
|
||||
if (!result) {
|
||||
throw new Error("memory broker sync is unavailable");
|
||||
@@ -273,6 +481,7 @@ export async function resolveBrokeredMemoryRuntime(
|
||||
method: "memory.export",
|
||||
payload: { context: params.context, plan: params.plan, handles: params.handles },
|
||||
expiresAtMs,
|
||||
...(params.signal ? { signal: params.signal } : {}),
|
||||
});
|
||||
if (!result) {
|
||||
throw new Error("memory broker export is unavailable");
|
||||
@@ -289,6 +498,7 @@ export async function resolveBrokeredMemoryRuntime(
|
||||
method: "memory.status",
|
||||
payload: { context: params.context, plan: params.plan },
|
||||
expiresAtMs,
|
||||
...(params.signal ? { signal: params.signal } : {}),
|
||||
});
|
||||
if (!result) {
|
||||
throw new Error("memory broker status is unavailable");
|
||||
@@ -298,6 +508,7 @@ export async function resolveBrokeredMemoryRuntime(
|
||||
async materializeAuthorizedVirtualView(params: {
|
||||
context: MemoryContentAccessContext<"read">;
|
||||
plan: AuthorizedMemoryContentPlan<"read">;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<AuthorizedMemoryVirtualView | undefined> {
|
||||
const expiresAtMs = hasPlanExpiry(params.plan);
|
||||
if (!expiresAtMs) {
|
||||
@@ -308,6 +519,7 @@ export async function resolveBrokeredMemoryRuntime(
|
||||
method: "memory.virtual-view",
|
||||
payload: { context: params.context, plan: params.plan },
|
||||
expiresAtMs,
|
||||
...(params.signal ? { signal: params.signal } : {}),
|
||||
});
|
||||
},
|
||||
async readAuthorizedVirtualFile(
|
||||
@@ -327,6 +539,7 @@ export async function resolveBrokeredMemoryRuntime(
|
||||
virtualPath: params.virtualPath,
|
||||
},
|
||||
expiresAtMs,
|
||||
...(params.signal ? { signal: params.signal } : {}),
|
||||
});
|
||||
if (!result) {
|
||||
throw new Error("memory broker virtual file is unavailable");
|
||||
@@ -346,39 +559,100 @@ export async function resolveBrokeredMemoryRuntime(
|
||||
}
|
||||
|
||||
export async function closeBrokeredMemoryRuntimes(): Promise<void> {
|
||||
const supervisors = [...state.supervisors];
|
||||
state.supervisors.clear();
|
||||
await Promise.all(supervisors.map((supervisor) => supervisor.stop()));
|
||||
const processes = [...state.processes.values()];
|
||||
state.processes.clear();
|
||||
await Promise.all(
|
||||
processes.map(async (process) => {
|
||||
try {
|
||||
await (await process).close();
|
||||
} catch {
|
||||
// Shutdown is best effort; a replacement process gets a fresh epoch and cannot reuse it.
|
||||
}
|
||||
}),
|
||||
);
|
||||
state.closing = true;
|
||||
try {
|
||||
// `stop()` retires through the lifecycle reader lease. Stop supervisors before taking the
|
||||
// shutdown writer so their in-flight probe cannot wait for the writer that waits for them.
|
||||
const supervisors = [...state.supervisors];
|
||||
state.supervisors.clear();
|
||||
await Promise.all(supervisors.map((supervisor) => supervisor.stop()));
|
||||
await withGatewayBrokeredMemoryMaintenanceLease(state.maintenance, async () => {
|
||||
const moduleUrls = [...state.processes.keys()].toSorted();
|
||||
await Promise.all(
|
||||
moduleUrls.map((moduleUrl) =>
|
||||
withBrokerLease(state.leases, moduleUrl, async () => {
|
||||
try {
|
||||
await retireProcessOnLease(state, moduleUrl);
|
||||
} catch {
|
||||
// Gateway shutdown still retires the map entry so a future lifecycle gets a new epoch.
|
||||
}
|
||||
}),
|
||||
),
|
||||
);
|
||||
});
|
||||
} finally {
|
||||
state.agentIdsByModule.clear();
|
||||
state.closing = false;
|
||||
}
|
||||
}
|
||||
|
||||
type BrokerMaintenanceProcess = Pick<MemoryBrokerProcess, "isRunning" | "quiesce" | "resume">;
|
||||
|
||||
async function runBrokeredMemoryMaintenance<T>(params: {
|
||||
processes: readonly BrokerMaintenanceProcess[];
|
||||
process: BrokerMaintenanceProcess;
|
||||
run: () => Promise<T>;
|
||||
retire: () => Promise<void>;
|
||||
}): Promise<T> {
|
||||
if (!params.process.isRunning()) {
|
||||
await params.retire();
|
||||
return await params.run();
|
||||
}
|
||||
try {
|
||||
await params.process.quiesce();
|
||||
} catch (error) {
|
||||
await rejectAfterBrokerRetirement(error, params.retire);
|
||||
}
|
||||
|
||||
let result!: T;
|
||||
let runError: unknown;
|
||||
try {
|
||||
result = await params.run();
|
||||
} catch (error) {
|
||||
runError = error;
|
||||
}
|
||||
try {
|
||||
await params.process.resume();
|
||||
} catch (resumeError) {
|
||||
await rejectAfterBrokerRetirement(
|
||||
runError === undefined ? resumeError : new AggregateError([runError, resumeError]),
|
||||
params.retire,
|
||||
);
|
||||
}
|
||||
if (runError !== undefined) {
|
||||
throw runError;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function rejectAfterBrokerRetirement(
|
||||
error: unknown,
|
||||
retire: () => Promise<void>,
|
||||
): Promise<never> {
|
||||
try {
|
||||
await retire();
|
||||
} catch (retireError) {
|
||||
throw new AggregateError([error, retireError], "memory broker retirement failed");
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
async function runBrokeredMemoryMaintenanceForModule<T>(params: {
|
||||
runtimeState: BrokerRuntimeState;
|
||||
moduleUrl: string;
|
||||
run: () => Promise<T>;
|
||||
}): Promise<T> {
|
||||
const running = params.processes.filter((process) => process.isRunning());
|
||||
const quiesced: BrokerMaintenanceProcess[] = [];
|
||||
try {
|
||||
for (const process of running) {
|
||||
await process.quiesce();
|
||||
quiesced.push(process);
|
||||
return await withBrokerLease(params.runtimeState.leases, params.moduleUrl, async () => {
|
||||
const pendingProcess = params.runtimeState.processes.get(params.moduleUrl);
|
||||
if (!pendingProcess) {
|
||||
return await params.run();
|
||||
}
|
||||
return await params.run();
|
||||
} finally {
|
||||
await Promise.all(quiesced.map((process) => process.resume().catch(() => undefined)));
|
||||
}
|
||||
const process = await pendingProcess;
|
||||
return await runBrokeredMemoryMaintenance({
|
||||
process,
|
||||
run: params.run,
|
||||
retire: () => retireProcessOnLease(params.runtimeState, params.moduleUrl),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -386,10 +660,29 @@ async function runBrokeredMemoryMaintenance<T>(params: {
|
||||
* their socket, SQLite handle, artifact root, or bootstrap credential in a worker process.
|
||||
*/
|
||||
export async function withBrokeredMemoryMaintenance<T>(run: () => Promise<T>): Promise<T> {
|
||||
const processes = await Promise.all([...state.processes.values()]);
|
||||
return await runBrokeredMemoryMaintenance({ processes, run });
|
||||
return await withGatewayBrokeredMemoryMaintenanceLease(state.maintenance, async () => {
|
||||
// The snapshot happens only after the write lease excludes broker resolution/replacement.
|
||||
const moduleUrls = [...state.processes.keys()].toSorted();
|
||||
const runNext = async (index: number): Promise<T> => {
|
||||
const moduleUrl = moduleUrls[index];
|
||||
if (!moduleUrl) {
|
||||
return await run();
|
||||
}
|
||||
return await runBrokeredMemoryMaintenanceForModule({
|
||||
runtimeState: state,
|
||||
moduleUrl,
|
||||
run: () => runNext(index + 1),
|
||||
});
|
||||
};
|
||||
return await runNext(0);
|
||||
});
|
||||
}
|
||||
|
||||
export const testing = {
|
||||
createBrokerMaintenanceGate,
|
||||
withBrokerLease,
|
||||
withBrokerLifecycleOperation,
|
||||
tryWithBrokerLifecycleOperation,
|
||||
withGatewayBrokeredMemoryMaintenanceLease,
|
||||
runBrokeredMemoryMaintenance,
|
||||
};
|
||||
|
||||
@@ -586,6 +586,7 @@ export async function createAuthorizedMemoryWriteInvocation(params: {
|
||||
export async function writeAuthorizedMemoryForInvocation(params: {
|
||||
invocation: AuthorizedMemoryWriteInvocation;
|
||||
mutation: AuthorizedMemoryMutation;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<MemoryWriteResult | MemoryInvocationUnavailable> {
|
||||
const state = writeInvocationStates.get(params.invocation);
|
||||
const context = state ? readCurrentWriteContext(state) : undefined;
|
||||
@@ -597,6 +598,7 @@ export async function writeAuthorizedMemoryForInvocation(params: {
|
||||
context,
|
||||
plan: state.plan,
|
||||
mutation: params.mutation,
|
||||
...(params.signal ? { signal: params.signal } : {}),
|
||||
} as never);
|
||||
} catch {
|
||||
logMemoryInvocationDiagnostic("authorization-failed");
|
||||
@@ -613,6 +615,7 @@ export async function stageAuthorizedMemorySealedCompactionForInvocation(params:
|
||||
invocation: AuthorizedMemoryWriteInvocation;
|
||||
content: string;
|
||||
transcriptSource: AuthorizedTranscriptDerivationSource;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<AuthorizedSealedCompactionArtifact | MemoryInvocationUnavailable> {
|
||||
const state = writeInvocationStates.get(params.invocation);
|
||||
const context = state ? readCurrentWriteContext(state) : undefined;
|
||||
@@ -632,6 +635,7 @@ export async function stageAuthorizedMemorySealedCompactionForInvocation(params:
|
||||
plan: state.plan,
|
||||
content: params.content,
|
||||
transcriptSource: params.transcriptSource,
|
||||
...(params.signal ? { signal: params.signal } : {}),
|
||||
});
|
||||
} catch {
|
||||
logMemoryInvocationDiagnostic("authorization-failed");
|
||||
@@ -646,6 +650,7 @@ export async function stageAuthorizedMemorySealedCompactionForInvocation(params:
|
||||
*/
|
||||
export async function materializeAuthorizedMemoryVirtualView(params: {
|
||||
invocation: AuthorizedMemoryReadInvocation;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<AuthorizedMemoryVirtualView | MemoryInvocationUnavailable> {
|
||||
const state = readState(params.invocation);
|
||||
const context = state ? readCurrentContext(state) : undefined;
|
||||
@@ -664,6 +669,7 @@ export async function materializeAuthorizedMemoryVirtualView(params: {
|
||||
const view = await state.virtualView.materializeAuthorizedVirtualView({
|
||||
context,
|
||||
plan: state.plan,
|
||||
...(params.signal ? { signal: params.signal } : {}),
|
||||
});
|
||||
const canonical = view
|
||||
? canonicalizeAuthorizedVirtualView({ view, context, plan: state.plan })
|
||||
@@ -687,6 +693,7 @@ export async function readAuthorizedMemoryVirtualFile(params: {
|
||||
invocation: AuthorizedMemoryReadInvocation;
|
||||
view: AuthorizedMemoryVirtualView;
|
||||
virtualPath: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<MemoryReadResult | MemoryInvocationUnavailable> {
|
||||
const state = readState(params.invocation);
|
||||
const context = state ? readCurrentContext(state) : undefined;
|
||||
@@ -708,6 +715,7 @@ export async function readAuthorizedMemoryVirtualFile(params: {
|
||||
plan: state.plan,
|
||||
view: params.view,
|
||||
virtualPath: params.virtualPath,
|
||||
...(params.signal ? { signal: params.signal } : {}),
|
||||
});
|
||||
if (
|
||||
!validateEnvelope({
|
||||
@@ -781,6 +789,7 @@ export async function readAuthorizedMemoryForInvocation(params: {
|
||||
handleId: string;
|
||||
from?: number;
|
||||
lines?: number;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<MemoryReadResult | MemoryInvocationUnavailable> {
|
||||
const state = readState(params.invocation);
|
||||
const context = state ? readCurrentContext(state) : undefined;
|
||||
@@ -800,6 +809,7 @@ export async function readAuthorizedMemoryForInvocation(params: {
|
||||
handle,
|
||||
...(params.from !== undefined ? { from: params.from } : {}),
|
||||
...(params.lines !== undefined ? { lines: params.lines } : {}),
|
||||
...(params.signal ? { signal: params.signal } : {}),
|
||||
} as never);
|
||||
if (
|
||||
!validateEnvelope({
|
||||
|
||||
@@ -353,6 +353,7 @@ export type MemoryPluginVirtualViewProvider = {
|
||||
materializeAuthorizedVirtualView(params: {
|
||||
context: MemoryContentAccessContext<"read">;
|
||||
plan: AuthorizedMemoryContentPlan<"read">;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<AuthorizedMemoryVirtualView | undefined>;
|
||||
readAuthorizedVirtualFile(params: {
|
||||
context: MemoryContentAccessContext<"read">;
|
||||
@@ -360,6 +361,7 @@ export type MemoryPluginVirtualViewProvider = {
|
||||
view: AuthorizedMemoryVirtualView;
|
||||
/** Virtual-only slash-separated path, never a host filesystem path. */
|
||||
virtualPath: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<AuthorizedMemoryResultEnvelope<MemoryReadResult>>;
|
||||
};
|
||||
|
||||
|
||||
@@ -53,6 +53,7 @@ export type AuthorizedMemoryReadHost = Readonly<{
|
||||
handleId: string;
|
||||
from?: number;
|
||||
lines?: number;
|
||||
signal?: AbortSignal;
|
||||
}) => Promise<MemoryReadResult | AuthorizedMemoryReadUnavailable>;
|
||||
}>;
|
||||
|
||||
|
||||
@@ -271,9 +271,9 @@ describe("createChildAdapter", () => {
|
||||
expect(firstSpawnWithFallbackParams().fallbacks).toEqual([]);
|
||||
expect(firstSpawnWithFallbackParams().options?.stdio).toEqual(["pipe", "pipe", "pipe", "ipc"]);
|
||||
|
||||
await adapter.openStartGate?.();
|
||||
await adapter.openStartGate?.({ launchId: "turn-1", planHash: "a".repeat(64) });
|
||||
expect(sendMock).toHaveBeenCalledWith(
|
||||
{ type: "openclaw-worker-start-v1" },
|
||||
{ type: "openclaw-worker-start-v1", launchId: "turn-1", planHash: "a".repeat(64) },
|
||||
expect.any(Function),
|
||||
);
|
||||
adapter.closeStartGate?.();
|
||||
|
||||
@@ -72,12 +72,16 @@ function resolveChildInvocation(params: {
|
||||
}
|
||||
|
||||
type ChildAdapter = SpawnProcessAdapter<NodeJS.Signals | null>;
|
||||
type WorkerStartIdentity = {
|
||||
launchId: string;
|
||||
planHash: string;
|
||||
};
|
||||
type WorkerChildAdapter = ChildAdapter & {
|
||||
closeStartGate?: () => void;
|
||||
openStartGate?: () => Promise<void>;
|
||||
openStartGate?: (identity: WorkerStartIdentity) => Promise<void>;
|
||||
};
|
||||
|
||||
const WORKER_START_MESSAGE = { type: "openclaw-worker-start-v1" } as const;
|
||||
const WORKER_START_MESSAGE_TYPE = "openclaw-worker-start-v1";
|
||||
|
||||
function isServiceManagedRuntime(): boolean {
|
||||
return Boolean(process.env.OPENCLAW_SERVICE_MARKER?.trim());
|
||||
@@ -516,7 +520,7 @@ export async function createChildAdapter(params: {
|
||||
|
||||
let startGateOpened = false;
|
||||
const openStartGate = params.ownedWorker
|
||||
? async () => {
|
||||
? async (identity: WorkerStartIdentity) => {
|
||||
if (startGateOpened) {
|
||||
return;
|
||||
}
|
||||
@@ -527,7 +531,7 @@ export async function createChildAdapter(params: {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
child.send(WORKER_START_MESSAGE, (error) => {
|
||||
child.send({ type: WORKER_START_MESSAGE_TYPE, ...identity }, (error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
return;
|
||||
|
||||
@@ -14,6 +14,8 @@ export const FIRST_USE_STATE_TABLES = [
|
||||
"cron_job_runtime_authorities",
|
||||
"execution_identity_contexts",
|
||||
"mcp_oauth_pending_authorizations",
|
||||
"node_worker_container_launches",
|
||||
"node_worker_container_leases",
|
||||
"node_worker_launches",
|
||||
"operator_approval_execution_identities",
|
||||
"execution_decision_facts",
|
||||
|
||||
+14
@@ -1154,6 +1154,18 @@ export interface NodeHostConfig {
|
||||
version: number;
|
||||
}
|
||||
|
||||
export interface NodeWorkerContainerLaunches {
|
||||
container_engine: string;
|
||||
launch_id: string;
|
||||
plan_hash: string;
|
||||
}
|
||||
|
||||
export interface NodeWorkerContainerLeases {
|
||||
expires_at_ms: number;
|
||||
launch_id: string;
|
||||
plan_hash: string;
|
||||
}
|
||||
|
||||
export interface NodeWorkerLaunches {
|
||||
completed_at_ms: number | null;
|
||||
created_at_ms: number;
|
||||
@@ -2031,6 +2043,8 @@ export interface DB {
|
||||
model_catalog_remote: ModelCatalogRemote;
|
||||
native_hook_relay_bridges: NativeHookRelayBridges;
|
||||
node_host_config: NodeHostConfig;
|
||||
node_worker_container_launches: NodeWorkerContainerLaunches;
|
||||
node_worker_container_leases: NodeWorkerContainerLeases;
|
||||
node_worker_launches: NodeWorkerLaunches;
|
||||
official_external_plugin_catalog_snapshots: OfficialExternalPluginCatalogSnapshots;
|
||||
onboarding_recommendations: OnboardingRecommendations;
|
||||
|
||||
@@ -1350,6 +1350,30 @@ CREATE TABLE IF NOT EXISTS node_worker_launches (
|
||||
)
|
||||
) STRICT;
|
||||
|
||||
-- Container launch metadata is deliberately separate from the public receipt.
|
||||
-- Recovery needs the selected engine to prove cleanup; worker descriptors and
|
||||
-- credentials remain process memory only.
|
||||
CREATE TABLE IF NOT EXISTS node_worker_container_launches (
|
||||
launch_id TEXT NOT NULL PRIMARY KEY
|
||||
CHECK (length(launch_id) BETWEEN 1 AND 256 AND instr(launch_id, char(0)) = 0),
|
||||
plan_hash TEXT NOT NULL
|
||||
CHECK (length(plan_hash) = 64 AND plan_hash NOT GLOB '*[^0-9a-f]*'),
|
||||
container_engine TEXT NOT NULL CHECK (container_engine IN ('docker', 'podman')),
|
||||
FOREIGN KEY (launch_id) REFERENCES node_worker_launches(launch_id) ON DELETE CASCADE
|
||||
) STRICT;
|
||||
|
||||
-- The container lease is separate from the launch receipt and engine metadata.
|
||||
-- It lets a restarted node withdraw an expired projection without retaining the
|
||||
-- descriptor, projection content, or any Gateway credential in durable state.
|
||||
CREATE TABLE IF NOT EXISTS node_worker_container_leases (
|
||||
launch_id TEXT NOT NULL PRIMARY KEY
|
||||
CHECK (length(launch_id) BETWEEN 1 AND 256 AND instr(launch_id, char(0)) = 0),
|
||||
plan_hash TEXT NOT NULL
|
||||
CHECK (length(plan_hash) = 64 AND plan_hash NOT GLOB '*[^0-9a-f]*'),
|
||||
expires_at_ms INTEGER NOT NULL CHECK (expires_at_ms BETWEEN 0 AND 9007199254740991),
|
||||
FOREIGN KEY (launch_id) REFERENCES node_worker_launches(launch_id) ON DELETE CASCADE
|
||||
) STRICT;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_node_worker_launches_terminal_completed
|
||||
ON node_worker_launches(completed_at_ms, launch_id)
|
||||
WHERE completed_at_ms IS NOT NULL;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user