feat(node-host): supervise durable worker launches (#122829)

* feat(node-host): add worker launch supervision

* fix(node-host): harden worker lifecycle ownership

* fix(node-host): harden worker execution boundary

* fix(node-host): preserve worker trust settings

* chore(plugin-sdk): refresh worker lifecycle baselines

* docs(plan): track runner implementation slices

* test(node-host): await runtime shutdown owner
This commit is contained in:
Peter Steinberger
2026-08-12 15:24:05 -07:00
committed by GitHub
parent 93f5e0f1f6
commit 6b9bea84f0
38 changed files with 2896 additions and 88 deletions
+51
View File
@@ -0,0 +1,51 @@
const POSIX_WORKER_ENV_KEYS = new Set([
"PATH",
"HOME",
"TMPDIR",
"TMP",
"TEMP",
"LANG",
"LANGUAGE",
"TZ",
"NODE_EXTRA_CA_CERTS",
"NODE_USE_SYSTEM_CA",
"OPENCLAW_ALLOW_INSECURE_PRIVATE_WS",
]);
const WINDOWS_WORKER_ENV_KEYS = new Set([
...POSIX_WORKER_ENV_KEYS,
"USERPROFILE",
"HOMEDRIVE",
"HOMEPATH",
"SYSTEMROOT",
"WINDIR",
"COMSPEC",
"PATHEXT",
]);
/** Freeze the minimal non-secret environment inherited by node-host workers. */
export function snapshotNodeWorkerEnv(source: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
const windows = process.platform === "win32";
const snapshot: NodeJS.ProcessEnv = {};
const retainedWindowsKeys = new Map<string, string>();
for (const [key, value] of Object.entries(source)) {
if (value === undefined) {
continue;
}
const normalized = windows ? key.toUpperCase() : key;
const allowed =
(windows ? WINDOWS_WORKER_ENV_KEYS : POSIX_WORKER_ENV_KEYS).has(normalized) ||
normalized.startsWith("LC_");
if (!allowed) {
continue;
}
if (windows) {
const previousKey = retainedWindowsKeys.get(normalized);
if (previousKey) {
delete snapshot[previousKey];
}
retainedWindowsKeys.set(normalized, key);
}
snapshot[key] = value;
}
return snapshot;
}
+434
View File
@@ -0,0 +1,434 @@
import type { DatabaseSync } from "node:sqlite";
import type { Selectable } from "kysely";
import {
executeSqliteQuerySync,
executeSqliteQueryTakeFirstSync,
getNodeSqliteKysely,
} from "../infra/kysely-sync.js";
import type { DB as OpenClawStateDatabase } from "../state/openclaw-state-db.generated.js";
import {
runOpenClawStateWriteTransaction,
type OpenClawStateDatabaseOptions,
} from "../state/openclaw-state-db.js";
import { OPENCLAW_STATE_SCHEMA_SQL } from "../state/openclaw-state-schema.js";
import {
inspectNodeWorkerProcessIdentity,
type NodeWorkerProcessIdentity,
} from "./node-worker-process-identity.js";
type NodeWorkerLaunchState =
| "pending"
| "running"
| "completed"
| "failed"
| "interrupted"
| "cancelled";
export type NodeWorkerTerminalState = Exclude<NodeWorkerLaunchState, "pending" | "running">;
type NodeWorkerLaunchDatabase = Pick<OpenClawStateDatabase, "node_worker_launches">;
type NodeWorkerLaunchRow = Selectable<NodeWorkerLaunchDatabase["node_worker_launches"]>;
export type NodeWorkerLaunchReceipt = {
launchId: string;
planHash: string;
gatewayNamespace: string;
environmentId: string;
sessionId: string;
ownerEpoch: number;
placementGeneration: number;
runId: string;
state: NodeWorkerLaunchState;
supervisor: NodeWorkerProcessIdentity;
worker: NodeWorkerProcessIdentity | null;
resultJson: string | null;
errorText: string | null;
completedAtMs: number | null;
createdAtMs: number;
updatedAtMs: number;
};
type NodeWorkerLaunchClaim = Pick<
NodeWorkerLaunchReceipt,
| "environmentId"
| "gatewayNamespace"
| "launchId"
| "ownerEpoch"
| "placementGeneration"
| "planHash"
| "runId"
| "sessionId"
>;
type NodeWorkerLaunchClaimResult = {
action: "start" | "replay" | "recover";
receipt: NodeWorkerLaunchReceipt;
};
const NODE_WORKER_LAUNCH_SCHEMA_START = "CREATE TABLE IF NOT EXISTS node_worker_launches (";
const NODE_WORKER_LAUNCH_SCHEMA_END = "\n) STRICT;";
const initializedDatabases = new WeakSet<DatabaseSync>();
const TERMINAL_STATES: ReadonlySet<string> = new Set([
"completed",
"failed",
"interrupted",
"cancelled",
]);
function ensureNodeWorkerLaunchSchema(database: DatabaseSync): void {
const start = OPENCLAW_STATE_SCHEMA_SQL.indexOf(NODE_WORKER_LAUNCH_SCHEMA_START);
const end =
start >= 0 ? OPENCLAW_STATE_SCHEMA_SQL.indexOf(NODE_WORKER_LAUNCH_SCHEMA_END, start) : -1;
if (start < 0 || end < start) {
throw new Error("OpenClaw node worker launch schema marker is missing.");
}
database.exec(OPENCLAW_STATE_SCHEMA_SQL.slice(start, end + NODE_WORKER_LAUNCH_SCHEMA_END.length)); // sqlite-allow-raw -- Canonical feature-local additive DDL only.
}
function query(database: DatabaseSync) {
return getNodeSqliteKysely<NodeWorkerLaunchDatabase>(database);
}
function readRow(database: DatabaseSync, launchId: string): NodeWorkerLaunchRow | undefined {
return executeSqliteQueryTakeFirstSync(
database,
query(database)
.selectFrom("node_worker_launches")
.selectAll()
.where("launch_id", "=", launchId),
);
}
function processIdentity(pid: number, startTime: number): NodeWorkerProcessIdentity {
return { pid, startTime };
}
function receiptFromRow(row: NodeWorkerLaunchRow): NodeWorkerLaunchReceipt {
if (!isNodeWorkerLaunchState(row.state)) {
throw new Error(`invalid node worker launch state ${row.state}`);
}
return {
launchId: row.launch_id,
planHash: row.plan_hash,
gatewayNamespace: row.gateway_namespace,
environmentId: row.environment_id,
sessionId: row.session_id,
ownerEpoch: row.owner_epoch,
placementGeneration: row.placement_generation,
runId: row.run_id,
state: row.state,
supervisor: processIdentity(row.supervisor_pid, row.supervisor_start_time),
worker:
row.worker_pid === null || row.worker_start_time === null
? null
: processIdentity(row.worker_pid, row.worker_start_time),
resultJson: row.result_json,
errorText: row.error_text,
completedAtMs: row.completed_at_ms,
createdAtMs: row.created_at_ms,
updatedAtMs: row.updated_at_ms,
};
}
function isNodeWorkerLaunchState(value: string): value is NodeWorkerLaunchState {
return value === "pending" || value === "running" || TERMINAL_STATES.has(value);
}
function validateIdentifier(value: string, label: string): void {
if (!value || value.trim() !== value || value.length > 256 || value.includes("\0")) {
throw new Error(`${label} must be a bounded non-empty identifier`);
}
}
function validatePlanHash(value: string): void {
if (!/^[a-f0-9]{64}$/u.test(value)) {
throw new Error("node worker plan hash must be 64 lowercase hexadecimal characters");
}
}
function validateTimestamp(value: number): void {
if (!Number.isSafeInteger(value) || value < 0) {
throw new Error("node worker launch timestamp must be a non-negative safe integer");
}
}
function validateProcessIdentity(identity: NodeWorkerProcessIdentity): void {
if (
!Number.isSafeInteger(identity.pid) ||
identity.pid <= 0 ||
identity.pid > 2_147_483_647 ||
!Number.isSafeInteger(identity.startTime) ||
identity.startTime < 0
) {
throw new Error("node worker process identity must contain a bounded pid and start time");
}
}
function requireMatchingRow(
database: DatabaseSync,
launchId: string,
planHash: string,
): NodeWorkerLaunchRow {
const row = readRow(database, launchId);
if (!row) {
throw new Error(`node worker launch ${launchId} does not exist`);
}
if (row.plan_hash !== planHash) {
throw new Error(`node worker launch ${launchId} was replayed with a different plan`);
}
return row;
}
function rowHasSupervisor(row: NodeWorkerLaunchRow, identity: NodeWorkerProcessIdentity): boolean {
return row.supervisor_pid === identity.pid && row.supervisor_start_time === identity.startTime;
}
function rowHasWorker(
row: NodeWorkerLaunchRow,
identity: NodeWorkerProcessIdentity | null,
): boolean {
return identity === null
? row.worker_pid === null && row.worker_start_time === null
: row.worker_pid === identity.pid && row.worker_start_time === identity.startTime;
}
function sameObservedOwner(current: NodeWorkerLaunchRow, observed: NodeWorkerLaunchRow): boolean {
return (
current.state === observed.state &&
current.supervisor_pid === observed.supervisor_pid &&
current.supervisor_start_time === observed.supervisor_start_time &&
current.worker_pid === observed.worker_pid &&
current.worker_start_time === observed.worker_start_time
);
}
/** Synchronous shared-state owner for durable node worker launch supervision. */
export class NodeWorkerLaunchStore {
private readonly databaseOptions: OpenClawStateDatabaseOptions;
constructor(options: { env?: NodeJS.ProcessEnv } = {}) {
this.databaseOptions = options.env ? { env: options.env } : {};
}
private write<T>(operationLabel: string, operation: (database: DatabaseSync) => T): T {
let initializedDatabase: DatabaseSync | undefined;
const result = runOpenClawStateWriteTransaction(
({ db }) => {
if (!initializedDatabases.has(db)) {
ensureNodeWorkerLaunchSchema(db);
initializedDatabase = db;
}
return operation(db);
},
this.databaseOptions,
{ operationLabel },
);
if (initializedDatabase) {
initializedDatabases.add(initializedDatabase);
}
return result;
}
claim(
claim: NodeWorkerLaunchClaim,
supervisor: NodeWorkerProcessIdentity,
nowMs = Date.now(),
): NodeWorkerLaunchClaimResult {
validateIdentifier(claim.launchId, "node worker launch id");
validatePlanHash(claim.planHash);
validateTimestamp(nowMs);
validateProcessIdentity(supervisor);
// Process inspection is intentionally outside SQLite. The second transaction
// re-reads the exact owner tuple before an adoption or recovery decision.
const observed = this.write("node-worker-launch.claim-inspect", (database) =>
readRow(database, claim.launchId),
);
if (observed && observed.plan_hash !== claim.planHash) {
throw new Error(`node worker launch ${claim.launchId} was replayed with a different plan`);
}
const observedSupervisorState = observed
? inspectNodeWorkerProcessIdentity(
processIdentity(observed.supervisor_pid, observed.supervisor_start_time),
)
: undefined;
return this.write("node-worker-launch.claim", (database) => {
let current = readRow(database, claim.launchId);
if (!current) {
executeSqliteQuerySync(
database,
query(database).insertInto("node_worker_launches").values({
launch_id: claim.launchId,
plan_hash: claim.planHash,
gateway_namespace: claim.gatewayNamespace,
environment_id: claim.environmentId,
session_id: claim.sessionId,
owner_epoch: claim.ownerEpoch,
placement_generation: claim.placementGeneration,
run_id: claim.runId,
state: "pending",
supervisor_pid: supervisor.pid,
supervisor_start_time: supervisor.startTime,
worker_pid: null,
worker_start_time: null,
result_json: null,
error_text: null,
completed_at_ms: null,
created_at_ms: nowMs,
updated_at_ms: nowMs,
}),
);
return {
action: "start",
receipt: receiptFromRow(requireMatchingRow(database, claim.launchId, claim.planHash)),
};
}
if (current.plan_hash !== claim.planHash) {
throw new Error(`node worker launch ${claim.launchId} was replayed with a different plan`);
}
const previousOwnerDefinitelyStale =
observedSupervisorState === "dead" || observedSupervisorState === "reused";
if (
current.state === "pending" &&
observed &&
sameObservedOwner(current, observed) &&
previousOwnerDefinitelyStale
) {
const updatedAtMs = Math.max(nowMs, current.created_at_ms, current.updated_at_ms);
executeSqliteQuerySync(
database,
query(database)
.updateTable("node_worker_launches")
.set({
supervisor_pid: supervisor.pid,
supervisor_start_time: supervisor.startTime,
updated_at_ms: updatedAtMs,
})
.where("launch_id", "=", claim.launchId)
.where("plan_hash", "=", claim.planHash)
.where("state", "=", "pending")
.where("supervisor_pid", "=", observed.supervisor_pid)
.where("supervisor_start_time", "=", observed.supervisor_start_time)
.where("worker_pid", "is", null)
.where("worker_start_time", "is", null),
);
current = requireMatchingRow(database, claim.launchId, claim.planHash);
return {
action: rowHasSupervisor(current, supervisor) ? "start" : "replay",
receipt: receiptFromRow(current),
};
}
if (
current.state === "running" &&
observed &&
sameObservedOwner(current, observed) &&
previousOwnerDefinitelyStale
) {
return { action: "recover", receipt: receiptFromRow(current) };
}
return { action: "replay", receipt: receiptFromRow(current) };
});
}
get(launchId: string): NodeWorkerLaunchReceipt | undefined {
validateIdentifier(launchId, "node worker launch id");
return this.write("node-worker-launch.get", (database) => {
const row = readRow(database, launchId);
return row ? receiptFromRow(row) : undefined;
});
}
markRunning(params: {
launchId: string;
planHash: string;
supervisor: NodeWorkerProcessIdentity;
worker: NodeWorkerProcessIdentity;
nowMs?: number;
}): NodeWorkerLaunchReceipt {
const nowMs = params.nowMs ?? Date.now();
validateTimestamp(nowMs);
validateProcessIdentity(params.supervisor);
validateProcessIdentity(params.worker);
return this.write("node-worker-launch.mark-running", (database) => {
const current = requireMatchingRow(database, params.launchId, params.planHash);
if (TERMINAL_STATES.has(current.state)) {
return receiptFromRow(current);
}
if (current.state === "running") {
return receiptFromRow(current);
}
if (!rowHasSupervisor(current, params.supervisor) || !rowHasWorker(current, null)) {
return receiptFromRow(current);
}
const updatedAtMs = Math.max(nowMs, current.created_at_ms, current.updated_at_ms);
executeSqliteQuerySync(
database,
query(database)
.updateTable("node_worker_launches")
.set({
state: "running",
worker_pid: params.worker.pid,
worker_start_time: params.worker.startTime,
updated_at_ms: updatedAtMs,
})
.where("launch_id", "=", params.launchId)
.where("plan_hash", "=", params.planHash)
.where("state", "=", "pending")
.where("supervisor_pid", "=", params.supervisor.pid)
.where("supervisor_start_time", "=", params.supervisor.startTime)
.where("worker_pid", "is", null)
.where("worker_start_time", "is", null),
);
return receiptFromRow(requireMatchingRow(database, params.launchId, params.planHash));
});
}
finish(params: {
launchId: string;
planHash: string;
supervisor: NodeWorkerProcessIdentity;
worker: NodeWorkerProcessIdentity | null;
state: NodeWorkerTerminalState;
resultJson?: string;
errorText?: string;
nowMs?: number;
}): NodeWorkerLaunchReceipt {
const nowMs = params.nowMs ?? Date.now();
validateTimestamp(nowMs);
validateProcessIdentity(params.supervisor);
if (params.worker) {
validateProcessIdentity(params.worker);
}
return this.write("node-worker-launch.finish", (database) => {
const current = requireMatchingRow(database, params.launchId, params.planHash);
if (TERMINAL_STATES.has(current.state)) {
return receiptFromRow(current);
}
if (!rowHasSupervisor(current, params.supervisor) || !rowHasWorker(current, params.worker)) {
return receiptFromRow(current);
}
const completedAtMs = Math.max(nowMs, current.created_at_ms, current.updated_at_ms);
let update = query(database)
.updateTable("node_worker_launches")
.set({
state: params.state,
result_json: params.state === "completed" ? (params.resultJson ?? null) : null,
error_text: params.state === "completed" ? null : (params.errorText ?? null),
completed_at_ms: completedAtMs,
updated_at_ms: completedAtMs,
})
.where("launch_id", "=", params.launchId)
.where("plan_hash", "=", params.planHash)
.where("state", "in", ["pending", "running"])
.where("supervisor_pid", "=", params.supervisor.pid)
.where("supervisor_start_time", "=", params.supervisor.startTime);
update = params.worker
? update
.where("worker_pid", "=", params.worker.pid)
.where("worker_start_time", "=", params.worker.startTime)
: update.where("worker_pid", "is", null).where("worker_start_time", "is", null);
executeSqliteQuerySync(database, update);
return receiptFromRow(requireMatchingRow(database, params.launchId, params.planHash));
});
}
}
@@ -0,0 +1,36 @@
import { readWindowsProcessStartTimeSync } from "../infra/windows-port-pids.js";
import { getFileLockProcessStartTime, isPidDefinitelyDead } from "../shared/pid-alive.js";
export type NodeWorkerProcessIdentity = {
pid: number;
startTime: number;
};
type NodeWorkerProcessIdentityState = "live" | "dead" | "reused" | "unknown";
function readNodeWorkerProcessStartTime(pid: number): number | null {
return process.platform === "win32"
? readWindowsProcessStartTimeSync(pid)
: getFileLockProcessStartTime(pid);
}
export function requireNodeWorkerProcessIdentity(pid: number): NodeWorkerProcessIdentity {
const startTime = readNodeWorkerProcessStartTime(pid);
if (startTime === null) {
throw new Error(`cannot establish PID-reuse-safe identity for process ${pid}`);
}
return { pid, startTime };
}
export function inspectNodeWorkerProcessIdentity(
identity: NodeWorkerProcessIdentity,
): NodeWorkerProcessIdentityState {
const observedStartTime = readNodeWorkerProcessStartTime(identity.pid);
if (observedStartTime !== null) {
if (observedStartTime !== identity.startTime) {
return "reused";
}
return isPidDefinitelyDead(identity.pid) ? "dead" : "live";
}
return isPidDefinitelyDead(identity.pid) ? "dead" : "unknown";
}
@@ -0,0 +1,362 @@
import { spawn, type ChildProcess } from "node:child_process";
import { createHash } from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { stableStringify } from "@openclaw/normalization-core";
import { afterEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import {
closeOpenClawStateDatabaseForTest,
openOpenClawStateDatabase,
} from "../state/openclaw-state-db.js";
import type { NodeWorkerLaunchReceipt } from "./node-worker-launch-store.js";
import {
inspectNodeWorkerProcessIdentity,
requireNodeWorkerProcessIdentity,
type NodeWorkerProcessIdentity,
} from "./node-worker-process-identity.js";
import { createNodeWorkerSupervisor } from "./node-worker-supervisor.js";
import {
testWorkerLaunchInput,
writeNodeWorkerFixture,
} from "./node-worker-supervisor.test-support.js";
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
const spawned = new Set<ChildProcess>();
const ownedProcessGroups: NodeWorkerProcessIdentity[] = [];
afterEach(async () => {
for (const child of spawned) {
if (child.exitCode === null && child.signalCode === null) {
child.kill("SIGKILL");
}
}
if (process.platform !== "win32") {
for (const identity of ownedProcessGroups) {
if (inspectNodeWorkerProcessIdentity(identity) === "reused") {
continue;
}
try {
process.kill(-identity.pid, "SIGKILL");
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ESRCH") {
throw error;
}
}
}
}
spawned.clear();
ownedProcessGroups.length = 0;
closeOpenClawStateDatabaseForTest();
});
function fixture(label: string) {
return writeNodeWorkerFixture(tempDirs.make(label));
}
function planHash(input: ReturnType<typeof testWorkerLaunchInput>): string {
return createHash("sha256")
.update(
stableStringify({
bundleHash: input.bundleHash,
descriptor: input.descriptor,
gatewayNamespace: input.gatewayNamespace,
placementGeneration: input.placementGeneration,
}),
)
.digest("hex");
}
function insertLaunch(params: {
env: NodeJS.ProcessEnv;
input: ReturnType<typeof testWorkerLaunchInput>;
state: "pending" | "running";
supervisor: NodeWorkerProcessIdentity;
worker?: NodeWorkerProcessIdentity;
}) {
const database = openOpenClawStateDatabase({ env: params.env }).db;
database
.prepare(
`INSERT INTO node_worker_launches (
launch_id, plan_hash, gateway_namespace, environment_id, session_id,
owner_epoch, placement_generation, run_id, state,
supervisor_pid, supervisor_start_time, worker_pid, worker_start_time,
result_json, error_text, completed_at_ms, created_at_ms, updated_at_ms
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, 1, 1)`,
)
.run(
params.input.launchId,
planHash(params.input),
params.input.gatewayNamespace,
params.input.descriptor.admission.environmentId,
params.input.descriptor.admission.sessionId,
params.input.descriptor.admission.ownerEpoch,
params.input.placementGeneration,
params.input.descriptor.assignment.runId,
params.state,
params.supervisor.pid,
params.supervisor.startTime,
params.worker?.pid ?? null,
params.worker?.startTime ?? null,
);
}
function waitForChildLine(child: ChildProcess): Promise<string> {
return new Promise((resolve, reject) => {
let stdout = "";
let stderr = "";
const onData = (chunk: Buffer) => {
stdout += chunk.toString("utf8");
const newline = stdout.indexOf("\n");
if (newline >= 0) {
resolve(stdout.slice(0, newline));
}
};
child.stdout?.on("data", onData);
child.stderr?.on("data", (chunk: Buffer) => {
stderr += chunk.toString("utf8");
});
child.once("error", reject);
child.once("close", (code, signal) => {
reject(new Error(`owner exited before ready (${code ?? signal}): ${stderr}`));
});
});
}
function waitForChildExit(child: ChildProcess): Promise<void> {
if (child.exitCode !== null || child.signalCode !== null) {
return Promise.resolve();
}
return new Promise((resolve, reject) => {
child.once("error", reject);
child.once("close", () => resolve());
});
}
function writeSupervisorOwnerScript(root: string): string {
const supervisorUrl = pathToFileURL(path.resolve("src/node-host/node-worker-supervisor.ts")).href;
const scriptPath = path.join(root, "supervisor-owner.mts");
fs.writeFileSync(
scriptPath,
`
import fs from "node:fs";
import { createNodeWorkerSupervisor } from ${JSON.stringify(supervisorUrl)};
const [bundleRoot, stateDir, inputPath] = process.argv.slice(2);
const supervisor = createNodeWorkerSupervisor({
bundleRoot,
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
});
const shutdown = async () => {
await supervisor.close();
process.exit(0);
};
process.once("SIGTERM", () => void shutdown());
const input = JSON.parse(fs.readFileSync(inputPath, "utf8"));
const receipt = await supervisor.launch(input);
process.stdout.write(JSON.stringify(receipt) + "\\n");
setInterval(() => {}, 1000);
`,
);
return scriptPath;
}
function spawnSupervisorOwner(params: {
bundleRoot: string;
env: NodeJS.ProcessEnv;
input: ReturnType<typeof testWorkerLaunchInput>;
root: string;
}): ChildProcess {
const inputPath = path.join(params.root, `${params.input.launchId}.json`);
fs.writeFileSync(inputPath, JSON.stringify(params.input));
const child = spawn(
process.execPath,
[
"--import",
"tsx",
writeSupervisorOwnerScript(params.root),
params.bundleRoot,
params.env.OPENCLAW_STATE_DIR!,
inputPath,
],
{ stdio: ["ignore", "pipe", "pipe"] },
);
spawned.add(child);
return child;
}
async function waitForIdentityDeath(identity: NodeWorkerProcessIdentity) {
await vi.waitFor(() => expect(inspectNodeWorkerProcessIdentity(identity)).not.toBe("live"), {
timeout: 5_000,
});
}
describe("node worker supervisor recovery", () => {
it("atomically adopts pending work only after the previous supervisor is stale", async () => {
const { bundleRoot, env, workspaceDir } = fixture("node-worker-stale-pending-");
const supervisor = createNodeWorkerSupervisor({ bundleRoot, env });
await supervisor.status("schema-probe");
const input = testWorkerLaunchInput(workspaceDir, "stale-pending-launch");
insertLaunch({
env,
input,
state: "pending",
supervisor: { pid: 2_147_483_647, startTime: 1 },
});
const running = await supervisor.launch(input);
expect(running).toMatchObject({
state: "running",
supervisor: requireNodeWorkerProcessIdentity(process.pid),
worker: { pid: expect.any(Number), startTime: expect.any(Number) },
});
await supervisor.close();
});
it.runIf(process.platform !== "win32")(
"kills the exact stale-owner worker group before marking it interrupted",
async () => {
const { bundleRoot, env, root, workspaceDir } = fixture("node-worker-stale-running-");
const marker = path.join(root, "recovery-grandchild.pid");
const workerSource = `
const { spawn } = require("node:child_process");
const fs = require("node:fs");
const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { stdio: "ignore" });
fs.writeFileSync(process.argv[1], String(child.pid));
setInterval(() => {}, 1000);
`;
const workerProcess = spawn(process.execPath, ["-e", workerSource, marker], {
detached: true,
stdio: "ignore",
});
spawned.add(workerProcess);
const worker = requireNodeWorkerProcessIdentity(workerProcess.pid!);
ownedProcessGroups.push(worker);
await vi.waitFor(() => expect(fs.existsSync(marker)).toBe(true));
const grandchild = requireNodeWorkerProcessIdentity(Number(fs.readFileSync(marker, "utf8")));
const input = testWorkerLaunchInput(workspaceDir, "stale-running-launch", "wait");
const supervisor = createNodeWorkerSupervisor({ bundleRoot, env });
await supervisor.status("schema-probe");
insertLaunch({
env,
input,
state: "running",
supervisor: { pid: 2_147_483_647, startTime: 1 },
worker,
});
const recovered = await supervisor.launch(input);
expect(recovered).toMatchObject({ state: "interrupted", worker });
await waitForIdentityDeath(worker);
await waitForIdentityDeath(grandchild);
expect((await supervisor.status(input.launchId))?.worker).toEqual(worker);
await supervisor.close();
},
);
it("returns a live foreign running receipt from a real second process without mutation", async () => {
const { bundleRoot, env, root, workspaceDir } = fixture("node-worker-live-replay-");
const input = testWorkerLaunchInput(workspaceDir, "live-running-launch", "wait");
const owner = spawnSupervisorOwner({ bundleRoot, env, input, root });
const owned = JSON.parse(await waitForChildLine(owner)) as NodeWorkerLaunchReceipt;
if (owned.worker) {
ownedProcessGroups.push(owned.worker);
}
const second = createNodeWorkerSupervisor({ bundleRoot, env });
const replay = await second.launch(input);
expect(replay).toEqual(owned);
expect(inspectNodeWorkerProcessIdentity(owned.supervisor)).toBe("live");
expect(inspectNodeWorkerProcessIdentity(owned.worker!)).toBe("live");
owner.kill("SIGTERM");
await waitForChildExit(owner);
await second.close();
});
it.runIf(process.platform !== "win32")(
"uses IPC disconnect after owner SIGKILL, then reconciles only after exact tree death",
async () => {
const { bundleRoot, env, root, workspaceDir } = fixture("node-worker-owner-kill-");
const input = testWorkerLaunchInput(workspaceDir, "owner-kill-launch", "tree");
const owner = spawnSupervisorOwner({ bundleRoot, env, input, root });
const owned = JSON.parse(await waitForChildLine(owner)) as NodeWorkerLaunchReceipt;
ownedProcessGroups.push(owned.worker!);
const grandchildPath = path.join(workspaceDir, "grandchild.pid");
await vi.waitFor(() => expect(fs.existsSync(grandchildPath)).toBe(true));
const grandchild = requireNodeWorkerProcessIdentity(
Number(fs.readFileSync(grandchildPath, "utf8")),
);
owner.kill("SIGKILL");
await waitForChildExit(owner);
await waitForIdentityDeath(owned.supervisor);
await waitForIdentityDeath(owned.worker!);
await waitForIdentityDeath(grandchild);
const restarted = createNodeWorkerSupervisor({ bundleRoot, env });
const reconciled = await restarted.launch(input);
expect(reconciled).toMatchObject({
state: "interrupted",
supervisor: owned.supervisor,
worker: owned.worker,
});
await restarted.close();
},
);
it("keeps a live foreign pending claim unchanged across real processes", async () => {
const { bundleRoot, env, root, workspaceDir } = fixture("node-worker-live-pending-");
const input = testWorkerLaunchInput(workspaceDir, "live-pending-launch", "wait");
const claim = {
launchId: input.launchId,
planHash: planHash(input),
gatewayNamespace: input.gatewayNamespace,
environmentId: input.descriptor.admission.environmentId,
sessionId: input.descriptor.admission.sessionId,
ownerEpoch: input.descriptor.admission.ownerEpoch,
placementGeneration: input.placementGeneration,
runId: input.descriptor.assignment.runId,
};
const storeUrl = pathToFileURL(path.resolve("src/node-host/node-worker-launch-store.ts")).href;
const identityUrl = pathToFileURL(
path.resolve("src/node-host/node-worker-process-identity.ts"),
).href;
const claimPath = path.join(root, "claim.json");
const scriptPath = path.join(root, "pending-owner.mts");
fs.writeFileSync(claimPath, JSON.stringify(claim));
fs.writeFileSync(
scriptPath,
`
import fs from "node:fs";
import { NodeWorkerLaunchStore } from ${JSON.stringify(storeUrl)};
import { requireNodeWorkerProcessIdentity } from ${JSON.stringify(identityUrl)};
const [stateDir, claimPath] = process.argv.slice(2);
const store = new NodeWorkerLaunchStore({ env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } });
const result = store.claim(
JSON.parse(fs.readFileSync(claimPath, "utf8")),
requireNodeWorkerProcessIdentity(process.pid),
);
process.stdout.write(JSON.stringify(result.receipt) + "\\n");
setInterval(() => {}, 1000);
`,
);
const owner = spawn(
process.execPath,
["--import", "tsx", scriptPath, env.OPENCLAW_STATE_DIR!, claimPath],
{ stdio: ["ignore", "pipe", "pipe"] },
);
spawned.add(owner);
const owned = JSON.parse(await waitForChildLine(owner)) as NodeWorkerLaunchReceipt;
const second = createNodeWorkerSupervisor({ bundleRoot, env });
const replay = await second.launch(input);
expect(replay).toEqual(owned);
owner.kill("SIGKILL");
await waitForChildExit(owner);
await second.close();
});
});
@@ -0,0 +1,184 @@
import fs from "node:fs";
import path from "node:path";
import {
WORKER_PROTOCOL_FEATURES,
WORKER_RPC_SET_VERSION,
} from "../../packages/gateway-protocol/src/schema/worker-admission.js";
import type { WorkerLaunchDescriptor } from "../worker/launch-descriptor.js";
const TEST_BUNDLE_HASH = "a".repeat(64);
export const TEST_WORKER_CREDENTIAL = 'node worker/"credential\\secret?';
export const TEST_WORKER_SOURCE = String.raw`
import fs from "node:fs";
import path from "node:path";
import { spawn } from "node:child_process";
let input = "";
for await (const chunk of process.stdin) input += chunk;
const descriptor = JSON.parse(input);
if (descriptor.assignment.prompt === "exit-before-start") {
fs.writeFileSync(path.join(descriptor.assignment.workspaceDir, "prestart-exited"), "exited");
process.exit(23);
}
if (!process.connected || !process.channel || !process.argv.includes("--internal-worker-ipc")) {
process.exit(24);
}
let grandchild;
let disposed = false;
let started = false;
let resolveStart;
const start = new Promise((resolve) => { resolveStart = resolve; });
const hardTerminate = () => {
if (process.platform === "win32") {
spawn("taskkill", ["/F", "/T", "/PID", String(process.pid)], {
detached: true,
stdio: "ignore",
windowsHide: true,
});
return;
}
process.kill(-process.pid, "SIGKILL");
};
const onMessage = (message) => {
if (
started ||
typeof message !== "object" ||
message === null ||
Array.isArray(message) ||
Object.keys(message).length !== 1 ||
message.type !== "openclaw-worker-start-v1"
) {
hardTerminate();
return;
}
started = true;
resolveStart();
};
const onDisconnect = () => {
if (disposed) return;
if (!started) process.exit(0);
hardTerminate();
};
process.on("message", onMessage);
process.once("disconnect", onDisconnect);
await start;
const exitWorker = (code) => {
disposed = true;
process.off("message", onMessage);
process.off("disconnect", onDisconnect);
if (process.connected) process.disconnect();
process.exit(code);
};
const writeResultAndExit = (value) => {
fs.writeSync(1, value);
exitWorker(0);
};
const mode = descriptor.assignment.prompt;
if (mode === "wait") {
setInterval(() => {}, 1000);
} else if (mode === "tree") {
grandchild = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { stdio: "ignore" });
fs.writeFileSync(path.join(descriptor.assignment.workspaceDir, "grandchild.pid"), String(grandchild.pid));
setInterval(() => {}, 1000);
} else if (mode === "secret-fail") {
await new Promise((resolve) => setTimeout(resolve, 500));
const credential = descriptor.admission.credential;
const escaped = JSON.stringify(credential).slice(1, -1);
process.stderr.write(
"failure " + "x".repeat(5000) + " " + credential + " " + encodeURIComponent(credential) + " " + escaped,
);
exitWorker(7);
} else if (mode.startsWith("secret-cutoff-")) {
const credential = descriptor.admission.credential;
const representations = {
"secret-cutoff-raw": credential,
"secret-cutoff-url": encodeURIComponent(credential),
"secret-cutoff-json": JSON.stringify(credential).slice(1, -1),
};
const representation = representations[mode];
const suffixBytes = 4096 - Math.floor(Buffer.byteLength(representation, "utf8") / 2);
process.stderr.write("x".repeat(5000) + representation + "y".repeat(suffixBytes));
exitWorker(7);
} else if (mode === "secret-success") {
await new Promise((resolve) => setTimeout(resolve, 500));
const credential = descriptor.admission.credential;
writeResultAndExit(
JSON.stringify({ raw: credential, encoded: encodeURIComponent(credential), status: "completed" }) + "\n",
);
} else if (mode === "overflow") {
writeResultAndExit("x".repeat(70 * 1024));
} else if (mode === "fast-terminal") {
const marker = path.join(descriptor.assignment.workspaceDir, "fast-terminal-marker");
process.once("SIGTERM", () => {
fs.writeFileSync(marker, "signal");
process.exit(143);
});
await new Promise((resolve) => setTimeout(resolve, 100));
fs.writeFileSync(marker, "normal");
writeResultAndExit(JSON.stringify({ status: "completed" }) + "\n");
} else if (mode === "env") {
writeResultAndExit(JSON.stringify(process.env) + "\n");
} else {
await new Promise((resolve) => setTimeout(resolve, 25));
writeResultAndExit(JSON.stringify({ argv: process.argv.slice(2), status: "completed" }) + "\n");
}
`;
export function testWorkerDescriptor(
workspaceDir: string,
prompt = "success",
): WorkerLaunchDescriptor {
return {
version: 3,
connectionEndpoint: { kind: "unix", socketPath: "/tmp/openclaw-worker/gateway.sock" },
admission: {
environmentId: "environment-1",
credential: TEST_WORKER_CREDENTIAL,
sessionId: "session-1",
ownerEpoch: 3,
rpcSetVersion: WORKER_RPC_SET_VERSION,
handshake: {
bundleHash: TEST_BUNDLE_HASH,
openclawVersion: "2026.8.1",
protocolFeatures: [...WORKER_PROTOCOL_FEATURES],
},
},
assignment: {
agentId: "agent-1",
operationalRunInstance: { instanceId: "instance-1", runId: "run-1" },
agentRuntimeIdentityToken: "signed-runtime-token",
runId: "run-1",
turnId: "turn-1",
prompt,
suppressPromptTranscript: false,
workspaceDir,
modelRef: { provider: "provider-1", model: "model-1" },
inferenceOptions: {},
initialMessages: [],
transcript: { baseLeafId: null, nextSeq: 1 },
liveEvents: { ackedSeq: 0, nextSeq: 1 },
toolAuthority: { allowedToolNames: [] },
},
};
}
export function writeNodeWorkerFixture(root: string) {
const stateDir = path.join(root, "state-root");
const bundleRoot = path.join(root, "bundles-root");
const workspaceDir = path.join(root, "workspace");
const bundleDir = path.join(bundleRoot, "gateway-1", "bundles", TEST_BUNDLE_HASH);
fs.mkdirSync(bundleDir, { recursive: true });
fs.mkdirSync(workspaceDir, { recursive: true });
fs.writeFileSync(path.join(bundleDir, "openclaw.mjs"), TEST_WORKER_SOURCE);
return { bundleRoot, env: { OPENCLAW_STATE_DIR: stateDir }, root, stateDir, workspaceDir };
}
export function testWorkerLaunchInput(workspaceDir: string, launchId: string, prompt = "success") {
return {
launchId,
gatewayNamespace: "gateway-1",
bundleHash: TEST_BUNDLE_HASH,
placementGeneration: 4,
descriptor: testWorkerDescriptor(workspaceDir, prompt),
};
}
@@ -0,0 +1,513 @@
import childProcess from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { registerSecretValueForRedaction } from "../logging/secret-redaction-registry.js";
import { resetSecretRedactionRegistryForTest } from "../logging/secret-redaction-registry.test-support.js";
import {
closeOpenClawStateDatabaseForTest,
openOpenClawStateDatabase,
} from "../state/openclaw-state-db.js";
import { withEnvAsync } from "../test-utils/env.js";
import { NodeWorkerLaunchStore } from "./node-worker-launch-store.js";
import {
inspectNodeWorkerProcessIdentity,
requireNodeWorkerProcessIdentity,
} from "./node-worker-process-identity.js";
import { createNodeWorkerSupervisor } from "./node-worker-supervisor.js";
import {
TEST_WORKER_CREDENTIAL,
TEST_WORKER_SOURCE,
testWorkerDescriptor,
testWorkerLaunchInput,
writeNodeWorkerFixture,
} from "./node-worker-supervisor.test-support.js";
type NodeWorkerSupervisor = ReturnType<typeof createNodeWorkerSupervisor>;
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
afterEach(() => {
vi.restoreAllMocks();
resetSecretRedactionRegistryForTest();
closeOpenClawStateDatabaseForTest();
});
function fixture() {
const root = tempDirs.make("node-worker-supervisor-");
const { bundleRoot, env, stateDir, workspaceDir } = writeNodeWorkerFixture(root);
const supervisor = createNodeWorkerSupervisor({ bundleRoot, env });
return { bundleRoot, env, root, stateDir, supervisor, workspaceDir };
}
function launchInput(workspaceDir: string, launchId: string, prompt = "success") {
return testWorkerLaunchInput(workspaceDir, launchId, prompt);
}
async function waitForTerminal(supervisor: NodeWorkerSupervisor, launchId: string) {
await vi.waitFor(
async () => {
expect((await supervisor.status(launchId))?.state).not.toMatch(/^(?:pending|running)$/u);
},
{ timeout: 5_000 },
);
const receipt = await supervisor.status(launchId);
if (!receipt) {
throw new Error(`missing launch receipt ${launchId}`);
}
return receipt;
}
describe("node worker supervisor", () => {
it("keeps construction and close inert without resolving process identity", async () => {
const root = tempDirs.make("node-worker-inert-");
const { bundleRoot, env } = writeNodeWorkerFixture(root);
const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform");
const spawnSync = vi.spyOn(childProcess, "spawnSync");
const execFileSync = vi.spyOn(childProcess, "execFileSync");
Object.defineProperty(process, "platform", { configurable: true, value: "win32" });
try {
const supervisor = createNodeWorkerSupervisor({ bundleRoot, env });
await supervisor.close();
expect(spawnSync).not.toHaveBeenCalled();
expect(execFileSync).not.toHaveBeenCalled();
} finally {
if (originalPlatform) {
Object.defineProperty(process, "platform", originalPlatform);
}
}
});
it("keeps the additive table absent until the first stateful operation", async () => {
const { bundleRoot, env, supervisor } = fixture();
const database = openOpenClawStateDatabase({ env });
const findTable = () =>
database.db
.prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?")
.get("node_worker_launches");
expect(findTable()).toBeUndefined();
await supervisor.close();
expect(findTable()).toBeUndefined();
const active = createNodeWorkerSupervisor({ bundleRoot, env });
expect(await active.status("missing-launch")).toBeUndefined();
expect(
database.db
.prepare("SELECT strict FROM pragma_table_list WHERE name = ?")
.get("node_worker_launches"),
).toEqual({ strict: 1 });
await active.close();
});
it("keeps pending and running launches owned by a live supervisor unchanged", async () => {
const { bundleRoot, env, supervisor } = fixture();
await supervisor.status("schema-probe");
const database = openOpenClawStateDatabase({ env }).db;
const supervisorIdentity = requireNodeWorkerProcessIdentity(process.pid);
const insert = database.prepare(`
INSERT INTO node_worker_launches (
launch_id, plan_hash, gateway_namespace, environment_id, session_id,
owner_epoch, placement_generation, run_id, state,
supervisor_pid, supervisor_start_time, worker_pid, worker_start_time,
result_json, error_text, completed_at_ms, created_at_ms, updated_at_ms
) VALUES (?, ?, 'gateway-1', 'environment-1', 'session-1', 3, 4, 'run-1', ?, ?, ?, ?, ?, NULL, NULL, NULL, 1, 1)
`);
insert.run(
"pending-launch",
"b".repeat(64),
"pending",
supervisorIdentity.pid,
supervisorIdentity.startTime,
null,
null,
);
insert.run(
"running-launch",
"c".repeat(64),
"running",
supervisorIdentity.pid,
supervisorIdentity.startTime,
process.pid,
supervisorIdentity.startTime,
);
const sameHandle = createNodeWorkerSupervisor({ bundleRoot, env });
expect(await sameHandle.status("pending-launch")).toMatchObject({
state: "pending",
worker: null,
});
expect(await sameHandle.status("running-launch")).toMatchObject({
state: "running",
worker: supervisorIdentity,
});
await supervisor.close();
await sameHandle.close();
closeOpenClawStateDatabaseForTest();
openOpenClawStateDatabase({ env });
const recovered = createNodeWorkerSupervisor({ bundleRoot, env });
expect(await recovered.status("pending-launch")).toMatchObject({
state: "pending",
worker: null,
});
expect(await recovered.status("running-launch")).toMatchObject({
state: "running",
worker: supervisorIdentity,
});
await recovered.close();
});
it("launches idempotently and persists only bounded non-secret facts", async () => {
const { env, supervisor, workspaceDir } = fixture();
const input = launchInput(workspaceDir, "success-launch");
expect(await supervisor.launch(input)).toMatchObject({
launchId: "success-launch",
state: "running",
environmentId: "environment-1",
sessionId: "session-1",
ownerEpoch: 3,
placementGeneration: 4,
runId: "run-1",
});
const completed = await waitForTerminal(supervisor, input.launchId);
expect(completed).toMatchObject({ state: "completed", errorText: null });
expect(JSON.parse(completed.resultJson ?? "null")).toEqual({
argv: ["worker", "--internal-worker-ipc"],
status: "completed",
});
expect(await supervisor.launch(input)).toEqual(completed);
await expect(
supervisor.launch({
...input,
descriptor: testWorkerDescriptor(workspaceDir, "different-plan"),
}),
).rejects.toThrow("replayed with a different plan");
const row = openOpenClawStateDatabase({ env })
.db.prepare("SELECT * FROM node_worker_launches WHERE launch_id = ?")
.get(input.launchId);
expect(JSON.stringify(row)).not.toContain(TEST_WORKER_CREDENTIAL);
await supervisor.close();
});
it.each(["status", "launch", "cancel", "close"] as const)(
"retains an observed terminal outcome when %s reconciliation keeps failing",
async (operation) => {
const { env, supervisor, workspaceDir } = fixture();
const input = launchInput(workspaceDir, `finish-failure-${operation}`);
const store = (supervisor as unknown as { store: NodeWorkerLaunchStore }).store;
const originalFinish = store.finish.bind(store);
let persistenceUnavailable = true;
const finish = vi.spyOn(store, "finish").mockImplementation((params) => {
if (persistenceUnavailable) {
throw new Error("injected finish failure");
}
return originalFinish(params);
});
const invoke = async () => {
switch (operation) {
case "status":
return await supervisor.status(input.launchId);
case "launch":
return await supervisor.launch(input);
case "cancel":
return await supervisor.cancel(input.launchId);
case "close":
await supervisor.close();
return new NodeWorkerLaunchStore({ env }).get(input.launchId);
default:
throw new Error("unsupported reconciliation operation");
}
};
expect(await supervisor.launch(input)).toMatchObject({ state: "running" });
await vi.waitFor(() => expect(finish).toHaveBeenCalled(), { timeout: 5_000 });
expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)?.state).toBe("running");
await expect(invoke()).rejects.toThrow("injected finish failure");
expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)?.state).toBe("running");
persistenceUnavailable = false;
const completed = await invoke();
expect(completed).toMatchObject({
state: "completed",
resultJson: expect.stringContaining('"status":"completed"'),
});
expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)?.state).toBe("completed");
await supervisor.close();
},
);
it("spawns workers with only supplied runtime essentials", async () => {
const root = tempDirs.make("node-worker-env-");
const { bundleRoot, env, workspaceDir } = writeNodeWorkerFixture(root);
const suppliedPathKey = process.platform === "win32" ? "Path" : "PATH";
const suppliedEnv: NodeJS.ProcessEnv = {
...env,
[suppliedPathKey]: process.env.PATH,
HOME: path.join(root, "worker-home"),
LANG: "en_US.UTF-8",
LC_TIME: "de_DE.UTF-8",
NODE_EXTRA_CA_CERTS: path.join(root, "private-ca.pem"),
NODE_USE_SYSTEM_CA: "1",
OPENCLAW_ALLOW_INSECURE_PRIVATE_WS: "1",
OPENCLAW_SUPPLIED_SECRET: "supplied-openclaw-secret",
NODE_OPTIONS: "--title=forbidden-worker-title",
BASH_ENV: path.join(root, "forbidden-shell-init"),
DYLD_INSERT_LIBRARIES: path.join(root, "forbidden-runtime-injection"),
HTTPS_PROXY: "http://supplied-proxy.invalid",
SUPPLIED_SECRET: "supplied-secret",
};
await withEnvAsync(
{
AMBIENT_SECRET: "ambient-secret",
OPENCLAW_AMBIENT_SECRET: "ambient-openclaw-secret",
HTTP_PROXY: "http://ambient-proxy.invalid",
NODE_OPTIONS: undefined,
},
async () => {
const expectedWorkerEnv: NodeJS.ProcessEnv = {
HOME: suppliedEnv.HOME,
LANG: suppliedEnv.LANG,
LC_TIME: suppliedEnv.LC_TIME,
NODE_EXTRA_CA_CERTS: suppliedEnv.NODE_EXTRA_CA_CERTS,
NODE_USE_SYSTEM_CA: suppliedEnv.NODE_USE_SYSTEM_CA,
OPENCLAW_ALLOW_INSECURE_PRIVATE_WS: suppliedEnv.OPENCLAW_ALLOW_INSECURE_PRIVATE_WS,
[suppliedPathKey]: suppliedEnv[suppliedPathKey],
};
const supervisor = createNodeWorkerSupervisor({ bundleRoot, env: suppliedEnv });
suppliedEnv.HOME = path.join(root, "mutated-home");
suppliedEnv.LANG = "mutated-locale";
const input = launchInput(workspaceDir, "env-launch", "env");
await supervisor.launch(input);
const completed = await waitForTerminal(supervisor, input.launchId);
const workerEnv = JSON.parse(completed.resultJson ?? "null") as Record<string, string>;
expect(workerEnv).toMatchObject(expectedWorkerEnv);
expect(workerEnv).not.toHaveProperty("AMBIENT_SECRET");
expect(workerEnv).not.toHaveProperty("OPENCLAW_AMBIENT_SECRET");
expect(workerEnv).not.toHaveProperty("OPENCLAW_STATE_DIR");
expect(workerEnv).not.toHaveProperty("OPENCLAW_SUPPLIED_SECRET");
expect(workerEnv).not.toHaveProperty("NODE_OPTIONS");
expect(workerEnv).not.toHaveProperty("BASH_ENV");
expect(workerEnv).not.toHaveProperty("DYLD_INSERT_LIBRARIES");
expect(workerEnv).not.toHaveProperty("HTTP_PROXY");
expect(workerEnv).not.toHaveProperty("HTTPS_PROXY");
expect(workerEnv).not.toHaveProperty("SUPPLIED_SECRET");
expect(JSON.stringify(workerEnv)).not.toContain(TEST_WORKER_CREDENTIAL);
const platformInjectedKeys =
process.platform === "darwin" ? ["__CF_USER_TEXT_ENCODING"] : [];
expect(Object.keys(workerEnv).toSorted()).toEqual(
[...Object.keys(expectedWorkerEnv), ...platformInjectedKeys]
.filter(
(key) => expectedWorkerEnv[key] !== undefined || platformInjectedKeys.includes(key),
)
.toSorted(),
);
await supervisor.close();
},
);
});
it("bounds output and scrubs launch credentials after registry eviction", async () => {
const { supervisor, workspaceDir } = fixture();
const successInput = launchInput(workspaceDir, "secret-success-launch", "secret-success");
const failureInput = launchInput(workspaceDir, "failure-launch", "secret-fail");
const overflowInput = launchInput(workspaceDir, "overflow-launch", "overflow");
await supervisor.launch(successInput);
await supervisor.launch(failureInput);
await supervisor.launch(overflowInput);
for (let index = 0; index < 600; index += 1) {
registerSecretValueForRedaction(`eviction-secret-${index}`);
}
const success = await waitForTerminal(supervisor, successInput.launchId);
const failure = await waitForTerminal(supervisor, failureInput.launchId);
const overflow = await waitForTerminal(supervisor, overflowInput.launchId);
const representations = [
TEST_WORKER_CREDENTIAL,
encodeURIComponent(TEST_WORKER_CREDENTIAL),
JSON.stringify(TEST_WORKER_CREDENTIAL).slice(1, -1),
];
expect(success.state).toBe("completed");
expect(JSON.parse(success.resultJson ?? "null")).toEqual({
raw: "[REDACTED]",
encoded: "[REDACTED]",
status: "completed",
});
expect(failure.state).toBe("failed");
expect(Buffer.byteLength(failure.errorText ?? "", "utf8")).toBeLessThanOrEqual(4 * 1024);
for (const representation of representations) {
expect(success.resultJson).not.toContain(representation);
expect(failure.errorText).not.toContain(representation);
}
expect(overflow).toMatchObject({
state: "failed",
errorText: expect.stringContaining("stdout exceeded 65536 bytes"),
});
await supervisor.close();
});
it.each([
["raw", "secret-cutoff-raw", TEST_WORKER_CREDENTIAL],
["URL", "secret-cutoff-url", encodeURIComponent(TEST_WORKER_CREDENTIAL)],
["JSON-escaped", "secret-cutoff-json", JSON.stringify(TEST_WORKER_CREDENTIAL).slice(1, -1)],
])(
"redacts a %s credential representation across the stderr cutoff",
async (_, prompt, representation) => {
const { supervisor, workspaceDir } = fixture();
const input = launchInput(workspaceDir, `cutoff-${prompt}`, prompt);
await supervisor.launch(input);
const failure = await waitForTerminal(supervisor, input.launchId);
expect(failure.state).toBe("failed");
expect(Buffer.byteLength(failure.errorText ?? "", "utf8")).toBeLessThanOrEqual(4 * 1024);
expect(failure.errorText).not.toContain(representation);
expect(failure.errorText).not.toContain(representation.slice(-8));
await supervisor.close();
},
);
it("does not open or signal a child after markRunning observes its terminal receipt", async () => {
const { supervisor, workspaceDir } = fixture();
const input = launchInput(workspaceDir, "fast-terminal-launch", "fast-terminal");
vi.spyOn(NodeWorkerLaunchStore.prototype, "markRunning").mockImplementation(
function (this: NodeWorkerLaunchStore, params) {
return this.finish({
launchId: params.launchId,
planHash: params.planHash,
supervisor: params.supervisor,
worker: null,
state: "completed",
resultJson: '{"status":"completed"}',
});
},
);
expect(await supervisor.launch(input)).toMatchObject({ state: "completed" });
const marker = path.join(workspaceDir, "fast-terminal-marker");
await new Promise((resolve) => {
setTimeout(resolve, 150);
});
expect(fs.existsSync(marker)).toBe(false);
await supervisor.close();
});
it("records a gated child that exits before journal readiness as terminal", async () => {
const { supervisor, workspaceDir } = fixture();
const input = launchInput(workspaceDir, "prestart-exit-launch", "exit-before-start");
const exitedPath = path.join(workspaceDir, "prestart-exited");
await supervisor.launch(input);
const terminal = await waitForTerminal(supervisor, input.launchId);
expect(fs.existsSync(exitedPath)).toBe(true);
expect(terminal.state).toBe("failed");
await supervisor.close();
});
it.each([
["cancel", "cancelled"],
["close", "interrupted"],
] as const)("records %s while awaiting the owned child", async (operation, state) => {
const { supervisor, workspaceDir } = fixture();
const input = launchInput(workspaceDir, `${operation}-launch`, "wait");
expect(await supervisor.launch(input)).toMatchObject({ state: "running" });
if (operation === "cancel") {
await supervisor.cancel(input.launchId);
} else {
await supervisor.close();
}
expect(await supervisor.status(input.launchId)).toMatchObject({
state,
worker: { pid: expect.any(Number), startTime: expect.any(Number) },
});
await supervisor.close();
});
it.each([
["cancel", "cancelled"],
["close", "interrupted"],
] as const)(
"%s during startup closes the gate before worker code runs",
async (operation, state) => {
const { supervisor, workspaceDir } = fixture();
const input = launchInput(workspaceDir, `${operation}-startup-launch`, "tree");
const originalMarkRunning = Object.getOwnPropertyDescriptor(
NodeWorkerLaunchStore.prototype,
"markRunning",
)?.value as NodeWorkerLaunchStore["markRunning"];
let stopping: Promise<unknown> | undefined;
vi.spyOn(NodeWorkerLaunchStore.prototype, "markRunning").mockImplementation(
function (this: NodeWorkerLaunchStore, params) {
const receipt = Reflect.apply(originalMarkRunning, this, [params]);
stopping =
operation === "cancel" ? supervisor.cancel(input.launchId) : supervisor.close();
return receipt;
},
);
await supervisor.launch(input);
await stopping;
expect((await supervisor.status(input.launchId))?.state).toBe(state);
expect(fs.existsSync(path.join(workspaceDir, "grandchild.pid"))).toBe(false);
await supervisor.close();
},
);
it.each([
["cancel", "cancelled"],
["close", "interrupted"],
] as const)("%s terminates the worker-owned grandchild", async (operation, state) => {
const { supervisor, workspaceDir } = fixture();
const input = launchInput(workspaceDir, `${operation}-tree-launch`, "tree");
const running = await supervisor.launch(input);
expect(running.state).toBe("running");
const grandchildPath = path.join(workspaceDir, "grandchild.pid");
await vi.waitFor(() => expect(fs.existsSync(grandchildPath)).toBe(true));
const grandchildPid = Number(fs.readFileSync(grandchildPath, "utf8"));
const grandchild = requireNodeWorkerProcessIdentity(grandchildPid);
expect(inspectNodeWorkerProcessIdentity(grandchild)).toBe("live");
if (operation === "cancel") {
await supervisor.cancel(input.launchId);
} else {
await supervisor.close();
}
const terminal = await supervisor.status(input.launchId);
expect(terminal).toMatchObject({ state, worker: running.worker });
await vi.waitFor(() => {
expect(inspectNodeWorkerProcessIdentity(running.worker!)).not.toBe("live");
expect(inspectNodeWorkerProcessIdentity(grandchild)).not.toBe("live");
});
await supervisor.close();
});
it("fails closed when the bundle entry resolves outside its namespaced bundle", async () => {
const { bundleRoot, root, supervisor, workspaceDir } = fixture();
const escapedHash = "b".repeat(64);
const escapedBundle = path.join(bundleRoot, "gateway-1", "bundles", escapedHash);
const outsideEntry = path.join(root, "outside.mjs");
fs.mkdirSync(escapedBundle, { recursive: true });
fs.writeFileSync(outsideEntry, TEST_WORKER_SOURCE);
fs.symlinkSync(outsideEntry, path.join(escapedBundle, "openclaw.mjs"));
const input = launchInput(workspaceDir, "escaped-entry");
input.bundleHash = escapedHash;
input.descriptor.admission.handshake.bundleHash = escapedHash;
expect(await supervisor.launch(input)).toMatchObject({
state: "failed",
errorText: expect.stringContaining("inside its bundle"),
});
await supervisor.close();
});
});
+692
View File
@@ -0,0 +1,692 @@
import { createHash } from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import { stableStringify } from "@openclaw/normalization-core";
import { resolveStateDir } from "../config/paths.js";
import { formatErrorMessage } from "../infra/errors.js";
import { isPathInside } from "../infra/path-guards.js";
import { redactToolPayloadText } from "../logging/redact.js";
import {
redactRegisteredSecretValues,
registerSecretValueForRedaction,
} from "../logging/secret-redaction-registry.js";
import {
appendCapturedOutput,
createCapturedOutputBuffers,
finalizeCapturedOutput,
} from "../process/exec-output.js";
import { signalProcessTree } from "../process/kill-tree.js";
import { createChildAdapter } from "../process/supervisor/adapters/child.js";
import { truncateUtf8Suffix } from "../utils/utf8-truncate.js";
import {
parseWorkerLaunchDescriptor,
type WorkerLaunchDescriptor,
} from "../worker/launch-descriptor.js";
import { snapshotNodeWorkerEnv } from "./node-worker-environment.js";
import {
NodeWorkerLaunchStore,
type NodeWorkerLaunchReceipt,
type NodeWorkerTerminalState,
} from "./node-worker-launch-store.js";
import {
inspectNodeWorkerProcessIdentity,
requireNodeWorkerProcessIdentity,
type NodeWorkerProcessIdentity,
} from "./node-worker-process-identity.js";
const STDOUT_MAX_BYTES = 64 * 1024;
const STDERR_MAX_BYTES = 4 * 1024;
const STOP_GRACE_MS = 1_000;
const FORCE_STOP_WAIT_MS = 4_000;
const RECOVERY_POLL_MS = 25;
const GATEWAY_NAMESPACE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u;
const BUNDLE_HASH_PATTERN = /^[a-f0-9]{64}$/u;
type NodeWorkerLaunchInput = {
launchId: string;
gatewayNamespace: string;
bundleHash: string;
placementGeneration: number;
descriptor: WorkerLaunchDescriptor;
};
type ChildAdapter = Awaited<ReturnType<typeof createChildAdapter>>;
type StopState = Extract<NodeWorkerTerminalState, "cancelled" | "interrupted">;
type OwnedTreeState = "live" | "dead" | "unknown";
type CredentialScrubber = {
maxRepresentationBytes: number;
scrub: (text: string) => string;
};
type ActiveBase = {
launchId: string;
planHash: string;
supervisor: NodeWorkerProcessIdentity;
worker: NodeWorkerProcessIdentity;
};
type RunningChild = ActiveBase & {
state: "running";
adapter: ChildAdapter;
done: Promise<void>;
journalReady: Promise<void>;
releaseJournal: () => void;
scrubber: CredentialScrubber;
stopState?: StopState;
};
type TerminalOutcome = Readonly<{
state: NodeWorkerTerminalState;
resultJson?: string;
errorText?: string;
}>;
type ObservedTerminal = ActiveBase & {
state: "observed";
outcome: TerminalOutcome;
persistenceError?: unknown;
};
type ActiveOwnership = RunningChild | ObservedTerminal;
function nodeWorkerPlanHash(params: {
bundleHash: string;
descriptor: WorkerLaunchDescriptor;
gatewayNamespace: string;
placementGeneration: number;
}): string {
return createHash("sha256").update(stableStringify(params)).digest("hex");
}
function resolveWorkerEntry(params: {
bundleRoot: string;
bundleHash: string;
gatewayNamespace: string;
}): string {
const root = fs.realpathSync.native(params.bundleRoot);
const bundle = fs.realpathSync.native(
path.join(root, params.gatewayNamespace, "bundles", params.bundleHash),
);
if (!isPathInside(root, bundle)) {
throw new Error("node worker bundle resolves outside its configured root");
}
const entry = fs.realpathSync.native(path.join(bundle, "openclaw.mjs"));
if (!isPathInside(bundle, entry) || !fs.statSync(entry).isFile()) {
throw new Error("node worker entry must be a regular file inside its bundle");
}
return entry;
}
function createCredentialScrubber(credential: string): CredentialScrubber {
const representations = new Set([
credential,
encodeURIComponent(credential),
JSON.stringify(credential).slice(1, -1),
]);
const ordered = [...representations].toSorted((left, right) => right.length - left.length);
return {
maxRepresentationBytes: Math.max(
...ordered.map((representation) => Buffer.byteLength(representation, "utf8")),
),
scrub: (text) => {
let scrubbed = text;
for (const representation of ordered) {
scrubbed = scrubbed.replaceAll(representation, "[REDACTED]");
}
return scrubbed;
},
};
}
function redactLaunchText(value: string, scrubCredential: (text: string) => string): string {
const launchRedacted = scrubCredential(value);
const exactRedacted = redactRegisteredSecretValues(launchRedacted, () => "[REDACTED]");
return redactToolPayloadText(exactRedacted);
}
function sanitizeDiagnostic(
value: string,
fallback: string,
scrubCredential: (text: string) => string,
): string {
const oneLine = redactLaunchText(value, scrubCredential).replace(/\s+/gu, " ").trim();
return truncateUtf8Suffix(oneLine || fallback, STDERR_MAX_BYTES);
}
function successfulResult(
stdout: ReturnType<typeof createCapturedOutputBuffers>,
scrubCredential: (text: string) => string,
): string {
if (stdout.truncatedBytes > 0) {
throw new Error(`worker stdout exceeded ${STDOUT_MAX_BYTES} bytes`);
}
const raw = finalizeCapturedOutput(stdout, "head", true).toString("utf8").trim();
const redacted = redactLaunchText(raw, scrubCredential);
let parsed: unknown;
try {
parsed = JSON.parse(redacted) as unknown;
} catch (error) {
throw new Error("worker returned invalid JSON output", { cause: error });
}
const result = JSON.stringify(parsed);
if (Buffer.byteLength(result, "utf8") > STDOUT_MAX_BYTES) {
throw new Error(`worker result exceeded ${STDOUT_MAX_BYTES} bytes`);
}
return result;
}
function inspectPosixProcessGroup(pid: number): OwnedTreeState {
try {
process.kill(-pid, 0);
return "live";
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
return code === "ESRCH" ? "dead" : "unknown";
}
}
function inspectOwnedWorkerTree(worker: NodeWorkerProcessIdentity): OwnedTreeState {
const root = inspectNodeWorkerProcessIdentity(worker);
if (root === "reused") {
return "dead";
}
if (root === "live") {
return "live";
}
if (root === "unknown") {
return "unknown";
}
return process.platform === "win32" ? "dead" : inspectPosixProcessGroup(worker.pid);
}
async function signalOwnedWorkerTree(
worker: NodeWorkerProcessIdentity,
signal: "SIGTERM" | "SIGKILL",
): Promise<void> {
const root = inspectNodeWorkerProcessIdentity(worker);
if (root === "reused" || root === "unknown") {
return;
}
await new Promise<void>((resolve) => {
signalProcessTree(worker.pid, signal, { detached: true, onComplete: resolve });
});
}
async function waitForOwnedWorkerTreeDeath(
worker: NodeWorkerProcessIdentity,
timeoutMs: number,
): Promise<OwnedTreeState> {
const deadline = Date.now() + timeoutMs;
let state = inspectOwnedWorkerTree(worker);
while (state === "live" && Date.now() < deadline) {
await delay(RECOVERY_POLL_MS);
state = inspectOwnedWorkerTree(worker);
}
return state;
}
/** Owns worker process groups, lifetime gates, and the durable node-host launch journal. */
class NodeWorkerSupervisor {
private readonly active = new Map<string, ActiveOwnership>();
private readonly starting = new Map<string, Promise<NodeWorkerLaunchReceipt>>();
private readonly bundleRoot: string;
private readonly store: NodeWorkerLaunchStore;
private readonly workerEnv: NodeJS.ProcessEnv;
private supervisorIdentity?: NodeWorkerProcessIdentity;
private closed = false;
private closePromise?: Promise<void>;
constructor(options: { bundleRoot?: string; env?: NodeJS.ProcessEnv } = {}) {
const env = options.env ?? process.env;
this.bundleRoot = path.resolve(
options.bundleRoot ?? path.join(resolveStateDir(env), "node-host"),
);
this.store = new NodeWorkerLaunchStore({ env });
this.workerEnv = snapshotNodeWorkerEnv(env);
}
private requireSupervisorIdentity(): NodeWorkerProcessIdentity {
return (this.supervisorIdentity ??= requireNodeWorkerProcessIdentity(process.pid));
}
async launch(input: NodeWorkerLaunchInput): Promise<NodeWorkerLaunchReceipt> {
if (!GATEWAY_NAMESPACE_PATTERN.test(input.gatewayNamespace)) {
throw new Error("gateway namespace must be a safe bounded path component");
}
if (!BUNDLE_HASH_PATTERN.test(input.bundleHash)) {
throw new Error("node worker bundle hash must be 64 lowercase hexadecimal characters");
}
if (!Number.isSafeInteger(input.placementGeneration) || input.placementGeneration < 0) {
throw new Error("node worker placement generation must be a non-negative safe integer");
}
const descriptor = parseWorkerLaunchDescriptor(structuredClone(input.descriptor));
if (descriptor.admission.handshake.bundleHash !== input.bundleHash) {
throw new Error("node worker descriptor bundle hash does not match the launch bundle");
}
const planHash = nodeWorkerPlanHash({
bundleHash: input.bundleHash,
descriptor,
gatewayNamespace: input.gatewayNamespace,
placementGeneration: input.placementGeneration,
});
const local = this.active.get(input.launchId);
if (local) {
if (local.planHash !== planHash) {
throw new Error(`node worker launch ${input.launchId} was replayed with a different plan`);
}
if (local.state === "observed") {
return this.reconcileActiveTerminal(local);
}
const receipt = this.store.get(input.launchId);
if (receipt) {
return receipt;
}
}
if (this.closed) {
throw new Error("node worker supervisor is closed");
}
const supervisor = this.requireSupervisorIdentity();
const claim = this.store.claim(
{
launchId: input.launchId,
planHash,
gatewayNamespace: input.gatewayNamespace,
environmentId: descriptor.admission.environmentId,
sessionId: descriptor.admission.sessionId,
ownerEpoch: descriptor.admission.ownerEpoch,
placementGeneration: input.placementGeneration,
runId: descriptor.assignment.runId,
},
supervisor,
);
if (claim.action === "recover") {
return await this.recoverRunning(claim.receipt);
}
if (claim.action === "replay") {
const replay = this.active.get(input.launchId);
if (replay?.planHash === planHash && replay.state === "observed") {
return this.reconcileActiveTerminal(replay);
}
const startup = this.starting.get(input.launchId);
return startup && claim.receipt.state === "pending" ? await startup : claim.receipt;
}
const startup = this.startClaimed({ input, descriptor, planHash, supervisor });
this.starting.set(input.launchId, startup);
try {
return await startup;
} finally {
if (this.starting.get(input.launchId) === startup) {
this.starting.delete(input.launchId);
}
}
}
async status(launchId: string): Promise<NodeWorkerLaunchReceipt | undefined> {
const active = this.active.get(launchId);
if (active?.state === "observed") {
return this.reconcileActiveTerminal(active);
}
return this.store.get(launchId);
}
async cancel(launchId: string): Promise<NodeWorkerLaunchReceipt | undefined> {
const active = this.active.get(launchId);
if (active) {
if (active.state === "running") {
await this.stopChild(active, "cancelled");
}
const observed = this.active.get(launchId);
if (observed?.state === "observed") {
return this.reconcileActiveTerminal(observed);
}
return this.store.get(launchId);
}
const startup = this.starting.get(launchId);
const receipt = this.store.get(launchId);
if (!receipt || receipt.state === "completed" || receipt.state === "failed") {
return receipt;
}
if (receipt.state === "interrupted" || receipt.state === "cancelled") {
return receipt;
}
if (!startup || receipt.state !== "pending" || receipt.supervisor.pid !== process.pid) {
return receipt;
}
const cancelled = this.store.finish({
launchId,
planHash: receipt.planHash,
supervisor: this.requireSupervisorIdentity(),
worker: null,
state: "cancelled",
errorText: "node worker launch cancelled",
});
await startup;
return this.store.get(launchId) ?? cancelled;
}
close(): Promise<void> {
if (this.closePromise) {
return this.closePromise;
}
this.closed = true;
const operation = (async () => {
await Promise.allSettled(this.starting.values());
await Promise.all(
[...this.active.values()]
.filter((active): active is RunningChild => active.state === "running")
.map(async (active) => await this.stopChild(active, "interrupted")),
);
const errors: unknown[] = [];
for (const active of this.active.values()) {
if (active.state !== "observed") {
continue;
}
try {
this.reconcileActiveTerminal(active);
} catch (error) {
errors.push(error);
}
}
if (errors.length === 1) {
throw errors[0];
}
if (errors.length > 1) {
throw new AggregateError(errors, "node worker terminal reconciliation failed");
}
})();
const closePromise = operation.finally(() => {
if (this.closePromise === closePromise) {
this.closePromise = undefined;
}
});
this.closePromise = closePromise;
return closePromise;
}
private reconcileActiveTerminal(active: ObservedTerminal): NodeWorkerLaunchReceipt {
try {
const receipt = this.store.finish({
launchId: active.launchId,
planHash: active.planHash,
supervisor: active.supervisor,
worker: active.worker,
...active.outcome,
});
if (receipt.state === "pending" || receipt.state === "running") {
throw new Error(`node worker launch ${active.launchId} terminal state was not persisted`);
}
if (this.active.get(active.launchId) === active) {
this.active.delete(active.launchId);
}
return receipt;
} catch (error) {
active.persistenceError = error;
throw error;
}
}
private async recoverRunning(receipt: NodeWorkerLaunchReceipt): Promise<NodeWorkerLaunchReceipt> {
if (receipt.state !== "running" || !receipt.worker) {
return receipt;
}
const previousSupervisor = inspectNodeWorkerProcessIdentity(receipt.supervisor);
if (previousSupervisor !== "dead" && previousSupervisor !== "reused") {
return this.store.get(receipt.launchId) ?? receipt;
}
let workerState = inspectOwnedWorkerTree(receipt.worker);
if (workerState === "unknown") {
return this.store.get(receipt.launchId) ?? receipt;
}
if (workerState === "live") {
await signalOwnedWorkerTree(receipt.worker, "SIGTERM");
workerState = await waitForOwnedWorkerTreeDeath(receipt.worker, STOP_GRACE_MS);
}
if (workerState === "live") {
await signalOwnedWorkerTree(receipt.worker, "SIGKILL");
workerState = await waitForOwnedWorkerTreeDeath(receipt.worker, FORCE_STOP_WAIT_MS);
}
if (workerState !== "dead") {
return this.store.get(receipt.launchId) ?? receipt;
}
return this.store.finish({
launchId: receipt.launchId,
planHash: receipt.planHash,
supervisor: receipt.supervisor,
worker: receipt.worker,
state: "interrupted",
errorText: "node host stopped before the worker launch completed",
});
}
private async startClaimed(params: {
input: NodeWorkerLaunchInput;
descriptor: WorkerLaunchDescriptor;
planHash: string;
supervisor: NodeWorkerProcessIdentity;
}): Promise<NodeWorkerLaunchReceipt> {
const credential = params.descriptor.admission.credential;
const scrubber = createCredentialScrubber(credential);
registerSecretValueForRedaction(credential);
let adapter: ChildAdapter;
try {
const entry = resolveWorkerEntry({
bundleRoot: this.bundleRoot,
bundleHash: params.input.bundleHash,
gatewayNamespace: params.input.gatewayNamespace,
});
adapter = await createChildAdapter({
argv: [process.execPath, entry, "worker", "--internal-worker-ipc"],
env: this.workerEnv,
exactEnv: true,
ownedWorker: true,
input: JSON.stringify(params.descriptor),
});
} catch (error) {
return this.store.finish({
launchId: params.input.launchId,
planHash: params.planHash,
supervisor: params.supervisor,
worker: null,
state: "failed",
errorText: sanitizeDiagnostic(
formatErrorMessage(error),
"node worker spawn failed",
scrubber.scrub,
),
});
}
if (!adapter.pid) {
adapter.kill("SIGKILL");
adapter.dispose();
return this.store.finish({
launchId: params.input.launchId,
planHash: params.planHash,
supervisor: params.supervisor,
worker: null,
state: "failed",
errorText: "node worker spawn did not return a process id",
});
}
let worker: NodeWorkerProcessIdentity;
try {
worker = requireNodeWorkerProcessIdentity(adapter.pid);
} catch (error) {
adapter.kill("SIGKILL");
await adapter.wait().catch(() => undefined);
adapter.dispose();
return this.store.finish({
launchId: params.input.launchId,
planHash: params.planHash,
supervisor: params.supervisor,
worker: null,
state: "failed",
errorText: sanitizeDiagnostic(
formatErrorMessage(error),
"node worker process identity unavailable",
scrubber.scrub,
),
});
}
let journalReleased = false;
let releaseJournalPromise!: () => void;
const journalReady = new Promise<void>((resolve) => {
releaseJournalPromise = resolve;
});
const releaseJournal = () => {
if (!journalReleased) {
journalReleased = true;
releaseJournalPromise();
}
};
const active = {
state: "running",
adapter,
journalReady,
launchId: params.input.launchId,
planHash: params.planHash,
releaseJournal,
scrubber,
supervisor: params.supervisor,
worker,
} as RunningChild;
active.done = this.observeChild(active);
this.active.set(active.launchId, active);
void active.done.catch(() => undefined);
let running: NodeWorkerLaunchReceipt;
try {
running = this.store.markRunning({
launchId: active.launchId,
planHash: active.planHash,
supervisor: params.supervisor,
worker,
});
} catch (error) {
active.releaseJournal();
await this.stopChild(active, "interrupted").catch(() => undefined);
throw error;
}
active.releaseJournal();
if (running.state === "cancelled" || running.state === "interrupted") {
await this.stopChild(active, running.state);
return this.store.get(active.launchId) ?? running;
}
if (running.state !== "running") {
adapter.closeStartGate?.();
return running;
}
if (this.closed) {
await this.stopChild(active, "interrupted");
return this.store.get(active.launchId) ?? running;
}
try {
await adapter.openStartGate?.();
} catch {
await this.stopChild(active, "interrupted");
return this.store.get(active.launchId) ?? running;
}
return running;
}
private async observeChild(active: RunningChild): Promise<void> {
const stdout = createCapturedOutputBuffers();
const stderr = createCapturedOutputBuffers();
active.adapter.onStdout((chunk) =>
appendCapturedOutput(stdout, chunk, STDOUT_MAX_BYTES, "head"),
);
active.adapter.onStderr((chunk) =>
appendCapturedOutput(
stderr,
chunk,
STDERR_MAX_BYTES + active.scrubber.maxRepresentationBytes,
"tail",
),
);
let outcome: TerminalOutcome;
try {
const exit = await active.adapter.wait();
await active.journalReady;
if (active.stopState) {
outcome = Object.freeze({
state: active.stopState,
errorText:
active.stopState === "cancelled"
? "node worker launch cancelled"
: "node worker launch interrupted during node-host shutdown",
});
} else if (exit.code === 0 && exit.signal === null) {
try {
outcome = Object.freeze({
state: "completed",
resultJson: successfulResult(stdout, active.scrubber.scrub),
});
} catch (error) {
outcome = Object.freeze({
state: "failed",
errorText: sanitizeDiagnostic(
formatErrorMessage(error),
"invalid worker result",
active.scrubber.scrub,
),
});
}
} else {
const detail = finalizeCapturedOutput(stderr, "tail", true).toString("utf8");
const exitLabel = exit.signal ? `signal ${exit.signal}` : `exit code ${String(exit.code)}`;
outcome = Object.freeze({
state: "failed",
errorText: sanitizeDiagnostic(
`node worker failed with ${exitLabel}${detail ? `: ${detail}` : ""}`,
"node worker failed",
active.scrubber.scrub,
),
});
}
} catch (error) {
await active.journalReady;
outcome = Object.freeze({
state: active.stopState ?? "failed",
errorText: sanitizeDiagnostic(
formatErrorMessage(error),
"node worker wait failed",
active.scrubber.scrub,
),
});
} finally {
active.adapter.dispose();
}
const observed: ObservedTerminal = {
state: "observed",
launchId: active.launchId,
planHash: active.planHash,
supervisor: active.supervisor,
worker: active.worker,
outcome,
};
if (this.active.get(active.launchId) !== active) {
return;
}
this.active.set(active.launchId, observed);
try {
this.reconcileActiveTerminal(observed);
} catch {
// The observed outcome stays owned in memory for the next supervisor operation.
}
}
private async stopChild(active: RunningChild, state: StopState): Promise<void> {
active.stopState ??= state;
active.adapter.kill("SIGTERM");
const forceKill = setTimeout(() => active.adapter.kill("SIGKILL"), STOP_GRACE_MS);
forceKill.unref?.();
try {
await active.done;
} finally {
clearTimeout(forceKill);
}
}
}
export function createNodeWorkerSupervisor(
options: {
bundleRoot?: string;
env?: NodeJS.ProcessEnv;
} = {},
): NodeWorkerSupervisor {
return new NodeWorkerSupervisor(options);
}
+2
View File
@@ -545,6 +545,8 @@ describe("runNodeHost", () => {
await vi.waitFor(() => expect(mocks.capturedGatewayClients[0]?.stop).toHaveBeenCalledOnce());
expect(clearIntervalSpy).not.toHaveBeenCalled();
await vi.waitFor(() => expect(mocks.closeMcpManager).toHaveBeenCalledOnce());
expect(resolveCloseMcp).toBeTypeOf("function");
resolveCloseMcp?.();
await running;
+6
View File
@@ -8,6 +8,7 @@ import { prepareNodeHostRuntime } from "./runtime.js";
const mocks = vi.hoisted(() => ({
closeMcp: vi.fn(async () => undefined),
closeWorkerSupervisor: vi.fn(async () => undefined),
handleInvoke: vi.fn(async () => undefined),
progressStartHeartbeats: vi.fn(),
progressWrite: vi.fn(async () => undefined),
@@ -39,6 +40,10 @@ vi.mock("./node-invoke-progress.js", () => ({
})),
}));
vi.mock("./node-worker-supervisor.js", () => ({
createNodeWorkerSupervisor: vi.fn(() => ({ close: mocks.closeWorkerSupervisor })),
}));
vi.mock("./plugin-node-host.js", () => ({
ensureNodeHostPluginRegistry: vi.fn(async () => undefined),
isRegisteredNodeHostCommandDuplex: vi.fn((command: string) => command === "test.duplex"),
@@ -178,6 +183,7 @@ describe("node-host invocation cancellation", () => {
await runtime.close();
expect(held.signal?.aborted).toBe(true);
expect(mocks.closeWorkerSupervisor).toHaveBeenCalledOnce();
held.release();
await invoking;
});
+3
View File
@@ -26,6 +26,7 @@ import { handleInvoke, type NodeInvokeRequestPayload, type SkillBinsProvider } f
import { startNodeHostMcpManager, type NodeHostMcpManager } from "./mcp.js";
import { buildNodeEventParams } from "./node-event-params.js";
import { createNodeInvokeProgressWriter } from "./node-invoke-progress.js";
import { createNodeWorkerSupervisor } from "./node-worker-supervisor.js";
import {
ensureNodeHostPluginRegistry,
isRegisteredNodeHostCommandDuplex,
@@ -305,6 +306,7 @@ export async function prepareNodeHostRuntime(params?: {
initialInventory,
start({ client, onInventoryChanged, onManifestChanged }) {
const mcpAbort = new AbortController();
const workerSupervisor = createNodeWorkerSupervisor({ env });
const skillBins = new SkillBinsCache(client, pathEnv);
const activeInvokes = new Map<string, ActiveNodeInvoke>();
const pluginCommandContext: OpenClawPluginNodeHostCommandContext = {
@@ -448,6 +450,7 @@ export async function prepareNodeHostRuntime(params?: {
async close() {
this.cancelAll();
stopAvailabilityWatch();
await workerSupervisor.close();
mcpAbort.abort();
const resolved = manager ?? (await startup.catch(() => undefined));
await resolved?.close();