mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-24 11:25:50 -06:00
feat(node-host): supervise durable worker launches (#122829)
* feat(node-host): add worker launch supervision * fix(node-host): harden worker lifecycle ownership * fix(node-host): harden worker execution boundary * fix(node-host): preserve worker trust settings * chore(plugin-sdk): refresh worker lifecycle baselines * docs(plan): track runner implementation slices * test(node-host): await runtime shutdown owner
This commit is contained in:
committed by
GitHub
parent
93f5e0f1f6
commit
6b9bea84f0
@@ -0,0 +1,51 @@
|
||||
const POSIX_WORKER_ENV_KEYS = new Set([
|
||||
"PATH",
|
||||
"HOME",
|
||||
"TMPDIR",
|
||||
"TMP",
|
||||
"TEMP",
|
||||
"LANG",
|
||||
"LANGUAGE",
|
||||
"TZ",
|
||||
"NODE_EXTRA_CA_CERTS",
|
||||
"NODE_USE_SYSTEM_CA",
|
||||
"OPENCLAW_ALLOW_INSECURE_PRIVATE_WS",
|
||||
]);
|
||||
const WINDOWS_WORKER_ENV_KEYS = new Set([
|
||||
...POSIX_WORKER_ENV_KEYS,
|
||||
"USERPROFILE",
|
||||
"HOMEDRIVE",
|
||||
"HOMEPATH",
|
||||
"SYSTEMROOT",
|
||||
"WINDIR",
|
||||
"COMSPEC",
|
||||
"PATHEXT",
|
||||
]);
|
||||
|
||||
/** Freeze the minimal non-secret environment inherited by node-host workers. */
|
||||
export function snapshotNodeWorkerEnv(source: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
|
||||
const windows = process.platform === "win32";
|
||||
const snapshot: NodeJS.ProcessEnv = {};
|
||||
const retainedWindowsKeys = new Map<string, string>();
|
||||
for (const [key, value] of Object.entries(source)) {
|
||||
if (value === undefined) {
|
||||
continue;
|
||||
}
|
||||
const normalized = windows ? key.toUpperCase() : key;
|
||||
const allowed =
|
||||
(windows ? WINDOWS_WORKER_ENV_KEYS : POSIX_WORKER_ENV_KEYS).has(normalized) ||
|
||||
normalized.startsWith("LC_");
|
||||
if (!allowed) {
|
||||
continue;
|
||||
}
|
||||
if (windows) {
|
||||
const previousKey = retainedWindowsKeys.get(normalized);
|
||||
if (previousKey) {
|
||||
delete snapshot[previousKey];
|
||||
}
|
||||
retainedWindowsKeys.set(normalized, key);
|
||||
}
|
||||
snapshot[key] = value;
|
||||
}
|
||||
return snapshot;
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type { Selectable } from "kysely";
|
||||
import {
|
||||
executeSqliteQuerySync,
|
||||
executeSqliteQueryTakeFirstSync,
|
||||
getNodeSqliteKysely,
|
||||
} from "../infra/kysely-sync.js";
|
||||
import type { DB as OpenClawStateDatabase } from "../state/openclaw-state-db.generated.js";
|
||||
import {
|
||||
runOpenClawStateWriteTransaction,
|
||||
type OpenClawStateDatabaseOptions,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import { OPENCLAW_STATE_SCHEMA_SQL } from "../state/openclaw-state-schema.js";
|
||||
import {
|
||||
inspectNodeWorkerProcessIdentity,
|
||||
type NodeWorkerProcessIdentity,
|
||||
} from "./node-worker-process-identity.js";
|
||||
|
||||
type NodeWorkerLaunchState =
|
||||
| "pending"
|
||||
| "running"
|
||||
| "completed"
|
||||
| "failed"
|
||||
| "interrupted"
|
||||
| "cancelled";
|
||||
export type NodeWorkerTerminalState = Exclude<NodeWorkerLaunchState, "pending" | "running">;
|
||||
|
||||
type NodeWorkerLaunchDatabase = Pick<OpenClawStateDatabase, "node_worker_launches">;
|
||||
type NodeWorkerLaunchRow = Selectable<NodeWorkerLaunchDatabase["node_worker_launches"]>;
|
||||
|
||||
export type NodeWorkerLaunchReceipt = {
|
||||
launchId: string;
|
||||
planHash: string;
|
||||
gatewayNamespace: string;
|
||||
environmentId: string;
|
||||
sessionId: string;
|
||||
ownerEpoch: number;
|
||||
placementGeneration: number;
|
||||
runId: string;
|
||||
state: NodeWorkerLaunchState;
|
||||
supervisor: NodeWorkerProcessIdentity;
|
||||
worker: NodeWorkerProcessIdentity | null;
|
||||
resultJson: string | null;
|
||||
errorText: string | null;
|
||||
completedAtMs: number | null;
|
||||
createdAtMs: number;
|
||||
updatedAtMs: number;
|
||||
};
|
||||
|
||||
type NodeWorkerLaunchClaim = Pick<
|
||||
NodeWorkerLaunchReceipt,
|
||||
| "environmentId"
|
||||
| "gatewayNamespace"
|
||||
| "launchId"
|
||||
| "ownerEpoch"
|
||||
| "placementGeneration"
|
||||
| "planHash"
|
||||
| "runId"
|
||||
| "sessionId"
|
||||
>;
|
||||
|
||||
type NodeWorkerLaunchClaimResult = {
|
||||
action: "start" | "replay" | "recover";
|
||||
receipt: NodeWorkerLaunchReceipt;
|
||||
};
|
||||
|
||||
const NODE_WORKER_LAUNCH_SCHEMA_START = "CREATE TABLE IF NOT EXISTS node_worker_launches (";
|
||||
const NODE_WORKER_LAUNCH_SCHEMA_END = "\n) STRICT;";
|
||||
const initializedDatabases = new WeakSet<DatabaseSync>();
|
||||
const TERMINAL_STATES: ReadonlySet<string> = new Set([
|
||||
"completed",
|
||||
"failed",
|
||||
"interrupted",
|
||||
"cancelled",
|
||||
]);
|
||||
|
||||
function ensureNodeWorkerLaunchSchema(database: DatabaseSync): void {
|
||||
const start = OPENCLAW_STATE_SCHEMA_SQL.indexOf(NODE_WORKER_LAUNCH_SCHEMA_START);
|
||||
const end =
|
||||
start >= 0 ? OPENCLAW_STATE_SCHEMA_SQL.indexOf(NODE_WORKER_LAUNCH_SCHEMA_END, start) : -1;
|
||||
if (start < 0 || end < start) {
|
||||
throw new Error("OpenClaw node worker launch schema marker is missing.");
|
||||
}
|
||||
database.exec(OPENCLAW_STATE_SCHEMA_SQL.slice(start, end + NODE_WORKER_LAUNCH_SCHEMA_END.length)); // sqlite-allow-raw -- Canonical feature-local additive DDL only.
|
||||
}
|
||||
|
||||
function query(database: DatabaseSync) {
|
||||
return getNodeSqliteKysely<NodeWorkerLaunchDatabase>(database);
|
||||
}
|
||||
|
||||
function readRow(database: DatabaseSync, launchId: string): NodeWorkerLaunchRow | undefined {
|
||||
return executeSqliteQueryTakeFirstSync(
|
||||
database,
|
||||
query(database)
|
||||
.selectFrom("node_worker_launches")
|
||||
.selectAll()
|
||||
.where("launch_id", "=", launchId),
|
||||
);
|
||||
}
|
||||
|
||||
function processIdentity(pid: number, startTime: number): NodeWorkerProcessIdentity {
|
||||
return { pid, startTime };
|
||||
}
|
||||
|
||||
function receiptFromRow(row: NodeWorkerLaunchRow): NodeWorkerLaunchReceipt {
|
||||
if (!isNodeWorkerLaunchState(row.state)) {
|
||||
throw new Error(`invalid node worker launch state ${row.state}`);
|
||||
}
|
||||
return {
|
||||
launchId: row.launch_id,
|
||||
planHash: row.plan_hash,
|
||||
gatewayNamespace: row.gateway_namespace,
|
||||
environmentId: row.environment_id,
|
||||
sessionId: row.session_id,
|
||||
ownerEpoch: row.owner_epoch,
|
||||
placementGeneration: row.placement_generation,
|
||||
runId: row.run_id,
|
||||
state: row.state,
|
||||
supervisor: processIdentity(row.supervisor_pid, row.supervisor_start_time),
|
||||
worker:
|
||||
row.worker_pid === null || row.worker_start_time === null
|
||||
? null
|
||||
: processIdentity(row.worker_pid, row.worker_start_time),
|
||||
resultJson: row.result_json,
|
||||
errorText: row.error_text,
|
||||
completedAtMs: row.completed_at_ms,
|
||||
createdAtMs: row.created_at_ms,
|
||||
updatedAtMs: row.updated_at_ms,
|
||||
};
|
||||
}
|
||||
|
||||
function isNodeWorkerLaunchState(value: string): value is NodeWorkerLaunchState {
|
||||
return value === "pending" || value === "running" || TERMINAL_STATES.has(value);
|
||||
}
|
||||
|
||||
function validateIdentifier(value: string, label: string): void {
|
||||
if (!value || value.trim() !== value || value.length > 256 || value.includes("\0")) {
|
||||
throw new Error(`${label} must be a bounded non-empty identifier`);
|
||||
}
|
||||
}
|
||||
|
||||
function validatePlanHash(value: string): void {
|
||||
if (!/^[a-f0-9]{64}$/u.test(value)) {
|
||||
throw new Error("node worker plan hash must be 64 lowercase hexadecimal characters");
|
||||
}
|
||||
}
|
||||
|
||||
function validateTimestamp(value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new Error("node worker launch timestamp must be a non-negative safe integer");
|
||||
}
|
||||
}
|
||||
|
||||
function validateProcessIdentity(identity: NodeWorkerProcessIdentity): void {
|
||||
if (
|
||||
!Number.isSafeInteger(identity.pid) ||
|
||||
identity.pid <= 0 ||
|
||||
identity.pid > 2_147_483_647 ||
|
||||
!Number.isSafeInteger(identity.startTime) ||
|
||||
identity.startTime < 0
|
||||
) {
|
||||
throw new Error("node worker process identity must contain a bounded pid and start time");
|
||||
}
|
||||
}
|
||||
|
||||
function requireMatchingRow(
|
||||
database: DatabaseSync,
|
||||
launchId: string,
|
||||
planHash: string,
|
||||
): NodeWorkerLaunchRow {
|
||||
const row = readRow(database, launchId);
|
||||
if (!row) {
|
||||
throw new Error(`node worker launch ${launchId} does not exist`);
|
||||
}
|
||||
if (row.plan_hash !== planHash) {
|
||||
throw new Error(`node worker launch ${launchId} was replayed with a different plan`);
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
function rowHasSupervisor(row: NodeWorkerLaunchRow, identity: NodeWorkerProcessIdentity): boolean {
|
||||
return row.supervisor_pid === identity.pid && row.supervisor_start_time === identity.startTime;
|
||||
}
|
||||
|
||||
function rowHasWorker(
|
||||
row: NodeWorkerLaunchRow,
|
||||
identity: NodeWorkerProcessIdentity | null,
|
||||
): boolean {
|
||||
return identity === null
|
||||
? row.worker_pid === null && row.worker_start_time === null
|
||||
: row.worker_pid === identity.pid && row.worker_start_time === identity.startTime;
|
||||
}
|
||||
|
||||
function sameObservedOwner(current: NodeWorkerLaunchRow, observed: NodeWorkerLaunchRow): boolean {
|
||||
return (
|
||||
current.state === observed.state &&
|
||||
current.supervisor_pid === observed.supervisor_pid &&
|
||||
current.supervisor_start_time === observed.supervisor_start_time &&
|
||||
current.worker_pid === observed.worker_pid &&
|
||||
current.worker_start_time === observed.worker_start_time
|
||||
);
|
||||
}
|
||||
|
||||
/** Synchronous shared-state owner for durable node worker launch supervision. */
|
||||
export class NodeWorkerLaunchStore {
|
||||
private readonly databaseOptions: OpenClawStateDatabaseOptions;
|
||||
|
||||
constructor(options: { env?: NodeJS.ProcessEnv } = {}) {
|
||||
this.databaseOptions = options.env ? { env: options.env } : {};
|
||||
}
|
||||
|
||||
private write<T>(operationLabel: string, operation: (database: DatabaseSync) => T): T {
|
||||
let initializedDatabase: DatabaseSync | undefined;
|
||||
const result = runOpenClawStateWriteTransaction(
|
||||
({ db }) => {
|
||||
if (!initializedDatabases.has(db)) {
|
||||
ensureNodeWorkerLaunchSchema(db);
|
||||
initializedDatabase = db;
|
||||
}
|
||||
return operation(db);
|
||||
},
|
||||
this.databaseOptions,
|
||||
{ operationLabel },
|
||||
);
|
||||
if (initializedDatabase) {
|
||||
initializedDatabases.add(initializedDatabase);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
claim(
|
||||
claim: NodeWorkerLaunchClaim,
|
||||
supervisor: NodeWorkerProcessIdentity,
|
||||
nowMs = Date.now(),
|
||||
): NodeWorkerLaunchClaimResult {
|
||||
validateIdentifier(claim.launchId, "node worker launch id");
|
||||
validatePlanHash(claim.planHash);
|
||||
validateTimestamp(nowMs);
|
||||
validateProcessIdentity(supervisor);
|
||||
|
||||
// Process inspection is intentionally outside SQLite. The second transaction
|
||||
// re-reads the exact owner tuple before an adoption or recovery decision.
|
||||
const observed = this.write("node-worker-launch.claim-inspect", (database) =>
|
||||
readRow(database, claim.launchId),
|
||||
);
|
||||
if (observed && observed.plan_hash !== claim.planHash) {
|
||||
throw new Error(`node worker launch ${claim.launchId} was replayed with a different plan`);
|
||||
}
|
||||
const observedSupervisorState = observed
|
||||
? inspectNodeWorkerProcessIdentity(
|
||||
processIdentity(observed.supervisor_pid, observed.supervisor_start_time),
|
||||
)
|
||||
: undefined;
|
||||
|
||||
return this.write("node-worker-launch.claim", (database) => {
|
||||
let current = readRow(database, claim.launchId);
|
||||
if (!current) {
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
query(database).insertInto("node_worker_launches").values({
|
||||
launch_id: claim.launchId,
|
||||
plan_hash: claim.planHash,
|
||||
gateway_namespace: claim.gatewayNamespace,
|
||||
environment_id: claim.environmentId,
|
||||
session_id: claim.sessionId,
|
||||
owner_epoch: claim.ownerEpoch,
|
||||
placement_generation: claim.placementGeneration,
|
||||
run_id: claim.runId,
|
||||
state: "pending",
|
||||
supervisor_pid: supervisor.pid,
|
||||
supervisor_start_time: supervisor.startTime,
|
||||
worker_pid: null,
|
||||
worker_start_time: null,
|
||||
result_json: null,
|
||||
error_text: null,
|
||||
completed_at_ms: null,
|
||||
created_at_ms: nowMs,
|
||||
updated_at_ms: nowMs,
|
||||
}),
|
||||
);
|
||||
return {
|
||||
action: "start",
|
||||
receipt: receiptFromRow(requireMatchingRow(database, claim.launchId, claim.planHash)),
|
||||
};
|
||||
}
|
||||
if (current.plan_hash !== claim.planHash) {
|
||||
throw new Error(`node worker launch ${claim.launchId} was replayed with a different plan`);
|
||||
}
|
||||
const previousOwnerDefinitelyStale =
|
||||
observedSupervisorState === "dead" || observedSupervisorState === "reused";
|
||||
if (
|
||||
current.state === "pending" &&
|
||||
observed &&
|
||||
sameObservedOwner(current, observed) &&
|
||||
previousOwnerDefinitelyStale
|
||||
) {
|
||||
const updatedAtMs = Math.max(nowMs, current.created_at_ms, current.updated_at_ms);
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
query(database)
|
||||
.updateTable("node_worker_launches")
|
||||
.set({
|
||||
supervisor_pid: supervisor.pid,
|
||||
supervisor_start_time: supervisor.startTime,
|
||||
updated_at_ms: updatedAtMs,
|
||||
})
|
||||
.where("launch_id", "=", claim.launchId)
|
||||
.where("plan_hash", "=", claim.planHash)
|
||||
.where("state", "=", "pending")
|
||||
.where("supervisor_pid", "=", observed.supervisor_pid)
|
||||
.where("supervisor_start_time", "=", observed.supervisor_start_time)
|
||||
.where("worker_pid", "is", null)
|
||||
.where("worker_start_time", "is", null),
|
||||
);
|
||||
current = requireMatchingRow(database, claim.launchId, claim.planHash);
|
||||
return {
|
||||
action: rowHasSupervisor(current, supervisor) ? "start" : "replay",
|
||||
receipt: receiptFromRow(current),
|
||||
};
|
||||
}
|
||||
if (
|
||||
current.state === "running" &&
|
||||
observed &&
|
||||
sameObservedOwner(current, observed) &&
|
||||
previousOwnerDefinitelyStale
|
||||
) {
|
||||
return { action: "recover", receipt: receiptFromRow(current) };
|
||||
}
|
||||
return { action: "replay", receipt: receiptFromRow(current) };
|
||||
});
|
||||
}
|
||||
|
||||
get(launchId: string): NodeWorkerLaunchReceipt | undefined {
|
||||
validateIdentifier(launchId, "node worker launch id");
|
||||
return this.write("node-worker-launch.get", (database) => {
|
||||
const row = readRow(database, launchId);
|
||||
return row ? receiptFromRow(row) : undefined;
|
||||
});
|
||||
}
|
||||
|
||||
markRunning(params: {
|
||||
launchId: string;
|
||||
planHash: string;
|
||||
supervisor: NodeWorkerProcessIdentity;
|
||||
worker: NodeWorkerProcessIdentity;
|
||||
nowMs?: number;
|
||||
}): NodeWorkerLaunchReceipt {
|
||||
const nowMs = params.nowMs ?? Date.now();
|
||||
validateTimestamp(nowMs);
|
||||
validateProcessIdentity(params.supervisor);
|
||||
validateProcessIdentity(params.worker);
|
||||
return this.write("node-worker-launch.mark-running", (database) => {
|
||||
const current = requireMatchingRow(database, params.launchId, params.planHash);
|
||||
if (TERMINAL_STATES.has(current.state)) {
|
||||
return receiptFromRow(current);
|
||||
}
|
||||
if (current.state === "running") {
|
||||
return receiptFromRow(current);
|
||||
}
|
||||
if (!rowHasSupervisor(current, params.supervisor) || !rowHasWorker(current, null)) {
|
||||
return receiptFromRow(current);
|
||||
}
|
||||
const updatedAtMs = Math.max(nowMs, current.created_at_ms, current.updated_at_ms);
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
query(database)
|
||||
.updateTable("node_worker_launches")
|
||||
.set({
|
||||
state: "running",
|
||||
worker_pid: params.worker.pid,
|
||||
worker_start_time: params.worker.startTime,
|
||||
updated_at_ms: updatedAtMs,
|
||||
})
|
||||
.where("launch_id", "=", params.launchId)
|
||||
.where("plan_hash", "=", params.planHash)
|
||||
.where("state", "=", "pending")
|
||||
.where("supervisor_pid", "=", params.supervisor.pid)
|
||||
.where("supervisor_start_time", "=", params.supervisor.startTime)
|
||||
.where("worker_pid", "is", null)
|
||||
.where("worker_start_time", "is", null),
|
||||
);
|
||||
return receiptFromRow(requireMatchingRow(database, params.launchId, params.planHash));
|
||||
});
|
||||
}
|
||||
|
||||
finish(params: {
|
||||
launchId: string;
|
||||
planHash: string;
|
||||
supervisor: NodeWorkerProcessIdentity;
|
||||
worker: NodeWorkerProcessIdentity | null;
|
||||
state: NodeWorkerTerminalState;
|
||||
resultJson?: string;
|
||||
errorText?: string;
|
||||
nowMs?: number;
|
||||
}): NodeWorkerLaunchReceipt {
|
||||
const nowMs = params.nowMs ?? Date.now();
|
||||
validateTimestamp(nowMs);
|
||||
validateProcessIdentity(params.supervisor);
|
||||
if (params.worker) {
|
||||
validateProcessIdentity(params.worker);
|
||||
}
|
||||
return this.write("node-worker-launch.finish", (database) => {
|
||||
const current = requireMatchingRow(database, params.launchId, params.planHash);
|
||||
if (TERMINAL_STATES.has(current.state)) {
|
||||
return receiptFromRow(current);
|
||||
}
|
||||
if (!rowHasSupervisor(current, params.supervisor) || !rowHasWorker(current, params.worker)) {
|
||||
return receiptFromRow(current);
|
||||
}
|
||||
const completedAtMs = Math.max(nowMs, current.created_at_ms, current.updated_at_ms);
|
||||
let update = query(database)
|
||||
.updateTable("node_worker_launches")
|
||||
.set({
|
||||
state: params.state,
|
||||
result_json: params.state === "completed" ? (params.resultJson ?? null) : null,
|
||||
error_text: params.state === "completed" ? null : (params.errorText ?? null),
|
||||
completed_at_ms: completedAtMs,
|
||||
updated_at_ms: completedAtMs,
|
||||
})
|
||||
.where("launch_id", "=", params.launchId)
|
||||
.where("plan_hash", "=", params.planHash)
|
||||
.where("state", "in", ["pending", "running"])
|
||||
.where("supervisor_pid", "=", params.supervisor.pid)
|
||||
.where("supervisor_start_time", "=", params.supervisor.startTime);
|
||||
update = params.worker
|
||||
? update
|
||||
.where("worker_pid", "=", params.worker.pid)
|
||||
.where("worker_start_time", "=", params.worker.startTime)
|
||||
: update.where("worker_pid", "is", null).where("worker_start_time", "is", null);
|
||||
executeSqliteQuerySync(database, update);
|
||||
return receiptFromRow(requireMatchingRow(database, params.launchId, params.planHash));
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import { readWindowsProcessStartTimeSync } from "../infra/windows-port-pids.js";
|
||||
import { getFileLockProcessStartTime, isPidDefinitelyDead } from "../shared/pid-alive.js";
|
||||
|
||||
export type NodeWorkerProcessIdentity = {
|
||||
pid: number;
|
||||
startTime: number;
|
||||
};
|
||||
|
||||
type NodeWorkerProcessIdentityState = "live" | "dead" | "reused" | "unknown";
|
||||
|
||||
function readNodeWorkerProcessStartTime(pid: number): number | null {
|
||||
return process.platform === "win32"
|
||||
? readWindowsProcessStartTimeSync(pid)
|
||||
: getFileLockProcessStartTime(pid);
|
||||
}
|
||||
|
||||
export function requireNodeWorkerProcessIdentity(pid: number): NodeWorkerProcessIdentity {
|
||||
const startTime = readNodeWorkerProcessStartTime(pid);
|
||||
if (startTime === null) {
|
||||
throw new Error(`cannot establish PID-reuse-safe identity for process ${pid}`);
|
||||
}
|
||||
return { pid, startTime };
|
||||
}
|
||||
|
||||
export function inspectNodeWorkerProcessIdentity(
|
||||
identity: NodeWorkerProcessIdentity,
|
||||
): NodeWorkerProcessIdentityState {
|
||||
const observedStartTime = readNodeWorkerProcessStartTime(identity.pid);
|
||||
if (observedStartTime !== null) {
|
||||
if (observedStartTime !== identity.startTime) {
|
||||
return "reused";
|
||||
}
|
||||
return isPidDefinitelyDead(identity.pid) ? "dead" : "live";
|
||||
}
|
||||
return isPidDefinitelyDead(identity.pid) ? "dead" : "unknown";
|
||||
}
|
||||
@@ -0,0 +1,362 @@
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { stableStringify } from "@openclaw/normalization-core";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import type { NodeWorkerLaunchReceipt } from "./node-worker-launch-store.js";
|
||||
import {
|
||||
inspectNodeWorkerProcessIdentity,
|
||||
requireNodeWorkerProcessIdentity,
|
||||
type NodeWorkerProcessIdentity,
|
||||
} from "./node-worker-process-identity.js";
|
||||
import { createNodeWorkerSupervisor } from "./node-worker-supervisor.js";
|
||||
import {
|
||||
testWorkerLaunchInput,
|
||||
writeNodeWorkerFixture,
|
||||
} from "./node-worker-supervisor.test-support.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
const spawned = new Set<ChildProcess>();
|
||||
const ownedProcessGroups: NodeWorkerProcessIdentity[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
for (const child of spawned) {
|
||||
if (child.exitCode === null && child.signalCode === null) {
|
||||
child.kill("SIGKILL");
|
||||
}
|
||||
}
|
||||
if (process.platform !== "win32") {
|
||||
for (const identity of ownedProcessGroups) {
|
||||
if (inspectNodeWorkerProcessIdentity(identity) === "reused") {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
process.kill(-identity.pid, "SIGKILL");
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ESRCH") {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
spawned.clear();
|
||||
ownedProcessGroups.length = 0;
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
});
|
||||
|
||||
function fixture(label: string) {
|
||||
return writeNodeWorkerFixture(tempDirs.make(label));
|
||||
}
|
||||
|
||||
function planHash(input: ReturnType<typeof testWorkerLaunchInput>): string {
|
||||
return createHash("sha256")
|
||||
.update(
|
||||
stableStringify({
|
||||
bundleHash: input.bundleHash,
|
||||
descriptor: input.descriptor,
|
||||
gatewayNamespace: input.gatewayNamespace,
|
||||
placementGeneration: input.placementGeneration,
|
||||
}),
|
||||
)
|
||||
.digest("hex");
|
||||
}
|
||||
|
||||
function insertLaunch(params: {
|
||||
env: NodeJS.ProcessEnv;
|
||||
input: ReturnType<typeof testWorkerLaunchInput>;
|
||||
state: "pending" | "running";
|
||||
supervisor: NodeWorkerProcessIdentity;
|
||||
worker?: NodeWorkerProcessIdentity;
|
||||
}) {
|
||||
const database = openOpenClawStateDatabase({ env: params.env }).db;
|
||||
database
|
||||
.prepare(
|
||||
`INSERT INTO node_worker_launches (
|
||||
launch_id, plan_hash, gateway_namespace, environment_id, session_id,
|
||||
owner_epoch, placement_generation, run_id, state,
|
||||
supervisor_pid, supervisor_start_time, worker_pid, worker_start_time,
|
||||
result_json, error_text, completed_at_ms, created_at_ms, updated_at_ms
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, 1, 1)`,
|
||||
)
|
||||
.run(
|
||||
params.input.launchId,
|
||||
planHash(params.input),
|
||||
params.input.gatewayNamespace,
|
||||
params.input.descriptor.admission.environmentId,
|
||||
params.input.descriptor.admission.sessionId,
|
||||
params.input.descriptor.admission.ownerEpoch,
|
||||
params.input.placementGeneration,
|
||||
params.input.descriptor.assignment.runId,
|
||||
params.state,
|
||||
params.supervisor.pid,
|
||||
params.supervisor.startTime,
|
||||
params.worker?.pid ?? null,
|
||||
params.worker?.startTime ?? null,
|
||||
);
|
||||
}
|
||||
|
||||
function waitForChildLine(child: ChildProcess): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
const onData = (chunk: Buffer) => {
|
||||
stdout += chunk.toString("utf8");
|
||||
const newline = stdout.indexOf("\n");
|
||||
if (newline >= 0) {
|
||||
resolve(stdout.slice(0, newline));
|
||||
}
|
||||
};
|
||||
child.stdout?.on("data", onData);
|
||||
child.stderr?.on("data", (chunk: Buffer) => {
|
||||
stderr += chunk.toString("utf8");
|
||||
});
|
||||
child.once("error", reject);
|
||||
child.once("close", (code, signal) => {
|
||||
reject(new Error(`owner exited before ready (${code ?? signal}): ${stderr}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function waitForChildExit(child: ChildProcess): Promise<void> {
|
||||
if (child.exitCode !== null || child.signalCode !== null) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
child.once("error", reject);
|
||||
child.once("close", () => resolve());
|
||||
});
|
||||
}
|
||||
|
||||
function writeSupervisorOwnerScript(root: string): string {
|
||||
const supervisorUrl = pathToFileURL(path.resolve("src/node-host/node-worker-supervisor.ts")).href;
|
||||
const scriptPath = path.join(root, "supervisor-owner.mts");
|
||||
fs.writeFileSync(
|
||||
scriptPath,
|
||||
`
|
||||
import fs from "node:fs";
|
||||
import { createNodeWorkerSupervisor } from ${JSON.stringify(supervisorUrl)};
|
||||
const [bundleRoot, stateDir, inputPath] = process.argv.slice(2);
|
||||
const supervisor = createNodeWorkerSupervisor({
|
||||
bundleRoot,
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
|
||||
});
|
||||
const shutdown = async () => {
|
||||
await supervisor.close();
|
||||
process.exit(0);
|
||||
};
|
||||
process.once("SIGTERM", () => void shutdown());
|
||||
const input = JSON.parse(fs.readFileSync(inputPath, "utf8"));
|
||||
const receipt = await supervisor.launch(input);
|
||||
process.stdout.write(JSON.stringify(receipt) + "\\n");
|
||||
setInterval(() => {}, 1000);
|
||||
`,
|
||||
);
|
||||
return scriptPath;
|
||||
}
|
||||
|
||||
function spawnSupervisorOwner(params: {
|
||||
bundleRoot: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
input: ReturnType<typeof testWorkerLaunchInput>;
|
||||
root: string;
|
||||
}): ChildProcess {
|
||||
const inputPath = path.join(params.root, `${params.input.launchId}.json`);
|
||||
fs.writeFileSync(inputPath, JSON.stringify(params.input));
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
writeSupervisorOwnerScript(params.root),
|
||||
params.bundleRoot,
|
||||
params.env.OPENCLAW_STATE_DIR!,
|
||||
inputPath,
|
||||
],
|
||||
{ stdio: ["ignore", "pipe", "pipe"] },
|
||||
);
|
||||
spawned.add(child);
|
||||
return child;
|
||||
}
|
||||
|
||||
async function waitForIdentityDeath(identity: NodeWorkerProcessIdentity) {
|
||||
await vi.waitFor(() => expect(inspectNodeWorkerProcessIdentity(identity)).not.toBe("live"), {
|
||||
timeout: 5_000,
|
||||
});
|
||||
}
|
||||
|
||||
describe("node worker supervisor recovery", () => {
|
||||
it("atomically adopts pending work only after the previous supervisor is stale", async () => {
|
||||
const { bundleRoot, env, workspaceDir } = fixture("node-worker-stale-pending-");
|
||||
const supervisor = createNodeWorkerSupervisor({ bundleRoot, env });
|
||||
await supervisor.status("schema-probe");
|
||||
const input = testWorkerLaunchInput(workspaceDir, "stale-pending-launch");
|
||||
insertLaunch({
|
||||
env,
|
||||
input,
|
||||
state: "pending",
|
||||
supervisor: { pid: 2_147_483_647, startTime: 1 },
|
||||
});
|
||||
|
||||
const running = await supervisor.launch(input);
|
||||
|
||||
expect(running).toMatchObject({
|
||||
state: "running",
|
||||
supervisor: requireNodeWorkerProcessIdentity(process.pid),
|
||||
worker: { pid: expect.any(Number), startTime: expect.any(Number) },
|
||||
});
|
||||
await supervisor.close();
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"kills the exact stale-owner worker group before marking it interrupted",
|
||||
async () => {
|
||||
const { bundleRoot, env, root, workspaceDir } = fixture("node-worker-stale-running-");
|
||||
const marker = path.join(root, "recovery-grandchild.pid");
|
||||
const workerSource = `
|
||||
const { spawn } = require("node:child_process");
|
||||
const fs = require("node:fs");
|
||||
const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { stdio: "ignore" });
|
||||
fs.writeFileSync(process.argv[1], String(child.pid));
|
||||
setInterval(() => {}, 1000);
|
||||
`;
|
||||
const workerProcess = spawn(process.execPath, ["-e", workerSource, marker], {
|
||||
detached: true,
|
||||
stdio: "ignore",
|
||||
});
|
||||
spawned.add(workerProcess);
|
||||
const worker = requireNodeWorkerProcessIdentity(workerProcess.pid!);
|
||||
ownedProcessGroups.push(worker);
|
||||
await vi.waitFor(() => expect(fs.existsSync(marker)).toBe(true));
|
||||
const grandchild = requireNodeWorkerProcessIdentity(Number(fs.readFileSync(marker, "utf8")));
|
||||
const input = testWorkerLaunchInput(workspaceDir, "stale-running-launch", "wait");
|
||||
const supervisor = createNodeWorkerSupervisor({ bundleRoot, env });
|
||||
await supervisor.status("schema-probe");
|
||||
insertLaunch({
|
||||
env,
|
||||
input,
|
||||
state: "running",
|
||||
supervisor: { pid: 2_147_483_647, startTime: 1 },
|
||||
worker,
|
||||
});
|
||||
|
||||
const recovered = await supervisor.launch(input);
|
||||
|
||||
expect(recovered).toMatchObject({ state: "interrupted", worker });
|
||||
await waitForIdentityDeath(worker);
|
||||
await waitForIdentityDeath(grandchild);
|
||||
expect((await supervisor.status(input.launchId))?.worker).toEqual(worker);
|
||||
await supervisor.close();
|
||||
},
|
||||
);
|
||||
|
||||
it("returns a live foreign running receipt from a real second process without mutation", async () => {
|
||||
const { bundleRoot, env, root, workspaceDir } = fixture("node-worker-live-replay-");
|
||||
const input = testWorkerLaunchInput(workspaceDir, "live-running-launch", "wait");
|
||||
const owner = spawnSupervisorOwner({ bundleRoot, env, input, root });
|
||||
const owned = JSON.parse(await waitForChildLine(owner)) as NodeWorkerLaunchReceipt;
|
||||
if (owned.worker) {
|
||||
ownedProcessGroups.push(owned.worker);
|
||||
}
|
||||
const second = createNodeWorkerSupervisor({ bundleRoot, env });
|
||||
|
||||
const replay = await second.launch(input);
|
||||
|
||||
expect(replay).toEqual(owned);
|
||||
expect(inspectNodeWorkerProcessIdentity(owned.supervisor)).toBe("live");
|
||||
expect(inspectNodeWorkerProcessIdentity(owned.worker!)).toBe("live");
|
||||
owner.kill("SIGTERM");
|
||||
await waitForChildExit(owner);
|
||||
await second.close();
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"uses IPC disconnect after owner SIGKILL, then reconciles only after exact tree death",
|
||||
async () => {
|
||||
const { bundleRoot, env, root, workspaceDir } = fixture("node-worker-owner-kill-");
|
||||
const input = testWorkerLaunchInput(workspaceDir, "owner-kill-launch", "tree");
|
||||
const owner = spawnSupervisorOwner({ bundleRoot, env, input, root });
|
||||
const owned = JSON.parse(await waitForChildLine(owner)) as NodeWorkerLaunchReceipt;
|
||||
ownedProcessGroups.push(owned.worker!);
|
||||
const grandchildPath = path.join(workspaceDir, "grandchild.pid");
|
||||
await vi.waitFor(() => expect(fs.existsSync(grandchildPath)).toBe(true));
|
||||
const grandchild = requireNodeWorkerProcessIdentity(
|
||||
Number(fs.readFileSync(grandchildPath, "utf8")),
|
||||
);
|
||||
|
||||
owner.kill("SIGKILL");
|
||||
await waitForChildExit(owner);
|
||||
await waitForIdentityDeath(owned.supervisor);
|
||||
await waitForIdentityDeath(owned.worker!);
|
||||
await waitForIdentityDeath(grandchild);
|
||||
|
||||
const restarted = createNodeWorkerSupervisor({ bundleRoot, env });
|
||||
const reconciled = await restarted.launch(input);
|
||||
expect(reconciled).toMatchObject({
|
||||
state: "interrupted",
|
||||
supervisor: owned.supervisor,
|
||||
worker: owned.worker,
|
||||
});
|
||||
await restarted.close();
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps a live foreign pending claim unchanged across real processes", async () => {
|
||||
const { bundleRoot, env, root, workspaceDir } = fixture("node-worker-live-pending-");
|
||||
const input = testWorkerLaunchInput(workspaceDir, "live-pending-launch", "wait");
|
||||
const claim = {
|
||||
launchId: input.launchId,
|
||||
planHash: planHash(input),
|
||||
gatewayNamespace: input.gatewayNamespace,
|
||||
environmentId: input.descriptor.admission.environmentId,
|
||||
sessionId: input.descriptor.admission.sessionId,
|
||||
ownerEpoch: input.descriptor.admission.ownerEpoch,
|
||||
placementGeneration: input.placementGeneration,
|
||||
runId: input.descriptor.assignment.runId,
|
||||
};
|
||||
const storeUrl = pathToFileURL(path.resolve("src/node-host/node-worker-launch-store.ts")).href;
|
||||
const identityUrl = pathToFileURL(
|
||||
path.resolve("src/node-host/node-worker-process-identity.ts"),
|
||||
).href;
|
||||
const claimPath = path.join(root, "claim.json");
|
||||
const scriptPath = path.join(root, "pending-owner.mts");
|
||||
fs.writeFileSync(claimPath, JSON.stringify(claim));
|
||||
fs.writeFileSync(
|
||||
scriptPath,
|
||||
`
|
||||
import fs from "node:fs";
|
||||
import { NodeWorkerLaunchStore } from ${JSON.stringify(storeUrl)};
|
||||
import { requireNodeWorkerProcessIdentity } from ${JSON.stringify(identityUrl)};
|
||||
const [stateDir, claimPath] = process.argv.slice(2);
|
||||
const store = new NodeWorkerLaunchStore({ env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } });
|
||||
const result = store.claim(
|
||||
JSON.parse(fs.readFileSync(claimPath, "utf8")),
|
||||
requireNodeWorkerProcessIdentity(process.pid),
|
||||
);
|
||||
process.stdout.write(JSON.stringify(result.receipt) + "\\n");
|
||||
setInterval(() => {}, 1000);
|
||||
`,
|
||||
);
|
||||
const owner = spawn(
|
||||
process.execPath,
|
||||
["--import", "tsx", scriptPath, env.OPENCLAW_STATE_DIR!, claimPath],
|
||||
{ stdio: ["ignore", "pipe", "pipe"] },
|
||||
);
|
||||
spawned.add(owner);
|
||||
const owned = JSON.parse(await waitForChildLine(owner)) as NodeWorkerLaunchReceipt;
|
||||
const second = createNodeWorkerSupervisor({ bundleRoot, env });
|
||||
|
||||
const replay = await second.launch(input);
|
||||
|
||||
expect(replay).toEqual(owned);
|
||||
owner.kill("SIGKILL");
|
||||
await waitForChildExit(owner);
|
||||
await second.close();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,184 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import {
|
||||
WORKER_PROTOCOL_FEATURES,
|
||||
WORKER_RPC_SET_VERSION,
|
||||
} from "../../packages/gateway-protocol/src/schema/worker-admission.js";
|
||||
import type { WorkerLaunchDescriptor } from "../worker/launch-descriptor.js";
|
||||
|
||||
const TEST_BUNDLE_HASH = "a".repeat(64);
|
||||
export const TEST_WORKER_CREDENTIAL = 'node worker/"credential\\secret?';
|
||||
|
||||
export const TEST_WORKER_SOURCE = String.raw`
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { spawn } from "node:child_process";
|
||||
let input = "";
|
||||
for await (const chunk of process.stdin) input += chunk;
|
||||
const descriptor = JSON.parse(input);
|
||||
if (descriptor.assignment.prompt === "exit-before-start") {
|
||||
fs.writeFileSync(path.join(descriptor.assignment.workspaceDir, "prestart-exited"), "exited");
|
||||
process.exit(23);
|
||||
}
|
||||
if (!process.connected || !process.channel || !process.argv.includes("--internal-worker-ipc")) {
|
||||
process.exit(24);
|
||||
}
|
||||
let grandchild;
|
||||
let disposed = false;
|
||||
let started = false;
|
||||
let resolveStart;
|
||||
const start = new Promise((resolve) => { resolveStart = resolve; });
|
||||
const hardTerminate = () => {
|
||||
if (process.platform === "win32") {
|
||||
spawn("taskkill", ["/F", "/T", "/PID", String(process.pid)], {
|
||||
detached: true,
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
});
|
||||
return;
|
||||
}
|
||||
process.kill(-process.pid, "SIGKILL");
|
||||
};
|
||||
const onMessage = (message) => {
|
||||
if (
|
||||
started ||
|
||||
typeof message !== "object" ||
|
||||
message === null ||
|
||||
Array.isArray(message) ||
|
||||
Object.keys(message).length !== 1 ||
|
||||
message.type !== "openclaw-worker-start-v1"
|
||||
) {
|
||||
hardTerminate();
|
||||
return;
|
||||
}
|
||||
started = true;
|
||||
resolveStart();
|
||||
};
|
||||
const onDisconnect = () => {
|
||||
if (disposed) return;
|
||||
if (!started) process.exit(0);
|
||||
hardTerminate();
|
||||
};
|
||||
process.on("message", onMessage);
|
||||
process.once("disconnect", onDisconnect);
|
||||
await start;
|
||||
const exitWorker = (code) => {
|
||||
disposed = true;
|
||||
process.off("message", onMessage);
|
||||
process.off("disconnect", onDisconnect);
|
||||
if (process.connected) process.disconnect();
|
||||
process.exit(code);
|
||||
};
|
||||
const writeResultAndExit = (value) => {
|
||||
fs.writeSync(1, value);
|
||||
exitWorker(0);
|
||||
};
|
||||
const mode = descriptor.assignment.prompt;
|
||||
if (mode === "wait") {
|
||||
setInterval(() => {}, 1000);
|
||||
} else if (mode === "tree") {
|
||||
grandchild = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { stdio: "ignore" });
|
||||
fs.writeFileSync(path.join(descriptor.assignment.workspaceDir, "grandchild.pid"), String(grandchild.pid));
|
||||
setInterval(() => {}, 1000);
|
||||
} else if (mode === "secret-fail") {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
const credential = descriptor.admission.credential;
|
||||
const escaped = JSON.stringify(credential).slice(1, -1);
|
||||
process.stderr.write(
|
||||
"failure " + "x".repeat(5000) + " " + credential + " " + encodeURIComponent(credential) + " " + escaped,
|
||||
);
|
||||
exitWorker(7);
|
||||
} else if (mode.startsWith("secret-cutoff-")) {
|
||||
const credential = descriptor.admission.credential;
|
||||
const representations = {
|
||||
"secret-cutoff-raw": credential,
|
||||
"secret-cutoff-url": encodeURIComponent(credential),
|
||||
"secret-cutoff-json": JSON.stringify(credential).slice(1, -1),
|
||||
};
|
||||
const representation = representations[mode];
|
||||
const suffixBytes = 4096 - Math.floor(Buffer.byteLength(representation, "utf8") / 2);
|
||||
process.stderr.write("x".repeat(5000) + representation + "y".repeat(suffixBytes));
|
||||
exitWorker(7);
|
||||
} else if (mode === "secret-success") {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
const credential = descriptor.admission.credential;
|
||||
writeResultAndExit(
|
||||
JSON.stringify({ raw: credential, encoded: encodeURIComponent(credential), status: "completed" }) + "\n",
|
||||
);
|
||||
} else if (mode === "overflow") {
|
||||
writeResultAndExit("x".repeat(70 * 1024));
|
||||
} else if (mode === "fast-terminal") {
|
||||
const marker = path.join(descriptor.assignment.workspaceDir, "fast-terminal-marker");
|
||||
process.once("SIGTERM", () => {
|
||||
fs.writeFileSync(marker, "signal");
|
||||
process.exit(143);
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
fs.writeFileSync(marker, "normal");
|
||||
writeResultAndExit(JSON.stringify({ status: "completed" }) + "\n");
|
||||
} else if (mode === "env") {
|
||||
writeResultAndExit(JSON.stringify(process.env) + "\n");
|
||||
} else {
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
writeResultAndExit(JSON.stringify({ argv: process.argv.slice(2), status: "completed" }) + "\n");
|
||||
}
|
||||
`;
|
||||
|
||||
export function testWorkerDescriptor(
|
||||
workspaceDir: string,
|
||||
prompt = "success",
|
||||
): WorkerLaunchDescriptor {
|
||||
return {
|
||||
version: 3,
|
||||
connectionEndpoint: { kind: "unix", socketPath: "/tmp/openclaw-worker/gateway.sock" },
|
||||
admission: {
|
||||
environmentId: "environment-1",
|
||||
credential: TEST_WORKER_CREDENTIAL,
|
||||
sessionId: "session-1",
|
||||
ownerEpoch: 3,
|
||||
rpcSetVersion: WORKER_RPC_SET_VERSION,
|
||||
handshake: {
|
||||
bundleHash: TEST_BUNDLE_HASH,
|
||||
openclawVersion: "2026.8.1",
|
||||
protocolFeatures: [...WORKER_PROTOCOL_FEATURES],
|
||||
},
|
||||
},
|
||||
assignment: {
|
||||
agentId: "agent-1",
|
||||
operationalRunInstance: { instanceId: "instance-1", runId: "run-1" },
|
||||
agentRuntimeIdentityToken: "signed-runtime-token",
|
||||
runId: "run-1",
|
||||
turnId: "turn-1",
|
||||
prompt,
|
||||
suppressPromptTranscript: false,
|
||||
workspaceDir,
|
||||
modelRef: { provider: "provider-1", model: "model-1" },
|
||||
inferenceOptions: {},
|
||||
initialMessages: [],
|
||||
transcript: { baseLeafId: null, nextSeq: 1 },
|
||||
liveEvents: { ackedSeq: 0, nextSeq: 1 },
|
||||
toolAuthority: { allowedToolNames: [] },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function writeNodeWorkerFixture(root: string) {
|
||||
const stateDir = path.join(root, "state-root");
|
||||
const bundleRoot = path.join(root, "bundles-root");
|
||||
const workspaceDir = path.join(root, "workspace");
|
||||
const bundleDir = path.join(bundleRoot, "gateway-1", "bundles", TEST_BUNDLE_HASH);
|
||||
fs.mkdirSync(bundleDir, { recursive: true });
|
||||
fs.mkdirSync(workspaceDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(bundleDir, "openclaw.mjs"), TEST_WORKER_SOURCE);
|
||||
return { bundleRoot, env: { OPENCLAW_STATE_DIR: stateDir }, root, stateDir, workspaceDir };
|
||||
}
|
||||
|
||||
export function testWorkerLaunchInput(workspaceDir: string, launchId: string, prompt = "success") {
|
||||
return {
|
||||
launchId,
|
||||
gatewayNamespace: "gateway-1",
|
||||
bundleHash: TEST_BUNDLE_HASH,
|
||||
placementGeneration: 4,
|
||||
descriptor: testWorkerDescriptor(workspaceDir, prompt),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,513 @@
|
||||
import childProcess from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import { registerSecretValueForRedaction } from "../logging/secret-redaction-registry.js";
|
||||
import { resetSecretRedactionRegistryForTest } from "../logging/secret-redaction-registry.test-support.js";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import { withEnvAsync } from "../test-utils/env.js";
|
||||
import { NodeWorkerLaunchStore } from "./node-worker-launch-store.js";
|
||||
import {
|
||||
inspectNodeWorkerProcessIdentity,
|
||||
requireNodeWorkerProcessIdentity,
|
||||
} from "./node-worker-process-identity.js";
|
||||
import { createNodeWorkerSupervisor } from "./node-worker-supervisor.js";
|
||||
import {
|
||||
TEST_WORKER_CREDENTIAL,
|
||||
TEST_WORKER_SOURCE,
|
||||
testWorkerDescriptor,
|
||||
testWorkerLaunchInput,
|
||||
writeNodeWorkerFixture,
|
||||
} from "./node-worker-supervisor.test-support.js";
|
||||
|
||||
type NodeWorkerSupervisor = ReturnType<typeof createNodeWorkerSupervisor>;
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
resetSecretRedactionRegistryForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
});
|
||||
|
||||
function fixture() {
|
||||
const root = tempDirs.make("node-worker-supervisor-");
|
||||
const { bundleRoot, env, stateDir, workspaceDir } = writeNodeWorkerFixture(root);
|
||||
const supervisor = createNodeWorkerSupervisor({ bundleRoot, env });
|
||||
return { bundleRoot, env, root, stateDir, supervisor, workspaceDir };
|
||||
}
|
||||
|
||||
function launchInput(workspaceDir: string, launchId: string, prompt = "success") {
|
||||
return testWorkerLaunchInput(workspaceDir, launchId, prompt);
|
||||
}
|
||||
|
||||
async function waitForTerminal(supervisor: NodeWorkerSupervisor, launchId: string) {
|
||||
await vi.waitFor(
|
||||
async () => {
|
||||
expect((await supervisor.status(launchId))?.state).not.toMatch(/^(?:pending|running)$/u);
|
||||
},
|
||||
{ timeout: 5_000 },
|
||||
);
|
||||
const receipt = await supervisor.status(launchId);
|
||||
if (!receipt) {
|
||||
throw new Error(`missing launch receipt ${launchId}`);
|
||||
}
|
||||
return receipt;
|
||||
}
|
||||
|
||||
describe("node worker supervisor", () => {
|
||||
it("keeps construction and close inert without resolving process identity", async () => {
|
||||
const root = tempDirs.make("node-worker-inert-");
|
||||
const { bundleRoot, env } = writeNodeWorkerFixture(root);
|
||||
const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform");
|
||||
const spawnSync = vi.spyOn(childProcess, "spawnSync");
|
||||
const execFileSync = vi.spyOn(childProcess, "execFileSync");
|
||||
Object.defineProperty(process, "platform", { configurable: true, value: "win32" });
|
||||
try {
|
||||
const supervisor = createNodeWorkerSupervisor({ bundleRoot, env });
|
||||
await supervisor.close();
|
||||
expect(spawnSync).not.toHaveBeenCalled();
|
||||
expect(execFileSync).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
if (originalPlatform) {
|
||||
Object.defineProperty(process, "platform", originalPlatform);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the additive table absent until the first stateful operation", async () => {
|
||||
const { bundleRoot, env, supervisor } = fixture();
|
||||
const database = openOpenClawStateDatabase({ env });
|
||||
const findTable = () =>
|
||||
database.db
|
||||
.prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?")
|
||||
.get("node_worker_launches");
|
||||
|
||||
expect(findTable()).toBeUndefined();
|
||||
await supervisor.close();
|
||||
expect(findTable()).toBeUndefined();
|
||||
|
||||
const active = createNodeWorkerSupervisor({ bundleRoot, env });
|
||||
expect(await active.status("missing-launch")).toBeUndefined();
|
||||
expect(
|
||||
database.db
|
||||
.prepare("SELECT strict FROM pragma_table_list WHERE name = ?")
|
||||
.get("node_worker_launches"),
|
||||
).toEqual({ strict: 1 });
|
||||
await active.close();
|
||||
});
|
||||
|
||||
it("keeps pending and running launches owned by a live supervisor unchanged", async () => {
|
||||
const { bundleRoot, env, supervisor } = fixture();
|
||||
await supervisor.status("schema-probe");
|
||||
const database = openOpenClawStateDatabase({ env }).db;
|
||||
const supervisorIdentity = requireNodeWorkerProcessIdentity(process.pid);
|
||||
const insert = database.prepare(`
|
||||
INSERT INTO node_worker_launches (
|
||||
launch_id, plan_hash, gateway_namespace, environment_id, session_id,
|
||||
owner_epoch, placement_generation, run_id, state,
|
||||
supervisor_pid, supervisor_start_time, worker_pid, worker_start_time,
|
||||
result_json, error_text, completed_at_ms, created_at_ms, updated_at_ms
|
||||
) VALUES (?, ?, 'gateway-1', 'environment-1', 'session-1', 3, 4, 'run-1', ?, ?, ?, ?, ?, NULL, NULL, NULL, 1, 1)
|
||||
`);
|
||||
insert.run(
|
||||
"pending-launch",
|
||||
"b".repeat(64),
|
||||
"pending",
|
||||
supervisorIdentity.pid,
|
||||
supervisorIdentity.startTime,
|
||||
null,
|
||||
null,
|
||||
);
|
||||
insert.run(
|
||||
"running-launch",
|
||||
"c".repeat(64),
|
||||
"running",
|
||||
supervisorIdentity.pid,
|
||||
supervisorIdentity.startTime,
|
||||
process.pid,
|
||||
supervisorIdentity.startTime,
|
||||
);
|
||||
|
||||
const sameHandle = createNodeWorkerSupervisor({ bundleRoot, env });
|
||||
expect(await sameHandle.status("pending-launch")).toMatchObject({
|
||||
state: "pending",
|
||||
worker: null,
|
||||
});
|
||||
expect(await sameHandle.status("running-launch")).toMatchObject({
|
||||
state: "running",
|
||||
worker: supervisorIdentity,
|
||||
});
|
||||
await supervisor.close();
|
||||
await sameHandle.close();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
|
||||
openOpenClawStateDatabase({ env });
|
||||
const recovered = createNodeWorkerSupervisor({ bundleRoot, env });
|
||||
expect(await recovered.status("pending-launch")).toMatchObject({
|
||||
state: "pending",
|
||||
worker: null,
|
||||
});
|
||||
expect(await recovered.status("running-launch")).toMatchObject({
|
||||
state: "running",
|
||||
worker: supervisorIdentity,
|
||||
});
|
||||
await recovered.close();
|
||||
});
|
||||
|
||||
it("launches idempotently and persists only bounded non-secret facts", async () => {
|
||||
const { env, supervisor, workspaceDir } = fixture();
|
||||
const input = launchInput(workspaceDir, "success-launch");
|
||||
|
||||
expect(await supervisor.launch(input)).toMatchObject({
|
||||
launchId: "success-launch",
|
||||
state: "running",
|
||||
environmentId: "environment-1",
|
||||
sessionId: "session-1",
|
||||
ownerEpoch: 3,
|
||||
placementGeneration: 4,
|
||||
runId: "run-1",
|
||||
});
|
||||
const completed = await waitForTerminal(supervisor, input.launchId);
|
||||
expect(completed).toMatchObject({ state: "completed", errorText: null });
|
||||
expect(JSON.parse(completed.resultJson ?? "null")).toEqual({
|
||||
argv: ["worker", "--internal-worker-ipc"],
|
||||
status: "completed",
|
||||
});
|
||||
expect(await supervisor.launch(input)).toEqual(completed);
|
||||
await expect(
|
||||
supervisor.launch({
|
||||
...input,
|
||||
descriptor: testWorkerDescriptor(workspaceDir, "different-plan"),
|
||||
}),
|
||||
).rejects.toThrow("replayed with a different plan");
|
||||
|
||||
const row = openOpenClawStateDatabase({ env })
|
||||
.db.prepare("SELECT * FROM node_worker_launches WHERE launch_id = ?")
|
||||
.get(input.launchId);
|
||||
expect(JSON.stringify(row)).not.toContain(TEST_WORKER_CREDENTIAL);
|
||||
await supervisor.close();
|
||||
});
|
||||
|
||||
it.each(["status", "launch", "cancel", "close"] as const)(
|
||||
"retains an observed terminal outcome when %s reconciliation keeps failing",
|
||||
async (operation) => {
|
||||
const { env, supervisor, workspaceDir } = fixture();
|
||||
const input = launchInput(workspaceDir, `finish-failure-${operation}`);
|
||||
const store = (supervisor as unknown as { store: NodeWorkerLaunchStore }).store;
|
||||
const originalFinish = store.finish.bind(store);
|
||||
let persistenceUnavailable = true;
|
||||
const finish = vi.spyOn(store, "finish").mockImplementation((params) => {
|
||||
if (persistenceUnavailable) {
|
||||
throw new Error("injected finish failure");
|
||||
}
|
||||
return originalFinish(params);
|
||||
});
|
||||
const invoke = async () => {
|
||||
switch (operation) {
|
||||
case "status":
|
||||
return await supervisor.status(input.launchId);
|
||||
case "launch":
|
||||
return await supervisor.launch(input);
|
||||
case "cancel":
|
||||
return await supervisor.cancel(input.launchId);
|
||||
case "close":
|
||||
await supervisor.close();
|
||||
return new NodeWorkerLaunchStore({ env }).get(input.launchId);
|
||||
default:
|
||||
throw new Error("unsupported reconciliation operation");
|
||||
}
|
||||
};
|
||||
|
||||
expect(await supervisor.launch(input)).toMatchObject({ state: "running" });
|
||||
await vi.waitFor(() => expect(finish).toHaveBeenCalled(), { timeout: 5_000 });
|
||||
expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)?.state).toBe("running");
|
||||
|
||||
await expect(invoke()).rejects.toThrow("injected finish failure");
|
||||
expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)?.state).toBe("running");
|
||||
|
||||
persistenceUnavailable = false;
|
||||
const completed = await invoke();
|
||||
expect(completed).toMatchObject({
|
||||
state: "completed",
|
||||
resultJson: expect.stringContaining('"status":"completed"'),
|
||||
});
|
||||
expect(new NodeWorkerLaunchStore({ env }).get(input.launchId)?.state).toBe("completed");
|
||||
await supervisor.close();
|
||||
},
|
||||
);
|
||||
|
||||
it("spawns workers with only supplied runtime essentials", async () => {
|
||||
const root = tempDirs.make("node-worker-env-");
|
||||
const { bundleRoot, env, workspaceDir } = writeNodeWorkerFixture(root);
|
||||
const suppliedPathKey = process.platform === "win32" ? "Path" : "PATH";
|
||||
const suppliedEnv: NodeJS.ProcessEnv = {
|
||||
...env,
|
||||
[suppliedPathKey]: process.env.PATH,
|
||||
HOME: path.join(root, "worker-home"),
|
||||
LANG: "en_US.UTF-8",
|
||||
LC_TIME: "de_DE.UTF-8",
|
||||
NODE_EXTRA_CA_CERTS: path.join(root, "private-ca.pem"),
|
||||
NODE_USE_SYSTEM_CA: "1",
|
||||
OPENCLAW_ALLOW_INSECURE_PRIVATE_WS: "1",
|
||||
OPENCLAW_SUPPLIED_SECRET: "supplied-openclaw-secret",
|
||||
NODE_OPTIONS: "--title=forbidden-worker-title",
|
||||
BASH_ENV: path.join(root, "forbidden-shell-init"),
|
||||
DYLD_INSERT_LIBRARIES: path.join(root, "forbidden-runtime-injection"),
|
||||
HTTPS_PROXY: "http://supplied-proxy.invalid",
|
||||
SUPPLIED_SECRET: "supplied-secret",
|
||||
};
|
||||
|
||||
await withEnvAsync(
|
||||
{
|
||||
AMBIENT_SECRET: "ambient-secret",
|
||||
OPENCLAW_AMBIENT_SECRET: "ambient-openclaw-secret",
|
||||
HTTP_PROXY: "http://ambient-proxy.invalid",
|
||||
NODE_OPTIONS: undefined,
|
||||
},
|
||||
async () => {
|
||||
const expectedWorkerEnv: NodeJS.ProcessEnv = {
|
||||
HOME: suppliedEnv.HOME,
|
||||
LANG: suppliedEnv.LANG,
|
||||
LC_TIME: suppliedEnv.LC_TIME,
|
||||
NODE_EXTRA_CA_CERTS: suppliedEnv.NODE_EXTRA_CA_CERTS,
|
||||
NODE_USE_SYSTEM_CA: suppliedEnv.NODE_USE_SYSTEM_CA,
|
||||
OPENCLAW_ALLOW_INSECURE_PRIVATE_WS: suppliedEnv.OPENCLAW_ALLOW_INSECURE_PRIVATE_WS,
|
||||
[suppliedPathKey]: suppliedEnv[suppliedPathKey],
|
||||
};
|
||||
const supervisor = createNodeWorkerSupervisor({ bundleRoot, env: suppliedEnv });
|
||||
suppliedEnv.HOME = path.join(root, "mutated-home");
|
||||
suppliedEnv.LANG = "mutated-locale";
|
||||
const input = launchInput(workspaceDir, "env-launch", "env");
|
||||
await supervisor.launch(input);
|
||||
const completed = await waitForTerminal(supervisor, input.launchId);
|
||||
const workerEnv = JSON.parse(completed.resultJson ?? "null") as Record<string, string>;
|
||||
|
||||
expect(workerEnv).toMatchObject(expectedWorkerEnv);
|
||||
expect(workerEnv).not.toHaveProperty("AMBIENT_SECRET");
|
||||
expect(workerEnv).not.toHaveProperty("OPENCLAW_AMBIENT_SECRET");
|
||||
expect(workerEnv).not.toHaveProperty("OPENCLAW_STATE_DIR");
|
||||
expect(workerEnv).not.toHaveProperty("OPENCLAW_SUPPLIED_SECRET");
|
||||
expect(workerEnv).not.toHaveProperty("NODE_OPTIONS");
|
||||
expect(workerEnv).not.toHaveProperty("BASH_ENV");
|
||||
expect(workerEnv).not.toHaveProperty("DYLD_INSERT_LIBRARIES");
|
||||
expect(workerEnv).not.toHaveProperty("HTTP_PROXY");
|
||||
expect(workerEnv).not.toHaveProperty("HTTPS_PROXY");
|
||||
expect(workerEnv).not.toHaveProperty("SUPPLIED_SECRET");
|
||||
expect(JSON.stringify(workerEnv)).not.toContain(TEST_WORKER_CREDENTIAL);
|
||||
const platformInjectedKeys =
|
||||
process.platform === "darwin" ? ["__CF_USER_TEXT_ENCODING"] : [];
|
||||
expect(Object.keys(workerEnv).toSorted()).toEqual(
|
||||
[...Object.keys(expectedWorkerEnv), ...platformInjectedKeys]
|
||||
.filter(
|
||||
(key) => expectedWorkerEnv[key] !== undefined || platformInjectedKeys.includes(key),
|
||||
)
|
||||
.toSorted(),
|
||||
);
|
||||
await supervisor.close();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("bounds output and scrubs launch credentials after registry eviction", async () => {
|
||||
const { supervisor, workspaceDir } = fixture();
|
||||
const successInput = launchInput(workspaceDir, "secret-success-launch", "secret-success");
|
||||
const failureInput = launchInput(workspaceDir, "failure-launch", "secret-fail");
|
||||
const overflowInput = launchInput(workspaceDir, "overflow-launch", "overflow");
|
||||
|
||||
await supervisor.launch(successInput);
|
||||
await supervisor.launch(failureInput);
|
||||
await supervisor.launch(overflowInput);
|
||||
for (let index = 0; index < 600; index += 1) {
|
||||
registerSecretValueForRedaction(`eviction-secret-${index}`);
|
||||
}
|
||||
const success = await waitForTerminal(supervisor, successInput.launchId);
|
||||
const failure = await waitForTerminal(supervisor, failureInput.launchId);
|
||||
const overflow = await waitForTerminal(supervisor, overflowInput.launchId);
|
||||
const representations = [
|
||||
TEST_WORKER_CREDENTIAL,
|
||||
encodeURIComponent(TEST_WORKER_CREDENTIAL),
|
||||
JSON.stringify(TEST_WORKER_CREDENTIAL).slice(1, -1),
|
||||
];
|
||||
expect(success.state).toBe("completed");
|
||||
expect(JSON.parse(success.resultJson ?? "null")).toEqual({
|
||||
raw: "[REDACTED]",
|
||||
encoded: "[REDACTED]",
|
||||
status: "completed",
|
||||
});
|
||||
expect(failure.state).toBe("failed");
|
||||
expect(Buffer.byteLength(failure.errorText ?? "", "utf8")).toBeLessThanOrEqual(4 * 1024);
|
||||
for (const representation of representations) {
|
||||
expect(success.resultJson).not.toContain(representation);
|
||||
expect(failure.errorText).not.toContain(representation);
|
||||
}
|
||||
expect(overflow).toMatchObject({
|
||||
state: "failed",
|
||||
errorText: expect.stringContaining("stdout exceeded 65536 bytes"),
|
||||
});
|
||||
await supervisor.close();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["raw", "secret-cutoff-raw", TEST_WORKER_CREDENTIAL],
|
||||
["URL", "secret-cutoff-url", encodeURIComponent(TEST_WORKER_CREDENTIAL)],
|
||||
["JSON-escaped", "secret-cutoff-json", JSON.stringify(TEST_WORKER_CREDENTIAL).slice(1, -1)],
|
||||
])(
|
||||
"redacts a %s credential representation across the stderr cutoff",
|
||||
async (_, prompt, representation) => {
|
||||
const { supervisor, workspaceDir } = fixture();
|
||||
const input = launchInput(workspaceDir, `cutoff-${prompt}`, prompt);
|
||||
|
||||
await supervisor.launch(input);
|
||||
const failure = await waitForTerminal(supervisor, input.launchId);
|
||||
|
||||
expect(failure.state).toBe("failed");
|
||||
expect(Buffer.byteLength(failure.errorText ?? "", "utf8")).toBeLessThanOrEqual(4 * 1024);
|
||||
expect(failure.errorText).not.toContain(representation);
|
||||
expect(failure.errorText).not.toContain(representation.slice(-8));
|
||||
await supervisor.close();
|
||||
},
|
||||
);
|
||||
|
||||
it("does not open or signal a child after markRunning observes its terminal receipt", async () => {
|
||||
const { supervisor, workspaceDir } = fixture();
|
||||
const input = launchInput(workspaceDir, "fast-terminal-launch", "fast-terminal");
|
||||
vi.spyOn(NodeWorkerLaunchStore.prototype, "markRunning").mockImplementation(
|
||||
function (this: NodeWorkerLaunchStore, params) {
|
||||
return this.finish({
|
||||
launchId: params.launchId,
|
||||
planHash: params.planHash,
|
||||
supervisor: params.supervisor,
|
||||
worker: null,
|
||||
state: "completed",
|
||||
resultJson: '{"status":"completed"}',
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
expect(await supervisor.launch(input)).toMatchObject({ state: "completed" });
|
||||
const marker = path.join(workspaceDir, "fast-terminal-marker");
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, 150);
|
||||
});
|
||||
expect(fs.existsSync(marker)).toBe(false);
|
||||
await supervisor.close();
|
||||
});
|
||||
|
||||
it("records a gated child that exits before journal readiness as terminal", async () => {
|
||||
const { supervisor, workspaceDir } = fixture();
|
||||
const input = launchInput(workspaceDir, "prestart-exit-launch", "exit-before-start");
|
||||
const exitedPath = path.join(workspaceDir, "prestart-exited");
|
||||
|
||||
await supervisor.launch(input);
|
||||
const terminal = await waitForTerminal(supervisor, input.launchId);
|
||||
|
||||
expect(fs.existsSync(exitedPath)).toBe(true);
|
||||
expect(terminal.state).toBe("failed");
|
||||
await supervisor.close();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["cancel", "cancelled"],
|
||||
["close", "interrupted"],
|
||||
] as const)("records %s while awaiting the owned child", async (operation, state) => {
|
||||
const { supervisor, workspaceDir } = fixture();
|
||||
const input = launchInput(workspaceDir, `${operation}-launch`, "wait");
|
||||
expect(await supervisor.launch(input)).toMatchObject({ state: "running" });
|
||||
|
||||
if (operation === "cancel") {
|
||||
await supervisor.cancel(input.launchId);
|
||||
} else {
|
||||
await supervisor.close();
|
||||
}
|
||||
|
||||
expect(await supervisor.status(input.launchId)).toMatchObject({
|
||||
state,
|
||||
worker: { pid: expect.any(Number), startTime: expect.any(Number) },
|
||||
});
|
||||
await supervisor.close();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["cancel", "cancelled"],
|
||||
["close", "interrupted"],
|
||||
] as const)(
|
||||
"%s during startup closes the gate before worker code runs",
|
||||
async (operation, state) => {
|
||||
const { supervisor, workspaceDir } = fixture();
|
||||
const input = launchInput(workspaceDir, `${operation}-startup-launch`, "tree");
|
||||
const originalMarkRunning = Object.getOwnPropertyDescriptor(
|
||||
NodeWorkerLaunchStore.prototype,
|
||||
"markRunning",
|
||||
)?.value as NodeWorkerLaunchStore["markRunning"];
|
||||
let stopping: Promise<unknown> | undefined;
|
||||
vi.spyOn(NodeWorkerLaunchStore.prototype, "markRunning").mockImplementation(
|
||||
function (this: NodeWorkerLaunchStore, params) {
|
||||
const receipt = Reflect.apply(originalMarkRunning, this, [params]);
|
||||
stopping =
|
||||
operation === "cancel" ? supervisor.cancel(input.launchId) : supervisor.close();
|
||||
return receipt;
|
||||
},
|
||||
);
|
||||
|
||||
await supervisor.launch(input);
|
||||
await stopping;
|
||||
|
||||
expect((await supervisor.status(input.launchId))?.state).toBe(state);
|
||||
expect(fs.existsSync(path.join(workspaceDir, "grandchild.pid"))).toBe(false);
|
||||
await supervisor.close();
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
["cancel", "cancelled"],
|
||||
["close", "interrupted"],
|
||||
] as const)("%s terminates the worker-owned grandchild", async (operation, state) => {
|
||||
const { supervisor, workspaceDir } = fixture();
|
||||
const input = launchInput(workspaceDir, `${operation}-tree-launch`, "tree");
|
||||
const running = await supervisor.launch(input);
|
||||
expect(running.state).toBe("running");
|
||||
const grandchildPath = path.join(workspaceDir, "grandchild.pid");
|
||||
await vi.waitFor(() => expect(fs.existsSync(grandchildPath)).toBe(true));
|
||||
const grandchildPid = Number(fs.readFileSync(grandchildPath, "utf8"));
|
||||
const grandchild = requireNodeWorkerProcessIdentity(grandchildPid);
|
||||
expect(inspectNodeWorkerProcessIdentity(grandchild)).toBe("live");
|
||||
|
||||
if (operation === "cancel") {
|
||||
await supervisor.cancel(input.launchId);
|
||||
} else {
|
||||
await supervisor.close();
|
||||
}
|
||||
|
||||
const terminal = await supervisor.status(input.launchId);
|
||||
expect(terminal).toMatchObject({ state, worker: running.worker });
|
||||
await vi.waitFor(() => {
|
||||
expect(inspectNodeWorkerProcessIdentity(running.worker!)).not.toBe("live");
|
||||
expect(inspectNodeWorkerProcessIdentity(grandchild)).not.toBe("live");
|
||||
});
|
||||
await supervisor.close();
|
||||
});
|
||||
|
||||
it("fails closed when the bundle entry resolves outside its namespaced bundle", async () => {
|
||||
const { bundleRoot, root, supervisor, workspaceDir } = fixture();
|
||||
const escapedHash = "b".repeat(64);
|
||||
const escapedBundle = path.join(bundleRoot, "gateway-1", "bundles", escapedHash);
|
||||
const outsideEntry = path.join(root, "outside.mjs");
|
||||
fs.mkdirSync(escapedBundle, { recursive: true });
|
||||
fs.writeFileSync(outsideEntry, TEST_WORKER_SOURCE);
|
||||
fs.symlinkSync(outsideEntry, path.join(escapedBundle, "openclaw.mjs"));
|
||||
const input = launchInput(workspaceDir, "escaped-entry");
|
||||
input.bundleHash = escapedHash;
|
||||
input.descriptor.admission.handshake.bundleHash = escapedHash;
|
||||
|
||||
expect(await supervisor.launch(input)).toMatchObject({
|
||||
state: "failed",
|
||||
errorText: expect.stringContaining("inside its bundle"),
|
||||
});
|
||||
await supervisor.close();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,692 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { stableStringify } from "@openclaw/normalization-core";
|
||||
import { resolveStateDir } from "../config/paths.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import { isPathInside } from "../infra/path-guards.js";
|
||||
import { redactToolPayloadText } from "../logging/redact.js";
|
||||
import {
|
||||
redactRegisteredSecretValues,
|
||||
registerSecretValueForRedaction,
|
||||
} from "../logging/secret-redaction-registry.js";
|
||||
import {
|
||||
appendCapturedOutput,
|
||||
createCapturedOutputBuffers,
|
||||
finalizeCapturedOutput,
|
||||
} from "../process/exec-output.js";
|
||||
import { signalProcessTree } from "../process/kill-tree.js";
|
||||
import { createChildAdapter } from "../process/supervisor/adapters/child.js";
|
||||
import { truncateUtf8Suffix } from "../utils/utf8-truncate.js";
|
||||
import {
|
||||
parseWorkerLaunchDescriptor,
|
||||
type WorkerLaunchDescriptor,
|
||||
} from "../worker/launch-descriptor.js";
|
||||
import { snapshotNodeWorkerEnv } from "./node-worker-environment.js";
|
||||
import {
|
||||
NodeWorkerLaunchStore,
|
||||
type NodeWorkerLaunchReceipt,
|
||||
type NodeWorkerTerminalState,
|
||||
} from "./node-worker-launch-store.js";
|
||||
import {
|
||||
inspectNodeWorkerProcessIdentity,
|
||||
requireNodeWorkerProcessIdentity,
|
||||
type NodeWorkerProcessIdentity,
|
||||
} from "./node-worker-process-identity.js";
|
||||
|
||||
const STDOUT_MAX_BYTES = 64 * 1024;
|
||||
const STDERR_MAX_BYTES = 4 * 1024;
|
||||
const STOP_GRACE_MS = 1_000;
|
||||
const FORCE_STOP_WAIT_MS = 4_000;
|
||||
const RECOVERY_POLL_MS = 25;
|
||||
const GATEWAY_NAMESPACE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u;
|
||||
const BUNDLE_HASH_PATTERN = /^[a-f0-9]{64}$/u;
|
||||
|
||||
type NodeWorkerLaunchInput = {
|
||||
launchId: string;
|
||||
gatewayNamespace: string;
|
||||
bundleHash: string;
|
||||
placementGeneration: number;
|
||||
descriptor: WorkerLaunchDescriptor;
|
||||
};
|
||||
|
||||
type ChildAdapter = Awaited<ReturnType<typeof createChildAdapter>>;
|
||||
type StopState = Extract<NodeWorkerTerminalState, "cancelled" | "interrupted">;
|
||||
type OwnedTreeState = "live" | "dead" | "unknown";
|
||||
type CredentialScrubber = {
|
||||
maxRepresentationBytes: number;
|
||||
scrub: (text: string) => string;
|
||||
};
|
||||
type ActiveBase = {
|
||||
launchId: string;
|
||||
planHash: string;
|
||||
supervisor: NodeWorkerProcessIdentity;
|
||||
worker: NodeWorkerProcessIdentity;
|
||||
};
|
||||
type RunningChild = ActiveBase & {
|
||||
state: "running";
|
||||
adapter: ChildAdapter;
|
||||
done: Promise<void>;
|
||||
journalReady: Promise<void>;
|
||||
releaseJournal: () => void;
|
||||
scrubber: CredentialScrubber;
|
||||
stopState?: StopState;
|
||||
};
|
||||
type TerminalOutcome = Readonly<{
|
||||
state: NodeWorkerTerminalState;
|
||||
resultJson?: string;
|
||||
errorText?: string;
|
||||
}>;
|
||||
type ObservedTerminal = ActiveBase & {
|
||||
state: "observed";
|
||||
outcome: TerminalOutcome;
|
||||
persistenceError?: unknown;
|
||||
};
|
||||
type ActiveOwnership = RunningChild | ObservedTerminal;
|
||||
|
||||
function nodeWorkerPlanHash(params: {
|
||||
bundleHash: string;
|
||||
descriptor: WorkerLaunchDescriptor;
|
||||
gatewayNamespace: string;
|
||||
placementGeneration: number;
|
||||
}): string {
|
||||
return createHash("sha256").update(stableStringify(params)).digest("hex");
|
||||
}
|
||||
|
||||
function resolveWorkerEntry(params: {
|
||||
bundleRoot: string;
|
||||
bundleHash: string;
|
||||
gatewayNamespace: string;
|
||||
}): string {
|
||||
const root = fs.realpathSync.native(params.bundleRoot);
|
||||
const bundle = fs.realpathSync.native(
|
||||
path.join(root, params.gatewayNamespace, "bundles", params.bundleHash),
|
||||
);
|
||||
if (!isPathInside(root, bundle)) {
|
||||
throw new Error("node worker bundle resolves outside its configured root");
|
||||
}
|
||||
const entry = fs.realpathSync.native(path.join(bundle, "openclaw.mjs"));
|
||||
if (!isPathInside(bundle, entry) || !fs.statSync(entry).isFile()) {
|
||||
throw new Error("node worker entry must be a regular file inside its bundle");
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
function createCredentialScrubber(credential: string): CredentialScrubber {
|
||||
const representations = new Set([
|
||||
credential,
|
||||
encodeURIComponent(credential),
|
||||
JSON.stringify(credential).slice(1, -1),
|
||||
]);
|
||||
const ordered = [...representations].toSorted((left, right) => right.length - left.length);
|
||||
return {
|
||||
maxRepresentationBytes: Math.max(
|
||||
...ordered.map((representation) => Buffer.byteLength(representation, "utf8")),
|
||||
),
|
||||
scrub: (text) => {
|
||||
let scrubbed = text;
|
||||
for (const representation of ordered) {
|
||||
scrubbed = scrubbed.replaceAll(representation, "[REDACTED]");
|
||||
}
|
||||
return scrubbed;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function redactLaunchText(value: string, scrubCredential: (text: string) => string): string {
|
||||
const launchRedacted = scrubCredential(value);
|
||||
const exactRedacted = redactRegisteredSecretValues(launchRedacted, () => "[REDACTED]");
|
||||
return redactToolPayloadText(exactRedacted);
|
||||
}
|
||||
|
||||
function sanitizeDiagnostic(
|
||||
value: string,
|
||||
fallback: string,
|
||||
scrubCredential: (text: string) => string,
|
||||
): string {
|
||||
const oneLine = redactLaunchText(value, scrubCredential).replace(/\s+/gu, " ").trim();
|
||||
return truncateUtf8Suffix(oneLine || fallback, STDERR_MAX_BYTES);
|
||||
}
|
||||
|
||||
function successfulResult(
|
||||
stdout: ReturnType<typeof createCapturedOutputBuffers>,
|
||||
scrubCredential: (text: string) => string,
|
||||
): string {
|
||||
if (stdout.truncatedBytes > 0) {
|
||||
throw new Error(`worker stdout exceeded ${STDOUT_MAX_BYTES} bytes`);
|
||||
}
|
||||
const raw = finalizeCapturedOutput(stdout, "head", true).toString("utf8").trim();
|
||||
const redacted = redactLaunchText(raw, scrubCredential);
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(redacted) as unknown;
|
||||
} catch (error) {
|
||||
throw new Error("worker returned invalid JSON output", { cause: error });
|
||||
}
|
||||
const result = JSON.stringify(parsed);
|
||||
if (Buffer.byteLength(result, "utf8") > STDOUT_MAX_BYTES) {
|
||||
throw new Error(`worker result exceeded ${STDOUT_MAX_BYTES} bytes`);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function inspectPosixProcessGroup(pid: number): OwnedTreeState {
|
||||
try {
|
||||
process.kill(-pid, 0);
|
||||
return "live";
|
||||
} catch (error) {
|
||||
const code = (error as NodeJS.ErrnoException).code;
|
||||
return code === "ESRCH" ? "dead" : "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
function inspectOwnedWorkerTree(worker: NodeWorkerProcessIdentity): OwnedTreeState {
|
||||
const root = inspectNodeWorkerProcessIdentity(worker);
|
||||
if (root === "reused") {
|
||||
return "dead";
|
||||
}
|
||||
if (root === "live") {
|
||||
return "live";
|
||||
}
|
||||
if (root === "unknown") {
|
||||
return "unknown";
|
||||
}
|
||||
return process.platform === "win32" ? "dead" : inspectPosixProcessGroup(worker.pid);
|
||||
}
|
||||
|
||||
async function signalOwnedWorkerTree(
|
||||
worker: NodeWorkerProcessIdentity,
|
||||
signal: "SIGTERM" | "SIGKILL",
|
||||
): Promise<void> {
|
||||
const root = inspectNodeWorkerProcessIdentity(worker);
|
||||
if (root === "reused" || root === "unknown") {
|
||||
return;
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
signalProcessTree(worker.pid, signal, { detached: true, onComplete: resolve });
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForOwnedWorkerTreeDeath(
|
||||
worker: NodeWorkerProcessIdentity,
|
||||
timeoutMs: number,
|
||||
): Promise<OwnedTreeState> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let state = inspectOwnedWorkerTree(worker);
|
||||
while (state === "live" && Date.now() < deadline) {
|
||||
await delay(RECOVERY_POLL_MS);
|
||||
state = inspectOwnedWorkerTree(worker);
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
/** Owns worker process groups, lifetime gates, and the durable node-host launch journal. */
|
||||
class NodeWorkerSupervisor {
|
||||
private readonly active = new Map<string, ActiveOwnership>();
|
||||
private readonly starting = new Map<string, Promise<NodeWorkerLaunchReceipt>>();
|
||||
private readonly bundleRoot: string;
|
||||
private readonly store: NodeWorkerLaunchStore;
|
||||
private readonly workerEnv: NodeJS.ProcessEnv;
|
||||
private supervisorIdentity?: NodeWorkerProcessIdentity;
|
||||
private closed = false;
|
||||
private closePromise?: Promise<void>;
|
||||
|
||||
constructor(options: { bundleRoot?: string; env?: NodeJS.ProcessEnv } = {}) {
|
||||
const env = options.env ?? process.env;
|
||||
this.bundleRoot = path.resolve(
|
||||
options.bundleRoot ?? path.join(resolveStateDir(env), "node-host"),
|
||||
);
|
||||
this.store = new NodeWorkerLaunchStore({ env });
|
||||
this.workerEnv = snapshotNodeWorkerEnv(env);
|
||||
}
|
||||
|
||||
private requireSupervisorIdentity(): NodeWorkerProcessIdentity {
|
||||
return (this.supervisorIdentity ??= requireNodeWorkerProcessIdentity(process.pid));
|
||||
}
|
||||
|
||||
async launch(input: NodeWorkerLaunchInput): Promise<NodeWorkerLaunchReceipt> {
|
||||
if (!GATEWAY_NAMESPACE_PATTERN.test(input.gatewayNamespace)) {
|
||||
throw new Error("gateway namespace must be a safe bounded path component");
|
||||
}
|
||||
if (!BUNDLE_HASH_PATTERN.test(input.bundleHash)) {
|
||||
throw new Error("node worker bundle hash must be 64 lowercase hexadecimal characters");
|
||||
}
|
||||
if (!Number.isSafeInteger(input.placementGeneration) || input.placementGeneration < 0) {
|
||||
throw new Error("node worker placement generation must be a non-negative safe integer");
|
||||
}
|
||||
const descriptor = parseWorkerLaunchDescriptor(structuredClone(input.descriptor));
|
||||
if (descriptor.admission.handshake.bundleHash !== input.bundleHash) {
|
||||
throw new Error("node worker descriptor bundle hash does not match the launch bundle");
|
||||
}
|
||||
const planHash = nodeWorkerPlanHash({
|
||||
bundleHash: input.bundleHash,
|
||||
descriptor,
|
||||
gatewayNamespace: input.gatewayNamespace,
|
||||
placementGeneration: input.placementGeneration,
|
||||
});
|
||||
const local = this.active.get(input.launchId);
|
||||
if (local) {
|
||||
if (local.planHash !== planHash) {
|
||||
throw new Error(`node worker launch ${input.launchId} was replayed with a different plan`);
|
||||
}
|
||||
if (local.state === "observed") {
|
||||
return this.reconcileActiveTerminal(local);
|
||||
}
|
||||
const receipt = this.store.get(input.launchId);
|
||||
if (receipt) {
|
||||
return receipt;
|
||||
}
|
||||
}
|
||||
if (this.closed) {
|
||||
throw new Error("node worker supervisor is closed");
|
||||
}
|
||||
const supervisor = this.requireSupervisorIdentity();
|
||||
const claim = this.store.claim(
|
||||
{
|
||||
launchId: input.launchId,
|
||||
planHash,
|
||||
gatewayNamespace: input.gatewayNamespace,
|
||||
environmentId: descriptor.admission.environmentId,
|
||||
sessionId: descriptor.admission.sessionId,
|
||||
ownerEpoch: descriptor.admission.ownerEpoch,
|
||||
placementGeneration: input.placementGeneration,
|
||||
runId: descriptor.assignment.runId,
|
||||
},
|
||||
supervisor,
|
||||
);
|
||||
if (claim.action === "recover") {
|
||||
return await this.recoverRunning(claim.receipt);
|
||||
}
|
||||
if (claim.action === "replay") {
|
||||
const replay = this.active.get(input.launchId);
|
||||
if (replay?.planHash === planHash && replay.state === "observed") {
|
||||
return this.reconcileActiveTerminal(replay);
|
||||
}
|
||||
const startup = this.starting.get(input.launchId);
|
||||
return startup && claim.receipt.state === "pending" ? await startup : claim.receipt;
|
||||
}
|
||||
const startup = this.startClaimed({ input, descriptor, planHash, supervisor });
|
||||
this.starting.set(input.launchId, startup);
|
||||
try {
|
||||
return await startup;
|
||||
} finally {
|
||||
if (this.starting.get(input.launchId) === startup) {
|
||||
this.starting.delete(input.launchId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async status(launchId: string): Promise<NodeWorkerLaunchReceipt | undefined> {
|
||||
const active = this.active.get(launchId);
|
||||
if (active?.state === "observed") {
|
||||
return this.reconcileActiveTerminal(active);
|
||||
}
|
||||
return this.store.get(launchId);
|
||||
}
|
||||
|
||||
async cancel(launchId: string): Promise<NodeWorkerLaunchReceipt | undefined> {
|
||||
const active = this.active.get(launchId);
|
||||
if (active) {
|
||||
if (active.state === "running") {
|
||||
await this.stopChild(active, "cancelled");
|
||||
}
|
||||
const observed = this.active.get(launchId);
|
||||
if (observed?.state === "observed") {
|
||||
return this.reconcileActiveTerminal(observed);
|
||||
}
|
||||
return this.store.get(launchId);
|
||||
}
|
||||
const startup = this.starting.get(launchId);
|
||||
const receipt = this.store.get(launchId);
|
||||
if (!receipt || receipt.state === "completed" || receipt.state === "failed") {
|
||||
return receipt;
|
||||
}
|
||||
if (receipt.state === "interrupted" || receipt.state === "cancelled") {
|
||||
return receipt;
|
||||
}
|
||||
if (!startup || receipt.state !== "pending" || receipt.supervisor.pid !== process.pid) {
|
||||
return receipt;
|
||||
}
|
||||
const cancelled = this.store.finish({
|
||||
launchId,
|
||||
planHash: receipt.planHash,
|
||||
supervisor: this.requireSupervisorIdentity(),
|
||||
worker: null,
|
||||
state: "cancelled",
|
||||
errorText: "node worker launch cancelled",
|
||||
});
|
||||
await startup;
|
||||
return this.store.get(launchId) ?? cancelled;
|
||||
}
|
||||
|
||||
close(): Promise<void> {
|
||||
if (this.closePromise) {
|
||||
return this.closePromise;
|
||||
}
|
||||
this.closed = true;
|
||||
const operation = (async () => {
|
||||
await Promise.allSettled(this.starting.values());
|
||||
await Promise.all(
|
||||
[...this.active.values()]
|
||||
.filter((active): active is RunningChild => active.state === "running")
|
||||
.map(async (active) => await this.stopChild(active, "interrupted")),
|
||||
);
|
||||
const errors: unknown[] = [];
|
||||
for (const active of this.active.values()) {
|
||||
if (active.state !== "observed") {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
this.reconcileActiveTerminal(active);
|
||||
} catch (error) {
|
||||
errors.push(error);
|
||||
}
|
||||
}
|
||||
if (errors.length === 1) {
|
||||
throw errors[0];
|
||||
}
|
||||
if (errors.length > 1) {
|
||||
throw new AggregateError(errors, "node worker terminal reconciliation failed");
|
||||
}
|
||||
})();
|
||||
const closePromise = operation.finally(() => {
|
||||
if (this.closePromise === closePromise) {
|
||||
this.closePromise = undefined;
|
||||
}
|
||||
});
|
||||
this.closePromise = closePromise;
|
||||
return closePromise;
|
||||
}
|
||||
|
||||
private reconcileActiveTerminal(active: ObservedTerminal): NodeWorkerLaunchReceipt {
|
||||
try {
|
||||
const receipt = this.store.finish({
|
||||
launchId: active.launchId,
|
||||
planHash: active.planHash,
|
||||
supervisor: active.supervisor,
|
||||
worker: active.worker,
|
||||
...active.outcome,
|
||||
});
|
||||
if (receipt.state === "pending" || receipt.state === "running") {
|
||||
throw new Error(`node worker launch ${active.launchId} terminal state was not persisted`);
|
||||
}
|
||||
if (this.active.get(active.launchId) === active) {
|
||||
this.active.delete(active.launchId);
|
||||
}
|
||||
return receipt;
|
||||
} catch (error) {
|
||||
active.persistenceError = error;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async recoverRunning(receipt: NodeWorkerLaunchReceipt): Promise<NodeWorkerLaunchReceipt> {
|
||||
if (receipt.state !== "running" || !receipt.worker) {
|
||||
return receipt;
|
||||
}
|
||||
const previousSupervisor = inspectNodeWorkerProcessIdentity(receipt.supervisor);
|
||||
if (previousSupervisor !== "dead" && previousSupervisor !== "reused") {
|
||||
return this.store.get(receipt.launchId) ?? receipt;
|
||||
}
|
||||
let workerState = inspectOwnedWorkerTree(receipt.worker);
|
||||
if (workerState === "unknown") {
|
||||
return this.store.get(receipt.launchId) ?? receipt;
|
||||
}
|
||||
if (workerState === "live") {
|
||||
await signalOwnedWorkerTree(receipt.worker, "SIGTERM");
|
||||
workerState = await waitForOwnedWorkerTreeDeath(receipt.worker, STOP_GRACE_MS);
|
||||
}
|
||||
if (workerState === "live") {
|
||||
await signalOwnedWorkerTree(receipt.worker, "SIGKILL");
|
||||
workerState = await waitForOwnedWorkerTreeDeath(receipt.worker, FORCE_STOP_WAIT_MS);
|
||||
}
|
||||
if (workerState !== "dead") {
|
||||
return this.store.get(receipt.launchId) ?? receipt;
|
||||
}
|
||||
return this.store.finish({
|
||||
launchId: receipt.launchId,
|
||||
planHash: receipt.planHash,
|
||||
supervisor: receipt.supervisor,
|
||||
worker: receipt.worker,
|
||||
state: "interrupted",
|
||||
errorText: "node host stopped before the worker launch completed",
|
||||
});
|
||||
}
|
||||
|
||||
private async startClaimed(params: {
|
||||
input: NodeWorkerLaunchInput;
|
||||
descriptor: WorkerLaunchDescriptor;
|
||||
planHash: string;
|
||||
supervisor: NodeWorkerProcessIdentity;
|
||||
}): Promise<NodeWorkerLaunchReceipt> {
|
||||
const credential = params.descriptor.admission.credential;
|
||||
const scrubber = createCredentialScrubber(credential);
|
||||
registerSecretValueForRedaction(credential);
|
||||
let adapter: ChildAdapter;
|
||||
try {
|
||||
const entry = resolveWorkerEntry({
|
||||
bundleRoot: this.bundleRoot,
|
||||
bundleHash: params.input.bundleHash,
|
||||
gatewayNamespace: params.input.gatewayNamespace,
|
||||
});
|
||||
adapter = await createChildAdapter({
|
||||
argv: [process.execPath, entry, "worker", "--internal-worker-ipc"],
|
||||
env: this.workerEnv,
|
||||
exactEnv: true,
|
||||
ownedWorker: true,
|
||||
input: JSON.stringify(params.descriptor),
|
||||
});
|
||||
} catch (error) {
|
||||
return this.store.finish({
|
||||
launchId: params.input.launchId,
|
||||
planHash: params.planHash,
|
||||
supervisor: params.supervisor,
|
||||
worker: null,
|
||||
state: "failed",
|
||||
errorText: sanitizeDiagnostic(
|
||||
formatErrorMessage(error),
|
||||
"node worker spawn failed",
|
||||
scrubber.scrub,
|
||||
),
|
||||
});
|
||||
}
|
||||
if (!adapter.pid) {
|
||||
adapter.kill("SIGKILL");
|
||||
adapter.dispose();
|
||||
return this.store.finish({
|
||||
launchId: params.input.launchId,
|
||||
planHash: params.planHash,
|
||||
supervisor: params.supervisor,
|
||||
worker: null,
|
||||
state: "failed",
|
||||
errorText: "node worker spawn did not return a process id",
|
||||
});
|
||||
}
|
||||
let worker: NodeWorkerProcessIdentity;
|
||||
try {
|
||||
worker = requireNodeWorkerProcessIdentity(adapter.pid);
|
||||
} catch (error) {
|
||||
adapter.kill("SIGKILL");
|
||||
await adapter.wait().catch(() => undefined);
|
||||
adapter.dispose();
|
||||
return this.store.finish({
|
||||
launchId: params.input.launchId,
|
||||
planHash: params.planHash,
|
||||
supervisor: params.supervisor,
|
||||
worker: null,
|
||||
state: "failed",
|
||||
errorText: sanitizeDiagnostic(
|
||||
formatErrorMessage(error),
|
||||
"node worker process identity unavailable",
|
||||
scrubber.scrub,
|
||||
),
|
||||
});
|
||||
}
|
||||
let journalReleased = false;
|
||||
let releaseJournalPromise!: () => void;
|
||||
const journalReady = new Promise<void>((resolve) => {
|
||||
releaseJournalPromise = resolve;
|
||||
});
|
||||
const releaseJournal = () => {
|
||||
if (!journalReleased) {
|
||||
journalReleased = true;
|
||||
releaseJournalPromise();
|
||||
}
|
||||
};
|
||||
const active = {
|
||||
state: "running",
|
||||
adapter,
|
||||
journalReady,
|
||||
launchId: params.input.launchId,
|
||||
planHash: params.planHash,
|
||||
releaseJournal,
|
||||
scrubber,
|
||||
supervisor: params.supervisor,
|
||||
worker,
|
||||
} as RunningChild;
|
||||
active.done = this.observeChild(active);
|
||||
this.active.set(active.launchId, active);
|
||||
void active.done.catch(() => undefined);
|
||||
let running: NodeWorkerLaunchReceipt;
|
||||
try {
|
||||
running = this.store.markRunning({
|
||||
launchId: active.launchId,
|
||||
planHash: active.planHash,
|
||||
supervisor: params.supervisor,
|
||||
worker,
|
||||
});
|
||||
} catch (error) {
|
||||
active.releaseJournal();
|
||||
await this.stopChild(active, "interrupted").catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
active.releaseJournal();
|
||||
if (running.state === "cancelled" || running.state === "interrupted") {
|
||||
await this.stopChild(active, running.state);
|
||||
return this.store.get(active.launchId) ?? running;
|
||||
}
|
||||
if (running.state !== "running") {
|
||||
adapter.closeStartGate?.();
|
||||
return running;
|
||||
}
|
||||
if (this.closed) {
|
||||
await this.stopChild(active, "interrupted");
|
||||
return this.store.get(active.launchId) ?? running;
|
||||
}
|
||||
try {
|
||||
await adapter.openStartGate?.();
|
||||
} catch {
|
||||
await this.stopChild(active, "interrupted");
|
||||
return this.store.get(active.launchId) ?? running;
|
||||
}
|
||||
return running;
|
||||
}
|
||||
|
||||
private async observeChild(active: RunningChild): Promise<void> {
|
||||
const stdout = createCapturedOutputBuffers();
|
||||
const stderr = createCapturedOutputBuffers();
|
||||
active.adapter.onStdout((chunk) =>
|
||||
appendCapturedOutput(stdout, chunk, STDOUT_MAX_BYTES, "head"),
|
||||
);
|
||||
active.adapter.onStderr((chunk) =>
|
||||
appendCapturedOutput(
|
||||
stderr,
|
||||
chunk,
|
||||
STDERR_MAX_BYTES + active.scrubber.maxRepresentationBytes,
|
||||
"tail",
|
||||
),
|
||||
);
|
||||
let outcome: TerminalOutcome;
|
||||
try {
|
||||
const exit = await active.adapter.wait();
|
||||
await active.journalReady;
|
||||
if (active.stopState) {
|
||||
outcome = Object.freeze({
|
||||
state: active.stopState,
|
||||
errorText:
|
||||
active.stopState === "cancelled"
|
||||
? "node worker launch cancelled"
|
||||
: "node worker launch interrupted during node-host shutdown",
|
||||
});
|
||||
} else if (exit.code === 0 && exit.signal === null) {
|
||||
try {
|
||||
outcome = Object.freeze({
|
||||
state: "completed",
|
||||
resultJson: successfulResult(stdout, active.scrubber.scrub),
|
||||
});
|
||||
} catch (error) {
|
||||
outcome = Object.freeze({
|
||||
state: "failed",
|
||||
errorText: sanitizeDiagnostic(
|
||||
formatErrorMessage(error),
|
||||
"invalid worker result",
|
||||
active.scrubber.scrub,
|
||||
),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
const detail = finalizeCapturedOutput(stderr, "tail", true).toString("utf8");
|
||||
const exitLabel = exit.signal ? `signal ${exit.signal}` : `exit code ${String(exit.code)}`;
|
||||
outcome = Object.freeze({
|
||||
state: "failed",
|
||||
errorText: sanitizeDiagnostic(
|
||||
`node worker failed with ${exitLabel}${detail ? `: ${detail}` : ""}`,
|
||||
"node worker failed",
|
||||
active.scrubber.scrub,
|
||||
),
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
await active.journalReady;
|
||||
outcome = Object.freeze({
|
||||
state: active.stopState ?? "failed",
|
||||
errorText: sanitizeDiagnostic(
|
||||
formatErrorMessage(error),
|
||||
"node worker wait failed",
|
||||
active.scrubber.scrub,
|
||||
),
|
||||
});
|
||||
} finally {
|
||||
active.adapter.dispose();
|
||||
}
|
||||
const observed: ObservedTerminal = {
|
||||
state: "observed",
|
||||
launchId: active.launchId,
|
||||
planHash: active.planHash,
|
||||
supervisor: active.supervisor,
|
||||
worker: active.worker,
|
||||
outcome,
|
||||
};
|
||||
if (this.active.get(active.launchId) !== active) {
|
||||
return;
|
||||
}
|
||||
this.active.set(active.launchId, observed);
|
||||
try {
|
||||
this.reconcileActiveTerminal(observed);
|
||||
} catch {
|
||||
// The observed outcome stays owned in memory for the next supervisor operation.
|
||||
}
|
||||
}
|
||||
|
||||
private async stopChild(active: RunningChild, state: StopState): Promise<void> {
|
||||
active.stopState ??= state;
|
||||
active.adapter.kill("SIGTERM");
|
||||
const forceKill = setTimeout(() => active.adapter.kill("SIGKILL"), STOP_GRACE_MS);
|
||||
forceKill.unref?.();
|
||||
try {
|
||||
await active.done;
|
||||
} finally {
|
||||
clearTimeout(forceKill);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function createNodeWorkerSupervisor(
|
||||
options: {
|
||||
bundleRoot?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
} = {},
|
||||
): NodeWorkerSupervisor {
|
||||
return new NodeWorkerSupervisor(options);
|
||||
}
|
||||
@@ -545,6 +545,8 @@ describe("runNodeHost", () => {
|
||||
await vi.waitFor(() => expect(mocks.capturedGatewayClients[0]?.stop).toHaveBeenCalledOnce());
|
||||
|
||||
expect(clearIntervalSpy).not.toHaveBeenCalled();
|
||||
await vi.waitFor(() => expect(mocks.closeMcpManager).toHaveBeenCalledOnce());
|
||||
expect(resolveCloseMcp).toBeTypeOf("function");
|
||||
resolveCloseMcp?.();
|
||||
await running;
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import { prepareNodeHostRuntime } from "./runtime.js";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
closeMcp: vi.fn(async () => undefined),
|
||||
closeWorkerSupervisor: vi.fn(async () => undefined),
|
||||
handleInvoke: vi.fn(async () => undefined),
|
||||
progressStartHeartbeats: vi.fn(),
|
||||
progressWrite: vi.fn(async () => undefined),
|
||||
@@ -39,6 +40,10 @@ vi.mock("./node-invoke-progress.js", () => ({
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("./node-worker-supervisor.js", () => ({
|
||||
createNodeWorkerSupervisor: vi.fn(() => ({ close: mocks.closeWorkerSupervisor })),
|
||||
}));
|
||||
|
||||
vi.mock("./plugin-node-host.js", () => ({
|
||||
ensureNodeHostPluginRegistry: vi.fn(async () => undefined),
|
||||
isRegisteredNodeHostCommandDuplex: vi.fn((command: string) => command === "test.duplex"),
|
||||
@@ -178,6 +183,7 @@ describe("node-host invocation cancellation", () => {
|
||||
await runtime.close();
|
||||
|
||||
expect(held.signal?.aborted).toBe(true);
|
||||
expect(mocks.closeWorkerSupervisor).toHaveBeenCalledOnce();
|
||||
held.release();
|
||||
await invoking;
|
||||
});
|
||||
|
||||
@@ -26,6 +26,7 @@ import { handleInvoke, type NodeInvokeRequestPayload, type SkillBinsProvider } f
|
||||
import { startNodeHostMcpManager, type NodeHostMcpManager } from "./mcp.js";
|
||||
import { buildNodeEventParams } from "./node-event-params.js";
|
||||
import { createNodeInvokeProgressWriter } from "./node-invoke-progress.js";
|
||||
import { createNodeWorkerSupervisor } from "./node-worker-supervisor.js";
|
||||
import {
|
||||
ensureNodeHostPluginRegistry,
|
||||
isRegisteredNodeHostCommandDuplex,
|
||||
@@ -305,6 +306,7 @@ export async function prepareNodeHostRuntime(params?: {
|
||||
initialInventory,
|
||||
start({ client, onInventoryChanged, onManifestChanged }) {
|
||||
const mcpAbort = new AbortController();
|
||||
const workerSupervisor = createNodeWorkerSupervisor({ env });
|
||||
const skillBins = new SkillBinsCache(client, pathEnv);
|
||||
const activeInvokes = new Map<string, ActiveNodeInvoke>();
|
||||
const pluginCommandContext: OpenClawPluginNodeHostCommandContext = {
|
||||
@@ -448,6 +450,7 @@ export async function prepareNodeHostRuntime(params?: {
|
||||
async close() {
|
||||
this.cancelAll();
|
||||
stopAvailabilityWatch();
|
||||
await workerSupervisor.close();
|
||||
mcpAbort.abort();
|
||||
const resolved = manager ?? (await startup.catch(() => undefined));
|
||||
await resolved?.close();
|
||||
|
||||
Reference in New Issue
Block a user