ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-26 20:35:39 -06:00
Code Issues Packages Projects Releases Wiki Activity

refactor(tooling): unify managed child process cleanup (#127480)

Browse Source
This commit is contained in:
Peter Steinberger
2026-08-21 15:43:39 -07:00
committed by GitHub
parent ee468b8038
commit 74c1900e63
16 changed files with 514 additions and 841 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bench-cli-startup.ts
+26 -50
View File
@@ -5,6 +5,11 @@ import os from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { expectDefined } from "../packages/normalization-core/src/expect.js";
import {
inspectManagedProcessGroup,
terminateManagedChild,
waitForManagedProcessGroupExit,
} from "./lib/managed-child-process.mts";
type CommandCase = {
id: string;
@@ -116,7 +121,6 @@ const DEFAULT_WARMUP = 1;
const DEFAULT_TIMEOUT_MS = 30_000;
const DEFAULT_TIMEOUT_KILL_GRACE_MS = 1_000;
const TIMEOUT_KILL_GRACE_MS = resolveTimeoutKillGraceMs(process.env);
const PROCESS_GROUP_EXIT_POLL_MS = 25;
const DEFAULT_ENTRY = "openclaw.mjs";
const MAX_RSS_MARKER = "__OPENCLAW_MAX_RSS_KB__=";
@@ -799,10 +803,9 @@ async function runSample(params: {
try {
return await new Promise<Sample>((resolve) => {
const useProcessGroup = process.platform !== "win32";
const proc = spawn(process.execPath, nodeArgs, {
cwd: process.cwd(),
detached: useProcessGroup,
detached: process.platform !== "win32",
env: {
...process.env,
HOME: runRoot,
@@ -846,10 +849,10 @@ async function runSample(params: {
const timeout = setTimeout(() => {
timedOut = true;
signalSampleProcess(proc, "SIGTERM", useProcessGroup);
signalSampleProcess(proc, "SIGTERM");
forceKillAt = Date.now() + TIMEOUT_KILL_GRACE_MS;
forceKillTimer = setTimeout(() => {
signalSampleProcess(proc, "SIGKILL", useProcessGroup);
signalSampleProcess(proc, "SIGKILL");
}, TIMEOUT_KILL_GRACE_MS).unref?.();
}, params.timeoutMs);
timeout.unref?.();
@@ -889,12 +892,11 @@ async function runSample(params: {
stderrTail: tailLines(stderr, 20),
}),
});
if (timedOut && isSampleProcessGroupAlive(proc, useProcessGroup)) {
if (timedOut && isSampleProcessGroupAlive(proc)) {
void finishAfterTimeoutCleanup({
complete,
forceKillAt,
proc,
useProcessGroup,
});
return;
}
@@ -912,74 +914,48 @@ async function finishAfterTimeoutCleanup(params: {
complete: () => void;
forceKillAt: number | null;
proc: ReturnType<typeof spawn>;
useProcessGroup: boolean;
}): Promise<void> {
const graceRemainingMs =
params.forceKillAt === null
? TIMEOUT_KILL_GRACE_MS
: Math.max(0, params.forceKillAt - Date.now());
if (graceRemainingMs > 0) {
await waitForSampleProcessGroupExit(params.proc, params.useProcessGroup, graceRemainingMs);
await waitForSampleProcessGroupExit(params.proc, graceRemainingMs);
}
if (isSampleProcessGroupAlive(params.proc, params.useProcessGroup)) {
signalSampleProcess(params.proc, "SIGKILL", params.useProcessGroup);
if (isSampleProcessGroupAlive(params.proc)) {
signalSampleProcess(params.proc, "SIGKILL");
}
await waitForSampleProcessGroupExit(params.proc, params.useProcessGroup, TIMEOUT_KILL_GRACE_MS);
await waitForSampleProcessGroupExit(params.proc, TIMEOUT_KILL_GRACE_MS);
params.complete();
}
function signalSampleProcess(
proc: ReturnType<typeof spawn>,
signal: NodeJS.Signals,
useProcessGroup: boolean,
): void {
function signalSampleProcess(proc: ReturnType<typeof spawn>, signal: NodeJS.Signals): void {
if (!proc.pid) {
return;
}
try {
if (useProcessGroup) {
process.kill(-proc.pid, signal);
} else {
proc.kill(signal);
}
} catch (error) {
const handleSignalError = (error: unknown) => {
const code = (error as NodeJS.ErrnoException | undefined)?.code;
if (code !== "ESRCH" && code !== "EPERM") {
throw error;
}
}
};
terminateManagedChild(proc, signal, {
onChildSignalError: handleSignalError,
onProcessGroupSignalError: handleSignalError,
processGroupFallback: "never",
useWindowsTaskkill: false,
});
}
function isSampleProcessGroupAlive(
proc: ReturnType<typeof spawn>,
useProcessGroup: boolean,
): boolean {
if (!useProcessGroup || !proc.pid) {
return false;
}
try {
process.kill(-proc.pid, 0);
return true;
} catch (error) {
return (error as NodeJS.ErrnoException | undefined)?.code === "EPERM";
}
function isSampleProcessGroupAlive(proc: ReturnType<typeof spawn>): boolean {
return inspectManagedProcessGroup(proc, { errorPolicy: "alive-on-eperm" }) === "live";
}
async function waitForSampleProcessGroupExit(
function waitForSampleProcessGroupExit(
proc: ReturnType<typeof spawn>,
useProcessGroup: boolean,
timeoutMs: number,
): Promise<boolean> {
const deadlineAt = Date.now() + timeoutMs;
while (Date.now() < deadlineAt) {
if (!isSampleProcessGroupAlive(proc, useProcessGroup)) {
return true;
}
await new Promise((resolvePoll) => {
setTimeout(resolvePoll, PROCESS_GROUP_EXIT_POLL_MS);
});
}
return !isSampleProcessGroupAlive(proc, useProcessGroup);
return waitForManagedProcessGroupExit(proc, timeoutMs, { errorPolicy: "alive-on-eperm" });
}
async function runCase(params: {
scripts/dev/tui-pty-test-watch.ts
+11 -64
View File
@@ -1,11 +1,11 @@
// Tui Pty Test Watch script supports OpenClaw repository automation.
import { spawn, spawnSync } from "node:child_process";
import { spawn } from "node:child_process";
import { mkdir, open, writeFile } from "node:fs/promises";
import { createRequire } from "node:module";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { terminateManagedChild } from "../lib/managed-child-process.mts";
import { sleep as delay } from "../lib/sleep.mjs";
import { resolveWindowsTaskkillPath } from "../lib/windows-taskkill.mjs";
type Options = {
altScreen: boolean;
@@ -48,12 +48,6 @@ type ChildStopper = {
type SignalChild = (child: KillableChild, signal: NodeJS.Signals) => void;
type RunTaskkill = (
command: string,
args: string[],
options: { stdio: "ignore" },
) => { error?: unknown; status?: number | null } | undefined;
function unrefTimer(timer: ReturnType<typeof setTimeout>): void {
(timer as { unref?: () => void }).unref?.();
}
@@ -133,60 +127,6 @@ function currentTerminalDimension(value: number | undefined, fallback: number):
return String(value && value > 0 ? value : fallback);
}
function signalWindowsProcessTree(
pid: number,
signal: NodeJS.Signals,
runTaskkill: RunTaskkill = spawnSync,
): boolean {
const args = ["/PID", String(pid), "/T"];
if (signal === "SIGKILL") {
args.push("/F");
}
const result = runTaskkill(resolveWindowsTaskkillPath(), args, { stdio: "ignore" });
return !result?.error && result?.status === 0;
}
function signalWindowsProcessTreeOrForce(
pid: number,
signal: NodeJS.Signals,
runTaskkill: RunTaskkill = spawnSync,
): boolean {
if (signalWindowsProcessTree(pid, signal, runTaskkill)) {
return true;
}
return signal !== "SIGKILL" && signalWindowsProcessTree(pid, "SIGKILL", runTaskkill);
}
function signalChildProcessTree(
child: KillableChild,
signal: NodeJS.Signals,
{
platform = process.platform,
runTaskkill = spawnSync,
useProcessGroup = platform !== "win32",
}: {
platform?: NodeJS.Platform;
runTaskkill?: RunTaskkill;
useProcessGroup?: boolean;
} = {},
): void {
if (useProcessGroup && typeof child.pid === "number") {
try {
process.kill(-child.pid, signal);
return;
} catch {
// Non-detached fallback or already-exited group; direct child signaling is
// still useful on platforms without process groups.
}
}
if (platform === "win32" && typeof child.pid === "number") {
if (signalWindowsProcessTreeOrForce(child.pid, signal, runTaskkill)) {
return;
}
}
child.kill(signal);
}
function createChildStopper(
child: KillableChild,
options: {
@@ -195,7 +135,15 @@ function createChildStopper(
sigkillGraceMs?: number;
} = {},
): ChildStopper {
const signalChild = options.signalChild ?? signalChildProcessTree;
const signalChild =
options.signalChild ??
((targetChild, signal) =>
terminateManagedChild(targetChild, signal, {
onChildSignalError(error) {
throw error;
},
taskkillTimeoutMs: null,
}));
const sigtermGraceMs = options.sigtermGraceMs ?? CHILD_SIGTERM_GRACE_MS;
const sigkillGraceMs = options.sigkillGraceMs ?? CHILD_SIGKILL_GRACE_MS;
let stopping = false;
@@ -523,5 +471,4 @@ export const testing = {
drainNewMirrorData,
parseOptions,
readNewMirrorData,
signalChildProcessTree,
};
scripts/e2e/parallels/host-command.ts
+40 -65
View File
@@ -8,6 +8,11 @@ import {
addTimerTimeoutGraceMs,
clampTimerTimeoutMs,
} from "@openclaw/normalization-core/number-coercion";
import {
inspectManagedProcessGroup,
terminateManagedChild,
waitForManagedProcessGroupExit,
} from "../../lib/managed-child-process.mts";
import { resolveNpmRunner } from "../../npm-runner.mts";
import { resolvePnpmRunner } from "../../pnpm-runner.mts";
import { buildCmdExeCommandLine, resolveWindowsCmdExePath } from "../../windows-cmd-helpers.mjs";
@@ -20,7 +25,6 @@ const HOST_COMMAND_WRAPPER_EXTRA_BUFFER_BYTES = 1024 * 1024;
const HOST_COMMAND_WRAPPER_BACKSTOP_MS = 5_000;
const HOST_COMMAND_TIMEOUT_KILL_GRACE_MS = 100;
const HOST_COMMAND_STREAMING_TIMEOUT_KILL_GRACE_MS = 2_000;
const HOST_COMMAND_PROCESS_GROUP_EXIT_POLL_MS = 25;
const HOST_COMMAND_POST_FORCE_KILL_WAIT_MS = 100;
const HOST_COMMAND_CHILD_PID_PREFIX = "__OPENCLAW_HOST_COMMAND_CHILD_PID__";
const HOST_COMMAND_SPAWN_ERROR_PREFIX = "__OPENCLAW_HOST_COMMAND_SPAWN_ERROR__";
@@ -79,36 +83,31 @@ function signalHostCommandProcess(pid: number | undefined, signal: NodeJS.Signal
if (!pid) {
return;
}
if (process.platform === "win32") {
try {
process.kill(pid, signal);
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (code !== "ESRCH") {
warn(`failed to send ${signal} to host command process ${pid}: ${code ?? String(error)}`);
}
}
return;
}
try {
process.kill(-pid, signal);
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (code === "ESRCH") {
return;
}
try {
process.kill(pid, signal);
} catch (fallbackError) {
const fallbackCode = (fallbackError as NodeJS.ErrnoException).code;
if (fallbackCode === "ESRCH") {
return;
}
warn(
`failed to send ${signal} to host command process ${pid}: group ${code ?? String(error)}, leader ${fallbackCode ?? String(fallbackError)}`,
);
}
}
let processGroupError: NodeJS.ErrnoException | undefined;
terminateManagedChild(
{
kill: (childSignal) => process.kill(pid, childSignal),
pid,
},
signal,
{
onChildSignalError(error) {
const code = (error as NodeJS.ErrnoException).code;
if (code === "ESRCH") {
return;
}
const reason = processGroupError
? `group ${processGroupError.code ?? processGroupError.toString()}, leader ${code ?? String(error)}`
: (code ?? String(error));
warn(`failed to send ${signal} to host command process ${pid}: ${reason}`);
},
onProcessGroupSignalError(error) {
processGroupError = error as NodeJS.ErrnoException;
},
processGroupFallback: "nonmissing",
useWindowsTaskkill: false,
},
);
}
const POSIX_TIMEOUT_WRAPPER = String.raw`
@@ -604,40 +603,16 @@ export async function runStreaming(
}
}
};
const streamingProcessGroupAlive = (): boolean => {
if (!detached || !childPid) {
return false;
}
try {
process.kill(-childPid, 0);
return true;
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "EPERM") {
return false;
}
if (child.exitCode !== null || child.signalCode !== null) {
return false;
}
try {
process.kill(childPid, 0);
return true;
} catch {
return false;
}
}
};
const waitForStreamingProcessGroupExit = async (timeoutBudgetMs: number): Promise<boolean> => {
const deadlineAt = Date.now() + timeoutBudgetMs;
while (Date.now() < deadlineAt) {
if (!streamingProcessGroupAlive()) {
return true;
}
await new Promise((resolvePoll) => {
setTimeout(resolvePoll, HOST_COMMAND_PROCESS_GROUP_EXIT_POLL_MS);
});
}
return !streamingProcessGroupAlive();
};
const streamingProcessGroupAlive = (): boolean =>
inspectManagedProcessGroup(child, {
errorPolicy: "verify-leader",
useProcessGroup: detached,
}) === "live";
const waitForStreamingProcessGroupExit = (timeoutBudgetMs: number): Promise<boolean> =>
waitForManagedProcessGroupExit(child, timeoutBudgetMs, {
errorPolicy: "verify-leader",
useProcessGroup: detached,
});
logStream?.on("error", (error) => {
logStreamError = error;
signalStreamingChild("SIGTERM");
scripts/lib/cross-os-release-checks/process.ts
+7 -9
View File
@@ -16,6 +16,7 @@ import { dirname } from "node:path";
import { StringDecoder } from "node:string_decoder";
import { buildCmdExeCommandLine, resolveWindowsCmdExePath } from "../../windows-cmd-helpers.mjs";
import { toStringifiedError } from "../error-format.mts";
import { terminateManagedChild } from "../managed-child-process.mts";
import { resolveWindowsTaskkillPath } from "../windows-taskkill.mjs";
import type {
Cleanup,
@@ -195,15 +196,12 @@ export async function stopGateway(gateway: GatewayHandle | null) {
}
function signalChildProcessTree(child: ChildProcess, signal: NodeJS.Signals) {
if (process.platform !== "win32" && child.pid) {
try {
process.kill(-child.pid, signal);
return;
} catch {
// The child may have exited before its process group was signaled.
}
}
child.kill(signal);
terminateManagedChild(child, signal, {
onChildSignalError(error) {
throw error;
},
useWindowsTaskkill: false,
});
}
export function registerActiveChildProcessTree(child: ChildProcess) {
scripts/lib/gateway-bench-child.ts
+40 -77
View File
@@ -1,7 +1,11 @@
// Gateway Bench Child script supports OpenClaw repository automation.
import { spawnSync, type ChildProcessWithoutNullStreams } from "node:child_process";
import type { ChildProcessWithoutNullStreams } from "node:child_process";
import {
inspectManagedProcessGroup,
terminateManagedChild,
waitForManagedProcessGroupExit,
} from "./managed-child-process.mts";
import { sleep as delay } from "./sleep.mjs";
import { resolveWindowsTaskkillPath } from "./windows-taskkill.mjs";
export { delay };
@@ -20,8 +24,6 @@ export type StopChildResult = ChildExit & {
type StopChildOptions = {
killGraceMs?: number;
platform?: NodeJS.Platform;
runTaskkill?: typeof spawnSync;
teardownGraceMs?: number;
};
@@ -31,9 +33,27 @@ export async function stopChild(
): Promise<StopChildResult> {
const teardownGraceMs = options.teardownGraceMs ?? TEARDOWN_GRACE_MS;
const killGraceMs = options.killGraceMs ?? TEARDOWN_KILL_GRACE_MS;
const processTreeOptions = {
platform: options.platform ?? process.platform,
runTaskkill: options.runTaskkill ?? spawnSync,
const processTreeAlive = () =>
inspectManagedProcessGroup(child, { errorPolicy: "alive-on-eperm" }) === "live";
const signalProcessTree = (signal: NodeJS.Signals): boolean => {
let delivered = true;
terminateManagedChild(
{
kill(childSignal) {
delivered = child.kill(childSignal);
return delivered;
},
pid: child.pid,
},
signal,
{
onChildSignalError(error) {
throw error;
},
taskkillTimeoutMs: null,
},
);
return delivered;
};
let observedExit: ChildExit | null = null;
const directExit = (): ChildExit | null =>
@@ -43,34 +63,30 @@ export async function stopChild(
: null);
const currentExit = (): ChildExit | null => {
const exit = directExit();
if (exit == null || isProcessTreeAlive(child, processTreeOptions)) {
if (exit == null || processTreeAlive()) {
return null;
}
return exit;
};
const waitForProcessTreeExit = async (ms: number): Promise<boolean> => {
const deadlineAt = Date.now() + ms;
while (Date.now() < deadlineAt) {
if (!isProcessTreeAlive(child, processTreeOptions)) {
return true;
}
await delay(Math.min(EXIT_POLL_MS, deadlineAt - Date.now()));
}
return !isProcessTreeAlive(child, processTreeOptions);
};
const waitForProcessTreeExit = (ms: number): Promise<boolean> =>
waitForManagedProcessGroupExit(child, ms, {
clampPollToDeadline: true,
errorPolicy: "alive-on-eperm",
pollIntervalMs: EXIT_POLL_MS,
});
const cleanupExitedProcessTree = async (
exit: ChildExit,
exitedBeforeTeardown: boolean,
): Promise<StopChildResult> => {
if (!isProcessTreeAlive(child, processTreeOptions)) {
if (!processTreeAlive()) {
return { ...exit, exitedBeforeTeardown };
}
const sentTeardownSignal = killProcessTree(child, "SIGTERM", processTreeOptions);
const sentTeardownSignal = signalProcessTree("SIGTERM");
if (sentTeardownSignal) {
await waitForProcessTreeExit(teardownGraceMs);
}
if (sentTeardownSignal && isProcessTreeAlive(child, processTreeOptions)) {
killProcessTree(child, "SIGKILL", processTreeOptions);
if (sentTeardownSignal && processTreeAlive()) {
signalProcessTree("SIGKILL");
await waitForProcessTreeExit(killGraceMs);
}
if (!sentTeardownSignal) {
@@ -115,7 +131,7 @@ export async function stopChild(
return await cleanupExitedProcessTree(queuedExit, true);
}
const sentTeardownSignal = killProcessTree(child, "SIGTERM", processTreeOptions);
const sentTeardownSignal = signalProcessTree("SIGTERM");
const gracefulExit = await waitForExit(teardownGraceMs);
if (gracefulExit != null) {
return { ...gracefulExit, exitedBeforeTeardown: !sentTeardownSignal };
@@ -130,7 +146,7 @@ export async function stopChild(
return { exitCode: null, exitedBeforeTeardown: true, signal: null };
}
killProcessTree(child, "SIGKILL", processTreeOptions);
signalProcessTree("SIGKILL");
const killedExit = await waitForExit(killGraceMs);
const finalExit = killedExit ?? currentExit();
if (finalExit != null) {
@@ -147,56 +163,3 @@ function releaseUnsettledChild(child: ChildProcessWithoutNullStreams): void {
child.stderr.destroy();
child.unref();
}
function isProcessTreeAlive(
child: ChildProcessWithoutNullStreams,
{ platform = process.platform }: Pick<StopChildOptions, "platform"> = {},
): boolean {
if (platform === "win32" || child.pid === undefined) {
return false;
}
try {
process.kill(-child.pid, 0);
return true;
} catch (error) {
return isProcessStillExistsError(error);
}
}
function isProcessStillExistsError(error: unknown): boolean {
const code = (error as { code?: unknown }).code;
return code === "EPERM";
}
function killProcessTree(
child: ChildProcessWithoutNullStreams,
signal: NodeJS.Signals,
{ platform = process.platform, runTaskkill = spawnSync }: StopChildOptions = {},
): boolean {
if (platform !== "win32" && child.pid !== undefined) {
try {
process.kill(-child.pid, signal);
return true;
} catch {
// Fall back to the direct child below.
}
}
if (platform === "win32" && child.pid !== undefined) {
const args = ["/PID", String(child.pid), "/T"];
if (signal === "SIGKILL") {
args.push("/F");
}
const taskkillPath = resolveWindowsTaskkillPath();
const result = runTaskkill(taskkillPath, args, { stdio: "ignore" });
if (!result?.error && result?.status === 0) {
return true;
}
if (signal !== "SIGKILL") {
const forceResult = runTaskkill(taskkillPath, [...args, "/F"], { stdio: "ignore" });
if (!forceResult?.error && forceResult?.status === 0) {
return true;
}
}
}
return child.kill(signal);
}
scripts/lib/managed-child-process.mts
+147 -54
View File
@@ -12,11 +12,33 @@ const PROCESS_GROUP_POLL_MS = 25;
const TASKKILL_TIMEOUT_MS = 10_000;
type ProcessTreeState = "indeterminate" | "live" | "signaled" | "terminated";
type ManagedChildTermination = { processTreeState: Exclude<ProcessTreeState, "live"> };
type ManagedProcessGroupErrorPolicy = "alive-on-eperm" | "indeterminate" | "verify-leader";
type ManagedProcessGroupChild = {
exitCode?: number | null;
pid?: number;
signalCode?: string | null;
};
type ManagedProcessGroupOptions = {
errorPolicy: ManagedProcessGroupErrorPolicy;
inspectLeaderWhenNoGroup?: boolean;
platform?: NodeJS.Platform;
useProcessGroup?: boolean;
};
type TaskkillRunner = (
command: string,
args: string[],
options: { killSignal?: NodeJS.Signals; stdio?: StdioOptions; timeout?: number },
) => { error?: Error; status: number | null } | undefined;
type ManagedChildTerminationOptions = {
onChildSignalError?: (error: unknown) => void;
onProcessGroupSignalError?: (error: unknown) => void;
platform?: NodeJS.Platform;
processGroupFallback?: "always" | "never" | "nonmissing";
runTaskkill?: TaskkillRunner;
taskkillTimeoutMs?: number | null;
useProcessGroup?: boolean;
useWindowsTaskkill?: boolean;
};
type ManagedCommandOptions = {
bin: string;
@@ -60,28 +82,22 @@ export function signalExitCode(signal: NodeJS.Signals) {
/**
* @param {import("node:child_process").ChildProcess} child
* @param {NodeJS.Signals} [signal]
* @param {{
* onProcessGroupSignalError?: (error: unknown) => void;
* platform?: NodeJS.Platform;
* runTaskkill?: typeof spawnSync;
* useProcessGroup?: boolean;
* }} [options]
* @param {ManagedChildTerminationOptions} [options]
* @returns {{ processTreeState: "indeterminate" | "signaled" | "terminated" } | undefined}
*/
export function terminateManagedChild(
child: { kill(signal?: NodeJS.Signals): unknown; pid?: number },
child: { kill(signal: NodeJS.Signals): unknown; pid?: number },
signal: NodeJS.Signals = "SIGTERM",
{
onChildSignalError,
onProcessGroupSignalError,
platform = process.platform,
processGroupFallback = "always",
runTaskkill = spawnSync,
taskkillTimeoutMs = TASKKILL_TIMEOUT_MS,
useProcessGroup = platform !== "win32",
}: {
onProcessGroupSignalError?: (error: unknown) => void;
platform?: NodeJS.Platform;
runTaskkill?: TaskkillRunner;
useProcessGroup?: boolean;
} = {},
useWindowsTaskkill = true,
}: ManagedChildTerminationOptions = {},
): ManagedChildTermination | undefined {
if (!child.pid) {
try {
@@ -89,7 +105,8 @@ export function terminateManagedChild(
if (platform !== "win32") {
return { processTreeState: delivered === false ? "terminated" : "signaled" };
}
} catch {
} catch (error) {
onChildSignalError?.(error);
// A child that never acquired a PID may already have failed to spawn.
}
return platform === "win32" ? { processTreeState: "indeterminate" } : undefined;
@@ -101,49 +118,130 @@ export function terminateManagedChild(
return { processTreeState: "signaled" };
}
} catch (error) {
if (!isMissingProcessError(error)) {
const processGroupIsMissing = isMissingProcessError(error);
if (!processGroupIsMissing) {
onProcessGroupSignalError?.(error);
}
if (
processGroupFallback === "never" ||
(processGroupFallback === "nonmissing" && processGroupIsMissing)
) {
return processGroupIsMissing ? { processTreeState: "terminated" } : undefined;
}
}
if (platform !== "win32") {
if (platform !== "win32" || !useWindowsTaskkill) {
try {
const delivered = child.kill(signal);
return { processTreeState: delivered === false ? "terminated" : "signaled" };
} catch (error) {
onChildSignalError?.(error);
return isMissingProcessError(error) ? { processTreeState: "terminated" } : undefined;
}
}
if (platform === "win32") {
const taskkillPath = resolveWindowsTaskkillPath();
const args = ["/PID", String(child.pid), "/T"];
if (signal === "SIGKILL") {
args.push("/F");
}
const taskkillOptions = {
killSignal: "SIGKILL",
stdio: "ignore",
timeout: TASKKILL_TIMEOUT_MS,
} satisfies Parameters<TaskkillRunner>[2];
const result = runTaskkill(taskkillPath, args, taskkillOptions);
if (!result?.error && result?.status === 0) {
const taskkillPath = resolveWindowsTaskkillPath();
const args = ["/PID", String(child.pid), "/T"];
if (signal === "SIGKILL") {
args.push("/F");
}
const taskkillOptions: Parameters<TaskkillRunner>[2] =
taskkillTimeoutMs === null
? { stdio: "ignore" }
: { killSignal: "SIGKILL", stdio: "ignore", timeout: taskkillTimeoutMs };
const result = runTaskkill(taskkillPath, args, taskkillOptions);
if (!result?.error && result?.status === 0) {
return { processTreeState: "terminated" };
}
if (signal !== "SIGKILL") {
const forceResult = runTaskkill(taskkillPath, [...args, "/F"], taskkillOptions);
if (!forceResult?.error && forceResult?.status === 0) {
return { processTreeState: "terminated" };
}
if (signal !== "SIGKILL") {
const forceResult = runTaskkill(taskkillPath, [...args, "/F"], taskkillOptions);
if (!forceResult?.error && forceResult?.status === 0) {
return { processTreeState: "terminated" };
}
}
try {
child.kill(signal);
} catch (error) {
onChildSignalError?.(error);
// The leader may already be gone, but failed taskkill leaves descendants unverified.
}
return { processTreeState: "indeterminate" };
}
export function inspectManagedProcessGroup(
child: ManagedProcessGroupChild,
{
errorPolicy,
inspectLeaderWhenNoGroup = false,
platform = process.platform,
useProcessGroup = platform !== "win32",
}: ManagedProcessGroupOptions,
): "dead" | "indeterminate" | "live" {
if (!useProcessGroup) {
return inspectLeaderWhenNoGroup &&
child.pid &&
child.exitCode === null &&
child.signalCode === null
? "live"
: "dead";
}
const { pid } = child;
if (typeof pid !== "number" || !Number.isSafeInteger(pid) || pid <= 1 || pid > 0x7fffffff) {
return "indeterminate";
}
try {
process.kill(-pid, 0);
return "live";
} catch (error) {
if (isMissingProcessError(error)) {
return "dead";
}
if (errorPolicy === "indeterminate") {
return "indeterminate";
}
if (!hasProcessErrorCode(error, "EPERM")) {
return "dead";
}
if (errorPolicy === "alive-on-eperm") {
return "live";
}
if (child.exitCode != null || child.signalCode != null) {
return "dead";
}
try {
child.kill(signal);
process.kill(pid, 0);
return "live";
} catch {
// The leader may already be gone, but failed taskkill leaves descendants unverified.
return "dead";
}
return { processTreeState: "indeterminate" };
}
return undefined;
}
export async function waitForManagedProcessGroupExit(
child: ManagedProcessGroupChild,
timeoutMs: number,
{
clampPollToDeadline = false,
pollIntervalMs = PROCESS_GROUP_POLL_MS,
...groupOptions
}: ManagedProcessGroupOptions & {
clampPollToDeadline?: boolean;
pollIntervalMs?: number;
},
): Promise<boolean> {
const deadlineAt = Date.now() + timeoutMs;
while (Date.now() < deadlineAt) {
if (inspectManagedProcessGroup(child, groupOptions) !== "live") {
return true;
}
const waitMs = clampPollToDeadline
? Math.min(pollIntervalMs, deadlineAt - Date.now())
: pollIntervalMs;
await new Promise((resolve) => {
setTimeout(resolve, waitMs);
});
}
return inspectManagedProcessGroup(child, groupOptions) !== "live";
}
/**
@@ -318,18 +416,6 @@ function createManagedCommandSetupCleanupError(error: unknown, cleanupError: unk
);
}
function processGroupStatus(pid: number | undefined): "dead" | "indeterminate" | "live" {
if (typeof pid !== "number" || !Number.isSafeInteger(pid) || pid <= 1 || pid > 0x7fffffff) {
return "indeterminate";
}
try {
process.kill(-pid, 0);
return "live";
} catch (error) {
return isMissingProcessError(error) ? "dead" : "indeterminate";
}
}
async function ensureManagedProcessTreeExit(
child: ChildProcess,
platform: NodeJS.Platform,
@@ -349,11 +435,14 @@ async function ensureManagedProcessTreeExit(
}
return;
}
const initialStatus = processGroupStatus(child.pid);
const initialStatus = inspectManagedProcessGroup(child, {
errorPolicy: "indeterminate",
platform,
});
if (initialStatus === "dead") {
return;
}
let status: ReturnType<typeof processGroupStatus> = initialStatus;
let status: ReturnType<typeof inspectManagedProcessGroup> = initialStatus;
// A missing group at signal time supersedes the earlier racy liveness probe.
const termination = terminateIfLive
? terminateManagedChild(child, "SIGKILL", { platform })
@@ -363,7 +452,7 @@ async function ensureManagedProcessTreeExit(
await new Promise((resolve) => {
setTimeout(resolve, PROCESS_GROUP_POLL_MS);
});
status = processGroupStatus(child.pid);
status = inspectManagedProcessGroup(child, { errorPolicy: "indeterminate", platform });
if (status === "dead") {
if (terminateIfLive && termination?.processTreeState !== "terminated") {
throw createManagedCommandCleanupError(
@@ -568,5 +657,9 @@ function signalNumberFor(signal: NodeJS.Signals) {
}
function isMissingProcessError(error: unknown) {
return Boolean(error && typeof error === "object" && "code" in error && error.code === "ESRCH");
return hasProcessErrorCode(error, "ESRCH");
}
function hasProcessErrorCode(error: unknown, code: string) {
return Boolean(error && typeof error === "object" && "code" in error && error.code === code);
}
scripts/run-additional-boundary-checks.mts
+14 -28
View File
@@ -10,12 +10,16 @@ import {
resolveTimerTimeoutMs,
} from "../packages/normalization-core/src/number-coercion.ts";
import { isDirectRunUrl } from "./lib/direct-run.mjs";
import {
inspectManagedProcessGroup,
terminateManagedChild,
waitForManagedProcessGroupExit,
} from "./lib/managed-child-process.mts";
const DEFAULT_CHECK_TIMEOUT_MS = 10 * 60 * 1000;
const DEFAULT_OUTPUT_MAX_BYTES = 512 * 1024;
// Boundary checks are disposable subprocesses; bound descendant cleanup after timeout.
const TIMEOUT_KILL_GRACE_MS = 250;
const PROCESS_GROUP_EXIT_POLL_MS = 25;
const POST_FORCE_KILL_WAIT_MS = 250;
type ProcessSignal = `SIG${string}`;
@@ -294,38 +298,20 @@ export function createBoundedOutputBuffer(maxBytes = DEFAULT_OUTPUT_MAX_BYTES) {
}
function terminateChild(child: ChildProcess, signal: ProcessSignal) {
if (process.platform !== "win32" && child.pid) {
try {
process.kill(-child.pid, signal as NodeJS.Signals);
return;
} catch {}
}
child.kill(signal as NodeJS.Signals);
terminateManagedChild(child, signal as NodeJS.Signals, {
onChildSignalError(error) {
throw error;
},
useWindowsTaskkill: false,
});
}
function processGroupAlive(child: ChildProcess) {
if (process.platform === "win32" || !child.pid) {
return false;
}
try {
process.kill(-child.pid, 0);
return true;
} catch (error) {
return typeof error === "object" && error !== null && "code" in error && error.code === "EPERM";
}
return inspectManagedProcessGroup(child, { errorPolicy: "alive-on-eperm" }) === "live";
}
async function waitForProcessGroupExit(child: ChildProcess, timeoutMs: number) {
const deadlineAt = Date.now() + timeoutMs;
while (Date.now() < deadlineAt) {
if (!processGroupAlive(child)) {
return true;
}
await new Promise((resolvePoll) => {
setTimeout(resolvePoll, PROCESS_GROUP_EXIT_POLL_MS);
});
}
return !processGroupAlive(child);
function waitForProcessGroupExit(child: ChildProcess, timeoutMs: number) {
return waitForManagedProcessGroupExit(child, timeoutMs, { errorPolicy: "alive-on-eperm" });
}
async function finishTerminatedProcessTree(
scripts/run-oxlint-shards.mts
+31 -59
View File
@@ -8,6 +8,11 @@ import {
resolveLocalCheckEnv,
resolveRepoToolBinPath,
} from "./lib/local-check-runtime.mts";
import {
inspectManagedProcessGroup,
terminateManagedChild,
waitForManagedProcessGroupExit,
} from "./lib/managed-child-process.mts";
import { shouldPrepareExtensionPackageBoundaryArtifacts } from "./run-oxlint.mts";
const DEFAULT_WINDOWS_EXTENSION_CHUNK_SIZE = 8;
@@ -15,7 +20,6 @@ const DEFAULT_SHARD_HEARTBEAT_MS = 30_000;
const DEFAULT_SHARD_TIMEOUT_MS = 15 * 60_000;
const DEFAULT_SHARD_KILL_GRACE_MS = 5_000;
const POST_FORCE_KILL_WAIT_MS = 1_000;
const PROCESS_GROUP_EXIT_POLL_MS = 25;
const DEFAULT_SPLIT_CORE_SHARD_CONCURRENCY = 4;
const FAST_LOCAL_CHECK_MIN_CPUS = 12;
const FAST_LOCAL_CHECK_MIN_MEMORY_BYTES = 48 * 1024 ** 3;
@@ -46,13 +50,10 @@ type RunnerOptions = {
};
type ShardRunnerOptions = RunnerOptions & { shard: OxlintShard };
type ShardBatchOptions = RunnerOptions & { concurrency: number; entries: OxlintShard[] };
type ChildProcessGroupOptions = { child: ChildProcess; useProcessGroup: boolean };
type ActiveShardChild = ChildProcessGroupOptions & { killGraceMs: number };
type SignalOptions = ChildProcessGroupOptions & { signal: NodeJS.Signals };
type WaitOptions = ChildProcessGroupOptions & { timeoutMs: number };
type ActiveShardChild = { child: ChildProcess; killGraceMs: number };
const ACTIVE_SHARD_CHILDREN = new Set<ActiveShardChild>();
let parentTerminationSignal: NodeJS.Signals | null = null;
let parentTerminationSignal: (typeof PARENT_TERMINATION_SIGNALS)[number] | null = null;
let parentTerminationForceKill: ReturnType<typeof setTimeout> | null = null;
let parentSignalForwardingInstalled = false;
@@ -508,16 +509,15 @@ export async function runShard({ env, extraArgs, runner, shard }: ShardRunnerOpt
const heartbeatMs = resolveShardHeartbeatMs(env);
const timeoutMs = resolveShardTimeoutMs(env);
const killGraceMs = resolveShardKillGraceMs(env);
const useProcessGroup = process.platform !== "win32";
const child = spawn(process.execPath, [runner, ...shard.args, ...extraArgs], {
stdio: "inherit",
detached: useProcessGroup,
detached: process.platform !== "win32",
env: {
...env,
OPENCLAW_OXLINT_SKIP_PREPARE: "1",
},
});
const unregisterShardChild = registerShardChild({ child, killGraceMs, useProcessGroup });
const unregisterShardChild = registerShardChild({ child, killGraceMs });
return await new Promise<number>((resolve) => {
let finished = false;
@@ -540,16 +540,16 @@ export async function runShard({ env, extraArgs, runner, shard }: ShardRunnerOpt
console.error(
`[oxlint:${shard.name}] timed out after ${elapsedSeconds}s; terminating shard`,
);
signalChildProcess({ child, signal: "SIGTERM", useProcessGroup });
signalChildProcess(child, "SIGTERM");
if (killGraceMs > 0) {
forceKillAt = Date.now() + killGraceMs;
forceKill = setTimeout(() => {
console.error(`[oxlint:${shard.name}] did not exit cleanly; killing shard`);
signalChildProcess({ child, signal: "SIGKILL", useProcessGroup });
signalChildProcess(child, "SIGKILL");
}, killGraceMs);
forceKill.unref();
} else {
signalChildProcess({ child, signal: "SIGKILL", useProcessGroup });
signalChildProcess(child, "SIGKILL");
}
}, timeoutMs)
: null;
@@ -577,20 +577,12 @@ export async function runShard({ env, extraArgs, runner, shard }: ShardRunnerOpt
const graceRemainingMs =
forceKillAt === null ? killGraceMs : Math.max(0, forceKillAt - Date.now());
if (graceRemainingMs > 0) {
await waitForChildProcessGroupExit({
child,
timeoutMs: graceRemainingMs,
useProcessGroup,
});
await waitForChildProcessGroupExit(child, graceRemainingMs);
}
if (isChildProcessGroupAlive({ child, useProcessGroup })) {
signalChildProcess({ child, signal: "SIGKILL", useProcessGroup });
if (isChildProcessGroupAlive(child)) {
signalChildProcess(child, "SIGKILL");
}
await waitForChildProcessGroupExit({
child,
timeoutMs: POST_FORCE_KILL_WAIT_MS,
useProcessGroup,
});
await waitForChildProcessGroupExit(child, POST_FORCE_KILL_WAIT_MS);
finish(status);
};
child.once("error", (error) => {
@@ -603,10 +595,7 @@ export async function runShard({ env, extraArgs, runner, shard }: ShardRunnerOpt
: timedOut
? 124
: (status ?? 1);
if (
(timedOut || parentTerminationSignal) &&
isChildProcessGroupAlive({ child, useProcessGroup })
) {
if ((timedOut || parentTerminationSignal) && isChildProcessGroupAlive(child)) {
void finishAfterForcedTeardown(exitStatus);
return;
}
@@ -699,47 +688,30 @@ function parsePositiveEnvInt(rawValue: string, key: string) {
return parsedValue;
}
function signalChildProcess({ child, signal, useProcessGroup }: SignalOptions) {
function signalChildProcess(child: ChildProcess, signal: NodeJS.Signals) {
if (!child.pid) {
return;
}
try {
if (useProcessGroup) {
process.kill(-child.pid, signal);
} else {
child.kill(signal);
}
} catch (error) {
const reportSignalError = (error: unknown) => {
if (!isNodeErrorCode(error, "ESRCH")) {
console.error(error);
}
}
};
terminateManagedChild(child, signal, {
onChildSignalError: reportSignalError,
onProcessGroupSignalError: reportSignalError,
processGroupFallback: "never",
useWindowsTaskkill: false,
});
}
function isChildProcessGroupAlive({ child, useProcessGroup }: ChildProcessGroupOptions) {
if (!useProcessGroup || !child.pid) {
return false;
}
try {
process.kill(-child.pid, 0);
return true;
} catch (error) {
return isNodeErrorCode(error, "EPERM");
}
function isChildProcessGroupAlive(child: ChildProcess) {
return inspectManagedProcessGroup(child, { errorPolicy: "alive-on-eperm" }) === "live";
}
async function waitForChildProcessGroupExit({ child, timeoutMs, useProcessGroup }: WaitOptions) {
const deadlineAt = Date.now() + timeoutMs;
while (Date.now() < deadlineAt) {
if (!isChildProcessGroupAlive({ child, useProcessGroup })) {
return true;
}
await new Promise((resolvePoll) => {
setTimeout(resolvePoll, PROCESS_GROUP_EXIT_POLL_MS);
});
}
return !isChildProcessGroupAlive({ child, useProcessGroup });
function waitForChildProcessGroupExit(child: ChildProcess, timeoutMs: number) {
return waitForManagedProcessGroupExit(child, timeoutMs, { errorPolicy: "alive-on-eperm" });
}
function registerShardChild(entry: ActiveShardChild) {
@@ -781,7 +753,7 @@ function isParentTerminationRequested() {
function signalActiveShardChildren(signal: NodeJS.Signals) {
for (const entry of ACTIVE_SHARD_CHILDREN) {
signalChildProcess({ ...entry, signal });
signalChildProcess(entry.child, signal);
}
}
scripts/test-docker-all.mts
+16 -32
View File
@@ -31,6 +31,11 @@ import {
resolveDockerE2ePlan,
} from "./lib/docker-e2e-plan.mts";
import type { DockerE2eLane } from "./lib/docker-e2e-scenarios.mts";
import {
inspectManagedProcessGroup,
terminateManagedChild,
waitForManagedProcessGroupExit,
} from "./lib/managed-child-process.mts";
import { sleep } from "./lib/sleep.mjs";
import {
createPrepublishPluginRegistryArtifact,
@@ -54,7 +59,6 @@ export const SHELL_CAPTURE_MAX_CHARS = 1024 * 1024;
export const LOG_TAIL_MAX_BYTES = 1024 * 1024;
const SHELL_TIMEOUT_KILL_GRACE_MS = 10_000;
const SHELL_POST_FORCE_KILL_WAIT_MS = 1_000;
const SHELL_PROCESS_GROUP_EXIT_POLL_MS = 25;
const MAX_TIMER_TIMEOUT_MS = 2_147_000_000;
const DEFAULT_TIMINGS_FILE = path.join(ROOT_DIR, ".artifacts/docker-tests/lane-timings.json");
const DEFAULT_GITHUB_WORKFLOW = "openclaw-live-and-e2e-checks-reusable.yml";
@@ -82,10 +86,10 @@ type SchedulerLane = Pick<DockerE2eLane, "name"> &
type TimingStore = Awaited<ReturnType<typeof loadTimingStore>>;
type ShellCommandResult = Omit<ReturnType<typeof shellCommandSkippedForShutdown>, "signal"> & {
signal: NodeJS.Signals | null;
signal: ChildProcess["signalCode"];
};
type ShellCaptureResult = Omit<ReturnType<typeof shellCaptureSkippedForShutdown>, "signal"> & {
signal: NodeJS.Signals | null;
signal: ChildProcess["signalCode"];
};
type ShellCommandOptions = {
@@ -1625,28 +1629,11 @@ function shellCaptureSkippedForShutdown(label: string, signal: ShutdownSignal |
}
function shellProcessGroupAlive(child: ChildProcess) {
if (process.platform === "win32" || !child.pid) {
return false;
}
try {
process.kill(-child.pid, 0);
return true;
} catch (error) {
return error instanceof Error && "code" in error && error.code === "EPERM";
}
return inspectManagedProcessGroup(child, { errorPolicy: "alive-on-eperm" }) === "live";
}
async function waitForShellProcessGroupExit(child: ChildProcess, timeoutMs: number) {
const deadlineAt = Date.now() + timeoutMs;
while (Date.now() < deadlineAt) {
if (!shellProcessGroupAlive(child)) {
return true;
}
await new Promise((resolvePoll) => {
setTimeout(resolvePoll, SHELL_PROCESS_GROUP_EXIT_POLL_MS);
});
}
return !shellProcessGroupAlive(child);
function waitForShellProcessGroupExit(child: ChildProcess, timeoutMs: number) {
return waitForManagedProcessGroupExit(child, timeoutMs, { errorPolicy: "alive-on-eperm" });
}
async function finishTimedOutShellProcessTree(
@@ -1669,15 +1656,12 @@ async function finishTimedOutShellProcessTree(
}
function terminateChild(child: ChildProcess, signal: ShutdownSignal) {
if (process.platform !== "win32" && child.pid) {
try {
process.kill(-child.pid, signal);
return;
} catch {
// Fall back to killing the direct child below.
}
}
child.kill(signal);
terminateManagedChild(child, signal, {
onChildSignalError(error) {
throw error;
},
useWindowsTaskkill: false,
});
}
function terminateActiveChildren(signal: ShutdownSignal) {
scripts/test-group-report.mts
+22 -85
View File
@@ -1,5 +1,5 @@
// Builds grouped Vitest duration reports or compares two grouped reports.
import { spawn, spawnSync, type ChildProcess } from "node:child_process";
import { spawn } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
@@ -7,6 +7,11 @@ import { pathToFileURL } from "node:url";
import { isRecord } from "@openclaw/normalization-core/record-coerce";
import pMap from "p-map";
import { coerceErrorMessage } from "./lib/error-format.mts";
import {
inspectManagedProcessGroup,
terminateManagedChild,
waitForManagedProcessGroupExit,
} from "./lib/managed-child-process.mts";
import { parsePositiveInt } from "./lib/numeric-options.mjs";
import {
buildGroupedTestComparison,
@@ -17,7 +22,6 @@ import {
renderGroupedTestReport,
} from "./lib/test-group-report.mts";
import { formatMs } from "./lib/vitest-report-cli-utils.mts";
import { resolveWindowsTaskkillPath } from "./lib/windows-taskkill.mjs";
import { resolveVitestNodeArgs } from "./run-vitest.mts";
import {
applyParallelVitestCachePaths,
@@ -31,7 +35,6 @@ const DEFAULT_TIMEOUT_KILL_GRACE_MS = 10_000;
const DEFAULT_SPAWN_LOG_MAX_BYTES = 1024 * 1024 * 256;
const DEFAULT_SPAWN_OUTPUT_MAX_BYTES = 1024 * 1024 * 64;
const DEFAULT_SPAWN_OUTPUT_TAIL_BYTES = 1024 * 256;
const PROCESS_GROUP_EXIT_POLL_MS = 25;
type ProcessSignal = `SIG${string}`;
type TimerHandle = ReturnType<typeof setTimeout>;
@@ -97,12 +100,6 @@ type RunVitestParams = TestGroupRunSpec &
reportPath: string;
};
type TaskkillRunner = (
command: string,
args: string[],
options: { stdio: "ignore" },
) => { error?: Error; status: number | null };
function usage() {
return [
"Usage: node --import tsx scripts/test-group-report.mts [options] [-- <vitest args>]",
@@ -335,57 +332,6 @@ function parseMaxRssBytes(output: string) {
return null;
}
function hasErrorCode(error: unknown, code: string) {
return isRecord(error) && error.code === code;
}
export function signalTestGroupReportChild(
child: Pick<ChildProcess, "kill" | "pid">,
signal: ProcessSignal,
{
appendDiagnostic = () => {},
platform = process.platform,
runTaskkill = spawnSync,
useProcessGroup = platform !== "win32",
}: {
appendDiagnostic?: (message: string) => void;
platform?: typeof process.platform;
runTaskkill?: TaskkillRunner;
useProcessGroup?: boolean;
} = {},
) {
if (useProcessGroup && typeof child.pid === "number") {
try {
process.kill(-child.pid, signal as NodeJS.Signals);
return;
} catch (error) {
if (error && !hasErrorCode(error, "ESRCH")) {
appendDiagnostic(
`[test-group-report] failed to send ${signal} to process group: ${coerceErrorMessage(error)}\n`,
);
}
}
}
if (platform === "win32" && typeof child.pid === "number") {
const args = ["/PID", String(child.pid), "/T"];
if (signal === "SIGKILL") {
args.push("/F");
}
const taskkillPath = resolveWindowsTaskkillPath();
const result = runTaskkill(taskkillPath, args, { stdio: "ignore" });
if (!result?.error && result?.status === 0) {
return;
}
if (signal !== "SIGKILL") {
const forceResult = runTaskkill(taskkillPath, [...args, "/F"], { stdio: "ignore" });
if (!forceResult?.error && forceResult?.status === 0) {
return;
}
}
}
child.kill(signal as NodeJS.Signals);
}
/**
* Runs a command, captures text output, and terminates timed-out process groups.
*/
@@ -422,7 +368,17 @@ export function spawnText(command: string, args: readonly string[], options: Spa
let childClosedResult: SpawnTextResult | null = null;
let waitingForKillGrace = false;
const signalChild = (signal: ProcessSignal) =>
signalTestGroupReportChild(child, signal, { appendDiagnostic, useProcessGroup });
terminateManagedChild(child, signal as NodeJS.Signals, {
onChildSignalError(error) {
throw error;
},
onProcessGroupSignalError(error) {
appendDiagnostic(
`[test-group-report] failed to send ${signal} to process group: ${coerceErrorMessage(error)}\n`,
);
},
taskkillTimeoutMs: null,
});
const parentSignalHandlers: { signal: ProcessSignal; handler: () => void }[] = [];
const cleanupParentSignalHandlers = () => {
for (const { signal, handler } of parentSignalHandlers) {
@@ -448,34 +404,15 @@ export function spawnText(command: string, args: readonly string[], options: Spa
relayParentSignal("SIGINT");
relayParentSignal("SIGTERM");
}
const processGroupIsAlive = () => {
if (!useProcessGroup || typeof child.pid !== "number") {
return false;
}
try {
process.kill(-child.pid, 0);
return true;
} catch (error) {
return Boolean(error && hasErrorCode(error, "EPERM"));
}
};
const waitForProcessGroupExit = async (timeoutMsToWait: number) => {
const deadlineAt = Date.now() + timeoutMsToWait;
while (Date.now() < deadlineAt) {
if (!processGroupIsAlive()) {
return true;
}
await new Promise((resolvePoll) => {
setTimeout(resolvePoll, PROCESS_GROUP_EXIT_POLL_MS);
});
}
return !processGroupIsAlive();
};
const processGroupIsAlive = () =>
inspectManagedProcessGroup(child, { errorPolicy: "alive-on-eperm" }) === "live";
const finishAfterProcessGroupCleanup = async (result: SpawnTextResult) => {
const graceRemainingMs =
killGraceDeadline === null ? killGraceMs : Math.max(0, killGraceDeadline - Date.now());
if (graceRemainingMs > 0) {
await waitForProcessGroupExit(graceRemainingMs);
await waitForManagedProcessGroupExit(child, graceRemainingMs, {
errorPolicy: "alive-on-eperm",
});
}
if (settled) {
return;
scripts/tsdown-build.mts
+17 -31
View File
@@ -13,7 +13,11 @@ import fs from "node:fs";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { BUNDLED_PLUGIN_PATH_PREFIX } from "./lib/bundled-plugin-paths.mjs";
import { terminateManagedChild } from "./lib/managed-child-process.mts";
import {
inspectManagedProcessGroup,
terminateManagedChild,
waitForManagedProcessGroupExit,
} from "./lib/managed-child-process.mts";
import { parsePositiveInt } from "./lib/numeric-options.mjs";
import { assertRealOutputRoot } from "./lib/output-root-guard.mjs";
import {
@@ -51,7 +55,6 @@ const PROC_MEMINFO_PATH = "/proc/meminfo";
const tsdownStdio = () => ["ignore", "pipe", "pipe"] satisfies ["ignore", "pipe", "pipe"];
// Build descendants get a short cleanup window; a timed-out build must not hold CI for seconds.
const TERMINATION_GRACE_MS = 250;
const PROCESS_GROUP_EXIT_POLL_MS = 25;
const POST_FORCE_KILL_WAIT_MS = 250;
const ROOT_TSDOWN_OUTPUT_ROOTS = ["dist", "dist-runtime"];
const PRESERVED_TSDOWN_OUTPUT_FILES = ["dist/cli-startup-metadata.json"];
@@ -899,35 +902,18 @@ export async function runTsdownBuildInvocation(
relayParentSignal("SIGHUP");
}
function processTreeAlive() {
if (!child.pid) {
return false;
}
if (!useProcessGroup) {
return child.exitCode === null && child.signalCode === null;
}
try {
process.kill(-child.pid, 0);
return true;
} catch (error) {
return (
typeof error === "object" && error !== null && "code" in error && error.code === "EPERM"
);
}
}
async function waitForProcessTreeExit(timeoutMsToWait: number) {
const deadlineAt = Date.now() + timeoutMsToWait;
while (Date.now() < deadlineAt) {
if (!processTreeAlive()) {
return true;
}
await new Promise((resolvePoll) => {
setTimeout(resolvePoll, PROCESS_GROUP_EXIT_POLL_MS);
});
}
return !processTreeAlive();
}
const processTreeAlive = () =>
inspectManagedProcessGroup(child, {
errorPolicy: "alive-on-eperm",
inspectLeaderWhenNoGroup: true,
platform,
}) === "live";
const waitForProcessTreeExit = (timeoutMsToWait: number) =>
waitForManagedProcessGroupExit(child, timeoutMsToWait, {
errorPolicy: "alive-on-eperm",
inspectLeaderWhenNoGroup: true,
platform,
});
async function finishTimedOutProcessTree() {
const graceRemainingMs =
test/scripts/bench-gateway-child-test-support.ts
-118
View File
@@ -1,11 +1,6 @@
// Gateway benchmark child test support simulates child process behavior for script tests.
import { EventEmitter } from "node:events";
import { expect, it, vi } from "vitest";
import { resolveWindowsTaskkillPath } from "../../scripts/lib/windows-taskkill.mjs";
function expectedTaskkillPath(): string {
return resolveWindowsTaskkillPath();
}
type StopChildResult = {
exitedBeforeTeardown: boolean;
@@ -17,8 +12,6 @@ type StopChild<TChild> = (
child: TChild,
options?: {
killGraceMs?: number;
platform?: NodeJS.Platform;
runTaskkill?: typeof spawnSync;
teardownGraceMs?: number;
},
) => Promise<StopChildResult>;
@@ -111,116 +104,6 @@ export function registerStopChildBehaviorTests<TChild>(params: {
expect(child.unref).toHaveBeenCalledOnce();
});
it("signals Windows child process trees with taskkill", async () => {
const child = new EventEmitter() as EventEmitter & {
exitCode: number | null;
kill: ReturnType<typeof vi.fn>;
pid: number;
signalCode: NodeJS.Signals | null;
stderr: { destroy: ReturnType<typeof vi.fn> };
stdin: { destroy: ReturnType<typeof vi.fn> };
stdout: { destroy: ReturnType<typeof vi.fn> };
unref: ReturnType<typeof vi.fn>;
};
child.exitCode = null;
child.kill = vi.fn(() => true);
child.pid = 4450;
child.signalCode = null;
child.stderr = { destroy: vi.fn() };
child.stdin = { destroy: vi.fn() };
child.stdout = { destroy: vi.fn() };
child.unref = vi.fn();
const runTaskkill = vi.fn(() => ({ error: undefined, status: 0 }));
await expect(
params.stopChild(child as unknown as TChild, {
killGraceMs: 1,
platform: "win32",
runTaskkill: runTaskkill as unknown as typeof spawnSync,
teardownGraceMs: 1,
}),
).resolves.toEqual({
exitedBeforeTeardown: false,
exitCode: null,
signal: "SIGKILL",
});
expect(runTaskkill).toHaveBeenNthCalledWith(1, expectedTaskkillPath(), ["/PID", "4450", "/T"], {
stdio: "ignore",
});
expect(runTaskkill).toHaveBeenNthCalledWith(
2,
expectedTaskkillPath(),
["/PID", "4450", "/T", "/F"],
{
stdio: "ignore",
},
);
expect(child.kill).not.toHaveBeenCalled();
expect(child.stdin.destroy).toHaveBeenCalledOnce();
expect(child.stdout.destroy).toHaveBeenCalledOnce();
expect(child.stderr.destroy).toHaveBeenCalledOnce();
expect(child.unref).toHaveBeenCalledOnce();
});
it("force-kills Windows child process trees when graceful taskkill fails", async () => {
const child = new EventEmitter() as EventEmitter & {
exitCode: number | null;
kill: ReturnType<typeof vi.fn>;
pid: number;
signalCode: NodeJS.Signals | null;
stderr: { destroy: ReturnType<typeof vi.fn> };
stdin: { destroy: ReturnType<typeof vi.fn> };
stdout: { destroy: ReturnType<typeof vi.fn> };
unref: ReturnType<typeof vi.fn>;
};
child.exitCode = null;
child.kill = vi.fn(() => true);
child.pid = 4450;
child.signalCode = null;
child.stderr = { destroy: vi.fn() };
child.stdin = { destroy: vi.fn() };
child.stdout = { destroy: vi.fn() };
child.unref = vi.fn();
const runTaskkill = vi
.fn()
.mockReturnValueOnce({ error: undefined, status: 1 })
.mockReturnValueOnce({ error: undefined, status: 0 })
.mockReturnValueOnce({ error: undefined, status: 0 });
await expect(
params.stopChild(child as unknown as TChild, {
killGraceMs: 1,
platform: "win32",
runTaskkill,
teardownGraceMs: 1,
}),
).resolves.toEqual({
exitedBeforeTeardown: false,
exitCode: null,
signal: "SIGKILL",
});
expect(runTaskkill).toHaveBeenNthCalledWith(1, expectedTaskkillPath(), ["/PID", "4450", "/T"], {
stdio: "ignore",
});
expect(runTaskkill).toHaveBeenNthCalledWith(
2,
expectedTaskkillPath(),
["/PID", "4450", "/T", "/F"],
{
stdio: "ignore",
},
);
expect(runTaskkill).toHaveBeenNthCalledWith(
3,
expectedTaskkillPath(),
["/PID", "4450", "/T", "/F"],
{
stdio: "ignore",
},
);
expect(child.kill).not.toHaveBeenCalled();
});
it.skipIf(process.platform === "win32")(
"preserves pre-teardown wrapper exits while cleaning the process group",
async () => {
@@ -348,4 +231,3 @@ export function registerStopChildBehaviorTests<TChild>(params: {
},
);
}
import type { spawnSync } from "node:child_process";
test/scripts/dev-tooling-safety.test.ts
-94
View File
@@ -22,14 +22,9 @@ import {
redactHomePath,
redactJsonValueForDevToolLog,
} from "../../scripts/lib/dev-tooling-safety.ts";
import { resolveWindowsTaskkillPath } from "../../scripts/lib/windows-taskkill.mjs";
const tempDirs: string[] = [];
function expectedTaskkillPath(): string {
return resolveWindowsTaskkillPath();
}
async function waitForCondition(predicate: () => boolean, timeoutMs = 5_000): Promise<void> {
const started = Date.now();
while (Date.now() - started < timeoutMs) {
@@ -501,95 +496,6 @@ describe("script-specific dev tooling hardening", () => {
expect(retained.toString("utf8")).toBe("89abcdef");
});
it.runIf(process.platform !== "win32")(
"signals the TUI PTY watch process group before falling back to the child",
() => {
const kill = vi.spyOn(process, "kill").mockReturnValue(true);
const childKill = vi.fn(() => true);
try {
tuiPtyWatchTesting.signalChildProcessTree({ pid: 123, kill: childKill }, "SIGTERM");
expect(kill).toHaveBeenCalledWith(-123, "SIGTERM");
expect(childKill).not.toHaveBeenCalled();
} finally {
kill.mockRestore();
}
},
);
it.runIf(process.platform !== "win32")(
"falls back to direct TUI PTY watch child signaling when the process group is gone",
() => {
const kill = vi.spyOn(process, "kill").mockImplementation(() => {
const error = new Error("missing process group") as NodeJS.ErrnoException;
error.code = "ESRCH";
throw error;
});
const childKill = vi.fn(() => true);
try {
tuiPtyWatchTesting.signalChildProcessTree({ pid: 123, kill: childKill }, "SIGTERM");
expect(kill).toHaveBeenCalledWith(-123, "SIGTERM");
expect(childKill).toHaveBeenCalledWith("SIGTERM");
} finally {
kill.mockRestore();
}
},
);
it("signals Windows TUI PTY watch process trees with taskkill", () => {
const childKill = vi.fn(() => true);
const runTaskkill = vi.fn(() => ({ error: undefined, status: 0 }));
tuiPtyWatchTesting.signalChildProcessTree({ pid: 123, kill: childKill }, "SIGTERM", {
platform: "win32",
runTaskkill,
});
expect(runTaskkill).toHaveBeenNthCalledWith(1, expectedTaskkillPath(), ["/PID", "123", "/T"], {
stdio: "ignore",
});
tuiPtyWatchTesting.signalChildProcessTree({ pid: 123, kill: childKill }, "SIGKILL", {
platform: "win32",
runTaskkill,
});
expect(runTaskkill).toHaveBeenNthCalledWith(
2,
expectedTaskkillPath(),
["/PID", "123", "/T", "/F"],
{
stdio: "ignore",
},
);
expect(childKill).not.toHaveBeenCalled();
});
it("force-kills Windows TUI PTY watch process trees when graceful taskkill fails", () => {
const childKill = vi.fn(() => true);
const runTaskkill = vi
.fn()
.mockReturnValueOnce({ error: undefined, status: 1 })
.mockReturnValueOnce({ error: undefined, status: 0 });
tuiPtyWatchTesting.signalChildProcessTree({ pid: 123, kill: childKill }, "SIGTERM", {
platform: "win32",
runTaskkill,
});
expect(runTaskkill).toHaveBeenNthCalledWith(1, expectedTaskkillPath(), ["/PID", "123", "/T"], {
stdio: "ignore",
});
expect(runTaskkill).toHaveBeenNthCalledWith(
2,
expectedTaskkillPath(),
["/PID", "123", "/T", "/F"],
{
stdio: "ignore",
},
);
expect(childKill).not.toHaveBeenCalled();
});
it("aborts stalled OpenAI realtime smoke fetches at the request timeout", async () => {
let signal: AbortSignal | undefined;
const request = realtimeSmokeTesting.createOpenAIClientSecret("test-key", {
test/scripts/docker-all-scheduler.test.ts
+2
View File
@@ -627,9 +627,11 @@ describe("scripts/test-docker-all scheduler", () => {
for (const fileName of [
"docker-e2e-plan.mts",
"docker-e2e-scenarios.mts",
"managed-child-process.mts",
"official-external-channel-catalog.json",
"release-version.mjs",
"sleep.mjs",
"windows-taskkill.mjs",
]) {
copyFileSync(path.join("scripts/lib", fileName), path.join(libDir, fileName));
}
test/scripts/managed-child-process.test.ts
+141
View File
@@ -7,9 +7,11 @@ import { pathToFileURL } from "node:url";
import { describe, expect, it, vi } from "vitest";
import {
createManagedCommandSpawnSpec,
inspectManagedProcessGroup,
runManagedCommand,
signalExitCode,
terminateManagedChild,
waitForManagedProcessGroupExit,
} from "../../scripts/lib/managed-child-process.mts";
import { createScriptTestHarness } from "./test-helpers.js";
@@ -199,6 +201,145 @@ describe("managed-child-process", () => {
});
});
it("preserves stdio-only taskkill and falls back after both trusted attempts fail", () => {
withDefaultWindowsSystemRoot(() => {
const child = { kill: vi.fn(() => true), pid: 12345 };
const runTaskkill = vi.fn(() => ({ error: undefined, status: 1 }));
expect(
terminateManagedChild(child, "SIGTERM", {
platform: "win32",
runTaskkill,
taskkillTimeoutMs: null,
}),
).toEqual({ processTreeState: "indeterminate" });
expect(runTaskkill).toHaveBeenNthCalledWith(1, taskkillPath, ["/PID", "12345", "/T"], {
stdio: "ignore",
});
expect(runTaskkill).toHaveBeenNthCalledWith(2, taskkillPath, ["/PID", "12345", "/T", "/F"], {
stdio: "ignore",
});
expect(child.kill).toHaveBeenCalledWith("SIGTERM");
});
});
it("preserves direct Windows signaling when a caller does not own taskkill", () => {
const child = { kill: vi.fn(() => true), pid: 12345 };
const runTaskkill = vi.fn();
expect(
terminateManagedChild(child, "SIGTERM", {
platform: "win32",
runTaskkill,
useWindowsTaskkill: false,
}),
).toEqual({ processTreeState: "signaled" });
expect(child.kill).toHaveBeenCalledWith("SIGTERM");
expect(runTaskkill).not.toHaveBeenCalled();
});
it("signals POSIX process groups without signaling their leaders twice", () => {
const kill = vi.spyOn(process, "kill").mockReturnValue(true);
const child = { kill: vi.fn(), pid: 12345 };
try {
expect(terminateManagedChild(child, "SIGTERM", { platform: "linux" })).toEqual({
processTreeState: "signaled",
});
expect(kill).toHaveBeenCalledWith(-12345, "SIGTERM");
expect(child.kill).not.toHaveBeenCalled();
} finally {
kill.mockRestore();
}
});
it.each([
{ code: "ESRCH", processGroupFallback: "nonmissing" as const },
{ code: "EPERM", processGroupFallback: "never" as const },
])("preserves caller-owned direct fallback for $code", ({ code, processGroupFallback }) => {
const error = Object.assign(new Error("process group unavailable"), { code });
const kill = vi.spyOn(process, "kill").mockImplementation(() => {
throw error;
});
const child = { kill: vi.fn(), pid: 12345 };
try {
terminateManagedChild(child, "SIGTERM", { platform: "linux", processGroupFallback });
expect(child.kill).not.toHaveBeenCalled();
} finally {
kill.mockRestore();
}
});
it("preserves distinct group permission policies and verifies the leader when requested", () => {
const permissionError = Object.assign(new Error("group signal denied"), { code: "EPERM" });
const child = { exitCode: null, pid: 12345, signalCode: null };
const kill = vi.spyOn(process, "kill").mockImplementation((pid) => {
if (pid === -12345) {
throw permissionError;
}
return true;
});
try {
expect(
inspectManagedProcessGroup(child, { errorPolicy: "alive-on-eperm", platform: "linux" }),
).toBe("live");
expect(
inspectManagedProcessGroup(child, { errorPolicy: "indeterminate", platform: "linux" }),
).toBe("indeterminate");
expect(
inspectManagedProcessGroup(child, { errorPolicy: "verify-leader", platform: "linux" }),
).toBe("live");
expect(kill).toHaveBeenCalledWith(12345, 0);
expect(
inspectManagedProcessGroup(
{ ...child, exitCode: 0 },
{ errorPolicy: "verify-leader", platform: "linux" },
),
).toBe("dead");
} finally {
kill.mockRestore();
}
});
it("inspects direct child liveness only when nongroup cleanup explicitly requires it", () => {
const child = { exitCode: null, pid: 12345, signalCode: null };
expect(
inspectManagedProcessGroup(child, { errorPolicy: "alive-on-eperm", platform: "win32" }),
).toBe("dead");
expect(
inspectManagedProcessGroup(child, {
errorPolicy: "alive-on-eperm",
inspectLeaderWhenNoGroup: true,
platform: "win32",
}),
).toBe("live");
expect(
inspectManagedProcessGroup(
{ ...child, exitCode: 0 },
{ errorPolicy: "alive-on-eperm", inspectLeaderWhenNoGroup: true, platform: "win32" },
),
).toBe("dead");
});
it("bounds process-group waiting when the group remains live", async () => {
const kill = vi.spyOn(process, "kill").mockReturnValue(true);
try {
await expect(
waitForManagedProcessGroupExit({ pid: 12345 }, 5, {
errorPolicy: "alive-on-eperm",
platform: "linux",
pollIntervalMs: 1,
}),
).resolves.toBe(false);
} finally {
kill.mockRestore();
}
});
it("signals the direct child when process-group ownership is disabled", () => {
const child = { kill: vi.fn(() => true), pid: 12345 };
test/scripts/test-group-report.test.ts
-75
View File
@@ -13,7 +13,6 @@ import {
resolveGroupKey,
resolveTestArea,
} from "../../scripts/lib/test-group-report.mts";
import { resolveWindowsTaskkillPath } from "../../scripts/lib/windows-taskkill.mjs";
import {
parseTestGroupReportArgs,
resolveFullSuiteVitestEnv,
@@ -23,7 +22,6 @@ import {
resolveRunPlanConcurrency,
resolveRunPlans,
runReportPlans,
signalTestGroupReportChild,
spawnText,
} from "../../scripts/test-group-report.mts";
import { withEnv } from "../../src/test-utils/env.js";
@@ -73,10 +71,6 @@ async function waitForDead(pid: number, timeoutMs: number): Promise<void> {
throw new Error(`timed out waiting for pid ${pid} to exit`);
}
function expectedTaskkillPath(): string {
return resolveWindowsTaskkillPath();
}
function waitForChildClose(
child: ReturnType<typeof spawn>,
timeoutMs = 5_000,
@@ -936,75 +930,6 @@ describe("scripts/test-group-report arg parsing", () => {
});
describe("scripts/test-group-report child process guard", () => {
it("signals Windows child process trees with taskkill", () => {
const child = {
kill: vi.fn(),
pid: 12345,
};
const runTaskkill = vi.fn(() => ({ error: undefined, status: 0 }));
signalTestGroupReportChild(child, "SIGTERM", {
platform: "win32",
runTaskkill,
});
expect(runTaskkill).toHaveBeenNthCalledWith(
1,
expectedTaskkillPath(),
["/PID", "12345", "/T"],
{
stdio: "ignore",
},
);
signalTestGroupReportChild(child, "SIGKILL", {
platform: "win32",
runTaskkill,
});
expect(runTaskkill).toHaveBeenNthCalledWith(
2,
expectedTaskkillPath(),
["/PID", "12345", "/T", "/F"],
{
stdio: "ignore",
},
);
expect(child.kill).not.toHaveBeenCalled();
});
it("force-kills Windows child process trees when graceful taskkill fails", () => {
const child = {
kill: vi.fn(),
pid: 12345,
};
const runTaskkill = vi
.fn()
.mockReturnValueOnce({ error: undefined, status: 1 })
.mockReturnValueOnce({ error: undefined, status: 0 });
signalTestGroupReportChild(child, "SIGTERM", {
platform: "win32",
runTaskkill,
});
expect(runTaskkill).toHaveBeenNthCalledWith(
1,
expectedTaskkillPath(),
["/PID", "12345", "/T"],
{
stdio: "ignore",
},
);
expect(runTaskkill).toHaveBeenNthCalledWith(
2,
expectedTaskkillPath(),
["/PID", "12345", "/T", "/F"],
{
stdio: "ignore",
},
);
expect(child.kill).not.toHaveBeenCalled();
});
it.concurrent("times out a child that ignores SIGTERM", async () => {
if (process.platform === "win32") {
return;