mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-26 04:15:48 -06:00
fix(tooling): arm prompt cleanup before spawn (#129755)
This commit is contained in:
@@ -21,6 +21,11 @@ import {
|
||||
parseStrictIntegerOption,
|
||||
redactForDevToolLog,
|
||||
} from "./lib/dev-tooling-safety.ts";
|
||||
import {
|
||||
inspectManagedProcessGroup,
|
||||
signalExitCode,
|
||||
terminateManagedChild,
|
||||
} from "./lib/managed-child-process.mts";
|
||||
|
||||
const TRANSPORT = process.env.OPENCLAW_PROMPT_TRANSPORT?.trim() === "direct" ? "direct" : "gateway";
|
||||
const GATEWAY_PROMPT_MODE = "extra";
|
||||
@@ -56,11 +61,6 @@ const GATEWAY_TIMEOUT_MS = parseStrictIntegerOption({
|
||||
min: 1,
|
||||
raw: process.env.OPENCLAW_PROMPT_GATEWAY_TIMEOUT_MS,
|
||||
});
|
||||
const GATEWAY_PARENT_SIGNAL_EXIT_CODES = new Map<NodeJS.Signals, number>([
|
||||
["SIGHUP", 129],
|
||||
["SIGINT", 130],
|
||||
["SIGTERM", 143],
|
||||
]);
|
||||
const CAPTURE_PROXY_MAX_BODY_BYTES = parseStrictIntegerOption({
|
||||
fallback: 2 * 1024 * 1024,
|
||||
label: "OPENCLAW_PROMPT_CAPTURE_MAX_BODY_BYTES",
|
||||
@@ -497,6 +497,7 @@ async function runDirectPrompt(
|
||||
timeoutMs,
|
||||
})
|
||||
: undefined;
|
||||
const parentSignalController = createPromptProbeParentSignalController();
|
||||
|
||||
try {
|
||||
const stdout: string[] = [];
|
||||
@@ -524,18 +525,23 @@ async function runDirectPrompt(
|
||||
child.once("exit", (code, signal) => resolve({ code, signal }));
|
||||
},
|
||||
);
|
||||
const stopDirectChild = async (signal: NodeJS.Signals = "SIGKILL") => {
|
||||
signalGatewayPromptChildTree(child, signal);
|
||||
await waitForGatewayPromptChildTreeExit(
|
||||
child,
|
||||
exitPromise.then(() => undefined),
|
||||
options.shutdownWaitMs ?? 1_500,
|
||||
);
|
||||
let stopPromise: Promise<void> | undefined;
|
||||
const stopDirectChild = (signal: NodeJS.Signals = "SIGKILL") => {
|
||||
if (!stopPromise) {
|
||||
terminateManagedChild(child, signal, { useWindowsTaskkill: false });
|
||||
stopPromise = waitForGatewayPromptChildTreeExit(
|
||||
child,
|
||||
options.shutdownWaitMs ?? 1_500,
|
||||
).then(() => undefined);
|
||||
}
|
||||
return stopPromise;
|
||||
};
|
||||
const removeParentSignalHandlers = installGatewayPromptParentSignalHandlers(
|
||||
child,
|
||||
stopDirectChild,
|
||||
);
|
||||
parentSignalController.attach({
|
||||
stop: stopDirectChild,
|
||||
forceKill: () => {
|
||||
terminateManagedChild(child, "SIGKILL", { useWindowsTaskkill: false });
|
||||
},
|
||||
});
|
||||
let timeoutTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
const exit = await Promise.race([
|
||||
exitPromise,
|
||||
@@ -550,7 +556,6 @@ async function runDirectPrompt(
|
||||
if (timeoutTimer) {
|
||||
clearTimeout(timeoutTimer);
|
||||
}
|
||||
removeParentSignalHandlers();
|
||||
});
|
||||
const joinedStdout = stdout.join("");
|
||||
const joinedStderr = stderr.join("");
|
||||
@@ -567,6 +572,7 @@ async function runDirectPrompt(
|
||||
...promptProbeTmpResult(tmpDir),
|
||||
};
|
||||
} finally {
|
||||
parentSignalController.dispose();
|
||||
await proxy?.stop().catch(() => {});
|
||||
await cleanupPromptProbeTmpDir(tmpDir).catch(() => {});
|
||||
}
|
||||
@@ -582,6 +588,7 @@ async function startGatewayProcess(params: {
|
||||
logPath: string;
|
||||
}) {
|
||||
const logFile = await fs.open(params.logPath, "a");
|
||||
const parentSignalController = createPromptProbeParentSignalController();
|
||||
const child = spawn(
|
||||
NODE_BIN,
|
||||
["openclaw.mjs", "gateway", "--port", String(params.port), "--bind", "loopback", "--force"],
|
||||
@@ -625,7 +632,6 @@ async function startGatewayProcess(params: {
|
||||
child.stdout.on("data", trackLogWrite);
|
||||
child.stderr.on("data", trackLogWrite);
|
||||
let stopPromise: Promise<boolean> | undefined;
|
||||
let removeParentSignalHandlers = () => {};
|
||||
const stopOnce = async (): Promise<boolean> => {
|
||||
stopPromise ??= stopGatewayPromptChild(
|
||||
child,
|
||||
@@ -635,11 +641,16 @@ async function startGatewayProcess(params: {
|
||||
pendingLogWrites,
|
||||
logWriteErrors,
|
||||
).finally(() => {
|
||||
removeParentSignalHandlers();
|
||||
parentSignalController.dispose();
|
||||
});
|
||||
return await stopPromise;
|
||||
};
|
||||
removeParentSignalHandlers = installGatewayPromptParentSignalHandlers(child, stopOnce);
|
||||
parentSignalController.attach({
|
||||
stop: stopOnce,
|
||||
forceKill: () => {
|
||||
terminateManagedChild(child, "SIGKILL", { useWindowsTaskkill: false });
|
||||
},
|
||||
});
|
||||
return {
|
||||
async stop(): Promise<boolean> {
|
||||
return await stopOnce();
|
||||
@@ -656,25 +667,18 @@ async function stopGatewayPromptChild(
|
||||
logWriteErrors: readonly unknown[] = [],
|
||||
): Promise<boolean> {
|
||||
let exited = child.exitCode !== null || child.signalCode !== null;
|
||||
const exitPromise = exited
|
||||
? Promise.resolve()
|
||||
: new Promise<void>((resolve) => {
|
||||
child.once("exit", () => {
|
||||
exited = true;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
if (!exited) {
|
||||
signalGatewayPromptChildTree(child, "SIGINT");
|
||||
child.once("exit", () => {
|
||||
exited = true;
|
||||
});
|
||||
}
|
||||
const exitedAfterSigint = await waitForGatewayPromptChildTreeExit(
|
||||
child,
|
||||
exitPromise,
|
||||
sigintTimeoutMs,
|
||||
);
|
||||
if (!exited) {
|
||||
terminateManagedChild(child, "SIGINT", { useWindowsTaskkill: false });
|
||||
}
|
||||
const exitedAfterSigint = await waitForGatewayPromptChildTreeExit(child, sigintTimeoutMs);
|
||||
if (!exitedAfterSigint) {
|
||||
signalGatewayPromptChildTree(child, "SIGKILL");
|
||||
await waitForGatewayPromptChildTreeExit(child, exitPromise, sigkillTimeoutMs);
|
||||
terminateManagedChild(child, "SIGKILL", { useWindowsTaskkill: false });
|
||||
await waitForGatewayPromptChildTreeExit(child, sigkillTimeoutMs);
|
||||
}
|
||||
const failedLogWrite = (await Promise.allSettled(pendingLogWrites)).find(
|
||||
(result): result is PromiseRejectedResult => result.status === "rejected",
|
||||
@@ -687,91 +691,87 @@ async function stopGatewayPromptChild(
|
||||
return exited;
|
||||
}
|
||||
|
||||
function installGatewayPromptParentSignalHandlers(
|
||||
child: StoppableGatewayChild,
|
||||
stopGateway: () => Promise<unknown>,
|
||||
): () => void {
|
||||
let parentSignalShutdownStarted = false;
|
||||
// Arm before spawn so a parent signal is retained until child cleanup can attach.
|
||||
// The first signal owns the exit code; later signals only escalate cleanup.
|
||||
function createPromptProbeParentSignalController() {
|
||||
let attachment: { forceKill(): void; stop(): Promise<unknown> } | undefined;
|
||||
let forceKillPending = false;
|
||||
let receivedSignal: NodeJS.Signals | undefined;
|
||||
let shutdownPromise: Promise<unknown> | undefined;
|
||||
const handlers = new Map<NodeJS.Signals, () => void>();
|
||||
const removeHandlers = () => {
|
||||
const dispose = () => {
|
||||
for (const [signal, handler] of handlers) {
|
||||
process.off(signal, handler);
|
||||
}
|
||||
handlers.clear();
|
||||
};
|
||||
for (const signal of GATEWAY_PARENT_SIGNAL_EXIT_CODES.keys()) {
|
||||
const forceKill = () => {
|
||||
try {
|
||||
attachment?.forceKill();
|
||||
} catch {}
|
||||
};
|
||||
const startShutdown = () => {
|
||||
if (!attachment || !receivedSignal || shutdownPromise) {
|
||||
return;
|
||||
}
|
||||
const attached = attachment;
|
||||
const exitCode = signalExitCode(receivedSignal);
|
||||
if (forceKillPending) {
|
||||
forceKillPending = false;
|
||||
forceKill();
|
||||
}
|
||||
shutdownPromise = Promise.resolve()
|
||||
.then(() => attached.stop())
|
||||
.catch(() => undefined)
|
||||
.finally(() => {
|
||||
dispose();
|
||||
process.exit(exitCode);
|
||||
});
|
||||
};
|
||||
for (const signal of ["SIGHUP", "SIGINT", "SIGTERM"] satisfies NodeJS.Signals[]) {
|
||||
const handler = () => {
|
||||
if (parentSignalShutdownStarted) {
|
||||
signalGatewayPromptChildTree(child, "SIGKILL");
|
||||
if (receivedSignal) {
|
||||
if (attachment) {
|
||||
forceKill();
|
||||
} else {
|
||||
forceKillPending = true;
|
||||
}
|
||||
return;
|
||||
}
|
||||
parentSignalShutdownStarted = true;
|
||||
void stopGateway()
|
||||
.catch(() => undefined)
|
||||
.finally(() => {
|
||||
removeHandlers();
|
||||
process.exit(GATEWAY_PARENT_SIGNAL_EXIT_CODES.get(signal) ?? 1);
|
||||
});
|
||||
receivedSignal = signal;
|
||||
startShutdown();
|
||||
};
|
||||
handlers.set(signal, handler);
|
||||
process.on(signal, handler);
|
||||
}
|
||||
return removeHandlers;
|
||||
return {
|
||||
attach(nextAttachment: NonNullable<typeof attachment>) {
|
||||
attachment ??= nextAttachment;
|
||||
startShutdown();
|
||||
},
|
||||
dispose,
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForGatewayPromptChildTreeExit(
|
||||
child: StoppableGatewayChild,
|
||||
exitPromise: Promise<void>,
|
||||
timeoutMs: number,
|
||||
): Promise<boolean> {
|
||||
let leaderExited = child.exitCode !== null || child.signalCode !== null;
|
||||
const trackedExit = exitPromise.then(() => {
|
||||
leaderExited = true;
|
||||
});
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const groupOptions = {
|
||||
errorPolicy: "alive-on-eperm",
|
||||
inspectLeaderWhenNoGroup: true,
|
||||
} as const;
|
||||
const childTreeExited = () =>
|
||||
(child.exitCode !== null || child.signalCode !== null) &&
|
||||
inspectManagedProcessGroup(child, groupOptions) !== "live";
|
||||
while (Date.now() < deadline) {
|
||||
if (leaderExited && !gatewayPromptChildTreeIsAlive(child)) {
|
||||
if (childTreeExited()) {
|
||||
return true;
|
||||
}
|
||||
const waitMs = Math.min(50, Math.max(0, deadline - Date.now()));
|
||||
if (leaderExited) {
|
||||
await sleep(waitMs);
|
||||
} else {
|
||||
await Promise.race([trackedExit, sleep(waitMs)]);
|
||||
}
|
||||
await sleep(Math.min(25, Math.max(0, deadline - Date.now())));
|
||||
}
|
||||
return leaderExited && !gatewayPromptChildTreeIsAlive(child);
|
||||
}
|
||||
|
||||
function signalGatewayPromptChildTree(
|
||||
child: StoppableGatewayChild,
|
||||
signal: NodeJS.Signals,
|
||||
): boolean {
|
||||
if (process.platform !== "win32" && typeof child.pid === "number") {
|
||||
try {
|
||||
process.kill(-child.pid, signal);
|
||||
return true;
|
||||
} catch {
|
||||
return child.kill(signal);
|
||||
}
|
||||
}
|
||||
return child.kill(signal);
|
||||
}
|
||||
|
||||
function gatewayPromptChildTreeIsAlive(child: StoppableGatewayChild): boolean {
|
||||
if (process.platform === "win32" || typeof child.pid !== "number") {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
process.kill(-child.pid, 0);
|
||||
return true;
|
||||
} catch (error) {
|
||||
return !isMissingProcessError(error);
|
||||
}
|
||||
}
|
||||
|
||||
function isMissingProcessError(error: unknown): boolean {
|
||||
return typeof error === "object" && error !== null && "code" in error && error.code === "ESRCH";
|
||||
return childTreeExited();
|
||||
}
|
||||
|
||||
async function waitForGatewayReady(url: string, token: string): Promise<void> {
|
||||
@@ -1006,7 +1006,7 @@ async function main() {
|
||||
|
||||
export const testing = {
|
||||
cleanupPromptProbeTmpDir,
|
||||
installGatewayPromptParentSignalHandlers,
|
||||
createPromptProbeParentSignalController,
|
||||
promptProbeTmpResult,
|
||||
readLogTail,
|
||||
readRequestBody,
|
||||
|
||||
@@ -940,7 +940,7 @@ describe("script-specific dev tooling hardening", () => {
|
||||
);
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"cleans Anthropic prompt gateway descendants on parent signal",
|
||||
"cleans Anthropic prompt gateway descendants when the child attaches after parent signal",
|
||||
async () => {
|
||||
const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-prompt-parent-signal-"));
|
||||
tempDirs.push(tempRoot);
|
||||
@@ -972,13 +972,21 @@ describe("script-specific dev tooling hardening", () => {
|
||||
`const { testing } = await import(${JSON.stringify(
|
||||
pathToFileURL(path.resolve("scripts/anthropic-prompt-probe.ts")).href,
|
||||
)});`,
|
||||
"const signalController = testing.createPromptProbeParentSignalController();",
|
||||
`const child = childProcess.spawn(process.execPath, ['--input-type=module', '--eval', ${JSON.stringify(leaderScript)}], { detached: true, stdio: 'ignore' });`,
|
||||
"let stopPromise;",
|
||||
"const stopGateway = () => {",
|
||||
" stopPromise ??= testing.stopGatewayPromptChild(child, { close: async () => {} }, 50, 100);",
|
||||
" return stopPromise;",
|
||||
"};",
|
||||
"testing.installGatewayPromptParentSignalHandlers(child, stopGateway);",
|
||||
"process.on('SIGTERM', () => {",
|
||||
" signalController.attach({",
|
||||
" stop: stopGateway,",
|
||||
" forceKill: () => {",
|
||||
" try { process.kill(-child.pid, 'SIGKILL'); } catch { child.kill('SIGKILL'); }",
|
||||
" },",
|
||||
" });",
|
||||
"});",
|
||||
`fs.writeFileSync(${JSON.stringify(readyPath)}, String(process.pid));`,
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("\n"),
|
||||
|
||||
Reference in New Issue
Block a user