mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-28 13:26:04 -06:00
fix(qa): handle zombie proc stats in upgrade survivor (#125829)
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
4589d8514c
commit
1ecd53fe27
@@ -930,13 +930,15 @@ command="${filtered[0]:-status}"
|
||||
|
||||
is_running() {
|
||||
[ -s "$pid_file" ] || return 1
|
||||
local pid
|
||||
local process_state
|
||||
local pid stat_line stat_tail
|
||||
pid="$(cat "$pid_file" 2>/dev/null || true)"
|
||||
[ -n "$pid" ] || return 1
|
||||
kill -0 "$pid" >/dev/null 2>&1 || return 1
|
||||
process_state="$(awk '{ print $3 }' "/proc/$pid/stat" 2>/dev/null || true)"
|
||||
[ "$process_state" != "Z" ]
|
||||
stat_line="$(cat "/proc/$pid/stat" 2>/dev/null || true)"
|
||||
stat_tail="${stat_line##*) }"
|
||||
[[ "$stat_line" == "$pid ("*") $stat_tail" &&
|
||||
"$stat_tail" =~ ^Z([[:space:]]+-?[0-9]+){49,}$ ]] && return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
stop_gateway() {
|
||||
|
||||
@@ -41,13 +41,15 @@ command="${filtered[0]:-status}"
|
||||
|
||||
is_running() {
|
||||
[ -s "$pid_file" ] || return 1
|
||||
local pid
|
||||
local process_state
|
||||
local pid stat_line stat_tail
|
||||
pid="$(cat "$pid_file" 2>/dev/null || true)"
|
||||
[ -n "$pid" ] || return 1
|
||||
kill -0 "$pid" >/dev/null 2>&1 || return 1
|
||||
process_state="$(awk '{ print $3 }' "/proc/$pid/stat" 2>/dev/null || true)"
|
||||
[ "$process_state" != "Z" ]
|
||||
stat_line="$(cat "/proc/$pid/stat" 2>/dev/null || true)"
|
||||
stat_tail="${stat_line##*) }"
|
||||
[[ "$stat_line" == "$pid ("*") $stat_tail" &&
|
||||
"$stat_tail" =~ ^Z([[:space:]]+-?[0-9]+){49,}$ ]] && return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
stop_gateway() {
|
||||
|
||||
@@ -216,6 +216,15 @@ function extractUpgradeSurvivorSupervisor(script: string): string {
|
||||
return source;
|
||||
}
|
||||
|
||||
function extractUpgradeSurvivorSystemctlShim(script: string): string {
|
||||
const match = script.match(/cat >"\$shim_dir\/systemctl" <<'SHIM'\n(?<source>[\s\S]*?)\nSHIM/u);
|
||||
const source = match?.groups?.source;
|
||||
if (!source) {
|
||||
throw new Error("upgrade survivor systemctl shim source not found");
|
||||
}
|
||||
return source;
|
||||
}
|
||||
|
||||
async function waitForProcessExit(child: ChildProcess, timeoutMs = 5_000): Promise<number | null> {
|
||||
if (child.exitCode !== null || child.signalCode !== null) {
|
||||
return child.exitCode;
|
||||
@@ -254,6 +263,75 @@ setInterval(() => {}, 1_000);
|
||||
return descendantPath;
|
||||
}
|
||||
|
||||
async function forEachUpgradeSurvivorSystemctlShim(
|
||||
callback: (fixture: {
|
||||
pid: number;
|
||||
run: (command: "is-active" | "stop", procStat?: string) => number | null;
|
||||
scriptPath: string;
|
||||
}) => void | Promise<void>,
|
||||
): Promise<void> {
|
||||
for (const scriptPath of [
|
||||
UPGRADE_SURVIVOR_RUN_SCRIPT,
|
||||
UPGRADE_SURVIVOR_UPDATE_RESTART_AUTH_PATH,
|
||||
]) {
|
||||
const workDir = tempDirs.make("openclaw-systemctl-shim-");
|
||||
const binDir = join(workDir, "bin");
|
||||
const pidPath = join(workDir, "gateway.pid");
|
||||
const childPidPath = join(workDir, "child.pid");
|
||||
const child = spawn(process.execPath, [writeTermIgnoringDescendant(workDir)], {
|
||||
env: { ...process.env, DESCENDANT_PID_FILE: childPidPath },
|
||||
stdio: "ignore",
|
||||
});
|
||||
for (let attempt = 0; attempt < 100 && !existsSync(childPidPath); attempt += 1) {
|
||||
await delay(10);
|
||||
}
|
||||
const pid = Number.parseInt(readFileSync(childPidPath, "utf8"), 10);
|
||||
writeFileSync(pidPath, `${pid}\n`);
|
||||
const shimPath = join(workDir, "systemctl");
|
||||
writeFileSync(shimPath, extractUpgradeSurvivorSystemctlShim(readFileSync(scriptPath, "utf8")), {
|
||||
mode: 0o755,
|
||||
});
|
||||
writeExecutables(binDir, {
|
||||
awk: `#!/usr/bin/env bash
|
||||
[ "$FAKE_PROC_STAT_MODE" != "unreadable" ] || exit 1
|
||||
set -- $FAKE_PROC_STAT
|
||||
printf '%s\\n' "\${3:-}"
|
||||
`,
|
||||
cat: `#!/usr/bin/env bash
|
||||
case "\${1:-}" in
|
||||
/proc/*/stat)
|
||||
[ "$FAKE_PROC_STAT_MODE" != "unreadable" ] || exit 1
|
||||
printf '%s\\n' "$FAKE_PROC_STAT"
|
||||
;;
|
||||
*) exec /bin/cat "$@" ;;
|
||||
esac
|
||||
`,
|
||||
sleep: "#!/usr/bin/env bash\nexit 97\n",
|
||||
});
|
||||
const run = (command: "is-active" | "stop", procStat?: string) =>
|
||||
spawnSync("bash", [shimPath, "--user", command, "openclaw-gateway.service"], {
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
FAKE_PROC_STAT: procStat ?? "",
|
||||
FAKE_PROC_STAT_MODE: procStat === undefined ? "unreadable" : "readable",
|
||||
OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_LOG: join(workDir, "systemctl.log"),
|
||||
OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE: pidPath,
|
||||
PATH: `${binDir}:${process.env.PATH ?? ""}`,
|
||||
},
|
||||
}).status;
|
||||
|
||||
try {
|
||||
await callback({ pid, run, scriptPath });
|
||||
} finally {
|
||||
if (child.exitCode === null && child.signalCode === null) {
|
||||
child.kill("SIGKILL");
|
||||
}
|
||||
await waitForProcessExit(child).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function cleanupSmokeLogTailHelpers(): string {
|
||||
const script = readFileSync(CLEANUP_SMOKE_RUN_PATH, "utf8");
|
||||
const match = script.match(
|
||||
@@ -3044,7 +3122,6 @@ fi
|
||||
for (const script of [publishedRunner, updateRestartAuth]) {
|
||||
expectTextToIncludeAll(script, [
|
||||
'supervisor_script="${pid_file}.supervisor.mjs"',
|
||||
'process_state="$(awk \'{ print $3 }\' "/proc/$pid/stat" 2>/dev/null || true)"',
|
||||
'OPENCLAW_SYSTEMCTL_SHIM_EXEC_START="$exec_start"',
|
||||
'nohup node "$supervisor_script"',
|
||||
'if (key.startsWith("OPENCLAW_UPDATE_")) {',
|
||||
@@ -3070,6 +3147,27 @@ fi
|
||||
}
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"stops promptly when the systemctl target is a zombie with spaces and parentheses in comm",
|
||||
async () => {
|
||||
await forEachUpgradeSurvivorSystemctlShim(({ pid, run, scriptPath }) => {
|
||||
const procTail = Array.from({ length: 49 }, (_, field) => field + 1).join(" ");
|
||||
expect(run("stop", `${pid} (gateway (old) worker) Z ${procTail}`), scriptPath).toBe(0);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"keeps a killable systemctl target active when proc stat is unreadable or malformed",
|
||||
async () => {
|
||||
await forEachUpgradeSurvivorSystemctlShim(({ pid, run, scriptPath }) => {
|
||||
for (const procStat of [undefined, `${pid} (gateway) Z`]) {
|
||||
expect(run("is-active", procStat), `${scriptPath}: ${procStat ?? "unreadable"}`).toBe(0);
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("stops supervised gateway restarts after the systemd burst limit", async () => {
|
||||
const workDir = tempDirs.make("openclaw-update-restart-supervisor-");
|
||||
const scripts = [
|
||||
|
||||
Reference in New Issue
Block a user