fix(qa): prevent false update-restart package failures (#120300)

* test(qa): expose update restart process exits * test(qa): supervise update restart gateway * test(qa): isolate supervised gateway environment * test(qa): diagnose restart plugin convergence * test(qa): mount trusted upgrade harness * test(qa): include upgrade runtime companions * test(qa): remove temporary restart diagnostics * test(qa): enforce systemd restart budget * test(qa): honor systemd stop timeout * test(qa): tighten package fixture ownership * test(qa): preserve unrelated lane shapes * test(qa): complete service fixture boundaries
2026-08-28 05:16:23 -06:00 · 2026-08-07 14:46:59 -07:00
parent 9195bd55c2
commit fb0812c857
9 changed files with 681 additions and 28 deletions
@@ -230,6 +230,9 @@ cleanup() {
  if [ -n "${plugin_registry_pid:-}" ]; then
    kill "$plugin_registry_pid" >/dev/null 2>&1 || true
  fi
+  if [ -s "$SYSTEMCTL_SHIM_PID_FILE" ]; then
+    systemctl --user stop openclaw-gateway.service >/dev/null 2>&1 || true
+  fi
  openclaw_e2e_terminate_gateways "${gateway_pid:-}"
  if [ -s "$SYSTEMCTL_SHIM_PID_FILE" ]; then
    local shim_pid
@@ -784,6 +787,7 @@ set -euo pipefail
 log_file="${OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_LOG:-/tmp/openclaw-systemctl-shim.log}"
 pid_file="${OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE:-/tmp/openclaw-systemctl-shim.pid}"
 daemon_log="${OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_DAEMON_LOG:-/tmp/openclaw-systemctl-shim-gateway.log}"
+supervisor_script="${pid_file}.supervisor.mjs"
 printf '%s\n' "$*" >>"$log_file"

 filtered=()
@@ -815,24 +819,27 @@ command="${filtered[0]:-status}"
 is_running() {
  [ -s "$pid_file" ] || return 1
  local pid
+  local process_state
  pid="$(cat "$pid_file" 2>/dev/null || true)"
  [ -n "$pid" ] || return 1
-  kill -0 "$pid" >/dev/null 2>&1
+  kill -0 "$pid" >/dev/null 2>&1 || return 1
+  process_state="$(awk '{ print $3 }' "/proc/$pid/stat" 2>/dev/null || true)"
+  [ "$process_state" != "Z" ]
 }

 stop_gateway() {
-  [ -s "$pid_file" ] || return 0
-  local pid
+  local pid=""
  pid="$(cat "$pid_file" 2>/dev/null || true)"
  if [[ "$pid" =~ ^[0-9]+$ ]] && [ "$pid" -gt 1 ] && kill -0 "$pid" >/dev/null 2>&1; then
    kill "$pid" >/dev/null 2>&1 || true
-    for _ in $(seq 1 100); do
-      kill -0 "$pid" >/dev/null 2>&1 || break
+    # The supervisor gives its child 30s, so keep this outer deadline comfortably longer.
+    for _ in $(seq 1 350); do
+      is_running || break
      sleep 0.1
    done
    kill -9 "$pid" >/dev/null 2>&1 || true
  fi
-  rm -f "$pid_file"
+  rm -f "$pid_file" "$supervisor_script"
 }

 unit_path() {
@@ -873,9 +880,149 @@ start_gateway() {
    echo "systemctl shim could not find ExecStart in $unit" >&2
    return 1
  }
+  rm -f "$pid_file" "$supervisor_script"
+  cat >"$supervisor_script" <<'SUPERVISOR'
+import fs from "node:fs";
+import { spawn } from "node:child_process";
+
+const command = process.env.OPENCLAW_SYSTEMCTL_SHIM_EXEC_START;
+const daemonLog = process.env.OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG;
+if (!command || !daemonLog) {
+  process.exit(2);
+}
+
+const output = fs.openSync(daemonLog, "a");
+const childEnv = { ...process.env };
+delete childEnv.OPENCLAW_SYSTEMCTL_SHIM_EXEC_START;
+delete childEnv.OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG;
+// systemd does not pass transient systemctl-caller update state into the service.
+for (const key of Object.keys(childEnv)) {
+  if (key.startsWith("OPENCLAW_UPDATE_")) {
+    delete childEnv[key];
+  }
+}
+delete childEnv.OPENCLAW_COMPATIBILITY_HOST_VERSION;
+const restartDelayMs = 5_000;
+const restartWindowMs = 60_000;
+const restartBurst = 5;
+const stopTimeoutMs = 30_000;
+const starts = [];
+let child;
+let activeGroupPid;
+let drainingGroupPid;
+let stopping = false;
+
+const finish = () => {
+  try {
+    fs.closeSync(output);
+  } catch {}
+  process.exit(0);
+};
+
+const signalProcessGroup = (pid, signal) => {
+  try {
+    process.kill(-pid, signal);
+  } catch (error) {
+    if (error?.code !== "ESRCH") {
+      fs.writeSync(output, `[systemctl-shim] gateway process group ${signal} failed: ${String(error)}\n`);
+    }
+  }
+};
+
+const isProcessGroupRunning = (pid) => {
+  try {
+    process.kill(-pid, 0);
+    return true;
+  } catch (error) {
+    return error?.code !== "ESRCH";
+  }
+};
+
+const drainProcessGroup = (pid, onStopped) => {
+  if (!pid) return onStopped();
+  if (drainingGroupPid === pid) return;
+  drainingGroupPid = pid;
+  let completed = false;
+  const complete = () => {
+    if (completed) return;
+    completed = true;
+    if (drainingGroupPid === pid) drainingGroupPid = undefined;
+    if (activeGroupPid === pid) activeGroupPid = undefined;
+    onStopped();
+  };
+  signalProcessGroup(pid, "SIGTERM");
+  const forceKill = setTimeout(() => {
+    signalProcessGroup(pid, "SIGKILL");
+    complete();
+  }, stopTimeoutMs);
+  const finishWhenStopped = () => {
+    if (completed) return;
+    if (isProcessGroupRunning(pid)) {
+      setTimeout(finishWhenStopped, 25);
+      return;
+    }
+    clearTimeout(forceKill);
+    complete();
+  };
+  finishWhenStopped();
+};
+
+const stop = () => {
+  if (stopping) return;
+  stopping = true;
+  if (drainingGroupPid) return;
+  if (activeGroupPid) {
+    drainProcessGroup(activeGroupPid, finish);
+    return;
+  }
+  if (child) {
+    child.kill("SIGTERM");
+    return;
+  }
+  finish();
+};
+
+const start = () => {
+  if (stopping) return finish();
+  const now = Date.now();
+  while (starts.length > 0 && starts[0] <= now - restartWindowMs) {
+    starts.shift();
+  }
+  if (starts.length >= restartBurst) {
+    fs.writeSync(output, "[systemctl-shim] gateway restart limit reached\n");
+    return finish();
+  }
+  starts.push(now);
+  child = spawn("bash", ["-lc", `exec ${command}`], {
+    detached: true,
+    env: childEnv,
+    stdio: ["ignore", output, output],
+  });
+  activeGroupPid = child.pid;
+  const childGroupPid = activeGroupPid;
+  child.on("error", (error) => {
+    fs.writeSync(output, `[systemctl-shim] gateway spawn failed: ${String(error)}\n`);
+  });
+  child.once("close", (code) => {
+    child = undefined;
+    drainProcessGroup(childGroupPid, () => {
+      if (stopping) return finish();
+      // Match the generated systemd unit's RestartPreventExitStatus contract.
+      if (code === 78) return finish();
+      setTimeout(start, restartDelayMs);
+    });
+  });
+};
+
+process.on("SIGINT", stop);
+process.on("SIGTERM", stop);
+start();
+SUPERVISOR
  (
    load_unit_environment "$unit"
-    nohup bash -lc "exec $exec_start" >>"$daemon_log" 2>&1 &
+    OPENCLAW_SYSTEMCTL_SHIM_EXEC_START="$exec_start" \
+      OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG="$daemon_log" \
+      nohup node "$supervisor_script" </dev/null >/dev/null 2>&1 &
    printf '%s\n' "$!" >"$pid_file"
  )
 }
@@ -10,6 +10,7 @@ set -euo pipefail
 log_file="${OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_LOG:-/tmp/openclaw-systemctl-shim.log}"
 pid_file="${OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE:-/tmp/openclaw-systemctl-shim.pid}"
 daemon_log="${OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_DAEMON_LOG:-/tmp/openclaw-systemctl-shim-gateway.log}"
+supervisor_script="${pid_file}.supervisor.mjs"
 printf '%s\n' "$*" >>"$log_file"

 filtered=()
@@ -41,24 +42,27 @@ command="${filtered[0]:-status}"
 is_running() {
  [ -s "$pid_file" ] || return 1
  local pid
+  local process_state
  pid="$(cat "$pid_file" 2>/dev/null || true)"
  [ -n "$pid" ] || return 1
-  kill -0 "$pid" >/dev/null 2>&1
+  kill -0 "$pid" >/dev/null 2>&1 || return 1
+  process_state="$(awk '{ print $3 }' "/proc/$pid/stat" 2>/dev/null || true)"
+  [ "$process_state" != "Z" ]
 }

 stop_gateway() {
-  [ -s "$pid_file" ] || return 0
-  local pid
+  local pid=""
  pid="$(cat "$pid_file" 2>/dev/null || true)"
  if [[ "$pid" =~ ^[0-9]+$ ]] && [ "$pid" -gt 1 ] && kill -0 "$pid" >/dev/null 2>&1; then
    kill "$pid" >/dev/null 2>&1 || true
-    for _ in $(seq 1 100); do
-      kill -0 "$pid" >/dev/null 2>&1 || break
+    # The supervisor gives its child 30s, so keep this outer deadline comfortably longer.
+    for _ in $(seq 1 350); do
+      is_running || break
      sleep 0.1
    done
    kill -9 "$pid" >/dev/null 2>&1 || true
  fi
-  rm -f "$pid_file"
+  rm -f "$pid_file" "$supervisor_script"
 }

 unit_path() {
@@ -99,9 +103,149 @@ start_gateway() {
    echo "systemctl shim could not find ExecStart in $unit" >&2
    return 1
  }
+  rm -f "$pid_file" "$supervisor_script"
+  cat >"$supervisor_script" <<'SUPERVISOR'
+import fs from "node:fs";
+import { spawn } from "node:child_process";
+
+const command = process.env.OPENCLAW_SYSTEMCTL_SHIM_EXEC_START;
+const daemonLog = process.env.OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG;
+if (!command || !daemonLog) {
+  process.exit(2);
+}
+
+const output = fs.openSync(daemonLog, "a");
+const childEnv = { ...process.env };
+delete childEnv.OPENCLAW_SYSTEMCTL_SHIM_EXEC_START;
+delete childEnv.OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG;
+// systemd does not pass transient systemctl-caller update state into the service.
+for (const key of Object.keys(childEnv)) {
+  if (key.startsWith("OPENCLAW_UPDATE_")) {
+    delete childEnv[key];
+  }
+}
+delete childEnv.OPENCLAW_COMPATIBILITY_HOST_VERSION;
+const restartDelayMs = 5_000;
+const restartWindowMs = 60_000;
+const restartBurst = 5;
+const stopTimeoutMs = 30_000;
+const starts = [];
+let child;
+let activeGroupPid;
+let drainingGroupPid;
+let stopping = false;
+
+const finish = () => {
+  try {
+    fs.closeSync(output);
+  } catch {}
+  process.exit(0);
+};
+
+const signalProcessGroup = (pid, signal) => {
+  try {
+    process.kill(-pid, signal);
+  } catch (error) {
+    if (error?.code !== "ESRCH") {
+      fs.writeSync(output, `[systemctl-shim] gateway process group ${signal} failed: ${String(error)}\n`);
+    }
+  }
+};
+
+const isProcessGroupRunning = (pid) => {
+  try {
+    process.kill(-pid, 0);
+    return true;
+  } catch (error) {
+    return error?.code !== "ESRCH";
+  }
+};
+
+const drainProcessGroup = (pid, onStopped) => {
+  if (!pid) return onStopped();
+  if (drainingGroupPid === pid) return;
+  drainingGroupPid = pid;
+  let completed = false;
+  const complete = () => {
+    if (completed) return;
+    completed = true;
+    if (drainingGroupPid === pid) drainingGroupPid = undefined;
+    if (activeGroupPid === pid) activeGroupPid = undefined;
+    onStopped();
+  };
+  signalProcessGroup(pid, "SIGTERM");
+  const forceKill = setTimeout(() => {
+    signalProcessGroup(pid, "SIGKILL");
+    complete();
+  }, stopTimeoutMs);
+  const finishWhenStopped = () => {
+    if (completed) return;
+    if (isProcessGroupRunning(pid)) {
+      setTimeout(finishWhenStopped, 25);
+      return;
+    }
+    clearTimeout(forceKill);
+    complete();
+  };
+  finishWhenStopped();
+};
+
+const stop = () => {
+  if (stopping) return;
+  stopping = true;
+  if (drainingGroupPid) return;
+  if (activeGroupPid) {
+    drainProcessGroup(activeGroupPid, finish);
+    return;
+  }
+  if (child) {
+    child.kill("SIGTERM");
+    return;
+  }
+  finish();
+};
+
+const start = () => {
+  if (stopping) return finish();
+  const now = Date.now();
+  while (starts.length > 0 && starts[0] <= now - restartWindowMs) {
+    starts.shift();
+  }
+  if (starts.length >= restartBurst) {
+    fs.writeSync(output, "[systemctl-shim] gateway restart limit reached\n");
+    return finish();
+  }
+  starts.push(now);
+  child = spawn("bash", ["-lc", `exec ${command}`], {
+    detached: true,
+    env: childEnv,
+    stdio: ["ignore", output, output],
+  });
+  activeGroupPid = child.pid;
+  const childGroupPid = activeGroupPid;
+  child.on("error", (error) => {
+    fs.writeSync(output, `[systemctl-shim] gateway spawn failed: ${String(error)}\n`);
+  });
+  child.once("close", (code) => {
+    child = undefined;
+    drainProcessGroup(childGroupPid, () => {
+      if (stopping) return finish();
+      // Match the generated systemd unit's RestartPreventExitStatus contract.
+      if (code === 78) return finish();
+      setTimeout(start, restartDelayMs);
+    });
+  });
+};
+
+process.on("SIGINT", stop);
+process.on("SIGTERM", stop);
+start();
+SUPERVISOR
  (
    load_unit_environment "$unit"
-    nohup bash -lc "exec $exec_start" >>"$daemon_log" 2>&1 &
+    OPENCLAW_SYSTEMCTL_SHIM_EXEC_START="$exec_start" \
+      OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG="$daemon_log" \
+      nohup node "$supervisor_script" </dev/null >/dev/null 2>&1 &
    printf '%s\n' "$!" >"$pid_file"
  )
 }
@@ -6,6 +6,7 @@ set -euo pipefail

 HARNESS_ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 ROOT_DIR="$(cd "${OPENCLAW_DOCKER_E2E_REPO_ROOT:-$HARNESS_ROOT_DIR}" && pwd)"
+DOCKER_E2E_HARNESS_ROOT_DIR="$HARNESS_ROOT_DIR"
 source "$HARNESS_ROOT_DIR/scripts/lib/docker-e2e-image.sh"
 source "$HARNESS_ROOT_DIR/scripts/lib/docker-e2e-package.sh"
 source "$HARNESS_ROOT_DIR/scripts/lib/openclaw-e2e-instance.sh"
@@ -268,6 +269,9 @@ cleanup() {
  if [ -n "${plugin_registry_pid:-}" ]; then
    kill "$plugin_registry_pid" >/dev/null 2>&1 || true
  fi
+  if [ -s "$SYSTEMCTL_SHIM_PID_FILE" ]; then
+    systemctl --user stop openclaw-gateway.service >/dev/null 2>&1 || true
+  fi
  openclaw_e2e_terminate_gateways "${gateway_pid:-}"
  if [ -s "$SYSTEMCTL_SHIM_PID_FILE" ]; then
    openclaw_e2e_terminate_gateways "$(cat "$SYSTEMCTL_SHIM_PID_FILE" 2>/dev/null || true)"
@@ -264,13 +264,14 @@ docker_e2e_cleanup_container_cidfile() {
 }

 docker_e2e_harness_mount_args() {
+  local harness_root="${DOCKER_E2E_HARNESS_ROOT_DIR:-$ROOT_DIR}"
  DOCKER_E2E_HARNESS_ARGS=(
-    -v "$ROOT_DIR/scripts/e2e:/app/scripts/e2e:ro"
-    -v "$ROOT_DIR/scripts/lib:/app/scripts/lib:ro"
-    -v "$ROOT_DIR/packages/normalization-core/src:/app/packages/normalization-core/src:ro"
-    -v "$ROOT_DIR/test/e2e/qa-lab:/app/test/e2e/qa-lab:ro"
-    -v "$ROOT_DIR/test/helpers:/app/test/helpers:ro"
-    -v "$ROOT_DIR/scripts/windows-cmd-helpers.mjs:/app/scripts/windows-cmd-helpers.mjs:ro"
+    -v "$harness_root/scripts/e2e:/app/scripts/e2e:ro"
+    -v "$harness_root/scripts/lib:/app/scripts/lib:ro"
+    -v "$harness_root/packages/normalization-core/src:/app/packages/normalization-core/src:ro"
+    -v "$harness_root/test/e2e/qa-lab:/app/test/e2e/qa-lab:ro"
+    -v "$harness_root/test/helpers:/app/test/helpers:ro"
+    -v "$harness_root/scripts/windows-cmd-helpers.mjs:/app/scripts/windows-cmd-helpers.mjs:ro"
  )
 }

@@ -674,7 +674,11 @@ function configuredChannelIdsForLane(poolLane, scenario) {

 export function requiredPrepublishPluginPackagesForLanes(poolLanes) {
  const configuredChannelIds = new Set();
+  const requiredPackages = new Set();
  for (const poolLane of poolLanes) {
+    for (const packageName of poolLane.prepublishPluginPackages ?? []) {
+      requiredPackages.add(packageName);
+    }
    const scenario = upgradeSurvivorScenarioForLane(poolLane);
    if (!scenario) {
      continue;
@@ -683,7 +687,7 @@ export function requiredPrepublishPluginPackagesForLanes(poolLanes) {
      configuredChannelIds.add(channelId);
    }
  }
-  return (officialExternalChannelCatalog.entries ?? [])
+  for (const packageName of (officialExternalChannelCatalog.entries ?? [])
    .filter((entry) => {
      const channelId = entry.openclaw?.channel?.id;
      const install = entry.openclaw?.install;
@@ -694,8 +698,10 @@ export function requiredPrepublishPluginPackagesForLanes(poolLanes) {
        install?.npmSpec === entry.name
      );
    })
-    .map((entry) => entry.name)
-    .toSorted((a, b) => a.localeCompare(b));
+    .map((entry) => entry.name)) {
+    requiredPackages.add(packageName);
+  }
+  return [...requiredPackages].toSorted((a, b) => a.localeCompare(b));
 }

 function buildPlanJson(params) {
@@ -11,6 +11,7 @@ export type DockerE2eLane = {
  name: string;
  needsLiveImage?: boolean;
  noOutputTimeoutMs?: number;
+  prepublishPluginPackages?: string[];
  resources: string[];
  retries: number;
  retryPatterns: RegExp[];
@@ -82,6 +82,9 @@ function lane(name, command, options = {}) {
    timeoutMs: options.timeoutMs,
    upgradeSurvivorScenario: options.upgradeSurvivorScenario,
    weight: options.weight ?? 1,
+    ...(options.prepublishPluginPackages
+      ? { prepublishPluginPackages: options.prepublishPluginPackages }
+      : {}),
  };
 }

@@ -198,6 +201,8 @@ function createPackageUpdateMaintenanceLanes() {
      weight: 3,
    }),
    npmLane("update-restart-auth", updateRestartAuthCommand, {
+      // Credential hydration auto-enables the candidate's Codex runtime during restart.
+      prepublishPluginPackages: ["@openclaw/codex"],
      stateScenario: "upgrade-survivor",
      timeoutMs: 25 * 60 * 1000,
      upgradeSurvivorScenario: "base",