fix(systemd): report service commands replaced by drop-ins (#128604)

* fix(systemd): report service commands replaced by drop-ins

* test(node-host): tolerate worker startup latency

* fix(systemd): bound effective command lookup

src/daemon/node-service.ts

@@ -43,7 +43,7 @@ export function resolveNodeService(): GatewayService {
     hasInstalledDefinition: hasInstalledDefinition
       ? (args) => hasInstalledDefinition({ ...args, env: withNodeServiceEnv(args.env ?? {}) })
       : undefined,
-    readCommand: (env) => base.readCommand(withNodeServiceEnv(env)),
+    readCommand: (env, opts) => base.readCommand(withNodeServiceEnv(env), opts),
     readRuntime: (env, opts) => base.readRuntime(withNodeServiceEnv(env), opts),
   };
 }

src/daemon/service.test.ts

@@ -186,8 +186,10 @@ describe("readGatewayServiceState", () => {
   });
 
   it("preserves runtime probe failures as an explicit unknown state", async () => {
+    const readCommand = vi.fn(async () => null);
     const service = createService({
       isLoaded: vi.fn(async () => true),
+      readCommand,
       readRuntime: vi.fn(async () => {
         throw new Error("systemctl show timed out");
       }),
@@ -195,6 +197,7 @@ describe("readGatewayServiceState", () => {
 
     const state = await readGatewayServiceState(service, { timeoutMs: 100 });
 
+    expect(readCommand).toHaveBeenCalledWith(process.env, { timeoutMs: 100 });
     expect(state.running).toBe(false);
     expect(state.runtime).toEqual({
       status: "unknown",

src/daemon/service.ts

@@ -87,7 +87,10 @@ export type GatewayService = {
   isLoaded: (args: GatewayServiceEnvArgs) => Promise<boolean>;
   isEnabled?: (args: GatewayServiceEnvArgs) => Promise<boolean>;
   hasInstalledDefinition?: (args: GatewayServiceEnvArgs) => Promise<boolean>;
-  readCommand: (env: GatewayServiceEnv) => Promise<GatewayServiceCommandConfig | null>;
+  readCommand: (
+    env: GatewayServiceEnv,
+    opts?: GatewayServiceReadOptions,
+  ) => Promise<GatewayServiceCommandConfig | null>;
   readRuntime: (
     env: GatewayServiceEnv,
     opts?: GatewayServiceReadOptions,
@@ -193,17 +196,16 @@ export async function readGatewayServiceState(
   args: ReadGatewayServiceStateArgs = {},
 ): Promise<GatewayServiceState> {
   const baseEnv = args.env ?? (process.env as GatewayServiceEnv);
-  const command = await service.readCommand(baseEnv).catch(() => null);
+  const { timeoutMs } = args;
+  // Keep command and status probes on the same fail-soft manager deadline.
+  const command = await service.readCommand(baseEnv, { timeoutMs }).catch(() => null);
   const env = mergeGatewayServiceEnv(baseEnv, command);
   // Callers that may mutate the selected service can reject persisted selector
   // drift before isLoaded/readRuntime invoke the native service manager.
   args.validateEnvBeforeStatusRead?.(env);
-  // Propagate the status read deadline so a wedged service manager fails soft
-  // instead of hanging both probes. readCommand parses local files and needs no
-  // bound; isLoaded/readRuntime can spawn service-manager subprocesses.
   const [loadState, runtime] = await Promise.all([
-    readGatewayServiceLoadState(service, { env, timeoutMs: args.timeoutMs }),
-    service.readRuntime(env, { timeoutMs: args.timeoutMs }).catch(
+    readGatewayServiceLoadState(service, { env, timeoutMs }),
+    service.readRuntime(env, { timeoutMs }).catch(
       (error: unknown) =>
         ({
           status: "unknown",

src/daemon/systemd-exec.ts

@@ -14,17 +14,26 @@ import {
 
 export type SystemdUnitScope = "system" | "user";
 
+async function execSystemdCommand(
+  command: "systemctl" | "busctl",
+  args: string[],
+  env?: GatewayServiceEnv,
+  timeoutMs?: number,
+): Promise<{ stdout: string; stderr: string; code: number }> {
+  return await execFileUtf8(command, args, {
+    env: env ? resolveSystemctlProcessEnv(env) : process.env,
+    // A wedged systemd socket can leave manager commands blocked forever; the timeout
+    // kills the child so status reads fail soft instead of hanging the command.
+    ...(timeoutMs && timeoutMs > 0 ? { timeout: timeoutMs, killSignal: "SIGKILL" as const } : {}),
+  });
+}
+
 export async function execSystemctl(
   args: string[],
   env?: GatewayServiceEnv,
   timeoutMs?: number,
 ): Promise<{ stdout: string; stderr: string; code: number }> {
-  return await execFileUtf8("systemctl", args, {
-    env: env ? resolveSystemctlProcessEnv(env) : process.env,
-    // A wedged systemd socket can leave `systemctl` blocked forever; the timeout
-    // kills the child so status reads fail soft instead of hanging the command.
-    ...(timeoutMs && timeoutMs > 0 ? { timeout: timeoutMs, killSignal: "SIGKILL" as const } : {}),
-  });
+  return await execSystemdCommand("systemctl", args, env, timeoutMs);
 }
 
 export function readSystemctlDetail(result: { stdout: string; stderr: string }): string {
@@ -229,27 +238,26 @@ function shouldFallbackToMachineUserScope(detail: string): boolean {
   return !detail.toLowerCase().includes("permission denied");
 }
 
-export async function execSystemctlUser(
+async function execSystemdUserCommand(
+  command: "systemctl" | "busctl",
   env: GatewayServiceEnv,
   args: string[],
   timeoutMs?: number,
 ): Promise<{ stdout: string; stderr: string; code: number }> {
   const { machineUser, preferMachineScope } = resolveSystemctlUserScope(env);
+  const run = (scopeArgs: string[]) =>
+    execSystemdCommand(command, [...scopeArgs, ...args], env, timeoutMs);
 
   // Under sudo-to-root, prefer the invoking non-root user's scope directly via machine scope.
   if (preferMachineScope && machineUser) {
     const machineScopeArgs = resolveSystemctlMachineUserScopeArgs(machineUser);
     if (machineScopeArgs.length > 0) {
       // Do not fall through to bare --user: under sudo that can target root's user manager.
-      return await execSystemctl([...machineScopeArgs, ...args], env, timeoutMs);
+      return await run(machineScopeArgs);
     }
   }
 
-  const directResult = await execSystemctl(
-    [...resolveSystemctlDirectUserScopeArgs(), ...args],
-    env,
-    timeoutMs,
-  );
+  const directResult = await run(resolveSystemctlDirectUserScopeArgs());
   if (directResult.code === 0) {
     return directResult;
   }
@@ -263,7 +271,23 @@ export async function execSystemctlUser(
   if (machineScopeArgs.length === 0) {
     return directResult;
   }
-  return await execSystemctl([...machineScopeArgs, ...args], env, timeoutMs);
+  return await run(machineScopeArgs);
+}
+
+export async function execSystemctlUser(
+  env: GatewayServiceEnv,
+  args: string[],
+  timeoutMs?: number,
+): Promise<{ stdout: string; stderr: string; code: number }> {
+  return await execSystemdUserCommand("systemctl", env, args, timeoutMs);
+}
+
+export async function execBusctlUser(
+  env: GatewayServiceEnv,
+  args: string[],
+  timeoutMs?: number,
+): Promise<{ stdout: string; stderr: string; code: number }> {
+  return await execSystemdUserCommand("busctl", env, args, timeoutMs);
 }
 
 export async function disableSystemdUserUnitForRemoval(

src/daemon/systemd-service-files.ts

@@ -1,6 +1,7 @@
 /** Linux systemd unit paths and environment-file parsing. */
 import fs from "node:fs/promises";
 import path from "node:path";
+import { asOptionalRecord } from "@openclaw/normalization-core/record-coerce";
 import { normalizeStringEntries } from "@openclaw/normalization-core/string-normalization";
 import { isUnresolvedShellReference } from "../config/state-dir-dotenv.js";
 import { splitArgsPreservingQuotes } from "./arg-split.js";
@@ -11,11 +12,14 @@ import type {
   GatewayServiceCommandConfig,
   GatewayServiceEnv,
   GatewayServiceEnvironmentValueSource,
+  GatewayServiceReadOptions,
 } from "./service-types.js";
+import { execBusctlUser } from "./systemd-exec.js";
 import { parseSystemdEnvAssignments, parseSystemdExecStart } from "./systemd-unit.js";
 
 const SYSTEMD_GATEWAY_DOTENV_FILENAME = "gateway.systemd.env";
 const SYSTEMD_NODE_DOTENV_FILENAME = "node.systemd.env";
+const SYSTEMD_MANAGER_QUERY_TIMEOUT_MS = 5_000;
 
 export function resolveSystemdUnitPathForName(env: GatewayServiceEnv, name: string): string {
   const home = normalizeWindowsPathSeparators(resolveDaemonHomeDir(env));
@@ -40,8 +44,57 @@ export function resolveSystemdUserUnitPath(env: GatewayServiceEnv): string {
 
 // Unit file parsing/rendering: see systemd-unit.ts
 
+async function readSystemdManagerExecStart(
+  env: GatewayServiceEnv,
+  opts?: GatewayServiceReadOptions,
+): Promise<string[] | null> {
+  const manager = "org.freedesktop.systemd1";
+  const timeoutMs =
+    opts?.timeoutMs && opts.timeoutMs > 0 ? opts.timeoutMs : SYSTEMD_MANAGER_QUERY_TIMEOUT_MS;
+  const deadlineAt = Date.now() + timeoutMs;
+  // Both D-Bus calls share one deadline so every caller reaches the local fallback promptly.
+  const query = (args: string[]) =>
+    execBusctlUser(env, ["--json=short", ...args], Math.max(1, deadlineAt - Date.now()));
+  const loaded = await query([
+    "call",
+    manager,
+    "/org/freedesktop/systemd1",
+    `${manager}.Manager`,
+    "LoadUnit",
+    "s",
+    `${resolveSystemdServiceName(env)}.service`,
+  ]);
+  if (loaded.code !== 0) {
+    return null;
+  }
+  const unit = asOptionalRecord(JSON.parse(loaded.stdout));
+  const unitPath = Array.isArray(unit?.data) && unit.data.length === 1 ? unit.data[0] : null;
+  if (unit?.type !== "o" || typeof unitPath !== "string" || !unitPath) {
+    return null;
+  }
+  const propertyArgs = ["get-property", manager, unitPath, `${manager}.Service`, "ExecStart"];
+  const result = await query(propertyArgs);
+  if (result.code !== 0) {
+    return null;
+  }
+  const property = asOptionalRecord(JSON.parse(result.stdout));
+  if (
+    property?.type !== "a(sasbttttuii)" ||
+    !Array.isArray(property.data) ||
+    property.data.length !== 1
+  ) {
+    return null;
+  }
+  const execution = property.data[0];
+  const argv = Array.isArray(execution) ? execution[1] : null;
+  return Array.isArray(argv) && argv.length > 0 && argv.every((arg) => typeof arg === "string")
+    ? argv
+    : null;
+}
+
 export async function readSystemdServiceExecStart(
   env: GatewayServiceEnv,
+  opts?: GatewayServiceReadOptions,
 ): Promise<GatewayServiceCommandConfig | null> {
   const unitPath = resolveSystemdUnitPath(env);
   try {
@@ -87,7 +140,10 @@ export async function readSystemdServiceExecStart(
       inlineEnvironment,
       environmentFromFiles.environment,
     );
-    const programArguments = parseSystemdExecStart(execStart);
+    // The loaded manager owns merged drop-ins; retain base-file metadata when D-Bus is offline.
+    const programArguments =
+      (await readSystemdManagerExecStart(env, opts).catch(() => null)) ??
+      parseSystemdExecStart(execStart);
     return {
       programArguments,
       ...(workingDirectory ? { workingDirectory } : {}),

src/daemon/systemd.test.ts

@@ -1355,6 +1355,93 @@ describe("readSystemdServiceExecStart", () => {
     vi.restoreAllMocks();
   });
 
+  it("reports the exact manager-effective argv when a drop-in replaces the base ExecStart", async () => {
+    const effectiveArguments = [
+      "/opt/stale/openclaw",
+      "gateway",
+      "--name",
+      "Stale Drop-In Gateway",
+    ];
+    const objectPath = "/org/freedesktop/systemd1/unit/openclaw_2dgateway_2eservice";
+    mockReadGatewayServiceFile([
+      "[Service]",
+      "ExecStart=/usr/bin/openclaw gateway run",
+      "WorkingDirectory=/srv/openclaw",
+      "Environment=OPENCLAW_GATEWAY_PORT=18789",
+    ]);
+    execFileMock.mockReset();
+    execFileMock.mockImplementation((command, args, options, callback) => {
+      expect(command).toBe("busctl");
+      expect(options.timeout).toEqual(expect.any(Number));
+      expect(options.timeout).toBeGreaterThan(0);
+      expect(options.timeout).toBeLessThanOrEqual(1234);
+      expect(options.killSignal).toBe("SIGKILL");
+      if (args.includes("LoadUnit")) {
+        expect(args).toEqual([
+          "--user",
+          "--json=short",
+          "call",
+          "org.freedesktop.systemd1",
+          "/org/freedesktop/systemd1",
+          "org.freedesktop.systemd1.Manager",
+          "LoadUnit",
+          "s",
+          GATEWAY_SERVICE,
+        ]);
+        callback(null, JSON.stringify({ type: "o", data: [objectPath] }), "");
+        return;
+      }
+      expect(args).toEqual([
+        "--user",
+        "--json=short",
+        "get-property",
+        "org.freedesktop.systemd1",
+        objectPath,
+        "org.freedesktop.systemd1.Service",
+        "ExecStart",
+      ]);
+      callback(
+        null,
+        JSON.stringify({
+          type: "a(sasbttttuii)",
+          data: [[effectiveArguments[0], effectiveArguments, false, 0, 0, 0, 0, 0, 0, 0]],
+        }),
+        "",
+      );
+    });
+
+    const command = await readSystemdServiceExecStart(
+      { HOME: TEST_SERVICE_HOME },
+      { timeoutMs: 1234 },
+    );
+
+    expect(command).toMatchObject({
+      programArguments: effectiveArguments,
+      workingDirectory: "/srv/openclaw",
+      environment: { OPENCLAW_GATEWAY_PORT: "18789" },
+      environmentValueSources: { OPENCLAW_GATEWAY_PORT: "inline" },
+      sourcePath: `${TEST_SERVICE_HOME}/.config/systemd/user/${GATEWAY_SERVICE}`,
+    });
+    expect(execFileMock).toHaveBeenCalledTimes(2);
+  });
+
+  it("bounds manager lookup for callers without a timeout before local fallback", async () => {
+    mockReadGatewayServiceFile(["[Service]", "ExecStart=/usr/bin/openclaw gateway run"]);
+    execFileMock.mockReset();
+    execFileMock.mockImplementation((command, _args, options, callback) => {
+      expect(command).toBe("busctl");
+      expect(options.timeout).toEqual(expect.any(Number));
+      expect(options.timeout).toBeGreaterThan(0);
+      expect(options.timeout).toBeLessThanOrEqual(5_000);
+      callback(createExecFileError("manager query timed out"), "", "manager query timed out");
+    });
+
+    const command = await readSystemdServiceExecStart({ HOME: TEST_SERVICE_HOME });
+
+    expect(command?.programArguments).toEqual(["/usr/bin/openclaw", "gateway", "run"]);
+    expect(execFileMock).toHaveBeenCalledTimes(1);
+  });
+
   it("loads OPENCLAW_GATEWAY_TOKEN from EnvironmentFile", async () => {
     const readFileSpy = mockReadGatewayServiceFile(
       ["[Service]", "ExecStart=/usr/bin/openclaw gateway run", "EnvironmentFile=%h/.openclaw/.env"],

src/node-host/node-worker-supervisor.container.test.ts

@@ -354,6 +354,13 @@ async function waitForTerminal(
   return await supervisor.status(launchId);
 }
 
+async function waitForWorkerStarted(workspaceDir: string): Promise<void> {
+  await vi.waitFor(
+    () => expect(fs.existsSync(path.join(workspaceDir, "worker-started"))).toBe(true),
+    { timeout: 5_000 },
+  );
+}
+
 function claimFixtureLaunch(
   fixture: ReturnType<typeof containerFixture>,
   launchId: string,
@@ -557,9 +564,7 @@ describe("node worker supervisor container isolation", () => {
 
       try {
         const running = await fixture.supervisor.launch(input, endpoint);
-        await vi.waitFor(() =>
-          expect(fs.existsSync(path.join(fixture.workspaceDir, "worker-started"))).toBe(true),
-        );
+        await waitForWorkerStarted(fixture.workspaceDir);
         if (operation === "cancel") {
           await fixture.supervisor.cancel(testNodeWorkerLaunchIdentity(input));
         } else {