diff --git a/src/daemon/node-service.ts b/src/daemon/node-service.ts index 82ab517ffaee..525adbdd11b3 100644 --- a/src/daemon/node-service.ts +++ b/src/daemon/node-service.ts @@ -43,7 +43,7 @@ export function resolveNodeService(): GatewayService { hasInstalledDefinition: hasInstalledDefinition ? (args) => hasInstalledDefinition({ ...args, env: withNodeServiceEnv(args.env ?? {}) }) : undefined, - readCommand: (env) => base.readCommand(withNodeServiceEnv(env)), + readCommand: (env, opts) => base.readCommand(withNodeServiceEnv(env), opts), readRuntime: (env, opts) => base.readRuntime(withNodeServiceEnv(env), opts), }; } diff --git a/src/daemon/service.test.ts b/src/daemon/service.test.ts index 5bac11b7cc27..22a54910329f 100644 --- a/src/daemon/service.test.ts +++ b/src/daemon/service.test.ts @@ -186,8 +186,10 @@ describe("readGatewayServiceState", () => { }); it("preserves runtime probe failures as an explicit unknown state", async () => { + const readCommand = vi.fn(async () => null); const service = createService({ isLoaded: vi.fn(async () => true), + readCommand, readRuntime: vi.fn(async () => { throw new Error("systemctl show timed out"); }), @@ -195,6 +197,7 @@ describe("readGatewayServiceState", () => { const state = await readGatewayServiceState(service, { timeoutMs: 100 }); + expect(readCommand).toHaveBeenCalledWith(process.env, { timeoutMs: 100 }); expect(state.running).toBe(false); expect(state.runtime).toEqual({ status: "unknown", diff --git a/src/daemon/service.ts b/src/daemon/service.ts index 1d37e0e8b30d..864b727dca88 100644 --- a/src/daemon/service.ts +++ b/src/daemon/service.ts @@ -87,7 +87,10 @@ export type GatewayService = { isLoaded: (args: GatewayServiceEnvArgs) => Promise; isEnabled?: (args: GatewayServiceEnvArgs) => Promise; hasInstalledDefinition?: (args: GatewayServiceEnvArgs) => Promise; - readCommand: (env: GatewayServiceEnv) => Promise; + readCommand: ( + env: GatewayServiceEnv, + opts?: GatewayServiceReadOptions, + ) => Promise; readRuntime: ( env: GatewayServiceEnv, opts?: GatewayServiceReadOptions, @@ -193,17 +196,16 @@ export async function readGatewayServiceState( args: ReadGatewayServiceStateArgs = {}, ): Promise { const baseEnv = args.env ?? (process.env as GatewayServiceEnv); - const command = await service.readCommand(baseEnv).catch(() => null); + const { timeoutMs } = args; + // Keep command and status probes on the same fail-soft manager deadline. + const command = await service.readCommand(baseEnv, { timeoutMs }).catch(() => null); const env = mergeGatewayServiceEnv(baseEnv, command); // Callers that may mutate the selected service can reject persisted selector // drift before isLoaded/readRuntime invoke the native service manager. args.validateEnvBeforeStatusRead?.(env); - // Propagate the status read deadline so a wedged service manager fails soft - // instead of hanging both probes. readCommand parses local files and needs no - // bound; isLoaded/readRuntime can spawn service-manager subprocesses. const [loadState, runtime] = await Promise.all([ - readGatewayServiceLoadState(service, { env, timeoutMs: args.timeoutMs }), - service.readRuntime(env, { timeoutMs: args.timeoutMs }).catch( + readGatewayServiceLoadState(service, { env, timeoutMs }), + service.readRuntime(env, { timeoutMs }).catch( (error: unknown) => ({ status: "unknown", diff --git a/src/daemon/systemd-exec.ts b/src/daemon/systemd-exec.ts index 194bbc0a06d7..21f9aa6c9c5d 100644 --- a/src/daemon/systemd-exec.ts +++ b/src/daemon/systemd-exec.ts @@ -14,17 +14,26 @@ import { export type SystemdUnitScope = "system" | "user"; +async function execSystemdCommand( + command: "systemctl" | "busctl", + args: string[], + env?: GatewayServiceEnv, + timeoutMs?: number, +): Promise<{ stdout: string; stderr: string; code: number }> { + return await execFileUtf8(command, args, { + env: env ? resolveSystemctlProcessEnv(env) : process.env, + // A wedged systemd socket can leave manager commands blocked forever; the timeout + // kills the child so status reads fail soft instead of hanging the command. + ...(timeoutMs && timeoutMs > 0 ? { timeout: timeoutMs, killSignal: "SIGKILL" as const } : {}), + }); +} + export async function execSystemctl( args: string[], env?: GatewayServiceEnv, timeoutMs?: number, ): Promise<{ stdout: string; stderr: string; code: number }> { - return await execFileUtf8("systemctl", args, { - env: env ? resolveSystemctlProcessEnv(env) : process.env, - // A wedged systemd socket can leave `systemctl` blocked forever; the timeout - // kills the child so status reads fail soft instead of hanging the command. - ...(timeoutMs && timeoutMs > 0 ? { timeout: timeoutMs, killSignal: "SIGKILL" as const } : {}), - }); + return await execSystemdCommand("systemctl", args, env, timeoutMs); } export function readSystemctlDetail(result: { stdout: string; stderr: string }): string { @@ -229,27 +238,26 @@ function shouldFallbackToMachineUserScope(detail: string): boolean { return !detail.toLowerCase().includes("permission denied"); } -export async function execSystemctlUser( +async function execSystemdUserCommand( + command: "systemctl" | "busctl", env: GatewayServiceEnv, args: string[], timeoutMs?: number, ): Promise<{ stdout: string; stderr: string; code: number }> { const { machineUser, preferMachineScope } = resolveSystemctlUserScope(env); + const run = (scopeArgs: string[]) => + execSystemdCommand(command, [...scopeArgs, ...args], env, timeoutMs); // Under sudo-to-root, prefer the invoking non-root user's scope directly via machine scope. if (preferMachineScope && machineUser) { const machineScopeArgs = resolveSystemctlMachineUserScopeArgs(machineUser); if (machineScopeArgs.length > 0) { // Do not fall through to bare --user: under sudo that can target root's user manager. - return await execSystemctl([...machineScopeArgs, ...args], env, timeoutMs); + return await run(machineScopeArgs); } } - const directResult = await execSystemctl( - [...resolveSystemctlDirectUserScopeArgs(), ...args], - env, - timeoutMs, - ); + const directResult = await run(resolveSystemctlDirectUserScopeArgs()); if (directResult.code === 0) { return directResult; } @@ -263,7 +271,23 @@ export async function execSystemctlUser( if (machineScopeArgs.length === 0) { return directResult; } - return await execSystemctl([...machineScopeArgs, ...args], env, timeoutMs); + return await run(machineScopeArgs); +} + +export async function execSystemctlUser( + env: GatewayServiceEnv, + args: string[], + timeoutMs?: number, +): Promise<{ stdout: string; stderr: string; code: number }> { + return await execSystemdUserCommand("systemctl", env, args, timeoutMs); +} + +export async function execBusctlUser( + env: GatewayServiceEnv, + args: string[], + timeoutMs?: number, +): Promise<{ stdout: string; stderr: string; code: number }> { + return await execSystemdUserCommand("busctl", env, args, timeoutMs); } export async function disableSystemdUserUnitForRemoval( diff --git a/src/daemon/systemd-service-files.ts b/src/daemon/systemd-service-files.ts index 745cee2ea752..2af8904d5359 100644 --- a/src/daemon/systemd-service-files.ts +++ b/src/daemon/systemd-service-files.ts @@ -1,6 +1,7 @@ /** Linux systemd unit paths and environment-file parsing. */ import fs from "node:fs/promises"; import path from "node:path"; +import { asOptionalRecord } from "@openclaw/normalization-core/record-coerce"; import { normalizeStringEntries } from "@openclaw/normalization-core/string-normalization"; import { isUnresolvedShellReference } from "../config/state-dir-dotenv.js"; import { splitArgsPreservingQuotes } from "./arg-split.js"; @@ -11,11 +12,14 @@ import type { GatewayServiceCommandConfig, GatewayServiceEnv, GatewayServiceEnvironmentValueSource, + GatewayServiceReadOptions, } from "./service-types.js"; +import { execBusctlUser } from "./systemd-exec.js"; import { parseSystemdEnvAssignments, parseSystemdExecStart } from "./systemd-unit.js"; const SYSTEMD_GATEWAY_DOTENV_FILENAME = "gateway.systemd.env"; const SYSTEMD_NODE_DOTENV_FILENAME = "node.systemd.env"; +const SYSTEMD_MANAGER_QUERY_TIMEOUT_MS = 5_000; export function resolveSystemdUnitPathForName(env: GatewayServiceEnv, name: string): string { const home = normalizeWindowsPathSeparators(resolveDaemonHomeDir(env)); @@ -40,8 +44,57 @@ export function resolveSystemdUserUnitPath(env: GatewayServiceEnv): string { // Unit file parsing/rendering: see systemd-unit.ts +async function readSystemdManagerExecStart( + env: GatewayServiceEnv, + opts?: GatewayServiceReadOptions, +): Promise { + const manager = "org.freedesktop.systemd1"; + const timeoutMs = + opts?.timeoutMs && opts.timeoutMs > 0 ? opts.timeoutMs : SYSTEMD_MANAGER_QUERY_TIMEOUT_MS; + const deadlineAt = Date.now() + timeoutMs; + // Both D-Bus calls share one deadline so every caller reaches the local fallback promptly. + const query = (args: string[]) => + execBusctlUser(env, ["--json=short", ...args], Math.max(1, deadlineAt - Date.now())); + const loaded = await query([ + "call", + manager, + "/org/freedesktop/systemd1", + `${manager}.Manager`, + "LoadUnit", + "s", + `${resolveSystemdServiceName(env)}.service`, + ]); + if (loaded.code !== 0) { + return null; + } + const unit = asOptionalRecord(JSON.parse(loaded.stdout)); + const unitPath = Array.isArray(unit?.data) && unit.data.length === 1 ? unit.data[0] : null; + if (unit?.type !== "o" || typeof unitPath !== "string" || !unitPath) { + return null; + } + const propertyArgs = ["get-property", manager, unitPath, `${manager}.Service`, "ExecStart"]; + const result = await query(propertyArgs); + if (result.code !== 0) { + return null; + } + const property = asOptionalRecord(JSON.parse(result.stdout)); + if ( + property?.type !== "a(sasbttttuii)" || + !Array.isArray(property.data) || + property.data.length !== 1 + ) { + return null; + } + const execution = property.data[0]; + const argv = Array.isArray(execution) ? execution[1] : null; + return Array.isArray(argv) && argv.length > 0 && argv.every((arg) => typeof arg === "string") + ? argv + : null; +} + export async function readSystemdServiceExecStart( env: GatewayServiceEnv, + opts?: GatewayServiceReadOptions, ): Promise { const unitPath = resolveSystemdUnitPath(env); try { @@ -87,7 +140,10 @@ export async function readSystemdServiceExecStart( inlineEnvironment, environmentFromFiles.environment, ); - const programArguments = parseSystemdExecStart(execStart); + // The loaded manager owns merged drop-ins; retain base-file metadata when D-Bus is offline. + const programArguments = + (await readSystemdManagerExecStart(env, opts).catch(() => null)) ?? + parseSystemdExecStart(execStart); return { programArguments, ...(workingDirectory ? { workingDirectory } : {}), diff --git a/src/daemon/systemd.test.ts b/src/daemon/systemd.test.ts index 69c6add440cb..a96fb31e681a 100644 --- a/src/daemon/systemd.test.ts +++ b/src/daemon/systemd.test.ts @@ -1355,6 +1355,93 @@ describe("readSystemdServiceExecStart", () => { vi.restoreAllMocks(); }); + it("reports the exact manager-effective argv when a drop-in replaces the base ExecStart", async () => { + const effectiveArguments = [ + "/opt/stale/openclaw", + "gateway", + "--name", + "Stale Drop-In Gateway", + ]; + const objectPath = "/org/freedesktop/systemd1/unit/openclaw_2dgateway_2eservice"; + mockReadGatewayServiceFile([ + "[Service]", + "ExecStart=/usr/bin/openclaw gateway run", + "WorkingDirectory=/srv/openclaw", + "Environment=OPENCLAW_GATEWAY_PORT=18789", + ]); + execFileMock.mockReset(); + execFileMock.mockImplementation((command, args, options, callback) => { + expect(command).toBe("busctl"); + expect(options.timeout).toEqual(expect.any(Number)); + expect(options.timeout).toBeGreaterThan(0); + expect(options.timeout).toBeLessThanOrEqual(1234); + expect(options.killSignal).toBe("SIGKILL"); + if (args.includes("LoadUnit")) { + expect(args).toEqual([ + "--user", + "--json=short", + "call", + "org.freedesktop.systemd1", + "/org/freedesktop/systemd1", + "org.freedesktop.systemd1.Manager", + "LoadUnit", + "s", + GATEWAY_SERVICE, + ]); + callback(null, JSON.stringify({ type: "o", data: [objectPath] }), ""); + return; + } + expect(args).toEqual([ + "--user", + "--json=short", + "get-property", + "org.freedesktop.systemd1", + objectPath, + "org.freedesktop.systemd1.Service", + "ExecStart", + ]); + callback( + null, + JSON.stringify({ + type: "a(sasbttttuii)", + data: [[effectiveArguments[0], effectiveArguments, false, 0, 0, 0, 0, 0, 0, 0]], + }), + "", + ); + }); + + const command = await readSystemdServiceExecStart( + { HOME: TEST_SERVICE_HOME }, + { timeoutMs: 1234 }, + ); + + expect(command).toMatchObject({ + programArguments: effectiveArguments, + workingDirectory: "/srv/openclaw", + environment: { OPENCLAW_GATEWAY_PORT: "18789" }, + environmentValueSources: { OPENCLAW_GATEWAY_PORT: "inline" }, + sourcePath: `${TEST_SERVICE_HOME}/.config/systemd/user/${GATEWAY_SERVICE}`, + }); + expect(execFileMock).toHaveBeenCalledTimes(2); + }); + + it("bounds manager lookup for callers without a timeout before local fallback", async () => { + mockReadGatewayServiceFile(["[Service]", "ExecStart=/usr/bin/openclaw gateway run"]); + execFileMock.mockReset(); + execFileMock.mockImplementation((command, _args, options, callback) => { + expect(command).toBe("busctl"); + expect(options.timeout).toEqual(expect.any(Number)); + expect(options.timeout).toBeGreaterThan(0); + expect(options.timeout).toBeLessThanOrEqual(5_000); + callback(createExecFileError("manager query timed out"), "", "manager query timed out"); + }); + + const command = await readSystemdServiceExecStart({ HOME: TEST_SERVICE_HOME }); + + expect(command?.programArguments).toEqual(["/usr/bin/openclaw", "gateway", "run"]); + expect(execFileMock).toHaveBeenCalledTimes(1); + }); + it("loads OPENCLAW_GATEWAY_TOKEN from EnvironmentFile", async () => { const readFileSpy = mockReadGatewayServiceFile( ["[Service]", "ExecStart=/usr/bin/openclaw gateway run", "EnvironmentFile=%h/.openclaw/.env"], diff --git a/src/node-host/node-worker-supervisor.container.test.ts b/src/node-host/node-worker-supervisor.container.test.ts index fdec32e87385..e694436d92f6 100644 --- a/src/node-host/node-worker-supervisor.container.test.ts +++ b/src/node-host/node-worker-supervisor.container.test.ts @@ -354,6 +354,13 @@ async function waitForTerminal( return await supervisor.status(launchId); } +async function waitForWorkerStarted(workspaceDir: string): Promise { + await vi.waitFor( + () => expect(fs.existsSync(path.join(workspaceDir, "worker-started"))).toBe(true), + { timeout: 5_000 }, + ); +} + function claimFixtureLaunch( fixture: ReturnType, launchId: string, @@ -557,9 +564,7 @@ describe("node worker supervisor container isolation", () => { try { const running = await fixture.supervisor.launch(input, endpoint); - await vi.waitFor(() => - expect(fs.existsSync(path.join(fixture.workspaceDir, "worker-started"))).toBe(true), - ); + await waitForWorkerStarted(fixture.workspaceDir); if (operation === "cancel") { await fixture.supervisor.cancel(testNodeWorkerLaunchIdentity(input)); } else {