From 5e8350fb043b5cd2b4eb6841903e2005f20059d8 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 14 Aug 2026 20:10:23 -0700 Subject: [PATCH] fix: report stopped LaunchAgents as stopped (#123961) * fix(daemon): report stopped launch agents as stopped * fix(daemon): uninstall stopped launch agents cleanly --- .../doctor-gateway-daemon-flow.test.ts | 7 ++- src/commands/doctor-gateway-daemon-flow.ts | 13 +++-- src/daemon/launchd-install.ts | 9 ++- src/daemon/launchd-runtime.ts | 53 ++++++++--------- src/daemon/launchd.test.ts | 40 +++++++++++-- src/daemon/runtime-format.test.ts | 9 +++ src/daemon/schtasks-runtime.ts | 4 +- src/daemon/schtasks.startup-fallback.test.ts | 32 ++++++++++ src/daemon/systemd-runtime.ts | 5 +- src/daemon/systemd.test.ts | 58 +++++++++++++++++++ src/infra/runtime-status.ts | 2 +- 11 files changed, 184 insertions(+), 48 deletions(-) diff --git a/src/commands/doctor-gateway-daemon-flow.test.ts b/src/commands/doctor-gateway-daemon-flow.test.ts index 336edc1cb210..a303e244c8cb 100644 --- a/src/commands/doctor-gateway-daemon-flow.test.ts +++ b/src/commands/doctor-gateway-daemon-flow.test.ts @@ -702,8 +702,9 @@ describe("maybeRepairGatewayDaemon", () => { it("skips LaunchAgent bootstrap repair when service repair policy is external", async () => { setPlatform("darwin"); service.isLoaded.mockResolvedValue(false); + service.readRuntime.mockResolvedValue({ status: "stopped" }); vi.mocked(launchd.isLaunchAgentLoaded).mockResolvedValue(false); - vi.mocked(launchd.launchAgentPlistExists).mockResolvedValue(true); + vi.mocked(launchd.launchAgentPlistExists).mockResolvedValueOnce(true).mockResolvedValue(false); await withEnvAsync({ OPENCLAW_SERVICE_REPAIR_POLICY: "external" }, async () => { await runAutoRepair(); @@ -712,6 +713,8 @@ describe("maybeRepairGatewayDaemon", () => { expect(launchd.repairLaunchAgentBootstrap).not.toHaveBeenCalled(); expect(service.install).not.toHaveBeenCalled(); expect(note).toHaveBeenCalledWith(EXTERNAL_SERVICE_REPAIR_NOTE, "Gateway LaunchAgent"); + expect(note).not.toHaveBeenCalledWith("Gateway service not installed.", "Gateway"); + expect(buildGatewayRuntimeHints).not.toHaveBeenCalled(); }); it("re-enables and bootstraps a parked LaunchAgent during non-interactive repair", async () => { @@ -753,7 +756,6 @@ describe("maybeRepairGatewayDaemon", () => { { status: "unknown", detail: "Bootstrap failed: 125: Domain does not support specified action", - missingSupervision: true, missingGuiSession: true, }, { platform: "darwin", env: process.env }, @@ -856,7 +858,6 @@ describe("maybeRepairGatewayDaemon", () => { { status: "unknown", detail: "Bootstrap failed: 125: Domain does not support specified action", - missingSupervision: true, missingGuiSession: true, }, { platform: "darwin", env: process.env }, diff --git a/src/commands/doctor-gateway-daemon-flow.ts b/src/commands/doctor-gateway-daemon-flow.ts index 3a461c0706e8..0a4d9fdab3f2 100644 --- a/src/commands/doctor-gateway-daemon-flow.ts +++ b/src/commands/doctor-gateway-daemon-flow.ts @@ -52,6 +52,7 @@ import { healthCommand } from "./health.js"; type LaunchAgentBootstrapDoctorOutcome = | { status: "skipped" } + | { status: "not-loaded" } | { status: "repaired" } | { status: "system-launchdaemon-blocked"; detail: string } | { status: "gui-session-unavailable"; detail: string }; @@ -101,7 +102,7 @@ async function maybeRepairLaunchAgentBootstrap(params: { note("LaunchAgent is installed but not loaded in launchd.", `${params.title} LaunchAgent`); if (params.serviceRepairExternal) { note(EXTERNAL_SERVICE_REPAIR_NOTE, `${params.title} LaunchAgent`); - return { status: "skipped" }; + return { status: "not-loaded" }; } const shouldFix = await confirmDoctorServiceRepair(params.prompter, { @@ -109,7 +110,7 @@ async function maybeRepairLaunchAgentBootstrap(params: { initialValue: true, }); if (!shouldFix) { - return { status: "skipped" }; + return { status: "not-loaded" }; } params.runtime.log(`Bootstrapping ${params.title} LaunchAgent...`); @@ -127,13 +128,13 @@ async function maybeRepairLaunchAgentBootstrap(params: { params.runtime.error( `${params.title} LaunchAgent bootstrap failed: ${repair.detail ?? "unknown error"}`, ); - return { status: "skipped" }; + return { status: "not-loaded" }; } const verified = await isLaunchAgentLoaded({ env: params.env }); if (!verified) { params.runtime.error(`${params.title} LaunchAgent still not loaded after repair.`); - return { status: "skipped" }; + return { status: "not-loaded" }; } note(`${params.title} LaunchAgent repaired.`, `${params.title} LaunchAgent`); @@ -279,6 +280,9 @@ export async function maybeRepairGatewayDaemon(params: { prompter: params.prompter, serviceRepairExternal, }); + if (gatewayRepair.status === "not-loaded") { + return; + } if (gatewayRepair.status === "system-launchdaemon-blocked") { note(gatewayRepair.detail, "Gateway"); return; @@ -287,7 +291,6 @@ export async function maybeRepairGatewayDaemon(params: { serviceRuntime = { status: "unknown", detail: gatewayRepair.detail || serviceRuntime?.detail, - missingSupervision: true, missingGuiSession: true, }; } diff --git a/src/daemon/launchd-install.ts b/src/daemon/launchd-install.ts index f72c55fc8efb..a2e29fb18bdd 100644 --- a/src/daemon/launchd-install.ts +++ b/src/daemon/launchd-install.ts @@ -44,9 +44,12 @@ export async function uninstallLaunchAgent({ const domain = resolveLaunchAgentGuiDomain(); const label = resolveLaunchAgentLabel(env); const plistPath = resolveLaunchAgentPlistPath(env); - const bootout = await execLaunchctl(["bootout", domain, plistPath]); - if (bootout.code !== 0 && !isLaunchctlNotLoaded(bootout)) { - throw new Error(`launchctl bootout failed: ${formatLaunchctlResultDetail(bootout)}`); + const probe = await probeLaunchAgentState(`${domain}/${label}`); + if (probe.state !== "not-loaded") { + const bootout = await execLaunchctl(["bootout", domain, plistPath]); + if (bootout.code !== 0 && !isLaunchctlNotLoaded(bootout)) { + throw new Error(`launchctl bootout failed: ${formatLaunchctlResultDetail(bootout)}`); + } } try { diff --git a/src/daemon/launchd-runtime.ts b/src/daemon/launchd-runtime.ts index e7885e8ed742..ce15124ff409 100644 --- a/src/daemon/launchd-runtime.ts +++ b/src/daemon/launchd-runtime.ts @@ -208,14 +208,14 @@ export async function isLaunchAgentEnabled(args: GatewayServiceEnvArgs): Promise export async function isLaunchAgentLoaded(args: GatewayServiceEnvArgs): Promise { const domain = resolveLaunchAgentGuiDomain(); const label = resolveLaunchAgentLabel(args.env); - const res = await execLaunchctl(["print", `${domain}/${label}`]); - if (res.code === 0) { + const probe = await probeLaunchAgentState(`${domain}/${label}`); + if (probe.state === "running" || probe.state === "stopped") { return true; } - if (isLaunchctlNotLoaded(res)) { + if (probe.state === "not-loaded") { return false; } - throw new Error(`launchctl print failed: ${formatLaunchctlResultDetail(res)}`); + throw new Error(`launchctl print failed: ${probe.detail ?? "unknown error"}`); } export async function launchAgentPlistExists(env: GatewayServiceEnv): Promise { @@ -233,8 +233,8 @@ export async function readLaunchAgentRuntime( ): Promise { const domain = resolveLaunchAgentGuiDomain(); const label = resolveLaunchAgentLabel(env); - const [res, systemOwnership] = await Promise.all([ - execLaunchctl(["print", `${domain}/${label}`]), + const [probe, systemOwnership] = await Promise.all([ + probeLaunchAgentState(`${domain}/${label}`), inspectSystemLaunchDaemonOwnership(label, { scanInstalledPlists: false }), ]); if (systemOwnership.status !== "absent") { @@ -248,24 +248,27 @@ export async function readLaunchAgentRuntime( }, }; } - if (res.code !== 0) { + if (probe.state === "not-loaded") { const plistExists = await launchAgentPlistExists(env); - const detail = (res.stderr || res.stdout).trim() || undefined; - const missingGuiSession = plistExists && isUnsupportedGuiDomain(detail ?? ""); + return plistExists ? { status: "stopped" } : { status: "unknown", missingUnit: true }; + } + if (probe.state === "unknown") { + const plistExists = await launchAgentPlistExists(env); + const missingGuiSession = plistExists && isUnsupportedGuiDomain(probe.detail ?? ""); return { status: "unknown", - detail, + detail: probe.detail, ...(plistExists - ? { missingSupervision: true, ...(missingGuiSession ? { missingGuiSession } : {}) } + ? missingGuiSession + ? { missingGuiSession: true } + : {} : { missingUnit: true }), }; } - const parsed = parseLaunchctlPrint(res.stdout || res.stderr || ""); + const parsed = probe.runtime; const plistExists = await launchAgentPlistExists(env); - const state = normalizeLowercaseStringOrEmpty(parsed.state); - const status = state === "running" || parsed.pid ? "running" : state ? "stopped" : "unknown"; return { - status, + status: probe.state, state: parsed.state, pid: parsed.pid, lastExitStatus: parsed.lastExitStatus, @@ -319,16 +322,16 @@ function isLaunchctlBootstrapPendingTeardown(res: { return normalized.includes("bootstrap failed: 5") || normalized.includes("input/output error"); } type LaunchAgentProbeResult = - | { state: "running" } - | { state: "stopped" } + | { state: "running"; runtime: LaunchctlPrintInfo } + | { state: "stopped"; runtime: LaunchctlPrintInfo } | { state: "not-loaded" } | { state: "unknown"; detail?: string }; export async function probeLaunchAgentState( serviceTarget: string, ): Promise { - // `launchctl print` output is not a stable API, so this is only a stop - // confirmation probe. Unknown output falls back to bootout instead of success. + // `launchctl print` output is not a stable API. Keep expected absence and + // unexpected failures distinct so every caller applies one classification. const probe = await execLaunchctl(["print", serviceTarget]); if (probe.code !== 0) { if (isLaunchctlNotLoaded(probe)) { @@ -344,26 +347,24 @@ export async function probeLaunchAgentState( normalizeLowercaseStringOrEmpty(runtime.state) === "running" || (typeof runtime.pid === "number" && runtime.pid > 1) ) { - return { state: "running" }; + return { state: "running", runtime }; } - return { state: "stopped" }; + return { state: "stopped", runtime }; } export async function waitForLaunchAgentStopped( serviceTarget: string, ): Promise { - let lastUnknown: LaunchAgentProbeResult | null = null; + let lastProbe: LaunchAgentProbeResult = { state: "unknown" }; for (let attempt = 0; attempt < 10; attempt += 1) { const probe = await probeLaunchAgentState(serviceTarget); + lastProbe = probe; if (probe.state === "stopped" || probe.state === "not-loaded") { return probe; } - if (probe.state === "unknown") { - lastUnknown = probe; - } await new Promise((resolve) => { setTimeout(resolve, 100); }); } - return lastUnknown ?? { state: "running" }; + return lastProbe; } diff --git a/src/daemon/launchd.test.ts b/src/daemon/launchd.test.ts index 108b192e32b6..866af23df944 100644 --- a/src/daemon/launchd.test.ts +++ b/src/daemon/launchd.test.ts @@ -752,15 +752,18 @@ describe("launchd runtime parsing", () => { }); describe("launchd runtime state", () => { - it("marks installed plist split-brain when launchd no longer has the job", async () => { + it("reports an installed but unloaded LaunchAgent as stopped", async () => { const env = createDefaultLaunchdEnv(); state.files.set(resolveLaunchAgentPlistPath(env), ""); - state.serviceLoaded = false; + state.printError = [ + "Bad request.", + 'Could not find service "ai.openclaw.gateway" in domain for user gui: 501', + ].join("\n"); + state.printFailuresRemaining = 1; const runtime = await readLaunchAgentRuntime(env); - expect(runtime.status).toBe("unknown"); - expect(runtime.missingSupervision).toBe(true); - expect(runtime.detail).toBe("Could not find service"); + + expect(runtime).toEqual({ status: "stopped" }); }); it.each([ @@ -775,11 +778,24 @@ describe("launchd runtime state", () => { const runtime = await readLaunchAgentRuntime(env); expect(runtime.status).toBe("unknown"); - expect(runtime.missingSupervision).toBe(true); expect(runtime.missingGuiSession).toBe(true); expect(runtime.detail).toBe(detail); }); + it("keeps unexpected launchctl failures visible without claiming missing supervision", async () => { + const env = createDefaultLaunchdEnv(); + state.files.set(resolveLaunchAgentPlistPath(env), ""); + state.printError = "Operation not permitted\nwhile reading launchd state"; + state.printFailuresRemaining = 1; + + const runtime = await readLaunchAgentRuntime(env); + + expect(runtime).toEqual({ + status: "unknown", + detail: "Operation not permitted while reading launchd state", + }); + }); + it("marks a missing unit when launchd has no job and no plist exists", async () => { const env = createDefaultLaunchdEnv(); state.serviceLoaded = false; @@ -1574,6 +1590,18 @@ describe("launchd uninstall", () => { await expect(uninstallLaunchAgent({ env, stdout: new PassThrough() })).resolves.toBeUndefined(); }); + it("uninstalls an already stopped LaunchAgent without booting it out again", async () => { + const env = createDefaultLaunchdEnv(); + const plistPath = resolveLaunchAgentPlistPath(env); + state.files.set(plistPath, "RunAtLoad=true"); + state.serviceLoaded = false; + state.bootoutError = "Boot-out failed: 5: Input/output error"; + + await expect(uninstallLaunchAgent({ env, stdout: new PassThrough() })).resolves.toBeUndefined(); + expect(state.files.has(plistPath)).toBe(false); + expect(state.launchctlCalls.some((call) => call[0] === "bootout")).toBe(false); + }); + it("removes dangling LaunchAgent symlinks instead of treating their targets as missing", async () => { const env = createDefaultLaunchdEnv(); const plistPath = resolveLaunchAgentPlistPath(env); diff --git a/src/daemon/runtime-format.test.ts b/src/daemon/runtime-format.test.ts index 5581dfbeff96..82fd77c65747 100644 --- a/src/daemon/runtime-format.test.ts +++ b/src/daemon/runtime-format.test.ts @@ -8,4 +8,13 @@ describe("formatRuntimeStatus", () => { "stopped (last exit 134 (SIGABRT/abort))", ); }); + + it("keeps multiline runtime details on one line", () => { + expect( + formatRuntimeStatus({ + status: "unknown", + detail: "Operation failed.\nService manager returned more detail.", + }), + ).toBe("unknown (Operation failed. Service manager returned more detail.)"); + }); }); diff --git a/src/daemon/schtasks-runtime.ts b/src/daemon/schtasks-runtime.ts index 24ab5cfab071..8fd615159f64 100644 --- a/src/daemon/schtasks-runtime.ts +++ b/src/daemon/schtasks-runtime.ts @@ -512,10 +512,10 @@ export async function readScheduledTaskRuntime( return resolveFallbackRuntime(env); } const detail = (res.stderr || res.stdout).trim(); - const missing = normalizeLowercaseStringOrEmpty(detail).includes("cannot find the file"); + const missing = probeScheduledTaskExists(taskName) === false; return { status: missing ? "stopped" : "unknown", - detail: detail || undefined, + ...(!missing && detail ? { detail } : {}), missingUnit: missing, }; } diff --git a/src/daemon/schtasks.startup-fallback.test.ts b/src/daemon/schtasks.startup-fallback.test.ts index d754f29fd979..2a0792f1a9ed 100644 --- a/src/daemon/schtasks.startup-fallback.test.ts +++ b/src/daemon/schtasks.startup-fallback.test.ts @@ -368,6 +368,38 @@ afterEach(() => { }); describe("Windows startup fallback", () => { + it("uses the locale-independent task probe when a scheduled task is missing", async () => { + await withWindowsEnv("openclaw-win-startup-", async ({ env }) => { + schtasksResponses.push( + { code: 0, stdout: "", stderr: "" }, + { code: 1, stdout: "", stderr: "FEHLER: Die angegebene Datei wurde nicht gefunden." }, + ); + spawnSync.mockReturnValue(makeSpawnSyncResult({ status: 1, stdout: "-2147024894" })); + + await expect(readScheduledTaskRuntime(env)).resolves.toEqual({ + status: "stopped", + missingUnit: true, + }); + }); + }); + + it("keeps unexpected scheduled-task query failures visible", async () => { + await withWindowsEnv("openclaw-win-startup-", async ({ env }) => { + const detail = "Zugriff verweigert"; + schtasksResponses.push( + { code: 0, stdout: "", stderr: "" }, + { code: 1, stdout: "", stderr: detail }, + ); + spawnSync.mockReturnValue(makeSpawnSyncResult({ status: 1, stdout: "-2147024891" })); + + await expect(readScheduledTaskRuntime(env)).resolves.toEqual({ + status: "unknown", + detail, + missingUnit: false, + }); + }); + }); + it("reports login item removal failures without leaking the item path", async () => { await withWindowsEnv("openclaw-win-startup-", async ({ env }) => { const startupEntryPath = await writeStartupFallbackEntry(env); diff --git a/src/daemon/systemd-runtime.ts b/src/daemon/systemd-runtime.ts index fb5e61b75a77..e8940340661e 100644 --- a/src/daemon/systemd-runtime.ts +++ b/src/daemon/systemd-runtime.ts @@ -18,6 +18,7 @@ import { execSystemctl, execSystemctlUser, isSystemctlMissing, + isSystemdUnitMissingDetail, isSystemdUnitNotEnabled, readSystemctlDetail, } from "./systemd-exec.js"; @@ -161,10 +162,10 @@ export async function readSystemdServiceRuntime( : await execSystemctlUser(env, showArgs, timeoutMs); if (res.code !== 0) { const detail = (res.stderr || res.stdout).trim(); - const missing = normalizeLowercaseStringOrEmpty(detail).includes("not found"); + const missing = !installed && isSystemdUnitMissingDetail(detail); return { status: missing ? "stopped" : "unknown", - detail: detail || undefined, + ...(!missing && detail ? { detail } : {}), missingUnit: missing, }; } diff --git a/src/daemon/systemd.test.ts b/src/daemon/systemd.test.ts index 756c9b5d36a7..bcc1b55a6eb9 100644 --- a/src/daemon/systemd.test.ts +++ b/src/daemon/systemd.test.ts @@ -1009,6 +1009,64 @@ describe("readSystemdServiceRuntime", () => { }); }); + it("reports a missing unit without surfacing routine systemctl stderr", async () => { + execFileMock + .mockImplementationOnce((_cmd, args, _opts, cb) => { + assertUserSystemctlArgs(args, "status"); + cb(null, "", ""); + }) + .mockImplementationOnce((_cmd, _args, _opts, cb) => { + const detail = "Unit openclaw-gateway.service could not be found."; + cb(createExecFileError(detail, { stderr: detail }), "", detail); + }); + + await expect(readSystemdServiceRuntime({ HOME: TEST_MANAGED_HOME })).resolves.toEqual({ + status: "stopped", + missingUnit: true, + }); + }); + + it("keeps unexpected systemctl failures visible", async () => { + execFileMock + .mockImplementationOnce((_cmd, args, _opts, cb) => { + assertUserSystemctlArgs(args, "status"); + cb(null, "", ""); + }) + .mockImplementationOnce((_cmd, _args, _opts, cb) => { + const detail = "Permission denied while reading systemd state"; + cb(createExecFileError(detail, { stderr: detail }), "", detail); + }); + + await expect(readSystemdServiceRuntime({ HOME: TEST_MANAGED_HOME })).resolves.toEqual({ + status: "unknown", + detail: "Permission denied while reading systemd state", + missingUnit: false, + }); + }); + + it("does not call an installed unit missing when systemd disagrees with its definition", async () => { + const accessSpy = vi.spyOn(fs, "access").mockImplementation(async (pathArg) => { + if (pathLikeToString(pathArg) === "/etc/systemd/system/openclaw-gateway.service") { + return; + } + throw Object.assign(new Error("ENOENT"), { code: "ENOENT" }); + }); + execFileMock.mockImplementationOnce((_cmd, _args, _opts, cb) => { + const detail = "Unit openclaw-gateway.service could not be found."; + cb(createExecFileError(detail, { stderr: detail }), "", detail); + }); + + try { + await expect(readSystemdServiceRuntime({ HOME: TEST_MANAGED_HOME })).resolves.toEqual({ + status: "unknown", + detail: "Unit openclaw-gateway.service could not be found.", + missingUnit: false, + }); + } finally { + accessSpy.mockRestore(); + } + }); + it("parses Result and the restart counter for crash-loop give-up detection", async () => { // Real systemd 249 give-up shape: a crash-looped unit keeps Result=exit-code // (start-limit-hit never overwrites an exec failure), so the counter reaching diff --git a/src/infra/runtime-status.ts b/src/infra/runtime-status.ts index acc6babfa486..1c71f8892661 100644 --- a/src/infra/runtime-status.ts +++ b/src/infra/runtime-status.ts @@ -31,7 +31,7 @@ export function formatRuntimeStatusWithDetails({ fullDetails.push(`state ${normalizedState}`); } for (const detail of details) { - const normalizedDetail = detail.trim(); + const normalizedDetail = detail.replace(/\s+/g, " ").trim(); if (normalizedDetail) { fullDetails.push(normalizedDetail); }