fix(macos): keep the node channel alive and visible when the node-host worker cannot start (#129925)

* fix(macos): keep the node channel alive and visible when the node-host worker cannot start A node-host worker that exited before its ready manifest never notified the retry policy, so the coordinator respawned the broken CLI forever and the whole node channel silently never dialed the gateway. Startup exits now consume the crash retry budget and carry the worker's stderr into the start error; every non-transient worker failure degrades the connect to native capabilities instead of aborting it; and the menu bar surfaces the recorded node-channel state with the concrete reason. * fix(macos): render node-channel status as a top-level menu view The native-menu extra style flattens multi-view Toggle labels to their first Text, so status sublines inside the label never rendered — the original 'zero indication' report. Top-level menu views render (exec-approval error pattern).
2026-08-28 13:26:04 -06:00 · 2026-08-26 00:02:52 -07:00
parent 4fcbf7c20d
commit af9b0c7616
6 changed files with 258 additions and 17 deletions
@@ -17,6 +17,7 @@ struct MenuContent: View {
    private let dashboardManager = DashboardManager.shared
    private let activityStore = WorkActivityStore.shared
    private let nodesStore = NodesStore.shared
+    private let nodeChannelStatus = MacNodeChannelStatusStore.shared
    @Bindable private var pairingPrompter = NodePairingApprovalPrompter.shared
    @Bindable private var devicePairingPrompter = DevicePairingApprovalPrompter.shared
    @State private var availableMics: [AudioInputDevice] = []
@@ -49,9 +50,6 @@ struct MenuContent: View {
                VStack(alignment: .leading, spacing: 2) {
                    Text(self.connectionLabel)
                    self.statusLine(label: self.healthStatus.label, color: self.healthStatus.color)
-                    if let macNodeStatus = self.macNodeStatus {
-                        self.statusLine(label: macNodeStatus.label, color: macNodeStatus.color)
-                    }
                    if self.pairingPrompter.pendingCount > 0 {
                        self.pairingStatusLine(
                            label: "Pairing approval pending (\(self.pairingPrompter.pendingCount))")
@@ -65,6 +63,15 @@ struct MenuContent: View {
                }
            }
            .disabled(self.state.connectionMode == .unconfigured)
+            // The native-menu extra style flattens multi-view Toggle labels to
+            // their first Text, so status sublines inside the label above never
+            // render. Node-channel state must be a top-level menu view to stay
+            // operator-visible (same pattern as the exec-approval error lines).
+            if let macNodeStatus = self.macNodeStatus {
+                Text(macNodeStatus.label)
+                    .font(.caption)
+                    .foregroundStyle(macNodeStatus.color)
+            }

            Divider()
            Toggle(isOn: self.heartbeatsBinding) {
@@ -366,6 +373,12 @@ struct MenuContent: View {
        guard self.state.connectionMode != .unconfigured else { return nil }
        guard case .connected = self.controlChannel.state else { return nil }

+        // The coordinator records why the node channel is down at the connect
+        // boundary; prefer that recorded fact over inferring from node listings.
+        if let line = self.nodeChannelStatus.state.operatorStatusLine {
+            return (line.label, line.isDegraded ? .orange : .red)
+        }
+
        let deviceId: String
        switch self.nodesStore.localNodeIdentityState {
        case .loading:
@@ -0,0 +1,50 @@
+import Foundation
+import Observation
+
+/// Recorded fact for the Mac node channel, written by MacNodeModeCoordinator at
+/// the connect boundary. The menu bar reads this instead of inferring node
+/// health from gateway node listings, so a node channel that never dials still
+/// surfaces its reason to the operator.
+enum MacNodeChannelState: Equatable, Sendable {
+    /// Node mode is paused, stopped, or not configured to run.
+    case idle
+    /// The channel connected. A non-nil reason means the node-host worker is
+    /// unavailable and only native capabilities are advertised.
+    case connected(workerUnavailableReason: String?)
+    /// The last connect attempt failed; the coordinator keeps retrying.
+    case unavailable(reason: String)
+
+    var operatorStatusLine: (label: String, isDegraded: Bool)? {
+        switch self {
+        case .idle, .connected(workerUnavailableReason: nil):
+            nil
+        case let .connected(workerUnavailableReason: .some(reason)):
+            ("Mac node degraded — \(Self.condense(reason))", true)
+        case let .unavailable(reason):
+            ("Mac node unavailable — \(Self.condense(reason))", false)
+        }
+    }
+
+    /// Menu status lines are single-line; keep the leading reason sentence and
+    /// bound it so a CLI stack trace cannot flood the menu.
+    private static func condense(_ reason: String) -> String {
+        let firstLine = reason
+            .split(separator: "\n", omittingEmptySubsequences: true)
+            .first
+            .map { $0.trimmingCharacters(in: .whitespaces) } ?? reason
+        return firstLine.count > 220 ? firstLine.prefix(220) + "…" : firstLine
+    }
+}
+
+@MainActor
+@Observable
+final class MacNodeChannelStatusStore {
+    static let shared = MacNodeChannelStatusStore()
+
+    private(set) var state: MacNodeChannelState = .idle
+
+    func record(_ state: MacNodeChannelState) {
+        guard self.state != state else { return }
+        self.state = state
+    }
+}
@@ -102,6 +102,10 @@ final class MacNodeHostWorker: MacNodeHostWorking, @unchecked Sendable {
    private var processGeneration: UUID?
    private var launchedWorker: MacNodeHostWorkerLaunch?
    private var stdoutBuffer = Data()
+    // Bounded head of worker stderr. CLI startup failures print their cause
+    // first; without this the operator-visible error is just "exited(1)".
+    private var stderrHead = ""
+    private static let maxStderrHeadLength = 700
    private var manifest: MacNodeHostManifest?
    private var inventoryData: Data?
    private var route: GatewayNodeSessionRoute?
@@ -377,7 +381,7 @@ final class MacNodeHostWorker: MacNodeHostWorking, @unchecked Sendable {
            let state = self.process?.isRunning == true ? "running" : "exited"
            self.finishStartLocked(.failure(WorkerError.unavailable(
                "node-host worker startup timed out (process \(state), buffered \(self.stdoutBuffer.count) bytes)")))
-            self.stopLocked(reason: "worker startup timed out")
+            self.stopLocked(reason: "worker startup timed out", notifyUnexpectedExit: true)
        }
        self.startTimer = timer
        timer.resume()
@@ -448,6 +452,10 @@ final class MacNodeHostWorker: MacNodeHostWorking, @unchecked Sendable {
                !message.isEmpty
            {
                self.logger.error("node-host worker stderr: \(message, privacy: .private)")
+                if self.stderrHead.count < Self.maxStderrHeadLength {
+                    self.stderrHead.append(self.stderrHead.isEmpty ? message : "\n" + message)
+                    self.stderrHead = String(self.stderrHead.prefix(Self.maxStderrHeadLength))
+                }
            }
        }
        self.stderrSource = stderrSource
@@ -720,8 +728,12 @@ final class MacNodeHostWorker: MacNodeHostWorking, @unchecked Sendable {
        preserveStart: Bool = false,
        notifyUnexpectedExit: Bool = false) -> Task<Void, Never>?
    {
-        let wasReady = self.manifest != nil
        let stoppedWorker = self.launchedWorker
+        // A worker that dies before its ready manifest still needs its stderr
+        // surfaced: the raw exit status alone cannot explain a CLI bootstrap
+        // refusal (missing runtime, incompatible state database, bad install).
+        let detailedReason = self.stderrHead.isEmpty ? reason : "\(reason): \(self.stderrHead)"
+        self.stderrHead = ""
        self.startTimer?.cancel()
        self.startTimer = nil
        self.launchedWorker = nil
@@ -730,7 +742,7 @@ final class MacNodeHostWorker: MacNodeHostWorking, @unchecked Sendable {
        self.inventoryData = nil
        self.route = nil
        if !preserveStart {
-            self.finishStartLocked(.failure(WorkerError.unavailable(reason)))
+            self.finishStartLocked(.failure(WorkerError.unavailable(detailedReason)))
        }
        if let processCleanupTask = self.processCleanupTask { return processCleanupTask }
        let pending = self.invokeContinuations
@@ -740,7 +752,10 @@ final class MacNodeHostWorker: MacNodeHostWorking, @unchecked Sendable {
        for (id, continuation) in pending {
            continuation.resume(returning: Self.unavailableResponse(id, "UNAVAILABLE: node-host worker stopped"))
        }
-        if notifyUnexpectedExit, wasReady, let stoppedWorker {
+        // Startup-time exits count too: without this, a worker that dies before
+        // its ready manifest never consumes retry budget and the coordinator
+        // respawns a broken CLI forever instead of latching retry exhaustion.
+        if notifyUnexpectedExit, let stoppedWorker {
            self.onUnexpectedExit(stoppedWorker.configurationGeneration)
        }
        guard let process = self.process else {
@@ -55,6 +55,7 @@ private struct ConnectionAttempt {
    let routeAuthorityGeneration: UInt64
    let codexThreadCatalogAdvertised: Bool
    let claudeSessionCatalogAdvertised: Bool
+    let workerUnavailableReason: String?
    let endpoint: GatewayConnection.EndpointSnapshot
    let options: GatewayConnectOptions
    let sessionBox: WebSocketSessionBox?
@@ -114,11 +115,13 @@ final class MacNodeModeCoordinator: NSObject {
    private var nodeHostWorkerRetryTaskGeneration: UInt64 = 0
    private var pendingEndpoint: GatewayConnection.EndpointSnapshot?
    private var activeNodeHostWorkerInput: MacNodeHostWorkerRetryPolicy.Input?
+    private var lastNodeHostWorkerStartFailure: String?
    private var lastObservedPaused: Bool
    private var lastObservedComputerControlEnabled: Bool
    private var lastObservedComputerControlProvider: ComputerControlProvider
    private let runtime: MacNodeRuntime
    private let session: GatewayNodeSession
+    private let channelStatus: MacNodeChannelStatusStore
    private let nodeHostWorker: (any MacNodeHostWorking)?
    private let presenceReporter: MacNodePresenceReporter
    private let notificationCenter: NotificationCenter
@@ -153,6 +156,7 @@ final class MacNodeModeCoordinator: NSObject {
        runtime: MacNodeRuntime,
        nodeHostWorker: (any MacNodeHostWorking)? = nil,
        presenceReporter: MacNodePresenceReporter = MacNodePresenceReporter(),
+        channelStatus: MacNodeChannelStatusStore = .shared,
        notificationCenter: NotificationCenter = .default,
        observeNotifications: Bool = false,
        initialPaused: Bool? = nil,
@@ -168,6 +172,7 @@ final class MacNodeModeCoordinator: NSObject {
        self.runtime = runtime
        self.nodeHostWorker = nodeHostWorker
        self.presenceReporter = presenceReporter
+        self.channelStatus = channelStatus
        self.notificationCenter = notificationCenter
        self.nodeHostWorkerRetrySleep = nodeHostWorkerRetrySleep
        self.nodeHostWorkerRetryPolicy = nodeHostWorkerRetryPolicy
@@ -273,6 +278,7 @@ final class MacNodeModeCoordinator: NSObject {
    }

    private func cancelCoordinatorTasks() {
+        self.channelStatus.record(.idle)
        self.task?.cancel()
        self.task = nil
        self.endpointRefreshTask?.cancel()
@@ -474,6 +480,7 @@ final class MacNodeModeCoordinator: NSObject {
            if Self.pausedStateRequiresDisconnect(isPaused) {
                // Pause revokes the node route, not only the outer retry loop. A
                // connected gateway was revoked before this refresh wake was emitted.
+                self.channelStatus.record(.idle)
                guard await refreshIterator.next() != nil else { return }
                continue
            }
@@ -520,16 +527,11 @@ final class MacNodeModeCoordinator: NSObject {
                if error is MacNodeHostWorkerRetryPolicy.RetryBackoffPending {
                    // The lifecycle-owned delayed wake is the only event allowed
                    // to admit this same worker input after an unexpected exit.
+                    self.channelStatus.record(.unavailable(
+                        reason: self.lastNodeHostWorkerStartFailure ?? error.localizedDescription))
                    guard await refreshIterator.next() != nil else { return }
                    continue
                }
-                if error is MacNodeHostWorkerRetryPolicy.RetryBudgetExhausted {
-                    // Only a new worker command or startup-scoped configuration
-                    // generation can re-arm a terminally exhausted worker.
-                    guard await refreshIterator.next() != nil else { return }
-                    retryDelay = 1_000_000_000
-                    continue
-                }
                if let tlsError = error as? GatewayTLSValidationError,
                   let attemptedEndpoint,
                   await GatewayTLSRepairCoordinator.shared.repair(
@@ -542,6 +544,7 @@ final class MacNodeModeCoordinator: NSObject {
                    continue
                }
                self.logger.error("mac node gateway connect failed: \(error.localizedDescription, privacy: .public)")
+                self.channelStatus.record(.unavailable(reason: error.localizedDescription))
                try? await Task.sleep(nanoseconds: min(retryDelay, 10_000_000_000))
                retryDelay = min(retryDelay * 2, 10_000_000_000)
            }
@@ -559,9 +562,8 @@ final class MacNodeModeCoordinator: NSObject {
    {
        let config = endpoint.config
        let provider = ComputerControlProvider.current()
-        let workerManifest = try await Self.workerManifest(
-            self.startNodeHostWorkerIfConfigured(provider: provider),
-            for: provider)
+        let (workerManifest, workerUnavailableReason) =
+            try await self.resolveWorkerManifestForConnection(provider: provider)
        let nativeCaps = self.currentCaps(
            browserControlEnabled: browserControlEnabled,
            cameraEnabled: cameraEnabled,
@@ -633,6 +635,7 @@ final class MacNodeModeCoordinator: NSObject {
                MacNodeCodexThreadCatalogContract.listCommand),
            claudeSessionCatalogAdvertised: commands.contains(
                MacNodeClaudeSessionCatalogContract.listCommand),
+            workerUnavailableReason: workerUnavailableReason,
            endpoint: endpoint,
            options: options,
            sessionBox: sessionBox,
@@ -660,6 +663,8 @@ final class MacNodeModeCoordinator: NSObject {
                guard workerRouteInstalled else { return }
                await self.nodeHostWorker?.publishInventory(ifCurrentRoute: installedRoute)
                await self.cancelReconnectProbe()
+                await self.channelStatus.record(.connected(
+                    workerUnavailableReason: attempt.workerUnavailableReason))
                self.logger.info("mac node connected to gateway")
                // The node hello owns this route's session defaults. Reusing the operator
                // connection here can trigger remote-tunnel recovery while the node connects.
@@ -692,6 +697,7 @@ final class MacNodeModeCoordinator: NSObject {
            },
            onDisconnected: { [weak self] reason in
                guard let self else { return }
+                await self.channelStatus.record(.unavailable(reason: reason))
                await self.invalidateRuntimeRoute(authorityGeneration: attempt.routeAuthorityGeneration)
                await self.scheduleReconnectProbe()
                self.logger.error("mac node disconnected: \(reason, privacy: .public)")
@@ -864,6 +870,13 @@ final class MacNodeModeCoordinator: NSObject {
    func handleNodeHostConfigurationChangeForTesting() async {
        await self.handleNodeHostConfigurationChange().value
    }
+
+    func resolveWorkerManifestForConnectionForTesting(
+        provider: ComputerControlProvider = .peekaboo) async throws
+        -> (manifest: MacNodeHostManifest?, unavailableReason: String?)
+    {
+        try await self.resolveWorkerManifestForConnection(provider: provider)
+    }
    #endif

    private func cancelReconnectProbe() {
@@ -931,6 +944,26 @@ extension MacNodeModeCoordinator {
        Self.resolvedCommands(caps: caps, computerControlProvider: computerControlProvider)
    }

+    /// The node-host worker is a capability superset, not a connect
+    /// precondition. Backoff-pending is transient (its lifecycle-owned wake
+    /// retries shortly); every other worker failure connects this Mac with
+    /// native capabilities only and surfaces the reason to the operator.
+    private func resolveWorkerManifestForConnection(
+        provider: ComputerControlProvider) async throws
+        -> (manifest: MacNodeHostManifest?, unavailableReason: String?)
+    {
+        do {
+            let manifest = try await Self.workerManifest(
+                self.startNodeHostWorkerIfConfigured(provider: provider),
+                for: provider)
+            return (manifest, nil)
+        } catch let backoff as MacNodeHostWorkerRetryPolicy.RetryBackoffPending {
+            throw backoff
+        } catch {
+            return (nil, self.recordNodeHostWorkerStartFailure(error))
+        }
+    }
+
    private func startNodeHostWorkerIfConfigured(
        provider: ComputerControlProvider) async throws -> MacNodeHostManifest?
    {
@@ -975,6 +1008,18 @@ extension MacNodeModeCoordinator {
        return try await nodeHostWorker.start(launch: effectiveLaunch)
    }

+    /// Retry exhaustion keeps the concrete worker error it exhausted on; the
+    /// bare "stopped after N unexpected exits" text cannot guide the operator.
+    private func recordNodeHostWorkerStartFailure(_ error: Error) -> String {
+        if error is MacNodeHostWorkerRetryPolicy.RetryBudgetExhausted {
+            let detail = self.lastNodeHostWorkerStartFailure.map { " — \($0)" } ?? ""
+            return error.localizedDescription + detail
+        }
+        let reason = error.localizedDescription
+        self.lastNodeHostWorkerStartFailure = reason
+        return reason
+    }
+
    private func handleNodeHostWorkerFailure(configurationGeneration: UInt64) {
        guard configurationGeneration == self.nodeHostWorkerConfigurationGeneration else { return }
        guard let input = self.activeNodeHostWorkerInput else {
@@ -1014,6 +1059,13 @@ extension MacNodeModeCoordinator {
                name: .openclawNodeHostWorkerRetryExhausted,
                object: self,
                userInfo: ["unexpectedExitCount": unexpectedExitCount])
+            // The exhausted worker must not take the node channel with it. Wake
+            // the connect loop after the disconnect drains so the next attempt
+            // reconnects with native capabilities and a visible degraded reason.
+            Task { @MainActor [weak self] in
+                await invalidation.value
+                self?.refreshContinuation.yield()
+            }
        }
    }

@@ -1026,6 +1078,7 @@ extension MacNodeModeCoordinator {
    private func resetNodeHostWorkerRetryState() {
        self.cancelNodeHostWorkerRetryTask()
        self.activeNodeHostWorkerInput = nil
+        self.lastNodeHostWorkerStartFailure = nil
        self.nodeHostWorkerRetryPolicy.reset()
    }

@@ -740,4 +740,39 @@ struct MacNodeHostWorkerTests {
            throw error
        }
    }
+
+    // Regression: a worker that exits before its ready manifest must consume the
+    // crash retry budget and carry its stderr into the start error. Before this,
+    // startup-time CLI refusals (for example a state database schema mismatch)
+    // never notified the retry policy, so the coordinator respawned the broken
+    // CLI forever and the operator only ever saw "exited(1)" in os_log.
+    @Test func `startup exit consumes retry budget and surfaces worker stderr`() async throws {
+        let exitGate = AsyncTestGate()
+        let exitGeneration = OSAllocatedUnfairLock<UInt64?>(initialState: nil)
+        let worker = MacNodeHostWorker(
+            session: GatewayNodeSession(),
+            startupTimeout: 5,
+            onUnexpectedExit: { generation in
+                exitGeneration.withLock { $0 = generation }
+                exitGate.open()
+            })
+        let script = """
+        echo 'refused: state database uses newer schema version' >&2
+        sleep 0.2
+        exit 7
+        """
+
+        do {
+            _ = try await worker.start(launch: MacNodeHostWorkerLaunch(
+                command: ["/bin/sh", "-c", script],
+                configurationGeneration: 3))
+            Issue.record("worker start unexpectedly succeeded")
+        } catch {
+            #expect(error.localizedDescription.contains("state database uses newer schema version"))
+        }
+
+        await exitGate.wait()
+        #expect(exitGeneration.withLock { $0 } == 3)
+        await worker.stop()
+    }
 }
@@ -123,6 +123,37 @@ private actor CoordinatorNodeHostWorkerProbe: MacNodeHostWorking {
    }
 }

+private actor CoordinatorFailingStartWorkerProbe: MacNodeHostWorking {
+    private var startCalls = 0
+
+    func start(launch _: MacNodeHostWorkerLaunch) async throws -> MacNodeHostManifest {
+        self.startCalls += 1
+        throw MacNodeHostWorker.WorkerError.unavailable(
+            "state database uses newer schema version 10")
+    }
+
+    func supports(_: String) async -> Bool {
+        false
+    }
+
+    func invoke(_ request: BridgeInvokeRequest) async -> BridgeInvokeResponse {
+        BridgeInvokeResponse(id: request.id, ok: false)
+    }
+
+    func handleInput(invokeId _: String, seq _: Int, payloadJSON _: String) async {}
+    func cancel(invokeId _: String) async {}
+    func setRoute(_: GatewayNodeSessionRoute?, authorityGeneration _: UInt64) async -> Bool {
+        true
+    }
+
+    func publishInventory(ifCurrentRoute _: GatewayNodeSessionRoute) async {}
+    func stop() async {}
+
+    func startCallCount() -> Int {
+        self.startCalls
+    }
+}
+
 private final class CoordinatorRetrySleeperProbe: @unchecked Sendable {
    private let entered = AsyncTestGate()
    private let releaseGate = AsyncTestGate()
@@ -325,6 +356,50 @@ struct MacNodeModeCoordinatorTests {
        await coordinator.stopAndWait()
    }

+    // Regression: an exhausted node-host worker must degrade the connect to
+    // native capabilities with a visible reason. Before this, retry exhaustion
+    // (and any startup-scoped worker failure) aborted the whole connection
+    // attempt, so the node channel never dialed and the operator saw nothing.
+    @Test @MainActor func `worker retry exhaustion degrades the node connect instead of blocking it`() async throws {
+        let worker = CoordinatorFailingStartWorkerProbe()
+        let session = GatewayNodeSession()
+        let coordinator = MacNodeModeCoordinator(
+            session: session,
+            runtime: MacNodeRuntime(nodeHostWorker: worker),
+            nodeHostWorker: worker,
+            notificationCenter: NotificationCenter(),
+            nodeHostWorkerRetryPolicy: MacNodeHostWorkerRetryPolicy(maximumRetryCount: 0))
+
+        try coordinator.prepareNodeHostWorkerRetryForTesting(
+            command: ["/usr/local/bin/openclaw", "node", "worker"])
+        coordinator.handleNodeHostWorkerFailureForTesting()
+        await coordinator.waitForRouteInvalidationForTesting()
+
+        let resolved = try await coordinator.resolveWorkerManifestForConnectionForTesting()
+        #expect(resolved.manifest == nil)
+        #expect(resolved.unavailableReason?.contains("unexpected exits") == true)
+        // The exhausted budget must also stop worker respawn attempts.
+        #expect(await worker.startCallCount() == 0)
+        await coordinator.stopAndWait()
+    }
+
+    @Test func `node channel states map to operator status lines`() {
+        #expect(MacNodeChannelState.idle.operatorStatusLine == nil)
+        #expect(MacNodeChannelState.connected(workerUnavailableReason: nil).operatorStatusLine == nil)
+
+        let degraded = MacNodeChannelState
+            .connected(workerUnavailableReason: "worker exited: schema mismatch")
+            .operatorStatusLine
+        #expect(degraded?.label == "Mac node degraded — worker exited: schema mismatch")
+        #expect(degraded?.isDegraded == true)
+
+        let unavailable = MacNodeChannelState
+            .unavailable(reason: "state database uses newer schema version 10\nTry: openclaw doctor")
+            .operatorStatusLine
+        #expect(unavailable?.label == "Mac node unavailable — state database uses newer schema version 10")
+        #expect(unavailable?.isDegraded == false)
+    }
+
    @Test func `paused node state requires route disconnect`() {
        #expect(MacNodeModeCoordinator.pausedStateRequiresDisconnect(true))
        #expect(!MacNodeModeCoordinator.pausedStateRequiresDisconnect(false))