From 167a8ef20a2a60594cd3fae1e96e3e95ffad9e4c Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 17:24:53 -0700 Subject: [PATCH] fix(onboarding): harden fresh local startup and activation (#108764) * fix(macos): preserve gateway during fresh setup * fix(macos): validate reusable gateway service * chore(macos): sync native i18n inventory * fix(macos): preserve latest gateway service request * test(macos): isolate gateway port concurrency coverage * fix(macos): recover gateways after failed readiness * fix(macos): repair wedged gateway listeners * fix(macos): bound gateway startup probes * chore(macos): sync native i18n inventory * fix(onboarding): activate verified inference immediately * fix(macos): isolate readiness failure generations * chore(macos): refresh native i18n inventory * fix(macos): serialize gateway service lifecycle * fix(macos): sequence gateway startup persistence * fix(macos): close gateway readiness races * fix(macos): publish recovered gateway state * fix(macos): make gateway readiness generation-safe * chore(macos): refresh native i18n inventory * fix(macos): preserve gateway endpoint identity * fix(macos): refresh gateway ownership after attach * fix(macos): distinguish transient gateway readiness * refactor(macos): split gateway readiness lifecycle * chore(macos): refresh native i18n inventory * chore(release): keep changelog release-owned --- apps/.i18n/native-source.json | 10 +- .../OpenClaw/ConnectionModeCoordinator.swift | 17 +- .../OpenClaw/GatewayLaunchAgentManager.swift | 90 +- .../OpenClaw/GatewayProcessManager.swift | 878 ++++++++- .../GatewayProcessManagerTests.swift | 1702 ++++++++++++++++- src/system-agent/setup-inference.test.ts | 5 +- src/system-agent/setup-inference.ts | 4 +- 7 files changed, 2624 insertions(+), 82 deletions(-) diff --git a/apps/.i18n/native-source.json b/apps/.i18n/native-source.json index 57841d398e47..bc43f4473e85 100644 --- a/apps/.i18n/native-source.json +++ b/apps/.i18n/native-source.json @@ -29883,7 +29883,7 @@ }, { "kind": "conditional-branch", - "line": 18, + "line": 78, "path": "apps/macos/Sources/OpenClaw/GatewayProcessManager.swift", "source": "Stopped", "surface": "apple", @@ -29891,7 +29891,7 @@ }, { "kind": "conditional-branch", - "line": 19, + "line": 79, "path": "apps/macos/Sources/OpenClaw/GatewayProcessManager.swift", "source": "Starting…", "surface": "apple", @@ -29899,7 +29899,7 @@ }, { "kind": "conditional-branch", - "line": 28, + "line": 88, "path": "apps/macos/Sources/OpenClaw/GatewayProcessManager.swift", "source": "Failed: \\(reason)", "surface": "apple", @@ -29907,7 +29907,7 @@ }, { "kind": "conditional-branch", - "line": 257, + "line": 571, "path": "apps/macos/Sources/OpenClaw/GatewayProcessManager.swift", "source": "not linked", "surface": "apple", @@ -29915,7 +29915,7 @@ }, { "kind": "conditional-branch", - "line": 270, + "line": 584, "path": "apps/macos/Sources/OpenClaw/GatewayProcessManager.swift", "source": "unknown error", "surface": "apple", diff --git a/apps/macos/Sources/OpenClaw/ConnectionModeCoordinator.swift b/apps/macos/Sources/OpenClaw/ConnectionModeCoordinator.swift index b1c5eab1dbb4..fc476e4f0780 100644 --- a/apps/macos/Sources/OpenClaw/ConnectionModeCoordinator.swift +++ b/apps/macos/Sources/OpenClaw/ConnectionModeCoordinator.swift @@ -7,10 +7,13 @@ final class ConnectionModeCoordinator { private let logger = Logger(subsystem: "ai.openclaw", category: "connection") private var lastMode: AppState.ConnectionMode? + private var applyGeneration: UInt64 = 0 /// Apply the requested connection mode by starting/stopping local gateway, /// managing the control-channel SSH tunnel, and cleaning up chat windows/panels. func apply(mode: AppState.ConnectionMode, paused: Bool) async { + self.applyGeneration &+= 1 + let applyGeneration = self.applyGeneration if let lastMode = self.lastMode, lastMode != mode { GatewayProcessManager.shared.clearLastFailure() NodesStore.shared.lastError = nil @@ -29,19 +32,29 @@ final class ConnectionModeCoordinator { case .local: _ = await NodeServiceManager.stop() + guard self.applyGeneration == applyGeneration else { return } NodesStore.shared.lastError = nil await RemoteTunnelManager.shared.stopAll() + guard self.applyGeneration == applyGeneration else { return } WebChatManager.shared.resetTunnels() let shouldStart = GatewayAutostartPolicy.shouldStartGateway(mode: .local, paused: paused) if shouldStart { GatewayProcessManager.shared.setActive(true) + await GatewayProcessManager.shared.waitForStartupAttempt() + guard self.applyGeneration == applyGeneration else { return } + var launchAgentInstalled = false if GatewayAutostartPolicy.shouldEnsureLaunchAgent( mode: .local, paused: paused) { - Task { await GatewayProcessManager.shared.ensureLaunchAgentEnabledIfNeeded() } + launchAgentInstalled = await GatewayProcessManager.shared.ensureLaunchAgentEnabledIfNeeded() } - _ = await GatewayProcessManager.shared.waitForGatewayReady() + guard self.applyGeneration == applyGeneration else { return } + // Always finish the generation-aware health audit after persistence work. A newer + // inactive lifecycle makes this return false without touching its repair marker. + _ = await GatewayProcessManager.shared.waitForGatewayReady( + launchAgentInstalled: launchAgentInstalled) + guard self.applyGeneration == applyGeneration else { return } } else { GatewayProcessManager.shared.stop() } diff --git a/apps/macos/Sources/OpenClaw/GatewayLaunchAgentManager.swift b/apps/macos/Sources/OpenClaw/GatewayLaunchAgentManager.swift index a61ff241f872..b4f0de888bfb 100644 --- a/apps/macos/Sources/OpenClaw/GatewayLaunchAgentManager.swift +++ b/apps/macos/Sources/OpenClaw/GatewayLaunchAgentManager.swift @@ -1,6 +1,11 @@ import Foundation enum GatewayLaunchAgentManager { + struct LoadedGatewayState: Equatable, Sendable { + let runningPID: Int32? + let reusablePID: Int32? + } + private static let logger = Logger(subsystem: "ai.openclaw", category: "gateway.launchd") private static let disableLaunchAgentMarker = ".openclaw/disable-launchagent" @@ -58,9 +63,24 @@ enum GatewayLaunchAgentManager { return nil } - static func isLoaded() async -> Bool { - guard let loaded = await self.readDaemonLoaded() else { return false } - return loaded + static func reusableLoadedGatewayPID(port: Int) async -> Int32? { + await self.loadedGatewayState(port: port).reusablePID + } + + static func loadedGatewayState(port: Int) async -> LoadedGatewayState { + guard let service = await self.readDaemonService() else { + return LoadedGatewayState(runningPID: nil, reusablePID: nil) + } + let runningPID = self.runningGatewayPID(from: service) + let configAudit = service["configAudit"] as? [String: Any] + let reusablePID: Int32? = if configAudit?["ok"] as? Bool == true, + self.gatewayPort(from: service) == port + { + runningPID + } else { + nil + } + return LoadedGatewayState(runningPID: runningPID, reusablePID: reusablePID) } static func runningGatewayPID() async -> Int32? { @@ -70,7 +90,7 @@ enum GatewayLaunchAgentManager { static func set(enabled: Bool, bundlePath: String, port: Int) async -> String? { _ = bundlePath - guard !CommandResolver.connectionModeIsRemote() else { + if enabled, CommandResolver.connectionModeIsRemote() { self.logger.info("launchd change skipped (remote mode)") return nil } @@ -136,10 +156,6 @@ enum GatewayLaunchAgentManager { } extension GatewayLaunchAgentManager { - private static func readDaemonLoaded() async -> Bool? { - await self.readDaemonService()?["loaded"] as? Bool - } - private static func readDaemonService() async -> [String: Any]? { let result = await self.runDaemonCommandResult( ["status", "--json", "--no-probe"], @@ -155,6 +171,33 @@ extension GatewayLaunchAgentManager { return service } + private static func gatewayPort(from service: [String: Any]) -> Int? { + guard let command = service["command"] as? [String: Any] else { return nil } + if let arguments = command["programArguments"] as? [String] { + for (index, argument) in arguments.enumerated() { + if argument == "--port" { + guard arguments.indices.contains(index + 1) else { return nil } + return self.validGatewayPort(arguments[index + 1]) + } + if argument.hasPrefix("--port=") { + return self.validGatewayPort(String(argument.dropFirst("--port=".count))) + } + } + } + let environment = command["environment"] as? [String: Any] + return self.validGatewayPort(environment?["OPENCLAW_GATEWAY_PORT"] as? String) + } + + private static func validGatewayPort(_ raw: String?) -> Int? { + guard let raw, + let port = Int(raw.trimmingCharacters(in: .whitespacesAndNewlines)), + (1...65535).contains(port) + else { + return nil + } + return port + } + private static func runningGatewayPID(from service: [String: Any]) -> Int32? { guard service["loaded"] as? Bool == true, let runtime = service["runtime"] as? [String: Any], @@ -197,9 +240,21 @@ extension GatewayLaunchAgentManager { #if DEBUG if self.testingInterceptDaemonCommands { self.testingDaemonCommandCalls.append(args) + if self.testingDaemonCommandDelayNanoseconds > 0 { + try? await Task.sleep(nanoseconds: self.testingDaemonCommandDelayNanoseconds) + } + let payload = if args.first == "status" { + if self.testingDaemonStatusPayloads.isEmpty { + self.testingDaemonStatusPayload ?? "{\"ok\":true}" + } else { + self.testingDaemonStatusPayloads.removeFirst() + } + } else { + "{\"ok\":true}" + } return CommandResult( success: true, - payload: Data("{\"ok\":true}".utf8), + payload: Data(payload.utf8), message: nil) } #endif @@ -251,6 +306,9 @@ extension GatewayLaunchAgentManager { private nonisolated(unsafe) static var testingDisableLaunchAgentMarkerURL: URL? private nonisolated(unsafe) static var testingInterceptDaemonCommands = false private nonisolated(unsafe) static var testingDaemonCommandCalls: [[String]] = [] + private nonisolated(unsafe) static var testingDaemonStatusPayload: String? + private nonisolated(unsafe) static var testingDaemonStatusPayloads: [String] = [] + private nonisolated(unsafe) static var testingDaemonCommandDelayNanoseconds: UInt64 = 0 static func setTestingDisableLaunchAgentMarkerURL(_ url: URL?) { self.testingDisableLaunchAgentMarkerURL = url @@ -260,6 +318,20 @@ extension GatewayLaunchAgentManager { self.testingInterceptDaemonCommands = intercept } + static func setTestingDaemonStatusPayload(_ payload: String?) { + self.testingDaemonStatusPayload = payload + self.testingDaemonStatusPayloads = [] + } + + static func setTestingDaemonStatusPayloads(_ payloads: [String]) { + self.testingDaemonStatusPayload = nil + self.testingDaemonStatusPayloads = payloads + } + + static func setTestingDaemonCommandDelayNanoseconds(_ nanoseconds: UInt64) { + self.testingDaemonCommandDelayNanoseconds = nanoseconds + } + static func clearTestingDaemonCommandCalls() { self.testingDaemonCommandCalls.removeAll(keepingCapacity: false) } diff --git a/apps/macos/Sources/OpenClaw/GatewayProcessManager.swift b/apps/macos/Sources/OpenClaw/GatewayProcessManager.swift index c4595182d8db..087ef2d85eff 100644 --- a/apps/macos/Sources/OpenClaw/GatewayProcessManager.swift +++ b/apps/macos/Sources/OpenClaw/GatewayProcessManager.swift @@ -1,11 +1,71 @@ import Foundation import Observation +import OpenClawKit + +private struct GatewayHealthProbeTimeout: LocalizedError, Sendable { + let timeoutMs: Double + + var errorDescription: String? { + "Gateway health probe timed out after \(Int(self.timeoutMs))ms" + } +} @MainActor @Observable final class GatewayProcessManager { static let shared = GatewayProcessManager() + private struct LaunchAgentEnableRequest: Sendable { + let bundlePath: String + let port: Int + let generation: UInt64 + var invocationIDs: [UInt64] + + func hasSameConfiguration(as other: LaunchAgentEnableRequest) -> Bool { + self.bundlePath == other.bundlePath && + self.port == other.port && + self.generation == other.generation + } + } + + private struct LaunchAgentReadinessFailure: Equatable { + let port: Int + let pid: Int32 + } + + private struct LaunchAgentReadinessCandidate: Equatable { + let failure: LaunchAgentReadinessFailure + let generation: UInt64 + } + + private struct LaunchAgentEnableResult: Sendable { + let error: String? + let installed: Bool + + static let skipped = LaunchAgentEnableResult(error: nil, installed: false) + + static func failed(_ error: String) -> LaunchAgentEnableResult { + LaunchAgentEnableResult(error: error, installed: false) + } + + static func installed() -> LaunchAgentEnableResult { + LaunchAgentEnableResult(error: nil, installed: true) + } + } + + private struct LaunchAgentStartupContext { + let port: Int + let enableResult: LaunchAgentEnableResult + let readinessPID: Int32? + let readinessRevision: UInt64 + } + + private enum GatewayProbeFailureDisposition: Equatable { + case retryWithoutRepair + case retryWithRepair + case fail + } + enum Status: Equatable { case stopped case starting @@ -42,9 +102,27 @@ final class GatewayProcessManager { private var environmentRefreshTask: Task? private var lastEnvironmentRefresh: Date? private var logRefreshTask: Task? + private var launchAgentEnableTask: Task<[UInt64: LaunchAgentEnableResult], Never>? + private var launchAgentEnableCurrentRequest: LaunchAgentEnableRequest? + private var launchAgentEnablePendingRequest: LaunchAgentEnableRequest? + private var launchAgentEnableSupersededInvocationIDs: Set = [] + private var launchAgentEnableNextInvocationID: UInt64 = 0 + private var launchAgentDisableTask: Task? + private var launchAgentDisableGeneration: UInt64? + private var launchAgentReadinessFailure: LaunchAgentReadinessFailure? + private var launchAgentReadinessCandidate: LaunchAgentReadinessCandidate? + private var launchAgentReadinessRevision: UInt64 = 0 + private var launchAgentInstallGeneration: UInt64? + private var lastObservedGatewayPID: Int32? + /// Async readiness audits may outlive stop/restart. Only the current generation may publish + /// their failure state or retain a PID for a later repair. + private var gatewayStartGeneration: UInt64 = 0 + private var gatewayStartTask: Task? + private var gatewayStartTaskGeneration: UInt64? #if DEBUG private var testingConnection: GatewayConnection? private var testingSkipControlChannelRefresh = false + private var testingControlChannelRefreshForces: [Bool] = [] #endif private let logger = Logger(subsystem: "ai.openclaw", category: "gateway.process") @@ -78,22 +156,187 @@ final class GatewayProcessManager { } } - func ensureLaunchAgentEnabledIfNeeded() async { - guard !CommandResolver.connectionModeIsRemote() else { return } + func ensureLaunchAgentEnabledIfNeeded() async -> Bool { + guard !CommandResolver.connectionModeIsRemote() else { return false } + guard self.desiredActive else { return false } if GatewayLaunchAgentManager.isLaunchAgentWriteDisabled() { self.appendLog("[gateway] launchd auto-enable skipped (attach-only)\n") self.logger.info("gateway launchd auto-enable skipped (disable marker set)") - return + return false } - let enabled = await GatewayLaunchAgentManager.isLoaded() - guard !enabled else { return } let bundlePath = Bundle.main.bundleURL.path let port = GatewayEnvironment.gatewayPort() - self.appendLog("[gateway] auto-enabling launchd job (\(gatewayLaunchdLabel)) on port \(port)\n") - let err = await GatewayLaunchAgentManager.set(enabled: true, bundlePath: bundlePath, port: port) - if let err { + let result = await self.enableLaunchAgentIfNeeded( + bundlePath: bundlePath, + port: port, + generation: self.gatewayStartGeneration) + if let err = result.error { self.appendLog("[gateway] launchd auto-enable failed: \(err)\n") } + return result.installed + } + + private func enableLaunchAgentIfNeeded( + bundlePath: String, + port: Int, + generation expectedGeneration: UInt64? = nil) async -> LaunchAgentEnableResult + { + let generation = expectedGeneration ?? self.gatewayStartGeneration + await self.waitForPendingLaunchAgentDisable() + guard generation == self.gatewayStartGeneration else { return .skipped } + self.launchAgentEnableNextInvocationID &+= 1 + let invocationID = self.launchAgentEnableNextInvocationID + let request = LaunchAgentEnableRequest( + bundlePath: bundlePath, + port: port, + generation: generation, + invocationIDs: [invocationID]) + if let task = self.launchAgentEnableTask { + if var current = self.launchAgentEnableCurrentRequest, + current.hasSameConfiguration(as: request) + { + // The in-flight request already represents the newest configuration. Drop an + // older queued change so A -> B -> A cannot finish on B. + current.invocationIDs.append(invocationID) + self.launchAgentEnableCurrentRequest = current + if let pending = self.launchAgentEnablePendingRequest { + self.launchAgentEnableSupersededInvocationIDs.formUnion(pending.invocationIDs) + } + self.launchAgentEnablePendingRequest = nil + } else if var pending = self.launchAgentEnablePendingRequest, + pending.hasSameConfiguration(as: request) + { + pending.invocationIDs.append(invocationID) + self.launchAgentEnablePendingRequest = pending + } else { + if let pending = self.launchAgentEnablePendingRequest { + self.launchAgentEnableSupersededInvocationIDs.formUnion(pending.invocationIDs) + } + self.launchAgentEnablePendingRequest = request + } + let results = await task.value + return results[invocationID] ?? .skipped + } + + self.launchAgentEnableSupersededInvocationIDs.removeAll(keepingCapacity: true) + self.launchAgentEnablePendingRequest = request + let task = Task { @MainActor in + await self.drainLaunchAgentEnableRequests() + } + self.launchAgentEnableTask = task + let results = await task.value + return results[invocationID] ?? .skipped + } + + private func waitForPendingLaunchAgentDisable() async { + // A stop may already be uninstalling launchd. Wait until it finishes so a newer start's + // attach/install is ordered last; loop because another stop can supersede it while waiting. + while let disableTask = self.launchAgentDisableTask { + await disableTask.value + } + } + + private func drainLaunchAgentEnableRequests() + async -> [UInt64: LaunchAgentEnableResult] + { + var results: [UInt64: LaunchAgentEnableResult] = [:] + while let request = self.launchAgentEnablePendingRequest { + self.launchAgentEnablePendingRequest = nil + self.launchAgentEnableSupersededInvocationIDs.subtract(request.invocationIDs) + self.launchAgentEnableCurrentRequest = request + let result = await self.performLaunchAgentEnable(request) + let completedRequest = self.launchAgentEnableCurrentRequest ?? request + for invocationID in completedRequest.invocationIDs { + results[invocationID] = result + } + self.launchAgentEnableCurrentRequest = nil + } + for invocationID in self.launchAgentEnableSupersededInvocationIDs { + guard results[invocationID] == nil else { continue } + results[invocationID] = .skipped + } + self.launchAgentEnableSupersededInvocationIDs.removeAll(keepingCapacity: true) + // Clear the task before returning. A later caller then starts a fresh drain instead of + // joining a completed task after the final pending-request check. + self.launchAgentEnableTask = nil + return results + } + + private func performLaunchAgentEnable(_ request: LaunchAgentEnableRequest) async -> LaunchAgentEnableResult { + // App startup and onboarding can request persistence together. One drain owns all installs; + // a second forced install would kill the first Gateway during startup migrations. + let launchAgent = await GatewayLaunchAgentManager.loadedGatewayState(port: request.port) + // Pair one launchd snapshot with a current listener read. A PID that starts after the + // status read cannot look reusable, so the ownership guard preserves it instead of forcing + // an install; a reusable PID from this same snapshot receives its readiness cycle below. + let listener = await PortGuardian.shared.describe(port: request.port) + if let listener { + guard listener.pid == launchAgent.runningPID else { + // A healthy manually started Gateway may be attached without becoming app-owned. + // Persistence checks and retained repair markers must not replace it. + return .skipped + } + } + + if let pid = launchAgent.reusablePID { + let failure = LaunchAgentReadinessFailure(port: request.port, pid: pid) + if self.launchAgentReadinessFailure != failure { + // A new launchd PID may still be running migrations. It must fail one complete + // readiness cycle before a later retry is allowed to replace it. + self.setLaunchAgentReadinessState( + candidate: LaunchAgentReadinessCandidate( + failure: failure, + generation: request.generation), + failure: nil) + return .skipped + } + + self.appendLog( + "[gateway] launchd pid \(pid) failed readiness on port \(request.port); repairing\n") + self.logger.warning( + "gateway launchd pid=\(pid) failed readiness on port=\(request.port); repairing") + } + self.setLaunchAgentReadinessState(candidate: nil, failure: nil) + self.appendLog( + "[gateway] enabling launchd job (\(gatewayLaunchdLabel)) on port \(request.port)\n") + if let error = await GatewayLaunchAgentManager.set( + enabled: true, + bundlePath: request.bundlePath, + port: request.port) + { + return .failed(error) + } + // Keep replacement evidence until a healthy audit refreshes the control channel. Startup + // and persistence calls coalesce, so the later caller may not receive `installed` itself. + self.launchAgentInstallGeneration = request.generation + return .installed() + } + + private func resolveLaunchAgentReadinessFailure( + port: Int, + startingPID: Int32?) async -> LaunchAgentReadinessFailure? + { + guard let startingPID, + let pid = await GatewayLaunchAgentManager.reusableLoadedGatewayPID(port: port), + pid == startingPID + else { + return nil + } + // A stable launchd PID that owns the port can still have a wedged health RPC. A listener + // owned by anyone else is protected and surfaced through the attach path instead. + if let listener = await PortGuardian.shared.describe(port: port), listener.pid != pid { + return nil + } + return LaunchAgentReadinessFailure(port: port, pid: pid) + } + + private func setLaunchAgentReadinessState( + candidate: LaunchAgentReadinessCandidate?, + failure: LaunchAgentReadinessFailure?) + { + self.launchAgentReadinessCandidate = candidate + self.launchAgentReadinessFailure = failure + self.launchAgentReadinessRevision &+= 1 } func startIfNeeded() { @@ -112,34 +355,68 @@ final class GatewayProcessManager { break } self.status = .starting + self.gatewayStartGeneration &+= 1 + let startGeneration = self.gatewayStartGeneration self.logger.debug("gateway start requested") // First try to latch onto an already-running gateway to avoid spawning a duplicate. - Task { [weak self] in + let task = Task { [weak self] in guard let self else { return } - if await self.attachExistingGatewayIfAvailable() { + defer { + if self.gatewayStartTaskGeneration == startGeneration { + self.gatewayStartTask = nil + self.gatewayStartTaskGeneration = nil + } + } + if await self.attachExistingGatewayAfterPendingDisable(startGeneration: startGeneration) { return } - await self.enableLaunchdGateway() + await self.enableLaunchdGateway(startGeneration: startGeneration) + } + self.gatewayStartTaskGeneration = startGeneration + self.gatewayStartTask = task + } + + func waitForStartupAttempt() async { + // Persistence/repair follows the complete attach-or-start decision. This prevents the + // automatic ensure path from replacing a PID while startup is accepting that same PID. + while let task = self.gatewayStartTask { + await task.value } } func stop() { + self.gatewayStartGeneration &+= 1 + let stopGeneration = self.gatewayStartGeneration self.desiredActive = false self.existingGatewayDetails = nil self.lastFailureReason = nil + self.setLaunchAgentReadinessState(candidate: nil, failure: nil) + self.launchAgentInstallGeneration = nil + // Queued work belongs to the previous lifecycle. The active enable cannot be cancelled + // safely, so the disable waits for its drain and wins unless a newer start supersedes it. + self.launchAgentEnablePendingRequest = nil + let enableTask = self.launchAgentEnableTask self.status = .stopped self.logger.info("gateway stop requested") - if CommandResolver.connectionModeIsRemote() { - return - } let bundlePath = Bundle.main.bundleURL.path - Task { - _ = await GatewayLaunchAgentManager.set( - enabled: false, - bundlePath: bundlePath, - port: GatewayEnvironment.gatewayPort()) + let priorDisableTask = self.launchAgentDisableTask + let disableTask = Task { @MainActor in + _ = await priorDisableTask?.value + _ = await enableTask?.value + if self.launchAgentDisableGeneration == stopGeneration { + _ = await GatewayLaunchAgentManager.set( + enabled: false, + bundlePath: bundlePath, + port: GatewayEnvironment.gatewayPort()) + } + if self.launchAgentDisableGeneration == stopGeneration { + self.launchAgentDisableTask = nil + self.launchAgentDisableGeneration = nil + } } + self.launchAgentDisableGeneration = stopGeneration + self.launchAgentDisableTask = disableTask } func clearLastFailure() { @@ -189,32 +466,69 @@ final class GatewayProcessManager { // MARK: - Internals + private func isCurrentGatewayStart(_ generation: UInt64?) -> Bool { + guard let generation else { return true } + return self.desiredActive && self.gatewayStartGeneration == generation + } + + private func attachExistingGatewayAfterPendingDisable( + port requestedPort: Int? = nil, + startGeneration: UInt64) async -> Bool + { + // A gateway that is still reachable during uninstall is not reusable. Let the stop finish + // before attachment so the new lifecycle cannot latch onto a process launchd then removes. + await self.waitForPendingLaunchAgentDisable() + guard self.isCurrentGatewayStart(startGeneration) else { return true } + return await self.attachExistingGatewayIfAvailable( + port: requestedPort, + startGeneration: startGeneration) + } + /// Attempt to connect to an already-running gateway on the configured port. /// If successful, mark status as attached and skip spawning a new process. - private func attachExistingGatewayIfAvailable() async -> Bool { - let port = GatewayEnvironment.gatewayPort() + private func attachExistingGatewayIfAvailable( + port requestedPort: Int? = nil, + startGeneration: UInt64? = nil) async -> Bool + { + let port = requestedPort ?? GatewayEnvironment.gatewayPort() let instance = await PortGuardian.shared.describe(port: port) + guard self.isCurrentGatewayStart(startGeneration) else { return true } let instanceText = instance.map { self.describe(instance: $0) } let hasListener = instance != nil let attemptAttach = { - try await self.connection.requestRaw(method: .health, timeoutMs: 2000) + try await self.probeGatewayHealth(timeoutMs: 2000) } for attempt in 0..<(hasListener ? 3 : 1) { + guard self.isCurrentGatewayStart(startGeneration) else { return true } do { let data = try await attemptAttach() + guard self.isCurrentGatewayStart(startGeneration) else { return true } + let attachedInstance = await PortGuardian.shared.describe(port: port) + guard self.isCurrentGatewayStart(startGeneration) else { return true } let snap = decodeHealthSnapshot(from: data) - let details = self.describe(details: instanceText, port: port, snap: snap) + let attachedInstanceText = attachedInstance.map { self.describe(instance: $0) } + let details = self.describe(details: attachedInstanceText, port: port, snap: snap) + let endpointPIDChanged = Self.gatewayPIDChanged( + from: self.lastObservedGatewayPID, + to: attachedInstance?.pid) || Self.gatewayPIDChanged( + from: instance?.pid, + to: attachedInstance?.pid) self.existingGatewayDetails = details + self.setLaunchAgentReadinessState(candidate: nil, failure: nil) self.clearLastFailure() self.status = .attachedExisting(details: details) self.appendLog("[gateway] using existing instance: \(details)\n") self.logger.info("gateway using existing instance details=\(details)") - self.refreshControlChannelIfNeeded(reason: "attach existing") + self.refreshControlChannelIfNeeded( + reason: "attach existing", + force: endpointPIDChanged) + self.lastObservedGatewayPID = attachedInstance?.pid ?? self.lastObservedGatewayPID self.refreshLog() return true } catch { + guard self.isCurrentGatewayStart(startGeneration) else { return true } if attempt < 2, hasListener { try? await Task.sleep(nanoseconds: 250_000_000) continue @@ -297,19 +611,23 @@ final class GatewayProcessManager { let lower = ns.localizedDescription.lowercased() return lower.contains("unauthorized") || lower.contains("auth") } +} - private func enableLaunchdGateway() async { +extension GatewayProcessManager { + private func prepareLaunchdGatewayStart(startGeneration: UInt64) async -> LaunchAgentStartupContext? { + guard self.isCurrentGatewayStart(startGeneration) else { return nil } self.existingGatewayDetails = nil let resolution = await Task.detached(priority: .utility) { GatewayEnvironment.resolveGatewayCommand() }.value + guard self.isCurrentGatewayStart(startGeneration) else { return nil } await MainActor.run { self.environmentStatus = resolution.status } guard resolution.command != nil else { await MainActor.run { self.status = .failed(resolution.status.message) } self.logger.error("gateway command resolve failed: \(resolution.status.message)") - return + return nil } if GatewayLaunchAgentManager.isLaunchAgentWriteDisabled() { @@ -318,45 +636,233 @@ final class GatewayProcessManager { self.lastFailureReason = "launchd disabled" self.appendLog("[gateway] launchd disabled; skipping auto-start\n") self.logger.info("gateway launchd enable skipped (disable marker set)") - return + return nil } let bundlePath = Bundle.main.bundleURL.path let port = GatewayEnvironment.gatewayPort() - self.appendLog("[gateway] enabling launchd job (\(gatewayLaunchdLabel)) on port \(port)\n") - self.logger.info("gateway enabling launchd port=\(port)") - let err = await GatewayLaunchAgentManager.set(enabled: true, bundlePath: bundlePath, port: port) - if let err { + self.logger.info("gateway ensuring launchd port=\(port)") + let enableResult = await self.enableLaunchAgentIfNeeded( + bundlePath: bundlePath, + port: port, + generation: startGeneration) + guard self.isCurrentGatewayStart(startGeneration) else { return nil } + if let err = enableResult.error { self.status = .failed(err) self.lastFailureReason = err self.logger.error("gateway launchd enable failed: \(err)") - return + return nil } + let readinessPID = await GatewayLaunchAgentManager.reusableLoadedGatewayPID(port: port) + guard self.isCurrentGatewayStart(startGeneration) else { return nil } + return LaunchAgentStartupContext( + port: port, + enableResult: enableResult, + readinessPID: readinessPID, + readinessRevision: self.launchAgentReadinessRevision) + } + + private func enableLaunchdGateway(startGeneration: UInt64) async { + guard let context = await self.prepareLaunchdGatewayStart(startGeneration: startGeneration) else { + return + } // Best-effort: wait for the gateway to accept connections. let deadline = Date().addingTimeInterval(6) + var latestRetryDisposition: GatewayProbeFailureDisposition? while Date() < deadline { - if !self.desiredActive { return } + guard !Task.isCancelled else { return } + guard self.isCurrentGatewayStart(startGeneration) else { return } do { - _ = try await self.connection.requestRaw(method: .health, timeoutMs: 1500) - let instance = await PortGuardian.shared.describe(port: port) - let details = instance.map { "pid \($0.pid)" } - self.clearLastFailure() - self.status = .running(details: details) - self.logger.info("gateway started details=\(details ?? "ok")") - self.refreshControlChannelIfNeeded(reason: "gateway started") - self.refreshLog() + let remainingMs = max(1, deadline.timeIntervalSinceNow * 1000) + _ = try await self.probeGatewayHealth(timeoutMs: min(1500, remainingMs)) + guard !Task.isCancelled else { return } + let instance = await PortGuardian.shared.describe(port: context.port) + guard self.publishLaunchdGatewayReady( + instance: instance, + context: context, + startGeneration: startGeneration) + else { return } return } catch { - try? await Task.sleep(nanoseconds: 400_000_000) + if Task.isCancelled || !self.isCurrentGatewayStart(startGeneration) { + return + } + switch self.probeFailureDisposition(error) { + case .fail: + await self.finishResponsiveGatewayProbeFailure( + error, + port: context.port, + startGeneration: startGeneration, + expectedCandidate: self.launchAgentReadinessCandidate, + expectedReadinessRevision: context.readinessRevision) + return + case .retryWithRepair: + latestRetryDisposition = .retryWithRepair + case .retryWithoutRepair: + // A responsive transient invalidates older connection-failure evidence. + latestRetryDisposition = .retryWithoutRepair + } + let retryDelay = min(0.4, max(0, deadline.timeIntervalSinceNow)) + if retryDelay > 0 { + try? await Task.sleep(nanoseconds: UInt64(retryDelay * 1_000_000_000)) + } } } + // Only a PID that survived this entire readiness cycle may be replaced later. launchd can + // restart the service while polling; that replacement needs its own full startup chance. + guard !Task.isCancelled else { return } + if latestRetryDisposition == .retryWithRepair { + await self.finishLaunchAgentReadinessFailure( + port: context.port, + startingPID: context.readinessPID, + startGeneration: startGeneration, + expectedReadinessRevision: context.readinessRevision) + } else { + self.finishGatewayReadinessDeadlineWithoutRepair( + startGeneration: startGeneration, + expectedReadinessRevision: context.readinessRevision) + } + } + + private func publishLaunchdGatewayReady( + instance: PortGuardian.Descriptor?, + context: LaunchAgentStartupContext, + startGeneration: UInt64) -> Bool + { + guard !Task.isCancelled else { return false } + guard self.isCurrentGatewayStart(startGeneration) else { return false } + guard self.launchAgentReadinessRevision == context.readinessRevision else { return false } + let details = instance.map { "pid \($0.pid)" } + let endpointPIDChanged = if let readinessPID = context.readinessPID, + let observedPID = instance?.pid + { + readinessPID != observedPID + } else { + false + } + let previouslyObservedPIDChanged = Self.gatewayPIDChanged( + from: self.lastObservedGatewayPID, + to: instance?.pid) + let launchAgentReplaced = context.enableResult.installed || + self.launchAgentInstallGeneration == startGeneration || + endpointPIDChanged || + previouslyObservedPIDChanged + self.setLaunchAgentReadinessState(candidate: nil, failure: nil) + self.clearLastFailure() + self.status = .running(details: details) + self.logger.info("gateway started details=\(details ?? "ok")") + self.refreshControlChannelIfNeeded( + reason: "gateway started", + force: launchAgentReplaced) + self.lastObservedGatewayPID = instance?.pid ?? self.lastObservedGatewayPID + if self.launchAgentInstallGeneration == startGeneration { + self.launchAgentInstallGeneration = nil + } + self.refreshLog() + return true + } + + private func finishLaunchAgentReadinessFailure( + port: Int, + startingPID: Int32?, + startGeneration: UInt64, + expectedCandidate: LaunchAgentReadinessCandidate? = nil, + candidateMustMatch: Bool = false, + expectedReadinessRevision: UInt64? = nil) async + { + let failure = await self.resolveLaunchAgentReadinessFailure( + port: port, + startingPID: startingPID) + guard !Task.isCancelled else { return } + guard self.isCurrentGatewayStart(startGeneration) else { return } + if let expectedReadinessRevision, + self.launchAgentReadinessRevision != expectedReadinessRevision + { + return + } + if candidateMustMatch, self.launchAgentReadinessCandidate != expectedCandidate { + return + } + self.setLaunchAgentReadinessState(candidate: nil, failure: failure) self.status = .failed("Gateway did not start in time") self.lastFailureReason = "launchd start timeout" self.logger.warning("gateway start timed out") } + private func finishResponsiveGatewayProbeFailure( + _ error: Error, + port: Int, + startGeneration: UInt64, + expectedCandidate: LaunchAgentReadinessCandidate?, + expectedReadinessRevision: UInt64) async + { + let instance = await PortGuardian.shared.describe(port: port) + guard !Task.isCancelled else { return } + guard self.isCurrentGatewayStart(startGeneration) else { return } + guard self.launchAgentReadinessRevision == expectedReadinessRevision else { return } + guard self.launchAgentReadinessCandidate == expectedCandidate else { return } + let reason = self.describeAttachFailure(error, port: port, instance: instance) + self.setLaunchAgentReadinessState(candidate: nil, failure: nil) + self.status = .failed(reason) + self.lastFailureReason = reason + self.appendLog("[gateway] responsive health probe failed: \(reason)\n") + self.logger.warning("gateway responsive health probe failed reason=\(reason)") + } + + private func finishGatewayReadinessDeadlineWithoutRepair( + startGeneration: UInt64, + expectedReadinessRevision: UInt64) + { + guard self.isCurrentGatewayStart(startGeneration) else { return } + guard self.launchAgentReadinessRevision == expectedReadinessRevision else { return } + // Transient RPC/cancellation responses do not prove the endpoint is unreachable. End the + // startup cleanly, but do not retain a PID that would authorize destructive repair. + self.setLaunchAgentReadinessState( + candidate: nil, + failure: self.launchAgentReadinessFailure) + self.status = .failed("Gateway did not become ready in time") + self.lastFailureReason = "gateway readiness deadline elapsed" + self.logger.warning("gateway readiness deadline elapsed without endpoint failure") + } + + private func probeFailureDisposition(_ error: Error) -> GatewayProbeFailureDisposition { + if self.probeFailureIsCancellation(error) { return .retryWithoutRepair } + if let response = error as? GatewayResponseError, + response.code.uppercased() == "UNAVAILABLE" + { + return .retryWithoutRepair + } + if error is GatewayHealthProbeTimeout { return .retryWithRepair } + let nsError = error as NSError + guard nsError.domain == NSURLErrorDomain else { return .fail } + switch URLError.Code(rawValue: nsError.code) { + case .timedOut, + .cannotFindHost, + .cannotConnectToHost, + .networkConnectionLost, + .dnsLookupFailed, + .notConnectedToInternet, + .resourceUnavailable: + return .retryWithRepair + default: + return .fail + } + } + + private func probeFailureIsCancellation(_ error: Error) -> Bool { + if error is CancellationError { return true } + let nsError = error as NSError + return nsError.domain == NSURLErrorDomain && + nsError.code == URLError.cancelled.rawValue + } + + private static func gatewayPIDChanged(from previousPID: Int32?, to observedPID: Int32?) -> Bool { + guard let previousPID, let observedPID else { return false } + return previousPID != observedPID + } + private func appendLog(_ chunk: String) { self.log.append(chunk) if self.log.count > self.logLimit { @@ -364,40 +870,187 @@ final class GatewayProcessManager { } } - private func refreshControlChannelIfNeeded(reason: String) { + private func refreshControlChannelIfNeeded(reason: String, force: Bool = false) { #if DEBUG + self.testingControlChannelRefreshForces.append(force) if self.testingSkipControlChannelRefresh { return } #endif - switch ControlChannel.shared.state { - case .connected, .connecting: - return - case .disconnected, .degraded: - break + if !force { + switch ControlChannel.shared.state { + case .connected, .connecting: + return + case .disconnected, .degraded: + break + } } self.appendLog("[gateway] refreshing control channel (\(reason))\n") self.logger.debug("gateway control channel refresh reason=\(reason)") Task { await ControlChannel.shared.configure() } } - func waitForGatewayReady(timeout: TimeInterval = 6) async -> Bool { + func waitForGatewayReady( + timeout: TimeInterval = 6, + launchAgentInstalled: Bool = false) async -> Bool + { + let startGeneration = self.gatewayStartGeneration + let readinessCandidate = self.launchAgentReadinessCandidate + let readinessFailure = self.launchAgentReadinessFailure + let readinessRevision = self.launchAgentReadinessRevision + let readinessPort = readinessCandidate?.failure.port + ?? GatewayEnvironment.gatewayPort() let deadline = Date().addingTimeInterval(timeout) + let endpointPIDBeforeProbe = self.lastObservedGatewayPID + var latestRetryDisposition: GatewayProbeFailureDisposition? while Date() < deadline { - if !self.desiredActive { return false } + guard !Task.isCancelled else { return false } + guard self.isCurrentGatewayStart(startGeneration) else { return false } do { - _ = try await self.connection.requestRaw(method: .health, timeoutMs: 1500) - self.clearLastFailure() - return true + let remainingMs = max(1, deadline.timeIntervalSinceNow * 1000) + _ = try await self.probeGatewayHealth(timeoutMs: min(1500, remainingMs)) + guard !Task.isCancelled else { return false } + let instance = await PortGuardian.shared.describe(port: readinessPort) + return self.publishGatewayReadinessSuccess( + instance: instance, + startGeneration: startGeneration, + readinessCandidate: readinessCandidate, + readinessRevision: readinessRevision, + launchAgentInstalled: launchAgentInstalled, + endpointPIDBeforeProbe: endpointPIDBeforeProbe) } catch { - try? await Task.sleep(nanoseconds: 300_000_000) + if Task.isCancelled || !self.isCurrentGatewayStart(startGeneration) { + return false + } + switch self.probeFailureDisposition(error) { + case .fail: + await self.finishResponsiveGatewayProbeFailure( + error, + port: readinessCandidate?.failure.port ?? GatewayEnvironment.gatewayPort(), + startGeneration: startGeneration, + expectedCandidate: readinessCandidate, + expectedReadinessRevision: readinessRevision) + return false + case .retryWithRepair: + latestRetryDisposition = .retryWithRepair + case .retryWithoutRepair: + // A responsive transient invalidates older connection-failure evidence. + latestRetryDisposition = .retryWithoutRepair + } + let retryDelay = min(0.3, max(0, deadline.timeIntervalSinceNow)) + if retryDelay > 0 { + try? await Task.sleep(nanoseconds: UInt64(retryDelay * 1_000_000_000)) + } } } - self.appendLog("[gateway] readiness wait timed out\n") - self.logger.warning("gateway readiness wait timed out") + await self.finishGatewayReadinessTimeout( + startGeneration: startGeneration, + readinessCandidate: readinessCandidate, + readinessFailure: readinessFailure, + readinessRevision: readinessRevision, + latestRetryDisposition: latestRetryDisposition) return false } + private func publishGatewayReadinessSuccess( + instance: PortGuardian.Descriptor?, + startGeneration: UInt64, + readinessCandidate: LaunchAgentReadinessCandidate?, + readinessRevision: UInt64, + launchAgentInstalled: Bool, + endpointPIDBeforeProbe: Int32?) -> Bool + { + guard !Task.isCancelled else { return false } + guard self.desiredActive, self.gatewayStartGeneration == startGeneration else { return false } + guard self.launchAgentReadinessRevision == readinessRevision else { return false } + guard self.launchAgentReadinessCandidate == readinessCandidate else { return false } + let details = instance.map { "pid \($0.pid)" } + let launchAgentReplaced = launchAgentInstalled || + self.launchAgentInstallGeneration == startGeneration + self.setLaunchAgentReadinessState(candidate: nil, failure: nil) + self.clearLastFailure() + if case .attachedExisting = self.status { + self.status = launchAgentReplaced + ? .running(details: details) + : .attachedExisting(details: details) + } else { + self.status = .running(details: details) + } + let endpointPIDChanged = Self.gatewayPIDChanged( + from: endpointPIDBeforeProbe, + to: instance?.pid) || Self.gatewayPIDChanged( + from: readinessCandidate?.failure.pid, + to: instance?.pid) + // A replaced process can leave the old socket briefly marked connected. Routine audits + // retain the connected channel; only replacement evidence forces refresh. + self.refreshControlChannelIfNeeded( + reason: "gateway readiness recovered", + force: launchAgentReplaced || endpointPIDChanged) + self.lastObservedGatewayPID = instance?.pid ?? self.lastObservedGatewayPID + if self.launchAgentInstallGeneration == startGeneration { + self.launchAgentInstallGeneration = nil + } + self.refreshLog() + return true + } + + private func finishGatewayReadinessTimeout( + startGeneration: UInt64, + readinessCandidate: LaunchAgentReadinessCandidate?, + readinessFailure: LaunchAgentReadinessFailure?, + readinessRevision: UInt64, + latestRetryDisposition: GatewayProbeFailureDisposition?) async + { + guard !Task.isCancelled else { return } + guard self.isCurrentGatewayStart(startGeneration) else { return } + guard self.launchAgentReadinessRevision == readinessRevision else { return } + guard self.launchAgentReadinessCandidate == readinessCandidate else { return } + self.appendLog("[gateway] readiness wait timed out\n") + guard latestRetryDisposition == .retryWithRepair else { + self.logger.warning("gateway readiness wait ended without endpoint failure evidence") + return + } + if let readinessCandidate, + readinessCandidate.generation == startGeneration + { + await self.finishLaunchAgentReadinessFailure( + port: readinessCandidate.failure.port, + startingPID: readinessCandidate.failure.pid, + startGeneration: startGeneration, + expectedCandidate: readinessCandidate, + candidateMustMatch: true, + expectedReadinessRevision: readinessRevision) + return + } + + self.setLaunchAgentReadinessState(candidate: nil, failure: readinessFailure) + if case .failed = self.status { + // Startup or persistence already published a concrete launchd/configuration error. + // A follow-up reachability timeout must not replace that actionable diagnosis. + self.logger.warning("gateway readiness wait timed out; preserving existing failure") + } else { + self.status = .failed("Gateway did not start in time") + self.lastFailureReason = "gateway readiness timeout" + self.logger.warning("gateway readiness wait timed out") + } + } + + private func probeGatewayHealth(timeoutMs: Double) async throws -> Data { + let connection = self.connection + // Startup owns recovery and its wall-clock deadline. A normal request can recursively + // start the Gateway and spend several 30-second connect retries before its RPC timer begins. + return try await AsyncTimeout.withTimeout( + seconds: max(0.001, timeoutMs / 1000), + onTimeout: { GatewayHealthProbeTimeout(timeoutMs: timeoutMs) }, + operation: { + try await connection.request( + method: GatewayConnection.Method.health.rawValue, + params: nil, + timeoutMs: timeoutMs, + retryTransportFailures: false) + }) + } + func clearLog() { self.log = "" try? FileManager().removeItem(atPath: GatewayLaunchAgentManager.launchdGatewayLogPath()) @@ -431,6 +1084,41 @@ extension GatewayProcessManager { self.testingSkipControlChannelRefresh = skip } + func _testControlChannelRefreshForces() -> [Bool] { + self.testingControlChannelRefreshForces + } + + func _testClearControlChannelRefreshForces() { + self.testingControlChannelRefreshForces.removeAll(keepingCapacity: true) + } + + func _testClearLaunchAgentInstallEvidence() { + self.launchAgentInstallGeneration = nil + } + + func _testSetLastObservedGatewayPID(_ pid: Int32?) { + self.lastObservedGatewayPID = pid + } + + func _testProbeFailureMayNeedLaunchAgentRepair(_ code: URLError.Code) -> Bool { + if case .retryWithRepair = self.probeFailureDisposition(URLError(code)) { + return true + } + return false + } + + func _testGatewayResponseRetriesWithoutRepair(_ code: String) -> Bool { + let error = GatewayResponseError( + method: "health", + code: code, + message: "test", + details: nil) + if case .retryWithoutRepair = self.probeFailureDisposition(error) { + return true + } + return false + } + func setTestingDesiredActive(_ active: Bool) { self.desiredActive = active } @@ -439,8 +1127,82 @@ extension GatewayProcessManager { self.lastFailureReason = reason } - func _testAttachExistingGatewayIfAvailable() async -> Bool { - await self.attachExistingGatewayIfAvailable() + func setTestingStatus(_ status: Status) { + self.status = status + } + + func _testAttachExistingGatewayIfAvailable(port: Int) async -> Bool { + await self.attachExistingGatewayIfAvailable(port: port) + } + + func _testAttachExistingGatewayAfterPendingDisable(port: Int) async -> Bool { + await self.attachExistingGatewayAfterPendingDisable( + port: port, + startGeneration: self.gatewayStartGeneration) + } + + func _testEnableLaunchAgentIfNeeded(bundlePath: String, port: Int) async -> String? { + await self.enableLaunchAgentIfNeeded(bundlePath: bundlePath, port: port).error + } + + func _testEnableLaunchAgentIfNeededInstalled(bundlePath: String, port: Int) async -> Bool { + await self.enableLaunchAgentIfNeeded(bundlePath: bundlePath, port: port).installed + } + + func _testRecordLaunchAgentReadinessFailure(port: Int, startingPID: Int32?) async { + let failure = await self.resolveLaunchAgentReadinessFailure( + port: port, + startingPID: startingPID) + self.setLaunchAgentReadinessState( + candidate: self.launchAgentReadinessCandidate, + failure: failure) + } + + func _testFinishLaunchAgentReadinessFailure(port: Int, startingPID: Int32?) async { + let startGeneration = self.gatewayStartGeneration + await self.finishLaunchAgentReadinessFailure( + port: port, + startingPID: startingPID, + startGeneration: startGeneration) + } + + func _testClearLaunchAgentReadinessFailure() { + self.setLaunchAgentReadinessState(candidate: nil, failure: nil) + } + + func _testSetLaunchAgentReadinessFailure(port: Int, pid: Int32) { + self.setLaunchAgentReadinessState( + candidate: self.launchAgentReadinessCandidate, + failure: LaunchAgentReadinessFailure(port: port, pid: pid)) + } + + func _testSetLaunchAgentReadinessCandidate(port: Int, pid: Int32) { + self.setLaunchAgentReadinessState( + candidate: LaunchAgentReadinessCandidate( + failure: LaunchAgentReadinessFailure(port: port, pid: pid), + generation: self.gatewayStartGeneration), + failure: self.launchAgentReadinessFailure) + } + + func _testHasLaunchAgentReadinessFailure() -> Bool { + self.launchAgentReadinessFailure != nil + } + + func _testHasLaunchAgentReadinessCandidate() -> Bool { + self.launchAgentReadinessCandidate != nil + } + + func _testLaunchAgentReadinessCandidatePID() -> Int32? { + self.launchAgentReadinessCandidate?.failure.pid + } + + func _testBeginGatewayStartGeneration() { + self.desiredActive = true + self.gatewayStartGeneration &+= 1 + } + + func _testPendingLaunchAgentPort() -> Int? { + self.launchAgentEnablePendingRequest?.port } } #endif diff --git a/apps/macos/Tests/OpenClawIPCTests/GatewayProcessManagerTests.swift b/apps/macos/Tests/OpenClawIPCTests/GatewayProcessManagerTests.swift index c55d8ed46c8f..e19b07abbed0 100644 --- a/apps/macos/Tests/OpenClawIPCTests/GatewayProcessManagerTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/GatewayProcessManagerTests.swift @@ -6,6 +6,950 @@ import Testing @Suite(.serialized) @MainActor struct GatewayProcessManagerTests { + private func withGatewayConfig( + mode: String, + _ body: () async throws -> T) async throws -> T + { + let configPath = TestIsolation.tempConfigPath() + try Data(#"{"gateway":{"mode":"\#(mode)"}}"#.utf8) + .write(to: URL(fileURLWithPath: configPath)) + defer { try? FileManager.default.removeItem(atPath: configPath) } + return try await TestIsolation.withEnvValues(["OPENCLAW_CONFIG_PATH": configPath], body) + } + + private func withLocalGatewayConfig( + _ body: () async throws -> T) async throws -> T + { + try await self.withGatewayConfig(mode: "local", body) + } + + @Test func `coalesces concurrent launch agent enable requests`() async throws { + let port = 19081 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload( + #"{"ok":true,"service":{"loaded":false}}"#) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload(nil) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + } + + let manager = GatewayProcessManager.shared + async let first: String? = manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: port) + async let second: String? = manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: port) + _ = await (first, second) + + let calls = GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + #expect(calls.filter { $0.first == "status" }.count == 1) + #expect(calls.filter { $0.first == "install" }.count == 1) + } + } + + @Test func `queues a changed launch agent request behind an in-flight request`() async throws { + let firstPort = 19091 + let secondPort = 19092 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload( + #"{"ok":true,"service":{"loaded":false}}"#) + GatewayLaunchAgentManager.setTestingDaemonCommandDelayNanoseconds(100_000_000) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload(nil) + GatewayLaunchAgentManager.setTestingDaemonCommandDelayNanoseconds(0) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + } + + let manager = GatewayProcessManager.shared + let first = Task { @MainActor in + await manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: firstPort) + } + for _ in 0..<100 { + if !GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot().isEmpty { + break + } + try? await Task.sleep(nanoseconds: 1_000_000) + } + #expect(!GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot().isEmpty) + + let second = Task { @MainActor in + await manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: secondPort) + } + #expect(await first.value == nil) + #expect(await second.value == nil) + + let calls = GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + let installPorts = calls.compactMap { arguments -> String? in + guard arguments.first == "install", + let portIndex = arguments.firstIndex(of: "--port"), + arguments.indices.contains(portIndex + 1) + else { + return nil + } + return arguments[portIndex + 1] + } + #expect(installPorts == [String(firstPort), String(secondPort)]) + + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + let newestPort = 19093 + let stalePort = 19094 + let current = Task { @MainActor in + await manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: newestPort) + } + for _ in 0..<100 { + if !GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot().isEmpty { + break + } + try? await Task.sleep(nanoseconds: 1_000_000) + } + let stale = Task { @MainActor in + await manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: stalePort) + } + for _ in 0..<100 { + if manager._testPendingLaunchAgentPort() == stalePort { + break + } + try? await Task.sleep(nanoseconds: 1_000_000) + } + #expect(manager._testPendingLaunchAgentPort() == stalePort) + let newest = Task { @MainActor in + await manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: newestPort) + } + #expect(await current.value == nil) + #expect(await stale.value == nil) + #expect(await newest.value == nil) + + let finalInstallPorts = GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + .compactMap { arguments -> String? in + guard arguments.first == "install", + let portIndex = arguments.firstIndex(of: "--port"), + arguments.indices.contains(portIndex + 1) + else { + return nil + } + return arguments[portIndex + 1] + } + #expect(finalInstallPorts == [String(newestPort)]) + } + } + + @Test func `coalesced drain returns each request installation result`() async throws { + let firstPort = 19107 + let secondPort = 19108 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonStatusPayloads([ + #"{"ok":true,"service":{"loaded":false}}"#, + """ + {"ok":true,"service":{ + "loaded":true, + "runtime":{"status":"running","pid":4242}, + "command":{"programArguments":["openclaw","gateway","--port","\(secondPort)"]}, + "configAudit":{"ok":true,"issues":[]} + }} + """, + ]) + GatewayLaunchAgentManager.setTestingDaemonCommandDelayNanoseconds(100_000_000) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayloads([]) + GatewayLaunchAgentManager.setTestingDaemonCommandDelayNanoseconds(0) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + GatewayProcessManager.shared._testClearLaunchAgentReadinessFailure() + } + + let manager = GatewayProcessManager.shared + let first = Task { @MainActor in + await manager._testEnableLaunchAgentIfNeededInstalled( + bundlePath: "/Applications/OpenClaw.app", + port: firstPort) + } + for _ in 0..<1000 { + if GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + .contains(where: { $0.first == "install" }) + { + break + } + try? await Task.sleep(nanoseconds: 1_000_000) + } + #expect(GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + .contains(where: { $0.first == "install" })) + + let second = Task { @MainActor in + await manager._testEnableLaunchAgentIfNeededInstalled( + bundlePath: "/Applications/OpenClaw.app", + port: secondPort) + } + + #expect(await first.value) + #expect(await second.value == false) + let calls = GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + #expect(calls.filter { $0.first == "install" }.count == 1) + } + } + + @Test func `stop discards queued enables and disables after the active request`() async throws { + let firstPort = 19095 + let secondPort = 19096 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload( + #"{"ok":true,"service":{"loaded":false}}"#) + GatewayLaunchAgentManager.setTestingDaemonCommandDelayNanoseconds(100_000_000) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload(nil) + GatewayLaunchAgentManager.setTestingDaemonCommandDelayNanoseconds(0) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + GatewayProcessManager.shared.setTestingDesiredActive(false) + } + + let manager = GatewayProcessManager.shared + manager.setTestingDesiredActive(true) + let first = Task { @MainActor in + await manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: firstPort) + } + for _ in 0..<100 { + if !GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot().isEmpty { + break + } + try? await Task.sleep(nanoseconds: 1_000_000) + } + let second = Task { @MainActor in + await manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: secondPort) + } + for _ in 0..<100 { + if manager._testPendingLaunchAgentPort() == secondPort { + break + } + try? await Task.sleep(nanoseconds: 1_000_000) + } + #expect(manager._testPendingLaunchAgentPort() == secondPort) + + manager.stop() + _ = await (first.value, second.value) + try? await Task.sleep(nanoseconds: 150_000_000) + + let calls = GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + let installPorts = calls.compactMap { arguments -> String? in + guard arguments.first == "install", + let portIndex = arguments.firstIndex(of: "--port"), + arguments.indices.contains(portIndex + 1) + else { + return nil + } + return arguments[portIndex + 1] + } + #expect(installPorts == [String(firstPort)]) + #expect(calls.filter { $0.first == "uninstall" }.count == 1) + #expect(manager._testPendingLaunchAgentPort() == nil) + #expect(manager.status == .stopped) + } + } + + @Test func `restart waits for an in-progress disable`() async throws { + let port = 19098 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload( + #"{"ok":true,"service":{"loaded":false}}"#) + GatewayLaunchAgentManager.setTestingDaemonCommandDelayNanoseconds(100_000_000) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload(nil) + GatewayLaunchAgentManager.setTestingDaemonCommandDelayNanoseconds(0) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + GatewayProcessManager.shared.setTestingDesiredActive(false) + } + + let manager = GatewayProcessManager.shared + manager.setTestingDesiredActive(true) + manager.stop() + for _ in 0..<100 { + if GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + .contains(where: { $0.first == "uninstall" }) + { + break + } + try? await Task.sleep(nanoseconds: 1_000_000) + } + #expect(GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + .contains(where: { $0.first == "uninstall" })) + + manager._testBeginGatewayStartGeneration() + _ = await manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: port) + + let calls = GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + #expect(calls.map(\.first) == ["uninstall", "status", "install"]) + } + } + + @Test func `restart waits for disable before attaching`() async throws { + let port = 19099 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + let session = GatewayTestWebSocketSession( + taskFactory: { + GatewayTestWebSocketTask( + sendHook: { task, message, sendIndex in + guard sendIndex > 0 else { return } + guard let id = GatewayWebSocketTestSupport.requestID(from: message) else { return } + task.emitReceiveSuccess(.data(GatewayWebSocketTestSupport.okResponseData(id: id))) + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let connection = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + let descriptor = PortGuardian.Descriptor( + pid: 4242, + command: "openclaw-gateway", + executablePath: "/tmp/openclaw-gateway") + + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonCommandDelayNanoseconds(100_000_000) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + let manager = GatewayProcessManager.shared + manager.setTestingConnection(connection) + manager.setTestingSkipControlChannelRefresh(true) + manager.setTestingDesiredActive(true) + await PortGuardian.shared.setTestingDescriptor(descriptor, forPort: port) + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonCommandDelayNanoseconds(0) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + manager.setTestingConnection(nil) + manager.setTestingSkipControlChannelRefresh(false) + manager.setTestingDesiredActive(false) + } + + manager.stop() + for _ in 0..<100 { + if GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + .contains(where: { $0.first == "uninstall" }) + { + break + } + try? await Task.sleep(nanoseconds: 1_000_000) + } + manager._testBeginGatewayStartGeneration() + + let startedAt = Date() + let attached = await manager._testAttachExistingGatewayAfterPendingDisable(port: port) + let elapsed = Date().timeIntervalSince(startedAt) + + #expect(attached) + #expect(elapsed >= 0.05) + #expect(GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + .filter { $0.first == "uninstall" }.count == 1) + guard case .attachedExisting = manager.status else { + Issue.record("expected attachedExisting status") + await PortGuardian.shared.setTestingDescriptor(nil, forPort: port) + await connection.shutdown() + return + } + await PortGuardian.shared.setTestingDescriptor(nil, forPort: port) + await connection.shutdown() + } + } + + @Test func `remote mode still removes the local launch agent`() async throws { + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withGatewayConfig(mode: "remote") { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + GatewayProcessManager.shared.setTestingDesiredActive(false) + } + + let manager = GatewayProcessManager.shared + manager.setTestingDesiredActive(true) + manager.stop() + for _ in 0..<100 { + if GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + .contains(where: { $0.first == "uninstall" }) + { + break + } + try? await Task.sleep(nanoseconds: 1_000_000) + } + + let calls = GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + #expect(calls.filter { $0.first == "uninstall" }.count == 1) + #expect(manager.status == .stopped) + } + } + + @Test func `inactive lifecycle skips persistence ensure`() async throws { + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + } + + let manager = GatewayProcessManager.shared + manager.setTestingDesiredActive(false) + _ = await manager.ensureLaunchAgentEnabledIfNeeded() + + #expect(GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot().isEmpty) + } + } + + @Test func `newer inactive lifecycle retains the pending disable`() async throws { + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonCommandDelayNanoseconds(100_000_000) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonCommandDelayNanoseconds(0) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + GatewayProcessManager.shared.setTestingDesiredActive(false) + } + + let manager = GatewayProcessManager.shared + manager.setTestingDesiredActive(true) + manager.stop() + manager.stop() + for _ in 0..<200 { + if GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + .contains(where: { $0.first == "uninstall" }) + { + break + } + try? await Task.sleep(nanoseconds: 1_000_000) + } + try? await Task.sleep(nanoseconds: 150_000_000) + + let calls = GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + #expect(calls.filter { $0.first == "uninstall" }.count == 1) + #expect(manager.status == .stopped) + } + } + + @Test func `keeps a reusable launch agent running`() async throws { + let port = 19082 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload( + """ + {"ok":true,"service":{ + "loaded":true, + "runtime":{"status":"running","pid":4242}, + "command":{"programArguments":["openclaw","gateway","--port","\(port)"]}, + "configAudit":{"ok":true,"issues":[]} + }} + """) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload(nil) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + GatewayProcessManager.shared._testClearLaunchAgentReadinessFailure() + } + + _ = await GatewayProcessManager.shared._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: port) + + let calls = GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + #expect(calls.filter { $0.first == "status" }.count == 1) + #expect(calls.allSatisfy { $0.first != "install" }) + } + } + + @Test func `repairs only a stable launch agent PID after readiness fails`() async throws { + let port = 19085 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload( + """ + {"ok":true,"service":{ + "loaded":true, + "runtime":{"status":"running","pid":4242}, + "command":{"programArguments":["openclaw","gateway","--port","\(port)"]}, + "configAudit":{"ok":true,"issues":[]} + }} + """) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload(nil) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + GatewayProcessManager.shared._testClearLaunchAgentReadinessFailure() + } + + let manager = GatewayProcessManager.shared + _ = await manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: port) + var calls = GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + #expect(calls.filter { $0.first == "install" }.isEmpty) + + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + await manager._testRecordLaunchAgentReadinessFailure(port: port, startingPID: 4242) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + + _ = await manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: port) + + calls = GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + #expect(calls.filter { $0.first == "status" }.count == 1) + #expect(calls.filter { $0.first == "install" }.count == 1) + } + } + + @Test func `gives a replacement launch agent PID a full readiness cycle`() async throws { + let port = 19086 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload( + """ + {"ok":true,"service":{ + "loaded":true, + "runtime":{"status":"running","pid":4243}, + "command":{"programArguments":["openclaw","gateway","--port","\(port)"]}, + "configAudit":{"ok":true,"issues":[]} + }} + """) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload(nil) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + GatewayProcessManager.shared._testClearLaunchAgentReadinessFailure() + } + + let manager = GatewayProcessManager.shared + await manager._testRecordLaunchAgentReadinessFailure(port: port, startingPID: 4242) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + + _ = await manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: port) + + let calls = GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + #expect(calls.filter { $0.first == "status" }.count == 1) + #expect(calls.filter { $0.first == "install" }.isEmpty) + } + } + + @Test func `stop wins while a readiness failure audit is pending`() async throws { + let port = 19089 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload( + """ + {"ok":true,"service":{ + "loaded":true, + "runtime":{"status":"running","pid":4242}, + "command":{"programArguments":["openclaw","gateway","--port","\(port)"]}, + "configAudit":{"ok":true,"issues":[]} + }} + """) + GatewayLaunchAgentManager.setTestingDaemonCommandDelayNanoseconds(100_000_000) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload(nil) + GatewayLaunchAgentManager.setTestingDaemonCommandDelayNanoseconds(0) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + GatewayProcessManager.shared.setTestingDesiredActive(false) + GatewayProcessManager.shared._testClearLaunchAgentReadinessFailure() + } + + let manager = GatewayProcessManager.shared + manager.setTestingDesiredActive(true) + let finish = Task { @MainActor in + await manager._testFinishLaunchAgentReadinessFailure( + port: port, + startingPID: 4242) + } + for _ in 0..<100 { + if GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + .contains(where: { $0.first == "status" }) + { + break + } + try? await Task.sleep(nanoseconds: 1_000_000) + } + #expect(GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + .contains(where: { $0.first == "status" })) + + manager.stop() + await finish.value + try? await Task.sleep(nanoseconds: 150_000_000) + + #expect(manager.status == .stopped) + #expect(!manager._testHasLaunchAgentReadinessFailure()) + } + } + + @Test func `stale readiness audit cannot clear a restarted generation`() async throws { + let port = 19090 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload( + """ + {"ok":true,"service":{ + "loaded":true, + "runtime":{"status":"running","pid":4242}, + "command":{"programArguments":["openclaw","gateway","--port","\(port)"]}, + "configAudit":{"ok":true,"issues":[]} + }} + """) + GatewayLaunchAgentManager.setTestingDaemonCommandDelayNanoseconds(200_000_000) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload(nil) + GatewayLaunchAgentManager.setTestingDaemonCommandDelayNanoseconds(0) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + GatewayProcessManager.shared.setTestingDesiredActive(false) + GatewayProcessManager.shared._testClearLaunchAgentReadinessFailure() + } + + let manager = GatewayProcessManager.shared + manager.setTestingDesiredActive(true) + let staleFinish = Task { @MainActor in + await manager._testFinishLaunchAgentReadinessFailure( + port: port, + startingPID: 4242) + } + for _ in 0..<100 { + if GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + .contains(where: { $0.first == "status" }) + { + break + } + try? await Task.sleep(nanoseconds: 1_000_000) + } + #expect(GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + .contains(where: { $0.first == "status" })) + + GatewayLaunchAgentManager.setTestingDaemonCommandDelayNanoseconds(0) + manager.stop() + manager._testBeginGatewayStartGeneration() + await manager._testFinishLaunchAgentReadinessFailure( + port: port, + startingPID: 4242) + #expect(manager._testHasLaunchAgentReadinessFailure()) + + await staleFinish.value + + #expect(manager.status == .failed("Gateway did not start in time")) + #expect(manager._testHasLaunchAgentReadinessFailure()) + } + } + + @Test func `repairs a stable launch agent PID with a wedged listener`() async throws { + let port = 19087 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload( + """ + {"ok":true,"service":{ + "loaded":true, + "runtime":{"status":"running","pid":4242}, + "command":{"programArguments":["openclaw","gateway","--port","\(port)"]}, + "configAudit":{"ok":true,"issues":[]} + }} + """) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload(nil) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + GatewayProcessManager.shared._testClearLaunchAgentReadinessFailure() + } + + let manager = GatewayProcessManager.shared + let listener = PortGuardian.Descriptor( + pid: 4242, + command: "openclaw-gateway", + executablePath: "/tmp/openclaw-gateway") + await PortGuardian.shared.setTestingDescriptor(listener, forPort: port) + await manager._testRecordLaunchAgentReadinessFailure(port: port, startingPID: 4242) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + + _ = await manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: port) + + let calls = GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + #expect(calls.filter { $0.first == "status" }.count == 1) + #expect(calls.filter { $0.first == "install" }.count == 1) + await PortGuardian.shared.setTestingDescriptor(nil, forPort: port) + } + } + + @Test func `protects a foreign listener after launch agent readiness fails`() async throws { + let port = 19088 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload( + """ + {"ok":true,"service":{ + "loaded":true, + "runtime":{"status":"running","pid":4242}, + "command":{"programArguments":["openclaw","gateway","--port","\(port)"]}, + "configAudit":{"ok":true,"issues":[]} + }} + """) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload(nil) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + GatewayProcessManager.shared._testClearLaunchAgentReadinessFailure() + } + + let manager = GatewayProcessManager.shared + await manager._testRecordLaunchAgentReadinessFailure(port: port, startingPID: 4242) + let listener = PortGuardian.Descriptor( + pid: 4243, + command: "foreign-listener", + executablePath: "/tmp/foreign-listener") + await PortGuardian.shared.setTestingDescriptor(listener, forPort: port) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + + _ = await manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: port) + + let calls = GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + #expect(calls.filter { $0.first == "status" }.count == 1) + #expect(calls.filter { $0.first == "install" }.isEmpty) + await PortGuardian.shared.setTestingDescriptor(nil, forPort: port) + } + } + + @Test func `protects an unmanaged listener during persistence ensure`() async throws { + let port = 19100 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload( + #"{"ok":true,"service":{"loaded":false}}"#) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload(nil) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + } + + let listener = PortGuardian.Descriptor( + pid: 4243, + command: "manual-gateway", + executablePath: "/tmp/manual-gateway") + await PortGuardian.shared.setTestingDescriptor(listener, forPort: port) + + _ = await GatewayProcessManager.shared._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: port) + + let calls = GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + #expect(calls.filter { $0.first == "status" }.count == 1) + #expect(calls.filter { $0.first == "install" }.isEmpty) + await PortGuardian.shared.setTestingDescriptor(nil, forPort: port) + } + } + + @Test func `does not force install when launchd starts during ownership inspection`() async throws { + let port = 19102 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonStatusPayloads([ + #"{"ok":true,"service":{"loaded":false}}"#, + """ + {"ok":true,"service":{ + "loaded":true, + "runtime":{"status":"running","pid":4242}, + "command":{"programArguments":["openclaw","gateway","--port","\(port)"]}, + "configAudit":{"ok":true,"issues":[]} + }} + """, + ]) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayloads([]) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + } + + let listener = PortGuardian.Descriptor( + pid: 4242, + command: "openclaw-gateway", + executablePath: "/tmp/openclaw-gateway") + await PortGuardian.shared.setTestingDescriptor(listener, forPort: port) + + _ = await GatewayProcessManager.shared._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: port) + + let calls = GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + #expect(calls.filter { $0.first == "status" }.count == 1) + #expect(calls.filter { $0.first == "install" }.isEmpty) + await PortGuardian.shared.setTestingDescriptor(nil, forPort: port) + } + } + + @Test func `repairs loaded launch agents that are not reusable`() async throws { + let port = 19083 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload(nil) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + } + + let staleStatuses = [ + """ + {"ok":true,"service":{ + "loaded":true, + "runtime":{"status":"stopped"}, + "command":{"programArguments":["openclaw","gateway","--port","\(port)"]}, + "configAudit":{"ok":true,"issues":[]} + }} + """, + """ + {"ok":true,"service":{ + "loaded":true, + "runtime":{"status":"running","pid":4242}, + "command":{"programArguments":["openclaw","gateway","--port","19084"]}, + "configAudit":{"ok":true,"issues":[]} + }} + """, + """ + {"ok":true,"service":{ + "loaded":true, + "runtime":{"status":"running","pid":4242}, + "command":{"programArguments":["openclaw","gateway","--port","\(port)"]}, + "configAudit":{"ok":false,"issues":[{"code":"gateway-service-version-mismatch"}]} + }} + """, + ] + + for status in staleStatuses { + GatewayLaunchAgentManager.setTestingDaemonStatusPayload(status) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + + _ = await GatewayProcessManager.shared._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: port) + + let calls = GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + #expect(calls.filter { $0.first == "status" }.count == 1) + #expect(calls.filter { $0.first == "install" }.count == 1) + } + } + } + @Test func `clears last failure when health succeeds`() async throws { let session = GatewayTestWebSocketSession( taskFactory: { @@ -24,21 +968,754 @@ struct GatewayProcessManagerTests { let manager = GatewayProcessManager.shared manager.setTestingConnection(connection) manager.setTestingDesiredActive(true) + manager.setTestingSkipControlChannelRefresh(true) manager.setTestingLastFailureReason("health failed") + manager.setTestingStatus(.failed("Gateway did not start in time")) + let readinessPort = GatewayEnvironment.gatewayPort() + manager._testSetLaunchAgentReadinessFailure(port: readinessPort, pid: 4242) + let descriptor = PortGuardian.Descriptor( + pid: 4343, + command: "openclaw-gateway", + executablePath: "/tmp/openclaw-gateway") + await PortGuardian.shared.setTestingDescriptor(descriptor, forPort: readinessPort) defer { manager.setTestingConnection(nil) manager.setTestingDesiredActive(false) + manager.setTestingSkipControlChannelRefresh(false) manager.setTestingLastFailureReason(nil) + manager._testClearLaunchAgentReadinessFailure() } let ready = await manager.waitForGatewayReady(timeout: 0.5) #expect(ready) #expect(manager.lastFailureReason == nil) + #expect(!manager._testHasLaunchAgentReadinessFailure()) + #expect(manager.status == .running(details: "pid 4343")) + await PortGuardian.shared.setTestingDescriptor(nil, forPort: readinessPort) + } + + @Test func `startup install forces the recovered control channel refresh`() async throws { + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + let session = GatewayTestWebSocketSession( + taskFactory: { + GatewayTestWebSocketTask( + sendHook: { task, message, sendIndex in + guard sendIndex > 0 else { return } + guard let id = GatewayWebSocketTestSupport.requestID(from: message) else { return } + task.emitReceiveSuccess(.data(GatewayWebSocketTestSupport.okResponseData(id: id))) + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let connection = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + try await self.withLocalGatewayConfig { + let port = GatewayEnvironment.gatewayPort() + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload( + #"{"ok":true,"service":{"loaded":false}}"#) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + let manager = GatewayProcessManager.shared + manager.setTestingConnection(connection) + manager.setTestingDesiredActive(true) + manager.setTestingStatus(.attachedExisting(details: "old pid")) + manager.setTestingSkipControlChannelRefresh(true) + manager._testClearControlChannelRefreshForces() + manager._testClearLaunchAgentReadinessFailure() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload(nil) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + manager.setTestingConnection(nil) + manager.setTestingDesiredActive(false) + manager.setTestingSkipControlChannelRefresh(false) + manager._testClearControlChannelRefreshForces() + manager._testClearLaunchAgentReadinessFailure() + } + + #expect(await manager._testEnableLaunchAgentIfNeededInstalled( + bundlePath: "/Applications/OpenClaw.app", + port: port)) + let descriptor = PortGuardian.Descriptor( + pid: 4242, + command: "openclaw-gateway", + executablePath: "/tmp/openclaw-gateway") + await PortGuardian.shared.setTestingDescriptor(descriptor, forPort: port) + + #expect(await manager.waitForGatewayReady(timeout: 0.5)) + #expect(manager._testControlChannelRefreshForces().last == true) + #expect(manager.status == .running(details: "pid 4242")) + + await connection.shutdown() + await PortGuardian.shared.setTestingDescriptor(nil, forPort: port) + } + } + + @Test func `readiness refreshes when the endpoint pid changes during the probe`() async throws { + let port = GatewayEnvironment.gatewayPort() + let session = GatewayTestWebSocketSession( + taskFactory: { + GatewayTestWebSocketTask( + sendHook: { task, message, sendIndex in + guard sendIndex > 0 else { return } + guard let id = GatewayWebSocketTestSupport.requestID(from: message) else { return } + let replacement = PortGuardian.Descriptor( + pid: 4343, + command: "openclaw-gateway", + executablePath: "/tmp/openclaw-gateway") + await PortGuardian.shared.setTestingDescriptor(replacement, forPort: port) + task.emitReceiveSuccess(.data(GatewayWebSocketTestSupport.okResponseData(id: id))) + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let connection = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + let manager = GatewayProcessManager.shared + manager.setTestingConnection(connection) + manager.setTestingDesiredActive(true) + manager.setTestingStatus(.running(details: "pid 4242")) + manager.setTestingSkipControlChannelRefresh(true) + manager._testClearControlChannelRefreshForces() + manager._testClearLaunchAgentReadinessFailure() + manager._testClearLaunchAgentInstallEvidence() + manager._testSetLastObservedGatewayPID(4242) + defer { + manager.setTestingConnection(nil) + manager.setTestingDesiredActive(false) + manager.setTestingSkipControlChannelRefresh(false) + manager._testClearControlChannelRefreshForces() + manager._testClearLaunchAgentReadinessFailure() + manager._testClearLaunchAgentInstallEvidence() + manager._testSetLastObservedGatewayPID(nil) + } + + #expect(await manager.waitForGatewayReady(timeout: 0.5)) + #expect(manager._testControlChannelRefreshForces().last == true) + #expect(manager.status == .running(details: "pid 4343")) + + await connection.shutdown() + await PortGuardian.shared.setTestingDescriptor(nil, forPort: port) + } + + @Test func `readiness retains the endpoint pid from before a launchd candidate`() async throws { + let port = GatewayEnvironment.gatewayPort() + let session = GatewayTestWebSocketSession( + taskFactory: { + GatewayTestWebSocketTask( + sendHook: { task, message, sendIndex in + guard sendIndex > 0 else { return } + guard let id = GatewayWebSocketTestSupport.requestID(from: message) else { return } + task.emitReceiveSuccess(.data(GatewayWebSocketTestSupport.okResponseData(id: id))) + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let connection = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + let descriptor = PortGuardian.Descriptor( + pid: 4242, + command: "openclaw-gateway", + executablePath: "/tmp/openclaw-gateway") + + let manager = GatewayProcessManager.shared + manager.setTestingConnection(connection) + manager.setTestingDesiredActive(true) + manager.setTestingStatus(.running(details: "pid 4141")) + manager.setTestingSkipControlChannelRefresh(true) + manager._testClearControlChannelRefreshForces() + manager._testClearLaunchAgentReadinessFailure() + manager._testClearLaunchAgentInstallEvidence() + manager._testSetLastObservedGatewayPID(4141) + manager._testSetLaunchAgentReadinessCandidate(port: port, pid: 4242) + await PortGuardian.shared.setTestingDescriptor(descriptor, forPort: port) + defer { + manager.setTestingConnection(nil) + manager.setTestingDesiredActive(false) + manager.setTestingSkipControlChannelRefresh(false) + manager._testClearControlChannelRefreshForces() + manager._testClearLaunchAgentReadinessFailure() + manager._testClearLaunchAgentInstallEvidence() + manager._testSetLastObservedGatewayPID(nil) + } + + #expect(await manager.waitForGatewayReady(timeout: 0.5)) + #expect(manager._testControlChannelRefreshForces().last == true) + #expect(manager.status == .running(details: "pid 4242")) + + await connection.shutdown() + await PortGuardian.shared.setTestingDescriptor(nil, forPort: port) + } + + @Test func `responsive health rejection does not arm launchd repair`() async throws { + let port = 19105 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + let session = GatewayTestWebSocketSession( + taskFactory: { + GatewayTestWebSocketTask( + sendHook: { task, message, sendIndex in + guard sendIndex > 0 else { return } + guard let id = GatewayWebSocketTestSupport.requestID(from: message) else { return } + let response = Data( + """ + {"type":"res","id":"\(id)","ok":false, + "error":{"code":"INVALID_REQUEST","message":"health rejected"}} + """.utf8) + task.emitReceiveSuccess(.data(response)) + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let connection = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload( + """ + {"ok":true,"service":{ + "loaded":true, + "runtime":{"status":"running","pid":4242}, + "command":{"programArguments":["openclaw","gateway","--port","\(port)"]}, + "configAudit":{"ok":true,"issues":[]} + }} + """) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + let manager = GatewayProcessManager.shared + manager.setTestingConnection(connection) + manager.setTestingDesiredActive(true) + manager.setTestingStatus(.attachedExisting(details: "pid 4242")) + manager._testClearLaunchAgentReadinessFailure() + manager._testSetLaunchAgentReadinessCandidate(port: port, pid: 4242) + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload(nil) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + manager.setTestingConnection(nil) + manager.setTestingDesiredActive(false) + manager.setTestingLastFailureReason(nil) + manager._testClearLaunchAgentReadinessFailure() + } + + let descriptor = PortGuardian.Descriptor( + pid: 4242, + command: "openclaw-gateway", + executablePath: "/tmp/openclaw-gateway") + await PortGuardian.shared.setTestingDescriptor(descriptor, forPort: port) + + #expect(await manager.waitForGatewayReady(timeout: 0.5) == false) + #expect(!manager._testHasLaunchAgentReadinessFailure()) + guard case let .failed(reason) = manager.status else { + Issue.record("expected responsive health failure") + await connection.shutdown() + await PortGuardian.shared.setTestingDescriptor(nil, forPort: port) + return + } + #expect(reason.contains("health rejected")) + + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + _ = await manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: port) + #expect(GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + .filter { $0.first == "install" }.isEmpty) + + await connection.shutdown() + await PortGuardian.shared.setTestingDescriptor(nil, forPort: port) + } + } + + @Test func `transient unavailable health response retries until ready`() async throws { + let port = GatewayEnvironment.gatewayPort() + let session = GatewayTestWebSocketSession( + taskFactory: { + GatewayTestWebSocketTask( + sendHook: { task, message, sendIndex in + guard sendIndex > 0 else { return } + guard let id = GatewayWebSocketTestSupport.requestID(from: message) else { return } + if sendIndex == 1 { + let response = Data( + """ + {"type":"res","id":"\(id)","ok":false, + "error":{"code":"UNAVAILABLE","message":"gateway restarting"}} + """.utf8) + task.emitReceiveSuccess(.data(response)) + return + } + task.emitReceiveSuccess(.data(GatewayWebSocketTestSupport.okResponseData(id: id))) + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let connection = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + let descriptor = PortGuardian.Descriptor( + pid: 4242, + command: "openclaw-gateway", + executablePath: "/tmp/openclaw-gateway") + + let manager = GatewayProcessManager.shared + manager.setTestingConnection(connection) + manager.setTestingDesiredActive(true) + manager.setTestingStatus(.starting) + manager._testClearLaunchAgentReadinessFailure() + await PortGuardian.shared.setTestingDescriptor(descriptor, forPort: port) + defer { + manager.setTestingConnection(nil) + manager.setTestingDesiredActive(false) + manager.setTestingLastFailureReason(nil) + manager._testClearLaunchAgentReadinessFailure() + manager._testSetLastObservedGatewayPID(nil) + } + + #expect(await manager.waitForGatewayReady(timeout: 1)) + #expect(manager.status == .running(details: "pid 4242")) + #expect(!manager._testHasLaunchAgentReadinessFailure()) + + await connection.shutdown() + await PortGuardian.shared.setTestingDescriptor(nil, forPort: port) + } + + @Test func `cancelled readiness probe preserves lifecycle state`() async throws { + let session = GatewayTestWebSocketSession( + taskFactory: { + GatewayTestWebSocketTask( + receiveHook: { _, receiveIndex in + if receiveIndex == 0 { + try await Task.sleep(nanoseconds: 30 * 1_000_000_000) + } + throw URLError(.cancelled) + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let connection = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + let manager = GatewayProcessManager.shared + manager.setTestingConnection(connection) + manager.setTestingDesiredActive(true) + manager.setTestingStatus(.running(details: "pid 4242")) + manager.setTestingLastFailureReason("keep newer state") + manager._testClearLaunchAgentReadinessFailure() + manager._testSetLaunchAgentReadinessCandidate(port: 19106, pid: 4242) + defer { + manager.setTestingConnection(nil) + manager.setTestingDesiredActive(false) + manager.setTestingLastFailureReason(nil) + manager._testClearLaunchAgentReadinessFailure() + } + + let readiness = Task { @MainActor in + await manager.waitForGatewayReady(timeout: 0.5) + } + for _ in 0..<100 { + if session.snapshotMakeCount() > 0 { break } + try? await Task.sleep(nanoseconds: 1_000_000) + } + #expect(session.snapshotMakeCount() == 1) + readiness.cancel() + + #expect(await readiness.value == false) + #expect(manager.status == .running(details: "pid 4242")) + #expect(manager.lastFailureReason == "keep newer state") + #expect(manager._testHasLaunchAgentReadinessCandidate()) + await connection.shutdown() + } + + @Test func `transport cancellation does not publish readiness failure`() async throws { + let session = GatewayTestWebSocketSession( + taskFactory: { + GatewayTestWebSocketTask( + receiveHook: { _, _ in + throw URLError(.cancelled) + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let connection = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + let manager = GatewayProcessManager.shared + manager.setTestingConnection(connection) + manager.setTestingDesiredActive(true) + manager.setTestingStatus(.running(details: "pid 4242")) + manager.setTestingLastFailureReason("keep current state") + manager._testClearLaunchAgentReadinessFailure() + manager._testSetLaunchAgentReadinessCandidate(port: 19113, pid: 4242) + defer { + manager.setTestingConnection(nil) + manager.setTestingDesiredActive(false) + manager.setTestingLastFailureReason(nil) + manager._testClearLaunchAgentReadinessFailure() + } + + let startedAt = Date() + #expect(await manager.waitForGatewayReady(timeout: 0.5) == false) + #expect(Date().timeIntervalSince(startedAt) < 1.5) + #expect(manager.status == .running(details: "pid 4242")) + #expect(manager.lastFailureReason == "keep current state") + #expect(manager._testHasLaunchAgentReadinessCandidate()) + await connection.shutdown() + } + + @Test func `only endpoint reachability failures arm launchd repair`() { + let manager = GatewayProcessManager.shared + #expect(manager._testProbeFailureMayNeedLaunchAgentRepair(.timedOut)) + #expect(manager._testProbeFailureMayNeedLaunchAgentRepair(.cannotConnectToHost)) + #expect(manager._testProbeFailureMayNeedLaunchAgentRepair(.networkConnectionLost)) + #expect(!manager._testProbeFailureMayNeedLaunchAgentRepair(.cancelled)) + #expect(!manager._testProbeFailureMayNeedLaunchAgentRepair(.badServerResponse)) + #expect(!manager._testProbeFailureMayNeedLaunchAgentRepair(.dataNotAllowed)) + #expect(manager._testGatewayResponseRetriesWithoutRepair("UNAVAILABLE")) + #expect(!manager._testGatewayResponseRetriesWithoutRepair("INVALID_REQUEST")) + } + + @Test func `stale readiness wait cannot clear a newer launch failure`() async throws { + let session = GatewayTestWebSocketSession( + taskFactory: { + GatewayTestWebSocketTask( + sendHook: { task, message, sendIndex in + guard sendIndex > 0 else { return } + guard let id = GatewayWebSocketTestSupport.requestID(from: message) else { return } + task.emitReceiveSuccess(.data(GatewayWebSocketTestSupport.okResponseData(id: id))) + }, + receiveHook: { _, receiveIndex in + if receiveIndex == 0 { + try await Task.sleep(nanoseconds: 100_000_000) + } + return .data(GatewayWebSocketTestSupport.connectChallengeData()) + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let connection = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + let manager = GatewayProcessManager.shared + manager.setTestingConnection(connection) + manager._testBeginGatewayStartGeneration() + defer { + manager.setTestingConnection(nil) + manager.setTestingDesiredActive(false) + manager._testClearLaunchAgentReadinessFailure() + } + + let staleWait = Task { @MainActor in + await manager.waitForGatewayReady(timeout: 0.5) + } + for _ in 0..<100 { + if session.snapshotMakeCount() > 0 { break } + try? await Task.sleep(nanoseconds: 1_000_000) + } + #expect(session.snapshotMakeCount() == 1) + manager._testBeginGatewayStartGeneration() + manager._testSetLaunchAgentReadinessFailure(port: 19101, pid: 4242) + + #expect(await staleWait.value == false) + #expect(manager._testHasLaunchAgentReadinessFailure()) + await connection.shutdown() + } + + @Test func `same generation stale probe preserves a newer readiness candidate`() async throws { + let session = GatewayTestWebSocketSession( + taskFactory: { + GatewayTestWebSocketTask( + sendHook: { task, message, sendIndex in + guard sendIndex > 0 else { return } + guard let id = GatewayWebSocketTestSupport.requestID(from: message) else { return } + task.emitReceiveSuccess(.data(GatewayWebSocketTestSupport.okResponseData(id: id))) + }, + receiveHook: { _, receiveIndex in + if receiveIndex == 0 { + try await Task.sleep(nanoseconds: 100_000_000) + } + return .data(GatewayWebSocketTestSupport.connectChallengeData()) + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let connection = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + let manager = GatewayProcessManager.shared + manager.setTestingConnection(connection) + manager.setTestingDesiredActive(true) + manager._testClearLaunchAgentReadinessFailure() + manager._testSetLaunchAgentReadinessCandidate(port: 19109, pid: 4242) + defer { + manager.setTestingConnection(nil) + manager.setTestingDesiredActive(false) + manager._testClearLaunchAgentReadinessFailure() + } + + let staleWait = Task { @MainActor in + await manager.waitForGatewayReady(timeout: 0.5) + } + for _ in 0..<100 { + if session.snapshotMakeCount() > 0 { break } + try? await Task.sleep(nanoseconds: 1_000_000) + } + #expect(session.snapshotMakeCount() == 1) + manager._testSetLaunchAgentReadinessCandidate(port: 19109, pid: 4243) + + #expect(await staleWait.value == false) + #expect(manager._testLaunchAgentReadinessCandidatePID() == 4243) + await connection.shutdown() + } + + @Test func `same generation stale timeout preserves a newer readiness failure`() async throws { + let session = GatewayTestWebSocketSession( + taskFactory: { + GatewayTestWebSocketTask( + receiveHook: { _, receiveIndex in + if receiveIndex == 0 { + try await Task.sleep(nanoseconds: 30 * 1_000_000_000) + } + return .data(GatewayWebSocketTestSupport.connectChallengeData()) + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let connection = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + let manager = GatewayProcessManager.shared + manager.setTestingConnection(connection) + manager.setTestingDesiredActive(true) + manager.setTestingLastFailureReason(nil) + manager._testClearLaunchAgentReadinessFailure() + defer { + manager.setTestingConnection(nil) + manager.setTestingDesiredActive(false) + manager.setTestingLastFailureReason(nil) + manager._testClearLaunchAgentReadinessFailure() + } + + let staleWait = Task { @MainActor in + await manager.waitForGatewayReady(timeout: 0.2) + } + for _ in 0..<100 { + if session.snapshotMakeCount() > 0 { break } + try? await Task.sleep(nanoseconds: 1_000_000) + } + #expect(session.snapshotMakeCount() == 1) + manager.setTestingLastFailureReason("newer same-generation failure") + manager._testSetLaunchAgentReadinessFailure(port: 19110, pid: 4244) + + #expect(await staleWait.value == false) + #expect(manager.lastFailureReason == "newer same-generation failure") + #expect(manager._testHasLaunchAgentReadinessFailure()) + await connection.shutdown() + } + + @Test func `stale readiness timeout cannot replace a newer launch failure`() async throws { + let session = GatewayTestWebSocketSession( + taskFactory: { + GatewayTestWebSocketTask( + receiveHook: { _, receiveIndex in + if receiveIndex == 0 { + try await Task.sleep(nanoseconds: 30 * 1_000_000_000) + } + return .data(GatewayWebSocketTestSupport.connectChallengeData()) + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let connection = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + let manager = GatewayProcessManager.shared + manager.setTestingConnection(connection) + manager._testBeginGatewayStartGeneration() + defer { + manager.setTestingConnection(nil) + manager.setTestingDesiredActive(false) + manager.setTestingLastFailureReason(nil) + manager._testClearLaunchAgentReadinessFailure() + } + + let staleWait = Task { @MainActor in + await manager.waitForGatewayReady(timeout: 0.5) + } + for _ in 0..<100 { + if session.snapshotMakeCount() > 0 { break } + try? await Task.sleep(nanoseconds: 1_000_000) + } + #expect(session.snapshotMakeCount() == 1) + manager._testBeginGatewayStartGeneration() + manager.setTestingLastFailureReason("newer command resolution failure") + manager._testSetLaunchAgentReadinessFailure(port: 19103, pid: 4243) + + #expect(await staleWait.value == false) + #expect(manager.lastFailureReason == "newer command resolution failure") + #expect(manager._testHasLaunchAgentReadinessFailure()) + await connection.shutdown() + } + + @Test func `readiness timeout includes a stalled socket connect`() async throws { + let session = GatewayTestWebSocketSession( + taskFactory: { + GatewayTestWebSocketTask( + receiveHook: { _, receiveIndex in + if receiveIndex == 0 { + try await Task.sleep(nanoseconds: 30 * 1_000_000_000) + } + return .data(GatewayWebSocketTestSupport.connectChallengeData()) + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let connection = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + let manager = GatewayProcessManager.shared + manager.setTestingConnection(connection) + manager.setTestingDesiredActive(true) + manager.setTestingStatus(.attachedExisting(details: "pid 3131")) + manager.setTestingLastFailureReason(nil) + manager._testClearLaunchAgentReadinessFailure() + manager._testSetLaunchAgentReadinessFailure(port: 19111, pid: 4245) + defer { + manager.setTestingConnection(nil) + manager.setTestingDesiredActive(false) + manager.setTestingLastFailureReason(nil) + manager._testClearLaunchAgentReadinessFailure() + } + + let startedAt = Date() + let ready = await manager.waitForGatewayReady(timeout: 0.1) + let elapsed = Date().timeIntervalSince(startedAt) + await connection.shutdown() + + #expect(!ready) + #expect(elapsed < 1) + #expect(session.snapshotMakeCount() == 1) + #expect(manager.status == .failed("Gateway did not start in time")) + #expect(manager.lastFailureReason == "gateway readiness timeout") + #expect(manager._testHasLaunchAgentReadinessFailure()) + } + + @Test func `readiness timeout preserves a concrete launch failure`() async throws { + let session = GatewayTestWebSocketSession( + taskFactory: { + GatewayTestWebSocketTask( + receiveHook: { _, receiveIndex in + if receiveIndex == 0 { + try await Task.sleep(nanoseconds: 30 * 1_000_000_000) + } + return .data(GatewayWebSocketTestSupport.connectChallengeData()) + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let connection = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + let manager = GatewayProcessManager.shared + manager.setTestingConnection(connection) + manager.setTestingDesiredActive(true) + manager.setTestingStatus(.failed("launchd install denied")) + manager.setTestingLastFailureReason("launchd install denied") + manager._testClearLaunchAgentReadinessFailure() + defer { + manager.setTestingConnection(nil) + manager.setTestingDesiredActive(false) + manager.setTestingLastFailureReason(nil) + manager._testClearLaunchAgentReadinessFailure() + } + + #expect(await manager.waitForGatewayReady(timeout: 0.1) == false) + #expect(manager.status == .failed("launchd install denied")) + #expect(manager.lastFailureReason == "launchd install denied") + await connection.shutdown() + } + + @Test func `replacement readiness timeout records the pid for the next repair`() async throws { + let port = 19104 + let marker = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-launchagent-marker-\(UUID().uuidString)") + let session = GatewayTestWebSocketSession( + taskFactory: { + GatewayTestWebSocketTask( + receiveHook: { _, receiveIndex in + if receiveIndex == 0 { + try await Task.sleep(nanoseconds: 30 * 1_000_000_000) + } + return .data(GatewayWebSocketTestSupport.connectChallengeData()) + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let connection = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + + try await self.withLocalGatewayConfig { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(marker) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(true) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload( + """ + {"ok":true,"service":{ + "loaded":true, + "runtime":{"status":"running","pid":4242}, + "command":{"programArguments":["openclaw","gateway","--port","\(port)"]}, + "configAudit":{"ok":true,"issues":[]} + }} + """) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + let manager = GatewayProcessManager.shared + manager.setTestingConnection(connection) + manager.setTestingDesiredActive(true) + manager._testClearLaunchAgentReadinessFailure() + defer { + GatewayLaunchAgentManager.setTestingDisableLaunchAgentMarkerURL(nil) + GatewayLaunchAgentManager.setTestingInterceptDaemonCommands(false) + GatewayLaunchAgentManager.setTestingDaemonStatusPayload(nil) + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + manager.setTestingConnection(nil) + manager.setTestingDesiredActive(false) + manager.setTestingLastFailureReason(nil) + manager._testClearLaunchAgentReadinessFailure() + } + + let listener = PortGuardian.Descriptor( + pid: 4242, + command: "openclaw-gateway", + executablePath: "/tmp/openclaw-gateway") + await PortGuardian.shared.setTestingDescriptor(listener, forPort: port) + + _ = await manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: port) + #expect(GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + .filter { $0.first == "install" }.isEmpty) + + #expect(await manager.waitForGatewayReady(timeout: 0.1) == false) + #expect(manager._testHasLaunchAgentReadinessFailure()) + + GatewayLaunchAgentManager.clearTestingDaemonCommandCalls() + _ = await manager._testEnableLaunchAgentIfNeeded( + bundlePath: "/Applications/OpenClaw.app", + port: port) + #expect(GatewayLaunchAgentManager.testingDaemonCommandCallsSnapshot() + .filter { $0.first == "install" }.count == 1) + + await connection.shutdown() + await PortGuardian.shared.setTestingDescriptor(nil, forPort: port) + } } @Test func `attaches to existing gateway without spawning launchd`() async throws { let port = 19097 - try await TestIsolation.withEnvValues(["OPENCLAW_GATEWAY_PORT": "\(port)"]) { + do { let healthData = Data( """ { @@ -70,6 +1747,20 @@ struct GatewayProcessManagerTests { sendHook: { task, message, sendIndex in guard sendIndex > 0 else { return } guard let id = GatewayWebSocketTestSupport.requestID(from: message) else { return } + if sendIndex == 1 { + let response = Data( + """ + {"type":"res","id":"\(id)","ok":false, + "error":{"code":"UNAVAILABLE","message":"gateway restarting"}} + """.utf8) + task.emitReceiveSuccess(.data(response)) + return + } + let replacement = PortGuardian.Descriptor( + pid: 4343, + command: "openclaw-gateway", + executablePath: "/tmp/openclaw-gateway") + await PortGuardian.shared.setTestingDescriptor(replacement, forPort: port) let json = """ { "type": "res", @@ -95,6 +1786,8 @@ struct GatewayProcessManagerTests { manager.setTestingConnection(connection) manager.setTestingSkipControlChannelRefresh(true) manager.setTestingLastFailureReason("stale") + manager._testClearControlChannelRefreshForces() + manager._testSetLastObservedGatewayPID(4242) @MainActor func cleanup() async { @@ -102,11 +1795,13 @@ struct GatewayProcessManagerTests { manager.setTestingSkipControlChannelRefresh(false) manager.setTestingDesiredActive(false) manager.setTestingLastFailureReason(nil) + manager._testClearControlChannelRefreshForces() + manager._testSetLastObservedGatewayPID(nil) await PortGuardian.shared.setTestingDescriptor(nil, forPort: port) } do { - let attached = await manager._testAttachExistingGatewayIfAvailable() + let attached = await manager._testAttachExistingGatewayIfAvailable(port: port) #expect(attached) #expect(manager.lastFailureReason == nil) guard case let .attachedExisting(statusDetails) = manager.status else { @@ -118,7 +1813,8 @@ struct GatewayProcessManagerTests { #expect(details.contains("port \(port)")) #expect(details.contains("Telegram linked")) #expect(details.contains("auth 1m")) - #expect(details.contains("pid 4242 openclaw-gateway @ /tmp/openclaw-gateway")) + #expect(details.contains("pid 4343 openclaw-gateway @ /tmp/openclaw-gateway")) + #expect(manager._testControlChannelRefreshForces().last == true) await cleanup() } catch { await cleanup() diff --git a/src/system-agent/setup-inference.test.ts b/src/system-agent/setup-inference.test.ts index acd478c7be02..b3734a680c3f 100644 --- a/src/system-agent/setup-inference.test.ts +++ b/src/system-agent/setup-inference.test.ts @@ -3614,10 +3614,7 @@ describe("activateSetupInference", () => { ]); expect(transformConfig).toHaveBeenCalledWith( expect.objectContaining({ - afterWrite: { - mode: "none", - reason: "OpenClaw activates verified inference", - }, + afterWrite: { mode: "auto" }, }), ); expect(refreshPluginRegistry).toHaveBeenCalledWith({ diff --git a/src/system-agent/setup-inference.ts b/src/system-agent/setup-inference.ts index a5ff44a5142d..38457aa1b4d8 100644 --- a/src/system-agent/setup-inference.ts +++ b/src/system-agent/setup-inference.ts @@ -2036,7 +2036,9 @@ async function activateSetupInferenceUnredacted( base: "source", // The transform stays side-effect free so a config conflict can retry // without replaying credential writes in another agent directory. - afterWrite: { mode: "none", reason: "OpenClaw activates verified inference" }, + // Setup changes only hot-reloadable model, agent, and plugin-entry surfaces. + // Publish the verified route now so the next turn cannot reuse the old harness. + afterWrite: { mode: "auto" }, transform: async (current, context) => { const latestRuntime = context.snapshot.runtimeConfig ?? context.snapshot.config; // Validate that the candidate is still admissible before reporting