From 19ace6830b17a20cf5b103c1893a2c5d7bca6bc3 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 14 Aug 2026 10:24:08 -0700 Subject: [PATCH] feat(macos): add embedded CUA computer provider (#123635) * feat(macos): embed CUA computer provider * fix(macos): clarify embedded CUA trust posture * fix(macos): contain embedded CUA daemon lifecycle * fix(macos): reap orphaned CUA daemons * fix(macos): record the spawned CUA daemon pid so reaping can terminate orphans * chore(macos): refresh native i18n baseline for the computer control provider picker * style(macos): satisfy swiftlint on the embedded CUA host and connect params * refactor(gateway): move optional connect params to GatewayConnectOptions --- apps/.i18n/native-source.json | 37 + apps/macos/Sources/OpenClaw/AppState.swift | 38 +- .../OpenClaw/ComputerControlProvider.swift | 59 ++ apps/macos/Sources/OpenClaw/Constants.swift | 1 + .../OpenClaw/CuaDriverHostCoordinator.swift | 686 ++++++++++++++++++ .../OpenClaw/CuaDriverHostLifecycle.swift | 314 ++++++++ .../Sources/OpenClaw/GeneralSettings.swift | 50 +- apps/macos/Sources/OpenClaw/MenuBar.swift | 5 +- .../OpenClaw/NodeMode/MacNodeHostWorker.swift | 47 +- .../NodeMode/MacNodeModeCoordinator.swift | 135 +++- .../OpenClaw/NodeMode/MacNodeRuntime.swift | 31 + .../ComputerControlSettingsTests.swift | 31 + .../CuaDriverHostCoordinatorTests.swift | 571 +++++++++++++++ .../GatewayChannelConnectTests.swift | 41 ++ .../MacNodeHostWorkerTests.swift | 78 ++ .../MacNodeModeCoordinatorTests.swift | 7 + .../MacNodeRuntimeTests.swift | 97 +++ .../Sources/OpenClawKit/GatewayChannel.swift | 10 +- .../OpenClawKit/GatewayConnectOptions.swift | 26 + docs/nodes/computer-use.md | 53 +- extensions/cua-computer/index.test.ts | 9 + extensions/cua-computer/index.ts | 2 +- extensions/cua-computer/openclaw.plugin.json | 3 +- extensions/cua-computer/package.json | 2 +- extensions/cua-computer/src/commands.test.ts | 109 ++- extensions/cua-computer/src/commands.ts | 70 +- .../src/mcp-driver-client.test.ts | 288 ++++++++ .../cua-computer/src/mcp-driver-client.ts | 682 +++++++++++++++++ scripts/check-changed.mts | 2 +- scripts/ci-changed-scope.mjs | 2 +- scripts/codesign-mac-app.sh | 5 + scripts/package-mac-app.sh | 3 + scripts/stage-cua-driver-macos.sh | 59 ++ scripts/test-projects.test-support.mts | 1 + test/scripts/package-mac-app.test.ts | 23 + 35 files changed, 3497 insertions(+), 80 deletions(-) create mode 100644 apps/macos/Sources/OpenClaw/ComputerControlProvider.swift create mode 100644 apps/macos/Sources/OpenClaw/CuaDriverHostCoordinator.swift create mode 100644 apps/macos/Sources/OpenClaw/CuaDriverHostLifecycle.swift create mode 100644 apps/macos/Tests/OpenClawIPCTests/CuaDriverHostCoordinatorTests.swift create mode 100644 extensions/cua-computer/src/mcp-driver-client.test.ts create mode 100644 extensions/cua-computer/src/mcp-driver-client.ts create mode 100755 scripts/stage-cua-driver-macos.sh diff --git a/apps/.i18n/native-source.json b/apps/.i18n/native-source.json index b9b50c36c726..8b00cf6ff9ff 100644 --- a/apps/.i18n/native-source.json +++ b/apps/.i18n/native-source.json @@ -23152,6 +23152,17 @@ } ] }, + { + "id": "native.apple.02f52db82466e98a", + "source": "CUA (driver not bundled)", + "surface": "apple", + "sites": [ + { + "kind": "conditional-branch", + "path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift" + } + ] + }, { "id": "native.apple.cfbd34b9f70ab403", "source": "Cache", @@ -24118,6 +24129,17 @@ } ] }, + { + "id": "native.apple.f0969ec829f56eed", + "source": "Choose the node-local automation backend for snapshots and actions.", + "surface": "apple", + "sites": [ + { + "kind": "ui-named-argument", + "path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift" + } + ] + }, { "id": "native.apple.e4b13f49bb435884", "source": "Choose where the Gateway runs and how this Mac app reaches it.", @@ -24624,6 +24646,21 @@ } ] }, + { + "id": "native.apple.7b46ae17b22676bf", + "source": "Computer Control provider", + "surface": "apple", + "sites": [ + { + "kind": "ui-call", + "path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift" + }, + { + "kind": "ui-named-argument", + "path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift" + } + ] + }, { "id": "native.apple.ab225ee73fa3efc2", "source": "Config", diff --git a/apps/macos/Sources/OpenClaw/AppState.swift b/apps/macos/Sources/OpenClaw/AppState.swift index 0efcd9b12000..03788178178b 100644 --- a/apps/macos/Sources/OpenClaw/AppState.swift +++ b/apps/macos/Sources/OpenClaw/AppState.swift @@ -113,6 +113,8 @@ final class AppState { @ObservationIgnored private let voiceWakeGlobalSyncScheduler = VoiceWakeGlobalSyncScheduler() @ObservationIgnored private var activeComputerPresenceTask: Task? @ObservationIgnored private var activeComputerPresenceUpdateGeneration: UInt64 = 0 + @ObservationIgnored private var computerControlHostReconciliationTask: Task? + @ObservationIgnored private var computerControlHostGeneration: UInt64 = 0 var isPaused: Bool { didSet { self.ifNotPreview { AppDefaults.standard.set(self.isPaused, forKey: pauseDefaultsKey) } } @@ -364,18 +366,40 @@ final class AppState { self.ifNotPreview { AppDefaults.standard.set(self.peekabooBridgeEnabled, forKey: peekabooBridgeEnabledKey) } - self.applyPeekabooBridgeHostState() + self.applyComputerControlHostState() } } - /// PeekabooBridge shares Computer Control's local UI-automation surface, so the host only - /// runs while Computer Control is enabled. With Computer Control off, users drive Peekaboo - /// via its own Mac app instead of a second, separately toggled bridge here. - func applyPeekabooBridgeHostState() { + /// The selected provider owns the complete Computer Control execution surface. + /// Keep the unselected host stopped so one node execution never mixes backends. + func applyComputerControlHostState() { self.ifNotPreview { let computerControlEnabled = isComputerControlEnabled() - let shouldRun = self.peekabooBridgeEnabled && computerControlEnabled - Task { await PeekabooBridgeHostCoordinator.shared.setEnabled(shouldRun) } + let provider = ComputerControlProvider.current() + let peekabooBridgeEnabled = self.peekabooBridgeEnabled + self.computerControlHostGeneration &+= 1 + let generation = self.computerControlHostGeneration + let predecessor = self.computerControlHostReconciliationTask + let task = Task { @MainActor [weak self] in + await predecessor?.value + guard let self, generation == self.computerControlHostGeneration else { return } + switch provider { + case .cua where computerControlEnabled: + await PeekabooBridgeHostCoordinator.shared.setEnabled(false) + guard generation == self.computerControlHostGeneration else { return } + await CuaDriverHostCoordinator.shared.setEnabled(true) + case .peekaboo: + await CuaDriverHostCoordinator.shared.setEnabled(false) + guard generation == self.computerControlHostGeneration else { return } + await PeekabooBridgeHostCoordinator.shared.setEnabled( + peekabooBridgeEnabled && computerControlEnabled) + case .cua: + await CuaDriverHostCoordinator.shared.setEnabled(false) + guard generation == self.computerControlHostGeneration else { return } + await PeekabooBridgeHostCoordinator.shared.setEnabled(false) + } + } + self.computerControlHostReconciliationTask = task } } diff --git a/apps/macos/Sources/OpenClaw/ComputerControlProvider.swift b/apps/macos/Sources/OpenClaw/ComputerControlProvider.swift new file mode 100644 index 000000000000..03b93d382746 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ComputerControlProvider.swift @@ -0,0 +1,59 @@ +import Foundation + +enum ComputerControlProvider: String, CaseIterable, Sendable { + case peekaboo + case cua + + static func current( + defaults: UserDefaults = AppDefaults.standard, + cuaAvailable: Bool = CuaDriverArtifact.bundledExecutableURL != nil) -> Self + { + guard let rawValue = defaults.string(forKey: computerControlProviderKey), + let provider = Self(rawValue: rawValue) + else { return .peekaboo } + if provider == .cua, !cuaAvailable { return .peekaboo } + return provider + } + + var displayName: String { + switch self { + case .peekaboo: "Peekaboo" + case .cua: "CUA" + } + } +} + +struct CuaDriverWorkerEndpoint: Equatable, Sendable { + let socketPath: String + let binaryPath: String +} + +enum CuaDriverWorkerEnvironment { + static let socketPath = "OPENCLAW_CUA_DRIVER_SOCKET_PATH" + static let binaryPath = "OPENCLAW_CUA_DRIVER_BINARY_PATH" +} + +enum CuaDriverArtifact { + static let resourceName = "cua-driver" + + static var bundledExecutableURL: URL? { + self.executableURL(in: Bundle.main.resourceURL) + } + + static func executableURL( + in resourceURL: URL?, + fileManager: FileManager = .default) -> URL? + { + guard let resourceURL else { return nil } + let candidate = resourceURL.appendingPathComponent(self.resourceName, isDirectory: false) + guard let values = try? candidate.resourceValues(forKeys: [ + .isRegularFileKey, + .isSymbolicLinkKey, + ]), + values.isRegularFile == true, + values.isSymbolicLink != true, + fileManager.isExecutableFile(atPath: candidate.path) + else { return nil } + return candidate + } +} diff --git a/apps/macos/Sources/OpenClaw/Constants.swift b/apps/macos/Sources/OpenClaw/Constants.swift index 3275ae3507cf..d9543d1a9e24 100644 --- a/apps/macos/Sources/OpenClaw/Constants.swift +++ b/apps/macos/Sources/OpenClaw/Constants.swift @@ -43,6 +43,7 @@ let canvasEnabledKey = "openclaw.canvasEnabled" let quickChatEnabledKey = "openclaw.quickChatEnabled" let cameraEnabledKey = "openclaw.cameraEnabled" let computerControlEnabledKey = "openclaw.computerControlEnabled" +let computerControlProviderKey = "openclaw.computerControlProvider" let cookieSyncEnabledKey = "openclaw.cookieSyncEnabled" let cookieSyncIntoProfileKey = "openclaw.cookieSyncIntoProfile" let cookieSyncDomainsKey = "openclaw.cookieSyncDomains" diff --git a/apps/macos/Sources/OpenClaw/CuaDriverHostCoordinator.swift b/apps/macos/Sources/OpenClaw/CuaDriverHostCoordinator.swift new file mode 100644 index 000000000000..e2d0075ebc78 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CuaDriverHostCoordinator.swift @@ -0,0 +1,686 @@ +import AppKit +import Darwin +import Foundation +import OpenClawIPC +import OSLog + +extension Notification.Name { + static let openclawCuaDriverAvailabilityChanged = Notification.Name( + "openclaw.cua-driver.availability-changed") +} + +struct CuaDriverProcessLaunch: Sendable { + let executableURL: URL + let arguments: [String] + let environment: [String: String] +} + +enum CuaDriverStderrEvent: Equatable, Sendable { + case notice(String) + case error(String) +} + +final class CuaDriverStderrRelay: @unchecked Sendable { + static let managedModeNotice = + """ + CUA embedded driver running in managed unrestricted mode; \ + OpenClaw command arming and pairing are the authorization boundary. + """ + + private static let dangerBannerPrefix = "DANGER: Cua Driver is running in unrestricted mode" + private static let maximumBufferedBytes = 32 * 1024 + private static let readChunkBytes = 4 * 1024 + + let pipe = Pipe() + + private let lock = NSLock() + private let emit: @Sendable (CuaDriverStderrEvent) -> Void + private var buffer = Data() + private var started = false + private var stopped = false + private var emittedManagedModeNotice = false + + init(emit: @escaping @Sendable (CuaDriverStderrEvent) -> Void) { + self.emit = emit + } + + func startReading() { + let shouldStart = self.lock.withLock { + guard !self.started, !self.stopped else { return false } + self.started = true + return true + } + guard shouldStart else { return } + self.pipe.fileHandleForReading.readabilityHandler = { [weak self] handle in + guard let self else { return } + let data = handle.readSafely(upToCount: Self.readChunkBytes) + guard !data.isEmpty else { + self.stop() + return + } + self.consume(data) + } + } + + func reportManagedMode() { + let shouldEmit = self.lock.withLock { + guard !self.stopped, !self.emittedManagedModeNotice else { return false } + self.emittedManagedModeNotice = true + return true + } + if shouldEmit { + self.emit(.notice(Self.managedModeNotice)) + } + } + + func stop() { + let tail = self.lock.withLock { () -> Data? in + guard !self.stopped else { return nil } + self.stopped = true + defer { self.buffer.removeAll(keepingCapacity: false) } + return self.buffer.isEmpty ? nil : self.buffer + } + self.pipe.fileHandleForReading.readabilityHandler = nil + try? self.pipe.fileHandleForReading.close() + try? self.pipe.fileHandleForWriting.close() + if let tail { + self.forward(tail) + } + } + + private func consume(_ data: Data) { + let lines = self.lock.withLock { () -> [Data] in + guard !self.stopped else { return [] } + self.buffer.append(data) + if self.buffer.count > Self.maximumBufferedBytes { + self.buffer = Data(self.buffer.suffix(Self.maximumBufferedBytes)) + } + var lines: [Data] = [] + while let newline = self.buffer.firstIndex(of: 0x0A) { + lines.append(Data(self.buffer[.. Void) throws -> any CuaDriverProcessControlling + typealias ReadinessProbe = @Sendable (String) async -> Bool + + static let shared = CuaDriverHostCoordinator( + observeNotifications: true, + beforeDaemonStop: { + await MacNodeModeCoordinator.shared.prepareForCuaDaemonStop() + }) + + private static let maximumRestartAttempts = 5 + private static let restartDelays: [Duration] = [ + .seconds(1), + .seconds(2), + .seconds(4), + .seconds(8), + .seconds(10), + ] + + private struct RunningChild { + let generation: UInt64 + let process: any CuaDriverProcessControlling + let socketDirectory: CuaDriverSocketDirectory + let executableURL: URL + } + + private let logger = Logger(subsystem: "ai.openclaw", category: "cua-driver-host") + private let notificationCenter: NotificationCenter + private let artifactURL: @MainActor () -> URL? + private let applicationSupportURL: @MainActor () -> URL + private let bundleIdentifier: @MainActor () -> String? + private let processLauncher: ProcessLauncher + private let readinessProbe: ReadinessProbe + private let restartSleep: @Sendable (Duration) async -> Void + private let permissionSnapshot: @MainActor () async -> [Capability: CapabilityAuthorizationStatus] + private let beforeDaemonStop: @MainActor () async -> Void + + private var desiredEnabled = false + private var runningChild: RunningChild? + private var readyEndpoint: CuaDriverWorkerEndpoint? + private var generation: UInt64 = 0 + private var stoppingGenerations = Set() + private var restartAttempt = 0 + private var restartTask: Task? + private var reconciliationTail: Task? + private var lastPermissionSnapshot: [Capability: CapabilityAuthorizationStatus]? + + init( + notificationCenter: NotificationCenter = .default, + observeNotifications: Bool = false, + artifactURL: @escaping @MainActor () -> URL? = { CuaDriverArtifact.bundledExecutableURL }, + applicationSupportURL: @escaping @MainActor () -> URL = { + let fileManager = FileManager.default + return fileManager.urls(for: .applicationSupportDirectory, in: .userDomainMask).first + ?? fileManager.homeDirectoryForCurrentUser.appendingPathComponent("Library/Application Support") + }, + bundleIdentifier: @escaping @MainActor () -> String? = { Bundle.main.bundleIdentifier }, + processLauncher: @escaping ProcessLauncher = CuaDriverHostCoordinator.launchProcess, + readinessProbe: @escaping ReadinessProbe = { path in + await Task.detached { CuaDriverHostCoordinator.socketAcceptsConnections(path) }.value + }, + restartSleep: @escaping @Sendable (Duration) async -> Void = { delay in + try? await Task.sleep(for: delay) + }, + permissionSnapshot: @escaping @MainActor () async -> [Capability: CapabilityAuthorizationStatus] = { + await PermissionManager.authorizationStatus([.accessibility, .screenRecording]) + }, + beforeDaemonStop: @escaping @MainActor () async -> Void = {}) + { + self.notificationCenter = notificationCenter + self.artifactURL = artifactURL + self.applicationSupportURL = applicationSupportURL + self.bundleIdentifier = bundleIdentifier + self.processLauncher = processLauncher + self.readinessProbe = readinessProbe + self.restartSleep = restartSleep + self.permissionSnapshot = permissionSnapshot + self.beforeDaemonStop = beforeDaemonStop + + guard observeNotifications else { return } + notificationCenter.addObserver( + self, + selector: #selector(self.permissionsMayHaveChanged), + name: .openclawPermissionsChanged, + object: nil) + notificationCenter.addObserver( + self, + selector: #selector(self.permissionsMayHaveChanged), + name: NSApplication.didBecomeActiveNotification, + object: nil) + } + + deinit { + self.restartTask?.cancel() + self.reconciliationTail?.cancel() + self.notificationCenter.removeObserver(self) + } + + var workerEndpoint: CuaDriverWorkerEndpoint? { + self.readyEndpoint + } + + func setEnabled(_ enabled: Bool) async { + let wasEnabled = self.desiredEnabled + self.desiredEnabled = enabled + if enabled, !wasEnabled { + self.restartAttempt = 0 + } + if !enabled { + self.restartTask?.cancel() + self.restartTask = nil + self.restartAttempt = 0 + } + await self.enqueueReconciliation(restart: false).value + } + + func shutdown() async { + await self.setEnabled(false) + } + + private func enqueueReconciliation(restart: Bool) -> Task { + let predecessor = self.reconciliationTail + let task = Task { @MainActor [weak self] in + await predecessor?.value + guard let self else { return } + if restart { + await self.ensureStopped() + } + if self.desiredEnabled { + await self.ensureStarted() + } else { + await self.ensureStopped() + } + } + self.reconciliationTail = task + return task + } + + private func ensureStarted() async { + guard self.runningChild == nil else { return } + let applicationSupportURL = self.applicationSupportURL() + let executableURL = self.artifactURL() + await Self.reapStaleSocketDirectories( + in: applicationSupportURL, + expectedExecutableURL: executableURL) + guard let executableURL else { + self.logger.info("embedded CUA remains unavailable because the driver is not bundled") + return + } + guard let hostBundleID = self.bundleIdentifier()?.trimmingCharacters(in: .whitespacesAndNewlines), + !hostBundleID.isEmpty + else { + self.logger.error("embedded CUA cannot start without a host bundle identifier") + return + } + + let socketDirectory: CuaDriverSocketDirectory + do { + socketDirectory = try Self.createSocketDirectory(in: applicationSupportURL) + } catch { + self.logger.error("\(error.localizedDescription, privacy: .public)") + self.scheduleRestartIfNeeded() + return + } + + self.generation &+= 1 + let generation = self.generation + let launch = Self.makeProcessLaunch( + executableURL: executableURL, + socketPath: socketDirectory.socketPath, + hostBundleID: hostBundleID) + do { + let process = try self.processLauncher(launch) { [weak self] status in + Task { @MainActor [weak self] in + self?.processExited(generation: generation, status: status) + } + } + // Record the pid we spawned so startup/teardown reaping can attribute and + // terminate exactly this daemon; without it a reaper can only delete the + // directory and would leave an orphaned privileged process running. + Self.writeProcessIdentifier(process.processIdentifier, to: socketDirectory) + self.runningChild = RunningChild( + generation: generation, + process: process, + socketDirectory: socketDirectory, + executableURL: executableURL) + } catch { + Self.cleanupSocketDirectory(socketDirectory) + self.logger.error("embedded CUA launch failed: \(error.localizedDescription, privacy: .public)") + self.scheduleRestartIfNeeded() + return + } + + let deadline = ContinuousClock.now + .seconds(10) + while ContinuousClock.now < deadline { + guard self.desiredEnabled, + let child = self.runningChild, + child.generation == generation, + child.process.isRunning + else { + await self.ensureStopped() + return + } + if await self.readinessProbe(socketDirectory.socketPath) { + self.lastPermissionSnapshot = await self.permissionSnapshot() + self.setReadyEndpoint(CuaDriverWorkerEndpoint( + socketPath: socketDirectory.socketPath, + binaryPath: executableURL.path)) + self.logger.info("embedded CUA ready at \(socketDirectory.socketPath, privacy: .public)") + return + } + try? await Task.sleep(for: .milliseconds(50)) + } + + self.logger.error("embedded CUA startup timed out") + await self.ensureStopped() + self.scheduleRestartIfNeeded() + } + + private func ensureStopped() async { + self.setReadyEndpoint(nil) + let applicationSupportURL = self.applicationSupportURL() + guard let child = self.runningChild else { + await Self.reapStaleSocketDirectories( + in: applicationSupportURL, + expectedExecutableURL: self.artifactURL()) + return + } + // The worker owns the MCP proxy. Drain it before the app closes the + // privileged daemon so an execution can never cross generations. + await self.beforeDaemonStop() + self.stoppingGenerations.insert(child.generation) + child.process.closeLiveness() + await Self.waitUntilStopped(child.process, timeout: .seconds(2)) + if child.process.isRunning { + child.process.terminate() + await Self.waitUntilStopped(child.process, timeout: .seconds(1)) + } + if child.process.isRunning { + child.process.forceKill() + await Self.waitUntilStopped(child.process, timeout: .seconds(1)) + } + if self.runningChild?.generation == child.generation { + self.runningChild = nil + } + self.stoppingGenerations.remove(child.generation) + if !child.process.isRunning { + Self.cleanupSocketDirectory(child.socketDirectory) + } + await Self.reapStaleSocketDirectories( + in: applicationSupportURL, + expectedExecutableURL: child.executableURL) + } + + private func processExited(generation: UInt64, status: Int32) { + guard let child = self.runningChild, child.generation == generation else { return } + let expected = self.stoppingGenerations.contains(generation) || !self.desiredEnabled + self.setReadyEndpoint(nil) + child.process.closeLiveness() + self.runningChild = nil + Self.cleanupSocketDirectory(child.socketDirectory) + if expected { + self.stoppingGenerations.remove(generation) + return + } + self.logger.error("embedded CUA exited unexpectedly with status \(status, privacy: .public)") + self.scheduleRestartIfNeeded() + } + + private func scheduleRestartIfNeeded() { + guard self.desiredEnabled, + self.restartTask == nil, + self.restartAttempt < Self.maximumRestartAttempts + else { return } + let delay = Self.restartDelays[self.restartAttempt] + self.restartAttempt += 1 + let restartSleep = self.restartSleep + self.restartTask = Task { @MainActor [weak self] in + await restartSleep(delay) + guard !Task.isCancelled, let self, self.desiredEnabled else { return } + self.restartTask = nil + await self.enqueueReconciliation(restart: false).value + } + } + + private func setReadyEndpoint(_ endpoint: CuaDriverWorkerEndpoint?) { + guard self.readyEndpoint != endpoint else { return } + self.readyEndpoint = endpoint + self.notificationCenter.post(name: .openclawCuaDriverAvailabilityChanged, object: nil) + } + + @objc private nonisolated func permissionsMayHaveChanged(_: Notification) { + Task { @MainActor [weak self] in + await self?.restartAfterPermissionChangeIfNeeded() + } + } + + private func restartAfterPermissionChangeIfNeeded() async { + let latest = await self.permissionSnapshot() + guard let previous = self.lastPermissionSnapshot else { + self.lastPermissionSnapshot = latest + return + } + guard latest != previous else { return } + self.lastPermissionSnapshot = latest + guard self.desiredEnabled, self.runningChild != nil else { return } + self.restartTask?.cancel() + self.restartTask = nil + await self.enqueueReconciliation(restart: true).value + } + + static func makeProcessLaunch( + executableURL: URL, + socketPath: String, + hostBundleID: String, + inheritedEnvironment: [String: String] = ProcessInfo.processInfo.environment) -> CuaDriverProcessLaunch + { + var environment = inheritedEnvironment.filter { key, _ in + !key.hasPrefix("CUA_DRIVER_") && key != "CUA_TELEMETRY_ENABLED" + } + environment["CUA_DRIVER_EMBEDDED"] = "1" + environment["CUA_DRIVER_HOST_BUNDLE_ID"] = hostBundleID + environment["CUA_DRIVER_PERMISSION_MODE"] = "unrestricted" + environment["CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS"] = "1" + environment["CUA_DRIVER_RS_TELEMETRY_ENABLED"] = "false" + environment["CUA_DRIVER_RS_UPDATE_CHECK"] = "false" + environment["CUA_DRIVER_EMBEDDED_HOST_PID"] = String(ProcessInfo.processInfo.processIdentifier) + // Unrestricted is deliberate: CUA bounded mode accepts only exact launch-time resource grants + // (cua-driver-core/src/session_manifest.rs), not arbitrary runtime-discovered windows/elements. + // OpenClaw command arming, pairing, and tool policy own authorization upstream, matching the + // shipped Peekaboo fulfiller; the owner-only 0700 socket directory is the local trust boundary. + return CuaDriverProcessLaunch( + executableURL: executableURL, + arguments: [ + "serve", + "--embedded", + "--parent-liveness-stdio", + "--no-permissions-gate", + "--socket", + socketPath, + // No --pid-file: `serve` ignores it and always writes the driver's + // global default path, which every cua-driver on the machine shares. + // OpenClaw records the spawned pid itself so reaping can attribute + // exactly the daemon this app owns. + "--host-bundle-id", + hostBundleID, + "--permission-mode", + "unrestricted", + "--dangerously-bypass-approvals", + ], + environment: environment) + } + + private static func launchProcess( + _ launch: CuaDriverProcessLaunch, + onTermination: @escaping @Sendable (Int32) -> Void) throws -> any CuaDriverProcessControlling + { + let process = Process() + let livenessPipe = try Self.makeLivenessPipe() + let logger = Logger(subsystem: "ai.openclaw", category: "cua-driver-host") + let stderrRelay = CuaDriverStderrRelay { event in + switch event { + case let .notice(message): + logger.notice("\(message, privacy: .public)") + case let .error(message): + logger.error("CUA driver stderr: \(message, privacy: .public)") + } + } + process.executableURL = launch.executableURL + process.arguments = launch.arguments + process.environment = launch.environment + process.standardInput = livenessPipe.fileHandleForReading + process.standardOutput = FileHandle.nullDevice + process.standardError = stderrRelay.pipe + process.terminationHandler = { terminated in + stderrRelay.stop() + onTermination(terminated.terminationStatus) + } + stderrRelay.startReading() + do { + try process.run() + } catch { + stderrRelay.stop() + throw error + } + stderrRelay.reportManagedMode() + return FoundationCuaDriverProcess( + process: process, + livenessPipe: livenessPipe, + stderrRelay: stderrRelay) + } + + private static func waitUntilStopped( + _ process: any CuaDriverProcessControlling, + timeout: Duration) async + { + let deadline = ContinuousClock.now + timeout + while process.isRunning, ContinuousClock.now < deadline { + try? await Task.sleep(for: .milliseconds(25)) + } + } + + static func createSocketDirectory(in applicationSupportURL: URL) throws -> CuaDriverSocketDirectory { + let openClawRoot = applicationSupportURL.appendingPathComponent("OpenClaw", isDirectory: true) + let root = openClawRoot.appendingPathComponent("cua", isDirectory: true) + for directory in [applicationSupportURL, openClawRoot, root] { + var status = stat() + if lstat(directory.path, &status) != 0 { + guard errno == ENOENT, Darwin.mkdir(directory.path, 0o700) == 0 else { + throw CuaDriverHostError.socketDirectory(String(cString: strerror(errno))) + } + guard lstat(directory.path, &status) == 0 else { + throw CuaDriverHostError.socketDirectory(String(cString: strerror(errno))) + } + } + guard status.st_mode & mode_t(S_IFMT) == mode_t(S_IFDIR), + status.st_uid == geteuid() + else { + throw CuaDriverHostError.socketDirectory("support roots must be owned directories") + } + } + + for _ in 0..<8 { + // Keep the endpoint well below sockaddr_un.sun_path even for long account names. + let token = UUID().uuidString.replacingOccurrences(of: "-", with: "").lowercased().prefix(16) + let directory = root.appendingPathComponent(String(token), isDirectory: true) + guard Darwin.mkdir(directory.path, 0o700) == 0 else { + if errno == EEXIST { continue } + throw CuaDriverHostError.socketDirectory(String(cString: strerror(errno))) + } + var status = stat() + guard lstat(directory.path, &status) == 0, + status.st_mode & mode_t(S_IFMT) == mode_t(S_IFDIR), + status.st_uid == geteuid(), + status.st_mode & 0o777 == 0o700 + else { + _ = Darwin.rmdir(directory.path) + throw CuaDriverHostError.socketDirectory("created directory failed ownership checks") + } + let socketPath = directory.appendingPathComponent("cua.sock", isDirectory: false).path + guard socketPath.utf8.count < MemoryLayout.size(ofValue: sockaddr_un().sun_path) else { + _ = Darwin.rmdir(directory.path) + throw CuaDriverHostError.socketPathTooLong + } + var socketStatus = stat() + guard lstat(socketPath, &socketStatus) != 0, errno == ENOENT else { + _ = Darwin.rmdir(directory.path) + throw CuaDriverHostError.socketDirectory("socket path already exists") + } + return CuaDriverSocketDirectory( + url: directory, + socketPath: socketPath, + device: UInt64(status.st_dev), + inode: UInt64(status.st_ino)) + } + throw CuaDriverHostError.socketDirectory("could not allocate a unique directory") + } + + static func cleanupSocketDirectory(_ directory: CuaDriverSocketDirectory) { + var status = stat() + guard lstat(directory.url.path, &status) == 0, + status.st_mode & mode_t(S_IFMT) == mode_t(S_IFDIR), + status.st_uid == geteuid(), + UInt64(status.st_dev) == directory.device, + UInt64(status.st_ino) == directory.inode + else { return } + + var socketStatus = stat() + if lstat(directory.socketPath, &socketStatus) == 0, + socketStatus.st_mode & mode_t(S_IFMT) == mode_t(S_IFSOCK), + socketStatus.st_uid == geteuid() + { + _ = Darwin.unlink(directory.socketPath) + } + var pidStatus = stat() + if lstat(directory.pidFilePath, &pidStatus) == 0, + pidStatus.st_mode & mode_t(S_IFMT) == mode_t(S_IFREG), + pidStatus.st_uid == geteuid() + { + _ = Darwin.unlink(directory.pidFilePath) + } + _ = Darwin.rmdir(directory.url.path) + } + + nonisolated static func socketAcceptsConnections(_ socketPath: String) -> Bool { + guard let descriptor = self.connectUnixSocket(socketPath) else { return false } + defer { close(descriptor) } + return true + } +} diff --git a/apps/macos/Sources/OpenClaw/CuaDriverHostLifecycle.swift b/apps/macos/Sources/OpenClaw/CuaDriverHostLifecycle.swift new file mode 100644 index 000000000000..799aba8b2ea4 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/CuaDriverHostLifecycle.swift @@ -0,0 +1,314 @@ +import Darwin +import Foundation +import OSLog + +extension CuaDriverHostCoordinator { + static func makeLivenessPipe() throws -> Pipe { + let pipe = Pipe() + let descriptor = pipe.fileHandleForWriting.fileDescriptor + let flags = fcntl(descriptor, F_GETFD) + guard flags >= 0, fcntl(descriptor, F_SETFD, flags | FD_CLOEXEC) >= 0 else { + let code = errno + try? pipe.fileHandleForReading.close() + try? pipe.fileHandleForWriting.close() + throw NSError(domain: NSPOSIXErrorDomain, code: Int(code)) + } + return pipe + } + + static func reapStaleSocketDirectories( + in applicationSupportURL: URL, + expectedExecutableURL: URL?) async + { + let logger = Logger(subsystem: "ai.openclaw", category: "cua-driver-host") + for directory in self.ownedSocketDirectories(in: applicationSupportURL) { + guard let processIdentifier = self.readProcessIdentifier(in: directory) else { + self.cleanupSocketDirectory(directory) + continue + } + guard self.processIsAlive(processIdentifier) else { + self.cleanupSocketDirectory(directory) + continue + } + guard let expectedExecutableURL, + self.processExecutableMatches( + processIdentifier, + expectedExecutableURL: expectedExecutableURL) + else { continue } + + if let hostPID = self.processEnvironmentValue( + processIdentifier, + key: "CUA_DRIVER_EMBEDDED_HOST_PID").flatMap(pid_t.init), + !self.processIsAlive(hostPID) + { + logger.error( + """ + reaping orphaned embedded CUA daemon \(processIdentifier, privacy: .public) \ + whose host \(hostPID, privacy: .public) is gone + """) + } else { + logger.error( + "reaping owned embedded CUA daemon \(processIdentifier, privacy: .public) during lifecycle cleanup") + } + if await self.terminateProcess( + processIdentifier, + directory: directory, + expectedExecutableURL: expectedExecutableURL) + { + self.cleanupSocketDirectory(directory) + } + } + } + + private static func ownedSocketDirectories( + in applicationSupportURL: URL) -> [CuaDriverSocketDirectory] + { + let openClawRoot = applicationSupportURL.appendingPathComponent("OpenClaw", isDirectory: true) + let root = openClawRoot.appendingPathComponent("cua", isDirectory: true) + for ancestor in [applicationSupportURL, openClawRoot, root] { + var status = stat() + guard lstat(ancestor.path, &status) == 0, + status.st_mode & mode_t(S_IFMT) == mode_t(S_IFDIR), + status.st_uid == geteuid() + else { return [] } + } + guard let children = try? FileManager.default.contentsOfDirectory( + at: root, + includingPropertiesForKeys: nil, + options: [.skipsHiddenFiles]) + else { return [] } + + return children.compactMap { child in + let name = child.lastPathComponent + guard name.utf8.count == 16, + name.utf8.allSatisfy({ (48...57).contains($0) || (97...102).contains($0) }) + else { return nil } + var status = stat() + guard lstat(child.path, &status) == 0, + status.st_mode & mode_t(S_IFMT) == mode_t(S_IFDIR), + status.st_uid == geteuid(), + status.st_mode & 0o777 == 0o700 + else { return nil } + return CuaDriverSocketDirectory( + url: child, + socketPath: child.appendingPathComponent("cua.sock").path, + device: UInt64(status.st_dev), + inode: UInt64(status.st_ino)) + } + } + + /// Records the spawned daemon pid inside its own owner-only socket directory. + /// `serve` ignores `--pid-file` and writes only a machine-global path shared by + /// every cua-driver, so the spawning app is the one authoritative source. + @discardableResult + static func writeProcessIdentifier( + _ processIdentifier: pid_t, + to directory: CuaDriverSocketDirectory) -> Bool + { + guard self.directoryIsUnchangedAndOwned(directory) else { return false } + let descriptor = Darwin.open( + directory.pidFilePath, + O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, + 0o600) + guard descriptor >= 0 else { return false } + defer { close(descriptor) } + let contents = Array("\(processIdentifier)".utf8) + return contents.withUnsafeBytes { bytes in + Darwin.write(descriptor, bytes.baseAddress, bytes.count) == bytes.count + } + } + + private static func readProcessIdentifier(in directory: CuaDriverSocketDirectory) -> pid_t? { + guard self.directoryIsUnchangedAndOwned(directory) else { return nil } + let descriptor = Darwin.open(directory.pidFilePath, O_RDONLY | O_CLOEXEC | O_NOFOLLOW) + guard descriptor >= 0 else { return nil } + defer { close(descriptor) } + + var status = stat() + guard fstat(descriptor, &status) == 0, + status.st_mode & mode_t(S_IFMT) == mode_t(S_IFREG), + status.st_uid == geteuid(), + status.st_size > 0, + status.st_size <= 32 + else { return nil } + var buffer = [UInt8](repeating: 0, count: Int(status.st_size)) + let count = buffer.withUnsafeMutableBytes { bytes in + Darwin.read(descriptor, bytes.baseAddress, bytes.count) + } + guard count == buffer.count, + let contents = String(bytes: buffer, encoding: .utf8), + let processIdentifier = pid_t(contents.trimmingCharacters(in: .whitespacesAndNewlines)), + processIdentifier > 1, + processIdentifier != getpid() + else { return nil } + return processIdentifier + } + + private static func directoryIsUnchangedAndOwned(_ directory: CuaDriverSocketDirectory) -> Bool { + var status = stat() + return lstat(directory.url.path, &status) == 0 && + status.st_mode & mode_t(S_IFMT) == mode_t(S_IFDIR) && + status.st_uid == geteuid() && + status.st_mode & 0o777 == 0o700 && + UInt64(status.st_dev) == directory.device && + UInt64(status.st_ino) == directory.inode + } + + nonisolated static func connectUnixSocket(_ socketPath: String) -> Int32? { + let descriptor = socket(AF_UNIX, SOCK_STREAM, 0) + guard descriptor >= 0 else { return nil } + var address = sockaddr_un() + address.sun_family = sa_family_t(AF_UNIX) + let maximumLength = MemoryLayout.size(ofValue: address.sun_path) + guard socketPath.utf8.count < maximumLength else { + close(descriptor) + return nil + } + socketPath.withCString { source in + withUnsafeMutablePointer(to: &address.sun_path) { pointer in + let bytes = UnsafeMutableRawPointer(pointer).assumingMemoryBound(to: Int8.self) + memset(bytes, 0, maximumLength) + strncpy(bytes, source, maximumLength - 1) + } + } + let addressSize = socklen_t(MemoryLayout.size(ofValue: address)) + let connected = withUnsafePointer(to: &address) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { rebound in + connect(descriptor, rebound, addressSize) == 0 + } + } + guard connected else { + close(descriptor) + return nil + } + return descriptor + } + + private static func processIsAlive(_ processIdentifier: pid_t) -> Bool { + guard processIdentifier > 1 else { return false } + if Darwin.kill(processIdentifier, 0) == 0 { return true } + return errno == EPERM + } + + private static func processExecutableURL(_ processIdentifier: pid_t) -> URL? { + var buffer = [CChar](repeating: 0, count: Int(PATH_MAX)) + let length = proc_pidpath(processIdentifier, &buffer, UInt32(buffer.count)) + guard length > 0 else { return nil } + let bytes = buffer.prefix { $0 != 0 }.map { UInt8(bitPattern: $0) } + guard let path = String(bytes: bytes, encoding: .utf8) else { return nil } + return URL(fileURLWithPath: path) + } + + private static func processExecutableMatches( + _ processIdentifier: pid_t, + expectedExecutableURL: URL) -> Bool + { + guard let actualExecutableURL = self.processExecutableURL(processIdentifier) else { return false } + let actualPath = actualExecutableURL.resolvingSymlinksInPath().standardizedFileURL.path + let expectedPath = expectedExecutableURL.resolvingSymlinksInPath().standardizedFileURL.path + return actualPath == expectedPath + } + + private static func processEnvironmentValue( + _ processIdentifier: pid_t, + key: String) -> String? + { + var argumentMaximum: Int32 = 0 + var argumentMaximumSize = MemoryLayout.size + var argumentMaximumMIB: [Int32] = [CTL_KERN, KERN_ARGMAX] + guard sysctl( + &argumentMaximumMIB, + u_int(argumentMaximumMIB.count), + &argumentMaximum, + &argumentMaximumSize, + nil, + 0) == 0, + argumentMaximum > 0, + argumentMaximum <= 4 * 1024 * 1024 + else { return nil } + + var buffer = [UInt8](repeating: 0, count: Int(argumentMaximum)) + var bufferSize = buffer.count + var processMIB: [Int32] = [CTL_KERN, KERN_PROCARGS2, processIdentifier] + let readSucceeded = buffer.withUnsafeMutableBytes { bytes in + sysctl( + &processMIB, + u_int(processMIB.count), + bytes.baseAddress, + &bufferSize, + nil, + 0) == 0 + } + guard readSucceeded, bufferSize >= MemoryLayout.size else { return nil } + + var argumentCount: Int32 = 0 + withUnsafeMutableBytes(of: &argumentCount) { destination in + destination.copyBytes(from: buffer.prefix(destination.count)) + } + guard argumentCount > 0 else { return nil } + + var offset = MemoryLayout.size + func skipString() -> Bool { + guard offset < bufferSize else { return false } + while offset < bufferSize, buffer[offset] != 0 { + offset += 1 + } + guard offset < bufferSize else { return false } + offset += 1 + return true + } + + guard skipString() else { return nil } + while offset < bufferSize, buffer[offset] == 0 { + offset += 1 + } + for _ in 0.. Bool + { + guard self.readProcessIdentifier(in: directory) == processIdentifier, + self.processExecutableMatches( + processIdentifier, + expectedExecutableURL: expectedExecutableURL) + else { return false } + if Darwin.kill(processIdentifier, SIGTERM) != 0, errno != ESRCH { return false } + if await self.waitForProcessExit(processIdentifier) { return true } + // Recheck the executable immediately before escalation so PID reuse can + // never redirect SIGKILL to an unrelated process. + guard self.processExecutableMatches( + processIdentifier, + expectedExecutableURL: expectedExecutableURL) + else { return false } + if Darwin.kill(processIdentifier, SIGKILL) != 0, errno != ESRCH { return false } + return await self.waitForProcessExit(processIdentifier) + } + + private static func waitForProcessExit(_ processIdentifier: pid_t) async -> Bool { + let deadline = ContinuousClock.now + .seconds(1) + while self.processIsAlive(processIdentifier), ContinuousClock.now < deadline { + try? await Task.sleep(for: .milliseconds(25)) + } + return !self.processIsAlive(processIdentifier) + } +} diff --git a/apps/macos/Sources/OpenClaw/GeneralSettings.swift b/apps/macos/Sources/OpenClaw/GeneralSettings.swift index 6f19e67db2bc..22c7497dfa67 100644 --- a/apps/macos/Sources/OpenClaw/GeneralSettings.swift +++ b/apps/macos/Sources/OpenClaw/GeneralSettings.swift @@ -16,7 +16,10 @@ struct GeneralSettings: View { @Bindable var state: AppState @AppStorage(cameraEnabledKey) private var cameraEnabled: Bool = false - @AppStorage(computerControlEnabledKey) private var computerControlEnabled: Bool = true + @AppStorage(computerControlEnabledKey, store: AppDefaults.standard) + private var computerControlEnabled: Bool = true + @AppStorage(computerControlProviderKey, store: AppDefaults.standard) + private var computerControlProviderRaw: String = ComputerControlProvider.peekaboo.rawValue let page: Page let isActive: Bool private let healthStore = HealthStore.shared @@ -66,8 +69,10 @@ struct GeneralSettings: View { QuickChatController.shared.setEnabled(enabled) } .onChange(of: self.computerControlEnabled) { _, _ in - // Turning Computer Control on/off must start or stop the gated PeekabooBridge host. - self.state.applyPeekabooBridgeHostState() + self.state.applyComputerControlHostState() + } + .onChange(of: self.computerControlProviderRaw) { _, _ in + self.state.applyComputerControlHostState() } .onReceive(NotificationCenter.default.publisher(for: NSApplication.didBecomeActiveNotification)) { _ in self.refreshComputerControlPermissions() @@ -141,6 +146,8 @@ struct GeneralSettings: View { """, binding: self.$computerControlEnabled) + self.computerControlProviderRow + SettingsCardRow( title: "Computer Control access", subtitle: .verbatim(self.computerControlPermissions.diagnostic.detailText)) @@ -874,6 +881,43 @@ struct GeneralSettings: View { } extension GeneralSettings { + @ViewBuilder + private var computerControlProviderRow: some View { + if self.computerControlEnabled { + SettingsCardRow( + title: "Computer Control provider", + subtitle: "Choose the node-local automation backend for snapshots and actions.") + { + Picker("Computer Control provider", selection: self.computerControlProviderBinding) { + Text(ComputerControlProvider.peekaboo.displayName) + .tag(ComputerControlProvider.peekaboo) + Text(self.cuaDriverBundled ? "CUA" : "CUA (driver not bundled)") + .tag(ComputerControlProvider.cua) + .disabled(!self.cuaDriverBundled) + } + .labelsHidden() + .pickerStyle(.menu) + .frame(width: 220, alignment: .trailing) + } + } + } + + private var cuaDriverBundled: Bool { + CuaDriverArtifact.bundledExecutableURL != nil + } + + private var computerControlProviderBinding: Binding { + Binding( + get: { + let selected = ComputerControlProvider(rawValue: self.computerControlProviderRaw) ?? .peekaboo + return selected == .cua && !self.cuaDriverBundled ? .peekaboo : selected + }, + set: { provider in + guard provider != .cua || self.cuaDriverBundled else { return } + self.computerControlProviderRaw = provider.rawValue + }) + } + private var cookieSyncStatusTitle: String { switch self.cookieSyncManager.state { case .stopped: "Stopped" diff --git a/apps/macos/Sources/OpenClaw/MenuBar.swift b/apps/macos/Sources/OpenClaw/MenuBar.swift index 1cb7a8defaea..cecb37b3eeba 100644 --- a/apps/macos/Sources/OpenClaw/MenuBar.swift +++ b/apps/macos/Sources/OpenClaw/MenuBar.swift @@ -408,6 +408,9 @@ final class AppDelegate: NSObject, NSApplicationDelegate { var nodeTerminationCleanup: @MainActor () async -> Void = { await TalkMLXSpeechSynthesizer.shared.shutdown() await MacNodeModeCoordinator.shared.stopAndWait() + // The worker owns the MCP proxy. Stop it before closing the app-owned + // daemon socket so an in-flight completion cannot be mistaken for retryable. + await CuaDriverHostCoordinator.shared.shutdown() } var peekabooBridgeTerminationCleanup: @MainActor () async -> Void = { @@ -625,7 +628,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate { Task { PresenceReporter.shared.start() } Task { await HealthStore.shared.refresh(onDemand: true) } Task { await PortGuardian.shared.sweep(mode: AppStateStore.shared.connectionMode) } - AppStateStore.shared.applyPeekabooBridgeHostState() + AppStateStore.shared.applyComputerControlHostState() if launchPlan.allowsAutomaticPresentation { DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { if !PostUpdateController.shared.startIfNeeded() { diff --git a/apps/macos/Sources/OpenClaw/NodeMode/MacNodeHostWorker.swift b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeHostWorker.swift index 64be8f534349..f0395e429c39 100644 --- a/apps/macos/Sources/OpenClaw/NodeMode/MacNodeHostWorker.swift +++ b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeHostWorker.swift @@ -1,6 +1,7 @@ import Darwin import Foundation import OpenClawKit +import OpenClawProtocol import OSLog import Subprocess @@ -14,16 +15,37 @@ struct MacNodeHostManifest: Equatable, Sendable { let version: String let caps: [String] let commands: [String] + let computerUse: AnyCodable? let pathEnv: String + + init( + version: String, + caps: [String], + commands: [String], + computerUse: AnyCodable? = nil, + pathEnv: String) + { + self.version = version + self.caps = caps + self.commands = commands + self.computerUse = computerUse + self.pathEnv = pathEnv + } } struct MacNodeHostWorkerLaunch: Equatable, Sendable { let command: [String] let currentDirectoryURL: URL? + let environment: [String: String] - init(command: [String], currentDirectoryURL: URL? = nil) { + init( + command: [String], + currentDirectoryURL: URL? = nil, + environment: [String: String] = [:]) + { self.command = command self.currentDirectoryURL = currentDirectoryURL + self.environment = environment } } @@ -321,6 +343,9 @@ final class MacNodeHostWorker: MacNodeHostWorking, @unchecked Sendable { return } var environment = ProcessInfo.processInfo.environment + environment.removeValue(forKey: CuaDriverWorkerEnvironment.socketPath) + environment.removeValue(forKey: CuaDriverWorkerEnvironment.binaryPath) + environment.merge(launch.environment, uniquingKeysWith: { _, explicit in explicit }) environment["PATH"] = CommandResolver.preferredPaths().joined(separator: ":") environment["OPENCLAW_NODE_EXEC_HOST"] = "app" environment["OPENCLAW_NODE_EXEC_FALLBACK"] = "0" @@ -459,7 +484,25 @@ final class MacNodeHostWorker: MacNodeHostWorking, @unchecked Sendable { self.stopLocked(reason: "worker returned invalid manifest") return } - let manifest = MacNodeHostManifest(version: version, caps: caps, commands: commands, pathEnv: pathEnv) + let computerUse: AnyCodable? + if let rawComputerUse = rawManifest["computerUse"] { + guard let rawComputerUse = rawComputerUse as? [String: Any], + let data = try? JSONSerialization.data(withJSONObject: rawComputerUse), + let decoded = try? JSONDecoder().decode(AnyCodable.self, from: data) + else { + self.stopLocked(reason: "worker returned invalid computer-use descriptor") + return + } + computerUse = decoded + } else { + computerUse = nil + } + let manifest = MacNodeHostManifest( + version: version, + caps: caps, + commands: commands, + computerUse: computerUse, + pathEnv: pathEnv) self.manifest = manifest self.inventoryData = (message["inventory"] as? [String: Any]).flatMap(Self.jsonData) self.finishStartLocked(.success(manifest)) diff --git a/apps/macos/Sources/OpenClaw/NodeMode/MacNodeModeCoordinator.swift b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeModeCoordinator.swift index 7f36ef817a79..3445e50321e5 100644 --- a/apps/macos/Sources/OpenClaw/NodeMode/MacNodeModeCoordinator.swift +++ b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeModeCoordinator.swift @@ -2,6 +2,7 @@ import AppKit import Foundation import OpenClawIPC import OpenClawKit +import OpenClawProtocol import OSLog struct MacNodeGatewayTLSSessionCache { @@ -115,6 +116,7 @@ final class MacNodeModeCoordinator: NSObject { private var activeNodeHostWorkerInput: MacNodeHostWorkerRetryPolicy.Input? private var lastObservedPaused: Bool private var lastObservedComputerControlEnabled: Bool + private var lastObservedComputerControlProvider: ComputerControlProvider private let runtime: MacNodeRuntime private let session: GatewayNodeSession private let nodeHostWorker: (any MacNodeHostWorking)? @@ -155,6 +157,7 @@ final class MacNodeModeCoordinator: NSObject { observeNotifications: Bool = false, initialPaused: Bool? = nil, initialComputerControlEnabled: Bool? = nil, + initialComputerControlProvider: ComputerControlProvider? = nil, nodeHostWorkerRetrySleep: @escaping @Sendable (UInt64) async throws -> Void = { try await Task.sleep(nanoseconds: $0) }, @@ -173,6 +176,8 @@ final class MacNodeModeCoordinator: NSObject { self.lastObservedPaused = initialPaused ?? AppDefaults.standard.bool(forKey: pauseDefaultsKey) self.lastObservedComputerControlEnabled = initialComputerControlEnabled ?? isComputerControlEnabled() + self.lastObservedComputerControlProvider = initialComputerControlProvider ?? + ComputerControlProvider.current() super.init() guard observeNotifications else { return } @@ -206,6 +211,11 @@ final class MacNodeModeCoordinator: NSObject { selector: #selector(self.nodeHostConfigurationChanged), name: .openclawCLIInstalled, object: nil) + self.notificationCenter.addObserver( + self, + selector: #selector(self.nodeHostConfigurationChanged), + name: .openclawCuaDriverAvailabilityChanged, + object: nil) } deinit { @@ -248,6 +258,11 @@ final class MacNodeModeCoordinator: NSObject { await self.beginTerminalStop().value } + func prepareForCuaDaemonStop() async { + self.resetNodeHostWorkerRetryState() + await self.enqueueRouteInvalidation(mode: .workerRestart).value + } + private func beginTerminalStop() -> Task { if let terminalStopTask = self.terminalStopTask { return terminalStopTask @@ -285,7 +300,8 @@ final class MacNodeModeCoordinator: NSObject { func refresh() { self.refresh( isPaused: AppDefaults.standard.bool(forKey: pauseDefaultsKey), - computerControlEnabled: isComputerControlEnabled()) + computerControlEnabled: isComputerControlEnabled(), + computerControlProvider: ComputerControlProvider.current()) } func currentCanvasPluginSurfaceRoute() async -> GatewayCanvasHostRoute? { @@ -321,17 +337,25 @@ final class MacNodeModeCoordinator: NSObject { await self.session.refreshCanvasHostRoute(replacing: observedURL) } - private func refresh(isPaused: Bool, computerControlEnabled: Bool) { + private func refresh( + isPaused: Bool, + computerControlEnabled: Bool, + computerControlProvider: ComputerControlProvider) + { + let providerChanged = self.lastObservedComputerControlProvider != computerControlProvider let shouldRevoke = Self.controlTransitionRequiresRouteInvalidation( previousPaused: self.lastObservedPaused, nextPaused: isPaused, previousComputerControlEnabled: self.lastObservedComputerControlEnabled, - nextComputerControlEnabled: computerControlEnabled) + nextComputerControlEnabled: computerControlEnabled, + previousComputerControlProvider: self.lastObservedComputerControlProvider, + nextComputerControlProvider: computerControlProvider) self.lastObservedPaused = isPaused self.lastObservedComputerControlEnabled = computerControlEnabled + self.lastObservedComputerControlProvider = computerControlProvider if shouldRevoke { - self.enqueueRouteInvalidation(mode: .reconnectRefresh) + self.enqueueRouteInvalidation(mode: providerChanged ? .workerRestart : .reconnectRefresh) } else { // Routine permission/foreground/defaults refreshes invalidate only // suspended setup. The installed route remains authoritative. @@ -525,10 +549,14 @@ final class MacNodeModeCoordinator: NSObject { claudeSessionCatalogEnabled: Bool) async throws -> ConnectionAttempt? { let config = endpoint.config - let workerManifest = try await self.startNodeHostWorkerIfConfigured() + let provider = ComputerControlProvider.current() + let workerManifest = try await Self.workerManifest( + self.startNodeHostWorkerIfConfigured(provider: provider), + for: provider) let nativeCaps = self.currentCaps( browserControlEnabled: browserControlEnabled, cameraEnabled: cameraEnabled, + computerControlProvider: provider, codexThreadCatalogEnabled: codexThreadCatalogEnabled, claudeSessionCatalogEnabled: claudeSessionCatalogEnabled) // If Computer Control was turned off, release any button the @@ -540,7 +568,7 @@ final class MacNodeModeCoordinator: NSObject { } let caps = Self.mergingUnique(nativeCaps, workerManifest?.caps ?? []) let commands = Self.mergingUnique( - self.currentCommands(caps: nativeCaps), + self.currentCommands(caps: nativeCaps, computerControlProvider: provider), workerManifest?.commands ?? []) let permissions = await self.currentPermissions() // TCC queries suspend. An endpoint loss/replacement during that @@ -559,6 +587,10 @@ final class MacNodeModeCoordinator: NSObject { scopes: [], caps: caps, commands: commands, + computerUse: Self.computerUseDescriptor( + provider: provider, + commands: commands, + workerManifest: workerManifest), pathEnv: workerManifest?.pathEnv, permissions: permissions, clientId: "openclaw-macos", @@ -769,10 +801,15 @@ final class MacNodeModeCoordinator: NSObject { await self.awaitStableRouteInvalidationDrain(onPendingSnapshot: onPendingSnapshot) } - func refreshForTesting(isPaused: Bool, computerControlEnabled: Bool) { + func refreshForTesting( + isPaused: Bool, + computerControlEnabled: Bool, + computerControlProvider: ComputerControlProvider = .peekaboo) + { self.refresh( isPaused: isPaused, - computerControlEnabled: computerControlEnabled) + computerControlEnabled: computerControlEnabled, + computerControlProvider: computerControlProvider) } func enqueueRouteInvalidationForTesting() { @@ -849,10 +886,13 @@ final class MacNodeModeCoordinator: NSObject { // Replace the process before reconnecting so updates cannot leave a stale route. return self.enqueueRouteInvalidation(mode: .workerRestart) } +} +extension MacNodeModeCoordinator { private func currentCaps( browserControlEnabled: Bool, cameraEnabled: Bool, + computerControlProvider: ComputerControlProvider, codexThreadCatalogEnabled: Bool, claudeSessionCatalogEnabled: Bool) -> [String] { @@ -862,6 +902,7 @@ final class MacNodeModeCoordinator: NSObject { browserControlEnabled: browserControlEnabled, cameraEnabled: cameraEnabled, computerControlEnabled: computerControlEnabled, + computerControlProvider: computerControlProvider, locationMode: OpenClawLocationMode(rawValue: rawLocationMode) ?? .off, connectionMode: AppStateStore.shared.connectionMode, codexThreadCatalogEnabled: codexThreadCatalogEnabled, @@ -873,11 +914,16 @@ final class MacNodeModeCoordinator: NSObject { return Self.advertisedPermissions(statuses) } - private func currentCommands(caps: [String]) -> [String] { - Self.resolvedCommands(caps: caps) + private func currentCommands( + caps: [String], + computerControlProvider: ComputerControlProvider) -> [String] + { + Self.resolvedCommands(caps: caps, computerControlProvider: computerControlProvider) } - private func startNodeHostWorkerIfConfigured() async throws -> MacNodeHostManifest? { + private func startNodeHostWorkerIfConfigured( + provider: ComputerControlProvider) async throws -> MacNodeHostManifest? + { guard let nodeHostWorker else { return nil } guard self.nodeHostWorkerRetryTask == nil else { throw MacNodeHostWorkerRetryPolicy.RetryBackoffPending() @@ -898,12 +944,21 @@ final class MacNodeModeCoordinator: NSObject { } catch let error as RuntimeResolutionError { throw MacNodeHostWorker.WorkerError.unavailable(RuntimeLocator.describeFailure(error)) } + var workerEnvironment: [String: String] = [:] + if provider == .cua, let endpoint = CuaDriverHostCoordinator.shared.workerEndpoint { + workerEnvironment[CuaDriverWorkerEnvironment.socketPath] = endpoint.socketPath + workerEnvironment[CuaDriverWorkerEnvironment.binaryPath] = endpoint.binaryPath + } + let effectiveLaunch = MacNodeHostWorkerLaunch( + command: launch.command, + currentDirectoryURL: launch.currentDirectoryURL, + environment: workerEnvironment) let input = MacNodeHostWorkerRetryPolicy.Input( - launch: launch, + launch: effectiveLaunch, configurationGeneration: self.nodeHostWorkerConfigurationGeneration) try self.nodeHostWorkerRetryPolicy.prepareForStart(input) self.activeNodeHostWorkerInput = input - return try await nodeHostWorker.start(launch: launch) + return try await nodeHostWorker.start(launch: effectiveLaunch) } private func handleNodeHostWorkerFailure() { @@ -991,10 +1046,13 @@ extension MacNodeModeCoordinator { previousPaused: Bool, nextPaused: Bool, previousComputerControlEnabled: Bool, - nextComputerControlEnabled: Bool) -> Bool + nextComputerControlEnabled: Bool, + previousComputerControlProvider: ComputerControlProvider = .peekaboo, + nextComputerControlProvider: ComputerControlProvider = .peekaboo) -> Bool { (!previousPaused && nextPaused) || - (previousComputerControlEnabled && !nextComputerControlEnabled) + (previousComputerControlEnabled && !nextComputerControlEnabled) || + previousComputerControlProvider != nextComputerControlProvider } nonisolated static func endpointState( @@ -1101,6 +1159,7 @@ extension MacNodeModeCoordinator { browserControlEnabled: Bool, cameraEnabled: Bool, computerControlEnabled: Bool, + computerControlProvider: ComputerControlProvider = .peekaboo, locationMode: OpenClawLocationMode, connectionMode: AppState.ConnectionMode, codexThreadCatalogEnabled: Bool = false, @@ -1114,7 +1173,7 @@ extension MacNodeModeCoordinator { if cameraEnabled { caps.append(OpenClawCapability.camera.rawValue) } // Advertised only when the operator has enabled Computer Control; the // command is dangerous and stays disarmed until allowlisted on the gateway. - if computerControlEnabled { + if computerControlEnabled, computerControlProvider == .peekaboo { caps.append(OpenClawCapability.computer.rawValue) } if locationMode != .off { caps.append(OpenClawCapability.location.rawValue) } @@ -1129,7 +1188,10 @@ extension MacNodeModeCoordinator { return caps } - nonisolated static func resolvedCommands(caps: [String]) -> [String] { + nonisolated static func resolvedCommands( + caps: [String], + computerControlProvider: ComputerControlProvider = .peekaboo) -> [String] + { var commands: [String] = [ OpenClawCanvasCommand.present.rawValue, OpenClawCanvasCommand.hide.rawValue, @@ -1139,11 +1201,14 @@ extension MacNodeModeCoordinator { OpenClawCanvasA2UICommand.push.rawValue, OpenClawCanvasA2UICommand.pushJSONL.rawValue, OpenClawCanvasA2UICommand.reset.rawValue, - MacNodeScreenCommand.snapshot.rawValue, - MacNodeScreenCommand.record.rawValue, - OpenClawSystemCommand.notify.rawValue, ] + if computerControlProvider == .peekaboo { + commands.append(MacNodeScreenCommand.snapshot.rawValue) + } + commands.append(MacNodeScreenCommand.record.rawValue) + commands.append(OpenClawSystemCommand.notify.rawValue) + let capsSet = Set(caps) if capsSet.contains(OpenClawCapability.camera.rawValue) { commands.append(OpenClawCameraCommand.list.rawValue) @@ -1168,6 +1233,36 @@ extension MacNodeModeCoordinator { return commands } + nonisolated static func workerManifest( + _ manifest: MacNodeHostManifest?, + for provider: ComputerControlProvider) -> MacNodeHostManifest? + { + guard let manifest else { return nil } + guard provider == .peekaboo else { return manifest } + let providerCommands = Set([ + MacNodeScreenCommand.snapshot.rawValue, + OpenClawComputerCommand.act.rawValue, + ]) + return MacNodeHostManifest( + version: manifest.version, + caps: manifest.caps.filter { $0 != OpenClawCapability.computer.rawValue }, + commands: manifest.commands.filter { !providerCommands.contains($0) }, + computerUse: nil, + pathEnv: manifest.pathEnv) + } + + nonisolated static func computerUseDescriptor( + provider: ComputerControlProvider, + commands: [String], + workerManifest: MacNodeHostManifest?) -> OpenClawProtocol.AnyCodable? + { + guard provider == .cua, + commands.contains(MacNodeScreenCommand.snapshot.rawValue), + commands.contains(OpenClawComputerCommand.act.rawValue) + else { return nil } + return workerManifest?.computerUse + } + nonisolated static func mergingUnique(_ primary: [String], _ additional: [String]) -> [String] { var seen = Set() return (primary + additional).filter { seen.insert($0).inserted } diff --git a/apps/macos/Sources/OpenClaw/NodeMode/MacNodeRuntime.swift b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeRuntime.swift index 4489eea98496..ec93d95baf4b 100644 --- a/apps/macos/Sources/OpenClaw/NodeMode/MacNodeRuntime.swift +++ b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeRuntime.swift @@ -112,12 +112,17 @@ actor MacNodeClaudeSessionCatalogWorker { actor MacNodeRuntime { private static let maxGatewayPayloadBytes = 25 * 1024 * 1024 private static let maxScreenSnapshotRawBytesBeforeBase64 = (maxGatewayPayloadBytes / 4) * 3 + private static let cuaOwnedCommands = Set([ + MacNodeScreenCommand.snapshot.rawValue, + OpenClawComputerCommand.act.rawValue, + ]) private let cameraCapture = CameraCaptureService() private let cameraPTZ: any CameraPTZServicing private let nodeHostWorker: (any MacNodeHostWorking)? private let makeMainActorServices: @Sendable () async -> any MacNodeRuntimeMainActorServices // Injectable so tests pin the gate instead of racing on process-global UserDefaults. private let computerControlEnabled: @Sendable () -> Bool + private let computerControlProvider: @Sendable () -> ComputerControlProvider private let canvasHostedSurfaceResolver: MacNodeCanvasHostedSurfaceResolver private let codexThreadCatalogEnabled: @Sendable () -> Bool private let codexThreadCatalogClient: MacNodeCodexThreadCatalogClient @@ -144,6 +149,9 @@ actor MacNodeRuntime { computerControlEnabled: @escaping @Sendable () -> Bool = { MacNodeRuntime.computerControlEnabledDefault() }, + computerControlProvider: @escaping @Sendable () -> ComputerControlProvider = { + ComputerControlProvider.current() + }, canvasSurfaceUrl: @escaping @Sendable () async -> String? = { await GatewayConnection.shared.canvasPluginSurfaceUrl() }, @@ -168,6 +176,7 @@ actor MacNodeRuntime { self.cameraPTZ = cameraPTZ self.makeMainActorServices = makeMainActorServices self.computerControlEnabled = computerControlEnabled + self.computerControlProvider = computerControlProvider self.canvasHostedSurfaceResolver = MacNodeCanvasHostedSurfaceResolver( currentSurfaceURL: canvasSurfaceUrl, refreshSurfaceURL: refreshCanvasSurfaceUrl) @@ -197,6 +206,9 @@ actor MacNodeRuntime { code: .unavailable, message: "CANVAS_DISABLED: enable Canvas in Settings")) } + if let cuaResponse = await self.handleCuaInvokeIfSelected(req) { + return cuaResponse + } do { switch command { case OpenClawCanvasCommand.present.rawValue, @@ -262,6 +274,25 @@ actor MacNodeRuntime { command.hasPrefix("canvas.") || command.hasPrefix("canvas.a2ui.") } + private func handleCuaInvokeIfSelected(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse? { + guard self.computerControlProvider() == .cua, + Self.cuaOwnedCommands.contains(req.command) + else { return nil } + guard self.computerControlEnabled() else { + return Self.errorResponse( + req, + code: .unavailable, + message: "COMPUTER_DISABLED: enable Computer Control in Settings") + } + guard let nodeHostWorker, await nodeHostWorker.supports(req.command) else { + return Self.errorResponse( + req, + code: .unavailable, + message: "UNAVAILABLE: selected CUA provider is not ready") + } + return await nodeHostWorker.invoke(req) + } + private func handleCodexThreadInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { guard self.codexThreadCatalogEnabled() else { return Self.errorResponse( diff --git a/apps/macos/Tests/OpenClawIPCTests/ComputerControlSettingsTests.swift b/apps/macos/Tests/OpenClawIPCTests/ComputerControlSettingsTests.swift index c9c2fcec9f63..d05cfdfe7312 100644 --- a/apps/macos/Tests/OpenClawIPCTests/ComputerControlSettingsTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/ComputerControlSettingsTests.swift @@ -16,4 +16,35 @@ struct ComputerControlSettingsTests { defaults.set(true, forKey: computerControlEnabledKey) #expect(isComputerControlEnabled(defaults: defaults)) } + + @Test func `computer control provider defaults to Peekaboo and preserves CUA selection`() throws { + let suiteName = "ComputerControlProviderSettingsTests.\(UUID().uuidString)" + let defaults = try #require(UserDefaults(suiteName: suiteName)) + defer { defaults.removePersistentDomain(forName: suiteName) } + + #expect(ComputerControlProvider.current(defaults: defaults, cuaAvailable: true) == .peekaboo) + defaults.set(ComputerControlProvider.cua.rawValue, forKey: computerControlProviderKey) + #expect(ComputerControlProvider.current(defaults: defaults, cuaAvailable: true) == .cua) + #expect(ComputerControlProvider.current(defaults: defaults, cuaAvailable: false) == .peekaboo) + defaults.set("retired-provider", forKey: computerControlProviderKey) + #expect(ComputerControlProvider.current(defaults: defaults, cuaAvailable: true) == .peekaboo) + } + + @Test func `bundled CUA locator accepts only a regular executable and never follows a symlink`() throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-cua-artifact-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: root) } + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + let binary = root.appendingPathComponent(CuaDriverArtifact.resourceName) + try Data("driver".utf8).write(to: binary) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: binary.path) + #expect(CuaDriverArtifact.executableURL(in: root) == binary) + + try FileManager.default.removeItem(at: binary) + let target = root.appendingPathComponent("real-driver") + try Data("driver".utf8).write(to: target) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: target.path) + try FileManager.default.createSymbolicLink(at: binary, withDestinationURL: target) + #expect(CuaDriverArtifact.executableURL(in: root) == nil) + } } diff --git a/apps/macos/Tests/OpenClawIPCTests/CuaDriverHostCoordinatorTests.swift b/apps/macos/Tests/OpenClawIPCTests/CuaDriverHostCoordinatorTests.swift new file mode 100644 index 000000000000..21d456549bfb --- /dev/null +++ b/apps/macos/Tests/OpenClawIPCTests/CuaDriverHostCoordinatorTests.swift @@ -0,0 +1,571 @@ +import Darwin +import Foundation +import OpenClawIPC +import Testing +@testable import OpenClaw + +@Suite(.serialized) +@MainActor +struct CuaDriverHostCoordinatorTests { + private func waitForReadyLaunch( + _ expected: Int, + launcher: CuaProcessLauncherProbe, + coordinator: CuaDriverHostCoordinator) async -> Bool + { + let deadline = ContinuousClock.now + .seconds(2) + while ContinuousClock.now < deadline { + if launcher.launches.count >= expected, coordinator.workerEndpoint != nil { return true } + try? await Task.sleep(for: .milliseconds(1)) + } + return launcher.launches.count >= expected && coordinator.workerEndpoint != nil + } + + @Test func `disabled host never spawns and enabled host publishes only a ready endpoint`() async throws { + let root = self.shortTemporaryDirectory("host") + defer { try? FileManager.default.removeItem(at: root) } + let executable = root.appendingPathComponent("cua-driver") + let launcher = CuaProcessLauncherProbe() + var workerStops = 0 + let coordinator = CuaDriverHostCoordinator( + artifactURL: { executable }, + applicationSupportURL: { root }, + bundleIdentifier: { "ai.openclaw.test" }, + processLauncher: { launch, onTermination in + launcher.launch(launch, onTermination: onTermination) + }, + readinessProbe: { _ in true }, + beforeDaemonStop: { + let allRunning = launcher.processes.allSatisfy(\.isRunning) + #expect(allRunning) + workerStops += 1 + }) + + await coordinator.setEnabled(false) + #expect(launcher.launches.isEmpty) + #expect(coordinator.workerEndpoint == nil) + + await coordinator.setEnabled(true) + let launch = try #require(launcher.launches.first) + let endpoint = try #require(coordinator.workerEndpoint) + #expect(launch.executableURL == executable) + #expect(endpoint.binaryPath == executable.path) + let socketArgument = try #require(launch.arguments.firstIndex(of: "--socket")) + 1 + #expect(endpoint.socketPath == launch.arguments[socketArgument]) + + await coordinator.setEnabled(false) + #expect(coordinator.workerEndpoint == nil) + #expect(workerStops == 1) + #expect(launcher.processes.allSatisfy { !$0.isRunning }) + } + + @Test func `socket directory is random owner-only and cleanup removes only its owned leaf`() throws { + let root = self.shortTemporaryDirectory("socket") + defer { try? FileManager.default.removeItem(at: root) } + + let first = try CuaDriverHostCoordinator.createSocketDirectory(in: root) + let second = try CuaDriverHostCoordinator.createSocketDirectory(in: root) + #expect(first.url != second.url) + for directory in [first, second] { + let attributes = try FileManager.default.attributesOfItem(atPath: directory.url.path) + let permissions = (attributes[.posixPermissions] as? NSNumber)?.intValue + #expect(permissions == 0o700) + #expect(!FileManager.default.fileExists(atPath: directory.socketPath)) + } + + CuaDriverHostCoordinator.cleanupSocketDirectory(first) + #expect(!FileManager.default.fileExists(atPath: first.url.path)) + #expect(FileManager.default.fileExists(atPath: second.url.path)) + CuaDriverHostCoordinator.cleanupSocketDirectory(second) + } + + @Test func `liveness write end is close on exec and absent from a subsequently spawned child`() throws { + let livenessPipe = try CuaDriverHostCoordinator.makeLivenessPipe() + let descriptor = livenessPipe.fileHandleForWriting.fileDescriptor + #expect(fcntl(descriptor, F_GETFD) & FD_CLOEXEC == FD_CLOEXEC) + + let child = Process() + child.executableURL = URL(fileURLWithPath: "/bin/sleep") + child.arguments = ["1"] + try child.run() + defer { + if child.isRunning { child.terminate() } + child.waitUntilExit() + } + #expect(!Self.process(child.processIdentifier, hasDescriptor: descriptor)) + } + + @Test func `liveness read end remains daemon standard input and observes writer EOF`() throws { + let livenessPipe = try CuaDriverHostCoordinator.makeLivenessPipe() + let child = Process() + child.executableURL = URL(fileURLWithPath: "/bin/cat") + child.standardInput = livenessPipe.fileHandleForReading + child.standardOutput = FileHandle.nullDevice + child.standardError = FileHandle.nullDevice + try child.run() + try livenessPipe.fileHandleForWriting.close() + for _ in 0..<1000 where child.isRunning { + usleep(1000) + } + if child.isRunning { child.terminate() } + child.waitUntilExit() + + #expect(child.terminationStatus == 0) + } + + @Test func `unexpected exit closes liveness and removes its socket directory`() async throws { + let root = self.shortTemporaryDirectory("exit-cleanup") + defer { try? FileManager.default.removeItem(at: root) } + let launcher = CuaProcessLauncherProbe() + let coordinator = CuaDriverHostCoordinator( + artifactURL: { root.appendingPathComponent("cua-driver") }, + applicationSupportURL: { root }, + bundleIdentifier: { "ai.openclaw.test" }, + processLauncher: { launch, onTermination in + launcher.launch(launch, onTermination: onTermination) + }, + readinessProbe: { _ in true }) + + await coordinator.setEnabled(true) + let endpoint = try #require(coordinator.workerEndpoint) + let process = try #require(launcher.processes.first) + process.crash(status: 7) + for _ in 0..<1000 where coordinator.workerEndpoint != nil { + await Task.yield() + } + + #expect(process.closeLivenessCount == 1) + #expect(!FileManager.default.fileExists(atPath: URL(fileURLWithPath: endpoint.socketPath) + .deletingLastPathComponent().path)) + await coordinator.setEnabled(false) + } + + @Test func `startup removes a preexisting owned directory without a live daemon`() async throws { + let root = self.shortTemporaryDirectory("startup-reap") + defer { try? FileManager.default.removeItem(at: root) } + let stale = try CuaDriverHostCoordinator.createSocketDirectory(in: root) + let launcher = CuaProcessLauncherProbe() + let coordinator = CuaDriverHostCoordinator( + artifactURL: { root.appendingPathComponent("cua-driver") }, + applicationSupportURL: { root }, + bundleIdentifier: { "ai.openclaw.test" }, + processLauncher: { launch, onTermination in + launcher.launch(launch, onTermination: onTermination) + }, + readinessProbe: { _ in true }) + + await coordinator.setEnabled(true) + #expect(!FileManager.default.fileExists(atPath: stale.url.path)) + await coordinator.setEnabled(false) + } + + @Test func `startup terminates a live owned daemon after its host is gone`() async throws { + let root = self.shortTemporaryDirectory("startup-orphan") + let executable = try self.expectedExecutable(in: root, target: "/bin/sleep") + let orphan = try self.startFakeDaemon(executable: executable, hostPID: Int32.max) + defer { + self.stopIfRunning(orphan) + try? FileManager.default.removeItem(at: root) + } + let stale = try CuaDriverHostCoordinator.createSocketDirectory(in: root) + try String(orphan.processIdentifier).write( + to: stale.url.appendingPathComponent("cua.pid"), + atomically: true, + encoding: .utf8) + let launcher = CuaProcessLauncherProbe() + let coordinator = CuaDriverHostCoordinator( + artifactURL: { executable }, + applicationSupportURL: { root }, + bundleIdentifier: { "ai.openclaw.test" }, + processLauncher: { launch, onTermination in + launcher.launch(launch, onTermination: onTermination) + }, + readinessProbe: { _ in true }) + + await coordinator.setEnabled(true) + + #expect(await self.waitUntilExited(orphan)) + #expect(!FileManager.default.fileExists(atPath: stale.url.path)) + await coordinator.setEnabled(false) + } + + @Test func `launch records the spawned daemon pid for later reaping`() async throws { + // Without this record the reaper can only delete the directory and leaves the + // privileged daemon running: `serve` ignores --pid-file and writes a global path. + let root = self.shortTemporaryDirectory("launch-pidfile") + let executable = try self.expectedExecutable(in: root, target: "/bin/sleep") + defer { try? FileManager.default.removeItem(at: root) } + let launcher = CuaProcessLauncherProbe() + let coordinator = CuaDriverHostCoordinator( + artifactURL: { executable }, + applicationSupportURL: { root }, + bundleIdentifier: { "ai.openclaw.test" }, + processLauncher: { launch, onTermination in + launcher.launch(launch, onTermination: onTermination) + }, + readinessProbe: { _ in true }) + + await coordinator.setEnabled(true) + + let directories = try FileManager.default.contentsOfDirectory( + at: root.appendingPathComponent("OpenClaw", isDirectory: true) + .appendingPathComponent("cua", isDirectory: true), + includingPropertiesForKeys: nil) + let pidFile = try #require(directories.first?.appendingPathComponent("cua.pid")) + let recorded = try String(contentsOf: pidFile, encoding: .utf8) + .trimmingCharacters(in: .whitespacesAndNewlines) + #expect(recorded == String(launcher.lastProcessIdentifier)) + await coordinator.setEnabled(false) + } + + @Test func `startup refuses to signal a pid owned by another executable`() async throws { + let root = self.shortTemporaryDirectory("startup-pid-reuse") + let expectedExecutable = try self.expectedExecutable(in: root, target: "/bin/cat") + let unrelated = try self.startSleep(executable: URL(fileURLWithPath: "/bin/sleep")) + defer { + self.stopIfRunning(unrelated) + try? FileManager.default.removeItem(at: root) + } + let stale = try CuaDriverHostCoordinator.createSocketDirectory(in: root) + try String(unrelated.processIdentifier).write( + to: stale.url.appendingPathComponent("cua.pid"), + atomically: true, + encoding: .utf8) + let launcher = CuaProcessLauncherProbe() + let coordinator = CuaDriverHostCoordinator( + artifactURL: { expectedExecutable }, + applicationSupportURL: { root }, + bundleIdentifier: { "ai.openclaw.test" }, + processLauncher: { launch, onTermination in + launcher.launch(launch, onTermination: onTermination) + }, + readinessProbe: { _ in true }) + + await coordinator.setEnabled(true) + + #expect(unrelated.isRunning) + #expect(FileManager.default.fileExists(atPath: stale.url.path)) + let launch = try #require(launcher.launches.first) + // `serve` ignores --pid-file (it always writes the machine-global default), + // so OpenClaw must never pass it and records the pid itself instead. + #expect(!launch.arguments.contains("--pid-file")) + await coordinator.setEnabled(false) + #expect(unrelated.isRunning) + } + + @Test func `teardown leaves no owned directories or live owned daemons`() async throws { + let root = self.shortTemporaryDirectory("teardown-reap") + let executable = try self.expectedExecutable(in: root, target: "/bin/sleep") + let orphan = try self.startFakeDaemon(executable: executable, hostPID: Int32.max) + defer { + self.stopIfRunning(orphan) + try? FileManager.default.removeItem(at: root) + } + let live = try CuaDriverHostCoordinator.createSocketDirectory(in: root) + try String(orphan.processIdentifier).write( + to: live.url.appendingPathComponent("cua.pid"), + atomically: true, + encoding: .utf8) + _ = try CuaDriverHostCoordinator.createSocketDirectory(in: root) + let coordinator = CuaDriverHostCoordinator( + artifactURL: { executable }, + applicationSupportURL: { root }, + bundleIdentifier: { "ai.openclaw.test" }) + + await coordinator.shutdown() + + #expect(await self.waitUntilExited(orphan)) + let cuaRoot = root.appendingPathComponent("OpenClaw/cua", isDirectory: true) + #expect(try FileManager.default.contentsOfDirectory(atPath: cuaRoot.path).isEmpty) + } + + @Test func `socket directory rejects a symlinked CUA root`() throws { + let root = self.shortTemporaryDirectory("unsafe") + defer { try? FileManager.default.removeItem(at: root) } + let openClaw = root.appendingPathComponent("OpenClaw", isDirectory: true) + let redirected = root.appendingPathComponent("redirected", isDirectory: true) + try FileManager.default.createDirectory(at: openClaw, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: redirected, withIntermediateDirectories: true) + try FileManager.default.createSymbolicLink( + at: openClaw.appendingPathComponent("cua", isDirectory: true), + withDestinationURL: redirected) + + #expect(throws: CuaDriverHostError.self) { + try CuaDriverHostCoordinator.createSocketDirectory(in: root) + } + #expect(try (FileManager.default.contentsOfDirectory(atPath: redirected.path)).isEmpty) + } + + @Test func `socket directory rejects a symlinked OpenClaw support root`() throws { + let root = self.shortTemporaryDirectory("unsafe-parent") + defer { try? FileManager.default.removeItem(at: root) } + let redirected = root.appendingPathComponent("redirected", isDirectory: true) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: redirected, withIntermediateDirectories: true) + try FileManager.default.createSymbolicLink( + at: root.appendingPathComponent("OpenClaw", isDirectory: true), + withDestinationURL: redirected) + + #expect(throws: CuaDriverHostError.self) { + try CuaDriverHostCoordinator.createSocketDirectory(in: root) + } + #expect(try (FileManager.default.contentsOfDirectory(atPath: redirected.path)).isEmpty) + } + + @Test func `embedded launch carries unrestricted acknowledgement and disables network reporting`() { + let launch = CuaDriverHostCoordinator.makeProcessLaunch( + executableURL: URL(fileURLWithPath: "/Applications/OpenClaw.app/Contents/Resources/cua-driver"), + socketPath: "/tmp/openclaw-cua-test.sock", + hostBundleID: "ai.openclaw.mac", + inheritedEnvironment: [ + "PATH": "/usr/bin:/bin", + "CUA_DRIVER_SOCKET": "/tmp/ambient.sock", + "CUA_DRIVER_PERMISSION_MODE": "bounded", + "CUA_TELEMETRY_ENABLED": "true", + ]) + + #expect(launch.arguments.contains("--embedded")) + #expect(launch.arguments.contains("--parent-liveness-stdio")) + // The embedded host cannot predeclare arbitrary runtime-discovered targets in a bounded manifest. + let permissionModeIndex = launch.arguments.firstIndex(of: "--permission-mode") + let permissionMode = permissionModeIndex.flatMap { index in + launch.arguments.indices.contains(index + 1) ? launch.arguments[index + 1] : nil + } + #expect(permissionMode == "unrestricted") + #expect(launch.arguments.contains("--dangerously-bypass-approvals")) + #expect(launch.environment["CUA_DRIVER_EMBEDDED"] == "1") + #expect(launch.environment["CUA_DRIVER_PERMISSION_MODE"] == "unrestricted") + #expect(launch.environment["CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS"] == "1") + #expect(launch.environment["CUA_DRIVER_RS_TELEMETRY_ENABLED"] == "false") + #expect(launch.environment["CUA_DRIVER_RS_UPDATE_CHECK"] == "false") + #expect(launch.environment["CUA_DRIVER_SOCKET"] == nil) + #expect(launch.environment["CUA_TELEMETRY_ENABLED"] == nil) + #expect(launch.environment["PATH"] == "/usr/bin:/bin") + } + + @Test func `stderr relay replaces the raw danger banner with one managed mode notice`() async throws { + let probe = CuaDriverStderrProbe() + let relay = CuaDriverStderrRelay { probe.append($0) } + relay.startReading() + relay.reportManagedMode() + relay.reportManagedMode() + + let driverOutput = """ + DANGER: Cua Driver is running in unrestricted mode. Runtime approval prompts are disabled. + driver diagnostic + + """ + try relay.pipe.fileHandleForWriting.write(contentsOf: Data(driverOutput.utf8)) + try relay.pipe.fileHandleForWriting.close() + for _ in 0..<1000 where probe.events.count < 2 { + await Task.yield() + } + relay.stop() + + #expect(probe.events == [ + .notice(CuaDriverStderrRelay.managedModeNotice), + .error("driver diagnostic"), + ]) + } + + @Test func `unexpected exits retry with a bounded budget while advertising unavailable`() async throws { + let root = self.shortTemporaryDirectory("restart") + defer { try? FileManager.default.removeItem(at: root) } + let launcher = CuaProcessLauncherProbe() + let coordinator = CuaDriverHostCoordinator( + artifactURL: { root.appendingPathComponent("cua-driver") }, + applicationSupportURL: { root }, + bundleIdentifier: { "ai.openclaw.test" }, + processLauncher: { launch, onTermination in + launcher.launch(launch, onTermination: onTermination) + }, + readinessProbe: { _ in true }, + restartSleep: { _ in }) + + await coordinator.setEnabled(true) + for expectedLaunchCount in 2...6 { + try #require(launcher.processes.last).crash(status: 7) + #expect(await self.waitForReadyLaunch( + expectedLaunchCount, + launcher: launcher, + coordinator: coordinator)) + } + try #require(launcher.processes.last).crash(status: 7) + for _ in 0..<100 { + await Task.yield() + } + #expect(launcher.launches.count == 6) + #expect(coordinator.workerEndpoint == nil) + await coordinator.setEnabled(false) + } + + @Test func `permission changes replace the daemon generation and endpoint`() async throws { + let root = self.shortTemporaryDirectory("permissions") + defer { try? FileManager.default.removeItem(at: root) } + let notifications = NotificationCenter() + let permissions = CuaPermissionSnapshotProbe() + let launcher = CuaProcessLauncherProbe() + let coordinator = CuaDriverHostCoordinator( + notificationCenter: notifications, + observeNotifications: true, + artifactURL: { root.appendingPathComponent("cua-driver") }, + applicationSupportURL: { root }, + bundleIdentifier: { "ai.openclaw.test" }, + processLauncher: { launch, onTermination in + launcher.launch(launch, onTermination: onTermination) + }, + readinessProbe: { _ in true }, + permissionSnapshot: { permissions.value }) + + await coordinator.setEnabled(true) + let originalEndpoint = try #require(coordinator.workerEndpoint) + permissions.value[.accessibility] = .granted + notifications.post(name: .openclawPermissionsChanged, object: nil) + #expect(await self.waitForReadyLaunch(2, launcher: launcher, coordinator: coordinator)) + let replacementEndpoint = try #require(coordinator.workerEndpoint) + #expect(replacementEndpoint.socketPath != originalEndpoint.socketPath) + #expect(!launcher.processes[0].isRunning) + await coordinator.setEnabled(false) + } + + private func shortTemporaryDirectory(_ label: String) -> URL { + URL(fileURLWithPath: "/tmp/oc-cua-\(label)-\(UUID().uuidString.prefix(8))", isDirectory: true) + } + + private func expectedExecutable(in root: URL, target: String) throws -> URL { + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + let executable = root.appendingPathComponent("cua-driver") + try FileManager.default.createSymbolicLink( + at: executable, + withDestinationURL: URL(fileURLWithPath: target)) + return executable + } + + private func startFakeDaemon(executable: URL, hostPID: Int32) throws -> Process { + let process = self.makeSleepProcess(executable: executable) + var environment = ProcessInfo.processInfo.environment + environment["CUA_DRIVER_EMBEDDED_HOST_PID"] = String(hostPID) + process.environment = environment + try process.run() + return process + } + + private func startSleep(executable: URL) throws -> Process { + let process = self.makeSleepProcess(executable: executable) + try process.run() + return process + } + + private func makeSleepProcess(executable: URL) -> Process { + let process = Process() + process.executableURL = executable + process.arguments = ["60"] + return process + } + + private func waitUntilExited(_ process: Process) async -> Bool { + let deadline = ContinuousClock.now + .seconds(2) + while process.isRunning, ContinuousClock.now < deadline { + try? await Task.sleep(for: .milliseconds(10)) + } + return !process.isRunning + } + + private func stopIfRunning(_ process: Process) { + if process.isRunning { + process.terminate() + process.waitUntilExit() + } + } + + private static func process(_ processIdentifier: pid_t, hasDescriptor descriptor: Int32) -> Bool { + var descriptors = [proc_fdinfo](repeating: proc_fdinfo(), count: 64) + let populatedBytes = descriptors.withUnsafeMutableBytes { buffer in + proc_pidinfo( + processIdentifier, + PROC_PIDLISTFDS, + 0, + buffer.baseAddress, + Int32(buffer.count)) + } + guard populatedBytes > 0 else { return false } + let count = min( + descriptors.count, + Int(populatedBytes) / MemoryLayout.stride) + return descriptors.prefix(count).contains { $0.proc_fd == descriptor } + } +} + +private final class CuaDriverStderrProbe: @unchecked Sendable { + private let lock = NSLock() + private var captured: [CuaDriverStderrEvent] = [] + + var events: [CuaDriverStderrEvent] { + self.lock.withLock { self.captured } + } + + func append(_ event: CuaDriverStderrEvent) { + self.lock.withLock { self.captured.append(event) } + } +} + +@MainActor +private final class CuaPermissionSnapshotProbe { + var value: [Capability: CapabilityAuthorizationStatus] = [ + .accessibility: .notGranted, + .screenRecording: .notGranted, + ] +} + +@MainActor +private final class CuaProcessLauncherProbe { + private(set) var launches: [CuaDriverProcessLaunch] = [] + private(set) var processes: [CuaProcessProbe] = [] + + var lastProcessIdentifier: pid_t { + self.processes.last?.processIdentifier ?? 0 + } + + func launch( + _ launch: CuaDriverProcessLaunch, + onTermination: @escaping @Sendable (Int32) -> Void) -> CuaProcessProbe + { + self.launches.append(launch) + let process = CuaProcessProbe(onTermination: onTermination) + self.processes.append(process) + return process + } +} + +@MainActor +private final class CuaProcessProbe: CuaDriverProcessControlling { + private(set) var isRunning = true + private(set) var closeLivenessCount = 0 + let processIdentifier: pid_t + private let onTermination: @Sendable (Int32) -> Void + + init(processIdentifier: pid_t = 424_242, onTermination: @escaping @Sendable (Int32) -> Void) { + self.processIdentifier = processIdentifier + self.onTermination = onTermination + } + + func closeLiveness() { + self.closeLivenessCount += 1 + guard self.isRunning else { return } + self.isRunning = false + self.onTermination(0) + } + + func terminate() { + self.closeLiveness() + } + + func forceKill() { + self.closeLiveness() + } + + func crash(status: Int32) { + guard self.isRunning else { return } + self.isRunning = false + self.onTermination(status) + } +} diff --git a/apps/macos/Tests/OpenClawIPCTests/GatewayChannelConnectTests.swift b/apps/macos/Tests/OpenClawIPCTests/GatewayChannelConnectTests.swift index 8b2d9d56d788..1a74eb34cdfa 100644 --- a/apps/macos/Tests/OpenClawIPCTests/GatewayChannelConnectTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/GatewayChannelConnectTests.swift @@ -293,6 +293,47 @@ struct GatewayChannelConnectTests { #expect(params["pathEnv"] as? String == "/opt/homebrew/bin:/usr/bin:/bin") } + @Test func `node connect forwards the selected computer-use descriptor`() async throws { + let recorder = ConnectParamsRecorder() + let session = GatewayTestWebSocketSession( + taskFactory: { + GatewayTestWebSocketTask( + sendHook: { _, message, sendIndex in + guard sendIndex == 0 else { return } + recorder.record(message) + }) + }) + let descriptor = OpenClawProtocol.AnyCodable([ + "contractVersion": OpenClawProtocol.AnyCodable(2), + "provider": OpenClawProtocol.AnyCodable([ + "id": OpenClawProtocol.AnyCodable("cua-computer"), + ]), + ]) + let options = GatewayConnectOptions( + role: "node", + scopes: [], + caps: ["screen", "computer"], + commands: ["screen.snapshot", "computer.act"], + computerUse: descriptor, + permissions: [:], + clientId: "openclaw-macos", + clientMode: "node", + clientDisplayName: "macOS Test", + includeDeviceIdentity: false) + let channel = try GatewayChannelActor( + url: #require(URL(string: "ws://example.invalid")), + token: nil, + session: WebSocketSessionBox(session: session), + connectOptions: options) + + try await channel.connect() + + let params = try #require(recorder.snapshot()) + let computerUse = try #require(params["computerUse"] as? [String: Any]) + #expect(computerUse["contractVersion"] as? Int == 2) + #expect((computerUse["provider"] as? [String: Any])?["id"] as? String == "cua-computer") + } + @Test func `concurrent connect shares failure`() async throws { let session = self.makeSession(response: .invalid(delayMs: 200)) let channel = try GatewayChannelActor( diff --git a/apps/macos/Tests/OpenClawIPCTests/MacNodeHostWorkerTests.swift b/apps/macos/Tests/OpenClawIPCTests/MacNodeHostWorkerTests.swift index 2ffd8e62f2bd..c1a743a996d5 100644 --- a/apps/macos/Tests/OpenClawIPCTests/MacNodeHostWorkerTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/MacNodeHostWorkerTests.swift @@ -1,6 +1,7 @@ import Darwin import Foundation import OpenClawKit +import OpenClawProtocol import Testing @testable import OpenClaw @@ -146,6 +147,40 @@ struct MacNodeHostWorkerTests { #expect(await worker.invokedCommands() == [command]) } + @Test(arguments: [MacNodeScreenCommand.snapshot.rawValue, OpenClawComputerCommand.act.rawValue]) + func `selected CUA provider gives the command pair exclusively to the worker`(command: String) async { + let worker = StubMacNodeHostWorker(commands: [command]) + let runtime = MacNodeRuntime( + nodeHostWorker: worker, + computerControlEnabled: { true }, + computerControlProvider: { .cua }) + + let response = await runtime.handleInvoke(BridgeInvokeRequest( + id: "cua-owned", + command: command, + paramsJSON: "{}")) + + #expect(response.ok) + #expect(response.payloadJSON == #"{"owner":"cli"}"#) + #expect(await worker.invokedCommands() == [command]) + } + + @Test func `selected CUA provider never falls back to native snapshot`() async { + let worker = StubMacNodeHostWorker(commands: []) + let runtime = MacNodeRuntime( + nodeHostWorker: worker, + computerControlEnabled: { true }, + computerControlProvider: { .cua }) + + let response = await runtime.handleInvoke(BridgeInvokeRequest( + id: "cua-unavailable", + command: MacNodeScreenCommand.snapshot.rawValue)) + + #expect(!response.ok) + #expect(response.error?.message == "UNAVAILABLE: selected CUA provider is not ready") + #expect(await worker.invokedCommands().isEmpty) + } + @Test(arguments: [ MacNodeCodexThreadCatalogContract.listCommand, MacNodeCodexThreadCatalogContract.turnsCommand, @@ -232,6 +267,30 @@ struct MacNodeHostWorkerTests { ["system", "mcp"]) == ["canvas", "screen", "system", "mcp"]) } + @Test func `provider selection filters command ownership and publishes only the CUA descriptor`() throws { + let descriptor = OpenClawProtocol.AnyCodable([ + "contractVersion": OpenClawProtocol.AnyCodable(2), + ]) + let manifest = MacNodeHostManifest( + version: "test", + caps: ["screen", "computer"], + commands: [MacNodeScreenCommand.snapshot.rawValue, OpenClawComputerCommand.act.rawValue], + computerUse: descriptor, + pathEnv: "/usr/bin:/bin") + + let peekaboo = try #require(MacNodeModeCoordinator.workerManifest(manifest, for: .peekaboo)) + #expect(!peekaboo.commands.contains(MacNodeScreenCommand.snapshot.rawValue)) + #expect(!peekaboo.commands.contains(OpenClawComputerCommand.act.rawValue)) + #expect(peekaboo.computerUse == nil) + + let cua = try #require(MacNodeModeCoordinator.workerManifest(manifest, for: .cua)) + #expect(cua.commands == manifest.commands) + #expect(MacNodeModeCoordinator.computerUseDescriptor( + provider: .cua, + commands: cua.commands, + workerManifest: cua) == descriptor) + } + @Test func `stale route updates cannot replace newer worker authority`() { #expect(MacNodeHostWorker.routeUpdateIsCurrent(candidateGeneration: 4, currentGeneration: 4)) #expect(MacNodeHostWorker.routeUpdateIsCurrent(candidateGeneration: 5, currentGeneration: 4)) @@ -267,6 +326,25 @@ struct MacNodeHostWorkerTests { await worker.stop() } + @Test func `worker receives only the app-provided CUA endpoint`() async throws { + let worker = MacNodeHostWorker(session: GatewayNodeSession()) + let script = """ + test "$OPENCLAW_CUA_DRIVER_SOCKET_PATH" = "/private/test/cua.sock" || exit 41 + test "$OPENCLAW_CUA_DRIVER_BINARY_PATH" = "/Applications/OpenClaw.app/Contents/Resources/cua-driver" || exit 42 + printf '%s\\n' '{"type":"ready","version":"test","manifest":{"caps":[],"commands":[],"pathEnv":"/usr/bin:/bin"},"inventory":{"skills":null,"pluginTools":[]}}' + while IFS= read -r line; do :; done + """ + + _ = try await worker.start(launch: MacNodeHostWorkerLaunch( + command: ["/bin/sh", "-c", script], + environment: [ + CuaDriverWorkerEnvironment.socketPath: "/private/test/cua.sock", + CuaDriverWorkerEnvironment.binaryPath: + "/Applications/OpenClaw.app/Contents/Resources/cua-driver", + ])) + await worker.stop() + } + @Test func `worker forwards terminal input and cancellation frames`() async throws { let worker = MacNodeHostWorker(session: GatewayNodeSession()) let script = """ diff --git a/apps/macos/Tests/OpenClawIPCTests/MacNodeModeCoordinatorTests.swift b/apps/macos/Tests/OpenClawIPCTests/MacNodeModeCoordinatorTests.swift index 82ef3d5a9889..7702fb0c1d6f 100644 --- a/apps/macos/Tests/OpenClawIPCTests/MacNodeModeCoordinatorTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/MacNodeModeCoordinatorTests.swift @@ -307,6 +307,13 @@ struct MacNodeModeCoordinatorTests { nextPaused: false, previousComputerControlEnabled: true, nextComputerControlEnabled: true)) + #expect(MacNodeModeCoordinator.controlTransitionRequiresRouteInvalidation( + previousPaused: false, + nextPaused: false, + previousComputerControlEnabled: true, + nextComputerControlEnabled: true, + previousComputerControlProvider: .peekaboo, + nextComputerControlProvider: .cua)) } @Test func `first endpoint snapshot rejects a stale captured endpoint`() throws { diff --git a/apps/macos/Tests/OpenClawIPCTests/MacNodeRuntimeTests.swift b/apps/macos/Tests/OpenClawIPCTests/MacNodeRuntimeTests.swift index c2227b330f73..2c9b90403e94 100644 --- a/apps/macos/Tests/OpenClawIPCTests/MacNodeRuntimeTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/MacNodeRuntimeTests.swift @@ -6,6 +6,41 @@ import Testing @testable import OpenClaw struct MacNodeRuntimeTests { + private actor ComputerProviderWorkerProbe: MacNodeHostWorking { + private let commands: Set + private(set) var invokedCommands: [String] = [] + + init(commands: Set) { + self.commands = commands + } + + func start(launch _: MacNodeHostWorkerLaunch) async throws -> MacNodeHostManifest { + MacNodeHostManifest( + version: "test", + caps: ["screen", "computer"], + commands: Array(self.commands), + pathEnv: "/usr/bin:/bin") + } + + func supports(_ command: String) -> Bool { + self.commands.contains(command) + } + + func invoke(_ request: BridgeInvokeRequest) -> BridgeInvokeResponse { + self.invokedCommands.append(request.command) + return BridgeInvokeResponse(id: request.id, ok: true, payloadJSON: #"{"owner":"worker"}"#) + } + + func handleInput(invokeId _: String, seq _: Int, payloadJSON _: String) {} + func cancel(invokeId _: String) {} + func setRoute(_: GatewayNodeSessionRoute?, authorityGeneration _: UInt64) -> Bool { + true + } + + func publishInventory(ifCurrentRoute _: GatewayNodeSessionRoute) {} + func stop() {} + } + private final class LockedCounter: @unchecked Sendable { private let lock = NSLock() private var count = 0 @@ -625,6 +660,68 @@ struct MacNodeRuntimeTests { #expect(result.cursorX == 12) } + @Test func `provider selection owns both snapshot and action without cross-provider fallback`() async throws { + let commands: Set = [MacNodeScreenCommand.snapshot.rawValue, OpenClawComputerCommand.act.rawValue] + let cuaWorker = ComputerProviderWorkerProbe(commands: commands) + let cuaServices = await MainActor.run { MainActorServicesProbe() } + let cuaRuntime = MacNodeRuntime( + nodeHostWorker: cuaWorker, + makeMainActorServices: { cuaServices }, + computerControlEnabled: { true }, + computerControlProvider: { .cua }) + let action = OpenClawComputerActParams(action: .leftClick, x: 12, y: 34, refWidth: 1280) + + #expect(await (self.invoke( + cuaRuntime, + "cua-snapshot", + MacNodeScreenCommand.snapshot.rawValue)).ok) + #expect(try await (self.invoke( + cuaRuntime, + "cua-action", + OpenClawComputerCommand.act.rawValue, + params: action)).ok) + #expect(await cuaWorker.invokedCommands == [ + MacNodeScreenCommand.snapshot.rawValue, + OpenClawComputerCommand.act.rawValue, + ]) + #expect(await MainActor.run { cuaServices.snapshotCallCount == 0 && cuaServices.performCallCount == 0 }) + + let peekabooWorker = ComputerProviderWorkerProbe(commands: commands) + let peekabooServices = await MainActor.run { MainActorServicesProbe() } + let peekabooRuntime = MacNodeRuntime( + nodeHostWorker: peekabooWorker, + makeMainActorServices: { peekabooServices }, + computerControlEnabled: { true }, + computerControlProvider: { .peekaboo }) + #expect(await (self.invoke( + peekabooRuntime, + "peekaboo-snapshot", + MacNodeScreenCommand.snapshot.rawValue)).ok) + #expect(try await (self.invoke( + peekabooRuntime, + "peekaboo-action", + OpenClawComputerCommand.act.rawValue, + params: action)).ok) + #expect(await peekabooWorker.invokedCommands.isEmpty) + #expect(await MainActor.run { + peekabooServices.snapshotCallCount == 1 && peekabooServices.performCallCount == 1 + }) + + let unavailableWorker = ComputerProviderWorkerProbe(commands: []) + let unavailableServices = await MainActor.run { MainActorServicesProbe() } + let unavailableRuntime = MacNodeRuntime( + nodeHostWorker: unavailableWorker, + makeMainActorServices: { unavailableServices }, + computerControlEnabled: { true }, + computerControlProvider: { .cua }) + let unavailable = await self.invoke( + unavailableRuntime, + "cua-unavailable", + MacNodeScreenCommand.snapshot.rawValue) + #expect(!unavailable.ok) + #expect(await MainActor.run { unavailableServices.snapshotCallCount == 0 }) + } + @Test func `concurrent invokes share one main actor services initialization`() async throws { let services = await MainActor.run { MainActorServicesProbe() } let factoryGate = AsyncTestGate() diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayChannel.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayChannel.swift index bee2f838571d..3415bd5d107e 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayChannel.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayChannel.swift @@ -490,15 +490,7 @@ public actor GatewayChannelActor { "role": ProtoAnyCodable(role), "scopes": ProtoAnyCodable(scopes), ] - if !options.commands.isEmpty { - params["commands"] = ProtoAnyCodable(options.commands) - } - if let pathEnv = options.pathEnv?.trimmingCharacters(in: .whitespacesAndNewlines), !pathEnv.isEmpty { - params["pathEnv"] = ProtoAnyCodable(pathEnv) - } - if !options.permissions.isEmpty { - params["permissions"] = ProtoAnyCodable(options.permissions) - } + options.applyOptionalConnectParams(to: ¶ms) self.applyConnectAuth( selectedAuth, deviceId: identity?.deviceId, diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayConnectOptions.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayConnectOptions.swift index e71eade9c781..599e3a409023 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayConnectOptions.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayConnectOptions.swift @@ -1,3 +1,5 @@ +import OpenClawProtocol + public enum OpenClawGatewayClientCapability { public static let agentKind = "agent-kind" public static let inlineWidgets = "inline-widgets" @@ -9,6 +11,7 @@ public struct GatewayConnectOptions: Sendable { public var scopesAreExplicit: Bool public var caps: [String] public var commands: [String] + public var computerUse: AnyCodable? public var pathEnv: String? public var permissions: [String: Bool] public var clientId: String @@ -32,6 +35,7 @@ public struct GatewayConnectOptions: Sendable { scopesAreExplicit: Bool = false, caps: [String], commands: [String], + computerUse: AnyCodable? = nil, pathEnv: String? = nil, permissions: [String: Bool], clientId: String, @@ -47,6 +51,7 @@ public struct GatewayConnectOptions: Sendable { self.scopesAreExplicit = scopesAreExplicit self.caps = caps self.commands = commands + self.computerUse = computerUse self.pathEnv = pathEnv self.permissions = permissions self.clientId = clientId @@ -73,3 +78,24 @@ public struct GatewayAuthBinding: Equatable, Sendable { public let source: GatewayAuthSource public let credentialFingerprint: String? } + +extension GatewayConnectOptions { + /// Additive connect-frame fields, sent only when this node declares them. + /// Lives here so `GatewayChannel.sendConnect` stays within its body budget. + func applyOptionalConnectParams(to params: inout [String: OpenClawProtocol.AnyCodable]) { + if !self.commands.isEmpty { + params["commands"] = OpenClawProtocol.AnyCodable(self.commands) + } + if let computerUse = self.computerUse { + params["computerUse"] = computerUse + } + if let pathEnv = self.pathEnv?.trimmingCharacters(in: .whitespacesAndNewlines), + !pathEnv.isEmpty + { + params["pathEnv"] = OpenClawProtocol.AnyCodable(pathEnv) + } + if !self.permissions.isEmpty { + params["permissions"] = OpenClawProtocol.AnyCodable(self.permissions) + } + } +} diff --git a/docs/nodes/computer-use.md b/docs/nodes/computer-use.md index 39b5962dbfdf..19969a62782d 100644 --- a/docs/nodes/computer-use.md +++ b/docs/nodes/computer-use.md @@ -7,15 +7,18 @@ read_when: title: "Computer use" --- -Computer use lets the gateway agent see and control a capable paired desktop. Eligibility is capability-based: the connected node must advertise both `computer.act` and `screen.snapshot`, whose result must include a `displayFrameId`. The tool captures a screenshot as its reference frame, then drives the pointer and keyboard through `computer.act`. The action set follows the core Anthropic computer-use actions; optional `computer_20251124` zoom is not exposed. A vision-capable model drives it through the built-in `computer` agent tool. +Computer use lets the gateway agent see and control a capable paired desktop. Eligibility is capability-based: the connected node must advertise both `computer.act` and `screen.snapshot`. The node's descriptor identifies the supported v2 action, target, observation, and delivery families, so the built-in `computer` tool exposes only what that provider can faithfully execute. Coordinate actions bind to a node-issued reference frame; capable providers can also address windows and elements, request background delivery, and return structured effect or refusal evidence. A vision-capable model drives the surface through the built-in `computer` agent tool. -The agent emits one uniform command, `computer.act`; it cannot tell how a node fulfills it. The bundled macOS app handles the command in-process with embedded Peekaboo services plus narrow CoreGraphics primitives (correct TCC permissions, no extra process). Windows and Linux can use the optional, experimental `cua-computer` plugin, which calls the packaged CUA Driver SDK directly. Both fulfillers use the same durable local enablement and pairing policy. +The agent emits one uniform command, `computer.act`; it cannot choose how a node fulfills it. On macOS, **Settings → General → Capabilities** selects the node-local provider: Peekaboo is the default and preserves the existing in-process coordinate-action path, while CUA uses a driver daemon embedded in `OpenClaw.app`. The app spawns that daemon directly so it inherits OpenClaw's Accessibility and Screen Recording grants, and the app-owned node worker connects through a private socket. Windows and Linux can use the optional, experimental `cua-computer` plugin, which calls the packaged CUA Driver SDK directly. + +Provider selection never falls back per action. Switching providers closes the active execution surface, rotates the provider generation, and re-advertises the node commands. A CUA failure therefore becomes an unavailable result instead of silently running the same action through Peekaboo. ## Requirements - A paired, connected node advertising both `computer.act` and `screen.snapshot`, with `screen.snapshot` returning `displayFrameId`. - **macOS fulfiller:** app setting **Allow Computer Control** enabled. It defaults on; an explicit off choice stays off. -- **macOS fulfiller:** **Accessibility** and Event Posting access granted to OpenClaw (for pointer/keyboard injection), plus **Screen Recording** permission (for `screen.snapshot`). +- **macOS fulfiller:** choose **Peekaboo** (default) or **CUA**. CUA is selectable only when the pinned driver is present in the signed app bundle; development builds without that artifact show **driver not bundled**. +- **macOS fulfiller:** **Accessibility** and **Screen Recording** granted to OpenClaw. The native Peekaboo path also requires Event Posting access for its CoreGraphics input primitives. - **Windows/Linux fulfiller:** bundled `cua-computer` plugin enabled. Its package includes the pinned CUA Driver SDK 0.19.3 runtime; no `cua-driver` executable, daemon, or MCP server is configured. - The pairing update that includes `computer.act` approved on the gateway. - A vision-capable agent model. @@ -31,11 +34,27 @@ The built-in `computer` tool takes one action per call. Coordinates are non-nega - Keyboard: `type` (text), `key` (combo such as `cmd+shift+t` or `Return`), `hold_key` (`text` combo held for `duration` seconds). - Pacing: `wait` (`duration` seconds). +Providers with the v2 window/element family can additionally expose `list_apps`, `list_windows`, `get_accessibility_tree`, `get_cursor_position`, `get_window_state`, `launch_app`, `kill_app`, `bring_to_front`, `set_value`, `zoom`, `escalate_scope`, and `invoke_menu`. The provider descriptor is authoritative; unavailable actions are omitted rather than emulated through another provider. + Modifier keys ride the `text` field on click and scroll actions (`shift`, `ctrl`, `alt`, `cmd`). After an input action the tool returns a fresh screenshot so the model can observe the result. If more than one computer-capable node is connected, pass `node` explicitly. Screenshots are kept **model-only**: they are never auto-delivered to the chat channel. Treat all on-screen content as untrusted input; the tool warns the model not to follow on-screen instructions that conflict with the user's request. -## Windows and Linux (experimental, via CUA Driver SDK) +## CUA Driver provider + +### macOS app-owned daemon + +The signed macOS app bundles the universal `cua-driver` 0.19.3 executable and offers **CUA** in the Computer Control provider picker. OpenClaw creates a private, owner-only socket directory under Application Support and starts `cua-driver serve --embedded` as a direct app child. It does not launch through the Gateway, the TypeScript worker, `open(1)`, or `NSWorkspace`; those paths would break macOS's TCC responsibility chain and create a second permission identity. + +The app waits until the private socket accepts connections before advertising CUA readiness. Its TypeScript node worker starts only the unprivileged MCP proxy against that socket and maps the same typed `computer.act` v2 actions used on other platforms. Permission changes restart the daemon, and provider changes, disabling Computer Control, app shutdown, or an unexpected child exit remove the advertised CUA commands until a fresh generation is ready. + +#### Trust model + +The embedded CUA daemon runs in unrestricted mode because bounded CUA grants require exact launch-time resources and cannot represent OpenClaw's runtime-discovered windows and elements. OpenClaw command arming, pairing approval, and tool policy are the authoritative authorization gate, identical to the shipped Peekaboo fulfiller. The app owns the daemon and its macOS TCC identity, and the daemon accepts local connections only through an owner-only socket directory. + +The CUA descriptor advertises window and element targets, background and foreground delivery, screenshots, and accessibility observations. Peekaboo remains the default in this release and advertises the existing coordinate-action family; its v2 adapter is separate work. + +### Windows and Linux (experimental, direct SDK) The bundled `cua-computer` plugin provides an experimental fulfiller for Windows and Linux node hosts. It is disabled by default and uses the pinned CUA Driver SDK 0.19.3 contract directly: @@ -63,21 +82,21 @@ This fulfiller currently controls only the primary display. `hold_key`, `left_mo The plugin calls `CuaDriver.createConfigured`, never bare `create()`. Its authorization ceiling, trusted session identifier, TTLs, and desktop scope are fixed by OpenClaw; model-facing `screen.snapshot` and `computer.act` inputs cannot select a session or widen that authority. Because the driver reports no stable display identity, frame authorization binds to the trusted session generation plus live primary-display geometry. A new session invalidates outstanding frames, but a same-geometry primary-display substitution inside one session cannot be detected; prefer a stable single-display session for this fulfiller. -This is a hard replacement of the former 0.10 daemon/MCP integration. OpenClaw does not spawn a CUA process, proxy an MCP client, or fall back to another CUA runtime. +On Windows and Linux this is a hard replacement of the former 0.10 daemon/MCP integration: OpenClaw does not spawn a CUA process or proxy an MCP client. macOS deliberately uses the app-owned embedded daemon described above so the driver remains in `OpenClaw.app`'s TCC responsibility chain. Neither path falls back to another provider for an individual action. ### Troubleshooting The `cua-computer` fulfiller surfaces typed error codes in the tool result and node logs. Common ones: -| Code | Cause | Fix | -| ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `COMPUTER_DRIVER_UNAVAILABLE` | The CUA Driver SDK runtime cannot initialize, the node is not Windows/Linux, or its desktop permissions/session are unavailable. | Run `openclaw node run` inside the interactive desktop session and check the platform desktop permissions. Reinstall OpenClaw if its bundled CUA Driver SDK package is missing. | -| `COMPUTER_REFUSED_` | The driver refused the action with a structured code such as `background_unavailable`, `background_occluded`, or `foreground_unavailable` (KDE/KWin Wayland). | Bring the target window forward, switch to X11, or use a supported compositor. See the compatibility notes above. | -| `COMPUTER_STALE_FRAME` | The coordinates referenced a screenshot that is no longer current (context compaction, a display geometry change, or a reference-width change). | Take a fresh `screenshot` before the coordinate action. | -| `COMPUTER_UNSUPPORTED_ACTION` | An action this fulfiller cannot faithfully deliver: `hold_key`, `left_mouse_down`, `left_mouse_up`, or modifier-held click/drag/scroll. | Use a supported action. The typed CUA Driver desktop contract has no held-input or modifier argument for these calls. | -| `COMPUTER_UNSUPPORTED_DISPLAY` | A non-primary `screenIndex`, a capture/screen geometry mismatch, or a cursor outside the primary display. | Drive the primary display only. | -| `COMPUTER_UNSUPPORTED_KEY` | A `key` value the driver cannot reproduce reliably: a digit or punctuation key whose shift state is layout-dependent, or an unknown key. | Send that text through the `type` action instead. | -| `COMPUTER_DRIVER_ERROR` / `COMPUTER_INVALID_REQUEST` | The driver failed without a structured code, or the action arguments were malformed. | Check the driver state and retake a screenshot; correct the action arguments. | +| Code | Cause | Fix | +| ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `COMPUTER_DRIVER_UNAVAILABLE` | The CUA runtime cannot initialize, the macOS app-owned endpoint is absent, or the desktop permissions/session are unavailable. | On macOS, verify CUA is selected and the bundled driver is ready; on Windows/Linux, run `openclaw node run` inside the interactive desktop session. Reinstall OpenClaw if the pinned runtime is missing. | +| `COMPUTER_REFUSED_` | The driver refused the action with a structured code such as `background_unavailable`, `background_occluded`, or `foreground_unavailable` (KDE/KWin Wayland). | Bring the target window forward, switch to X11, or use a supported compositor. See the compatibility notes above. | +| `COMPUTER_STALE_FRAME` | The coordinates referenced a screenshot that is no longer current (context compaction, a display geometry change, or a reference-width change). | Take a fresh `screenshot` before the coordinate action. | +| `COMPUTER_UNSUPPORTED_ACTION` | An action this fulfiller cannot faithfully deliver: `hold_key`, `left_mouse_down`, `left_mouse_up`, or modifier-held click/drag/scroll. | Use a supported action. The typed CUA Driver desktop contract has no held-input or modifier argument for these calls. | +| `COMPUTER_UNSUPPORTED_DISPLAY` | A non-primary `screenIndex`, a capture/screen geometry mismatch, or a cursor outside the primary display. | Drive the primary display only. | +| `COMPUTER_UNSUPPORTED_KEY` | A `key` value the driver cannot reproduce reliably: a digit or punctuation key whose shift state is layout-dependent, or an unknown key. | Send that text through the `type` action instead. | +| `COMPUTER_DRIVER_ERROR` / `COMPUTER_INVALID_REQUEST` | The driver failed without a structured code, or the action arguments were malformed. | Check the driver state and retake a screenshot; correct the action arguments. | ## The `computer.act` node command @@ -90,7 +109,7 @@ Reads reuse `screen.snapshot`; there is no second capture path. See [Camera and ## Authorization -1. Enable the platform fulfiller: on macOS, **Settings → Allow Computer Control** starts enabled, then grant **Accessibility** and **Screen Recording** under **Settings → Permissions**; on Windows/Linux, follow the experimental `cua-computer` setup above. +1. Enable the platform fulfiller: on macOS, **Settings → General → Capabilities → Allow Computer Control** starts enabled, then choose Peekaboo or CUA and grant **Accessibility** and **Screen Recording** under **Settings → Permissions**; on Windows/Linux, follow the experimental `cua-computer` setup above. 2. Approve the pairing update on the gateway (a new command forces re-pairing). 3. Expose the tool to the vision-capable agent. For the default `coding` profile: @@ -108,11 +127,11 @@ Once the node-local control is enabled and the pairing update is approved, `comp On macOS, default-on means a paired gateway can drive pointer and keyboard input as soon as the required macOS grants exist. There is no per-action confirmation. Turn off **Allow Computer Control** before pairing, or at any later time, to stop advertising and accepting `computer.act`. -`gateway.nodes.commands.deny` remains an explicit global revocation and always wins. For the macOS fulfiller, `computer.act` does not need a `gateway.nodes.commands.allow` entry. The experimental `cua-computer` plugin registers `computer.act` as a dangerous plugin node command, so once that plugin is enabled the operator must add it to `gateway.nodes.commands.allow` (see the Windows/Linux setup above); the plugin registration excludes it from the default allowlist regardless of platform. An authenticated operator with `operator.write` can invoke an enabled, paired command through `node.invoke`; there is no per-action admin check. +`gateway.nodes.commands.deny` remains an explicit global revocation and always wins. The native macOS Peekaboo fulfiller does not need a `gateway.nodes.commands.allow` entry. CUA registers `computer.act` as a dangerous plugin node command on every platform, so selecting CUA on macOS or enabling the plugin on Windows/Linux also requires an explicit `gateway.nodes.commands.allow` entry. An authenticated operator with `operator.write` can invoke an enabled, paired command through `node.invoke`; there is no per-action admin check. ## Safety -- Every layer (tool policy, gateway command policy, pairing, node-app setting, and platform permissions) must agree. For the current macOS fulfiller, that includes **Allow Computer Control**, Accessibility, and Screen Recording. Actions execute while those durable controls remain enabled; there is no per-action confirmation. +- Every layer (tool policy, gateway command policy, pairing, node-app setting, and platform permissions) must agree. On macOS that includes **Allow Computer Control**, Accessibility, and Screen Recording; the native Peekaboo path also requires Event Posting. Actions execute while those durable controls remain enabled; there is no per-action confirmation. - The macOS fulfiller posts text one grapheme at a time, so cancellation, disconnect, pause, disable, or endpoint replacement stops it before the next grapheme. The experimental CUA Driver fulfiller passes node cancellation to the SDK for each call. - Screenshots are model-only and never auto-sent to chat (issue [#44759](https://github.com/openclaw/openclaw/issues/44759)). - Treat screen content as untrusted; it can carry prompt injection. diff --git a/extensions/cua-computer/index.test.ts b/extensions/cua-computer/index.test.ts index 95276981a739..5dca38cc3ba9 100644 --- a/extensions/cua-computer/index.test.ts +++ b/extensions/cua-computer/index.test.ts @@ -24,6 +24,15 @@ function validateManifestConfig(value: unknown) { } describe("cua-computer plugin registration", () => { + it("defaults on only for the app-gated macOS provider path", () => { + const manifest = JSON.parse( + fs.readFileSync(new URL("./openclaw.plugin.json", import.meta.url), "utf8"), + ) as { enabledByDefault?: boolean; enabledByDefaultOnPlatforms?: string[] }; + + expect(manifest.enabledByDefault).toBe(false); + expect(manifest.enabledByDefaultOnPlatforms).toEqual(["darwin"]); + }); + it("registers the screen and dangerous computer node-host commands", () => { const commands: OpenClawPluginNodeHostCommand[] = []; const policies: OpenClawPluginNodeInvokePolicy[] = []; diff --git a/extensions/cua-computer/index.ts b/extensions/cua-computer/index.ts index b497400287fb..c769583e26bc 100644 --- a/extensions/cua-computer/index.ts +++ b/extensions/cua-computer/index.ts @@ -14,7 +14,7 @@ const configSchema = buildPluginConfigSchema(CuaComputerConfigSchema); export default definePluginEntry({ id: "cua-computer", name: "CUA Computer", - description: "Experimental CUA Driver SDK computer control for Windows and Linux node hosts.", + description: "Experimental CUA Driver computer control for macOS, Windows, and Linux node hosts.", configSchema, register(api) { const parsed = CuaComputerConfigSchema.safeParse(api.pluginConfig ?? {}); diff --git a/extensions/cua-computer/openclaw.plugin.json b/extensions/cua-computer/openclaw.plugin.json index 06deab12d316..5308ed5423bd 100644 --- a/extensions/cua-computer/openclaw.plugin.json +++ b/extensions/cua-computer/openclaw.plugin.json @@ -4,8 +4,9 @@ "onStartup": true }, "enabledByDefault": false, + "enabledByDefaultOnPlatforms": ["darwin"], "name": "CUA Computer", - "description": "Experimental CUA Driver SDK computer control for Windows and Linux node hosts.", + "description": "Experimental CUA Driver computer control for macOS, Windows, and Linux node hosts.", "configSchema": { "type": "object", "additionalProperties": false, diff --git a/extensions/cua-computer/package.json b/extensions/cua-computer/package.json index a5d014fd8aeb..b4e4a7de410e 100644 --- a/extensions/cua-computer/package.json +++ b/extensions/cua-computer/package.json @@ -1,7 +1,7 @@ { "name": "@openclaw/cua-computer", "version": "2026.8.1", - "description": "Experimental CUA Driver SDK computer control for Windows and Linux node hosts", + "description": "Experimental CUA Driver computer control for macOS, Windows, and Linux node hosts", "type": "module", "dependencies": { "@trycua/cua-driver": "0.19.3", diff --git a/extensions/cua-computer/src/commands.test.ts b/extensions/cua-computer/src/commands.test.ts index d404ba34e8cd..11570a317c54 100644 --- a/extensions/cua-computer/src/commands.test.ts +++ b/extensions/cua-computer/src/commands.test.ts @@ -35,10 +35,24 @@ function result(structured: Record, image = false): CuaToolResu }; } -function driver() { +function driver( + options: { + geometry?: typeof geometry; + screenSize?: { width: number; height: number; scale_factor: number }; + } = {}, +) { let generation = "execution-1"; - const getDesktopState = vi.fn(async () => result(geometry, true)); - const getScreenSize = vi.fn(async () => result({ width: 100, height: 50, scale_factor: 1 })); + const activeGeometry = options.geometry ?? geometry; + const getDesktopState = vi.fn(async () => result(activeGeometry, true)); + const getScreenSize = vi.fn(async () => + result( + options.screenSize ?? { + width: activeGeometry.screen_width, + height: activeGeometry.screen_height, + scale_factor: activeGeometry.scale_factor, + }, + ), + ); const click = vi.fn(async () => result({})); const drag = vi.fn(async () => result({})); const moveCursor = vi.fn(async () => result({})); @@ -157,6 +171,95 @@ describe("cua-computer provider", () => { expect(actions).toContain("get_window_state"); }); + it("advertises the macOS mapping only with a complete app-provided endpoint", () => { + const { session } = driver(); + const endpoint = { + OPENCLAW_CUA_DRIVER_SOCKET_PATH: "/tmp/openclaw-cua-test/driver.sock", + OPENCLAW_CUA_DRIVER_BINARY_PATH: process.execPath, + }; + const provider = createCuaComputerProvider({ + platform: "darwin", + env: endpoint, + driver: session, + }); + + expect(provider.isAvailable()).toBe(true); + expect(provider.capabilities().actions).toContain("get_window_state"); + expect(provider.capabilities().actions).not.toContain("left_mouse_down"); + expect(provider.capabilities().features).toEqual({ + recording: false, + agentCursor: false, + multiDisplay: false, + }); + + const createDriver = vi.fn(() => session); + expect( + createCuaComputerProvider({ + platform: "darwin", + env: endpoint, + createDriver, + }).isAvailable(), + ).toBe(true); + expect(createDriver).not.toHaveBeenCalled(); + + for (const env of [ + {}, + { OPENCLAW_CUA_DRIVER_SOCKET_PATH: endpoint.OPENCLAW_CUA_DRIVER_SOCKET_PATH }, + { OPENCLAW_CUA_DRIVER_BINARY_PATH: endpoint.OPENCLAW_CUA_DRIVER_BINARY_PATH }, + { ...endpoint, OPENCLAW_CUA_DRIVER_SOCKET_PATH: "relative.sock" }, + { ...endpoint, OPENCLAW_CUA_DRIVER_BINARY_PATH: "/missing/cua-driver" }, + ]) { + expect( + createCuaComputerProvider({ platform: "darwin", env, driver: session }).isAvailable(), + ).toBe(false); + } + }); + + it("keeps macOS Retina screenshots in native-pixel action coordinates", async () => { + const retina = driver({ + geometry: { + platform: "macos", + display: "primary", + screenshot_width: 200, + screenshot_height: 100, + screen_width: 100, + screen_height: 50, + scale_factor: 2, + }, + screenSize: { width: 100, height: 50, scale_factor: 2 }, + }); + const computer = await createCuaComputerProvider({ + platform: "darwin", + env: { + OPENCLAW_CUA_DRIVER_SOCKET_PATH: "/tmp/openclaw-cua-test/driver.sock", + OPENCLAW_CUA_DRIVER_BINARY_PATH: process.execPath, + }, + driver: retina.session, + imageProcessor: { + encode: vi.fn(async () => ({ data: Buffer.from("png"), width: 100, height: 50 })), + }, + }).openExecution({}); + const screen = JSON.parse(await computer.snapshot('{"format":"png","maxWidth":100}')) as { + displayFrameId: string; + width: number; + }; + + await computer.act( + JSON.stringify({ + action: "left_click", + displayFrameId: screen.displayFrameId, + refWidth: screen.width, + x: 10, + y: 10, + }), + ); + + expect(retina.click).toHaveBeenCalledWith( + { x: 20, y: 20, button: ClickButton.Left, count: 1 }, + undefined, + ); + }); + it("uses one typed session for snapshot and frame-authorized click", async () => { const { session, getDesktopState, getScreenSize, click } = driver(); const computer = await execution(session); diff --git a/extensions/cua-computer/src/commands.ts b/extensions/cua-computer/src/commands.ts index 4e060d17b740..785a3728207f 100644 --- a/extensions/cua-computer/src/commands.ts +++ b/extensions/cua-computer/src/commands.ts @@ -30,6 +30,7 @@ import { type CuaLastFrame, type CuaScreenSize, } from "./frame.js"; +import { createCuaMcpDriver } from "./mcp-driver-client.js"; import { handleV2Act, type CuaComputerActParams } from "./v2-actions.js"; const AVAILABILITY_POLL_MS = 5_000; @@ -38,6 +39,8 @@ const CUA_WIRE_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(1, 14); // capture, not the delivered frame. 8K (7680x4320 = ~33.2M) is a valid primary // display; budget above it so full-resolution snapshots reach the downscaler. const MAX_IMAGE_PIXELS = 40_000_000; +const CUA_DRIVER_SOCKET_PATH_ENV = "OPENCLAW_CUA_DRIVER_SOCKET_PATH"; +const CUA_DRIVER_BINARY_PATH_ENV = "OPENCLAW_CUA_DRIVER_BINARY_PATH"; const DesktopStateSchema = z.object({ platform: z.string().min(1), @@ -76,6 +79,30 @@ type CuaComputerProviderOptions = { clearInterval?: typeof clearInterval; }; +function resolveMacOsMcpEndpoint( + env: NodeJS.ProcessEnv, +): { socketPath: string; binaryPath: string } | undefined { + const socketPath = env[CUA_DRIVER_SOCKET_PATH_ENV]?.trim(); + const binaryPath = env[CUA_DRIVER_BINARY_PATH_ENV]?.trim(); + if (!socketPath || !binaryPath) { + return undefined; + } + if ( + socketPath.includes("\0") || + binaryPath.includes("\0") || + !path.isAbsolute(socketPath) || + !path.isAbsolute(binaryPath) + ) { + return undefined; + } + try { + fs.accessSync(binaryPath, fs.constants.X_OK); + } catch { + return undefined; + } + return { socketPath, binaryPath }; +} + class PromiseQueue { private tail: Promise = Promise.resolve(); @@ -204,7 +231,7 @@ function clickArgs( const modifiers = normalizeModifiers(params.modifiers); if (modifiers.length > 0) { throw new Error( - "COMPUTER_UNSUPPORTED_ACTION: modifier-held clicks are unsupported by cua-driver on Linux", + "COMPUTER_UNSUPPORTED_ACTION: modifier-held desktop clicks are unsupported by cua-driver", ); } return { @@ -397,6 +424,7 @@ export function createCuaComputerProvider( ): ComputerUseProvider { const platform = options.platform ?? process.platform; const env = options.env ?? process.env; + const macOsEndpoint = platform === "darwin" ? resolveMacOsMcpEndpoint(env) : undefined; let ownedDriver: CuaDriverSession | undefined; let stopped = false; // The node host owns one trusted SDK session for this command execution. @@ -405,7 +433,13 @@ export function createCuaComputerProvider( if (stopped) { throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-computer is stopping"); } - return options.driver ?? (ownedDriver ??= (options.createDriver ?? createCuaDriver)()); + return ( + options.driver ?? + (ownedDriver ??= ( + options.createDriver ?? + (macOsEndpoint ? () => createCuaMcpDriver({ ...macOsEndpoint, env }) : createCuaDriver) + )()) + ); }; const disposeOwnedDriver = async () => { stopped = true; @@ -416,8 +450,14 @@ export function createCuaComputerProvider( const imageProcessor = options.imageProcessor ?? createImageProcessor(env); const interval = options.setInterval ?? setInterval; const clear = options.clearInterval ?? clearInterval; - const isSupportedPlatform = platform === "linux" || platform === "win32"; - const isAvailable = () => isSupportedPlatform && driver().isAvailable(); + const isSupportedPlatform = + platform === "linux" || platform === "win32" || macOsEndpoint !== undefined; + // The app injects the endpoint only after the host-owned daemon socket is + // accepting connections. Node-host manifests are one-shot, so the validated + // endpoint pair is the synchronous macOS readiness lease; invocation still + // awaits the MCP initialize handshake and fails visibly if it cannot attach. + const isAvailable = () => + macOsEndpoint !== undefined || (isSupportedPlatform && driver().isAvailable()); return { id: "cua-computer", @@ -462,7 +502,9 @@ export function createCuaComputerProvider( await queue.run(async () => { if (!isSupportedPlatform) { throw new Error( - "COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux", + platform === "darwin" + ? `COMPUTER_DRIVER_UNAVAILABLE: cua-computer requires app-provided ${CUA_DRIVER_SOCKET_PATH_ENV} and ${CUA_DRIVER_BINARY_PATH_ENV}` + : "COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports macOS, Windows, and Linux", ); } const params = parseScreenSnapshotParamsJSON(paramsJSON); @@ -472,14 +514,14 @@ export function createCuaComputerProvider( const quality = Math.min(1, Math.max(0.05, params.quality ?? 0.72)); const desktop = await driver().getDesktopState(signal); const geometry = desktopGeometry(desktop); - // cua-driver desktop input consumes native get_desktop_state PNG pixels, - // and on every supported backend the driver reports screen geometry in - // that same physical-pixel space (Windows PMv2, Linux X11/Wayland). If a - // capture ever diverges from screen geometry, our screenshot->native - // scaling would mis-target input, so refuse rather than click blind. + // Windows and Linux report capture and input geometry in the same + // physical-pixel space. macOS intentionally reports logical screen + // points plus native Retina pixels; its desktop tools consume the + // native screenshot coordinates and undo that scale internally. if ( - geometry.screenWidth !== geometry.screenshotWidth || - geometry.screenHeight !== geometry.screenshotHeight + platform !== "darwin" && + (geometry.screenWidth !== geometry.screenshotWidth || + geometry.screenHeight !== geometry.screenshotHeight) ) { throw new Error( "COMPUTER_UNSUPPORTED_DISPLAY: cua-driver reported capture and screen geometry in different pixel spaces", @@ -514,7 +556,9 @@ export function createCuaComputerProvider( await queue.run(async () => { if (!isSupportedPlatform) { throw new Error( - "COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux", + platform === "darwin" + ? `COMPUTER_DRIVER_UNAVAILABLE: cua-computer requires app-provided ${CUA_DRIVER_SOCKET_PATH_ENV} and ${CUA_DRIVER_BINARY_PATH_ENV}` + : "COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports macOS, Windows, and Linux", ); } return await handleV2Act( diff --git a/extensions/cua-computer/src/mcp-driver-client.test.ts b/extensions/cua-computer/src/mcp-driver-client.test.ts new file mode 100644 index 000000000000..21f784270ea6 --- /dev/null +++ b/extensions/cua-computer/src/mcp-driver-client.test.ts @@ -0,0 +1,288 @@ +import fs from "node:fs/promises"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { ClickButton } from "./driver-client.js"; +import { createCuaMcpDriver } from "./mcp-driver-client.js"; + +type RpcRequest = { + id?: number; + method: string; + params?: { name?: string; arguments?: Record }; +}; + +type FakeEndpoint = { + binaryPath: string; + socketPath: string; + requests: RpcRequest[]; + respond: (request: RpcRequest, result: unknown) => void; + writeRaw: (request: RpcRequest, value: string) => void; + close: () => Promise; +}; + +async function createFakeEndpoint( + handle: (request: RpcRequest, endpoint: FakeEndpoint) => void, +): Promise { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-cua-mcp-test-")); + const socketPath = path.join(directory, "daemon.sock"); + const binaryPath = path.join(directory, "cua-driver"); + await fs.writeFile( + binaryPath, + `#!/usr/bin/env node +const net = require("node:net"); +const args = process.argv.slice(2); +if (args.length !== 4 || args[0] !== "mcp" || args[1] !== "--embedded" || args[2] !== "--socket") process.exit(64); +if (process.env.CUA_DRIVER_EMBEDDED !== undefined || process.env.CUA_DRIVER_PERMISSION_MODE !== undefined || process.env.CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS !== undefined || process.env.CUA_DRIVER_RS_TELEMETRY_ENABLED !== "false" || process.env.CUA_DRIVER_RS_UPDATE_CHECK !== "false") process.exit(65); +const socket = net.createConnection(args[3]); +process.stdin.pipe(socket); +socket.pipe(process.stdout); +socket.on("error", (error) => { process.stderr.write(error.message); process.exit(66); }); +socket.on("close", () => process.exit(0)); +`, + { mode: 0o700 }, + ); + + const requests: RpcRequest[] = []; + const connections = new Set(); + const writers = new Map(); + const endpoint = {} as FakeEndpoint; + const server = net.createServer((socket) => { + connections.add(socket); + socket.once("close", () => connections.delete(socket)); + let buffer = Buffer.alloc(0); + socket.on("data", (chunk: Buffer) => { + buffer = Buffer.concat([buffer, chunk]); + while (true) { + const newline = buffer.indexOf(0x0a); + if (newline < 0) { + break; + } + const line = buffer.subarray(0, newline); + buffer = buffer.subarray(newline + 1); + if (line.length === 0) { + continue; + } + const request = JSON.parse(line.toString("utf8")) as RpcRequest; + requests.push(request); + if (typeof request.id === "number") { + writers.set(request.id, socket); + } + handle(request, endpoint); + } + }); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(socketPath, () => { + resolve(); + }); + }); + + Object.assign(endpoint, { + binaryPath, + socketPath, + requests, + respond: (request: RpcRequest, result: unknown) => { + const writer = request.id === undefined ? undefined : writers.get(request.id); + writer?.write(`${JSON.stringify({ jsonrpc: "2.0", id: request.id, result })}\n`); + }, + writeRaw: (request: RpcRequest, value: string) => { + const writer = request.id === undefined ? undefined : writers.get(request.id); + writer?.write(value); + }, + close: async () => { + for (const connection of connections) { + connection.destroy(); + } + await new Promise((resolve) => { + server.close(() => { + resolve(); + }); + }); + await fs.rm(directory, { recursive: true, force: true }); + }, + }); + return endpoint; +} + +function toolResult(structuredContent: Record, image = false) { + return { + content: [ + { type: "text", text: "ok" }, + ...(image + ? [{ type: "image", mimeType: "image/png", data: Buffer.from("png").toString("base64") }] + : []), + ], + isError: false, + structuredContent, + }; +} + +function sessionState(scope: "window" | "desktop") { + return toolResult({ + session: "openclaw-test", + capture_scope: scope, + effective_scope: scope, + desktop_unlocked: scope === "desktop", + escalation_reason: null, + escalation_detail: null, + }); +} + +describe.runIf(process.platform !== "win32")("CUA MCP proxy transport", () => { + it("initializes the bundled proxy and translates tool results through CuaDriverSession", async () => { + let closed = false; + const endpoint = await createFakeEndpoint((request, fake) => { + if (request.method === "initialize") { + fake.respond(request, { + protocolVersion: "2025-06-18", + capabilities: { tools: {} }, + serverInfo: { name: "fake-cua-driver", version: "0.19.3" }, + }); + return; + } + if (request.method !== "tools/call") { + return; + } + switch (request.params?.name) { + case "start_session": + fake.respond(request, sessionState("desktop")); + break; + case "get_desktop_state": + fake.respond( + request, + toolResult( + { + platform: "macos", + display: "primary", + screenshot_width: 200, + screenshot_height: 100, + screen_width: 100, + screen_height: 50, + scale_factor: 2, + }, + true, + ), + ); + break; + case "click": + fake.respond( + request, + toolResult({ + effect: "confirmed", + route: "accessibility", + delivery: { mode: "background", delivered_count: 1 }, + evidence: [{ kind: "value_readback" }], + }), + ); + break; + case "end_session": + fake.respond(request, toolResult({ session: "openclaw-test", active: false })); + break; + default: + break; + } + }); + try { + const driver = createCuaMcpDriver({ + ...endpoint, + env: { + ...process.env, + CUA_DRIVER_EMBEDDED: "1", + CUA_DRIVER_PERMISSION_MODE: "unrestricted", + CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS: "1", + }, + }); + await vi.waitFor(() => expect(driver.isAvailable()).toBe(true)); + + const desktop = await driver.getDesktopState(); + expect(JSON.parse(desktop.structuredJson!)).toMatchObject({ + platform: "macos", + screenshot_width: 200, + }); + expect(desktop.images).toHaveLength(1); + const click = await driver.click({ x: 20, y: 30, button: ClickButton.Left, count: 1 }); + expect(click.action).toEqual({ + effect: 0, + route: 0, + delivery: { mode: 0, deliveredCount: 1 }, + evidence: [{ kind: 0 }], + }); + + await driver.dispose(); + await vi.waitFor(() => { + closed = endpoint.requests.some( + (request) => request.method === "tools/call" && request.params?.name === "end_session", + ); + expect(closed).toBe(true); + }); + expect(endpoint.requests.map((request) => request.method)).toContain( + "notifications/initialized", + ); + expect( + endpoint.requests.find( + (request) => request.method === "tools/call" && request.params?.name === "click", + )?.params?.arguments, + ).toMatchObject({ x: 20, y: 30, button: "left", count: 1, scope: "desktop" }); + } finally { + await endpoint.close(); + } + }); + + it("fails closed on invalid proxy JSON", async () => { + const endpoint = await createFakeEndpoint((request, fake) => { + if (request.method === "initialize" && request.id !== undefined) { + fake.writeRaw(request, "not-json\n"); + } + }); + try { + const driver = createCuaMcpDriver(endpoint); + await expect(driver.getDesktopState()).rejects.toThrow("COMPUTER_DRIVER_ERROR"); + expect(driver.isAvailable()).toBe(false); + await driver.dispose(); + } finally { + await endpoint.close(); + } + }); + + it("bounds pending calls and tears down the proxy on cancellation", async () => { + const held: RpcRequest[] = []; + const endpoint = await createFakeEndpoint((request, fake) => { + if (request.method === "initialize") { + fake.respond(request, { + protocolVersion: "2025-06-18", + capabilities: { tools: {} }, + serverInfo: { name: "fake-cua-driver", version: "0.19.3" }, + }); + } else if (request.method === "tools/call" && request.params?.name === "start_session") { + fake.respond(request, sessionState("window")); + } else if (request.method === "tools/call" && request.params?.name === "list_windows") { + held.push(request); + } else if (request.method === "tools/call" && request.params?.name === "end_session") { + fake.respond(request, toolResult({ session: "openclaw-test", active: false })); + } + }); + try { + const driver = createCuaMcpDriver(endpoint); + await vi.waitFor(() => expect(driver.isAvailable()).toBe(true)); + const calls = Array.from({ length: 65 }, () => driver.callTool("list_windows", {})); + await expect(calls[64]).rejects.toThrow("too many pending requests"); + await vi.waitFor(() => expect(held).toHaveLength(64)); + for (const request of held) { + endpoint.respond(request, toolResult({ windows: [] })); + } + await Promise.all(calls.slice(0, 64)); + + const controller = new AbortController(); + const cancelled = driver.callTool("list_windows", {}, controller.signal); + await vi.waitFor(() => expect(held).toHaveLength(65)); + controller.abort(new Error("test cancellation")); + await expect(cancelled).rejects.toThrow("request was cancelled"); + expect(driver.isAvailable()).toBe(false); + await driver.dispose(); + } finally { + await endpoint.close(); + } + }); +}); diff --git a/extensions/cua-computer/src/mcp-driver-client.ts b/extensions/cua-computer/src/mcp-driver-client.ts new file mode 100644 index 000000000000..eae2780b07e5 --- /dev/null +++ b/extensions/cua-computer/src/mcp-driver-client.ts @@ -0,0 +1,682 @@ +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import type { ActionResult } from "@trycua/cua-driver"; +import { + ClickButton, + EscalationReason, + ScrollDirection, + type CuaDriverSession, + type CuaToolResult, +} from "./driver-client.js"; + +const MCP_PROTOCOL_VERSION = "2025-06-18"; +const MCP_STARTUP_TIMEOUT_MS = 10_000; +const MCP_REQUEST_TIMEOUT_MS = 120_000; +const MCP_SHUTDOWN_TIMEOUT_MS = 2_000; +const MAX_MCP_LINE_BYTES = 256 * 1024 * 1024; +const MAX_PENDING_REQUESTS = 64; +const MAX_STDERR_BYTES = 32 * 1024; + +const ACTION_RESULT_TOOLS = new Set([ + "click", + "double_click", + "right_click", + "scroll", + "drag", + "mouse_drag", + "parallel_mouse_drag", + "move_cursor", + "mouse_button_down", + "mouse_button_up", + "type_text", + "type_text_chars", + "press_key", + "hotkey", + "set_value", + "set_window_frame", + "invoke_menu", + "browser_click", + "browser_pointer", + "browser_type", +]); + +type JsonRpcResponse = { + jsonrpc?: unknown; + id?: unknown; + result?: unknown; + error?: { code?: unknown; message?: unknown }; +}; + +type PendingRequest = { + resolve: (value: unknown) => void; + reject: (error: Error) => void; + timer: NodeJS.Timeout; + signal?: AbortSignal; + onAbort?: () => void; +}; + +type McpToolResult = { + content?: Array<{ type?: unknown; text?: unknown; data?: unknown; mimeType?: unknown }>; + isError?: unknown; + structuredContent?: unknown; +}; + +function driverUnavailable(message: string, cause?: unknown): Error { + return new Error(`COMPUTER_DRIVER_UNAVAILABLE: ${message}`, { cause }); +} + +function driverProtocolError(message: string, cause?: unknown): Error { + return new Error(`COMPUTER_DRIVER_ERROR: ${message}`, { cause }); +} + +function record(value: unknown): Record | undefined { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function mappedEnum(value: unknown, values: readonly string[], label: string): number { + if (typeof value !== "string") { + throw driverProtocolError(`CUA MCP ${label} is missing`); + } + const index = values.indexOf(value); + if (index < 0) { + throw driverProtocolError(`CUA MCP ${label} is invalid`); + } + return index; +} + +function mcpActionResult(tool: string, structured: unknown): ActionResult | undefined { + if (!ACTION_RESULT_TOOLS.has(tool)) { + return undefined; + } + const value = record(structured); + if (!value) { + throw driverProtocolError(`CUA MCP ${tool} returned no ActionResult`); + } + const delivery = record(value.delivery); + const escalation = record(value.escalation); + const evidence = Array.isArray(value.evidence) ? value.evidence : undefined; + return { + effect: mappedEnum( + value.effect, + ["confirmed", "partial", "unverifiable", "suspected_noop", "refused"], + "action effect", + ), + route: mappedEnum( + value.route, + ["accessibility", "synthetic_events", "global_input", "system_api", "dom", "trusted_input"], + "action route", + ), + ...(delivery + ? { + delivery: { + mode: mappedEnum( + delivery.mode, + ["background", "foreground", "not_applicable", "unknown"], + "delivery mode", + ), + ...(typeof delivery.delivered_count === "number" + ? { deliveredCount: delivery.delivered_count } + : {}), + }, + } + : {}), + ...(evidence + ? { + evidence: evidence.map((entry) => ({ + kind: mappedEnum( + record(entry)?.kind, + ["value_readback", "window_change"], + "evidence kind", + ), + })), + } + : {}), + ...(escalation + ? { + escalation: { + target: mappedEnum( + escalation.target, + ["pixel", "foreground", "page", "session"], + "escalation target", + ), + reason: mappedEnum( + escalation.reason, + [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required", + ], + "escalation reason", + ), + }, + } + : {}), + } as ActionResult; +} + +function normalizeMcpToolResult(tool: string, raw: unknown): CuaToolResult { + const value = record(raw) as McpToolResult | undefined; + if (!value) { + throw driverProtocolError(`CUA MCP ${tool} returned a non-object result`); + } + const content = Array.isArray(value.content) ? value.content : []; + const text = content.flatMap((entry) => + entry?.type === "text" && typeof entry.text === "string" ? [entry.text] : [], + ); + const images = content.flatMap((entry) => + entry?.type === "image" && typeof entry.data === "string" && typeof entry.mimeType === "string" + ? [{ dataBase64: entry.data, mimeType: entry.mimeType }] + : [], + ); + const structured = record(value.structuredContent); + const errorCode = + typeof structured?.code === "string" + ? structured.code + : typeof record(structured?.refusal)?.code === "string" + ? (record(structured?.refusal)?.code as string) + : undefined; + const isError = value.isError === true; + return { + text: text.join("\n"), + images, + ...(structured ? { structuredJson: JSON.stringify(structured) } : {}), + isError, + ...(errorCode ? { errorCode } : {}), + ...(!isError ? { action: mcpActionResult(tool, structured) } : {}), + degraded: structured?.degraded === true, + rawJson: JSON.stringify(raw), + }; +} + +class CuaMcpProxyClient { + private readonly child: ChildProcessWithoutNullStreams; + private readonly pending = new Map(); + private readonly ready: Promise; + private nextId = 0; + private stdout = Buffer.alloc(0); + private stderr = Buffer.alloc(0); + private available = false; + private failure: Error | undefined; + private stopped = false; + + constructor(binaryPath: string, socketPath: string, env: NodeJS.ProcessEnv) { + const proxyEnvironment = { ...env }; + for (const key of Object.keys(proxyEnvironment)) { + if (key.startsWith("CUA_DRIVER_") || key === "CUA_TELEMETRY_ENABLED") { + delete proxyEnvironment[key]; + } + } + this.child = spawn(binaryPath, ["mcp", "--embedded", "--socket", socketPath], { + env: { + ...proxyEnvironment, + CUA_DRIVER_RS_TELEMETRY_ENABLED: "false", + CUA_DRIVER_RS_UPDATE_CHECK: "false", + }, + stdio: ["pipe", "pipe", "pipe"], + }); + this.child.stdout.on("data", (chunk: Buffer) => this.handleStdout(chunk)); + this.child.stderr.on("data", (chunk: Buffer) => { + this.stderr = Buffer.concat([this.stderr, chunk]).subarray(-MAX_STDERR_BYTES); + }); + this.child.once("error", (error) => + this.fail(driverUnavailable("failed to start CUA MCP proxy", error)), + ); + this.child.once("exit", (code, signal) => { + if (!this.stopped) { + const detail = this.stderr.toString("utf8").trim(); + this.fail( + driverUnavailable( + `CUA MCP proxy exited (${signal ?? code ?? "unknown"})${detail ? `: ${detail}` : ""}`, + ), + ); + } + }); + this.ready = this.initialize(); + void this.ready.catch(() => {}); + } + + isAvailable(): boolean { + return this.available && !this.failure && !this.stopped; + } + + async callTool( + name: string, + args: Record, + signal?: AbortSignal, + ): Promise { + await this.ready; + return normalizeMcpToolResult( + name, + await this.request("tools/call", { name, arguments: args }, MCP_REQUEST_TIMEOUT_MS, signal), + ); + } + + async stop(): Promise { + if (this.stopped) { + return; + } + this.stopped = true; + this.available = false; + this.rejectPending(driverUnavailable("CUA MCP proxy is stopping")); + this.child.stdin.end(); + if (await this.waitForExit(MCP_SHUTDOWN_TIMEOUT_MS)) { + return; + } + this.child.kill("SIGTERM"); + if (await this.waitForExit(MCP_SHUTDOWN_TIMEOUT_MS)) { + return; + } + this.child.kill("SIGKILL"); + } + + private async initialize(): Promise { + const initialized = record( + await this.request( + "initialize", + { + protocolVersion: MCP_PROTOCOL_VERSION, + capabilities: {}, + clientInfo: { name: "openclaw-cua-computer", version: "1" }, + }, + MCP_STARTUP_TIMEOUT_MS, + ), + ); + if (initialized?.protocolVersion !== MCP_PROTOCOL_VERSION) { + throw driverProtocolError("CUA MCP proxy returned an incompatible protocol version"); + } + this.notify("notifications/initialized", {}); + this.available = true; + } + + private request( + method: string, + params: Record, + timeoutMs: number, + signal?: AbortSignal, + ): Promise { + if (this.failure) { + return Promise.reject(this.failure); + } + if (this.stopped) { + return Promise.reject(driverUnavailable("CUA MCP proxy is stopping")); + } + if (signal?.aborted) { + return Promise.reject(driverUnavailable("CUA MCP request was cancelled", signal.reason)); + } + if (this.pending.size >= MAX_PENDING_REQUESTS) { + return Promise.reject(driverUnavailable("CUA MCP proxy has too many pending requests")); + } + const id = ++this.nextId; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.fail(driverUnavailable(`CUA MCP ${method} timed out after ${timeoutMs}ms`)); + }, timeoutMs); + timer.unref?.(); + const pending: PendingRequest = { resolve, reject, timer, signal }; + if (signal) { + pending.onAbort = () => + this.fail(driverUnavailable("CUA MCP request was cancelled", signal.reason)); + signal.addEventListener("abort", pending.onAbort, { once: true }); + } + this.pending.set(id, pending); + this.write({ jsonrpc: "2.0", id, method, params }); + }); + } + + private notify(method: string, params: Record): void { + this.write({ jsonrpc: "2.0", method, params }); + } + + private write(value: Record): void { + this.child.stdin.write(`${JSON.stringify(value)}\n`, (error) => { + if (error) { + this.fail(driverUnavailable("failed writing to CUA MCP proxy", error)); + } + }); + } + + private handleStdout(chunk: Buffer): void { + if (this.failure || this.stopped) { + return; + } + this.stdout = Buffer.concat([this.stdout, chunk]); + if (this.stdout.length > MAX_MCP_LINE_BYTES) { + this.fail(driverProtocolError("CUA MCP response exceeded the line-size limit")); + return; + } + while (true) { + const newline = this.stdout.indexOf(0x0a); + if (newline < 0) { + return; + } + const line = this.stdout.subarray(0, newline); + this.stdout = this.stdout.subarray(newline + 1); + if (line.length === 0) { + continue; + } + let response: JsonRpcResponse; + try { + response = JSON.parse(line.toString("utf8")) as JsonRpcResponse; + } catch (error) { + this.fail(driverProtocolError("CUA MCP proxy returned invalid JSON", error)); + return; + } + if (response.jsonrpc !== "2.0") { + this.fail(driverProtocolError("CUA MCP proxy returned an invalid JSON-RPC version")); + return; + } + if (typeof response.id !== "number" || !Number.isSafeInteger(response.id)) { + this.fail(driverProtocolError("CUA MCP proxy returned an invalid response id")); + return; + } + const pending = this.pending.get(response.id); + if (!pending) { + continue; + } + this.pending.delete(response.id); + this.clearPending(pending); + if (response.error) { + const message = + typeof response.error.message === "string" + ? response.error.message + : "unknown JSON-RPC error"; + pending.reject(driverProtocolError(`CUA MCP request failed: ${message}`)); + } else { + pending.resolve(response.result); + } + } + } + + private fail(error: Error): void { + if (this.failure || this.stopped) { + return; + } + this.failure = error; + this.available = false; + this.rejectPending(error); + this.child.kill("SIGTERM"); + } + + private rejectPending(error: Error): void { + for (const pending of this.pending.values()) { + this.clearPending(pending); + pending.reject(error); + } + this.pending.clear(); + } + + private clearPending(pending: PendingRequest): void { + clearTimeout(pending.timer); + if (pending.signal && pending.onAbort) { + pending.signal.removeEventListener("abort", pending.onAbort); + } + } + + private async waitForExit(timeoutMs: number): Promise { + if (this.child.exitCode !== null || this.child.signalCode !== null) { + return true; + } + return await new Promise((resolve) => { + const onExit = () => { + clearTimeout(timer); + resolve(true); + }; + const timer = setTimeout(() => { + this.child.removeListener("exit", onExit); + resolve(false); + }, timeoutMs); + timer.unref?.(); + this.child.once("exit", onExit); + }); + } +} + +function sessionState(value: CuaToolResult): import("@trycua/cua-driver").SessionStateOutput { + if (value.isError || !value.structuredJson) { + throw driverProtocolError(value.text || "CUA MCP session operation failed"); + } + let structured: Record | undefined; + try { + structured = record(JSON.parse(value.structuredJson)); + } catch (error) { + throw driverProtocolError("CUA MCP session operation returned invalid JSON", error); + } + if (!structured) { + throw driverProtocolError("CUA MCP session operation returned invalid state"); + } + return { + session: typeof structured.session === "string" ? structured.session : "", + captureScope: mappedEnum( + structured.capture_scope, + ["auto", "window", "desktop"], + "capture scope", + ), + effectiveScope: mappedEnum( + structured.effective_scope, + ["window", "desktop"], + "effective scope", + ), + desktopUnlocked: structured.desktop_unlocked === true, + ...(typeof structured.escalation_reason === "string" + ? { + escalationReason: mappedEnum( + structured.escalation_reason, + [ + "ax_tree_pixel_mismatch", + "background_delivery_failed", + "foreground_ineffective", + "no_window_target", + "other", + ], + "escalation reason", + ), + } + : {}), + ...(typeof structured.escalation_detail === "string" + ? { escalationDetail: structured.escalation_detail } + : {}), + } as import("@trycua/cua-driver").SessionStateOutput; +} + +class McpCuaDriverSession implements CuaDriverSession { + readonly generation = randomUUID(); + private readonly publicSession = `openclaw-${randomUUID()}`; + private startPromise: Promise | undefined; + private captureScope: "window" | "desktop" | undefined; + private started = false; + private disposed = false; + + constructor(private readonly client: CuaMcpProxyClient) {} + + isAvailable(): boolean { + return !this.disposed && this.client.isAvailable(); + } + + resetAvailabilityCache(): void {} + + async callTool(name: string, args: Record, signal?: AbortSignal) { + await this.ensureStarted("window", signal); + return await this.client.callTool(name, { ...args, session: this.publicSession }, signal); + } + + async escalateScope(reason: EscalationReason, signal?: AbortSignal) { + await this.ensureStarted("window", signal); + const result = await this.client.callTool( + "escalate_session", + { + session: this.publicSession, + reason: [ + "ax_tree_pixel_mismatch", + "background_delivery_failed", + "foreground_ineffective", + "no_window_target", + "other", + ][reason], + }, + signal, + ); + this.captureScope = "desktop"; + return sessionState(result); + } + + async getDesktopState(signal?: AbortSignal) { + return await this.desktopTool("get_desktop_state", {}, signal); + } + + async getScreenSize(signal?: AbortSignal) { + return await this.desktopTool("get_screen_size", {}, signal); + } + + async click( + input: { x: number; y: number; button: ClickButton; count: number }, + signal?: AbortSignal, + ) { + return await this.desktopTool( + "click", + { + x: input.x, + y: input.y, + button: ["left", "right", "middle"][input.button], + count: input.count, + scope: "desktop", + }, + signal, + ); + } + + async drag( + input: { fromX: number; fromY: number; toX: number; toY: number; durationMs?: bigint }, + signal?: AbortSignal, + ) { + return await this.desktopTool( + "drag", + { + from_x: input.fromX, + from_y: input.fromY, + to_x: input.toX, + to_y: input.toY, + ...(input.durationMs === undefined ? {} : { duration_ms: Number(input.durationMs) }), + scope: "desktop", + }, + signal, + ); + } + + async moveCursor(input: { x: number; y: number }, signal?: AbortSignal) { + return await this.desktopTool( + "move_cursor", + { x: input.x, y: input.y, scope: "desktop" }, + signal, + ); + } + + async scroll( + input: { x: number; y: number; direction: ScrollDirection; amount: bigint }, + signal?: AbortSignal, + ) { + return await this.desktopTool( + "scroll", + { + x: input.x, + y: input.y, + direction: ["up", "down", "left", "right"][input.direction], + by: "line", + amount: Number(input.amount), + scope: "desktop", + }, + signal, + ); + } + + async typeText(text: string, signal?: AbortSignal) { + return await this.desktopTool("type_text", { text, scope: "desktop" }, signal); + } + + async pressKey(input: { key: string; modifiers: string[] }, signal?: AbortSignal) { + return await this.desktopTool( + "press_key", + { key: input.key, modifiers: input.modifiers, scope: "desktop" }, + signal, + ); + } + + async dispose(): Promise { + if (this.disposed) { + return; + } + this.disposed = true; + let failure: unknown; + try { + await this.startPromise; + if (this.started && this.client.isAvailable()) { + await this.client.callTool("end_session", { session: this.publicSession }); + } + } catch (error) { + failure = error; + } + try { + await this.client.stop(); + } catch (error) { + failure ??= error; + } + if (failure) { + throw failure instanceof Error + ? failure + : driverUnavailable("CUA MCP cleanup failed", failure); + } + } + + private async desktopTool( + name: string, + args: Record, + signal?: AbortSignal, + ): Promise { + await this.ensureStarted("desktop", signal); + return await this.client.callTool(name, { ...args, session: this.publicSession }, signal); + } + + private async ensureStarted(scope: "window" | "desktop", signal?: AbortSignal): Promise { + if (this.disposed) { + throw driverUnavailable("cua-computer is stopping"); + } + if (!this.startPromise) { + this.captureScope = scope; + const start = this.client + .callTool("start_session", { session: this.publicSession, capture_scope: scope }, signal) + .then((result) => { + if (result.isError) { + throw driverProtocolError(result.text || "CUA MCP start_session failed"); + } + this.started = true; + }); + this.startPromise = start; + try { + await start; + } catch (error) { + if (this.startPromise === start) { + this.startPromise = undefined; + } + throw error; + } + return; + } + await this.startPromise; + if (scope === "desktop" && this.captureScope !== "desktop") { + await this.escalateScope(EscalationReason.Other, signal); + } + } +} + +export function createCuaMcpDriver(options: { + binaryPath: string; + socketPath: string; + env?: NodeJS.ProcessEnv; +}): CuaDriverSession { + return new McpCuaDriverSession( + new CuaMcpProxyClient(options.binaryPath, options.socketPath, options.env ?? process.env), + ); +} diff --git a/scripts/check-changed.mts b/scripts/check-changed.mts index a47eb9e1879d..d004c552ab3e 100644 --- a/scripts/check-changed.mts +++ b/scripts/check-changed.mts @@ -132,7 +132,7 @@ const ANDROID_VERSION_SYNC_PATHS = new Set([ "apps/android/version.json", ]); const MACOS_APP_CI_PATH_RE = - /^(?:apps\/(?:macos|macos-mlx-tts|shared|swabble)\/|Swabble\/|src\/(?:worker\/workspace-rsync-receiver\.ts|gateway\/worker-environments\/workspace-(?:accepted-(?:remote-script|sync)|mutation-remote-script|rsync-path\.test|sync(?:-helpers)?)\.ts)$|scripts\/(?:codesign-mac-app|create-dmg|mac-elevation-host|notarize-mac-artifact|package-mac-app|package-mac-dist)\.sh$|scripts\/lib\/(?:plistbuddy|swift-toolchain)\.sh$|test\/scripts\/(?:codesign-mac-app|create-dmg|mac-elevation-host|notarize-mac-artifact|package-mac-app|package-mac-dist)\.test\.ts$)/u; + /^(?:apps\/(?:macos|macos-mlx-tts|shared|swabble)\/|Swabble\/|src\/(?:worker\/workspace-rsync-receiver\.ts|gateway\/worker-environments\/workspace-(?:accepted-(?:remote-script|sync)|mutation-remote-script|rsync-path\.test|sync(?:-helpers)?)\.ts)$|scripts\/(?:codesign-mac-app|create-dmg|mac-elevation-host|notarize-mac-artifact|package-mac-app|package-mac-dist|stage-cua-driver-macos)\.sh$|scripts\/lib\/(?:plistbuddy|swift-toolchain)\.sh$|test\/scripts\/(?:codesign-mac-app|create-dmg|mac-elevation-host|notarize-mac-artifact|package-mac-app|package-mac-dist)\.test\.ts$)/u; let corepackPnpmShimDir: string | undefined; let corepackPnpmShimCleanupRegistered = false; let cachedGeneratedExtensionAssetPaths: Set | undefined; diff --git a/scripts/ci-changed-scope.mjs b/scripts/ci-changed-scope.mjs index c58ccd601c90..305f103eef1b 100644 --- a/scripts/ci-changed-scope.mjs +++ b/scripts/ci-changed-scope.mjs @@ -46,7 +46,7 @@ const APPLE_SHARED_CONTRACT_FIXTURE_RE = const MACOS_NATIVE_RE = /^(apps\/macos\/|apps\/macos-mlx-tts\/|apps\/shared\/|apps\/swabble\/|Swabble\/)/; const MACOS_SCRIPT_SCOPE_RE = - /^(?:scripts\/(?:check-swift-tools|codesign-mac-app|create-dmg|format-swift|install-swift-tools|install-xcodegen|lint-swift|mac-elevation-host|notarize-mac-artifact|package-mac-app|package-mac-dist)\.sh|scripts\/lib\/(?:plistbuddy|swift-toolchain)\.sh|test\/scripts\/(?:codesign-mac-app|create-dmg|mac-elevation-host|notarize-mac-artifact|package-mac-app|package-mac-dist)\.test\.ts)$/; + /^(?:scripts\/(?:check-swift-tools|codesign-mac-app|create-dmg|format-swift|install-swift-tools|install-xcodegen|lint-swift|mac-elevation-host|notarize-mac-artifact|package-mac-app|package-mac-dist|stage-cua-driver-macos)\.sh|scripts\/lib\/(?:plistbuddy|swift-toolchain)\.sh|test\/scripts\/(?:codesign-mac-app|create-dmg|mac-elevation-host|notarize-mac-artifact|package-mac-app|package-mac-dist)\.test\.ts)$/; const WORKSPACE_RSYNC_RECEIVER_SCOPE_RE = /^src\/(?:worker\/workspace-rsync-receiver\.ts|gateway\/worker-environments\/workspace-(?:accepted-(?:remote-script|sync)|mutation-remote-script|rsync-path\.test|sync(?:-helpers)?)\.ts)$/; const IOS_BUILD_RE = diff --git a/scripts/codesign-mac-app.sh b/scripts/codesign-mac-app.sh index 63c070c96a6c..6792c3d3a388 100755 --- a/scripts/codesign-mac-app.sh +++ b/scripts/codesign-mac-app.sh @@ -361,6 +361,11 @@ if [ -f "$MLX_TTS_HELPER" ]; then echo "Signing MLX TTS helper"; sign_plain_item "$MLX_TTS_HELPER" fi +CUA_DRIVER="$APP_BUNDLE/Contents/Resources/cua-driver" +if [ -f "$CUA_DRIVER" ]; then + echo "Signing embedded CUA driver"; sign_plain_item "$CUA_DRIVER" +fi + # Sign main binary if [ -f "$APP_BUNDLE/Contents/MacOS/OpenClaw" ]; then echo "Signing main binary"; sign_item "$APP_BUNDLE/Contents/MacOS/OpenClaw" "$APP_ENTITLEMENTS" diff --git a/scripts/package-mac-app.sh b/scripts/package-mac-app.sh index e7a2cf4cd33c..ee5075ccfdfa 100755 --- a/scripts/package-mac-app.sh +++ b/scripts/package-mac-app.sh @@ -566,6 +566,9 @@ fi rm -rf "$APP_ROOT/Contents/Resources/ProviderIcons" cp -R "$PROVIDER_ICONS_SRC" "$APP_ROOT/Contents/Resources/ProviderIcons" +echo "🖥 Staging embedded CUA driver" +"$ROOT_DIR/scripts/stage-cua-driver-macos.sh" "$APP_ROOT/Contents/Resources/cua-driver" + echo "📦 Copying CLI installer" INSTALL_CLI_SRC="$ROOT_DIR/scripts/install-cli.sh" if [ ! -f "$INSTALL_CLI_SRC" ]; then diff --git a/scripts/stage-cua-driver-macos.sh b/scripts/stage-cua-driver-macos.sh new file mode 100755 index 000000000000..8151d301022f --- /dev/null +++ b/scripts/stage-cua-driver-macos.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +VERSION="0.19.3" +TAG="cua-driver-rs-v${VERSION}" +ASSET="cua-driver-rs-${VERSION}-darwin-universal-binary.tar.gz" +EXPECTED_SHA256="733e28a3782ac8d325f8fce8b5d97486c1054af755b40dfd086151b34c79377e" +DOWNLOAD_URL="https://github.com/trycua/cua/releases/download/${TAG}/${ASSET}" +CACHE_DIR="$ROOT_DIR/apps/macos/.build/cua-driver/${TAG}" +ARCHIVE="$CACHE_DIR/$ASSET" +DESTINATION="${1:-}" + +if [[ -z "$DESTINATION" || "$DESTINATION" == -* ]]; then + echo "Usage: scripts/stage-cua-driver-macos.sh " >&2 + exit 2 +fi + +verify_archive() { + [[ -f "$ARCHIVE" ]] || return 1 + local actual + actual="$(shasum -a 256 "$ARCHIVE" | awk '{print $1}')" + [[ "$actual" == "$EXPECTED_SHA256" ]] +} + +mkdir -p "$CACHE_DIR" +if ! verify_archive; then + rm -f "$ARCHIVE" + partial="$ARCHIVE.partial.$$" + trap 'rm -f "$partial"' EXIT + curl --fail --location --retry 3 --retry-delay 2 --output "$partial" "$DOWNLOAD_URL" + actual="$(shasum -a 256 "$partial" | awk '{print $1}')" + if [[ "$actual" != "$EXPECTED_SHA256" ]]; then + echo "ERROR: CUA driver archive sha256 mismatch: got $actual" >&2 + exit 1 + fi + mv "$partial" "$ARCHIVE" + trap - EXIT +fi + +extract_dir="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-cua-driver.XXXXXX")" +trap 'rm -rf "$extract_dir"' EXIT +tar -xzf "$ARCHIVE" -C "$extract_dir" cua-driver +source_binary="$extract_dir/cua-driver" +if [[ ! -f "$source_binary" || -L "$source_binary" ]]; then + echo "ERROR: CUA driver archive did not contain a regular cua-driver executable" >&2 + exit 1 +fi + +archs="$(/usr/bin/lipo -archs "$source_binary")" +if [[ " $archs " != *" arm64 "* || " $archs " != *" x86_64 "* ]]; then + echo "ERROR: CUA driver is not universal (architectures: $archs)" >&2 + exit 1 +fi + +mkdir -p "$(dirname "$DESTINATION")" +cp "$source_binary" "$DESTINATION" +chmod 0755 "$DESTINATION" +echo "Staged cua-driver $VERSION at $DESTINATION" diff --git a/scripts/test-projects.test-support.mts b/scripts/test-projects.test-support.mts index 0dccb576dcda..da8589903145 100644 --- a/scripts/test-projects.test-support.mts +++ b/scripts/test-projects.test-support.mts @@ -2510,6 +2510,7 @@ const SEMANTIC_TOOLING_TARGET_PATTERNS: Array<[RegExp, string[]]> = [ ], [/^scripts\/lib\/plistbuddy\.sh$/u, ["create-dmg", "package-mac-app", "package-mac-dist"]], [/^scripts\/lib\/swift-toolchain\.sh$/u, ["package-mac-app", "package-mac-dist"]], + [/^scripts\/stage-cua-driver-macos\.sh$/u, ["package-mac-app"]], [ /^scripts\/lib\/npm-publish-plan\.mjs$/u, [ diff --git a/test/scripts/package-mac-app.test.ts b/test/scripts/package-mac-app.test.ts index 0f640d51d1c7..669897a90703 100644 --- a/test/scripts/package-mac-app.test.ts +++ b/test/scripts/package-mac-app.test.ts @@ -1342,6 +1342,29 @@ describe("package-mac-app plist stamping", () => { ); }); + it("stages the pinned universal CUA driver before nested-code signing", () => { + const packageScript = readFileSync(scriptPath, "utf8"); + const stageScript = readFileSync("scripts/stage-cua-driver-macos.sh", "utf8"); + const codesignScript = readFileSync("scripts/codesign-mac-app.sh", "utf8"); + + expect(stageScript).toContain('TAG="cua-driver-rs-v${VERSION}"'); + expect(stageScript).toContain( + 'EXPECTED_SHA256="733e28a3782ac8d325f8fce8b5d97486c1054af755b40dfd086151b34c79377e"', + ); + expect(packageScript).toContain( + '"$ROOT_DIR/scripts/stage-cua-driver-macos.sh" "$APP_ROOT/Contents/Resources/cua-driver"', + ); + expect(packageScript.indexOf("Staging embedded CUA driver")).toBeLessThan( + packageScript.indexOf('echo "🔏 Signing bundle'), + ); + expect(codesignScript).toContain( + 'echo "Signing embedded CUA driver"; sign_plain_item "$CUA_DRIVER"', + ); + expect(codesignScript.indexOf("Signing embedded CUA driver")).toBeLessThan( + codesignScript.indexOf("# Finally sign the bundle"), + ); + }); + it("does not mask required Info.plist stamp failures", () => { const script = readFileSync(scriptPath, "utf8"); const stampBlock = script.slice(