feat(macos): add embedded CUA computer provider (#123635)

* feat(macos): embed CUA computer provider

* fix(macos): clarify embedded CUA trust posture

* fix(macos): contain embedded CUA daemon lifecycle

* fix(macos): reap orphaned CUA daemons

* fix(macos): record the spawned CUA daemon pid so reaping can terminate orphans

* chore(macos): refresh native i18n baseline for the computer control provider picker

* style(macos): satisfy swiftlint on the embedded CUA host and connect params

* refactor(gateway): move optional connect params to GatewayConnectOptions
This commit is contained in:
Peter Steinberger
2026-08-14 10:24:08 -07:00
committed by GitHub
parent 0803505259
commit 19ace6830b
35 changed files with 3497 additions and 80 deletions
+37
View File
@@ -23152,6 +23152,17 @@
}
]
},
{
"id": "native.apple.02f52db82466e98a",
"source": "CUA (driver not bundled)",
"surface": "apple",
"sites": [
{
"kind": "conditional-branch",
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift"
}
]
},
{
"id": "native.apple.cfbd34b9f70ab403",
"source": "Cache",
@@ -24118,6 +24129,17 @@
}
]
},
{
"id": "native.apple.f0969ec829f56eed",
"source": "Choose the node-local automation backend for snapshots and actions.",
"surface": "apple",
"sites": [
{
"kind": "ui-named-argument",
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift"
}
]
},
{
"id": "native.apple.e4b13f49bb435884",
"source": "Choose where the Gateway runs and how this Mac app reaches it.",
@@ -24624,6 +24646,21 @@
}
]
},
{
"id": "native.apple.7b46ae17b22676bf",
"source": "Computer Control provider",
"surface": "apple",
"sites": [
{
"kind": "ui-call",
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift"
},
{
"kind": "ui-named-argument",
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift"
}
]
},
{
"id": "native.apple.ab225ee73fa3efc2",
"source": "Config",
+31 -7
View File
@@ -113,6 +113,8 @@ final class AppState {
@ObservationIgnored private let voiceWakeGlobalSyncScheduler = VoiceWakeGlobalSyncScheduler()
@ObservationIgnored private var activeComputerPresenceTask: Task<Void, Never>?
@ObservationIgnored private var activeComputerPresenceUpdateGeneration: UInt64 = 0
@ObservationIgnored private var computerControlHostReconciliationTask: Task<Void, Never>?
@ObservationIgnored private var computerControlHostGeneration: UInt64 = 0
var isPaused: Bool {
didSet { self.ifNotPreview { AppDefaults.standard.set(self.isPaused, forKey: pauseDefaultsKey) } }
@@ -364,18 +366,40 @@ final class AppState {
self.ifNotPreview {
AppDefaults.standard.set(self.peekabooBridgeEnabled, forKey: peekabooBridgeEnabledKey)
}
self.applyPeekabooBridgeHostState()
self.applyComputerControlHostState()
}
}
/// PeekabooBridge shares Computer Control's local UI-automation surface, so the host only
/// runs while Computer Control is enabled. With Computer Control off, users drive Peekaboo
/// via its own Mac app instead of a second, separately toggled bridge here.
func applyPeekabooBridgeHostState() {
/// The selected provider owns the complete Computer Control execution surface.
/// Keep the unselected host stopped so one node execution never mixes backends.
func applyComputerControlHostState() {
self.ifNotPreview {
let computerControlEnabled = isComputerControlEnabled()
let shouldRun = self.peekabooBridgeEnabled && computerControlEnabled
Task { await PeekabooBridgeHostCoordinator.shared.setEnabled(shouldRun) }
let provider = ComputerControlProvider.current()
let peekabooBridgeEnabled = self.peekabooBridgeEnabled
self.computerControlHostGeneration &+= 1
let generation = self.computerControlHostGeneration
let predecessor = self.computerControlHostReconciliationTask
let task = Task { @MainActor [weak self] in
await predecessor?.value
guard let self, generation == self.computerControlHostGeneration else { return }
switch provider {
case .cua where computerControlEnabled:
await PeekabooBridgeHostCoordinator.shared.setEnabled(false)
guard generation == self.computerControlHostGeneration else { return }
await CuaDriverHostCoordinator.shared.setEnabled(true)
case .peekaboo:
await CuaDriverHostCoordinator.shared.setEnabled(false)
guard generation == self.computerControlHostGeneration else { return }
await PeekabooBridgeHostCoordinator.shared.setEnabled(
peekabooBridgeEnabled && computerControlEnabled)
case .cua:
await CuaDriverHostCoordinator.shared.setEnabled(false)
guard generation == self.computerControlHostGeneration else { return }
await PeekabooBridgeHostCoordinator.shared.setEnabled(false)
}
}
self.computerControlHostReconciliationTask = task
}
}
@@ -0,0 +1,59 @@
import Foundation
enum ComputerControlProvider: String, CaseIterable, Sendable {
case peekaboo
case cua
static func current(
defaults: UserDefaults = AppDefaults.standard,
cuaAvailable: Bool = CuaDriverArtifact.bundledExecutableURL != nil) -> Self
{
guard let rawValue = defaults.string(forKey: computerControlProviderKey),
let provider = Self(rawValue: rawValue)
else { return .peekaboo }
if provider == .cua, !cuaAvailable { return .peekaboo }
return provider
}
var displayName: String {
switch self {
case .peekaboo: "Peekaboo"
case .cua: "CUA"
}
}
}
struct CuaDriverWorkerEndpoint: Equatable, Sendable {
let socketPath: String
let binaryPath: String
}
enum CuaDriverWorkerEnvironment {
static let socketPath = "OPENCLAW_CUA_DRIVER_SOCKET_PATH"
static let binaryPath = "OPENCLAW_CUA_DRIVER_BINARY_PATH"
}
enum CuaDriverArtifact {
static let resourceName = "cua-driver"
static var bundledExecutableURL: URL? {
self.executableURL(in: Bundle.main.resourceURL)
}
static func executableURL(
in resourceURL: URL?,
fileManager: FileManager = .default) -> URL?
{
guard let resourceURL else { return nil }
let candidate = resourceURL.appendingPathComponent(self.resourceName, isDirectory: false)
guard let values = try? candidate.resourceValues(forKeys: [
.isRegularFileKey,
.isSymbolicLinkKey,
]),
values.isRegularFile == true,
values.isSymbolicLink != true,
fileManager.isExecutableFile(atPath: candidate.path)
else { return nil }
return candidate
}
}
@@ -43,6 +43,7 @@ let canvasEnabledKey = "openclaw.canvasEnabled"
let quickChatEnabledKey = "openclaw.quickChatEnabled"
let cameraEnabledKey = "openclaw.cameraEnabled"
let computerControlEnabledKey = "openclaw.computerControlEnabled"
let computerControlProviderKey = "openclaw.computerControlProvider"
let cookieSyncEnabledKey = "openclaw.cookieSyncEnabled"
let cookieSyncIntoProfileKey = "openclaw.cookieSyncIntoProfile"
let cookieSyncDomainsKey = "openclaw.cookieSyncDomains"
@@ -0,0 +1,686 @@
import AppKit
import Darwin
import Foundation
import OpenClawIPC
import OSLog
extension Notification.Name {
static let openclawCuaDriverAvailabilityChanged = Notification.Name(
"openclaw.cua-driver.availability-changed")
}
struct CuaDriverProcessLaunch: Sendable {
let executableURL: URL
let arguments: [String]
let environment: [String: String]
}
enum CuaDriverStderrEvent: Equatable, Sendable {
case notice(String)
case error(String)
}
final class CuaDriverStderrRelay: @unchecked Sendable {
static let managedModeNotice =
"""
CUA embedded driver running in managed unrestricted mode; \
OpenClaw command arming and pairing are the authorization boundary.
"""
private static let dangerBannerPrefix = "DANGER: Cua Driver is running in unrestricted mode"
private static let maximumBufferedBytes = 32 * 1024
private static let readChunkBytes = 4 * 1024
let pipe = Pipe()
private let lock = NSLock()
private let emit: @Sendable (CuaDriverStderrEvent) -> Void
private var buffer = Data()
private var started = false
private var stopped = false
private var emittedManagedModeNotice = false
init(emit: @escaping @Sendable (CuaDriverStderrEvent) -> Void) {
self.emit = emit
}
func startReading() {
let shouldStart = self.lock.withLock {
guard !self.started, !self.stopped else { return false }
self.started = true
return true
}
guard shouldStart else { return }
self.pipe.fileHandleForReading.readabilityHandler = { [weak self] handle in
guard let self else { return }
let data = handle.readSafely(upToCount: Self.readChunkBytes)
guard !data.isEmpty else {
self.stop()
return
}
self.consume(data)
}
}
func reportManagedMode() {
let shouldEmit = self.lock.withLock {
guard !self.stopped, !self.emittedManagedModeNotice else { return false }
self.emittedManagedModeNotice = true
return true
}
if shouldEmit {
self.emit(.notice(Self.managedModeNotice))
}
}
func stop() {
let tail = self.lock.withLock { () -> Data? in
guard !self.stopped else { return nil }
self.stopped = true
defer { self.buffer.removeAll(keepingCapacity: false) }
return self.buffer.isEmpty ? nil : self.buffer
}
self.pipe.fileHandleForReading.readabilityHandler = nil
try? self.pipe.fileHandleForReading.close()
try? self.pipe.fileHandleForWriting.close()
if let tail {
self.forward(tail)
}
}
private func consume(_ data: Data) {
let lines = self.lock.withLock { () -> [Data] in
guard !self.stopped else { return [] }
self.buffer.append(data)
if self.buffer.count > Self.maximumBufferedBytes {
self.buffer = Data(self.buffer.suffix(Self.maximumBufferedBytes))
}
var lines: [Data] = []
while let newline = self.buffer.firstIndex(of: 0x0A) {
lines.append(Data(self.buffer[..<newline]))
self.buffer.removeSubrange(...newline)
}
return lines
}
lines.forEach(self.forward)
}
private func forward(_ data: Data) {
let line = (String(bytes: data, encoding: .utf8) ?? "")
.trimmingCharacters(in: .whitespacesAndNewlines)
guard !line.isEmpty, !line.hasPrefix(Self.dangerBannerPrefix) else { return }
self.emit(.error(line))
}
}
@MainActor
protocol CuaDriverProcessControlling: AnyObject {
var isRunning: Bool { get }
/// Spawned daemon pid. OpenClaw records this itself because `serve` ignores
/// `--pid-file` and writes only the machine-global default path.
var processIdentifier: pid_t { get }
func closeLiveness()
func terminate()
func forceKill()
}
@MainActor
private final class FoundationCuaDriverProcess: CuaDriverProcessControlling {
let process: Process
private let livenessPipe: Pipe
private let stderrRelay: CuaDriverStderrRelay
init(process: Process, livenessPipe: Pipe, stderrRelay: CuaDriverStderrRelay) {
self.process = process
self.livenessPipe = livenessPipe
self.stderrRelay = stderrRelay
}
deinit {
try? self.livenessPipe.fileHandleForWriting.close()
self.stderrRelay.stop()
}
var isRunning: Bool {
self.process.isRunning
}
var processIdentifier: pid_t {
self.process.processIdentifier
}
func closeLiveness() {
try? self.livenessPipe.fileHandleForWriting.close()
}
func terminate() {
guard self.process.isRunning else { return }
self.process.terminate()
}
func forceKill() {
guard self.process.isRunning else { return }
_ = Darwin.kill(self.process.processIdentifier, SIGKILL)
}
}
struct CuaDriverSocketDirectory: Equatable, Sendable {
let url: URL
let socketPath: String
let device: UInt64
let inode: UInt64
var pidFilePath: String {
self.url.appendingPathComponent("cua.pid", isDirectory: false).path
}
}
enum CuaDriverHostError: LocalizedError {
case socketDirectory(String)
case socketPathTooLong
var errorDescription: String? {
switch self {
case let .socketDirectory(message):
"Could not prepare the CUA socket directory: \(message)"
case .socketPathTooLong:
"The private CUA socket path is too long"
}
}
}
/// Owns the embedded CUA daemon as a direct OpenClaw.app child so macOS TCC
/// attributes Accessibility and Screen Recording checks to this signed app.
@MainActor
final class CuaDriverHostCoordinator {
typealias ProcessLauncher = @MainActor (
CuaDriverProcessLaunch,
@escaping @Sendable (Int32) -> Void) throws -> any CuaDriverProcessControlling
typealias ReadinessProbe = @Sendable (String) async -> Bool
static let shared = CuaDriverHostCoordinator(
observeNotifications: true,
beforeDaemonStop: {
await MacNodeModeCoordinator.shared.prepareForCuaDaemonStop()
})
private static let maximumRestartAttempts = 5
private static let restartDelays: [Duration] = [
.seconds(1),
.seconds(2),
.seconds(4),
.seconds(8),
.seconds(10),
]
private struct RunningChild {
let generation: UInt64
let process: any CuaDriverProcessControlling
let socketDirectory: CuaDriverSocketDirectory
let executableURL: URL
}
private let logger = Logger(subsystem: "ai.openclaw", category: "cua-driver-host")
private let notificationCenter: NotificationCenter
private let artifactURL: @MainActor () -> URL?
private let applicationSupportURL: @MainActor () -> URL
private let bundleIdentifier: @MainActor () -> String?
private let processLauncher: ProcessLauncher
private let readinessProbe: ReadinessProbe
private let restartSleep: @Sendable (Duration) async -> Void
private let permissionSnapshot: @MainActor () async -> [Capability: CapabilityAuthorizationStatus]
private let beforeDaemonStop: @MainActor () async -> Void
private var desiredEnabled = false
private var runningChild: RunningChild?
private var readyEndpoint: CuaDriverWorkerEndpoint?
private var generation: UInt64 = 0
private var stoppingGenerations = Set<UInt64>()
private var restartAttempt = 0
private var restartTask: Task<Void, Never>?
private var reconciliationTail: Task<Void, Never>?
private var lastPermissionSnapshot: [Capability: CapabilityAuthorizationStatus]?
init(
notificationCenter: NotificationCenter = .default,
observeNotifications: Bool = false,
artifactURL: @escaping @MainActor () -> URL? = { CuaDriverArtifact.bundledExecutableURL },
applicationSupportURL: @escaping @MainActor () -> URL = {
let fileManager = FileManager.default
return fileManager.urls(for: .applicationSupportDirectory, in: .userDomainMask).first
?? fileManager.homeDirectoryForCurrentUser.appendingPathComponent("Library/Application Support")
},
bundleIdentifier: @escaping @MainActor () -> String? = { Bundle.main.bundleIdentifier },
processLauncher: @escaping ProcessLauncher = CuaDriverHostCoordinator.launchProcess,
readinessProbe: @escaping ReadinessProbe = { path in
await Task.detached { CuaDriverHostCoordinator.socketAcceptsConnections(path) }.value
},
restartSleep: @escaping @Sendable (Duration) async -> Void = { delay in
try? await Task.sleep(for: delay)
},
permissionSnapshot: @escaping @MainActor () async -> [Capability: CapabilityAuthorizationStatus] = {
await PermissionManager.authorizationStatus([.accessibility, .screenRecording])
},
beforeDaemonStop: @escaping @MainActor () async -> Void = {})
{
self.notificationCenter = notificationCenter
self.artifactURL = artifactURL
self.applicationSupportURL = applicationSupportURL
self.bundleIdentifier = bundleIdentifier
self.processLauncher = processLauncher
self.readinessProbe = readinessProbe
self.restartSleep = restartSleep
self.permissionSnapshot = permissionSnapshot
self.beforeDaemonStop = beforeDaemonStop
guard observeNotifications else { return }
notificationCenter.addObserver(
self,
selector: #selector(self.permissionsMayHaveChanged),
name: .openclawPermissionsChanged,
object: nil)
notificationCenter.addObserver(
self,
selector: #selector(self.permissionsMayHaveChanged),
name: NSApplication.didBecomeActiveNotification,
object: nil)
}
deinit {
self.restartTask?.cancel()
self.reconciliationTail?.cancel()
self.notificationCenter.removeObserver(self)
}
var workerEndpoint: CuaDriverWorkerEndpoint? {
self.readyEndpoint
}
func setEnabled(_ enabled: Bool) async {
let wasEnabled = self.desiredEnabled
self.desiredEnabled = enabled
if enabled, !wasEnabled {
self.restartAttempt = 0
}
if !enabled {
self.restartTask?.cancel()
self.restartTask = nil
self.restartAttempt = 0
}
await self.enqueueReconciliation(restart: false).value
}
func shutdown() async {
await self.setEnabled(false)
}
private func enqueueReconciliation(restart: Bool) -> Task<Void, Never> {
let predecessor = self.reconciliationTail
let task = Task { @MainActor [weak self] in
await predecessor?.value
guard let self else { return }
if restart {
await self.ensureStopped()
}
if self.desiredEnabled {
await self.ensureStarted()
} else {
await self.ensureStopped()
}
}
self.reconciliationTail = task
return task
}
private func ensureStarted() async {
guard self.runningChild == nil else { return }
let applicationSupportURL = self.applicationSupportURL()
let executableURL = self.artifactURL()
await Self.reapStaleSocketDirectories(
in: applicationSupportURL,
expectedExecutableURL: executableURL)
guard let executableURL else {
self.logger.info("embedded CUA remains unavailable because the driver is not bundled")
return
}
guard let hostBundleID = self.bundleIdentifier()?.trimmingCharacters(in: .whitespacesAndNewlines),
!hostBundleID.isEmpty
else {
self.logger.error("embedded CUA cannot start without a host bundle identifier")
return
}
let socketDirectory: CuaDriverSocketDirectory
do {
socketDirectory = try Self.createSocketDirectory(in: applicationSupportURL)
} catch {
self.logger.error("\(error.localizedDescription, privacy: .public)")
self.scheduleRestartIfNeeded()
return
}
self.generation &+= 1
let generation = self.generation
let launch = Self.makeProcessLaunch(
executableURL: executableURL,
socketPath: socketDirectory.socketPath,
hostBundleID: hostBundleID)
do {
let process = try self.processLauncher(launch) { [weak self] status in
Task { @MainActor [weak self] in
self?.processExited(generation: generation, status: status)
}
}
// Record the pid we spawned so startup/teardown reaping can attribute and
// terminate exactly this daemon; without it a reaper can only delete the
// directory and would leave an orphaned privileged process running.
Self.writeProcessIdentifier(process.processIdentifier, to: socketDirectory)
self.runningChild = RunningChild(
generation: generation,
process: process,
socketDirectory: socketDirectory,
executableURL: executableURL)
} catch {
Self.cleanupSocketDirectory(socketDirectory)
self.logger.error("embedded CUA launch failed: \(error.localizedDescription, privacy: .public)")
self.scheduleRestartIfNeeded()
return
}
let deadline = ContinuousClock.now + .seconds(10)
while ContinuousClock.now < deadline {
guard self.desiredEnabled,
let child = self.runningChild,
child.generation == generation,
child.process.isRunning
else {
await self.ensureStopped()
return
}
if await self.readinessProbe(socketDirectory.socketPath) {
self.lastPermissionSnapshot = await self.permissionSnapshot()
self.setReadyEndpoint(CuaDriverWorkerEndpoint(
socketPath: socketDirectory.socketPath,
binaryPath: executableURL.path))
self.logger.info("embedded CUA ready at \(socketDirectory.socketPath, privacy: .public)")
return
}
try? await Task.sleep(for: .milliseconds(50))
}
self.logger.error("embedded CUA startup timed out")
await self.ensureStopped()
self.scheduleRestartIfNeeded()
}
private func ensureStopped() async {
self.setReadyEndpoint(nil)
let applicationSupportURL = self.applicationSupportURL()
guard let child = self.runningChild else {
await Self.reapStaleSocketDirectories(
in: applicationSupportURL,
expectedExecutableURL: self.artifactURL())
return
}
// The worker owns the MCP proxy. Drain it before the app closes the
// privileged daemon so an execution can never cross generations.
await self.beforeDaemonStop()
self.stoppingGenerations.insert(child.generation)
child.process.closeLiveness()
await Self.waitUntilStopped(child.process, timeout: .seconds(2))
if child.process.isRunning {
child.process.terminate()
await Self.waitUntilStopped(child.process, timeout: .seconds(1))
}
if child.process.isRunning {
child.process.forceKill()
await Self.waitUntilStopped(child.process, timeout: .seconds(1))
}
if self.runningChild?.generation == child.generation {
self.runningChild = nil
}
self.stoppingGenerations.remove(child.generation)
if !child.process.isRunning {
Self.cleanupSocketDirectory(child.socketDirectory)
}
await Self.reapStaleSocketDirectories(
in: applicationSupportURL,
expectedExecutableURL: child.executableURL)
}
private func processExited(generation: UInt64, status: Int32) {
guard let child = self.runningChild, child.generation == generation else { return }
let expected = self.stoppingGenerations.contains(generation) || !self.desiredEnabled
self.setReadyEndpoint(nil)
child.process.closeLiveness()
self.runningChild = nil
Self.cleanupSocketDirectory(child.socketDirectory)
if expected {
self.stoppingGenerations.remove(generation)
return
}
self.logger.error("embedded CUA exited unexpectedly with status \(status, privacy: .public)")
self.scheduleRestartIfNeeded()
}
private func scheduleRestartIfNeeded() {
guard self.desiredEnabled,
self.restartTask == nil,
self.restartAttempt < Self.maximumRestartAttempts
else { return }
let delay = Self.restartDelays[self.restartAttempt]
self.restartAttempt += 1
let restartSleep = self.restartSleep
self.restartTask = Task { @MainActor [weak self] in
await restartSleep(delay)
guard !Task.isCancelled, let self, self.desiredEnabled else { return }
self.restartTask = nil
await self.enqueueReconciliation(restart: false).value
}
}
private func setReadyEndpoint(_ endpoint: CuaDriverWorkerEndpoint?) {
guard self.readyEndpoint != endpoint else { return }
self.readyEndpoint = endpoint
self.notificationCenter.post(name: .openclawCuaDriverAvailabilityChanged, object: nil)
}
@objc private nonisolated func permissionsMayHaveChanged(_: Notification) {
Task { @MainActor [weak self] in
await self?.restartAfterPermissionChangeIfNeeded()
}
}
private func restartAfterPermissionChangeIfNeeded() async {
let latest = await self.permissionSnapshot()
guard let previous = self.lastPermissionSnapshot else {
self.lastPermissionSnapshot = latest
return
}
guard latest != previous else { return }
self.lastPermissionSnapshot = latest
guard self.desiredEnabled, self.runningChild != nil else { return }
self.restartTask?.cancel()
self.restartTask = nil
await self.enqueueReconciliation(restart: true).value
}
static func makeProcessLaunch(
executableURL: URL,
socketPath: String,
hostBundleID: String,
inheritedEnvironment: [String: String] = ProcessInfo.processInfo.environment) -> CuaDriverProcessLaunch
{
var environment = inheritedEnvironment.filter { key, _ in
!key.hasPrefix("CUA_DRIVER_") && key != "CUA_TELEMETRY_ENABLED"
}
environment["CUA_DRIVER_EMBEDDED"] = "1"
environment["CUA_DRIVER_HOST_BUNDLE_ID"] = hostBundleID
environment["CUA_DRIVER_PERMISSION_MODE"] = "unrestricted"
environment["CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS"] = "1"
environment["CUA_DRIVER_RS_TELEMETRY_ENABLED"] = "false"
environment["CUA_DRIVER_RS_UPDATE_CHECK"] = "false"
environment["CUA_DRIVER_EMBEDDED_HOST_PID"] = String(ProcessInfo.processInfo.processIdentifier)
// Unrestricted is deliberate: CUA bounded mode accepts only exact launch-time resource grants
// (cua-driver-core/src/session_manifest.rs), not arbitrary runtime-discovered windows/elements.
// OpenClaw command arming, pairing, and tool policy own authorization upstream, matching the
// shipped Peekaboo fulfiller; the owner-only 0700 socket directory is the local trust boundary.
return CuaDriverProcessLaunch(
executableURL: executableURL,
arguments: [
"serve",
"--embedded",
"--parent-liveness-stdio",
"--no-permissions-gate",
"--socket",
socketPath,
// No --pid-file: `serve` ignores it and always writes the driver's
// global default path, which every cua-driver on the machine shares.
// OpenClaw records the spawned pid itself so reaping can attribute
// exactly the daemon this app owns.
"--host-bundle-id",
hostBundleID,
"--permission-mode",
"unrestricted",
"--dangerously-bypass-approvals",
],
environment: environment)
}
private static func launchProcess(
_ launch: CuaDriverProcessLaunch,
onTermination: @escaping @Sendable (Int32) -> Void) throws -> any CuaDriverProcessControlling
{
let process = Process()
let livenessPipe = try Self.makeLivenessPipe()
let logger = Logger(subsystem: "ai.openclaw", category: "cua-driver-host")
let stderrRelay = CuaDriverStderrRelay { event in
switch event {
case let .notice(message):
logger.notice("\(message, privacy: .public)")
case let .error(message):
logger.error("CUA driver stderr: \(message, privacy: .public)")
}
}
process.executableURL = launch.executableURL
process.arguments = launch.arguments
process.environment = launch.environment
process.standardInput = livenessPipe.fileHandleForReading
process.standardOutput = FileHandle.nullDevice
process.standardError = stderrRelay.pipe
process.terminationHandler = { terminated in
stderrRelay.stop()
onTermination(terminated.terminationStatus)
}
stderrRelay.startReading()
do {
try process.run()
} catch {
stderrRelay.stop()
throw error
}
stderrRelay.reportManagedMode()
return FoundationCuaDriverProcess(
process: process,
livenessPipe: livenessPipe,
stderrRelay: stderrRelay)
}
private static func waitUntilStopped(
_ process: any CuaDriverProcessControlling,
timeout: Duration) async
{
let deadline = ContinuousClock.now + timeout
while process.isRunning, ContinuousClock.now < deadline {
try? await Task.sleep(for: .milliseconds(25))
}
}
static func createSocketDirectory(in applicationSupportURL: URL) throws -> CuaDriverSocketDirectory {
let openClawRoot = applicationSupportURL.appendingPathComponent("OpenClaw", isDirectory: true)
let root = openClawRoot.appendingPathComponent("cua", isDirectory: true)
for directory in [applicationSupportURL, openClawRoot, root] {
var status = stat()
if lstat(directory.path, &status) != 0 {
guard errno == ENOENT, Darwin.mkdir(directory.path, 0o700) == 0 else {
throw CuaDriverHostError.socketDirectory(String(cString: strerror(errno)))
}
guard lstat(directory.path, &status) == 0 else {
throw CuaDriverHostError.socketDirectory(String(cString: strerror(errno)))
}
}
guard status.st_mode & mode_t(S_IFMT) == mode_t(S_IFDIR),
status.st_uid == geteuid()
else {
throw CuaDriverHostError.socketDirectory("support roots must be owned directories")
}
}
for _ in 0..<8 {
// Keep the endpoint well below sockaddr_un.sun_path even for long account names.
let token = UUID().uuidString.replacingOccurrences(of: "-", with: "").lowercased().prefix(16)
let directory = root.appendingPathComponent(String(token), isDirectory: true)
guard Darwin.mkdir(directory.path, 0o700) == 0 else {
if errno == EEXIST { continue }
throw CuaDriverHostError.socketDirectory(String(cString: strerror(errno)))
}
var status = stat()
guard lstat(directory.path, &status) == 0,
status.st_mode & mode_t(S_IFMT) == mode_t(S_IFDIR),
status.st_uid == geteuid(),
status.st_mode & 0o777 == 0o700
else {
_ = Darwin.rmdir(directory.path)
throw CuaDriverHostError.socketDirectory("created directory failed ownership checks")
}
let socketPath = directory.appendingPathComponent("cua.sock", isDirectory: false).path
guard socketPath.utf8.count < MemoryLayout.size(ofValue: sockaddr_un().sun_path) else {
_ = Darwin.rmdir(directory.path)
throw CuaDriverHostError.socketPathTooLong
}
var socketStatus = stat()
guard lstat(socketPath, &socketStatus) != 0, errno == ENOENT else {
_ = Darwin.rmdir(directory.path)
throw CuaDriverHostError.socketDirectory("socket path already exists")
}
return CuaDriverSocketDirectory(
url: directory,
socketPath: socketPath,
device: UInt64(status.st_dev),
inode: UInt64(status.st_ino))
}
throw CuaDriverHostError.socketDirectory("could not allocate a unique directory")
}
static func cleanupSocketDirectory(_ directory: CuaDriverSocketDirectory) {
var status = stat()
guard lstat(directory.url.path, &status) == 0,
status.st_mode & mode_t(S_IFMT) == mode_t(S_IFDIR),
status.st_uid == geteuid(),
UInt64(status.st_dev) == directory.device,
UInt64(status.st_ino) == directory.inode
else { return }
var socketStatus = stat()
if lstat(directory.socketPath, &socketStatus) == 0,
socketStatus.st_mode & mode_t(S_IFMT) == mode_t(S_IFSOCK),
socketStatus.st_uid == geteuid()
{
_ = Darwin.unlink(directory.socketPath)
}
var pidStatus = stat()
if lstat(directory.pidFilePath, &pidStatus) == 0,
pidStatus.st_mode & mode_t(S_IFMT) == mode_t(S_IFREG),
pidStatus.st_uid == geteuid()
{
_ = Darwin.unlink(directory.pidFilePath)
}
_ = Darwin.rmdir(directory.url.path)
}
nonisolated static func socketAcceptsConnections(_ socketPath: String) -> Bool {
guard let descriptor = self.connectUnixSocket(socketPath) else { return false }
defer { close(descriptor) }
return true
}
}
@@ -0,0 +1,314 @@
import Darwin
import Foundation
import OSLog
extension CuaDriverHostCoordinator {
static func makeLivenessPipe() throws -> Pipe {
let pipe = Pipe()
let descriptor = pipe.fileHandleForWriting.fileDescriptor
let flags = fcntl(descriptor, F_GETFD)
guard flags >= 0, fcntl(descriptor, F_SETFD, flags | FD_CLOEXEC) >= 0 else {
let code = errno
try? pipe.fileHandleForReading.close()
try? pipe.fileHandleForWriting.close()
throw NSError(domain: NSPOSIXErrorDomain, code: Int(code))
}
return pipe
}
static func reapStaleSocketDirectories(
in applicationSupportURL: URL,
expectedExecutableURL: URL?) async
{
let logger = Logger(subsystem: "ai.openclaw", category: "cua-driver-host")
for directory in self.ownedSocketDirectories(in: applicationSupportURL) {
guard let processIdentifier = self.readProcessIdentifier(in: directory) else {
self.cleanupSocketDirectory(directory)
continue
}
guard self.processIsAlive(processIdentifier) else {
self.cleanupSocketDirectory(directory)
continue
}
guard let expectedExecutableURL,
self.processExecutableMatches(
processIdentifier,
expectedExecutableURL: expectedExecutableURL)
else { continue }
if let hostPID = self.processEnvironmentValue(
processIdentifier,
key: "CUA_DRIVER_EMBEDDED_HOST_PID").flatMap(pid_t.init),
!self.processIsAlive(hostPID)
{
logger.error(
"""
reaping orphaned embedded CUA daemon \(processIdentifier, privacy: .public) \
whose host \(hostPID, privacy: .public) is gone
""")
} else {
logger.error(
"reaping owned embedded CUA daemon \(processIdentifier, privacy: .public) during lifecycle cleanup")
}
if await self.terminateProcess(
processIdentifier,
directory: directory,
expectedExecutableURL: expectedExecutableURL)
{
self.cleanupSocketDirectory(directory)
}
}
}
private static func ownedSocketDirectories(
in applicationSupportURL: URL) -> [CuaDriverSocketDirectory]
{
let openClawRoot = applicationSupportURL.appendingPathComponent("OpenClaw", isDirectory: true)
let root = openClawRoot.appendingPathComponent("cua", isDirectory: true)
for ancestor in [applicationSupportURL, openClawRoot, root] {
var status = stat()
guard lstat(ancestor.path, &status) == 0,
status.st_mode & mode_t(S_IFMT) == mode_t(S_IFDIR),
status.st_uid == geteuid()
else { return [] }
}
guard let children = try? FileManager.default.contentsOfDirectory(
at: root,
includingPropertiesForKeys: nil,
options: [.skipsHiddenFiles])
else { return [] }
return children.compactMap { child in
let name = child.lastPathComponent
guard name.utf8.count == 16,
name.utf8.allSatisfy({ (48...57).contains($0) || (97...102).contains($0) })
else { return nil }
var status = stat()
guard lstat(child.path, &status) == 0,
status.st_mode & mode_t(S_IFMT) == mode_t(S_IFDIR),
status.st_uid == geteuid(),
status.st_mode & 0o777 == 0o700
else { return nil }
return CuaDriverSocketDirectory(
url: child,
socketPath: child.appendingPathComponent("cua.sock").path,
device: UInt64(status.st_dev),
inode: UInt64(status.st_ino))
}
}
/// Records the spawned daemon pid inside its own owner-only socket directory.
/// `serve` ignores `--pid-file` and writes only a machine-global path shared by
/// every cua-driver, so the spawning app is the one authoritative source.
@discardableResult
static func writeProcessIdentifier(
_ processIdentifier: pid_t,
to directory: CuaDriverSocketDirectory) -> Bool
{
guard self.directoryIsUnchangedAndOwned(directory) else { return false }
let descriptor = Darwin.open(
directory.pidFilePath,
O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW,
0o600)
guard descriptor >= 0 else { return false }
defer { close(descriptor) }
let contents = Array("\(processIdentifier)".utf8)
return contents.withUnsafeBytes { bytes in
Darwin.write(descriptor, bytes.baseAddress, bytes.count) == bytes.count
}
}
private static func readProcessIdentifier(in directory: CuaDriverSocketDirectory) -> pid_t? {
guard self.directoryIsUnchangedAndOwned(directory) else { return nil }
let descriptor = Darwin.open(directory.pidFilePath, O_RDONLY | O_CLOEXEC | O_NOFOLLOW)
guard descriptor >= 0 else { return nil }
defer { close(descriptor) }
var status = stat()
guard fstat(descriptor, &status) == 0,
status.st_mode & mode_t(S_IFMT) == mode_t(S_IFREG),
status.st_uid == geteuid(),
status.st_size > 0,
status.st_size <= 32
else { return nil }
var buffer = [UInt8](repeating: 0, count: Int(status.st_size))
let count = buffer.withUnsafeMutableBytes { bytes in
Darwin.read(descriptor, bytes.baseAddress, bytes.count)
}
guard count == buffer.count,
let contents = String(bytes: buffer, encoding: .utf8),
let processIdentifier = pid_t(contents.trimmingCharacters(in: .whitespacesAndNewlines)),
processIdentifier > 1,
processIdentifier != getpid()
else { return nil }
return processIdentifier
}
private static func directoryIsUnchangedAndOwned(_ directory: CuaDriverSocketDirectory) -> Bool {
var status = stat()
return lstat(directory.url.path, &status) == 0 &&
status.st_mode & mode_t(S_IFMT) == mode_t(S_IFDIR) &&
status.st_uid == geteuid() &&
status.st_mode & 0o777 == 0o700 &&
UInt64(status.st_dev) == directory.device &&
UInt64(status.st_ino) == directory.inode
}
nonisolated static func connectUnixSocket(_ socketPath: String) -> Int32? {
let descriptor = socket(AF_UNIX, SOCK_STREAM, 0)
guard descriptor >= 0 else { return nil }
var address = sockaddr_un()
address.sun_family = sa_family_t(AF_UNIX)
let maximumLength = MemoryLayout.size(ofValue: address.sun_path)
guard socketPath.utf8.count < maximumLength else {
close(descriptor)
return nil
}
socketPath.withCString { source in
withUnsafeMutablePointer(to: &address.sun_path) { pointer in
let bytes = UnsafeMutableRawPointer(pointer).assumingMemoryBound(to: Int8.self)
memset(bytes, 0, maximumLength)
strncpy(bytes, source, maximumLength - 1)
}
}
let addressSize = socklen_t(MemoryLayout.size(ofValue: address))
let connected = withUnsafePointer(to: &address) { pointer in
pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { rebound in
connect(descriptor, rebound, addressSize) == 0
}
}
guard connected else {
close(descriptor)
return nil
}
return descriptor
}
private static func processIsAlive(_ processIdentifier: pid_t) -> Bool {
guard processIdentifier > 1 else { return false }
if Darwin.kill(processIdentifier, 0) == 0 { return true }
return errno == EPERM
}
private static func processExecutableURL(_ processIdentifier: pid_t) -> URL? {
var buffer = [CChar](repeating: 0, count: Int(PATH_MAX))
let length = proc_pidpath(processIdentifier, &buffer, UInt32(buffer.count))
guard length > 0 else { return nil }
let bytes = buffer.prefix { $0 != 0 }.map { UInt8(bitPattern: $0) }
guard let path = String(bytes: bytes, encoding: .utf8) else { return nil }
return URL(fileURLWithPath: path)
}
private static func processExecutableMatches(
_ processIdentifier: pid_t,
expectedExecutableURL: URL) -> Bool
{
guard let actualExecutableURL = self.processExecutableURL(processIdentifier) else { return false }
let actualPath = actualExecutableURL.resolvingSymlinksInPath().standardizedFileURL.path
let expectedPath = expectedExecutableURL.resolvingSymlinksInPath().standardizedFileURL.path
return actualPath == expectedPath
}
private static func processEnvironmentValue(
_ processIdentifier: pid_t,
key: String) -> String?
{
var argumentMaximum: Int32 = 0
var argumentMaximumSize = MemoryLayout<Int32>.size
var argumentMaximumMIB: [Int32] = [CTL_KERN, KERN_ARGMAX]
guard sysctl(
&argumentMaximumMIB,
u_int(argumentMaximumMIB.count),
&argumentMaximum,
&argumentMaximumSize,
nil,
0) == 0,
argumentMaximum > 0,
argumentMaximum <= 4 * 1024 * 1024
else { return nil }
var buffer = [UInt8](repeating: 0, count: Int(argumentMaximum))
var bufferSize = buffer.count
var processMIB: [Int32] = [CTL_KERN, KERN_PROCARGS2, processIdentifier]
let readSucceeded = buffer.withUnsafeMutableBytes { bytes in
sysctl(
&processMIB,
u_int(processMIB.count),
bytes.baseAddress,
&bufferSize,
nil,
0) == 0
}
guard readSucceeded, bufferSize >= MemoryLayout<Int32>.size else { return nil }
var argumentCount: Int32 = 0
withUnsafeMutableBytes(of: &argumentCount) { destination in
destination.copyBytes(from: buffer.prefix(destination.count))
}
guard argumentCount > 0 else { return nil }
var offset = MemoryLayout<Int32>.size
func skipString() -> Bool {
guard offset < bufferSize else { return false }
while offset < bufferSize, buffer[offset] != 0 {
offset += 1
}
guard offset < bufferSize else { return false }
offset += 1
return true
}
guard skipString() else { return nil }
while offset < bufferSize, buffer[offset] == 0 {
offset += 1
}
for _ in 0..<argumentCount where offset < bufferSize {
guard skipString() else { return nil }
}
let prefix = Data("\(key)=".utf8)
while offset < bufferSize {
while offset < bufferSize, buffer[offset] == 0 {
offset += 1
}
guard offset < bufferSize else { break }
let start = offset
guard skipString() else { break }
let entry = Data(buffer[start..<(offset - 1)])
if entry.starts(with: prefix) {
return String(bytes: entry.dropFirst(prefix.count), encoding: .utf8)
}
}
return nil
}
private static func terminateProcess(
_ processIdentifier: pid_t,
directory: CuaDriverSocketDirectory,
expectedExecutableURL: URL) async -> Bool
{
guard self.readProcessIdentifier(in: directory) == processIdentifier,
self.processExecutableMatches(
processIdentifier,
expectedExecutableURL: expectedExecutableURL)
else { return false }
if Darwin.kill(processIdentifier, SIGTERM) != 0, errno != ESRCH { return false }
if await self.waitForProcessExit(processIdentifier) { return true }
// Recheck the executable immediately before escalation so PID reuse can
// never redirect SIGKILL to an unrelated process.
guard self.processExecutableMatches(
processIdentifier,
expectedExecutableURL: expectedExecutableURL)
else { return false }
if Darwin.kill(processIdentifier, SIGKILL) != 0, errno != ESRCH { return false }
return await self.waitForProcessExit(processIdentifier)
}
private static func waitForProcessExit(_ processIdentifier: pid_t) async -> Bool {
let deadline = ContinuousClock.now + .seconds(1)
while self.processIsAlive(processIdentifier), ContinuousClock.now < deadline {
try? await Task.sleep(for: .milliseconds(25))
}
return !self.processIsAlive(processIdentifier)
}
}
@@ -16,7 +16,10 @@ struct GeneralSettings: View {
@Bindable var state: AppState
@AppStorage(cameraEnabledKey) private var cameraEnabled: Bool = false
@AppStorage(computerControlEnabledKey) private var computerControlEnabled: Bool = true
@AppStorage(computerControlEnabledKey, store: AppDefaults.standard)
private var computerControlEnabled: Bool = true
@AppStorage(computerControlProviderKey, store: AppDefaults.standard)
private var computerControlProviderRaw: String = ComputerControlProvider.peekaboo.rawValue
let page: Page
let isActive: Bool
private let healthStore = HealthStore.shared
@@ -66,8 +69,10 @@ struct GeneralSettings: View {
QuickChatController.shared.setEnabled(enabled)
}
.onChange(of: self.computerControlEnabled) { _, _ in
// Turning Computer Control on/off must start or stop the gated PeekabooBridge host.
self.state.applyPeekabooBridgeHostState()
self.state.applyComputerControlHostState()
}
.onChange(of: self.computerControlProviderRaw) { _, _ in
self.state.applyComputerControlHostState()
}
.onReceive(NotificationCenter.default.publisher(for: NSApplication.didBecomeActiveNotification)) { _ in
self.refreshComputerControlPermissions()
@@ -141,6 +146,8 @@ struct GeneralSettings: View {
""",
binding: self.$computerControlEnabled)
self.computerControlProviderRow
SettingsCardRow(
title: "Computer Control access",
subtitle: .verbatim(self.computerControlPermissions.diagnostic.detailText))
@@ -874,6 +881,43 @@ struct GeneralSettings: View {
}
extension GeneralSettings {
@ViewBuilder
private var computerControlProviderRow: some View {
if self.computerControlEnabled {
SettingsCardRow(
title: "Computer Control provider",
subtitle: "Choose the node-local automation backend for snapshots and actions.")
{
Picker("Computer Control provider", selection: self.computerControlProviderBinding) {
Text(ComputerControlProvider.peekaboo.displayName)
.tag(ComputerControlProvider.peekaboo)
Text(self.cuaDriverBundled ? "CUA" : "CUA (driver not bundled)")
.tag(ComputerControlProvider.cua)
.disabled(!self.cuaDriverBundled)
}
.labelsHidden()
.pickerStyle(.menu)
.frame(width: 220, alignment: .trailing)
}
}
}
private var cuaDriverBundled: Bool {
CuaDriverArtifact.bundledExecutableURL != nil
}
private var computerControlProviderBinding: Binding<ComputerControlProvider> {
Binding(
get: {
let selected = ComputerControlProvider(rawValue: self.computerControlProviderRaw) ?? .peekaboo
return selected == .cua && !self.cuaDriverBundled ? .peekaboo : selected
},
set: { provider in
guard provider != .cua || self.cuaDriverBundled else { return }
self.computerControlProviderRaw = provider.rawValue
})
}
private var cookieSyncStatusTitle: String {
switch self.cookieSyncManager.state {
case .stopped: "Stopped"
+4 -1
View File
@@ -408,6 +408,9 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
var nodeTerminationCleanup: @MainActor () async -> Void = {
await TalkMLXSpeechSynthesizer.shared.shutdown()
await MacNodeModeCoordinator.shared.stopAndWait()
// The worker owns the MCP proxy. Stop it before closing the app-owned
// daemon socket so an in-flight completion cannot be mistaken for retryable.
await CuaDriverHostCoordinator.shared.shutdown()
}
var peekabooBridgeTerminationCleanup: @MainActor () async -> Void = {
@@ -625,7 +628,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
Task { PresenceReporter.shared.start() }
Task { await HealthStore.shared.refresh(onDemand: true) }
Task { await PortGuardian.shared.sweep(mode: AppStateStore.shared.connectionMode) }
AppStateStore.shared.applyPeekabooBridgeHostState()
AppStateStore.shared.applyComputerControlHostState()
if launchPlan.allowsAutomaticPresentation {
DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) {
if !PostUpdateController.shared.startIfNeeded() {
@@ -1,6 +1,7 @@
import Darwin
import Foundation
import OpenClawKit
import OpenClawProtocol
import OSLog
import Subprocess
@@ -14,16 +15,37 @@ struct MacNodeHostManifest: Equatable, Sendable {
let version: String
let caps: [String]
let commands: [String]
let computerUse: AnyCodable?
let pathEnv: String
init(
version: String,
caps: [String],
commands: [String],
computerUse: AnyCodable? = nil,
pathEnv: String)
{
self.version = version
self.caps = caps
self.commands = commands
self.computerUse = computerUse
self.pathEnv = pathEnv
}
}
struct MacNodeHostWorkerLaunch: Equatable, Sendable {
let command: [String]
let currentDirectoryURL: URL?
let environment: [String: String]
init(command: [String], currentDirectoryURL: URL? = nil) {
init(
command: [String],
currentDirectoryURL: URL? = nil,
environment: [String: String] = [:])
{
self.command = command
self.currentDirectoryURL = currentDirectoryURL
self.environment = environment
}
}
@@ -321,6 +343,9 @@ final class MacNodeHostWorker: MacNodeHostWorking, @unchecked Sendable {
return
}
var environment = ProcessInfo.processInfo.environment
environment.removeValue(forKey: CuaDriverWorkerEnvironment.socketPath)
environment.removeValue(forKey: CuaDriverWorkerEnvironment.binaryPath)
environment.merge(launch.environment, uniquingKeysWith: { _, explicit in explicit })
environment["PATH"] = CommandResolver.preferredPaths().joined(separator: ":")
environment["OPENCLAW_NODE_EXEC_HOST"] = "app"
environment["OPENCLAW_NODE_EXEC_FALLBACK"] = "0"
@@ -459,7 +484,25 @@ final class MacNodeHostWorker: MacNodeHostWorking, @unchecked Sendable {
self.stopLocked(reason: "worker returned invalid manifest")
return
}
let manifest = MacNodeHostManifest(version: version, caps: caps, commands: commands, pathEnv: pathEnv)
let computerUse: AnyCodable?
if let rawComputerUse = rawManifest["computerUse"] {
guard let rawComputerUse = rawComputerUse as? [String: Any],
let data = try? JSONSerialization.data(withJSONObject: rawComputerUse),
let decoded = try? JSONDecoder().decode(AnyCodable.self, from: data)
else {
self.stopLocked(reason: "worker returned invalid computer-use descriptor")
return
}
computerUse = decoded
} else {
computerUse = nil
}
let manifest = MacNodeHostManifest(
version: version,
caps: caps,
commands: commands,
computerUse: computerUse,
pathEnv: pathEnv)
self.manifest = manifest
self.inventoryData = (message["inventory"] as? [String: Any]).flatMap(Self.jsonData)
self.finishStartLocked(.success(manifest))
@@ -2,6 +2,7 @@ import AppKit
import Foundation
import OpenClawIPC
import OpenClawKit
import OpenClawProtocol
import OSLog
struct MacNodeGatewayTLSSessionCache {
@@ -115,6 +116,7 @@ final class MacNodeModeCoordinator: NSObject {
private var activeNodeHostWorkerInput: MacNodeHostWorkerRetryPolicy.Input?
private var lastObservedPaused: Bool
private var lastObservedComputerControlEnabled: Bool
private var lastObservedComputerControlProvider: ComputerControlProvider
private let runtime: MacNodeRuntime
private let session: GatewayNodeSession
private let nodeHostWorker: (any MacNodeHostWorking)?
@@ -155,6 +157,7 @@ final class MacNodeModeCoordinator: NSObject {
observeNotifications: Bool = false,
initialPaused: Bool? = nil,
initialComputerControlEnabled: Bool? = nil,
initialComputerControlProvider: ComputerControlProvider? = nil,
nodeHostWorkerRetrySleep: @escaping @Sendable (UInt64) async throws -> Void = {
try await Task.sleep(nanoseconds: $0)
},
@@ -173,6 +176,8 @@ final class MacNodeModeCoordinator: NSObject {
self.lastObservedPaused = initialPaused ?? AppDefaults.standard.bool(forKey: pauseDefaultsKey)
self.lastObservedComputerControlEnabled = initialComputerControlEnabled ??
isComputerControlEnabled()
self.lastObservedComputerControlProvider = initialComputerControlProvider ??
ComputerControlProvider.current()
super.init()
guard observeNotifications else { return }
@@ -206,6 +211,11 @@ final class MacNodeModeCoordinator: NSObject {
selector: #selector(self.nodeHostConfigurationChanged),
name: .openclawCLIInstalled,
object: nil)
self.notificationCenter.addObserver(
self,
selector: #selector(self.nodeHostConfigurationChanged),
name: .openclawCuaDriverAvailabilityChanged,
object: nil)
}
deinit {
@@ -248,6 +258,11 @@ final class MacNodeModeCoordinator: NSObject {
await self.beginTerminalStop().value
}
func prepareForCuaDaemonStop() async {
self.resetNodeHostWorkerRetryState()
await self.enqueueRouteInvalidation(mode: .workerRestart).value
}
private func beginTerminalStop() -> Task<Void, Never> {
if let terminalStopTask = self.terminalStopTask {
return terminalStopTask
@@ -285,7 +300,8 @@ final class MacNodeModeCoordinator: NSObject {
func refresh() {
self.refresh(
isPaused: AppDefaults.standard.bool(forKey: pauseDefaultsKey),
computerControlEnabled: isComputerControlEnabled())
computerControlEnabled: isComputerControlEnabled(),
computerControlProvider: ComputerControlProvider.current())
}
func currentCanvasPluginSurfaceRoute() async -> GatewayCanvasHostRoute? {
@@ -321,17 +337,25 @@ final class MacNodeModeCoordinator: NSObject {
await self.session.refreshCanvasHostRoute(replacing: observedURL)
}
private func refresh(isPaused: Bool, computerControlEnabled: Bool) {
private func refresh(
isPaused: Bool,
computerControlEnabled: Bool,
computerControlProvider: ComputerControlProvider)
{
let providerChanged = self.lastObservedComputerControlProvider != computerControlProvider
let shouldRevoke = Self.controlTransitionRequiresRouteInvalidation(
previousPaused: self.lastObservedPaused,
nextPaused: isPaused,
previousComputerControlEnabled: self.lastObservedComputerControlEnabled,
nextComputerControlEnabled: computerControlEnabled)
nextComputerControlEnabled: computerControlEnabled,
previousComputerControlProvider: self.lastObservedComputerControlProvider,
nextComputerControlProvider: computerControlProvider)
self.lastObservedPaused = isPaused
self.lastObservedComputerControlEnabled = computerControlEnabled
self.lastObservedComputerControlProvider = computerControlProvider
if shouldRevoke {
self.enqueueRouteInvalidation(mode: .reconnectRefresh)
self.enqueueRouteInvalidation(mode: providerChanged ? .workerRestart : .reconnectRefresh)
} else {
// Routine permission/foreground/defaults refreshes invalidate only
// suspended setup. The installed route remains authoritative.
@@ -525,10 +549,14 @@ final class MacNodeModeCoordinator: NSObject {
claudeSessionCatalogEnabled: Bool) async throws -> ConnectionAttempt?
{
let config = endpoint.config
let workerManifest = try await self.startNodeHostWorkerIfConfigured()
let provider = ComputerControlProvider.current()
let workerManifest = try await Self.workerManifest(
self.startNodeHostWorkerIfConfigured(provider: provider),
for: provider)
let nativeCaps = self.currentCaps(
browserControlEnabled: browserControlEnabled,
cameraEnabled: cameraEnabled,
computerControlProvider: provider,
codexThreadCatalogEnabled: codexThreadCatalogEnabled,
claudeSessionCatalogEnabled: claudeSessionCatalogEnabled)
// If Computer Control was turned off, release any button the
@@ -540,7 +568,7 @@ final class MacNodeModeCoordinator: NSObject {
}
let caps = Self.mergingUnique(nativeCaps, workerManifest?.caps ?? [])
let commands = Self.mergingUnique(
self.currentCommands(caps: nativeCaps),
self.currentCommands(caps: nativeCaps, computerControlProvider: provider),
workerManifest?.commands ?? [])
let permissions = await self.currentPermissions()
// TCC queries suspend. An endpoint loss/replacement during that
@@ -559,6 +587,10 @@ final class MacNodeModeCoordinator: NSObject {
scopes: [],
caps: caps,
commands: commands,
computerUse: Self.computerUseDescriptor(
provider: provider,
commands: commands,
workerManifest: workerManifest),
pathEnv: workerManifest?.pathEnv,
permissions: permissions,
clientId: "openclaw-macos",
@@ -769,10 +801,15 @@ final class MacNodeModeCoordinator: NSObject {
await self.awaitStableRouteInvalidationDrain(onPendingSnapshot: onPendingSnapshot)
}
func refreshForTesting(isPaused: Bool, computerControlEnabled: Bool) {
func refreshForTesting(
isPaused: Bool,
computerControlEnabled: Bool,
computerControlProvider: ComputerControlProvider = .peekaboo)
{
self.refresh(
isPaused: isPaused,
computerControlEnabled: computerControlEnabled)
computerControlEnabled: computerControlEnabled,
computerControlProvider: computerControlProvider)
}
func enqueueRouteInvalidationForTesting() {
@@ -849,10 +886,13 @@ final class MacNodeModeCoordinator: NSObject {
// Replace the process before reconnecting so updates cannot leave a stale route.
return self.enqueueRouteInvalidation(mode: .workerRestart)
}
}
extension MacNodeModeCoordinator {
private func currentCaps(
browserControlEnabled: Bool,
cameraEnabled: Bool,
computerControlProvider: ComputerControlProvider,
codexThreadCatalogEnabled: Bool,
claudeSessionCatalogEnabled: Bool) -> [String]
{
@@ -862,6 +902,7 @@ final class MacNodeModeCoordinator: NSObject {
browserControlEnabled: browserControlEnabled,
cameraEnabled: cameraEnabled,
computerControlEnabled: computerControlEnabled,
computerControlProvider: computerControlProvider,
locationMode: OpenClawLocationMode(rawValue: rawLocationMode) ?? .off,
connectionMode: AppStateStore.shared.connectionMode,
codexThreadCatalogEnabled: codexThreadCatalogEnabled,
@@ -873,11 +914,16 @@ final class MacNodeModeCoordinator: NSObject {
return Self.advertisedPermissions(statuses)
}
private func currentCommands(caps: [String]) -> [String] {
Self.resolvedCommands(caps: caps)
private func currentCommands(
caps: [String],
computerControlProvider: ComputerControlProvider) -> [String]
{
Self.resolvedCommands(caps: caps, computerControlProvider: computerControlProvider)
}
private func startNodeHostWorkerIfConfigured() async throws -> MacNodeHostManifest? {
private func startNodeHostWorkerIfConfigured(
provider: ComputerControlProvider) async throws -> MacNodeHostManifest?
{
guard let nodeHostWorker else { return nil }
guard self.nodeHostWorkerRetryTask == nil else {
throw MacNodeHostWorkerRetryPolicy.RetryBackoffPending()
@@ -898,12 +944,21 @@ final class MacNodeModeCoordinator: NSObject {
} catch let error as RuntimeResolutionError {
throw MacNodeHostWorker.WorkerError.unavailable(RuntimeLocator.describeFailure(error))
}
var workerEnvironment: [String: String] = [:]
if provider == .cua, let endpoint = CuaDriverHostCoordinator.shared.workerEndpoint {
workerEnvironment[CuaDriverWorkerEnvironment.socketPath] = endpoint.socketPath
workerEnvironment[CuaDriverWorkerEnvironment.binaryPath] = endpoint.binaryPath
}
let effectiveLaunch = MacNodeHostWorkerLaunch(
command: launch.command,
currentDirectoryURL: launch.currentDirectoryURL,
environment: workerEnvironment)
let input = MacNodeHostWorkerRetryPolicy.Input(
launch: launch,
launch: effectiveLaunch,
configurationGeneration: self.nodeHostWorkerConfigurationGeneration)
try self.nodeHostWorkerRetryPolicy.prepareForStart(input)
self.activeNodeHostWorkerInput = input
return try await nodeHostWorker.start(launch: launch)
return try await nodeHostWorker.start(launch: effectiveLaunch)
}
private func handleNodeHostWorkerFailure() {
@@ -991,10 +1046,13 @@ extension MacNodeModeCoordinator {
previousPaused: Bool,
nextPaused: Bool,
previousComputerControlEnabled: Bool,
nextComputerControlEnabled: Bool) -> Bool
nextComputerControlEnabled: Bool,
previousComputerControlProvider: ComputerControlProvider = .peekaboo,
nextComputerControlProvider: ComputerControlProvider = .peekaboo) -> Bool
{
(!previousPaused && nextPaused) ||
(previousComputerControlEnabled && !nextComputerControlEnabled)
(previousComputerControlEnabled && !nextComputerControlEnabled) ||
previousComputerControlProvider != nextComputerControlProvider
}
nonisolated static func endpointState(
@@ -1101,6 +1159,7 @@ extension MacNodeModeCoordinator {
browserControlEnabled: Bool,
cameraEnabled: Bool,
computerControlEnabled: Bool,
computerControlProvider: ComputerControlProvider = .peekaboo,
locationMode: OpenClawLocationMode,
connectionMode: AppState.ConnectionMode,
codexThreadCatalogEnabled: Bool = false,
@@ -1114,7 +1173,7 @@ extension MacNodeModeCoordinator {
if cameraEnabled { caps.append(OpenClawCapability.camera.rawValue) }
// Advertised only when the operator has enabled Computer Control; the
// command is dangerous and stays disarmed until allowlisted on the gateway.
if computerControlEnabled {
if computerControlEnabled, computerControlProvider == .peekaboo {
caps.append(OpenClawCapability.computer.rawValue)
}
if locationMode != .off { caps.append(OpenClawCapability.location.rawValue) }
@@ -1129,7 +1188,10 @@ extension MacNodeModeCoordinator {
return caps
}
nonisolated static func resolvedCommands(caps: [String]) -> [String] {
nonisolated static func resolvedCommands(
caps: [String],
computerControlProvider: ComputerControlProvider = .peekaboo) -> [String]
{
var commands: [String] = [
OpenClawCanvasCommand.present.rawValue,
OpenClawCanvasCommand.hide.rawValue,
@@ -1139,11 +1201,14 @@ extension MacNodeModeCoordinator {
OpenClawCanvasA2UICommand.push.rawValue,
OpenClawCanvasA2UICommand.pushJSONL.rawValue,
OpenClawCanvasA2UICommand.reset.rawValue,
MacNodeScreenCommand.snapshot.rawValue,
MacNodeScreenCommand.record.rawValue,
OpenClawSystemCommand.notify.rawValue,
]
if computerControlProvider == .peekaboo {
commands.append(MacNodeScreenCommand.snapshot.rawValue)
}
commands.append(MacNodeScreenCommand.record.rawValue)
commands.append(OpenClawSystemCommand.notify.rawValue)
let capsSet = Set(caps)
if capsSet.contains(OpenClawCapability.camera.rawValue) {
commands.append(OpenClawCameraCommand.list.rawValue)
@@ -1168,6 +1233,36 @@ extension MacNodeModeCoordinator {
return commands
}
nonisolated static func workerManifest(
_ manifest: MacNodeHostManifest?,
for provider: ComputerControlProvider) -> MacNodeHostManifest?
{
guard let manifest else { return nil }
guard provider == .peekaboo else { return manifest }
let providerCommands = Set([
MacNodeScreenCommand.snapshot.rawValue,
OpenClawComputerCommand.act.rawValue,
])
return MacNodeHostManifest(
version: manifest.version,
caps: manifest.caps.filter { $0 != OpenClawCapability.computer.rawValue },
commands: manifest.commands.filter { !providerCommands.contains($0) },
computerUse: nil,
pathEnv: manifest.pathEnv)
}
nonisolated static func computerUseDescriptor(
provider: ComputerControlProvider,
commands: [String],
workerManifest: MacNodeHostManifest?) -> OpenClawProtocol.AnyCodable?
{
guard provider == .cua,
commands.contains(MacNodeScreenCommand.snapshot.rawValue),
commands.contains(OpenClawComputerCommand.act.rawValue)
else { return nil }
return workerManifest?.computerUse
}
nonisolated static func mergingUnique(_ primary: [String], _ additional: [String]) -> [String] {
var seen = Set<String>()
return (primary + additional).filter { seen.insert($0).inserted }
@@ -112,12 +112,17 @@ actor MacNodeClaudeSessionCatalogWorker {
actor MacNodeRuntime {
private static let maxGatewayPayloadBytes = 25 * 1024 * 1024
private static let maxScreenSnapshotRawBytesBeforeBase64 = (maxGatewayPayloadBytes / 4) * 3
private static let cuaOwnedCommands = Set([
MacNodeScreenCommand.snapshot.rawValue,
OpenClawComputerCommand.act.rawValue,
])
private let cameraCapture = CameraCaptureService()
private let cameraPTZ: any CameraPTZServicing
private let nodeHostWorker: (any MacNodeHostWorking)?
private let makeMainActorServices: @Sendable () async -> any MacNodeRuntimeMainActorServices
// Injectable so tests pin the gate instead of racing on process-global UserDefaults.
private let computerControlEnabled: @Sendable () -> Bool
private let computerControlProvider: @Sendable () -> ComputerControlProvider
private let canvasHostedSurfaceResolver: MacNodeCanvasHostedSurfaceResolver
private let codexThreadCatalogEnabled: @Sendable () -> Bool
private let codexThreadCatalogClient: MacNodeCodexThreadCatalogClient
@@ -144,6 +149,9 @@ actor MacNodeRuntime {
computerControlEnabled: @escaping @Sendable () -> Bool = {
MacNodeRuntime.computerControlEnabledDefault()
},
computerControlProvider: @escaping @Sendable () -> ComputerControlProvider = {
ComputerControlProvider.current()
},
canvasSurfaceUrl: @escaping @Sendable () async -> String? = {
await GatewayConnection.shared.canvasPluginSurfaceUrl()
},
@@ -168,6 +176,7 @@ actor MacNodeRuntime {
self.cameraPTZ = cameraPTZ
self.makeMainActorServices = makeMainActorServices
self.computerControlEnabled = computerControlEnabled
self.computerControlProvider = computerControlProvider
self.canvasHostedSurfaceResolver = MacNodeCanvasHostedSurfaceResolver(
currentSurfaceURL: canvasSurfaceUrl,
refreshSurfaceURL: refreshCanvasSurfaceUrl)
@@ -197,6 +206,9 @@ actor MacNodeRuntime {
code: .unavailable,
message: "CANVAS_DISABLED: enable Canvas in Settings"))
}
if let cuaResponse = await self.handleCuaInvokeIfSelected(req) {
return cuaResponse
}
do {
switch command {
case OpenClawCanvasCommand.present.rawValue,
@@ -262,6 +274,25 @@ actor MacNodeRuntime {
command.hasPrefix("canvas.") || command.hasPrefix("canvas.a2ui.")
}
private func handleCuaInvokeIfSelected(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse? {
guard self.computerControlProvider() == .cua,
Self.cuaOwnedCommands.contains(req.command)
else { return nil }
guard self.computerControlEnabled() else {
return Self.errorResponse(
req,
code: .unavailable,
message: "COMPUTER_DISABLED: enable Computer Control in Settings")
}
guard let nodeHostWorker, await nodeHostWorker.supports(req.command) else {
return Self.errorResponse(
req,
code: .unavailable,
message: "UNAVAILABLE: selected CUA provider is not ready")
}
return await nodeHostWorker.invoke(req)
}
private func handleCodexThreadInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
guard self.codexThreadCatalogEnabled() else {
return Self.errorResponse(
@@ -16,4 +16,35 @@ struct ComputerControlSettingsTests {
defaults.set(true, forKey: computerControlEnabledKey)
#expect(isComputerControlEnabled(defaults: defaults))
}
@Test func `computer control provider defaults to Peekaboo and preserves CUA selection`() throws {
let suiteName = "ComputerControlProviderSettingsTests.\(UUID().uuidString)"
let defaults = try #require(UserDefaults(suiteName: suiteName))
defer { defaults.removePersistentDomain(forName: suiteName) }
#expect(ComputerControlProvider.current(defaults: defaults, cuaAvailable: true) == .peekaboo)
defaults.set(ComputerControlProvider.cua.rawValue, forKey: computerControlProviderKey)
#expect(ComputerControlProvider.current(defaults: defaults, cuaAvailable: true) == .cua)
#expect(ComputerControlProvider.current(defaults: defaults, cuaAvailable: false) == .peekaboo)
defaults.set("retired-provider", forKey: computerControlProviderKey)
#expect(ComputerControlProvider.current(defaults: defaults, cuaAvailable: true) == .peekaboo)
}
@Test func `bundled CUA locator accepts only a regular executable and never follows a symlink`() throws {
let root = FileManager.default.temporaryDirectory
.appendingPathComponent("openclaw-cua-artifact-\(UUID().uuidString)", isDirectory: true)
defer { try? FileManager.default.removeItem(at: root) }
try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)
let binary = root.appendingPathComponent(CuaDriverArtifact.resourceName)
try Data("driver".utf8).write(to: binary)
try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: binary.path)
#expect(CuaDriverArtifact.executableURL(in: root) == binary)
try FileManager.default.removeItem(at: binary)
let target = root.appendingPathComponent("real-driver")
try Data("driver".utf8).write(to: target)
try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: target.path)
try FileManager.default.createSymbolicLink(at: binary, withDestinationURL: target)
#expect(CuaDriverArtifact.executableURL(in: root) == nil)
}
}
@@ -0,0 +1,571 @@
import Darwin
import Foundation
import OpenClawIPC
import Testing
@testable import OpenClaw
@Suite(.serialized)
@MainActor
struct CuaDriverHostCoordinatorTests {
private func waitForReadyLaunch(
_ expected: Int,
launcher: CuaProcessLauncherProbe,
coordinator: CuaDriverHostCoordinator) async -> Bool
{
let deadline = ContinuousClock.now + .seconds(2)
while ContinuousClock.now < deadline {
if launcher.launches.count >= expected, coordinator.workerEndpoint != nil { return true }
try? await Task.sleep(for: .milliseconds(1))
}
return launcher.launches.count >= expected && coordinator.workerEndpoint != nil
}
@Test func `disabled host never spawns and enabled host publishes only a ready endpoint`() async throws {
let root = self.shortTemporaryDirectory("host")
defer { try? FileManager.default.removeItem(at: root) }
let executable = root.appendingPathComponent("cua-driver")
let launcher = CuaProcessLauncherProbe()
var workerStops = 0
let coordinator = CuaDriverHostCoordinator(
artifactURL: { executable },
applicationSupportURL: { root },
bundleIdentifier: { "ai.openclaw.test" },
processLauncher: { launch, onTermination in
launcher.launch(launch, onTermination: onTermination)
},
readinessProbe: { _ in true },
beforeDaemonStop: {
let allRunning = launcher.processes.allSatisfy(\.isRunning)
#expect(allRunning)
workerStops += 1
})
await coordinator.setEnabled(false)
#expect(launcher.launches.isEmpty)
#expect(coordinator.workerEndpoint == nil)
await coordinator.setEnabled(true)
let launch = try #require(launcher.launches.first)
let endpoint = try #require(coordinator.workerEndpoint)
#expect(launch.executableURL == executable)
#expect(endpoint.binaryPath == executable.path)
let socketArgument = try #require(launch.arguments.firstIndex(of: "--socket")) + 1
#expect(endpoint.socketPath == launch.arguments[socketArgument])
await coordinator.setEnabled(false)
#expect(coordinator.workerEndpoint == nil)
#expect(workerStops == 1)
#expect(launcher.processes.allSatisfy { !$0.isRunning })
}
@Test func `socket directory is random owner-only and cleanup removes only its owned leaf`() throws {
let root = self.shortTemporaryDirectory("socket")
defer { try? FileManager.default.removeItem(at: root) }
let first = try CuaDriverHostCoordinator.createSocketDirectory(in: root)
let second = try CuaDriverHostCoordinator.createSocketDirectory(in: root)
#expect(first.url != second.url)
for directory in [first, second] {
let attributes = try FileManager.default.attributesOfItem(atPath: directory.url.path)
let permissions = (attributes[.posixPermissions] as? NSNumber)?.intValue
#expect(permissions == 0o700)
#expect(!FileManager.default.fileExists(atPath: directory.socketPath))
}
CuaDriverHostCoordinator.cleanupSocketDirectory(first)
#expect(!FileManager.default.fileExists(atPath: first.url.path))
#expect(FileManager.default.fileExists(atPath: second.url.path))
CuaDriverHostCoordinator.cleanupSocketDirectory(second)
}
@Test func `liveness write end is close on exec and absent from a subsequently spawned child`() throws {
let livenessPipe = try CuaDriverHostCoordinator.makeLivenessPipe()
let descriptor = livenessPipe.fileHandleForWriting.fileDescriptor
#expect(fcntl(descriptor, F_GETFD) & FD_CLOEXEC == FD_CLOEXEC)
let child = Process()
child.executableURL = URL(fileURLWithPath: "/bin/sleep")
child.arguments = ["1"]
try child.run()
defer {
if child.isRunning { child.terminate() }
child.waitUntilExit()
}
#expect(!Self.process(child.processIdentifier, hasDescriptor: descriptor))
}
@Test func `liveness read end remains daemon standard input and observes writer EOF`() throws {
let livenessPipe = try CuaDriverHostCoordinator.makeLivenessPipe()
let child = Process()
child.executableURL = URL(fileURLWithPath: "/bin/cat")
child.standardInput = livenessPipe.fileHandleForReading
child.standardOutput = FileHandle.nullDevice
child.standardError = FileHandle.nullDevice
try child.run()
try livenessPipe.fileHandleForWriting.close()
for _ in 0..<1000 where child.isRunning {
usleep(1000)
}
if child.isRunning { child.terminate() }
child.waitUntilExit()
#expect(child.terminationStatus == 0)
}
@Test func `unexpected exit closes liveness and removes its socket directory`() async throws {
let root = self.shortTemporaryDirectory("exit-cleanup")
defer { try? FileManager.default.removeItem(at: root) }
let launcher = CuaProcessLauncherProbe()
let coordinator = CuaDriverHostCoordinator(
artifactURL: { root.appendingPathComponent("cua-driver") },
applicationSupportURL: { root },
bundleIdentifier: { "ai.openclaw.test" },
processLauncher: { launch, onTermination in
launcher.launch(launch, onTermination: onTermination)
},
readinessProbe: { _ in true })
await coordinator.setEnabled(true)
let endpoint = try #require(coordinator.workerEndpoint)
let process = try #require(launcher.processes.first)
process.crash(status: 7)
for _ in 0..<1000 where coordinator.workerEndpoint != nil {
await Task.yield()
}
#expect(process.closeLivenessCount == 1)
#expect(!FileManager.default.fileExists(atPath: URL(fileURLWithPath: endpoint.socketPath)
.deletingLastPathComponent().path))
await coordinator.setEnabled(false)
}
@Test func `startup removes a preexisting owned directory without a live daemon`() async throws {
let root = self.shortTemporaryDirectory("startup-reap")
defer { try? FileManager.default.removeItem(at: root) }
let stale = try CuaDriverHostCoordinator.createSocketDirectory(in: root)
let launcher = CuaProcessLauncherProbe()
let coordinator = CuaDriverHostCoordinator(
artifactURL: { root.appendingPathComponent("cua-driver") },
applicationSupportURL: { root },
bundleIdentifier: { "ai.openclaw.test" },
processLauncher: { launch, onTermination in
launcher.launch(launch, onTermination: onTermination)
},
readinessProbe: { _ in true })
await coordinator.setEnabled(true)
#expect(!FileManager.default.fileExists(atPath: stale.url.path))
await coordinator.setEnabled(false)
}
@Test func `startup terminates a live owned daemon after its host is gone`() async throws {
let root = self.shortTemporaryDirectory("startup-orphan")
let executable = try self.expectedExecutable(in: root, target: "/bin/sleep")
let orphan = try self.startFakeDaemon(executable: executable, hostPID: Int32.max)
defer {
self.stopIfRunning(orphan)
try? FileManager.default.removeItem(at: root)
}
let stale = try CuaDriverHostCoordinator.createSocketDirectory(in: root)
try String(orphan.processIdentifier).write(
to: stale.url.appendingPathComponent("cua.pid"),
atomically: true,
encoding: .utf8)
let launcher = CuaProcessLauncherProbe()
let coordinator = CuaDriverHostCoordinator(
artifactURL: { executable },
applicationSupportURL: { root },
bundleIdentifier: { "ai.openclaw.test" },
processLauncher: { launch, onTermination in
launcher.launch(launch, onTermination: onTermination)
},
readinessProbe: { _ in true })
await coordinator.setEnabled(true)
#expect(await self.waitUntilExited(orphan))
#expect(!FileManager.default.fileExists(atPath: stale.url.path))
await coordinator.setEnabled(false)
}
@Test func `launch records the spawned daemon pid for later reaping`() async throws {
// Without this record the reaper can only delete the directory and leaves the
// privileged daemon running: `serve` ignores --pid-file and writes a global path.
let root = self.shortTemporaryDirectory("launch-pidfile")
let executable = try self.expectedExecutable(in: root, target: "/bin/sleep")
defer { try? FileManager.default.removeItem(at: root) }
let launcher = CuaProcessLauncherProbe()
let coordinator = CuaDriverHostCoordinator(
artifactURL: { executable },
applicationSupportURL: { root },
bundleIdentifier: { "ai.openclaw.test" },
processLauncher: { launch, onTermination in
launcher.launch(launch, onTermination: onTermination)
},
readinessProbe: { _ in true })
await coordinator.setEnabled(true)
let directories = try FileManager.default.contentsOfDirectory(
at: root.appendingPathComponent("OpenClaw", isDirectory: true)
.appendingPathComponent("cua", isDirectory: true),
includingPropertiesForKeys: nil)
let pidFile = try #require(directories.first?.appendingPathComponent("cua.pid"))
let recorded = try String(contentsOf: pidFile, encoding: .utf8)
.trimmingCharacters(in: .whitespacesAndNewlines)
#expect(recorded == String(launcher.lastProcessIdentifier))
await coordinator.setEnabled(false)
}
@Test func `startup refuses to signal a pid owned by another executable`() async throws {
let root = self.shortTemporaryDirectory("startup-pid-reuse")
let expectedExecutable = try self.expectedExecutable(in: root, target: "/bin/cat")
let unrelated = try self.startSleep(executable: URL(fileURLWithPath: "/bin/sleep"))
defer {
self.stopIfRunning(unrelated)
try? FileManager.default.removeItem(at: root)
}
let stale = try CuaDriverHostCoordinator.createSocketDirectory(in: root)
try String(unrelated.processIdentifier).write(
to: stale.url.appendingPathComponent("cua.pid"),
atomically: true,
encoding: .utf8)
let launcher = CuaProcessLauncherProbe()
let coordinator = CuaDriverHostCoordinator(
artifactURL: { expectedExecutable },
applicationSupportURL: { root },
bundleIdentifier: { "ai.openclaw.test" },
processLauncher: { launch, onTermination in
launcher.launch(launch, onTermination: onTermination)
},
readinessProbe: { _ in true })
await coordinator.setEnabled(true)
#expect(unrelated.isRunning)
#expect(FileManager.default.fileExists(atPath: stale.url.path))
let launch = try #require(launcher.launches.first)
// `serve` ignores --pid-file (it always writes the machine-global default),
// so OpenClaw must never pass it and records the pid itself instead.
#expect(!launch.arguments.contains("--pid-file"))
await coordinator.setEnabled(false)
#expect(unrelated.isRunning)
}
@Test func `teardown leaves no owned directories or live owned daemons`() async throws {
let root = self.shortTemporaryDirectory("teardown-reap")
let executable = try self.expectedExecutable(in: root, target: "/bin/sleep")
let orphan = try self.startFakeDaemon(executable: executable, hostPID: Int32.max)
defer {
self.stopIfRunning(orphan)
try? FileManager.default.removeItem(at: root)
}
let live = try CuaDriverHostCoordinator.createSocketDirectory(in: root)
try String(orphan.processIdentifier).write(
to: live.url.appendingPathComponent("cua.pid"),
atomically: true,
encoding: .utf8)
_ = try CuaDriverHostCoordinator.createSocketDirectory(in: root)
let coordinator = CuaDriverHostCoordinator(
artifactURL: { executable },
applicationSupportURL: { root },
bundleIdentifier: { "ai.openclaw.test" })
await coordinator.shutdown()
#expect(await self.waitUntilExited(orphan))
let cuaRoot = root.appendingPathComponent("OpenClaw/cua", isDirectory: true)
#expect(try FileManager.default.contentsOfDirectory(atPath: cuaRoot.path).isEmpty)
}
@Test func `socket directory rejects a symlinked CUA root`() throws {
let root = self.shortTemporaryDirectory("unsafe")
defer { try? FileManager.default.removeItem(at: root) }
let openClaw = root.appendingPathComponent("OpenClaw", isDirectory: true)
let redirected = root.appendingPathComponent("redirected", isDirectory: true)
try FileManager.default.createDirectory(at: openClaw, withIntermediateDirectories: true)
try FileManager.default.createDirectory(at: redirected, withIntermediateDirectories: true)
try FileManager.default.createSymbolicLink(
at: openClaw.appendingPathComponent("cua", isDirectory: true),
withDestinationURL: redirected)
#expect(throws: CuaDriverHostError.self) {
try CuaDriverHostCoordinator.createSocketDirectory(in: root)
}
#expect(try (FileManager.default.contentsOfDirectory(atPath: redirected.path)).isEmpty)
}
@Test func `socket directory rejects a symlinked OpenClaw support root`() throws {
let root = self.shortTemporaryDirectory("unsafe-parent")
defer { try? FileManager.default.removeItem(at: root) }
let redirected = root.appendingPathComponent("redirected", isDirectory: true)
try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)
try FileManager.default.createDirectory(at: redirected, withIntermediateDirectories: true)
try FileManager.default.createSymbolicLink(
at: root.appendingPathComponent("OpenClaw", isDirectory: true),
withDestinationURL: redirected)
#expect(throws: CuaDriverHostError.self) {
try CuaDriverHostCoordinator.createSocketDirectory(in: root)
}
#expect(try (FileManager.default.contentsOfDirectory(atPath: redirected.path)).isEmpty)
}
@Test func `embedded launch carries unrestricted acknowledgement and disables network reporting`() {
let launch = CuaDriverHostCoordinator.makeProcessLaunch(
executableURL: URL(fileURLWithPath: "/Applications/OpenClaw.app/Contents/Resources/cua-driver"),
socketPath: "/tmp/openclaw-cua-test.sock",
hostBundleID: "ai.openclaw.mac",
inheritedEnvironment: [
"PATH": "/usr/bin:/bin",
"CUA_DRIVER_SOCKET": "/tmp/ambient.sock",
"CUA_DRIVER_PERMISSION_MODE": "bounded",
"CUA_TELEMETRY_ENABLED": "true",
])
#expect(launch.arguments.contains("--embedded"))
#expect(launch.arguments.contains("--parent-liveness-stdio"))
// The embedded host cannot predeclare arbitrary runtime-discovered targets in a bounded manifest.
let permissionModeIndex = launch.arguments.firstIndex(of: "--permission-mode")
let permissionMode = permissionModeIndex.flatMap { index in
launch.arguments.indices.contains(index + 1) ? launch.arguments[index + 1] : nil
}
#expect(permissionMode == "unrestricted")
#expect(launch.arguments.contains("--dangerously-bypass-approvals"))
#expect(launch.environment["CUA_DRIVER_EMBEDDED"] == "1")
#expect(launch.environment["CUA_DRIVER_PERMISSION_MODE"] == "unrestricted")
#expect(launch.environment["CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS"] == "1")
#expect(launch.environment["CUA_DRIVER_RS_TELEMETRY_ENABLED"] == "false")
#expect(launch.environment["CUA_DRIVER_RS_UPDATE_CHECK"] == "false")
#expect(launch.environment["CUA_DRIVER_SOCKET"] == nil)
#expect(launch.environment["CUA_TELEMETRY_ENABLED"] == nil)
#expect(launch.environment["PATH"] == "/usr/bin:/bin")
}
@Test func `stderr relay replaces the raw danger banner with one managed mode notice`() async throws {
let probe = CuaDriverStderrProbe()
let relay = CuaDriverStderrRelay { probe.append($0) }
relay.startReading()
relay.reportManagedMode()
relay.reportManagedMode()
let driverOutput = """
DANGER: Cua Driver is running in unrestricted mode. Runtime approval prompts are disabled.
driver diagnostic
"""
try relay.pipe.fileHandleForWriting.write(contentsOf: Data(driverOutput.utf8))
try relay.pipe.fileHandleForWriting.close()
for _ in 0..<1000 where probe.events.count < 2 {
await Task.yield()
}
relay.stop()
#expect(probe.events == [
.notice(CuaDriverStderrRelay.managedModeNotice),
.error("driver diagnostic"),
])
}
@Test func `unexpected exits retry with a bounded budget while advertising unavailable`() async throws {
let root = self.shortTemporaryDirectory("restart")
defer { try? FileManager.default.removeItem(at: root) }
let launcher = CuaProcessLauncherProbe()
let coordinator = CuaDriverHostCoordinator(
artifactURL: { root.appendingPathComponent("cua-driver") },
applicationSupportURL: { root },
bundleIdentifier: { "ai.openclaw.test" },
processLauncher: { launch, onTermination in
launcher.launch(launch, onTermination: onTermination)
},
readinessProbe: { _ in true },
restartSleep: { _ in })
await coordinator.setEnabled(true)
for expectedLaunchCount in 2...6 {
try #require(launcher.processes.last).crash(status: 7)
#expect(await self.waitForReadyLaunch(
expectedLaunchCount,
launcher: launcher,
coordinator: coordinator))
}
try #require(launcher.processes.last).crash(status: 7)
for _ in 0..<100 {
await Task.yield()
}
#expect(launcher.launches.count == 6)
#expect(coordinator.workerEndpoint == nil)
await coordinator.setEnabled(false)
}
@Test func `permission changes replace the daemon generation and endpoint`() async throws {
let root = self.shortTemporaryDirectory("permissions")
defer { try? FileManager.default.removeItem(at: root) }
let notifications = NotificationCenter()
let permissions = CuaPermissionSnapshotProbe()
let launcher = CuaProcessLauncherProbe()
let coordinator = CuaDriverHostCoordinator(
notificationCenter: notifications,
observeNotifications: true,
artifactURL: { root.appendingPathComponent("cua-driver") },
applicationSupportURL: { root },
bundleIdentifier: { "ai.openclaw.test" },
processLauncher: { launch, onTermination in
launcher.launch(launch, onTermination: onTermination)
},
readinessProbe: { _ in true },
permissionSnapshot: { permissions.value })
await coordinator.setEnabled(true)
let originalEndpoint = try #require(coordinator.workerEndpoint)
permissions.value[.accessibility] = .granted
notifications.post(name: .openclawPermissionsChanged, object: nil)
#expect(await self.waitForReadyLaunch(2, launcher: launcher, coordinator: coordinator))
let replacementEndpoint = try #require(coordinator.workerEndpoint)
#expect(replacementEndpoint.socketPath != originalEndpoint.socketPath)
#expect(!launcher.processes[0].isRunning)
await coordinator.setEnabled(false)
}
private func shortTemporaryDirectory(_ label: String) -> URL {
URL(fileURLWithPath: "/tmp/oc-cua-\(label)-\(UUID().uuidString.prefix(8))", isDirectory: true)
}
private func expectedExecutable(in root: URL, target: String) throws -> URL {
try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)
let executable = root.appendingPathComponent("cua-driver")
try FileManager.default.createSymbolicLink(
at: executable,
withDestinationURL: URL(fileURLWithPath: target))
return executable
}
private func startFakeDaemon(executable: URL, hostPID: Int32) throws -> Process {
let process = self.makeSleepProcess(executable: executable)
var environment = ProcessInfo.processInfo.environment
environment["CUA_DRIVER_EMBEDDED_HOST_PID"] = String(hostPID)
process.environment = environment
try process.run()
return process
}
private func startSleep(executable: URL) throws -> Process {
let process = self.makeSleepProcess(executable: executable)
try process.run()
return process
}
private func makeSleepProcess(executable: URL) -> Process {
let process = Process()
process.executableURL = executable
process.arguments = ["60"]
return process
}
private func waitUntilExited(_ process: Process) async -> Bool {
let deadline = ContinuousClock.now + .seconds(2)
while process.isRunning, ContinuousClock.now < deadline {
try? await Task.sleep(for: .milliseconds(10))
}
return !process.isRunning
}
private func stopIfRunning(_ process: Process) {
if process.isRunning {
process.terminate()
process.waitUntilExit()
}
}
private static func process(_ processIdentifier: pid_t, hasDescriptor descriptor: Int32) -> Bool {
var descriptors = [proc_fdinfo](repeating: proc_fdinfo(), count: 64)
let populatedBytes = descriptors.withUnsafeMutableBytes { buffer in
proc_pidinfo(
processIdentifier,
PROC_PIDLISTFDS,
0,
buffer.baseAddress,
Int32(buffer.count))
}
guard populatedBytes > 0 else { return false }
let count = min(
descriptors.count,
Int(populatedBytes) / MemoryLayout<proc_fdinfo>.stride)
return descriptors.prefix(count).contains { $0.proc_fd == descriptor }
}
}
private final class CuaDriverStderrProbe: @unchecked Sendable {
private let lock = NSLock()
private var captured: [CuaDriverStderrEvent] = []
var events: [CuaDriverStderrEvent] {
self.lock.withLock { self.captured }
}
func append(_ event: CuaDriverStderrEvent) {
self.lock.withLock { self.captured.append(event) }
}
}
@MainActor
private final class CuaPermissionSnapshotProbe {
var value: [Capability: CapabilityAuthorizationStatus] = [
.accessibility: .notGranted,
.screenRecording: .notGranted,
]
}
@MainActor
private final class CuaProcessLauncherProbe {
private(set) var launches: [CuaDriverProcessLaunch] = []
private(set) var processes: [CuaProcessProbe] = []
var lastProcessIdentifier: pid_t {
self.processes.last?.processIdentifier ?? 0
}
func launch(
_ launch: CuaDriverProcessLaunch,
onTermination: @escaping @Sendable (Int32) -> Void) -> CuaProcessProbe
{
self.launches.append(launch)
let process = CuaProcessProbe(onTermination: onTermination)
self.processes.append(process)
return process
}
}
@MainActor
private final class CuaProcessProbe: CuaDriverProcessControlling {
private(set) var isRunning = true
private(set) var closeLivenessCount = 0
let processIdentifier: pid_t
private let onTermination: @Sendable (Int32) -> Void
init(processIdentifier: pid_t = 424_242, onTermination: @escaping @Sendable (Int32) -> Void) {
self.processIdentifier = processIdentifier
self.onTermination = onTermination
}
func closeLiveness() {
self.closeLivenessCount += 1
guard self.isRunning else { return }
self.isRunning = false
self.onTermination(0)
}
func terminate() {
self.closeLiveness()
}
func forceKill() {
self.closeLiveness()
}
func crash(status: Int32) {
guard self.isRunning else { return }
self.isRunning = false
self.onTermination(status)
}
}
@@ -293,6 +293,47 @@ struct GatewayChannelConnectTests {
#expect(params["pathEnv"] as? String == "/opt/homebrew/bin:/usr/bin:/bin")
}
@Test func `node connect forwards the selected computer-use descriptor`() async throws {
let recorder = ConnectParamsRecorder()
let session = GatewayTestWebSocketSession(
taskFactory: {
GatewayTestWebSocketTask(
sendHook: { _, message, sendIndex in
guard sendIndex == 0 else { return }
recorder.record(message)
})
})
let descriptor = OpenClawProtocol.AnyCodable([
"contractVersion": OpenClawProtocol.AnyCodable(2),
"provider": OpenClawProtocol.AnyCodable([
"id": OpenClawProtocol.AnyCodable("cua-computer"),
]),
])
let options = GatewayConnectOptions(
role: "node",
scopes: [],
caps: ["screen", "computer"],
commands: ["screen.snapshot", "computer.act"],
computerUse: descriptor,
permissions: [:],
clientId: "openclaw-macos",
clientMode: "node",
clientDisplayName: "macOS Test",
includeDeviceIdentity: false)
let channel = try GatewayChannelActor(
url: #require(URL(string: "ws://example.invalid")),
token: nil,
session: WebSocketSessionBox(session: session),
connectOptions: options)
try await channel.connect()
let params = try #require(recorder.snapshot())
let computerUse = try #require(params["computerUse"] as? [String: Any])
#expect(computerUse["contractVersion"] as? Int == 2)
#expect((computerUse["provider"] as? [String: Any])?["id"] as? String == "cua-computer")
}
@Test func `concurrent connect shares failure`() async throws {
let session = self.makeSession(response: .invalid(delayMs: 200))
let channel = try GatewayChannelActor(
@@ -1,6 +1,7 @@
import Darwin
import Foundation
import OpenClawKit
import OpenClawProtocol
import Testing
@testable import OpenClaw
@@ -146,6 +147,40 @@ struct MacNodeHostWorkerTests {
#expect(await worker.invokedCommands() == [command])
}
@Test(arguments: [MacNodeScreenCommand.snapshot.rawValue, OpenClawComputerCommand.act.rawValue])
func `selected CUA provider gives the command pair exclusively to the worker`(command: String) async {
let worker = StubMacNodeHostWorker(commands: [command])
let runtime = MacNodeRuntime(
nodeHostWorker: worker,
computerControlEnabled: { true },
computerControlProvider: { .cua })
let response = await runtime.handleInvoke(BridgeInvokeRequest(
id: "cua-owned",
command: command,
paramsJSON: "{}"))
#expect(response.ok)
#expect(response.payloadJSON == #"{"owner":"cli"}"#)
#expect(await worker.invokedCommands() == [command])
}
@Test func `selected CUA provider never falls back to native snapshot`() async {
let worker = StubMacNodeHostWorker(commands: [])
let runtime = MacNodeRuntime(
nodeHostWorker: worker,
computerControlEnabled: { true },
computerControlProvider: { .cua })
let response = await runtime.handleInvoke(BridgeInvokeRequest(
id: "cua-unavailable",
command: MacNodeScreenCommand.snapshot.rawValue))
#expect(!response.ok)
#expect(response.error?.message == "UNAVAILABLE: selected CUA provider is not ready")
#expect(await worker.invokedCommands().isEmpty)
}
@Test(arguments: [
MacNodeCodexThreadCatalogContract.listCommand,
MacNodeCodexThreadCatalogContract.turnsCommand,
@@ -232,6 +267,30 @@ struct MacNodeHostWorkerTests {
["system", "mcp"]) == ["canvas", "screen", "system", "mcp"])
}
@Test func `provider selection filters command ownership and publishes only the CUA descriptor`() throws {
let descriptor = OpenClawProtocol.AnyCodable([
"contractVersion": OpenClawProtocol.AnyCodable(2),
])
let manifest = MacNodeHostManifest(
version: "test",
caps: ["screen", "computer"],
commands: [MacNodeScreenCommand.snapshot.rawValue, OpenClawComputerCommand.act.rawValue],
computerUse: descriptor,
pathEnv: "/usr/bin:/bin")
let peekaboo = try #require(MacNodeModeCoordinator.workerManifest(manifest, for: .peekaboo))
#expect(!peekaboo.commands.contains(MacNodeScreenCommand.snapshot.rawValue))
#expect(!peekaboo.commands.contains(OpenClawComputerCommand.act.rawValue))
#expect(peekaboo.computerUse == nil)
let cua = try #require(MacNodeModeCoordinator.workerManifest(manifest, for: .cua))
#expect(cua.commands == manifest.commands)
#expect(MacNodeModeCoordinator.computerUseDescriptor(
provider: .cua,
commands: cua.commands,
workerManifest: cua) == descriptor)
}
@Test func `stale route updates cannot replace newer worker authority`() {
#expect(MacNodeHostWorker.routeUpdateIsCurrent(candidateGeneration: 4, currentGeneration: 4))
#expect(MacNodeHostWorker.routeUpdateIsCurrent(candidateGeneration: 5, currentGeneration: 4))
@@ -267,6 +326,25 @@ struct MacNodeHostWorkerTests {
await worker.stop()
}
@Test func `worker receives only the app-provided CUA endpoint`() async throws {
let worker = MacNodeHostWorker(session: GatewayNodeSession())
let script = """
test "$OPENCLAW_CUA_DRIVER_SOCKET_PATH" = "/private/test/cua.sock" || exit 41
test "$OPENCLAW_CUA_DRIVER_BINARY_PATH" = "/Applications/OpenClaw.app/Contents/Resources/cua-driver" || exit 42
printf '%s\\n' '{"type":"ready","version":"test","manifest":{"caps":[],"commands":[],"pathEnv":"/usr/bin:/bin"},"inventory":{"skills":null,"pluginTools":[]}}'
while IFS= read -r line; do :; done
"""
_ = try await worker.start(launch: MacNodeHostWorkerLaunch(
command: ["/bin/sh", "-c", script],
environment: [
CuaDriverWorkerEnvironment.socketPath: "/private/test/cua.sock",
CuaDriverWorkerEnvironment.binaryPath:
"/Applications/OpenClaw.app/Contents/Resources/cua-driver",
]))
await worker.stop()
}
@Test func `worker forwards terminal input and cancellation frames`() async throws {
let worker = MacNodeHostWorker(session: GatewayNodeSession())
let script = """
@@ -307,6 +307,13 @@ struct MacNodeModeCoordinatorTests {
nextPaused: false,
previousComputerControlEnabled: true,
nextComputerControlEnabled: true))
#expect(MacNodeModeCoordinator.controlTransitionRequiresRouteInvalidation(
previousPaused: false,
nextPaused: false,
previousComputerControlEnabled: true,
nextComputerControlEnabled: true,
previousComputerControlProvider: .peekaboo,
nextComputerControlProvider: .cua))
}
@Test func `first endpoint snapshot rejects a stale captured endpoint`() throws {
@@ -6,6 +6,41 @@ import Testing
@testable import OpenClaw
struct MacNodeRuntimeTests {
private actor ComputerProviderWorkerProbe: MacNodeHostWorking {
private let commands: Set<String>
private(set) var invokedCommands: [String] = []
init(commands: Set<String>) {
self.commands = commands
}
func start(launch _: MacNodeHostWorkerLaunch) async throws -> MacNodeHostManifest {
MacNodeHostManifest(
version: "test",
caps: ["screen", "computer"],
commands: Array(self.commands),
pathEnv: "/usr/bin:/bin")
}
func supports(_ command: String) -> Bool {
self.commands.contains(command)
}
func invoke(_ request: BridgeInvokeRequest) -> BridgeInvokeResponse {
self.invokedCommands.append(request.command)
return BridgeInvokeResponse(id: request.id, ok: true, payloadJSON: #"{"owner":"worker"}"#)
}
func handleInput(invokeId _: String, seq _: Int, payloadJSON _: String) {}
func cancel(invokeId _: String) {}
func setRoute(_: GatewayNodeSessionRoute?, authorityGeneration _: UInt64) -> Bool {
true
}
func publishInventory(ifCurrentRoute _: GatewayNodeSessionRoute) {}
func stop() {}
}
private final class LockedCounter: @unchecked Sendable {
private let lock = NSLock()
private var count = 0
@@ -625,6 +660,68 @@ struct MacNodeRuntimeTests {
#expect(result.cursorX == 12)
}
@Test func `provider selection owns both snapshot and action without cross-provider fallback`() async throws {
let commands: Set<String> = [MacNodeScreenCommand.snapshot.rawValue, OpenClawComputerCommand.act.rawValue]
let cuaWorker = ComputerProviderWorkerProbe(commands: commands)
let cuaServices = await MainActor.run { MainActorServicesProbe() }
let cuaRuntime = MacNodeRuntime(
nodeHostWorker: cuaWorker,
makeMainActorServices: { cuaServices },
computerControlEnabled: { true },
computerControlProvider: { .cua })
let action = OpenClawComputerActParams(action: .leftClick, x: 12, y: 34, refWidth: 1280)
#expect(await (self.invoke(
cuaRuntime,
"cua-snapshot",
MacNodeScreenCommand.snapshot.rawValue)).ok)
#expect(try await (self.invoke(
cuaRuntime,
"cua-action",
OpenClawComputerCommand.act.rawValue,
params: action)).ok)
#expect(await cuaWorker.invokedCommands == [
MacNodeScreenCommand.snapshot.rawValue,
OpenClawComputerCommand.act.rawValue,
])
#expect(await MainActor.run { cuaServices.snapshotCallCount == 0 && cuaServices.performCallCount == 0 })
let peekabooWorker = ComputerProviderWorkerProbe(commands: commands)
let peekabooServices = await MainActor.run { MainActorServicesProbe() }
let peekabooRuntime = MacNodeRuntime(
nodeHostWorker: peekabooWorker,
makeMainActorServices: { peekabooServices },
computerControlEnabled: { true },
computerControlProvider: { .peekaboo })
#expect(await (self.invoke(
peekabooRuntime,
"peekaboo-snapshot",
MacNodeScreenCommand.snapshot.rawValue)).ok)
#expect(try await (self.invoke(
peekabooRuntime,
"peekaboo-action",
OpenClawComputerCommand.act.rawValue,
params: action)).ok)
#expect(await peekabooWorker.invokedCommands.isEmpty)
#expect(await MainActor.run {
peekabooServices.snapshotCallCount == 1 && peekabooServices.performCallCount == 1
})
let unavailableWorker = ComputerProviderWorkerProbe(commands: [])
let unavailableServices = await MainActor.run { MainActorServicesProbe() }
let unavailableRuntime = MacNodeRuntime(
nodeHostWorker: unavailableWorker,
makeMainActorServices: { unavailableServices },
computerControlEnabled: { true },
computerControlProvider: { .cua })
let unavailable = await self.invoke(
unavailableRuntime,
"cua-unavailable",
MacNodeScreenCommand.snapshot.rawValue)
#expect(!unavailable.ok)
#expect(await MainActor.run { unavailableServices.snapshotCallCount == 0 })
}
@Test func `concurrent invokes share one main actor services initialization`() async throws {
let services = await MainActor.run { MainActorServicesProbe() }
let factoryGate = AsyncTestGate()
@@ -490,15 +490,7 @@ public actor GatewayChannelActor {
"role": ProtoAnyCodable(role),
"scopes": ProtoAnyCodable(scopes),
]
if !options.commands.isEmpty {
params["commands"] = ProtoAnyCodable(options.commands)
}
if let pathEnv = options.pathEnv?.trimmingCharacters(in: .whitespacesAndNewlines), !pathEnv.isEmpty {
params["pathEnv"] = ProtoAnyCodable(pathEnv)
}
if !options.permissions.isEmpty {
params["permissions"] = ProtoAnyCodable(options.permissions)
}
options.applyOptionalConnectParams(to: &params)
self.applyConnectAuth(
selectedAuth,
deviceId: identity?.deviceId,
@@ -1,3 +1,5 @@
import OpenClawProtocol
public enum OpenClawGatewayClientCapability {
public static let agentKind = "agent-kind"
public static let inlineWidgets = "inline-widgets"
@@ -9,6 +11,7 @@ public struct GatewayConnectOptions: Sendable {
public var scopesAreExplicit: Bool
public var caps: [String]
public var commands: [String]
public var computerUse: AnyCodable?
public var pathEnv: String?
public var permissions: [String: Bool]
public var clientId: String
@@ -32,6 +35,7 @@ public struct GatewayConnectOptions: Sendable {
scopesAreExplicit: Bool = false,
caps: [String],
commands: [String],
computerUse: AnyCodable? = nil,
pathEnv: String? = nil,
permissions: [String: Bool],
clientId: String,
@@ -47,6 +51,7 @@ public struct GatewayConnectOptions: Sendable {
self.scopesAreExplicit = scopesAreExplicit
self.caps = caps
self.commands = commands
self.computerUse = computerUse
self.pathEnv = pathEnv
self.permissions = permissions
self.clientId = clientId
@@ -73,3 +78,24 @@ public struct GatewayAuthBinding: Equatable, Sendable {
public let source: GatewayAuthSource
public let credentialFingerprint: String?
}
extension GatewayConnectOptions {
/// Additive connect-frame fields, sent only when this node declares them.
/// Lives here so `GatewayChannel.sendConnect` stays within its body budget.
func applyOptionalConnectParams(to params: inout [String: OpenClawProtocol.AnyCodable]) {
if !self.commands.isEmpty {
params["commands"] = OpenClawProtocol.AnyCodable(self.commands)
}
if let computerUse = self.computerUse {
params["computerUse"] = computerUse
}
if let pathEnv = self.pathEnv?.trimmingCharacters(in: .whitespacesAndNewlines),
!pathEnv.isEmpty
{
params["pathEnv"] = OpenClawProtocol.AnyCodable(pathEnv)
}
if !self.permissions.isEmpty {
params["permissions"] = OpenClawProtocol.AnyCodable(self.permissions)
}
}
}
+36 -17
View File
@@ -7,15 +7,18 @@ read_when:
title: "Computer use"
---
Computer use lets the gateway agent see and control a capable paired desktop. Eligibility is capability-based: the connected node must advertise both `computer.act` and `screen.snapshot`, whose result must include a `displayFrameId`. The tool captures a screenshot as its reference frame, then drives the pointer and keyboard through `computer.act`. The action set follows the core Anthropic computer-use actions; optional `computer_20251124` zoom is not exposed. A vision-capable model drives it through the built-in `computer` agent tool.
Computer use lets the gateway agent see and control a capable paired desktop. Eligibility is capability-based: the connected node must advertise both `computer.act` and `screen.snapshot`. The node's descriptor identifies the supported v2 action, target, observation, and delivery families, so the built-in `computer` tool exposes only what that provider can faithfully execute. Coordinate actions bind to a node-issued reference frame; capable providers can also address windows and elements, request background delivery, and return structured effect or refusal evidence. A vision-capable model drives the surface through the built-in `computer` agent tool.
The agent emits one uniform command, `computer.act`; it cannot tell how a node fulfills it. The bundled macOS app handles the command in-process with embedded Peekaboo services plus narrow CoreGraphics primitives (correct TCC permissions, no extra process). Windows and Linux can use the optional, experimental `cua-computer` plugin, which calls the packaged CUA Driver SDK directly. Both fulfillers use the same durable local enablement and pairing policy.
The agent emits one uniform command, `computer.act`; it cannot choose how a node fulfills it. On macOS, **Settings → General → Capabilities** selects the node-local provider: Peekaboo is the default and preserves the existing in-process coordinate-action path, while CUA uses a driver daemon embedded in `OpenClaw.app`. The app spawns that daemon directly so it inherits OpenClaw's Accessibility and Screen Recording grants, and the app-owned node worker connects through a private socket. Windows and Linux can use the optional, experimental `cua-computer` plugin, which calls the packaged CUA Driver SDK directly.
Provider selection never falls back per action. Switching providers closes the active execution surface, rotates the provider generation, and re-advertises the node commands. A CUA failure therefore becomes an unavailable result instead of silently running the same action through Peekaboo.
## Requirements
- A paired, connected node advertising both `computer.act` and `screen.snapshot`, with `screen.snapshot` returning `displayFrameId`.
- **macOS fulfiller:** app setting **Allow Computer Control** enabled. It defaults on; an explicit off choice stays off.
- **macOS fulfiller:** **Accessibility** and Event Posting access granted to OpenClaw (for pointer/keyboard injection), plus **Screen Recording** permission (for `screen.snapshot`).
- **macOS fulfiller:** choose **Peekaboo** (default) or **CUA**. CUA is selectable only when the pinned driver is present in the signed app bundle; development builds without that artifact show **driver not bundled**.
- **macOS fulfiller:** **Accessibility** and **Screen Recording** granted to OpenClaw. The native Peekaboo path also requires Event Posting access for its CoreGraphics input primitives.
- **Windows/Linux fulfiller:** bundled `cua-computer` plugin enabled. Its package includes the pinned CUA Driver SDK 0.19.3 runtime; no `cua-driver` executable, daemon, or MCP server is configured.
- The pairing update that includes `computer.act` approved on the gateway.
- A vision-capable agent model.
@@ -31,11 +34,27 @@ The built-in `computer` tool takes one action per call. Coordinates are non-nega
- Keyboard: `type` (text), `key` (combo such as `cmd+shift+t` or `Return`), `hold_key` (`text` combo held for `duration` seconds).
- Pacing: `wait` (`duration` seconds).
Providers with the v2 window/element family can additionally expose `list_apps`, `list_windows`, `get_accessibility_tree`, `get_cursor_position`, `get_window_state`, `launch_app`, `kill_app`, `bring_to_front`, `set_value`, `zoom`, `escalate_scope`, and `invoke_menu`. The provider descriptor is authoritative; unavailable actions are omitted rather than emulated through another provider.
Modifier keys ride the `text` field on click and scroll actions (`shift`, `ctrl`, `alt`, `cmd`). After an input action the tool returns a fresh screenshot so the model can observe the result. If more than one computer-capable node is connected, pass `node` explicitly.
Screenshots are kept **model-only**: they are never auto-delivered to the chat channel. Treat all on-screen content as untrusted input; the tool warns the model not to follow on-screen instructions that conflict with the user's request.
## Windows and Linux (experimental, via CUA Driver SDK)
## CUA Driver provider
### macOS app-owned daemon
The signed macOS app bundles the universal `cua-driver` 0.19.3 executable and offers **CUA** in the Computer Control provider picker. OpenClaw creates a private, owner-only socket directory under Application Support and starts `cua-driver serve --embedded` as a direct app child. It does not launch through the Gateway, the TypeScript worker, `open(1)`, or `NSWorkspace`; those paths would break macOS's TCC responsibility chain and create a second permission identity.
The app waits until the private socket accepts connections before advertising CUA readiness. Its TypeScript node worker starts only the unprivileged MCP proxy against that socket and maps the same typed `computer.act` v2 actions used on other platforms. Permission changes restart the daemon, and provider changes, disabling Computer Control, app shutdown, or an unexpected child exit remove the advertised CUA commands until a fresh generation is ready.
#### Trust model
The embedded CUA daemon runs in unrestricted mode because bounded CUA grants require exact launch-time resources and cannot represent OpenClaw's runtime-discovered windows and elements. OpenClaw command arming, pairing approval, and tool policy are the authoritative authorization gate, identical to the shipped Peekaboo fulfiller. The app owns the daemon and its macOS TCC identity, and the daemon accepts local connections only through an owner-only socket directory.
The CUA descriptor advertises window and element targets, background and foreground delivery, screenshots, and accessibility observations. Peekaboo remains the default in this release and advertises the existing coordinate-action family; its v2 adapter is separate work.
### Windows and Linux (experimental, direct SDK)
The bundled `cua-computer` plugin provides an experimental fulfiller for Windows and Linux node hosts. It is disabled by default and uses the pinned CUA Driver SDK 0.19.3 contract directly:
@@ -63,21 +82,21 @@ This fulfiller currently controls only the primary display. `hold_key`, `left_mo
The plugin calls `CuaDriver.createConfigured`, never bare `create()`. Its authorization ceiling, trusted session identifier, TTLs, and desktop scope are fixed by OpenClaw; model-facing `screen.snapshot` and `computer.act` inputs cannot select a session or widen that authority. Because the driver reports no stable display identity, frame authorization binds to the trusted session generation plus live primary-display geometry. A new session invalidates outstanding frames, but a same-geometry primary-display substitution inside one session cannot be detected; prefer a stable single-display session for this fulfiller.
This is a hard replacement of the former 0.10 daemon/MCP integration. OpenClaw does not spawn a CUA process, proxy an MCP client, or fall back to another CUA runtime.
On Windows and Linux this is a hard replacement of the former 0.10 daemon/MCP integration: OpenClaw does not spawn a CUA process or proxy an MCP client. macOS deliberately uses the app-owned embedded daemon described above so the driver remains in `OpenClaw.app`'s TCC responsibility chain. Neither path falls back to another provider for an individual action.
### Troubleshooting
The `cua-computer` fulfiller surfaces typed error codes in the tool result and node logs. Common ones:
| Code | Cause | Fix |
| ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `COMPUTER_DRIVER_UNAVAILABLE` | The CUA Driver SDK runtime cannot initialize, the node is not Windows/Linux, or its desktop permissions/session are unavailable. | Run `openclaw node run` inside the interactive desktop session and check the platform desktop permissions. Reinstall OpenClaw if its bundled CUA Driver SDK package is missing. |
| `COMPUTER_REFUSED_<code>` | The driver refused the action with a structured code such as `background_unavailable`, `background_occluded`, or `foreground_unavailable` (KDE/KWin Wayland). | Bring the target window forward, switch to X11, or use a supported compositor. See the compatibility notes above. |
| `COMPUTER_STALE_FRAME` | The coordinates referenced a screenshot that is no longer current (context compaction, a display geometry change, or a reference-width change). | Take a fresh `screenshot` before the coordinate action. |
| `COMPUTER_UNSUPPORTED_ACTION` | An action this fulfiller cannot faithfully deliver: `hold_key`, `left_mouse_down`, `left_mouse_up`, or modifier-held click/drag/scroll. | Use a supported action. The typed CUA Driver desktop contract has no held-input or modifier argument for these calls. |
| `COMPUTER_UNSUPPORTED_DISPLAY` | A non-primary `screenIndex`, a capture/screen geometry mismatch, or a cursor outside the primary display. | Drive the primary display only. |
| `COMPUTER_UNSUPPORTED_KEY` | A `key` value the driver cannot reproduce reliably: a digit or punctuation key whose shift state is layout-dependent, or an unknown key. | Send that text through the `type` action instead. |
| `COMPUTER_DRIVER_ERROR` / `COMPUTER_INVALID_REQUEST` | The driver failed without a structured code, or the action arguments were malformed. | Check the driver state and retake a screenshot; correct the action arguments. |
| Code | Cause | Fix |
| ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `COMPUTER_DRIVER_UNAVAILABLE` | The CUA runtime cannot initialize, the macOS app-owned endpoint is absent, or the desktop permissions/session are unavailable. | On macOS, verify CUA is selected and the bundled driver is ready; on Windows/Linux, run `openclaw node run` inside the interactive desktop session. Reinstall OpenClaw if the pinned runtime is missing. |
| `COMPUTER_REFUSED_<code>` | The driver refused the action with a structured code such as `background_unavailable`, `background_occluded`, or `foreground_unavailable` (KDE/KWin Wayland). | Bring the target window forward, switch to X11, or use a supported compositor. See the compatibility notes above. |
| `COMPUTER_STALE_FRAME` | The coordinates referenced a screenshot that is no longer current (context compaction, a display geometry change, or a reference-width change). | Take a fresh `screenshot` before the coordinate action. |
| `COMPUTER_UNSUPPORTED_ACTION` | An action this fulfiller cannot faithfully deliver: `hold_key`, `left_mouse_down`, `left_mouse_up`, or modifier-held click/drag/scroll. | Use a supported action. The typed CUA Driver desktop contract has no held-input or modifier argument for these calls. |
| `COMPUTER_UNSUPPORTED_DISPLAY` | A non-primary `screenIndex`, a capture/screen geometry mismatch, or a cursor outside the primary display. | Drive the primary display only. |
| `COMPUTER_UNSUPPORTED_KEY` | A `key` value the driver cannot reproduce reliably: a digit or punctuation key whose shift state is layout-dependent, or an unknown key. | Send that text through the `type` action instead. |
| `COMPUTER_DRIVER_ERROR` / `COMPUTER_INVALID_REQUEST` | The driver failed without a structured code, or the action arguments were malformed. | Check the driver state and retake a screenshot; correct the action arguments. |
## The `computer.act` node command
@@ -90,7 +109,7 @@ Reads reuse `screen.snapshot`; there is no second capture path. See [Camera and
## Authorization
1. Enable the platform fulfiller: on macOS, **Settings → Allow Computer Control** starts enabled, then grant **Accessibility** and **Screen Recording** under **Settings → Permissions**; on Windows/Linux, follow the experimental `cua-computer` setup above.
1. Enable the platform fulfiller: on macOS, **Settings → General → Capabilities → Allow Computer Control** starts enabled, then choose Peekaboo or CUA and grant **Accessibility** and **Screen Recording** under **Settings → Permissions**; on Windows/Linux, follow the experimental `cua-computer` setup above.
2. Approve the pairing update on the gateway (a new command forces re-pairing).
3. Expose the tool to the vision-capable agent. For the default `coding` profile:
@@ -108,11 +127,11 @@ Once the node-local control is enabled and the pairing update is approved, `comp
On macOS, default-on means a paired gateway can drive pointer and keyboard input as soon as the required macOS grants exist. There is no per-action confirmation. Turn off **Allow Computer Control** before pairing, or at any later time, to stop advertising and accepting `computer.act`.
`gateway.nodes.commands.deny` remains an explicit global revocation and always wins. For the macOS fulfiller, `computer.act` does not need a `gateway.nodes.commands.allow` entry. The experimental `cua-computer` plugin registers `computer.act` as a dangerous plugin node command, so once that plugin is enabled the operator must add it to `gateway.nodes.commands.allow` (see the Windows/Linux setup above); the plugin registration excludes it from the default allowlist regardless of platform. An authenticated operator with `operator.write` can invoke an enabled, paired command through `node.invoke`; there is no per-action admin check.
`gateway.nodes.commands.deny` remains an explicit global revocation and always wins. The native macOS Peekaboo fulfiller does not need a `gateway.nodes.commands.allow` entry. CUA registers `computer.act` as a dangerous plugin node command on every platform, so selecting CUA on macOS or enabling the plugin on Windows/Linux also requires an explicit `gateway.nodes.commands.allow` entry. An authenticated operator with `operator.write` can invoke an enabled, paired command through `node.invoke`; there is no per-action admin check.
## Safety
- Every layer (tool policy, gateway command policy, pairing, node-app setting, and platform permissions) must agree. For the current macOS fulfiller, that includes **Allow Computer Control**, Accessibility, and Screen Recording. Actions execute while those durable controls remain enabled; there is no per-action confirmation.
- Every layer (tool policy, gateway command policy, pairing, node-app setting, and platform permissions) must agree. On macOS that includes **Allow Computer Control**, Accessibility, and Screen Recording; the native Peekaboo path also requires Event Posting. Actions execute while those durable controls remain enabled; there is no per-action confirmation.
- The macOS fulfiller posts text one grapheme at a time, so cancellation, disconnect, pause, disable, or endpoint replacement stops it before the next grapheme. The experimental CUA Driver fulfiller passes node cancellation to the SDK for each call.
- Screenshots are model-only and never auto-sent to chat (issue [#44759](https://github.com/openclaw/openclaw/issues/44759)).
- Treat screen content as untrusted; it can carry prompt injection.
+9
View File
@@ -24,6 +24,15 @@ function validateManifestConfig(value: unknown) {
}
describe("cua-computer plugin registration", () => {
it("defaults on only for the app-gated macOS provider path", () => {
const manifest = JSON.parse(
fs.readFileSync(new URL("./openclaw.plugin.json", import.meta.url), "utf8"),
) as { enabledByDefault?: boolean; enabledByDefaultOnPlatforms?: string[] };
expect(manifest.enabledByDefault).toBe(false);
expect(manifest.enabledByDefaultOnPlatforms).toEqual(["darwin"]);
});
it("registers the screen and dangerous computer node-host commands", () => {
const commands: OpenClawPluginNodeHostCommand[] = [];
const policies: OpenClawPluginNodeInvokePolicy[] = [];
+1 -1
View File
@@ -14,7 +14,7 @@ const configSchema = buildPluginConfigSchema(CuaComputerConfigSchema);
export default definePluginEntry({
id: "cua-computer",
name: "CUA Computer",
description: "Experimental CUA Driver SDK computer control for Windows and Linux node hosts.",
description: "Experimental CUA Driver computer control for macOS, Windows, and Linux node hosts.",
configSchema,
register(api) {
const parsed = CuaComputerConfigSchema.safeParse(api.pluginConfig ?? {});
+2 -1
View File
@@ -4,8 +4,9 @@
"onStartup": true
},
"enabledByDefault": false,
"enabledByDefaultOnPlatforms": ["darwin"],
"name": "CUA Computer",
"description": "Experimental CUA Driver SDK computer control for Windows and Linux node hosts.",
"description": "Experimental CUA Driver computer control for macOS, Windows, and Linux node hosts.",
"configSchema": {
"type": "object",
"additionalProperties": false,
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@openclaw/cua-computer",
"version": "2026.8.1",
"description": "Experimental CUA Driver SDK computer control for Windows and Linux node hosts",
"description": "Experimental CUA Driver computer control for macOS, Windows, and Linux node hosts",
"type": "module",
"dependencies": {
"@trycua/cua-driver": "0.19.3",
+106 -3
View File
@@ -35,10 +35,24 @@ function result(structured: Record<string, unknown>, image = false): CuaToolResu
};
}
function driver() {
function driver(
options: {
geometry?: typeof geometry;
screenSize?: { width: number; height: number; scale_factor: number };
} = {},
) {
let generation = "execution-1";
const getDesktopState = vi.fn(async () => result(geometry, true));
const getScreenSize = vi.fn(async () => result({ width: 100, height: 50, scale_factor: 1 }));
const activeGeometry = options.geometry ?? geometry;
const getDesktopState = vi.fn(async () => result(activeGeometry, true));
const getScreenSize = vi.fn(async () =>
result(
options.screenSize ?? {
width: activeGeometry.screen_width,
height: activeGeometry.screen_height,
scale_factor: activeGeometry.scale_factor,
},
),
);
const click = vi.fn(async () => result({}));
const drag = vi.fn(async () => result({}));
const moveCursor = vi.fn(async () => result({}));
@@ -157,6 +171,95 @@ describe("cua-computer provider", () => {
expect(actions).toContain("get_window_state");
});
it("advertises the macOS mapping only with a complete app-provided endpoint", () => {
const { session } = driver();
const endpoint = {
OPENCLAW_CUA_DRIVER_SOCKET_PATH: "/tmp/openclaw-cua-test/driver.sock",
OPENCLAW_CUA_DRIVER_BINARY_PATH: process.execPath,
};
const provider = createCuaComputerProvider({
platform: "darwin",
env: endpoint,
driver: session,
});
expect(provider.isAvailable()).toBe(true);
expect(provider.capabilities().actions).toContain("get_window_state");
expect(provider.capabilities().actions).not.toContain("left_mouse_down");
expect(provider.capabilities().features).toEqual({
recording: false,
agentCursor: false,
multiDisplay: false,
});
const createDriver = vi.fn(() => session);
expect(
createCuaComputerProvider({
platform: "darwin",
env: endpoint,
createDriver,
}).isAvailable(),
).toBe(true);
expect(createDriver).not.toHaveBeenCalled();
for (const env of [
{},
{ OPENCLAW_CUA_DRIVER_SOCKET_PATH: endpoint.OPENCLAW_CUA_DRIVER_SOCKET_PATH },
{ OPENCLAW_CUA_DRIVER_BINARY_PATH: endpoint.OPENCLAW_CUA_DRIVER_BINARY_PATH },
{ ...endpoint, OPENCLAW_CUA_DRIVER_SOCKET_PATH: "relative.sock" },
{ ...endpoint, OPENCLAW_CUA_DRIVER_BINARY_PATH: "/missing/cua-driver" },
]) {
expect(
createCuaComputerProvider({ platform: "darwin", env, driver: session }).isAvailable(),
).toBe(false);
}
});
it("keeps macOS Retina screenshots in native-pixel action coordinates", async () => {
const retina = driver({
geometry: {
platform: "macos",
display: "primary",
screenshot_width: 200,
screenshot_height: 100,
screen_width: 100,
screen_height: 50,
scale_factor: 2,
},
screenSize: { width: 100, height: 50, scale_factor: 2 },
});
const computer = await createCuaComputerProvider({
platform: "darwin",
env: {
OPENCLAW_CUA_DRIVER_SOCKET_PATH: "/tmp/openclaw-cua-test/driver.sock",
OPENCLAW_CUA_DRIVER_BINARY_PATH: process.execPath,
},
driver: retina.session,
imageProcessor: {
encode: vi.fn(async () => ({ data: Buffer.from("png"), width: 100, height: 50 })),
},
}).openExecution({});
const screen = JSON.parse(await computer.snapshot('{"format":"png","maxWidth":100}')) as {
displayFrameId: string;
width: number;
};
await computer.act(
JSON.stringify({
action: "left_click",
displayFrameId: screen.displayFrameId,
refWidth: screen.width,
x: 10,
y: 10,
}),
);
expect(retina.click).toHaveBeenCalledWith(
{ x: 20, y: 20, button: ClickButton.Left, count: 1 },
undefined,
);
});
it("uses one typed session for snapshot and frame-authorized click", async () => {
const { session, getDesktopState, getScreenSize, click } = driver();
const computer = await execution(session);
+57 -13
View File
@@ -30,6 +30,7 @@ import {
type CuaLastFrame,
type CuaScreenSize,
} from "./frame.js";
import { createCuaMcpDriver } from "./mcp-driver-client.js";
import { handleV2Act, type CuaComputerActParams } from "./v2-actions.js";
const AVAILABILITY_POLL_MS = 5_000;
@@ -38,6 +39,8 @@ const CUA_WIRE_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(1, 14);
// capture, not the delivered frame. 8K (7680x4320 = ~33.2M) is a valid primary
// display; budget above it so full-resolution snapshots reach the downscaler.
const MAX_IMAGE_PIXELS = 40_000_000;
const CUA_DRIVER_SOCKET_PATH_ENV = "OPENCLAW_CUA_DRIVER_SOCKET_PATH";
const CUA_DRIVER_BINARY_PATH_ENV = "OPENCLAW_CUA_DRIVER_BINARY_PATH";
const DesktopStateSchema = z.object({
platform: z.string().min(1),
@@ -76,6 +79,30 @@ type CuaComputerProviderOptions = {
clearInterval?: typeof clearInterval;
};
function resolveMacOsMcpEndpoint(
env: NodeJS.ProcessEnv,
): { socketPath: string; binaryPath: string } | undefined {
const socketPath = env[CUA_DRIVER_SOCKET_PATH_ENV]?.trim();
const binaryPath = env[CUA_DRIVER_BINARY_PATH_ENV]?.trim();
if (!socketPath || !binaryPath) {
return undefined;
}
if (
socketPath.includes("\0") ||
binaryPath.includes("\0") ||
!path.isAbsolute(socketPath) ||
!path.isAbsolute(binaryPath)
) {
return undefined;
}
try {
fs.accessSync(binaryPath, fs.constants.X_OK);
} catch {
return undefined;
}
return { socketPath, binaryPath };
}
class PromiseQueue {
private tail: Promise<void> = Promise.resolve();
@@ -204,7 +231,7 @@ function clickArgs(
const modifiers = normalizeModifiers(params.modifiers);
if (modifiers.length > 0) {
throw new Error(
"COMPUTER_UNSUPPORTED_ACTION: modifier-held clicks are unsupported by cua-driver on Linux",
"COMPUTER_UNSUPPORTED_ACTION: modifier-held desktop clicks are unsupported by cua-driver",
);
}
return {
@@ -397,6 +424,7 @@ export function createCuaComputerProvider(
): ComputerUseProvider {
const platform = options.platform ?? process.platform;
const env = options.env ?? process.env;
const macOsEndpoint = platform === "darwin" ? resolveMacOsMcpEndpoint(env) : undefined;
let ownedDriver: CuaDriverSession | undefined;
let stopped = false;
// The node host owns one trusted SDK session for this command execution.
@@ -405,7 +433,13 @@ export function createCuaComputerProvider(
if (stopped) {
throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-computer is stopping");
}
return options.driver ?? (ownedDriver ??= (options.createDriver ?? createCuaDriver)());
return (
options.driver ??
(ownedDriver ??= (
options.createDriver ??
(macOsEndpoint ? () => createCuaMcpDriver({ ...macOsEndpoint, env }) : createCuaDriver)
)())
);
};
const disposeOwnedDriver = async () => {
stopped = true;
@@ -416,8 +450,14 @@ export function createCuaComputerProvider(
const imageProcessor = options.imageProcessor ?? createImageProcessor(env);
const interval = options.setInterval ?? setInterval;
const clear = options.clearInterval ?? clearInterval;
const isSupportedPlatform = platform === "linux" || platform === "win32";
const isAvailable = () => isSupportedPlatform && driver().isAvailable();
const isSupportedPlatform =
platform === "linux" || platform === "win32" || macOsEndpoint !== undefined;
// The app injects the endpoint only after the host-owned daemon socket is
// accepting connections. Node-host manifests are one-shot, so the validated
// endpoint pair is the synchronous macOS readiness lease; invocation still
// awaits the MCP initialize handshake and fails visibly if it cannot attach.
const isAvailable = () =>
macOsEndpoint !== undefined || (isSupportedPlatform && driver().isAvailable());
return {
id: "cua-computer",
@@ -462,7 +502,9 @@ export function createCuaComputerProvider(
await queue.run(async () => {
if (!isSupportedPlatform) {
throw new Error(
"COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux",
platform === "darwin"
? `COMPUTER_DRIVER_UNAVAILABLE: cua-computer requires app-provided ${CUA_DRIVER_SOCKET_PATH_ENV} and ${CUA_DRIVER_BINARY_PATH_ENV}`
: "COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports macOS, Windows, and Linux",
);
}
const params = parseScreenSnapshotParamsJSON(paramsJSON);
@@ -472,14 +514,14 @@ export function createCuaComputerProvider(
const quality = Math.min(1, Math.max(0.05, params.quality ?? 0.72));
const desktop = await driver().getDesktopState(signal);
const geometry = desktopGeometry(desktop);
// cua-driver desktop input consumes native get_desktop_state PNG pixels,
// and on every supported backend the driver reports screen geometry in
// that same physical-pixel space (Windows PMv2, Linux X11/Wayland). If a
// capture ever diverges from screen geometry, our screenshot->native
// scaling would mis-target input, so refuse rather than click blind.
// Windows and Linux report capture and input geometry in the same
// physical-pixel space. macOS intentionally reports logical screen
// points plus native Retina pixels; its desktop tools consume the
// native screenshot coordinates and undo that scale internally.
if (
geometry.screenWidth !== geometry.screenshotWidth ||
geometry.screenHeight !== geometry.screenshotHeight
platform !== "darwin" &&
(geometry.screenWidth !== geometry.screenshotWidth ||
geometry.screenHeight !== geometry.screenshotHeight)
) {
throw new Error(
"COMPUTER_UNSUPPORTED_DISPLAY: cua-driver reported capture and screen geometry in different pixel spaces",
@@ -514,7 +556,9 @@ export function createCuaComputerProvider(
await queue.run(async () => {
if (!isSupportedPlatform) {
throw new Error(
"COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux",
platform === "darwin"
? `COMPUTER_DRIVER_UNAVAILABLE: cua-computer requires app-provided ${CUA_DRIVER_SOCKET_PATH_ENV} and ${CUA_DRIVER_BINARY_PATH_ENV}`
: "COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports macOS, Windows, and Linux",
);
}
return await handleV2Act(
@@ -0,0 +1,288 @@
import fs from "node:fs/promises";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import { describe, expect, it, vi } from "vitest";
import { ClickButton } from "./driver-client.js";
import { createCuaMcpDriver } from "./mcp-driver-client.js";
type RpcRequest = {
id?: number;
method: string;
params?: { name?: string; arguments?: Record<string, unknown> };
};
type FakeEndpoint = {
binaryPath: string;
socketPath: string;
requests: RpcRequest[];
respond: (request: RpcRequest, result: unknown) => void;
writeRaw: (request: RpcRequest, value: string) => void;
close: () => Promise<void>;
};
async function createFakeEndpoint(
handle: (request: RpcRequest, endpoint: FakeEndpoint) => void,
): Promise<FakeEndpoint> {
const directory = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-cua-mcp-test-"));
const socketPath = path.join(directory, "daemon.sock");
const binaryPath = path.join(directory, "cua-driver");
await fs.writeFile(
binaryPath,
`#!/usr/bin/env node
const net = require("node:net");
const args = process.argv.slice(2);
if (args.length !== 4 || args[0] !== "mcp" || args[1] !== "--embedded" || args[2] !== "--socket") process.exit(64);
if (process.env.CUA_DRIVER_EMBEDDED !== undefined || process.env.CUA_DRIVER_PERMISSION_MODE !== undefined || process.env.CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS !== undefined || process.env.CUA_DRIVER_RS_TELEMETRY_ENABLED !== "false" || process.env.CUA_DRIVER_RS_UPDATE_CHECK !== "false") process.exit(65);
const socket = net.createConnection(args[3]);
process.stdin.pipe(socket);
socket.pipe(process.stdout);
socket.on("error", (error) => { process.stderr.write(error.message); process.exit(66); });
socket.on("close", () => process.exit(0));
`,
{ mode: 0o700 },
);
const requests: RpcRequest[] = [];
const connections = new Set<net.Socket>();
const writers = new Map<number, net.Socket>();
const endpoint = {} as FakeEndpoint;
const server = net.createServer((socket) => {
connections.add(socket);
socket.once("close", () => connections.delete(socket));
let buffer = Buffer.alloc(0);
socket.on("data", (chunk: Buffer) => {
buffer = Buffer.concat([buffer, chunk]);
while (true) {
const newline = buffer.indexOf(0x0a);
if (newline < 0) {
break;
}
const line = buffer.subarray(0, newline);
buffer = buffer.subarray(newline + 1);
if (line.length === 0) {
continue;
}
const request = JSON.parse(line.toString("utf8")) as RpcRequest;
requests.push(request);
if (typeof request.id === "number") {
writers.set(request.id, socket);
}
handle(request, endpoint);
}
});
});
await new Promise<void>((resolve, reject) => {
server.once("error", reject);
server.listen(socketPath, () => {
resolve();
});
});
Object.assign(endpoint, {
binaryPath,
socketPath,
requests,
respond: (request: RpcRequest, result: unknown) => {
const writer = request.id === undefined ? undefined : writers.get(request.id);
writer?.write(`${JSON.stringify({ jsonrpc: "2.0", id: request.id, result })}\n`);
},
writeRaw: (request: RpcRequest, value: string) => {
const writer = request.id === undefined ? undefined : writers.get(request.id);
writer?.write(value);
},
close: async () => {
for (const connection of connections) {
connection.destroy();
}
await new Promise<void>((resolve) => {
server.close(() => {
resolve();
});
});
await fs.rm(directory, { recursive: true, force: true });
},
});
return endpoint;
}
function toolResult(structuredContent: Record<string, unknown>, image = false) {
return {
content: [
{ type: "text", text: "ok" },
...(image
? [{ type: "image", mimeType: "image/png", data: Buffer.from("png").toString("base64") }]
: []),
],
isError: false,
structuredContent,
};
}
function sessionState(scope: "window" | "desktop") {
return toolResult({
session: "openclaw-test",
capture_scope: scope,
effective_scope: scope,
desktop_unlocked: scope === "desktop",
escalation_reason: null,
escalation_detail: null,
});
}
describe.runIf(process.platform !== "win32")("CUA MCP proxy transport", () => {
it("initializes the bundled proxy and translates tool results through CuaDriverSession", async () => {
let closed = false;
const endpoint = await createFakeEndpoint((request, fake) => {
if (request.method === "initialize") {
fake.respond(request, {
protocolVersion: "2025-06-18",
capabilities: { tools: {} },
serverInfo: { name: "fake-cua-driver", version: "0.19.3" },
});
return;
}
if (request.method !== "tools/call") {
return;
}
switch (request.params?.name) {
case "start_session":
fake.respond(request, sessionState("desktop"));
break;
case "get_desktop_state":
fake.respond(
request,
toolResult(
{
platform: "macos",
display: "primary",
screenshot_width: 200,
screenshot_height: 100,
screen_width: 100,
screen_height: 50,
scale_factor: 2,
},
true,
),
);
break;
case "click":
fake.respond(
request,
toolResult({
effect: "confirmed",
route: "accessibility",
delivery: { mode: "background", delivered_count: 1 },
evidence: [{ kind: "value_readback" }],
}),
);
break;
case "end_session":
fake.respond(request, toolResult({ session: "openclaw-test", active: false }));
break;
default:
break;
}
});
try {
const driver = createCuaMcpDriver({
...endpoint,
env: {
...process.env,
CUA_DRIVER_EMBEDDED: "1",
CUA_DRIVER_PERMISSION_MODE: "unrestricted",
CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS: "1",
},
});
await vi.waitFor(() => expect(driver.isAvailable()).toBe(true));
const desktop = await driver.getDesktopState();
expect(JSON.parse(desktop.structuredJson!)).toMatchObject({
platform: "macos",
screenshot_width: 200,
});
expect(desktop.images).toHaveLength(1);
const click = await driver.click({ x: 20, y: 30, button: ClickButton.Left, count: 1 });
expect(click.action).toEqual({
effect: 0,
route: 0,
delivery: { mode: 0, deliveredCount: 1 },
evidence: [{ kind: 0 }],
});
await driver.dispose();
await vi.waitFor(() => {
closed = endpoint.requests.some(
(request) => request.method === "tools/call" && request.params?.name === "end_session",
);
expect(closed).toBe(true);
});
expect(endpoint.requests.map((request) => request.method)).toContain(
"notifications/initialized",
);
expect(
endpoint.requests.find(
(request) => request.method === "tools/call" && request.params?.name === "click",
)?.params?.arguments,
).toMatchObject({ x: 20, y: 30, button: "left", count: 1, scope: "desktop" });
} finally {
await endpoint.close();
}
});
it("fails closed on invalid proxy JSON", async () => {
const endpoint = await createFakeEndpoint((request, fake) => {
if (request.method === "initialize" && request.id !== undefined) {
fake.writeRaw(request, "not-json\n");
}
});
try {
const driver = createCuaMcpDriver(endpoint);
await expect(driver.getDesktopState()).rejects.toThrow("COMPUTER_DRIVER_ERROR");
expect(driver.isAvailable()).toBe(false);
await driver.dispose();
} finally {
await endpoint.close();
}
});
it("bounds pending calls and tears down the proxy on cancellation", async () => {
const held: RpcRequest[] = [];
const endpoint = await createFakeEndpoint((request, fake) => {
if (request.method === "initialize") {
fake.respond(request, {
protocolVersion: "2025-06-18",
capabilities: { tools: {} },
serverInfo: { name: "fake-cua-driver", version: "0.19.3" },
});
} else if (request.method === "tools/call" && request.params?.name === "start_session") {
fake.respond(request, sessionState("window"));
} else if (request.method === "tools/call" && request.params?.name === "list_windows") {
held.push(request);
} else if (request.method === "tools/call" && request.params?.name === "end_session") {
fake.respond(request, toolResult({ session: "openclaw-test", active: false }));
}
});
try {
const driver = createCuaMcpDriver(endpoint);
await vi.waitFor(() => expect(driver.isAvailable()).toBe(true));
const calls = Array.from({ length: 65 }, () => driver.callTool("list_windows", {}));
await expect(calls[64]).rejects.toThrow("too many pending requests");
await vi.waitFor(() => expect(held).toHaveLength(64));
for (const request of held) {
endpoint.respond(request, toolResult({ windows: [] }));
}
await Promise.all(calls.slice(0, 64));
const controller = new AbortController();
const cancelled = driver.callTool("list_windows", {}, controller.signal);
await vi.waitFor(() => expect(held).toHaveLength(65));
controller.abort(new Error("test cancellation"));
await expect(cancelled).rejects.toThrow("request was cancelled");
expect(driver.isAvailable()).toBe(false);
await driver.dispose();
} finally {
await endpoint.close();
}
});
});
@@ -0,0 +1,682 @@
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
import { randomUUID } from "node:crypto";
import type { ActionResult } from "@trycua/cua-driver";
import {
ClickButton,
EscalationReason,
ScrollDirection,
type CuaDriverSession,
type CuaToolResult,
} from "./driver-client.js";
const MCP_PROTOCOL_VERSION = "2025-06-18";
const MCP_STARTUP_TIMEOUT_MS = 10_000;
const MCP_REQUEST_TIMEOUT_MS = 120_000;
const MCP_SHUTDOWN_TIMEOUT_MS = 2_000;
const MAX_MCP_LINE_BYTES = 256 * 1024 * 1024;
const MAX_PENDING_REQUESTS = 64;
const MAX_STDERR_BYTES = 32 * 1024;
const ACTION_RESULT_TOOLS = new Set([
"click",
"double_click",
"right_click",
"scroll",
"drag",
"mouse_drag",
"parallel_mouse_drag",
"move_cursor",
"mouse_button_down",
"mouse_button_up",
"type_text",
"type_text_chars",
"press_key",
"hotkey",
"set_value",
"set_window_frame",
"invoke_menu",
"browser_click",
"browser_pointer",
"browser_type",
]);
type JsonRpcResponse = {
jsonrpc?: unknown;
id?: unknown;
result?: unknown;
error?: { code?: unknown; message?: unknown };
};
type PendingRequest = {
resolve: (value: unknown) => void;
reject: (error: Error) => void;
timer: NodeJS.Timeout;
signal?: AbortSignal;
onAbort?: () => void;
};
type McpToolResult = {
content?: Array<{ type?: unknown; text?: unknown; data?: unknown; mimeType?: unknown }>;
isError?: unknown;
structuredContent?: unknown;
};
function driverUnavailable(message: string, cause?: unknown): Error {
return new Error(`COMPUTER_DRIVER_UNAVAILABLE: ${message}`, { cause });
}
function driverProtocolError(message: string, cause?: unknown): Error {
return new Error(`COMPUTER_DRIVER_ERROR: ${message}`, { cause });
}
function record(value: unknown): Record<string, unknown> | undefined {
return value !== null && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: undefined;
}
function mappedEnum(value: unknown, values: readonly string[], label: string): number {
if (typeof value !== "string") {
throw driverProtocolError(`CUA MCP ${label} is missing`);
}
const index = values.indexOf(value);
if (index < 0) {
throw driverProtocolError(`CUA MCP ${label} is invalid`);
}
return index;
}
function mcpActionResult(tool: string, structured: unknown): ActionResult | undefined {
if (!ACTION_RESULT_TOOLS.has(tool)) {
return undefined;
}
const value = record(structured);
if (!value) {
throw driverProtocolError(`CUA MCP ${tool} returned no ActionResult`);
}
const delivery = record(value.delivery);
const escalation = record(value.escalation);
const evidence = Array.isArray(value.evidence) ? value.evidence : undefined;
return {
effect: mappedEnum(
value.effect,
["confirmed", "partial", "unverifiable", "suspected_noop", "refused"],
"action effect",
),
route: mappedEnum(
value.route,
["accessibility", "synthetic_events", "global_input", "system_api", "dom", "trusted_input"],
"action route",
),
...(delivery
? {
delivery: {
mode: mappedEnum(
delivery.mode,
["background", "foreground", "not_applicable", "unknown"],
"delivery mode",
),
...(typeof delivery.delivered_count === "number"
? { deliveredCount: delivery.delivered_count }
: {}),
},
}
: {}),
...(evidence
? {
evidence: evidence.map((entry) => ({
kind: mappedEnum(
record(entry)?.kind,
["value_readback", "window_change"],
"evidence kind",
),
})),
}
: {}),
...(escalation
? {
escalation: {
target: mappedEnum(
escalation.target,
["pixel", "foreground", "page", "session"],
"escalation target",
),
reason: mappedEnum(
escalation.reason,
[
"route_unavailable",
"delivery_failed",
"effect_unconfirmed",
"suspected_noop",
"permission_required",
],
"escalation reason",
),
},
}
: {}),
} as ActionResult;
}
function normalizeMcpToolResult(tool: string, raw: unknown): CuaToolResult {
const value = record(raw) as McpToolResult | undefined;
if (!value) {
throw driverProtocolError(`CUA MCP ${tool} returned a non-object result`);
}
const content = Array.isArray(value.content) ? value.content : [];
const text = content.flatMap((entry) =>
entry?.type === "text" && typeof entry.text === "string" ? [entry.text] : [],
);
const images = content.flatMap((entry) =>
entry?.type === "image" && typeof entry.data === "string" && typeof entry.mimeType === "string"
? [{ dataBase64: entry.data, mimeType: entry.mimeType }]
: [],
);
const structured = record(value.structuredContent);
const errorCode =
typeof structured?.code === "string"
? structured.code
: typeof record(structured?.refusal)?.code === "string"
? (record(structured?.refusal)?.code as string)
: undefined;
const isError = value.isError === true;
return {
text: text.join("\n"),
images,
...(structured ? { structuredJson: JSON.stringify(structured) } : {}),
isError,
...(errorCode ? { errorCode } : {}),
...(!isError ? { action: mcpActionResult(tool, structured) } : {}),
degraded: structured?.degraded === true,
rawJson: JSON.stringify(raw),
};
}
class CuaMcpProxyClient {
private readonly child: ChildProcessWithoutNullStreams;
private readonly pending = new Map<number, PendingRequest>();
private readonly ready: Promise<void>;
private nextId = 0;
private stdout = Buffer.alloc(0);
private stderr = Buffer.alloc(0);
private available = false;
private failure: Error | undefined;
private stopped = false;
constructor(binaryPath: string, socketPath: string, env: NodeJS.ProcessEnv) {
const proxyEnvironment = { ...env };
for (const key of Object.keys(proxyEnvironment)) {
if (key.startsWith("CUA_DRIVER_") || key === "CUA_TELEMETRY_ENABLED") {
delete proxyEnvironment[key];
}
}
this.child = spawn(binaryPath, ["mcp", "--embedded", "--socket", socketPath], {
env: {
...proxyEnvironment,
CUA_DRIVER_RS_TELEMETRY_ENABLED: "false",
CUA_DRIVER_RS_UPDATE_CHECK: "false",
},
stdio: ["pipe", "pipe", "pipe"],
});
this.child.stdout.on("data", (chunk: Buffer) => this.handleStdout(chunk));
this.child.stderr.on("data", (chunk: Buffer) => {
this.stderr = Buffer.concat([this.stderr, chunk]).subarray(-MAX_STDERR_BYTES);
});
this.child.once("error", (error) =>
this.fail(driverUnavailable("failed to start CUA MCP proxy", error)),
);
this.child.once("exit", (code, signal) => {
if (!this.stopped) {
const detail = this.stderr.toString("utf8").trim();
this.fail(
driverUnavailable(
`CUA MCP proxy exited (${signal ?? code ?? "unknown"})${detail ? `: ${detail}` : ""}`,
),
);
}
});
this.ready = this.initialize();
void this.ready.catch(() => {});
}
isAvailable(): boolean {
return this.available && !this.failure && !this.stopped;
}
async callTool(
name: string,
args: Record<string, unknown>,
signal?: AbortSignal,
): Promise<CuaToolResult> {
await this.ready;
return normalizeMcpToolResult(
name,
await this.request("tools/call", { name, arguments: args }, MCP_REQUEST_TIMEOUT_MS, signal),
);
}
async stop(): Promise<void> {
if (this.stopped) {
return;
}
this.stopped = true;
this.available = false;
this.rejectPending(driverUnavailable("CUA MCP proxy is stopping"));
this.child.stdin.end();
if (await this.waitForExit(MCP_SHUTDOWN_TIMEOUT_MS)) {
return;
}
this.child.kill("SIGTERM");
if (await this.waitForExit(MCP_SHUTDOWN_TIMEOUT_MS)) {
return;
}
this.child.kill("SIGKILL");
}
private async initialize(): Promise<void> {
const initialized = record(
await this.request(
"initialize",
{
protocolVersion: MCP_PROTOCOL_VERSION,
capabilities: {},
clientInfo: { name: "openclaw-cua-computer", version: "1" },
},
MCP_STARTUP_TIMEOUT_MS,
),
);
if (initialized?.protocolVersion !== MCP_PROTOCOL_VERSION) {
throw driverProtocolError("CUA MCP proxy returned an incompatible protocol version");
}
this.notify("notifications/initialized", {});
this.available = true;
}
private request(
method: string,
params: Record<string, unknown>,
timeoutMs: number,
signal?: AbortSignal,
): Promise<unknown> {
if (this.failure) {
return Promise.reject(this.failure);
}
if (this.stopped) {
return Promise.reject(driverUnavailable("CUA MCP proxy is stopping"));
}
if (signal?.aborted) {
return Promise.reject(driverUnavailable("CUA MCP request was cancelled", signal.reason));
}
if (this.pending.size >= MAX_PENDING_REQUESTS) {
return Promise.reject(driverUnavailable("CUA MCP proxy has too many pending requests"));
}
const id = ++this.nextId;
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
this.fail(driverUnavailable(`CUA MCP ${method} timed out after ${timeoutMs}ms`));
}, timeoutMs);
timer.unref?.();
const pending: PendingRequest = { resolve, reject, timer, signal };
if (signal) {
pending.onAbort = () =>
this.fail(driverUnavailable("CUA MCP request was cancelled", signal.reason));
signal.addEventListener("abort", pending.onAbort, { once: true });
}
this.pending.set(id, pending);
this.write({ jsonrpc: "2.0", id, method, params });
});
}
private notify(method: string, params: Record<string, unknown>): void {
this.write({ jsonrpc: "2.0", method, params });
}
private write(value: Record<string, unknown>): void {
this.child.stdin.write(`${JSON.stringify(value)}\n`, (error) => {
if (error) {
this.fail(driverUnavailable("failed writing to CUA MCP proxy", error));
}
});
}
private handleStdout(chunk: Buffer): void {
if (this.failure || this.stopped) {
return;
}
this.stdout = Buffer.concat([this.stdout, chunk]);
if (this.stdout.length > MAX_MCP_LINE_BYTES) {
this.fail(driverProtocolError("CUA MCP response exceeded the line-size limit"));
return;
}
while (true) {
const newline = this.stdout.indexOf(0x0a);
if (newline < 0) {
return;
}
const line = this.stdout.subarray(0, newline);
this.stdout = this.stdout.subarray(newline + 1);
if (line.length === 0) {
continue;
}
let response: JsonRpcResponse;
try {
response = JSON.parse(line.toString("utf8")) as JsonRpcResponse;
} catch (error) {
this.fail(driverProtocolError("CUA MCP proxy returned invalid JSON", error));
return;
}
if (response.jsonrpc !== "2.0") {
this.fail(driverProtocolError("CUA MCP proxy returned an invalid JSON-RPC version"));
return;
}
if (typeof response.id !== "number" || !Number.isSafeInteger(response.id)) {
this.fail(driverProtocolError("CUA MCP proxy returned an invalid response id"));
return;
}
const pending = this.pending.get(response.id);
if (!pending) {
continue;
}
this.pending.delete(response.id);
this.clearPending(pending);
if (response.error) {
const message =
typeof response.error.message === "string"
? response.error.message
: "unknown JSON-RPC error";
pending.reject(driverProtocolError(`CUA MCP request failed: ${message}`));
} else {
pending.resolve(response.result);
}
}
}
private fail(error: Error): void {
if (this.failure || this.stopped) {
return;
}
this.failure = error;
this.available = false;
this.rejectPending(error);
this.child.kill("SIGTERM");
}
private rejectPending(error: Error): void {
for (const pending of this.pending.values()) {
this.clearPending(pending);
pending.reject(error);
}
this.pending.clear();
}
private clearPending(pending: PendingRequest): void {
clearTimeout(pending.timer);
if (pending.signal && pending.onAbort) {
pending.signal.removeEventListener("abort", pending.onAbort);
}
}
private async waitForExit(timeoutMs: number): Promise<boolean> {
if (this.child.exitCode !== null || this.child.signalCode !== null) {
return true;
}
return await new Promise<boolean>((resolve) => {
const onExit = () => {
clearTimeout(timer);
resolve(true);
};
const timer = setTimeout(() => {
this.child.removeListener("exit", onExit);
resolve(false);
}, timeoutMs);
timer.unref?.();
this.child.once("exit", onExit);
});
}
}
function sessionState(value: CuaToolResult): import("@trycua/cua-driver").SessionStateOutput {
if (value.isError || !value.structuredJson) {
throw driverProtocolError(value.text || "CUA MCP session operation failed");
}
let structured: Record<string, unknown> | undefined;
try {
structured = record(JSON.parse(value.structuredJson));
} catch (error) {
throw driverProtocolError("CUA MCP session operation returned invalid JSON", error);
}
if (!structured) {
throw driverProtocolError("CUA MCP session operation returned invalid state");
}
return {
session: typeof structured.session === "string" ? structured.session : "",
captureScope: mappedEnum(
structured.capture_scope,
["auto", "window", "desktop"],
"capture scope",
),
effectiveScope: mappedEnum(
structured.effective_scope,
["window", "desktop"],
"effective scope",
),
desktopUnlocked: structured.desktop_unlocked === true,
...(typeof structured.escalation_reason === "string"
? {
escalationReason: mappedEnum(
structured.escalation_reason,
[
"ax_tree_pixel_mismatch",
"background_delivery_failed",
"foreground_ineffective",
"no_window_target",
"other",
],
"escalation reason",
),
}
: {}),
...(typeof structured.escalation_detail === "string"
? { escalationDetail: structured.escalation_detail }
: {}),
} as import("@trycua/cua-driver").SessionStateOutput;
}
class McpCuaDriverSession implements CuaDriverSession {
readonly generation = randomUUID();
private readonly publicSession = `openclaw-${randomUUID()}`;
private startPromise: Promise<void> | undefined;
private captureScope: "window" | "desktop" | undefined;
private started = false;
private disposed = false;
constructor(private readonly client: CuaMcpProxyClient) {}
isAvailable(): boolean {
return !this.disposed && this.client.isAvailable();
}
resetAvailabilityCache(): void {}
async callTool(name: string, args: Record<string, unknown>, signal?: AbortSignal) {
await this.ensureStarted("window", signal);
return await this.client.callTool(name, { ...args, session: this.publicSession }, signal);
}
async escalateScope(reason: EscalationReason, signal?: AbortSignal) {
await this.ensureStarted("window", signal);
const result = await this.client.callTool(
"escalate_session",
{
session: this.publicSession,
reason: [
"ax_tree_pixel_mismatch",
"background_delivery_failed",
"foreground_ineffective",
"no_window_target",
"other",
][reason],
},
signal,
);
this.captureScope = "desktop";
return sessionState(result);
}
async getDesktopState(signal?: AbortSignal) {
return await this.desktopTool("get_desktop_state", {}, signal);
}
async getScreenSize(signal?: AbortSignal) {
return await this.desktopTool("get_screen_size", {}, signal);
}
async click(
input: { x: number; y: number; button: ClickButton; count: number },
signal?: AbortSignal,
) {
return await this.desktopTool(
"click",
{
x: input.x,
y: input.y,
button: ["left", "right", "middle"][input.button],
count: input.count,
scope: "desktop",
},
signal,
);
}
async drag(
input: { fromX: number; fromY: number; toX: number; toY: number; durationMs?: bigint },
signal?: AbortSignal,
) {
return await this.desktopTool(
"drag",
{
from_x: input.fromX,
from_y: input.fromY,
to_x: input.toX,
to_y: input.toY,
...(input.durationMs === undefined ? {} : { duration_ms: Number(input.durationMs) }),
scope: "desktop",
},
signal,
);
}
async moveCursor(input: { x: number; y: number }, signal?: AbortSignal) {
return await this.desktopTool(
"move_cursor",
{ x: input.x, y: input.y, scope: "desktop" },
signal,
);
}
async scroll(
input: { x: number; y: number; direction: ScrollDirection; amount: bigint },
signal?: AbortSignal,
) {
return await this.desktopTool(
"scroll",
{
x: input.x,
y: input.y,
direction: ["up", "down", "left", "right"][input.direction],
by: "line",
amount: Number(input.amount),
scope: "desktop",
},
signal,
);
}
async typeText(text: string, signal?: AbortSignal) {
return await this.desktopTool("type_text", { text, scope: "desktop" }, signal);
}
async pressKey(input: { key: string; modifiers: string[] }, signal?: AbortSignal) {
return await this.desktopTool(
"press_key",
{ key: input.key, modifiers: input.modifiers, scope: "desktop" },
signal,
);
}
async dispose(): Promise<void> {
if (this.disposed) {
return;
}
this.disposed = true;
let failure: unknown;
try {
await this.startPromise;
if (this.started && this.client.isAvailable()) {
await this.client.callTool("end_session", { session: this.publicSession });
}
} catch (error) {
failure = error;
}
try {
await this.client.stop();
} catch (error) {
failure ??= error;
}
if (failure) {
throw failure instanceof Error
? failure
: driverUnavailable("CUA MCP cleanup failed", failure);
}
}
private async desktopTool(
name: string,
args: Record<string, unknown>,
signal?: AbortSignal,
): Promise<CuaToolResult> {
await this.ensureStarted("desktop", signal);
return await this.client.callTool(name, { ...args, session: this.publicSession }, signal);
}
private async ensureStarted(scope: "window" | "desktop", signal?: AbortSignal): Promise<void> {
if (this.disposed) {
throw driverUnavailable("cua-computer is stopping");
}
if (!this.startPromise) {
this.captureScope = scope;
const start = this.client
.callTool("start_session", { session: this.publicSession, capture_scope: scope }, signal)
.then((result) => {
if (result.isError) {
throw driverProtocolError(result.text || "CUA MCP start_session failed");
}
this.started = true;
});
this.startPromise = start;
try {
await start;
} catch (error) {
if (this.startPromise === start) {
this.startPromise = undefined;
}
throw error;
}
return;
}
await this.startPromise;
if (scope === "desktop" && this.captureScope !== "desktop") {
await this.escalateScope(EscalationReason.Other, signal);
}
}
}
export function createCuaMcpDriver(options: {
binaryPath: string;
socketPath: string;
env?: NodeJS.ProcessEnv;
}): CuaDriverSession {
return new McpCuaDriverSession(
new CuaMcpProxyClient(options.binaryPath, options.socketPath, options.env ?? process.env),
);
}
+1 -1
View File
@@ -132,7 +132,7 @@ const ANDROID_VERSION_SYNC_PATHS = new Set([
"apps/android/version.json",
]);
const MACOS_APP_CI_PATH_RE =
/^(?:apps\/(?:macos|macos-mlx-tts|shared|swabble)\/|Swabble\/|src\/(?:worker\/workspace-rsync-receiver\.ts|gateway\/worker-environments\/workspace-(?:accepted-(?:remote-script|sync)|mutation-remote-script|rsync-path\.test|sync(?:-helpers)?)\.ts)$|scripts\/(?:codesign-mac-app|create-dmg|mac-elevation-host|notarize-mac-artifact|package-mac-app|package-mac-dist)\.sh$|scripts\/lib\/(?:plistbuddy|swift-toolchain)\.sh$|test\/scripts\/(?:codesign-mac-app|create-dmg|mac-elevation-host|notarize-mac-artifact|package-mac-app|package-mac-dist)\.test\.ts$)/u;
/^(?:apps\/(?:macos|macos-mlx-tts|shared|swabble)\/|Swabble\/|src\/(?:worker\/workspace-rsync-receiver\.ts|gateway\/worker-environments\/workspace-(?:accepted-(?:remote-script|sync)|mutation-remote-script|rsync-path\.test|sync(?:-helpers)?)\.ts)$|scripts\/(?:codesign-mac-app|create-dmg|mac-elevation-host|notarize-mac-artifact|package-mac-app|package-mac-dist|stage-cua-driver-macos)\.sh$|scripts\/lib\/(?:plistbuddy|swift-toolchain)\.sh$|test\/scripts\/(?:codesign-mac-app|create-dmg|mac-elevation-host|notarize-mac-artifact|package-mac-app|package-mac-dist)\.test\.ts$)/u;
let corepackPnpmShimDir: string | undefined;
let corepackPnpmShimCleanupRegistered = false;
let cachedGeneratedExtensionAssetPaths: Set<string> | undefined;
+1 -1
View File
@@ -46,7 +46,7 @@ const APPLE_SHARED_CONTRACT_FIXTURE_RE =
const MACOS_NATIVE_RE =
/^(apps\/macos\/|apps\/macos-mlx-tts\/|apps\/shared\/|apps\/swabble\/|Swabble\/)/;
const MACOS_SCRIPT_SCOPE_RE =
/^(?:scripts\/(?:check-swift-tools|codesign-mac-app|create-dmg|format-swift|install-swift-tools|install-xcodegen|lint-swift|mac-elevation-host|notarize-mac-artifact|package-mac-app|package-mac-dist)\.sh|scripts\/lib\/(?:plistbuddy|swift-toolchain)\.sh|test\/scripts\/(?:codesign-mac-app|create-dmg|mac-elevation-host|notarize-mac-artifact|package-mac-app|package-mac-dist)\.test\.ts)$/;
/^(?:scripts\/(?:check-swift-tools|codesign-mac-app|create-dmg|format-swift|install-swift-tools|install-xcodegen|lint-swift|mac-elevation-host|notarize-mac-artifact|package-mac-app|package-mac-dist|stage-cua-driver-macos)\.sh|scripts\/lib\/(?:plistbuddy|swift-toolchain)\.sh|test\/scripts\/(?:codesign-mac-app|create-dmg|mac-elevation-host|notarize-mac-artifact|package-mac-app|package-mac-dist)\.test\.ts)$/;
const WORKSPACE_RSYNC_RECEIVER_SCOPE_RE =
/^src\/(?:worker\/workspace-rsync-receiver\.ts|gateway\/worker-environments\/workspace-(?:accepted-(?:remote-script|sync)|mutation-remote-script|rsync-path\.test|sync(?:-helpers)?)\.ts)$/;
const IOS_BUILD_RE =
+5
View File
@@ -361,6 +361,11 @@ if [ -f "$MLX_TTS_HELPER" ]; then
echo "Signing MLX TTS helper"; sign_plain_item "$MLX_TTS_HELPER"
fi
CUA_DRIVER="$APP_BUNDLE/Contents/Resources/cua-driver"
if [ -f "$CUA_DRIVER" ]; then
echo "Signing embedded CUA driver"; sign_plain_item "$CUA_DRIVER"
fi
# Sign main binary
if [ -f "$APP_BUNDLE/Contents/MacOS/OpenClaw" ]; then
echo "Signing main binary"; sign_item "$APP_BUNDLE/Contents/MacOS/OpenClaw" "$APP_ENTITLEMENTS"
+3
View File
@@ -566,6 +566,9 @@ fi
rm -rf "$APP_ROOT/Contents/Resources/ProviderIcons"
cp -R "$PROVIDER_ICONS_SRC" "$APP_ROOT/Contents/Resources/ProviderIcons"
echo "🖥 Staging embedded CUA driver"
"$ROOT_DIR/scripts/stage-cua-driver-macos.sh" "$APP_ROOT/Contents/Resources/cua-driver"
echo "📦 Copying CLI installer"
INSTALL_CLI_SRC="$ROOT_DIR/scripts/install-cli.sh"
if [ ! -f "$INSTALL_CLI_SRC" ]; then
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
VERSION="0.19.3"
TAG="cua-driver-rs-v${VERSION}"
ASSET="cua-driver-rs-${VERSION}-darwin-universal-binary.tar.gz"
EXPECTED_SHA256="733e28a3782ac8d325f8fce8b5d97486c1054af755b40dfd086151b34c79377e"
DOWNLOAD_URL="https://github.com/trycua/cua/releases/download/${TAG}/${ASSET}"
CACHE_DIR="$ROOT_DIR/apps/macos/.build/cua-driver/${TAG}"
ARCHIVE="$CACHE_DIR/$ASSET"
DESTINATION="${1:-}"
if [[ -z "$DESTINATION" || "$DESTINATION" == -* ]]; then
echo "Usage: scripts/stage-cua-driver-macos.sh <destination>" >&2
exit 2
fi
verify_archive() {
[[ -f "$ARCHIVE" ]] || return 1
local actual
actual="$(shasum -a 256 "$ARCHIVE" | awk '{print $1}')"
[[ "$actual" == "$EXPECTED_SHA256" ]]
}
mkdir -p "$CACHE_DIR"
if ! verify_archive; then
rm -f "$ARCHIVE"
partial="$ARCHIVE.partial.$$"
trap 'rm -f "$partial"' EXIT
curl --fail --location --retry 3 --retry-delay 2 --output "$partial" "$DOWNLOAD_URL"
actual="$(shasum -a 256 "$partial" | awk '{print $1}')"
if [[ "$actual" != "$EXPECTED_SHA256" ]]; then
echo "ERROR: CUA driver archive sha256 mismatch: got $actual" >&2
exit 1
fi
mv "$partial" "$ARCHIVE"
trap - EXIT
fi
extract_dir="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-cua-driver.XXXXXX")"
trap 'rm -rf "$extract_dir"' EXIT
tar -xzf "$ARCHIVE" -C "$extract_dir" cua-driver
source_binary="$extract_dir/cua-driver"
if [[ ! -f "$source_binary" || -L "$source_binary" ]]; then
echo "ERROR: CUA driver archive did not contain a regular cua-driver executable" >&2
exit 1
fi
archs="$(/usr/bin/lipo -archs "$source_binary")"
if [[ " $archs " != *" arm64 "* || " $archs " != *" x86_64 "* ]]; then
echo "ERROR: CUA driver is not universal (architectures: $archs)" >&2
exit 1
fi
mkdir -p "$(dirname "$DESTINATION")"
cp "$source_binary" "$DESTINATION"
chmod 0755 "$DESTINATION"
echo "Staged cua-driver $VERSION at $DESTINATION"
+1
View File
@@ -2510,6 +2510,7 @@ const SEMANTIC_TOOLING_TARGET_PATTERNS: Array<[RegExp, string[]]> = [
],
[/^scripts\/lib\/plistbuddy\.sh$/u, ["create-dmg", "package-mac-app", "package-mac-dist"]],
[/^scripts\/lib\/swift-toolchain\.sh$/u, ["package-mac-app", "package-mac-dist"]],
[/^scripts\/stage-cua-driver-macos\.sh$/u, ["package-mac-app"]],
[
/^scripts\/lib\/npm-publish-plan\.mjs$/u,
[
+23
View File
@@ -1342,6 +1342,29 @@ describe("package-mac-app plist stamping", () => {
);
});
it("stages the pinned universal CUA driver before nested-code signing", () => {
const packageScript = readFileSync(scriptPath, "utf8");
const stageScript = readFileSync("scripts/stage-cua-driver-macos.sh", "utf8");
const codesignScript = readFileSync("scripts/codesign-mac-app.sh", "utf8");
expect(stageScript).toContain('TAG="cua-driver-rs-v${VERSION}"');
expect(stageScript).toContain(
'EXPECTED_SHA256="733e28a3782ac8d325f8fce8b5d97486c1054af755b40dfd086151b34c79377e"',
);
expect(packageScript).toContain(
'"$ROOT_DIR/scripts/stage-cua-driver-macos.sh" "$APP_ROOT/Contents/Resources/cua-driver"',
);
expect(packageScript.indexOf("Staging embedded CUA driver")).toBeLessThan(
packageScript.indexOf('echo "🔏 Signing bundle'),
);
expect(codesignScript).toContain(
'echo "Signing embedded CUA driver"; sign_plain_item "$CUA_DRIVER"',
);
expect(codesignScript.indexOf("Signing embedded CUA driver")).toBeLessThan(
codesignScript.indexOf("# Finally sign the bundle"),
);
});
it("does not mask required Info.plist stamp failures", () => {
const script = readFileSync(scriptPath, "utf8");
const stampBlock = script.slice(