From a4f61d135a48fbfc598a1b3a0d356d853718efca Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 15 Aug 2026 01:08:54 -0700 Subject: [PATCH] feat(macos): Computer Use provider readiness checklist (#124093) * feat(macos): show computer provider readiness * refactor(macos): drop the status string superseded by the readiness checklist * chore(macos): refresh i18n baseline after removing the superseded status string --- apps/.i18n/native-source.json | 163 +++++++++++--- .../OpenClaw/ComputerActionService.swift | 8 - .../ComputerControlReadinessView.swift | 208 ++++++++++++++++++ .../Sources/OpenClaw/GeneralSettings.swift | 52 +---- .../ComputerControlSettingsTests.swift | 65 ++++++ 5 files changed, 410 insertions(+), 86 deletions(-) create mode 100644 apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift diff --git a/apps/.i18n/native-source.json b/apps/.i18n/native-source.json index 904388057670..0e76c4445689 100644 --- a/apps/.i18n/native-source.json +++ b/apps/.i18n/native-source.json @@ -20552,6 +20552,17 @@ } ] }, + { + "id": "native.apple.1f848d6b8b8c9a5b", + "source": "Accessibility", + "surface": "apple", + "sites": [ + { + "kind": "ui-localized-call", + "path": "apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift" + } + ] + }, { "id": "native.apple.dad3a39ff4502cb0", "source": "Accessibility access is required to attach text from \\(appName).", @@ -20563,17 +20574,6 @@ } ] }, - { - "id": "native.apple.c8aea204934bc72d", - "source": "Accessibility grant may be stale", - "surface": "apple", - "sites": [ - { - "kind": "conditional-branch", - "path": "apps/macos/Sources/OpenClaw/ComputerActionService.swift" - } - ] - }, { "id": "native.apple.8749684a8c17f426", "source": "Accessibility permission is required to paste.", @@ -22703,6 +22703,10 @@ { "kind": "ui-call", "path": "apps/macos/Sources/OpenClaw/ChannelsSettings+View.swift" + }, + { + "kind": "ui-localized-call", + "path": "apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift" } ] }, @@ -23174,6 +23178,17 @@ } ] }, + { + "id": "native.apple.1b206a7a4565f60b", + "source": "CUA daemon", + "surface": "apple", + "sites": [ + { + "kind": "ui-localized-call", + "path": "apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift" + } + ] + }, { "id": "native.apple.cfbd34b9f70ab403", "source": "Cache", @@ -24646,17 +24661,6 @@ } ] }, - { - "id": "native.apple.173d1e07a2d80eb6", - "source": "Computer Control access", - "surface": "apple", - "sites": [ - { - "kind": "ui-named-argument", - "path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift" - } - ] - }, { "id": "native.apple.7b46ae17b22676bf", "source": "Computer Control provider", @@ -30875,14 +30879,36 @@ } ] }, + { + "id": "native.apple.a32c9e42458c9b1c", + "source": "Grant OpenClaw in System Settings → Privacy & Security → Accessibility, then reopen it.", + "surface": "apple", + "sites": [ + { + "kind": "ui-localized-call-multiline", + "path": "apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift" + } + ] + }, + { + "id": "native.apple.cdb8cc9716727acf", + "source": "Grant OpenClaw in System Settings → Privacy & Security → Screen Recording, then reopen it.", + "surface": "apple", + "sites": [ + { + "kind": "ui-localized-call-multiline", + "path": "apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift" + } + ] + }, { "id": "native.apple.a2b0395080861914", "source": "Granted", "surface": "apple", "sites": [ { - "kind": "conditional-branch", - "path": "apps/macos/Sources/OpenClaw/ComputerActionService.swift" + "kind": "ui-localized-call", + "path": "apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift" }, { "kind": "ui-call", @@ -31974,6 +32000,17 @@ } ] }, + { + "id": "native.apple.cff202e57912738d", + "source": "Keep OpenClaw open while it checks the embedded CUA daemon.", + "surface": "apple", + "sites": [ + { + "kind": "ui-localized-call", + "path": "apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift" + } + ] + }, { "id": "native.apple.b2a590882480d15e", "source": "Keep OpenClaw visible in the Dock. When off, windows still show the Dock icon while open.", @@ -33892,17 +33929,6 @@ } ] }, - { - "id": "native.apple.2f563e685fcd2e50", - "source": "Missing permission", - "surface": "apple", - "sites": [ - { - "kind": "conditional-branch", - "path": "apps/macos/Sources/OpenClaw/ComputerActionService.swift" - } - ] - }, { "id": "native.apple.b8b79ae74c5aa5ca", "source": "Missing: %@", @@ -35747,6 +35773,17 @@ } ] }, + { + "id": "native.apple.e0a495b711628049", + "source": "Not granted", + "surface": "apple", + "sites": [ + { + "kind": "ui-localized-call", + "path": "apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift" + } + ] + }, { "id": "native.apple.6141dea69ca38164", "source": "Not listening", @@ -35799,6 +35836,17 @@ } ] }, + { + "id": "native.apple.b4f3bc6fe500677b", + "source": "Not ready", + "surface": "apple", + "sites": [ + { + "kind": "ui-localized-call", + "path": "apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift" + } + ] + }, { "id": "native.apple.59d1d878b3170094", "source": "Not selected", @@ -40781,6 +40829,17 @@ } ] }, + { + "id": "native.apple.bb3054116d83e716", + "source": "Reopen OpenClaw to retry the embedded CUA daemon.", + "surface": "apple", + "sites": [ + { + "kind": "ui-localized-call", + "path": "apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift" + } + ] + }, { "id": "native.apple.d103d60717e1d702", "source": "Repairing the OpenClaw Gateway update…", @@ -42019,6 +42078,10 @@ "kind": "ui-localized-call", "path": "apps/ios/WatchApp/Sources/WatchInboxView.swift" }, + { + "kind": "ui-localized-call", + "path": "apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift" + }, { "kind": "conditional-branch", "path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift" @@ -42557,6 +42620,17 @@ } ] }, + { + "id": "native.apple.9c4ba1b5e037c818", + "source": "Screen Recording", + "surface": "apple", + "sites": [ + { + "kind": "ui-localized-call", + "path": "apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift" + } + ] + }, { "id": "native.apple.f7fb49bcff01c4bb", "source": "Screenshot: \\(label)", @@ -47830,6 +47904,10 @@ "kind": "ui-call", "path": "apps/macos/Sources/OpenClaw/AboutSettings.swift" }, + { + "kind": "ui-localized-call", + "path": "apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift" + }, { "kind": "conditional-branch", "path": "apps/macos/Sources/OpenClaw/VoiceWakeSettings.swift" @@ -47868,6 +47946,10 @@ "kind": "ui-localized-call", "path": "apps/ios/Sources/Design/SettingsProTabSupport.swift" }, + { + "kind": "ui-localized-call", + "path": "apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift" + }, { "kind": "conditional-branch", "path": "apps/macos/Sources/OpenClaw/DebugSettings.swift" @@ -48275,6 +48357,17 @@ } ] }, + { + "id": "native.apple.9fdb78560dae9056", + "source": "Use a packaged OpenClaw build that includes the CUA driver.", + "surface": "apple", + "sites": [ + { + "kind": "ui-localized-call", + "path": "apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift" + } + ] + }, { "id": "native.apple.927cfaa2ccf2e93d", "source": "Use a secure wss:// or Tailscale Serve Gateway, then scan a full-access setup code from the Control UI or openclaw qr and reconnect to enable settings and upgrades.", diff --git a/apps/macos/Sources/OpenClaw/ComputerActionService.swift b/apps/macos/Sources/OpenClaw/ComputerActionService.swift index affdb0f804d5..3af35c915c02 100644 --- a/apps/macos/Sources/OpenClaw/ComputerActionService.swift +++ b/apps/macos/Sources/OpenClaw/ComputerActionService.swift @@ -341,14 +341,6 @@ struct ComputerControlPermissionSnapshot: Equatable, Sendable { case missing([Bucket]) case accessibilityGrantMayBeStale - var statusText: String { - switch self { - case .granted: "Granted" - case .missing: "Missing permission" - case .accessibilityGrantMayBeStale: "Accessibility grant may be stale" - } - } - var detailText: String { switch self { case .granted: diff --git a/apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift b/apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift new file mode 100644 index 000000000000..e57a31a1ec1c --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ComputerControlReadinessView.swift @@ -0,0 +1,208 @@ +import AppKit +import OpenClawIPC +import SwiftUI + +struct ComputerControlReadinessRow: Equatable, Identifiable { + enum ID: Hashable { + case provider + case accessibility + case screenRecording + case cuaDaemon + } + + enum Status: Equatable { + case available + case granted + case running + case unavailable + case notGranted + case notReady + case unknown + + var text: String { + switch self { + case .available: String(localized: "Available") + case .granted: String(localized: "Granted") + case .running: String(localized: "Running") + case .unavailable: String(localized: "Unavailable") + case .notGranted: String(localized: "Not granted") + case .notReady: String(localized: "Not ready") + case .unknown: String(localized: "Unknown") + } + } + + var icon: String { + switch self { + case .available, .granted, .running: "checkmark.circle" + case .unavailable, .notGranted, .notReady: "exclamationmark.circle" + case .unknown: "questionmark.circle" + } + } + + var color: Color { + switch self { + case .available, .granted, .running: .green + case .unavailable, .notGranted, .notReady: .orange + case .unknown: .secondary + } + } + } + + let id: ID + let title: String + let status: Status + let nextStep: String? +} + +enum ComputerControlReadinessPresentation { + static func rows( + provider: ComputerControlProvider, + cuaDriverAvailable: Bool, + permissions: [Capability: CapabilityAuthorizationStatus], + cuaDaemonReady: Bool?) -> [ComputerControlReadinessRow] + { + let providerAvailable = provider == .peekaboo || cuaDriverAvailable + var rows = [ + ComputerControlReadinessRow( + id: .provider, + title: provider.displayName, + status: providerAvailable ? .available : .unavailable, + nextStep: providerAvailable + ? nil + : String(localized: "Use a packaged OpenClaw build that includes the CUA driver.")), + self.permissionRow( + id: .accessibility, + title: String(localized: "Accessibility"), + status: permissions[.accessibility], + nextStep: String( + localized: """ + Grant OpenClaw in System Settings → Privacy & Security → Accessibility, then reopen it. + """)), + self.permissionRow( + id: .screenRecording, + title: String(localized: "Screen Recording"), + status: permissions[.screenRecording], + nextStep: String( + localized: """ + Grant OpenClaw in System Settings → Privacy & Security → Screen Recording, then reopen it. + """)), + ] + + if provider == .cua { + let status: ComputerControlReadinessRow.Status = if !cuaDriverAvailable { + .unavailable + } else if let cuaDaemonReady { + cuaDaemonReady ? .running : .notReady + } else { + .unknown + } + let nextStep: String? = switch status { + case .running: + nil + case .unavailable: + String(localized: "Use a packaged OpenClaw build that includes the CUA driver.") + case .notReady: + String(localized: "Reopen OpenClaw to retry the embedded CUA daemon.") + case .unknown: + String(localized: "Keep OpenClaw open while it checks the embedded CUA daemon.") + case .available, .granted, .notGranted: + nil + } + rows.append(ComputerControlReadinessRow( + id: .cuaDaemon, + title: String(localized: "CUA daemon"), + status: status, + nextStep: nextStep)) + } + return rows + } + + private static func permissionRow( + id: ComputerControlReadinessRow.ID, + title: String, + status: CapabilityAuthorizationStatus?, + nextStep: String) -> ComputerControlReadinessRow + { + switch status ?? .unknown { + case .granted: + ComputerControlReadinessRow(id: id, title: title, status: .granted, nextStep: nil) + case .notGranted: + ComputerControlReadinessRow(id: id, title: title, status: .notGranted, nextStep: nextStep) + case .unknown: + ComputerControlReadinessRow(id: id, title: title, status: .unknown, nextStep: nextStep) + } + } +} + +@MainActor +struct ComputerControlReadinessView: View { + let provider: ComputerControlProvider + let cuaDriverAvailable: Bool + + @State private var permissions: [Capability: CapabilityAuthorizationStatus] = [:] + @State private var cuaDaemonReady: Bool? + + init(provider: ComputerControlProvider, cuaDriverAvailable: Bool) { + self.provider = provider + self.cuaDriverAvailable = cuaDriverAvailable + self._cuaDaemonReady = State(initialValue: Self.cuaDaemonReadiness( + provider: provider, + cuaDriverAvailable: cuaDriverAvailable)) + } + + var body: some View { + ForEach(self.rows) { row in + SettingsCardRow( + title: .verbatim(row.title), + subtitle: row.nextStep.map(SettingsTextValue.verbatim)) + { + Label { + Text(verbatim: row.status.text) + } icon: { + Image(systemName: row.status.icon) + } + .font(.caption.weight(.medium)) + .foregroundStyle(row.status.color) + } + } + .task(id: self.provider) { + await self.refreshPermissions() + self.refreshCuaDaemonReadiness() + } + .onReceive(NotificationCenter.default.publisher(for: NSApplication.didBecomeActiveNotification)) { _ in + Task { await self.refreshPermissions() } + } + .onReceive(NotificationCenter.default.publisher(for: .openclawPermissionsChanged)) { _ in + Task { await self.refreshPermissions() } + } + .onReceive(NotificationCenter.default.publisher(for: .openclawCuaDriverAvailabilityChanged)) { _ in + self.refreshCuaDaemonReadiness() + } + } + + private var rows: [ComputerControlReadinessRow] { + ComputerControlReadinessPresentation.rows( + provider: self.provider, + cuaDriverAvailable: self.cuaDriverAvailable, + permissions: self.permissions, + cuaDaemonReady: self.cuaDaemonReady) + } + + private func refreshPermissions() async { + self.permissions = await PermissionManager.authorizationStatus([.accessibility, .screenRecording]) + } + + private func refreshCuaDaemonReadiness() { + self.cuaDaemonReady = Self.cuaDaemonReadiness( + provider: self.provider, + cuaDriverAvailable: self.cuaDriverAvailable) + } + + private static func cuaDaemonReadiness( + provider: ComputerControlProvider, + cuaDriverAvailable: Bool) -> Bool? + { + guard provider == .cua, cuaDriverAvailable else { return nil } + return CuaDriverHostCoordinator.shared.workerEndpoint != nil + } +} diff --git a/apps/macos/Sources/OpenClaw/GeneralSettings.swift b/apps/macos/Sources/OpenClaw/GeneralSettings.swift index 22c7497dfa67..9edc20494f09 100644 --- a/apps/macos/Sources/OpenClaw/GeneralSettings.swift +++ b/apps/macos/Sources/OpenClaw/GeneralSettings.swift @@ -29,7 +29,6 @@ struct GeneralSettings: View { @State private var gatewayStatus: GatewayEnvironmentStatus = .checking @State private var remoteStatus: RemoteStatus = .idle @State private var showRemoteAdvanced = false - @State private var computerControlPermissions = ComputerControlPermissionSnapshot.probe() @State private var cookieSyncManager = CookieSyncManager.shared private let isPreview = ProcessInfo.processInfo.isPreview private var isNixMode: Bool { @@ -74,12 +73,6 @@ struct GeneralSettings: View { .onChange(of: self.computerControlProviderRaw) { _, _ in self.state.applyComputerControlHostState() } - .onReceive(NotificationCenter.default.publisher(for: NSApplication.didBecomeActiveNotification)) { _ in - self.refreshComputerControlPermissions() - } - .onReceive(NotificationCenter.default.publisher(for: .openclawPermissionsChanged)) { _ in - self.refreshComputerControlPermissions() - } .onDisappear { self.gatewayDiscovery.stop() } } @@ -148,17 +141,10 @@ struct GeneralSettings: View { self.computerControlProviderRow - SettingsCardRow( - title: "Computer Control access", - subtitle: .verbatim(self.computerControlPermissions.diagnostic.detailText)) - { - Label { - Text(verbatim: self.computerControlPermissions.diagnostic.statusText) - } icon: { - Image(systemName: self.computerControlPermissionIcon) - } - .font(.caption.weight(.medium)) - .foregroundStyle(self.computerControlPermissionColor) + if self.computerControlEnabled { + ComputerControlReadinessView( + provider: self.selectedComputerControlProvider, + cuaDriverAvailable: self.cuaDriverBundled) } SettingsCardToggleRow( @@ -377,7 +363,6 @@ struct GeneralSettings: View { private func updateActiveWork(active: Bool) { guard !self.isPreview else { return } if active { - self.refreshComputerControlPermissions() self.refreshGatewayStatus() if self.page == .connection { self.gatewayDiscovery.start() @@ -387,26 +372,6 @@ struct GeneralSettings: View { } } - private func refreshComputerControlPermissions() { - guard self.page == .general, self.isActive, !self.isPreview else { return } - self.computerControlPermissions = .probe() - } - - private var computerControlPermissionIcon: String { - switch self.computerControlPermissions.diagnostic { - case .granted: "checkmark.circle.fill" - case .missing: "exclamationmark.circle.fill" - case .accessibilityGrantMayBeStale: "exclamationmark.triangle.fill" - } - } - - private var computerControlPermissionColor: Color { - switch self.computerControlPermissions.diagnostic { - case .granted: .green - case .missing, .accessibilityGrantMayBeStale: .orange - } - } - private var connectionStatusPanel: some View { HStack(alignment: .center, spacing: 14) { ZStack { @@ -906,12 +871,13 @@ extension GeneralSettings { CuaDriverArtifact.bundledExecutableURL != nil } + private var selectedComputerControlProvider: ComputerControlProvider { + ComputerControlProvider.current() + } + private var computerControlProviderBinding: Binding { Binding( - get: { - let selected = ComputerControlProvider(rawValue: self.computerControlProviderRaw) ?? .peekaboo - return selected == .cua && !self.cuaDriverBundled ? .peekaboo : selected - }, + get: { self.selectedComputerControlProvider }, set: { provider in guard provider != .cua || self.cuaDriverBundled else { return } self.computerControlProviderRaw = provider.rawValue diff --git a/apps/macos/Tests/OpenClawIPCTests/ComputerControlSettingsTests.swift b/apps/macos/Tests/OpenClawIPCTests/ComputerControlSettingsTests.swift index 91e5310ad5f7..b6d176babf00 100644 --- a/apps/macos/Tests/OpenClawIPCTests/ComputerControlSettingsTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/ComputerControlSettingsTests.swift @@ -1,4 +1,5 @@ import Foundation +import OpenClawIPC import Testing @testable import OpenClaw @@ -62,4 +63,68 @@ struct ComputerControlSettingsTests { #expect(decoded["socketPath"] as? String == endpoint.socketPath) #expect(decoded["binaryPath"] as? String == endpoint.binaryPath) } + + @Test func `readiness rows never promote unknown or unavailable state`() { + struct Scenario { + let provider: ComputerControlProvider + let cuaDriverAvailable: Bool + let permissions: [Capability: CapabilityAuthorizationStatus] + let cuaDaemonReady: Bool? + let expectedStatuses: [ComputerControlReadinessRow.Status] + } + + let granted: [Capability: CapabilityAuthorizationStatus] = [ + .accessibility: .granted, + .screenRecording: .granted, + ] + let scenarios = [ + Scenario( + provider: .peekaboo, + cuaDriverAvailable: false, + permissions: granted, + cuaDaemonReady: nil, + expectedStatuses: [.available, .granted, .granted]), + Scenario( + provider: .cua, + cuaDriverAvailable: true, + permissions: granted, + cuaDaemonReady: true, + expectedStatuses: [.available, .granted, .granted, .running]), + Scenario( + provider: .cua, + cuaDriverAvailable: true, + permissions: [:], + cuaDaemonReady: nil, + expectedStatuses: [.available, .unknown, .unknown, .unknown]), + Scenario( + provider: .cua, + cuaDriverAvailable: true, + permissions: [.accessibility: .notGranted, .screenRecording: .notGranted], + cuaDaemonReady: false, + expectedStatuses: [.available, .notGranted, .notGranted, .notReady]), + Scenario( + provider: .cua, + cuaDriverAvailable: false, + permissions: granted, + cuaDaemonReady: true, + expectedStatuses: [.unavailable, .granted, .granted, .unavailable]), + ] + + for scenario in scenarios { + let rows = ComputerControlReadinessPresentation.rows( + provider: scenario.provider, + cuaDriverAvailable: scenario.cuaDriverAvailable, + permissions: scenario.permissions, + cuaDaemonReady: scenario.cuaDaemonReady) + #expect(rows.map(\.status) == scenario.expectedStatuses) + for row in rows { + switch row.status { + case .available, .granted, .running: + #expect(row.nextStep == nil) + case .unavailable, .notGranted, .notReady, .unknown: + #expect(row.nextStep?.isEmpty == false) + } + } + } + } }