diff --git a/apps/macos/Sources/OpenClaw/ComputerActionService.swift b/apps/macos/Sources/OpenClaw/ComputerActionService.swift index f6a691d53470..affdb0f804d5 100644 --- a/apps/macos/Sources/OpenClaw/ComputerActionService.swift +++ b/apps/macos/Sources/OpenClaw/ComputerActionService.swift @@ -452,6 +452,10 @@ final class ComputerActionService { case buttonNotHeld case eventCreationFailed case lifecycleChanged + case invalidV2Request(String) + case staleObservation + case unsupportedAction(OpenClawComputerAction) + case refused(String) var errorDescription: String? { switch self { @@ -491,6 +495,14 @@ final class ComputerActionService { "Failed to synthesize input event" case .lifecycleChanged: "Computer control lifecycle changed while the action was pending" + case let .invalidV2Request(message): + "COMPUTER_INVALID_REQUEST: \(message)" + case .staleObservation: + "COMPUTER_STALE_OBSERVATION: take a fresh observation and retry" + case let .unsupportedAction(action): + "COMPUTER_UNSUPPORTED_ACTION: \(action.rawValue)" + case let .refused(message): + "COMPUTER_REFUSED_action_refused: \(message)" } } } @@ -500,6 +512,7 @@ final class ComputerActionService { private let mouseEventFactory: MouseEventFactory private let mouseEventPoster: MouseEventPoster private let textGraphemePoster: TextGraphemePoster + private lazy var v2 = ComputerActionServiceV2() /// Tracks whether a left_mouse_down is outstanding so mouse_move emits /// drag events (state persists across invokes on the shared instance). private var leftButtonDown = false @@ -592,6 +605,27 @@ final class ComputerActionService { lifecycleGeneration: UInt64) async throws -> OpenClawComputerActResult { try self.executionQueue.checkExecutionAllowed(lifecycleGeneration: lifecycleGeneration) + if params.deliveryMode == .background, + params.windowRef == nil, + !params.action.isComputerActV2Only + { + return OpenClawComputerActResult( + ok: false, + effect: .suspectedNoop, + escalation: OpenClawComputerEscalation( + recommended: "window-pixel", + reasonCode: "no_window_target")) + } + if params.isV2Request { + return try await self.v2.perform( + params, + lifecycleGeneration: lifecycleGeneration, + checkExecutionAllowed: { [weak self] in + guard let self else { throw CancellationError() } + try self.executionQueue.checkExecutionAllowed( + lifecycleGeneration: lifecycleGeneration) + }) + } try Self.validateInputPermissions(ComputerControlPermissionSnapshot.probe()) let display = try await resolveDisplay(params: params) try executionQueue.checkExecutionAllowed(lifecycleGeneration: lifecycleGeneration) @@ -704,6 +738,8 @@ final class ComputerActionService { let keys = try requireKeys(params.keys) let holdMs = min(Self.maxHoldMs, max(0, params.durationMs ?? 1000)) try await self.automation.hotkey(keys: keys, holdDuration: holdMs) + default: + throw ComputerActionError.unsupportedAction(params.action) } } diff --git a/apps/macos/Sources/OpenClaw/ComputerActionServiceV2.swift b/apps/macos/Sources/OpenClaw/ComputerActionServiceV2.swift new file mode 100644 index 000000000000..cf79ff5a1e2f --- /dev/null +++ b/apps/macos/Sources/OpenClaw/ComputerActionServiceV2.swift @@ -0,0 +1,970 @@ +import AppKit +import CoreGraphics +import Foundation +import OpenClawKit +import PeekabooAutomationKit +import PeekabooFoundation + +extension CGRect { + fileprivate var centerPoint: CGPoint { + CGPoint(x: self.midX, y: self.midY) + } +} + +extension OpenClawComputerActParams { + var isV2Request: Bool { + self.action.isComputerActV2Only || self.windowRef != nil || self.elementRef != nil || + self.observationId != nil + } +} + +extension OpenClawComputerAction { + var isComputerActV2Only: Bool { + switch self { + case .leftClick, .rightClick, .middleClick, .doubleClick, .tripleClick, .mouseMove, + .leftClickDrag, .leftMouseDown, .leftMouseUp, .scroll, .type, .key, .holdKey: + false + default: + true + } + } +} + +@MainActor +struct ComputerActionExecutionAuthority { + let check: @MainActor () throws -> Void + + func run(_ operation: () async throws -> Result) async throws -> Result { + try self.check() + let result = try await operation() + try self.check() + return result + } +} + +/// Implements the additive computer.act v2 surface without changing the v1 +/// coordinate path. All authority-bearing references are process-local, +/// execution-local, and invalidated when the native lifecycle generation moves. +@MainActor +final class ComputerActionServiceV2 { + private struct WindowTarget { + let app: ServiceApplicationInfo + let window: ServiceWindowInfo + } + + private struct ElementTarget { + let id: String + let bounds: CGRect + } + + private struct ObservationState { + let id: String + let windowRef: String + let snapshotId: String + let elements: [String: ElementTarget] + } + + private let executionID = UUID().uuidString.lowercased() + private let automation: UIAutomationService + private let applications: ApplicationService + private let windows: WindowManagementService + private let menu: MenuService + private let observationService: DesktopObservationService + private var lifecycleGeneration: UInt64? + private var appRefs: [String: ServiceApplicationInfo] = [:] + private var windowRefs: [String: WindowTarget] = [:] + private var observation: ObservationState? + private var executionAuthority: ComputerActionExecutionAuthority? + + init() { + let snapshotManager = SnapshotManager() + let automation = UIAutomationService(snapshotManager: snapshotManager) + let applications = ApplicationService() + let menu = MenuService(applicationService: applications) + self.automation = automation + self.applications = applications + self.windows = WindowManagementService(applicationService: applications) + self.menu = menu + self.observationService = DesktopObservationService( + screenCapture: ScreenCaptureService(loggingService: LoggingService()), + automation: automation, + applications: applications, + menu: menu, + snapshotManager: snapshotManager) + } + + func perform( + _ params: OpenClawComputerActParams, + lifecycleGeneration: UInt64, + checkExecutionAllowed: @escaping @MainActor () throws -> Void) async throws + -> OpenClawComputerActResult + { + precondition(self.executionAuthority == nil) + let authority = ComputerActionExecutionAuthority(check: checkExecutionAllowed) + self.executionAuthority = authority + defer { self.executionAuthority = nil } + + return try await authority.run { + self.adoptLifecycleGeneration(lifecycleGeneration) + switch params.action { + case .listApps: + return try await self.listApps() + case .listWindows: + return try await self.listWindows() + case .getAccessibilityTree: + return try await self.getAccessibilityTree(params) + case .getCursorPosition: + return self.getCursorPosition() + case .getWindowState: + return try await self.getWindowState(params) + case .launchApp: + return try await self.launchApp(params) + case .killApp: + return try await self.killApp(params) + case .bringToFront: + return try await self.bringToFront(params) + case .setValue: + return try await self.setValue(params) + case .invokeMenu: + return try await self.invokeMenu(params) + case .leftClick, .rightClick, .middleClick, .doubleClick, .tripleClick: + return try await self.click(params) + case .type: + return try await self.type(params) + case .key: + return try await self.key(params) + case .scroll: + return try await self.scroll(params) + case .mouseMove, .leftClickDrag, .leftMouseDown, .leftMouseUp, .holdKey: + throw ComputerActionService.ComputerActionError.unsupportedAction(params.action) + case .screenshot, .wait, .zoom, .getBrowserState, .browserPrepare, .browserNavigate, + .browserClick, .browserType, .browserDialog, .browserSetInputFiles, .browserDownload, + .browserPointer, .escalateScope, .getRecordingState, .startRecording, .stopRecording, + .replayTrajectory: + throw ComputerActionService.ComputerActionError.unsupportedAction(params.action) + } + } + } + + private func withExecutionAuthority( + _ operation: () async throws -> Result) async throws -> Result + { + guard let executionAuthority = self.executionAuthority else { + preconditionFailure("Computer action executed outside its queue authority") + } + return try await executionAuthority.run(operation) + } + + // MARK: - Discovery and observation + + private func listApps() async throws -> OpenClawComputerActResult { + let output = try await self.withExecutionAuthority { + try await self.applications.listApplications() + } + self.appRefs.removeAll(keepingCapacity: true) + let rows: [[String: Any]] = output.data.applications.prefix(500).map { app in + let ref = self.issueRef("app") + self.appRefs[ref] = app + var row: [String: Any] = [ + "app": ref, + "name": app.name, + "running": true, + "active": app.isActive, + "pid": Int(app.processIdentifier), + ] + if let bundleIdentifier = app.bundleIdentifier { + row["bundleId"] = bundleIdentifier + } + return row + } + return OpenClawComputerActResult(ok: true, details: [ + "apps": AnyCodable(rows), + "totalApps": AnyCodable(output.data.applications.count), + "truncatedApps": AnyCodable(max(0, output.data.applications.count - rows.count)), + ]) + } + + private func listWindows() async throws -> OpenClawComputerActResult { + let appOutput = try await self.withExecutionAuthority { + try await self.applications.listApplications() + } + self.windowRefs.removeAll(keepingCapacity: true) + var rows: [[String: Any]] = [] + var warnings = appOutput.metadata.warnings + outer: for app in appOutput.data.applications + where app.windowCount > 0 || app.windowIDs?.isEmpty == false + { + do { + let output = try await self.withExecutionAuthority { + try await self.applications.listWindows( + for: "PID:\(app.processIdentifier)", + timeout: nil) + } + warnings.append(contentsOf: output.metadata.warnings) + for window in output.data.windows where window.layer == 0 { + let ref = self.issueWindowRef(app: app, window: window) + rows.append([ + "windowRef": ref, + "appName": app.name, + "title": window.title, + "bounds": Self.boundsDictionary(window.bounds), + "isOnScreen": window.isOnScreen, + "minimized": window.isMinimized, + ]) + if rows.count == 500 { break outer } + } + } catch { + warnings.append("\(app.name): \(error.localizedDescription)") + } + } + var details: [String: AnyCodable] = ["windows": AnyCodable(rows)] + if !warnings.isEmpty { + details["warnings"] = AnyCodable(Array(warnings.prefix(64))) + } + return OpenClawComputerActResult(ok: true, details: details) + } + + private func getAccessibilityTree( + _ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult + { + let limits = try Self.observationLimits(params) + let context: WindowContext? = try params.windowRef.map { ref in + let target = try self.resolveWindow(ref) + return Self.windowContext(target, limits: limits) + } + let result = try await self.withExecutionAuthority { + try await self.automation.inspectAccessibilityTree(windowContext: context) + } + let projected = Self.projectElements( + result.elements.all, + query: params.query, + limit: limits.maxElements) + var details: [String: AnyCodable] = [ + "elements": AnyCodable(projected.elements), + "totalElementCount": AnyCodable(result.elements.all.count), + ] + if projected.truncated > 0 { + details["truncatedElements"] = AnyCodable(projected.truncated) + } + if !result.metadata.warnings.isEmpty { + details["warnings"] = AnyCodable(Array(result.metadata.warnings.prefix(64))) + } + return OpenClawComputerActResult(ok: true, details: details) + } + + private func getCursorPosition() -> OpenClawComputerActResult { + let point = self.automation.currentMouseLocation() ?? .zero + return OpenClawComputerActResult(ok: true, details: [ + "x": AnyCodable(Double(point.x)), + "y": AnyCodable(Double(point.y)), + ]) + } + + private func getWindowState( + _ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult + { + let windowRef = try Self.require(params.windowRef, field: "windowRef") + let target = try self.resolveWindow(windowRef) + let limits = try Self.observationLimits(params) + guard let windowID = CGWindowID(exactly: target.window.windowID) else { + throw ComputerActionService.ComputerActionError.staleObservation + } + let request = DesktopObservationRequest( + target: .windowID(windowID), + capture: DesktopCaptureOptions(focus: .background), + detection: DesktopDetectionOptions( + mode: .accessibility, + traversalBudget: AXTraversalBudget( + maxDepth: limits.depth, + maxElementCount: limits.maxElements, + maxChildrenPerNode: AXTraversalBudget.defaultMaxChildrenPerNode))) + let result = try await self.withExecutionAuthority { + try await self.observationService.observe(request) + } + let observedWindow = result.capture.metadata.windowInfo ?? target.window + guard observedWindow.windowID == target.window.windowID else { + throw ComputerActionService.ComputerActionError.staleObservation + } + self.windowRefs[windowRef] = WindowTarget(app: target.app, window: observedWindow) + let detected = result.elements?.elements.all ?? [] + let filtered = Self.filterElements(detected, query: params.query) + let bounded = Array(filtered.prefix(limits.maxElements)) + let observationID = self.issueRef("observation") + var elementTargets: [String: ElementTarget] = [:] + let elements = bounded.map { element in + let ref = self.issueRef("element") + elementTargets[ref] = ElementTarget(id: element.id, bounds: element.bounds) + return OpenClawComputerObservationElement( + elementRef: ref, + role: element.type.rawValue, + label: element.label, + value: element.value, + bounds: Self.bounds(element.bounds)) + } + let snapshotID = result.elements?.snapshotId ?? "" + guard !snapshotID.isEmpty else { + throw ComputerActionService.ComputerActionError.refused( + "Peekaboo observation returned no snapshot receipt") + } + self.observation = ObservationState( + id: observationID, + windowRef: windowRef, + snapshotId: snapshotID, + elements: elementTargets) + var details: [String: AnyCodable] = [ + "totalElementCount": AnyCodable(detected.count), + "coordinateSpace": AnyCodable("global-logical-points"), + ] + if filtered.count > bounded.count { + details["truncatedElements"] = AnyCodable(filtered.count - bounded.count) + } + if !result.diagnostics.warnings.isEmpty { + details["warnings"] = AnyCodable(Array(result.diagnostics.warnings.prefix(64))) + } + let size = result.capture.metadata.size + guard size.width >= 1, size.height >= 1 else { + throw ComputerActionService.ComputerActionError.refused( + "Peekaboo observation returned an invalid image size") + } + return OpenClawComputerActResult( + ok: true, + observation: OpenClawComputerObservation( + kind: "window", + base64: result.capture.imageData.base64EncodedString(), + format: "png", + width: Int(size.width), + height: Int(size.height), + observationId: observationID, + elements: elements.isEmpty ? nil : elements), + details: details) + } + + // MARK: - Lifecycle + + private func launchApp(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult { + let appValue = try Self.require(params.app, field: "app") + let target = self.appRefs[appValue] + if appValue.hasPrefix("peekaboo:v2:app:"), target == nil { + throw ComputerActionService.ComputerActionError.staleObservation + } + let launched = try await self.withExecutionAuthority { + try await self.applications.launchApplication(request: ApplicationLaunchRequest( + applicationIdentifier: target?.bundleIdentifier ?? target?.name ?? appValue, + activates: true, + waitUntilReady: true, + waitForWindow: true)) + } + return OpenClawComputerActResult(ok: true, effect: .confirmed, details: [ + "name": AnyCodable(launched.name), + "pid": AnyCodable(Int(launched.processIdentifier)), + ]) + } + + private func killApp(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult { + let appRef = try Self.require(params.app, field: "app") + guard let app = self.appRefs[appRef], let identity = app.processIdentity else { + throw ComputerActionService.ComputerActionError.invalidV2Request( + "kill_app requires a running app reference from list_apps") + } + let quit = try await self.withExecutionAuthority { + try await self.applications.quitApplication(request: ApplicationQuitRequest( + identifier: "PID:\(app.processIdentifier)", + force: false, + expectedIdentity: identity)) + } + return OpenClawComputerActResult( + ok: quit, + effect: quit ? .confirmed : .suspectedNoop, + details: ["app": AnyCodable(appRef)]) + } + + private func bringToFront( + _ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult + { + let target = try self.requiredWindow(params) + try await self.focus(target) + let focused = try await self.withExecutionAuthority { + try await self.windows.getFocusedWindow() + } + let confirmed = focused?.windowID == target.window.windowID + return OpenClawComputerActResult( + ok: confirmed, + effect: confirmed ? .confirmed : .suspectedNoop, + escalation: confirmed ? nil : OpenClawComputerEscalation( + recommended: "desktop", + reasonCode: "foreground_ineffective")) + } + + // MARK: - Input + + private func click(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult { + let target = try self.requiredWindow(params) + let resolved = try self.clickTarget(params, windowRef: Self.require(params.windowRef, field: "windowRef")) + let mode = params.deliveryMode ?? .background + if mode == .foreground { + try await self.focus(target) + guard target.window.bounds.contains(resolved.point) else { + throw ComputerActionService.ComputerActionError.invalidV2Request( + "foreground click target is outside the selected window") + } + try Self.postForegroundClick( + at: resolved.point, + action: params.action, + modifiers: params.modifiers) + return OpenClawComputerActResult(ok: true, effect: .unverifiable, details: [ + "deliveryMode": AnyCodable("foreground"), + "route": AnyCodable("global_events"), + ]) + } + guard params.action != .middleClick, params.action != .tripleClick else { + return Self.backgroundEscalation(reason: "background_click_variant_unavailable") + } + guard let identity = target.window.mutationIdentity else { + throw ComputerActionService.ComputerActionError.staleObservation + } + let clickType: ClickType = switch params.action { + case .rightClick: .right + case .doubleClick: .double + default: .single + } + do { + let result = try await self.withExecutionAuthority { + try await self.automation.clickWithOutcome( + target: resolved.target, + clickType: clickType, + snapshotId: resolved.snapshotId, + expectedWindowIdentity: identity, + expectedWindowBounds: target.window.bounds) + } + return Self.result(from: result.outcome, background: true) + } catch let failure as DesktopActionFailure { + return Self.failureResult(failure, background: true) + } + } + + private func type(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult { + let text = try Self.require(params.text, field: "text", allowEmpty: false) + let target = try self.requiredWindow(params) + let mode = params.deliveryMode ?? .background + if let elementRef = params.elementRef { + let focusResult = try await self.focusElement( + elementRef, + params: params, + target: target, + foreground: mode == .foreground) + if !focusResult.ok { return focusResult } + } + do { + let result: UIAutomationActionResult + if mode == .foreground { + try await self.focus(target) + result = try await self.withExecutionAuthority { + try await self.automation.typeActionsWithOutcome( + [.text(text)], cadence: .fixed(milliseconds: 0), snapshotId: nil) + } + } else { + guard let identity = target.window.mutationIdentity else { + throw ComputerActionService.ComputerActionError.staleObservation + } + result = try await self.withExecutionAuthority { + try await self.automation.typeActionsWithOutcome( + [.text(text)], + cadence: .fixed(milliseconds: 0), + snapshotId: params.elementRef == nil ? nil : self.observation?.snapshotId, + expectedWindowIdentity: identity, + expectedWindowBounds: target.window.bounds) + } + } + return Self.result(from: result.outcome, background: mode == .background) + } catch let failure as DesktopActionFailure { + return Self.failureResult(failure, background: mode == .background) + } + } + + private func key(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult { + let keys = try Self.require(params.keys, field: "keys", allowEmpty: false) + let target = try self.requiredWindow(params) + let mode = params.deliveryMode ?? .background + if let elementRef = params.elementRef { + let focusResult = try await self.focusElement( + elementRef, + params: params, + target: target, + foreground: mode == .foreground) + if !focusResult.ok { return focusResult } + } + do { + let result: UIAutomationActionResult + if mode == .foreground { + try await self.focus(target) + result = try await self.withExecutionAuthority { + try await self.automation.hotkeyWithOutcome(keys: keys, holdDuration: 0) + } + } else { + guard let identity = target.window.mutationIdentity else { + throw ComputerActionService.ComputerActionError.staleObservation + } + result = try await self.withExecutionAuthority { + try await self.automation.hotkeyWithOutcome( + keys: keys, + holdDuration: 0, + expectedWindowIdentity: identity, + expectedWindowBounds: target.window.bounds) + } + } + return Self.result(from: result.outcome, background: mode == .background) + } catch let failure as DesktopActionFailure { + return Self.failureResult(failure, background: mode == .background) + } + } + + private func setValue(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult { + guard params.deliveryMode != .foreground else { + throw ComputerActionService.ComputerActionError.unsupportedAction(params.action) + } + let windowRef = try Self.require(params.windowRef, field: "windowRef") + _ = try self.resolveWindow(windowRef) + let element = try self.requiredElement(params, windowRef: windowRef) + let value = try Self.require(params.value, field: "value", allowEmpty: true) + do { + let result = try await self.withExecutionAuthority { + try await self.automation.setValueWithOutcome( + target: element.id, + value: .string(value), + snapshotId: self.observation?.snapshotId) + } + return Self.result(from: result.outcome, background: true) + } catch let failure as DesktopActionFailure { + return Self.failureResult(failure, background: true) + } + } + + private func invokeMenu(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult { + guard params.deliveryMode != .foreground else { + throw ComputerActionService.ComputerActionError.unsupportedAction(params.action) + } + let target = try self.requiredWindow(params) + guard let path = params.path, !path.isEmpty, path.count <= 16, + path.allSatisfy({ !$0.isEmpty && $0.count <= 200 }) + else { + throw ComputerActionService.ComputerActionError.invalidV2Request( + "path must contain 1 to 16 non-empty menu components") + } + try await self.withExecutionAuthority { + try await self.menu.clickMenuItem( + app: "PID:\(target.app.processIdentifier)", + itemPath: path.joined(separator: " > ")) + } + return OpenClawComputerActResult(ok: true, effect: .unverifiable, details: [ + "deliveryMode": AnyCodable("background"), + "route": AnyCodable("accessibility_action"), + ]) + } + + private func scroll(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult { + let windowRef = try Self.require(params.windowRef, field: "windowRef") + let target = try self.resolveWindow(windowRef) + guard let direction = params.scrollDirection else { + throw ComputerActionService.ComputerActionError.invalidV2Request( + "scrollDirection is required for scroll") + } + let mode = params.deliveryMode ?? .background + let element = try params.elementRef.map { _ in try self.requiredElement(params, windowRef: windowRef) } + if mode == .background, element == nil { + return Self.backgroundEscalation(reason: "background_scroll_requires_element") + } + if mode == .foreground { + try await self.focus(target) + if let point = try self.optionalPoint(params, windowRef: windowRef) { + try await self.withExecutionAuthority { + try await self.automation.moveMouse(to: point, duration: 0, steps: 1, profile: .linear) + } + } + } + let result = try await self.withExecutionAuthority { + try await self.automation.scrollWithOutcome(ScrollRequest( + direction: Self.scrollDirection(direction), + amount: min(100, max(1, params.scrollAmount ?? 3)), + target: element?.id, + snapshotId: element == nil ? nil : self.observation?.snapshotId, + foreground: mode == .foreground)) + } + return Self.result(from: result.outcome, background: mode == .background) + } + + // MARK: - Reference and target helpers + + private func adoptLifecycleGeneration(_ generation: UInt64) { + guard self.lifecycleGeneration != generation else { return } + self.lifecycleGeneration = generation + self.appRefs.removeAll() + self.windowRefs.removeAll() + self.observation = nil + } + + private func issueRef(_ kind: String) -> String { + "peekaboo:v2:\(kind):\(self.executionID):\(self.lifecycleGeneration ?? 0):" + + UUID().uuidString.lowercased() + } + + private func issueWindowRef(app: ServiceApplicationInfo, window: ServiceWindowInfo) -> String { + if let existing = self.windowRefs.first(where: { + $0.value.app.processIdentifier == app.processIdentifier && + $0.value.window.windowID == window.windowID && + $0.value.window.mutationIdentity == window.mutationIdentity + })?.key { + return existing + } + let ref = self.issueRef("window") + self.windowRefs[ref] = WindowTarget(app: app, window: window) + return ref + } + + private func resolveWindow(_ ref: String) throws -> WindowTarget { + guard let target = self.windowRefs[ref] else { + throw ComputerActionService.ComputerActionError.staleObservation + } + return target + } + + private func requiredWindow(_ params: OpenClawComputerActParams) throws -> WindowTarget { + try self.resolveWindow(Self.require(params.windowRef, field: "windowRef")) + } + + private func requiredObservation(_ params: OpenClawComputerActParams, windowRef: String) throws + -> ObservationState + { + guard let observation = self.observation, + observation.id == params.observationId, + observation.windowRef == windowRef + else { + throw ComputerActionService.ComputerActionError.staleObservation + } + return observation + } + + private func requiredElement( + _ params: OpenClawComputerActParams, + windowRef: String) throws -> ElementTarget + { + let elementRef = try Self.require(params.elementRef, field: "elementRef") + let observation = try self.requiredObservation(params, windowRef: windowRef) + guard let element = observation.elements[elementRef] else { + throw ComputerActionService.ComputerActionError.staleObservation + } + return element + } + + private func clickTarget( + _ params: OpenClawComputerActParams, + windowRef: String) throws -> (target: ClickTarget, point: CGPoint, snapshotId: String) + { + let observation = try self.requiredObservation(params, windowRef: windowRef) + if params.elementRef != nil { + let element = try self.requiredElement(params, windowRef: windowRef) + return (.elementId(element.id), element.bounds.centerPoint, observation.snapshotId) + } + guard let x = params.x, let y = params.y else { + throw ComputerActionService.ComputerActionError.invalidV2Request( + "coordinates or elementRef are required for \(params.action.rawValue)") + } + guard x >= 0, y >= 0 else { + throw ComputerActionService.ComputerActionError.invalidV2Request( + "coordinates must be nonnegative") + } + let point = CGPoint(x: x, y: y) + return (.coordinates(point), point, observation.snapshotId) + } + + private func optionalPoint( + _ params: OpenClawComputerActParams, + windowRef: String) throws -> CGPoint? + { + if params.elementRef != nil { + return try self.requiredElement(params, windowRef: windowRef).bounds.centerPoint + } + if params.x == nil, params.y == nil { return nil } + guard let x = params.x, let y = params.y else { + throw ComputerActionService.ComputerActionError.invalidV2Request( + "both x and y are required") + } + guard x >= 0, y >= 0 else { + throw ComputerActionService.ComputerActionError.invalidV2Request( + "coordinates must be nonnegative") + } + _ = try self.requiredObservation(params, windowRef: windowRef) + return CGPoint(x: x, y: y) + } + + private func focusElement( + _ elementRef: String, + params: OpenClawComputerActParams, + target: WindowTarget, + foreground: Bool) async throws -> OpenClawComputerActResult + { + let windowRef = try Self.require(params.windowRef, field: "windowRef") + let observation = try self.requiredObservation(params, windowRef: windowRef) + guard let element = observation.elements[elementRef] else { + throw ComputerActionService.ComputerActionError.staleObservation + } + if foreground { + try await self.focus(target) + try Self.postForegroundClick(at: element.bounds.centerPoint, action: .leftClick, modifiers: nil) + return OpenClawComputerActResult(ok: true, effect: .unverifiable) + } + guard let identity = target.window.mutationIdentity else { + throw ComputerActionService.ComputerActionError.staleObservation + } + let result = try await self.withExecutionAuthority { + try await self.automation.clickWithOutcome( + target: .elementId(element.id), + clickType: .single, + snapshotId: observation.snapshotId, + expectedWindowIdentity: identity, + expectedWindowBounds: target.window.bounds) + } + return Self.result(from: result.outcome, background: true) + } + + private func focus(_ target: WindowTarget) async throws { + try await self.withExecutionAuthority { + try await self.applications.activateApplication(request: ApplicationActivationRequest( + application: target.app)) + } + try await self.withExecutionAuthority { + try await self.windows.focusWindow(target: .windowId(target.window.windowID)) + } + } +} + +// MARK: - Projection and outcomes + +extension ComputerActionServiceV2 { + private static func result( + from outcome: DesktopActionOutcome?, + background: Bool) -> OpenClawComputerActResult + { + guard let outcome else { + return OpenClawComputerActResult(ok: true, effect: .unverifiable) + } + if outcome.effect == .refused { + return background + ? self.backgroundEscalation(reason: outcome.refusalReason?.rawValue ?? "action_refused") + : OpenClawComputerActResult( + ok: false, + effect: .suspectedNoop, + escalation: OpenClawComputerEscalation( + recommended: "desktop", + reasonCode: "foreground_ineffective")) + } + let effect: OpenClawComputerActionEffect = switch outcome.effect { + case .confirmed: .confirmed + case .suspectedNoop: .suspectedNoop + case .partial, .unverifiable, .refused: .unverifiable + } + let escalation = background && outcome.effect == .suspectedNoop + ? OpenClawComputerEscalation(recommended: "foreground", reasonCode: "suspected_noop") + : nil + var details: [String: AnyCodable] = [ + "route": AnyCodable(outcome.delivery?.mechanism.rawValue ?? "unknown"), + "evidence": AnyCodable(outcome.evidence.rawValue), + ] + if let delivery = outcome.delivery { + details["deliveryMode"] = AnyCodable(delivery.mode.rawValue) + } + return OpenClawComputerActResult( + ok: outcome.effect != .suspectedNoop, + effect: effect, + escalation: escalation, + details: details) + } + + private static func failureResult( + _ failure: DesktopActionFailure, + background: Bool) -> OpenClawComputerActResult + { + let result = self.result(from: failure.outcome, background: background) + return OpenClawComputerActResult( + ok: false, + effect: result.effect ?? .unverifiable, + escalation: background + ? OpenClawComputerEscalation( + recommended: "foreground", + reasonCode: failure.outcome.effect == .suspectedNoop + ? "suspected_noop" + : "background_delivery_failed") + : nil, + details: ["message": AnyCodable(failure.message)]) + } + + private static func backgroundEscalation(reason: String) -> OpenClawComputerActResult { + OpenClawComputerActResult( + ok: false, + effect: .suspectedNoop, + escalation: OpenClawComputerEscalation( + recommended: "foreground", + reasonCode: reason)) + } + + private static func filterElements( + _ elements: [DetectedElement], + query: String?) -> [DetectedElement] + { + guard let query = query?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased(), + !query.isEmpty + else { return elements } + return elements.filter { element in + [element.id, element.type.rawValue, element.label, element.value] + .compactMap { $0?.lowercased() } + .contains { $0.contains(query) } + } + } + + private static func projectElements( + _ elements: [DetectedElement], + query: String?, + limit: Int) -> (elements: [[String: Any]], truncated: Int) + { + let filtered = self.filterElements(elements, query: query) + return ( + Array(filtered.prefix(limit)).map { element in + var row: [String: Any] = [ + "role": element.type.rawValue, + "bounds": self.boundsDictionary(element.bounds), + ] + if let label = element.label { row["label"] = label } + if let value = element.value { row["value"] = value } + return row + }, + max(0, filtered.count - limit)) + } + + private static func windowContext( + _ target: WindowTarget, + limits: (depth: Int, maxElements: Int)) -> WindowContext + { + WindowContext( + applicationName: target.app.name, + applicationBundleId: target.app.bundleIdentifier, + applicationProcessId: target.app.processIdentifier, + windowTitle: target.window.title, + windowID: target.window.windowID, + windowBounds: target.window.bounds, + windowMutationIdentity: target.window.mutationIdentity, + traversalBudget: AXTraversalBudget( + maxDepth: limits.depth, + maxElementCount: limits.maxElements, + maxChildrenPerNode: AXTraversalBudget.defaultMaxChildrenPerNode), + requiresFreshAccessibilityTree: true) + } + + private static func observationLimits( + _ params: OpenClawComputerActParams) throws -> (depth: Int, maxElements: Int) + { + let depth = params.depth ?? AXTraversalBudget.defaultMaxDepth + let maxElements = params.maxElements ?? AXTraversalBudget.defaultMaxElementCount + guard (0...64).contains(depth), (1...2000).contains(maxElements) else { + throw ComputerActionService.ComputerActionError.invalidV2Request( + "depth must be 0...64 and maxElements must be 1...2000") + } + return (depth, maxElements) + } + + private static func require( + _ value: String?, + field: String, + allowEmpty: Bool = false) throws -> String + { + guard let value, allowEmpty || !value.isEmpty else { + throw ComputerActionService.ComputerActionError.invalidV2Request( + "\(field) is required") + } + return value + } + + private static func bounds(_ rect: CGRect) -> OpenClawComputerBounds { + OpenClawComputerBounds( + x: Double(rect.origin.x), + y: Double(rect.origin.y), + width: max(0, Double(rect.width)), + height: max(0, Double(rect.height))) + } + + private static func boundsDictionary(_ rect: CGRect) -> [String: Any] { + [ + "x": Double(rect.origin.x), + "y": Double(rect.origin.y), + "width": max(0, Double(rect.width)), + "height": max(0, Double(rect.height)), + ] + } + + private static func scrollDirection( + _ direction: OpenClawComputerScrollDirection) -> PeekabooFoundation.ScrollDirection + { + switch direction { + case .up: .up + case .down: .down + case .left: .left + case .right: .right + } + } + + private static func postForegroundClick( + at point: CGPoint, + action: OpenClawComputerAction, + modifiers: String?) throws + { + guard CGPreflightPostEventAccess() else { + throw ComputerActionService.ComputerActionError.postEventAccessDenied + } + let button: CGMouseButton = action == .middleClick ? .center : action == .rightClick ? .right : .left + let downType: CGEventType = button == .center ? .otherMouseDown : button == .right ? .rightMouseDown : + .leftMouseDown + let upType: CGEventType = button == .center ? .otherMouseUp : button == .right ? .rightMouseUp : .leftMouseUp + let count = action == .tripleClick ? 3 : action == .doubleClick ? 2 : 1 + let flags = try self.modifierFlags(modifiers) + for index in 1...count { + guard let down = CGEvent( + mouseEventSource: nil, + mouseType: downType, + mouseCursorPosition: point, + mouseButton: button), + let up = CGEvent( + mouseEventSource: nil, + mouseType: upType, + mouseCursorPosition: point, + mouseButton: button) + else { + throw ComputerActionService.ComputerActionError.refused( + "failed to construct foreground click") + } + down.flags = flags + up.flags = flags + down.setIntegerValueField(.mouseEventClickState, value: Int64(index)) + up.setIntegerValueField(.mouseEventClickState, value: Int64(index)) + down.post(tap: .cghidEventTap) + up.post(tap: .cghidEventTap) + } + } + + private static func modifierFlags(_ raw: String?) throws -> CGEventFlags { + var flags: CGEventFlags = [] + for token in (raw ?? "") + .lowercased() + .split(whereSeparator: { $0 == "," || $0 == "+" || $0.isWhitespace }) + { + switch token { + case "cmd", "command", "meta": flags.insert(.maskCommand) + case "shift": flags.insert(.maskShift) + case "ctrl", "control": flags.insert(.maskControl) + case "alt", "option": flags.insert(.maskAlternate) + case "fn", "function": flags.insert(.maskSecondaryFn) + default: + throw ComputerActionService.ComputerActionError.invalidV2Request( + "unsupported modifier '\(token)'") + } + } + return flags + } +} diff --git a/apps/macos/Sources/OpenClaw/ComputerControlProvider.swift b/apps/macos/Sources/OpenClaw/ComputerControlProvider.swift index 8097b6dcf372..f26157506ff2 100644 --- a/apps/macos/Sources/OpenClaw/ComputerControlProvider.swift +++ b/apps/macos/Sources/OpenClaw/ComputerControlProvider.swift @@ -1,9 +1,55 @@ import Foundation +import OpenClawProtocol enum ComputerControlProvider: String, CaseIterable, Sendable { case peekaboo case cua + static var peekabooComputerUseDescriptor: OpenClawProtocol.AnyCodable { + OpenClawProtocol.AnyCodable([ + "contractVersion": 2, + "provider": [ + "id": "peekaboo", + "label": "Peekaboo", + "generation": "peekaboo-v2:\(UUID().uuidString.lowercased())", + ], + "actions": [ + "screenshot", + "left_click", + "right_click", + "middle_click", + "double_click", + "triple_click", + "mouse_move", + "left_click_drag", + "left_mouse_down", + "left_mouse_up", + "scroll", + "type", + "key", + "hold_key", + "list_apps", + "list_windows", + "get_accessibility_tree", + "get_cursor_position", + "get_window_state", + "launch_app", + "kill_app", + "bring_to_front", + "set_value", + "invoke_menu", + ], + "targets": ["screen", "window", "element"], + "deliveryModes": ["background", "foreground"], + "observations": ["image", "accessibility"], + "features": [ + "recording": false, + "agentCursor": false, + "multiDisplay": true, + ], + ] as [String: Any]) + } + static func current( defaults: UserDefaults = AppDefaults.standard, cuaAvailable: Bool = CuaDriverArtifact.bundledExecutableURL != nil) -> Self diff --git a/apps/macos/Sources/OpenClaw/NodeMode/MacNodeModeCoordinator.swift b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeModeCoordinator.swift index 3445e50321e5..9991278aff81 100644 --- a/apps/macos/Sources/OpenClaw/NodeMode/MacNodeModeCoordinator.swift +++ b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeModeCoordinator.swift @@ -1256,11 +1256,13 @@ extension MacNodeModeCoordinator { commands: [String], workerManifest: MacNodeHostManifest?) -> OpenClawProtocol.AnyCodable? { - guard provider == .cua, - commands.contains(MacNodeScreenCommand.snapshot.rawValue), + guard commands.contains(MacNodeScreenCommand.snapshot.rawValue), commands.contains(OpenClawComputerCommand.act.rawValue) else { return nil } - return workerManifest?.computerUse + return switch provider { + case .peekaboo: ComputerControlProvider.peekabooComputerUseDescriptor + case .cua: workerManifest?.computerUse + } } nonisolated static func mergingUnique(_ primary: [String], _ additional: [String]) -> [String] { diff --git a/apps/macos/Sources/OpenClaw/NodeMode/MacNodeRuntime.swift b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeRuntime.swift index ec93d95baf4b..e99728fea06c 100644 --- a/apps/macos/Sources/OpenClaw/NodeMode/MacNodeRuntime.swift +++ b/apps/macos/Sources/OpenClaw/NodeMode/MacNodeRuntime.swift @@ -631,16 +631,21 @@ extension MacNodeRuntime { + "under System Settings → Privacy & Security → Accessibility") case .noDisplays, .invalidScreenIndex, .missingDisplayFrameId, .displayFrameChanged, .missingCoordinate, .coordinateOutOfBounds, .invalidReferenceWidth, .missingKeys, - .emptyText, .invalidScroll, .invalidModifier, .buttonAlreadyHeld, .buttonNotHeld: + .emptyText, .invalidScroll, .invalidModifier, .buttonAlreadyHeld, .buttonNotHeld, + .invalidV2Request, .staleObservation, .unsupportedAction: return Self.errorResponse( req, code: .invalidRequest, - message: "INVALID_REQUEST: \(error.localizedDescription)") - case .eventCreationFailed, .lifecycleChanged: + message: error.localizedDescription.hasPrefix("COMPUTER_") + ? error.localizedDescription + : "INVALID_REQUEST: \(error.localizedDescription)") + case .eventCreationFailed, .lifecycleChanged, .refused: return Self.errorResponse( req, code: .unavailable, - message: "UNAVAILABLE: \(error.localizedDescription)") + message: error.localizedDescription.hasPrefix("COMPUTER_") + ? error.localizedDescription + : "UNAVAILABLE: \(error.localizedDescription)") } } } diff --git a/apps/macos/Tests/OpenClawIPCTests/ComputerActionServiceTests.swift b/apps/macos/Tests/OpenClawIPCTests/ComputerActionServiceTests.swift index 62355a8cbb82..343391469376 100644 --- a/apps/macos/Tests/OpenClawIPCTests/ComputerActionServiceTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/ComputerActionServiceTests.swift @@ -114,6 +114,14 @@ struct ComputerActionServiceTests { return false } + private func unsupportedAction(_ error: Error?) -> OpenClawComputerAction? { + guard let error = error as? ComputerActionService.ComputerActionError else { return nil } + if case let .unsupportedAction(action) = error { + return action + } + return nil + } + private func validationError( _ operation: () throws -> Void) -> ComputerActionService.ComputerActionError? { @@ -431,6 +439,76 @@ struct ComputerActionServiceTests { #expect(probe.maximumActiveActionCount == 1) } + @Test func `v2 authority rejects a result after lifecycle revocation`() async { + let service = ComputerActionServiceV2() + var checks = 0 + let outcomeError: Error? + do { + _ = try await service.perform( + OpenClawComputerActParams(action: .getCursorPosition), + lifecycleGeneration: 0, + checkExecutionAllowed: { + checks += 1 + if checks > 1 { + throw ComputerActionService.ComputerActionError.lifecycleChanged + } + }) + outcomeError = nil + } catch { + outcomeError = error + } + + #expect(self.isLifecycleChanged(outcomeError)) + #expect(checks == 2) + } + + @Test func `execution authority revalidates after awaited preparation`() async { + let started = AsyncSignal() + let resume = AsyncSignal() + let releaseProbe = LifecycleReleaseProbe(allowed: true) + let authority = ComputerActionExecutionAuthority { + guard releaseProbe.allowed else { + throw ComputerActionService.ComputerActionError.lifecycleChanged + } + } + let operation = Task { @MainActor in + try await authority.run { + await started.signal() + await resume.wait() + return true + } + } + await started.wait() + releaseProbe.allowed = false + await resume.signal() + + let outcomeError: Error? + do { + _ = try await operation.value + outcomeError = nil + } catch { + outcomeError = error + } + #expect(self.isLifecycleChanged(outcomeError)) + } + + @Test func `v2 rejects foreground accessibility-only actions`() async { + let service = ComputerActionServiceV2() + for action in [OpenClawComputerAction.setValue, .invokeMenu] { + let outcomeError: Error? + do { + _ = try await service.perform( + OpenClawComputerActParams(action: action, deliveryMode: .foreground), + lifecycleGeneration: 0, + checkExecutionAllowed: {}) + outcomeError = nil + } catch { + outcomeError = error + } + #expect(self.unsupportedAction(outcomeError) == action) + } + } + @Test func `cancelled queued action never executes`() async throws { let probe = ActionProbe() let queue = ComputerActionExecutionQueue(onLifecycleRelease: { true }) diff --git a/apps/macos/Tests/OpenClawIPCTests/MacNodeHostWorkerTests.swift b/apps/macos/Tests/OpenClawIPCTests/MacNodeHostWorkerTests.swift index 73d0f61d7903..e8f46653a640 100644 --- a/apps/macos/Tests/OpenClawIPCTests/MacNodeHostWorkerTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/MacNodeHostWorkerTests.swift @@ -267,7 +267,7 @@ struct MacNodeHostWorkerTests { ["system", "mcp"]) == ["canvas", "screen", "system", "mcp"]) } - @Test func `provider selection filters command ownership and publishes only the CUA descriptor`() throws { + @Test func `provider selection filters command ownership and publishes each provider descriptor`() throws { let descriptor = OpenClawProtocol.AnyCodable([ "contractVersion": OpenClawProtocol.AnyCodable(2), ]) @@ -282,6 +282,25 @@ struct MacNodeHostWorkerTests { #expect(!peekaboo.commands.contains(MacNodeScreenCommand.snapshot.rawValue)) #expect(!peekaboo.commands.contains(OpenClawComputerCommand.act.rawValue)) #expect(peekaboo.computerUse == nil) + let peekabooDescriptor = try #require(MacNodeModeCoordinator.computerUseDescriptor( + provider: .peekaboo, + commands: [MacNodeScreenCommand.snapshot.rawValue, OpenClawComputerCommand.act.rawValue], + workerManifest: peekaboo)) + let peekabooJSON = try JSONEncoder().encode(peekabooDescriptor) + let peekabooObject = try #require( + JSONSerialization.jsonObject(with: peekabooJSON) as? [String: Any]) + #expect(peekabooObject["contractVersion"] as? Int == 2) + #expect((peekabooObject["provider"] as? [String: Any])?["id"] as? String == "peekaboo") + let actions = try #require(peekabooObject["actions"] as? [String]) + #expect(actions.contains("get_window_state")) + #expect(actions.contains("invoke_menu")) + #expect(!actions.contains("zoom")) + #expect(!actions.contains("get_browser_state")) + #expect(!actions.contains("start_recording")) + let features = try #require(peekabooObject["features"] as? [String: Any]) + #expect(features["recording"] as? Bool == false) + #expect(features["agentCursor"] as? Bool == false) + #expect(features["multiDisplay"] as? Bool == true) let cua = try #require(MacNodeModeCoordinator.workerManifest(manifest, for: .cua)) #expect(cua.commands == manifest.commands) diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/ComputerCommands.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/ComputerCommands.swift index 96053a9ec8e7..912254b1792f 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawKit/ComputerCommands.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/ComputerCommands.swift @@ -12,7 +12,8 @@ public enum OpenClawComputerCommand: String, Codable, Sendable { /// onto the embedded Peekaboo automation engine plus a narrow CoreGraphics /// path for primitives Peekaboo does not express (middle/triple click, /// separate mouse down/up, modifier-held clicks/scroll). -public enum OpenClawComputerAction: String, Codable, Sendable { +public enum OpenClawComputerAction: String, Codable, CaseIterable, Sendable { + case screenshot case leftClick = "left_click" case rightClick = "right_click" case middleClick = "middle_click" @@ -26,6 +27,60 @@ public enum OpenClawComputerAction: String, Codable, Sendable { case type case key case holdKey = "hold_key" + case wait + case listApps = "list_apps" + case listWindows = "list_windows" + case getAccessibilityTree = "get_accessibility_tree" + case getCursorPosition = "get_cursor_position" + case getWindowState = "get_window_state" + case launchApp = "launch_app" + case killApp = "kill_app" + case bringToFront = "bring_to_front" + case setValue = "set_value" + case zoom + case getBrowserState = "get_browser_state" + case browserPrepare = "browser_prepare" + case browserNavigate = "browser_navigate" + case browserClick = "browser_click" + case browserType = "browser_type" + case browserDialog = "browser_dialog" + case browserSetInputFiles = "browser_set_input_files" + case browserDownload = "browser_download" + case browserPointer = "browser_pointer" + case escalateScope = "escalate_scope" + case getRecordingState = "get_recording_state" + case startRecording = "start_recording" + case stopRecording = "stop_recording" + case replayTrajectory = "replay_trajectory" + case invokeMenu = "invoke_menu" + + private var isNativeWireAction: Bool { + switch self { + case .wait, .zoom, .getBrowserState, .browserPrepare, .browserNavigate, + .browserClick, .browserType, .browserDialog, .browserSetInputFiles, + .browserDownload, .browserPointer, .escalateScope, .getRecordingState, + .startRecording, .stopRecording, .replayTrajectory: + false + default: + true + } + } + + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + let rawValue = try container.decode(String.self) + guard let action = Self(rawValue: rawValue), action.isNativeWireAction else { + throw DecodingError.dataCorruptedError( + in: container, + debugDescription: "Unsupported native computer action: \(rawValue)") + } + self = action + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(self.rawValue) + } } public enum OpenClawComputerScrollDirection: String, Codable, Sendable { @@ -35,6 +90,19 @@ public enum OpenClawComputerScrollDirection: String, Codable, Sendable { case right } +public enum OpenClawComputerDeliveryMode: String, Codable, Sendable { + case background + case foreground +} + +public enum OpenClawComputerEscalationReason: String, Codable, Sendable { + case axTreePixelMismatch = "ax_tree_pixel_mismatch" + case backgroundDeliveryFailed = "background_delivery_failed" + case foregroundIneffective = "foreground_ineffective" + case noWindowTarget = "no_window_target" + case other +} + /// Wire params for `computer.act`. All coordinate fields are reference-screenshot /// pixels at `refWidth`; `keys` is a chord for key/hold_key; `modifiers` are /// modifier keys held during pointer actions; `scrollAmount` is wheel ticks. @@ -54,6 +122,21 @@ public struct OpenClawComputerActParams: Codable, Sendable, Equatable { public var durationMs: Int? public var screenIndex: Int? public var refWidth: Int? + public var windowRef: String? + public var elementRef: String? + public var observationId: String? + public var deliveryMode: OpenClawComputerDeliveryMode? + public var query: String? + public var depth: Int? + public var maxElements: Int? + public var app: String? + public var value: String? + public var path: [String]? + public var x1: Double? + public var y1: Double? + public var x2: Double? + public var y2: Double? + public var reason: OpenClawComputerEscalationReason? public init( action: OpenClawComputerAction, @@ -69,7 +152,22 @@ public struct OpenClawComputerActParams: Codable, Sendable, Equatable { scrollAmount: Int? = nil, durationMs: Int? = nil, screenIndex: Int? = nil, - refWidth: Int? = nil) + refWidth: Int? = nil, + windowRef: String? = nil, + elementRef: String? = nil, + observationId: String? = nil, + deliveryMode: OpenClawComputerDeliveryMode? = nil, + query: String? = nil, + depth: Int? = nil, + maxElements: Int? = nil, + app: String? = nil, + value: String? = nil, + path: [String]? = nil, + x1: Double? = nil, + y1: Double? = nil, + x2: Double? = nil, + y2: Double? = nil, + reason: OpenClawComputerEscalationReason? = nil) { self.action = action self.displayFrameId = displayFrameId @@ -85,18 +183,138 @@ public struct OpenClawComputerActParams: Codable, Sendable, Equatable { self.durationMs = durationMs self.screenIndex = screenIndex self.refWidth = refWidth + self.windowRef = windowRef + self.elementRef = elementRef + self.observationId = observationId + self.deliveryMode = deliveryMode + self.query = query + self.depth = depth + self.maxElements = maxElements + self.app = app + self.value = value + self.path = path + self.x1 = x1 + self.y1 = y1 + self.x2 = x2 + self.y2 = y2 + self.reason = reason } } -/// Result of a `computer.act` input action. +public enum OpenClawComputerActionEffect: String, Codable, Sendable { + case confirmed + case unverifiable + case suspectedNoop = "suspected_noop" +} + +public struct OpenClawComputerBounds: Codable, Sendable, Equatable { + public var x: Double + public var y: Double + public var width: Double + public var height: Double + + public init(x: Double, y: Double, width: Double, height: Double) { + self.x = x + self.y = y + self.width = width + self.height = height + } +} + +public struct OpenClawComputerObservationElement: Codable, Sendable, Equatable { + public var elementRef: String + public var role: String + public var label: String? + public var value: String? + public var bounds: OpenClawComputerBounds + + public init( + elementRef: String, + role: String, + label: String? = nil, + value: String? = nil, + bounds: OpenClawComputerBounds) + { + self.elementRef = elementRef + self.role = role + self.label = label + self.value = value + self.bounds = bounds + } +} + +public struct OpenClawComputerObservation: Codable, Sendable, Equatable { + public var kind: String + public var base64: String? + public var format: String? + public var width: Int? + public var height: Int? + public var observationId: String? + public var elements: [OpenClawComputerObservationElement]? + + public init( + kind: String, + base64: String? = nil, + format: String? = nil, + width: Int? = nil, + height: Int? = nil, + observationId: String? = nil, + elements: [OpenClawComputerObservationElement]? = nil) + { + self.kind = kind + self.base64 = base64 + self.format = format + self.width = width + self.height = height + self.observationId = observationId + self.elements = elements + } +} + +public struct OpenClawComputerEscalation: Codable, Sendable, Equatable { + public var recommended: String + public var reasonCode: String + + public init(recommended: String, reasonCode: String) { + self.recommended = recommended + self.reasonCode = reasonCode + } +} + +/// Result of a `computer.act` input action. Optional v2 fields are omitted for +/// v1 actions so their existing JSON payload remains byte-for-byte unchanged. public struct OpenClawComputerActResult: Codable, Sendable, Equatable { public var ok: Bool - public var cursorX: Double - public var cursorY: Double + public var cursorX: Double? + public var cursorY: Double? + public var effect: OpenClawComputerActionEffect? + public var observation: OpenClawComputerObservation? + public var escalation: OpenClawComputerEscalation? + public var details: [String: AnyCodable]? public init(ok: Bool, cursorX: Double, cursorY: Double) { self.ok = ok self.cursorX = cursorX self.cursorY = cursorY + self.effect = nil + self.observation = nil + self.escalation = nil + self.details = nil + } + + public init( + ok: Bool, + effect: OpenClawComputerActionEffect? = nil, + observation: OpenClawComputerObservation? = nil, + escalation: OpenClawComputerEscalation? = nil, + details: [String: AnyCodable]? = nil) + { + self.ok = ok + self.cursorX = nil + self.cursorY = nil + self.effect = effect + self.observation = observation + self.escalation = escalation + self.details = details } } diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ComputerCommandsWireDecodeTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ComputerCommandsWireDecodeTests.swift new file mode 100644 index 000000000000..dd71c6154255 --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ComputerCommandsWireDecodeTests.swift @@ -0,0 +1,143 @@ +import Foundation +import OpenClawKit +import Testing + +struct ComputerCommandsWireDecodeTests { + private struct DecodeCase { + let json: String + let action: OpenClawComputerAction + var app: String? + var windowRef: String? + var elementRef: String? + var observationId: String? + var deliveryMode: OpenClawComputerDeliveryMode? + var query: String? + var depth: Int? + var maxElements: Int? + var value: String? + var path: [String]? + var x: Double? + var y: Double? + var text: String? + var keys: String? + } + + private func decode(_ testCase: DecodeCase) throws -> OpenClawComputerActParams { + try JSONDecoder().decode(OpenClawComputerActParams.self, from: Data(testCase.json.utf8)) + } + + private func expectFields(_ params: OpenClawComputerActParams, match testCase: DecodeCase) { + #expect(params.action == testCase.action) + #expect(params.app == testCase.app) + #expect(params.windowRef == testCase.windowRef) + #expect(params.elementRef == testCase.elementRef) + #expect(params.observationId == testCase.observationId) + #expect(params.deliveryMode == testCase.deliveryMode) + #expect(params.query == testCase.query) + #expect(params.depth == testCase.depth) + #expect(params.maxElements == testCase.maxElements) + #expect(params.value == testCase.value) + #expect(params.path == testCase.path) + #expect(params.x == testCase.x) + #expect(params.y == testCase.y) + #expect(params.text == testCase.text) + #expect(params.keys == testCase.keys) + } + + @Test func `decodes every implemented v2 action family and delivery mode`() throws { + let cases = [ + DecodeCase( + json: #""" + {"action":"get_accessibility_tree","windowRef":"window-1","query":"Save", + "depth":4,"maxElements":250} + """#, + action: .getAccessibilityTree, + windowRef: "window-1", + query: "Save", + depth: 4, + maxElements: 250), + DecodeCase( + json: #""" + {"action":"get_window_state","windowRef":"window-2","observationId":"observation-2", + "deliveryMode":"background"} + """#, + action: .getWindowState, + windowRef: "window-2", + observationId: "observation-2", + deliveryMode: .background), + DecodeCase( + json: #"{"action":"launch_app","app":"TextEdit","deliveryMode":"foreground"}"#, + action: .launchApp, + app: "TextEdit", + deliveryMode: .foreground), + DecodeCase( + json: #""" + {"action":"set_value","elementRef":"element-3","observationId":"observation-3", + "value":"hello","deliveryMode":"background"} + """#, + action: .setValue, + elementRef: "element-3", + observationId: "observation-3", + deliveryMode: .background, + value: "hello"), + DecodeCase( + json: #"{"action":"invoke_menu","app":"app-4","path":["File","Save As…"],"deliveryMode":"foreground"}"#, + action: .invokeMenu, + app: "app-4", + deliveryMode: .foreground, + path: ["File", "Save As…"]), + DecodeCase( + json: #""" + {"action":"left_click","windowRef":"window-5","observationId":"observation-5", + "x":120,"y":240,"deliveryMode":"background"} + """#, + action: .leftClick, + windowRef: "window-5", + observationId: "observation-5", + deliveryMode: .background, + x: 120, + y: 240), + DecodeCase( + json: #"{"action":"type","windowRef":"window-6","text":"hello","deliveryMode":"foreground"}"#, + action: .type, + windowRef: "window-6", + deliveryMode: .foreground, + text: "hello"), + DecodeCase( + json: #"{"action":"key","elementRef":"element-7","keys":"cmd+return","deliveryMode":"background"}"#, + action: .key, + elementRef: "element-7", + deliveryMode: .background, + keys: "cmd+return"), + ] + + for testCase in cases { + try self.expectFields(self.decode(testCase), match: testCase) + } + } + + @Test func `rejects unknown and native-unimplemented action names`() throws { + for action in ["totally_unknown", "browser_click", "start_recording"] { + let data = Data(#"{"action":"\#(action)"}"#.utf8) + #expect(throws: DecodingError.self) { + _ = try JSONDecoder().decode(OpenClawComputerActParams.self, from: data) + } + } + } + + @Test func `action raw values match the frozen computer use v2 contract`() { + let frozenActionNames = [ + "screenshot", "left_click", "right_click", "middle_click", "double_click", + "triple_click", "mouse_move", "left_click_drag", "left_mouse_down", "left_mouse_up", + "scroll", "type", "key", "hold_key", "wait", "list_apps", "list_windows", + "get_accessibility_tree", "get_cursor_position", "get_window_state", "launch_app", + "kill_app", "bring_to_front", "set_value", "zoom", "get_browser_state", + "browser_prepare", "browser_navigate", "browser_click", "browser_type", + "browser_dialog", "browser_set_input_files", "browser_download", "browser_pointer", + "escalate_scope", "get_recording_state", "start_recording", "stop_recording", + "replay_trajectory", "invoke_menu", + ] + + #expect(OpenClawComputerAction.allCases.map(\.rawValue) == frozenActionNames) + } +} diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ComputerInputGeometryTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ComputerInputGeometryTests.swift index fd096152b756..8a126f733021 100644 --- a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ComputerInputGeometryTests.swift +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ComputerInputGeometryTests.swift @@ -270,4 +270,72 @@ struct ComputerInputGeometryTests { #expect(holdParams.keys == "space") #expect(holdParams.durationMs == 2000) } + + @Test func `v1 result encoding omits every additive v2 field`() throws { + let data = try JSONEncoder().encode(OpenClawComputerActResult( + ok: true, + cursorX: 12, + cursorY: 34)) + let object = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) + + #expect(object.keys.sorted() == ["cursorX", "cursorY", "ok"]) + #expect(object["ok"] as? Bool == true) + #expect(object["cursorX"] as? Double == 12) + #expect(object["cursorY"] as? Double == 34) + } + + @Test func `decodes v2 window observation and element delivery fields`() throws { + let data = try #require(""" + { + "action":"set_value", + "windowRef":"window-1", + "elementRef":"element-1", + "observationId":"observation-1", + "value":"hello", + "deliveryMode":"background" + } + """.data(using: .utf8)) + let params = try JSONDecoder().decode(OpenClawComputerActParams.self, from: data) + + #expect(params.action == .setValue) + #expect(params.windowRef == "window-1") + #expect(params.elementRef == "element-1") + #expect(params.observationId == "observation-1") + #expect(params.value == "hello") + #expect(params.deliveryMode == .background) + } + + @Test func `v2 result encoding matches the shared result envelope`() throws { + let result = OpenClawComputerActResult( + ok: false, + effect: .suspectedNoop, + observation: OpenClawComputerObservation( + kind: "window", + base64: "cG5n", + format: "png", + width: 10, + height: 20, + observationId: "observation-1", + elements: [OpenClawComputerObservationElement( + elementRef: "element-1", + role: "button", + label: "Save", + bounds: OpenClawComputerBounds(x: 1, y: 2, width: 3, height: 4))]), + escalation: OpenClawComputerEscalation( + recommended: "foreground", + reasonCode: "background_delivery_failed"), + details: ["deliveryMode": AnyCodable("background")]) + let data = try JSONEncoder().encode(result) + let object = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) + + #expect(object["ok"] as? Bool == false) + #expect(object["effect"] as? String == "suspected_noop") + let observation = try #require(object["observation"] as? [String: Any]) + #expect(observation["observationId"] as? String == "observation-1") + let elements = try #require(observation["elements"] as? [[String: Any]]) + #expect(elements.first?["elementRef"] as? String == "element-1") + let escalation = try #require(object["escalation"] as? [String: Any]) + #expect(escalation["recommended"] as? String == "foreground") + #expect(escalation["reasonCode"] as? String == "background_delivery_failed") + } }