feat(macos): fulfill computer.act v2 via Peekaboo (#123801)

2026-08-15 23:24:03 -06:00 · 2026-08-14 14:12:25 -07:00
parent b3f1cd36db
commit 4a6edc0d27
10 changed files with 1598 additions and 13 deletions
@@ -452,6 +452,10 @@ final class ComputerActionService {
        case buttonNotHeld
        case eventCreationFailed
        case lifecycleChanged
+        case invalidV2Request(String)
+        case staleObservation
+        case unsupportedAction(OpenClawComputerAction)
+        case refused(String)

        var errorDescription: String? {
            switch self {
@@ -491,6 +495,14 @@ final class ComputerActionService {
                "Failed to synthesize input event"
            case .lifecycleChanged:
                "Computer control lifecycle changed while the action was pending"
+            case let .invalidV2Request(message):
+                "COMPUTER_INVALID_REQUEST: \(message)"
+            case .staleObservation:
+                "COMPUTER_STALE_OBSERVATION: take a fresh observation and retry"
+            case let .unsupportedAction(action):
+                "COMPUTER_UNSUPPORTED_ACTION: \(action.rawValue)"
+            case let .refused(message):
+                "COMPUTER_REFUSED_action_refused: \(message)"
            }
        }
    }
@@ -500,6 +512,7 @@ final class ComputerActionService {
    private let mouseEventFactory: MouseEventFactory
    private let mouseEventPoster: MouseEventPoster
    private let textGraphemePoster: TextGraphemePoster
+    private lazy var v2 = ComputerActionServiceV2()
    /// Tracks whether a left_mouse_down is outstanding so mouse_move emits
    /// drag events (state persists across invokes on the shared instance).
    private var leftButtonDown = false
@@ -592,6 +605,27 @@ final class ComputerActionService {
        lifecycleGeneration: UInt64) async throws -> OpenClawComputerActResult
    {
        try self.executionQueue.checkExecutionAllowed(lifecycleGeneration: lifecycleGeneration)
+        if params.deliveryMode == .background,
+           params.windowRef == nil,
+           !params.action.isComputerActV2Only
+        {
+            return OpenClawComputerActResult(
+                ok: false,
+                effect: .suspectedNoop,
+                escalation: OpenClawComputerEscalation(
+                    recommended: "window-pixel",
+                    reasonCode: "no_window_target"))
+        }
+        if params.isV2Request {
+            return try await self.v2.perform(
+                params,
+                lifecycleGeneration: lifecycleGeneration,
+                checkExecutionAllowed: { [weak self] in
+                    guard let self else { throw CancellationError() }
+                    try self.executionQueue.checkExecutionAllowed(
+                        lifecycleGeneration: lifecycleGeneration)
+                })
+        }
        try Self.validateInputPermissions(ComputerControlPermissionSnapshot.probe())
        let display = try await resolveDisplay(params: params)
        try executionQueue.checkExecutionAllowed(lifecycleGeneration: lifecycleGeneration)
@@ -704,6 +738,8 @@ final class ComputerActionService {
            let keys = try requireKeys(params.keys)
            let holdMs = min(Self.maxHoldMs, max(0, params.durationMs ?? 1000))
            try await self.automation.hotkey(keys: keys, holdDuration: holdMs)
+        default:
+            throw ComputerActionError.unsupportedAction(params.action)
        }
    }

@@ -0,0 +1,970 @@
+import AppKit
+import CoreGraphics
+import Foundation
+import OpenClawKit
+import PeekabooAutomationKit
+import PeekabooFoundation
+
+extension CGRect {
+    fileprivate var centerPoint: CGPoint {
+        CGPoint(x: self.midX, y: self.midY)
+    }
+}
+
+extension OpenClawComputerActParams {
+    var isV2Request: Bool {
+        self.action.isComputerActV2Only || self.windowRef != nil || self.elementRef != nil ||
+            self.observationId != nil
+    }
+}
+
+extension OpenClawComputerAction {
+    var isComputerActV2Only: Bool {
+        switch self {
+        case .leftClick, .rightClick, .middleClick, .doubleClick, .tripleClick, .mouseMove,
+             .leftClickDrag, .leftMouseDown, .leftMouseUp, .scroll, .type, .key, .holdKey:
+            false
+        default:
+            true
+        }
+    }
+}
+
+@MainActor
+struct ComputerActionExecutionAuthority {
+    let check: @MainActor () throws -> Void
+
+    func run<Result>(_ operation: () async throws -> Result) async throws -> Result {
+        try self.check()
+        let result = try await operation()
+        try self.check()
+        return result
+    }
+}
+
+/// Implements the additive computer.act v2 surface without changing the v1
+/// coordinate path. All authority-bearing references are process-local,
+/// execution-local, and invalidated when the native lifecycle generation moves.
+@MainActor
+final class ComputerActionServiceV2 {
+    private struct WindowTarget {
+        let app: ServiceApplicationInfo
+        let window: ServiceWindowInfo
+    }
+
+    private struct ElementTarget {
+        let id: String
+        let bounds: CGRect
+    }
+
+    private struct ObservationState {
+        let id: String
+        let windowRef: String
+        let snapshotId: String
+        let elements: [String: ElementTarget]
+    }
+
+    private let executionID = UUID().uuidString.lowercased()
+    private let automation: UIAutomationService
+    private let applications: ApplicationService
+    private let windows: WindowManagementService
+    private let menu: MenuService
+    private let observationService: DesktopObservationService
+    private var lifecycleGeneration: UInt64?
+    private var appRefs: [String: ServiceApplicationInfo] = [:]
+    private var windowRefs: [String: WindowTarget] = [:]
+    private var observation: ObservationState?
+    private var executionAuthority: ComputerActionExecutionAuthority?
+
+    init() {
+        let snapshotManager = SnapshotManager()
+        let automation = UIAutomationService(snapshotManager: snapshotManager)
+        let applications = ApplicationService()
+        let menu = MenuService(applicationService: applications)
+        self.automation = automation
+        self.applications = applications
+        self.windows = WindowManagementService(applicationService: applications)
+        self.menu = menu
+        self.observationService = DesktopObservationService(
+            screenCapture: ScreenCaptureService(loggingService: LoggingService()),
+            automation: automation,
+            applications: applications,
+            menu: menu,
+            snapshotManager: snapshotManager)
+    }
+
+    func perform(
+        _ params: OpenClawComputerActParams,
+        lifecycleGeneration: UInt64,
+        checkExecutionAllowed: @escaping @MainActor () throws -> Void) async throws
+        -> OpenClawComputerActResult
+    {
+        precondition(self.executionAuthority == nil)
+        let authority = ComputerActionExecutionAuthority(check: checkExecutionAllowed)
+        self.executionAuthority = authority
+        defer { self.executionAuthority = nil }
+
+        return try await authority.run {
+            self.adoptLifecycleGeneration(lifecycleGeneration)
+            switch params.action {
+            case .listApps:
+                return try await self.listApps()
+            case .listWindows:
+                return try await self.listWindows()
+            case .getAccessibilityTree:
+                return try await self.getAccessibilityTree(params)
+            case .getCursorPosition:
+                return self.getCursorPosition()
+            case .getWindowState:
+                return try await self.getWindowState(params)
+            case .launchApp:
+                return try await self.launchApp(params)
+            case .killApp:
+                return try await self.killApp(params)
+            case .bringToFront:
+                return try await self.bringToFront(params)
+            case .setValue:
+                return try await self.setValue(params)
+            case .invokeMenu:
+                return try await self.invokeMenu(params)
+            case .leftClick, .rightClick, .middleClick, .doubleClick, .tripleClick:
+                return try await self.click(params)
+            case .type:
+                return try await self.type(params)
+            case .key:
+                return try await self.key(params)
+            case .scroll:
+                return try await self.scroll(params)
+            case .mouseMove, .leftClickDrag, .leftMouseDown, .leftMouseUp, .holdKey:
+                throw ComputerActionService.ComputerActionError.unsupportedAction(params.action)
+            case .screenshot, .wait, .zoom, .getBrowserState, .browserPrepare, .browserNavigate,
+                 .browserClick, .browserType, .browserDialog, .browserSetInputFiles, .browserDownload,
+                 .browserPointer, .escalateScope, .getRecordingState, .startRecording, .stopRecording,
+                 .replayTrajectory:
+                throw ComputerActionService.ComputerActionError.unsupportedAction(params.action)
+            }
+        }
+    }
+
+    private func withExecutionAuthority<Result>(
+        _ operation: () async throws -> Result) async throws -> Result
+    {
+        guard let executionAuthority = self.executionAuthority else {
+            preconditionFailure("Computer action executed outside its queue authority")
+        }
+        return try await executionAuthority.run(operation)
+    }
+
+    // MARK: - Discovery and observation
+
+    private func listApps() async throws -> OpenClawComputerActResult {
+        let output = try await self.withExecutionAuthority {
+            try await self.applications.listApplications()
+        }
+        self.appRefs.removeAll(keepingCapacity: true)
+        let rows: [[String: Any]] = output.data.applications.prefix(500).map { app in
+            let ref = self.issueRef("app")
+            self.appRefs[ref] = app
+            var row: [String: Any] = [
+                "app": ref,
+                "name": app.name,
+                "running": true,
+                "active": app.isActive,
+                "pid": Int(app.processIdentifier),
+            ]
+            if let bundleIdentifier = app.bundleIdentifier {
+                row["bundleId"] = bundleIdentifier
+            }
+            return row
+        }
+        return OpenClawComputerActResult(ok: true, details: [
+            "apps": AnyCodable(rows),
+            "totalApps": AnyCodable(output.data.applications.count),
+            "truncatedApps": AnyCodable(max(0, output.data.applications.count - rows.count)),
+        ])
+    }
+
+    private func listWindows() async throws -> OpenClawComputerActResult {
+        let appOutput = try await self.withExecutionAuthority {
+            try await self.applications.listApplications()
+        }
+        self.windowRefs.removeAll(keepingCapacity: true)
+        var rows: [[String: Any]] = []
+        var warnings = appOutput.metadata.warnings
+        outer: for app in appOutput.data.applications
+            where app.windowCount > 0 || app.windowIDs?.isEmpty == false
+        {
+            do {
+                let output = try await self.withExecutionAuthority {
+                    try await self.applications.listWindows(
+                        for: "PID:\(app.processIdentifier)",
+                        timeout: nil)
+                }
+                warnings.append(contentsOf: output.metadata.warnings)
+                for window in output.data.windows where window.layer == 0 {
+                    let ref = self.issueWindowRef(app: app, window: window)
+                    rows.append([
+                        "windowRef": ref,
+                        "appName": app.name,
+                        "title": window.title,
+                        "bounds": Self.boundsDictionary(window.bounds),
+                        "isOnScreen": window.isOnScreen,
+                        "minimized": window.isMinimized,
+                    ])
+                    if rows.count == 500 { break outer }
+                }
+            } catch {
+                warnings.append("\(app.name): \(error.localizedDescription)")
+            }
+        }
+        var details: [String: AnyCodable] = ["windows": AnyCodable(rows)]
+        if !warnings.isEmpty {
+            details["warnings"] = AnyCodable(Array(warnings.prefix(64)))
+        }
+        return OpenClawComputerActResult(ok: true, details: details)
+    }
+
+    private func getAccessibilityTree(
+        _ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult
+    {
+        let limits = try Self.observationLimits(params)
+        let context: WindowContext? = try params.windowRef.map { ref in
+            let target = try self.resolveWindow(ref)
+            return Self.windowContext(target, limits: limits)
+        }
+        let result = try await self.withExecutionAuthority {
+            try await self.automation.inspectAccessibilityTree(windowContext: context)
+        }
+        let projected = Self.projectElements(
+            result.elements.all,
+            query: params.query,
+            limit: limits.maxElements)
+        var details: [String: AnyCodable] = [
+            "elements": AnyCodable(projected.elements),
+            "totalElementCount": AnyCodable(result.elements.all.count),
+        ]
+        if projected.truncated > 0 {
+            details["truncatedElements"] = AnyCodable(projected.truncated)
+        }
+        if !result.metadata.warnings.isEmpty {
+            details["warnings"] = AnyCodable(Array(result.metadata.warnings.prefix(64)))
+        }
+        return OpenClawComputerActResult(ok: true, details: details)
+    }
+
+    private func getCursorPosition() -> OpenClawComputerActResult {
+        let point = self.automation.currentMouseLocation() ?? .zero
+        return OpenClawComputerActResult(ok: true, details: [
+            "x": AnyCodable(Double(point.x)),
+            "y": AnyCodable(Double(point.y)),
+        ])
+    }
+
+    private func getWindowState(
+        _ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult
+    {
+        let windowRef = try Self.require(params.windowRef, field: "windowRef")
+        let target = try self.resolveWindow(windowRef)
+        let limits = try Self.observationLimits(params)
+        guard let windowID = CGWindowID(exactly: target.window.windowID) else {
+            throw ComputerActionService.ComputerActionError.staleObservation
+        }
+        let request = DesktopObservationRequest(
+            target: .windowID(windowID),
+            capture: DesktopCaptureOptions(focus: .background),
+            detection: DesktopDetectionOptions(
+                mode: .accessibility,
+                traversalBudget: AXTraversalBudget(
+                    maxDepth: limits.depth,
+                    maxElementCount: limits.maxElements,
+                    maxChildrenPerNode: AXTraversalBudget.defaultMaxChildrenPerNode)))
+        let result = try await self.withExecutionAuthority {
+            try await self.observationService.observe(request)
+        }
+        let observedWindow = result.capture.metadata.windowInfo ?? target.window
+        guard observedWindow.windowID == target.window.windowID else {
+            throw ComputerActionService.ComputerActionError.staleObservation
+        }
+        self.windowRefs[windowRef] = WindowTarget(app: target.app, window: observedWindow)
+        let detected = result.elements?.elements.all ?? []
+        let filtered = Self.filterElements(detected, query: params.query)
+        let bounded = Array(filtered.prefix(limits.maxElements))
+        let observationID = self.issueRef("observation")
+        var elementTargets: [String: ElementTarget] = [:]
+        let elements = bounded.map { element in
+            let ref = self.issueRef("element")
+            elementTargets[ref] = ElementTarget(id: element.id, bounds: element.bounds)
+            return OpenClawComputerObservationElement(
+                elementRef: ref,
+                role: element.type.rawValue,
+                label: element.label,
+                value: element.value,
+                bounds: Self.bounds(element.bounds))
+        }
+        let snapshotID = result.elements?.snapshotId ?? ""
+        guard !snapshotID.isEmpty else {
+            throw ComputerActionService.ComputerActionError.refused(
+                "Peekaboo observation returned no snapshot receipt")
+        }
+        self.observation = ObservationState(
+            id: observationID,
+            windowRef: windowRef,
+            snapshotId: snapshotID,
+            elements: elementTargets)
+        var details: [String: AnyCodable] = [
+            "totalElementCount": AnyCodable(detected.count),
+            "coordinateSpace": AnyCodable("global-logical-points"),
+        ]
+        if filtered.count > bounded.count {
+            details["truncatedElements"] = AnyCodable(filtered.count - bounded.count)
+        }
+        if !result.diagnostics.warnings.isEmpty {
+            details["warnings"] = AnyCodable(Array(result.diagnostics.warnings.prefix(64)))
+        }
+        let size = result.capture.metadata.size
+        guard size.width >= 1, size.height >= 1 else {
+            throw ComputerActionService.ComputerActionError.refused(
+                "Peekaboo observation returned an invalid image size")
+        }
+        return OpenClawComputerActResult(
+            ok: true,
+            observation: OpenClawComputerObservation(
+                kind: "window",
+                base64: result.capture.imageData.base64EncodedString(),
+                format: "png",
+                width: Int(size.width),
+                height: Int(size.height),
+                observationId: observationID,
+                elements: elements.isEmpty ? nil : elements),
+            details: details)
+    }
+
+    // MARK: - Lifecycle
+
+    private func launchApp(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult {
+        let appValue = try Self.require(params.app, field: "app")
+        let target = self.appRefs[appValue]
+        if appValue.hasPrefix("peekaboo:v2:app:"), target == nil {
+            throw ComputerActionService.ComputerActionError.staleObservation
+        }
+        let launched = try await self.withExecutionAuthority {
+            try await self.applications.launchApplication(request: ApplicationLaunchRequest(
+                applicationIdentifier: target?.bundleIdentifier ?? target?.name ?? appValue,
+                activates: true,
+                waitUntilReady: true,
+                waitForWindow: true))
+        }
+        return OpenClawComputerActResult(ok: true, effect: .confirmed, details: [
+            "name": AnyCodable(launched.name),
+            "pid": AnyCodable(Int(launched.processIdentifier)),
+        ])
+    }
+
+    private func killApp(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult {
+        let appRef = try Self.require(params.app, field: "app")
+        guard let app = self.appRefs[appRef], let identity = app.processIdentity else {
+            throw ComputerActionService.ComputerActionError.invalidV2Request(
+                "kill_app requires a running app reference from list_apps")
+        }
+        let quit = try await self.withExecutionAuthority {
+            try await self.applications.quitApplication(request: ApplicationQuitRequest(
+                identifier: "PID:\(app.processIdentifier)",
+                force: false,
+                expectedIdentity: identity))
+        }
+        return OpenClawComputerActResult(
+            ok: quit,
+            effect: quit ? .confirmed : .suspectedNoop,
+            details: ["app": AnyCodable(appRef)])
+    }
+
+    private func bringToFront(
+        _ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult
+    {
+        let target = try self.requiredWindow(params)
+        try await self.focus(target)
+        let focused = try await self.withExecutionAuthority {
+            try await self.windows.getFocusedWindow()
+        }
+        let confirmed = focused?.windowID == target.window.windowID
+        return OpenClawComputerActResult(
+            ok: confirmed,
+            effect: confirmed ? .confirmed : .suspectedNoop,
+            escalation: confirmed ? nil : OpenClawComputerEscalation(
+                recommended: "desktop",
+                reasonCode: "foreground_ineffective"))
+    }
+
+    // MARK: - Input
+
+    private func click(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult {
+        let target = try self.requiredWindow(params)
+        let resolved = try self.clickTarget(params, windowRef: Self.require(params.windowRef, field: "windowRef"))
+        let mode = params.deliveryMode ?? .background
+        if mode == .foreground {
+            try await self.focus(target)
+            guard target.window.bounds.contains(resolved.point) else {
+                throw ComputerActionService.ComputerActionError.invalidV2Request(
+                    "foreground click target is outside the selected window")
+            }
+            try Self.postForegroundClick(
+                at: resolved.point,
+                action: params.action,
+                modifiers: params.modifiers)
+            return OpenClawComputerActResult(ok: true, effect: .unverifiable, details: [
+                "deliveryMode": AnyCodable("foreground"),
+                "route": AnyCodable("global_events"),
+            ])
+        }
+        guard params.action != .middleClick, params.action != .tripleClick else {
+            return Self.backgroundEscalation(reason: "background_click_variant_unavailable")
+        }
+        guard let identity = target.window.mutationIdentity else {
+            throw ComputerActionService.ComputerActionError.staleObservation
+        }
+        let clickType: ClickType = switch params.action {
+        case .rightClick: .right
+        case .doubleClick: .double
+        default: .single
+        }
+        do {
+            let result = try await self.withExecutionAuthority {
+                try await self.automation.clickWithOutcome(
+                    target: resolved.target,
+                    clickType: clickType,
+                    snapshotId: resolved.snapshotId,
+                    expectedWindowIdentity: identity,
+                    expectedWindowBounds: target.window.bounds)
+            }
+            return Self.result(from: result.outcome, background: true)
+        } catch let failure as DesktopActionFailure {
+            return Self.failureResult(failure, background: true)
+        }
+    }
+
+    private func type(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult {
+        let text = try Self.require(params.text, field: "text", allowEmpty: false)
+        let target = try self.requiredWindow(params)
+        let mode = params.deliveryMode ?? .background
+        if let elementRef = params.elementRef {
+            let focusResult = try await self.focusElement(
+                elementRef,
+                params: params,
+                target: target,
+                foreground: mode == .foreground)
+            if !focusResult.ok { return focusResult }
+        }
+        do {
+            let result: UIAutomationActionResult<TypeResult>
+            if mode == .foreground {
+                try await self.focus(target)
+                result = try await self.withExecutionAuthority {
+                    try await self.automation.typeActionsWithOutcome(
+                        [.text(text)], cadence: .fixed(milliseconds: 0), snapshotId: nil)
+                }
+            } else {
+                guard let identity = target.window.mutationIdentity else {
+                    throw ComputerActionService.ComputerActionError.staleObservation
+                }
+                result = try await self.withExecutionAuthority {
+                    try await self.automation.typeActionsWithOutcome(
+                        [.text(text)],
+                        cadence: .fixed(milliseconds: 0),
+                        snapshotId: params.elementRef == nil ? nil : self.observation?.snapshotId,
+                        expectedWindowIdentity: identity,
+                        expectedWindowBounds: target.window.bounds)
+                }
+            }
+            return Self.result(from: result.outcome, background: mode == .background)
+        } catch let failure as DesktopActionFailure {
+            return Self.failureResult(failure, background: mode == .background)
+        }
+    }
+
+    private func key(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult {
+        let keys = try Self.require(params.keys, field: "keys", allowEmpty: false)
+        let target = try self.requiredWindow(params)
+        let mode = params.deliveryMode ?? .background
+        if let elementRef = params.elementRef {
+            let focusResult = try await self.focusElement(
+                elementRef,
+                params: params,
+                target: target,
+                foreground: mode == .foreground)
+            if !focusResult.ok { return focusResult }
+        }
+        do {
+            let result: UIAutomationActionResult<Void>
+            if mode == .foreground {
+                try await self.focus(target)
+                result = try await self.withExecutionAuthority {
+                    try await self.automation.hotkeyWithOutcome(keys: keys, holdDuration: 0)
+                }
+            } else {
+                guard let identity = target.window.mutationIdentity else {
+                    throw ComputerActionService.ComputerActionError.staleObservation
+                }
+                result = try await self.withExecutionAuthority {
+                    try await self.automation.hotkeyWithOutcome(
+                        keys: keys,
+                        holdDuration: 0,
+                        expectedWindowIdentity: identity,
+                        expectedWindowBounds: target.window.bounds)
+                }
+            }
+            return Self.result(from: result.outcome, background: mode == .background)
+        } catch let failure as DesktopActionFailure {
+            return Self.failureResult(failure, background: mode == .background)
+        }
+    }
+
+    private func setValue(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult {
+        guard params.deliveryMode != .foreground else {
+            throw ComputerActionService.ComputerActionError.unsupportedAction(params.action)
+        }
+        let windowRef = try Self.require(params.windowRef, field: "windowRef")
+        _ = try self.resolveWindow(windowRef)
+        let element = try self.requiredElement(params, windowRef: windowRef)
+        let value = try Self.require(params.value, field: "value", allowEmpty: true)
+        do {
+            let result = try await self.withExecutionAuthority {
+                try await self.automation.setValueWithOutcome(
+                    target: element.id,
+                    value: .string(value),
+                    snapshotId: self.observation?.snapshotId)
+            }
+            return Self.result(from: result.outcome, background: true)
+        } catch let failure as DesktopActionFailure {
+            return Self.failureResult(failure, background: true)
+        }
+    }
+
+    private func invokeMenu(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult {
+        guard params.deliveryMode != .foreground else {
+            throw ComputerActionService.ComputerActionError.unsupportedAction(params.action)
+        }
+        let target = try self.requiredWindow(params)
+        guard let path = params.path, !path.isEmpty, path.count <= 16,
+              path.allSatisfy({ !$0.isEmpty && $0.count <= 200 })
+        else {
+            throw ComputerActionService.ComputerActionError.invalidV2Request(
+                "path must contain 1 to 16 non-empty menu components")
+        }
+        try await self.withExecutionAuthority {
+            try await self.menu.clickMenuItem(
+                app: "PID:\(target.app.processIdentifier)",
+                itemPath: path.joined(separator: " > "))
+        }
+        return OpenClawComputerActResult(ok: true, effect: .unverifiable, details: [
+            "deliveryMode": AnyCodable("background"),
+            "route": AnyCodable("accessibility_action"),
+        ])
+    }
+
+    private func scroll(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult {
+        let windowRef = try Self.require(params.windowRef, field: "windowRef")
+        let target = try self.resolveWindow(windowRef)
+        guard let direction = params.scrollDirection else {
+            throw ComputerActionService.ComputerActionError.invalidV2Request(
+                "scrollDirection is required for scroll")
+        }
+        let mode = params.deliveryMode ?? .background
+        let element = try params.elementRef.map { _ in try self.requiredElement(params, windowRef: windowRef) }
+        if mode == .background, element == nil {
+            return Self.backgroundEscalation(reason: "background_scroll_requires_element")
+        }
+        if mode == .foreground {
+            try await self.focus(target)
+            if let point = try self.optionalPoint(params, windowRef: windowRef) {
+                try await self.withExecutionAuthority {
+                    try await self.automation.moveMouse(to: point, duration: 0, steps: 1, profile: .linear)
+                }
+            }
+        }
+        let result = try await self.withExecutionAuthority {
+            try await self.automation.scrollWithOutcome(ScrollRequest(
+                direction: Self.scrollDirection(direction),
+                amount: min(100, max(1, params.scrollAmount ?? 3)),
+                target: element?.id,
+                snapshotId: element == nil ? nil : self.observation?.snapshotId,
+                foreground: mode == .foreground))
+        }
+        return Self.result(from: result.outcome, background: mode == .background)
+    }
+
+    // MARK: - Reference and target helpers
+
+    private func adoptLifecycleGeneration(_ generation: UInt64) {
+        guard self.lifecycleGeneration != generation else { return }
+        self.lifecycleGeneration = generation
+        self.appRefs.removeAll()
+        self.windowRefs.removeAll()
+        self.observation = nil
+    }
+
+    private func issueRef(_ kind: String) -> String {
+        "peekaboo:v2:\(kind):\(self.executionID):\(self.lifecycleGeneration ?? 0):" +
+            UUID().uuidString.lowercased()
+    }
+
+    private func issueWindowRef(app: ServiceApplicationInfo, window: ServiceWindowInfo) -> String {
+        if let existing = self.windowRefs.first(where: {
+            $0.value.app.processIdentifier == app.processIdentifier &&
+                $0.value.window.windowID == window.windowID &&
+                $0.value.window.mutationIdentity == window.mutationIdentity
+        })?.key {
+            return existing
+        }
+        let ref = self.issueRef("window")
+        self.windowRefs[ref] = WindowTarget(app: app, window: window)
+        return ref
+    }
+
+    private func resolveWindow(_ ref: String) throws -> WindowTarget {
+        guard let target = self.windowRefs[ref] else {
+            throw ComputerActionService.ComputerActionError.staleObservation
+        }
+        return target
+    }
+
+    private func requiredWindow(_ params: OpenClawComputerActParams) throws -> WindowTarget {
+        try self.resolveWindow(Self.require(params.windowRef, field: "windowRef"))
+    }
+
+    private func requiredObservation(_ params: OpenClawComputerActParams, windowRef: String) throws
+        -> ObservationState
+    {
+        guard let observation = self.observation,
+              observation.id == params.observationId,
+              observation.windowRef == windowRef
+        else {
+            throw ComputerActionService.ComputerActionError.staleObservation
+        }
+        return observation
+    }
+
+    private func requiredElement(
+        _ params: OpenClawComputerActParams,
+        windowRef: String) throws -> ElementTarget
+    {
+        let elementRef = try Self.require(params.elementRef, field: "elementRef")
+        let observation = try self.requiredObservation(params, windowRef: windowRef)
+        guard let element = observation.elements[elementRef] else {
+            throw ComputerActionService.ComputerActionError.staleObservation
+        }
+        return element
+    }
+
+    private func clickTarget(
+        _ params: OpenClawComputerActParams,
+        windowRef: String) throws -> (target: ClickTarget, point: CGPoint, snapshotId: String)
+    {
+        let observation = try self.requiredObservation(params, windowRef: windowRef)
+        if params.elementRef != nil {
+            let element = try self.requiredElement(params, windowRef: windowRef)
+            return (.elementId(element.id), element.bounds.centerPoint, observation.snapshotId)
+        }
+        guard let x = params.x, let y = params.y else {
+            throw ComputerActionService.ComputerActionError.invalidV2Request(
+                "coordinates or elementRef are required for \(params.action.rawValue)")
+        }
+        guard x >= 0, y >= 0 else {
+            throw ComputerActionService.ComputerActionError.invalidV2Request(
+                "coordinates must be nonnegative")
+        }
+        let point = CGPoint(x: x, y: y)
+        return (.coordinates(point), point, observation.snapshotId)
+    }
+
+    private func optionalPoint(
+        _ params: OpenClawComputerActParams,
+        windowRef: String) throws -> CGPoint?
+    {
+        if params.elementRef != nil {
+            return try self.requiredElement(params, windowRef: windowRef).bounds.centerPoint
+        }
+        if params.x == nil, params.y == nil { return nil }
+        guard let x = params.x, let y = params.y else {
+            throw ComputerActionService.ComputerActionError.invalidV2Request(
+                "both x and y are required")
+        }
+        guard x >= 0, y >= 0 else {
+            throw ComputerActionService.ComputerActionError.invalidV2Request(
+                "coordinates must be nonnegative")
+        }
+        _ = try self.requiredObservation(params, windowRef: windowRef)
+        return CGPoint(x: x, y: y)
+    }
+
+    private func focusElement(
+        _ elementRef: String,
+        params: OpenClawComputerActParams,
+        target: WindowTarget,
+        foreground: Bool) async throws -> OpenClawComputerActResult
+    {
+        let windowRef = try Self.require(params.windowRef, field: "windowRef")
+        let observation = try self.requiredObservation(params, windowRef: windowRef)
+        guard let element = observation.elements[elementRef] else {
+            throw ComputerActionService.ComputerActionError.staleObservation
+        }
+        if foreground {
+            try await self.focus(target)
+            try Self.postForegroundClick(at: element.bounds.centerPoint, action: .leftClick, modifiers: nil)
+            return OpenClawComputerActResult(ok: true, effect: .unverifiable)
+        }
+        guard let identity = target.window.mutationIdentity else {
+            throw ComputerActionService.ComputerActionError.staleObservation
+        }
+        let result = try await self.withExecutionAuthority {
+            try await self.automation.clickWithOutcome(
+                target: .elementId(element.id),
+                clickType: .single,
+                snapshotId: observation.snapshotId,
+                expectedWindowIdentity: identity,
+                expectedWindowBounds: target.window.bounds)
+        }
+        return Self.result(from: result.outcome, background: true)
+    }
+
+    private func focus(_ target: WindowTarget) async throws {
+        try await self.withExecutionAuthority {
+            try await self.applications.activateApplication(request: ApplicationActivationRequest(
+                application: target.app))
+        }
+        try await self.withExecutionAuthority {
+            try await self.windows.focusWindow(target: .windowId(target.window.windowID))
+        }
+    }
+}
+
+// MARK: - Projection and outcomes
+
+extension ComputerActionServiceV2 {
+    private static func result(
+        from outcome: DesktopActionOutcome?,
+        background: Bool) -> OpenClawComputerActResult
+    {
+        guard let outcome else {
+            return OpenClawComputerActResult(ok: true, effect: .unverifiable)
+        }
+        if outcome.effect == .refused {
+            return background
+                ? self.backgroundEscalation(reason: outcome.refusalReason?.rawValue ?? "action_refused")
+                : OpenClawComputerActResult(
+                    ok: false,
+                    effect: .suspectedNoop,
+                    escalation: OpenClawComputerEscalation(
+                        recommended: "desktop",
+                        reasonCode: "foreground_ineffective"))
+        }
+        let effect: OpenClawComputerActionEffect = switch outcome.effect {
+        case .confirmed: .confirmed
+        case .suspectedNoop: .suspectedNoop
+        case .partial, .unverifiable, .refused: .unverifiable
+        }
+        let escalation = background && outcome.effect == .suspectedNoop
+            ? OpenClawComputerEscalation(recommended: "foreground", reasonCode: "suspected_noop")
+            : nil
+        var details: [String: AnyCodable] = [
+            "route": AnyCodable(outcome.delivery?.mechanism.rawValue ?? "unknown"),
+            "evidence": AnyCodable(outcome.evidence.rawValue),
+        ]
+        if let delivery = outcome.delivery {
+            details["deliveryMode"] = AnyCodable(delivery.mode.rawValue)
+        }
+        return OpenClawComputerActResult(
+            ok: outcome.effect != .suspectedNoop,
+            effect: effect,
+            escalation: escalation,
+            details: details)
+    }
+
+    private static func failureResult(
+        _ failure: DesktopActionFailure,
+        background: Bool) -> OpenClawComputerActResult
+    {
+        let result = self.result(from: failure.outcome, background: background)
+        return OpenClawComputerActResult(
+            ok: false,
+            effect: result.effect ?? .unverifiable,
+            escalation: background
+                ? OpenClawComputerEscalation(
+                    recommended: "foreground",
+                    reasonCode: failure.outcome.effect == .suspectedNoop
+                        ? "suspected_noop"
+                        : "background_delivery_failed")
+                : nil,
+            details: ["message": AnyCodable(failure.message)])
+    }
+
+    private static func backgroundEscalation(reason: String) -> OpenClawComputerActResult {
+        OpenClawComputerActResult(
+            ok: false,
+            effect: .suspectedNoop,
+            escalation: OpenClawComputerEscalation(
+                recommended: "foreground",
+                reasonCode: reason))
+    }
+
+    private static func filterElements(
+        _ elements: [DetectedElement],
+        query: String?) -> [DetectedElement]
+    {
+        guard let query = query?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased(),
+              !query.isEmpty
+        else { return elements }
+        return elements.filter { element in
+            [element.id, element.type.rawValue, element.label, element.value]
+                .compactMap { $0?.lowercased() }
+                .contains { $0.contains(query) }
+        }
+    }
+
+    private static func projectElements(
+        _ elements: [DetectedElement],
+        query: String?,
+        limit: Int) -> (elements: [[String: Any]], truncated: Int)
+    {
+        let filtered = self.filterElements(elements, query: query)
+        return (
+            Array(filtered.prefix(limit)).map { element in
+                var row: [String: Any] = [
+                    "role": element.type.rawValue,
+                    "bounds": self.boundsDictionary(element.bounds),
+                ]
+                if let label = element.label { row["label"] = label }
+                if let value = element.value { row["value"] = value }
+                return row
+            },
+            max(0, filtered.count - limit))
+    }
+
+    private static func windowContext(
+        _ target: WindowTarget,
+        limits: (depth: Int, maxElements: Int)) -> WindowContext
+    {
+        WindowContext(
+            applicationName: target.app.name,
+            applicationBundleId: target.app.bundleIdentifier,
+            applicationProcessId: target.app.processIdentifier,
+            windowTitle: target.window.title,
+            windowID: target.window.windowID,
+            windowBounds: target.window.bounds,
+            windowMutationIdentity: target.window.mutationIdentity,
+            traversalBudget: AXTraversalBudget(
+                maxDepth: limits.depth,
+                maxElementCount: limits.maxElements,
+                maxChildrenPerNode: AXTraversalBudget.defaultMaxChildrenPerNode),
+            requiresFreshAccessibilityTree: true)
+    }
+
+    private static func observationLimits(
+        _ params: OpenClawComputerActParams) throws -> (depth: Int, maxElements: Int)
+    {
+        let depth = params.depth ?? AXTraversalBudget.defaultMaxDepth
+        let maxElements = params.maxElements ?? AXTraversalBudget.defaultMaxElementCount
+        guard (0...64).contains(depth), (1...2000).contains(maxElements) else {
+            throw ComputerActionService.ComputerActionError.invalidV2Request(
+                "depth must be 0...64 and maxElements must be 1...2000")
+        }
+        return (depth, maxElements)
+    }
+
+    private static func require(
+        _ value: String?,
+        field: String,
+        allowEmpty: Bool = false) throws -> String
+    {
+        guard let value, allowEmpty || !value.isEmpty else {
+            throw ComputerActionService.ComputerActionError.invalidV2Request(
+                "\(field) is required")
+        }
+        return value
+    }
+
+    private static func bounds(_ rect: CGRect) -> OpenClawComputerBounds {
+        OpenClawComputerBounds(
+            x: Double(rect.origin.x),
+            y: Double(rect.origin.y),
+            width: max(0, Double(rect.width)),
+            height: max(0, Double(rect.height)))
+    }
+
+    private static func boundsDictionary(_ rect: CGRect) -> [String: Any] {
+        [
+            "x": Double(rect.origin.x),
+            "y": Double(rect.origin.y),
+            "width": max(0, Double(rect.width)),
+            "height": max(0, Double(rect.height)),
+        ]
+    }
+
+    private static func scrollDirection(
+        _ direction: OpenClawComputerScrollDirection) -> PeekabooFoundation.ScrollDirection
+    {
+        switch direction {
+        case .up: .up
+        case .down: .down
+        case .left: .left
+        case .right: .right
+        }
+    }
+
+    private static func postForegroundClick(
+        at point: CGPoint,
+        action: OpenClawComputerAction,
+        modifiers: String?) throws
+    {
+        guard CGPreflightPostEventAccess() else {
+            throw ComputerActionService.ComputerActionError.postEventAccessDenied
+        }
+        let button: CGMouseButton = action == .middleClick ? .center : action == .rightClick ? .right : .left
+        let downType: CGEventType = button == .center ? .otherMouseDown : button == .right ? .rightMouseDown :
+            .leftMouseDown
+        let upType: CGEventType = button == .center ? .otherMouseUp : button == .right ? .rightMouseUp : .leftMouseUp
+        let count = action == .tripleClick ? 3 : action == .doubleClick ? 2 : 1
+        let flags = try self.modifierFlags(modifiers)
+        for index in 1...count {
+            guard let down = CGEvent(
+                mouseEventSource: nil,
+                mouseType: downType,
+                mouseCursorPosition: point,
+                mouseButton: button),
+                let up = CGEvent(
+                    mouseEventSource: nil,
+                    mouseType: upType,
+                    mouseCursorPosition: point,
+                    mouseButton: button)
+            else {
+                throw ComputerActionService.ComputerActionError.refused(
+                    "failed to construct foreground click")
+            }
+            down.flags = flags
+            up.flags = flags
+            down.setIntegerValueField(.mouseEventClickState, value: Int64(index))
+            up.setIntegerValueField(.mouseEventClickState, value: Int64(index))
+            down.post(tap: .cghidEventTap)
+            up.post(tap: .cghidEventTap)
+        }
+    }
+
+    private static func modifierFlags(_ raw: String?) throws -> CGEventFlags {
+        var flags: CGEventFlags = []
+        for token in (raw ?? "")
+            .lowercased()
+            .split(whereSeparator: { $0 == "," || $0 == "+" || $0.isWhitespace })
+        {
+            switch token {
+            case "cmd", "command", "meta": flags.insert(.maskCommand)
+            case "shift": flags.insert(.maskShift)
+            case "ctrl", "control": flags.insert(.maskControl)
+            case "alt", "option": flags.insert(.maskAlternate)
+            case "fn", "function": flags.insert(.maskSecondaryFn)
+            default:
+                throw ComputerActionService.ComputerActionError.invalidV2Request(
+                    "unsupported modifier '\(token)'")
+            }
+        }
+        return flags
+    }
+}
@@ -1,9 +1,55 @@
 import Foundation
+import OpenClawProtocol

 enum ComputerControlProvider: String, CaseIterable, Sendable {
    case peekaboo
    case cua

+    static var peekabooComputerUseDescriptor: OpenClawProtocol.AnyCodable {
+        OpenClawProtocol.AnyCodable([
+            "contractVersion": 2,
+            "provider": [
+                "id": "peekaboo",
+                "label": "Peekaboo",
+                "generation": "peekaboo-v2:\(UUID().uuidString.lowercased())",
+            ],
+            "actions": [
+                "screenshot",
+                "left_click",
+                "right_click",
+                "middle_click",
+                "double_click",
+                "triple_click",
+                "mouse_move",
+                "left_click_drag",
+                "left_mouse_down",
+                "left_mouse_up",
+                "scroll",
+                "type",
+                "key",
+                "hold_key",
+                "list_apps",
+                "list_windows",
+                "get_accessibility_tree",
+                "get_cursor_position",
+                "get_window_state",
+                "launch_app",
+                "kill_app",
+                "bring_to_front",
+                "set_value",
+                "invoke_menu",
+            ],
+            "targets": ["screen", "window", "element"],
+            "deliveryModes": ["background", "foreground"],
+            "observations": ["image", "accessibility"],
+            "features": [
+                "recording": false,
+                "agentCursor": false,
+                "multiDisplay": true,
+            ],
+        ] as [String: Any])
+    }
+
    static func current(
        defaults: UserDefaults = AppDefaults.standard,
        cuaAvailable: Bool = CuaDriverArtifact.bundledExecutableURL != nil) -> Self
@@ -1256,11 +1256,13 @@ extension MacNodeModeCoordinator {
        commands: [String],
        workerManifest: MacNodeHostManifest?) -> OpenClawProtocol.AnyCodable?
    {
-        guard provider == .cua,
-              commands.contains(MacNodeScreenCommand.snapshot.rawValue),
+        guard commands.contains(MacNodeScreenCommand.snapshot.rawValue),
              commands.contains(OpenClawComputerCommand.act.rawValue)
        else { return nil }
-        return workerManifest?.computerUse
+        return switch provider {
+        case .peekaboo: ComputerControlProvider.peekabooComputerUseDescriptor
+        case .cua: workerManifest?.computerUse
+        }
    }

    nonisolated static func mergingUnique(_ primary: [String], _ additional: [String]) -> [String] {
@@ -631,16 +631,21 @@ extension MacNodeRuntime {
                        + "under System Settings → Privacy & Security → Accessibility")
            case .noDisplays, .invalidScreenIndex, .missingDisplayFrameId, .displayFrameChanged,
                 .missingCoordinate, .coordinateOutOfBounds, .invalidReferenceWidth, .missingKeys,
-                 .emptyText, .invalidScroll, .invalidModifier, .buttonAlreadyHeld, .buttonNotHeld:
+                 .emptyText, .invalidScroll, .invalidModifier, .buttonAlreadyHeld, .buttonNotHeld,
+                 .invalidV2Request, .staleObservation, .unsupportedAction:
                return Self.errorResponse(
                    req,
                    code: .invalidRequest,
-                    message: "INVALID_REQUEST: \(error.localizedDescription)")
-            case .eventCreationFailed, .lifecycleChanged:
+                    message: error.localizedDescription.hasPrefix("COMPUTER_")
+                        ? error.localizedDescription
+                        : "INVALID_REQUEST: \(error.localizedDescription)")
+            case .eventCreationFailed, .lifecycleChanged, .refused:
                return Self.errorResponse(
                    req,
                    code: .unavailable,
-                    message: "UNAVAILABLE: \(error.localizedDescription)")
+                    message: error.localizedDescription.hasPrefix("COMPUTER_")
+                        ? error.localizedDescription
+                        : "UNAVAILABLE: \(error.localizedDescription)")
            }
        }
    }
@@ -114,6 +114,14 @@ struct ComputerActionServiceTests {
        return false
    }

+    private func unsupportedAction(_ error: Error?) -> OpenClawComputerAction? {
+        guard let error = error as? ComputerActionService.ComputerActionError else { return nil }
+        if case let .unsupportedAction(action) = error {
+            return action
+        }
+        return nil
+    }
+
    private func validationError(
        _ operation: () throws -> Void) -> ComputerActionService.ComputerActionError?
    {
@@ -431,6 +439,76 @@ struct ComputerActionServiceTests {
        #expect(probe.maximumActiveActionCount == 1)
    }

+    @Test func `v2 authority rejects a result after lifecycle revocation`() async {
+        let service = ComputerActionServiceV2()
+        var checks = 0
+        let outcomeError: Error?
+        do {
+            _ = try await service.perform(
+                OpenClawComputerActParams(action: .getCursorPosition),
+                lifecycleGeneration: 0,
+                checkExecutionAllowed: {
+                    checks += 1
+                    if checks > 1 {
+                        throw ComputerActionService.ComputerActionError.lifecycleChanged
+                    }
+                })
+            outcomeError = nil
+        } catch {
+            outcomeError = error
+        }
+
+        #expect(self.isLifecycleChanged(outcomeError))
+        #expect(checks == 2)
+    }
+
+    @Test func `execution authority revalidates after awaited preparation`() async {
+        let started = AsyncSignal()
+        let resume = AsyncSignal()
+        let releaseProbe = LifecycleReleaseProbe(allowed: true)
+        let authority = ComputerActionExecutionAuthority {
+            guard releaseProbe.allowed else {
+                throw ComputerActionService.ComputerActionError.lifecycleChanged
+            }
+        }
+        let operation = Task { @MainActor in
+            try await authority.run {
+                await started.signal()
+                await resume.wait()
+                return true
+            }
+        }
+        await started.wait()
+        releaseProbe.allowed = false
+        await resume.signal()
+
+        let outcomeError: Error?
+        do {
+            _ = try await operation.value
+            outcomeError = nil
+        } catch {
+            outcomeError = error
+        }
+        #expect(self.isLifecycleChanged(outcomeError))
+    }
+
+    @Test func `v2 rejects foreground accessibility-only actions`() async {
+        let service = ComputerActionServiceV2()
+        for action in [OpenClawComputerAction.setValue, .invokeMenu] {
+            let outcomeError: Error?
+            do {
+                _ = try await service.perform(
+                    OpenClawComputerActParams(action: action, deliveryMode: .foreground),
+                    lifecycleGeneration: 0,
+                    checkExecutionAllowed: {})
+                outcomeError = nil
+            } catch {
+                outcomeError = error
+            }
+            #expect(self.unsupportedAction(outcomeError) == action)
+        }
+    }
+
    @Test func `cancelled queued action never executes`() async throws {
        let probe = ActionProbe()
        let queue = ComputerActionExecutionQueue(onLifecycleRelease: { true })
@@ -267,7 +267,7 @@ struct MacNodeHostWorkerTests {
            ["system", "mcp"]) == ["canvas", "screen", "system", "mcp"])
    }

-    @Test func `provider selection filters command ownership and publishes only the CUA descriptor`() throws {
+    @Test func `provider selection filters command ownership and publishes each provider descriptor`() throws {
        let descriptor = OpenClawProtocol.AnyCodable([
            "contractVersion": OpenClawProtocol.AnyCodable(2),
        ])
@@ -282,6 +282,25 @@ struct MacNodeHostWorkerTests {
        #expect(!peekaboo.commands.contains(MacNodeScreenCommand.snapshot.rawValue))
        #expect(!peekaboo.commands.contains(OpenClawComputerCommand.act.rawValue))
        #expect(peekaboo.computerUse == nil)
+        let peekabooDescriptor = try #require(MacNodeModeCoordinator.computerUseDescriptor(
+            provider: .peekaboo,
+            commands: [MacNodeScreenCommand.snapshot.rawValue, OpenClawComputerCommand.act.rawValue],
+            workerManifest: peekaboo))
+        let peekabooJSON = try JSONEncoder().encode(peekabooDescriptor)
+        let peekabooObject = try #require(
+            JSONSerialization.jsonObject(with: peekabooJSON) as? [String: Any])
+        #expect(peekabooObject["contractVersion"] as? Int == 2)
+        #expect((peekabooObject["provider"] as? [String: Any])?["id"] as? String == "peekaboo")
+        let actions = try #require(peekabooObject["actions"] as? [String])
+        #expect(actions.contains("get_window_state"))
+        #expect(actions.contains("invoke_menu"))
+        #expect(!actions.contains("zoom"))
+        #expect(!actions.contains("get_browser_state"))
+        #expect(!actions.contains("start_recording"))
+        let features = try #require(peekabooObject["features"] as? [String: Any])
+        #expect(features["recording"] as? Bool == false)
+        #expect(features["agentCursor"] as? Bool == false)
+        #expect(features["multiDisplay"] as? Bool == true)

        let cua = try #require(MacNodeModeCoordinator.workerManifest(manifest, for: .cua))
        #expect(cua.commands == manifest.commands)
@@ -12,7 +12,8 @@ public enum OpenClawComputerCommand: String, Codable, Sendable {
 /// onto the embedded Peekaboo automation engine plus a narrow CoreGraphics
 /// path for primitives Peekaboo does not express (middle/triple click,
 /// separate mouse down/up, modifier-held clicks/scroll).
-public enum OpenClawComputerAction: String, Codable, Sendable {
+public enum OpenClawComputerAction: String, Codable, CaseIterable, Sendable {
+    case screenshot
    case leftClick = "left_click"
    case rightClick = "right_click"
    case middleClick = "middle_click"
@@ -26,6 +27,60 @@ public enum OpenClawComputerAction: String, Codable, Sendable {
    case type
    case key
    case holdKey = "hold_key"
+    case wait
+    case listApps = "list_apps"
+    case listWindows = "list_windows"
+    case getAccessibilityTree = "get_accessibility_tree"
+    case getCursorPosition = "get_cursor_position"
+    case getWindowState = "get_window_state"
+    case launchApp = "launch_app"
+    case killApp = "kill_app"
+    case bringToFront = "bring_to_front"
+    case setValue = "set_value"
+    case zoom
+    case getBrowserState = "get_browser_state"
+    case browserPrepare = "browser_prepare"
+    case browserNavigate = "browser_navigate"
+    case browserClick = "browser_click"
+    case browserType = "browser_type"
+    case browserDialog = "browser_dialog"
+    case browserSetInputFiles = "browser_set_input_files"
+    case browserDownload = "browser_download"
+    case browserPointer = "browser_pointer"
+    case escalateScope = "escalate_scope"
+    case getRecordingState = "get_recording_state"
+    case startRecording = "start_recording"
+    case stopRecording = "stop_recording"
+    case replayTrajectory = "replay_trajectory"
+    case invokeMenu = "invoke_menu"
+
+    private var isNativeWireAction: Bool {
+        switch self {
+        case .wait, .zoom, .getBrowserState, .browserPrepare, .browserNavigate,
+             .browserClick, .browserType, .browserDialog, .browserSetInputFiles,
+             .browserDownload, .browserPointer, .escalateScope, .getRecordingState,
+             .startRecording, .stopRecording, .replayTrajectory:
+            false
+        default:
+            true
+        }
+    }
+
+    public init(from decoder: Decoder) throws {
+        let container = try decoder.singleValueContainer()
+        let rawValue = try container.decode(String.self)
+        guard let action = Self(rawValue: rawValue), action.isNativeWireAction else {
+            throw DecodingError.dataCorruptedError(
+                in: container,
+                debugDescription: "Unsupported native computer action: \(rawValue)")
+        }
+        self = action
+    }
+
+    public func encode(to encoder: Encoder) throws {
+        var container = encoder.singleValueContainer()
+        try container.encode(self.rawValue)
+    }
 }

 public enum OpenClawComputerScrollDirection: String, Codable, Sendable {
@@ -35,6 +90,19 @@ public enum OpenClawComputerScrollDirection: String, Codable, Sendable {
    case right
 }

+public enum OpenClawComputerDeliveryMode: String, Codable, Sendable {
+    case background
+    case foreground
+}
+
+public enum OpenClawComputerEscalationReason: String, Codable, Sendable {
+    case axTreePixelMismatch = "ax_tree_pixel_mismatch"
+    case backgroundDeliveryFailed = "background_delivery_failed"
+    case foregroundIneffective = "foreground_ineffective"
+    case noWindowTarget = "no_window_target"
+    case other
+}
+
 /// Wire params for `computer.act`. All coordinate fields are reference-screenshot
 /// pixels at `refWidth`; `keys` is a chord for key/hold_key; `modifiers` are
 /// modifier keys held during pointer actions; `scrollAmount` is wheel ticks.
@@ -54,6 +122,21 @@ public struct OpenClawComputerActParams: Codable, Sendable, Equatable {
    public var durationMs: Int?
    public var screenIndex: Int?
    public var refWidth: Int?
+    public var windowRef: String?
+    public var elementRef: String?
+    public var observationId: String?
+    public var deliveryMode: OpenClawComputerDeliveryMode?
+    public var query: String?
+    public var depth: Int?
+    public var maxElements: Int?
+    public var app: String?
+    public var value: String?
+    public var path: [String]?
+    public var x1: Double?
+    public var y1: Double?
+    public var x2: Double?
+    public var y2: Double?
+    public var reason: OpenClawComputerEscalationReason?

    public init(
        action: OpenClawComputerAction,
@@ -69,7 +152,22 @@ public struct OpenClawComputerActParams: Codable, Sendable, Equatable {
        scrollAmount: Int? = nil,
        durationMs: Int? = nil,
        screenIndex: Int? = nil,
-        refWidth: Int? = nil)
+        refWidth: Int? = nil,
+        windowRef: String? = nil,
+        elementRef: String? = nil,
+        observationId: String? = nil,
+        deliveryMode: OpenClawComputerDeliveryMode? = nil,
+        query: String? = nil,
+        depth: Int? = nil,
+        maxElements: Int? = nil,
+        app: String? = nil,
+        value: String? = nil,
+        path: [String]? = nil,
+        x1: Double? = nil,
+        y1: Double? = nil,
+        x2: Double? = nil,
+        y2: Double? = nil,
+        reason: OpenClawComputerEscalationReason? = nil)
    {
        self.action = action
        self.displayFrameId = displayFrameId
@@ -85,18 +183,138 @@ public struct OpenClawComputerActParams: Codable, Sendable, Equatable {
        self.durationMs = durationMs
        self.screenIndex = screenIndex
        self.refWidth = refWidth
+        self.windowRef = windowRef
+        self.elementRef = elementRef
+        self.observationId = observationId
+        self.deliveryMode = deliveryMode
+        self.query = query
+        self.depth = depth
+        self.maxElements = maxElements
+        self.app = app
+        self.value = value
+        self.path = path
+        self.x1 = x1
+        self.y1 = y1
+        self.x2 = x2
+        self.y2 = y2
+        self.reason = reason
    }
 }

-/// Result of a `computer.act` input action.
+public enum OpenClawComputerActionEffect: String, Codable, Sendable {
+    case confirmed
+    case unverifiable
+    case suspectedNoop = "suspected_noop"
+}
+
+public struct OpenClawComputerBounds: Codable, Sendable, Equatable {
+    public var x: Double
+    public var y: Double
+    public var width: Double
+    public var height: Double
+
+    public init(x: Double, y: Double, width: Double, height: Double) {
+        self.x = x
+        self.y = y
+        self.width = width
+        self.height = height
+    }
+}
+
+public struct OpenClawComputerObservationElement: Codable, Sendable, Equatable {
+    public var elementRef: String
+    public var role: String
+    public var label: String?
+    public var value: String?
+    public var bounds: OpenClawComputerBounds
+
+    public init(
+        elementRef: String,
+        role: String,
+        label: String? = nil,
+        value: String? = nil,
+        bounds: OpenClawComputerBounds)
+    {
+        self.elementRef = elementRef
+        self.role = role
+        self.label = label
+        self.value = value
+        self.bounds = bounds
+    }
+}
+
+public struct OpenClawComputerObservation: Codable, Sendable, Equatable {
+    public var kind: String
+    public var base64: String?
+    public var format: String?
+    public var width: Int?
+    public var height: Int?
+    public var observationId: String?
+    public var elements: [OpenClawComputerObservationElement]?
+
+    public init(
+        kind: String,
+        base64: String? = nil,
+        format: String? = nil,
+        width: Int? = nil,
+        height: Int? = nil,
+        observationId: String? = nil,
+        elements: [OpenClawComputerObservationElement]? = nil)
+    {
+        self.kind = kind
+        self.base64 = base64
+        self.format = format
+        self.width = width
+        self.height = height
+        self.observationId = observationId
+        self.elements = elements
+    }
+}
+
+public struct OpenClawComputerEscalation: Codable, Sendable, Equatable {
+    public var recommended: String
+    public var reasonCode: String
+
+    public init(recommended: String, reasonCode: String) {
+        self.recommended = recommended
+        self.reasonCode = reasonCode
+    }
+}
+
+/// Result of a `computer.act` input action. Optional v2 fields are omitted for
+/// v1 actions so their existing JSON payload remains byte-for-byte unchanged.
 public struct OpenClawComputerActResult: Codable, Sendable, Equatable {
    public var ok: Bool
-    public var cursorX: Double
-    public var cursorY: Double
+    public var cursorX: Double?
+    public var cursorY: Double?
+    public var effect: OpenClawComputerActionEffect?
+    public var observation: OpenClawComputerObservation?
+    public var escalation: OpenClawComputerEscalation?
+    public var details: [String: AnyCodable]?

    public init(ok: Bool, cursorX: Double, cursorY: Double) {
        self.ok = ok
        self.cursorX = cursorX
        self.cursorY = cursorY
+        self.effect = nil
+        self.observation = nil
+        self.escalation = nil
+        self.details = nil
+    }
+
+    public init(
+        ok: Bool,
+        effect: OpenClawComputerActionEffect? = nil,
+        observation: OpenClawComputerObservation? = nil,
+        escalation: OpenClawComputerEscalation? = nil,
+        details: [String: AnyCodable]? = nil)
+    {
+        self.ok = ok
+        self.cursorX = nil
+        self.cursorY = nil
+        self.effect = effect
+        self.observation = observation
+        self.escalation = escalation
+        self.details = details
    }
 }
@@ -0,0 +1,143 @@
+import Foundation
+import OpenClawKit
+import Testing
+
+struct ComputerCommandsWireDecodeTests {
+    private struct DecodeCase {
+        let json: String
+        let action: OpenClawComputerAction
+        var app: String?
+        var windowRef: String?
+        var elementRef: String?
+        var observationId: String?
+        var deliveryMode: OpenClawComputerDeliveryMode?
+        var query: String?
+        var depth: Int?
+        var maxElements: Int?
+        var value: String?
+        var path: [String]?
+        var x: Double?
+        var y: Double?
+        var text: String?
+        var keys: String?
+    }
+
+    private func decode(_ testCase: DecodeCase) throws -> OpenClawComputerActParams {
+        try JSONDecoder().decode(OpenClawComputerActParams.self, from: Data(testCase.json.utf8))
+    }
+
+    private func expectFields(_ params: OpenClawComputerActParams, match testCase: DecodeCase) {
+        #expect(params.action == testCase.action)
+        #expect(params.app == testCase.app)
+        #expect(params.windowRef == testCase.windowRef)
+        #expect(params.elementRef == testCase.elementRef)
+        #expect(params.observationId == testCase.observationId)
+        #expect(params.deliveryMode == testCase.deliveryMode)
+        #expect(params.query == testCase.query)
+        #expect(params.depth == testCase.depth)
+        #expect(params.maxElements == testCase.maxElements)
+        #expect(params.value == testCase.value)
+        #expect(params.path == testCase.path)
+        #expect(params.x == testCase.x)
+        #expect(params.y == testCase.y)
+        #expect(params.text == testCase.text)
+        #expect(params.keys == testCase.keys)
+    }
+
+    @Test func `decodes every implemented v2 action family and delivery mode`() throws {
+        let cases = [
+            DecodeCase(
+                json: #"""
+                {"action":"get_accessibility_tree","windowRef":"window-1","query":"Save",
+                 "depth":4,"maxElements":250}
+                """#,
+                action: .getAccessibilityTree,
+                windowRef: "window-1",
+                query: "Save",
+                depth: 4,
+                maxElements: 250),
+            DecodeCase(
+                json: #"""
+                {"action":"get_window_state","windowRef":"window-2","observationId":"observation-2",
+                 "deliveryMode":"background"}
+                """#,
+                action: .getWindowState,
+                windowRef: "window-2",
+                observationId: "observation-2",
+                deliveryMode: .background),
+            DecodeCase(
+                json: #"{"action":"launch_app","app":"TextEdit","deliveryMode":"foreground"}"#,
+                action: .launchApp,
+                app: "TextEdit",
+                deliveryMode: .foreground),
+            DecodeCase(
+                json: #"""
+                {"action":"set_value","elementRef":"element-3","observationId":"observation-3",
+                 "value":"hello","deliveryMode":"background"}
+                """#,
+                action: .setValue,
+                elementRef: "element-3",
+                observationId: "observation-3",
+                deliveryMode: .background,
+                value: "hello"),
+            DecodeCase(
+                json: #"{"action":"invoke_menu","app":"app-4","path":["File","Save As…"],"deliveryMode":"foreground"}"#,
+                action: .invokeMenu,
+                app: "app-4",
+                deliveryMode: .foreground,
+                path: ["File", "Save As…"]),
+            DecodeCase(
+                json: #"""
+                {"action":"left_click","windowRef":"window-5","observationId":"observation-5",
+                 "x":120,"y":240,"deliveryMode":"background"}
+                """#,
+                action: .leftClick,
+                windowRef: "window-5",
+                observationId: "observation-5",
+                deliveryMode: .background,
+                x: 120,
+                y: 240),
+            DecodeCase(
+                json: #"{"action":"type","windowRef":"window-6","text":"hello","deliveryMode":"foreground"}"#,
+                action: .type,
+                windowRef: "window-6",
+                deliveryMode: .foreground,
+                text: "hello"),
+            DecodeCase(
+                json: #"{"action":"key","elementRef":"element-7","keys":"cmd+return","deliveryMode":"background"}"#,
+                action: .key,
+                elementRef: "element-7",
+                deliveryMode: .background,
+                keys: "cmd+return"),
+        ]
+
+        for testCase in cases {
+            try self.expectFields(self.decode(testCase), match: testCase)
+        }
+    }
+
+    @Test func `rejects unknown and native-unimplemented action names`() throws {
+        for action in ["totally_unknown", "browser_click", "start_recording"] {
+            let data = Data(#"{"action":"\#(action)"}"#.utf8)
+            #expect(throws: DecodingError.self) {
+                _ = try JSONDecoder().decode(OpenClawComputerActParams.self, from: data)
+            }
+        }
+    }
+
+    @Test func `action raw values match the frozen computer use v2 contract`() {
+        let frozenActionNames = [
+            "screenshot", "left_click", "right_click", "middle_click", "double_click",
+            "triple_click", "mouse_move", "left_click_drag", "left_mouse_down", "left_mouse_up",
+            "scroll", "type", "key", "hold_key", "wait", "list_apps", "list_windows",
+            "get_accessibility_tree", "get_cursor_position", "get_window_state", "launch_app",
+            "kill_app", "bring_to_front", "set_value", "zoom", "get_browser_state",
+            "browser_prepare", "browser_navigate", "browser_click", "browser_type",
+            "browser_dialog", "browser_set_input_files", "browser_download", "browser_pointer",
+            "escalate_scope", "get_recording_state", "start_recording", "stop_recording",
+            "replay_trajectory", "invoke_menu",
+        ]
+
+        #expect(OpenClawComputerAction.allCases.map(\.rawValue) == frozenActionNames)
+    }
+}
@@ -270,4 +270,72 @@ struct ComputerInputGeometryTests {
        #expect(holdParams.keys == "space")
        #expect(holdParams.durationMs == 2000)
    }
+
+    @Test func `v1 result encoding omits every additive v2 field`() throws {
+        let data = try JSONEncoder().encode(OpenClawComputerActResult(
+            ok: true,
+            cursorX: 12,
+            cursorY: 34))
+        let object = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any])
+
+        #expect(object.keys.sorted() == ["cursorX", "cursorY", "ok"])
+        #expect(object["ok"] as? Bool == true)
+        #expect(object["cursorX"] as? Double == 12)
+        #expect(object["cursorY"] as? Double == 34)
+    }
+
+    @Test func `decodes v2 window observation and element delivery fields`() throws {
+        let data = try #require("""
+        {
+          "action":"set_value",
+          "windowRef":"window-1",
+          "elementRef":"element-1",
+          "observationId":"observation-1",
+          "value":"hello",
+          "deliveryMode":"background"
+        }
+        """.data(using: .utf8))
+        let params = try JSONDecoder().decode(OpenClawComputerActParams.self, from: data)
+
+        #expect(params.action == .setValue)
+        #expect(params.windowRef == "window-1")
+        #expect(params.elementRef == "element-1")
+        #expect(params.observationId == "observation-1")
+        #expect(params.value == "hello")
+        #expect(params.deliveryMode == .background)
+    }
+
+    @Test func `v2 result encoding matches the shared result envelope`() throws {
+        let result = OpenClawComputerActResult(
+            ok: false,
+            effect: .suspectedNoop,
+            observation: OpenClawComputerObservation(
+                kind: "window",
+                base64: "cG5n",
+                format: "png",
+                width: 10,
+                height: 20,
+                observationId: "observation-1",
+                elements: [OpenClawComputerObservationElement(
+                    elementRef: "element-1",
+                    role: "button",
+                    label: "Save",
+                    bounds: OpenClawComputerBounds(x: 1, y: 2, width: 3, height: 4))]),
+            escalation: OpenClawComputerEscalation(
+                recommended: "foreground",
+                reasonCode: "background_delivery_failed"),
+            details: ["deliveryMode": AnyCodable("background")])
+        let data = try JSONEncoder().encode(result)
+        let object = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any])
+
+        #expect(object["ok"] as? Bool == false)
+        #expect(object["effect"] as? String == "suspected_noop")
+        let observation = try #require(object["observation"] as? [String: Any])
+        #expect(observation["observationId"] as? String == "observation-1")
+        let elements = try #require(observation["elements"] as? [[String: Any]])
+        #expect(elements.first?["elementRef"] as? String == "element-1")
+        let escalation = try #require(object["escalation"] as? [String: Any])
+        #expect(escalation["recommended"] as? String == "foreground")
+        #expect(escalation["reasonCode"] as? String == "background_delivery_failed")
+    }
 }