From 554fb212c95dbf1fc709fc9a78ea3c2a30050c47 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 24 Aug 2026 01:56:12 -0700 Subject: [PATCH] fix(nodes): report camera positions the hardware actually reached (#128595) * fix(nodes): report camera positions the hardware actually reached `camera.ptz.control` returned a position it never verified, and `camera.snap`/`camera.clip` could capture from a camera the caller did not ask for. Both told the agent an action succeeded when it had not. PTZ read its post-write status from the same UVC connection that issued the write. Gimbal cameras echo a pending setpoint back on the writing connection, so the check confirmed its own write. Those cameras also service camera-terminal controls only while a video stream is active, and no capture session was held, so writes could be discarded entirely while reads returned phantom values. Hold a frame-discarding capture session across every PTZ operation, close the writing controller, and verify through a fresh connection against each axis's advertised resolution. An axis that misses now reports through the existing CAMERA_PTZ_PARTIAL outcome with observed versus requested values and what to check next. Apple camera selection accepted an explicit deviceId and silently fell back to the default camera when nothing matched. Linux already rejected this, and CameraPTZService already rejected it in the same app. Centralize exact selection in OpenClawKit so macOS and iOS both fail with a device-not-found error; the facing/default fallback stays only for requests that supply no deviceId. camera.ptz.status now activates the camera and its privacy indicator for the duration of the read. That is the cost of returning real positions. * fix(nodes): tell callers how to recover from an unknown camera ID Device IDs change when cameras are reconnected, so a bare device-not-found error dead-ends the caller. Both Apple errors and the docs now point at camera.list for current IDs. Addresses the ClawSweeper P2 finding on #128595. --- .../ios/Sources/Camera/CameraController.swift | 35 +-- .../OpenClaw/CameraCaptureService.swift | 32 +-- .../OpenClaw/CameraPTZNativeController.swift | 27 ++ .../Sources/OpenClaw/CameraPTZService.swift | 97 +++++-- .../CameraPTZServiceTests.swift | 259 ++++++++++++++++-- .../CameraCapturePipelineSupport.swift | 41 +-- docs/nodes/camera.md | 8 +- docs/reference/AGENTS.default.md | 2 +- skills/camsnap/SKILL.md | 12 +- 9 files changed, 413 insertions(+), 100 deletions(-) diff --git a/apps/ios/Sources/Camera/CameraController.swift b/apps/ios/Sources/Camera/CameraController.swift index fc9bc163193d..f293369dc333 100644 --- a/apps/ios/Sources/Camera/CameraController.swift +++ b/apps/ios/Sources/Camera/CameraController.swift @@ -61,9 +61,8 @@ actor CameraController { preferFrontCamera: facing == .front, deviceId: params.deviceId, pickCamera: { preferFrontCamera, deviceId in - Self.pickCamera(facing: preferFrontCamera ? .front : .back, deviceId: deviceId) + try Self.pickCamera(facing: preferFrontCamera ? .front : .back, deviceId: deviceId) }, - cameraUnavailableError: CameraError.cameraUnavailable, mapSetupError: { setupError in CameraError.captureFailed(setupError.localizedDescription) }) @@ -141,9 +140,8 @@ actor CameraController { includeAudio: includeAudio, durationMs: durationMs), pickCamera: { preferFrontCamera, deviceId in - Self.pickCamera(facing: preferFrontCamera ? .front : .back, deviceId: deviceId) + try Self.pickCamera(facing: preferFrontCamera ? .front : .back, deviceId: deviceId) }, - cameraUnavailableError: CameraError.cameraUnavailable, mapSetupError: Self.mapMovieSetupError, operation: { output in let recording = CameraMovieRecordingOperation(output: output, outputURL: movURL) @@ -196,19 +194,24 @@ actor CameraController { private nonisolated static func pickCamera( facing: OpenClawCameraFacing, - deviceId: String?) -> AVCaptureDevice? + deviceId: String?) throws -> AVCaptureDevice { - if let deviceId, !deviceId.isEmpty { - if let match = discoverVideoDevices().first(where: { $0.uniqueID == deviceId }) { - return match - } - } - let position: AVCaptureDevice.Position = (facing == .front) ? .front : .back - if let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: position) { - return device - } - // Fall back to any default camera (e.g. simulator / unusual device configurations). - return AVCaptureDevice.default(for: .video) + try CameraCapturePipelineSupport.selectCamera( + deviceId: deviceId, + matching: { deviceId in + self.discoverVideoDevices().first { $0.uniqueID == deviceId } + }, + fallback: { + let position: AVCaptureDevice.Position = facing == .front ? .front : .back + return AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: position) ?? + AVCaptureDevice.default(for: .video) + }, + unavailableError: CameraError.cameraUnavailable, + deviceNotFoundError: { + CameraError.invalidParams( + "INVALID_REQUEST: camera device not found: \($0); " + + "run camera.list for current device IDs") + }) } private nonisolated static func mapMovieSetupError(_ setupError: CameraSessionConfigurationError) -> CameraError { diff --git a/apps/macos/Sources/OpenClaw/CameraCaptureService.swift b/apps/macos/Sources/OpenClaw/CameraCaptureService.swift index dc47fd0bc5f8..3505d91c55bb 100644 --- a/apps/macos/Sources/OpenClaw/CameraCaptureService.swift +++ b/apps/macos/Sources/OpenClaw/CameraCaptureService.swift @@ -68,9 +68,8 @@ actor CameraCaptureService { preferFrontCamera: facing == .front, deviceId: deviceId, pickCamera: { preferFrontCamera, deviceId in - Self.pickCamera(facing: preferFrontCamera ? .front : .back, deviceId: deviceId) + try Self.pickCamera(facing: preferFrontCamera ? .front : .back, deviceId: deviceId) }, - cameraUnavailableError: CameraError.cameraUnavailable, mapSetupError: { setupError in CameraError.captureFailed(setupError.localizedDescription) }) @@ -130,9 +129,8 @@ actor CameraCaptureService { includeAudio: includeAudio, durationMs: durationMs), pickCamera: { preferFrontCamera, deviceId in - Self.pickCamera(facing: preferFrontCamera ? .front : .back, deviceId: deviceId) + try Self.pickCamera(facing: preferFrontCamera ? .front : .back, deviceId: deviceId) }, - cameraUnavailableError: CameraError.cameraUnavailable, mapSetupError: Self.mapMovieSetupError) let session = prepared.session let output = prepared.output @@ -172,21 +170,19 @@ actor CameraCaptureService { private nonisolated static func pickCamera( facing: CameraFacing, - deviceId: String?) -> AVCaptureDevice? + deviceId: String?) throws -> AVCaptureDevice { - if let deviceId, !deviceId.isEmpty { - if let match = CameraDeviceResolver.camera(deviceId: deviceId) { - return match - } - } - let position: AVCaptureDevice.Position = (facing == .front) ? .front : .back - - if let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: position) { - return device - } - - // Many macOS cameras report `unspecified` position; fall back to any default. - return AVCaptureDevice.default(for: .video) + try CameraCapturePipelineSupport.selectCamera( + deviceId: deviceId, + matching: CameraDeviceResolver.camera, + fallback: { + let position: AVCaptureDevice.Position = facing == .front ? .front : .back + // Many macOS cameras report `unspecified` position; fall back only without an explicit device. + return AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: position) ?? + AVCaptureDevice.default(for: .video) + }, + unavailableError: CameraError.cameraUnavailable, + deviceNotFoundError: { CameraPTZError.deviceNotFound($0) }) } private nonisolated static func clampQuality(_ quality: Double?) -> Double { diff --git a/apps/macos/Sources/OpenClaw/CameraPTZNativeController.swift b/apps/macos/Sources/OpenClaw/CameraPTZNativeController.swift index 937d4e30a686..cfa1b48ed0b1 100644 --- a/apps/macos/Sources/OpenClaw/CameraPTZNativeController.swift +++ b/apps/macos/Sources/OpenClaw/CameraPTZNativeController.swift @@ -1,3 +1,4 @@ +import AVFoundation import Darwin import Foundation import OpenClawCameraPTZNative @@ -54,6 +55,32 @@ struct NativeCameraPTZBackend: CameraPTZBackend { func open(deviceId: String) throws -> any CameraPTZControlling { try NativeCameraPTZController(deviceId: deviceId) } + + func withCaptureSession(deviceId: String, body: () throws -> T) throws -> T { + guard let device = CameraDeviceResolver.camera(deviceId: deviceId) else { + throw CameraPTZError.deviceNotFound(deviceId) + } + let session = AVCaptureSession() + let input = try AVCaptureDeviceInput(device: device) + guard session.canAddInput(input) else { + throw CameraPTZError.unsupported("camera cannot start a video stream") + } + session.addInput(input) + let output = AVCaptureVideoDataOutput() + guard session.canAddOutput(output) else { + throw CameraPTZError.unsupported("camera cannot provide a video stream") + } + session.addOutput(output) + + // UVC controls require a live video stream, which briefly lights the camera privacy indicator. + // Without a sample-buffer delegate, video frames are neither delivered nor retained. + session.startRunning() + defer { session.stopRunning() } + guard session.isRunning else { + throw CameraPTZError.unsupported("camera video stream did not start") + } + return try body() + } } private final class NativeCameraPTZController: CameraPTZControlling { diff --git a/apps/macos/Sources/OpenClaw/CameraPTZService.swift b/apps/macos/Sources/OpenClaw/CameraPTZService.swift index add831190a71..5b0520db787b 100644 --- a/apps/macos/Sources/OpenClaw/CameraPTZService.swift +++ b/apps/macos/Sources/OpenClaw/CameraPTZService.swift @@ -49,7 +49,7 @@ enum CameraPTZError: LocalizedError, Equatable { case let .invalidRequest(message): "INVALID_REQUEST: \(message)" case let .deviceNotFound(deviceId): - "CAMERA_DEVICE_NOT_FOUND: \(deviceId)" + "CAMERA_DEVICE_NOT_FOUND: \(deviceId); run camera.list for current device IDs" case let .unsupported(message): "CAMERA_PTZ_UNSUPPORTED: \(message)" case let .axisUnsupported(axis): @@ -136,6 +136,7 @@ protocol CameraPTZControlling: AnyObject { protocol CameraPTZBackend: Sendable { func open(deviceId: String) throws -> any CameraPTZControlling + func withCaptureSession(deviceId: String, body: () throws -> T) throws -> T } actor CameraPTZService: CameraPTZServicing { @@ -160,28 +161,32 @@ actor CameraPTZService: CameraPTZServicing { func status(deviceId: String) throws -> CameraPTZStatusResponse { let deviceId = try self.resolveDeviceId(deviceId) - return try self.withController(deviceId: deviceId) { controller in - try Self.makeStatusResponse(deviceId: deviceId, raw: controller.status()) + return try self.backend.withCaptureSession(deviceId: deviceId) { + try self.withController(deviceId: deviceId) { controller in + try Self.makeStatusResponse(deviceId: deviceId, raw: controller.status()) + } } } func control(_ params: OpenClawCameraPTZControlParams) throws -> CameraPTZControlResponse { let deviceId = try self.resolveDeviceId(params.deviceId) let axes = try Self.validateControl(params) - return try self.withController(deviceId: deviceId) { controller in - let status = try Self.executableStatus(controller.status()) - let plan = switch params.operation { - case .home: try Self.planHome(status: status) - case .set, .move: try Self.planMotion( - status: status, - operation: params.operation, - axes: axes) + return try self.backend.withCaptureSession(deviceId: deviceId) { + try self.withController(deviceId: deviceId) { controller in + let status = try Self.executableStatus(controller.status()) + let plan = switch params.operation { + case .home: try Self.planHome(status: status) + case .set, .move: try Self.planMotion( + status: status, + operation: params.operation, + axes: axes) + } + return try self.execute( + plan: plan, + controller: controller, + deviceId: deviceId, + operation: params.operation) } - return try Self.execute( - plan: plan, - controller: controller, - deviceId: deviceId, - operation: params.operation) } } @@ -340,7 +345,7 @@ actor CameraPTZService: CameraPTZServicing { operation == .move ? current + value : value } - private static func execute( + private func execute( plan: WritePlan, controller: any CameraPTZControlling, deviceId: String, @@ -361,28 +366,72 @@ actor CameraPTZService: CameraPTZServicing { } } catch { guard !applied.isEmpty else { throw error } - throw self.partialError(applied: applied, controller: controller, failure: error) + controller.close() + throw self.partialError(applied: applied, deviceId: deviceId, failure: error) } + // A writing UVC connection can echo its pending setpoint instead of the committed camera position. + controller.close() let finalStatus: CameraPTZRawStatus + var statusFailure: Error? do { - finalStatus = try self.executableStatus(controller.status()) + finalStatus = try self.withController(deviceId: deviceId) { + do { + return try Self.executableStatus($0.status()) + } catch { + statusFailure = error + throw error + } + } } catch { - throw self.partialError(applied: applied, controller: controller, failure: error) + throw self.partialError(applied: applied, deviceId: deviceId, failure: statusFailure ?? error) + } + + let requestedAxes: [(String, Int32?, CameraPTZRawAxisStatus?)] = [ + ("panDegrees", plan.panTilt?.pan, finalStatus.pan), + ("tiltDegrees", plan.panTilt?.tilt, finalStatus.tilt), + ("zoomPercent", plan.zoom, finalStatus.zoom), + ] + var mismatches: [String] = [] + for (name, target, axis) in requestedAxes { + guard let target else { continue } + guard let axis else { + mismatches.append("\(name) requested=\(target) observed=unavailable") + continue + } + guard abs(Int64(axis.current) - Int64(target)) > Int64(max(0, axis.range.step)) else { + continue + } + let requested = name == "zoomPercent" + ? axis.range.percent(of: target) + : Self.arcsecondsToDegrees(target) + let observed = name == "zoomPercent" + ? axis.range.percent(of: axis.current) + : Self.arcsecondsToDegrees(axis.current) + mismatches.append("\(name) requested=\(requested) observed=\(observed)") + } + guard mismatches.isEmpty else { + throw CameraPTZError.partial( + applied: applied, + state: Self.makeState(finalStatus), + failure: mismatches.joined(separator: "; ") + + "; confirm a video stream reaches the camera and disable on-camera AI framing/tracking") } return CameraPTZControlResponse( deviceId: deviceId, operation: operation, - state: self.makeState(finalStatus), + state: Self.makeState(finalStatus), adjusted: plan.adjusted) } - private static func partialError( + private func partialError( applied: [String], - controller: any CameraPTZControlling, + deviceId: String, failure: Error) -> CameraPTZError { - let state = try? self.makeState(self.executableStatus(controller.status())) + let state = try? self.withController(deviceId: deviceId) { + try Self.makeState(Self.executableStatus($0.status())) + } return .partial( applied: applied, state: state, diff --git a/apps/macos/Tests/OpenClawIPCTests/CameraPTZServiceTests.swift b/apps/macos/Tests/OpenClawIPCTests/CameraPTZServiceTests.swift index 1b55ca67b1bb..7a4b2dbeeae8 100644 --- a/apps/macos/Tests/OpenClawIPCTests/CameraPTZServiceTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/CameraPTZServiceTests.swift @@ -35,7 +35,7 @@ struct CameraPTZServiceTests { } } - private final class FakeController: CameraPTZControlling, @unchecked Sendable { + private final class FakeDevice: @unchecked Sendable { var rawStatus: CameraPTZRawStatus var statusHook: (() -> Void)? var panTiltHook: (() -> Void)? @@ -45,48 +45,168 @@ struct CameraPTZServiceTests { var panTiltError: Error? var zoomError: Error? var statusFailureCalls: Set = [] - private var statusCalls = 0 + var statusCalls = 0 + var openCount = 0 + var acceptsWrites = true + var panLandingOffset: Int32 = 0 + var events: [String] = [] init(status: CameraPTZRawStatus) { self.rawStatus = status } + } + + private final class FakeController: CameraPTZControlling, @unchecked Sendable { + private let device: FakeDevice + private let connection: Int + private var pendingPanTilt: (pan: Int32, tilt: Int32)? + private var pendingZoom: Int32? + private var closed = false + + var statusHook: (() -> Void)? { + get { self.device.statusHook } + set { self.device.statusHook = newValue } + } + + var panTiltHook: (() -> Void)? { + get { self.device.panTiltHook } + set { self.device.panTiltHook = newValue } + } + + var panTiltWrites: [(Int32, Int32)] { + self.device.panTiltWrites + } + + var zoomWrites: [Int32] { + self.device.zoomWrites + } + + var closeCount: Int { + self.device.closeCount + } + + var events: [String] { + self.device.events + } + + var acceptsWrites: Bool { + get { self.device.acceptsWrites } + set { self.device.acceptsWrites = newValue } + } + + var panLandingOffset: Int32 { + get { self.device.panLandingOffset } + set { self.device.panLandingOffset = newValue } + } + + var panTiltError: Error? { + get { self.device.panTiltError } + set { self.device.panTiltError = newValue } + } + + var zoomError: Error? { + get { self.device.zoomError } + set { self.device.zoomError = newValue } + } + + var statusFailureCalls: Set { + get { self.device.statusFailureCalls } + set { self.device.statusFailureCalls = newValue } + } + + init(status: CameraPTZRawStatus) { + self.device = FakeDevice(status: status) + self.connection = 0 + } + + private init(device: FakeDevice, connection: Int) { + self.device = device + self.connection = connection + } + + func openConnection() -> FakeController { + self.device.openCount += 1 + let connection = self.device.openCount + self.device.events.append("open:\(connection)") + return FakeController(device: self.device, connection: connection) + } + + func withCaptureSession(_ body: () throws -> T) rethrows -> T { + self.device.events.append("session:start") + defer { self.device.events.append("session:stop") } + return try body() + } func status() throws -> CameraPTZRawStatus { - self.statusHook?() - self.statusCalls += 1 - if self.statusFailureCalls.remove(self.statusCalls) != nil { + self.device.statusHook?() + self.device.statusCalls += 1 + self.device.events.append("status:\(self.connection)") + if self.device.statusFailureCalls.remove(self.device.statusCalls) != nil { throw FakeFailure.status } - return self.rawStatus + let committed = self.device.rawStatus + return CameraPTZRawStatus( + pan: committed.pan.map { + CameraPTZRawAxisStatus( + current: self.pendingPanTilt?.pan ?? $0.current, + range: $0.range, + canSet: $0.canSet) + }, + tilt: committed.tilt.map { + CameraPTZRawAxisStatus( + current: self.pendingPanTilt?.tilt ?? $0.current, + range: $0.range, + canSet: $0.canSet) + }, + zoom: committed.zoom.map { + CameraPTZRawAxisStatus( + current: self.pendingZoom ?? $0.current, + range: $0.range, + canSet: $0.canSet) + }) } func setPanTilt(pan: Int32, tilt: Int32) throws { - if let panTiltError { throw panTiltError } - self.panTiltHook?() - self.panTiltWrites.append((pan, tilt)) - self.rawStatus = CameraPTZRawStatus( - pan: self.rawStatus.pan.map { - CameraPTZRawAxisStatus(current: pan, range: $0.range, canSet: $0.canSet) + if let panTiltError = self.device.panTiltError { throw panTiltError } + self.device.panTiltHook?() + self.device.panTiltWrites.append((pan, tilt)) + self.device.events.append("write:panTilt:\(self.connection)") + self.pendingPanTilt = (pan, tilt) + guard self.device.acceptsWrites else { return } + let committed = self.device.rawStatus + self.device.rawStatus = CameraPTZRawStatus( + pan: committed.pan.map { + CameraPTZRawAxisStatus( + current: pan + self.device.panLandingOffset, + range: $0.range, + canSet: $0.canSet) }, - tilt: self.rawStatus.tilt.map { + tilt: committed.tilt.map { CameraPTZRawAxisStatus(current: tilt, range: $0.range, canSet: $0.canSet) }, - zoom: self.rawStatus.zoom) + zoom: committed.zoom) } func setZoom(_ zoom: Int32) throws { - if let zoomError { throw zoomError } - self.zoomWrites.append(zoom) - self.rawStatus = CameraPTZRawStatus( - pan: self.rawStatus.pan, - tilt: self.rawStatus.tilt, - zoom: self.rawStatus.zoom.map { + if let zoomError = self.device.zoomError { throw zoomError } + self.device.zoomWrites.append(zoom) + self.device.events.append("write:zoom:\(self.connection)") + self.pendingZoom = zoom + guard self.device.acceptsWrites else { return } + let committed = self.device.rawStatus + self.device.rawStatus = CameraPTZRawStatus( + pan: committed.pan, + tilt: committed.tilt, + zoom: committed.zoom.map { CameraPTZRawAxisStatus(current: zoom, range: $0.range, canSet: $0.canSet) }) } func close() { - self.closeCount += 1 + guard !self.closed else { return } + self.closed = true + self.device.closeCount += 1 + self.device.events.append("close:\(self.connection)") } } @@ -94,7 +214,11 @@ struct CameraPTZServiceTests { let controller: FakeController func open(deviceId _: String) -> any CameraPTZControlling { - self.controller + self.controller.openConnection() + } + + func withCaptureSession(deviceId _: String, body: () throws -> T) rethrows -> T { + try self.controller.withCaptureSession(body) } } @@ -181,6 +305,36 @@ struct CameraPTZServiceTests { #expect(range.value(percent: 200) == 500) } + @Test func `unknown explicit capture device never falls back to another camera`() { + var selectedFallback = false + + #expect(throws: CameraPTZError.deviceNotFound("missing")) { + try CameraCapturePipelineSupport.selectCamera( + deviceId: "missing", + matching: { _ in nil as String? }, + fallback: { + selectedFallback = true + return "different-camera" + }, + unavailableError: CameraCaptureService.CameraError.cameraUnavailable, + deviceNotFoundError: { CameraPTZError.deviceNotFound($0) }) + } + #expect(!selectedFallback) + } + + @Test func `capture without an explicit device preserves facing camera fallback`() throws { + for deviceId in [nil, ""] as [String?] { + let selected = try CameraCapturePipelineSupport.selectCamera( + deviceId: deviceId, + matching: { _ in nil as String? }, + fallback: { "facing-camera" }, + unavailableError: CameraCaptureService.CameraError.cameraUnavailable, + deviceNotFoundError: { CameraPTZError.deviceNotFound($0) }) + + #expect(selected == "facing-camera") + } + } + @Test func `USB identity uses AVFoundation packed identifier`() throws { let identity = try CameraUSBIdentity.parse(deviceId: "0x21100002e1a4c06") @@ -265,6 +419,67 @@ struct CameraPTZServiceTests { #expect(controller.closeCount == 1) } + @Test func `status keeps the capture stream active around all UVC access`() async throws { + let controller = self.makeController() + + _ = try await self.makeService(controller).status(deviceId: "camera-id") + + #expect(controller.events == [ + "session:start", "open:1", "status:1", "close:1", "session:stop", + ]) + } + + @Test func `control closes its writer before verifying on a fresh connection`() async throws { + let controller = self.makeController() + + let response = try await self.makeService(controller).control(OpenClawCameraPTZControlParams( + deviceId: "camera-id", + operation: .set, + target: OpenClawCameraPTZAxisValues(panDegrees: 5))) + + #expect(response.state.panDegrees == 5) + #expect(controller.events == [ + "session:start", "open:1", "status:1", "write:panTilt:1", "close:1", + "open:2", "status:2", "close:2", "session:stop", + ]) + } + + @Test func `connection local setpoint echoes never turn ignored motion into success`() async { + let controller = self.makeController() + controller.acceptsWrites = false + + do { + _ = try await self.makeService(controller).control(OpenClawCameraPTZControlParams( + deviceId: "camera-id", + operation: .set, + target: OpenClawCameraPTZAxisValues(panDegrees: 5, zoomPercent: 75))) + Issue.record("the writer echoed ignored setpoints as a successful physical move") + } catch let CameraPTZError.partial(applied, state, failure) { + #expect(applied == ["panTilt", "zoom"]) + #expect(state == CameraPTZState(panDegrees: 0, tiltDegrees: 0, zoomPercent: 50)) + #expect(failure.contains("panDegrees requested=5.0 observed=0.0")) + #expect(failure.contains("zoomPercent requested=75.0 observed=50.0")) + #expect(failure.contains("video stream")) + #expect(failure.contains("AI framing/tracking")) + #expect(controller.events.contains("open:2")) + #expect(controller.events.last == "session:stop") + } catch { + Issue.record("unexpected error: \(error)") + } + } + + @Test func `verified motion within one advertised axis step still succeeds`() async throws { + let controller = self.makeController() + controller.panLandingOffset = 1800 + + let response = try await self.makeService(controller).control(OpenClawCameraPTZControlParams( + deviceId: "camera-id", + operation: .set, + target: OpenClawCameraPTZAxisValues(panDegrees: 5))) + + #expect(response.state.panDegrees == 5.5) + } + @Test func `readable axes without SET capability are not exposed or writable`() async throws { let controller = self.makeController(canSet: false) let service = self.makeService(controller) diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/CameraCapturePipelineSupport.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/CameraCapturePipelineSupport.swift index bf5693a8bdc7..45e1aac3b3fa 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawKit/CameraCapturePipelineSupport.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/CameraCapturePipelineSupport.swift @@ -22,20 +22,36 @@ public struct CameraMovieSessionOptions: Sendable { } public enum CameraCapturePipelineSupport { + public static func selectCamera( + deviceId: String?, + matching: (String) -> Device?, + fallback: () -> Device?, + unavailableError: @autoclosure () -> Error, + deviceNotFoundError: (String) -> Error) throws -> Device + { + if let deviceId, !deviceId.isEmpty { + guard let device = matching(deviceId) else { + throw deviceNotFoundError(deviceId) + } + return device + } + guard let device = fallback() else { + throw unavailableError() + } + return device + } + public static func preparePhotoSession( preferFrontCamera: Bool, deviceId: String?, - pickCamera: (_ preferFrontCamera: Bool, _ deviceId: String?) -> AVCaptureDevice?, - cameraUnavailableError: @autoclosure () -> Error, + pickCamera: (_ preferFrontCamera: Bool, _ deviceId: String?) throws -> AVCaptureDevice, mapSetupError: (CameraSessionConfigurationError) -> Error) throws -> (session: AVCaptureSession, device: AVCaptureDevice, output: AVCapturePhotoOutput) { let session = AVCaptureSession() session.sessionPreset = .photo - guard let device = pickCamera(preferFrontCamera, deviceId) else { - throw cameraUnavailableError() - } + let device = try pickCamera(preferFrontCamera, deviceId) do { try CameraSessionConfiguration.addCameraInput(session: session, camera: device) @@ -48,17 +64,14 @@ public enum CameraCapturePipelineSupport { public static func prepareMovieSession( options: CameraMovieSessionOptions, - pickCamera: (_ preferFrontCamera: Bool, _ deviceId: String?) -> AVCaptureDevice?, - cameraUnavailableError: @autoclosure () -> Error, + pickCamera: (_ preferFrontCamera: Bool, _ deviceId: String?) throws -> AVCaptureDevice, mapSetupError: (CameraSessionConfigurationError) -> Error) throws -> (session: AVCaptureSession, output: AVCaptureMovieFileOutput) { let session = AVCaptureSession() session.sessionPreset = .high - guard let camera = pickCamera(options.preferFrontCamera, options.deviceId) else { - throw cameraUnavailableError() - } + let camera = try pickCamera(options.preferFrontCamera, options.deviceId) do { try CameraSessionConfiguration.addCameraInput(session: session, camera: camera) @@ -74,8 +87,7 @@ public enum CameraCapturePipelineSupport { public static func prepareWarmMovieSession( options: CameraMovieSessionOptions, - pickCamera: (_ preferFrontCamera: Bool, _ deviceId: String?) -> AVCaptureDevice?, - cameraUnavailableError: @autoclosure () -> Error, + pickCamera: (_ preferFrontCamera: Bool, _ deviceId: String?) throws -> AVCaptureDevice, mapSetupError: (CameraSessionConfigurationError) -> Error) async throws -> (session: AVCaptureSession, output: AVCaptureMovieFileOutput) { @@ -83,7 +95,6 @@ public enum CameraCapturePipelineSupport { let prepared = try self.prepareMovieSession( options: options, pickCamera: pickCamera, - cameraUnavailableError: cameraUnavailableError(), mapSetupError: mapSetupError) try Task.checkCancellation() prepared.session.startRunning() @@ -99,8 +110,7 @@ public enum CameraCapturePipelineSupport { public static func withWarmMovieSession( options: CameraMovieSessionOptions, - pickCamera: (_ preferFrontCamera: Bool, _ deviceId: String?) -> AVCaptureDevice?, - cameraUnavailableError: @autoclosure () -> Error, + pickCamera: (_ preferFrontCamera: Bool, _ deviceId: String?) throws -> AVCaptureDevice, mapSetupError: (CameraSessionConfigurationError) -> Error, operation: (AVCaptureMovieFileOutput) async throws -> T) async throws -> T { @@ -108,7 +118,6 @@ public enum CameraCapturePipelineSupport { let prepared = try self.prepareMovieSession( options: options, pickCamera: pickCamera, - cameraUnavailableError: cameraUnavailableError(), mapSetupError: mapSetupError) return try await self.withCaptureSessionLifecycle( start: { prepared.session.startRunning() }, diff --git a/docs/nodes/camera.md b/docs/nodes/camera.md index 827445a17aa0..1474a1b04a0b 100644 --- a/docs/nodes/camera.md +++ b/docs/nodes/camera.md @@ -9,6 +9,8 @@ title: "Camera capture" OpenClaw supports camera capture for agent workflows on paired **iOS**, **Android**, **macOS**, and **Linux** nodes: capture a photo (`jpg`) or a short video clip (`mp4`, with optional audio) via Gateway `node.invoke`. +When a capture request includes `deviceId`, the selected camera must match that ID exactly. An unknown ID fails instead of capturing from a different camera; run `camera.list` to refresh device IDs, which change when cameras are reconnected. + The macOS app can also physically pan, tilt, and zoom supported USB UVC cameras. PTZ moves the camera hardware; it does not rotate, crop, or otherwise transform a captured image. All camera access is gated behind a user-controlled setting per platform. @@ -134,7 +136,7 @@ Physical PTZ is implemented by the Mac app for USB cameras that expose standard Always pass an explicit `deviceId` returned by `camera.list`. OpenClaw never chooses a default camera for physical movement. -- `camera.ptz.status` is a safe read command. Request: `{ "deviceId": "" }`. +- `camera.ptz.status` reads the current position without moving the camera. Request: `{ "deviceId": "" }`. - The response contains only executable `pan`, `tilt`, and `zoom` axes under `axes`. - Pan and tilt values are degrees. Zoom values are percentages. - Each axis reports `current`, `min`, `max`, `step`, `unit`, `canSet`, and `canMove`. `default` appears only when the camera successfully reports a device default. @@ -146,7 +148,9 @@ Always pass an explicit `deviceId` returned by `camera.list`. OpenClaw never cho `set` and `move` require at least one finite axis value. Omitted axes remain unchanged, and move deltas for zoom are percentage points. `home` restores the device-advertised defaults; it returns `CAMERA_PTZ_UNSUPPORTED` without moving the camera when `canHome` is false. The Mac app clamps and snaps requested values to the camera's range and resolution; the response returns the post-operation `state` and lists changed request fields in `adjusted`. Requesting an unsupported axis returns `CAMERA_PTZ_AXIS_UNSUPPORTED`. -Pan/tilt and zoom use separate hardware writes and cannot be atomic. If an earlier control group succeeds but a later write or final status read fails, `CAMERA_PTZ_PARTIAL` names the applied groups, includes best-effort resulting state when readable, and tells the caller to run `camera.ptz.status` before retrying. +Both PTZ commands briefly open a live camera stream because supported cameras only service UVC controls while streaming. This activates the camera and its privacy indicator for the duration, including when `camera.ptz.status` only reads the position. Frames are not retained, and no photo, video, or file is produced. + +Pan/tilt and zoom use separate hardware writes and cannot be atomic. OpenClaw verifies the resulting position through a fresh control connection. If a later write or final status read fails, or an axis does not reach its requested position within the camera's reported resolution, `CAMERA_PTZ_PARTIAL` names the acknowledged control groups, includes the independently observed state when readable, and tells the caller to run `camera.ptz.status` before retrying. Position failures also report the requested and observed values; check that a video stream reaches the camera and disable on-camera AI framing or tracking that can override UVC controls. `camera.ptz.control` is dangerous and remains disarmed until the operator explicitly adds it to `gateway.nodes.commands.allow`: diff --git a/docs/reference/AGENTS.default.md b/docs/reference/AGENTS.default.md index 13cb91116730..08772a0b5bd2 100644 --- a/docs/reference/AGENTS.default.md +++ b/docs/reference/AGENTS.default.md @@ -106,7 +106,7 @@ Example roster for a personal-assistant workspace; swap in whichever skills fit - **mcporter** - tool server runtime/CLI for managing external skill backends. - **Peekaboo** - fast macOS screenshots with optional AI vision analysis. -- **camsnap** - capture frames, clips, or motion alerts from RTSP/ONVIF security cams. +- **camsnap** - capture frames, clips, or motion alerts from RTSP/ONVIF security cams and local webcams, including USB pan/tilt/zoom control. - **oracle** - OpenAI-ready agent CLI with session replay and browser control. - **eightctl** - control your sleep, from the terminal. - **imsg** - send, read, stream iMessage & SMS. diff --git a/skills/camsnap/SKILL.md b/skills/camsnap/SKILL.md index f7ad9adfb871..1284829fd2fc 100644 --- a/skills/camsnap/SKILL.md +++ b/skills/camsnap/SKILL.md @@ -1,6 +1,6 @@ --- name: camsnap -description: "Capture frames or clips from RTSP/ONVIF cameras." +description: "Capture frames or clips from RTSP/ONVIF cameras and local webcams, including USB pan/tilt/zoom control." homepage: https://camsnap.ai metadata: { @@ -39,7 +39,17 @@ Common commands - Motion watch: `camsnap watch kitchen --threshold 0.2 --action '...'` - Doctor: `camsnap doctor --probe` +Local webcams (macOS) + +- List devices: `camsnap devices` +- Snapshot from a local camera: `camsnap snap --device 0 --out webcam.jpg` +- PTZ position and ranges: `camsnap ptz status --device 0` +- Move a gimbal camera: `camsnap ptz goto --device 0 --pan 45 --tilt -18` +- Also `camsnap ptz move` (relative deltas) and `camsnap ptz home`. + Notes - Requires `ffmpeg` on PATH. - Prefer a short test capture before longer clips. +- PTZ needs macOS Camera permission: these cameras only service UVC controls while streaming, so `camsnap` holds a capture session open for the operation. +- Motion commands verify the settled position and exit non-zero if the camera did not reach it. On-camera AI framing can override manual positioning; disable tracking if moves keep missing.