diff --git a/apps/ios/Sources/Camera/CameraController.swift b/apps/ios/Sources/Camera/CameraController.swift index fc9bc163193d..f293369dc333 100644 --- a/apps/ios/Sources/Camera/CameraController.swift +++ b/apps/ios/Sources/Camera/CameraController.swift @@ -61,9 +61,8 @@ actor CameraController { preferFrontCamera: facing == .front, deviceId: params.deviceId, pickCamera: { preferFrontCamera, deviceId in - Self.pickCamera(facing: preferFrontCamera ? .front : .back, deviceId: deviceId) + try Self.pickCamera(facing: preferFrontCamera ? .front : .back, deviceId: deviceId) }, - cameraUnavailableError: CameraError.cameraUnavailable, mapSetupError: { setupError in CameraError.captureFailed(setupError.localizedDescription) }) @@ -141,9 +140,8 @@ actor CameraController { includeAudio: includeAudio, durationMs: durationMs), pickCamera: { preferFrontCamera, deviceId in - Self.pickCamera(facing: preferFrontCamera ? .front : .back, deviceId: deviceId) + try Self.pickCamera(facing: preferFrontCamera ? .front : .back, deviceId: deviceId) }, - cameraUnavailableError: CameraError.cameraUnavailable, mapSetupError: Self.mapMovieSetupError, operation: { output in let recording = CameraMovieRecordingOperation(output: output, outputURL: movURL) @@ -196,19 +194,24 @@ actor CameraController { private nonisolated static func pickCamera( facing: OpenClawCameraFacing, - deviceId: String?) -> AVCaptureDevice? + deviceId: String?) throws -> AVCaptureDevice { - if let deviceId, !deviceId.isEmpty { - if let match = discoverVideoDevices().first(where: { $0.uniqueID == deviceId }) { - return match - } - } - let position: AVCaptureDevice.Position = (facing == .front) ? .front : .back - if let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: position) { - return device - } - // Fall back to any default camera (e.g. simulator / unusual device configurations). - return AVCaptureDevice.default(for: .video) + try CameraCapturePipelineSupport.selectCamera( + deviceId: deviceId, + matching: { deviceId in + self.discoverVideoDevices().first { $0.uniqueID == deviceId } + }, + fallback: { + let position: AVCaptureDevice.Position = facing == .front ? .front : .back + return AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: position) ?? + AVCaptureDevice.default(for: .video) + }, + unavailableError: CameraError.cameraUnavailable, + deviceNotFoundError: { + CameraError.invalidParams( + "INVALID_REQUEST: camera device not found: \($0); " + + "run camera.list for current device IDs") + }) } private nonisolated static func mapMovieSetupError(_ setupError: CameraSessionConfigurationError) -> CameraError { diff --git a/apps/macos/Sources/OpenClaw/CameraCaptureService.swift b/apps/macos/Sources/OpenClaw/CameraCaptureService.swift index dc47fd0bc5f8..3505d91c55bb 100644 --- a/apps/macos/Sources/OpenClaw/CameraCaptureService.swift +++ b/apps/macos/Sources/OpenClaw/CameraCaptureService.swift @@ -68,9 +68,8 @@ actor CameraCaptureService { preferFrontCamera: facing == .front, deviceId: deviceId, pickCamera: { preferFrontCamera, deviceId in - Self.pickCamera(facing: preferFrontCamera ? .front : .back, deviceId: deviceId) + try Self.pickCamera(facing: preferFrontCamera ? .front : .back, deviceId: deviceId) }, - cameraUnavailableError: CameraError.cameraUnavailable, mapSetupError: { setupError in CameraError.captureFailed(setupError.localizedDescription) }) @@ -130,9 +129,8 @@ actor CameraCaptureService { includeAudio: includeAudio, durationMs: durationMs), pickCamera: { preferFrontCamera, deviceId in - Self.pickCamera(facing: preferFrontCamera ? .front : .back, deviceId: deviceId) + try Self.pickCamera(facing: preferFrontCamera ? .front : .back, deviceId: deviceId) }, - cameraUnavailableError: CameraError.cameraUnavailable, mapSetupError: Self.mapMovieSetupError) let session = prepared.session let output = prepared.output @@ -172,21 +170,19 @@ actor CameraCaptureService { private nonisolated static func pickCamera( facing: CameraFacing, - deviceId: String?) -> AVCaptureDevice? + deviceId: String?) throws -> AVCaptureDevice { - if let deviceId, !deviceId.isEmpty { - if let match = CameraDeviceResolver.camera(deviceId: deviceId) { - return match - } - } - let position: AVCaptureDevice.Position = (facing == .front) ? .front : .back - - if let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: position) { - return device - } - - // Many macOS cameras report `unspecified` position; fall back to any default. - return AVCaptureDevice.default(for: .video) + try CameraCapturePipelineSupport.selectCamera( + deviceId: deviceId, + matching: CameraDeviceResolver.camera, + fallback: { + let position: AVCaptureDevice.Position = facing == .front ? .front : .back + // Many macOS cameras report `unspecified` position; fall back only without an explicit device. + return AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: position) ?? + AVCaptureDevice.default(for: .video) + }, + unavailableError: CameraError.cameraUnavailable, + deviceNotFoundError: { CameraPTZError.deviceNotFound($0) }) } private nonisolated static func clampQuality(_ quality: Double?) -> Double { diff --git a/apps/macos/Sources/OpenClaw/CameraPTZNativeController.swift b/apps/macos/Sources/OpenClaw/CameraPTZNativeController.swift index 937d4e30a686..cfa1b48ed0b1 100644 --- a/apps/macos/Sources/OpenClaw/CameraPTZNativeController.swift +++ b/apps/macos/Sources/OpenClaw/CameraPTZNativeController.swift @@ -1,3 +1,4 @@ +import AVFoundation import Darwin import Foundation import OpenClawCameraPTZNative @@ -54,6 +55,32 @@ struct NativeCameraPTZBackend: CameraPTZBackend { func open(deviceId: String) throws -> any CameraPTZControlling { try NativeCameraPTZController(deviceId: deviceId) } + + func withCaptureSession(deviceId: String, body: () throws -> T) throws -> T { + guard let device = CameraDeviceResolver.camera(deviceId: deviceId) else { + throw CameraPTZError.deviceNotFound(deviceId) + } + let session = AVCaptureSession() + let input = try AVCaptureDeviceInput(device: device) + guard session.canAddInput(input) else { + throw CameraPTZError.unsupported("camera cannot start a video stream") + } + session.addInput(input) + let output = AVCaptureVideoDataOutput() + guard session.canAddOutput(output) else { + throw CameraPTZError.unsupported("camera cannot provide a video stream") + } + session.addOutput(output) + + // UVC controls require a live video stream, which briefly lights the camera privacy indicator. + // Without a sample-buffer delegate, video frames are neither delivered nor retained. + session.startRunning() + defer { session.stopRunning() } + guard session.isRunning else { + throw CameraPTZError.unsupported("camera video stream did not start") + } + return try body() + } } private final class NativeCameraPTZController: CameraPTZControlling { diff --git a/apps/macos/Sources/OpenClaw/CameraPTZService.swift b/apps/macos/Sources/OpenClaw/CameraPTZService.swift index add831190a71..5b0520db787b 100644 --- a/apps/macos/Sources/OpenClaw/CameraPTZService.swift +++ b/apps/macos/Sources/OpenClaw/CameraPTZService.swift @@ -49,7 +49,7 @@ enum CameraPTZError: LocalizedError, Equatable { case let .invalidRequest(message): "INVALID_REQUEST: \(message)" case let .deviceNotFound(deviceId): - "CAMERA_DEVICE_NOT_FOUND: \(deviceId)" + "CAMERA_DEVICE_NOT_FOUND: \(deviceId); run camera.list for current device IDs" case let .unsupported(message): "CAMERA_PTZ_UNSUPPORTED: \(message)" case let .axisUnsupported(axis): @@ -136,6 +136,7 @@ protocol CameraPTZControlling: AnyObject { protocol CameraPTZBackend: Sendable { func open(deviceId: String) throws -> any CameraPTZControlling + func withCaptureSession(deviceId: String, body: () throws -> T) throws -> T } actor CameraPTZService: CameraPTZServicing { @@ -160,28 +161,32 @@ actor CameraPTZService: CameraPTZServicing { func status(deviceId: String) throws -> CameraPTZStatusResponse { let deviceId = try self.resolveDeviceId(deviceId) - return try self.withController(deviceId: deviceId) { controller in - try Self.makeStatusResponse(deviceId: deviceId, raw: controller.status()) + return try self.backend.withCaptureSession(deviceId: deviceId) { + try self.withController(deviceId: deviceId) { controller in + try Self.makeStatusResponse(deviceId: deviceId, raw: controller.status()) + } } } func control(_ params: OpenClawCameraPTZControlParams) throws -> CameraPTZControlResponse { let deviceId = try self.resolveDeviceId(params.deviceId) let axes = try Self.validateControl(params) - return try self.withController(deviceId: deviceId) { controller in - let status = try Self.executableStatus(controller.status()) - let plan = switch params.operation { - case .home: try Self.planHome(status: status) - case .set, .move: try Self.planMotion( - status: status, - operation: params.operation, - axes: axes) + return try self.backend.withCaptureSession(deviceId: deviceId) { + try self.withController(deviceId: deviceId) { controller in + let status = try Self.executableStatus(controller.status()) + let plan = switch params.operation { + case .home: try Self.planHome(status: status) + case .set, .move: try Self.planMotion( + status: status, + operation: params.operation, + axes: axes) + } + return try self.execute( + plan: plan, + controller: controller, + deviceId: deviceId, + operation: params.operation) } - return try Self.execute( - plan: plan, - controller: controller, - deviceId: deviceId, - operation: params.operation) } } @@ -340,7 +345,7 @@ actor CameraPTZService: CameraPTZServicing { operation == .move ? current + value : value } - private static func execute( + private func execute( plan: WritePlan, controller: any CameraPTZControlling, deviceId: String, @@ -361,28 +366,72 @@ actor CameraPTZService: CameraPTZServicing { } } catch { guard !applied.isEmpty else { throw error } - throw self.partialError(applied: applied, controller: controller, failure: error) + controller.close() + throw self.partialError(applied: applied, deviceId: deviceId, failure: error) } + // A writing UVC connection can echo its pending setpoint instead of the committed camera position. + controller.close() let finalStatus: CameraPTZRawStatus + var statusFailure: Error? do { - finalStatus = try self.executableStatus(controller.status()) + finalStatus = try self.withController(deviceId: deviceId) { + do { + return try Self.executableStatus($0.status()) + } catch { + statusFailure = error + throw error + } + } } catch { - throw self.partialError(applied: applied, controller: controller, failure: error) + throw self.partialError(applied: applied, deviceId: deviceId, failure: statusFailure ?? error) + } + + let requestedAxes: [(String, Int32?, CameraPTZRawAxisStatus?)] = [ + ("panDegrees", plan.panTilt?.pan, finalStatus.pan), + ("tiltDegrees", plan.panTilt?.tilt, finalStatus.tilt), + ("zoomPercent", plan.zoom, finalStatus.zoom), + ] + var mismatches: [String] = [] + for (name, target, axis) in requestedAxes { + guard let target else { continue } + guard let axis else { + mismatches.append("\(name) requested=\(target) observed=unavailable") + continue + } + guard abs(Int64(axis.current) - Int64(target)) > Int64(max(0, axis.range.step)) else { + continue + } + let requested = name == "zoomPercent" + ? axis.range.percent(of: target) + : Self.arcsecondsToDegrees(target) + let observed = name == "zoomPercent" + ? axis.range.percent(of: axis.current) + : Self.arcsecondsToDegrees(axis.current) + mismatches.append("\(name) requested=\(requested) observed=\(observed)") + } + guard mismatches.isEmpty else { + throw CameraPTZError.partial( + applied: applied, + state: Self.makeState(finalStatus), + failure: mismatches.joined(separator: "; ") + + "; confirm a video stream reaches the camera and disable on-camera AI framing/tracking") } return CameraPTZControlResponse( deviceId: deviceId, operation: operation, - state: self.makeState(finalStatus), + state: Self.makeState(finalStatus), adjusted: plan.adjusted) } - private static func partialError( + private func partialError( applied: [String], - controller: any CameraPTZControlling, + deviceId: String, failure: Error) -> CameraPTZError { - let state = try? self.makeState(self.executableStatus(controller.status())) + let state = try? self.withController(deviceId: deviceId) { + try Self.makeState(Self.executableStatus($0.status())) + } return .partial( applied: applied, state: state, diff --git a/apps/macos/Tests/OpenClawIPCTests/CameraPTZServiceTests.swift b/apps/macos/Tests/OpenClawIPCTests/CameraPTZServiceTests.swift index 1b55ca67b1bb..7a4b2dbeeae8 100644 --- a/apps/macos/Tests/OpenClawIPCTests/CameraPTZServiceTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/CameraPTZServiceTests.swift @@ -35,7 +35,7 @@ struct CameraPTZServiceTests { } } - private final class FakeController: CameraPTZControlling, @unchecked Sendable { + private final class FakeDevice: @unchecked Sendable { var rawStatus: CameraPTZRawStatus var statusHook: (() -> Void)? var panTiltHook: (() -> Void)? @@ -45,48 +45,168 @@ struct CameraPTZServiceTests { var panTiltError: Error? var zoomError: Error? var statusFailureCalls: Set = [] - private var statusCalls = 0 + var statusCalls = 0 + var openCount = 0 + var acceptsWrites = true + var panLandingOffset: Int32 = 0 + var events: [String] = [] init(status: CameraPTZRawStatus) { self.rawStatus = status } + } + + private final class FakeController: CameraPTZControlling, @unchecked Sendable { + private let device: FakeDevice + private let connection: Int + private var pendingPanTilt: (pan: Int32, tilt: Int32)? + private var pendingZoom: Int32? + private var closed = false + + var statusHook: (() -> Void)? { + get { self.device.statusHook } + set { self.device.statusHook = newValue } + } + + var panTiltHook: (() -> Void)? { + get { self.device.panTiltHook } + set { self.device.panTiltHook = newValue } + } + + var panTiltWrites: [(Int32, Int32)] { + self.device.panTiltWrites + } + + var zoomWrites: [Int32] { + self.device.zoomWrites + } + + var closeCount: Int { + self.device.closeCount + } + + var events: [String] { + self.device.events + } + + var acceptsWrites: Bool { + get { self.device.acceptsWrites } + set { self.device.acceptsWrites = newValue } + } + + var panLandingOffset: Int32 { + get { self.device.panLandingOffset } + set { self.device.panLandingOffset = newValue } + } + + var panTiltError: Error? { + get { self.device.panTiltError } + set { self.device.panTiltError = newValue } + } + + var zoomError: Error? { + get { self.device.zoomError } + set { self.device.zoomError = newValue } + } + + var statusFailureCalls: Set { + get { self.device.statusFailureCalls } + set { self.device.statusFailureCalls = newValue } + } + + init(status: CameraPTZRawStatus) { + self.device = FakeDevice(status: status) + self.connection = 0 + } + + private init(device: FakeDevice, connection: Int) { + self.device = device + self.connection = connection + } + + func openConnection() -> FakeController { + self.device.openCount += 1 + let connection = self.device.openCount + self.device.events.append("open:\(connection)") + return FakeController(device: self.device, connection: connection) + } + + func withCaptureSession(_ body: () throws -> T) rethrows -> T { + self.device.events.append("session:start") + defer { self.device.events.append("session:stop") } + return try body() + } func status() throws -> CameraPTZRawStatus { - self.statusHook?() - self.statusCalls += 1 - if self.statusFailureCalls.remove(self.statusCalls) != nil { + self.device.statusHook?() + self.device.statusCalls += 1 + self.device.events.append("status:\(self.connection)") + if self.device.statusFailureCalls.remove(self.device.statusCalls) != nil { throw FakeFailure.status } - return self.rawStatus + let committed = self.device.rawStatus + return CameraPTZRawStatus( + pan: committed.pan.map { + CameraPTZRawAxisStatus( + current: self.pendingPanTilt?.pan ?? $0.current, + range: $0.range, + canSet: $0.canSet) + }, + tilt: committed.tilt.map { + CameraPTZRawAxisStatus( + current: self.pendingPanTilt?.tilt ?? $0.current, + range: $0.range, + canSet: $0.canSet) + }, + zoom: committed.zoom.map { + CameraPTZRawAxisStatus( + current: self.pendingZoom ?? $0.current, + range: $0.range, + canSet: $0.canSet) + }) } func setPanTilt(pan: Int32, tilt: Int32) throws { - if let panTiltError { throw panTiltError } - self.panTiltHook?() - self.panTiltWrites.append((pan, tilt)) - self.rawStatus = CameraPTZRawStatus( - pan: self.rawStatus.pan.map { - CameraPTZRawAxisStatus(current: pan, range: $0.range, canSet: $0.canSet) + if let panTiltError = self.device.panTiltError { throw panTiltError } + self.device.panTiltHook?() + self.device.panTiltWrites.append((pan, tilt)) + self.device.events.append("write:panTilt:\(self.connection)") + self.pendingPanTilt = (pan, tilt) + guard self.device.acceptsWrites else { return } + let committed = self.device.rawStatus + self.device.rawStatus = CameraPTZRawStatus( + pan: committed.pan.map { + CameraPTZRawAxisStatus( + current: pan + self.device.panLandingOffset, + range: $0.range, + canSet: $0.canSet) }, - tilt: self.rawStatus.tilt.map { + tilt: committed.tilt.map { CameraPTZRawAxisStatus(current: tilt, range: $0.range, canSet: $0.canSet) }, - zoom: self.rawStatus.zoom) + zoom: committed.zoom) } func setZoom(_ zoom: Int32) throws { - if let zoomError { throw zoomError } - self.zoomWrites.append(zoom) - self.rawStatus = CameraPTZRawStatus( - pan: self.rawStatus.pan, - tilt: self.rawStatus.tilt, - zoom: self.rawStatus.zoom.map { + if let zoomError = self.device.zoomError { throw zoomError } + self.device.zoomWrites.append(zoom) + self.device.events.append("write:zoom:\(self.connection)") + self.pendingZoom = zoom + guard self.device.acceptsWrites else { return } + let committed = self.device.rawStatus + self.device.rawStatus = CameraPTZRawStatus( + pan: committed.pan, + tilt: committed.tilt, + zoom: committed.zoom.map { CameraPTZRawAxisStatus(current: zoom, range: $0.range, canSet: $0.canSet) }) } func close() { - self.closeCount += 1 + guard !self.closed else { return } + self.closed = true + self.device.closeCount += 1 + self.device.events.append("close:\(self.connection)") } } @@ -94,7 +214,11 @@ struct CameraPTZServiceTests { let controller: FakeController func open(deviceId _: String) -> any CameraPTZControlling { - self.controller + self.controller.openConnection() + } + + func withCaptureSession(deviceId _: String, body: () throws -> T) rethrows -> T { + try self.controller.withCaptureSession(body) } } @@ -181,6 +305,36 @@ struct CameraPTZServiceTests { #expect(range.value(percent: 200) == 500) } + @Test func `unknown explicit capture device never falls back to another camera`() { + var selectedFallback = false + + #expect(throws: CameraPTZError.deviceNotFound("missing")) { + try CameraCapturePipelineSupport.selectCamera( + deviceId: "missing", + matching: { _ in nil as String? }, + fallback: { + selectedFallback = true + return "different-camera" + }, + unavailableError: CameraCaptureService.CameraError.cameraUnavailable, + deviceNotFoundError: { CameraPTZError.deviceNotFound($0) }) + } + #expect(!selectedFallback) + } + + @Test func `capture without an explicit device preserves facing camera fallback`() throws { + for deviceId in [nil, ""] as [String?] { + let selected = try CameraCapturePipelineSupport.selectCamera( + deviceId: deviceId, + matching: { _ in nil as String? }, + fallback: { "facing-camera" }, + unavailableError: CameraCaptureService.CameraError.cameraUnavailable, + deviceNotFoundError: { CameraPTZError.deviceNotFound($0) }) + + #expect(selected == "facing-camera") + } + } + @Test func `USB identity uses AVFoundation packed identifier`() throws { let identity = try CameraUSBIdentity.parse(deviceId: "0x21100002e1a4c06") @@ -265,6 +419,67 @@ struct CameraPTZServiceTests { #expect(controller.closeCount == 1) } + @Test func `status keeps the capture stream active around all UVC access`() async throws { + let controller = self.makeController() + + _ = try await self.makeService(controller).status(deviceId: "camera-id") + + #expect(controller.events == [ + "session:start", "open:1", "status:1", "close:1", "session:stop", + ]) + } + + @Test func `control closes its writer before verifying on a fresh connection`() async throws { + let controller = self.makeController() + + let response = try await self.makeService(controller).control(OpenClawCameraPTZControlParams( + deviceId: "camera-id", + operation: .set, + target: OpenClawCameraPTZAxisValues(panDegrees: 5))) + + #expect(response.state.panDegrees == 5) + #expect(controller.events == [ + "session:start", "open:1", "status:1", "write:panTilt:1", "close:1", + "open:2", "status:2", "close:2", "session:stop", + ]) + } + + @Test func `connection local setpoint echoes never turn ignored motion into success`() async { + let controller = self.makeController() + controller.acceptsWrites = false + + do { + _ = try await self.makeService(controller).control(OpenClawCameraPTZControlParams( + deviceId: "camera-id", + operation: .set, + target: OpenClawCameraPTZAxisValues(panDegrees: 5, zoomPercent: 75))) + Issue.record("the writer echoed ignored setpoints as a successful physical move") + } catch let CameraPTZError.partial(applied, state, failure) { + #expect(applied == ["panTilt", "zoom"]) + #expect(state == CameraPTZState(panDegrees: 0, tiltDegrees: 0, zoomPercent: 50)) + #expect(failure.contains("panDegrees requested=5.0 observed=0.0")) + #expect(failure.contains("zoomPercent requested=75.0 observed=50.0")) + #expect(failure.contains("video stream")) + #expect(failure.contains("AI framing/tracking")) + #expect(controller.events.contains("open:2")) + #expect(controller.events.last == "session:stop") + } catch { + Issue.record("unexpected error: \(error)") + } + } + + @Test func `verified motion within one advertised axis step still succeeds`() async throws { + let controller = self.makeController() + controller.panLandingOffset = 1800 + + let response = try await self.makeService(controller).control(OpenClawCameraPTZControlParams( + deviceId: "camera-id", + operation: .set, + target: OpenClawCameraPTZAxisValues(panDegrees: 5))) + + #expect(response.state.panDegrees == 5.5) + } + @Test func `readable axes without SET capability are not exposed or writable`() async throws { let controller = self.makeController(canSet: false) let service = self.makeService(controller) diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/CameraCapturePipelineSupport.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/CameraCapturePipelineSupport.swift index bf5693a8bdc7..45e1aac3b3fa 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawKit/CameraCapturePipelineSupport.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/CameraCapturePipelineSupport.swift @@ -22,20 +22,36 @@ public struct CameraMovieSessionOptions: Sendable { } public enum CameraCapturePipelineSupport { + public static func selectCamera( + deviceId: String?, + matching: (String) -> Device?, + fallback: () -> Device?, + unavailableError: @autoclosure () -> Error, + deviceNotFoundError: (String) -> Error) throws -> Device + { + if let deviceId, !deviceId.isEmpty { + guard let device = matching(deviceId) else { + throw deviceNotFoundError(deviceId) + } + return device + } + guard let device = fallback() else { + throw unavailableError() + } + return device + } + public static func preparePhotoSession( preferFrontCamera: Bool, deviceId: String?, - pickCamera: (_ preferFrontCamera: Bool, _ deviceId: String?) -> AVCaptureDevice?, - cameraUnavailableError: @autoclosure () -> Error, + pickCamera: (_ preferFrontCamera: Bool, _ deviceId: String?) throws -> AVCaptureDevice, mapSetupError: (CameraSessionConfigurationError) -> Error) throws -> (session: AVCaptureSession, device: AVCaptureDevice, output: AVCapturePhotoOutput) { let session = AVCaptureSession() session.sessionPreset = .photo - guard let device = pickCamera(preferFrontCamera, deviceId) else { - throw cameraUnavailableError() - } + let device = try pickCamera(preferFrontCamera, deviceId) do { try CameraSessionConfiguration.addCameraInput(session: session, camera: device) @@ -48,17 +64,14 @@ public enum CameraCapturePipelineSupport { public static func prepareMovieSession( options: CameraMovieSessionOptions, - pickCamera: (_ preferFrontCamera: Bool, _ deviceId: String?) -> AVCaptureDevice?, - cameraUnavailableError: @autoclosure () -> Error, + pickCamera: (_ preferFrontCamera: Bool, _ deviceId: String?) throws -> AVCaptureDevice, mapSetupError: (CameraSessionConfigurationError) -> Error) throws -> (session: AVCaptureSession, output: AVCaptureMovieFileOutput) { let session = AVCaptureSession() session.sessionPreset = .high - guard let camera = pickCamera(options.preferFrontCamera, options.deviceId) else { - throw cameraUnavailableError() - } + let camera = try pickCamera(options.preferFrontCamera, options.deviceId) do { try CameraSessionConfiguration.addCameraInput(session: session, camera: camera) @@ -74,8 +87,7 @@ public enum CameraCapturePipelineSupport { public static func prepareWarmMovieSession( options: CameraMovieSessionOptions, - pickCamera: (_ preferFrontCamera: Bool, _ deviceId: String?) -> AVCaptureDevice?, - cameraUnavailableError: @autoclosure () -> Error, + pickCamera: (_ preferFrontCamera: Bool, _ deviceId: String?) throws -> AVCaptureDevice, mapSetupError: (CameraSessionConfigurationError) -> Error) async throws -> (session: AVCaptureSession, output: AVCaptureMovieFileOutput) { @@ -83,7 +95,6 @@ public enum CameraCapturePipelineSupport { let prepared = try self.prepareMovieSession( options: options, pickCamera: pickCamera, - cameraUnavailableError: cameraUnavailableError(), mapSetupError: mapSetupError) try Task.checkCancellation() prepared.session.startRunning() @@ -99,8 +110,7 @@ public enum CameraCapturePipelineSupport { public static func withWarmMovieSession( options: CameraMovieSessionOptions, - pickCamera: (_ preferFrontCamera: Bool, _ deviceId: String?) -> AVCaptureDevice?, - cameraUnavailableError: @autoclosure () -> Error, + pickCamera: (_ preferFrontCamera: Bool, _ deviceId: String?) throws -> AVCaptureDevice, mapSetupError: (CameraSessionConfigurationError) -> Error, operation: (AVCaptureMovieFileOutput) async throws -> T) async throws -> T { @@ -108,7 +118,6 @@ public enum CameraCapturePipelineSupport { let prepared = try self.prepareMovieSession( options: options, pickCamera: pickCamera, - cameraUnavailableError: cameraUnavailableError(), mapSetupError: mapSetupError) return try await self.withCaptureSessionLifecycle( start: { prepared.session.startRunning() }, diff --git a/docs/nodes/camera.md b/docs/nodes/camera.md index 827445a17aa0..1474a1b04a0b 100644 --- a/docs/nodes/camera.md +++ b/docs/nodes/camera.md @@ -9,6 +9,8 @@ title: "Camera capture" OpenClaw supports camera capture for agent workflows on paired **iOS**, **Android**, **macOS**, and **Linux** nodes: capture a photo (`jpg`) or a short video clip (`mp4`, with optional audio) via Gateway `node.invoke`. +When a capture request includes `deviceId`, the selected camera must match that ID exactly. An unknown ID fails instead of capturing from a different camera; run `camera.list` to refresh device IDs, which change when cameras are reconnected. + The macOS app can also physically pan, tilt, and zoom supported USB UVC cameras. PTZ moves the camera hardware; it does not rotate, crop, or otherwise transform a captured image. All camera access is gated behind a user-controlled setting per platform. @@ -134,7 +136,7 @@ Physical PTZ is implemented by the Mac app for USB cameras that expose standard Always pass an explicit `deviceId` returned by `camera.list`. OpenClaw never chooses a default camera for physical movement. -- `camera.ptz.status` is a safe read command. Request: `{ "deviceId": "" }`. +- `camera.ptz.status` reads the current position without moving the camera. Request: `{ "deviceId": "" }`. - The response contains only executable `pan`, `tilt`, and `zoom` axes under `axes`. - Pan and tilt values are degrees. Zoom values are percentages. - Each axis reports `current`, `min`, `max`, `step`, `unit`, `canSet`, and `canMove`. `default` appears only when the camera successfully reports a device default. @@ -146,7 +148,9 @@ Always pass an explicit `deviceId` returned by `camera.list`. OpenClaw never cho `set` and `move` require at least one finite axis value. Omitted axes remain unchanged, and move deltas for zoom are percentage points. `home` restores the device-advertised defaults; it returns `CAMERA_PTZ_UNSUPPORTED` without moving the camera when `canHome` is false. The Mac app clamps and snaps requested values to the camera's range and resolution; the response returns the post-operation `state` and lists changed request fields in `adjusted`. Requesting an unsupported axis returns `CAMERA_PTZ_AXIS_UNSUPPORTED`. -Pan/tilt and zoom use separate hardware writes and cannot be atomic. If an earlier control group succeeds but a later write or final status read fails, `CAMERA_PTZ_PARTIAL` names the applied groups, includes best-effort resulting state when readable, and tells the caller to run `camera.ptz.status` before retrying. +Both PTZ commands briefly open a live camera stream because supported cameras only service UVC controls while streaming. This activates the camera and its privacy indicator for the duration, including when `camera.ptz.status` only reads the position. Frames are not retained, and no photo, video, or file is produced. + +Pan/tilt and zoom use separate hardware writes and cannot be atomic. OpenClaw verifies the resulting position through a fresh control connection. If a later write or final status read fails, or an axis does not reach its requested position within the camera's reported resolution, `CAMERA_PTZ_PARTIAL` names the acknowledged control groups, includes the independently observed state when readable, and tells the caller to run `camera.ptz.status` before retrying. Position failures also report the requested and observed values; check that a video stream reaches the camera and disable on-camera AI framing or tracking that can override UVC controls. `camera.ptz.control` is dangerous and remains disarmed until the operator explicitly adds it to `gateway.nodes.commands.allow`: diff --git a/docs/reference/AGENTS.default.md b/docs/reference/AGENTS.default.md index 13cb91116730..08772a0b5bd2 100644 --- a/docs/reference/AGENTS.default.md +++ b/docs/reference/AGENTS.default.md @@ -106,7 +106,7 @@ Example roster for a personal-assistant workspace; swap in whichever skills fit - **mcporter** - tool server runtime/CLI for managing external skill backends. - **Peekaboo** - fast macOS screenshots with optional AI vision analysis. -- **camsnap** - capture frames, clips, or motion alerts from RTSP/ONVIF security cams. +- **camsnap** - capture frames, clips, or motion alerts from RTSP/ONVIF security cams and local webcams, including USB pan/tilt/zoom control. - **oracle** - OpenAI-ready agent CLI with session replay and browser control. - **eightctl** - control your sleep, from the terminal. - **imsg** - send, read, stream iMessage & SMS. diff --git a/skills/camsnap/SKILL.md b/skills/camsnap/SKILL.md index f7ad9adfb871..1284829fd2fc 100644 --- a/skills/camsnap/SKILL.md +++ b/skills/camsnap/SKILL.md @@ -1,6 +1,6 @@ --- name: camsnap -description: "Capture frames or clips from RTSP/ONVIF cameras." +description: "Capture frames or clips from RTSP/ONVIF cameras and local webcams, including USB pan/tilt/zoom control." homepage: https://camsnap.ai metadata: { @@ -39,7 +39,17 @@ Common commands - Motion watch: `camsnap watch kitchen --threshold 0.2 --action '...'` - Doctor: `camsnap doctor --probe` +Local webcams (macOS) + +- List devices: `camsnap devices` +- Snapshot from a local camera: `camsnap snap --device 0 --out webcam.jpg` +- PTZ position and ranges: `camsnap ptz status --device 0` +- Move a gimbal camera: `camsnap ptz goto --device 0 --pan 45 --tilt -18` +- Also `camsnap ptz move` (relative deltas) and `camsnap ptz home`. + Notes - Requires `ffmpeg` on PATH. - Prefer a short test capture before longer clips. +- PTZ needs macOS Camera permission: these cameras only service UVC controls while streaming, so `camsnap` holds a capture session open for the operation. +- Motion commands verify the settled position and exit non-zero if the camera did not reach it. On-camera AI framing can override manual positioning; disable tracking if moves keep missing.