diff --git a/apps/.i18n/native-source.json b/apps/.i18n/native-source.json index bdc89f907c94..26db55e1e574 100644 --- a/apps/.i18n/native-source.json +++ b/apps/.i18n/native-source.json @@ -22611,7 +22611,7 @@ }, { "kind": "ui-localized-call-multiline", - "line": 1281, + "line": 1283, "path": "apps/ios/Sources/Gateway/GatewayConnectionController.swift", "source": "Can't reach gateway at %1$@:%2$@. Verify Tailscale Serve is enabled and publishes this Gateway.", "surface": "apple", @@ -22619,7 +22619,7 @@ }, { "kind": "ui-localized-call", - "line": 1290, + "line": 1292, "path": "apps/ios/Sources/Gateway/GatewayConnectionController.swift", "source": "Can't reach gateway at %1$@:%2$@. Check Tailscale or LAN.", "surface": "apple", @@ -22627,7 +22627,7 @@ }, { "kind": "ui-localized-call-multiline", - "line": 1296, + "line": 1298, "path": "apps/ios/Sources/Gateway/GatewayConnectionController.swift", "source": "TLS fingerprint verification timed out for %1$@:%2$@. Secure endpoint was reached, but TLS did not finish in time.", "surface": "apple", @@ -22635,7 +22635,7 @@ }, { "kind": "ui-localized-call-multiline", - "line": 1304, + "line": 1306, "path": "apps/ios/Sources/Gateway/GatewayConnectionController.swift", "source": "No secure gateway endpoint was detected at %1$@:%2$@. Enable gateway TLS or Tailscale Serve, or use a trusted private LAN address with Unencrypted selected.", "surface": "apple", @@ -22643,7 +22643,7 @@ }, { "kind": "ui-localized-call", - "line": 1314, + "line": 1316, "path": "apps/ios/Sources/Gateway/GatewayConnectionController.swift", "source": "Could not read the TLS certificate from %1$@:%2$@.", "surface": "apple", @@ -23507,7 +23507,7 @@ }, { "kind": "conditional-branch", - "line": 4909, + "line": 4917, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Connected", "surface": "apple", @@ -23515,7 +23515,7 @@ }, { "kind": "conditional-branch", - "line": 4909, + "line": 4917, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Offline", "surface": "apple", @@ -23523,7 +23523,7 @@ }, { "kind": "conditional-branch", - "line": 6075, + "line": 6087, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "No chat messages yet", "surface": "apple", @@ -23531,7 +23531,7 @@ }, { "kind": "conditional-branch", - "line": 6274, + "line": 6286, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Connecting…", "surface": "apple", @@ -23539,7 +23539,7 @@ }, { "kind": "conditional-branch", - "line": 6274, + "line": 6286, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Reconnecting…", "surface": "apple", @@ -23547,7 +23547,7 @@ }, { "kind": "conditional-branch", - "line": 8683, + "line": 8695, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Approval", "surface": "apple", @@ -23555,7 +23555,7 @@ }, { "kind": "conditional-branch", - "line": 8683, + "line": 8695, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "This approval was already", "surface": "apple", @@ -23563,7 +23563,7 @@ }, { "kind": "conditional-branch", - "line": 8689, + "line": 8701, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "This approval was already set to Always Allow.", "surface": "apple", @@ -23571,7 +23571,7 @@ }, { "kind": "conditional-branch", - "line": 8690, + "line": 8702, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Approval set to Always Allow.", "surface": "apple", @@ -23579,7 +23579,7 @@ }, { "kind": "conditional-branch", - "line": 10028, + "line": 10040, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "\\(urlText.prefix(500))…", "surface": "apple", @@ -31595,7 +31595,7 @@ }, { "kind": "conditional-branch", - "line": 385, + "line": 389, "path": "apps/macos/Sources/OpenClaw/MenuContentView.swift", "source": "Main", "surface": "apple", @@ -31603,7 +31603,7 @@ }, { "kind": "conditional-branch", - "line": 385, + "line": 389, "path": "apps/macos/Sources/OpenClaw/MenuContentView.swift", "source": "Other", "surface": "apple", @@ -31611,7 +31611,7 @@ }, { "kind": "ui-modifier", - "line": 459, + "line": 463, "path": "apps/macos/Sources/OpenClaw/MenuContentView.swift", "source": "Show pairing requests", "surface": "apple", @@ -31619,7 +31619,7 @@ }, { "kind": "ui-call", - "line": 484, + "line": 488, "path": "apps/macos/Sources/OpenClaw/MenuContentView.swift", "source": "Refreshing microphones…", "surface": "apple", @@ -31627,7 +31627,7 @@ }, { "kind": "ui-call", - "line": 491, + "line": 495, "path": "apps/macos/Sources/OpenClaw/MenuContentView.swift", "source": "Microphone", "surface": "apple", @@ -31635,7 +31635,7 @@ }, { "kind": "ui-call", - "line": 516, + "line": 520, "path": "apps/macos/Sources/OpenClaw/MenuContentView.swift", "source": "Disconnected (using System default)", "surface": "apple", @@ -31779,7 +31779,7 @@ }, { "kind": "conditional-branch", - "line": 435, + "line": 436, "path": "apps/macos/Sources/OpenClaw/NodePairingApprovalPrompter.swift", "source": "Node pairing approved", "surface": "apple", @@ -31787,7 +31787,7 @@ }, { "kind": "conditional-branch", - "line": 435, + "line": 436, "path": "apps/macos/Sources/OpenClaw/NodePairingApprovalPrompter.swift", "source": "Node pairing rejected", "surface": "apple", @@ -36235,7 +36235,7 @@ }, { "kind": "conditional-branch", - "line": 455, + "line": 460, "path": "apps/macos/Sources/OpenClawMacCLI/WizardCommand.swift", "source": " [\\(initial)]", "surface": "apple", @@ -36243,7 +36243,7 @@ }, { "kind": "conditional-branch", - "line": 502, + "line": 507, "path": "apps/macos/Sources/OpenClawMacCLI/WizardCommand.swift", "source": " — \\(option.hint!)", "surface": "apple", @@ -36251,7 +36251,7 @@ }, { "kind": "conditional-branch", - "line": 509, + "line": 514, "path": "apps/macos/Sources/OpenClawMacCLI/WizardCommand.swift", "source": " [\\(initialIndices.map(String.init).joined(separator: \",\"))]", "surface": "apple", diff --git a/apps/ios/Sources/Gateway/GatewayConnectionController+ManualAuth.swift b/apps/ios/Sources/Gateway/GatewayConnectionController+ManualAuth.swift index 17aa29e28c56..bfb54c339679 100644 --- a/apps/ios/Sources/Gateway/GatewayConnectionController+ManualAuth.swift +++ b/apps/ios/Sources/Gateway/GatewayConnectionController+ManualAuth.swift @@ -3,6 +3,10 @@ import OpenClawKit extension GatewayConnectionController { static func migrateLegacyDeviceAuth() { + guard + let primaryIdentity = DeviceIdentityStore.loadOrCreatePersisted(), + let shareIdentity = DeviceIdentityStore.loadOrCreatePersisted(profile: .shareExtension) + else { return } let migrationGatewayID = self.legacyDeviceAuthMigrationGatewayID() let relay = ShareGatewayRelaySettings.loadConfig() let instanceID = GatewaySettingsStore.currentInstanceID() @@ -15,8 +19,6 @@ extension GatewayConnectionController { } else { GatewaySettingsStore.discardUnscopedGatewayCredentials(instanceId: instanceID) } - let primaryIdentity = DeviceIdentityStore.loadOrCreate() - let shareIdentity = DeviceIdentityStore.loadOrCreate(profile: .shareExtension) // The extension connects independently, so the host's last route cannot prove who // issued its legacy token. Require one extension re-pair instead of guessing an owner. DeviceAuthStore.discardUnscopedTokens( diff --git a/apps/ios/Sources/Gateway/GatewayConnectionController.swift b/apps/ios/Sources/Gateway/GatewayConnectionController.swift index 204b1942fd4b..354317b5729d 100644 --- a/apps/ios/Sources/Gateway/GatewayConnectionController.swift +++ b/apps/ios/Sources/Gateway/GatewayConnectionController.swift @@ -609,20 +609,22 @@ final class GatewayConnectionController { } private static func clearDeviceAuthTokens(gatewayID: String) { - let primaryIdentity = DeviceIdentityStore.loadOrCreate() - DeviceAuthStore.clearToken(deviceId: primaryIdentity.deviceId, role: "node", gatewayID: gatewayID) - DeviceAuthStore.clearToken(deviceId: primaryIdentity.deviceId, role: "operator", gatewayID: gatewayID) - let shareIdentity = DeviceIdentityStore.loadOrCreate(profile: .shareExtension) - DeviceAuthStore.clearToken( - deviceId: shareIdentity.deviceId, - role: "node", - gatewayID: gatewayID, - profile: .shareExtension) - DeviceAuthStore.clearToken( - deviceId: shareIdentity.deviceId, - role: "operator", - gatewayID: gatewayID, - profile: .shareExtension) + if let primaryIdentity = DeviceIdentityStore.loadOrCreatePersisted() { + DeviceAuthStore.clearToken(deviceId: primaryIdentity.deviceId, role: "node", gatewayID: gatewayID) + DeviceAuthStore.clearToken(deviceId: primaryIdentity.deviceId, role: "operator", gatewayID: gatewayID) + } + if let shareIdentity = DeviceIdentityStore.loadOrCreatePersisted(profile: .shareExtension) { + DeviceAuthStore.clearToken( + deviceId: shareIdentity.deviceId, + role: "node", + gatewayID: gatewayID, + profile: .shareExtension) + DeviceAuthStore.clearToken( + deviceId: shareIdentity.deviceId, + role: "operator", + gatewayID: gatewayID, + profile: .shareExtension) + } } private func clearLegacyManualGatewayDefaults(matching stableID: String) { diff --git a/apps/ios/Sources/Model/NodeAppModel.swift b/apps/ios/Sources/Model/NodeAppModel.swift index a6879a2686dd..c72af0ec18c8 100644 --- a/apps/ios/Sources/Model/NodeAppModel.swift +++ b/apps/ios/Sources/Model/NodeAppModel.swift @@ -3918,7 +3918,7 @@ extension NodeAppModel { } private func hasStoredGatewayRoleToken(_ role: String, gatewayID: String) -> Bool { - let identity = DeviceIdentityStore.loadOrCreate() + guard let identity = DeviceIdentityStore.loadOrCreatePersisted() else { return false } return DeviceAuthStore.loadToken( deviceId: identity.deviceId, role: role, @@ -4788,9 +4788,13 @@ extension NodeAppModel { password: String?, forceTalkPermissionUpgradeRequest: Bool = false) -> Bool { - let identity = DeviceIdentityStore.loadOrCreate() - let storedOperatorScopes = DeviceAuthStore - .loadToken(deviceId: identity.deviceId, role: "operator", gatewayID: gatewayID)? + let storedOperatorScopes = DeviceIdentityStore.loadOrCreatePersisted() + .flatMap { identity in + DeviceAuthStore.loadToken( + deviceId: identity.deviceId, + role: "operator", + gatewayID: gatewayID) + }? .scopes ?? [] return Self.shouldRequestOperatorApprovalScope( token: token, @@ -4825,9 +4829,13 @@ extension NodeAppModel { password: String?, forceTalkPermissionUpgradeRequest: Bool = false) -> Bool { - let identity = DeviceIdentityStore.loadOrCreate() - let storedOperatorScopes = DeviceAuthStore - .loadToken(deviceId: identity.deviceId, role: "operator", gatewayID: gatewayID)? + let storedOperatorScopes = DeviceIdentityStore.loadOrCreatePersisted() + .flatMap { identity in + DeviceAuthStore.loadToken( + deviceId: identity.deviceId, + role: "operator", + gatewayID: gatewayID) + }? .scopes ?? [] return Self.shouldRequestOperatorAdminScope( token: token, @@ -4943,9 +4951,13 @@ extension NodeAppModel { return } let gatewayID = config.nodeOptions.deviceAuthGatewayID ?? config.effectiveStableID - let identity = DeviceIdentityStore.loadOrCreate() - self.hasOperatorAdminScope = DeviceAuthStore - .loadToken(deviceId: identity.deviceId, role: "operator", gatewayID: gatewayID)? + self.hasOperatorAdminScope = DeviceIdentityStore.loadOrCreatePersisted() + .flatMap { identity in + DeviceAuthStore.loadToken( + deviceId: identity.deviceId, + role: "operator", + gatewayID: gatewayID) + }? .scopes .contains("operator.admin") == true } diff --git a/apps/ios/Sources/Onboarding/GatewayOnboardingReset.swift b/apps/ios/Sources/Onboarding/GatewayOnboardingReset.swift index 16146aa25f2c..69810af8a4b6 100644 --- a/apps/ios/Sources/Onboarding/GatewayOnboardingReset.swift +++ b/apps/ios/Sources/Onboarding/GatewayOnboardingReset.swift @@ -10,7 +10,7 @@ enum GatewayOnboardingReset { disconnectGateway: Bool = true, defaults: UserDefaults = .standard) async { - await self.prepare( + _ = await self.prepare( appModel: appModel, instanceId: instanceId, gatewayStableID: gatewayStableID, @@ -24,12 +24,13 @@ enum GatewayOnboardingReset { instanceId: String, defaults: UserDefaults = .standard) async { - await self.prepare( + guard await self.prepare( appModel: appModel, instanceId: instanceId, gatewayStableID: nil, disconnectGateway: true, defaults: defaults) + else { return } self.clearOnboardingState(defaults: defaults) } @@ -42,12 +43,13 @@ enum GatewayOnboardingReset { defaults: UserDefaults = .standard) { appModel.purgeChatTranscriptCacheBeforeStartup() - self.preparePairingState( + guard self.preparePairingState( appModel: appModel, instanceId: instanceId, gatewayStableID: nil, disconnectGateway: true, defaults: defaults) + else { return } self.clearOnboardingState(defaults: defaults) } @@ -57,10 +59,10 @@ enum GatewayOnboardingReset { instanceId: String, gatewayStableID: String?, disconnectGateway: Bool, - defaults: UserDefaults) async + defaults: UserDefaults) async -> Bool { await appModel.purgeChatTranscriptCache(gatewayID: gatewayStableID) - self.preparePairingState( + return self.preparePairingState( appModel: appModel, instanceId: instanceId, gatewayStableID: gatewayStableID, @@ -74,8 +76,26 @@ enum GatewayOnboardingReset { instanceId: String, gatewayStableID: String?, disconnectGateway: Bool, - defaults: UserDefaults) + defaults: UserDefaults) -> Bool { + guard let deviceId = DeviceIdentityStore.loadOrCreatePersisted()?.deviceId else { + appModel.gatewayStatusText = "Could not access device identity" + return false + } + let shareDeviceId: String? + if gatewayStableID != nil { + guard let resolvedShareDeviceId = DeviceIdentityStore + .loadOrCreatePersisted(profile: .shareExtension)? + .deviceId + else { + appModel.gatewayStatusText = "Could not access share device identity" + return false + } + shareDeviceId = resolvedShareDeviceId + } else { + shareDeviceId = nil + } + if disconnectGateway { appModel.disconnectGateway() } @@ -91,25 +111,25 @@ enum GatewayOnboardingReset { } } - let deviceId = DeviceIdentityStore.loadOrCreate().deviceId if let gatewayStableID { let authenticationOwnerID = GatewaySettingsStore.authenticationOwnerID( routeStableID: gatewayStableID) - let shareDeviceId = DeviceIdentityStore.loadOrCreate(profile: .shareExtension).deviceId // Bootstrap replacement invalidates only the target. Other paired gateways remain // usable when the user switches back after reviewing or completing this setup. DeviceAuthStore.clearToken(deviceId: deviceId, role: "node", gatewayID: authenticationOwnerID) DeviceAuthStore.clearToken(deviceId: deviceId, role: "operator", gatewayID: authenticationOwnerID) - DeviceAuthStore.clearToken( - deviceId: shareDeviceId, - role: "node", - gatewayID: authenticationOwnerID, - profile: .shareExtension) - DeviceAuthStore.clearToken( - deviceId: shareDeviceId, - role: "operator", - gatewayID: authenticationOwnerID, - profile: .shareExtension) + if let shareDeviceId { + DeviceAuthStore.clearToken( + deviceId: shareDeviceId, + role: "node", + gatewayID: authenticationOwnerID, + profile: .shareExtension) + DeviceAuthStore.clearToken( + deviceId: shareDeviceId, + role: "operator", + gatewayID: authenticationOwnerID, + profile: .shareExtension) + } GatewayTLSStore.clearFingerprint(stableID: gatewayStableID) } else { // Full onboarding reset is the only path that intentionally forgets every gateway. @@ -126,6 +146,7 @@ enum GatewayOnboardingReset { GatewaySettingsStore.clearPreferredGatewayStableID(defaults: defaults) GatewaySettingsStore.clearLastDiscoveredGatewayStableID(defaults: defaults) defaults.set(false, forKey: "gateway.autoconnect") + return true } private static func clearOnboardingState(defaults: UserDefaults) { diff --git a/apps/ios/Sources/Terminal/TerminalHubScreen.swift b/apps/ios/Sources/Terminal/TerminalHubScreen.swift index 0bde6126dd2f..b4dc76edc7d8 100644 --- a/apps/ios/Sources/Terminal/TerminalHubScreen.swift +++ b/apps/ios/Sources/Terminal/TerminalHubScreen.swift @@ -165,7 +165,7 @@ struct TerminalHubScreen: View { // must honor that boundary or a stale token can override the supplied password. guard config.nodeOptions.allowStoredDeviceAuth else { return nil } let gatewayID = config.nodeOptions.deviceAuthGatewayID ?? config.effectiveStableID - let identity = DeviceIdentityStore.loadOrCreate() + guard let identity = DeviceIdentityStore.loadOrCreatePersisted() else { return nil } return DeviceAuthStore.loadToken( deviceId: identity.deviceId, role: "operator", diff --git a/apps/macos/Sources/OpenClaw/GatewayConnection.swift b/apps/macos/Sources/OpenClaw/GatewayConnection.swift index 40e91e6faef8..5111056bb11c 100644 --- a/apps/macos/Sources/OpenClaw/GatewayConnection.swift +++ b/apps/macos/Sources/OpenClaw/GatewayConnection.swift @@ -1177,7 +1177,7 @@ extension GatewayConnection { { return token } - let identity = DeviceIdentityStore.loadOrCreate() + guard let identity = DeviceIdentityStore.loadOrCreatePersisted() else { return nil } return DeviceAuthStore.loadToken( deviceId: identity.deviceId, role: "operator", diff --git a/apps/macos/Sources/OpenClaw/MenuContentView.swift b/apps/macos/Sources/OpenClaw/MenuContentView.swift index 6d845ad1721a..68e08cfcc6ef 100644 --- a/apps/macos/Sources/OpenClaw/MenuContentView.swift +++ b/apps/macos/Sources/OpenClaw/MenuContentView.swift @@ -357,8 +357,12 @@ struct MenuContent: View { guard self.state.connectionMode != .unconfigured else { return nil } guard case .connected = self.controlChannel.state else { return nil } - let deviceId = DeviceIdentityStore.loadOrCreate( - profile: MacNodeModeCoordinator.nodeIdentityProfile).deviceId + guard let identity = DeviceIdentityStore.loadOrCreatePersisted( + profile: MacNodeModeCoordinator.nodeIdentityProfile) + else { + return ("Mac identity unavailable", .red) + } + let deviceId = identity.deviceId if let entry = self.nodesStore.nodes.first(where: { $0.nodeId == deviceId }) { guard entry.isConnected else { return ("Mac capabilities offline", .orange) diff --git a/apps/macos/Sources/OpenClaw/NodePairingApprovalPrompter.swift b/apps/macos/Sources/OpenClaw/NodePairingApprovalPrompter.swift index a2169e7eb1d9..d0d05e831411 100644 --- a/apps/macos/Sources/OpenClaw/NodePairingApprovalPrompter.swift +++ b/apps/macos/Sources/OpenClaw/NodePairingApprovalPrompter.swift @@ -319,8 +319,9 @@ final class NodePairingApprovalPrompter { if req.silent == true { return true } - let localNodeId = DeviceIdentityStore.loadOrCreate( - profile: MacNodeModeCoordinator.nodeIdentityProfile).deviceId + guard let localNodeId = DeviceIdentityStore.loadOrCreatePersisted( + profile: MacNodeModeCoordinator.nodeIdentityProfile)?.deviceId + else { return false } return Self.shouldAutoApproveOwnLocalNode( connectionMode: AppStateStore.shared.connectionMode, requestNodeId: req.nodeId, @@ -451,8 +452,13 @@ final class NodePairingApprovalPrompter { } private func tryAutomaticApproveIfPossible(_ req: PendingRequest) async -> Bool { - let localNodeId = DeviceIdentityStore.loadOrCreate( - profile: MacNodeModeCoordinator.nodeIdentityProfile).deviceId + guard let localNodeId = DeviceIdentityStore.loadOrCreatePersisted( + profile: MacNodeModeCoordinator.nodeIdentityProfile)?.deviceId + else { + self.logger.error( + "automatic pairing skipped (device identity unavailable) requestId=\(req.requestId, privacy: .public)") + return false + } if Self.shouldAutoApproveOwnLocalNode( connectionMode: AppStateStore.shared.connectionMode, requestNodeId: req.nodeId, diff --git a/apps/macos/Sources/OpenClawMacCLI/WizardCommand.swift b/apps/macos/Sources/OpenClawMacCLI/WizardCommand.swift index e97418d9482a..919a5634599a 100644 --- a/apps/macos/Sources/OpenClawMacCLI/WizardCommand.swift +++ b/apps/macos/Sources/OpenClawMacCLI/WizardCommand.swift @@ -272,7 +272,12 @@ actor GatewayWizardClient { params["auth"] = ProtoAnyCodable(["password": ProtoAnyCodable(password)]) } let connectNonce = try await self.waitForConnectChallenge() - let identity = DeviceIdentityStore.loadOrCreate() + guard let identity = DeviceIdentityStore.loadOrCreatePersisted() else { + throw NSError( + domain: "OpenClawMacCLI", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Could not access the persisted device identity"]) + } let signedAtMs = Int64(Date().timeIntervalSince1970 * 1000) let payload = GatewayDeviceAuthPayload.buildConnectCompatibilityPayload( fields: .init( diff --git a/apps/macos/Tests/OpenClawIPCTests/GatewayProcessManagerTests.swift b/apps/macos/Tests/OpenClawIPCTests/GatewayProcessManagerTests.swift index e19b07abbed0..75e6cac9279e 100644 --- a/apps/macos/Tests/OpenClawIPCTests/GatewayProcessManagerTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/GatewayProcessManagerTests.swift @@ -1,7 +1,7 @@ import Foundation -import OpenClawKit import Testing @testable import OpenClaw +@testable import OpenClawKit @Suite(.serialized) @MainActor @@ -1234,54 +1234,59 @@ struct GatewayProcessManagerTests { } @Test func `transient unavailable health response retries until ready`() async throws { - let port = GatewayEnvironment.gatewayPort() - let session = GatewayTestWebSocketSession( - taskFactory: { - GatewayTestWebSocketTask( - sendHook: { task, message, sendIndex in - guard sendIndex > 0 else { return } - guard let id = GatewayWebSocketTestSupport.requestID(from: message) else { return } - if sendIndex == 1 { - let response = Data( - """ - {"type":"res","id":"\(id)","ok":false, - "error":{"code":"UNAVAILABLE","message":"gateway restarting"}} - """.utf8) - task.emitReceiveSuccess(.data(response)) - return - } - task.emitReceiveSuccess(.data(GatewayWebSocketTestSupport.okResponseData(id: id))) - }) - }) - let url = try #require(URL(string: "ws://example.invalid")) - let connection = GatewayConnection( - configProvider: { (url: url, token: nil, password: nil) }, - sessionBox: WebSocketSessionBox(session: session)) - let descriptor = PortGuardian.Descriptor( - pid: 4242, - command: "openclaw-gateway", - executablePath: "/tmp/openclaw-gateway") + let stateDir = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-gateway-ready-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: stateDir) } + try await DeviceIdentityStore.withStateDirectory(stateDir) { + let port = GatewayEnvironment.gatewayPort() + let session = GatewayTestWebSocketSession( + taskFactory: { + GatewayTestWebSocketTask( + sendHook: { task, message, sendIndex in + guard sendIndex > 0 else { return } + guard let id = GatewayWebSocketTestSupport.requestID(from: message) else { return } + if sendIndex == 1 { + let response = Data( + """ + {"type":"res","id":"\(id)","ok":false, + "error":{"code":"UNAVAILABLE","message":"gateway restarting"}} + """.utf8) + task.emitReceiveSuccess(.data(response)) + return + } + task.emitReceiveSuccess(.data(GatewayWebSocketTestSupport.okResponseData(id: id))) + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let connection = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + let descriptor = PortGuardian.Descriptor( + pid: 4242, + command: "openclaw-gateway", + executablePath: "/tmp/openclaw-gateway") - let manager = GatewayProcessManager.shared - manager.setTestingConnection(connection) - manager.setTestingDesiredActive(true) - manager.setTestingStatus(.starting) - manager._testClearLaunchAgentReadinessFailure() - await PortGuardian.shared.setTestingDescriptor(descriptor, forPort: port) - defer { - manager.setTestingConnection(nil) - manager.setTestingDesiredActive(false) - manager.setTestingLastFailureReason(nil) + let manager = GatewayProcessManager.shared + manager.setTestingConnection(connection) + manager.setTestingDesiredActive(true) + manager.setTestingStatus(.starting) manager._testClearLaunchAgentReadinessFailure() - manager._testSetLastObservedGatewayPID(nil) + await PortGuardian.shared.setTestingDescriptor(descriptor, forPort: port) + defer { + manager.setTestingConnection(nil) + manager.setTestingDesiredActive(false) + manager.setTestingLastFailureReason(nil) + manager._testClearLaunchAgentReadinessFailure() + manager._testSetLastObservedGatewayPID(nil) + } + + #expect(await manager.waitForGatewayReady(timeout: 1)) + #expect(manager.status == .running(details: "pid 4242")) + #expect(!manager._testHasLaunchAgentReadinessFailure()) + + await connection.shutdown() + await PortGuardian.shared.setTestingDescriptor(nil, forPort: port) } - - #expect(await manager.waitForGatewayReady(timeout: 1)) - #expect(manager.status == .running(details: "pid 4242")) - #expect(!manager._testHasLaunchAgentReadinessFailure()) - - await connection.shutdown() - await PortGuardian.shared.setTestingDescriptor(nil, forPort: port) } @Test func `cancelled readiness probe preserves lifecycle state`() async throws { diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/DeviceAuthStore.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/DeviceAuthStore.swift index 84294703c544..a1ddd7488999 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawKit/DeviceAuthStore.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/DeviceAuthStore.swift @@ -1,6 +1,6 @@ import Foundation -public struct DeviceAuthEntry: Codable, Sendable { +public struct DeviceAuthEntry: Codable, Sendable, Equatable { public let token: String public let role: String public let scopes: [String] @@ -16,7 +16,7 @@ public struct DeviceAuthEntry: Codable, Sendable { } } -private struct DeviceAuthStoreFile: Codable { +struct DeviceAuthStoreFile: Codable, Equatable { var version: Int var deviceId: String var tokens: [String: DeviceAuthEntry] @@ -215,9 +215,13 @@ public enum DeviceAuthStore { guard let decoded = try? JSONDecoder().decode(DeviceAuthStoreFile.self, from: data) else { return nil } + return self.normalizedStore(decoded) + } + + static func normalizedStore(_ decoded: DeviceAuthStoreFile) -> DeviceAuthStoreFile? { guard decoded.version == 1 else { return nil } - // Entries carry their owner, so legacy raw keys can be safely reindexed on read. - // The next mutation persists only byte-stable v2 keys without changing file shape. + // Entries carry their owner, so reads and identity migration compare one canonical + // role/scope/owner map instead of raw JSON key order or legacy dictionary keys. var tokens: [String: DeviceAuthEntry] = [:] for entry in decoded.tokens.values { let role = self.normalizeRole(entry.role) @@ -231,8 +235,13 @@ public enum DeviceAuthStore { scopes: self.normalizeScopes(entry.scopes), updatedAtMs: entry.updatedAtMs, gatewayID: gatewayID) - if let existing = tokens[key], existing.updatedAtMs > normalized.updatedAtMs { - continue + if let existing = tokens[key] { + if existing.updatedAtMs > normalized.updatedAtMs { + continue + } + if existing.updatedAtMs == normalized.updatedAtMs, existing != normalized { + return nil + } } tokens[key] = normalized } diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/DeviceIdentity.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/DeviceIdentity.swift index 1e21592b047b..b6360c5034de 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawKit/DeviceIdentity.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/DeviceIdentity.swift @@ -32,7 +32,7 @@ public enum GatewayDeviceIdentityProfile: String, Sendable { } } -public struct DeviceIdentity: Codable, Sendable { +public struct DeviceIdentity: Codable, Sendable, Equatable { public var deviceId: String public var publicKey: String public var privateKey: String @@ -87,7 +87,7 @@ enum DeviceIdentityPaths { private static func stateDirOverrideURL() -> URL? { // Test-scoped stores must win over the process environment. Parallel Swift tests // otherwise race whenever another suite temporarily swaps OPENCLAW_STATE_DIR. - if let scopedStateDirURL = self.scopedStateDirURL { + if let scopedStateDirURL { return scopedStateDirURL } for key in self.stateDirEnv { @@ -101,7 +101,7 @@ enum DeviceIdentityPaths { return nil } - private static func legacyStateDirURL() -> URL? { + static func legacyStateDirURL() -> URL? { if let appSupport = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first { return appSupport.appendingPathComponent("OpenClaw", isDirectory: true) } @@ -132,7 +132,7 @@ enum DeviceIdentityPaths { #endif } - private static func appGroupStateDirURL() -> URL? { + static func appGroupStateDirURL() -> URL? { guard let containerURL = FileManager.default .containerURL(forSecurityApplicationGroupIdentifier: OpenClawAppGroup.identifier) @@ -142,56 +142,64 @@ enum DeviceIdentityPaths { return containerURL.appendingPathComponent("OpenClaw", isDirectory: true) } - /// Files a one-time fallback migration may carry from the App Group container into the - /// selected store. Stored device tokens are keyed by deviceId, so the identity file is - /// only useful together with its auth sibling; migrating one without the other forces - /// an unnecessary re-pair even though the deviceId survived. - struct AppGroupMigrationSource { + struct LegacyIdentitySource: Equatable { + let stateDirURL: URL let identityURL: URL let authURL: URL } - static func appGroupMigrationSource( - profile: GatewayDeviceIdentityProfile) -> AppGroupMigrationSource? + static func legacyIdentitySources( + profile: GatewayDeviceIdentityProfile) -> [LegacyIdentitySource] { - self.appGroupMigrationSource( - appGroupStateDirURL: self.appGroupStateDirURL(), - appGroupStateDirAvailable: self.appGroupStateDirAvailable, - stateDirOverridden: self.stateDirOverrideURL() != nil, - profile: profile) - } - - /// Non-nil only for unentitled builds whose store selection fell back to legacy storage; - /// entitled builds keep using the App Group container and must never migrate out of it. - /// An explicit OPENCLAW_STATE_DIR override selects a caller-chosen store, not the legacy - /// fallback; importing container identity/tokens there would leak the machine's real - /// pairing into unrelated stores (test dirs, relocated installs). - static func appGroupMigrationSource( - appGroupStateDirURL: URL?, - appGroupStateDirAvailable: Bool, - stateDirOverridden: Bool, - profile: GatewayDeviceIdentityProfile) -> AppGroupMigrationSource? - { - guard !stateDirOverridden, !appGroupStateDirAvailable, let appGroupStateDirURL else { - return nil + // Node doctor cannot traverse sandboxed Apple App Group/Application Support containers. + // Native startup therefore owns this one-time import before runtime becomes SQLite-only. + let selectedStateDirURL = self.stateDirURL() + let roots: [URL] = if self.scopedStateDirURL != nil || self.stateDirOverrideURL() != nil { + // Explicit and task-local stores must never import the machine's real identity. + [selectedStateDirURL] + } else { + // Unentitled macOS builds intentionally probe the former App Group once: shipped + // installs must carry their identity and auth together instead of rotating/re-pairing. + [selectedStateDirURL, self.appGroupStateDirURL(), self.legacyStateDirURL()] + .compactMap(\.self) + } + + var seen = Set() + return roots.compactMap { root in + let standardizedRoot = root.standardizedFileURL + guard seen.insert(standardizedRoot.path).inserted else { return nil } + let identityDirURL = standardizedRoot.appendingPathComponent("identity", isDirectory: true) + return LegacyIdentitySource( + stateDirURL: standardizedRoot, + identityURL: identityDirURL.appendingPathComponent(profile.identityFileName, isDirectory: false), + authURL: identityDirURL.appendingPathComponent(profile.authFileName, isDirectory: false)) } - let identityDirURL = appGroupStateDirURL.appendingPathComponent("identity", isDirectory: true) - return AppGroupMigrationSource( - identityURL: identityDirURL.appendingPathComponent(profile.identityFileName, isDirectory: false), - authURL: identityDirURL.appendingPathComponent(profile.authFileName, isDirectory: false)) } } +struct DeviceIdentityMaterial: Equatable { + let identity: DeviceIdentity + let publicKeyPEM: String + let privateKeyPEM: String +} + public enum DeviceIdentityStore { - private static let ed25519SPKIPrefix = Data([ + static let ed25519SPKIPrefix = Data([ 0x30, 0x2A, 0x30, 0x05, 0x06, 0x03, 0x2B, 0x65, 0x70, 0x03, 0x21, 0x00, ]) - private static let ed25519PKCS8PrivatePrefix = Data([ + static let ed25519PKCS8PrivatePrefix = Data([ 0x30, 0x2E, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06, 0x03, 0x2B, 0x65, 0x70, 0x04, 0x22, 0x04, 0x20, ]) + static func storageError(_ message: String) -> NSError { + NSError( + domain: "ai.openclaw.device-identity-store", + code: 1, + userInfo: [NSLocalizedDescriptionKey: message]) + } + public static func loadOrCreate() -> DeviceIdentity { self.loadOrCreate(profile: .primary) } @@ -219,131 +227,22 @@ public enum DeviceIdentityStore { #endif public static func loadOrCreate(profile: GatewayDeviceIdentityProfile) -> DeviceIdentity { - self.loadOrCreate( - fileURL: self.fileURL(profile: profile), - migrationSource: DeviceIdentityPaths.appGroupMigrationSource(profile: profile)) + guard let identity = loadOrCreatePersisted(profile: profile) else { + preconditionFailure("Could not persist the OpenClaw device identity") + } + return identity } /// Loads or creates an identity, returning nil unless its key material was durably persisted. public static func loadOrCreatePersisted( profile: GatewayDeviceIdentityProfile = .primary) -> DeviceIdentity? { - self.loadOrCreatePersisted( - fileURL: self.fileURL(profile: profile), - migrationSource: DeviceIdentityPaths.appGroupMigrationSource(profile: profile)) - } - - static func loadOrCreate( - fileURL url: URL, - migrationSource: DeviceIdentityPaths.AppGroupMigrationSource? = nil) -> DeviceIdentity - { - if let data = try? Data(contentsOf: url) { - switch self.decodeStoredIdentity(data) { - case let .identity(decoded): - return decoded - case .recognizedInvalid, .unknown: - // Existing bytes may hold a newer schema or recoverable key material; never - // overwrite them. Callers run with a transient identity instead. - return self.generate() - } - } - if FileManager.default.fileExists(atPath: url.path) { - return self.generate() - } - if let migrated = self.migratedIdentity(from: migrationSource, to: url) { - return migrated - } - let identity = self.generate() - self.save(identity, to: url) - return identity - } - - static func loadOrCreatePersisted( - fileURL url: URL, - migrationSource: DeviceIdentityPaths.AppGroupMigrationSource? = nil) -> DeviceIdentity? - { - let identity = self.loadOrCreate(fileURL: url, migrationSource: migrationSource) - guard let data = try? Data(contentsOf: url), - case let .identity(stored) = self.decodeStoredIdentity(data), - stored.deviceId == identity.deviceId, - stored.publicKey == identity.publicKey, - stored.privateKey == identity.privateKey - else { - return nil - } - return stored - } - - /// One-time upgrade path for builds that lost App Group storage: it runs only while the - /// selected store has no identity file, so steady state never re-reads the old container. - private static func migratedIdentity( - from source: DeviceIdentityPaths.AppGroupMigrationSource?, - to destinationURL: URL) -> DeviceIdentity? - { - guard - let source, - let data = try? Data(contentsOf: source.identityURL), - case let .identity(identity) = self.decodeStoredIdentity(data) - else { - return nil - } - self.save(identity, to: destinationURL) - // Stored device tokens only load when their store's deviceId matches (DeviceAuthStore), - // so they must move together with the identity or the install re-pairs for no reason. - // A mismatched copy is inert behind that same check; no validation needed here. - self.copyAuthStoreFile( - from: source.authURL, - toDirectory: destinationURL.deletingLastPathComponent()) - return identity - } - - private static func copyAuthStoreFile(from sourceURL: URL, toDirectory directoryURL: URL) { - let fileManager = FileManager.default - let destinationURL = directoryURL - .appendingPathComponent(sourceURL.lastPathComponent, isDirectory: false) - guard - !fileManager.fileExists(atPath: destinationURL.path), - fileManager.fileExists(atPath: sourceURL.path) - else { - return - } - try? fileManager.copyItem(at: sourceURL, to: destinationURL) - try? fileManager.setAttributes( - [.posixPermissions: 0o600], - ofItemAtPath: destinationURL.path) - } - - private enum DecodeResult { - case identity(DeviceIdentity) - case recognizedInvalid - case unknown - } - - private static func decodeStoredIdentity(_ data: Data) -> DecodeResult { - let decoder = JSONDecoder() - if let decoded = try? decoder.decode(DeviceIdentity.self, from: data) { - guard let identity = self.normalizedRawIdentity(decoded) else { - return .recognizedInvalid - } - return .identity(identity) - } - - if let decoded = try? decoder.decode(PemDeviceIdentity.self, from: data) { - guard decoded.version == 1, - let publicKeyData = self.rawPublicKey(fromPEM: decoded.publicKeyPem), - let privateKeyData = self.rawPrivateKey(fromPEM: decoded.privateKeyPem), - self.keyPairMatches(publicKeyData: publicKeyData, privateKeyData: privateKeyData) - else { - return .recognizedInvalid - } - return .identity(DeviceIdentity( - deviceId: self.deviceId(publicKeyData: publicKeyData), - publicKey: publicKeyData.base64EncodedString(), - privateKey: privateKeyData.base64EncodedString(), - createdAtMs: decoded.createdAtMs)) - } - - return self.hasRecognizedIdentityShape(data) ? .recognizedInvalid : .unknown + let stateDirURL = DeviceIdentityPaths.stateDirURL() + return try? DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: self.databaseURL(stateDirURL: stateDirURL), + destinationStateDirURL: stateDirURL, + profile: profile, + legacySources: DeviceIdentityPaths.legacyIdentitySources(profile: profile)) } public static func signPayload(_ payload: String, identity: DeviceIdentity) -> String? { @@ -357,17 +256,21 @@ public enum DeviceIdentityStore { } } - private static func generate() -> DeviceIdentity { + static func generateMaterial() -> DeviceIdentityMaterial { let privateKey = Curve25519.Signing.PrivateKey() let publicKey = privateKey.publicKey let publicKeyData = publicKey.rawRepresentation let privateKeyData = privateKey.rawRepresentation let deviceId = self.deviceId(publicKeyData: publicKeyData) - return DeviceIdentity( + let identity = DeviceIdentity( deviceId: deviceId, publicKey: publicKeyData.base64EncodedString(), privateKey: privateKeyData.base64EncodedString(), createdAtMs: Int64(Date().timeIntervalSince1970 * 1000)) + return DeviceIdentityMaterial( + identity: identity, + publicKeyPEM: self.pem(label: "PUBLIC KEY", der: self.ed25519SPKIPrefix + publicKeyData), + privateKeyPEM: self.pem(label: "PRIVATE KEY", der: self.ed25519PKCS8PrivatePrefix + privateKeyData)) } private static func base64UrlEncode(_ data: Data) -> String { @@ -383,10 +286,77 @@ public enum DeviceIdentityStore { return self.base64UrlEncode(data) } - private static func normalizedRawIdentity(_ identity: DeviceIdentity) -> DeviceIdentity? { - guard !identity.deviceId.isEmpty, - let publicKeyData = Data(base64Encoded: identity.publicKey), - let privateKeyData = Data(base64Encoded: identity.privateKey) + static func material(fromLegacyData data: Data) throws -> DeviceIdentityMaterial { + guard let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw DeviceIdentityStore.storageError("Legacy device identity is not a JSON object") + } + let keys = Set(object.keys) + let decoder = JSONDecoder() + if keys == ["deviceId", "publicKey", "privateKey", "createdAtMs"], + let decoded = try? decoder.decode(DeviceIdentity.self, from: data), + decoded.createdAtMs >= 0 + { + guard let normalized = normalizedRawIdentity(decoded), + let publicKeyData = Data(base64Encoded: normalized.publicKey), + let privateKeyData = Data(base64Encoded: normalized.privateKey) + else { + throw DeviceIdentityStore + .storageError("Legacy raw device identity has invalid key material or deviceId") + } + return DeviceIdentityMaterial( + identity: normalized, + publicKeyPEM: self.pem(label: "PUBLIC KEY", der: self.ed25519SPKIPrefix + publicKeyData), + privateKeyPEM: self.pem(label: "PRIVATE KEY", der: self.ed25519PKCS8PrivatePrefix + privateKeyData)) + } + if keys == ["version", "deviceId", "publicKeyPem", "privateKeyPem", "createdAtMs"], + let decoded = try? decoder.decode(PemDeviceIdentity.self, from: data) + { + guard decoded.version == 1, decoded.createdAtMs >= 0, + let publicKeyData = rawPublicKey(fromPEM: decoded.publicKeyPem), + let privateKeyData = rawPrivateKey(fromPEM: decoded.privateKeyPem), + keyPairMatches(publicKeyData: publicKeyData, privateKeyData: privateKeyData) + else { + throw DeviceIdentityStore.storageError("Legacy PEM device identity has invalid key material") + } + return self.material( + publicKeyData: publicKeyData, + privateKeyData: privateKeyData, + createdAtMs: decoded.createdAtMs) + } + throw DeviceIdentityStore.storageError("Legacy device identity has an unsupported shape") + } + + static func material( + deviceId: String, + publicKeyPEM: String, + privateKeyPEM: String, + createdAtMs: Int64) throws -> DeviceIdentityMaterial + { + guard createdAtMs >= 0, + let publicKeyData = rawPublicKey(fromPEM: publicKeyPEM), + let privateKeyData = rawPrivateKey(fromPEM: privateKeyPEM), + keyPairMatches(publicKeyData: publicKeyData, privateKeyData: privateKeyData) + else { + throw DeviceIdentityStore.storageError("SQLite device identity has invalid key material") + } + let canonical = self.material( + publicKeyData: publicKeyData, + privateKeyData: privateKeyData, + createdAtMs: createdAtMs) + guard canonical.identity.deviceId == deviceId else { + throw DeviceIdentityStore.storageError("SQLite device identity deviceId does not match its public key") + } + guard canonical.publicKeyPEM == publicKeyPEM, canonical.privateKeyPEM == privateKeyPEM else { + throw DeviceIdentityStore.storageError("SQLite device identity PEM is not canonical") + } + return canonical + } + + private static func normalizedRawIdentity(_ rawIdentity: DeviceIdentity) -> DeviceIdentity? { + let rawKey = rawIdentity.privateKey + guard !rawIdentity.deviceId.isEmpty, + let publicKeyData = Data(base64Encoded: rawIdentity.publicKey), + let privateKeyData = Data(base64Encoded: rawKey) else { return nil } guard publicKeyData.count == 32, privateKeyData.count == 32, @@ -394,28 +364,28 @@ public enum DeviceIdentityStore { else { return nil } return DeviceIdentity( deviceId: self.deviceId(publicKeyData: publicKeyData), - publicKey: identity.publicKey, - privateKey: identity.privateKey, - createdAtMs: identity.createdAtMs) + publicKey: rawIdentity.publicKey, + privateKey: rawKey, + createdAtMs: rawIdentity.createdAtMs) } - private static func rawPublicKey(fromPEM pem: String) -> Data? { - guard let der = self.derData(fromPEM: pem), + static func rawPublicKey(fromPEM pem: String) -> Data? { + guard let der = derData(fromPEM: pem, label: "PUBLIC KEY"), der.count == self.ed25519SPKIPrefix.count + 32, der.prefix(self.ed25519SPKIPrefix.count) == self.ed25519SPKIPrefix else { return nil } return der.suffix(32) } - private static func rawPrivateKey(fromPEM pem: String) -> Data? { - guard let der = self.derData(fromPEM: pem), + static func rawPrivateKey(fromPEM pem: String) -> Data? { + guard let der = derData(fromPEM: pem, label: "PRIVATE KEY"), der.count == self.ed25519PKCS8PrivatePrefix.count + 32, der.prefix(self.ed25519PKCS8PrivatePrefix.count) == self.ed25519PKCS8PrivatePrefix else { return nil } return der.suffix(32) } - private static func keyPairMatches(publicKeyData: Data, privateKeyData: Data) -> Bool { + static func keyPairMatches(publicKeyData: Data, privateKeyData: Data) -> Bool { guard let privateKey = try? Curve25519.Signing.PrivateKey(rawRepresentation: privateKeyData) else { return false @@ -423,45 +393,53 @@ public enum DeviceIdentityStore { return privateKey.publicKey.rawRepresentation == publicKeyData } - private static func derData(fromPEM pem: String) -> Data? { - let body = pem - .split(whereSeparator: \.isNewline) - .filter { !$0.hasPrefix("-----") } - .joined() - return Data(base64Encoded: body) + private static func derData(fromPEM pem: String, label: String) -> Data? { + let lines = pem.split(separator: "\n", omittingEmptySubsequences: false) + guard lines.count >= 4, + lines.first == "-----BEGIN \(label)-----", + lines[lines.count - 2] == "-----END \(label)-----", + lines.last?.isEmpty == true + else { return nil } + let body = lines.dropFirst().dropLast(2) + guard !body.isEmpty, body.allSatisfy({ !$0.isEmpty && $0.count <= 64 }) else { return nil } + return Data(base64Encoded: body.joined()) } - private static func hasRecognizedIdentityShape(_ data: Data) -> Bool { - guard let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { - return false - } - return object.keys.contains("publicKeyPem") - || object.keys.contains("privateKeyPem") - || object.keys.contains("publicKey") - || object.keys.contains("privateKey") - } - - private static func deviceId(publicKeyData: Data) -> String { + static func deviceId(publicKeyData: Data) -> String { SHA256.hash(data: publicKeyData).compactMap { String(format: "%02x", $0) }.joined() } - private static func save(_ identity: DeviceIdentity, to url: URL) { - do { - try FileManager.default.createDirectory( - at: url.deletingLastPathComponent(), - withIntermediateDirectories: true) - let data = try JSONEncoder().encode(identity) - try data.write(to: url, options: [.atomic]) - } catch { - // best-effort only - } + static func material( + publicKeyData: Data, + privateKeyData: Data, + createdAtMs: Int64) -> DeviceIdentityMaterial + { + let identity = DeviceIdentity( + deviceId: deviceId(publicKeyData: publicKeyData), + publicKey: publicKeyData.base64EncodedString(), + privateKey: privateKeyData.base64EncodedString(), + createdAtMs: createdAtMs) + return DeviceIdentityMaterial( + identity: identity, + publicKeyPEM: self.pem(label: "PUBLIC KEY", der: self.ed25519SPKIPrefix + publicKeyData), + privateKeyPEM: self.pem(label: "PRIVATE KEY", der: self.ed25519PKCS8PrivatePrefix + privateKeyData)) } - private static func fileURL(profile: GatewayDeviceIdentityProfile) -> URL { - let base = DeviceIdentityPaths.stateDirURL() - return base - .appendingPathComponent("identity", isDirectory: true) - .appendingPathComponent(profile.identityFileName, isDirectory: false) + private static func pem(label: String, der: Data) -> String { + let base64 = der.base64EncodedString() + let fence = String(repeating: "-", count: 5) + let lines = stride(from: 0, to: base64.count, by: 64).map { offset -> String in + let start = base64.index(base64.startIndex, offsetBy: offset) + let end = base64.index(start, offsetBy: min(64, base64.distance(from: start, to: base64.endIndex))) + return String(base64[start.. URL { + stateDirURL + .appendingPathComponent("state", isDirectory: true) + .appendingPathComponent("openclaw.sqlite", isDirectory: false) } } diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/DeviceIdentitySQLiteStore.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/DeviceIdentitySQLiteStore.swift new file mode 100644 index 000000000000..d68f6c7b2f27 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/DeviceIdentitySQLiteStore.swift @@ -0,0 +1,984 @@ +import CryptoKit +import Darwin +import Foundation +import SQLite3 + +enum DeviceIdentitySQLiteStore { + // Keep aligned with OPENCLAW_STATE_SCHEMA_VERSION. Swift never upgrades this database. + private static let maximumSupportedSchemaVersion: Int64 = 4 + private static let busyTimeoutMilliseconds: Int32 = 5000 + private static let maximumLegacyIdentityBytes = 64 * 1024 + private static let maximumLegacyAuthBytes = 4 * 1024 * 1024 + private static let doctorClaimSuffix = ".doctor-importing" + private static let nativeClaimSuffix = ".native-importing" + private static let tableName = "device_identities" + private static let indexName = "idx_device_identities_device" + + private static let createSchemaSQL = """ + CREATE TABLE IF NOT EXISTS device_identities ( + identity_key TEXT NOT NULL PRIMARY KEY, + device_id TEXT NOT NULL, + public_key_pem TEXT NOT NULL, + private_key_pem TEXT NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL + ) STRICT; + + CREATE INDEX IF NOT EXISTS idx_device_identities_device + ON device_identities(device_id, updated_at_ms DESC); + """ + + private struct LegacyClaim { + let source: DeviceIdentityPaths.LegacyIdentitySource + let identityURL: URL + let data: Data + let snapshot: LegacyFileSnapshot + let material: DeviceIdentityMaterial + } + + private struct LegacyFileSnapshot: Equatable { + let device: UInt64 + let inode: UInt64 + let size: UInt64 + let modifiedAt: Date? + } + + private struct LegacyAuthCandidate { + let data: Data + let store: DeviceAuthStoreFile + } + + private struct Column: Equatable { + let name: String + let type: String + let notNull: Bool + let primaryKeyPosition: Int32 + let hidden: Int32 + } + + private final class IdentityCoordinator { + private var database: OpaquePointer? + + init(database: OpaquePointer) { + self.database = database + } + + func release() throws { + guard let database else { return } + self.database = nil + var releaseError: NSError? + if sqlite3_exec(database, "ROLLBACK", nil, nil, nil) != SQLITE_OK { + releaseError = DeviceIdentityStore.storageError( + "Could not release device identity coordinator: \(String(cString: sqlite3_errmsg(database)))") + } + if sqlite3_close(database) != SQLITE_OK, releaseError == nil { + releaseError = DeviceIdentityStore.storageError("Could not close device identity coordinator") + } + if let releaseError { throw releaseError } + } + } + + static func loadOrCreate( + databaseURL: URL, + destinationStateDirURL: URL, + profile: GatewayDeviceIdentityProfile, + legacySources: [DeviceIdentityPaths.LegacyIdentitySource] = [], + beforeLegacyClaim: ((DeviceIdentityPaths.LegacyIdentitySource) throws -> Void)? = nil, + afterLegacyCommit: (() throws -> Void)? = nil) throws + -> DeviceIdentity + { + let coordinator = try self.acquireIdentityCoordinator(databaseURL: databaseURL) + do { + let identity = try self.loadOrCreateOwned( + databaseURL: databaseURL, + destinationStateDirURL: destinationStateDirURL, + profile: profile, + legacySources: legacySources, + beforeLegacyClaim: beforeLegacyClaim, + afterLegacyCommit: afterLegacyCommit) + try coordinator.release() + return identity + } catch { + do { + try coordinator.release() + } catch let releaseError { + throw DeviceIdentityStore.storageError( + "Device identity operation failed: \(error.localizedDescription); " + + "coordinator release failed: \(releaseError.localizedDescription)") + } + throw error + } + } + + private static func loadOrCreateOwned( + databaseURL: URL, + destinationStateDirURL: URL, + profile: GatewayDeviceIdentityProfile, + legacySources: [DeviceIdentityPaths.LegacyIdentitySource], + beforeLegacyClaim: ((DeviceIdentityPaths.LegacyIdentitySource) throws -> Void)?, + afterLegacyCommit: (() throws -> Void)?) throws -> DeviceIdentity + { + try self.secureDirectory(destinationStateDirURL) + try self.secureDirectory(databaseURL.deletingLastPathComponent()) + var claims: [LegacyClaim] = [] + do { + for source in legacySources { + if let claim = try self.claimLegacyIdentity(source, beforeClaim: beforeLegacyClaim) { + claims.append(claim) + } + } + return try self.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: destinationStateDirURL, + profile: profile, + claims: claims, + afterLegacyCommit: afterLegacyCommit) + } catch { + do { + try self.restoreClaimedLegacyIdentities(claims) + } catch let restoreError { + throw DeviceIdentityStore.storageError( + "Device identity migration failed: \(error.localizedDescription); " + + "native claim restoration failed: \(restoreError.localizedDescription)") + } + throw error + } + } + + private static func loadOrCreate( + databaseURL: URL, + destinationStateDirURL: URL, + profile: GatewayDeviceIdentityProfile, + claims: [LegacyClaim], + afterLegacyCommit: (() throws -> Void)?) throws -> DeviceIdentity + { + try self.requireConsistentClaims(claims) + let generatedMaterial = claims.isEmpty ? DeviceIdentityStore.generateMaterial() : nil + let writeTimestampMs = Int64(Date().timeIntervalSince1970 * 1000) + + var database: OpaquePointer? + let flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX + let openResult = sqlite3_open_v2(databaseURL.path, &database, flags, nil) + guard openResult == SQLITE_OK, let database else { + let message = database.map { String(cString: sqlite3_errmsg($0)) } ?? "unknown SQLite error" + if let database { sqlite3_close(database) } + throw DeviceIdentityStore.storageError("Could not open device identity database: \(message)") + } + defer { + sqlite3_close(database) + try? self.secureDatabaseFiles(databaseURL) + } + guard sqlite3_busy_timeout(database, self.busyTimeoutMilliseconds) == SQLITE_OK else { + throw self.databaseError(database, operation: "configure SQLite busy timeout") + } + try self.secureDatabaseFiles(databaseURL) + + try self.execute(database, sql: "BEGIN IMMEDIATE") + var committed = false + defer { + if !committed { + try? self.execute(database, sql: "ROLLBACK") + } + } + + try self.ensureSchema(database, allowFreshCreation: true) + let existing = try self.readIdentity(database, key: profile.rawValue) + let selected: DeviceIdentityMaterial + if let existing { + if let migrated = claims.first?.material, + !self.hasSameKeyMaterial(migrated, existing) + { + throw DeviceIdentityStore.storageError( + "Legacy device identity conflicts with SQLite identity key \(profile.rawValue); source preserved") + } + selected = existing + } else { + guard let candidate = claims.first?.material ?? generatedMaterial else { + throw DeviceIdentityStore.storageError("Device identity candidate is unavailable") + } + selected = candidate + try self.insertIdentity( + database, + key: profile.rawValue, + material: selected, + updatedAtMs: writeTimestampMs) + } + + // The row reread under the write transaction is authoritative. Never return generated + // or migrated key material unless SQLite reports the exact canonical receipt. + guard let authoritative = try self.readIdentity(database, key: profile.rawValue), + authoritative == selected + else { + throw DeviceIdentityStore.storageError("SQLite did not preserve the authoritative device identity") + } + try self.ensureSchema(database, allowFreshCreation: false) + try self.execute(database, sql: "COMMIT") + committed = true + try self.secureDatabaseFiles(databaseURL) + + if !claims.isEmpty { + try afterLegacyCommit?() + // The committed reread is the destructive-cleanup receipt. Doctor cannot alter the + // row while the native claim remains visible to every Node identity entry point. + guard let committedIdentity = try self.readIdentity(database, key: profile.rawValue), + committedIdentity == authoritative + else { + throw DeviceIdentityStore.storageError( + "Committed SQLite identity changed before legacy cleanup; native claim preserved") + } + try self.relocateLegacyAuthIfNeeded( + claims: claims, + destinationStateDirURL: destinationStateDirURL, + profile: profile, + deviceId: authoritative.identity.deviceId) + try self.removeClaimedLegacyIdentities(claims) + } + return authoritative.identity + } + + private static func acquireIdentityCoordinator(databaseURL: URL) throws -> IdentityCoordinator { + let canonicalPath = self.canonicalDatabasePath(databaseURL) + let digest = SHA256.hash(data: Data(canonicalPath.utf8)) + let pathHash = digest.prefix(4).map { String(format: "%02x", $0) }.joined() + let lockDirectoryURL = FileManager.default.temporaryDirectory + .appendingPathComponent("openclaw-\(getuid())", isDirectory: true) + try self.secureCoordinatorDirectory(lockDirectoryURL) + let coordinatorURL = lockDirectoryURL.appendingPathComponent( + "device-identity.\(pathHash).lock.sqlite", + isDirectory: false) + + var database: OpaquePointer? + let flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX + let openResult = sqlite3_open_v2(coordinatorURL.path, &database, flags, nil) + guard openResult == SQLITE_OK, let database else { + let message = database.map { String(cString: sqlite3_errmsg($0)) } ?? "unknown SQLite error" + if let database { sqlite3_close(database) } + throw DeviceIdentityStore.storageError("Could not open device identity coordinator: \(message)") + } + do { + guard sqlite3_busy_timeout(database, self.busyTimeoutMilliseconds) == SQLITE_OK else { + throw self.databaseError(database, operation: "configure device identity coordinator timeout") + } + try self.execute(database, sql: "BEGIN EXCLUSIVE") + try self.secureFile(coordinatorURL) + return IdentityCoordinator(database: database) + } catch { + sqlite3_close(database) + throw error + } + } + + private static func secureCoordinatorDirectory(_ url: URL) throws { + var info = stat() + if lstat(url.path, &info) != 0 { + let inspectError = errno + guard inspectError == ENOENT else { + throw POSIXError(POSIXErrorCode(rawValue: inspectError) ?? .EIO) + } + if mkdir(url.path, mode_t(0o700)) != 0, errno != EEXIST { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + guard lstat(url.path, &info) == 0 else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + } + guard info.st_mode & mode_t(S_IFMT) == mode_t(S_IFDIR), + info.st_uid == geteuid() + else { + throw DeviceIdentityStore.storageError( + "Device identity coordinator directory must be a user-owned real directory") + } + guard chmod(url.path, mode_t(0o700)) == 0 else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + guard lstat(url.path, &info) == 0, + info.st_mode & mode_t(S_IFMT) == mode_t(S_IFDIR), + info.st_uid == geteuid(), + info.st_mode & mode_t(0o077) == 0 + else { + throw DeviceIdentityStore.storageError( + "Device identity coordinator directory permissions are not private") + } + } + + private static func canonicalDatabasePath(_ databaseURL: URL) -> String { + let fileManager = FileManager.default + let resolved = databaseURL.standardizedFileURL + var current = resolved + var missingSegments: [String] = [] + while !fileManager.fileExists(atPath: current.path) { + let parent = current.deletingLastPathComponent() + guard parent.path != current.path else { return resolved.path } + missingSegments.insert(current.lastPathComponent, at: 0) + current = parent + } + var canonical = current.resolvingSymlinksInPath().standardizedFileURL + for segment in missingSegments { + canonical.appendPathComponent(segment) + } + return canonical.standardizedFileURL.path + } + + private static func ensureSchema(_ database: OpaquePointer, allowFreshCreation: Bool) throws { + let userVersion = try self.queryInt64(database, sql: "PRAGMA user_version") + guard userVersion <= self.maximumSupportedSchemaVersion else { + let message = + "Device identity database uses newer schema version \(userVersion); " + + "this build supports \(self.maximumSupportedSchemaVersion)" + throw DeviceIdentityStore.storageError(message) + } + + if try !self.schemaObjectExists(database, type: "table", name: self.tableName) { + let objectCount = try self.queryInt64( + database, + sql: "SELECT COUNT(*) FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%'") + guard allowFreshCreation, userVersion == 0, objectCount == 0 else { + throw DeviceIdentityStore.storageError("Nonempty OpenClaw database is missing device_identities") + } + try self.execute(database, sql: self.createSchemaSQL) + } + try self.validateDatabaseOwnership(database, userVersion: userVersion) + + let expectedColumns = [ + Column(name: "identity_key", type: "TEXT", notNull: true, primaryKeyPosition: 1, hidden: 0), + Column(name: "device_id", type: "TEXT", notNull: true, primaryKeyPosition: 0, hidden: 0), + Column(name: "public_key_pem", type: "TEXT", notNull: true, primaryKeyPosition: 0, hidden: 0), + Column(name: "private_key_pem", type: "TEXT", notNull: true, primaryKeyPosition: 0, hidden: 0), + Column(name: "created_at_ms", type: "INTEGER", notNull: true, primaryKeyPosition: 0, hidden: 0), + Column(name: "updated_at_ms", type: "INTEGER", notNull: true, primaryKeyPosition: 0, hidden: 0), + ] + guard try self.tableColumns(database) == expectedColumns else { + throw DeviceIdentityStore.storageError("device_identities has an incompatible schema") + } + let tableSQL = try self.queryText( + database, + sql: "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = 'device_identities'") ?? "" + let normalizedTableSQL = tableSQL + .split(whereSeparator: \.isWhitespace) + .joined(separator: " ") + .uppercased() + guard normalizedTableSQL.hasSuffix(") STRICT") else { + throw DeviceIdentityStore.storageError("device_identities must be a STRICT table") + } + guard try self.validRequiredIndex(database) else { + throw DeviceIdentityStore.storageError("idx_device_identities_device has an incompatible schema") + } + } + + private static func validateDatabaseOwnership( + _ database: OpaquePointer, + userVersion: Int64) throws + { + if userVersion == 0 { + let statement = try self.prepare( + database, + sql: """ + SELECT type, name + FROM sqlite_schema + WHERE name NOT LIKE 'sqlite_%' + ORDER BY type, name + """) + defer { sqlite3_finalize(statement) } + var objects: [(String, String)] = [] + while true { + let result = sqlite3_step(statement) + if result == SQLITE_DONE { break } + guard result == SQLITE_ROW else { + throw self.databaseError(database, operation: "validate Swift identity database ownership") + } + try objects.append(( + self.requiredText(statement, column: 0, field: "schema object type"), + self.requiredText(statement, column: 1, field: "schema object name"))) + } + guard objects.count == 2, + objects[0].0 == "index", objects[0].1 == self.indexName, + objects[1].0 == "table", objects[1].1 == self.tableName + else { + throw DeviceIdentityStore.storageError( + "Schema version zero database contains objects not owned by the Swift identity store") + } + return + } + + let statement = try self.prepare( + database, + sql: "SELECT role, schema_version FROM schema_meta WHERE meta_key = 'primary' LIMIT 1") + defer { sqlite3_finalize(statement) } + guard sqlite3_step(statement) == SQLITE_ROW, + try self.requiredText(statement, column: 0, field: "schema role") == "global", + sqlite3_column_type(statement, 1) == SQLITE_INTEGER, + sqlite3_column_int64(statement, 1) == userVersion, + sqlite3_step(statement) == SQLITE_DONE + else { + throw DeviceIdentityStore.storageError( + "OpenClaw state database schema metadata does not match its global schema version") + } + } + + private static func tableColumns(_ database: OpaquePointer) throws -> [Column] { + let statement = try self.prepare(database, sql: "PRAGMA table_xinfo('device_identities')") + defer { sqlite3_finalize(statement) } + var columns: [Column] = [] + while true { + let result = sqlite3_step(statement) + if result == SQLITE_DONE { return columns } + guard result == SQLITE_ROW else { + throw self.databaseError(database, operation: "inspect device identity columns") + } + try columns.append(Column( + name: self.requiredText(statement, column: 1, field: "column name"), + type: self.requiredText(statement, column: 2, field: "column type").uppercased(), + notNull: sqlite3_column_int(statement, 3) == 1, + primaryKeyPosition: sqlite3_column_int(statement, 5), + hidden: sqlite3_column_int(statement, 6))) + } + } + + private static func validRequiredIndex(_ database: OpaquePointer) throws -> Bool { + let list = try self.prepare(database, sql: "PRAGMA index_list('device_identities')") + defer { sqlite3_finalize(list) } + var found = false + while true { + let result = sqlite3_step(list) + if result == SQLITE_DONE { break } + guard result == SQLITE_ROW else { + throw self.databaseError(database, operation: "inspect device identity indexes") + } + let name = try self.requiredText(list, column: 1, field: "index name") + if name == self.indexName { + found = sqlite3_column_int(list, 2) == 0 && sqlite3_column_int(list, 4) == 0 + } + } + guard found else { return false } + + let details = try self.prepare(database, sql: "PRAGMA index_xinfo('idx_device_identities_device')") + defer { sqlite3_finalize(details) } + var keyColumns: [(String, Bool)] = [] + while true { + let result = sqlite3_step(details) + if result == SQLITE_DONE { break } + guard result == SQLITE_ROW else { + throw self.databaseError(database, operation: "inspect device identity index columns") + } + guard sqlite3_column_int(details, 5) == 1 else { continue } + try keyColumns.append(( + self.requiredText(details, column: 2, field: "index column"), + sqlite3_column_int(details, 3) == 1)) + } + return keyColumns.count == 2 + && keyColumns[0].0 == "device_id" && !keyColumns[0].1 + && keyColumns[1].0 == "updated_at_ms" && keyColumns[1].1 + } + + private static func readIdentity( + _ database: OpaquePointer, + key: String) throws -> DeviceIdentityMaterial? + { + let statement = try self.prepare( + database, + sql: """ + SELECT device_id, public_key_pem, private_key_pem, created_at_ms, updated_at_ms + FROM device_identities + WHERE identity_key = ? + """) + defer { sqlite3_finalize(statement) } + try self.bindText(statement, index: 1, value: key, database: database) + let result = sqlite3_step(statement) + if result == SQLITE_DONE { return nil } + guard result == SQLITE_ROW else { + throw self.databaseError(database, operation: "read device identity") + } + guard sqlite3_column_type(statement, 3) == SQLITE_INTEGER, + sqlite3_column_type(statement, 4) == SQLITE_INTEGER, + sqlite3_column_int64(statement, 4) >= 0 + else { + throw DeviceIdentityStore.storageError("SQLite device identity timestamps must be integers") + } + let material = try DeviceIdentityStore.material( + deviceId: self.requiredText(statement, column: 0, field: "device_id"), + publicKeyPEM: self.requiredText(statement, column: 1, field: "public_key_pem"), + privateKeyPEM: self.requiredText(statement, column: 2, field: "private_key_pem"), + createdAtMs: sqlite3_column_int64(statement, 3)) + guard sqlite3_step(statement) == SQLITE_DONE else { + throw DeviceIdentityStore.storageError("SQLite returned duplicate device identity keys") + } + return material + } + + private static func insertIdentity( + _ database: OpaquePointer, + key: String, + material: DeviceIdentityMaterial, + updatedAtMs: Int64) throws + { + let statement = try self.prepare( + database, + sql: """ + INSERT INTO device_identities ( + identity_key, device_id, public_key_pem, private_key_pem, created_at_ms, updated_at_ms + ) VALUES (?, ?, ?, ?, ?, ?) + """) + defer { sqlite3_finalize(statement) } + try self.bindText(statement, index: 1, value: key, database: database) + try self.bindText(statement, index: 2, value: material.identity.deviceId, database: database) + try self.bindText(statement, index: 3, value: material.publicKeyPEM, database: database) + try self.bindText(statement, index: 4, value: material.privateKeyPEM, database: database) + guard sqlite3_bind_int64(statement, 5, material.identity.createdAtMs) == SQLITE_OK, + sqlite3_bind_int64(statement, 6, updatedAtMs) == SQLITE_OK + else { + throw self.databaseError(database, operation: "bind device identity timestamps") + } + guard sqlite3_step(statement) == SQLITE_DONE, sqlite3_changes(database) == 1 else { + throw self.databaseError(database, operation: "insert device identity") + } + } + + private static func claimLegacyIdentity( + _ source: DeviceIdentityPaths.LegacyIdentitySource, + beforeClaim: ((DeviceIdentityPaths.LegacyIdentitySource) throws -> Void)?) throws -> LegacyClaim? + { + let doctorClaimURL = self.claimURL(source.identityURL, suffix: self.doctorClaimSuffix) + let nativeClaimURL = self.claimURL(source.identityURL, suffix: self.nativeClaimSuffix) + try beforeClaim?(source) + + // Doctor and native startup atomically rename the same source to distinct claims. + // The loser observes a durable winner instead of treating an in-flight identity as absent. + var ownsNativeClaim = false + for _ in 0..<3 { + if self.pathMayExist(doctorClaimURL) { + throw DeviceIdentityStore.storageError( + "Device identity Doctor import is pending; run openclaw doctor --fix before starting the app") + } + if self.pathMayExist(nativeClaimURL) { + guard !self.pathMayExist(source.identityURL) else { + throw DeviceIdentityStore.storageError( + "Legacy device identity source and interrupted native claim both exist") + } + ownsNativeClaim = true + break + } + // Claims first, source last: a Doctor restore moves claim -> source atomically. + guard self.pathMayExist(source.identityURL) else { return nil } + + let renameResult = source.identityURL.path.withCString { sourcePath in + nativeClaimURL.path.withCString { destinationPath in + renamex_np(sourcePath, destinationPath, UInt32(RENAME_EXCL)) + } + } + if renameResult == 0 { + ownsNativeClaim = true + break + } + + let renameError = errno + guard renameError == ENOENT || renameError == EEXIST else { + throw DeviceIdentityStore.storageError( + "Could not claim legacy device identity: \(String(cString: strerror(renameError)))") + } + } + guard ownsNativeClaim else { + throw DeviceIdentityStore.storageError("Legacy device identity changed while being claimed") + } + + do { + let before = try self.legacyFileSnapshot( + nativeClaimURL, + beneath: source.stateDirURL, + maximumBytes: self.maximumLegacyIdentityBytes) + let data = try Data(contentsOf: nativeClaimURL, options: [.mappedIfSafe]) + guard data.count <= self.maximumLegacyIdentityBytes else { + throw DeviceIdentityStore.storageError("Legacy device identity exceeds the maximum supported size") + } + let after = try self.legacyFileSnapshot( + nativeClaimURL, + beneath: source.stateDirURL, + maximumBytes: self.maximumLegacyIdentityBytes) + guard before == after, UInt64(data.count) == before.size else { + throw DeviceIdentityStore.storageError("Legacy device identity changed while being claimed") + } + let material = try DeviceIdentityStore.material(fromLegacyData: data) + return LegacyClaim( + source: source, + identityURL: nativeClaimURL, + data: data, + snapshot: before, + material: material) + } catch { + do { + try self.restoreClaimedLegacyIdentity( + identityURL: nativeClaimURL, + sourceURL: source.identityURL) + } catch let restoreError { + throw DeviceIdentityStore.storageError( + "Legacy device identity validation failed: \(error.localizedDescription); " + + "native claim restoration failed: \(restoreError.localizedDescription)") + } + throw error + } + } + + private static func claimURL(_ sourceURL: URL, suffix: String) -> URL { + URL( + fileURLWithPath: sourceURL.path + suffix, + isDirectory: false) + } + + private static func pathMayExist(_ url: URL) -> Bool { + let fileManager = FileManager.default + return fileManager.fileExists(atPath: url.path) + || (try? fileManager.destinationOfSymbolicLink(atPath: url.path)) != nil + } + + private static func legacyFileSnapshot( + _ url: URL, + beneath rootURL: URL, + maximumBytes: Int) throws -> LegacyFileSnapshot + { + try self.requireNoSymlinkTraversal(url, beneath: rootURL) + let resourceValues = try url.resourceValues(forKeys: [.isSymbolicLinkKey, .isRegularFileKey]) + guard resourceValues.isSymbolicLink != true, resourceValues.isRegularFile == true else { + throw DeviceIdentityStore.storageError("Legacy device identity source must be a regular non-symbolic file") + } + let attributes = try FileManager.default.attributesOfItem(atPath: url.path) + guard attributes[.type] as? FileAttributeType == .typeRegular, + let linkCount = attributes[.referenceCount] as? NSNumber, + linkCount.uint64Value == 1, + let device = attributes[.systemNumber] as? NSNumber, + let inode = attributes[.systemFileNumber] as? NSNumber, + let size = attributes[.size] as? NSNumber, + size.uint64Value <= UInt64(maximumBytes) + else { + throw DeviceIdentityStore.storageError( + "Legacy device identity source must be a bounded regular file with exactly one link") + } + return LegacyFileSnapshot( + device: device.uint64Value, + inode: inode.uint64Value, + size: size.uint64Value, + modifiedAt: attributes[.modificationDate] as? Date) + } + + private static func requireNoSymlinkTraversal(_ url: URL, beneath rootURL: URL) throws { + let root = rootURL.standardizedFileURL + let candidate = url.standardizedFileURL + let rootPrefix = root.path.hasSuffix("/") ? root.path : root.path + "/" + guard candidate.path.hasPrefix(rootPrefix) else { + throw DeviceIdentityStore.storageError("Legacy device identity path escaped its state directory") + } + let relativePath = String(candidate.path.dropFirst(rootPrefix.count)) + let expected = root.resolvingSymlinksInPath() + .appendingPathComponent(relativePath, isDirectory: false) + .standardizedFileURL + guard candidate.resolvingSymlinksInPath().standardizedFileURL == expected else { + throw DeviceIdentityStore.storageError("Legacy device identity path must not traverse symbolic links") + } + } + + private static func requireConsistentClaims(_ claims: [LegacyClaim]) throws { + guard let first = claims.first else { return } + guard claims.dropFirst().allSatisfy({ self.hasSameKeyMaterial($0.material, first.material) }) else { + throw DeviceIdentityStore.storageError("Legacy device identity sources conflict; all sources preserved") + } + } + + private static func hasSameKeyMaterial( + _ lhs: DeviceIdentityMaterial, + _ rhs: DeviceIdentityMaterial) -> Bool + { + lhs.identity.deviceId == rhs.identity.deviceId + && lhs.identity.publicKey == rhs.identity.publicKey + && lhs.identity.privateKey == rhs.identity.privateKey + } + + private static func relocateLegacyAuthIfNeeded( + claims: [LegacyClaim], + destinationStateDirURL: URL, + profile: GatewayDeviceIdentityProfile, + deviceId: String) throws + { + let fileManager = FileManager.default + let destinationIdentityDirURL = destinationStateDirURL + .appendingPathComponent("identity", isDirectory: true) + let destinationAuthURL = destinationIdentityDirURL + .appendingPathComponent(profile.authFileName, isDirectory: false) + let sourceAuth = try claims.compactMap { claim -> LegacyAuthCandidate? in + let source = claim.source + guard source.stateDirURL.standardizedFileURL != destinationStateDirURL.standardizedFileURL, + fileManager.fileExists(atPath: source.authURL.path) + else { return nil } + return try self.readDeviceAuth( + source.authURL, + beneath: source.stateDirURL, + deviceId: deviceId) + } + if let firstSourceAuth = sourceAuth.first, + !sourceAuth.dropFirst().allSatisfy({ $0.store == firstSourceAuth.store }) + { + throw DeviceIdentityStore.storageError( + "Legacy device auth sources conflict; all identity sources preserved") + } + if fileManager.fileExists(atPath: destinationAuthURL.path) { + let destinationAuth = try self.readDeviceAuth( + destinationAuthURL, + beneath: destinationStateDirURL, + deviceId: deviceId) + guard sourceAuth.allSatisfy({ $0.store == destinationAuth.store }) else { + throw DeviceIdentityStore.storageError( + "Destination device auth differs from legacy auth; identity source preserved") + } + return + } + guard let selectedAuth = sourceAuth.first else { return } + + // DeviceAuthStore remains file-backed. Copy it when identity ownership moves between + // Apple containers, but never delete or rewrite the source auth file. + try self.secureDirectory(destinationIdentityDirURL) + let temporaryAuthURL = destinationIdentityDirURL.appendingPathComponent( + ".\(profile.authFileName).identity-migrating-\(UUID().uuidString)", + isDirectory: false) + defer { try? fileManager.removeItem(at: temporaryAuthURL) } + try selectedAuth.data.write(to: temporaryAuthURL, options: [.atomic]) + try self.secureFile(temporaryAuthURL) + + // Publish only complete bytes, and never replace a token another process won first. + // Foundation rejects atomic + withoutOverwriting, so use Darwin's exclusive rename. + let renameResult = temporaryAuthURL.path.withCString { sourcePath in + destinationAuthURL.path.withCString { destinationPath in + renamex_np(sourcePath, destinationPath, UInt32(RENAME_EXCL)) + } + } + if renameResult != 0 { + let renameError = errno + guard renameError == EEXIST else { + throw DeviceIdentityStore.storageError( + "Could not publish migrated device auth: \(String(cString: strerror(renameError)))") + } + let destinationAuth = try self.readDeviceAuth( + destinationAuthURL, + beneath: destinationStateDirURL, + deviceId: deviceId) + guard destinationAuth.store == selectedAuth.store else { + throw DeviceIdentityStore.storageError( + "Concurrently created device auth differs from legacy auth; identity source preserved") + } + return + } + try self.secureFile(destinationAuthURL) + } + + private static func readDeviceAuth( + _ url: URL, + beneath stateDirURL: URL, + deviceId: String) throws -> LegacyAuthCandidate + { + let before = try self.legacyFileSnapshot( + url, + beneath: stateDirURL, + maximumBytes: self.maximumLegacyAuthBytes) + let data = try Data(contentsOf: url, options: [.mappedIfSafe]) + let after = try self.legacyFileSnapshot( + url, + beneath: stateDirURL, + maximumBytes: self.maximumLegacyAuthBytes) + guard before == after, UInt64(data.count) == before.size else { + throw DeviceIdentityStore.storageError("Device auth changed during identity migration") + } + guard let decoded = try? JSONDecoder().decode(DeviceAuthStoreFile.self, from: data), + let normalized = DeviceAuthStore.normalizedStore(decoded), + normalized.deviceId == deviceId + else { + throw DeviceIdentityStore.storageError( + "Device auth does not belong to the migrated device identity; source preserved") + } + return LegacyAuthCandidate(data: data, store: normalized) + } + + private static func removeClaimedLegacyIdentities(_ claims: [LegacyClaim]) throws { + let fileManager = FileManager.default + for claim in claims { + guard !self.pathMayExist(claim.source.identityURL) else { + throw DeviceIdentityStore.storageError( + "Legacy device identity source reappeared during migration; native claim preserved") + } + guard fileManager.fileExists(atPath: claim.identityURL.path) else { + if (try? fileManager.destinationOfSymbolicLink(atPath: claim.identityURL.path)) != nil { + throw DeviceIdentityStore.storageError( + "Legacy device identity changed to a symbolic link; source preserved") + } + continue + } + let snapshot = try self.legacyFileSnapshot( + claim.identityURL, + beneath: claim.source.stateDirURL, + maximumBytes: self.maximumLegacyIdentityBytes) + let current = try Data(contentsOf: claim.identityURL, options: [.mappedIfSafe]) + guard snapshot == claim.snapshot, current == claim.data else { + throw DeviceIdentityStore + .storageError("Legacy device identity changed during migration; source preserved") + } + } + for claim in claims where fileManager.fileExists(atPath: claim.identityURL.path) { + try fileManager.removeItem(at: claim.identityURL) + } + } + + private static func restoreClaimedLegacyIdentities(_ claims: [LegacyClaim]) throws { + var restorationErrors: [String] = [] + for claim in claims.reversed() { + do { + try self.restoreClaimedLegacyIdentity( + identityURL: claim.identityURL, + sourceURL: claim.source.identityURL) + } catch { + restorationErrors.append("\(claim.source.identityURL.path): \(error.localizedDescription)") + } + } + if !restorationErrors.isEmpty { + throw DeviceIdentityStore.storageError( + "Could not restore every native device identity claim: " + + restorationErrors.joined(separator: "; ")) + } + } + + private static func restoreClaimedLegacyIdentity(identityURL: URL, sourceURL: URL) throws { + guard self.pathMayExist(identityURL) else { return } + let renameResult = identityURL.path.withCString { claimedPath in + sourceURL.path.withCString { destinationPath in + renamex_np(claimedPath, destinationPath, UInt32(RENAME_EXCL)) + } + } + guard renameResult == 0 else { + let renameError = errno + if renameError == ENOENT, !self.pathMayExist(identityURL) { + return + } + throw DeviceIdentityStore.storageError( + "Could not restore legacy device identity: \(String(cString: strerror(renameError)))") + } + } + + private static func schemaObjectExists( + _ database: OpaquePointer, + type: String, + name: String) throws -> Bool + { + let statement = try self.prepare( + database, + sql: "SELECT 1 FROM sqlite_schema WHERE type = ? AND name = ? LIMIT 1") + defer { sqlite3_finalize(statement) } + try self.bindText(statement, index: 1, value: type, database: database) + try self.bindText(statement, index: 2, value: name, database: database) + let result = sqlite3_step(statement) + if result == SQLITE_ROW { return true } + if result == SQLITE_DONE { return false } + throw self.databaseError(database, operation: "inspect SQLite schema") + } + + private static func queryInt64(_ database: OpaquePointer, sql: String) throws -> Int64 { + let statement = try self.prepare(database, sql: sql) + defer { sqlite3_finalize(statement) } + guard sqlite3_step(statement) == SQLITE_ROW, + sqlite3_column_type(statement, 0) == SQLITE_INTEGER + else { + throw self.databaseError(database, operation: "read SQLite integer") + } + let value = sqlite3_column_int64(statement, 0) + guard sqlite3_step(statement) == SQLITE_DONE else { + throw DeviceIdentityStore.storageError("SQLite integer query returned multiple rows") + } + return value + } + + private static func queryText(_ database: OpaquePointer, sql: String) throws -> String? { + let statement = try self.prepare(database, sql: sql) + defer { sqlite3_finalize(statement) } + let result = sqlite3_step(statement) + if result == SQLITE_DONE { return nil } + guard result == SQLITE_ROW else { + throw self.databaseError(database, operation: "read SQLite text") + } + let value = try self.requiredText(statement, column: 0, field: "query result") + guard sqlite3_step(statement) == SQLITE_DONE else { + throw DeviceIdentityStore.storageError("SQLite text query returned multiple rows") + } + return value + } + + private static func execute(_ database: OpaquePointer, sql: String) throws { + var errorMessage: UnsafeMutablePointer? + let result = sqlite3_exec(database, sql, nil, nil, &errorMessage) + guard result == SQLITE_OK else { + let detail = errorMessage.map { String(cString: $0) } ?? String(cString: sqlite3_errmsg(database)) + sqlite3_free(errorMessage) + throw DeviceIdentityStore.storageError("SQLite operation failed: \(detail)") + } + } + + private static func prepare(_ database: OpaquePointer, sql: String) throws -> OpaquePointer { + var statement: OpaquePointer? + guard sqlite3_prepare_v2(database, sql, -1, &statement, nil) == SQLITE_OK, let statement else { + throw self.databaseError(database, operation: "prepare SQLite statement") + } + return statement + } + + private static func bindText( + _ statement: OpaquePointer, + index: Int32, + value: String, + database: OpaquePointer) throws + { + let transient = unsafeBitCast(-1, to: sqlite3_destructor_type.self) + guard sqlite3_bind_text(statement, index, value, -1, transient) == SQLITE_OK else { + throw self.databaseError(database, operation: "bind SQLite text") + } + } + + private static func requiredText( + _ statement: OpaquePointer, + column: Int32, + field: String) throws -> String + { + guard sqlite3_column_type(statement, column) == SQLITE_TEXT, + let pointer = sqlite3_column_text(statement, column) + else { + throw DeviceIdentityStore.storageError("SQLite \(field) must be text") + } + return String(cString: pointer) + } + + private static func databaseError( + _ database: OpaquePointer, + operation: String) -> NSError + { + DeviceIdentityStore.storageError("Could not \(operation): \(String(cString: sqlite3_errmsg(database)))") + } +} + +extension DeviceIdentitySQLiteStore { + private static func secureDirectory(_ url: URL) throws { + let fileManager = FileManager.default + try fileManager.createDirectory(at: url, withIntermediateDirectories: true) + var attributes: [FileAttributeKey: Any] = [.posixPermissions: 0o700] + #if os(iOS) || os(watchOS) + attributes[.protectionKey] = FileProtectionType.completeUntilFirstUserAuthentication + #endif + try fileManager.setAttributes(attributes, ofItemAtPath: url.path) + } + + private static func secureFile(_ url: URL) throws { + guard FileManager.default.fileExists(atPath: url.path) else { return } + var attributes: [FileAttributeKey: Any] = [.posixPermissions: 0o600] + #if os(iOS) || os(watchOS) + attributes[.protectionKey] = FileProtectionType.completeUntilFirstUserAuthentication + #endif + try FileManager.default.setAttributes(attributes, ofItemAtPath: url.path) + } + + private static func secureDatabaseFiles(_ databaseURL: URL) throws { + try self.secureFile(databaseURL) + for suffix in ["-wal", "-shm", "-journal"] { + try self.secureFile(URL(fileURLWithPath: databaseURL.path + suffix, isDirectory: false)) + } + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayChannel.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayChannel.swift index 26f69bb939ed..c569fda36bf0 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayChannel.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayChannel.swift @@ -430,6 +430,20 @@ public actor GatewayChannelActor { } } + private static func loadDeviceIdentityForConnect( + includeDeviceIdentity: Bool, + profile: GatewayDeviceIdentityProfile) throws -> DeviceIdentity? + { + guard includeDeviceIdentity else { return nil } + guard let identity = DeviceIdentityStore.loadOrCreatePersisted(profile: profile) else { + throw NSError( + domain: "Gateway", + code: 3, + userInfo: [NSLocalizedDescriptionKey: "Could not access the persisted device identity"]) + } + return identity + } + private func sendConnect( task: WebSocketTaskBox, attemptID: UUID, @@ -460,7 +474,9 @@ public actor GatewayChannelActor { let includeDeviceIdentity = options.includeDeviceIdentity let allowStoredDeviceAuth = options.allowStoredDeviceAuth let deviceAuthGatewayID = options.deviceAuthGatewayID - let identity = includeDeviceIdentity ? DeviceIdentityStore.loadOrCreate(profile: deviceIdentityProfile) : nil + let identity = try Self.loadDeviceIdentityForConnect( + includeDeviceIdentity: includeDeviceIdentity, + profile: deviceIdentityProfile) let selectedAuth = self.selectConnectAuth( role: role, includeDeviceIdentity: includeDeviceIdentity, diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/ShareGatewayRelaySettings.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/ShareGatewayRelaySettings.swift index b31a2eb65423..f7489b4cfb2c 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawKit/ShareGatewayRelaySettings.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/ShareGatewayRelaySettings.swift @@ -52,7 +52,9 @@ public enum ShareGatewayRelaySettings { if config.gatewayStableID?.isEmpty == false { return config } - let identity = DeviceIdentityStore.loadOrCreate(profile: .shareExtension) + guard let identity = DeviceIdentityStore.loadOrCreatePersisted(profile: .shareExtension) else { + return config + } DeviceAuthStore.discardUnscopedTokens( deviceId: identity.deviceId, profile: .shareExtension) diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/DeviceAuthPayloadTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/DeviceAuthPayloadTests.swift index 6c1a641b0182..63c67b30a72d 100644 --- a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/DeviceAuthPayloadTests.swift +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/DeviceAuthPayloadTests.swift @@ -46,9 +46,12 @@ struct DeviceAuthPayloadTests { func `signed device dictionary preserves 64-bit timestamp`() throws { let tempDir = FileManager.default.temporaryDirectory .appendingPathComponent(UUID().uuidString, isDirectory: true) - let identityURL = tempDir.appendingPathComponent("device.json", isDirectory: false) + let databaseURL = tempDir.appendingPathComponent("openclaw.sqlite", isDirectory: false) defer { try? FileManager.default.removeItem(at: tempDir) } - let identity = DeviceIdentityStore.loadOrCreate(fileURL: identityURL) + let identity = try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: tempDir, + profile: .primary) let signedAtMs: Int64 = 1_800_000_000_000 let payload = GatewayDeviceAuthPayload.buildV3( fields: .init( diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/DeviceIdentityStoreTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/DeviceIdentityStoreTests.swift index f3fb7f5a2495..f4780102b113 100644 --- a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/DeviceIdentityStoreTests.swift +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/DeviceIdentityStoreTests.swift @@ -1,5 +1,6 @@ import CryptoKit import Foundation +import SQLite3 import Testing @testable import OpenClawKit @@ -35,7 +36,7 @@ struct DeviceIdentityStoreTests { #expect(observed.count == stateDirectories.count) for stateDirectory in stateDirectories { #expect(FileManager.default.fileExists( - atPath: stateDirectory.appendingPathComponent("identity/device.json").path)) + atPath: stateDirectory.appendingPathComponent("state/openclaw.sqlite").path)) } } @@ -91,11 +92,18 @@ struct DeviceIdentityStoreTests { func `durable identity creation verifies persisted key material`() throws { let tempDir = FileManager.default.temporaryDirectory .appendingPathComponent(UUID().uuidString, isDirectory: true) - let identityURL = tempDir.appendingPathComponent("device.json", isDirectory: false) + let databaseURL = tempDir.appendingPathComponent("openclaw.sqlite", isDirectory: false) defer { try? FileManager.default.removeItem(at: tempDir) } - let identity = try #require(DeviceIdentityStore.loadOrCreatePersisted(fileURL: identityURL)) - let reloaded = DeviceIdentityStore.loadOrCreate(fileURL: identityURL) + let identity = try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: tempDir, + profile: .primary, + legacySources: []) + let reloaded = try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: tempDir, + profile: .primary) #expect(reloaded.deviceId == identity.deviceId) #expect(reloaded.publicKey == identity.publicKey) @@ -353,8 +361,19 @@ struct DeviceIdentityStoreTests { #expect(selected == legacyURL) } + @Test + func `task scoped state directory never probes machine legacy roots`() async { + let scopedURL = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + + await DeviceIdentityStore.withStateDirectory(scopedURL) { + let sources = DeviceIdentityPaths.legacyIdentitySources(profile: .primary) + #expect(sources.map(\.stateDirURL) == [scopedURL.standardizedFileURL]) + } + } + @Test(.stateDirectoryIsolated) - func `secondary profiles use separate identity and auth files`() throws { + func `secondary profiles use separate identity rows and auth files`() throws { let primaryIdentity = DeviceIdentityStore.loadOrCreate() let nodeIdentity = DeviceIdentityStore.loadOrCreate(profile: .node) let shareIdentity = DeviceIdentityStore.loadOrCreate(profile: .shareExtension) @@ -376,13 +395,13 @@ struct DeviceIdentityStoreTests { // getenv, not ProcessInfo: the trait pins OPENCLAW_STATE_DIR via setenv and // ProcessInfo.environment can serve a stale snapshot on Darwin. let stateDirPath = try #require(getenv("OPENCLAW_STATE_DIR").map { String(cString: $0) }) - let identityDir = URL(fileURLWithPath: stateDirPath, isDirectory: true) - .appendingPathComponent("identity", isDirectory: true) + let stateDir = URL(fileURLWithPath: stateDirPath, isDirectory: true) + let identityDir = stateDir.appendingPathComponent("identity", isDirectory: true) #expect(primaryIdentity.deviceId != nodeIdentity.deviceId) #expect(primaryIdentity.deviceId != shareIdentity.deviceId) - #expect(FileManager.default.fileExists(atPath: identityDir.appendingPathComponent("device.json").path)) - #expect(FileManager.default.fileExists(atPath: identityDir.appendingPathComponent("node-device.json").path)) - #expect(FileManager.default.fileExists(atPath: identityDir.appendingPathComponent("share-device.json").path)) + #expect(try Self.scalarInt( + stateDir.appendingPathComponent("state/openclaw.sqlite"), + "SELECT COUNT(*) FROM device_identities") == 3) #expect(FileManager.default.fileExists(atPath: identityDir.appendingPathComponent("device-auth.json").path)) #expect(FileManager.default .fileExists(atPath: identityDir.appendingPathComponent("node-device-auth.json").path)) @@ -410,257 +429,633 @@ struct DeviceIdentityStoreTests { } @Test - func `loads TypeScript PEM identity schema without rewriting or regenerating`() throws { + func `fresh database creates only canonical identity schema and leaves user version zero`() throws { let tempDir = FileManager.default.temporaryDirectory .appendingPathComponent(UUID().uuidString, isDirectory: true) - let identityURL = tempDir - .appendingPathComponent("identity", isDirectory: true) - .appendingPathComponent("device.json", isDirectory: false) defer { try? FileManager.default.removeItem(at: tempDir) } - try FileManager.default.createDirectory( - at: identityURL.deletingLastPathComponent(), - withIntermediateDirectories: true) - let stored = try Self.identityJSON( - publicKeyPem: Self.pem( - label: "PUBLIC KEY", - body: "MCowBQYDK2VwAyEAA6EHv/POEL4dcN0Y50vAmWfk1jCbpQ1fHdyGZBJVMbg="), - privateKeyPem: Self.pem( - label: "PRIVATE KEY", - body: "MC4CAQAwBQYDK2VwBCIEIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4f")) - try stored.write(to: identityURL, atomically: true, encoding: .utf8) - let before = try String(contentsOf: identityURL, encoding: .utf8) + let databaseURL = tempDir.appendingPathComponent("openclaw.sqlite", isDirectory: false) - let identity = DeviceIdentityStore.loadOrCreate(fileURL: identityURL) + _ = try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: tempDir, + profile: .primary) - #expect(identity.deviceId == "56475aa75463474c0285df5dbf2bcab73da651358839e9b77481b2eab107708c") - #expect(identity.publicKey == "A6EHv/POEL4dcN0Y50vAmWfk1jCbpQ1fHdyGZBJVMbg=") - #expect(identity.privateKey == "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=") + #expect(try Self.scalarInt(databaseURL, "PRAGMA user_version") == 0) + #expect(try Self.scalarText( + databaseURL, + """ + SELECT group_concat(type || ':' || name, ',') + FROM (SELECT type, name FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name) + """) == "index:idx_device_identities_device,table:device_identities") + let directoryMode = try #require( + FileManager.default.attributesOfItem(atPath: tempDir.path)[.posixPermissions] as? NSNumber) + let databaseMode = try #require( + FileManager.default.attributesOfItem(atPath: databaseURL.path)[.posixPermissions] as? NSNumber) + #expect(directoryMode.intValue & 0o777 == 0o700) + #expect(databaseMode.intValue & 0o777 == 0o600) + } + + @Test + func `Node PEM fixture repairs a stale device id and remains signing compatible`() throws { + let tempDir = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let sourceRoot = tempDir.appendingPathComponent("Application Support/OpenClaw", isDirectory: true) + let source = try Self.writeLegacyIdentity( + stateDirURL: sourceRoot, + profile: .primary, + contents: Self.nodePEMIdentityJSON(deviceId: "stale-device-id")) + let databaseURL = tempDir.appendingPathComponent("destination/openclaw.sqlite", isDirectory: false) + let destination = databaseURL.deletingLastPathComponent() + + let identity = try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: destination, + profile: .primary, + legacySources: [source]) + + #expect(identity.deviceId == Self.fixtureDeviceID) + #expect(identity.publicKey == Self.fixturePublicKeyRaw) + #expect(identity.privateKey == Self.fixturePrivateKeyRaw) #expect(identity.createdAtMs == 1_800_000_000_000) + #expect(!FileManager.default.fileExists(atPath: source.identityURL.path)) + #expect(try Self.scalarText( + databaseURL, + "SELECT public_key_pem FROM device_identities WHERE identity_key = 'primary'") == Self.fixturePublicKeyPEM) + #expect(try Self.scalarText( + databaseURL, + "SELECT private_key_pem FROM device_identities WHERE identity_key = 'primary'") == Self + .fixturePrivateKeyPEM) #expect(DeviceIdentityStore.publicKeyBase64Url(identity) == "A6EHv_POEL4dcN0Y50vAmWfk1jCbpQ1fHdyGZBJVMbg") let signature = try #require(DeviceIdentityStore.signPayload("hello", identity: identity)) - let publicKeyData = try #require(Data(base64Encoded: identity.publicKey)) - let signatureData = try #require(Self.base64UrlDecode(signature)) - let publicKey = try Curve25519.Signing.PublicKey(rawRepresentation: publicKeyData) - #expect(publicKey.isValidSignature(signatureData, for: Data("hello".utf8))) - #expect(try String(contentsOf: identityURL, encoding: .utf8) == before) + let publicKey = try Curve25519.Signing.PublicKey( + rawRepresentation: #require(Data(base64Encoded: identity.publicKey))) + #expect(try publicKey.isValidSignature( + #require(Self.base64UrlDecode(signature)), + for: Data("hello".utf8))) } @Test - func `does not overwrite a recognized invalid TypeScript identity schema`() throws { + func `reads a canonical Node SQLite row without rewriting it`() throws { let tempDir = FileManager.default.temporaryDirectory .appendingPathComponent(UUID().uuidString, isDirectory: true) - let identityURL = tempDir - .appendingPathComponent("identity", isDirectory: true) - .appendingPathComponent("device.json", isDirectory: false) defer { try? FileManager.default.removeItem(at: tempDir) } - try FileManager.default.createDirectory( - at: identityURL.deletingLastPathComponent(), - withIntermediateDirectories: true) - let stored = """ - { - "version": 1, - "deviceId": "stale-device-id", - "publicKeyPem": "not-a-valid-public-key", - "privateKeyPem": "not-a-valid-private-key", - "createdAtMs": 1700000000000 + let databaseURL = tempDir.appendingPathComponent("openclaw.sqlite", isDirectory: false) + try Self.seedCanonicalSchema(databaseURL, nodeOwned: true) + try Self.execute(databaseURL, """ + INSERT INTO device_identities ( + identity_key, device_id, public_key_pem, private_key_pem, created_at_ms, updated_at_ms + ) VALUES ( + 'node', '\(Self.fixtureDeviceID)', '\(Self.sql(Self.fixturePublicKeyPEM))', + '\(Self.sql(Self.fixturePrivateKeyPEM))', 1800000000000, 1800000000123 + ) + """) + + let identity = try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: tempDir, + profile: .node) + + #expect(identity.deviceId == Self.fixtureDeviceID) + #expect(identity.createdAtMs == 1_800_000_000_000) + #expect(try Self.scalarInt( + databaseURL, + "SELECT updated_at_ms FROM device_identities WHERE identity_key = 'node'") == 1_800_000_000_123) + } + + @Test + func `same key migration preserves the authoritative SQLite timestamp`() throws { + let tempDir = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let databaseURL = tempDir.appendingPathComponent("openclaw.sqlite", isDirectory: false) + try Self.seedCanonicalSchema(databaseURL, nodeOwned: true) + try Self.execute(databaseURL, """ + INSERT INTO device_identities ( + identity_key, device_id, public_key_pem, private_key_pem, created_at_ms, updated_at_ms + ) VALUES ( + 'primary', '\(Self.fixtureDeviceID)', '\(Self.sql(Self.fixturePublicKeyPEM))', + '\(Self.sql(Self.fixturePrivateKeyPEM))', 1700000000000, 1700000000123 + ) + """) + let source = try Self.writeLegacyIdentity( + stateDirURL: tempDir.appendingPathComponent("legacy", isDirectory: true), + profile: .primary, + contents: Self.nodePEMIdentityJSON()) + + let identity = try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: tempDir, + profile: .primary, + legacySources: [source]) + + #expect(identity.deviceId == Self.fixtureDeviceID) + #expect(identity.createdAtMs == 1_700_000_000_000) + #expect(try Self.scalarInt( + databaseURL, + "SELECT updated_at_ms FROM device_identities WHERE identity_key = 'primary'") == 1_700_000_000_123) + #expect(!FileManager.default.fileExists(atPath: source.identityURL.path)) + } + + @Test + func `strict legacy validation preserves invalid source and creates no row`() throws { + let tempDir = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let invalidJSON = try Self.nodePEMIdentityJSON(privateKeyPem: "not-a-private-key") + let source = try Self.writeLegacyIdentity( + stateDirURL: tempDir.appendingPathComponent("shared", isDirectory: true), + profile: .primary, + contents: invalidJSON) + let databaseURL = tempDir.appendingPathComponent("destination/openclaw.sqlite") + + #expect(throws: NSError.self) { + try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: databaseURL.deletingLastPathComponent(), + profile: .primary, + legacySources: [source]) } - """ - try stored.write(to: identityURL, atomically: true, encoding: .utf8) - let before = try String(contentsOf: identityURL, encoding: .utf8) - - let identity = DeviceIdentityStore.loadOrCreate(fileURL: identityURL) - - #expect(identity.deviceId != "stale-device-id") - #expect(try String(contentsOf: identityURL, encoding: .utf8) == before) + #expect(FileManager.default.fileExists(atPath: source.identityURL.path)) + #expect(!FileManager.default.fileExists(atPath: databaseURL.path)) } @Test - func `does not overwrite an existing unrecognized identity file`() throws { + func `pending Doctor claim blocks identity generation`() throws { let tempDir = FileManager.default.temporaryDirectory .appendingPathComponent(UUID().uuidString, isDirectory: true) - let identityURL = tempDir - .appendingPathComponent("identity", isDirectory: true) - .appendingPathComponent("device.json", isDirectory: false) defer { try? FileManager.default.removeItem(at: tempDir) } - try FileManager.default.createDirectory( - at: identityURL.deletingLastPathComponent(), - withIntermediateDirectories: true) - let stored = """ - { - "schema": "future-openclaw-device-identity", - "stableDeviceId": "app-group-device-id" + let sourceRoot = tempDir.appendingPathComponent("legacy", isDirectory: true) + let source = try Self.writeLegacyIdentity( + stateDirURL: sourceRoot, + profile: .primary, + contents: Self.nodePEMIdentityJSON()) + let claimURL = URL( + fileURLWithPath: source.identityURL.path + ".doctor-importing", + isDirectory: false) + try FileManager.default.moveItem(at: source.identityURL, to: claimURL) + let destination = tempDir.appendingPathComponent("destination", isDirectory: true) + let databaseURL = destination.appendingPathComponent("openclaw.sqlite", isDirectory: false) + + #expect(throws: NSError.self) { + try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: destination, + profile: .primary, + legacySources: [source]) } + + #expect(FileManager.default.fileExists(atPath: claimURL.path)) + #expect(!FileManager.default.fileExists(atPath: databaseURL.path)) + } + + @Test + func `Doctor rename winning native claim race blocks identity generation`() throws { + let tempDir = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let source = try Self.writeLegacyIdentity( + stateDirURL: tempDir.appendingPathComponent("legacy", isDirectory: true), + profile: .primary, + contents: Self.nodePEMIdentityJSON()) + let claimURL = URL( + fileURLWithPath: source.identityURL.path + ".doctor-importing", + isDirectory: false) + let destination = tempDir.appendingPathComponent("destination", isDirectory: true) + let databaseURL = destination.appendingPathComponent("openclaw.sqlite", isDirectory: false) + + #expect(throws: NSError.self) { + try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: destination, + profile: .primary, + legacySources: [source], + beforeLegacyClaim: { _ in + try FileManager.default.moveItem(at: source.identityURL, to: claimURL) + }) + } + + #expect(FileManager.default.fileExists(atPath: claimURL.path)) + #expect(!FileManager.default.fileExists(atPath: databaseURL.path)) + } + + @Test + func `interrupted native claim resumes without rotating identity`() throws { + let tempDir = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let source = try Self.writeLegacyIdentity( + stateDirURL: tempDir.appendingPathComponent("legacy", isDirectory: true), + profile: .primary, + contents: Self.nodePEMIdentityJSON()) + let claimURL = URL( + fileURLWithPath: source.identityURL.path + ".native-importing", + isDirectory: false) + try FileManager.default.moveItem(at: source.identityURL, to: claimURL) + let destination = tempDir.appendingPathComponent("destination", isDirectory: true) + let databaseURL = destination.appendingPathComponent("openclaw.sqlite", isDirectory: false) + + let identity = try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: destination, + profile: .primary, + legacySources: [source]) + + #expect(identity.deviceId == Self.fixtureDeviceID) + #expect(!FileManager.default.fileExists(atPath: source.identityURL.path)) + #expect(!FileManager.default.fileExists(atPath: claimURL.path)) + } + + @Test + func `source reappearance preserves both native claim and recreated source`() throws { + let tempDir = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let legacyData = try Self.nodePEMIdentityJSON() + let source = try Self.writeLegacyIdentity( + stateDirURL: tempDir.appendingPathComponent("legacy", isDirectory: true), + profile: .primary, + contents: legacyData) + let claimURL = URL( + fileURLWithPath: source.identityURL.path + ".native-importing", + isDirectory: false) + let destination = tempDir.appendingPathComponent("destination", isDirectory: true) + let databaseURL = destination.appendingPathComponent("openclaw.sqlite", isDirectory: false) + + #expect(throws: NSError.self) { + try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: destination, + profile: .primary, + legacySources: [source], + afterLegacyCommit: { + try legacyData.write(to: source.identityURL, atomically: true, encoding: .utf8) + }) + } + + #expect(FileManager.default.fileExists(atPath: source.identityURL.path)) + #expect(FileManager.default.fileExists(atPath: claimURL.path)) + #expect(try Self.scalarText( + databaseURL, + "SELECT device_id FROM device_identities WHERE identity_key = 'primary'") == Self.fixtureDeviceID) + } + + @Test + func `multi source rollback restores every independent native claim`() throws { + let tempDir = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let legacyData = try Self.nodePEMIdentityJSON() + let first = try Self.writeLegacyIdentity( + stateDirURL: tempDir.appendingPathComponent("first", isDirectory: true), + profile: .primary, + contents: legacyData) + let second = try Self.writeLegacyIdentity( + stateDirURL: tempDir.appendingPathComponent("second", isDirectory: true), + profile: .primary, + contents: legacyData) + let firstClaimURL = URL( + fileURLWithPath: first.identityURL.path + ".native-importing", + isDirectory: false) + let secondClaimURL = URL( + fileURLWithPath: second.identityURL.path + ".native-importing", + isDirectory: false) + let destination = tempDir.appendingPathComponent("destination", isDirectory: true) + let databaseURL = destination.appendingPathComponent("openclaw.sqlite", isDirectory: false) + + #expect(throws: NSError.self) { + try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: destination, + profile: .primary, + legacySources: [first, second], + afterLegacyCommit: { + try legacyData.write(to: second.identityURL, atomically: true, encoding: .utf8) + }) + } + + #expect(FileManager.default.fileExists(atPath: first.identityURL.path)) + #expect(!FileManager.default.fileExists(atPath: firstClaimURL.path)) + #expect(FileManager.default.fileExists(atPath: second.identityURL.path)) + #expect(FileManager.default.fileExists(atPath: secondClaimURL.path)) + } + + @Test + func `post commit identity change restores native claim before cleanup`() throws { + let tempDir = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let source = try Self.writeLegacyIdentity( + stateDirURL: tempDir.appendingPathComponent("legacy", isDirectory: true), + profile: .primary, + contents: Self.nodePEMIdentityJSON()) + let claimURL = URL( + fileURLWithPath: source.identityURL.path + ".native-importing", + isDirectory: false) + let destination = tempDir.appendingPathComponent("destination", isDirectory: true) + let databaseURL = destination.appendingPathComponent("openclaw.sqlite", isDirectory: false) + + #expect(throws: NSError.self) { + try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: destination, + profile: .primary, + legacySources: [source], + afterLegacyCommit: { + try Self.execute( + databaseURL, + "UPDATE device_identities SET device_id = 'tampered' WHERE identity_key = 'primary'") + }) + } + + #expect(FileManager.default.fileExists(atPath: source.identityURL.path)) + #expect(!FileManager.default.fileExists(atPath: claimURL.path)) + #expect(try Self.scalarText( + databaseURL, + "SELECT device_id FROM device_identities WHERE identity_key = 'primary'") == "tampered") + } + + @Test + func `legacy claim rejects symbolic traversal and hard linked sources`() throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: root) } + let realRoot = root.appendingPathComponent("real", isDirectory: true) + let real = try Self.writeLegacyIdentity( + stateDirURL: realRoot, + profile: .primary, + contents: Self.nodePEMIdentityJSON()) + let symbolicRoot = root.appendingPathComponent("symbolic", isDirectory: true) + let symbolic = Self.legacyIdentitySource(stateDirURL: symbolicRoot, profile: .primary) + try FileManager.default.createDirectory( + at: symbolic.identityURL.deletingLastPathComponent(), + withIntermediateDirectories: true) + try FileManager.default.createSymbolicLink(at: symbolic.identityURL, withDestinationURL: real.identityURL) + #expect(throws: NSError.self) { + try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: root.appendingPathComponent("symbolic.sqlite"), + destinationStateDirURL: root, + profile: .primary, + legacySources: [symbolic]) + } + + let traversingRoot = root.appendingPathComponent("traversing", isDirectory: true) + let traversing = Self.legacyIdentitySource(stateDirURL: traversingRoot, profile: .primary) + try FileManager.default.createDirectory(at: traversingRoot, withIntermediateDirectories: true) + try FileManager.default.createSymbolicLink( + at: traversing.identityURL.deletingLastPathComponent(), + withDestinationURL: real.identityURL.deletingLastPathComponent()) + #expect(throws: NSError.self) { + try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: root.appendingPathComponent("traversing.sqlite"), + destinationStateDirURL: root, + profile: .primary, + legacySources: [traversing]) + } + + let hardRoot = root.appendingPathComponent("hard", isDirectory: true) + let hard = Self.legacyIdentitySource(stateDirURL: hardRoot, profile: .primary) + try FileManager.default.createDirectory( + at: hard.identityURL.deletingLastPathComponent(), + withIntermediateDirectories: true) + try FileManager.default.linkItem(at: real.identityURL, to: hard.identityURL) + #expect(throws: NSError.self) { + try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: root.appendingPathComponent("hard.sqlite"), + destinationStateDirURL: root, + profile: .primary, + legacySources: [hard]) + } + #expect(FileManager.default.fileExists(atPath: real.identityURL.path)) + #expect(FileManager.default.fileExists(atPath: hard.identityURL.path)) + } + + @Test + func `conflicting SQLite identity preserves the legacy source`() throws { + let tempDir = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let destination = tempDir.appendingPathComponent("destination", isDirectory: true) + let databaseURL = destination.appendingPathComponent("openclaw.sqlite") + let existing = try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: destination, + profile: .primary) + let source = try Self.writeLegacyIdentity( + stateDirURL: tempDir.appendingPathComponent("shared", isDirectory: true), + profile: .primary, + contents: Self.nodePEMIdentityJSON()) + + #expect(throws: NSError.self) { + try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: destination, + profile: .primary, + legacySources: [source]) + } + #expect(FileManager.default.fileExists(atPath: source.identityURL.path)) + #expect(try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: destination, + profile: .primary).deviceId == existing.deviceId) + } + + @Test + func `migration copies auth without clobbering or removing its source`() throws { + let tempDir = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let sourceRoot = tempDir.appendingPathComponent("shared", isDirectory: true) + let source = try Self.writeLegacyIdentity( + stateDirURL: sourceRoot, + profile: .primary, + contents: Self.nodePEMIdentityJSON()) + let auth = "{\"version\":1,\"deviceId\":\"\(Self.fixtureDeviceID)\",\"tokens\":{}}" + try auth.write(to: source.authURL, atomically: true, encoding: .utf8) + let destination = tempDir.appendingPathComponent("legacy", isDirectory: true) + let destinationAuthURL = destination.appendingPathComponent("identity/device-auth.json") + + _ = try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: destination.appendingPathComponent("openclaw.sqlite"), + destinationStateDirURL: destination, + profile: .primary, + legacySources: [source]) + + #expect(!FileManager.default.fileExists(atPath: source.identityURL.path)) + #expect(try String(contentsOf: source.authURL, encoding: .utf8) == auth) + #expect(try String(contentsOf: destinationAuthURL, encoding: .utf8) == auth) + } + + @Test + func `migration rejects destination auth owned by another device`() throws { + let tempDir = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let sourceRoot = tempDir.appendingPathComponent("shared", isDirectory: true) + let source = try Self.writeLegacyIdentity( + stateDirURL: sourceRoot, + profile: .primary, + contents: Self.nodePEMIdentityJSON()) + let sourceAuth = "{\"version\":1,\"deviceId\":\"\(Self.fixtureDeviceID)\",\"tokens\":{}}" + try sourceAuth.write(to: source.authURL, atomically: true, encoding: .utf8) + let destination = tempDir.appendingPathComponent("legacy", isDirectory: true) + let destinationAuthURL = destination.appendingPathComponent("identity/device-auth.json") + try FileManager.default.createDirectory( + at: destinationAuthURL.deletingLastPathComponent(), + withIntermediateDirectories: true) + let destinationAuth = #"{"version":1,"deviceId":"another-device","tokens":{}}"# + try destinationAuth.write(to: destinationAuthURL, atomically: true, encoding: .utf8) + + #expect(throws: NSError.self) { + try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: destination.appendingPathComponent("openclaw.sqlite"), + destinationStateDirURL: destination, + profile: .primary, + legacySources: [source]) + } + + #expect(FileManager.default.fileExists(atPath: source.identityURL.path)) + #expect(try String(contentsOf: source.authURL, encoding: .utf8) == sourceAuth) + #expect(try String(contentsOf: destinationAuthURL, encoding: .utf8) == destinationAuth) + } + + // swiftlint:disable line_length + @Test + func `migration rejects stale destination auth for the same device`() throws { + let tempDir = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let sourceRoot = tempDir.appendingPathComponent("shared", isDirectory: true) + let source = try Self.writeLegacyIdentity( + stateDirURL: sourceRoot, + profile: .primary, + contents: Self.nodePEMIdentityJSON()) + let sourceAuth = """ + {"version":1,"deviceId":"\(Self + .fixtureDeviceID)","tokens":{"node":{"token":"source-token","role":"node","scopes":[],"updatedAtMs":100}}} """ - try stored.write(to: identityURL, atomically: true, encoding: .utf8) - let before = try String(contentsOf: identityURL, encoding: .utf8) - - let identity = DeviceIdentityStore.loadOrCreate(fileURL: identityURL) - - #expect(identity.deviceId != "app-group-device-id") - #expect(try String(contentsOf: identityURL, encoding: .utf8) == before) - } - - @Test - func `migrates an existing app group identity and auth store when falling back to legacy storage`() throws { - let tempDir = FileManager.default.temporaryDirectory - .appendingPathComponent(UUID().uuidString, isDirectory: true) - defer { try? FileManager.default.removeItem(at: tempDir) } - let appGroupURL = tempDir.appendingPathComponent("shared", isDirectory: true) - let appGroupIdentityURL = appGroupURL - .appendingPathComponent("identity", isDirectory: true) - .appendingPathComponent("device.json", isDirectory: false) - let appGroupAuthURL = appGroupIdentityURL - .deletingLastPathComponent() - .appendingPathComponent("device-auth.json", isDirectory: false) - let legacyIdentityURL = tempDir - .appendingPathComponent("legacy", isDirectory: true) - .appendingPathComponent("identity", isDirectory: true) - .appendingPathComponent("device.json", isDirectory: false) - let legacyAuthURL = legacyIdentityURL - .deletingLastPathComponent() - .appendingPathComponent("device-auth.json", isDirectory: false) + try sourceAuth.write(to: source.authURL, atomically: true, encoding: .utf8) + let destination = tempDir.appendingPathComponent("legacy", isDirectory: true) + let destinationAuthURL = destination.appendingPathComponent("identity/device-auth.json") try FileManager.default.createDirectory( - at: appGroupIdentityURL.deletingLastPathComponent(), + at: destinationAuthURL.deletingLastPathComponent(), withIntermediateDirectories: true) - let stored = try Self.identityJSON( - publicKeyPem: Self.pem( - label: "PUBLIC KEY", - body: "MCowBQYDK2VwAyEAA6EHv/POEL4dcN0Y50vAmWfk1jCbpQ1fHdyGZBJVMbg="), - privateKeyPem: Self.pem( - label: "PRIVATE KEY", - body: "MC4CAQAwBQYDK2VwBCIEIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4f")) - try stored.write(to: appGroupIdentityURL, atomically: true, encoding: .utf8) - let storedAuth = #"{"version":1,"deviceId":"app-group-device-id","tokens":{}}"# - try storedAuth.write(to: appGroupAuthURL, atomically: true, encoding: .utf8) - let appGroupBefore = try String(contentsOf: appGroupIdentityURL, encoding: .utf8) + let destinationAuth = "{\"version\":1,\"deviceId\":\"\(Self.fixtureDeviceID)\",\"tokens\":{}}" + try destinationAuth.write(to: destinationAuthURL, atomically: true, encoding: .utf8) - let migrationSource = DeviceIdentityPaths.appGroupMigrationSource( - appGroupStateDirURL: appGroupURL, - appGroupStateDirAvailable: false, - stateDirOverridden: false, - profile: .primary) - let identity = DeviceIdentityStore.loadOrCreate( - fileURL: legacyIdentityURL, - migrationSource: migrationSource) + #expect(throws: NSError.self) { + try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: destination.appendingPathComponent("openclaw.sqlite"), + destinationStateDirURL: destination, + profile: .primary, + legacySources: [source]) + } - #expect(identity.deviceId == "56475aa75463474c0285df5dbf2bcab73da651358839e9b77481b2eab107708c") - #expect(FileManager.default.fileExists(atPath: legacyIdentityURL.path)) - let reloaded = DeviceIdentityStore.loadOrCreate(fileURL: legacyIdentityURL) - #expect(reloaded.deviceId == identity.deviceId) - #expect(try String(contentsOf: appGroupIdentityURL, encoding: .utf8) == appGroupBefore) - #expect(try String(contentsOf: legacyAuthURL, encoding: .utf8) == storedAuth) - #expect(try String(contentsOf: appGroupAuthURL, encoding: .utf8) == storedAuth) + #expect(FileManager.default.fileExists(atPath: source.identityURL.path)) + #expect(try String(contentsOf: source.authURL, encoding: .utf8) == sourceAuth) + #expect(try String(contentsOf: destinationAuthURL, encoding: .utf8) == destinationAuth) } @Test - func `does not clobber an existing auth store when migrating an app group identity`() throws { + func `migration accepts equivalent auth with reordered scopes`() throws { let tempDir = FileManager.default.temporaryDirectory .appendingPathComponent(UUID().uuidString, isDirectory: true) defer { try? FileManager.default.removeItem(at: tempDir) } - let appGroupIdentityDirURL = tempDir - .appendingPathComponent("shared", isDirectory: true) - .appendingPathComponent("identity", isDirectory: true) - let legacyIdentityDirURL = tempDir - .appendingPathComponent("legacy", isDirectory: true) - .appendingPathComponent("identity", isDirectory: true) - try FileManager.default.createDirectory(at: appGroupIdentityDirURL, withIntermediateDirectories: true) - try FileManager.default.createDirectory(at: legacyIdentityDirURL, withIntermediateDirectories: true) - let stored = try Self.identityJSON( - publicKeyPem: Self.pem( - label: "PUBLIC KEY", - body: "MCowBQYDK2VwAyEAA6EHv/POEL4dcN0Y50vAmWfk1jCbpQ1fHdyGZBJVMbg="), - privateKeyPem: Self.pem( - label: "PRIVATE KEY", - body: "MC4CAQAwBQYDK2VwBCIEIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4f")) - try stored.write( - to: appGroupIdentityDirURL.appendingPathComponent("device.json", isDirectory: false), - atomically: true, - encoding: .utf8) - try #"{"version":1,"deviceId":"app-group-device-id","tokens":{}}"#.write( - to: appGroupIdentityDirURL.appendingPathComponent("device-auth.json", isDirectory: false), - atomically: true, - encoding: .utf8) - let legacyAuthURL = legacyIdentityDirURL.appendingPathComponent("device-auth.json", isDirectory: false) - let existingLegacyAuth = #"{"version":1,"deviceId":"legacy-device-id","tokens":{}}"# - try existingLegacyAuth.write(to: legacyAuthURL, atomically: true, encoding: .utf8) - - let identity = DeviceIdentityStore.loadOrCreate( - fileURL: legacyIdentityDirURL.appendingPathComponent("device.json", isDirectory: false), - migrationSource: DeviceIdentityPaths.appGroupMigrationSource( - appGroupStateDirURL: tempDir.appendingPathComponent("shared", isDirectory: true), - appGroupStateDirAvailable: false, - stateDirOverridden: false, - profile: .primary)) - - #expect(identity.deviceId == "56475aa75463474c0285df5dbf2bcab73da651358839e9b77481b2eab107708c") - #expect(try String(contentsOf: legacyAuthURL, encoding: .utf8) == existingLegacyAuth) - } - - @Test - func `keeps an existing legacy identity instead of migrating the app group copy`() throws { - let tempDir = FileManager.default.temporaryDirectory - .appendingPathComponent(UUID().uuidString, isDirectory: true) - defer { try? FileManager.default.removeItem(at: tempDir) } - let appGroupURL = tempDir.appendingPathComponent("shared", isDirectory: true) - let appGroupIdentityURL = appGroupURL - .appendingPathComponent("identity", isDirectory: true) - .appendingPathComponent("device.json", isDirectory: false) - let legacyIdentityURL = tempDir - .appendingPathComponent("legacy", isDirectory: true) - .appendingPathComponent("identity", isDirectory: true) - .appendingPathComponent("device.json", isDirectory: false) + let sourceRoot = tempDir.appendingPathComponent("shared", isDirectory: true) + let source = try Self.writeLegacyIdentity( + stateDirURL: sourceRoot, + profile: .primary, + contents: Self.nodePEMIdentityJSON()) + let sourceAuth = """ + {"version":1,"deviceId":"\(Self + .fixtureDeviceID)","tokens":{"legacy":{"token":"source-token","role":"node","scopes":["write","read"],"updatedAtMs":100}}} + """ + try sourceAuth.write(to: source.authURL, atomically: true, encoding: .utf8) + let destination = tempDir.appendingPathComponent("legacy", isDirectory: true) + let destinationAuthURL = destination.appendingPathComponent("identity/device-auth.json") try FileManager.default.createDirectory( - at: appGroupIdentityURL.deletingLastPathComponent(), + at: destinationAuthURL.deletingLastPathComponent(), withIntermediateDirectories: true) - let stored = try Self.identityJSON( - publicKeyPem: Self.pem( - label: "PUBLIC KEY", - body: "MCowBQYDK2VwAyEAA6EHv/POEL4dcN0Y50vAmWfk1jCbpQ1fHdyGZBJVMbg="), - privateKeyPem: Self.pem( - label: "PRIVATE KEY", - body: "MC4CAQAwBQYDK2VwBCIEIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4f")) - try stored.write(to: appGroupIdentityURL, atomically: true, encoding: .utf8) + let destinationAuth = """ + {"version":1,"deviceId":"\(Self + .fixtureDeviceID)","tokens":{"node":{"token":"source-token","role":"node","scopes":["read","write"],"updatedAtMs":100}}} + """ + try destinationAuth.write(to: destinationAuthURL, atomically: true, encoding: .utf8) - let existingLegacy = DeviceIdentityStore.loadOrCreate(fileURL: legacyIdentityURL) - let migrationSource = DeviceIdentityPaths.appGroupMigrationSource( - appGroupStateDirURL: appGroupURL, - appGroupStateDirAvailable: false, - stateDirOverridden: false, + _ = try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: destination.appendingPathComponent("openclaw.sqlite"), + destinationStateDirURL: destination, + profile: .primary, + legacySources: [source]) + + #expect(!FileManager.default.fileExists(atPath: source.identityURL.path)) + #expect(try String(contentsOf: destinationAuthURL, encoding: .utf8) == destinationAuth) + } + + // swiftlint:enable line_length + + @Test + func `preserves WAL journal mode`() throws { + let tempDir = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let databaseURL = tempDir.appendingPathComponent("openclaw.sqlite") + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + try Self.seedCanonicalSchema(databaseURL) + #expect(try Self.scalarText(databaseURL, "PRAGMA journal_mode = WAL")?.lowercased() == "wal") + + _ = try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: databaseURL, + destinationStateDirURL: tempDir, profile: .primary) - let identity = DeviceIdentityStore.loadOrCreate( - fileURL: legacyIdentityURL, - migrationSource: migrationSource) - #expect(identity.deviceId == existingLegacy.deviceId) - #expect(identity.deviceId != "56475aa75463474c0285df5dbf2bcab73da651358839e9b77481b2eab107708c") + #expect(try Self.scalarText(databaseURL, "PRAGMA journal_mode")?.lowercased() == "wal") } @Test - func `provides no app group migration source when the entitlement is present`() { - let appGroupURL = FileManager.default.temporaryDirectory + func `fails closed for nonempty missing wrong and newer schemas`() throws { + let root = FileManager.default.temporaryDirectory .appendingPathComponent(UUID().uuidString, isDirectory: true) - #expect(DeviceIdentityPaths.appGroupMigrationSource( - appGroupStateDirURL: appGroupURL, - appGroupStateDirAvailable: true, - stateDirOverridden: false, - profile: .primary) == nil) - } + defer { try? FileManager.default.removeItem(at: root) } + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) - // Regression: an explicit OPENCLAW_STATE_DIR override must never import the - // machine's app-group identity/tokens; developer-Mac pairing state leaked - // into isolated test state dirs through this migration path. - @Test - func `provides no app group migration source when the state dir is overridden`() { - let appGroupURL = FileManager.default.temporaryDirectory - .appendingPathComponent(UUID().uuidString, isDirectory: true) - #expect(DeviceIdentityPaths.appGroupMigrationSource( - appGroupStateDirURL: appGroupURL, - appGroupStateDirAvailable: false, - stateDirOverridden: true, - profile: .primary) == nil) - } + let missing = root.appendingPathComponent("missing.sqlite") + try Self.execute(missing, "CREATE TABLE unrelated (id INTEGER PRIMARY KEY) STRICT") + #expect(throws: NSError.self) { + try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: missing, + destinationStateDirURL: root, + profile: .primary) + } - private static func base64UrlDecode(_ value: String) -> Data? { + let wrong = root.appendingPathComponent("wrong.sqlite") + try Self.execute(wrong, """ + CREATE TABLE device_identities ( + identity_key TEXT NOT NULL PRIMARY KEY, + device_id TEXT NOT NULL + ) STRICT; + CREATE INDEX idx_device_identities_device ON device_identities(device_id); + """) + #expect(throws: NSError.self) { + try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: wrong, + destinationStateDirURL: root, + profile: .primary) + } + + let newer = root.appendingPathComponent("newer.sqlite") + try Self.execute(newer, "PRAGMA user_version = 4") + #expect(throws: NSError.self) { + try DeviceIdentitySQLiteStore.loadOrCreate( + databaseURL: newer, + destinationStateDirURL: root, + profile: .primary) + } + } +} + +extension DeviceIdentityStoreTests { + fileprivate static func base64UrlDecode(_ value: String) -> Data? { let normalized = value .replacingOccurrences(of: "-", with: "+") .replacingOccurrences(of: "_", with: "/") @@ -668,19 +1063,151 @@ struct DeviceIdentityStoreTests { return Data(base64Encoded: padded) } - private static func identityJSON(publicKeyPem: String, privateKeyPem: String) throws -> String { + fileprivate static let fixtureDeviceID = "56475aa75463474c0285df5dbf2bcab73da651358839e9b77481b2eab107708c" + fileprivate static let fixturePublicKeyRaw = "A6EHv/POEL4dcN0Y50vAmWfk1jCbpQ1fHdyGZBJVMbg=" + fileprivate static let fixturePublicKeyData = Data(base64Encoded: Self.fixturePublicKeyRaw)! + fileprivate static let fixturePrivateKeyData = Data((0..<32).map { UInt8($0) }) + fileprivate static let fixturePrivateKeyRaw = Self.fixturePrivateKeyData.base64EncodedString() + fileprivate static let fixturePublicKeyPEM = Self.fixturePEM( + label: "PUBLIC KEY", + der: Data([ + 0x30, 0x2A, 0x30, 0x05, 0x06, 0x03, 0x2B, 0x65, + 0x70, 0x03, 0x21, 0x00, + ]) + Self.fixturePublicKeyData) + fileprivate static let fixturePrivateKeyPEM = Self.fixturePEM( + label: "PRIVATE KEY", + der: Data([ + 0x30, 0x2E, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06, + 0x03, 0x2B, 0x65, 0x70, 0x04, 0x22, 0x04, 0x20, + ]) + Self.fixturePrivateKeyData) + + fileprivate static func fixturePEM(label: String, der: Data) -> String { + let fence = String(repeating: "-", count: 5) + return "\(fence)BEGIN \(label)\(fence)\n\(der.base64EncodedString())\n\(fence)END \(label)\(fence)\n" + } + + fileprivate static func nodePEMIdentityJSON( + deviceId: String = Self.fixtureDeviceID, + privateKeyPem: String = Self.fixturePrivateKeyPEM) throws -> String + { let object: [String: Any] = [ "version": 1, - "deviceId": "stale-device-id", - "publicKeyPem": publicKeyPem, + "deviceId": deviceId, + "publicKeyPem": self.fixturePublicKeyPEM, "privateKeyPem": privateKeyPem, "createdAtMs": Int64(1_800_000_000_000), ] let data = try JSONSerialization.data(withJSONObject: object, options: [.prettyPrinted, .sortedKeys]) - return String(decoding: data, as: UTF8.self) + "\n" + guard let json = String(bytes: data, encoding: .utf8) else { + throw DeviceIdentityStore.storageError("Could not encode fixture identity JSON") + } + return json + "\n" } - private static func pem(label: String, body: String) -> String { - "-----BEGIN \(label)-----\n\(body)\n-----END \(label)-----\n" + fileprivate static func writeLegacyIdentity( + stateDirURL: URL, + profile: GatewayDeviceIdentityProfile, + contents: String) throws -> DeviceIdentityPaths.LegacyIdentitySource + { + let source = self.legacyIdentitySource(stateDirURL: stateDirURL, profile: profile) + try FileManager.default.createDirectory( + at: source.identityURL.deletingLastPathComponent(), + withIntermediateDirectories: true) + try contents.write(to: source.identityURL, atomically: true, encoding: .utf8) + return source + } + + fileprivate static func legacyIdentitySource( + stateDirURL: URL, + profile: GatewayDeviceIdentityProfile) -> DeviceIdentityPaths.LegacyIdentitySource + { + let stateDirURL = stateDirURL.standardizedFileURL + let identityDirURL = stateDirURL.appendingPathComponent("identity", isDirectory: true) + return DeviceIdentityPaths.LegacyIdentitySource( + stateDirURL: stateDirURL, + identityURL: identityDirURL.appendingPathComponent(profile.identityFileName, isDirectory: false), + authURL: identityDirURL.appendingPathComponent(profile.authFileName, isDirectory: false)) + } + + fileprivate static func seedCanonicalSchema(_ databaseURL: URL, nodeOwned: Bool = false) throws { + try FileManager.default.createDirectory( + at: databaseURL.deletingLastPathComponent(), + withIntermediateDirectories: true) + let metadataSQL = nodeOwned + ? """ + CREATE TABLE schema_meta ( + meta_key TEXT NOT NULL PRIMARY KEY, + role TEXT NOT NULL, + schema_version INTEGER NOT NULL, + agent_id TEXT, + app_version TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) STRICT; + INSERT INTO schema_meta ( + meta_key, role, schema_version, agent_id, app_version, created_at, updated_at + ) VALUES ('primary', 'global', 4, NULL, NULL, 1800000000000, 1800000000000); + PRAGMA user_version = 4; + """ + : "" + try self.execute(databaseURL, """ + CREATE TABLE device_identities ( + identity_key TEXT NOT NULL PRIMARY KEY, + device_id TEXT NOT NULL, + public_key_pem TEXT NOT NULL, + private_key_pem TEXT NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL + ) STRICT; + CREATE INDEX idx_device_identities_device + ON device_identities(device_id, updated_at_ms DESC); + \(metadataSQL) + """) + } + + fileprivate static func execute(_ databaseURL: URL, _ sql: String) throws { + var database: OpaquePointer? + guard sqlite3_open(databaseURL.path, &database) == SQLITE_OK, let database else { + throw DeviceIdentityStore.storageError("Could not open test database") + } + defer { sqlite3_close(database) } + guard sqlite3_exec(database, sql, nil, nil, nil) == SQLITE_OK else { + throw DeviceIdentityStore.storageError(String(cString: sqlite3_errmsg(database))) + } + } + + fileprivate static func scalarInt(_ databaseURL: URL, _ sql: String) throws -> Int64 { + try self.scalar(databaseURL, sql) { sqlite3_column_int64($0, 0) } + } + + fileprivate static func scalarText(_ databaseURL: URL, _ sql: String) throws -> String? { + try self.scalar(databaseURL, sql) { statement in + sqlite3_column_text(statement, 0).map { String(cString: $0) } + } + } + + fileprivate static func scalar( + _ databaseURL: URL, + _ sql: String, + transform: (OpaquePointer) -> T) throws -> T + { + var database: OpaquePointer? + guard sqlite3_open(databaseURL.path, &database) == SQLITE_OK, let database else { + throw DeviceIdentityStore.storageError("Could not open test database") + } + defer { sqlite3_close(database) } + var statement: OpaquePointer? + guard sqlite3_prepare_v2(database, sql, -1, &statement, nil) == SQLITE_OK, let statement else { + throw DeviceIdentityStore.storageError(String(cString: sqlite3_errmsg(database))) + } + defer { sqlite3_finalize(statement) } + guard sqlite3_step(statement) == SQLITE_ROW else { + throw DeviceIdentityStore.storageError(String(cString: sqlite3_errmsg(database))) + } + return transform(statement) + } + + fileprivate static func sql(_ value: String) -> String { + value.replacingOccurrences(of: "'", with: "''") } } diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/GatewayNodeSessionTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/GatewayNodeSessionTests.swift index bae1e954340b..9bc2d9f5d7ac 100644 --- a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/GatewayNodeSessionTests.swift +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/GatewayNodeSessionTests.swift @@ -3076,15 +3076,13 @@ struct GatewayNodeSessionTests { @Test(.stateDirectoryIsolated) func `failed device token write is not reported as an issued role`() async throws { - let tempDir = FileManager.default.temporaryDirectory - .appendingPathComponent(UUID().uuidString, isDirectory: true) - try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) - let blocker = tempDir.appendingPathComponent("not-a-directory", isDirectory: false) + let stateDir = try #require(ProcessInfo.processInfo.environment["OPENCLAW_STATE_DIR"]) + let blocker = URL(fileURLWithPath: stateDir, isDirectory: true) + .appendingPathComponent("identity", isDirectory: false) try Data().write(to: blocker) - // Repoint the pinned state dir at a plain file to force write failures; - // the isolation trait restores the env var after the test. - setenv("OPENCLAW_STATE_DIR", blocker.path, 1) - defer { try? FileManager.default.removeItem(at: tempDir) } + // Block only the legacy auth directory. SQLite identity creation must + // still succeed so this test reaches the token persistence failure. + defer { try? FileManager.default.removeItem(at: blocker) } let session = FakeGatewayWebSocketSession(helloAuth: [ "deviceToken": "node-device-token", diff --git a/docs/cli/node.md b/docs/cli/node.md index 84527096174e..920f006c5dec 100644 --- a/docs/cli/node.md +++ b/docs/cli/node.md @@ -170,8 +170,8 @@ Inspect the local node identity the Gateway verifies against: openclaw node identity --json ``` -It prints the device ID and public key from `identity/device.json` and never -creates or modifies identity files. +It prints the device ID and public key from the `primary` row in +`state/openclaw.sqlite` and never creates the database or a new identity. On tightly controlled node networks, the Gateway operator can explicitly opt in to auto-approving first-time node pairing from trusted CIDRs: @@ -204,11 +204,11 @@ identity that the Gateway uses for pairing and routing. This state lives in the OpenClaw state directory (`~/.openclaw` by default, or `$OPENCLAW_STATE_DIR` when set): -| State | Purpose | -| -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | -| `state/openclaw.sqlite` (`node_host_config`) | Client instance ID, display name, and Gateway connection metadata. The client sends this ID as `instanceId`. | -| `identity/device.json` | Signed Ed25519 keypair and derived device ID. For signed connections, this device ID is the routed node ID and pairing identity. | -| `identity/device-auth.json` | Paired device tokens, keyed by cryptographic device ID and role. | +| State | Purpose | +| -------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `state/openclaw.sqlite` (`node_host_config`) | Client instance ID, display name, and Gateway connection metadata. The client sends this ID as `instanceId`. | +| `state/openclaw.sqlite` (`device_identities`, `primary`) | Signed Ed25519 keypair and derived device ID. For signed connections, this device ID is the routed node ID and pairing identity. | +| `identity/device-auth.json` | Paired device tokens, keyed by cryptographic device ID and role. | `--node-id` changes only the client instance ID in shared SQLite state. It does not change the cryptographic device ID or clear pairing auth. Migrating a retired @@ -234,13 +234,15 @@ The two request IDs are distinct. An applicable trusted-CIDR policy can auto-approve the first-time device-pairing step; command-surface approval remains a separate check. -Older OpenClaw releases stored node-host state in `node.json` and could leave an -obsolete `token` field there. Stop the node host and run `openclaw doctor --fix` -once; Doctor imports the supported identity and connection fields into SQLite, -discards the unused token field, verifies the row, and removes the retired file. -Normal node commands fail closed with this repair instruction while the file or -an interrupted Doctor claim remains. Keep both files under `identity/` private; -they contain the device keypair and auth tokens. +Older OpenClaw releases stored node-host state in `node.json` and the signed +identity in `identity/device.json`. Stop the node host and run +`openclaw doctor --fix` once; Doctor claims each retired source, validates it, +imports and verifies the canonical SQLite row, then removes the old file. Normal +node commands fail closed with this repair instruction while either retired file +or an interrupted Doctor claim remains. Keep `state/openclaw.sqlite` and +`identity/device-auth.json` private; they contain the device keypair and auth +tokens. Device auth remains a separate store and is not rewritten by the +identity migration. ## Exec approvals diff --git a/docs/gateway/doctor.md b/docs/gateway/doctor.md index d9af40e546af..c3614fd7d850 100644 --- a/docs/gateway/doctor.md +++ b/docs/gateway/doctor.md @@ -365,6 +365,7 @@ That stages grounded durable candidates into the short-term dreaming store while - Sessions store + transcripts: from `~/.openclaw/sessions/` to `~/.openclaw/agents//sessions/` - Agent dir: from `~/.openclaw/agent/` to `~/.openclaw/agents//agent/` - WhatsApp auth state (Baileys): from legacy `~/.openclaw/credentials/*.json` (except `oauth.json`) to `~/.openclaw/credentials/whatsapp//...` (default account id: `default`) + - Signed device identity: from `~/.openclaw/identity/device.json` into the `primary` `device_identities` row in `state/openclaw.sqlite`; the separate device-auth file is left untouched These migrations are best-effort and idempotent; doctor emits warnings when it leaves any legacy folders behind as backups. The Gateway/CLI also auto-migrates the legacy sessions + agent dir on startup so history/auth/models land in the per-agent path without a manual doctor run. WhatsApp auth is intentionally only migrated via `openclaw doctor`. Talk provider/provider-map normalization compares by structural equality, so key-order-only diffs no longer trigger repeat no-op `doctor --fix` changes. diff --git a/docs/nodes/index.md b/docs/nodes/index.md index a9c49e932461..7023b5fd0cc9 100644 --- a/docs/nodes/index.md +++ b/docs/nodes/index.md @@ -234,7 +234,7 @@ operators can ignore skills from every paired node with The headless node keeps three separate state records: - `~/.openclaw/state/openclaw.sqlite` (`node_host_config`): the client instance ID, display name, and Gateway connection metadata. -- `~/.openclaw/identity/device.json`: the signed device keypair and derived cryptographic device ID. +- `~/.openclaw/state/openclaw.sqlite` (`device_identities`, key `primary`): the signed device keypair and derived cryptographic device ID. - `~/.openclaw/identity/device-auth.json`: paired device auth tokens keyed by cryptographic device ID and role. For a signed node, the Gateway uses the cryptographic device ID for pairing and @@ -243,6 +243,11 @@ node routing. The client instance ID is only connection metadata. Changing [Identity and pairing state](/cli/node#identity-and-pairing-state) for the supported revoke-and-re-pair flow and upgrade notes. +A retired `identity/device.json` file or interrupted Doctor claim blocks normal +identity use. Stop the node host and run `openclaw doctor --fix`; Doctor imports +the validated keypair into SQLite before removing the old file. The identity +migration leaves `identity/device-auth.json` untouched. + ### Allowlist the commands Exec approvals are **per node host**. Add allowlist entries from the gateway: @@ -747,7 +752,7 @@ openclaw node run --host --port 18789 Notes: - Pairing is still required (the Gateway will show a device pairing prompt). -- Client instance metadata, signed device identity, and pairing auth use separate files; see [Headless identity state](#headless-identity-state). +- Client instance metadata, signed device identity, and pairing auth use separate state records; see [Headless identity state](#headless-identity-state). - Exec approvals are enforced locally via `~/.openclaw/exec-approvals.json` (see [Exec approvals](/tools/exec-approvals)). - On macOS, the headless node host executes `system.run` locally by default. Set `OPENCLAW_NODE_EXEC_HOST=app` to route `system.run` through the companion app exec host; add `OPENCLAW_NODE_EXEC_FALLBACK=0` to require the app host and fail closed if it is unavailable. - Add `--tls` / `--tls-fingerprint` when the Gateway WS uses TLS. diff --git a/docs/refactor/database-first.md b/docs/refactor/database-first.md index bc8a3d8dc447..b4dfe7fa2847 100644 --- a/docs/refactor/database-first.md +++ b/docs/refactor/database-first.md @@ -475,13 +475,10 @@ The branch already has a real shared SQLite base: - The shared schema reserves an `exec_approvals_config` singleton row, but the runtime cutover remains pending. TypeScript and the macOS companion still use the state-scoped JSON file and must move to SQLite together. -- Device identity, device auth, and bootstrap runtime modules now keep their - SQLite snapshot readers/writers separate from doctor-only legacy JSON import - helpers. Device identity uses typed `device_identities` rows and device auth - tokens use typed `device_auth_tokens` rows. Device auth writes reconcile rows - by device/role instead of truncating the token table, and runtime no longer - routes single-token updates through the old whole-store adapter. The legacy - version-1 JSON payloads exist only as doctor import/export shapes. +- TypeScript device identity now uses typed `device_identities` rows, with + doctor-only legacy JSON import kept outside the runtime owner. Device auth is + still file-backed pending a coordinated schema and cross-runtime migration; + `device_auth_tokens` remains reserved for that follow-up. - GitHub Copilot token exchange cache uses the shared SQLite plugin-state table under `github-copilot/token-cache/default`. It is provider-owned cache state, so it intentionally does not add a host schema table. @@ -490,20 +487,13 @@ The branch already has a real shared SQLite base: tracked SDK session, and OpenClaw keeps durable session/transcript state in SQLite instead of compatibility marker files. - The shared Swift runtime (`OpenClawKit`) uses the same - `state/openclaw.sqlite` rows for device identity and device auth. macOS app - helpers import the shared SQLite helpers instead of owning a second JSON or - SQLite path. A leftover legacy `identity/device.json` blocks identity creation - until doctor imports it into SQLite, matching the TypeScript and Android - startup gate. -- Android device identity uses the same TypeScript-compatible key material - stored in typed `state/openclaw.sqlite#table/device_identities` rows. It never - reads or writes `openclaw/identity/device.json`; a leftover legacy file blocks - startup until doctor imports it into SQLite. -- Android cached device auth tokens also use typed - `state/openclaw.sqlite#table/device_auth_tokens` rows and share the same - version-1 token semantics as TypeScript and Swift. Runtime no longer reads `SecurePrefs` - `gateway.deviceToken*` compatibility keys; those belong to migration/doctor - logic only. + `state/openclaw.sqlite#table/device_identities` shape and row keys for device + identity. Apple-container legacy files are imported by the Swift migration + owner because the TypeScript Doctor cannot access those containers. Swift + device auth remains file-backed for the coordinated auth follow-up. +- Android device identity and cached device auth remain app-local stores. They + require a separate Android-owned migration; the host SQLite claims do not + describe current Android behavior. - Android notification recent-package history uses typed `android_notification_recent_packages` rows. Runtime no longer migrates or reads the old SharedPreferences CSV keys. diff --git a/scripts/check-database-first-legacy-stores.mjs b/scripts/check-database-first-legacy-stores.mjs index 173527c69fd3..6ca98af7b25c 100644 --- a/scripts/check-database-first-legacy-stores.mjs +++ b/scripts/check-database-first-legacy-stores.mjs @@ -121,6 +121,7 @@ const legacyStorePatterns = [ /\bpush\/(?:apns-registrations|web-push-subscriptions|vapid-keys)\.json\b/u, /\bmcp-oauth\/[^"'`]*\.json\b/u, /\bnode\.json\b/u, + /\bidentity\/device\.json\b/u, /\bsubagents\/runs\.json\b/u, /\btmp\/skill-uploads\b/u, /\b(?:crestodian|openclaw)\/rescue-pending\/[^"'`]*\.json\b/u, @@ -153,6 +154,7 @@ const allowedRuntimeMigrationPaths = [ "src/infra/state-migrations.workspace-setup.ts", "src/infra/state-migrations.web-push.ts", "src/infra/state-migrations.node-host.ts", + "src/infra/state-migrations.device-identity.ts", "src/infra/state-migrations.subagent-registry.ts", "src/infra/state-migrations.rescue-pending.ts", "src/commands/session-state-migration.ts", diff --git a/scripts/check-kysely-guardrails.mjs b/scripts/check-kysely-guardrails.mjs index e231479ae249..d629a70d9245 100644 --- a/scripts/check-kysely-guardrails.mjs +++ b/scripts/check-kysely-guardrails.mjs @@ -56,6 +56,7 @@ const rawSqliteAllowPathGroups = { "src/state/openclaw-state-db.ts", "src/state/sqlite-schema-shape.test-support.ts", ], + "cross-process SQLite coordination locks": ["src/infra/device-identity-coordinator.ts"], "backup snapshot maintenance": [ "src/commands/backup-verify.ts", "src/infra/backup-create.ts", diff --git a/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh b/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh index 8dd0551ab2fb..eeb2958d42a5 100644 --- a/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh +++ b/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh @@ -242,12 +242,18 @@ prepare_update_restart_probe_current_install() { local port="$1" local log_file="$2" local command_timeout="${OPENCLAW_UPGRADE_SURVIVOR_COMMAND_TIMEOUT:-900s}" + local doctor_log="${log_file}.doctor" local start_epoch local ready_epoch echo "Preparing candidate-auth gateway for automatic update restart." install_update_restart_systemctl_shim seed_update_restart_probe_device_auth + if ! openclaw_e2e_maybe_timeout "$command_timeout" openclaw doctor --fix --non-interactive >"$doctor_log" 2>&1; then + echo "candidate device identity migration failed" >&2 + cat "$doctor_log" >&2 || true + return 1 + fi start_epoch="$(node -e "process.stdout.write(String(Date.now()))")" env -u OPENCLAW_GATEWAY_TOKEN -u OPENCLAW_GATEWAY_PASSWORD openclaw gateway --port "$port" --bind loopback --allow-unconfigured >"$log_file" 2>&1 & gateway_pid="$!" diff --git a/src/agents/tools/gateway.test.ts b/src/agents/tools/gateway.test.ts index acc87b5ecb78..9eda2aa53cf8 100644 --- a/src/agents/tools/gateway.test.ts +++ b/src/agents/tools/gateway.test.ts @@ -923,47 +923,6 @@ describe("gateway tool defaults", () => { expect(mocks.callGateway).not.toHaveBeenCalled(); }); - it("fails remote approval calls when requester device identity is not persisted", async () => { - mocks.configState.value = { - gateway: { - mode: "remote", - remote: { - url: "ws://127.0.0.1:18789", - token: "remote-token", - }, - }, - }; - mocks.persistedDeviceIdentity = null; - mocks.callGateway.mockResolvedValueOnce({ decision: "allow-once" }); - - await expect( - callGatewayTool("exec.approval.waitDecision", {}, { id: "approval-id" }), - ).rejects.toThrow("remote approval gateway calls require a stable device identity"); - expect(mocks.callGateway).not.toHaveBeenCalled(); - }); - - it("fails remote approval calls when requester device identity readback differs", async () => { - mocks.configState.value = { - gateway: { - mode: "remote", - remote: { - url: "wss://gateway.example", - token: "remote-token", - }, - }, - }; - mocks.persistedDeviceIdentity = { - ...mocks.deviceIdentity, - deviceId: "other-device", - }; - mocks.callGateway.mockResolvedValueOnce({ decision: "allow-once" }); - - await expect( - callGatewayTool("exec.approval.waitDecision", {}, { id: "approval-id" }), - ).rejects.toThrow("remote approval gateway calls require a stable device identity"); - expect(mocks.callGateway).not.toHaveBeenCalled(); - }); - it("does not send the local approval runtime token to gatewayUrl overrides", async () => { // Approval runtime tokens are local IPC credentials, not bearer tokens for // user-supplied gateway URLs. diff --git a/src/agents/tools/gateway.ts b/src/agents/tools/gateway.ts index c424d4ff1c4b..3df790d93830 100644 --- a/src/agents/tools/gateway.ts +++ b/src/agents/tools/gateway.ts @@ -311,12 +311,6 @@ function resolveApprovalRequesterDeviceIdentityForGatewayTool(params: { return identity; } const identity = loadOrCreateDeviceIdentity(); - // Approval registration and wait can use separate gateway connections. - // Reject loadOrCreate's unpersisted fallback so both sides bind the same id. - const persistedIdentity = loadDeviceIdentityIfPresent(); - if (persistedIdentity?.deviceId !== identity.deviceId) { - throw new Error("device identity is not persisted"); - } return identity; } catch (error) { if (isNodeApprovalReplay) { diff --git a/src/cli/node-cli/identity.test.ts b/src/cli/node-cli/identity.test.ts index ebb0fce20880..f1daa370e3e3 100644 --- a/src/cli/node-cli/identity.test.ts +++ b/src/cli/node-cli/identity.test.ts @@ -54,11 +54,11 @@ describe("runNodeIdentityShow", () => { runNodeIdentityShow({}); expect(errorSpy).toHaveBeenCalledOnce(); expect(exitSpy).toHaveBeenCalledWith(1); - expect(fs.existsSync(path.join(stateDir, "identity", "device.json"))).toBe(false); + expect(fs.existsSync(path.join(stateDir, "state", "openclaw.sqlite"))).toBe(false); }); it("writes deviceId and raw public key JSON to stdout", () => { - const identity = loadOrCreateDeviceIdentity(path.join(stateDir, "identity", "device.json")); + const identity = loadOrCreateDeviceIdentity(); runNodeIdentityShow({ json: true }); expect(exitSpy).not.toHaveBeenCalled(); expect(logSpy).not.toHaveBeenCalled(); @@ -75,7 +75,7 @@ describe("runNodeIdentityShow", () => { }); it("prints human-readable lines without --json", () => { - const identity = loadOrCreateDeviceIdentity(path.join(stateDir, "identity", "device.json")); + const identity = loadOrCreateDeviceIdentity(); runNodeIdentityShow({}); expect(exitSpy).not.toHaveBeenCalled(); const output = logSpy.mock.calls.map((call: unknown[]) => String(call[0])).join("\n"); diff --git a/src/commands/doctor-config-flow.test.ts b/src/commands/doctor-config-flow.test.ts index 71757e910f6b..4d85bc387799 100644 --- a/src/commands/doctor-config-flow.test.ts +++ b/src/commands/doctor-config-flow.test.ts @@ -1552,6 +1552,25 @@ describe("doctor config flow", () => { }); }); + it("enables Doctor-only state migrations only for explicit repair", async () => { + await runDoctorConfigWithInput({ + config: {}, + run: loadAndMaybeMigrateDoctorConfig, + }); + expect(runDoctorConfigPreflightOptionsMock).toHaveBeenLastCalledWith( + expect.objectContaining({ doctorOnlyStateMigrations: false }), + ); + + await runDoctorConfigWithInput({ + config: {}, + repair: true, + run: loadAndMaybeMigrateDoctorConfig, + }); + expect(runDoctorConfigPreflightOptionsMock).toHaveBeenLastCalledWith( + expect.objectContaining({ doctorOnlyStateMigrations: true }), + ); + }); + it("collects plugin blocker previews from the pre-auto-enable config", async () => { await runDoctorConfigWithInput({ config: { diff --git a/src/commands/doctor-config-flow.ts b/src/commands/doctor-config-flow.ts index 4cd6fbebcc75..1d35fefe1f19 100644 --- a/src/commands/doctor-config-flow.ts +++ b/src/commands/doctor-config-flow.ts @@ -142,6 +142,7 @@ export async function loadAndMaybeMigrateDoctorConfig(params: { const preflight = await runDoctorConfigPreflight({ repairPrefixedConfig: shouldRepair, recoverCorruptTargetStore: shouldRepair, + doctorOnlyStateMigrations: shouldRepair, }); const snapshot = preflight.snapshot; const baseCfg = preflight.baseConfig; diff --git a/src/commands/doctor-config-preflight.state-migration.test.ts b/src/commands/doctor-config-preflight.state-migration.test.ts index 853b347113d8..a5616a6cf3f4 100644 --- a/src/commands/doctor-config-preflight.state-migration.test.ts +++ b/src/commands/doctor-config-preflight.state-migration.test.ts @@ -395,6 +395,7 @@ describe("runDoctorConfigPreflight state migration", () => { cfg: { gateway: { mode: "local", port: 19091 } }, env: process.env, recoverCorruptTargetStore: undefined, + doctorOnlyStateMigrations: undefined, }); expect(note).toHaveBeenCalledWith("- cron-imported", "Doctor changes"); expect(note).toHaveBeenCalledWith("- imported", "Doctor changes"); @@ -884,9 +885,22 @@ describe("runDoctorConfigPreflight state migration", () => { cfg: { gateway: { mode: "local", port: 19091 } }, env: process.env, recoverCorruptTargetStore: true, + doctorOnlyStateMigrations: undefined, }); }); + it("passes explicit Doctor-only migration authority only when requested", async () => { + await runDoctorConfigPreflight({ + migrateLegacyConfig: false, + invalidConfigNote: false, + doctorOnlyStateMigrations: true, + }); + + expect(autoMigrateLegacyState).toHaveBeenCalledWith( + expect.objectContaining({ doctorOnlyStateMigrations: true }), + ); + }); + it("runs plugin state migrations with resolved legacy config before config repair removes retired paths", async () => { const parsedConfig = { $include: "memory-search.json" }; const resolvedConfig = { @@ -955,6 +969,7 @@ describe("runDoctorConfigPreflight state migration", () => { pluginDoctorConfig: resolvedConfig, env: process.env, recoverCorruptTargetStore: undefined, + doctorOnlyStateMigrations: undefined, }); }); diff --git a/src/commands/doctor-config-preflight.ts b/src/commands/doctor-config-preflight.ts index 32d82146b3aa..67ee724cee5c 100644 --- a/src/commands/doctor-config-preflight.ts +++ b/src/commands/doctor-config-preflight.ts @@ -387,6 +387,8 @@ export async function runDoctorConfigPreflight( skipPristineCoreStateMigrations?: boolean; /** Prepared before Gateway bootstrap can create files under an otherwise pristine state root. */ skipPristineStartupStateMigrations?: boolean; + /** Enable migrations that may retire security-sensitive stores only during explicit repair. */ + doctorOnlyStateMigrations?: boolean; } = {}, ): Promise { const stateMigrationsRequested = options.migrateState !== false; @@ -590,6 +592,7 @@ export async function runDoctorConfigPreflight( : {}), env: process.env, recoverCorruptTargetStore: options.recoverCorruptTargetStore, + doctorOnlyStateMigrations: options.doctorOnlyStateMigrations, }), ); } else if (stateMigrationInput.pluginDoctorConfig) { diff --git a/src/commands/doctor-device-pairing.ts b/src/commands/doctor-device-pairing.ts index 8be1982c06c5..6630c6aabdea 100644 --- a/src/commands/doctor-device-pairing.ts +++ b/src/commands/doctor-device-pairing.ts @@ -9,6 +9,7 @@ import { resolveStateDir } from "../config/paths.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { HealthFinding } from "../flows/health-checks.js"; import { callGateway } from "../gateway/call.js"; +import { loadDeviceIdentityIfPresent } from "../infra/device-identity.js"; import { listApprovedPairedDeviceRoles, listDevicePairing, @@ -105,11 +106,6 @@ type LocalDeviceAuthIssue = { fixHint: string; }; -type StoredDeviceIdentity = { - version: 1; - deviceId: string; -}; - function hasNumberVersion(value: object): value is { version: number } { return "version" in value && typeof value.version === "number"; } @@ -407,24 +403,12 @@ function readJsonFile(filePath: string): unknown { return tryReadJsonSync(filePath); } -function readLocalIdentity(env: NodeJS.ProcessEnv = process.env): StoredDeviceIdentity | null { - const filePath = path.join(resolveStateDir(env), "identity", "device.json"); - const identity = readJsonFile(filePath); - if ( - !identity || - typeof identity !== "object" || - !hasNumberVersion(identity) || - identity.version !== 1 || - !("deviceId" in identity) || - typeof identity.deviceId !== "string" || - !identity.deviceId.trim() - ) { +function readLocalIdentity(env: NodeJS.ProcessEnv = process.env): { deviceId: string } | null { + try { + return loadDeviceIdentityIfPresent({ env }); + } catch { return null; } - return { - version: 1, - deviceId: identity.deviceId, - }; } function readLocalDeviceAuthStore(env: NodeJS.ProcessEnv = process.env): DeviceAuthStore | null { diff --git a/src/commands/doctor.e2e-harness.ts b/src/commands/doctor.e2e-harness.ts index 8a571fb8e4ff..aeb7c7a4edb2 100644 --- a/src/commands/doctor.e2e-harness.ts +++ b/src/commands/doctor.e2e-harness.ts @@ -208,6 +208,13 @@ function createLegacyStateMigrationDetectionResult(params?: { targetScope: undefined, stateDir: "/tmp/state", oauthDir: "/tmp/oauth", + deviceIdentity: { + sourcePath: "/tmp/state/identity/device.json", + claimPath: "/tmp/state/identity/device.json.doctor-importing", + nativeClaimPath: "/tmp/state/identity/device.json.native-importing", + hasLegacy: false, + hasInvalidCanonical: false, + }, mcpOauth: { sourceDir: "/tmp/state/mcp-oauth", sourcePaths: [], diff --git a/src/flows/doctor-health-contributions.test.ts b/src/flows/doctor-health-contributions.test.ts index 290b23500ea2..3872a5feab10 100644 --- a/src/flows/doctor-health-contributions.test.ts +++ b/src/flows/doctor-health-contributions.test.ts @@ -1473,6 +1473,29 @@ describe("doctor health contributions", () => { await contribution.run(ctx); + expect(mocks.detectLegacyStateMigrations).toHaveBeenCalledWith({ cfg }); + expect(mocks.runLegacyStateMigrations).toHaveBeenCalledWith({ + detected, + config: cfg, + recoverCorruptTargetStore: false, + }); + }); + + it("grants Doctor-only state migration authority only in repair mode", async () => { + const contribution = requireDoctorContribution("doctor:legacy-state"); + const cfg = { session: { store: "/tmp/shared-sessions.json" } }; + const detected = { preview: ["legacy sessions"], warnings: [], notices: [] }; + mocks.detectLegacyStateMigrations.mockResolvedValue(detected); + const ctx = { + cfg, + sourceConfigValid: true, + prompter: buildDoctorPrompter(true), + runtime: { log: vi.fn(), error: vi.fn(), exit: vi.fn() }, + options: { nonInteractive: true, repair: true }, + } as unknown as Parameters<(typeof contribution)["run"]>[0]; + + await contribution.run(ctx); + expect(mocks.detectLegacyStateMigrations).toHaveBeenCalledWith({ cfg, doctorOnlyStateMigrations: true, @@ -1480,7 +1503,8 @@ describe("doctor health contributions", () => { expect(mocks.runLegacyStateMigrations).toHaveBeenCalledWith({ detected, config: cfg, - recoverCorruptTargetStore: false, + doctorOnlyStateMigrations: true, + recoverCorruptTargetStore: true, }); }); diff --git a/src/flows/doctor-health-contributions.ts b/src/flows/doctor-health-contributions.ts index 38498cf0ea99..5234fffb795d 100644 --- a/src/flows/doctor-health-contributions.ts +++ b/src/flows/doctor-health-contributions.ts @@ -521,9 +521,10 @@ async function runLegacyStateHealth(ctx: DoctorHealthFlowContext): Promise // Settle retired-plugin state cleanup (may replace ctx.cfg) before the // legacy-state detect/migrate pair reads the config. await runCoreContributionHealthRepair(ctx, ["core/doctor/removed-workspaces-state"]); + const doctorOnlyStateMigrations = ctx.options.repair === true || ctx.options.yes === true; const legacyState = await detectLegacyStateMigrations({ cfg: ctx.cfg, - doctorOnlyStateMigrations: true, + ...(doctorOnlyStateMigrations ? { doctorOnlyStateMigrations: true } : {}), }); if (legacyState.warnings.length > 0) { note(legacyState.warnings.join("\n"), "Doctor warnings"); @@ -548,6 +549,7 @@ async function runLegacyStateHealth(ctx: DoctorHealthFlowContext): Promise const migrated = await runLegacyStateMigrations({ detected: legacyState, config: ctx.cfg, + ...(doctorOnlyStateMigrations ? { doctorOnlyStateMigrations: true } : {}), recoverCorruptTargetStore: ctx.options.repair === true || ctx.options.yes === true, }); if (migrated.changes.length > 0) { diff --git a/src/gateway/device-authz.test-helpers.ts b/src/gateway/device-authz.test-helpers.ts index a05018a54326..dbca6d3c2132 100644 --- a/src/gateway/device-authz.test-helpers.ts +++ b/src/gateway/device-authz.test-helpers.ts @@ -19,7 +19,7 @@ import { trackConnectChallengeNonce } from "./test-helpers.js"; export function resolveDeviceIdentityPath(name: string): string { const root = process.env.OPENCLAW_STATE_DIR ?? process.env.HOME ?? os.tmpdir(); - return path.join(root, "test-device-identities", `${name}.json`); + return path.join(root, "test-device-identities", `${name}.sqlite`); } export function loadDeviceIdentity(name: string): { @@ -28,7 +28,7 @@ export function loadDeviceIdentity(name: string): { publicKey: string; } { const identityPath = resolveDeviceIdentityPath(name); - const identity = loadOrCreateDeviceIdentity(identityPath); + const identity = loadOrCreateDeviceIdentity({ path: identityPath }); return { identityPath, identity, diff --git a/src/gateway/gateway-cli-backend.connect.test.ts b/src/gateway/gateway-cli-backend.connect.test.ts index 8f779620daae..1143cb768acd 100644 --- a/src/gateway/gateway-cli-backend.connect.test.ts +++ b/src/gateway/gateway-cli-backend.connect.test.ts @@ -22,7 +22,7 @@ const tempDirs = createSuiteTempRootTracker({ prefix: "openclaw-gateway-connect- async function createTempDeviceIdentity() { const tempRoot = await tempDirs.make("device"); - return loadOrCreateDeviceIdentity(path.join(tempRoot, "device.json")); + return loadOrCreateDeviceIdentity({ path: path.join(tempRoot, "device.sqlite") }); } async function startMinimalGatewayServer(params: { token: string }) { diff --git a/src/gateway/operator-approvals-client.e2e.test.ts b/src/gateway/operator-approvals-client.e2e.test.ts index 7f0456363330..bdf0f30c9034 100644 --- a/src/gateway/operator-approvals-client.e2e.test.ts +++ b/src/gateway/operator-approvals-client.e2e.test.ts @@ -192,15 +192,15 @@ describe("operator approval gateway client e2e", () => { setTestEnvValue("HOME", tempHome); setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); - const requesterIdentity = loadOrCreateDeviceIdentity( - path.join(stateDir, "test-device-identities", "approval-requester.json"), - ); - const reviewerIdentity = loadOrCreateDeviceIdentity( - path.join(stateDir, "test-device-identities", "approval-reviewer.json"), - ); - const underscopedIdentity = loadOrCreateDeviceIdentity( - path.join(stateDir, "test-device-identities", "approval-underscoped.json"), - ); + const requesterIdentity = loadOrCreateDeviceIdentity({ + path: path.join(stateDir, "test-device-identities", "approval-requester.sqlite"), + }); + const reviewerIdentity = loadOrCreateDeviceIdentity({ + path: path.join(stateDir, "test-device-identities", "approval-reviewer.sqlite"), + }); + const underscopedIdentity = loadOrCreateDeviceIdentity({ + path: path.join(stateDir, "test-device-identities", "approval-underscoped.sqlite"), + }); expect(requesterIdentity.deviceId).not.toBe(reviewerIdentity.deviceId); const port = await getFreeGatewayPort(); diff --git a/src/gateway/probe.test.ts b/src/gateway/probe.test.ts index bd6ef0fa37d9..a4d2cc08dbe8 100644 --- a/src/gateway/probe.test.ts +++ b/src/gateway/probe.test.ts @@ -162,8 +162,8 @@ vi.mock("../infra/device-identity.js", () => ({ } return deviceIdentityState.value; }, - loadDeviceIdentityIfPresent: (filePath: unknown) => { - deviceIdentityState.identityPaths.push(filePath); + loadDeviceIdentityIfPresent: (options: unknown) => { + deviceIdentityState.identityPaths.push(options); if (deviceIdentityState.throwOnLoad) { throw new Error("read-only identity dir"); } @@ -397,9 +397,7 @@ describe("probeGateway", () => { await runTokenProbe({ env }); - expect(deviceIdentityState.identityPaths).toEqual([ - "/tmp/openclaw-probe-service-state/identity/device.json", - ]); + expect(deviceIdentityState.identityPaths).toEqual([{ env }]); expect(deviceIdentityState.tokenParams).toEqual([ { deviceId: "test-device-identity", diff --git a/src/gateway/probe.ts b/src/gateway/probe.ts index 0c45ea331705..43a5e2ae3da6 100644 --- a/src/gateway/probe.ts +++ b/src/gateway/probe.ts @@ -1,12 +1,10 @@ // Gateway reachability probe client. // Connects to a gateway and summarizes auth, health, status, and presence. import { randomUUID } from "node:crypto"; -import path from "node:path"; import { GATEWAY_CLIENT_MODES, GATEWAY_CLIENT_NAMES, } from "../../packages/gateway-protocol/src/client-info.js"; -import { resolveStateDir } from "../config/paths.js"; import { loadDeviceAuthToken } from "../infra/device-auth-store.js"; import { formatErrorMessage } from "../infra/errors.js"; import type { SystemPresence } from "../infra/system-presence.js"; @@ -257,8 +255,7 @@ export async function probeGateway(opts: { return null; } const { loadDeviceIdentityIfPresent } = await import("../infra/device-identity.js"); - const stateDir = resolveStateDir(opts.env); - const identity = loadDeviceIdentityIfPresent(path.join(stateDir, "identity", "device.json")); + const identity = loadDeviceIdentityIfPresent({ env: opts.env }); if (!identity) { return null; } diff --git a/src/gateway/server.auth.browser-hardening.test.ts b/src/gateway/server.auth.browser-hardening.test.ts index 107da7c783b0..07b5d952beaf 100644 --- a/src/gateway/server.auth.browser-hardening.test.ts +++ b/src/gateway/server.auth.browser-hardening.test.ts @@ -64,7 +64,7 @@ async function createSignedDevice(params: { signedAtMs?: number; }) { const identity = params.identityPath - ? loadOrCreateDeviceIdentity(params.identityPath) + ? loadOrCreateDeviceIdentity({ path: params.identityPath }) : loadOrCreateDeviceIdentity(); const signedAtMs = params.signedAtMs ?? Date.now(); const payload = buildDeviceAuthPayload({ @@ -169,7 +169,7 @@ async function createSignedBrowserDevice( scopes: ["operator.admin"], clientId: client.id, clientMode: client.mode, - identityPath: path.join(os.tmpdir(), `openclaw-${identityName}-device-${randomUUID()}.json`), + identityPath: path.join(os.tmpdir(), `openclaw-${identityName}-device-${randomUUID()}.sqlite`), nonce: nonce ?? "", }); } diff --git a/src/gateway/server.auth.compat-baseline.test.ts b/src/gateway/server.auth.compat-baseline.test.ts index 2579a650a7d6..dc4c2f173641 100644 --- a/src/gateway/server.auth.compat-baseline.test.ts +++ b/src/gateway/server.auth.compat-baseline.test.ts @@ -236,14 +236,14 @@ describe("gateway auth compatibility baseline", () => { test("keeps local backend device-token reconnects out of pairing", async () => { const identityPath = path.join( os.tmpdir(), - `openclaw-backend-device-${process.pid}-${port}.json`, + `openclaw-backend-device-${process.pid}-${port}.sqlite`, ); const { loadOrCreateDeviceIdentity, publicKeyRawBase64UrlFromPem } = await import("../infra/device-identity.js"); const { approveDevicePairing, requestDevicePairing, rotateDeviceToken } = await import("../infra/device-pairing.js"); - const identity = loadOrCreateDeviceIdentity(identityPath); + const identity = loadOrCreateDeviceIdentity({ path: identityPath }); const pending = await requestDevicePairing({ deviceId: identity.deviceId, publicKey: publicKeyRawBase64UrlFromPem(identity.publicKeyPem), @@ -411,7 +411,7 @@ describe("gateway auth compatibility baseline", () => { try { const deviceIdentityPath = path.join( os.tmpdir(), - `openclaw-auth-none-control-ui-first-${process.pid}-${port}.json`, + `openclaw-auth-none-control-ui-first-${process.pid}-${port}.sqlite`, ); const res = await connectReq(ws, { skipDefaultAuth: true, @@ -443,11 +443,11 @@ describe("gateway auth compatibility baseline", () => { const nonce = await readConnectChallengeNonce(ws); const identityPath = path.join( os.tmpdir(), - `openclaw-auth-none-control-ui-${process.pid}-${port}.json`, + `openclaw-auth-none-control-ui-${process.pid}-${port}.sqlite`, ); const staleIdentityPath = path.join( os.tmpdir(), - `openclaw-auth-none-control-ui-stale-${process.pid}-${port}.json`, + `openclaw-auth-none-control-ui-stale-${process.pid}-${port}.sqlite`, ); const { identity, device } = await createSignedDevice({ token: null, @@ -457,7 +457,7 @@ describe("gateway auth compatibility baseline", () => { identityPath, nonce, }); - const staleIdentity = loadOrCreateDeviceIdentity(staleIdentityPath); + const staleIdentity = loadOrCreateDeviceIdentity({ path: staleIdentityPath }); const pending = await requestDevicePairing({ deviceId: identity.deviceId, publicKey: publicKeyRawBase64UrlFromPem(staleIdentity.publicKeyPem), diff --git a/src/gateway/server.auth.control-ui.suite.ts b/src/gateway/server.auth.control-ui.suite.ts index 9ae865b56b02..6eee33502c8c 100644 --- a/src/gateway/server.auth.control-ui.suite.ts +++ b/src/gateway/server.auth.control-ui.suite.ts @@ -174,10 +174,10 @@ export function registerControlUiAndPairingSuite(): void { let identityPath = operatorIdentityPathByPrefix.get(identityPrefix); if (!identityPath) { const poolId = process.env.VITEST_POOL_ID ?? "0"; - identityPath = path.join(os.tmpdir(), `${identityPrefix}${process.pid}-${poolId}.json`); + identityPath = path.join(os.tmpdir(), `${identityPrefix}${process.pid}-${poolId}.sqlite`); operatorIdentityPathByPrefix.set(identityPrefix, identityPath); } - const identity = loadOrCreateDeviceIdentity(identityPath); + const identity = loadOrCreateDeviceIdentity({ path: identityPath }); return { identityPath, identity, diff --git a/src/gateway/server.auth.default-token.suite.ts b/src/gateway/server.auth.default-token.suite.ts index 0a20141f91c5..fdad2e62fd14 100644 --- a/src/gateway/server.auth.default-token.suite.ts +++ b/src/gateway/server.auth.default-token.suite.ts @@ -353,7 +353,7 @@ export function registerDefaultAuthTokenSuite(): void { scopes: [], clientId: GATEWAY_CLIENT_NAMES.TEST, clientMode: GATEWAY_CLIENT_MODES.TEST, - identityPath: path.join(os.tmpdir(), `openclaw-test-device-${randomUUID()}.json`), + identityPath: path.join(os.tmpdir(), `openclaw-test-device-${randomUUID()}.sqlite`), nonce, }); diff --git a/src/gateway/server.auth.test-helpers.ts b/src/gateway/server.auth.test-helpers.ts index 5a0dfcd4ebe2..471a8140fa75 100644 --- a/src/gateway/server.auth.test-helpers.ts +++ b/src/gateway/server.auth.test-helpers.ts @@ -39,7 +39,7 @@ function nextAuthIdentityPath(prefix: string): string { poolId + "-" + String(authIdentityPathSeq++) + - ".json"; + ".sqlite"; return path.join(os.tmpdir(), fileName); } @@ -181,7 +181,7 @@ async function createSignedDevice(params: { const { loadOrCreateDeviceIdentity, publicKeyRawBase64UrlFromPem, signDevicePayload } = await import("../infra/device-identity.js"); const identity = params.identityPath - ? loadOrCreateDeviceIdentity(params.identityPath) + ? loadOrCreateDeviceIdentity({ path: params.identityPath }) : loadOrCreateDeviceIdentity(); const signedAtMs = params.signedAtMs ?? Date.now(); const payload = buildDeviceAuthPayload({ @@ -331,7 +331,7 @@ async function resolvePairedTokenForDeviceIdentityPath(deviceIdentityPath: strin const { loadOrCreateDeviceIdentity } = await import("../infra/device-identity.js"); const { getPairedDevice } = await import("../infra/device-pairing.js"); - const identity = loadOrCreateDeviceIdentity(deviceIdentityPath); + const identity = loadOrCreateDeviceIdentity({ path: deviceIdentityPath }); const paired = await getPairedDevice(identity.deviceId); const deviceToken = paired?.tokens?.operator?.token; expect(paired?.deviceId).toBe(identity.deviceId); diff --git a/src/gateway/server.node-pairing-authz.test.ts b/src/gateway/server.node-pairing-authz.test.ts index eeef7cfd0261..e6e027749138 100644 --- a/src/gateway/server.node-pairing-authz.test.ts +++ b/src/gateway/server.node-pairing-authz.test.ts @@ -210,7 +210,7 @@ async function expectRpcNodePairingApprovalRejected(params: { await connectOk(ws, { token: "secret", scopes: params.operatorScopes, - deviceIdentityPath: `${await makeNodePairingStateDir()}/${params.operatorName}.json`, + deviceIdentityPath: `${await makeNodePairingStateDir()}/${params.operatorName}.sqlite`, }); await seedNodeDevice(params.nodeId); const request = await requestNodePairing({ @@ -734,7 +734,7 @@ describe("gateway node pairing authorization", () => { await connectOk(ws, { token: "secret", scopes: ["operator.read"], - deviceIdentityPath: `${await makeNodePairingStateDir()}/read-only.json`, + deviceIdentityPath: `${await makeNodePairingStateDir()}/read-only.sqlite`, }); type NodeDiagnostics = { diff --git a/src/gateway/server.node-pairing-rate-limit.test.ts b/src/gateway/server.node-pairing-rate-limit.test.ts index 3dd2f951fce9..f861d2f34463 100644 --- a/src/gateway/server.node-pairing-rate-limit.test.ts +++ b/src/gateway/server.node-pairing-rate-limit.test.ts @@ -69,7 +69,7 @@ async function attemptNodePairing( } async function approveNodeIdentity(params: { identityPath: string; caps: string[] }) { - const identity = loadOrCreateDeviceIdentity(params.identityPath); + const identity = loadOrCreateDeviceIdentity({ path: params.identityPath }); // Node surfaces attach to paired devices, so device pairing comes first. // The stored key must match what the reconnect presents or the handshake // restarts pairing and burns the rate-limit budget under test. @@ -112,7 +112,7 @@ describe("node pairing rate limit", () => { const responses = await Promise.all( Array.from( { length: 8 }, - async (_, index) => await attemptNodePairing(port, `${identityPrefix}-${index}.json`), + async (_, index) => await attemptNodePairing(port, `${identityPrefix}-${index}.sqlite`), ), ); const rateLimited = responses.filter((res) => { @@ -146,7 +146,7 @@ describe("node pairing rate limit", () => { os.tmpdir(), `openclaw-node-pairing-upgrade-${randomUUID()}`, ); - const pairedIdentityPath = `${identityPrefix}-paired.json`; + const pairedIdentityPath = `${identityPrefix}-paired.sqlite`; const pairedIdentity = await approveNodeIdentity({ identityPath: pairedIdentityPath, caps: ["camera"], @@ -155,7 +155,7 @@ describe("node pairing rate limit", () => { const firstTimeResponses = await Promise.all( Array.from( { length: 3 }, - async (_, index) => await attemptNodePairing(port, `${identityPrefix}-${index}.json`), + async (_, index) => await attemptNodePairing(port, `${identityPrefix}-${index}.sqlite`), ), ); expect(firstTimeResponses.filter((res) => res.ok)).toHaveLength(3); @@ -203,7 +203,10 @@ describe("node pairing rate limit", () => { }, }; await withGatewayServer(async ({ port }) => { - const identityPath = path.join(os.tmpdir(), `openclaw-node-reapproval-${randomUUID()}.json`); + const identityPath = path.join( + os.tmpdir(), + `openclaw-node-reapproval-${randomUUID()}.sqlite`, + ); const identity = await approveNodeIdentity({ identityPath, caps: ["camera"] }); const responses = await Promise.all( diff --git a/src/gateway/server.roles-allowlist-update.test.ts b/src/gateway/server.roles-allowlist-update.test.ts index 541a09fb4130..00e47f75d4a4 100644 --- a/src/gateway/server.roles-allowlist-update.test.ts +++ b/src/gateway/server.roles-allowlist-update.test.ts @@ -339,9 +339,12 @@ async function respondToInvoke( } function createDeviceIdentityForTest(prefix: string) { - return loadOrCreateDeviceIdentity( - path.join(os.tmpdir(), `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2)}.json`), - ); + return loadOrCreateDeviceIdentity({ + path: path.join( + os.tmpdir(), + `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2)}.sqlite`, + ), + }); } describe("gateway role enforcement", () => { @@ -486,15 +489,18 @@ describe("gateway node command allowlist", () => { const invokeCapture = createInvokeCapture(); try { - const systemDeviceIdentity = loadOrCreateDeviceIdentity( - path.join(os.tmpdir(), `openclaw-node-system-run-${Date.now()}-${Math.random()}.json`), - ); - const emptyDeviceIdentity = loadOrCreateDeviceIdentity( - path.join(os.tmpdir(), `openclaw-node-empty-${Date.now()}-${Math.random()}.json`), - ); - const allowedDeviceIdentity = loadOrCreateDeviceIdentity( - path.join(os.tmpdir(), `openclaw-node-allowed-${Date.now()}-${Math.random()}.json`), - ); + const systemDeviceIdentity = loadOrCreateDeviceIdentity({ + path: path.join( + os.tmpdir(), + `openclaw-node-system-run-${Date.now()}-${Math.random()}.sqlite`, + ), + }); + const emptyDeviceIdentity = loadOrCreateDeviceIdentity({ + path: path.join(os.tmpdir(), `openclaw-node-empty-${Date.now()}-${Math.random()}.sqlite`), + }); + const allowedDeviceIdentity = loadOrCreateDeviceIdentity({ + path: path.join(os.tmpdir(), `openclaw-node-allowed-${Date.now()}-${Math.random()}.sqlite`), + }); systemClient = await connectNodeClientWithPairing({ port, diff --git a/src/gateway/server.shared-auth-rotation.test.ts b/src/gateway/server.shared-auth-rotation.test.ts index 7b919a76d2a8..c30fef932985 100644 --- a/src/gateway/server.shared-auth-rotation.test.ts +++ b/src/gateway/server.shared-auth-rotation.test.ts @@ -50,7 +50,7 @@ async function openDeviceTokenWsWithDetails( auth?: { deviceToken?: unknown }; }; }> { - const identityPath = path.join(os.tmpdir(), `openclaw-shared-auth-${process.pid}-${port}.json`); + const identityPath = path.join(os.tmpdir(), `openclaw-shared-auth-${process.pid}-${port}.sqlite`); const { loadOrCreateDeviceIdentity, publicKeyRawBase64UrlFromPem } = await import("../infra/device-identity.js"); const { approveDevicePairing, ensureDeviceToken, requestDevicePairing, rotateDeviceToken } = @@ -69,7 +69,7 @@ async function openDeviceTokenWsWithDetails( mode: "test", }; - const identity = loadOrCreateDeviceIdentity(identityPath); + const identity = loadOrCreateDeviceIdentity({ path: identityPath }); const pending = await requestDevicePairing({ deviceId: identity.deviceId, publicKey: publicKeyRawBase64UrlFromPem(identity.publicKeyPem), diff --git a/src/gateway/server.talk-config.test.ts b/src/gateway/server.talk-config.test.ts index 8cdcbe9fe827..c2b04ea1d43a 100644 --- a/src/gateway/server.talk-config.test.ts +++ b/src/gateway/server.talk-config.test.ts @@ -62,9 +62,12 @@ afterAll(async () => { }); async function createFreshOperatorDevice(scopes: string[], nonce: string) { - const identity = loadOrCreateDeviceIdentity( - path.join(os.tmpdir(), `openclaw-talk-config-device-${process.pid}-${talkConfigDeviceSeq++}`), - ); + const identity = loadOrCreateDeviceIdentity({ + path: path.join( + os.tmpdir(), + `openclaw-talk-config-device-${process.pid}-${talkConfigDeviceSeq++}.sqlite`, + ), + }); const signedAtMs = Date.now(); const payload = buildDeviceAuthPayload({ deviceId: identity.deviceId, diff --git a/src/gateway/test-helpers.e2e.ts b/src/gateway/test-helpers.e2e.ts index ba3e55cf575e..c50a7b21d32d 100644 --- a/src/gateway/test-helpers.e2e.ts +++ b/src/gateway/test-helpers.e2e.ts @@ -61,17 +61,17 @@ export async function connectGatewayClient(params: { const identityRoot = process.env.OPENCLAW_STATE_DIR ?? process.env.HOME ?? os.tmpdir(); const deviceIdentity = params.deviceIdentity ?? - loadOrCreateDeviceIdentity( - (() => { + loadOrCreateDeviceIdentity({ + path: (() => { const safe = normalizeLowercaseStringOrEmpty( `${params.clientName ?? GATEWAY_CLIENT_NAMES.TEST}-${params.mode ?? GATEWAY_CLIENT_MODES.TEST}-${platform}-${params.deviceFamily ?? "none"}-${role}`.replace( /[^a-zA-Z0-9._-]+/g, "_", ), ); - return path.join(identityRoot, "test-device-identities", `${safe}.json`); + return path.join(identityRoot, "test-device-identities", `${safe}.sqlite`); })(), - ); + }); return await new Promise>((resolve, reject) => { let settled = false; const stop = (err?: Error, connectedClient?: InstanceType) => { diff --git a/src/gateway/test-helpers.server.ts b/src/gateway/test-helpers.server.ts index 6c90935da8fd..e52322c77216 100644 --- a/src/gateway/test-helpers.server.ts +++ b/src/gateway/test-helpers.server.ts @@ -872,7 +872,7 @@ function resolveDefaultTestDeviceIdentityPath(params: { ), ); const suiteRoot = process.env.OPENCLAW_STATE_DIR ?? process.env.HOME ?? os.tmpdir(); - return path.join(suiteRoot, "test-device-identities", `${safe}.json`); + return path.join(suiteRoot, "test-device-identities", `${safe}.sqlite`); } export async function readConnectChallengeNonce( @@ -1095,7 +1095,7 @@ export async function connectReq( deviceFamily: client.deviceFamily, role, }); - const identity = loadOrCreateDeviceIdentity(identityPath); + const identity = loadOrCreateDeviceIdentity({ path: identityPath }); const signedAtMs = Date.now(); const payload = buildDeviceAuthPayloadV3({ deviceId: identity.deviceId, diff --git a/src/gateway/watch-node-http.test.ts b/src/gateway/watch-node-http.test.ts index 10a331df851e..b733f353e029 100644 --- a/src/gateway/watch-node-http.test.ts +++ b/src/gateway/watch-node-http.test.ts @@ -174,7 +174,9 @@ async function waitForLastConnectedMetadata(baseDir: string, nodeId: string): Pr describe("watch node HTTP transport", () => { it("rejects capabilities and identities outside the bounded watch surface", async () => { const baseDir = await tempDirs.make("openclaw-watch-node-surface-"); - const identity = loadOrCreateDeviceIdentity(path.join(baseDir, "watch-identity.json")); + const identity = loadOrCreateDeviceIdentity({ + path: path.join(baseDir, "watch-identity.sqlite"), + }); const issued = await issueDeviceBootstrapToken({ baseDir, profile: NODE_PAIRING_SETUP_BOOTSTRAP_PROFILE, @@ -231,7 +233,9 @@ describe("watch node HTTP transport", () => { it("accepts a supported notification permission set to false", async () => { const baseDir = await tempDirs.make("openclaw-watch-node-permissions-"); - const identity = loadOrCreateDeviceIdentity(path.join(baseDir, "watch-identity.json")); + const identity = loadOrCreateDeviceIdentity({ + path: path.join(baseDir, "watch-identity.sqlite"), + }); const issued = await issueDeviceBootstrapToken({ baseDir, profile: NODE_PAIRING_SETUP_BOOTSTRAP_PROFILE, @@ -258,7 +262,9 @@ describe("watch node HTTP transport", () => { it("does not let attacker challenges evict another client nonce", async () => { const baseDir = await tempDirs.make("openclaw-watch-node-challenge-eviction-"); - const identity = loadOrCreateDeviceIdentity(path.join(baseDir, "watch-identity.json")); + const identity = loadOrCreateDeviceIdentity({ + path: path.join(baseDir, "watch-identity.sqlite"), + }); const issued = await issueDeviceBootstrapToken({ baseDir, profile: NODE_PAIRING_SETUP_BOOTSTRAP_PROFILE, @@ -295,7 +301,9 @@ describe("watch node HTTP transport", () => { it("requires an authenticated disconnect and emits one lifecycle teardown", async () => { const baseDir = await tempDirs.make("openclaw-watch-node-disconnect-"); - const identity = loadOrCreateDeviceIdentity(path.join(baseDir, "watch-identity.json")); + const identity = loadOrCreateDeviceIdentity({ + path: path.join(baseDir, "watch-identity.sqlite"), + }); const issued = await issueDeviceBootstrapToken({ baseDir, profile: NODE_PAIRING_SETUP_BOOTSTRAP_PROFILE, @@ -353,7 +361,9 @@ describe("watch node HTTP transport", () => { it("rejects empty shadow credentials without consuming the challenge", async () => { const baseDir = await tempDirs.make("openclaw-watch-node-auth-fields-"); - const identity = loadOrCreateDeviceIdentity(path.join(baseDir, "watch-identity.json")); + const identity = loadOrCreateDeviceIdentity({ + path: path.join(baseDir, "watch-identity.sqlite"), + }); const issued = await issueDeviceBootstrapToken({ baseDir, profile: NODE_PAIRING_SETUP_BOOTSTRAP_PROFILE, @@ -397,9 +407,9 @@ describe("watch node HTTP transport", () => { }; const abortedBaseDir = await tempDirs.make("openclaw-watch-node-aborted-connect-"); - const abortedIdentity = loadOrCreateDeviceIdentity( - path.join(abortedBaseDir, "watch-identity.json"), - ); + const abortedIdentity = loadOrCreateDeviceIdentity({ + path: path.join(abortedBaseDir, "watch-identity.sqlite"), + }); const abortedBootstrap = await issueDeviceBootstrapToken({ baseDir: abortedBaseDir, profile: NODE_PAIRING_SETUP_BOOTSTRAP_PROFILE, @@ -433,9 +443,9 @@ describe("watch node HTTP transport", () => { } const completedBaseDir = await tempDirs.make("openclaw-watch-node-completed-connect-"); - const completedIdentity = loadOrCreateDeviceIdentity( - path.join(completedBaseDir, "watch-identity.json"), - ); + const completedIdentity = loadOrCreateDeviceIdentity({ + path: path.join(completedBaseDir, "watch-identity.sqlite"), + }); const completedBootstrap = await issueDeviceBootstrapToken({ baseDir: completedBaseDir, profile: NODE_PAIRING_SETUP_BOOTSTRAP_PROFILE, @@ -471,7 +481,9 @@ describe("watch node HTTP transport", () => { it("bootstraps, registers, polls an invoke, and accepts its result", async () => { const baseDir = await tempDirs.make("openclaw-watch-node-http-"); - const identity = loadOrCreateDeviceIdentity(path.join(baseDir, "watch-identity.json")); + const identity = loadOrCreateDeviceIdentity({ + path: path.join(baseDir, "watch-identity.sqlite"), + }); const issued = await issueDeviceBootstrapToken({ baseDir, profile: NODE_PAIRING_SETUP_BOOTSTRAP_PROFILE, diff --git a/src/infra/device-bootstrap.test.ts b/src/infra/device-bootstrap.test.ts index 7e14c1210b28..e3c1313661fb 100644 --- a/src/infra/device-bootstrap.test.ts +++ b/src/infra/device-bootstrap.test.ts @@ -511,7 +511,7 @@ describe("device bootstrap tokens", () => { it("accepts equivalent public key encodings after binding the bootstrap token", async () => { const baseDir = await createTempDir(); - const identity = loadOrCreateDeviceIdentity(path.join(baseDir, "device.json")); + const identity = loadOrCreateDeviceIdentity({ path: path.join(baseDir, "device.sqlite") }); const issued = await issueDeviceBootstrapToken({ baseDir }); const rawPublicKey = publicKeyRawBase64UrlFromPem(identity.publicKeyPem); diff --git a/src/infra/device-identity-coordinator.ts b/src/infra/device-identity-coordinator.ts new file mode 100644 index 000000000000..76247346efc7 --- /dev/null +++ b/src/infra/device-identity-coordinator.ts @@ -0,0 +1,138 @@ +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { resolveGatewayLockDir } from "../config/paths.js"; +import { requireNodeSqlite } from "./node-sqlite.js"; + +const DEFAULT_BUSY_TIMEOUT_MS = 5000; + +class DeviceIdentityCoordinatorError extends Error { + constructor( + message: string, + public override readonly cause?: unknown, + ) { + super(message); + this.name = "DeviceIdentityCoordinatorError"; + } +} + +function canonicalizeDatabasePath(databasePath: string): string { + const resolved = path.resolve(databasePath); + try { + return fs.realpathSync.native(resolved); + } catch { + const missingSegments: string[] = []; + let current = resolved; + while (true) { + const parent = path.dirname(current); + if (parent === current) { + return resolved; + } + missingSegments.push(path.basename(current)); + current = parent; + try { + return path.join(fs.realpathSync.native(current), ...missingSegments.toReversed()); + } catch { + // Existing ancestors can still contain aliases even when the database is absent. + } + } + } +} + +function resolveDeviceIdentityCoordinatorPath( + databasePath: string, + lockDir = resolveGatewayLockDir(), +): string { + const canonicalPath = canonicalizeDatabasePath(databasePath); + const databaseHash = crypto.createHash("sha256").update(canonicalPath).digest("hex").slice(0, 8); + return path.join(lockDir, `device-identity.${databaseHash}.lock.sqlite`); +} + +function ensurePrivateCoordinatorDirectory(lockDir: string): void { + let stats: fs.Stats; + try { + stats = fs.lstatSync(lockDir); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + try { + fs.mkdirSync(lockDir, { mode: 0o700 }); + } catch (mkdirError) { + if ((mkdirError as NodeJS.ErrnoException).code !== "EEXIST") { + throw mkdirError; + } + } + stats = fs.lstatSync(lockDir); + } + if (stats.isSymbolicLink() || !stats.isDirectory()) { + throw new DeviceIdentityCoordinatorError( + "device identity coordinator directory must be a real directory", + ); + } + const uid = typeof process.getuid === "function" ? process.getuid() : undefined; + if (uid !== undefined && stats.uid !== uid) { + throw new DeviceIdentityCoordinatorError( + "device identity coordinator directory belongs to another user", + ); + } + if (process.platform !== "win32") { + fs.chmodSync(lockDir, 0o700); + const secured = fs.lstatSync(lockDir); + if (secured.isSymbolicLink() || !secured.isDirectory() || (secured.mode & 0o077) !== 0) { + throw new DeviceIdentityCoordinatorError( + "device identity coordinator directory permissions are not private", + ); + } + } +} + +export function acquireDeviceIdentityCoordinator(params: { + databasePath: string; + busyTimeoutMs?: number; + lockDir?: string; +}): { release: () => void } { + const coordinatorPath = resolveDeviceIdentityCoordinatorPath(params.databasePath, params.lockDir); + ensurePrivateCoordinatorDirectory(path.dirname(coordinatorPath)); + const { DatabaseSync } = requireNodeSqlite(); + const database = new DatabaseSync(coordinatorPath); + try { + const timeout = Math.max(0, Math.trunc(params.busyTimeoutMs ?? DEFAULT_BUSY_TIMEOUT_MS)); + database.exec(`PRAGMA busy_timeout = ${timeout}; BEGIN EXCLUSIVE;`); + } catch (error) { + try { + database.close(); + } catch {} + throw new DeviceIdentityCoordinatorError( + "device identity migration or creation already owns this state database", + error, + ); + } + + let released = false; + return { + release: () => { + if (released) { + return; + } + released = true; + let releaseError: unknown; + try { + database.exec("ROLLBACK"); + } catch (error) { + releaseError = error; + } + try { + database.close(); + } catch (error) { + releaseError ??= error; + } + if (releaseError) { + throw new DeviceIdentityCoordinatorError( + "failed to release device identity coordinator", + releaseError, + ); + } + }, + }; +} diff --git a/src/infra/device-identity-legacy.ts b/src/infra/device-identity-legacy.ts new file mode 100644 index 000000000000..d20b342e3857 --- /dev/null +++ b/src/infra/device-identity-legacy.ts @@ -0,0 +1,97 @@ +// Canonicalizes retired Node and Swift identity payloads for Doctor import. +import { createHash } from "node:crypto"; +import { + validateStoredDeviceIdentity, + type StoredDeviceIdentity, +} from "./device-identity-store.js"; +import { + decodeCanonicalBase64OrBase64Url, + deriveEd25519PrivateKeyRaw, + deriveEd25519PublicKeyRaw, + ed25519PrivateKeyPemFromRaw, + ed25519PublicKeyPemFromRaw, +} from "./ed25519-signature.js"; + +export type NormalizedLegacyDeviceIdentity = StoredDeviceIdentity; + +function fingerprintPublicKey(publicKeyPem: string): string { + return createHash("sha256").update(deriveEd25519PublicKeyRaw(publicKeyPem)).digest("hex"); +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function isValidCreatedAtMs(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + +function normalizeLegacyCreatedAtMs(value: unknown): number { + // Shipped file readers accepted valid keypairs even when this metadata was + // missing or invalid. Doctor preserves that upgrade path without weakening SQLite. + return isValidCreatedAtMs(value) ? value : Date.now(); +} + +function normalizeLegacyKeyPair(params: { + createdAtMs: number; + privateKeyPem: string; + publicKeyPem: string; +}): NormalizedLegacyDeviceIdentity | null { + try { + const publicKeyRaw = deriveEd25519PublicKeyRaw(params.publicKeyPem); + const privateKeyRaw = deriveEd25519PrivateKeyRaw(params.privateKeyPem); + const publicKeyPem = ed25519PublicKeyPemFromRaw(publicKeyRaw); + const privateKeyPem = ed25519PrivateKeyPemFromRaw(privateKeyRaw); + // Legacy deviceId was derived metadata. Preserve the authoritative key bytes and + // recompute the fingerprint so stale metadata never rotates a shipped identity. + const normalized = { + deviceId: fingerprintPublicKey(publicKeyPem), + publicKeyPem, + privateKeyPem, + createdAtMs: params.createdAtMs, + }; + validateStoredDeviceIdentity(normalized); + return normalized; + } catch { + return null; + } +} + +/** Normalize a retired Node PEM or Swift raw-key payload for Doctor import. */ +export function normalizeLegacyDeviceIdentity( + value: unknown, +): NormalizedLegacyDeviceIdentity | null { + if ( + isRecord(value) && + value.version === 1 && + typeof value.deviceId === "string" && + typeof value.publicKeyPem === "string" && + typeof value.privateKeyPem === "string" + ) { + return normalizeLegacyKeyPair({ + createdAtMs: normalizeLegacyCreatedAtMs(value.createdAtMs), + privateKeyPem: value.privateKeyPem, + publicKeyPem: value.publicKeyPem, + }); + } + if ( + isRecord(value) && + !("version" in value) && + typeof value.deviceId === "string" && + typeof value.publicKey === "string" && + typeof value.privateKey === "string" + ) { + try { + const publicKeyRaw = decodeCanonicalBase64OrBase64Url(value.publicKey); + const privateKeyRaw = decodeCanonicalBase64OrBase64Url(value.privateKey); + return normalizeLegacyKeyPair({ + createdAtMs: normalizeLegacyCreatedAtMs(value.createdAtMs), + privateKeyPem: ed25519PrivateKeyPemFromRaw(privateKeyRaw), + publicKeyPem: ed25519PublicKeyPemFromRaw(publicKeyRaw), + }); + } catch { + return null; + } + } + return null; +} diff --git a/src/infra/device-identity-store.ts b/src/infra/device-identity-store.ts new file mode 100644 index 000000000000..da3b978c3e70 --- /dev/null +++ b/src/infra/device-identity-store.ts @@ -0,0 +1,426 @@ +// Canonical SQLite storage for gateway/device Ed25519 identities. +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import type { Insertable, Selectable } from "kysely"; +import { withOpenClawStateDatabaseReadOnly } from "../state/openclaw-state-db-readonly.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, + type OpenClawStateDatabaseOptions, +} from "../state/openclaw-state-db.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; +import { + deriveCanonicalEd25519PrivateKeyRaw, + deriveCanonicalEd25519PublicKeyRaw, +} from "./ed25519-signature.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; + +export const PRIMARY_DEVICE_IDENTITY_KEY = "primary"; + +export type DeviceIdentity = { + deviceId: string; + publicKeyPem: string; + privateKeyPem: string; +}; + +export type StoredDeviceIdentity = DeviceIdentity & { + createdAtMs: number; +}; + +export type DeviceIdentityStoreOptions = OpenClawStateDatabaseOptions & { + identityKey?: string; +}; + +type DeviceIdentityDatabase = Pick; +type DeviceIdentityRow = Selectable; +type DeviceIdentityInsert = Insertable; + +export class DeviceIdentityStorageError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = "DeviceIdentityStorageError"; + } +} + +function normalizeIdentityKey(key: string | undefined): string { + const normalized = key ?? PRIMARY_DEVICE_IDENTITY_KEY; + if (normalized.length === 0 || normalized !== normalized.trim()) { + throw new DeviceIdentityStorageError( + "Device identity key must be a non-empty string without surrounding whitespace.", + ); + } + if (normalized.length > 128) { + throw new DeviceIdentityStorageError("Device identity key exceeds 128 characters."); + } + return normalized; +} + +function invalidStoredIdentityError( + identityKey: string, + cause?: unknown, +): DeviceIdentityStorageError { + return new DeviceIdentityStorageError( + `SQLite contains an invalid persisted device identity "${identityKey}". Run "openclaw doctor --fix" before starting the gateway or connecting this client.`, + cause === undefined ? undefined : { cause }, + ); +} + +function fingerprintPublicKey(publicKeyPem: string): string { + const raw = deriveCanonicalEd25519PublicKeyRaw(publicKeyPem); + return crypto.createHash("sha256").update(raw).digest("hex"); +} + +/** Generate canonical Ed25519 material before entering a synchronous write transaction. */ +export function generateStoredDeviceIdentity(now = Date.now()): StoredDeviceIdentity { + const { publicKey, privateKey } = crypto.generateKeyPairSync("ed25519"); + const publicKeyPem = publicKey.export({ type: "spki", format: "pem" }); + const privateKeyPem = privateKey.export({ type: "pkcs8", format: "pem" }); + return { + deviceId: fingerprintPublicKey(publicKeyPem), + publicKeyPem, + privateKeyPem, + createdAtMs: now, + }; +} + +function keyPairMatches(publicKeyPem: string, privateKeyPem: string): boolean { + try { + deriveCanonicalEd25519PublicKeyRaw(publicKeyPem); + deriveCanonicalEd25519PrivateKeyRaw(privateKeyPem); + const publicKey = crypto.createPublicKey(publicKeyPem); + const privateKey = crypto.createPrivateKey(privateKeyPem); + if (publicKey.asymmetricKeyType !== "ed25519" || privateKey.asymmetricKeyType !== "ed25519") { + return false; + } + const derivedPublicKey = crypto + .createPublicKey(privateKeyPem) + .export({ type: "spki", format: "der" }); + const storedPublicKey = publicKey.export({ type: "spki", format: "der" }); + return Buffer.from(derivedPublicKey).equals(Buffer.from(storedPublicKey)); + } catch { + return false; + } +} + +function parseCreatedAtMs(value: unknown): number | null { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null; +} + +/** Validate persisted key material and return the canonical runtime shape. */ +export function validateStoredDeviceIdentity( + value: StoredDeviceIdentity, + identityKey = PRIMARY_DEVICE_IDENTITY_KEY, +): DeviceIdentity { + try { + if ( + !value.deviceId || + !/^[a-f0-9]{64}$/.test(value.deviceId) || + !value.publicKeyPem || + !value.privateKeyPem || + parseCreatedAtMs(value.createdAtMs) === null || + !keyPairMatches(value.publicKeyPem, value.privateKeyPem) + ) { + throw invalidStoredIdentityError(identityKey); + } + const derivedDeviceId = fingerprintPublicKey(value.publicKeyPem); + if (derivedDeviceId !== value.deviceId) { + throw invalidStoredIdentityError(identityKey); + } + return { + deviceId: value.deviceId, + publicKeyPem: value.publicKeyPem, + privateKeyPem: value.privateKeyPem, + }; + } catch (error) { + if (error instanceof DeviceIdentityStorageError) { + throw error; + } + throw invalidStoredIdentityError(identityKey, error); + } +} + +function rowToStoredIdentity( + row: DeviceIdentityRow, + expectedIdentityKey: string, +): StoredDeviceIdentity { + if ( + row.identity_key !== expectedIdentityKey || + typeof row.device_id !== "string" || + typeof row.public_key_pem !== "string" || + typeof row.private_key_pem !== "string" || + parseCreatedAtMs(row.created_at_ms) === null || + parseCreatedAtMs(row.updated_at_ms) === null + ) { + throw invalidStoredIdentityError(expectedIdentityKey); + } + return { + deviceId: row.device_id, + publicKeyPem: row.public_key_pem, + privateKeyPem: row.private_key_pem, + createdAtMs: row.created_at_ms, + }; +} + +function salvageStoredIdentityRow( + row: DeviceIdentityRow, + expectedIdentityKey: string, + repairedAtMs: number, +): StoredDeviceIdentity | null { + // Device ids, timestamps, and PEM framing are repairable metadata. Preserve matching + // Ed25519 key bytes because rotating them would invalidate pairing and stored auth. + if ( + row.identity_key !== expectedIdentityKey || + typeof row.public_key_pem !== "string" || + typeof row.private_key_pem !== "string" + ) { + return null; + } + try { + const publicKey = crypto.createPublicKey(row.public_key_pem); + const privateKey = crypto.createPrivateKey(row.private_key_pem); + if (publicKey.asymmetricKeyType !== "ed25519" || privateKey.asymmetricKeyType !== "ed25519") { + return null; + } + const canonicalPublicKeyPem = publicKey.export({ type: "spki", format: "pem" }); + const canonicalPrivateKeyPem = privateKey.export({ type: "pkcs8", format: "pem" }); + const derivedPublicKeyPem = crypto + .createPublicKey(canonicalPrivateKeyPem) + .export({ type: "spki", format: "pem" }); + if (derivedPublicKeyPem !== canonicalPublicKeyPem) { + return null; + } + const createdAtMs = + parseCreatedAtMs(row.created_at_ms) ?? parseCreatedAtMs(row.updated_at_ms) ?? repairedAtMs; + const salvaged = { + deviceId: fingerprintPublicKey(canonicalPublicKeyPem), + publicKeyPem: canonicalPublicKeyPem, + privateKeyPem: canonicalPrivateKeyPem, + createdAtMs, + }; + validateStoredDeviceIdentity(salvaged, expectedIdentityKey); + return salvaged; + } catch { + return null; + } +} + +function storedIdentityToRow( + identityKey: string, + stored: StoredDeviceIdentity, + updatedAtMs = stored.createdAtMs, +): DeviceIdentityInsert { + return { + identity_key: identityKey, + device_id: stored.deviceId, + public_key_pem: stored.publicKeyPem, + private_key_pem: stored.privateKeyPem, + created_at_ms: stored.createdAtMs, + updated_at_ms: updatedAtMs, + }; +} + +function readStoredIdentityRowFromDatabase( + database: { db: Parameters[0] }, + identityKey: string, +): DeviceIdentityRow | null { + const db = getNodeSqliteKysely(database.db); + return ( + executeSqliteQueryTakeFirstSync( + database.db, + db.selectFrom("device_identities").selectAll().where("identity_key", "=", identityKey), + ) ?? null + ); +} + +function readStoredIdentityFromDatabase( + database: { db: Parameters[0] }, + identityKey: string, +): StoredDeviceIdentity | null { + const row = readStoredIdentityRowFromDatabase(database, identityKey); + return row ? rowToStoredIdentity(row, identityKey) : null; +} + +/** Resolve the concrete database and row identity used by process caches and diagnostics. */ +export function resolveDeviceIdentityStore(options: DeviceIdentityStoreOptions = {}): { + databasePath: string; + identityKey: string; +} { + return { + databasePath: path.resolve( + options.path ?? resolveOpenClawStateSqlitePath(options.env ?? process.env), + ), + identityKey: normalizeIdentityKey(options.identityKey), + }; +} + +/** Read through the writable shared-state lifecycle, validating any existing row. */ +export function readStoredDeviceIdentity( + options: DeviceIdentityStoreOptions = {}, +): StoredDeviceIdentity | null { + const resolved = resolveDeviceIdentityStore(options); + const database = openOpenClawStateDatabase({ + env: options.env, + path: resolved.databasePath, + }); + const stored = readStoredIdentityFromDatabase(database, resolved.identityKey); + if (stored) { + validateStoredDeviceIdentity(stored, resolved.identityKey); + } + return stored; +} + +/** Read without creating, repairing, chmodding, or joining the writer lifecycle. */ +export function readStoredDeviceIdentityReadOnly( + options: DeviceIdentityStoreOptions = {}, +): StoredDeviceIdentity | null { + const resolved = resolveDeviceIdentityStore(options); + try { + fs.lstatSync(resolved.databasePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + return null; + } + return withOpenClawStateDatabaseReadOnly( + (database) => { + const stored = readStoredIdentityFromDatabase(database, resolved.identityKey); + if (stored) { + validateStoredDeviceIdentity(stored, resolved.identityKey); + } + return stored; + }, + { env: options.env, path: resolved.databasePath }, + ); +} + +/** Insert a candidate only when the key is still absent, then return the authoritative row. */ +export function insertStoredDeviceIdentityIfAbsent( + candidate: StoredDeviceIdentity, + options: DeviceIdentityStoreOptions = {}, +): StoredDeviceIdentity { + const resolved = resolveDeviceIdentityStore(options); + validateStoredDeviceIdentity(candidate, resolved.identityKey); + return runOpenClawStateWriteTransaction( + ({ db }) => { + const existing = readStoredIdentityFromDatabase({ db }, resolved.identityKey); + if (existing) { + validateStoredDeviceIdentity(existing, resolved.identityKey); + } else { + const kysely = getNodeSqliteKysely(db); + executeSqliteQuerySync( + db, + kysely + .insertInto("device_identities") + .values(storedIdentityToRow(resolved.identityKey, candidate)) + .onConflict((conflict) => conflict.column("identity_key").doNothing()), + ); + } + const authoritative = readStoredIdentityFromDatabase({ db }, resolved.identityKey); + if (!authoritative) { + throw new DeviceIdentityStorageError( + `SQLite device identity "${resolved.identityKey}" was not durable after insert.`, + ); + } + validateStoredDeviceIdentity(authoritative, resolved.identityKey); + return authoritative; + }, + { env: options.env, path: resolved.databasePath }, + { operationLabel: "device-identity.create" }, + ); +} + +/** Replace only an invalid authoritative row; preserve a valid concurrent winner. */ +export function repairInvalidStoredDeviceIdentity( + candidate: StoredDeviceIdentity, + options: DeviceIdentityStoreOptions = {}, +): { identity: StoredDeviceIdentity; repaired: boolean; rotated: boolean } { + const resolved = resolveDeviceIdentityStore(options); + validateStoredDeviceIdentity(candidate, resolved.identityKey); + return runOpenClawStateWriteTransaction( + ({ db }) => { + let repaired = false; + let rotated = false; + let existingRow: DeviceIdentityRow | null = null; + try { + existingRow = readStoredIdentityRowFromDatabase({ db }, resolved.identityKey); + const existing = existingRow + ? rowToStoredIdentity(existingRow, resolved.identityKey) + : null; + if (existing) { + validateStoredDeviceIdentity(existing, resolved.identityKey); + return { identity: existing, repaired, rotated }; + } + } catch (error) { + if (!(error instanceof DeviceIdentityStorageError)) { + throw error; + } + } + if (existingRow) { + const salvaged = salvageStoredIdentityRow( + existingRow, + resolved.identityKey, + candidate.createdAtMs, + ); + if (salvaged) { + executeSqliteQuerySync( + db, + getNodeSqliteKysely(db) + .updateTable("device_identities") + .set({ + device_id: salvaged.deviceId, + public_key_pem: salvaged.publicKeyPem, + private_key_pem: salvaged.privateKeyPem, + created_at_ms: salvaged.createdAtMs, + updated_at_ms: candidate.createdAtMs, + }) + .where("identity_key", "=", resolved.identityKey), + ); + const authoritative = readStoredIdentityFromDatabase({ db }, resolved.identityKey); + if (!authoritative) { + throw new DeviceIdentityStorageError( + `SQLite device identity "${resolved.identityKey}" was not durable after repair.`, + ); + } + validateStoredDeviceIdentity(authoritative, resolved.identityKey); + return { identity: authoritative, repaired: true, rotated }; + } + executeSqliteQuerySync( + db, + getNodeSqliteKysely(db) + .deleteFrom("device_identities") + .where("identity_key", "=", resolved.identityKey), + ); + } + + // An absent row after an invalid-row detection still means identity continuity was lost. + // Report the generated winner so Doctor always surfaces the required re-approval. + repaired = true; + rotated = true; + + executeSqliteQuerySync( + db, + getNodeSqliteKysely(db) + .insertInto("device_identities") + .values(storedIdentityToRow(resolved.identityKey, candidate)) + .onConflict((conflict) => conflict.column("identity_key").doNothing()), + ); + const authoritative = readStoredIdentityFromDatabase({ db }, resolved.identityKey); + if (!authoritative) { + throw new DeviceIdentityStorageError( + `SQLite device identity "${resolved.identityKey}" was not durable after repair.`, + ); + } + validateStoredDeviceIdentity(authoritative, resolved.identityKey); + return { identity: authoritative, repaired, rotated }; + }, + { env: options.env, path: resolved.databasePath }, + { operationLabel: "device-identity.doctor-repair" }, + ); +} diff --git a/src/infra/device-identity.state-dir.test.ts b/src/infra/device-identity.state-dir.test.ts index 968ab6d623cd..1f791006e7ec 100644 --- a/src/infra/device-identity.state-dir.test.ts +++ b/src/infra/device-identity.state-dir.test.ts @@ -1,76 +1,42 @@ -// Covers default device identity path under state dir. -import fs from "node:fs/promises"; +// Covers default device identity SQLite path under the state dir. +import fs from "node:fs"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { withStateDirEnv } from "../test-helpers/state-dir-env.js"; -import { loadOrCreateDeviceIdentity } from "./device-identity.js"; +import { loadDeviceIdentityIfPresent, loadOrCreateDeviceIdentity } from "./device-identity.js"; + +afterEach(() => { + closeOpenClawStateDatabaseForTest(); +}); describe("device identity state dir defaults", () => { - it("writes the default identity file under OPENCLAW_STATE_DIR", async () => { + it("writes the default identity to the shared state database", async () => { await withStateDirEnv("openclaw-identity-state-", async ({ stateDir }) => { const identity = loadOrCreateDeviceIdentity(); - const identityPath = path.join(stateDir, "identity", "device.json"); - const raw = JSON.parse(await fs.readFile(identityPath, "utf8")) as { deviceId?: string }; - expect(raw.deviceId).toBe(identity.deviceId); + const databasePath = path.join(stateDir, "state", "openclaw.sqlite"); + + expect(loadDeviceIdentityIfPresent()).toEqual(identity); + expect(fs.existsSync(databasePath)).toBe(true); + expect(fs.existsSync(path.join(stateDir, "identity", "device.json"))).toBe(false); }); }); it("reuses the stored identity on subsequent loads", async () => { - await withStateDirEnv("openclaw-identity-state-", async ({ stateDir }) => { + await withStateDirEnv("openclaw-identity-state-", async () => { const first = loadOrCreateDeviceIdentity(); const second = loadOrCreateDeviceIdentity(); - const identityPath = path.join(stateDir, "identity", "device.json"); - const raw = JSON.parse(await fs.readFile(identityPath, "utf8")) as { - deviceId?: string; - publicKeyPem?: string; - }; expect(second).toEqual(first); - expect(raw.deviceId).toBe(first.deviceId); - expect(raw.publicKeyPem).toBe(first.publicKeyPem); }); }); - it("repairs stored device IDs that no longer match the public key", async () => { + it("keeps read-only lookup non-creating when the default database is absent", async () => { await withStateDirEnv("openclaw-identity-state-", async ({ stateDir }) => { - const original = loadOrCreateDeviceIdentity(); - const identityPath = path.join(stateDir, "identity", "device.json"); - const raw = JSON.parse(await fs.readFile(identityPath, "utf8")) as Record; + const databasePath = path.join(stateDir, "state", "openclaw.sqlite"); - await fs.writeFile( - identityPath, - `${JSON.stringify({ ...raw, deviceId: "stale-device-id" }, null, 2)}\n`, - "utf8", - ); - - const repaired = loadOrCreateDeviceIdentity(); - const stored = JSON.parse(await fs.readFile(identityPath, "utf8")) as { deviceId?: string }; - - expect(repaired.deviceId).toBe(original.deviceId); - expect(stored.deviceId).toBe(original.deviceId); - }); - }); - - it("returns a transient identity without overwriting an invalid stored file", async () => { - await withStateDirEnv("openclaw-identity-state-", async ({ stateDir }) => { - const identityPath = path.join(stateDir, "identity", "device.json"); - await fs.mkdir(path.dirname(identityPath), { recursive: true }); - const before = [ - "{", - ' "version": 1,', - ' "deviceId": "broken",', - ' "publicKeyPem": "not-a-valid-public-key",', - ' "privateKeyPem": "not-a-valid-private-key"', - "}", - "", - ].join("\n"); - await fs.writeFile(identityPath, before, "utf8"); - - const regenerated = loadOrCreateDeviceIdentity(); - const stored = await fs.readFile(identityPath, "utf8"); - - expect(regenerated.deviceId).not.toBe("broken"); - expect(stored).toBe(before); + expect(loadDeviceIdentityIfPresent()).toBeNull(); + expect(fs.existsSync(databasePath)).toBe(false); }); }); }); diff --git a/src/infra/device-identity.test.ts b/src/infra/device-identity.test.ts index f5f7336051be..fba7c89dcbb2 100644 --- a/src/infra/device-identity.test.ts +++ b/src/infra/device-identity.test.ts @@ -1,8 +1,17 @@ -// Covers device identity creation, conversion, signing, and verification. +// Covers SQLite device identity creation, migration boundaries, and crypto helpers. +import { spawn, type ChildProcess } from "node:child_process"; +import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; +import { + closeOpenClawStateDatabaseForTest, + OPENCLAW_STATE_SCHEMA_VERSION, +} from "../state/openclaw-state-db.js"; import { withTempDir } from "../test-utils/temp-dir.js"; +import { acquireDeviceIdentityCoordinator } from "./device-identity-coordinator.js"; +import { normalizeLegacyDeviceIdentity } from "./device-identity-legacy.js"; +import type { DeviceIdentityStoreOptions } from "./device-identity-store.js"; import { deriveDeviceIdFromPublicKey, loadDeviceIdentityIfPresent, @@ -12,6 +21,7 @@ import { publicKeyRawBase64UrlFromPem, signDevicePayload, verifyDeviceSignature, + type DeviceIdentity, } from "./device-identity.js"; const SWIFT_RAW_DEVICE_ID = "56475aa75463474c0285df5dbf2bcab73da651358839e9b77481b2eab107708c"; @@ -19,218 +29,392 @@ const SWIFT_RAW_PUBLIC_KEY = "A6EHv/POEL4dcN0Y50vAmWfk1jCbpQ1fHdyGZBJVMbg="; const SWIFT_RAW_PRIVATE_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8="; // pragma: allowlist secret const MISMATCHED_SWIFT_RAW_PRIVATE_KEY = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE="; // pragma: allowlist secret -async function withIdentity( - run: (identity: ReturnType) => void, -) { - await withTempDir("openclaw-device-identity-", async (dir) => { - const identity = loadOrCreateDeviceIdentity(path.join(dir, "device.json")); - run(identity); +afterEach(() => { + closeOpenClawStateDatabaseForTest(); +}); + +function storeOptions(rootDir: string, identityKey?: string): DeviceIdentityStoreOptions { + return { + env: { ...process.env, OPENCLAW_STATE_DIR: rootDir }, + path: path.join(rootDir, "state", "openclaw.sqlite"), + ...(identityKey ? { identityKey } : {}), + }; +} + +function waitForChild(child: ChildProcess): Promise { + let stdout = ""; + let stderr = ""; + child.stdout?.on("data", (chunk) => (stdout += String(chunk))); + child.stderr?.on("data", (chunk) => (stderr += String(chunk))); + return new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => { + if (code !== 0) { + reject(new Error(`identity worker failed (${String(code ?? signal)}): ${stderr}`)); + return; + } + const resultLine = stdout.trim().split("\n").at(-1); + if (!resultLine) { + reject(new Error("identity worker produced no result")); + return; + } + resolve(JSON.parse(resultLine) as DeviceIdentity); + }); }); } -describe("device identity crypto helpers", () => { - it("loads an existing identity without creating a missing file", async () => { - await withTempDir("openclaw-device-identity-readonly-", async (dir) => { - const identityPath = path.join(dir, "identity", "device.json"); - - expect(loadDeviceIdentityIfPresent(identityPath)).toBeNull(); - expect(fs.existsSync(identityPath)).toBe(false); - - const created = loadOrCreateDeviceIdentity(identityPath); - - expect(loadDeviceIdentityIfPresent(identityPath)).toEqual(created); - }); - }); - - it("does not repair mismatched stored device ids in read-only mode", async () => { - await withTempDir("openclaw-device-identity-readonly-", async (dir) => { - const identityPath = path.join(dir, "identity", "device.json"); - loadOrCreateDeviceIdentity(identityPath); - const stored = JSON.parse(fs.readFileSync(identityPath, "utf8")) as Record; - fs.writeFileSync( - identityPath, - `${JSON.stringify({ ...stored, deviceId: "mismatched" }, null, 2)}\n`, - "utf8", - ); - const before = fs.readFileSync(identityPath, "utf8"); - - expect(loadDeviceIdentityIfPresent(identityPath)).toBeNull(); - expect(fs.readFileSync(identityPath, "utf8")).toBe(before); - }); - }); - - it("loads Swift raw-key identity files without generating a new device id", async () => { - await withTempDir("openclaw-device-identity-swift-", async (dir) => { - const identityPath = path.join(dir, "identity", "device.json"); - fs.mkdirSync(path.dirname(identityPath), { recursive: true }); - fs.writeFileSync( - identityPath, - `${JSON.stringify( - { - deviceId: SWIFT_RAW_DEVICE_ID, - publicKey: SWIFT_RAW_PUBLIC_KEY, - privateKey: SWIFT_RAW_PRIVATE_KEY, - createdAtMs: 1_700_000_000_000, - }, - null, - 2, - )}\n`, - "utf8", - ); - - const readonly = loadDeviceIdentityIfPresent(identityPath); - const loaded = loadOrCreateDeviceIdentity(identityPath); - const stored = JSON.parse(fs.readFileSync(identityPath, "utf8")) as Record; - - expect(readonly?.deviceId).toBe(SWIFT_RAW_DEVICE_ID); - expect(loaded.deviceId).toBe(SWIFT_RAW_DEVICE_ID); - expect(publicKeyRawBase64UrlFromPem(loaded.publicKeyPem)).toBe( - "A6EHv_POEL4dcN0Y50vAmWfk1jCbpQ1fHdyGZBJVMbg", - ); - expect( - verifyDeviceSignature( - loaded.publicKeyPem, - "hello", - signDevicePayload(loaded.privateKeyPem, "hello"), - ), - ).toBe(true); - expect(stored.version).toBe(1); - expect(stored.deviceId).toBe(SWIFT_RAW_DEVICE_ID); - expect(typeof stored.publicKeyPem).toBe("string"); - expect(typeof stored.privateKeyPem).toBe("string"); - const publicKeyPem = stored.publicKeyPem as string; - const privateKeyPem = stored.privateKeyPem as string; - expect(publicKeyPem.startsWith("-----BEGIN PUBLIC KEY-----\n")).toBe(true); - expect(publicKeyPem.endsWith("-----END PUBLIC KEY-----\n")).toBe(true); - expect(privateKeyPem.startsWith("-----BEGIN PRIVATE KEY-----\n")).toBe(true); - expect(privateKeyPem.endsWith("-----END PRIVATE KEY-----\n")).toBe(true); - expect(stored.createdAtMs).toBe(1_700_000_000_000); - expect(stored).not.toHaveProperty("publicKey"); - expect(stored).not.toHaveProperty("privateKey"); - }); - }); - - it("does not overwrite recognized invalid identity files", async () => { - await withTempDir("openclaw-device-identity-invalid-", async (dir) => { - const identityPath = path.join(dir, "identity", "device.json"); - fs.mkdirSync(path.dirname(identityPath), { recursive: true }); - fs.writeFileSync( - identityPath, - `${JSON.stringify( - { - version: 1, - deviceId: "stale-device-id", - publicKeyPem: "not-a-valid-public-key", - privateKeyPem: "not-a-valid-private-key", // pragma: allowlist secret - createdAtMs: 1_700_000_000_000, - }, - null, - 2, - )}\n`, - "utf8", - ); - const before = fs.readFileSync(identityPath, "utf8"); - - expect(loadDeviceIdentityIfPresent(identityPath)).toBeNull(); - const loaded = loadOrCreateDeviceIdentity(identityPath); - const processIdentity = loadOrCreateProcessDeviceIdentity(identityPath); - - expect(loaded.deviceId).not.toBe("stale-device-id"); - expect(loadOrCreateProcessDeviceIdentity(identityPath)).toBe(processIdentity); - expect(fs.readFileSync(identityPath, "utf8")).toBe(before); - }); - }); - - it("does not overwrite existing unrecognized identity files", async () => { - await withTempDir("openclaw-device-identity-unrecognized-", async (dir) => { - const identityPath = path.join(dir, "identity", "device.json"); - fs.mkdirSync(path.dirname(identityPath), { recursive: true }); - fs.writeFileSync( - identityPath, - `${JSON.stringify( - { - schema: "future-openclaw-device-identity", - stableDeviceId: "app-group-device-id", - }, - null, - 2, - )}\n`, - "utf8", - ); - const before = fs.readFileSync(identityPath, "utf8"); - - expect(loadDeviceIdentityIfPresent(identityPath)).toBeNull(); - const loaded = loadOrCreateDeviceIdentity(identityPath); - - expect(loaded.deviceId).not.toBe("app-group-device-id"); - expect(fs.readFileSync(identityPath, "utf8")).toBe(before); - }); - }); - - it("does not migrate Swift raw-key identity files with mismatched key material", async () => { - await withTempDir("openclaw-device-identity-swift-invalid-", async (dir) => { - const identityPath = path.join(dir, "identity", "device.json"); - fs.mkdirSync(path.dirname(identityPath), { recursive: true }); - fs.writeFileSync( - identityPath, - `${JSON.stringify( - { - deviceId: SWIFT_RAW_DEVICE_ID, - publicKey: SWIFT_RAW_PUBLIC_KEY, - privateKey: MISMATCHED_SWIFT_RAW_PRIVATE_KEY, - createdAtMs: 1_700_000_000_000, - }, - null, - 2, - )}\n`, - "utf8", - ); - const before = fs.readFileSync(identityPath, "utf8"); - - expect(loadDeviceIdentityIfPresent(identityPath)).toBeNull(); - const loaded = loadOrCreateDeviceIdentity(identityPath); - - expect(loaded.deviceId).not.toBe(SWIFT_RAW_DEVICE_ID); - expect(fs.readFileSync(identityPath, "utf8")).toBe(before); - }); - }); - - it("derives the same canonical raw key and device id from pem and encoded public keys", async () => { - await withIdentity((identity) => { - const publicKeyRaw = publicKeyRawBase64UrlFromPem(identity.publicKeyPem); - const paddedBase64 = `${publicKeyRaw.replaceAll("-", "+").replaceAll("_", "/")}==`; - - expect(normalizeDevicePublicKeyBase64Url(identity.publicKeyPem)).toBe(publicKeyRaw); - expect(normalizeDevicePublicKeyBase64Url(paddedBase64)).toBe(publicKeyRaw); - expect(deriveDeviceIdFromPublicKey(identity.publicKeyPem)).toBe(identity.deviceId); - expect(deriveDeviceIdFromPublicKey(publicKeyRaw)).toBe(identity.deviceId); - }); - }); - - it("signs payloads that verify against pem and raw public key forms", async () => { - await withIdentity((identity) => { - const payload = JSON.stringify({ - action: "system.run", - ts: 1234, +async function runConcurrentIdentityLoads(rootDir: string): Promise { + const startPath = path.join(rootDir, "identity-start"); + const moduleUrl = new URL("./device-identity.ts", import.meta.url).href; + const workerSource = ` + import fs from "node:fs"; + const { loadOrCreateDeviceIdentity } = await import(process.env.OPENCLAW_IDENTITY_MODULE); + fs.writeFileSync(process.env.OPENCLAW_IDENTITY_READY_PATH, "ready"); + const deadline = Date.now() + 15_000; + while (!fs.existsSync(process.env.OPENCLAW_IDENTITY_START_PATH)) { + if (Date.now() >= deadline) { + throw new Error("timed out waiting for concurrent identity start"); + } + await new Promise((resolve) => { + setTimeout(resolve, 2); }); - const signature = signDevicePayload(identity.privateKeyPem, payload); - const publicKeyRaw = publicKeyRawBase64UrlFromPem(identity.publicKeyPem); + } + const identity = loadOrCreateDeviceIdentity({ + env: { ...process.env, OPENCLAW_STATE_DIR: process.env.OPENCLAW_IDENTITY_STATE_DIR }, + path: process.env.OPENCLAW_IDENTITY_DATABASE_PATH, + }); + console.log(JSON.stringify(identity)); + `; + const workers = [0, 1].map((index) => { + const readyPath = path.join(rootDir, `identity-ready-${index}`); + const child = spawn( + process.execPath, + ["--import", "tsx", "--input-type=module", "-e", workerSource], + { + env: { + ...process.env, + OPENCLAW_IDENTITY_DATABASE_PATH: path.join(rootDir, "state", "openclaw.sqlite"), + OPENCLAW_IDENTITY_MODULE: moduleUrl, + OPENCLAW_IDENTITY_READY_PATH: readyPath, + OPENCLAW_IDENTITY_START_PATH: startPath, + OPENCLAW_IDENTITY_STATE_DIR: rootDir, + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + return { child, outcome: waitForChild(child), readyPath }; + }); - expect(verifyDeviceSignature(identity.publicKeyPem, payload, signature)).toBe(true); - expect(verifyDeviceSignature(publicKeyRaw, payload, signature)).toBe(true); - expect(verifyDeviceSignature(publicKeyRaw, `${payload}!`, signature)).toBe(false); + try { + const deadline = Date.now() + 15_000; + while (!workers.every((worker) => fs.existsSync(worker.readyPath))) { + if (workers.some(({ child }) => child.exitCode !== null || child.signalCode !== null)) { + break; + } + if (Date.now() >= deadline) { + throw new Error("timed out waiting for concurrent identity workers"); + } + await new Promise((resolve) => { + setTimeout(resolve, 2); + }); + } + fs.writeFileSync(startPath, "start"); + return await Promise.all(workers.map((worker) => worker.outcome)); + } finally { + for (const { child } of workers) { + if (child.exitCode === null && child.signalCode === null) { + child.kill(); + } + } + await Promise.allSettled(workers.map((worker) => worker.outcome)); + } +} + +describe("device identity SQLite store", () => { + it("serializes identity ownership with the shared SQLite coordinator", async () => { + await withTempDir("openclaw-device-identity-coordinator-", async (rootDir) => { + const databasePath = path.join(rootDir, "state", "openclaw.sqlite"); + const lockDir = path.join(rootDir, "locks"); + const first = acquireDeviceIdentityCoordinator({ databasePath, lockDir, busyTimeoutMs: 0 }); + try { + expect(() => + acquireDeviceIdentityCoordinator({ databasePath, lockDir, busyTimeoutMs: 0 }), + ).toThrow(/migration or creation already owns this state database/); + } finally { + first.release(); + } + + const next = acquireDeviceIdentityCoordinator({ databasePath, lockDir, busyTimeoutMs: 0 }); + next.release(); + + fs.chmodSync(lockDir, 0o755); + const secured = acquireDeviceIdentityCoordinator({ databasePath, lockDir, busyTimeoutMs: 0 }); + try { + expect(fs.statSync(lockDir).mode & 0o077).toBe(0); + } finally { + secured.release(); + } + + const symlinkLockDir = path.join(rootDir, "symlink-locks"); + fs.symlinkSync(lockDir, symlinkLockDir); + expect(() => + acquireDeviceIdentityCoordinator({ + databasePath, + lockDir: symlinkLockDir, + busyTimeoutMs: 0, + }), + ).toThrow(/real directory/); }); }); - it("fails closed for invalid public keys and signatures", async () => { - await withIdentity((identity) => { - const payload = "hello"; - const signature = signDevicePayload(identity.privateKeyPem, payload); + it("reads a missing database without creating files", async () => { + await withTempDir("openclaw-device-identity-readonly-", async (rootDir) => { + const options = storeOptions(rootDir); + expect(loadDeviceIdentityIfPresent(options)).toBeNull(); + expect(fs.existsSync(options.path!)).toBe(false); + expect(fs.existsSync(path.dirname(options.path!))).toBe(false); + }); + }); - expect(normalizeDevicePublicKeyBase64Url("-----BEGIN PUBLIC KEY-----broken")).toBeNull(); - expect(deriveDeviceIdFromPublicKey("%%%")).toBeNull(); - expect(verifyDeviceSignature("%%%invalid%%%", payload, signature)).toBe(false); - expect(verifyDeviceSignature(identity.publicKeyPem, payload, "%%%invalid%%%")).toBe(false); + it("creates and reuses the primary identity in SQLite", async () => { + await withTempDir("openclaw-device-identity-create-", async (rootDir) => { + const options = storeOptions(rootDir); + const created = loadOrCreateDeviceIdentity(options); + const loaded = loadOrCreateDeviceIdentity(options); + + expect(loaded).toEqual(created); + expect(loadDeviceIdentityIfPresent(options)).toEqual(created); + expect(fs.existsSync(options.path!)).toBe(true); + expect(fs.existsSync(path.join(rootDir, "identity", "device.json"))).toBe(false); + }); + }); + + it("adopts a Swift-created version-zero identity database and completes the shared schema", async () => { + await withTempDir("openclaw-device-identity-swift-db-", async (rootDir) => { + const options = storeOptions(rootDir); + const expected = normalizeLegacyDeviceIdentity({ + deviceId: SWIFT_RAW_DEVICE_ID, + publicKey: SWIFT_RAW_PUBLIC_KEY, + privateKey: SWIFT_RAW_PRIVATE_KEY, + createdAtMs: 1_700_000_000_000, + }); + if (!expected) { + throw new Error("Swift identity fixture must normalize"); + } + fs.mkdirSync(path.dirname(options.path!), { recursive: true }); + const sqlite = await import("node:sqlite"); + const database = new sqlite.DatabaseSync(options.path!); + database.exec(` + CREATE TABLE device_identities ( + identity_key TEXT NOT NULL PRIMARY KEY, + device_id TEXT NOT NULL, + public_key_pem TEXT NOT NULL, + private_key_pem TEXT NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL + ) STRICT; + CREATE INDEX idx_device_identities_device + ON device_identities(device_id, updated_at_ms DESC); + `); + database + .prepare(` + INSERT INTO device_identities ( + identity_key, device_id, public_key_pem, private_key_pem, created_at_ms, updated_at_ms + ) VALUES (?, ?, ?, ?, ?, ?) + `) + .run( + "primary", + expected.deviceId, + expected.publicKeyPem, + expected.privateKeyPem, + expected.createdAtMs, + expected.createdAtMs, + ); + database.close(); + + expect(loadOrCreateDeviceIdentity(options)).toEqual({ + deviceId: expected.deviceId, + publicKeyPem: expected.publicKeyPem, + privateKeyPem: expected.privateKeyPem, + }); + closeOpenClawStateDatabaseForTest(); + const verified = new sqlite.DatabaseSync(options.path!, { readOnly: true }); + expect(verified.prepare("PRAGMA user_version").get()).toEqual({ + user_version: OPENCLAW_STATE_SCHEMA_VERSION, + }); + expect( + verified + .prepare("SELECT role, schema_version FROM schema_meta WHERE meta_key = 'primary'") + .get(), + ).toEqual({ role: "global", schema_version: OPENCLAW_STATE_SCHEMA_VERSION }); + verified.close(); + }); + }); + + it("keeps process identities cached by database path and identity key", async () => { + await withTempDir("openclaw-device-identity-cache-", async (rootDir) => { + const primaryOptions = storeOptions(rootDir); + const secondaryOptions = storeOptions(rootDir, "secondary"); + const primary = loadOrCreateProcessDeviceIdentity(primaryOptions); + const secondary = loadOrCreateProcessDeviceIdentity(secondaryOptions); + + expect(loadOrCreateProcessDeviceIdentity(primaryOptions)).toBe(primary); + expect(loadOrCreateProcessDeviceIdentity(secondaryOptions)).toBe(secondary); + expect(secondary.deviceId).not.toBe(primary.deviceId); + + const claimPath = path.join(rootDir, "identity", "device.json.doctor-importing"); + fs.mkdirSync(path.dirname(claimPath), { recursive: true }); + fs.writeFileSync(claimPath, "{}\n"); + expect(() => loadOrCreateProcessDeviceIdentity(primaryOptions)).toThrow(/doctor --fix/); + }); + }); + + it("returns one authoritative winner to concurrent creators", async () => { + await withTempDir("openclaw-device-identity-concurrent-", async (rootDir) => { + const [first, second] = await runConcurrentIdentityLoads(rootDir); + + expect(second).toEqual(first); + expect(loadDeviceIdentityIfPresent(storeOptions(rootDir))).toEqual(first); + }); + }, 30_000); + + it("fails closed for a corrupt persisted row", async () => { + await withTempDir("openclaw-device-identity-corrupt-", async (rootDir) => { + const options = storeOptions(rootDir); + loadOrCreateDeviceIdentity(options); + closeOpenClawStateDatabaseForTest(); + + const sqlite = await import("node:sqlite"); + const database = new sqlite.DatabaseSync(options.path!); + database + .prepare("UPDATE device_identities SET device_id = ? WHERE identity_key = ?") + .run("corrupt-device-id", "primary"); + database.close(); + + expect(() => loadDeviceIdentityIfPresent(options)).toThrow( + /invalid persisted device identity/, + ); + expect(() => loadOrCreateDeviceIdentity(options)).toThrow( + /invalid persisted device identity/, + ); + }); + }); + + it.each(["device.json", "device.json.doctor-importing", "device.json.native-importing"])( + "blocks SQLite access while legacy %s may exist", + async (legacyName) => { + await withTempDir("openclaw-device-identity-legacy-", async (rootDir) => { + const options = storeOptions(rootDir); + const legacyPath = path.join(rootDir, "identity", legacyName); + fs.mkdirSync(path.dirname(legacyPath), { recursive: true }); + fs.writeFileSync(legacyPath, "{}\n"); + + expect(() => loadDeviceIdentityIfPresent(options)).toThrow(/doctor --fix/); + expect(() => loadOrCreateDeviceIdentity(options)).toThrow(/doctor --fix/); + expect(fs.existsSync(options.path!)).toBe(false); + }); + }, + ); + + it.each([ + ["canonical", (rootDir: string) => path.join(rootDir, "state", "openclaw.sqlite")], + ["arbitrary", (rootDir: string) => path.join(rootDir, "identity-state.sqlite")], + ])("derives the legacy root from an explicit %s database path", async (_label, dbPath) => { + await withTempDir("openclaw-device-identity-explicit-path-", async (rootDir) => { + const legacyPath = path.join(rootDir, "identity", "device.json"); + fs.mkdirSync(path.dirname(legacyPath), { recursive: true }); + fs.writeFileSync(legacyPath, "{}\n"); + + expect(() => loadOrCreateDeviceIdentity({ path: dbPath(rootDir) })).toThrow(/doctor --fix/); }); }); }); + +describe("legacy device identity normalization", () => { + it("normalizes valid Node PEM material and derives its canonical device id", () => { + const { publicKey, privateKey } = crypto.generateKeyPairSync("ed25519"); + const publicKeyPem = publicKey.export({ type: "spki", format: "pem" }); + const privateKeyPem = privateKey.export({ type: "pkcs8", format: "pem" }); + const normalized = normalizeLegacyDeviceIdentity({ + version: 1, + deviceId: "stale-device-id", + publicKeyPem, + privateKeyPem, + createdAtMs: 1_700_000_000_000, + }); + + expect(normalized).toMatchObject({ + deviceId: deriveDeviceIdFromPublicKey(publicKeyPem), + publicKeyPem, + privateKeyPem, + createdAtMs: 1_700_000_000_000, + }); + }); + + it("converts valid Swift raw-key material to PEM", () => { + const normalized = normalizeLegacyDeviceIdentity({ + deviceId: SWIFT_RAW_DEVICE_ID, + publicKey: SWIFT_RAW_PUBLIC_KEY, + privateKey: SWIFT_RAW_PRIVATE_KEY, + createdAtMs: 1_700_000_000_000, + }); + + expect(normalized?.deviceId).toBe(SWIFT_RAW_DEVICE_ID); + expect(normalized?.createdAtMs).toBe(1_700_000_000_000); + expect(crypto.createPublicKey(normalized?.publicKeyPem ?? "").asymmetricKeyType).toBe( + "ed25519", + ); + expect(crypto.createPrivateKey(normalized?.privateKeyPem ?? "").asymmetricKeyType).toBe( + "ed25519", + ); + }); + + it("rejects mismatched or malformed legacy key material", () => { + expect( + normalizeLegacyDeviceIdentity({ + deviceId: SWIFT_RAW_DEVICE_ID, + publicKey: SWIFT_RAW_PUBLIC_KEY, + privateKey: MISMATCHED_SWIFT_RAW_PRIVATE_KEY, + createdAtMs: 1_700_000_000_000, + }), + ).toBeNull(); + expect( + normalizeLegacyDeviceIdentity({ + version: 1, + deviceId: SWIFT_RAW_DEVICE_ID, + publicKeyPem: "not-a-key", + privateKeyPem: "not-a-key", + createdAtMs: Number.NaN, + }), + ).toBeNull(); + }); +}); + +describe("device identity crypto helpers", () => { + it("preserves existing public-key wire normalization", () => { + const { publicKey } = crypto.generateKeyPairSync("ed25519"); + const publicKeyPem = publicKey.export({ type: "spki", format: "pem" }); + const publicKeyRaw = publicKeyRawBase64UrlFromPem(publicKeyPem); + const standardBase64 = `${publicKeyRaw.replaceAll("-", "+").replaceAll("_", "/")}=`; + + expect(normalizeDevicePublicKeyBase64Url(publicKeyPem)).toBe(publicKeyRaw); + expect(normalizeDevicePublicKeyBase64Url(standardBase64)).toBe(publicKeyRaw); + expect(normalizeDevicePublicKeyBase64Url(`${standardBase64}=`)).toBe(publicKeyRaw); + expect(deriveDeviceIdFromPublicKey(publicKeyRaw)).toBe( + deriveDeviceIdFromPublicKey(publicKeyPem), + ); + }); + + it("signs payloads that verify against PEM and raw public key forms", () => { + const { publicKey, privateKey } = crypto.generateKeyPairSync("ed25519"); + const publicKeyPem = publicKey.export({ type: "spki", format: "pem" }); + const privateKeyPem = privateKey.export({ type: "pkcs8", format: "pem" }); + const payload = JSON.stringify({ action: "system.run", ts: 1234 }); + const signature = signDevicePayload(privateKeyPem, payload); + const publicKeyRaw = publicKeyRawBase64UrlFromPem(publicKeyPem); + + expect(verifyDeviceSignature(publicKeyPem, payload, signature)).toBe(true); + expect(verifyDeviceSignature(publicKeyRaw, payload, signature)).toBe(true); + expect(verifyDeviceSignature(publicKeyRaw, `${payload}!`, signature)).toBe(false); + }); +}); diff --git a/src/infra/device-identity.ts b/src/infra/device-identity.ts index d2011ee7dfeb..839f06ba379d 100644 --- a/src/infra/device-identity.ts +++ b/src/infra/device-identity.ts @@ -1,277 +1,182 @@ -// Stores and converts the gateway/device Ed25519 identity. +// Gateway/device Ed25519 identity API backed by canonical shared SQLite state. import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { resolveStateDir } from "../config/paths.js"; +import { acquireDeviceIdentityCoordinator } from "./device-identity-coordinator.js"; +import { + generateStoredDeviceIdentity, + insertStoredDeviceIdentityIfAbsent, + PRIMARY_DEVICE_IDENTITY_KEY, + readStoredDeviceIdentity, + readStoredDeviceIdentityReadOnly, + resolveDeviceIdentityStore, + type DeviceIdentity, + type DeviceIdentityStoreOptions, + type StoredDeviceIdentity, +} from "./device-identity-store.js"; import { - base64UrlDecode, - deriveEd25519PublicKeyRaw, - ed25519PrivateKeyPemFromRaw, - ed25519PublicKeyPemFromRaw, normalizeEd25519PublicKeyBase64Url, publicKeyRawBase64UrlFromEd25519Pem, signEd25519Payload, verifyEd25519Signature, } from "./ed25519-signature.js"; -import { privateFileStoreSync } from "./private-file-store.js"; -/** Gateway/device Ed25519 identity used for APNs relay and gateway authentication. */ -export type DeviceIdentity = { - deviceId: string; - publicKeyPem: string; - privateKeyPem: string; -}; +export type { DeviceIdentity } from "./device-identity-store.js"; -type StoredIdentity = { - version: 1; - deviceId: string; - publicKeyPem: string; - privateKeyPem: string; - createdAtMs: number; -}; +const LEGACY_DEVICE_IDENTITY_RELATIVE_PATH = path.join("identity", "device.json"); +const DOCTOR_CLAIM_SUFFIX = ".doctor-importing"; +const NATIVE_CLAIM_SUFFIX = ".native-importing"; -type StoredSwiftIdentity = { - deviceId: string; - publicKey: string; - privateKey: string; - createdAtMs: number; -}; - -function resolveDefaultIdentityPath(): string { - return path.join(resolveStateDir(), "identity", "device.json"); -} - -// Swift stores raw Ed25519 key bytes; Node crypto needs DER/PEM wrappers around them. -const publicKeyPemFromRaw = ed25519PublicKeyPemFromRaw; -const privateKeyPemFromRaw = ed25519PrivateKeyPemFromRaw; - -function fingerprintPublicKey(publicKeyPem: string): string { - const raw = deriveEd25519PublicKeyRaw(publicKeyPem); - return crypto.createHash("sha256").update(raw).digest("hex"); -} - -function tryFingerprintPublicKey(publicKeyPem: string): string | null { - try { - return fingerprintPublicKey(publicKeyPem); - } catch { - return null; +class DeviceIdentityMigrationRequiredError extends Error { + constructor(filePath: string) { + super( + `Legacy device identity exists at ${filePath}. Run "openclaw doctor --fix" before starting the gateway or connecting this client.`, + ); + this.name = "DeviceIdentityMigrationRequiredError"; } } -function keyPairMatches(publicKeyPem: string, privateKeyPem: string): boolean { +function toDeviceIdentity(stored: StoredDeviceIdentity): DeviceIdentity { + return { + deviceId: stored.deviceId, + publicKeyPem: stored.publicKeyPem, + privateKeyPem: stored.privateKeyPem, + }; +} + +function pathMayExist(filePath: string): boolean { try { - const payload = Buffer.from("openclaw-device-identity-self-check", "utf8"); - const signature = crypto.sign(null, payload, crypto.createPrivateKey(privateKeyPem)); - return crypto.verify(null, payload, crypto.createPublicKey(publicKeyPem), signature); - } catch { - return false; + fs.lstatSync(filePath); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== "ENOENT"; } } -function generateIdentity(): DeviceIdentity { - const { publicKey, privateKey } = crypto.generateKeyPairSync("ed25519"); - const publicKeyPem = publicKey.export({ type: "spki", format: "pem" }); - const privateKeyPem = privateKey.export({ type: "pkcs8", format: "pem" }); - const deviceId = fingerprintPublicKey(publicKeyPem); - return { deviceId, publicKeyPem, privateKeyPem }; +function resolveLegacyStateDir(options: DeviceIdentityStoreOptions): string { + if (options.env?.OPENCLAW_STATE_DIR?.trim()) { + return resolveStateDir(options.env); + } + if (options.path) { + const databaseDir = path.dirname(path.resolve(options.path)); + return path.basename(databaseDir) === "state" ? path.dirname(databaseDir) : databaseDir; + } + return resolveStateDir(options.env ?? process.env); } -type NormalizedStoredIdentity = - | { - kind: "identity"; - identity: DeviceIdentity; - stored?: StoredIdentity; - validForReadOnly: boolean; +/** Exact retired file owned by Doctor migration code. */ +function resolveLegacyDeviceIdentityPath(options: DeviceIdentityStoreOptions = {}): string { + return path.join(resolveLegacyStateDir(options), LEGACY_DEVICE_IDENTITY_RELATIVE_PATH); +} + +function assertNoPendingLegacyIdentity(options: DeviceIdentityStoreOptions): void { + const { identityKey } = resolveDeviceIdentityStore(options); + if (identityKey !== PRIMARY_DEVICE_IDENTITY_KEY) { + return; + } + const legacyPath = resolveLegacyDeviceIdentityPath(options); + if ( + // Claims first, source last: both migration owners restore claim -> source atomically. + pathMayExist(`${legacyPath}${DOCTOR_CLAIM_SUFFIX}`) || + pathMayExist(`${legacyPath}${NATIVE_CLAIM_SUFFIX}`) || + pathMayExist(legacyPath) + ) { + throw new DeviceIdentityMigrationRequiredError(legacyPath); + } +} + +function withDeviceIdentityCoordinator( + options: DeviceIdentityStoreOptions, + operation: ( + resolved: ReturnType, + resolvedOptions: DeviceIdentityStoreOptions, + ) => T, +): T { + const resolved = resolveDeviceIdentityStore(options); + const resolvedOptions: DeviceIdentityStoreOptions = { + ...options, + path: resolved.databasePath, + identityKey: resolved.identityKey, + }; + const coordinator = acquireDeviceIdentityCoordinator({ databasePath: resolved.databasePath }); + let result: T; + try { + result = operation(resolved, resolvedOptions); + } catch (operationError) { + try { + coordinator.release(); + } catch (releaseError) { + const aggregateError = new AggregateError( + [operationError, releaseError], + "device identity operation and coordinator release both failed", + { cause: releaseError }, + ); + throw aggregateError; } - | { kind: "recognized-invalid" }; - -function isRecord(value: unknown): value is Record { - return Boolean(value) && typeof value === "object"; + throw operationError; + } + coordinator.release(); + return result; } -function hasRecognizedIdentityShape(parsed: unknown): boolean { - return ( - isRecord(parsed) && - ("publicKeyPem" in parsed || - "privateKeyPem" in parsed || - "publicKey" in parsed || - "privateKey" in parsed) +function loadOrCreateDeviceIdentityOwned(options: DeviceIdentityStoreOptions): DeviceIdentity { + assertNoPendingLegacyIdentity(options); + const existing = readStoredDeviceIdentity(options); + if (existing) { + return toDeviceIdentity(existing); + } + + // Generate outside the write transaction. The transaction rereads the row + // before inserting so concurrent runtimes converge on one authoritative key. + const candidate = generateStoredDeviceIdentity(); + return toDeviceIdentity(insertStoredDeviceIdentityIfAbsent(candidate, options)); +} + +/** Load a valid canonical identity or atomically create its SQLite row. */ +export function loadOrCreateDeviceIdentity( + options: DeviceIdentityStoreOptions = {}, +): DeviceIdentity { + return withDeviceIdentityCoordinator(options, (_resolved, resolvedOptions) => + loadOrCreateDeviceIdentityOwned(resolvedOptions), ); } -function normalizeStoredIdentity(parsed: unknown): NormalizedStoredIdentity | null { - if ( - isRecord(parsed) && - "version" in parsed && - parsed.version === 1 && - "deviceId" in parsed && - typeof parsed.deviceId === "string" && - "publicKeyPem" in parsed && - typeof parsed.publicKeyPem === "string" && - "privateKeyPem" in parsed && - typeof parsed.privateKeyPem === "string" - ) { - const stored = parsed as StoredIdentity; - const derivedId = tryFingerprintPublicKey(stored.publicKeyPem); - if (!derivedId || !keyPairMatches(stored.publicKeyPem, stored.privateKeyPem)) { - return { kind: "recognized-invalid" }; - } - const identity = { - deviceId: derivedId, - publicKeyPem: stored.publicKeyPem, - privateKeyPem: stored.privateKeyPem, - }; - return derivedId === stored.deviceId - ? { kind: "identity", identity, validForReadOnly: true } - : { - kind: "identity", - identity, - validForReadOnly: false, - stored: { - ...stored, - deviceId: derivedId, - }, - }; - } +const processDeviceIdentities = new Map(); +const MAX_PROCESS_DEVICE_IDENTITIES = 32; - if ( - isRecord(parsed) && - !("version" in parsed) && - "deviceId" in parsed && - typeof parsed.deviceId === "string" && - "publicKey" in parsed && - typeof parsed.publicKey === "string" && - "privateKey" in parsed && - typeof parsed.privateKey === "string" - ) { - const stored = parsed as StoredSwiftIdentity; - const publicKeyRaw = base64UrlDecode(stored.publicKey); - const privateKeyRaw = base64UrlDecode(stored.privateKey); - if (publicKeyRaw.length !== 32 || privateKeyRaw.length !== 32) { - return { kind: "recognized-invalid" }; - } - const publicKeyPem = publicKeyPemFromRaw(publicKeyRaw); - const privateKeyPem = privateKeyPemFromRaw(privateKeyRaw); - if (!keyPairMatches(publicKeyPem, privateKeyPem)) { - return { kind: "recognized-invalid" }; - } - // Migrate the legacy Swift raw-key shape only after the key pair proves valid. - const derivedId = fingerprintPublicKey(publicKeyPem); - const validForReadOnly = derivedId === stored.deviceId; - const migrated: StoredIdentity = { - version: 1, - deviceId: derivedId, - publicKeyPem, - privateKeyPem, - createdAtMs: - typeof stored.createdAtMs === "number" && Number.isFinite(stored.createdAtMs) - ? stored.createdAtMs - : Date.now(), - }; - return { - kind: "identity", - identity: { - deviceId: derivedId, - publicKeyPem, - privateKeyPem, - }, - validForReadOnly, - stored: migrated, - }; - } - - return hasRecognizedIdentityShape(parsed) ? { kind: "recognized-invalid" } : null; -} - -function identityFileExists(filePath: string): boolean { - try { - return fs.statSync(filePath).isFile(); - } catch { - return false; - } -} - -/** Load a valid persisted identity, repair/migrate when safe, or create a new one. */ -export function loadOrCreateDeviceIdentity( - filePath: string = resolveDefaultIdentityPath(), -): DeviceIdentity { - try { - const store = privateFileStoreSync(path.dirname(filePath)); - const parsed = store.readJsonIfExists(path.basename(filePath)); - const normalized = normalizeStoredIdentity(parsed); - if (normalized?.kind === "identity") { - if (normalized.stored) { - try { - store.writeJson(path.basename(filePath), normalized.stored, { - trailingNewline: true, - }); - } catch { - // Keep using recognized OpenClaw key material even if best-effort normalization fails. - } - } - return normalized.identity; - } - if (normalized?.kind === "recognized-invalid") { - // Avoid overwriting recognizable but invalid identity files; callers can still use a fresh key. - return generateIdentity(); - } - if (identityFileExists(filePath)) { - // Unrecognized existing files may hold a newer schema; never overwrite them either. - return generateIdentity(); - } - } catch { - if (identityFileExists(filePath)) { - return generateIdentity(); - } - } - - const identity = generateIdentity(); - const stored: StoredIdentity = { - version: 1, - deviceId: identity.deviceId, - publicKeyPem: identity.publicKeyPem, - privateKeyPem: identity.privateKeyPem, - createdAtMs: Date.now(), - }; - privateFileStoreSync(path.dirname(filePath)).writeJson(path.basename(filePath), stored, { - trailingNewline: true, - }); - return identity; -} - -let processDeviceIdentity: { filePath: string; identity: DeviceIdentity } | undefined; - -/** - * Keep one identity stable for the lifetime of the active state-dir process. - * Recognizable invalid stores yield transient keys, so independent reloads would split gateway ownership. - */ +/** Keep one authoritative identity stable for the lifetime of a state-dir process. */ export function loadOrCreateProcessDeviceIdentity( - filePath: string = resolveDefaultIdentityPath(), + options: DeviceIdentityStoreOptions = {}, ): DeviceIdentity { - if (processDeviceIdentity?.filePath === filePath) { - return processDeviceIdentity.identity; - } - const identity = loadOrCreateDeviceIdentity(filePath); - processDeviceIdentity = { filePath, identity }; - return identity; + return withDeviceIdentityCoordinator(options, (resolved, resolvedOptions) => { + assertNoPendingLegacyIdentity(resolvedOptions); + const cacheKey = `${resolved.databasePath}\0${resolved.identityKey}`; + const cached = processDeviceIdentities.get(cacheKey); + if (cached) { + return cached; + } + const identity = loadOrCreateDeviceIdentityOwned(resolvedOptions); + if (processDeviceIdentities.size >= MAX_PROCESS_DEVICE_IDENTITIES) { + const oldestKey = processDeviceIdentities.keys().next().value; + if (oldestKey !== undefined) { + processDeviceIdentities.delete(oldestKey); + } + } + processDeviceIdentities.set(cacheKey, identity); + return identity; + }); } -/** Load a valid persisted device identity without creating, repairing, or migrating files. */ +/** Load a valid persisted identity without creating or mutating SQLite state. */ export function loadDeviceIdentityIfPresent( - filePath: string = resolveDefaultIdentityPath(), + options: DeviceIdentityStoreOptions = {}, ): DeviceIdentity | null { - try { - const parsed = privateFileStoreSync(path.dirname(filePath)).readJsonIfExists( - path.basename(filePath), - ); - const normalized = normalizeStoredIdentity(parsed); - if (normalized?.kind !== "identity" || !normalized.validForReadOnly) { - return null; - } - return normalized.identity; - } catch { - return null; - } + return withDeviceIdentityCoordinator(options, (_resolved, resolvedOptions) => { + assertNoPendingLegacyIdentity(resolvedOptions); + const stored = readStoredDeviceIdentityReadOnly(resolvedOptions); + return stored ? toDeviceIdentity(stored) : null; + }); } /** Sign a UTF-8 payload with a PEM Ed25519 private key and return base64url bytes. */ @@ -287,12 +192,11 @@ export function normalizeDevicePublicKeyBase64Url(publicKey: string): string | n /** Derive the stable device id from PEM or raw base64/base64url public key material. */ export function deriveDeviceIdFromPublicKey(publicKey: string): string | null { try { - const raw = publicKey.includes("BEGIN") - ? deriveEd25519PublicKeyRaw(publicKey) - : base64UrlDecode(publicKey); - if (raw.length === 0) { + const normalized = normalizeEd25519PublicKeyBase64Url(publicKey); + if (!normalized) { return null; } + const raw = Buffer.from(normalized, "base64url"); return crypto.createHash("sha256").update(raw).digest("hex"); } catch { return null; diff --git a/src/infra/ed25519-signature.test.ts b/src/infra/ed25519-signature.test.ts index 1cef7066359f..90f3878ab282 100644 --- a/src/infra/ed25519-signature.test.ts +++ b/src/infra/ed25519-signature.test.ts @@ -1,10 +1,87 @@ +import crypto from "node:crypto"; import { describe, expect, it } from "vitest"; -import { base64UrlDecode } from "./ed25519-signature.ts"; +import { + base64UrlDecode, + decodeCanonicalBase64OrBase64Url, + deriveCanonicalEd25519PrivateKeyRaw, + deriveCanonicalEd25519PublicKeyRaw, + deriveEd25519PrivateKeyRaw, + deriveEd25519PublicKeyRaw, + ed25519PrivateKeyPemFromRaw, + ed25519PublicKeyPemFromRaw, + normalizeEd25519PublicKeyBase64Url, +} from "./ed25519-signature.js"; + +describe("strict base64 decoding", () => { + it("accepts canonical unpadded base64url", () => { + const raw = Buffer.from([0xfb, 0xff, 0x01]); + expect(base64UrlDecode("-_8B")).toEqual(raw); + }); + + it("accepts canonical standard base64 through the strict mixed decoder", () => { + const raw = Buffer.from([0xfb, 0xff, 0x01]); + expect(decodeCanonicalBase64OrBase64Url("+/8B")).toEqual(raw); + }); + + it.each(["", "A", "AB==", "AA=", "AA===", "AA==junk", "-_8B="])( + "rejects noncanonical input %j", + (input) => { + expect(() => decodeCanonicalBase64OrBase64Url(input)).toThrow(); + }, + ); -describe("base64UrlDecode", () => { it("throws on input exceeding the maximum allowed length", () => { - // MAX_BASE64URL_DECODE_INPUT_LENGTH is 4096; 5000 is safely over it. - const oversized = "A".repeat(5000); - expect(() => base64UrlDecode(oversized)).toThrow(/maximum allowed length/); + expect(() => base64UrlDecode("A".repeat(5000))).toThrow(/maximum allowed length/); + }); +}); + +describe("strict Ed25519 keys", () => { + it("round-trips exact 32-byte raw keys", () => { + const raw = Buffer.alloc(32, 7); + const publicKeyPem = ed25519PublicKeyPemFromRaw(raw); + const privateKeyPem = ed25519PrivateKeyPemFromRaw(raw); + + expect(deriveEd25519PublicKeyRaw(publicKeyPem)).toEqual(raw); + expect(deriveEd25519PrivateKeyRaw(privateKeyPem)).toEqual(raw); + }); + + it.each([31, 33])("rejects %i-byte raw keys", (length) => { + const raw = Buffer.alloc(length); + expect(() => ed25519PublicKeyPemFromRaw(raw)).toThrow(/exactly 32 bytes/); + expect(() => ed25519PrivateKeyPemFromRaw(raw)).toThrow(/exactly 32 bytes/); + }); + + it("rejects non-Ed25519 key types", () => { + const { publicKey, privateKey } = crypto.generateKeyPairSync("rsa", { + modulusLength: 2048, + }); + const publicKeyPem = publicKey.export({ type: "spki", format: "pem" }); + const privateKeyPem = privateKey.export({ type: "pkcs8", format: "pem" }); + + expect(() => deriveEd25519PublicKeyRaw(publicKeyPem)).toThrow(/Ed25519/); + expect(() => deriveEd25519PrivateKeyRaw(privateKeyPem)).toThrow(/Ed25519/); + expect(normalizeEd25519PublicKeyBase64Url(publicKeyPem)).toBeNull(); + }); + + it("rejects alternate PEM formatting even when crypto can parse it", () => { + const { publicKey, privateKey } = crypto.generateKeyPairSync("ed25519"); + const publicKeyPem = publicKey.export({ type: "spki", format: "pem" }); + const privateKeyPem = privateKey.export({ type: "pkcs8", format: "pem" }); + const variants = [ + publicKeyPem.trimEnd(), + publicKeyPem.replaceAll("\n", "\r\n"), + publicKeyPem.replace(/\n([A-Za-z0-9+/=]{30})/, "\n$1\n"), + ]; + + for (const pem of variants) { + expect(() => crypto.createPublicKey(pem)).not.toThrow(); + expect(() => deriveCanonicalEd25519PublicKeyRaw(pem)).toThrow(/canonical PEM/); + expect(deriveEd25519PublicKeyRaw(pem)).toHaveLength(32); + expect(normalizeEd25519PublicKeyBase64Url(pem)).not.toBeNull(); + } + expect(() => deriveCanonicalEd25519PrivateKeyRaw(privateKeyPem.trimEnd())).toThrow( + /canonical PEM/, + ); + expect(deriveEd25519PrivateKeyRaw(privateKeyPem.trimEnd())).toHaveLength(32); }); }); diff --git a/src/infra/ed25519-signature.ts b/src/infra/ed25519-signature.ts index bdb3e13723ea..da59be28b3ce 100644 --- a/src/infra/ed25519-signature.ts +++ b/src/infra/ed25519-signature.ts @@ -1,10 +1,11 @@ import crypto from "node:crypto"; +const ED25519_RAW_KEY_LENGTH = 32; const ED25519_SPKI_PREFIX = Buffer.from("302a300506032b6570032100", "hex"); const ED25519_PKCS8_PRIVATE_PREFIX = Buffer.from("302e020100300506032b657004220420", "hex"); function base64UrlEncode(buf: Buffer): string { - return buf.toString("base64").replaceAll("+", "-").replaceAll("/", "_").replace(/=+$/g, ""); + return buf.toString("base64url"); } // Ed25519 public keys and signatures are fixed-size (<= ~86 base64url chars), @@ -12,6 +13,16 @@ function base64UrlEncode(buf: Buffer): string { // Bound the decoded buffer to keep a single request from allocating arbitrary memory. const MAX_BASE64URL_DECODE_INPUT_LENGTH = 4096; +function assertBoundedBase64Input(input: string): void { + if (input.length > MAX_BASE64URL_DECODE_INPUT_LENGTH) { + throw new Error("base64url input exceeds the maximum allowed length"); + } + if (input.length === 0) { + throw new Error("base64 input must not be empty"); + } +} + +/** Decode the existing permissive base64url wire shape. */ export function base64UrlDecode(input: string): Buffer { if (input.length > MAX_BASE64URL_DECODE_INPUT_LENGTH) { throw new Error("base64url input exceeds the maximum allowed length"); @@ -21,6 +32,26 @@ export function base64UrlDecode(input: string): Buffer { return Buffer.from(padded, "base64"); } +/** Decode a canonical standard-base64 or unpadded-base64url value. */ +export function decodeCanonicalBase64OrBase64Url(input: string): Buffer { + assertBoundedBase64Input(input); + if (/^[A-Za-z0-9_-]+$/.test(input)) { + const decoded = Buffer.from(input, "base64url"); + if (base64UrlEncode(decoded) !== input) { + throw new Error("invalid canonical base64url input"); + } + return decoded; + } + if (!/^[A-Za-z0-9+/]+={0,2}$/.test(input) || input.length % 4 !== 0) { + throw new Error("invalid canonical base64 input"); + } + const decoded = Buffer.from(input, "base64"); + if (decoded.toString("base64") !== input) { + throw new Error("invalid canonical base64 input"); + } + return decoded; +} + function pemEncode(label: "PUBLIC KEY" | "PRIVATE KEY", der: Buffer): string { const body = der @@ -30,24 +61,92 @@ function pemEncode(label: "PUBLIC KEY" | "PRIVATE KEY", der: Buffer): string { return `-----BEGIN ${label}-----\n${body}\n-----END ${label}-----\n`; } +function decodeCanonicalPem(label: "PUBLIC KEY" | "PRIVATE KEY", pem: string): Buffer { + const header = `-----BEGIN ${label}-----\n`; + const footer = `\n-----END ${label}-----\n`; + if (!pem.startsWith(header) || !pem.endsWith(footer)) { + throw new Error(`${label} must use canonical PEM framing`); + } + const body = pem.slice(header.length, -footer.length); + if (!body || !/^[A-Za-z0-9+/=\n]+$/.test(body)) { + throw new Error(`${label} contains invalid PEM body bytes`); + } + const der = Buffer.from(body.replaceAll("\n", ""), "base64"); + if (pemEncode(label, der) !== pem) { + throw new Error(`${label} must use canonical PEM base64 encoding`); + } + return der; +} + +function assertRawKeyLength(raw: Buffer, label: string): void { + if (raw.length !== ED25519_RAW_KEY_LENGTH) { + throw new Error(`${label} must contain exactly ${ED25519_RAW_KEY_LENGTH} bytes`); + } +} + export function ed25519PublicKeyPemFromRaw(publicKeyRaw: Buffer): string { + assertRawKeyLength(publicKeyRaw, "Ed25519 public key"); return pemEncode("PUBLIC KEY", Buffer.concat([ED25519_SPKI_PREFIX, publicKeyRaw])); } export function ed25519PrivateKeyPemFromRaw(privateKeyRaw: Buffer): string { + assertRawKeyLength(privateKeyRaw, "Ed25519 private key"); return pemEncode("PRIVATE KEY", Buffer.concat([ED25519_PKCS8_PRIVATE_PREFIX, privateKeyRaw])); } +function assertEd25519KeyType(key: crypto.KeyObject, label: string): void { + if (key.asymmetricKeyType !== "ed25519") { + throw new Error(`${label} must be an Ed25519 key`); + } +} + +function deriveRawKeyFromDer(params: { der: Buffer; label: string; prefix: Buffer }): Buffer { + const expectedLength = params.prefix.length + ED25519_RAW_KEY_LENGTH; + if ( + params.der.length !== expectedLength || + !params.der.subarray(0, params.prefix.length).equals(params.prefix) + ) { + throw new Error(`${params.label} has a noncanonical Ed25519 encoding`); + } + return params.der.subarray(params.prefix.length); +} + +export function deriveCanonicalEd25519PublicKeyRaw(publicKeyPem: string): Buffer { + const spki = decodeCanonicalPem("PUBLIC KEY", publicKeyPem); + const key = crypto.createPublicKey({ key: spki, type: "spki", format: "der" }); + assertEd25519KeyType(key, "public key"); + return deriveRawKeyFromDer({ der: spki, label: "public key", prefix: ED25519_SPKI_PREFIX }); +} + +export function deriveCanonicalEd25519PrivateKeyRaw(privateKeyPem: string): Buffer { + const pkcs8 = decodeCanonicalPem("PRIVATE KEY", privateKeyPem); + const key = crypto.createPrivateKey({ key: pkcs8, type: "pkcs8", format: "der" }); + assertEd25519KeyType(key, "private key"); + return deriveRawKeyFromDer({ + der: pkcs8, + label: "private key", + prefix: ED25519_PKCS8_PRIVATE_PREFIX, + }); +} + +/** Parse any Node-compatible Ed25519 PEM and return its canonical raw public key. */ export function deriveEd25519PublicKeyRaw(publicKeyPem: string): Buffer { const key = crypto.createPublicKey(publicKeyPem); - const spki = key.export({ type: "spki", format: "der" }) as Buffer; - if ( - spki.length === ED25519_SPKI_PREFIX.length + 32 && - spki.subarray(0, ED25519_SPKI_PREFIX.length).equals(ED25519_SPKI_PREFIX) - ) { - return spki.subarray(ED25519_SPKI_PREFIX.length); - } - return spki; + assertEd25519KeyType(key, "public key"); + const spki = key.export({ type: "spki", format: "der" }); + return deriveRawKeyFromDer({ der: spki, label: "public key", prefix: ED25519_SPKI_PREFIX }); +} + +/** Parse any Node-compatible Ed25519 PEM and return its canonical raw private key. */ +export function deriveEd25519PrivateKeyRaw(privateKeyPem: string): Buffer { + const key = crypto.createPrivateKey(privateKeyPem); + assertEd25519KeyType(key, "private key"); + const pkcs8 = key.export({ type: "pkcs8", format: "der" }); + return deriveRawKeyFromDer({ + der: pkcs8, + label: "private key", + prefix: ED25519_PKCS8_PRIVATE_PREFIX, + }); } export function publicKeyRawBase64UrlFromEd25519Pem(publicKeyPem: string): string { @@ -56,10 +155,9 @@ export function publicKeyRawBase64UrlFromEd25519Pem(publicKeyPem: string): strin export function normalizeEd25519PublicKeyBase64Url(publicKey: string): string | null { try { - if (publicKey.includes("BEGIN")) { - return publicKeyRawBase64UrlFromEd25519Pem(publicKey); - } - const raw = base64UrlDecode(publicKey); + const raw = publicKey.includes("BEGIN") + ? deriveEd25519PublicKeyRaw(publicKey) + : base64UrlDecode(publicKey); if (raw.length === 0) { return null; } @@ -71,18 +169,19 @@ export function normalizeEd25519PublicKeyBase64Url(publicKey: string): string | export function signEd25519Payload(privateKeyPem: string, payload: string): string { const key = crypto.createPrivateKey(privateKeyPem); - const sig = crypto.sign(null, Buffer.from(payload, "utf8"), key); - return base64UrlEncode(sig); + const signature = crypto.sign(null, Buffer.from(payload, "utf8"), key); + return base64UrlEncode(signature); } function createEd25519PublicKey(publicKey: string): crypto.KeyObject { - return publicKey.includes("BEGIN") - ? crypto.createPublicKey(publicKey) - : crypto.createPublicKey({ - key: Buffer.concat([ED25519_SPKI_PREFIX, base64UrlDecode(publicKey)]), - type: "spki", - format: "der", - }); + if (publicKey.includes("BEGIN")) { + return crypto.createPublicKey(publicKey); + } + return crypto.createPublicKey({ + key: Buffer.concat([ED25519_SPKI_PREFIX, base64UrlDecode(publicKey)]), + type: "spki", + format: "der", + }); } export function verifyEd25519Signature(params: { @@ -104,8 +203,8 @@ export function verifyEd25519SignatureBytes(params: { }): boolean { try { const key = createEd25519PublicKey(params.publicKey); - const sig = base64UrlDecode(params.signatureBase64Url); - return crypto.verify(null, params.payload, key, sig); + const signature = base64UrlDecode(params.signatureBase64Url); + return crypto.verify(null, params.payload, key, signature); } catch { return false; } diff --git a/src/infra/state-migrations.device-identity-repair.ts b/src/infra/state-migrations.device-identity-repair.ts new file mode 100644 index 000000000000..c6beec72f907 --- /dev/null +++ b/src/infra/state-migrations.device-identity-repair.ts @@ -0,0 +1,95 @@ +// Doctor-only detection and replacement for invalid canonical device identity rows. +import fs from "node:fs"; +import path from "node:path"; +import { + DeviceIdentityStorageError, + generateStoredDeviceIdentity, + readStoredDeviceIdentityReadOnly, + repairInvalidStoredDeviceIdentity, +} from "./device-identity-store.js"; +import { formatErrorMessage } from "./errors.js"; +import type { LegacyDeviceIdentityDetection } from "./state-migrations.device-identity.types.js"; +import type { MigrationMessages } from "./state-migrations.types.js"; + +const LEGACY_IDENTITY_RELATIVE_PATH = path.join("identity", "device.json"); +const DOCTOR_CLAIM_SUFFIX = ".doctor-importing"; +const NATIVE_CLAIM_SUFFIX = ".native-importing"; +const IDENTITY_KEY = "primary"; + +function pathMayExist(filePath: string): boolean { + try { + fs.lstatSync(filePath); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== "ENOENT"; + } +} + +/** Detect the exact retired paths and invalid canonical row only with Doctor authority. */ +export function detectLegacyDeviceIdentity(params: { + stateDir: string; + env?: NodeJS.ProcessEnv; + doctorOnlyStateMigrations?: boolean; +}): LegacyDeviceIdentityDetection { + const sourcePath = path.join(params.stateDir, LEGACY_IDENTITY_RELATIVE_PATH); + const claimPath = `${sourcePath}${DOCTOR_CLAIM_SUFFIX}`; + const nativeClaimPath = `${sourcePath}${NATIVE_CLAIM_SUFFIX}`; + const doctorAuthorized = params.doctorOnlyStateMigrations === true; + let hasInvalidCanonical = false; + if (doctorAuthorized) { + try { + readStoredDeviceIdentityReadOnly({ + env: { ...(params.env ?? process.env), OPENCLAW_STATE_DIR: params.stateDir }, + identityKey: IDENTITY_KEY, + }); + } catch (error) { + hasInvalidCanonical = error instanceof DeviceIdentityStorageError; + } + } + return { + sourcePath, + claimPath, + nativeClaimPath, + hasLegacy: + doctorAuthorized && + (pathMayExist(claimPath) || pathMayExist(nativeClaimPath) || pathMayExist(sourcePath)), + hasInvalidCanonical, + }; +} + +export function hasLegacyDeviceIdentityPath(detected: LegacyDeviceIdentityDetection): boolean { + return ( + pathMayExist(detected.claimPath) || + pathMayExist(detected.nativeClaimPath) || + pathMayExist(detected.sourcePath) + ); +} + +/** Generate a replacement only after the caller acquires Doctor's exclusive state lock. */ +export function repairInvalidCanonicalIdentity(env: NodeJS.ProcessEnv): MigrationMessages { + try { + const result = repairInvalidStoredDeviceIdentity(generateStoredDeviceIdentity(), { + env, + identityKey: IDENTITY_KEY, + }); + if (!result.repaired) { + return { changes: [], warnings: [] }; + } + if (!result.rotated) { + return { + changes: ["Repaired invalid primary device identity metadata in SQLite."], + warnings: [], + }; + } + return { + changes: ["Replaced invalid primary device identity in SQLite."], + warnings: [], + notices: ["The repaired device has a new identity and must be approved again."], + }; + } catch (error) { + return { + changes: [], + warnings: [`Failed repairing invalid SQLite device identity: ${formatErrorMessage(error)}`], + }; + } +} diff --git a/src/infra/state-migrations.device-identity.test.ts b/src/infra/state-migrations.device-identity.test.ts new file mode 100644 index 000000000000..e6c4ce1d2d48 --- /dev/null +++ b/src/infra/state-migrations.device-identity.test.ts @@ -0,0 +1,805 @@ +// Covers fail-closed Doctor import of the retired primary device identity JSON. +import { createHash, generateKeyPairSync } from "node:crypto"; +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; +import { + normalizeLegacyDeviceIdentity, + type NormalizedLegacyDeviceIdentity, +} from "./device-identity-legacy.js"; +import { deriveDeviceIdFromPublicKey } from "./device-identity.js"; +import { acquireGatewayLock } from "./gateway-lock.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; +import { + detectLegacyDeviceIdentity, + migrateLegacyDeviceIdentity, +} from "./state-migrations.device-identity.js"; + +type MigrationDatabase = Pick< + OpenClawStateKyselyDatabase, + "device_auth_tokens" | "device_identities" | "migration_sources" +>; + +const CREATED_AT_MS = 1_700_000_000_000; +const SWIFT_RAW_DEVICE_ID = "56475aa75463474c0285df5dbf2bcab73da651358839e9b77481b2eab107708c"; +const SWIFT_RAW_PUBLIC_KEY = "A6EHv/POEL4dcN0Y50vAmWfk1jCbpQ1fHdyGZBJVMbg="; +const SWIFT_RAW_PRIVATE_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8="; // pragma: allowlist secret + +describe("legacy device identity Doctor migration", () => { + const tempDirs = useAutoCleanupTempDirTracker((cleanup) => { + afterEach(() => { + closeOpenClawStateDatabaseForTest(); + cleanup(); + }); + }); + + function useStateDir(): { env: NodeJS.ProcessEnv; stateDir: string } { + const stateDir = tempDirs.make("openclaw-device-identity-migration-"); + return { + env: { ...process.env, HOME: stateDir, OPENCLAW_STATE_DIR: stateDir }, + stateDir, + }; + } + + function database(env: NodeJS.ProcessEnv) { + return openOpenClawStateDatabase({ env }).db; + } + + function swiftIdentity() { + return { + deviceId: SWIFT_RAW_DEVICE_ID, + publicKey: SWIFT_RAW_PUBLIC_KEY, + privateKey: SWIFT_RAW_PRIVATE_KEY, + createdAtMs: CREATED_AT_MS, + }; + } + + function normalizedSwift(): NormalizedLegacyDeviceIdentity { + const normalized = normalizeLegacyDeviceIdentity(swiftIdentity()); + if (!normalized) { + throw new Error("expected valid Swift identity fixture"); + } + return normalized; + } + + function nodeIdentity() { + const identity = normalizedSwift(); + return { + version: 1, + deviceId: identity.deviceId, + publicKeyPem: identity.publicKeyPem, + privateKeyPem: identity.privateKeyPem, + createdAtMs: identity.createdAtMs, + }; + } + + function anotherIdentity(): NormalizedLegacyDeviceIdentity { + const { publicKey, privateKey } = generateKeyPairSync("ed25519"); + const publicKeyPem = publicKey.export({ type: "spki", format: "pem" }); + const privateKeyPem = privateKey.export({ type: "pkcs8", format: "pem" }); + const deviceId = deriveDeviceIdFromPublicKey(publicKeyPem); + if (!deviceId) { + throw new Error("expected generated device id"); + } + return { deviceId, publicKeyPem, privateKeyPem, createdAtMs: CREATED_AT_MS + 1 }; + } + + function rewrapPem(pem: string): string { + const [header, ...rest] = pem.trim().split("\n"); + const footer = rest.pop(); + if (!header || !footer) { + throw new Error("expected PEM fixture"); + } + const lines = rest.join("").match(/.{1,20}/g) ?? []; + return `${header}\n${lines.join("\n")}\n${footer}\n`; + } + + async function writeLegacy(params: { + stateDir: string; + value?: unknown; + bytes?: Buffer; + }): Promise { + const sourcePath = path.join(params.stateDir, "identity", "device.json"); + await fsp.mkdir(path.dirname(sourcePath), { recursive: true }); + await fsp.writeFile( + sourcePath, + params.bytes ?? Buffer.from(`${JSON.stringify(params.value ?? nodeIdentity())}\n`, "utf8"), + ); + return sourcePath; + } + + function identityRow(env: NodeJS.ProcessEnv) { + const db = database(env); + return executeSqliteQueryTakeFirstSync( + db, + getNodeSqliteKysely(db) + .selectFrom("device_identities") + .selectAll() + .where("identity_key", "=", "primary"), + ); + } + + function receipt(env: NodeJS.ProcessEnv) { + const db = database(env); + return executeSqliteQueryTakeFirstSync( + db, + getNodeSqliteKysely(db) + .selectFrom("migration_sources") + .selectAll() + .where("migration_kind", "=", "legacy-device-identity-json"), + ); + } + + function seedCanonical(env: NodeJS.ProcessEnv, identity: NormalizedLegacyDeviceIdentity): void { + const db = database(env); + executeSqliteQuerySync( + db, + getNodeSqliteKysely(db) + .insertInto("device_identities") + .values({ + identity_key: "primary", + device_id: identity.deviceId, + public_key_pem: identity.publicKeyPem, + private_key_pem: identity.privateKeyPem, + created_at_ms: identity.createdAtMs, + updated_at_ms: identity.createdAtMs + 10, + }), + ); + } + + function seedInvalidCanonical(env: NodeJS.ProcessEnv): void { + const db = database(env); + executeSqliteQuerySync( + db, + getNodeSqliteKysely(db) + .insertInto("device_identities") + .values({ + identity_key: "primary", + device_id: "0".repeat(64), + public_key_pem: "invalid-public-key", + private_key_pem: "invalid-private-key", + created_at_ms: 1, + updated_at_ms: 2, + }), + ); + } + + async function migrate( + stateDir: string, + env: NodeJS.ProcessEnv, + overrides: { + beforeClaim?: (sourcePath: string) => void; + beforeCleanup?: () => void; + removeSource?: (sourcePath: string) => Promise | void; + } = {}, + ) { + return await migrateLegacyDeviceIdentity({ + detected: detectLegacyDeviceIdentity({ + stateDir, + env, + doctorOnlyStateMigrations: true, + }), + stateDir, + env, + doctorOnlyStateMigrations: true, + ...overrides, + }); + } + + it("detects exact source and claim paths only with explicit Doctor authority", async () => { + const { stateDir } = useStateDir(); + const sourcePath = await writeLegacy({ stateDir }); + const disabled = detectLegacyDeviceIdentity({ stateDir }); + expect(disabled).toEqual({ + sourcePath, + claimPath: `${sourcePath}.doctor-importing`, + nativeClaimPath: `${sourcePath}.native-importing`, + hasLegacy: false, + hasInvalidCanonical: false, + }); + + expect( + detectLegacyDeviceIdentity({ stateDir, doctorOnlyStateMigrations: true }).hasLegacy, + ).toBe(true); + await fsp.rename(sourcePath, `${sourcePath}.doctor-importing`); + expect( + detectLegacyDeviceIdentity({ stateDir, doctorOnlyStateMigrations: true }).hasLegacy, + ).toBe(true); + await fsp.rename(`${sourcePath}.doctor-importing`, `${sourcePath}.native-importing`); + expect( + detectLegacyDeviceIdentity({ stateDir, doctorOnlyStateMigrations: true }).hasLegacy, + ).toBe(true); + }); + + it("keeps normal migration read-only and imports only with Doctor authority", async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy({ stateDir }); + + const skipped = await migrateLegacyDeviceIdentity({ + detected: detectLegacyDeviceIdentity({ stateDir }), + env, + stateDir, + }); + + expect(skipped).toEqual({ changes: [], warnings: [] }); + expect(fs.existsSync(sourcePath)).toBe(true); + expect(identityRow(env)).toBeUndefined(); + closeOpenClawStateDatabaseForTest(); + + const repaired = await migrateLegacyDeviceIdentity({ + detected: detectLegacyDeviceIdentity({ stateDir, doctorOnlyStateMigrations: true }), + env, + stateDir, + doctorOnlyStateMigrations: true, + }); + + expect(repaired.changes).toContain("Migrated primary device identity to SQLite."); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(identityRow(env)?.device_id).toBe(SWIFT_RAW_DEVICE_ID); + }); + + for (const [label, value] of [ + ["Node PEM", nodeIdentity], + ["Swift raw-key", swiftIdentity], + ] as const) { + it(`imports a valid ${label} identity and preserves device auth bytes`, async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy({ stateDir, value: value() }); + const authPath = path.join(stateDir, "identity", "device-auth.json"); + const authBytes = Buffer.from([0x7b, 0x0a, 0xff, 0x00, 0x7d]); + await fsp.writeFile(authPath, authBytes); + + const result = await migrate(stateDir, env); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual(["Migrated primary device identity to SQLite."]); + expect(identityRow(env)).toMatchObject({ + identity_key: "primary", + device_id: SWIFT_RAW_DEVICE_ID, + created_at_ms: CREATED_AT_MS, + }); + expect(fs.existsSync(sourcePath)).toBe(false); + await expect(fsp.readFile(authPath)).resolves.toEqual(authBytes); + expect(receipt(env)).toMatchObject({ + removed_source: 1, + source_record_count: 1, + target_table: "device_identities", + }); + }); + } + + for (const [label, value] of [ + ["Node PEM identity with an invalid timestamp", { ...nodeIdentity(), createdAtMs: -1 }], + [ + "Swift raw-key identity without a timestamp", + (() => { + const legacy = { ...swiftIdentity() } as Record; + delete legacy.createdAtMs; + return legacy; + })(), + ], + ] as const) { + it(`imports a valid ${label}`, async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy({ stateDir, value }); + const startedAt = Date.now(); + + const result = await migrate(stateDir, env); + const finishedAt = Date.now(); + + expect(result.warnings).toEqual([]); + expect(identityRow(env)).toMatchObject({ + identity_key: "primary", + device_id: SWIFT_RAW_DEVICE_ID, + }); + expect(identityRow(env)?.created_at_ms).toBeGreaterThanOrEqual(startedAt); + expect(identityRow(env)?.created_at_ms).toBeLessThanOrEqual(finishedAt); + expect(fs.existsSync(sourcePath)).toBe(false); + }); + } + + it("repairs noncanonical PEM formatting before retiring JSON", async () => { + const { env, stateDir } = useStateDir(); + const expected = normalizedSwift(); + const preservedCreatedAtMs = expected.createdAtMs + 50; + seedCanonical(env, { + ...expected, + publicKeyPem: rewrapPem(expected.publicKeyPem), + privateKeyPem: rewrapPem(expected.privateKeyPem), + createdAtMs: preservedCreatedAtMs, + }); + const sourcePath = await writeLegacy({ stateDir, value: nodeIdentity() }); + + const result = await migrate(stateDir, env); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual(["Migrated primary device identity to SQLite."]); + expect(identityRow(env)).toMatchObject({ + device_id: expected.deviceId, + public_key_pem: expected.publicKeyPem, + private_key_pem: expected.privateKeyPem, + created_at_ms: expected.createdAtMs, + }); + expect(fs.existsSync(sourcePath)).toBe(false); + }); + + it("repairs an invalid canonical row from a validated legacy identity", async () => { + const { env, stateDir } = useStateDir(); + const expected = normalizedSwift(); + seedInvalidCanonical(env); + const sourcePath = await writeLegacy({ stateDir, value: nodeIdentity() }); + + const result = await migrate(stateDir, env); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual(["Migrated primary device identity to SQLite."]); + expect(identityRow(env)).toMatchObject({ + device_id: expected.deviceId, + public_key_pem: expected.publicKeyPem, + private_key_pem: expected.privateKeyPem, + created_at_ms: expected.createdAtMs, + }); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(JSON.parse(receipt(env)?.report_json ?? "null")).toMatchObject({ + repairedSqliteRecordCount: 1, + }); + }); + + it("replaces an invalid canonical row without legacy JSON only under Doctor authority", async () => { + const { env, stateDir } = useStateDir(); + seedInvalidCanonical(env); + + expect(detectLegacyDeviceIdentity({ stateDir, env }).hasInvalidCanonical).toBe(false); + const detected = detectLegacyDeviceIdentity({ + stateDir, + env, + doctorOnlyStateMigrations: true, + }); + expect(detected).toMatchObject({ hasLegacy: false, hasInvalidCanonical: true }); + + const skipped = await migrateLegacyDeviceIdentity({ detected, env, stateDir }); + expect(skipped).toEqual({ changes: [], warnings: [] }); + expect(identityRow(env)?.device_id).toBe("0".repeat(64)); + + const result = await migrateLegacyDeviceIdentity({ + detected, + env, + stateDir, + doctorOnlyStateMigrations: true, + }); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual(["Replaced invalid primary device identity in SQLite."]); + expect(result.notices).toEqual([ + "The repaired device has a new identity and must be approved again.", + ]); + expect(identityRow(env)).toMatchObject({ + identity_key: "primary", + device_id: expect.stringMatching(/^[a-f0-9]{64}$/), + }); + expect( + detectLegacyDeviceIdentity({ + stateDir, + env, + doctorOnlyStateMigrations: true, + }).hasInvalidCanonical, + ).toBe(false); + }); + + it("repairs canonical identity metadata without rotating valid key material", async () => { + const { env, stateDir } = useStateDir(); + const expected = normalizedSwift(); + seedCanonical(env, expected); + const db = database(env); + executeSqliteQuerySync( + db, + getNodeSqliteKysely(db) + .updateTable("device_identities") + .set({ + device_id: "0".repeat(64), + public_key_pem: rewrapPem(expected.publicKeyPem), + private_key_pem: rewrapPem(expected.privateKeyPem), + created_at_ms: -1, + updated_at_ms: -1, + }) + .where("identity_key", "=", "primary"), + ); + const detected = detectLegacyDeviceIdentity({ + stateDir, + env, + doctorOnlyStateMigrations: true, + }); + + const result = await migrateLegacyDeviceIdentity({ + detected, + env, + stateDir, + doctorOnlyStateMigrations: true, + }); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Repaired invalid primary device identity metadata in SQLite.", + ]); + expect(result.notices ?? []).toEqual([]); + expect(identityRow(env)).toMatchObject({ + device_id: expected.deviceId, + public_key_pem: expected.publicKeyPem, + private_key_pem: expected.privateKeyPem, + created_at_ms: expect.any(Number), + }); + }); + + it("prefers legacy key material that appears after invalid-row detection", async () => { + const { env, stateDir } = useStateDir(); + seedInvalidCanonical(env); + const detected = detectLegacyDeviceIdentity({ + stateDir, + env, + doctorOnlyStateMigrations: true, + }); + expect(detected).toMatchObject({ hasLegacy: false, hasInvalidCanonical: true }); + const sourcePath = await writeLegacy({ stateDir, value: nodeIdentity() }); + + const result = await migrateLegacyDeviceIdentity({ + detected, + env, + stateDir, + doctorOnlyStateMigrations: true, + }); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual(["Migrated primary device identity to SQLite."]); + expect(identityRow(env)?.device_id).toBe(SWIFT_RAW_DEVICE_ID); + expect(fs.existsSync(sourcePath)).toBe(false); + }); + + it("reports a generated identity when the invalid row disappears before repair", async () => { + const { env, stateDir } = useStateDir(); + seedInvalidCanonical(env); + const detected = detectLegacyDeviceIdentity({ + stateDir, + env, + doctorOnlyStateMigrations: true, + }); + const db = database(env); + executeSqliteQuerySync( + db, + getNodeSqliteKysely(db) + .deleteFrom("device_identities") + .where("identity_key", "=", "primary"), + ); + + const result = await migrateLegacyDeviceIdentity({ + detected, + env, + stateDir, + doctorOnlyStateMigrations: true, + }); + + expect(result.changes).toEqual(["Replaced invalid primary device identity in SQLite."]); + expect(result.notices).toEqual([ + "The repaired device has a new identity and must be approved again.", + ]); + expect(identityRow(env)?.device_id).toMatch(/^[a-f0-9]{64}$/); + }); + + it("requires mutation-time Doctor authority after canonical state becomes invalid", async () => { + const { env, stateDir } = useStateDir(); + seedCanonical(env, normalizedSwift()); + const sourcePath = await writeLegacy({ stateDir, value: nodeIdentity() }); + const detected = detectLegacyDeviceIdentity({ + stateDir, + env, + doctorOnlyStateMigrations: true, + }); + expect(detected).toMatchObject({ hasLegacy: true, hasInvalidCanonical: false }); + const db = database(env); + executeSqliteQuerySync( + db, + getNodeSqliteKysely(db) + .updateTable("device_identities") + .set({ device_id: "0".repeat(64) }) + .where("identity_key", "=", "primary"), + ); + + const result = await migrateLegacyDeviceIdentity({ detected, env, stateDir }); + + expect(result).toEqual({ changes: [], warnings: [] }); + expect(identityRow(env)?.device_id).toBe("0".repeat(64)); + expect(fs.existsSync(sourcePath)).toBe(true); + }); + + it("does not generate an identity from a stale legacy-only detection", async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy({ stateDir, value: nodeIdentity() }); + const detected = detectLegacyDeviceIdentity({ + stateDir, + env, + doctorOnlyStateMigrations: true, + }); + expect(detected).toMatchObject({ hasLegacy: true, hasInvalidCanonical: false }); + await fsp.unlink(sourcePath); + + const result = await migrateLegacyDeviceIdentity({ + detected, + env, + stateDir, + doctorOnlyStateMigrations: true, + }); + + expect(result).toEqual({ changes: [], warnings: [] }); + expect(identityRow(env)).toBeUndefined(); + }); + + it("repairs an invalid canonical update timestamp before retiring JSON", async () => { + const { env, stateDir } = useStateDir(); + const expected = normalizedSwift(); + seedCanonical(env, expected); + const db = database(env); + executeSqliteQuerySync( + db, + getNodeSqliteKysely(db) + .updateTable("device_identities") + .set({ updated_at_ms: -1 }) + .where("identity_key", "=", "primary"), + ); + const sourcePath = await writeLegacy({ stateDir, value: nodeIdentity() }); + + const result = await migrate(stateDir, env); + + expect(result.warnings).toEqual([]); + expect(identityRow(env)).toMatchObject({ + device_id: expected.deviceId, + public_key_pem: expected.publicKeyPem, + private_key_pem: expected.privateKeyPem, + created_at_ms: expected.createdAtMs, + }); + expect(identityRow(env)?.updated_at_ms).toBeGreaterThanOrEqual(expected.createdAtMs); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(JSON.parse(receipt(env)?.report_json ?? "null")).toMatchObject({ + repairedSqliteRecordCount: 1, + }); + }); + + it("blocks a different canonical identity and restores the source", async () => { + const { env, stateDir } = useStateDir(); + const winner = anotherIdentity(); + seedCanonical(env, winner); + const sourcePath = await writeLegacy({ stateDir }); + const before = await fsp.readFile(sourcePath); + + const result = await migrate(stateDir, env); + + expect(result.warnings.join("\n")).toContain("canonical SQLite device identity differs"); + expect(identityRow(env)?.device_id).toBe(winner.deviceId); + await expect(fsp.readFile(sourcePath)).resolves.toEqual(before); + expect(fs.existsSync(`${sourcePath}.doctor-importing`)).toBe(false); + expect(receipt(env)).toBeUndefined(); + }); + + it("restores a source changed before Doctor can claim it", async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy({ stateDir }); + + const result = await migrate(stateDir, env, { + beforeClaim: (candidate) => fs.appendFileSync(candidate, " "), + }); + + expect(result.warnings.join("\n")).toContain("changed before Doctor could claim it"); + expect(fs.existsSync(sourcePath)).toBe(true); + expect(fs.existsSync(`${sourcePath}.doctor-importing`)).toBe(false); + expect(identityRow(env)).toBeUndefined(); + expect(receipt(env)).toBeUndefined(); + }); + + it("imports an interrupted claim", async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy({ stateDir }); + const claimPath = `${sourcePath}.doctor-importing`; + await fsp.rename(sourcePath, claimPath); + + const result = await migrate(stateDir, env); + + expect(result.warnings).toEqual([]); + expect(identityRow(env)?.device_id).toBe(SWIFT_RAW_DEVICE_ID); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(fs.existsSync(claimPath)).toBe(false); + }); + + it("preserves an interrupted native claim for native startup", async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy({ stateDir }); + const nativeClaimPath = `${sourcePath}.native-importing`; + await fsp.rename(sourcePath, nativeClaimPath); + + const result = await migrate(stateDir, env); + + expect(result.warnings.join("\n")).toContain("Native device identity import is pending"); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(fs.existsSync(nativeClaimPath)).toBe(true); + expect(identityRow(env)).toBeUndefined(); + expect(receipt(env)).toBeUndefined(); + }); + + it("refuses source and interrupted claim together", async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy({ stateDir }); + await fsp.copyFile(sourcePath, `${sourcePath}.doctor-importing`); + + const result = await migrate(stateDir, env); + + expect(result.warnings.join("\n")).toContain("source and interrupted claim both exist"); + expect(identityRow(env)).toBeUndefined(); + expect(receipt(env)).toBeUndefined(); + }); + + it("rechecks the canonical row before deleting the claimed source", async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy({ stateDir }); + const replacement = anotherIdentity(); + + const result = await migrate(stateDir, env, { + beforeCleanup: () => { + const db = database(env); + executeSqliteQuerySync( + db, + getNodeSqliteKysely(db) + .updateTable("device_identities") + .set({ + device_id: replacement.deviceId, + public_key_pem: replacement.publicKeyPem, + private_key_pem: replacement.privateKeyPem, + created_at_ms: replacement.createdAtMs, + updated_at_ms: replacement.createdAtMs, + }) + .where("identity_key", "=", "primary"), + ); + }, + }); + + expect(result.warnings.join("\n")).toContain("legacy cleanup failed"); + expect(fs.existsSync(`${sourcePath}.doctor-importing`)).toBe(true); + expect(identityRow(env)?.device_id).toBe(replacement.deviceId); + expect(receipt(env)).toMatchObject({ removed_source: 0 }); + }); + + it("resumes an interrupted claim and cleanup receipt", async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy({ stateDir }); + const first = await migrate(stateDir, env, { + removeSource: () => { + throw new Error("simulated unlink failure"); + }, + }); + expect(first.warnings.join("\n")).toContain("legacy cleanup failed"); + expect(fs.existsSync(`${sourcePath}.doctor-importing`)).toBe(true); + expect(receipt(env)).toMatchObject({ removed_source: 0 }); + + closeOpenClawStateDatabaseForTest(); + const retry = await migrate(stateDir, env); + + expect(retry.warnings).toEqual([]); + expect(retry.changes).toEqual([ + "Removed retired device identity JSON covered by its SQLite receipt.", + ]); + expect(fs.existsSync(`${sourcePath}.doctor-importing`)).toBe(false); + expect(receipt(env)).toMatchObject({ removed_source: 1 }); + }); + + it("does not discard recreated bytes that differ from the receipt", async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy({ stateDir }); + await migrate(stateDir, env, { + removeSource: () => { + throw new Error("simulated unlink failure"); + }, + }); + const replacement = `${JSON.stringify({ ...nodeIdentity(), createdAtMs: CREATED_AT_MS + 1 })}\n`; + await fsp.writeFile(sourcePath, replacement, "utf8"); + + closeOpenClawStateDatabaseForTest(); + const retry = await migrate(stateDir, env); + + expect(retry.warnings.join("\n")).toContain("bytes differ from the migration receipt"); + await expect(fsp.readFile(sourcePath, "utf8")).resolves.toBe(replacement); + expect(identityRow(env)?.created_at_ms).toBe(CREATED_AT_MS); + }); + + it("rejects symlinked, hardlinked, oversized, non-UTF-8, and invalid sources", async () => { + const cases: Array<{ env: NodeJS.ProcessEnv; sourcePath: string; stateDir: string }> = []; + + const symlink = useStateDir(); + const symlinkTarget = path.join(symlink.stateDir, "outside.json"); + await fsp.writeFile(symlinkTarget, JSON.stringify(nodeIdentity()), "utf8"); + const symlinkPath = path.join(symlink.stateDir, "identity", "device.json"); + await fsp.mkdir(path.dirname(symlinkPath), { recursive: true }); + await fsp.symlink(symlinkTarget, symlinkPath); + cases.push({ ...symlink, sourcePath: symlinkPath }); + + const hardlink = useStateDir(); + const hardlinkTarget = path.join(hardlink.stateDir, "outside.json"); + await fsp.writeFile(hardlinkTarget, JSON.stringify(nodeIdentity()), "utf8"); + const hardlinkPath = path.join(hardlink.stateDir, "identity", "device.json"); + await fsp.mkdir(path.dirname(hardlinkPath), { recursive: true }); + await fsp.link(hardlinkTarget, hardlinkPath); + cases.push({ ...hardlink, sourcePath: hardlinkPath }); + + const oversized = useStateDir(); + const oversizedPath = await writeLegacy({ + stateDir: oversized.stateDir, + bytes: Buffer.alloc(128 * 1024 + 1, 0x20), + }); + cases.push({ ...oversized, sourcePath: oversizedPath }); + + const nonUtf8 = useStateDir(); + const nonUtf8Path = await writeLegacy({ + stateDir: nonUtf8.stateDir, + bytes: Buffer.from([0xff, 0xfe]), + }); + cases.push({ ...nonUtf8, sourcePath: nonUtf8Path }); + + const invalid = useStateDir(); + const invalidPath = await writeLegacy({ + stateDir: invalid.stateDir, + value: { version: 1, deviceId: "broken" }, + }); + cases.push({ ...invalid, sourcePath: invalidPath }); + + for (const testCase of cases) { + closeOpenClawStateDatabaseForTest(); + const result = await migrate(testCase.stateDir, testCase.env); + expect(result.warnings.join("\n")).toContain("Failed reading legacy device identity"); + expect(fs.existsSync(testCase.sourcePath)).toBe(true); + expect(identityRow(testCase.env)).toBeUndefined(); + expect(receipt(testCase.env)).toBeUndefined(); + } + }); + + it("requires exclusive state ownership", async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy({ stateDir }); + const gatewayLock = await acquireGatewayLock({ + allowInTests: true, + env, + pollIntervalMs: 10, + port: 18_790, + timeoutMs: 100, + }); + if (!gatewayLock) { + throw new Error("expected test Gateway lock"); + } + let result: Awaited>; + try { + result = await migrate(stateDir, env); + } finally { + await gatewayLock.release(); + } + + expect(result.warnings.join("\n")).toContain("Gateway or another SQLite maintenance command"); + expect(fs.existsSync(sourcePath)).toBe(true); + }); + + it("records the digest of the exact imported bytes", async () => { + const { env, stateDir } = useStateDir(); + const bytes = Buffer.from(`${JSON.stringify(nodeIdentity())}\n`, "utf8"); + await writeLegacy({ stateDir, bytes }); + + await migrate(stateDir, env); + + expect(receipt(env)).toMatchObject({ + source_sha256: createHash("sha256").update(bytes).digest("hex"), + }); + }); +}); diff --git a/src/infra/state-migrations.device-identity.ts b/src/infra/state-migrations.device-identity.ts new file mode 100644 index 000000000000..1f9dda3c5c1c --- /dev/null +++ b/src/infra/state-migrations.device-identity.ts @@ -0,0 +1,697 @@ +// Doctor-only import for the retired primary device identity JSON. +import { createHash } from "node:crypto"; +import path from "node:path"; +import { root, type Root } from "@openclaw/fs-safe"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, +} from "../state/openclaw-state-db.js"; +import { acquireDeviceIdentityCoordinator } from "./device-identity-coordinator.js"; +import { + normalizeLegacyDeviceIdentity, + type NormalizedLegacyDeviceIdentity, +} from "./device-identity-legacy.js"; +import { + resolveDeviceIdentityStore, + validateStoredDeviceIdentity, + type DeviceIdentity, +} from "./device-identity-store.js"; +import { deriveEd25519PrivateKeyRaw, deriveEd25519PublicKeyRaw } from "./ed25519-signature.js"; +import { formatErrorMessage } from "./errors.js"; +import { acquireGatewayLock, GatewayLockError } from "./gateway-lock.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; +import { + hasLegacyDeviceIdentityPath, + repairInvalidCanonicalIdentity, +} from "./state-migrations.device-identity-repair.js"; +import type { LegacyDeviceIdentityDetection } from "./state-migrations.device-identity.types.js"; +import type { MigrationMessages } from "./state-migrations.types.js"; + +const IDENTITY_KEY = "primary"; +const MIGRATION_KIND = "legacy-device-identity-json"; +const MIGRATION_LOCK_TIMEOUT_MS = 250; +const MIGRATION_LOCK_POLL_INTERVAL_MS = 25; +const MAX_LEGACY_IDENTITY_BYTES = 128 * 1024; +const utf8Decoder = new TextDecoder("utf-8", { fatal: true }); + +function isValidCreatedAtMs(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + +function deviceIdentityKeyMaterialMatches(left: DeviceIdentity, right: DeviceIdentity): boolean { + try { + return ( + deriveEd25519PublicKeyRaw(left.publicKeyPem).equals( + deriveEd25519PublicKeyRaw(right.publicKeyPem), + ) && + deriveEd25519PrivateKeyRaw(left.privateKeyPem).equals( + deriveEd25519PrivateKeyRaw(right.privateKeyPem), + ) + ); + } catch { + return false; + } +} + +type DeviceIdentityMigrationDatabase = Pick< + OpenClawStateKyselyDatabase, + "device_identities" | "migration_runs" | "migration_sources" +>; + +type LegacySourceSnapshot = { + sourcePath: string; + dev: number; + ino: number; + mtimeMs: number; + sha256: string; + size: number; + identity: NormalizedLegacyDeviceIdentity; +}; + +type MigrationReceipt = { + sourceKey: string; + sourceSha256: string | null; + removedSource: boolean; +}; + +export { detectLegacyDeviceIdentity } from "./state-migrations.device-identity-repair.js"; + +function relativeLegacyPath(stateDir: string, filePath: string): string { + const relativePath = path.relative(path.resolve(stateDir), path.resolve(filePath)); + if ( + !relativePath || + relativePath === ".." || + relativePath.startsWith(`..${path.sep}`) || + path.isAbsolute(relativePath) + ) { + throw new Error("legacy device identity path is outside the state directory"); + } + return relativePath; +} + +async function readLegacySourceSnapshot(params: { + stateRoot: Root; + stateDir: string; + sourcePath: string; +}): Promise { + const opened = await params.stateRoot.read( + relativeLegacyPath(params.stateDir, params.sourcePath), + { + hardlinks: "reject", + maxBytes: MAX_LEGACY_IDENTITY_BYTES, + symlinks: "reject", + }, + ); + if (opened.stat.size !== opened.buffer.byteLength) { + throw new Error("legacy device identity changed while it was being read"); + } + const identity = normalizeLegacyDeviceIdentity(JSON.parse(utf8Decoder.decode(opened.buffer))); + if (!identity) { + throw new Error("legacy device identity is invalid or unsupported"); + } + return { + sourcePath: params.sourcePath, + dev: opened.stat.dev, + ino: opened.stat.ino, + mtimeMs: opened.stat.mtimeMs, + sha256: createHash("sha256").update(opened.buffer).digest("hex"), + size: opened.stat.size, + identity, + }; +} + +function snapshotsMatch(left: LegacySourceSnapshot, right: LegacySourceSnapshot): boolean { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.mtimeMs === right.mtimeMs && + left.sha256 === right.sha256 && + left.size === right.size + ); +} + +function receiptSourceKey(sourcePath: string): string { + return `device-identity-json:${createHash("sha256").update(path.resolve(sourcePath)).digest("hex")}`; +} + +function readMigrationReceipt(sourcePath: string, env: NodeJS.ProcessEnv): MigrationReceipt | null { + const sourceKey = receiptSourceKey(sourcePath); + const { db } = openOpenClawStateDatabase({ env }); + const row = executeSqliteQueryTakeFirstSync( + db, + getNodeSqliteKysely(db) + .selectFrom("migration_sources") + .select(["removed_source", "source_sha256"]) + .where("source_key", "=", sourceKey), + ); + return row + ? { + sourceKey, + sourceSha256: row.source_sha256, + removedSource: row.removed_source === 1, + } + : null; +} + +type CanonicalIdentityRow = { + identity_key: string; + device_id: string; + public_key_pem: string; + private_key_pem: string; + created_at_ms: number; + updated_at_ms: number; +}; + +function classifyCanonicalRow( + row: CanonicalIdentityRow, + identity: NormalizedLegacyDeviceIdentity, +): "same" | "different" | "invalid" { + if (!isValidCreatedAtMs(row.updated_at_ms)) { + return "invalid"; + } + try { + validateStoredDeviceIdentity( + { + deviceId: row.device_id, + publicKeyPem: row.public_key_pem, + privateKeyPem: row.private_key_pem, + createdAtMs: row.created_at_ms, + }, + row.identity_key, + ); + } catch { + return "invalid"; + } + // Valid identities are equal by key fingerprint. PEM text and timestamps are + // serialization metadata, not a reason to rotate an already-canonical key. + return row.identity_key === IDENTITY_KEY && + row.device_id === identity.deviceId && + deviceIdentityKeyMaterialMatches( + { + deviceId: row.device_id, + publicKeyPem: row.public_key_pem, + privateKeyPem: row.private_key_pem, + }, + identity, + ) + ? "same" + : "different"; +} + +function readCanonicalIdentity( + db: ReturnType["db"], +): CanonicalIdentityRow | undefined { + return executeSqliteQueryTakeFirstSync( + db, + getNodeSqliteKysely(db) + .selectFrom("device_identities") + .selectAll() + .where("identity_key", "=", IDENTITY_KEY), + ); +} + +function verifyCanonicalIdentity( + identity: NormalizedLegacyDeviceIdentity, + env: NodeJS.ProcessEnv, +): void { + const { db } = openOpenClawStateDatabase({ env }); + const row = readCanonicalIdentity(db); + if (!row || classifyCanonicalRow(row, identity) !== "same") { + throw new Error("canonical SQLite device identity no longer matches the legacy source"); + } +} + +function importAndRecordReceipt(params: { + env: NodeJS.ProcessEnv; + sourcePath: string; + snapshot: LegacySourceSnapshot; +}): { sourceKey: string; imported: boolean } { + const sourceKey = receiptSourceKey(params.sourcePath); + const runId = `${sourceKey}:${params.snapshot.sha256.slice(0, 16)}`; + const now = Date.now(); + return runOpenClawStateWriteTransaction( + ({ db }) => { + const stateDb = getNodeSqliteKysely(db); + const existingReceipt = executeSqliteQueryTakeFirstSync( + db, + stateDb + .selectFrom("migration_sources") + .select("source_sha256") + .where("source_key", "=", sourceKey), + ); + if (existingReceipt) { + if (existingReceipt.source_sha256 !== params.snapshot.sha256) { + throw new Error("migration receipt belongs to different device identity bytes"); + } + const existing = readCanonicalIdentity(db); + if (!existing || classifyCanonicalRow(existing, params.snapshot.identity) !== "same") { + throw new Error("migration receipt does not match the canonical device identity"); + } + return { sourceKey, imported: false }; + } + + const existing = readCanonicalIdentity(db); + const existingState = existing + ? classifyCanonicalRow(existing, params.snapshot.identity) + : undefined; + if (existingState === "different") { + throw new Error("canonical SQLite device identity differs from the legacy identity"); + } + const imported = !existing || existingState === "invalid"; + const repaired = existingState === "invalid"; + if (!existing) { + executeSqliteQuerySync( + db, + stateDb.insertInto("device_identities").values({ + identity_key: IDENTITY_KEY, + device_id: params.snapshot.identity.deviceId, + public_key_pem: params.snapshot.identity.publicKeyPem, + private_key_pem: params.snapshot.identity.privateKeyPem, + created_at_ms: params.snapshot.identity.createdAtMs, + updated_at_ms: now, + }), + ); + } else if (repaired) { + executeSqliteQuerySync( + db, + stateDb + .updateTable("device_identities") + .set({ + device_id: params.snapshot.identity.deviceId, + public_key_pem: params.snapshot.identity.publicKeyPem, + private_key_pem: params.snapshot.identity.privateKeyPem, + created_at_ms: params.snapshot.identity.createdAtMs, + updated_at_ms: now, + }) + .where("identity_key", "=", IDENTITY_KEY), + ); + } + + const verified = readCanonicalIdentity(db); + if (!verified || classifyCanonicalRow(verified, params.snapshot.identity) !== "same") { + throw new Error("SQLite verification failed for the primary device identity"); + } + + const reportJson = JSON.stringify({ + source: MIGRATION_KIND, + target: "device_identities", + identityKey: IDENTITY_KEY, + deviceId: params.snapshot.identity.deviceId, + sourceSha256: params.snapshot.sha256, + importedRecordCount: imported ? 1 : 0, + preservedSqliteRecordCount: existing ? 1 : 0, + repairedSqliteRecordCount: repaired ? 1 : 0, + }); + executeSqliteQuerySync( + db, + stateDb.insertInto("migration_runs").values({ + id: runId, + started_at: now, + finished_at: now, + status: "completed", + report_json: reportJson, + }), + ); + executeSqliteQuerySync( + db, + stateDb.insertInto("migration_sources").values({ + source_key: sourceKey, + migration_kind: MIGRATION_KIND, + source_path: params.sourcePath, + target_table: "device_identities", + source_sha256: params.snapshot.sha256, + source_size_bytes: params.snapshot.size, + source_record_count: 1, + last_run_id: runId, + status: "completed", + imported_at: now, + removed_source: 0, + report_json: reportJson, + }), + ); + return { sourceKey, imported }; + }, + { env: params.env }, + ); +} + +function markSourceRemoved(sourceKey: string, env: NodeJS.ProcessEnv): void { + runOpenClawStateWriteTransaction( + ({ db }) => { + executeSqliteQuerySync( + db, + getNodeSqliteKysely(db) + .updateTable("migration_sources") + .set({ removed_source: 1 }) + .where("source_key", "=", sourceKey), + ); + }, + { env }, + ); +} + +async function removePath(params: { + stateRoot: Root; + stateDir: string; + sourcePath: string; + removeSource?: (sourcePath: string) => Promise | void; +}): Promise { + if (params.removeSource) { + await params.removeSource(params.sourcePath); + return; + } + await params.stateRoot.remove(relativeLegacyPath(params.stateDir, params.sourcePath)); +} + +async function restoreClaim(params: { + stateRoot: Root; + stateDir: string; + sourcePath: string; + claimPath: string; +}): Promise { + try { + if (!(await params.stateRoot.exists(relativeLegacyPath(params.stateDir, params.claimPath)))) { + return null; + } + if (await params.stateRoot.exists(relativeLegacyPath(params.stateDir, params.sourcePath))) { + return `source path already exists: ${params.sourcePath}`; + } + await params.stateRoot.move( + relativeLegacyPath(params.stateDir, params.claimPath), + relativeLegacyPath(params.stateDir, params.sourcePath), + ); + return null; + } catch (error) { + return String(error); + } +} + +async function cleanupReceiptSources(params: { + stateRoot: Root; + stateDir: string; + detected: LegacyDeviceIdentityDetection; + receipt: MigrationReceipt; + env: NodeJS.ProcessEnv; + removeSource?: (sourcePath: string) => Promise | void; +}): Promise { + if ( + await params.stateRoot.exists( + relativeLegacyPath(params.stateDir, params.detected.nativeClaimPath), + ) + ) { + return { + changes: [], + warnings: [ + "Native device identity import is pending; restart the native app before running Doctor cleanup.", + ], + }; + } + const changes: string[] = []; + const warnings: string[] = []; + let removed = 0; + for (const candidate of [params.detected.sourcePath, params.detected.claimPath]) { + if (!(await params.stateRoot.exists(relativeLegacyPath(params.stateDir, candidate)))) { + continue; + } + let snapshot: LegacySourceSnapshot; + try { + snapshot = await readLegacySourceSnapshot({ + stateRoot: params.stateRoot, + stateDir: params.stateDir, + sourcePath: candidate, + }); + } catch (error) { + warnings.push(`Retired device identity cleanup refused ${candidate}: ${String(error)}`); + continue; + } + if (snapshot.sha256 !== params.receipt.sourceSha256) { + warnings.push( + `Retired device identity cleanup preserved ${candidate}: bytes differ from the migration receipt.`, + ); + continue; + } + try { + verifyCanonicalIdentity(snapshot.identity, params.env); + await removePath({ ...params, sourcePath: candidate }); + removed += 1; + } catch (error) { + warnings.push(`Retired device identity cleanup failed for ${candidate}: ${String(error)}`); + } + } + if (warnings.length === 0 && (!params.receipt.removedSource || removed > 0)) { + markSourceRemoved(params.receipt.sourceKey, params.env); + } + if (removed > 0) { + changes.push("Removed retired device identity JSON covered by its SQLite receipt."); + } + return { changes, warnings }; +} + +async function migrateWithExclusiveStateOwnership(params: { + detected: LegacyDeviceIdentityDetection; + stateRoot: Root; + stateDir: string; + env: NodeJS.ProcessEnv; + beforeClaim?: (sourcePath: string) => void; + beforeCleanup?: () => void; + removeSource?: (sourcePath: string) => Promise | void; +}): Promise { + const receipt = readMigrationReceipt(params.detected.sourcePath, params.env); + if (receipt) { + return await cleanupReceiptSources({ ...params, receipt }); + } + + if ( + await params.stateRoot.exists( + relativeLegacyPath(params.stateDir, params.detected.nativeClaimPath), + ) + ) { + return { + changes: [], + warnings: [ + "Native device identity import is pending; restart the native app before running Doctor.", + ], + }; + } + + const hasSource = await params.stateRoot.exists( + relativeLegacyPath(params.stateDir, params.detected.sourcePath), + ); + const hasClaim = await params.stateRoot.exists( + relativeLegacyPath(params.stateDir, params.detected.claimPath), + ); + if (hasSource && hasClaim) { + return { + changes: [], + warnings: [ + "Failed migrating legacy device identity: source and interrupted claim both exist.", + ], + }; + } + const activePath = hasSource + ? params.detected.sourcePath + : hasClaim + ? params.detected.claimPath + : null; + if (!activePath) { + return { changes: [], warnings: [] }; + } + + let snapshot: LegacySourceSnapshot; + try { + snapshot = await readLegacySourceSnapshot({ + stateRoot: params.stateRoot, + stateDir: params.stateDir, + sourcePath: activePath, + }); + } catch (error) { + return { + changes: [], + warnings: [`Failed reading legacy device identity: ${String(error)}`], + }; + } + + if (activePath === params.detected.sourcePath) { + try { + params.beforeClaim?.(params.detected.sourcePath); + await params.stateRoot.move( + relativeLegacyPath(params.stateDir, params.detected.sourcePath), + relativeLegacyPath(params.stateDir, params.detected.claimPath), + ); + const claimed = await readLegacySourceSnapshot({ + stateRoot: params.stateRoot, + stateDir: params.stateDir, + sourcePath: params.detected.claimPath, + }); + if (!snapshotsMatch(snapshot, claimed)) { + throw new Error("legacy device identity changed before Doctor could claim it"); + } + snapshot = claimed; + } catch (error) { + const restoreError = await restoreClaim({ ...params, ...params.detected }); + return { + changes: [], + warnings: [ + `Failed migrating legacy device identity: ${String(error)}${restoreError ? `; restore failure: ${restoreError}` : ""}`, + ], + }; + } + } + + let result: ReturnType; + try { + result = importAndRecordReceipt({ + env: params.env, + sourcePath: params.detected.sourcePath, + snapshot, + }); + } catch (error) { + const restoreError = await restoreClaim({ ...params, ...params.detected }); + return { + changes: [], + warnings: [ + `Failed migrating legacy device identity: ${String(error)}${restoreError ? `; restore failure: ${restoreError}` : ""}`, + ], + }; + } + + try { + params.beforeCleanup?.(); + if ( + await params.stateRoot.exists(relativeLegacyPath(params.stateDir, params.detected.sourcePath)) + ) { + throw new Error("legacy device identity source reappeared during import"); + } + const finalSnapshot = await readLegacySourceSnapshot({ + stateRoot: params.stateRoot, + stateDir: params.stateDir, + sourcePath: params.detected.claimPath, + }); + if (!snapshotsMatch(snapshot, finalSnapshot)) { + throw new Error("legacy device identity claim changed after SQLite import"); + } + verifyCanonicalIdentity(finalSnapshot.identity, params.env); + await removePath({ ...params, sourcePath: params.detected.claimPath }); + if ( + await params.stateRoot.exists(relativeLegacyPath(params.stateDir, params.detected.claimPath)) + ) { + throw new Error("legacy device identity Doctor claim remains after cleanup"); + } + markSourceRemoved(result.sourceKey, params.env); + } catch (error) { + return { + changes: [], + warnings: [`Device identity is in SQLite, but legacy cleanup failed: ${String(error)}`], + }; + } + + return { + changes: [ + result.imported + ? "Migrated primary device identity to SQLite." + : "Preserved identical primary device identity already in SQLite.", + ], + warnings: [], + notices: ["Removed retired device identity JSON after verified SQLite import."], + }; +} + +/** Import the retired primary identity while excluding Gateways that can recreate it. */ +export async function migrateLegacyDeviceIdentity(params: { + detected: LegacyDeviceIdentityDetection; + stateDir: string; + env?: NodeJS.ProcessEnv; + doctorOnlyStateMigrations?: boolean; + beforeClaim?: (sourcePath: string) => void; + beforeCleanup?: () => void; + removeSource?: (sourcePath: string) => Promise | void; +}): Promise { + if (!params.detected.hasLegacy && !params.detected.hasInvalidCanonical) { + return { changes: [], warnings: [] }; + } + if (params.doctorOnlyStateMigrations !== true) { + return { changes: [], warnings: [] }; + } + const env = { ...(params.env ?? process.env), OPENCLAW_STATE_DIR: params.stateDir }; + let lock: Awaited>; + try { + lock = await acquireGatewayLock({ + allowInTests: true, + env, + pollIntervalMs: MIGRATION_LOCK_POLL_INTERVAL_MS, + role: "sqlite-maintenance", + timeoutMs: MIGRATION_LOCK_TIMEOUT_MS, + }); + } catch (error) { + const detail = + error instanceof GatewayLockError + ? "the Gateway or another SQLite maintenance command owns this state directory" + : String(error); + return { + changes: [], + warnings: [ + `Failed migrating legacy device identity: ${detail}. Stop the Gateway and run \`openclaw doctor --fix\` again.`, + ], + }; + } + if (!lock) { + return { + changes: [], + warnings: ["Failed migrating legacy device identity: exclusive state ownership unavailable."], + }; + } + + let result: MigrationMessages = { changes: [], warnings: [] }; + let releaseError: unknown; + let identityCoordinator: ReturnType | undefined; + try { + try { + identityCoordinator = acquireDeviceIdentityCoordinator({ + databasePath: resolveDeviceIdentityStore({ env, identityKey: IDENTITY_KEY }).databasePath, + }); + } catch (error) { + result.warnings.push( + `Failed migrating legacy device identity: identity state is busy (${formatErrorMessage(error)}).`, + ); + } + if (identityCoordinator) { + try { + const hasLegacyNow = hasLegacyDeviceIdentityPath(params.detected); + if (hasLegacyNow) { + const stateRoot = await root(params.stateDir, { + hardlinks: "reject", + maxBytes: MAX_LEGACY_IDENTITY_BYTES, + symlinks: "reject", + }); + result = await migrateWithExclusiveStateOwnership({ ...params, env, stateRoot }); + } else if (params.detected.hasInvalidCanonical) { + result = repairInvalidCanonicalIdentity(env); + } + } catch (error) { + result.warnings.push(`Failed reading legacy device identity state: ${String(error)}`); + } + } + } finally { + try { + identityCoordinator?.release(); + } catch (error) { + releaseError = error; + } + try { + await lock.release(); + } catch (error) { + releaseError ??= error; + } + } + if (releaseError) { + result.warnings.push( + `Device identity migration lock release failed: ${formatErrorMessage(releaseError)}`, + ); + } + return result; +} diff --git a/src/infra/state-migrations.device-identity.types.ts b/src/infra/state-migrations.device-identity.types.ts new file mode 100644 index 000000000000..a68350ba6248 --- /dev/null +++ b/src/infra/state-migrations.device-identity.types.ts @@ -0,0 +1,8 @@ +/** Doctor-only detection result for the retired primary device identity JSON. */ +export type LegacyDeviceIdentityDetection = { + sourcePath: string; + claimPath: string; + nativeClaimPath: string; + hasLegacy: boolean; + hasInvalidCanonical: boolean; +}; diff --git a/src/infra/state-migrations.doctor.ts b/src/infra/state-migrations.doctor.ts index fc31680fabb5..c0f0e6d1a8a7 100644 --- a/src/infra/state-migrations.doctor.ts +++ b/src/infra/state-migrations.doctor.ts @@ -51,6 +51,10 @@ import { detectLegacyDebugProxyCaptureSidecar, migrateLegacyDebugProxyCaptureSidecar, } from "./state-migrations.debug-proxy.js"; +import { + detectLegacyDeviceIdentity, + migrateLegacyDeviceIdentity, +} from "./state-migrations.device-identity.js"; import { existsDir, fileExists, @@ -442,6 +446,11 @@ export async function detectLegacyStateMigrations(params: { stateDir, doctorOnlyStateMigrations: params.doctorOnlyStateMigrations, }); + const deviceIdentity = detectLegacyDeviceIdentity({ + stateDir, + env, + doctorOnlyStateMigrations: params.doctorOnlyStateMigrations, + }); const mcpOauth = detectLegacyMcpOAuthStores({ stateDir, doctorOnlyStateMigrations: params.doctorOnlyStateMigrations, @@ -630,6 +639,12 @@ export async function detectLegacyStateMigrations(params: { if (apns.hasLegacy) { preview.push("- APNs registrations: legacy JSON → shared SQLite state"); } + if (deviceIdentity.hasLegacy) { + preview.push("- Primary device identity: legacy JSON → shared SQLite state"); + } + if (deviceIdentity.hasInvalidCanonical && !deviceIdentity.hasLegacy) { + preview.push("- Primary device identity: invalid SQLite row → new device identity"); + } if (mcpOauth.hasLegacy) { preview.push("- MCP OAuth credentials: legacy JSON → shared SQLite state"); } @@ -740,6 +755,7 @@ export async function detectLegacyStateMigrations(params: { acpReplayLedger, managedOutgoingImages, apns, + deviceIdentity, mcpOauth, restartSentinel, workspace, @@ -934,6 +950,7 @@ export async function runLegacyStateMigrations(params: { env?: NodeJS.ProcessEnv; now?: () => number; recoverCorruptTargetStore?: boolean; + doctorOnlyStateMigrations?: boolean; }): Promise { const now = params.now ?? (() => Date.now()); const detected = params.detected; @@ -1003,6 +1020,12 @@ export async function runLegacyStateMigrations(params: { env, stateDir: detected.stateDir, }); + const deviceIdentity = await migrateLegacyDeviceIdentity({ + detected: detected.deviceIdentity, + env, + stateDir: detected.stateDir, + doctorOnlyStateMigrations: params.doctorOnlyStateMigrations, + }); const mcpOauth = await migrateLegacyMcpOAuthStores({ detected: detected.mcpOauth, env, @@ -1072,6 +1095,7 @@ export async function runLegacyStateMigrations(params: { acpReplayLedger, managedOutgoingImages, apns, + deviceIdentity, mcpOauth, restartSentinel, workspace, @@ -1099,6 +1123,7 @@ export async function runLegacyStateMigrations(params: { ...acpReplayLedger.changes, ...managedOutgoingImages.changes, ...apns.changes, + ...deviceIdentity.changes, ...mcpOauth.changes, ...restartSentinel.changes, ...workspace.changes, @@ -1133,6 +1158,7 @@ export async function runLegacyStateMigrations(params: { ...acpReplayLedger.warnings, ...managedOutgoingImages.warnings, ...apns.warnings, + ...deviceIdentity.warnings, ...mcpOauth.warnings, ...restartSentinel.warnings, ...workspace.warnings, @@ -1171,6 +1197,7 @@ export async function autoMigrateLegacyState(params: { log?: MigrationLogger; now?: () => number; recoverCorruptTargetStore?: boolean; + doctorOnlyStateMigrations?: boolean; }): Promise<{ migrated: boolean; skipped: boolean; @@ -1258,6 +1285,13 @@ export async function autoMigrateLegacyState(params: { sessionStoreOwnership, env, homedir: params.homedir, + doctorOnlyStateMigrations: params.doctorOnlyStateMigrations, + }); + const deviceIdentity = await migrateLegacyDeviceIdentity({ + detected: detected.deviceIdentity, + env, + stateDir: detected.stateDir, + doctorOnlyStateMigrations: params.doctorOnlyStateMigrations, }); const hasCustomAgentDir = env.OPENCLAW_AGENT_DIR?.trim() || env.PI_CODING_AGENT_DIR?.trim(); if (hasCustomAgentDir) { @@ -1329,6 +1363,7 @@ export async function autoMigrateLegacyState(params: { ...configHealth.changes, ...pluginBindingApprovals.changes, ...currentConversationBindings.changes, + ...deviceIdentity.changes, ...restartSentinel.changes, ...channelPairing.changes, ...preSessionChannelPlans.changes, @@ -1350,6 +1385,7 @@ export async function autoMigrateLegacyState(params: { ...configHealth.warnings, ...pluginBindingApprovals.warnings, ...currentConversationBindings.warnings, + ...deviceIdentity.warnings, ...restartSentinel.warnings, ...channelPairing.warnings, ...preSessionChannelPlans.warnings, @@ -1360,6 +1396,7 @@ export async function autoMigrateLegacyState(params: { detected, pluginInstallIndex, updateCheck, + deviceIdentity, restartSentinel, pluginPlans, ]; @@ -1381,6 +1418,7 @@ export async function autoMigrateLegacyState(params: { configHealth.changes.length > 0 || pluginBindingApprovals.changes.length > 0 || currentConversationBindings.changes.length > 0 || + deviceIdentity.changes.length > 0 || restartSentinel.changes.length > 0 || channelPairing.changes.length > 0 || preSessionChannelPlans.changes.length > 0 || @@ -1416,6 +1454,7 @@ export async function autoMigrateLegacyState(params: { ...stateSchema.changes, ...orphanKeys.changes, ...acpSessionMetadata.changes, + ...deviceIdentity.changes, ]; const warnings = [ ...stateDirResult.warnings, @@ -1423,15 +1462,21 @@ export async function autoMigrateLegacyState(params: { ...detected.warnings, ...orphanKeys.warnings, ...acpSessionMetadata.warnings, + ...deviceIdentity.warnings, + ]; + const notices = [ + ...(stateDirResult.notices ?? []), + ...detected.notices, + ...(deviceIdentity.notices ?? []), ]; - const notices = [...(stateDirResult.notices ?? []), ...detected.notices]; logMigrationResults(changes, warnings, notices); return { migrated: stateDirResult.migrated || stateSchema.changes.length > 0 || orphanKeys.changes.length > 0 || - acpSessionMetadata.changes.length > 0, + acpSessionMetadata.changes.length > 0 || + deviceIdentity.changes.length > 0, skipped: false, changes, warnings, @@ -1521,6 +1566,7 @@ export async function autoMigrateLegacyState(params: { ...configHealth.changes, ...pluginBindingApprovals.changes, ...currentConversationBindings.changes, + ...deviceIdentity.changes, ...restartSentinel.changes, ...channelPairing.changes, ...preSessionChannelPlans.changes, @@ -1546,6 +1592,7 @@ export async function autoMigrateLegacyState(params: { ...configHealth.warnings, ...pluginBindingApprovals.warnings, ...currentConversationBindings.warnings, + ...deviceIdentity.warnings, ...restartSentinel.warnings, ...channelPairing.warnings, ...preSessionChannelPlans.warnings, @@ -1560,6 +1607,7 @@ export async function autoMigrateLegacyState(params: { detected, pluginInstallIndex, updateCheck, + deviceIdentity, restartSentinel, pluginPlans, ]; diff --git a/src/infra/state-migrations.types.ts b/src/infra/state-migrations.types.ts index 5b3e3dcca3f5..ff3703294ac3 100644 --- a/src/infra/state-migrations.types.ts +++ b/src/infra/state-migrations.types.ts @@ -3,6 +3,7 @@ import type { SessionScope } from "../config/sessions/types.js"; import type { PluginDoctorStateMigration } from "../plugins/doctor-contract-registry.js"; import type { LegacyAuditLogsDetection } from "./state-migrations.audit-logs.types.js"; import type { LegacyChannelPairingStateDetection } from "./state-migrations.channel-pairing.js"; +import type { LegacyDeviceIdentityDetection } from "./state-migrations.device-identity.types.js"; import type { LegacyMcpOAuthDetection } from "./state-migrations.mcp-oauth.types.js"; import type { LegacyRestartSentinelDetection } from "./state-migrations.restart-sentinel.types.js"; import type { LegacyWorkspaceStateDetection } from "./state-migrations.workspace-setup.types.js"; @@ -118,6 +119,7 @@ export type LegacyStateDetection = { sourcePath: string; hasLegacy: boolean; }; + deviceIdentity: LegacyDeviceIdentityDetection; mcpOauth: LegacyMcpOAuthDetection; restartSentinel?: LegacyRestartSentinelDetection; workspace: LegacyWorkspaceStateDetection; diff --git a/src/state/openclaw-state-db.test.ts b/src/state/openclaw-state-db.test.ts index 9c7ce90d9ab6..b164dd724539 100644 --- a/src/state/openclaw-state-db.test.ts +++ b/src/state/openclaw-state-db.test.ts @@ -847,6 +847,49 @@ describe("openclaw state database", () => { ).toBeUndefined(); }); + it("adopts a canonical device identity seed database without losing the identity", () => { + const stateDir = createTempStateDir(); + const databasePath = path.join(stateDir, "state", "openclaw.sqlite"); + fs.mkdirSync(path.dirname(databasePath), { recursive: true }); + const { DatabaseSync } = requireNodeSqlite(); + const seed = new DatabaseSync(databasePath); + seed.exec(` +CREATE TABLE device_identities ( + identity_key TEXT NOT NULL PRIMARY KEY, + device_id TEXT NOT NULL, + public_key_pem TEXT NOT NULL, + private_key_pem TEXT NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL +) STRICT; +CREATE INDEX idx_device_identities_device + ON device_identities(device_id, updated_at_ms DESC); +INSERT INTO device_identities VALUES ( + 'primary', 'device-1', 'public-key', 'private-key', 10, 20 +); +`); + seed.close(); + + const database = openOpenClawStateDatabase({ + env: { OPENCLAW_STATE_DIR: stateDir }, + }); + + expect( + database.db.prepare("SELECT * FROM device_identities WHERE identity_key = 'primary'").get(), + ).toEqual({ + identity_key: "primary", + device_id: "device-1", + public_key_pem: "public-key", + private_key_pem: "private-key", + created_at_ms: 10, + updated_at_ms: 20, + }); + expect(readSqliteNumberPragma(database.db, "user_version")).toBe(OPENCLAW_STATE_SCHEMA_VERSION); + expect(collectSqliteSchemaShape(database.db)).toEqual( + createSqliteSchemaShapeFromSql(new URL("./openclaw-state-schema.sql", import.meta.url)), + ); + }); + it("doctor migrates existing APNs tombstone tables to STRICT without losing rows", () => { const stateDir = createTempStateDir(); const options = { env: { OPENCLAW_STATE_DIR: stateDir } }; diff --git a/test/helpers/gateway-e2e-harness.ts b/test/helpers/gateway-e2e-harness.ts index 04a0db7ea9a5..1cd023fe48cc 100644 --- a/test/helpers/gateway-e2e-harness.ts +++ b/test/helpers/gateway-e2e-harness.ts @@ -121,8 +121,8 @@ export async function connectNode( inst: GatewayInstance, label: string, ): Promise<{ client: GatewayClient; nodeId: string }> { - const identityPath = path.join(inst.homeDir, `${label}-device.json`); - const deviceIdentity = loadOrCreateDeviceIdentity(identityPath); + const identityPath = path.join(inst.homeDir, `${label}-device.sqlite`); + const deviceIdentity = loadOrCreateDeviceIdentity({ path: identityPath }); const nodeId = deviceIdentity.deviceId; const client = await connectGatewayClient({ url: `ws://127.0.0.1:${inst.port}`, diff --git a/test/scripts/check-database-first-legacy-stores.test.ts b/test/scripts/check-database-first-legacy-stores.test.ts index 546c9c3db299..90d11ecf2daf 100644 --- a/test/scripts/check-database-first-legacy-stores.test.ts +++ b/test/scripts/check-database-first-legacy-stores.test.ts @@ -8707,6 +8707,33 @@ describe("check-database-first-legacy-stores", () => { expect(violations).toEqual([]); }); + it("blocks runtime writes to the retired device identity file", () => { + const violations = collectDatabaseFirstLegacyStoreViolations( + ` + import fs from "node:fs"; + fs.writeFileSync(path.join(stateDir, "identity/device.json"), "{}\\n"); + `, + "src/infra/device-identity.ts", + ); + + expect(violations).toEqual([{ kind: "legacy store filesystem write", line: 3 }]); + }); + + it("allows only the device identity migration owner to retire its legacy source", () => { + const violations = collectDatabaseFirstLegacyStoreViolations( + ` + import fs from "node:fs"; + fs.renameSync( + path.join(stateDir, "identity/device.json"), + path.join(stateDir, "identity/device.json.doctor-importing"), + ); + `, + "src/infra/state-migrations.device-identity.ts", + ); + + expect(violations).toEqual([]); + }); + it("allows the workspace Doctor migration owner to claim legacy sidecars", () => { const violations = collectDatabaseFirstLegacyStoreViolations( `