mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-25 03:45:46 -06:00
8624b9acb8
* feat(gateway): recover channels and health promptly after host sleep A dependency-free thaw detector rides the existing 30s maintenance tick: when the process resumes after being frozen >=45s beyond cadence (laptop sleep, VM pause, SIGSTOP), the gateway restarts running channel accounts (dead sockets otherwise take up to ~35 minutes to notice), refreshes health/presence, and resets the event-loop histogram so the freeze does not read as degradation. Admission is rechecked before every recovery side effect; a suspension beginning mid-recovery re-pends the thaw, and timed-out channel stops complete their two-call restart in one pass. The macOS app cooperates: NSWorkspace sleep/wake observers in GatewayConnectivityCoordinator best-effort prepare a local gateway suspension before sleep and resume it on wake, never blocking sleep. The lease is bound to the route that prepared it and always cleared on wake; route or mode changes across sleep drop it to self-expiry. Live proof: SIGSTOP 85s on an isolated dev gateway -> 'host thaw detected: process was frozen ~57683ms', channels restarted, health ok, eventLoop degraded=false after thaw. * fix(macos): resume a sleep lease whose prepare response arrives after wake A prepare completing after didWake previously discarded the lease id, fencing the gateway until the two-minute expiry after micro-sleeps; the late response now resumes immediately. Document the conservative route-token drift tradeoff. * fix(macos): retry wake resume after refreshing the dead post-sleep transport After real sleep the WebSocket is usually dead exactly when resume runs; refresh the endpoint first, then attempt resume up to three times with bounded delays, clearing the lease only on success or exhaustion. A new sleep cycle aborts in-flight retries. * fix(gateway): bound plugin stopAccount so channel stops cannot wedge recovery stopChannel awaited plugin stopAccount unbounded; a never-settling stop hung the thaw restart (and health-monitor sweeps) and held the single-flight recovery guard forever. Race it against the existing 5s stop timeout; the timed-out path flows into the established recoveryStopTimedOut two-call restart contract. Regression wedges pre-fix. * refactor(gateway): move thaw channel restart off ChannelManager and fence mid-pass restartRunningChannelAccounts is a standalone helper over the public manager surface with a shouldContinue probe checked before every stop and start, so a suspension committing while an account stop is awaited leaves later accounts untouched. Regression covers the mid-pass close. * fix(gateway): sanitize late writes from an abandoned stopAccount An abandoned (timed-out) stopAccount can settle after its replacement started; route its late setStatus writes through the existing stale-task sanitizer so they cannot repaint or tear down the replacement. Regression fails pre-fix.
111 lines
4.2 KiB
Swift
111 lines
4.2 KiB
Swift
import Foundation
|
|
|
|
enum GatewaySleepPrepareResult: Equatable {
|
|
case ready(suspensionID: String)
|
|
case busy
|
|
}
|
|
|
|
@MainActor
|
|
final class GatewaySleepCycleController {
|
|
typealias Prepare = (String) async throws -> GatewaySleepPrepareResult
|
|
typealias Resume = (String) async throws -> Void
|
|
typealias Refresh = () async -> Void
|
|
typealias CurrentRoute = () -> String?
|
|
typealias RetryDelay = (Duration) async -> Void
|
|
|
|
private static let resumeAttempts = 3
|
|
private static let resumeRetryDelay: Duration = .seconds(2)
|
|
|
|
private let requestID: String
|
|
private let currentRoute: CurrentRoute
|
|
private let prepare: Prepare
|
|
private let resume: Resume
|
|
private let refresh: Refresh
|
|
private let retryDelay: RetryDelay
|
|
private let log: (String) -> Void
|
|
private var suspension: (id: String, route: String?)?
|
|
private var cycleGeneration: UInt64 = 0
|
|
|
|
init(
|
|
requestID: String,
|
|
currentRoute: @escaping CurrentRoute,
|
|
prepare: @escaping Prepare,
|
|
resume: @escaping Resume,
|
|
refresh: @escaping Refresh,
|
|
retryDelay: @escaping RetryDelay = { try? await Task.sleep(for: $0) },
|
|
log: @escaping (String) -> Void)
|
|
{
|
|
self.requestID = requestID
|
|
self.currentRoute = currentRoute
|
|
self.prepare = prepare
|
|
self.resume = resume
|
|
self.refresh = refresh
|
|
self.retryDelay = retryDelay
|
|
self.log = log
|
|
}
|
|
|
|
func willSleep(mode: AppState.ConnectionMode?) async {
|
|
guard mode == .local else { return }
|
|
self.cycleGeneration &+= 1
|
|
let generation = self.cycleGeneration
|
|
do {
|
|
switch try await self.prepare(self.requestID) {
|
|
case let .ready(suspensionID):
|
|
guard generation == self.cycleGeneration else {
|
|
// The wake already happened; release the late lease right away
|
|
// instead of fencing the gateway until its two-minute expiry.
|
|
try await self.resume(suspensionID)
|
|
return
|
|
}
|
|
self.suspension = (id: suspensionID, route: self.currentRoute())
|
|
case .busy:
|
|
self.log("gateway sleep preparation skipped because the gateway is busy")
|
|
}
|
|
} catch {
|
|
self.log("gateway sleep preparation failed: \(error.localizedDescription)")
|
|
}
|
|
}
|
|
|
|
func didWake(mode: AppState.ConnectionMode?) async {
|
|
let suspension = self.suspension
|
|
self.suspension = nil
|
|
// Invalidate a prepare response that arrives after the wake notification;
|
|
// its short-lived lease must expire instead of surviving into a later cycle.
|
|
self.cycleGeneration &+= 1
|
|
guard mode == .local else {
|
|
if suspension != nil {
|
|
self.log("dropping gateway sleep lease: route/mode changed across sleep; lease will self-expire")
|
|
}
|
|
return
|
|
}
|
|
let generation = self.cycleGeneration
|
|
// Refresh first: after real sleep the transport is usually dead, and the
|
|
// resume RPC needs the re-established connection to succeed at all.
|
|
await self.refresh()
|
|
if let suspension {
|
|
if let route = suspension.route, self.currentRoute() == route {
|
|
await self.resumeWithRetries(suspension.id, generation: generation)
|
|
} else {
|
|
self.log("dropping gateway sleep lease: route/mode changed across sleep; lease will self-expire")
|
|
}
|
|
}
|
|
}
|
|
|
|
private func resumeWithRetries(_ suspensionID: String, generation: UInt64) async {
|
|
for attempt in 1...Self.resumeAttempts {
|
|
// A new sleep cycle owns the connection; abandoned leases self-expire.
|
|
guard generation == self.cycleGeneration else { return }
|
|
do {
|
|
try await self.resume(suspensionID)
|
|
return
|
|
} catch {
|
|
self.log("gateway wake resume attempt \(attempt) failed: \(error.localizedDescription)")
|
|
if attempt < Self.resumeAttempts {
|
|
await self.retryDelay(Self.resumeRetryDelay)
|
|
}
|
|
}
|
|
}
|
|
self.log("giving up on gateway wake resume; lease will self-expire")
|
|
}
|
|
}
|