mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-26 04:15:48 -06:00
Peter Steinberger
8624b9acb8
* feat(gateway): recover channels and health promptly after host sleep A dependency-free thaw detector rides the existing 30s maintenance tick: when the process resumes after being frozen >=45s beyond cadence (laptop sleep, VM pause, SIGSTOP), the gateway restarts running channel accounts (dead sockets otherwise take up to ~35 minutes to notice), refreshes health/presence, and resets the event-loop histogram so the freeze does not read as degradation. Admission is rechecked before every recovery side effect; a suspension beginning mid-recovery re-pends the thaw, and timed-out channel stops complete their two-call restart in one pass. The macOS app cooperates: NSWorkspace sleep/wake observers in GatewayConnectivityCoordinator best-effort prepare a local gateway suspension before sleep and resume it on wake, never blocking sleep. The lease is bound to the route that prepared it and always cleared on wake; route or mode changes across sleep drop it to self-expiry. Live proof: SIGSTOP 85s on an isolated dev gateway -> 'host thaw detected: process was frozen ~57683ms', channels restarted, health ok, eventLoop degraded=false after thaw. * fix(macos): resume a sleep lease whose prepare response arrives after wake A prepare completing after didWake previously discarded the lease id, fencing the gateway until the two-minute expiry after micro-sleeps; the late response now resumes immediately. Document the conservative route-token drift tradeoff. * fix(macos): retry wake resume after refreshing the dead post-sleep transport After real sleep the WebSocket is usually dead exactly when resume runs; refresh the endpoint first, then attempt resume up to three times with bounded delays, clearing the lease only on success or exhaustion. A new sleep cycle aborts in-flight retries. * fix(gateway): bound plugin stopAccount so channel stops cannot wedge recovery stopChannel awaited plugin stopAccount unbounded; a never-settling stop hung the thaw restart (and health-monitor sweeps) and held the single-flight recovery guard forever. Race it against the existing 5s stop timeout; the timed-out path flows into the established recoveryStopTimedOut two-call restart contract. Regression wedges pre-fix. * refactor(gateway): move thaw channel restart off ChannelManager and fence mid-pass restartRunningChannelAccounts is a standalone helper over the public manager surface with a shouldContinue probe checked before every stop and start, so a suspension committing while an account stop is awaited leaves later accounts untouched. Regression covers the mid-pass close. * fix(gateway): sanitize late writes from an abandoned stopAccount An abandoned (timed-out) stopAccount can settle after its replacement started; route its late setStatus writes through the existing stale-task sanitizer so they cannot repaint or tear down the replacement. Regression fails pre-fix.
231 lines
8.2 KiB
Swift
231 lines
8.2 KiB
Swift
import Testing
|
|
@testable import OpenClaw
|
|
|
|
private struct PrepareFailure: Error {}
|
|
|
|
@Suite(.serialized)
|
|
@MainActor
|
|
struct GatewaySleepCycleControllerTests {
|
|
@Test func `ready preparation resumes its suspension once and refreshes`() async {
|
|
var preparedRequestIDs: [String] = []
|
|
var resumedIDs: [String] = []
|
|
var refreshCount = 0
|
|
let route = "ws://127.0.0.1:18789"
|
|
let controller = GatewaySleepCycleController(
|
|
requestID: "macos-sleep-test-run",
|
|
currentRoute: { route },
|
|
prepare: { requestID in
|
|
preparedRequestIDs.append(requestID)
|
|
return .ready(suspensionID: "suspension-1")
|
|
},
|
|
resume: { resumedIDs.append($0) },
|
|
refresh: { refreshCount += 1 },
|
|
log: { _ in })
|
|
|
|
await controller.willSleep(mode: .local)
|
|
await controller.didWake(mode: .local)
|
|
await controller.didWake(mode: .local)
|
|
|
|
#expect(preparedRequestIDs == ["macos-sleep-test-run"])
|
|
#expect(resumedIDs == ["suspension-1"])
|
|
#expect(refreshCount == 2)
|
|
}
|
|
|
|
@Test func `prepare response arriving after wake resumes the late lease immediately`() async {
|
|
var resumedIDs: [String] = []
|
|
var releasePrepare: CheckedContinuation<Void, Never>?
|
|
let controller = GatewaySleepCycleController(
|
|
requestID: "macos-sleep-test-run",
|
|
currentRoute: { "ws://127.0.0.1:18789" },
|
|
prepare: { _ in
|
|
await withCheckedContinuation { releasePrepare = $0 }
|
|
return .ready(suspensionID: "late-suspension")
|
|
},
|
|
resume: { resumedIDs.append($0) },
|
|
refresh: {},
|
|
log: { _ in })
|
|
|
|
let sleepTask = Task { await controller.willSleep(mode: .local) }
|
|
// Let willSleep reach the suspended prepare before waking.
|
|
while releasePrepare == nil {
|
|
await Task.yield()
|
|
}
|
|
await controller.didWake(mode: .local)
|
|
releasePrepare?.resume()
|
|
await sleepTask.value
|
|
await controller.didWake(mode: .local)
|
|
|
|
#expect(resumedIDs == ["late-suspension"])
|
|
}
|
|
|
|
@Test func `resume retries after a transport failure and succeeds`() async {
|
|
var resumeAttempts = 0
|
|
var delays = 0
|
|
let controller = GatewaySleepCycleController(
|
|
requestID: "macos-sleep-test-run",
|
|
currentRoute: { "ws://127.0.0.1:18789" },
|
|
prepare: { _ in .ready(suspensionID: "suspension-retry") },
|
|
resume: { _ in
|
|
resumeAttempts += 1
|
|
if resumeAttempts == 1 { throw PrepareFailure() }
|
|
},
|
|
refresh: {},
|
|
retryDelay: { _ in delays += 1 },
|
|
log: { _ in })
|
|
|
|
await controller.willSleep(mode: .local)
|
|
await controller.didWake(mode: .local)
|
|
|
|
#expect(resumeAttempts == 2)
|
|
#expect(delays == 1)
|
|
}
|
|
|
|
@Test func `resume gives up after exhausting retries`() async {
|
|
var resumeAttempts = 0
|
|
var logs: [String] = []
|
|
let controller = GatewaySleepCycleController(
|
|
requestID: "macos-sleep-test-run",
|
|
currentRoute: { "ws://127.0.0.1:18789" },
|
|
prepare: { _ in .ready(suspensionID: "suspension-exhaust") },
|
|
resume: { _ in
|
|
resumeAttempts += 1
|
|
throw PrepareFailure()
|
|
},
|
|
refresh: {},
|
|
retryDelay: { _ in },
|
|
log: { logs.append($0) })
|
|
|
|
await controller.willSleep(mode: .local)
|
|
await controller.didWake(mode: .local)
|
|
|
|
#expect(resumeAttempts == 3)
|
|
#expect(logs.contains { $0.contains("giving up") })
|
|
}
|
|
|
|
@Test func `a new sleep cycle aborts in-flight resume retries`() async {
|
|
var resumeAttempts = 0
|
|
var beginNextSleep: (() async -> Void)?
|
|
let controller = GatewaySleepCycleController(
|
|
requestID: "macos-sleep-test-run",
|
|
currentRoute: { "ws://127.0.0.1:18789" },
|
|
prepare: { _ in .ready(suspensionID: "suspension-abort") },
|
|
resume: { _ in
|
|
resumeAttempts += 1
|
|
throw PrepareFailure()
|
|
},
|
|
refresh: {},
|
|
retryDelay: { _ in await beginNextSleep?() },
|
|
log: { _ in })
|
|
|
|
await controller.willSleep(mode: .local)
|
|
beginNextSleep = { await controller.willSleep(mode: .local) }
|
|
await controller.didWake(mode: .local)
|
|
|
|
#expect(resumeAttempts == 1)
|
|
}
|
|
|
|
@Test func `busy preparation does not resume but still refreshes`() async {
|
|
var resumeCount = 0
|
|
var refreshCount = 0
|
|
let controller = GatewaySleepCycleController(
|
|
requestID: "macos-sleep-test-run",
|
|
currentRoute: { "ws://127.0.0.1:18789" },
|
|
prepare: { _ in .busy },
|
|
resume: { _ in resumeCount += 1 },
|
|
refresh: { refreshCount += 1 },
|
|
log: { _ in })
|
|
|
|
await controller.willSleep(mode: .local)
|
|
await controller.didWake(mode: .local)
|
|
|
|
#expect(resumeCount == 0)
|
|
#expect(refreshCount == 1)
|
|
}
|
|
|
|
@Test func `failed preparation does not resume but still refreshes`() async {
|
|
var resumeCount = 0
|
|
var refreshCount = 0
|
|
let controller = GatewaySleepCycleController(
|
|
requestID: "macos-sleep-test-run",
|
|
currentRoute: { "ws://127.0.0.1:18789" },
|
|
prepare: { _ in throw PrepareFailure() },
|
|
resume: { _ in resumeCount += 1 },
|
|
refresh: { refreshCount += 1 },
|
|
log: { _ in })
|
|
|
|
await controller.willSleep(mode: .local)
|
|
await controller.didWake(mode: .local)
|
|
|
|
#expect(resumeCount == 0)
|
|
#expect(refreshCount == 1)
|
|
}
|
|
|
|
@Test func `remote mode performs no sleep or wake work`() async {
|
|
var prepareCount = 0
|
|
var resumeCount = 0
|
|
var refreshCount = 0
|
|
let controller = GatewaySleepCycleController(
|
|
requestID: "macos-sleep-test-run",
|
|
currentRoute: { "ws://127.0.0.1:18789" },
|
|
prepare: { _ in
|
|
prepareCount += 1
|
|
return .ready(suspensionID: "unused")
|
|
},
|
|
resume: { _ in resumeCount += 1 },
|
|
refresh: { refreshCount += 1 },
|
|
log: { _ in })
|
|
|
|
await controller.willSleep(mode: .remote)
|
|
await controller.didWake(mode: .remote)
|
|
|
|
#expect(prepareCount == 0)
|
|
#expect(resumeCount == 0)
|
|
#expect(refreshCount == 0)
|
|
}
|
|
|
|
@Test func `changed route drops the suspension and still refreshes`() async {
|
|
var route = "ws://127.0.0.1:18789"
|
|
var resumedIDs: [String] = []
|
|
var refreshCount = 0
|
|
var logs: [String] = []
|
|
let controller = GatewaySleepCycleController(
|
|
requestID: "macos-sleep-test-run",
|
|
currentRoute: { route },
|
|
prepare: { _ in .ready(suspensionID: "suspension-1") },
|
|
resume: { resumedIDs.append($0) },
|
|
refresh: { refreshCount += 1 },
|
|
log: { logs.append($0) })
|
|
|
|
await controller.willSleep(mode: .local)
|
|
route = "ws://127.0.0.1:19001"
|
|
await controller.didWake(mode: .local)
|
|
route = "ws://127.0.0.1:18789"
|
|
await controller.didWake(mode: .local)
|
|
|
|
#expect(resumedIDs.isEmpty)
|
|
#expect(refreshCount == 2)
|
|
#expect(logs == ["dropping gateway sleep lease: route/mode changed across sleep; lease will self-expire"])
|
|
}
|
|
|
|
@Test func `remote wake clears a held local suspension`() async {
|
|
var resumedIDs: [String] = []
|
|
var refreshCount = 0
|
|
var logs: [String] = []
|
|
let controller = GatewaySleepCycleController(
|
|
requestID: "macos-sleep-test-run",
|
|
currentRoute: { "ws://127.0.0.1:18789" },
|
|
prepare: { _ in .ready(suspensionID: "suspension-1") },
|
|
resume: { resumedIDs.append($0) },
|
|
refresh: { refreshCount += 1 },
|
|
log: { logs.append($0) })
|
|
|
|
await controller.willSleep(mode: .local)
|
|
await controller.didWake(mode: .remote)
|
|
await controller.didWake(mode: .local)
|
|
|
|
#expect(resumedIDs.isEmpty)
|
|
#expect(refreshCount == 1)
|
|
#expect(logs == ["dropping gateway sleep lease: route/mode changed across sleep; lease will self-expire"])
|
|
}
|
|
}
|