mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-17 16:12:21 -06:00
Peter Steinberger
7214577cf1
BoundedCommandTests and BoundedProcessTests failed nondeterministically: 4/20 runs idle, 7/8 under CPU saturation. Three separate causes. 1. #require inside a retry loop. waitForPID polled through readPID with `try?`, but #require records an issue even when its error is swallowed, so the first read of a created-but-not-yet-written pid file failed the test outright. Added a non-recording pollPID for the polling path and kept the recording read as the authoritative final attempt. The two single-read call sites now poll too - echo $$ > file creates and writes in two steps, so any single read can see a missing or empty file. 2. A 0.1s deadline racing process spawn. BoundedCommand starts its timeout concurrently with the spawn, so the deadline also bounded /bin/sh starting and publishing its pid; under load the child was killed before it ever wrote the file. Wait for the pid while the run is in flight and give the child a deadline well clear of spawn cost. 3. A 1s per-process budget on the concurrent fan-outs. Instrumenting the deadline showed a stalled run observed all 64 exits at ~3.1s, clustered within 100ms of each other - a global stall, not a straggler. Those tests assert that no exit is lost during monitor registration, not latency, so the timeout should not double as a performance assertion. A missed exit still fails: the 50ms pollUntilExit fallback would never complete. Also widened waitUntilGone, since reaping is asynchronous. No production code changed. Proof: 30/30 idle and 20/20 under full 32-core saturation, against 16/20 and 1/8 before.
83 lines
3.3 KiB
Swift
83 lines
3.3 KiB
Swift
import Darwin
|
|
import Foundation
|
|
import Testing
|
|
@testable import OpenClawDiscovery
|
|
|
|
struct BoundedCommandTests {
|
|
@Test func `drains output larger than a process pipe`() async throws {
|
|
let byteCount = 256 * 1024
|
|
let output = await BoundedCommand.run(
|
|
path: "/usr/bin/head",
|
|
arguments: ["-c", "\(byteCount)", "/dev/zero"],
|
|
timeout: 1.0)
|
|
|
|
let value = try #require(output)
|
|
#expect(value.utf8.count == byteCount)
|
|
}
|
|
|
|
@Test func `force kills and reaps a command that ignores termination`() async throws {
|
|
let pidFile = FileManager.default.temporaryDirectory
|
|
.appendingPathComponent("openclaw-bounded-command-\(UUID().uuidString).pid")
|
|
defer { try? FileManager.default.removeItem(at: pidFile) }
|
|
|
|
let clock = ContinuousClock()
|
|
let startedAt = clock.now
|
|
// BoundedCommand starts its deadline concurrently with the spawn, so the
|
|
// timeout also bounds `/bin/sh` starting up and publishing its pid. A
|
|
// deadline near spawn latency turns that into a coin flip: under load the
|
|
// child is killed before it ever writes the file. Keep it well clear of
|
|
// spawn cost; what this test asserts is the force-kill, not spawn speed.
|
|
let runTask = Task {
|
|
await BoundedCommand.run(
|
|
path: "/bin/sh",
|
|
arguments: ["-c", "echo $$ > \"$PID_FILE\"; trap '' TERM; exec /bin/sleep 30"],
|
|
environment: ["PID_FILE": pidFile.path],
|
|
timeout: 2.0)
|
|
}
|
|
|
|
let pid = try await Self.waitForPID(in: pidFile)
|
|
let output = await runTask.value
|
|
|
|
#expect(output == nil)
|
|
#expect(startedAt.duration(to: clock.now) < .seconds(10))
|
|
#expect(Self.waitUntilGone(pid))
|
|
}
|
|
|
|
/// Non-recording parse for polling. `#require` records an issue even when the
|
|
/// error it throws is swallowed by `try?`, so a retry loop must not use it or
|
|
/// the first not-yet-written read fails the test outright.
|
|
private static func pollPID(in file: URL) -> pid_t? {
|
|
guard let text = try? String(contentsOf: file, encoding: .utf8) else { return nil }
|
|
return pid_t(text.trimmingCharacters(in: .whitespacesAndNewlines))
|
|
}
|
|
|
|
/// The child creates the pid file and writes to it in two steps, so a single
|
|
/// read can observe a missing *or* empty file. Poll until it parses.
|
|
private static func waitForPID(in file: URL) async throws -> pid_t {
|
|
let deadline = ContinuousClock.now + .seconds(10)
|
|
while ContinuousClock.now < deadline {
|
|
if let pid = self.pollPID(in: file) {
|
|
return pid
|
|
}
|
|
try await Task.sleep(for: .milliseconds(10))
|
|
}
|
|
let text = try String(contentsOf: file, encoding: .utf8)
|
|
.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
return try #require(pid_t(text))
|
|
}
|
|
|
|
/// Reaping is asynchronous, so the process can still be visible for a moment
|
|
/// after `run` returns.
|
|
private static func waitUntilGone(_ pid: pid_t) -> Bool {
|
|
let deadline = Date().addingTimeInterval(5)
|
|
while Date() < deadline {
|
|
errno = 0
|
|
if kill(pid, 0) == -1, errno == ESRCH {
|
|
return true
|
|
}
|
|
usleep(10000)
|
|
}
|
|
return false
|
|
}
|
|
}
|