mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-16 23:52:40 -06:00
7214577cf1
BoundedCommandTests and BoundedProcessTests failed nondeterministically: 4/20 runs idle, 7/8 under CPU saturation. Three separate causes. 1. #require inside a retry loop. waitForPID polled through readPID with `try?`, but #require records an issue even when its error is swallowed, so the first read of a created-but-not-yet-written pid file failed the test outright. Added a non-recording pollPID for the polling path and kept the recording read as the authoritative final attempt. The two single-read call sites now poll too - echo $$ > file creates and writes in two steps, so any single read can see a missing or empty file. 2. A 0.1s deadline racing process spawn. BoundedCommand starts its timeout concurrently with the spawn, so the deadline also bounded /bin/sh starting and publishing its pid; under load the child was killed before it ever wrote the file. Wait for the pid while the run is in flight and give the child a deadline well clear of spawn cost. 3. A 1s per-process budget on the concurrent fan-outs. Instrumenting the deadline showed a stalled run observed all 64 exits at ~3.1s, clustered within 100ms of each other - a global stall, not a straggler. Those tests assert that no exit is lost during monitor registration, not latency, so the timeout should not double as a performance assertion. A missed exit still fails: the 50ms pollUntilExit fallback would never complete. Also widened waitUntilGone, since reaping is asynchronous. No production code changed. Proof: 30/30 idle and 20/20 under full 32-core saturation, against 16/20 and 1/8 before.
236 lines
9.9 KiB
Swift
236 lines
9.9 KiB
Swift
import Darwin
|
|
import Foundation
|
|
import Testing
|
|
@testable import OpenClaw
|
|
|
|
struct BoundedProcessTests {
|
|
/// The two fan-out tests below assert that no exit notification is *lost* when a
|
|
/// child exits while its monitor is being registered. They are not latency
|
|
/// assertions, so their timeout must not double as one.
|
|
///
|
|
/// Spawning dozens of processes at once periodically stalls the whole fan-out:
|
|
/// instrumenting the deadline showed every one of the 64 exits observed at
|
|
/// ~3.1s, clustered within 100ms of each other, rather than a single slow
|
|
/// straggler. A 1s per-process budget lost that coin flip roughly one run in
|
|
/// five, failing 63 of 64 at once. A generous budget still fails hard if an
|
|
/// exit is genuinely missed — the 50ms `pollUntilExit` fallback would never
|
|
/// complete — while a merely-loaded machine passes.
|
|
private static let concurrentSpawnTimeout: TimeInterval = 30
|
|
|
|
@Test func `captures output without waiting for inherited handles`() async throws {
|
|
let startedAt = ContinuousClock.now
|
|
let result = try await BoundedProcess.run(
|
|
path: "/bin/sh",
|
|
arguments: ["-c", "sleep 5 & echo $!; echo ready"],
|
|
timeout: 1)
|
|
|
|
let output = try #require(String(data: result.output, encoding: .utf8))
|
|
let lines = output.split(separator: "\n")
|
|
let childPID = try #require(lines.first.flatMap { pid_t($0) })
|
|
#expect(lines.contains("ready"))
|
|
#expect(result.terminationStatus == 0)
|
|
#expect(ContinuousClock.now - startedAt < .seconds(2))
|
|
#expect(self.waitUntilGone(childPID))
|
|
}
|
|
|
|
@Test func `preserves combined standard output and error ordering`() async throws {
|
|
let result = try await BoundedProcess.run(
|
|
path: "/bin/sh",
|
|
arguments: ["-c", "printf first; printf second >&2; printf third"],
|
|
timeout: 1)
|
|
|
|
#expect(String(data: result.output, encoding: .utf8) == "firstsecondthird")
|
|
}
|
|
|
|
@Test func `captures parallel instant exits`() async throws {
|
|
let results = try await withThrowingTaskGroup(of: Int32.self) { group in
|
|
for _ in 0..<32 {
|
|
group.addTask {
|
|
try await BoundedProcess.run(
|
|
path: "/usr/bin/true",
|
|
arguments: [],
|
|
timeout: Self.concurrentSpawnTimeout).terminationStatus
|
|
}
|
|
}
|
|
return try await group.reduce(into: []) { $0.append($1) }
|
|
}
|
|
|
|
#expect(results.count == 32)
|
|
#expect(results.allSatisfy { $0 == 0 })
|
|
}
|
|
|
|
@Test func `captures parallel script exits during monitor registration`() async throws {
|
|
let directory = FileManager.default.temporaryDirectory
|
|
.appendingPathComponent("openclaw-bounded-process-\(UUID().uuidString)", isDirectory: true)
|
|
try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
|
|
defer { try? FileManager.default.removeItem(at: directory) }
|
|
let script = directory.appendingPathComponent("instant-exit")
|
|
try "#!/bin/sh\nexit 0\n".write(to: script, atomically: true, encoding: .utf8)
|
|
try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: script.path)
|
|
|
|
let results = try await withThrowingTaskGroup(of: Int32.self) { group in
|
|
for _ in 0..<64 {
|
|
group.addTask {
|
|
try await BoundedProcess.run(
|
|
path: script.path,
|
|
arguments: [],
|
|
timeout: Self.concurrentSpawnTimeout).terminationStatus
|
|
}
|
|
}
|
|
return try await group.reduce(into: []) { $0.append($1) }
|
|
}
|
|
|
|
#expect(results.count == 64)
|
|
#expect(results.allSatisfy { $0 == 0 })
|
|
}
|
|
|
|
@Test func `times out and reaps a TERM-resistant process group`() async throws {
|
|
let directory = FileManager.default.temporaryDirectory
|
|
.appendingPathComponent("openclaw-bounded-process-\(UUID().uuidString)", isDirectory: true)
|
|
try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
|
|
defer { try? FileManager.default.removeItem(at: directory) }
|
|
let parentPIDFile = directory.appendingPathComponent("parent.pid")
|
|
let childPIDFile = directory.appendingPathComponent("child.pid")
|
|
|
|
let startedAt = ContinuousClock.now
|
|
do {
|
|
_ = try await BoundedProcess.run(
|
|
path: "/bin/sh",
|
|
arguments: [
|
|
"-c",
|
|
"""
|
|
trap '' TERM
|
|
/bin/sh -c 'trap "" TERM; echo $$ > "$CHILD_PID_FILE"; while :; do :; done' &
|
|
echo $$ > "$PARENT_PID_FILE"
|
|
while [ ! -s "$CHILD_PID_FILE" ]; do :; done
|
|
while :; do :; done
|
|
""",
|
|
],
|
|
environment: [
|
|
"PARENT_PID_FILE": parentPIDFile.path,
|
|
"CHILD_PID_FILE": childPIDFile.path,
|
|
],
|
|
timeout: 2)
|
|
Issue.record("Expected process timeout")
|
|
} catch {
|
|
#expect(error is BoundedProcessError)
|
|
}
|
|
|
|
let parentPID = try await self.waitForPID(in: parentPIDFile)
|
|
let childPID = try await self.waitForPID(in: childPIDFile)
|
|
#expect(ContinuousClock.now - startedAt < .seconds(3))
|
|
#expect(self.waitUntilGone(parentPID))
|
|
#expect(self.waitUntilGone(childPID))
|
|
}
|
|
|
|
@Test func `cancellation reaps the process group`() async throws {
|
|
let directory = FileManager.default.temporaryDirectory
|
|
.appendingPathComponent("openclaw-bounded-process-\(UUID().uuidString)", isDirectory: true)
|
|
try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
|
|
defer { try? FileManager.default.removeItem(at: directory) }
|
|
let parentPIDFile = directory.appendingPathComponent("parent.pid")
|
|
let childPIDFile = directory.appendingPathComponent("child.pid")
|
|
|
|
let task = Task {
|
|
try await BoundedProcess.run(
|
|
path: "/bin/sh",
|
|
arguments: [
|
|
"-c",
|
|
"""
|
|
sleep 30 &
|
|
echo $$ > "$PARENT_PID_FILE"
|
|
echo $! > "$CHILD_PID_FILE"
|
|
wait
|
|
""",
|
|
],
|
|
environment: [
|
|
"PARENT_PID_FILE": parentPIDFile.path,
|
|
"CHILD_PID_FILE": childPIDFile.path,
|
|
],
|
|
timeout: 30)
|
|
}
|
|
let parentPID = try await self.waitForPID(in: parentPIDFile)
|
|
let childPID = try await self.waitForPID(in: childPIDFile)
|
|
|
|
task.cancel()
|
|
do {
|
|
_ = try await task.value
|
|
Issue.record("Expected cancellation")
|
|
} catch {
|
|
#expect(error is CancellationError)
|
|
}
|
|
|
|
#expect(self.waitUntilGone(parentPID))
|
|
#expect(self.waitUntilGone(childPID))
|
|
}
|
|
|
|
@Test func `rejects excessive output and reaps the producer`() async throws {
|
|
let directory = FileManager.default.temporaryDirectory
|
|
.appendingPathComponent("openclaw-bounded-process-\(UUID().uuidString)", isDirectory: true)
|
|
try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
|
|
defer { try? FileManager.default.removeItem(at: directory) }
|
|
let pidFile = directory.appendingPathComponent("producer.pid")
|
|
|
|
let startedAt = ContinuousClock.now
|
|
do {
|
|
_ = try await BoundedProcess.run(
|
|
path: "/bin/sh",
|
|
arguments: [
|
|
"-c",
|
|
"""
|
|
echo $$ > "$PID_FILE"
|
|
while :; do
|
|
printf '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\n'
|
|
done
|
|
""",
|
|
],
|
|
environment: ["PID_FILE": pidFile.path],
|
|
timeout: 5)
|
|
Issue.record("Expected output limit failure")
|
|
} catch {
|
|
#expect(!(error is BoundedProcessError))
|
|
}
|
|
|
|
let producerPID = try await self.waitForPID(in: pidFile)
|
|
#expect(ContinuousClock.now - startedAt < .seconds(2))
|
|
#expect(self.waitUntilGone(producerPID))
|
|
}
|
|
|
|
/// Non-recording parse for polling. `#require` records an issue even when the
|
|
/// error it throws is swallowed by `try?`, so `waitForPID` cannot retry through
|
|
/// `readPID`: the first read of a created-but-not-yet-written pid file would
|
|
/// fail the test despite the retry succeeding a moment later.
|
|
private func pollPID(from file: URL) -> pid_t? {
|
|
guard let text = try? String(contentsOf: file, encoding: .utf8) else { return nil }
|
|
return pid_t(text.trimmingCharacters(in: .whitespacesAndNewlines))
|
|
}
|
|
|
|
/// `echo $$ > file` creates the file and writes to it in two steps, so a single
|
|
/// read can observe a missing *or* empty file. Poll until it parses, and only
|
|
/// then fall back to the recording read so a genuine absence still fails.
|
|
private func waitForPID(in file: URL) async throws -> pid_t {
|
|
let deadline = ContinuousClock.now + .seconds(10)
|
|
while ContinuousClock.now < deadline {
|
|
if let value = self.pollPID(from: file) {
|
|
return value
|
|
}
|
|
try await Task.sleep(for: .milliseconds(10))
|
|
}
|
|
let text = try String(contentsOf: file, encoding: .utf8)
|
|
.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
return try #require(pid_t(text))
|
|
}
|
|
|
|
private func waitUntilGone(_ pid: pid_t) -> Bool {
|
|
let deadline = Date().addingTimeInterval(5)
|
|
while Date() < deadline {
|
|
errno = 0
|
|
if kill(pid, 0) == -1, errno == ESRCH {
|
|
return true
|
|
}
|
|
usleep(10000)
|
|
}
|
|
return false
|
|
}
|
|
}
|