diff --git a/test/helpers/openclaw-test-instance.test.ts b/test/helpers/openclaw-test-instance.test.ts index 97c6374e8b75..8e67fd19f7ce 100644 --- a/test/helpers/openclaw-test-instance.test.ts +++ b/test/helpers/openclaw-test-instance.test.ts @@ -1,4 +1,5 @@ // OpenClaw test instance tests cover spawned test instance lifecycle. +import { EventEmitter } from "node:events"; import fs from "node:fs/promises"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; @@ -14,6 +15,16 @@ async function expectPathMissing(targetPath: string): Promise { throw new Error(`Expected missing path: ${targetPath}`); } +function createGatewayProcessState( + overrides: Partial<{ exitCode: number | null; signalCode: NodeJS.Signals | null }> = {}, +) { + return Object.assign(new EventEmitter(), { + exitCode: null as number | null, + signalCode: null as NodeJS.Signals | null, + ...overrides, + }); +} + describe("openclaw test instance", () => { it("keeps only bounded child output tails in helper logs", () => { const stdout = testing.createBoundedStringLog(); @@ -40,8 +51,67 @@ describe("openclaw test instance", () => { it("fails startup waits immediately after signaled gateway exits", async () => { await expect( - testing.waitForPortOpen({ exitCode: null, signalCode: "SIGTERM" }, [], [], 1, 10_000), - ).rejects.toThrow("gateway exited before listening"); + testing.waitForGatewayReady( + createGatewayProcessState({ signalCode: "SIGTERM" }), + [], + [], + 1, + 10_000, + ), + ).rejects.toThrow("gateway exited before readiness"); + }); + + it("waits until the gateway readiness probe reports ready", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValueOnce( + new Response('{"ready":false,"failing":["startup-sidecars"]}', { status: 503 }), + ) + .mockResolvedValueOnce(new Response('{"ready":true,"failing":[]}', { status: 200 })); + + await expect( + testing.waitForGatewayReady(createGatewayProcessState(), [], [], 12345, 1_000, fetchImpl), + ).resolves.toBeUndefined(); + + expect(fetchImpl).toHaveBeenCalledTimes(2); + expect(fetchImpl.mock.calls[0]?.[0]).toBe("http://127.0.0.1:12345/readyz"); + }); + + it("keeps stalled readiness probes inside the startup deadline", async () => { + const fetchImpl = vi.fn((_url, init) => { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => reject(init.signal?.reason), { once: true }); + }); + }); + const startedAt = Date.now(); + + await expect( + testing.waitForGatewayReady(createGatewayProcessState(), [], [], 12345, 25, fetchImpl), + ).rejects.toThrow("timeout waiting for gateway readiness"); + + expect(fetchImpl).toHaveBeenCalledOnce(); + expect(Date.now() - startedAt).toBeLessThan(500); + }); + + it("aborts a stalled readiness probe when the gateway exits", async () => { + const processState = createGatewayProcessState(); + const fetchImpl = vi.fn((_url, init) => { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => reject(init.signal?.reason), { once: true }); + }); + }); + const startedAt = Date.now(); + setTimeout(() => { + processState.signalCode = "SIGTERM"; + processState.emit("exit", null, "SIGTERM"); + }, 25); + + await expect( + testing.waitForGatewayReady(processState, [], [], 12345, 5_000, fetchImpl), + ).rejects.toThrow("gateway exited before readiness"); + + expect(fetchImpl).toHaveBeenCalledOnce(); + expect(Date.now() - startedAt).toBeLessThan(500); }); it("signals test instance process groups on POSIX", () => { diff --git a/test/helpers/openclaw-test-instance.ts b/test/helpers/openclaw-test-instance.ts index 17152734505b..2092d8d79dd2 100644 --- a/test/helpers/openclaw-test-instance.ts +++ b/test/helpers/openclaw-test-instance.ts @@ -78,6 +78,10 @@ type BoundedStringLog = string[] & { }; type OpenClawTestChildProcess = Pick; +type OpenClawTestProcessReadiness = Pick & { + once: (event: "exit", listener: () => void) => unknown; + off: (event: "exit", listener: () => void) => unknown; +}; function createBoundedStringLog(): string[] { const log = [] as BoundedStringLog; @@ -213,44 +217,83 @@ const getFreePort = async () => { return addr.port; }; -async function waitForPortOpen( - proc: Pick, +async function waitForGatewayReady( + proc: OpenClawTestProcessReadiness, chunksOut: string[], chunksErr: string[], port: number, timeoutMs: number, + fetchImpl: typeof fetch = fetch, ) { + const exitedBeforeReadinessError = () => + new Error( + `gateway exited before readiness (code=${String(proc.exitCode)} signal=${String( + proc.signalCode, + )})\n${formatLogs(chunksOut, chunksErr)}`, + ); const startedAt = Date.now(); while (Date.now() - startedAt < timeoutMs) { if (hasChildExited(proc)) { - throw new Error( - `gateway exited before listening (code=${String(proc.exitCode)} signal=${String( - proc.signalCode, - )})\n${formatLogs(chunksOut, chunksErr)}`, - ); + throw exitedBeforeReadinessError(); } + const remainingMs = timeoutMs - (Date.now() - startedAt); + const attemptTimeoutMs = Math.min(1_000, Math.max(1, remainingMs)); + const probeAbort = new AbortController(); + let attemptTimeout: ReturnType | undefined; + let handleExit = () => {}; + const exitPromise = new Promise((_resolve, reject) => { + handleExit = () => { + const error = exitedBeforeReadinessError(); + probeAbort.abort(error); + reject(error); + }; + proc.once("exit", handleExit); + }); + const timeoutPromise = new Promise((_resolve, reject) => { + attemptTimeout = setTimeout(() => { + const error = new Error("gateway readiness probe timed out"); + probeAbort.abort(error); + reject(error); + }, attemptTimeoutMs); + attemptTimeout.unref?.(); + }); try { - await new Promise((resolve, reject) => { - const socket = net.connect({ host: "127.0.0.1", port }); - socket.once("connect", () => { - socket.destroy(); - resolve(); - }); - socket.once("error", (err) => { - socket.destroy(); - reject(err); - }); - }); - return; + // A dead child cannot complete readiness. Race the owner lifecycle against + // both HTTP headers and body parsing so a stuck probe never hides its exit. + const ready = await Promise.race([ + (async () => { + const response = await fetchImpl(`http://127.0.0.1:${port}/readyz`, { + signal: probeAbort.signal, + }); + const readiness: unknown = await response.json(); + return response.ok && isRecord(readiness) && readiness.ready === true; + })(), + exitPromise, + timeoutPromise, + ]); + if (ready) { + return; + } } catch { + if (hasChildExited(proc)) { + throw exitedBeforeReadinessError(); + } // keep polling + } finally { + if (attemptTimeout) { + clearTimeout(attemptTimeout); + } + proc.off("exit", handleExit); } - await sleep(10); + const delayMs = Math.min(10, timeoutMs - (Date.now() - startedAt)); + if (delayMs > 0) { + await sleep(delayMs); + } } throw new Error( - `timeout waiting for gateway to listen on port ${port}\n${formatLogs(chunksOut, chunksErr)}`, + `timeout waiting for gateway readiness on port ${port}\n${formatLogs(chunksOut, chunksErr)}`, ); } @@ -415,7 +458,7 @@ export async function createOpenClawTestInstance( child.stderr?.on("data", (d) => appendLogChunk(stderr, d)); try { - await waitForPortOpen( + await waitForGatewayReady( child, stdout, stderr, @@ -539,5 +582,5 @@ export const testing = { formatLogs, hasChildExited, signalOpenClawTestProcess, - waitForPortOpen, + waitForGatewayReady, }; diff --git a/test/vitest/vitest.agents-paths.mjs b/test/vitest/vitest.agents-paths.mjs index 3159532cd485..92e548393907 100644 --- a/test/vitest/vitest.agents-paths.mjs +++ b/test/vitest/vitest.agents-paths.mjs @@ -14,6 +14,7 @@ const coreIsolatedFiles = [ "src/agents/model-selection.plugin-runtime.test.ts", "src/agents/models-config.runtime-source-snapshot.test.ts", "src/agents/openai-transport-stream.streaming.test.ts", + "src/agents/subagent-registry.announce-loop-guard.test.ts", "src/agents/subagent-registry-restart-recovery.test.ts", "src/agents/video-generation-task-status.test.ts", ];