mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(test): wait for gateway readiness and isolate reset-heavy suite (#117725)
* fix(ci): isolate gateway readiness and subagent reset tests * fix(ci): isolate gateway readiness and subagent reset tests * fix(test): wait for gateway readiness in shared fixture * test(agents): isolate announce loop guard suite * fix(test): observe gateway exit during readiness * test: narrow gateway readiness process contract
This commit is contained in:
committed by
GitHub
parent
37c4a55ce2
commit
c40faefcf5
@@ -1,4 +1,5 @@
|
||||
// OpenClaw test instance tests cover spawned test instance lifecycle.
|
||||
import { EventEmitter } from "node:events";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
@@ -14,6 +15,16 @@ async function expectPathMissing(targetPath: string): Promise<void> {
|
||||
throw new Error(`Expected missing path: ${targetPath}`);
|
||||
}
|
||||
|
||||
function createGatewayProcessState(
|
||||
overrides: Partial<{ exitCode: number | null; signalCode: NodeJS.Signals | null }> = {},
|
||||
) {
|
||||
return Object.assign(new EventEmitter(), {
|
||||
exitCode: null as number | null,
|
||||
signalCode: null as NodeJS.Signals | null,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
describe("openclaw test instance", () => {
|
||||
it("keeps only bounded child output tails in helper logs", () => {
|
||||
const stdout = testing.createBoundedStringLog();
|
||||
@@ -40,8 +51,67 @@ describe("openclaw test instance", () => {
|
||||
|
||||
it("fails startup waits immediately after signaled gateway exits", async () => {
|
||||
await expect(
|
||||
testing.waitForPortOpen({ exitCode: null, signalCode: "SIGTERM" }, [], [], 1, 10_000),
|
||||
).rejects.toThrow("gateway exited before listening");
|
||||
testing.waitForGatewayReady(
|
||||
createGatewayProcessState({ signalCode: "SIGTERM" }),
|
||||
[],
|
||||
[],
|
||||
1,
|
||||
10_000,
|
||||
),
|
||||
).rejects.toThrow("gateway exited before readiness");
|
||||
});
|
||||
|
||||
it("waits until the gateway readiness probe reports ready", async () => {
|
||||
const fetchImpl = vi
|
||||
.fn<typeof fetch>()
|
||||
.mockResolvedValueOnce(
|
||||
new Response('{"ready":false,"failing":["startup-sidecars"]}', { status: 503 }),
|
||||
)
|
||||
.mockResolvedValueOnce(new Response('{"ready":true,"failing":[]}', { status: 200 }));
|
||||
|
||||
await expect(
|
||||
testing.waitForGatewayReady(createGatewayProcessState(), [], [], 12345, 1_000, fetchImpl),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
expect(fetchImpl.mock.calls[0]?.[0]).toBe("http://127.0.0.1:12345/readyz");
|
||||
});
|
||||
|
||||
it("keeps stalled readiness probes inside the startup deadline", async () => {
|
||||
const fetchImpl = vi.fn<typeof fetch>((_url, init) => {
|
||||
return new Promise((_resolve, reject) => {
|
||||
init?.signal?.addEventListener("abort", () => reject(init.signal?.reason), { once: true });
|
||||
});
|
||||
});
|
||||
const startedAt = Date.now();
|
||||
|
||||
await expect(
|
||||
testing.waitForGatewayReady(createGatewayProcessState(), [], [], 12345, 25, fetchImpl),
|
||||
).rejects.toThrow("timeout waiting for gateway readiness");
|
||||
|
||||
expect(fetchImpl).toHaveBeenCalledOnce();
|
||||
expect(Date.now() - startedAt).toBeLessThan(500);
|
||||
});
|
||||
|
||||
it("aborts a stalled readiness probe when the gateway exits", async () => {
|
||||
const processState = createGatewayProcessState();
|
||||
const fetchImpl = vi.fn<typeof fetch>((_url, init) => {
|
||||
return new Promise((_resolve, reject) => {
|
||||
init?.signal?.addEventListener("abort", () => reject(init.signal?.reason), { once: true });
|
||||
});
|
||||
});
|
||||
const startedAt = Date.now();
|
||||
setTimeout(() => {
|
||||
processState.signalCode = "SIGTERM";
|
||||
processState.emit("exit", null, "SIGTERM");
|
||||
}, 25);
|
||||
|
||||
await expect(
|
||||
testing.waitForGatewayReady(processState, [], [], 12345, 5_000, fetchImpl),
|
||||
).rejects.toThrow("gateway exited before readiness");
|
||||
|
||||
expect(fetchImpl).toHaveBeenCalledOnce();
|
||||
expect(Date.now() - startedAt).toBeLessThan(500);
|
||||
});
|
||||
|
||||
it("signals test instance process groups on POSIX", () => {
|
||||
|
||||
@@ -78,6 +78,10 @@ type BoundedStringLog = string[] & {
|
||||
};
|
||||
|
||||
type OpenClawTestChildProcess = Pick<OpenClawTestProcess, "kill" | "pid">;
|
||||
type OpenClawTestProcessReadiness = Pick<OpenClawTestProcess, "exitCode" | "signalCode"> & {
|
||||
once: (event: "exit", listener: () => void) => unknown;
|
||||
off: (event: "exit", listener: () => void) => unknown;
|
||||
};
|
||||
|
||||
function createBoundedStringLog(): string[] {
|
||||
const log = [] as BoundedStringLog;
|
||||
@@ -213,44 +217,83 @@ const getFreePort = async () => {
|
||||
return addr.port;
|
||||
};
|
||||
|
||||
async function waitForPortOpen(
|
||||
proc: Pick<OpenClawTestProcess, "exitCode" | "signalCode">,
|
||||
async function waitForGatewayReady(
|
||||
proc: OpenClawTestProcessReadiness,
|
||||
chunksOut: string[],
|
||||
chunksErr: string[],
|
||||
port: number,
|
||||
timeoutMs: number,
|
||||
fetchImpl: typeof fetch = fetch,
|
||||
) {
|
||||
const exitedBeforeReadinessError = () =>
|
||||
new Error(
|
||||
`gateway exited before readiness (code=${String(proc.exitCode)} signal=${String(
|
||||
proc.signalCode,
|
||||
)})\n${formatLogs(chunksOut, chunksErr)}`,
|
||||
);
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
if (hasChildExited(proc)) {
|
||||
throw new Error(
|
||||
`gateway exited before listening (code=${String(proc.exitCode)} signal=${String(
|
||||
proc.signalCode,
|
||||
)})\n${formatLogs(chunksOut, chunksErr)}`,
|
||||
);
|
||||
throw exitedBeforeReadinessError();
|
||||
}
|
||||
|
||||
const remainingMs = timeoutMs - (Date.now() - startedAt);
|
||||
const attemptTimeoutMs = Math.min(1_000, Math.max(1, remainingMs));
|
||||
const probeAbort = new AbortController();
|
||||
let attemptTimeout: ReturnType<typeof setTimeout> | undefined;
|
||||
let handleExit = () => {};
|
||||
const exitPromise = new Promise<never>((_resolve, reject) => {
|
||||
handleExit = () => {
|
||||
const error = exitedBeforeReadinessError();
|
||||
probeAbort.abort(error);
|
||||
reject(error);
|
||||
};
|
||||
proc.once("exit", handleExit);
|
||||
});
|
||||
const timeoutPromise = new Promise<never>((_resolve, reject) => {
|
||||
attemptTimeout = setTimeout(() => {
|
||||
const error = new Error("gateway readiness probe timed out");
|
||||
probeAbort.abort(error);
|
||||
reject(error);
|
||||
}, attemptTimeoutMs);
|
||||
attemptTimeout.unref?.();
|
||||
});
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const socket = net.connect({ host: "127.0.0.1", port });
|
||||
socket.once("connect", () => {
|
||||
socket.destroy();
|
||||
resolve();
|
||||
});
|
||||
socket.once("error", (err) => {
|
||||
socket.destroy();
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
return;
|
||||
// A dead child cannot complete readiness. Race the owner lifecycle against
|
||||
// both HTTP headers and body parsing so a stuck probe never hides its exit.
|
||||
const ready = await Promise.race([
|
||||
(async () => {
|
||||
const response = await fetchImpl(`http://127.0.0.1:${port}/readyz`, {
|
||||
signal: probeAbort.signal,
|
||||
});
|
||||
const readiness: unknown = await response.json();
|
||||
return response.ok && isRecord(readiness) && readiness.ready === true;
|
||||
})(),
|
||||
exitPromise,
|
||||
timeoutPromise,
|
||||
]);
|
||||
if (ready) {
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
if (hasChildExited(proc)) {
|
||||
throw exitedBeforeReadinessError();
|
||||
}
|
||||
// keep polling
|
||||
} finally {
|
||||
if (attemptTimeout) {
|
||||
clearTimeout(attemptTimeout);
|
||||
}
|
||||
proc.off("exit", handleExit);
|
||||
}
|
||||
|
||||
await sleep(10);
|
||||
const delayMs = Math.min(10, timeoutMs - (Date.now() - startedAt));
|
||||
if (delayMs > 0) {
|
||||
await sleep(delayMs);
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
`timeout waiting for gateway to listen on port ${port}\n${formatLogs(chunksOut, chunksErr)}`,
|
||||
`timeout waiting for gateway readiness on port ${port}\n${formatLogs(chunksOut, chunksErr)}`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -415,7 +458,7 @@ export async function createOpenClawTestInstance(
|
||||
child.stderr?.on("data", (d) => appendLogChunk(stderr, d));
|
||||
|
||||
try {
|
||||
await waitForPortOpen(
|
||||
await waitForGatewayReady(
|
||||
child,
|
||||
stdout,
|
||||
stderr,
|
||||
@@ -539,5 +582,5 @@ export const testing = {
|
||||
formatLogs,
|
||||
hasChildExited,
|
||||
signalOpenClawTestProcess,
|
||||
waitForPortOpen,
|
||||
waitForGatewayReady,
|
||||
};
|
||||
|
||||
@@ -14,6 +14,7 @@ const coreIsolatedFiles = [
|
||||
"src/agents/model-selection.plugin-runtime.test.ts",
|
||||
"src/agents/models-config.runtime-source-snapshot.test.ts",
|
||||
"src/agents/openai-transport-stream.streaming.test.ts",
|
||||
"src/agents/subagent-registry.announce-loop-guard.test.ts",
|
||||
"src/agents/subagent-registry-restart-recovery.test.ts",
|
||||
"src/agents/video-generation-task-status.test.ts",
|
||||
];
|
||||
|
||||
Reference in New Issue
Block a user