diff --git a/extensions/qa-lab/src/suite-runtime-agent-process.integration.test.ts b/extensions/qa-lab/src/suite-runtime-agent-process.integration.test.ts index c5ab297f2416..374b50bbad93 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-process.integration.test.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-process.integration.test.ts @@ -72,4 +72,46 @@ describe("qa suite runtime CLI integration", () => { status: "ok", }); }); + + it("retains real child output when the qa cli times out", async () => { + const repoRoot = await mkdtemp(path.join(os.tmpdir(), "qa-cli-timeout-repo-")); + const tempRoot = await mkdtemp(path.join(os.tmpdir(), "qa-cli-timeout-runtime-")); + cleanups.push(async () => { + await rm(repoRoot, { recursive: true, force: true }); + await rm(tempRoot, { recursive: true, force: true }); + }); + const distDir = path.join(repoRoot, "dist"); + await mkdir(distDir, { recursive: true }); + await writeFile( + path.join(distDir, "index.js"), + [ + 'process.stdout.write("timeout stdout marker\\n");', + 'process.stderr.write("timeout stderr marker\\n");', + "setInterval(() => {}, 60_000);", + "", + ].join("\n"), + "utf8", + ); + + const error = await runQaCli( + { + repoRoot, + gateway: { + tempRoot, + runtimeEnv: process.env, + }, + primaryModel: "openai/gpt-5.6-luna", + alternateModel: "openai/gpt-5.6-luna", + providerMode: "mock-openai", + } as never, + ["qa", "suite"], + { timeoutMs: 1_000 }, + ).catch((value: unknown) => value); + + expect(error).toMatchObject({ code: "qa_cli_timeout" }); + const message = error instanceof Error ? error.message : String(error); + expect(message).toContain("qa cli timed out: openclaw qa suite"); + expect(message).toContain("stdout:\ntimeout stdout marker"); + expect(message).toContain("stderr:\ntimeout stderr marker"); + }); }); diff --git a/extensions/qa-lab/src/suite-runtime-agent-process.test.ts b/extensions/qa-lab/src/suite-runtime-agent-process.test.ts index ddbfebc5d091..047efc5d5256 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-process.test.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-process.test.ts @@ -174,6 +174,7 @@ describe("qa suite runtime agent process helpers", () => { it.runIf(process.platform !== "win32")("kills timed-out qa cli process groups", async () => { const killSpy = vi.spyOn(process, "kill").mockImplementation(() => true); + vi.useFakeTimers(); try { const child = createSpawnedProcess({ pid: 12345 }); const { pending } = startMockQaCli({ @@ -181,15 +182,37 @@ describe("qa suite runtime agent process helpers", () => { child, options: { timeoutMs: 1 }, }); - const timeoutAssertion = expect(pending).rejects.toThrow( - "qa cli timed out: openclaw qa suite", + const errorPromise = pending.catch((value: unknown) => value); + await Promise.resolve(); + expect(spawnMock).toHaveBeenCalledTimes(1); + child.stdout.emit( + "data", + Buffer.from( + `stdout-head-marker\n${"x".repeat(QA_CHILD_STDOUT_MAX_BYTES)}\nstdout-tail-marker`, + ), ); + child.stderr.emit( + "data", + Buffer.from( + `stderr-head-marker\n${"x".repeat(QA_CHILD_STDERR_TAIL_BYTES)}\nstderr-tail-marker`, + ), + ); + await vi.advanceTimersByTimeAsync(1); - await waitForSpawnCount(1); - await timeoutAssertion; + const error = await errorPromise; + expect(error).toMatchObject({ code: "qa_cli_timeout" }); + const message = error instanceof Error ? error.message : String(error); + expect(message).toContain("qa cli timed out: openclaw qa suite"); + expect(message).toContain("stdout:\n[qa cli stdout truncated to last"); + expect(message).toContain("stdout-tail-marker"); + expect(message).not.toContain("stdout-head-marker"); + expect(message).toContain("stderr:\n[qa cli stderr truncated to last"); + expect(message).toContain("stderr-tail-marker"); + expect(message).not.toContain("stderr-head-marker"); expect(killSpy).toHaveBeenCalledWith(-12345, "SIGKILL"); expect(child.kill).not.toHaveBeenCalled(); } finally { + vi.useRealTimers(); killSpy.mockRestore(); } }); diff --git a/extensions/qa-lab/src/suite-runtime-agent-process.ts b/extensions/qa-lab/src/suite-runtime-agent-process.ts index 3fbba57037fb..709fde09c9a1 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-process.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-process.ts @@ -265,6 +265,7 @@ async function runQaCli( opts?: { timeoutMs?: number; json?: boolean; env?: NodeJS.ProcessEnv }, ) { const stdout = createQaChildOutputCapture(); + const stdoutTail = createQaChildOutputTail(); const stderr = createQaChildOutputTail(); const distEntryPath = path.join(env.repoRoot, "dist", "index.js"); const nodeExecPath = await resolveQaNodeExecPath(); @@ -281,11 +282,25 @@ async function runQaCli( const timeoutMs = resolveTimerTimeoutMs(opts?.timeoutMs, 60_000); const timeout = setTimeout(() => { signalQaCliProcessTree(child, "SIGKILL"); + const stdoutText = formatQaChildOutputTail(stdoutTail, "qa cli stdout"); + const stderrText = formatQaChildOutputTail(stderr, "qa cli stderr"); + const diagnostics = [ + stdoutText ? `stdout:\n${stdoutText}` : "", + stderrText ? `stderr:\n${stderrText}` : "", + ] + .filter(Boolean) + .join("\n"); reject( - new QaSuiteInfraError("qa_cli_timeout", `qa cli timed out: openclaw ${args.join(" ")}`), + new QaSuiteInfraError( + "qa_cli_timeout", + `qa cli timed out: openclaw ${args.join(" ")}${diagnostics ? `\n${diagnostics}` : ""}`, + ), ); }, timeoutMs); - child.stdout.on("data", (chunk) => appendQaChildOutput(stdout, chunk)); + child.stdout.on("data", (chunk) => { + appendQaChildOutput(stdout, chunk); + appendQaChildOutputTail(stdoutTail, chunk); + }); child.stderr.on("data", (chunk) => appendQaChildOutputTail(stderr, chunk)); child.once("error", (error) => { clearTimeout(timeout);