diff --git a/extensions/qa-lab/src/confidence-report.test.ts b/extensions/qa-lab/src/confidence-report.test.ts index 903c6d967544..7254f52ac6ef 100644 --- a/extensions/qa-lab/src/confidence-report.test.ts +++ b/extensions/qa-lab/src/confidence-report.test.ts @@ -93,6 +93,43 @@ describe("qa confidence report", () => { expect(renderQaConfidenceMarkdownReport(report)).toContain("Global pass: no"); }); + it("uses suite lifecycle status before terminal outcome counts", async () => { + const manifest: QaConfidenceManifest = { + version: 1, + profile: "codex-100", + lanes: [ + { + id: "suite", + title: "Suite", + kind: "qa-suite-summary", + artifact: "suite/qa-suite-summary.json", + required: true, + }, + ], + }; + for (const [runStatus, expectedPass, expectedLaneStatus, expectedDetails] of [ + ["running", false, "unknown", "still running"], + ["completed", true, "pass", "counts.failed=0"], + ["paused", false, "unknown", "unsupported run.status=paused"], + ] as const) { + await writeJson("suite/qa-suite-summary.json", { + run: { status: runStatus }, + counts: { total: 1, passed: 1, skipped: 0, failed: 0 }, + scenarios: [{ name: "completed prefix", status: "pass" }], + }); + + const report = await buildQaConfidenceReport({ + manifest, + artifactRoot: tempRoot, + strictGlobalPass: true, + }); + + expect(report.pass).toBe(expectedPass); + expect(report.lanes[0]).toMatchObject({ status: expectedLaneStatus }); + expect(report.lanes[0]?.details).toContain(expectedDetails); + } + }); + it("does not let optional lanes block strict gates", async () => { await writeJson("required/qa-suite-summary.json", { counts: { total: 1, passed: 1, skipped: 0, failed: 0 }, diff --git a/extensions/qa-lab/src/confidence-report.ts b/extensions/qa-lab/src/confidence-report.ts index 583439d1fdbf..9140f25cfd79 100644 --- a/extensions/qa-lab/src/confidence-report.ts +++ b/extensions/qa-lab/src/confidence-report.ts @@ -365,6 +365,17 @@ function evaluateQaSuiteSummary(payload: unknown): QaConfidenceLaneEvaluation { details: "qa-suite-summary payload was not an object", }; } + const runStatus = isRecord(payload.run) ? payload.run.status : undefined; + if (runStatus !== undefined && runStatus !== "completed") { + return { + passed: false, + status: "unknown", + details: + runStatus === "running" + ? "qa-suite-summary is still running" + : `qa-suite-summary has unsupported run.status=${readString(runStatus) ?? typeof runStatus}`, + }; + } const accountingError = findQaSuiteSummaryAccountingError(payload); if (accountingError) { return { diff --git a/extensions/qa-lab/src/report.ts b/extensions/qa-lab/src/report.ts index d35d37d4dac0..7e5c45a14220 100644 --- a/extensions/qa-lab/src/report.ts +++ b/extensions/qa-lab/src/report.ts @@ -28,6 +28,7 @@ function formatQaReportCheck(check: QaReportCheck, indent = "") { export function renderQaMarkdownReport(params: { title: string; + inProgress?: boolean; startedAt: Date; finishedAt: Date; checks?: QaReportCheck[]; @@ -48,10 +49,11 @@ export function renderQaMarkdownReport(params: { scenarios.filter((scenario) => scenario.status === "skip").length; const lines = [ - `# ${params.title}`, + `# ${params.title}${params.inProgress ? " (In Progress)" : ""}`, "", + ...(params.inProgress ? ["- Status: running"] : []), `- Started: ${params.startedAt.toISOString()}`, - `- Finished: ${params.finishedAt.toISOString()}`, + `- ${params.inProgress ? "Updated" : "Finished"}: ${params.finishedAt.toISOString()}`, `- Duration ms: ${params.finishedAt.getTime() - params.startedAt.getTime()}`, `- Passed: ${passCount}`, `- Failed: ${failCount}`, diff --git a/extensions/qa-lab/src/suite-artifacts.ts b/extensions/qa-lab/src/suite-artifacts.ts index 8363c49fcc16..025daa03a697 100644 --- a/extensions/qa-lab/src/suite-artifacts.ts +++ b/extensions/qa-lab/src/suite-artifacts.ts @@ -49,6 +49,7 @@ export async function publishQaSuiteArtifactFiles(params: { } export type QaSuiteSummaryJsonParams = { + status?: QaSuiteSummaryJson["run"]["status"]; scenarios: QaSuiteScenarioResult[]; startedAt: Date; finishedAt: Date; @@ -108,6 +109,7 @@ export function buildQaSuiteSummaryJson(params: QaSuiteSummaryJsonParams): QaSui ...(params.metrics ? { metrics: params.metrics } : {}), ...(params.evidence ? { evidence: params.evidence } : {}), run: { + status: params.status ?? "completed", startedAt: params.startedAt.toISOString(), finishedAt: params.finishedAt.toISOString(), providerMode: params.providerMode, @@ -131,6 +133,7 @@ export function buildQaSuiteSummaryJson(params: QaSuiteSummaryJsonParams): QaSui } export async function writeQaSuiteArtifacts(params: { + status?: QaSuiteSummaryJson["run"]["status"]; repoRoot?: string; outputDir: string; startedAt: Date; @@ -195,6 +198,7 @@ export async function writeQaSuiteArtifacts(params: { : crablineChannelDriverSelection; const report = renderQaMarkdownReport({ title: "OpenClaw QA Scenario Suite", + inProgress: params.status === "running", startedAt: params.startedAt, finishedAt: params.finishedAt, checks: [], diff --git a/extensions/qa-lab/src/suite-process-lifecycle.test-support.ts b/extensions/qa-lab/src/suite-process-lifecycle.test-support.ts new file mode 100644 index 000000000000..c6eaf7e32821 --- /dev/null +++ b/extensions/qa-lab/src/suite-process-lifecycle.test-support.ts @@ -0,0 +1,35 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { resolveQaGatewayChildCommand } from "./gateway-child-command.js"; +import { runQaSuite } from "./suite-launch.runtime.js"; + +const repoRoot = fileURLToPath(new URL("../../../", import.meta.url)); +const outputDir = process.argv[2]; +const scenarioIds = process.argv.slice(3); + +if (!outputDir || scenarioIds.length === 0) { + throw new Error("suite process fixture requires an output directory and scenario ids"); +} + +try { + const sutOpenClawCommand = { + ...resolveQaGatewayChildCommand(repoRoot), + usePackagedPlugins: false, + }; + const result = await runQaSuite({ + repoRoot, + outputDir: path.relative(repoRoot, outputDir), + providerMode: "mock-openai", + scenarioIds, + concurrency: 4, + sutOpenClawCommand, + }); + const failed = result.result.scenarios.filter((scenario) => scenario.status !== "pass"); + if (failed.length > 0) { + throw new Error(`suite process fixture failed ${failed.length} scenario(s)`); + } +} catch (error) { + process.stderr.write(`${formatErrorMessage(error)}\n`); + process.exitCode = 1; +} diff --git a/extensions/qa-lab/src/suite-process-lifecycle.test.ts b/extensions/qa-lab/src/suite-process-lifecycle.test.ts new file mode 100644 index 000000000000..8e4783511d8e --- /dev/null +++ b/extensions/qa-lab/src/suite-process-lifecycle.test.ts @@ -0,0 +1,290 @@ +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; +import { existsSync } from "node:fs"; +import fs from "node:fs/promises"; +import net from "node:net"; +import path from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; +import { useAutoCleanupTempDirTracker } from "openclaw/plugin-sdk/test-env"; +import { afterEach, describe, expect, it } from "vitest"; +import type { QaSuiteSummaryJson } from "./suite-summary.js"; +import { runQaWindowsTaskkill } from "./windows-system-tools.js"; + +const repoRoot = fileURLToPath(new URL("../../../", import.meta.url)); +const fixturePath = fileURLToPath( + new URL("./suite-process-lifecycle.test-support.ts", import.meta.url), +); +const artifactsRoot = path.join(repoRoot, ".artifacts", "qa-e2e"); +const tempDirs = useAutoCleanupTempDirTracker(afterEach); +const activeChildren = new Set(); + +const PROCESS_LIFECYCLE_SCENARIO = "channel-chat-baseline"; +// Suite execution contends with the surrounding extension shard; only the bounded +// post-summary close window is the lifecycle contract this regression enforces. +const SUITE_COMPLETION_TIMEOUT_MS = 420_000; +const POST_SUMMARY_EXIT_TIMEOUT_MS = 45_000; + +function buildSuiteProcessEnv(outputDir: string) { + const home = path.join(outputDir, "process-home"); + const env: NodeJS.ProcessEnv = { + ...process.env, + HOME: home, + USERPROFILE: home, + OPENCLAW_HOME: home, + OPENCLAW_STATE_DIR: path.join(home, ".openclaw"), + OPENCLAW_CONFIG_PATH: path.join(home, ".openclaw", "openclaw.json"), + OPENCLAW_BUILD_PRIVATE_QA: "1", + OPENCLAW_QA_SUITE_PROGRESS: "1", + OPENCLAW_RUN_NODE_SKIP_DTS_BUILD: "1", + }; + if (!existsSync(path.join(repoRoot, "dist", "index.js"))) { + env.OPENCLAW_FORCE_BUILD = "1"; + } + delete env.VITEST; + delete env.VITEST_POOL_ID; + delete env.VITEST_WORKER_ID; + delete env.OPENCLAW_VITEST_FS_MODULE_CACHE_PATH; + delete env.OPENCLAW_VITEST_FS_MODULE_CACHE_WRITER; + delete env.NODE_COMPILE_CACHE; + delete env.NODE_DISABLE_COMPILE_CACHE; + delete env.OPENCLAW_NODE_COMPILE_CACHE_WRITER; + if (env.NODE_ENV === "test") { + delete env.NODE_ENV; + } + return env; +} + +function forceStopProcessTree(child: ChildProcess) { + if (!child.pid || child.exitCode !== null || child.signalCode !== null) { + return; + } + if (process.platform === "win32") { + if (!runQaWindowsTaskkill({ pid: child.pid, signal: "SIGKILL" })) { + child.kill("SIGKILL"); + } + return; + } + const rows = spawnSync("ps", ["-axo", "pid=,ppid="], { encoding: "utf8" }) + .stdout.trim() + .split("\n") + .flatMap((line) => { + const [pidText, parentPidText] = line.trim().split(/\s+/u); + const pid = Number(pidText); + const parentPid = Number(parentPidText); + return Number.isSafeInteger(pid) && Number.isSafeInteger(parentPid) + ? [{ pid, parentPid }] + : []; + }); + const owned = new Set([child.pid]); + let foundDescendant = true; + while (foundDescendant) { + foundDescendant = false; + for (const { pid, parentPid } of rows) { + if (owned.has(parentPid) && !owned.has(pid)) { + owned.add(pid); + foundDescendant = true; + } + } + } + for (const pid of [...owned].toReversed()) { + try { + process.kill(pid, "SIGKILL"); + } catch { + // Already gone. + } + } +} + +afterEach(async () => { + for (const child of activeChildren) { + forceStopProcessTree(child); + } + await Promise.all( + [...activeChildren].map( + (child) => + new Promise((resolve) => { + if (child.exitCode !== null || child.signalCode !== null) { + resolve(); + return; + } + child.once("close", () => resolve()); + }), + ), + ); + activeChildren.clear(); +}); + +function startSuiteProcess(outputDir: string, scenarioIds: readonly string[]) { + const child = spawn( + process.execPath, + ["--import", "tsx", fixturePath, outputDir, ...scenarioIds], + { + cwd: repoRoot, + env: buildSuiteProcessEnv(outputDir), + stdio: ["ignore", "pipe", "pipe"], + }, + ); + activeChildren.add(child); + let stdout = ""; + let stderr = ""; + const gatewayPorts = new Set(); + child.stdout?.setEncoding("utf8"); + child.stderr?.setEncoding("utf8"); + child.stdout?.on("data", (chunk: string) => { + stdout += chunk; + }); + child.stderr?.on("data", (chunk: string) => { + stderr += chunk; + for (const match of chunk.matchAll(/gateway ready: http:\/\/127\.0\.0\.1:(\d+)/gu)) { + gatewayPorts.add(Number(match[1])); + } + }); + const closed = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + (resolve, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => { + activeChildren.delete(child); + resolve({ code, signal }); + }); + }, + ); + return { + child, + closed, + gatewayPorts, + output: () => ({ stderr, stdout }), + }; +} + +async function isTcpPortOpen(port: number) { + return await new Promise((resolve) => { + const socket = net.createConnection({ host: "127.0.0.1", port }); + const finish = (open: boolean) => { + socket.destroy(); + resolve(open); + }; + socket.setTimeout(500, () => finish(false)); + socket.once("connect", () => finish(true)); + socket.once("error", () => finish(false)); + }); +} + +async function waitForCompletedSummary(params: { + outputDir: string; + timeoutMs: number; + closed: Promise<{ code: number | null; signal: NodeJS.Signals | null }>; + output: () => { stderr: string; stdout: string }; +}) { + const summaryPath = path.join(params.outputDir, "qa-suite-summary.json"); + const deadline = Date.now() + params.timeoutMs; + const processState: { + error?: unknown; + outcome?: { code: number | null; signal: NodeJS.Signals | null }; + } = {}; + void params.closed.then( + (outcome) => { + processState.outcome = outcome; + }, + (error: unknown) => { + processState.error = error; + }, + ); + const throwIfProcessClosed = () => { + if (!processState.error && !processState.outcome) { + return; + } + const output = params.output(); + throw new Error( + `QA suite process exited before writing a completed summary: ${JSON.stringify(processState.outcome ?? { error: String(processState.error) })}\nstdout:\n${output.stdout.slice(-8_000)}\nstderr:\n${output.stderr.slice(-8_000)}`, + ); + }; + while (Date.now() < deadline) { + let summary: QaSuiteSummaryJson; + try { + summary = JSON.parse(await fs.readFile(summaryPath, "utf8")) as QaSuiteSummaryJson; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + throwIfProcessClosed(); + await sleep(50); + continue; + } + const runStatus: unknown = summary.run.status; + if (runStatus === "completed") { + return summary; + } + if (runStatus !== "running") { + throw new Error(`QA suite summary is missing lifecycle status: ${String(runStatus)}`); + } + throwIfProcessClosed(); + await sleep(50); + } + const output = params.output(); + throw new Error( + `QA suite did not write a completed summary within ${params.timeoutMs}ms\nstdout:\n${output.stdout.slice(-8_000)}\nstderr:\n${output.stderr.slice(-8_000)}`, + ); +} + +async function waitForProcessClose( + closed: Promise<{ code: number | null; signal: NodeJS.Signals | null }>, + timeoutMs: number, +) { + let timeout: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + closed, + new Promise((_resolve, reject) => { + timeout = setTimeout( + () => + reject( + new Error( + `QA suite process did not exit within ${timeoutMs}ms of summary completion`, + ), + ), + timeoutMs, + ); + }), + ]); + } finally { + if (timeout) { + clearTimeout(timeout); + } + } +} + +describe("qa suite command process lifecycle", () => { + it( + "exits after the terminal summary and leaves no gateway listener", + { timeout: SUITE_COMPLETION_TIMEOUT_MS + POST_SUMMARY_EXIT_TIMEOUT_MS + 30_000 }, + async () => { + await fs.mkdir(artifactsRoot, { recursive: true }); + const outputDir = tempDirs.make("suite-process-lifecycle-", artifactsRoot); + const run = startSuiteProcess(outputDir, [PROCESS_LIFECYCLE_SCENARIO]); + const startedWaitingAt = Date.now(); + const heartbeat = setInterval(() => { + const output = run.output(); + process.stderr.write( + `[qa-process-lifecycle] waiting for completed summary elapsedMs=${Date.now() - startedWaitingAt} gatewayPorts=${run.gatewayPorts.size} stderrBytes=${Buffer.byteLength(output.stderr)}\n`, + ); + }, 30_000); + heartbeat.unref(); + const summary = await waitForCompletedSummary({ + outputDir, + timeoutMs: SUITE_COMPLETION_TIMEOUT_MS, + closed: run.closed, + output: run.output, + }).finally(() => clearInterval(heartbeat)); + const outcome = await waitForProcessClose(run.closed, POST_SUMMARY_EXIT_TIMEOUT_MS); + const output = run.output(); + + expect(outcome, output.stderr).toEqual({ code: 0, signal: null }); + expect(run.gatewayPorts.size, output.stderr).toBeGreaterThan(0); + expect(summary.run.status).toBe("completed"); + expect(summary.counts).toEqual({ total: 1, passed: 1, failed: 0, skipped: 0 }); + await expect( + Promise.all([...run.gatewayPorts].map((port) => isTcpPortOpen(port))), + ).resolves.toEqual([...run.gatewayPorts].map(() => false)); + }, + ); +}); diff --git a/extensions/qa-lab/src/suite-run-isolated.cleanup.test.ts b/extensions/qa-lab/src/suite-run-isolated.cleanup.test.ts index 6c19c814413e..1447b4b1905c 100644 --- a/extensions/qa-lab/src/suite-run-isolated.cleanup.test.ts +++ b/extensions/qa-lab/src/suite-run-isolated.cleanup.test.ts @@ -158,6 +158,13 @@ describe("isolated QA suite transport cleanup", () => { expect(lab.setLatestReport).toHaveBeenCalledWith( expect.objectContaining({ outputPath: "/qa-output/qa-suite-report.md" }), ); + expect(mocks.writeQaSuiteArtifacts).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ status: "running" }), + ); + expect(mocks.writeQaSuiteArtifacts).toHaveBeenLastCalledWith( + expect.not.objectContaining({ status: "running" }), + ); expect((thrown as Error).message.split("\n")[0]).toBe( "QA scenarios passed, but cleanup failed", ); diff --git a/extensions/qa-lab/src/suite-run-isolated.ts b/extensions/qa-lab/src/suite-run-isolated.ts index 8553f8f5aba6..966aea812020 100644 --- a/extensions/qa-lab/src/suite-run-isolated.ts +++ b/extensions/qa-lab/src/suite-run-isolated.ts @@ -98,6 +98,7 @@ export async function runQaFlowSuiteIsolated( .then(async () => { const partialFinishedAt = new Date(); const { report, reportPath } = await writeQaSuiteArtifacts({ + status: "running", repoRoot, outputDir, startedAt, diff --git a/extensions/qa-lab/src/suite-summary.ts b/extensions/qa-lab/src/suite-summary.ts index eb6c61d31867..a47b057abd7f 100644 --- a/extensions/qa-lab/src/suite-summary.ts +++ b/extensions/qa-lab/src/suite-summary.ts @@ -49,6 +49,7 @@ export type QaSuiteSummaryJson = { }; evidence?: QaEvidenceSummaryJson; run: { + status: "running" | "completed"; startedAt: string; finishedAt: string; providerMode: QaProviderMode; diff --git a/extensions/qa-lab/src/suite.summary-json.test.ts b/extensions/qa-lab/src/suite.summary-json.test.ts index 771d12ad872a..b8487f8911fb 100644 --- a/extensions/qa-lab/src/suite.summary-json.test.ts +++ b/extensions/qa-lab/src/suite.summary-json.test.ts @@ -23,6 +23,7 @@ describe("buildQaSuiteSummaryJson", () => { it("records provider/model/mode so parity gates can verify labels", () => { const json = buildQaSuiteSummaryJson(baseParams); + expect(json.run.status).toBe("completed"); expect(json.run.startedAt).toBe("2026-04-11T00:00:00.000Z"); expect(json.run.finishedAt).toBe("2026-04-11T00:05:00.000Z"); expect(json.run.providerMode).toBe("mock-openai"); @@ -41,6 +42,12 @@ describe("buildQaSuiteSummaryJson", () => { expect(json.run.scenarioIds).toBeNull(); }); + it("distinguishes an in-progress artifact from terminal suite output", () => { + const json = buildQaSuiteSummaryJson({ ...baseParams, status: "running" }); + + expect(json.run.status).toBe("running"); + }); + it("records Crabline channel-driver metadata when selected", () => { const json = buildQaSuiteSummaryJson({ ...baseParams, diff --git a/extensions/qa-lab/src/suite.test.ts b/extensions/qa-lab/src/suite.test.ts index 587a7d688486..5896a45ff343 100644 --- a/extensions/qa-lab/src/suite.test.ts +++ b/extensions/qa-lab/src/suite.test.ts @@ -618,6 +618,41 @@ describe("qa suite", () => { } }); + it("distinguishes partial Markdown from the terminal report shape", async () => { + const outputDir = await tempDirs.makeTempDir("qa-suite-report-lifecycle-"); + const baseParams = { + outputDir, + startedAt: new Date("2026-04-11T00:00:00.000Z"), + finishedAt: new Date("2026-04-11T00:01:00.000Z"), + scenarios: [{ name: "Baseline", status: "pass" as const, steps: [] }], + transport: { + id: "qa-channel", + createReportNotes: () => [], + } as unknown as QaTransportAdapter, + providerMode: "mock-openai" as const, + primaryModel: "mock-openai/gpt-5.6-luna", + alternateModel: "mock-openai/gpt-5.6-luna-alt", + fastMode: true, + concurrency: 1, + }; + + try { + const partial = await writeQaSuiteArtifacts({ ...baseParams, status: "running" }); + expect(partial.report).toContain("# OpenClaw QA Scenario Suite (In Progress)"); + expect(partial.report).toContain("- Status: running"); + expect(partial.report).toContain("- Updated: 2026-04-11T00:01:00.000Z"); + expect(partial.report).not.toContain("- Finished:"); + + const terminal = await writeQaSuiteArtifacts(baseParams); + expect(terminal.report).toContain("# OpenClaw QA Scenario Suite\n"); + expect(terminal.report).toContain("- Finished: 2026-04-11T00:01:00.000Z"); + expect(terminal.report).not.toContain("In Progress"); + expect(terminal.report).not.toContain("- Status: running"); + } finally { + await fs.rm(outputDir, { recursive: true, force: true }); + } + }); + it("writes the selected Crabline driver with an honest failed result", async () => { const outputDir = await tempDirs.makeTempDir("qa-suite-crabline-"); try {