From 7b8bba26c9539eb631a56ea65c2ff22771b2d0db Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Thu, 6 Aug 2026 05:16:35 +0800 Subject: [PATCH] fix(qa): report cleanup failures truthfully (#119727) * fix(qa): report cleanup failures truthfully Punchcard-Session: silver-valley-valley-dt * fix(qa): satisfy cleanup runner static checks Punchcard-Session: silver-meadow-lantern-vx --- .../qa-lab/src/suite-launch.runtime.test.ts | 31 ++- .../src/suite-run-isolated.cleanup.test.ts | 181 +++++++++++++- extensions/qa-lab/src/suite-run-isolated.ts | 72 +++--- .../suite-run-standard.parity-retry.test.ts | 51 +++- extensions/qa-lab/src/suite-run-standard.ts | 48 ++-- ...uite-runtime-parity-runner.cleanup.test.ts | 220 +++++++++++++++++- .../qa-lab/src/suite-runtime-parity-runner.ts | 32 ++- extensions/qa-lab/src/suite.test.ts | 102 +++++++- extensions/qa-lab/src/suite.ts | 115 ++++++--- 9 files changed, 736 insertions(+), 116 deletions(-) diff --git a/extensions/qa-lab/src/suite-launch.runtime.test.ts b/extensions/qa-lab/src/suite-launch.runtime.test.ts index 331a87876d78..462d552e9525 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.test.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.test.ts @@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { QaSuiteInfraError } from "./errors.js"; import type { QaLabServerHandle } from "./lab-server.types.js"; import type { QaSuiteScenarioResult } from "./suite.js"; +import { throwQaSuiteCleanupErrors } from "./suite.js"; import type { QaTestFileScenario, QaTestFileScenarioRunResult, @@ -31,7 +32,7 @@ vi.mock("./test-file-scenario-runner.js", async (importOriginal) => ({ runQaTestFileScenarios, })); -import { runQaSuite } from "./suite-launch.runtime.js"; +import { runQaSuite, runQaSuiteWithInfraRetry } from "./suite-launch.runtime.js"; const tempRoots: string[] = []; @@ -251,6 +252,34 @@ describe("qa suite runtime launcher", () => { } }); + it("retries a cleanup-only ECONNRESET through its preserved cause", async () => { + const cleanupError = Object.assign(new Error("cleanup socket reset"), { + code: "ECONNRESET", + }); + const stderrWrite = vi.spyOn(process.stderr, "write").mockReturnValue(true); + let attempts = 0; + + try { + const result = await runQaSuiteWithInfraRetry(async () => { + attempts += 1; + if (attempts === 1) { + throwQaSuiteCleanupErrors({ + cleanupFailures: [{ phase: "lab stop", error: cleanupError }], + runFailed: false, + runError: undefined, + }); + } + return "retried"; + }, 1); + + expect(result).toBe("retried"); + expect(attempts).toBe(2); + expect(stderrWrite.mock.calls.flat().join("")).toContain("[qa-suite] infra retry 1/1:"); + } finally { + stderrWrite.mockRestore(); + } + }); + it("partitions flow-only suites that request isolated workers", async () => { const repoRoot = await makeTempRepo("qa-suite-flow-only-isolated-"); const result = await runQaSuite({ diff --git a/extensions/qa-lab/src/suite-run-isolated.cleanup.test.ts b/extensions/qa-lab/src/suite-run-isolated.cleanup.test.ts index abd94549cf24..f711ae484a1e 100644 --- a/extensions/qa-lab/src/suite-run-isolated.cleanup.test.ts +++ b/extensions/qa-lab/src/suite-run-isolated.cleanup.test.ts @@ -1,10 +1,61 @@ -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import { createQaBusState } from "./bus-state.js"; import type { QaLabServerHandle } from "./lab-server.types.js"; import type { QaTransportAdapterFactory } from "./qa-transport-registry.js"; import { runQaFlowSuiteIsolated } from "./suite-run-isolated.js"; +import { runQaFlowSuiteStandard } from "./suite-run-standard.js"; import { makeQaSuiteTestScenario } from "./suite-test-helpers.js"; -import type { QaSuiteResolvedRunContext, QaSuiteRunner } from "./suite-types.js"; +import type { + QaSuiteResolvedRunContext, + QaSuiteRunner, + QaSuiteScenarioRunner, +} from "./suite-types.js"; + +const mocks = vi.hoisted(() => ({ + disposeRegisteredAgentHarnesses: vi.fn(async () => {}), + fetchWithSsrFGuard: vi.fn(async () => ({ + response: new Response(null, { status: 204 }), + release: vi.fn(async () => {}), + })), + startQaGatewayChild: vi.fn(async () => ({ + baseUrl: "http://127.0.0.1:18789", + token: "qa-test-token", + cfg: {}, + getProcessCpuMs: () => null, + getProcessRssBytes: () => null, + stop: vi.fn(async () => {}), + })), + writeQaSuiteArtifacts: vi.fn(async () => ({ + evidence: undefined, + evidencePath: "/qa-output/qa-evidence.json", + report: "", + reportPath: "/qa-output/qa-suite-report.md", + summaryPath: "/qa-output/qa-suite-summary.json", + })), +})); + +vi.mock("openclaw/plugin-sdk/agent-harness", () => ({ + disposeRegisteredAgentHarnesses: mocks.disposeRegisteredAgentHarnesses, +})); +vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({ + fetchWithSsrFGuard: mocks.fetchWithSsrFGuard, +})); +vi.mock("./gateway-child.js", () => ({ + startQaGatewayChild: mocks.startQaGatewayChild, +})); +vi.mock("./providers/server-runtime.js", () => ({ + startQaProviderServer: vi.fn(async () => undefined), +})); +vi.mock("./suite-artifacts.js", () => ({ + writeQaSuiteArtifacts: mocks.writeQaSuiteArtifacts, +})); +vi.mock("./suite-runtime-gateway.js", () => ({ + waitForGatewayHealthy: vi.fn(async () => {}), + waitForTransportReady: vi.fn(async () => {}), +})); +vi.mock("./web-runtime.js", () => ({ + closeQaWebSessions: vi.fn(async () => {}), +})); function createCleanupTestLab(): QaLabServerHandle { return { @@ -41,6 +92,132 @@ function createCleanupTestContext(): QaSuiteResolvedRunContext { } describe("isolated QA suite transport cleanup", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.disposeRegisteredAgentHarnesses.mockResolvedValue(undefined); + }); + + it("retains passing artifacts and finishes owned cleanup before reporting teardown failure", async () => { + const lab = createCleanupTestLab(); + const release = vi.fn(async () => {}); + const factory: QaTransportAdapterFactory = { + id: "leased", + matches: ({ channelId, driver }) => channelId === "leased" && driver === "live", + async create() { + return { + id: "leased", + label: "Leased channel", + accountId: "sut", + requiredPluginIds: [], + supportedActions: [], + sendInbound: async (input) => lab.state.addInboundMessage(input), + createGatewayConfig: () => ({}), + async waitReady() {}, + buildAgentDelivery: ({ target }) => ({ + channel: "leased", + to: target, + replyChannel: "leased", + replyTo: target, + }), + async handleAction() {}, + createReportNotes: () => [], + cleanup: release, + }; + }, + }; + const cleanupError = new Error("agent harness disposal failed"); + mocks.disposeRegisteredAgentHarnesses.mockRejectedValueOnce(cleanupError); + const stderrWrite = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + const runChild = vi.fn().mockResolvedValue({ + outputDir: "/qa-child", + evidencePath: "/qa-child/qa-evidence.json", + reportPath: "/qa-child/qa-suite-report.md", + summaryPath: "/qa-child/qa-suite-summary.json", + report: "", + scenarios: [{ name: "leased-channel-scenario", status: "pass", steps: [] }], + watchUrl: lab.baseUrl, + }); + const context = createCleanupTestContext(); + context.progressEnabled = true; + + const thrown = await runQaFlowSuiteIsolated( + { + adapterFactories: [factory], + channelDriver: "live", + channelId: "leased", + startLab: async () => lab, + }, + context, + runChild, + ).catch((error: unknown) => error); + + expect(release).toHaveBeenCalledOnce(); + expect(mocks.disposeRegisteredAgentHarnesses).toHaveBeenCalledOnce(); + expect(lab.stop).toHaveBeenCalledOnce(); + expect(lab.setLatestReport).toHaveBeenCalledWith( + expect.objectContaining({ outputPath: "/qa-output/qa-suite-report.md" }), + ); + expect((thrown as Error).message.split("\n")[0]).toBe( + "QA scenarios passed, but cleanup failed", + ); + expect((thrown as Error).message).toContain( + "failed cleanup phases: agent harnesses: agent harness disposal failed", + ); + expect((thrown as Error).message).toContain( + "retained artifacts: output=/qa-output report=/qa-output/qa-suite-report.md summary=/qa-output/qa-suite-summary.json", + ); + expect((thrown as Error).cause).toBe(cleanupError); + expect(stderrWrite.mock.calls.flat().join("")).not.toContain("run complete"); + stderrWrite.mockRestore(); + }); + + it("prints one generic completion after a real nested standard run and parent cleanup", async () => { + const parentLab = createCleanupTestLab(); + const childLab = createCleanupTestLab(); + const startLab = vi + .fn<() => Promise>() + .mockResolvedValueOnce(parentLab) + .mockResolvedValueOnce(childLab); + const context = createCleanupTestContext(); + context.channelDriver = undefined; + context.progressEnabled = true; + const runScenario = vi + .fn() + .mockResolvedValue({ name: "leased-channel-scenario", status: "pass", steps: [] }); + const runChild: QaSuiteRunner = async (childParams) => { + if (!childParams) { + throw new Error("expected nested standard run params"); + } + return await runQaFlowSuiteStandard( + childParams, + { + ...context, + startedAt: new Date("2026-08-04T00:00:01.000Z"), + outputDir: childParams.outputDir ?? "/qa-output/scenarios/leased-channel-scenario", + concurrency: 1, + }, + runScenario, + ); + }; + const stderrWrite = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + + try { + await runQaFlowSuiteIsolated({ startLab }, context, runChild); + + const completionLines = stderrWrite.mock.calls + .flat() + .join("") + .split("\n") + .filter((line) => line.startsWith("[qa-suite] run complete")); + expect(completionLines).toEqual(["[qa-suite] run complete"]); + expect(runScenario).toHaveBeenCalledOnce(); + expect(childLab.stop).toHaveBeenCalledOnce(); + expect(parentLab.stop).toHaveBeenCalledOnce(); + } finally { + stderrWrite.mockRestore(); + } + }); + it.each(["cleanup", "cleanupAfterGatewayStop"] as const)( "retries a failed parent %s phase before disposing its owned lab", async (cleanupPhase) => { diff --git a/extensions/qa-lab/src/suite-run-isolated.ts b/extensions/qa-lab/src/suite-run-isolated.ts index 9dce017c43bb..4ed11a0c09fd 100644 --- a/extensions/qa-lab/src/suite-run-isolated.ts +++ b/extensions/qa-lab/src/suite-run-isolated.ts @@ -15,6 +15,7 @@ import type { } from "./suite-types.js"; import { createQaSuiteTransportAdapter, + markQaSuiteNestedRun, requireQaSuiteStartLab, runQaSuiteCleanupSteps, throwQaSuiteCleanupErrors, @@ -135,6 +136,9 @@ export async function runQaFlowSuiteIsolated( let isolatedRunFailed = false; let isolatedRunError: unknown; let parentTransportCleaned = false; + let result: QaSuiteResult | undefined; + let completionProgress: string | undefined; + let evidenceWritten = false; try { if (params?.channelDriver === "live") { // The parent only renders aggregate artifacts. Release its live credentials @@ -164,24 +168,26 @@ export async function runQaFlowSuiteIsolated( updateScenarioRun(); try { const scenarioOutputDir = path.join(outputDir, "scenarios", scenario.id); - const result: QaSuiteResult = await runQaFlowSuite( - buildQaIsolatedScenarioWorkerParams({ - repoRoot, - outputDir: scenarioOutputDir, - providerMode, - transportId, - channelDriver: params?.channelDriver, - channelDriverSelection: params?.channelDriverSelection, - primaryModel, - alternateModel, - fastMode, - startLab, - scenario, - input: params, - }), + const childSuiteResult: QaSuiteResult = await runQaFlowSuite( + markQaSuiteNestedRun( + buildQaIsolatedScenarioWorkerParams({ + repoRoot, + outputDir: scenarioOutputDir, + providerMode, + transportId, + channelDriver: params?.channelDriver, + channelDriverSelection: params?.channelDriverSelection, + primaryModel, + alternateModel, + fastMode, + startLab, + scenario, + input: params, + }), + ), ); const scenarioResult: QaSuiteScenarioResult = - result.scenarios[0] ?? + childSuiteResult.scenarios[0] ?? ({ name: scenario.title, status: "fail", @@ -246,13 +252,12 @@ export async function runQaFlowSuiteIsolated( }, { startStaggerMs: workerStartStaggerMs, - shouldStop: (result) => params?.failFast === true && result.status === "fail", + shouldStop: (scenarioResult) => + params?.failFast === true && scenarioResult.status === "fail", }, ); await artifactWriteQueue; const finishedAt = new Date(); - const failedCount = scenarios.filter((scenario) => scenario.status === "fail").length; - const skippedCount = scenarios.filter((scenario) => scenario.status === "skip").length; lab.setScenarioRun({ kind: "suite", status: "completed", @@ -296,11 +301,9 @@ export async function runQaFlowSuiteIsolated( markdown: report, generatedAt: finishedAt.toISOString(), } satisfies QaLabLatestReport); - writeQaSuiteProgress( - progressEnabled, - `run complete: passed=${scenarios.length - failedCount - skippedCount} failed=${failedCount} skipped=${skippedCount} total=${scenarios.length}`, - ); - return { + completionProgress = "run complete"; + evidenceWritten = evidence !== undefined && (params?.writeEvidenceFile ?? true); + result = { outputDir, evidence, evidencePath, @@ -315,18 +318,27 @@ export async function runQaFlowSuiteIsolated( isolatedRunError = error; throw error; } finally { - const cleanupSteps: Array<() => Promise> = [ - ...(!parentTransportCleaned ? [() => transportFactoryResult.cleanupWithoutGateway()] : []), - () => disposeRegisteredAgentHarnesses(), + const cleanupSteps = [ + ...(!parentTransportCleaned + ? [{ phase: "parent transport", run: () => transportFactoryResult.cleanupWithoutGateway() }] + : []), + { phase: "agent harnesses", run: () => disposeRegisteredAgentHarnesses() }, ]; if (ownsLab) { - cleanupSteps.push(() => lab.stop()); + cleanupSteps.push({ phase: "lab stop", run: () => lab.stop() }); } - const cleanupErrors = await runQaSuiteCleanupSteps(cleanupSteps); + const cleanupFailures = await runQaSuiteCleanupSteps(cleanupSteps); throwQaSuiteCleanupErrors({ - cleanupErrors, + cleanupFailures, runFailed: isolatedRunFailed, runError: isolatedRunError, + result, + evidenceWritten, }); } + if (!result || !completionProgress) { + throw new Error("QA suite completed without terminal result metadata"); + } + writeQaSuiteProgress(progressEnabled, completionProgress); + return result; } diff --git a/extensions/qa-lab/src/suite-run-standard.parity-retry.test.ts b/extensions/qa-lab/src/suite-run-standard.parity-retry.test.ts index 075fec3bc83b..05c7202a06d7 100644 --- a/extensions/qa-lab/src/suite-run-standard.parity-retry.test.ts +++ b/extensions/qa-lab/src/suite-run-standard.parity-retry.test.ts @@ -7,6 +7,7 @@ import type { QaSuiteScenarioResult, QaSuiteScenarioRunner, } from "./suite-types.js"; +import type { runQaFlowSuiteCleanupPlan } from "./suite.js"; const mocks = vi.hoisted(() => ({ captureRuntimeParityCell: vi.fn(async (params: { runtime: "codex"; wallClockMs: number }) => ({ @@ -44,6 +45,8 @@ const mocks = vi.hoisted(() => ({ })), waitForGatewayHealthy: vi.fn(async () => {}), waitForTransportReady: vi.fn(async () => {}), + runQaFlowSuiteCleanupPlan: vi.fn(async () => []), + writeQaSuiteProgress: vi.fn(), })); vi.mock("openclaw/plugin-sdk/agent-harness", () => ({ @@ -65,7 +68,8 @@ vi.mock("./suite-runtime-gateway.js", () => ({ waitForGatewayHealthy: mocks.waitForGatewayHealthy, waitForTransportReady: mocks.waitForTransportReady, })); -vi.mock("./suite.js", () => ({ +vi.mock("./suite.js", async (importOriginal) => ({ + ...(await importOriginal()), buildQaSuiteRuntimeMetrics: vi.fn(() => ({ wallMs: 1 })), captureGatewayHeapSnapshotCheckpoint: vi.fn(async () => undefined), createQaSuiteTransportAdapter: vi.fn(async () => ({ @@ -75,10 +79,9 @@ vi.mock("./suite.js", () => ({ })), requireQaSuiteStartLab: vi.fn(), resolveQaSuiteTransportReadyTimeoutMs: vi.fn(() => 1_000), - runQaFlowSuiteCleanupPlan: vi.fn(async () => []), - throwQaSuiteCleanupErrors: vi.fn(), + runQaFlowSuiteCleanupPlan: mocks.runQaFlowSuiteCleanupPlan, waitForQaLabReadyOrStopOwned: vi.fn(async () => {}), - writeQaSuiteProgress: vi.fn(), + writeQaSuiteProgress: mocks.writeQaSuiteProgress, })); vi.mock("./web-runtime.js", () => ({ closeQaWebSessions: vi.fn(async () => {}), @@ -128,6 +131,7 @@ function makeRetryTestResult(status: "pass" | "fail"): QaSuiteScenarioResult { beforeEach(() => { vi.clearAllMocks(); + mocks.runQaFlowSuiteCleanupPlan.mockResolvedValue([]); }); describe("QA suite Control UI ownership", () => { @@ -196,6 +200,45 @@ describe("QA suite Control UI ownership", () => { }); describe("QA runtime parity scenario retry isolation", () => { + it("does not report terminal success when cleanup fails after writing artifacts", async () => { + const lab = makeRetryTestLab(); + const cleanupError = Object.assign(new Error("gateway shutdown socket reset"), { + code: "ECONNRESET", + }); + mocks.runQaFlowSuiteCleanupPlan.mockResolvedValueOnce([ + { phase: "gateway stop", error: cleanupError }, + ]); + + const thrown = await runQaFlowSuiteStandard( + { lab }, + makeRetryTestContext(), + vi.fn().mockResolvedValue(makeRetryTestResult("pass")), + ).catch((error: unknown) => error); + + expect(thrown).toBeInstanceOf(AggregateError); + expect((thrown as Error).message.split("\n")[0]).toBe( + "QA scenarios passed, but cleanup failed", + ); + expect((thrown as Error).message).toContain( + "scenario counts: passed=1 failed=0 skipped=0 total=1", + ); + expect((thrown as Error).message).toContain( + "failed cleanup phases: gateway stop: gateway shutdown socket reset", + ); + expect((thrown as Error).message).toContain( + "retained artifacts: output=/qa-output report=/qa-output/qa-suite-report.md summary=/qa-output/qa-suite-summary.json", + ); + expect((thrown as Error).cause).toBe(cleanupError); + expect(lab.setLatestReport).toHaveBeenCalledWith( + expect.objectContaining({ outputPath: "/qa-output/qa-suite-report.md" }), + ); + expect( + mocks.writeQaSuiteProgress.mock.calls.filter(([, message]) => + String(message).startsWith("run complete"), + ), + ).toHaveLength(0); + }); + it.each([ { forcedRuntime: undefined, expectedRuntime: "openclaw" }, { forcedRuntime: "codex" as const, expectedRuntime: "codex" }, diff --git a/extensions/qa-lab/src/suite-run-standard.ts b/extensions/qa-lab/src/suite-run-standard.ts index 2300396e667a..cf74d9d97d8f 100644 --- a/extensions/qa-lab/src/suite-run-standard.ts +++ b/extensions/qa-lab/src/suite-run-standard.ts @@ -39,6 +39,7 @@ import { createQaSuiteTransportAdapter, buildQaSuiteRuntimeMetrics, captureGatewayHeapSnapshotCheckpoint, + isQaSuiteNestedRun, requireQaSuiteStartLab, resolveQaSuiteTransportReadyTimeoutMs, runQaFlowSuiteCleanupPlan, @@ -107,6 +108,9 @@ export async function runQaFlowSuiteStandard( let preserveGatewayRuntimeDir: string | undefined; let runFailed = false; let runError: unknown; + let result: QaSuiteResult | undefined; + let completionProgress: string | undefined; + let evidenceWritten = false; try { writeQaSuiteProgress(progressEnabled, `provider start: ${providerMode}`); const activeMock = await startQaProviderServer(providerMode, { @@ -267,7 +271,7 @@ export async function runQaFlowSuiteStandard( }; const scenarioRetryCount = scenario.execution.kind === "flow" ? scenario.execution.retryCount : undefined; - let result: QaSuiteScenarioResult = + let scenarioResult: QaSuiteScenarioResult = params?.captureRuntimeParityCell || scenarioRetryCount === 0 ? await runSelectedScenario() : await runQaScenarioWithFlakeRetry(runSelectedScenario, () => @@ -276,19 +280,19 @@ export async function runQaFlowSuiteStandard( `scenario retry (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`, ), ); - if (result.status === "pass" && params?.roundTripProbe?.scenarioId === scenario.id) { + if (scenarioResult.status === "pass" && params?.roundTripProbe?.scenarioId === scenario.id) { const probeResult = await runQaSuiteRoundTripProbe({ probe: params.roundTripProbe, transport, }); const probePassed = probeResult.passed >= params.roundTripProbe.count; - result = { - ...result, + scenarioResult = { + ...scenarioResult, status: probePassed ? "pass" : "fail", - details: [result.details, probeResult.details].filter(Boolean).join(" | "), + details: [scenarioResult.details, probeResult.details].filter(Boolean).join(" | "), timing: probeResult.timing, steps: [ - ...result.steps, + ...scenarioResult.steps, { name: "Round-trip samples", status: probePassed ? "pass" : "fail", @@ -306,17 +310,17 @@ export async function runQaFlowSuiteStandard( }); } sampleGatewayProcessRss(`scenario:${scenario.id}:finish`); - scenarios.push(result); + scenarios.push(scenarioResult); writeQaSuiteProgress( progressEnabled, - `scenario ${result.status} (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`, + `scenario ${scenarioResult.status} (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`, ); liveScenarioOutcomes[index] = { id: scenario.id, name: scenario.title, - status: result.status, - details: result.details, - steps: result.steps, + status: scenarioResult.status, + details: scenarioResult.details, + steps: scenarioResult.steps, startedAt: liveScenarioOutcomes[index]?.startedAt, finishedAt: new Date().toISOString(), }; @@ -326,7 +330,7 @@ export async function runQaFlowSuiteStandard( startedAt: startedAt.toISOString(), scenarios: [...liveScenarioOutcomes], }); - if (params?.failFast === true && result.status === "fail") { + if (params?.failFast === true && scenarioResult.status === "fail") { break; } } @@ -407,12 +411,9 @@ export async function runQaFlowSuiteStandard( generatedAt: finishedAt.toISOString(), } satisfies QaLabLatestReport; lab.setLatestReport(latestReport); - writeQaSuiteProgress( - progressEnabled, - `run complete: passed=${scenarios.length - failedCount - skippedCount} failed=${failedCount} skipped=${skippedCount} total=${scenarios.length}`, - ); - - return { + completionProgress = `run complete: passed=${scenarios.length - failedCount - skippedCount} failed=${failedCount} skipped=${skippedCount} total=${scenarios.length}`; + evidenceWritten = evidence !== undefined && (params?.writeEvidenceFile ?? true); + result = { outputDir, evidence, evidencePath, @@ -433,7 +434,7 @@ export async function runQaFlowSuiteStandard( const keepTemp = process.env.OPENCLAW_QA_KEEP_TEMP === "1" || false; const activeGateway = gateway; const activeMock = mock; - const cleanupErrors = await runQaFlowSuiteCleanupPlan({ + const cleanupFailures = await runQaFlowSuiteCleanupPlan({ closeWebSessions: activeEnv ? () => closeQaWebSessions(activeEnv.webSessionIds) : undefined, cleanupTransportBeforeGatewayStop: () => transportFactoryResult.cleanupBeforeGatewayStop(), cleanupTransportAfterGatewayStop: () => transportFactoryResult.cleanupAfterGatewayStop(), @@ -457,6 +458,13 @@ export async function runQaFlowSuiteStandard( } }, }); - throwQaSuiteCleanupErrors({ cleanupErrors, runFailed, runError }); + throwQaSuiteCleanupErrors({ cleanupFailures, runFailed, runError, result, evidenceWritten }); } + if (!result || !completionProgress) { + throw new Error("QA suite completed without terminal result metadata"); + } + if (!params?.captureRuntimeParityCell && !isQaSuiteNestedRun(params)) { + writeQaSuiteProgress(progressEnabled, completionProgress); + } + return result; } diff --git a/extensions/qa-lab/src/suite-runtime-parity-runner.cleanup.test.ts b/extensions/qa-lab/src/suite-runtime-parity-runner.cleanup.test.ts index a96adf26547c..89a8f7bdefdd 100644 --- a/extensions/qa-lab/src/suite-runtime-parity-runner.cleanup.test.ts +++ b/extensions/qa-lab/src/suite-runtime-parity-runner.cleanup.test.ts @@ -5,9 +5,84 @@ import { createQaTransportAdapter, type QaTransportAdapterFactory, } from "./qa-transport-registry.js"; +import { runQaFlowSuiteStandard } from "./suite-run-standard.js"; import { runQaRuntimeParitySuite } from "./suite-runtime-parity-runner.js"; import { makeQaSuiteTestScenario } from "./suite-test-helpers.js"; -import type { QaSuiteRunner } from "./suite-types.js"; +import type { + QaSuiteResolvedRunContext, + QaSuiteRunner, + QaSuiteScenarioRunner, +} from "./suite-types.js"; + +const mocks = vi.hoisted(() => ({ + captureRuntimeParityCell: vi.fn( + async (params: { runtime: "openclaw" | "codex"; wallClockMs: number }) => ({ + runtime: params.runtime, + transcriptBytes: "", + toolCalls: [], + finalText: "ok", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + cacheDiagnostics: { + assistantTurns: 1, + cacheTelemetryTurns: 1, + cacheHitTurns: 0, + cacheWriteTurns: 0, + cacheMisses: [], + cacheMissInputTokens: 0, + unmeasuredPostWarmTurns: [], + }, + wallClockMs: params.wallClockMs, + bootStateLines: [], + }), + ), + disposeRegisteredAgentHarnesses: vi.fn(async () => {}), + fetchWithSsrFGuard: vi.fn(async () => ({ + response: new Response(null, { status: 204 }), + release: vi.fn(async () => {}), + })), + startQaGatewayChild: vi.fn(async () => ({ + baseUrl: "http://127.0.0.1:18789", + token: "qa-test-token", + cfg: {}, + getProcessCpuMs: () => null, + getProcessRssBytes: () => null, + stop: vi.fn(async () => {}), + })), + writeQaSuiteArtifacts: vi.fn(async () => ({ + evidence: { kind: "test" }, + evidencePath: "/qa-output/qa-evidence.json", + report: "", + reportPath: "/qa-output/qa-suite-report.md", + summaryPath: "/qa-output/qa-suite-summary.json", + })), +})); + +vi.mock("openclaw/plugin-sdk/agent-harness", () => ({ + disposeRegisteredAgentHarnesses: mocks.disposeRegisteredAgentHarnesses, +})); +vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({ + fetchWithSsrFGuard: mocks.fetchWithSsrFGuard, +})); +vi.mock("./gateway-child.js", () => ({ + startQaGatewayChild: mocks.startQaGatewayChild, +})); +vi.mock("./providers/server-runtime.js", () => ({ + startQaProviderServer: vi.fn(async () => undefined), +})); +vi.mock("./runtime-parity.js", async (importOriginal) => ({ + ...(await importOriginal()), + captureRuntimeParityCell: mocks.captureRuntimeParityCell, +})); +vi.mock("./suite-artifacts.js", () => ({ + writeQaSuiteArtifacts: mocks.writeQaSuiteArtifacts, +})); +vi.mock("./suite-runtime-gateway.js", () => ({ + waitForGatewayHealthy: vi.fn(async () => {}), + waitForTransportReady: vi.fn(async () => {}), +})); +vi.mock("./web-runtime.js", () => ({ + closeQaWebSessions: vi.fn(async () => {}), +})); function createCleanupTestLab(): QaLabServerHandle { return { @@ -62,6 +137,7 @@ function createCleanupTestFactory( function runCleanupTestSuite(params: { factory: QaTransportAdapterFactory; lab: QaLabServerHandle; + progressEnabled?: boolean; runChild: QaSuiteRunner; }) { return runQaRuntimeParitySuite({ @@ -80,12 +156,148 @@ function runCleanupTestSuite(params: { concurrency: 1, selectedScenarios: [makeQaSuiteTestScenario("runtime-cleanup")], startLab: async () => params.lab, - progressEnabled: false, + progressEnabled: params.progressEnabled ?? false, runtimePair: ["openclaw", "codex"], }); } describe("runtime parity suite transport cleanup", () => { + it("keeps parent artifacts discoverable when owned lab cleanup fails", async () => { + const cleanupError = Object.assign(new Error("owned lab shutdown reset"), { + code: "ECONNRESET", + }); + const setLatestReport = vi.fn(); + const stopLab = vi.fn(async () => { + throw cleanupError; + }); + const lab = createCleanupTestLab(); + lab.setLatestReport = setLatestReport; + lab.stop = stopLab; + const cleanup = vi.fn(async () => {}); + const factory = createCleanupTestFactory(lab, () => ({ cleanup })); + const stderrWrite = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + const runChild = vi.fn().mockImplementation(async (params) => ({ + outputDir: "/qa-child", + evidencePath: "/qa-child/qa-evidence.json", + reportPath: "/qa-child/qa-suite-report.md", + summaryPath: "/qa-child/qa-suite-summary.json", + report: "", + scenarios: [{ name: "runtime-cleanup", status: "pass", steps: [] }], + watchUrl: lab.baseUrl, + runtimeParityCell: { + runtime: params?.forcedRuntime ?? "openclaw", + transcriptBytes: "", + toolCalls: [], + finalText: "ok", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + wallClockMs: 1, + bootStateLines: [], + }, + })); + + try { + const thrown = await runCleanupTestSuite({ + factory, + lab, + progressEnabled: true, + runChild, + }).catch((error: unknown) => error); + + expect(cleanup).toHaveBeenCalledOnce(); + expect(setLatestReport).toHaveBeenCalledWith( + expect.objectContaining({ outputPath: "/qa-output/qa-suite-report.md" }), + ); + expect(setLatestReport.mock.invocationCallOrder[0]).toBeLessThan( + stopLab.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + expect((thrown as Error).message.split("\n")[0]).toBe( + "QA scenarios passed, but cleanup failed", + ); + expect((thrown as Error).message).toContain( + "failed cleanup phases: lab stop: owned lab shutdown reset", + ); + expect((thrown as Error).message).toContain( + "retained artifacts: output=/qa-output report=/qa-output/qa-suite-report.md summary=/qa-output/qa-suite-summary.json evidence=/qa-output/qa-evidence.json", + ); + expect((thrown as Error).cause).toBe(cleanupError); + expect(stderrWrite.mock.calls.flat().join("")).not.toContain("run complete"); + } finally { + stderrWrite.mockRestore(); + } + }); + + it("prints one generic completion after real nested standard cells and parent cleanup", async () => { + const scenario = makeQaSuiteTestScenario("runtime-cleanup"); + const parentLab = createCleanupTestLab(); + const openClawLab = createCleanupTestLab(); + const codexLab = createCleanupTestLab(); + const startLab = vi + .fn<() => Promise>() + .mockResolvedValueOnce(parentLab) + .mockResolvedValueOnce(openClawLab) + .mockResolvedValueOnce(codexLab); + const runScenario = vi + .fn() + .mockResolvedValue({ name: scenario.title, status: "pass", steps: [] }); + const runChild: QaSuiteRunner = async (childParams) => { + if (!childParams) { + throw new Error("expected nested standard run params"); + } + const context: QaSuiteResolvedRunContext = { + startedAt: new Date("2026-08-04T00:00:01.000Z"), + repoRoot: childParams.repoRoot ?? "/qa-repo", + outputDir: childParams.outputDir ?? "/qa-output/runtime-cell", + transportId: childParams.transportId ?? "qa-channel", + selectedScenarios: [scenario], + providerMode: childParams.providerMode ?? "mock-openai", + primaryModel: childParams.primaryModel ?? "mock-openai/test-model", + alternateModel: childParams.alternateModel ?? "mock-openai/test-model-alt", + fastMode: childParams.fastMode ?? true, + channelDriver: childParams.channelDriver, + enabledPluginIds: childParams.enabledPluginIds ?? [], + gatewayConfigPatch: undefined, + gatewayRuntimeOptions: undefined, + concurrency: 1, + progressEnabled: true, + gatewayHeapCheckpointsEnabled: false, + }; + return await runQaFlowSuiteStandard(childParams, context, runScenario); + }; + const stderrWrite = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + + try { + await runQaRuntimeParitySuite({ + runQaFlowSuite: runChild, + repoRoot: "/qa-repo", + outputDir: "/qa-output", + startedAt: new Date("2026-08-04T00:00:00.000Z"), + providerMode: "mock-openai", + transportId: "qa-channel", + primaryModel: "mock-openai/test-model", + alternateModel: "mock-openai/test-model-alt", + fastMode: true, + concurrency: 1, + selectedScenarios: [scenario], + startLab, + progressEnabled: true, + runtimePair: ["openclaw", "codex"], + }); + + const completionLines = stderrWrite.mock.calls + .flat() + .join("") + .split("\n") + .filter((line) => line.startsWith("[qa-suite] run complete")); + expect(completionLines).toEqual(["[qa-suite] run complete"]); + expect(runScenario).toHaveBeenCalledTimes(2); + expect(openClawLab.stop).toHaveBeenCalledOnce(); + expect(codexLab.stop).toHaveBeenCalledOnce(); + expect(parentLab.stop).toHaveBeenCalledOnce(); + } finally { + stderrWrite.mockRestore(); + } + }); + it("preserves the scenario error when its owned lab cleanup fails", async () => { const lab = createCleanupTestLab(); const scenarioError = new Error("runtime scenario failed"); @@ -98,7 +310,9 @@ describe("runtime parity suite transport cleanup", () => { const runChild = vi.fn().mockRejectedValueOnce(scenarioError); await expect(runCleanupTestSuite({ factory, lab, runChild })).rejects.toMatchObject({ - message: "QA suite and cleanup failed", + message: expect.stringContaining( + "failed cleanup phases: lab stop: owned lab shutdown failed", + ), cause: scenarioError, errors: [scenarioError, cleanupError], }); diff --git a/extensions/qa-lab/src/suite-runtime-parity-runner.ts b/extensions/qa-lab/src/suite-runtime-parity-runner.ts index 946b98124d75..c549811bd0ed 100644 --- a/extensions/qa-lab/src/suite-runtime-parity-runner.ts +++ b/extensions/qa-lab/src/suite-runtime-parity-runner.ts @@ -108,6 +108,8 @@ export async function runQaRuntimeParitySuite(params: { let runFailed = false; let runError: unknown; let parentTransportCleaned = false; + let result: QaSuiteResult | undefined; + let evidenceWritten = false; try { if (params.channelDriver === "live") { // The parent only contributes aggregate metadata; release its exclusive @@ -217,16 +219,16 @@ export async function runQaRuntimeParitySuite(params: { }, }); - const result = buildRuntimeParityScenarioResult({ + const parityScenarioResult = buildRuntimeParityScenarioResult({ scenarioName: scenario.title, result: parity, }); liveScenarioOutcomes[index] = { id: scenario.id, name: scenario.title, - status: result.status, - details: result.details, - steps: result.steps, + status: parityScenarioResult.status, + details: parityScenarioResult.details, + steps: parityScenarioResult.steps, startedAt: liveScenarioOutcomes[index]?.startedAt, finishedAt: new Date().toISOString(), }; @@ -238,9 +240,9 @@ export async function runQaRuntimeParitySuite(params: { }); writeQaSuiteProgress( params.progressEnabled, - `runtime pair ${result.status} (${index + 1}/${params.selectedScenarios.length}): ${scenarioIdForLog}`, + `runtime pair ${parityScenarioResult.status} (${index + 1}/${params.selectedScenarios.length}): ${scenarioIdForLog}`, ); - return result; + return parityScenarioResult; }, { startStaggerMs: resolveQaSuiteWorkerStartStaggerMs(params.concurrency), @@ -285,7 +287,8 @@ export async function runQaRuntimeParitySuite(params: { finishedAt: finishedAt.toISOString(), scenarios: [...liveScenarioOutcomes], }); - return { + evidenceWritten = evidence !== undefined && (params.writeEvidenceFile ?? true); + result = { outputDir: params.outputDir, evidence, evidencePath, @@ -300,10 +303,17 @@ export async function runQaRuntimeParitySuite(params: { runError = error; throw error; } finally { - const cleanupErrors = await runQaSuiteCleanupSteps([ - ...(!parentTransportCleaned ? [() => transportFactoryResult.cleanupWithoutGateway()] : []), - ...(ownsLab ? [() => lab.stop()] : []), + const cleanupFailures = await runQaSuiteCleanupSteps([ + ...(!parentTransportCleaned + ? [{ phase: "parent transport", run: () => transportFactoryResult.cleanupWithoutGateway() }] + : []), + ...(ownsLab ? [{ phase: "lab stop", run: () => lab.stop() }] : []), ]); - throwQaSuiteCleanupErrors({ cleanupErrors, runFailed, runError }); + throwQaSuiteCleanupErrors({ cleanupFailures, runFailed, runError, result, evidenceWritten }); } + if (!result) { + throw new Error("QA runtime parity suite completed without a result"); + } + writeQaSuiteProgress(params.progressEnabled, "run complete"); + return result; } diff --git a/extensions/qa-lab/src/suite.test.ts b/extensions/qa-lab/src/suite.test.ts index a24bda6999fe..060f281c030c 100644 --- a/extensions/qa-lab/src/suite.test.ts +++ b/extensions/qa-lab/src/suite.test.ts @@ -7,6 +7,7 @@ import { QA_EVIDENCE_FILENAME, QA_EVIDENCE_SUMMARY_KIND } from "./evidence-summa import type { QaLabServerHandle } from "./lab-server.types.js"; import type { QaTransportAdapter } from "./qa-transport.js"; import { makeQaSuiteTestScenario } from "./suite-test-helpers.js"; +import type { QaSuiteResult } from "./suite-types.js"; import { qaSuiteProgressTesting, runQaFlowSuite } from "./suite.js"; import { createTempDirHarness } from "./temp-dir.test-helper.js"; @@ -39,7 +40,8 @@ function makeQaSuiteTestLabHandle(): QaLabServerHandle { describe("qa suite", () => { it("runs the production cleanup plan in dependency order after a failure", async () => { const calls: string[] = []; - const failure = new Error("transport close failed"); + const transportFailure = new Error("transport close failed"); + const providerFailure = new Error("provider close failed"); const step = (name: string, error?: Error) => async () => { calls.push(name); if (error) { @@ -47,13 +49,13 @@ describe("qa suite", () => { } }; - const errors = await qaSuiteProgressTesting.runQaFlowSuiteCleanupPlan({ + const failures = await qaSuiteProgressTesting.runQaFlowSuiteCleanupPlan({ closeWebSessions: step("web sessions"), - cleanupTransportBeforeGatewayStop: step("transport before gateway", failure), + cleanupTransportBeforeGatewayStop: step("transport before gateway", transportFailure), cleanupTransportAfterGatewayStop: step("transport after gateway"), stopGateway: step("gateway"), disposeAgentHarnesses: step("agent harnesses"), - stopProvider: step("provider"), + stopProvider: step("provider", providerFailure), finishLab: step("lab"), }); @@ -66,19 +68,97 @@ describe("qa suite", () => { "provider", "lab", ]); - expect(errors).toEqual([failure]); + expect(failures).toEqual([ + { phase: "transport before gateway stop", error: transportFailure }, + { phase: "provider stop", error: providerFailure }, + ]); }); it("keeps the primary suite error as the cause of aggregated cleanup failures", () => { const runError = new Error("gateway infrastructure failed"); + const cleanupError = new Error("transport cleanup failed"); - expect(() => + let thrown: unknown; + try { qaSuiteProgressTesting.throwQaSuiteCleanupErrors({ - cleanupErrors: [new Error("transport cleanup failed")], + cleanupFailures: [{ phase: "transport before gateway stop", error: cleanupError }], runFailed: true, runError, - }), - ).toThrow(expect.objectContaining({ cause: runError })); + }); + } catch (error) { + thrown = error; + } + + expect(thrown).toMatchObject({ + cause: runError, + errors: [runError, cleanupError], + }); + expect((thrown as Error).message.split("\n")[0]).toBe("QA suite and cleanup failed"); + expect((thrown as Error).message).toContain( + "failed cleanup phases: transport before gateway stop: transport cleanup failed", + ); + }); + + it("reports cleanup failure before scenarios completed when no result exists", () => { + const cleanupError = new Error("stop failed"); + let thrown: unknown; + try { + qaSuiteProgressTesting.throwQaSuiteCleanupErrors({ + cleanupFailures: [{ phase: "lab stop", error: cleanupError }], + runFailed: false, + runError: undefined, + }); + } catch (error) { + thrown = error; + } + + expect((thrown as Error).message.split("\n")[0]).toBe( + "QA suite cleanup failed before scenarios completed", + ); + expect((thrown as Error).cause).toBe(cleanupError); + }); + + it("reports completed counts, labeled failures, and only written artifact paths", () => { + const result = { + outputDir: "/qa-output\nretained", + evidencePath: "/qa-output/qa-evidence.json", + reportPath: "/qa-output/qa-suite-report.md", + summaryPath: "/qa-output/qa-suite-summary.json", + report: "", + scenarios: [ + { name: "pass", status: "pass", steps: [] }, + { name: "fail", status: "fail", steps: [] }, + { name: "skip", status: "skip", steps: [] }, + ], + watchUrl: "http://127.0.0.1:43123", + } satisfies QaSuiteResult; + + let thrown: unknown; + try { + qaSuiteProgressTesting.throwQaSuiteCleanupErrors({ + cleanupFailures: [ + { phase: "agent\nharnesses", error: new Error("dispose failed") }, + { phase: "lab stop", error: new Error("stop failed") }, + ], + runFailed: false, + runError: undefined, + result, + evidenceWritten: false, + }); + } catch (error) { + thrown = error; + } + + expect((thrown as Error).message).toBe( + [ + "QA scenarios completed, but cleanup failed", + "scenario counts: passed=1 failed=1 skipped=1 total=3", + "failed cleanup phases: agent harnesses: dispose failed; lab stop: stop failed", + "retained artifacts: output=/qa-output retained report=/qa-output/qa-suite-report.md summary=/qa-output/qa-suite-summary.json", + ].join("\n"), + ); + expect("cause" in (thrown as object)).toBe(false); + expect((thrown as Error).message).not.toContain("evidence="); }); it("does not release transport credentials when gateway teardown fails", async () => { @@ -91,7 +171,7 @@ describe("qa suite", () => { } }; - const errors = await qaSuiteProgressTesting.runQaFlowSuiteCleanupPlan({ + const failures = await qaSuiteProgressTesting.runQaFlowSuiteCleanupPlan({ cleanupTransportBeforeGatewayStop: step("transport before gateway"), cleanupTransportAfterGatewayStop: step("transport after gateway"), stopGateway: step("gateway", gatewayFailure), @@ -100,7 +180,7 @@ describe("qa suite", () => { }); expect(calls).toEqual(["transport before gateway", "gateway", "agent harnesses", "lab"]); - expect(errors).toEqual([gatewayFailure]); + expect(failures).toEqual([{ phase: "gateway stop", error: gatewayFailure }]); }); it("rejects unsupported transport ids before starting the lab", async () => { diff --git a/extensions/qa-lab/src/suite.ts b/extensions/qa-lab/src/suite.ts index af5f4fb2d619..f9df38084ddd 100644 --- a/extensions/qa-lab/src/suite.ts +++ b/extensions/qa-lab/src/suite.ts @@ -202,6 +202,16 @@ export function writeQaSuiteProgress(enabled: boolean, message: string) { process.stderr.write(`[qa-suite] ${message}\n`); } +const qaSuiteNestedRuns = new WeakSet(); + +export function markQaSuiteNestedRun(params: T): T { + qaSuiteNestedRuns.add(params); + return params; +} + +export const isQaSuiteNestedRun = (params: object | undefined) => + params !== undefined && qaSuiteNestedRuns.has(params); + export function formatQaSuiteRunStartProgress(params: { selectedScenarioCount: number; concurrency: number; @@ -270,16 +280,19 @@ export async function waitForQaLabReadyOrStopOwned(params: { } } -export async function runQaSuiteCleanupSteps(steps: ReadonlyArray<() => Promise>) { - const errors: unknown[] = []; +type QaSuiteCleanupStep = { phase: string; run: () => Promise }; +type QaSuiteCleanupFailure = { phase: string; error: unknown }; + +export async function runQaSuiteCleanupSteps(steps: readonly QaSuiteCleanupStep[]) { + const failures: QaSuiteCleanupFailure[] = []; for (const step of steps) { try { - await step(); + await step.run(); } catch (error) { - errors.push(error); + failures.push({ phase: step.phase, error }); } } - return errors; + return failures; } export async function runQaFlowSuiteCleanupPlan(params: { @@ -291,47 +304,81 @@ export async function runQaFlowSuiteCleanupPlan(params: { stopProvider?: () => Promise; finishLab: () => Promise; }) { - const errors = await runQaSuiteCleanupSteps([ - ...(params.closeWebSessions ? [params.closeWebSessions] : []), + const stopGateway = params.stopGateway; + let gatewayStopped = !stopGateway; + const stopGatewayAndMark = async () => { + await stopGateway?.(); + gatewayStopped = true; + }; + const cleanupTransportAfterGatewayStop = async () => { + if (gatewayStopped) { + await params.cleanupTransportAfterGatewayStop(); + } + }; + return runQaSuiteCleanupSteps([ + ...(params.closeWebSessions ? [{ phase: "web sessions", run: params.closeWebSessions }] : []), // Drain transport HTTP work before stopping the gateway; otherwise a completed suite can // emit an unhandled response-close rejection during delivery. - params.cleanupTransportBeforeGatewayStop, + { phase: "transport before gateway stop", run: params.cleanupTransportBeforeGatewayStop }, + ...(stopGateway ? [{ phase: "gateway stop", run: stopGatewayAndMark }] : []), + // Never release a credential-backed transport until gateway teardown proves + // that the isolated runtime reached its terminal boundary. + { phase: "transport after gateway stop", run: cleanupTransportAfterGatewayStop }, + { phase: "agent harnesses", run: params.disposeAgentHarnesses }, + ...(params.stopProvider ? [{ phase: "provider stop", run: params.stopProvider }] : []), + { phase: "lab finish", run: params.finishLab }, ]); - let gatewayStopped = !params.stopGateway; - if (params.stopGateway) { - const gatewayErrors = await runQaSuiteCleanupSteps([params.stopGateway]); - errors.push(...gatewayErrors); - gatewayStopped = gatewayErrors.length === 0; - } - errors.push( - ...(await runQaSuiteCleanupSteps([ - // Never release a credential-backed transport until gateway teardown proves - // that the isolated runtime reached its terminal boundary. - ...(gatewayStopped ? [params.cleanupTransportAfterGatewayStop] : []), - params.disposeAgentHarnesses, - ...(params.stopProvider ? [params.stopProvider] : []), - params.finishLab, - ])), - ); - return errors; } export function throwQaSuiteCleanupErrors(params: { - cleanupErrors: unknown[]; + cleanupFailures: readonly QaSuiteCleanupFailure[]; runFailed: boolean; runError: unknown; + result?: QaSuiteResult; + evidenceWritten?: boolean; }) { - if (params.cleanupErrors.length === 0) { + if (params.cleanupFailures.length === 0) { return; } - if (params.cleanupErrors.length === 1 && !params.runFailed) { - throw params.cleanupErrors[0]; + const result = params.result; + const scenarios = result?.scenarios ?? []; + const failed = scenarios.filter((scenario) => scenario.status === "fail").length; + const skipped = scenarios.filter((scenario) => scenario.status === "skip").length; + const passed = scenarios.length - failed - skipped; + const cleanupHeadline = !result + ? "QA suite cleanup failed before scenarios completed" + : failed === 0 && skipped === 0 + ? "QA scenarios passed, but cleanup failed" + : "QA scenarios completed, but cleanup failed"; + const message = [ + params.runFailed ? "QA suite and cleanup failed" : cleanupHeadline, + ...(result + ? [ + `scenario counts: passed=${passed} failed=${failed} skipped=${skipped} total=${scenarios.length}`, + ] + : params.runFailed + ? ["scenarios did not complete"] + : []), + `failed cleanup phases: ${params.cleanupFailures + .map( + ({ phase, error }) => + `${sanitizeQaSuiteProgressValue(phase)}: ${sanitizeQaSuiteProgressValue(formatErrorMessage(error))}`, + ) + .join("; ")}`, + ...(result + ? [ + `retained artifacts: output=${sanitizeQaSuiteProgressValue(result.outputDir)} report=${sanitizeQaSuiteProgressValue(result.reportPath)} summary=${sanitizeQaSuiteProgressValue(result.summaryPath)}${params.evidenceWritten ? ` evidence=${sanitizeQaSuiteProgressValue(result.evidencePath)}` : ""}`, + ] + : []), + ].join("\n"); + const errors = params.cleanupFailures.map((failure) => failure.error); + if (params.runFailed) { + throw new AggregateError([params.runError, ...errors], message, { cause: params.runError }); } - throw new AggregateError( - params.runFailed ? [params.runError, ...params.cleanupErrors] : params.cleanupErrors, - params.runFailed ? "QA suite and cleanup failed" : "QA suite cleanup failed", - params.runFailed ? { cause: params.runError } : undefined, - ); + if (errors.length === 1) { + throw new AggregateError(errors, message, { cause: errors[0] }); + } + throw new AggregateError(errors, message); } export function requireQaSuiteStartLab(startLab: QaSuiteStartLabFn | undefined): QaSuiteStartLabFn {