fix(qa): report cleanup failures truthfully (#119727)

* fix(qa): report cleanup failures truthfully Punchcard-Session: silver-valley-valley-dt * fix(qa): satisfy cleanup runner static checks Punchcard-Session: silver-meadow-lantern-vx
2026-08-12 21:53:00 -06:00 · 2026-08-06 05:16:35 +08:00
parent e9e6ef3edb
commit 7b8bba26c9
9 changed files with 736 additions and 116 deletions
@@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { QaSuiteInfraError } from "./errors.js";
 import type { QaLabServerHandle } from "./lab-server.types.js";
 import type { QaSuiteScenarioResult } from "./suite.js";
+import { throwQaSuiteCleanupErrors } from "./suite.js";
 import type {
  QaTestFileScenario,
  QaTestFileScenarioRunResult,
@@ -31,7 +32,7 @@ vi.mock("./test-file-scenario-runner.js", async (importOriginal) => ({
  runQaTestFileScenarios,
 }));

-import { runQaSuite } from "./suite-launch.runtime.js";
+import { runQaSuite, runQaSuiteWithInfraRetry } from "./suite-launch.runtime.js";

 const tempRoots: string[] = [];

@@ -251,6 +252,34 @@ describe("qa suite runtime launcher", () => {
    }
  });

+  it("retries a cleanup-only ECONNRESET through its preserved cause", async () => {
+    const cleanupError = Object.assign(new Error("cleanup socket reset"), {
+      code: "ECONNRESET",
+    });
+    const stderrWrite = vi.spyOn(process.stderr, "write").mockReturnValue(true);
+    let attempts = 0;
+
+    try {
+      const result = await runQaSuiteWithInfraRetry(async () => {
+        attempts += 1;
+        if (attempts === 1) {
+          throwQaSuiteCleanupErrors({
+            cleanupFailures: [{ phase: "lab stop", error: cleanupError }],
+            runFailed: false,
+            runError: undefined,
+          });
+        }
+        return "retried";
+      }, 1);
+
+      expect(result).toBe("retried");
+      expect(attempts).toBe(2);
+      expect(stderrWrite.mock.calls.flat().join("")).toContain("[qa-suite] infra retry 1/1:");
+    } finally {
+      stderrWrite.mockRestore();
+    }
+  });
+
  it("partitions flow-only suites that request isolated workers", async () => {
    const repoRoot = await makeTempRepo("qa-suite-flow-only-isolated-");
    const result = await runQaSuite({
@@ -1,10 +1,61 @@
-import { describe, expect, it, vi } from "vitest";
+import { beforeEach, describe, expect, it, vi } from "vitest";
 import { createQaBusState } from "./bus-state.js";
 import type { QaLabServerHandle } from "./lab-server.types.js";
 import type { QaTransportAdapterFactory } from "./qa-transport-registry.js";
 import { runQaFlowSuiteIsolated } from "./suite-run-isolated.js";
+import { runQaFlowSuiteStandard } from "./suite-run-standard.js";
 import { makeQaSuiteTestScenario } from "./suite-test-helpers.js";
-import type { QaSuiteResolvedRunContext, QaSuiteRunner } from "./suite-types.js";
+import type {
+  QaSuiteResolvedRunContext,
+  QaSuiteRunner,
+  QaSuiteScenarioRunner,
+} from "./suite-types.js";
+
+const mocks = vi.hoisted(() => ({
+  disposeRegisteredAgentHarnesses: vi.fn(async () => {}),
+  fetchWithSsrFGuard: vi.fn(async () => ({
+    response: new Response(null, { status: 204 }),
+    release: vi.fn(async () => {}),
+  })),
+  startQaGatewayChild: vi.fn(async () => ({
+    baseUrl: "http://127.0.0.1:18789",
+    token: "qa-test-token",
+    cfg: {},
+    getProcessCpuMs: () => null,
+    getProcessRssBytes: () => null,
+    stop: vi.fn(async () => {}),
+  })),
+  writeQaSuiteArtifacts: vi.fn(async () => ({
+    evidence: undefined,
+    evidencePath: "/qa-output/qa-evidence.json",
+    report: "",
+    reportPath: "/qa-output/qa-suite-report.md",
+    summaryPath: "/qa-output/qa-suite-summary.json",
+  })),
+}));
+
+vi.mock("openclaw/plugin-sdk/agent-harness", () => ({
+  disposeRegisteredAgentHarnesses: mocks.disposeRegisteredAgentHarnesses,
+}));
+vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
+  fetchWithSsrFGuard: mocks.fetchWithSsrFGuard,
+}));
+vi.mock("./gateway-child.js", () => ({
+  startQaGatewayChild: mocks.startQaGatewayChild,
+}));
+vi.mock("./providers/server-runtime.js", () => ({
+  startQaProviderServer: vi.fn(async () => undefined),
+}));
+vi.mock("./suite-artifacts.js", () => ({
+  writeQaSuiteArtifacts: mocks.writeQaSuiteArtifacts,
+}));
+vi.mock("./suite-runtime-gateway.js", () => ({
+  waitForGatewayHealthy: vi.fn(async () => {}),
+  waitForTransportReady: vi.fn(async () => {}),
+}));
+vi.mock("./web-runtime.js", () => ({
+  closeQaWebSessions: vi.fn(async () => {}),
+}));

 function createCleanupTestLab(): QaLabServerHandle {
  return {
@@ -41,6 +92,132 @@ function createCleanupTestContext(): QaSuiteResolvedRunContext {
 }

 describe("isolated QA suite transport cleanup", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mocks.disposeRegisteredAgentHarnesses.mockResolvedValue(undefined);
+  });
+
+  it("retains passing artifacts and finishes owned cleanup before reporting teardown failure", async () => {
+    const lab = createCleanupTestLab();
+    const release = vi.fn(async () => {});
+    const factory: QaTransportAdapterFactory = {
+      id: "leased",
+      matches: ({ channelId, driver }) => channelId === "leased" && driver === "live",
+      async create() {
+        return {
+          id: "leased",
+          label: "Leased channel",
+          accountId: "sut",
+          requiredPluginIds: [],
+          supportedActions: [],
+          sendInbound: async (input) => lab.state.addInboundMessage(input),
+          createGatewayConfig: () => ({}),
+          async waitReady() {},
+          buildAgentDelivery: ({ target }) => ({
+            channel: "leased",
+            to: target,
+            replyChannel: "leased",
+            replyTo: target,
+          }),
+          async handleAction() {},
+          createReportNotes: () => [],
+          cleanup: release,
+        };
+      },
+    };
+    const cleanupError = new Error("agent harness disposal failed");
+    mocks.disposeRegisteredAgentHarnesses.mockRejectedValueOnce(cleanupError);
+    const stderrWrite = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
+    const runChild = vi.fn<QaSuiteRunner>().mockResolvedValue({
+      outputDir: "/qa-child",
+      evidencePath: "/qa-child/qa-evidence.json",
+      reportPath: "/qa-child/qa-suite-report.md",
+      summaryPath: "/qa-child/qa-suite-summary.json",
+      report: "",
+      scenarios: [{ name: "leased-channel-scenario", status: "pass", steps: [] }],
+      watchUrl: lab.baseUrl,
+    });
+    const context = createCleanupTestContext();
+    context.progressEnabled = true;
+
+    const thrown = await runQaFlowSuiteIsolated(
+      {
+        adapterFactories: [factory],
+        channelDriver: "live",
+        channelId: "leased",
+        startLab: async () => lab,
+      },
+      context,
+      runChild,
+    ).catch((error: unknown) => error);
+
+    expect(release).toHaveBeenCalledOnce();
+    expect(mocks.disposeRegisteredAgentHarnesses).toHaveBeenCalledOnce();
+    expect(lab.stop).toHaveBeenCalledOnce();
+    expect(lab.setLatestReport).toHaveBeenCalledWith(
+      expect.objectContaining({ outputPath: "/qa-output/qa-suite-report.md" }),
+    );
+    expect((thrown as Error).message.split("\n")[0]).toBe(
+      "QA scenarios passed, but cleanup failed",
+    );
+    expect((thrown as Error).message).toContain(
+      "failed cleanup phases: agent harnesses: agent harness disposal failed",
+    );
+    expect((thrown as Error).message).toContain(
+      "retained artifacts: output=/qa-output report=/qa-output/qa-suite-report.md summary=/qa-output/qa-suite-summary.json",
+    );
+    expect((thrown as Error).cause).toBe(cleanupError);
+    expect(stderrWrite.mock.calls.flat().join("")).not.toContain("run complete");
+    stderrWrite.mockRestore();
+  });
+
+  it("prints one generic completion after a real nested standard run and parent cleanup", async () => {
+    const parentLab = createCleanupTestLab();
+    const childLab = createCleanupTestLab();
+    const startLab = vi
+      .fn<() => Promise<QaLabServerHandle>>()
+      .mockResolvedValueOnce(parentLab)
+      .mockResolvedValueOnce(childLab);
+    const context = createCleanupTestContext();
+    context.channelDriver = undefined;
+    context.progressEnabled = true;
+    const runScenario = vi
+      .fn<QaSuiteScenarioRunner>()
+      .mockResolvedValue({ name: "leased-channel-scenario", status: "pass", steps: [] });
+    const runChild: QaSuiteRunner = async (childParams) => {
+      if (!childParams) {
+        throw new Error("expected nested standard run params");
+      }
+      return await runQaFlowSuiteStandard(
+        childParams,
+        {
+          ...context,
+          startedAt: new Date("2026-08-04T00:00:01.000Z"),
+          outputDir: childParams.outputDir ?? "/qa-output/scenarios/leased-channel-scenario",
+          concurrency: 1,
+        },
+        runScenario,
+      );
+    };
+    const stderrWrite = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
+
+    try {
+      await runQaFlowSuiteIsolated({ startLab }, context, runChild);
+
+      const completionLines = stderrWrite.mock.calls
+        .flat()
+        .join("")
+        .split("\n")
+        .filter((line) => line.startsWith("[qa-suite] run complete"));
+      expect(completionLines).toEqual(["[qa-suite] run complete"]);
+      expect(runScenario).toHaveBeenCalledOnce();
+      expect(childLab.stop).toHaveBeenCalledOnce();
+      expect(parentLab.stop).toHaveBeenCalledOnce();
+    } finally {
+      stderrWrite.mockRestore();
+    }
+  });
+
  it.each(["cleanup", "cleanupAfterGatewayStop"] as const)(
    "retries a failed parent %s phase before disposing its owned lab",
    async (cleanupPhase) => {
@@ -15,6 +15,7 @@ import type {
 } from "./suite-types.js";
 import {
  createQaSuiteTransportAdapter,
+  markQaSuiteNestedRun,
  requireQaSuiteStartLab,
  runQaSuiteCleanupSteps,
  throwQaSuiteCleanupErrors,
@@ -135,6 +136,9 @@ export async function runQaFlowSuiteIsolated(
  let isolatedRunFailed = false;
  let isolatedRunError: unknown;
  let parentTransportCleaned = false;
+  let result: QaSuiteResult | undefined;
+  let completionProgress: string | undefined;
+  let evidenceWritten = false;
  try {
    if (params?.channelDriver === "live") {
      // The parent only renders aggregate artifacts. Release its live credentials
@@ -164,7 +168,8 @@ export async function runQaFlowSuiteIsolated(
        updateScenarioRun();
        try {
          const scenarioOutputDir = path.join(outputDir, "scenarios", scenario.id);
-          const result: QaSuiteResult = await runQaFlowSuite(
+          const childSuiteResult: QaSuiteResult = await runQaFlowSuite(
+            markQaSuiteNestedRun(
              buildQaIsolatedScenarioWorkerParams({
                repoRoot,
                outputDir: scenarioOutputDir,
@@ -179,9 +184,10 @@ export async function runQaFlowSuiteIsolated(
                scenario,
                input: params,
              }),
+            ),
          );
          const scenarioResult: QaSuiteScenarioResult =
-            result.scenarios[0] ??
+            childSuiteResult.scenarios[0] ??
            ({
              name: scenario.title,
              status: "fail",
@@ -246,13 +252,12 @@ export async function runQaFlowSuiteIsolated(
      },
      {
        startStaggerMs: workerStartStaggerMs,
-        shouldStop: (result) => params?.failFast === true && result.status === "fail",
+        shouldStop: (scenarioResult) =>
+          params?.failFast === true && scenarioResult.status === "fail",
      },
    );
    await artifactWriteQueue;
    const finishedAt = new Date();
-    const failedCount = scenarios.filter((scenario) => scenario.status === "fail").length;
-    const skippedCount = scenarios.filter((scenario) => scenario.status === "skip").length;
    lab.setScenarioRun({
      kind: "suite",
      status: "completed",
@@ -296,11 +301,9 @@ export async function runQaFlowSuiteIsolated(
      markdown: report,
      generatedAt: finishedAt.toISOString(),
    } satisfies QaLabLatestReport);
-    writeQaSuiteProgress(
-      progressEnabled,
-      `run complete: passed=${scenarios.length - failedCount - skippedCount} failed=${failedCount} skipped=${skippedCount} total=${scenarios.length}`,
-    );
-    return {
+    completionProgress = "run complete";
+    evidenceWritten = evidence !== undefined && (params?.writeEvidenceFile ?? true);
+    result = {
      outputDir,
      evidence,
      evidencePath,
@@ -315,18 +318,27 @@ export async function runQaFlowSuiteIsolated(
    isolatedRunError = error;
    throw error;
  } finally {
-    const cleanupSteps: Array<() => Promise<void>> = [
-      ...(!parentTransportCleaned ? [() => transportFactoryResult.cleanupWithoutGateway()] : []),
-      () => disposeRegisteredAgentHarnesses(),
+    const cleanupSteps = [
+      ...(!parentTransportCleaned
+        ? [{ phase: "parent transport", run: () => transportFactoryResult.cleanupWithoutGateway() }]
+        : []),
+      { phase: "agent harnesses", run: () => disposeRegisteredAgentHarnesses() },
    ];
    if (ownsLab) {
-      cleanupSteps.push(() => lab.stop());
+      cleanupSteps.push({ phase: "lab stop", run: () => lab.stop() });
    }
-    const cleanupErrors = await runQaSuiteCleanupSteps(cleanupSteps);
+    const cleanupFailures = await runQaSuiteCleanupSteps(cleanupSteps);
    throwQaSuiteCleanupErrors({
-      cleanupErrors,
+      cleanupFailures,
      runFailed: isolatedRunFailed,
      runError: isolatedRunError,
+      result,
+      evidenceWritten,
    });
  }
+  if (!result || !completionProgress) {
+    throw new Error("QA suite completed without terminal result metadata");
+  }
+  writeQaSuiteProgress(progressEnabled, completionProgress);
+  return result;
 }
@@ -7,6 +7,7 @@ import type {
  QaSuiteScenarioResult,
  QaSuiteScenarioRunner,
 } from "./suite-types.js";
+import type { runQaFlowSuiteCleanupPlan } from "./suite.js";

 const mocks = vi.hoisted(() => ({
  captureRuntimeParityCell: vi.fn(async (params: { runtime: "codex"; wallClockMs: number }) => ({
@@ -44,6 +45,8 @@ const mocks = vi.hoisted(() => ({
  })),
  waitForGatewayHealthy: vi.fn(async () => {}),
  waitForTransportReady: vi.fn(async () => {}),
+  runQaFlowSuiteCleanupPlan: vi.fn<typeof runQaFlowSuiteCleanupPlan>(async () => []),
+  writeQaSuiteProgress: vi.fn(),
 }));

 vi.mock("openclaw/plugin-sdk/agent-harness", () => ({
@@ -65,7 +68,8 @@ vi.mock("./suite-runtime-gateway.js", () => ({
  waitForGatewayHealthy: mocks.waitForGatewayHealthy,
  waitForTransportReady: mocks.waitForTransportReady,
 }));
-vi.mock("./suite.js", () => ({
+vi.mock("./suite.js", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("./suite.js")>()),
  buildQaSuiteRuntimeMetrics: vi.fn(() => ({ wallMs: 1 })),
  captureGatewayHeapSnapshotCheckpoint: vi.fn(async () => undefined),
  createQaSuiteTransportAdapter: vi.fn(async () => ({
@@ -75,10 +79,9 @@ vi.mock("./suite.js", () => ({
  })),
  requireQaSuiteStartLab: vi.fn(),
  resolveQaSuiteTransportReadyTimeoutMs: vi.fn(() => 1_000),
-  runQaFlowSuiteCleanupPlan: vi.fn(async () => []),
-  throwQaSuiteCleanupErrors: vi.fn(),
+  runQaFlowSuiteCleanupPlan: mocks.runQaFlowSuiteCleanupPlan,
  waitForQaLabReadyOrStopOwned: vi.fn(async () => {}),
-  writeQaSuiteProgress: vi.fn(),
+  writeQaSuiteProgress: mocks.writeQaSuiteProgress,
 }));
 vi.mock("./web-runtime.js", () => ({
  closeQaWebSessions: vi.fn(async () => {}),
@@ -128,6 +131,7 @@ function makeRetryTestResult(status: "pass" | "fail"): QaSuiteScenarioResult {

 beforeEach(() => {
  vi.clearAllMocks();
+  mocks.runQaFlowSuiteCleanupPlan.mockResolvedValue([]);
 });

 describe("QA suite Control UI ownership", () => {
@@ -196,6 +200,45 @@ describe("QA suite Control UI ownership", () => {
 });

 describe("QA runtime parity scenario retry isolation", () => {
+  it("does not report terminal success when cleanup fails after writing artifacts", async () => {
+    const lab = makeRetryTestLab();
+    const cleanupError = Object.assign(new Error("gateway shutdown socket reset"), {
+      code: "ECONNRESET",
+    });
+    mocks.runQaFlowSuiteCleanupPlan.mockResolvedValueOnce([
+      { phase: "gateway stop", error: cleanupError },
+    ]);
+
+    const thrown = await runQaFlowSuiteStandard(
+      { lab },
+      makeRetryTestContext(),
+      vi.fn<QaSuiteScenarioRunner>().mockResolvedValue(makeRetryTestResult("pass")),
+    ).catch((error: unknown) => error);
+
+    expect(thrown).toBeInstanceOf(AggregateError);
+    expect((thrown as Error).message.split("\n")[0]).toBe(
+      "QA scenarios passed, but cleanup failed",
+    );
+    expect((thrown as Error).message).toContain(
+      "scenario counts: passed=1 failed=0 skipped=0 total=1",
+    );
+    expect((thrown as Error).message).toContain(
+      "failed cleanup phases: gateway stop: gateway shutdown socket reset",
+    );
+    expect((thrown as Error).message).toContain(
+      "retained artifacts: output=/qa-output report=/qa-output/qa-suite-report.md summary=/qa-output/qa-suite-summary.json",
+    );
+    expect((thrown as Error).cause).toBe(cleanupError);
+    expect(lab.setLatestReport).toHaveBeenCalledWith(
+      expect.objectContaining({ outputPath: "/qa-output/qa-suite-report.md" }),
+    );
+    expect(
+      mocks.writeQaSuiteProgress.mock.calls.filter(([, message]) =>
+        String(message).startsWith("run complete"),
+      ),
+    ).toHaveLength(0);
+  });
+
  it.each([
    { forcedRuntime: undefined, expectedRuntime: "openclaw" },
    { forcedRuntime: "codex" as const, expectedRuntime: "codex" },
@@ -39,6 +39,7 @@ import {
  createQaSuiteTransportAdapter,
  buildQaSuiteRuntimeMetrics,
  captureGatewayHeapSnapshotCheckpoint,
+  isQaSuiteNestedRun,
  requireQaSuiteStartLab,
  resolveQaSuiteTransportReadyTimeoutMs,
  runQaFlowSuiteCleanupPlan,
@@ -107,6 +108,9 @@ export async function runQaFlowSuiteStandard(
  let preserveGatewayRuntimeDir: string | undefined;
  let runFailed = false;
  let runError: unknown;
+  let result: QaSuiteResult | undefined;
+  let completionProgress: string | undefined;
+  let evidenceWritten = false;
  try {
    writeQaSuiteProgress(progressEnabled, `provider start: ${providerMode}`);
    const activeMock = await startQaProviderServer(providerMode, {
@@ -267,7 +271,7 @@ export async function runQaFlowSuiteStandard(
      };
      const scenarioRetryCount =
        scenario.execution.kind === "flow" ? scenario.execution.retryCount : undefined;
-      let result: QaSuiteScenarioResult =
+      let scenarioResult: QaSuiteScenarioResult =
        params?.captureRuntimeParityCell || scenarioRetryCount === 0
          ? await runSelectedScenario()
          : await runQaScenarioWithFlakeRetry(runSelectedScenario, () =>
@@ -276,19 +280,19 @@ export async function runQaFlowSuiteStandard(
                `scenario retry (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`,
              ),
            );
-      if (result.status === "pass" && params?.roundTripProbe?.scenarioId === scenario.id) {
+      if (scenarioResult.status === "pass" && params?.roundTripProbe?.scenarioId === scenario.id) {
        const probeResult = await runQaSuiteRoundTripProbe({
          probe: params.roundTripProbe,
          transport,
        });
        const probePassed = probeResult.passed >= params.roundTripProbe.count;
-        result = {
-          ...result,
+        scenarioResult = {
+          ...scenarioResult,
          status: probePassed ? "pass" : "fail",
-          details: [result.details, probeResult.details].filter(Boolean).join(" | "),
+          details: [scenarioResult.details, probeResult.details].filter(Boolean).join(" | "),
          timing: probeResult.timing,
          steps: [
-            ...result.steps,
+            ...scenarioResult.steps,
            {
              name: "Round-trip samples",
              status: probePassed ? "pass" : "fail",
@@ -306,17 +310,17 @@ export async function runQaFlowSuiteStandard(
        });
      }
      sampleGatewayProcessRss(`scenario:${scenario.id}:finish`);
-      scenarios.push(result);
+      scenarios.push(scenarioResult);
      writeQaSuiteProgress(
        progressEnabled,
-        `scenario ${result.status} (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`,
+        `scenario ${scenarioResult.status} (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`,
      );
      liveScenarioOutcomes[index] = {
        id: scenario.id,
        name: scenario.title,
-        status: result.status,
-        details: result.details,
-        steps: result.steps,
+        status: scenarioResult.status,
+        details: scenarioResult.details,
+        steps: scenarioResult.steps,
        startedAt: liveScenarioOutcomes[index]?.startedAt,
        finishedAt: new Date().toISOString(),
      };
@@ -326,7 +330,7 @@ export async function runQaFlowSuiteStandard(
        startedAt: startedAt.toISOString(),
        scenarios: [...liveScenarioOutcomes],
      });
-      if (params?.failFast === true && result.status === "fail") {
+      if (params?.failFast === true && scenarioResult.status === "fail") {
        break;
      }
    }
@@ -407,12 +411,9 @@ export async function runQaFlowSuiteStandard(
      generatedAt: finishedAt.toISOString(),
    } satisfies QaLabLatestReport;
    lab.setLatestReport(latestReport);
-    writeQaSuiteProgress(
-      progressEnabled,
-      `run complete: passed=${scenarios.length - failedCount - skippedCount} failed=${failedCount} skipped=${skippedCount} total=${scenarios.length}`,
-    );
-
-    return {
+    completionProgress = `run complete: passed=${scenarios.length - failedCount - skippedCount} failed=${failedCount} skipped=${skippedCount} total=${scenarios.length}`;
+    evidenceWritten = evidence !== undefined && (params?.writeEvidenceFile ?? true);
+    result = {
      outputDir,
      evidence,
      evidencePath,
@@ -433,7 +434,7 @@ export async function runQaFlowSuiteStandard(
    const keepTemp = process.env.OPENCLAW_QA_KEEP_TEMP === "1" || false;
    const activeGateway = gateway;
    const activeMock = mock;
-    const cleanupErrors = await runQaFlowSuiteCleanupPlan({
+    const cleanupFailures = await runQaFlowSuiteCleanupPlan({
      closeWebSessions: activeEnv ? () => closeQaWebSessions(activeEnv.webSessionIds) : undefined,
      cleanupTransportBeforeGatewayStop: () => transportFactoryResult.cleanupBeforeGatewayStop(),
      cleanupTransportAfterGatewayStop: () => transportFactoryResult.cleanupAfterGatewayStop(),
@@ -457,6 +458,13 @@ export async function runQaFlowSuiteStandard(
            }
          },
    });
-    throwQaSuiteCleanupErrors({ cleanupErrors, runFailed, runError });
+    throwQaSuiteCleanupErrors({ cleanupFailures, runFailed, runError, result, evidenceWritten });
  }
+  if (!result || !completionProgress) {
+    throw new Error("QA suite completed without terminal result metadata");
+  }
+  if (!params?.captureRuntimeParityCell && !isQaSuiteNestedRun(params)) {
+    writeQaSuiteProgress(progressEnabled, completionProgress);
+  }
+  return result;
 }
@@ -5,9 +5,84 @@ import {
  createQaTransportAdapter,
  type QaTransportAdapterFactory,
 } from "./qa-transport-registry.js";
+import { runQaFlowSuiteStandard } from "./suite-run-standard.js";
 import { runQaRuntimeParitySuite } from "./suite-runtime-parity-runner.js";
 import { makeQaSuiteTestScenario } from "./suite-test-helpers.js";
-import type { QaSuiteRunner } from "./suite-types.js";
+import type {
+  QaSuiteResolvedRunContext,
+  QaSuiteRunner,
+  QaSuiteScenarioRunner,
+} from "./suite-types.js";
+
+const mocks = vi.hoisted(() => ({
+  captureRuntimeParityCell: vi.fn(
+    async (params: { runtime: "openclaw" | "codex"; wallClockMs: number }) => ({
+      runtime: params.runtime,
+      transcriptBytes: "",
+      toolCalls: [],
+      finalText: "ok",
+      usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+      cacheDiagnostics: {
+        assistantTurns: 1,
+        cacheTelemetryTurns: 1,
+        cacheHitTurns: 0,
+        cacheWriteTurns: 0,
+        cacheMisses: [],
+        cacheMissInputTokens: 0,
+        unmeasuredPostWarmTurns: [],
+      },
+      wallClockMs: params.wallClockMs,
+      bootStateLines: [],
+    }),
+  ),
+  disposeRegisteredAgentHarnesses: vi.fn(async () => {}),
+  fetchWithSsrFGuard: vi.fn(async () => ({
+    response: new Response(null, { status: 204 }),
+    release: vi.fn(async () => {}),
+  })),
+  startQaGatewayChild: vi.fn(async () => ({
+    baseUrl: "http://127.0.0.1:18789",
+    token: "qa-test-token",
+    cfg: {},
+    getProcessCpuMs: () => null,
+    getProcessRssBytes: () => null,
+    stop: vi.fn(async () => {}),
+  })),
+  writeQaSuiteArtifacts: vi.fn(async () => ({
+    evidence: { kind: "test" },
+    evidencePath: "/qa-output/qa-evidence.json",
+    report: "",
+    reportPath: "/qa-output/qa-suite-report.md",
+    summaryPath: "/qa-output/qa-suite-summary.json",
+  })),
+}));
+
+vi.mock("openclaw/plugin-sdk/agent-harness", () => ({
+  disposeRegisteredAgentHarnesses: mocks.disposeRegisteredAgentHarnesses,
+}));
+vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
+  fetchWithSsrFGuard: mocks.fetchWithSsrFGuard,
+}));
+vi.mock("./gateway-child.js", () => ({
+  startQaGatewayChild: mocks.startQaGatewayChild,
+}));
+vi.mock("./providers/server-runtime.js", () => ({
+  startQaProviderServer: vi.fn(async () => undefined),
+}));
+vi.mock("./runtime-parity.js", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("./runtime-parity.js")>()),
+  captureRuntimeParityCell: mocks.captureRuntimeParityCell,
+}));
+vi.mock("./suite-artifacts.js", () => ({
+  writeQaSuiteArtifacts: mocks.writeQaSuiteArtifacts,
+}));
+vi.mock("./suite-runtime-gateway.js", () => ({
+  waitForGatewayHealthy: vi.fn(async () => {}),
+  waitForTransportReady: vi.fn(async () => {}),
+}));
+vi.mock("./web-runtime.js", () => ({
+  closeQaWebSessions: vi.fn(async () => {}),
+}));

 function createCleanupTestLab(): QaLabServerHandle {
  return {
@@ -62,6 +137,7 @@ function createCleanupTestFactory(
 function runCleanupTestSuite(params: {
  factory: QaTransportAdapterFactory;
  lab: QaLabServerHandle;
+  progressEnabled?: boolean;
  runChild: QaSuiteRunner;
 }) {
  return runQaRuntimeParitySuite({
@@ -80,12 +156,148 @@ function runCleanupTestSuite(params: {
    concurrency: 1,
    selectedScenarios: [makeQaSuiteTestScenario("runtime-cleanup")],
    startLab: async () => params.lab,
-    progressEnabled: false,
+    progressEnabled: params.progressEnabled ?? false,
    runtimePair: ["openclaw", "codex"],
  });
 }

 describe("runtime parity suite transport cleanup", () => {
+  it("keeps parent artifacts discoverable when owned lab cleanup fails", async () => {
+    const cleanupError = Object.assign(new Error("owned lab shutdown reset"), {
+      code: "ECONNRESET",
+    });
+    const setLatestReport = vi.fn<QaLabServerHandle["setLatestReport"]>();
+    const stopLab = vi.fn<QaLabServerHandle["stop"]>(async () => {
+      throw cleanupError;
+    });
+    const lab = createCleanupTestLab();
+    lab.setLatestReport = setLatestReport;
+    lab.stop = stopLab;
+    const cleanup = vi.fn(async () => {});
+    const factory = createCleanupTestFactory(lab, () => ({ cleanup }));
+    const stderrWrite = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
+    const runChild = vi.fn<QaSuiteRunner>().mockImplementation(async (params) => ({
+      outputDir: "/qa-child",
+      evidencePath: "/qa-child/qa-evidence.json",
+      reportPath: "/qa-child/qa-suite-report.md",
+      summaryPath: "/qa-child/qa-suite-summary.json",
+      report: "",
+      scenarios: [{ name: "runtime-cleanup", status: "pass", steps: [] }],
+      watchUrl: lab.baseUrl,
+      runtimeParityCell: {
+        runtime: params?.forcedRuntime ?? "openclaw",
+        transcriptBytes: "",
+        toolCalls: [],
+        finalText: "ok",
+        usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+        wallClockMs: 1,
+        bootStateLines: [],
+      },
+    }));
+
+    try {
+      const thrown = await runCleanupTestSuite({
+        factory,
+        lab,
+        progressEnabled: true,
+        runChild,
+      }).catch((error: unknown) => error);
+
+      expect(cleanup).toHaveBeenCalledOnce();
+      expect(setLatestReport).toHaveBeenCalledWith(
+        expect.objectContaining({ outputPath: "/qa-output/qa-suite-report.md" }),
+      );
+      expect(setLatestReport.mock.invocationCallOrder[0]).toBeLessThan(
+        stopLab.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY,
+      );
+      expect((thrown as Error).message.split("\n")[0]).toBe(
+        "QA scenarios passed, but cleanup failed",
+      );
+      expect((thrown as Error).message).toContain(
+        "failed cleanup phases: lab stop: owned lab shutdown reset",
+      );
+      expect((thrown as Error).message).toContain(
+        "retained artifacts: output=/qa-output report=/qa-output/qa-suite-report.md summary=/qa-output/qa-suite-summary.json evidence=/qa-output/qa-evidence.json",
+      );
+      expect((thrown as Error).cause).toBe(cleanupError);
+      expect(stderrWrite.mock.calls.flat().join("")).not.toContain("run complete");
+    } finally {
+      stderrWrite.mockRestore();
+    }
+  });
+
+  it("prints one generic completion after real nested standard cells and parent cleanup", async () => {
+    const scenario = makeQaSuiteTestScenario("runtime-cleanup");
+    const parentLab = createCleanupTestLab();
+    const openClawLab = createCleanupTestLab();
+    const codexLab = createCleanupTestLab();
+    const startLab = vi
+      .fn<() => Promise<QaLabServerHandle>>()
+      .mockResolvedValueOnce(parentLab)
+      .mockResolvedValueOnce(openClawLab)
+      .mockResolvedValueOnce(codexLab);
+    const runScenario = vi
+      .fn<QaSuiteScenarioRunner>()
+      .mockResolvedValue({ name: scenario.title, status: "pass", steps: [] });
+    const runChild: QaSuiteRunner = async (childParams) => {
+      if (!childParams) {
+        throw new Error("expected nested standard run params");
+      }
+      const context: QaSuiteResolvedRunContext = {
+        startedAt: new Date("2026-08-04T00:00:01.000Z"),
+        repoRoot: childParams.repoRoot ?? "/qa-repo",
+        outputDir: childParams.outputDir ?? "/qa-output/runtime-cell",
+        transportId: childParams.transportId ?? "qa-channel",
+        selectedScenarios: [scenario],
+        providerMode: childParams.providerMode ?? "mock-openai",
+        primaryModel: childParams.primaryModel ?? "mock-openai/test-model",
+        alternateModel: childParams.alternateModel ?? "mock-openai/test-model-alt",
+        fastMode: childParams.fastMode ?? true,
+        channelDriver: childParams.channelDriver,
+        enabledPluginIds: childParams.enabledPluginIds ?? [],
+        gatewayConfigPatch: undefined,
+        gatewayRuntimeOptions: undefined,
+        concurrency: 1,
+        progressEnabled: true,
+        gatewayHeapCheckpointsEnabled: false,
+      };
+      return await runQaFlowSuiteStandard(childParams, context, runScenario);
+    };
+    const stderrWrite = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
+
+    try {
+      await runQaRuntimeParitySuite({
+        runQaFlowSuite: runChild,
+        repoRoot: "/qa-repo",
+        outputDir: "/qa-output",
+        startedAt: new Date("2026-08-04T00:00:00.000Z"),
+        providerMode: "mock-openai",
+        transportId: "qa-channel",
+        primaryModel: "mock-openai/test-model",
+        alternateModel: "mock-openai/test-model-alt",
+        fastMode: true,
+        concurrency: 1,
+        selectedScenarios: [scenario],
+        startLab,
+        progressEnabled: true,
+        runtimePair: ["openclaw", "codex"],
+      });
+
+      const completionLines = stderrWrite.mock.calls
+        .flat()
+        .join("")
+        .split("\n")
+        .filter((line) => line.startsWith("[qa-suite] run complete"));
+      expect(completionLines).toEqual(["[qa-suite] run complete"]);
+      expect(runScenario).toHaveBeenCalledTimes(2);
+      expect(openClawLab.stop).toHaveBeenCalledOnce();
+      expect(codexLab.stop).toHaveBeenCalledOnce();
+      expect(parentLab.stop).toHaveBeenCalledOnce();
+    } finally {
+      stderrWrite.mockRestore();
+    }
+  });
+
  it("preserves the scenario error when its owned lab cleanup fails", async () => {
    const lab = createCleanupTestLab();
    const scenarioError = new Error("runtime scenario failed");
@@ -98,7 +310,9 @@ describe("runtime parity suite transport cleanup", () => {
    const runChild = vi.fn<QaSuiteRunner>().mockRejectedValueOnce(scenarioError);

    await expect(runCleanupTestSuite({ factory, lab, runChild })).rejects.toMatchObject({
-      message: "QA suite and cleanup failed",
+      message: expect.stringContaining(
+        "failed cleanup phases: lab stop: owned lab shutdown failed",
+      ),
      cause: scenarioError,
      errors: [scenarioError, cleanupError],
    });
@@ -108,6 +108,8 @@ export async function runQaRuntimeParitySuite(params: {
  let runFailed = false;
  let runError: unknown;
  let parentTransportCleaned = false;
+  let result: QaSuiteResult | undefined;
+  let evidenceWritten = false;
  try {
    if (params.channelDriver === "live") {
      // The parent only contributes aggregate metadata; release its exclusive
@@ -217,16 +219,16 @@ export async function runQaRuntimeParitySuite(params: {
          },
        });

-        const result = buildRuntimeParityScenarioResult({
+        const parityScenarioResult = buildRuntimeParityScenarioResult({
          scenarioName: scenario.title,
          result: parity,
        });
        liveScenarioOutcomes[index] = {
          id: scenario.id,
          name: scenario.title,
-          status: result.status,
-          details: result.details,
-          steps: result.steps,
+          status: parityScenarioResult.status,
+          details: parityScenarioResult.details,
+          steps: parityScenarioResult.steps,
          startedAt: liveScenarioOutcomes[index]?.startedAt,
          finishedAt: new Date().toISOString(),
        };
@@ -238,9 +240,9 @@ export async function runQaRuntimeParitySuite(params: {
        });
        writeQaSuiteProgress(
          params.progressEnabled,
-          `runtime pair ${result.status} (${index + 1}/${params.selectedScenarios.length}): ${scenarioIdForLog}`,
+          `runtime pair ${parityScenarioResult.status} (${index + 1}/${params.selectedScenarios.length}): ${scenarioIdForLog}`,
        );
-        return result;
+        return parityScenarioResult;
      },
      {
        startStaggerMs: resolveQaSuiteWorkerStartStaggerMs(params.concurrency),
@@ -285,7 +287,8 @@ export async function runQaRuntimeParitySuite(params: {
      finishedAt: finishedAt.toISOString(),
      scenarios: [...liveScenarioOutcomes],
    });
-    return {
+    evidenceWritten = evidence !== undefined && (params.writeEvidenceFile ?? true);
+    result = {
      outputDir: params.outputDir,
      evidence,
      evidencePath,
@@ -300,10 +303,17 @@ export async function runQaRuntimeParitySuite(params: {
    runError = error;
    throw error;
  } finally {
-    const cleanupErrors = await runQaSuiteCleanupSteps([
-      ...(!parentTransportCleaned ? [() => transportFactoryResult.cleanupWithoutGateway()] : []),
-      ...(ownsLab ? [() => lab.stop()] : []),
+    const cleanupFailures = await runQaSuiteCleanupSteps([
+      ...(!parentTransportCleaned
+        ? [{ phase: "parent transport", run: () => transportFactoryResult.cleanupWithoutGateway() }]
+        : []),
+      ...(ownsLab ? [{ phase: "lab stop", run: () => lab.stop() }] : []),
    ]);
-    throwQaSuiteCleanupErrors({ cleanupErrors, runFailed, runError });
+    throwQaSuiteCleanupErrors({ cleanupFailures, runFailed, runError, result, evidenceWritten });
  }
+  if (!result) {
+    throw new Error("QA runtime parity suite completed without a result");
+  }
+  writeQaSuiteProgress(params.progressEnabled, "run complete");
+  return result;
 }
@@ -7,6 +7,7 @@ import { QA_EVIDENCE_FILENAME, QA_EVIDENCE_SUMMARY_KIND } from "./evidence-summa
 import type { QaLabServerHandle } from "./lab-server.types.js";
 import type { QaTransportAdapter } from "./qa-transport.js";
 import { makeQaSuiteTestScenario } from "./suite-test-helpers.js";
+import type { QaSuiteResult } from "./suite-types.js";
 import { qaSuiteProgressTesting, runQaFlowSuite } from "./suite.js";
 import { createTempDirHarness } from "./temp-dir.test-helper.js";

@@ -39,7 +40,8 @@ function makeQaSuiteTestLabHandle(): QaLabServerHandle {
 describe("qa suite", () => {
  it("runs the production cleanup plan in dependency order after a failure", async () => {
    const calls: string[] = [];
-    const failure = new Error("transport close failed");
+    const transportFailure = new Error("transport close failed");
+    const providerFailure = new Error("provider close failed");
    const step = (name: string, error?: Error) => async () => {
      calls.push(name);
      if (error) {
@@ -47,13 +49,13 @@ describe("qa suite", () => {
      }
    };

-    const errors = await qaSuiteProgressTesting.runQaFlowSuiteCleanupPlan({
+    const failures = await qaSuiteProgressTesting.runQaFlowSuiteCleanupPlan({
      closeWebSessions: step("web sessions"),
-      cleanupTransportBeforeGatewayStop: step("transport before gateway", failure),
+      cleanupTransportBeforeGatewayStop: step("transport before gateway", transportFailure),
      cleanupTransportAfterGatewayStop: step("transport after gateway"),
      stopGateway: step("gateway"),
      disposeAgentHarnesses: step("agent harnesses"),
-      stopProvider: step("provider"),
+      stopProvider: step("provider", providerFailure),
      finishLab: step("lab"),
    });

@@ -66,19 +68,97 @@ describe("qa suite", () => {
      "provider",
      "lab",
    ]);
-    expect(errors).toEqual([failure]);
+    expect(failures).toEqual([
+      { phase: "transport before gateway stop", error: transportFailure },
+      { phase: "provider stop", error: providerFailure },
+    ]);
  });

  it("keeps the primary suite error as the cause of aggregated cleanup failures", () => {
    const runError = new Error("gateway infrastructure failed");
+    const cleanupError = new Error("transport cleanup failed");

-    expect(() =>
+    let thrown: unknown;
+    try {
      qaSuiteProgressTesting.throwQaSuiteCleanupErrors({
-        cleanupErrors: [new Error("transport cleanup failed")],
+        cleanupFailures: [{ phase: "transport before gateway stop", error: cleanupError }],
        runFailed: true,
        runError,
-      }),
-    ).toThrow(expect.objectContaining({ cause: runError }));
+      });
+    } catch (error) {
+      thrown = error;
+    }
+
+    expect(thrown).toMatchObject({
+      cause: runError,
+      errors: [runError, cleanupError],
+    });
+    expect((thrown as Error).message.split("\n")[0]).toBe("QA suite and cleanup failed");
+    expect((thrown as Error).message).toContain(
+      "failed cleanup phases: transport before gateway stop: transport cleanup failed",
+    );
+  });
+
+  it("reports cleanup failure before scenarios completed when no result exists", () => {
+    const cleanupError = new Error("stop failed");
+    let thrown: unknown;
+    try {
+      qaSuiteProgressTesting.throwQaSuiteCleanupErrors({
+        cleanupFailures: [{ phase: "lab stop", error: cleanupError }],
+        runFailed: false,
+        runError: undefined,
+      });
+    } catch (error) {
+      thrown = error;
+    }
+
+    expect((thrown as Error).message.split("\n")[0]).toBe(
+      "QA suite cleanup failed before scenarios completed",
+    );
+    expect((thrown as Error).cause).toBe(cleanupError);
+  });
+
+  it("reports completed counts, labeled failures, and only written artifact paths", () => {
+    const result = {
+      outputDir: "/qa-output\nretained",
+      evidencePath: "/qa-output/qa-evidence.json",
+      reportPath: "/qa-output/qa-suite-report.md",
+      summaryPath: "/qa-output/qa-suite-summary.json",
+      report: "",
+      scenarios: [
+        { name: "pass", status: "pass", steps: [] },
+        { name: "fail", status: "fail", steps: [] },
+        { name: "skip", status: "skip", steps: [] },
+      ],
+      watchUrl: "http://127.0.0.1:43123",
+    } satisfies QaSuiteResult;
+
+    let thrown: unknown;
+    try {
+      qaSuiteProgressTesting.throwQaSuiteCleanupErrors({
+        cleanupFailures: [
+          { phase: "agent\nharnesses", error: new Error("dispose failed") },
+          { phase: "lab stop", error: new Error("stop failed") },
+        ],
+        runFailed: false,
+        runError: undefined,
+        result,
+        evidenceWritten: false,
+      });
+    } catch (error) {
+      thrown = error;
+    }
+
+    expect((thrown as Error).message).toBe(
+      [
+        "QA scenarios completed, but cleanup failed",
+        "scenario counts: passed=1 failed=1 skipped=1 total=3",
+        "failed cleanup phases: agent harnesses: dispose failed; lab stop: stop failed",
+        "retained artifacts: output=/qa-output retained report=/qa-output/qa-suite-report.md summary=/qa-output/qa-suite-summary.json",
+      ].join("\n"),
+    );
+    expect("cause" in (thrown as object)).toBe(false);
+    expect((thrown as Error).message).not.toContain("evidence=");
  });

  it("does not release transport credentials when gateway teardown fails", async () => {
@@ -91,7 +171,7 @@ describe("qa suite", () => {
      }
    };

-    const errors = await qaSuiteProgressTesting.runQaFlowSuiteCleanupPlan({
+    const failures = await qaSuiteProgressTesting.runQaFlowSuiteCleanupPlan({
      cleanupTransportBeforeGatewayStop: step("transport before gateway"),
      cleanupTransportAfterGatewayStop: step("transport after gateway"),
      stopGateway: step("gateway", gatewayFailure),
@@ -100,7 +180,7 @@ describe("qa suite", () => {
    });

    expect(calls).toEqual(["transport before gateway", "gateway", "agent harnesses", "lab"]);
-    expect(errors).toEqual([gatewayFailure]);
+    expect(failures).toEqual([{ phase: "gateway stop", error: gatewayFailure }]);
  });

  it("rejects unsupported transport ids before starting the lab", async () => {
@@ -202,6 +202,16 @@ export function writeQaSuiteProgress(enabled: boolean, message: string) {
  process.stderr.write(`[qa-suite] ${message}\n`);
 }

+const qaSuiteNestedRuns = new WeakSet<object>();
+
+export function markQaSuiteNestedRun<T extends object>(params: T): T {
+  qaSuiteNestedRuns.add(params);
+  return params;
+}
+
+export const isQaSuiteNestedRun = (params: object | undefined) =>
+  params !== undefined && qaSuiteNestedRuns.has(params);
+
 export function formatQaSuiteRunStartProgress(params: {
  selectedScenarioCount: number;
  concurrency: number;
@@ -270,16 +280,19 @@ export async function waitForQaLabReadyOrStopOwned(params: {
  }
 }

-export async function runQaSuiteCleanupSteps(steps: ReadonlyArray<() => Promise<void>>) {
-  const errors: unknown[] = [];
+type QaSuiteCleanupStep = { phase: string; run: () => Promise<void> };
+type QaSuiteCleanupFailure = { phase: string; error: unknown };
+
+export async function runQaSuiteCleanupSteps(steps: readonly QaSuiteCleanupStep[]) {
+  const failures: QaSuiteCleanupFailure[] = [];
  for (const step of steps) {
    try {
-      await step();
+      await step.run();
    } catch (error) {
-      errors.push(error);
+      failures.push({ phase: step.phase, error });
    }
  }
-  return errors;
+  return failures;
 }

 export async function runQaFlowSuiteCleanupPlan(params: {
@@ -291,47 +304,81 @@ export async function runQaFlowSuiteCleanupPlan(params: {
  stopProvider?: () => Promise<void>;
  finishLab: () => Promise<void>;
 }) {
-  const errors = await runQaSuiteCleanupSteps([
-    ...(params.closeWebSessions ? [params.closeWebSessions] : []),
+  const stopGateway = params.stopGateway;
+  let gatewayStopped = !stopGateway;
+  const stopGatewayAndMark = async () => {
+    await stopGateway?.();
+    gatewayStopped = true;
+  };
+  const cleanupTransportAfterGatewayStop = async () => {
+    if (gatewayStopped) {
+      await params.cleanupTransportAfterGatewayStop();
+    }
+  };
+  return runQaSuiteCleanupSteps([
+    ...(params.closeWebSessions ? [{ phase: "web sessions", run: params.closeWebSessions }] : []),
    // Drain transport HTTP work before stopping the gateway; otherwise a completed suite can
    // emit an unhandled response-close rejection during delivery.
-    params.cleanupTransportBeforeGatewayStop,
-  ]);
-  let gatewayStopped = !params.stopGateway;
-  if (params.stopGateway) {
-    const gatewayErrors = await runQaSuiteCleanupSteps([params.stopGateway]);
-    errors.push(...gatewayErrors);
-    gatewayStopped = gatewayErrors.length === 0;
-  }
-  errors.push(
-    ...(await runQaSuiteCleanupSteps([
+    { phase: "transport before gateway stop", run: params.cleanupTransportBeforeGatewayStop },
+    ...(stopGateway ? [{ phase: "gateway stop", run: stopGatewayAndMark }] : []),
    // Never release a credential-backed transport until gateway teardown proves
    // that the isolated runtime reached its terminal boundary.
-      ...(gatewayStopped ? [params.cleanupTransportAfterGatewayStop] : []),
-      params.disposeAgentHarnesses,
-      ...(params.stopProvider ? [params.stopProvider] : []),
-      params.finishLab,
-    ])),
-  );
-  return errors;
+    { phase: "transport after gateway stop", run: cleanupTransportAfterGatewayStop },
+    { phase: "agent harnesses", run: params.disposeAgentHarnesses },
+    ...(params.stopProvider ? [{ phase: "provider stop", run: params.stopProvider }] : []),
+    { phase: "lab finish", run: params.finishLab },
+  ]);
 }

 export function throwQaSuiteCleanupErrors(params: {
-  cleanupErrors: unknown[];
+  cleanupFailures: readonly QaSuiteCleanupFailure[];
  runFailed: boolean;
  runError: unknown;
+  result?: QaSuiteResult;
+  evidenceWritten?: boolean;
 }) {
-  if (params.cleanupErrors.length === 0) {
+  if (params.cleanupFailures.length === 0) {
    return;
  }
-  if (params.cleanupErrors.length === 1 && !params.runFailed) {
-    throw params.cleanupErrors[0];
+  const result = params.result;
+  const scenarios = result?.scenarios ?? [];
+  const failed = scenarios.filter((scenario) => scenario.status === "fail").length;
+  const skipped = scenarios.filter((scenario) => scenario.status === "skip").length;
+  const passed = scenarios.length - failed - skipped;
+  const cleanupHeadline = !result
+    ? "QA suite cleanup failed before scenarios completed"
+    : failed === 0 && skipped === 0
+      ? "QA scenarios passed, but cleanup failed"
+      : "QA scenarios completed, but cleanup failed";
+  const message = [
+    params.runFailed ? "QA suite and cleanup failed" : cleanupHeadline,
+    ...(result
+      ? [
+          `scenario counts: passed=${passed} failed=${failed} skipped=${skipped} total=${scenarios.length}`,
+        ]
+      : params.runFailed
+        ? ["scenarios did not complete"]
+        : []),
+    `failed cleanup phases: ${params.cleanupFailures
+      .map(
+        ({ phase, error }) =>
+          `${sanitizeQaSuiteProgressValue(phase)}: ${sanitizeQaSuiteProgressValue(formatErrorMessage(error))}`,
+      )
+      .join("; ")}`,
+    ...(result
+      ? [
+          `retained artifacts: output=${sanitizeQaSuiteProgressValue(result.outputDir)} report=${sanitizeQaSuiteProgressValue(result.reportPath)} summary=${sanitizeQaSuiteProgressValue(result.summaryPath)}${params.evidenceWritten ? ` evidence=${sanitizeQaSuiteProgressValue(result.evidencePath)}` : ""}`,
+        ]
+      : []),
+  ].join("\n");
+  const errors = params.cleanupFailures.map((failure) => failure.error);
+  if (params.runFailed) {
+    throw new AggregateError([params.runError, ...errors], message, { cause: params.runError });
  }
-  throw new AggregateError(
-    params.runFailed ? [params.runError, ...params.cleanupErrors] : params.cleanupErrors,
-    params.runFailed ? "QA suite and cleanup failed" : "QA suite cleanup failed",
-    params.runFailed ? { cause: params.runError } : undefined,
-  );
+  if (errors.length === 1) {
+    throw new AggregateError(errors, message, { cause: errors[0] });
+  }
+  throw new AggregateError(errors, message);
 }

 export function requireQaSuiteStartLab(startLab: QaSuiteStartLabFn | undefined): QaSuiteStartLabFn {