fix(qa): mark partial suite artifacts as running (#125924)

* fix(qa): mark partial suite artifacts as running

Isolated QA suite progress artifacts now identify themselves as running in JSON and Markdown, so completed-prefix results cannot be mistaken for terminal teardown. Final artifacts keep their existing completed shape, with process regression coverage for clean exit and closed Gateway listeners.

* test(qa): decouple suite runtime from teardown deadline

Allow the real QA scenario to finish under a contended extension shard while keeping the post-summary process exit requirement fixed at 45 seconds.

* test(qa): keep lifecycle proof observable

Emit a bounded progress heartbeat while the real QA child is still producing its terminal summary so the extension shard watchdog does not mistake a long, active process proof for a stalled Vitest run.

* test(qa): isolate lifecycle process environment

Run the real QA child outside Vitest and shared compile-cache markers, and fail fast with bounded process output when it exits before publishing a terminal summary.

* test(qa): run lifecycle proof on repo gateway

Build and launch the real repository Gateway when dist is absent, avoid package-candidate auth bootstrap, and keep the teardown regression bounded and observable in unbuilt extension-test jobs.

* test(qa): terminate Windows lifecycle process trees

* fix(qa): reject running confidence summaries

extensions/qa-lab/src/confidence-report.test.ts

@@ -93,6 +93,43 @@ describe("qa confidence report", () => {
     expect(renderQaConfidenceMarkdownReport(report)).toContain("Global pass: no");
   });
 
+  it("uses suite lifecycle status before terminal outcome counts", async () => {
+    const manifest: QaConfidenceManifest = {
+      version: 1,
+      profile: "codex-100",
+      lanes: [
+        {
+          id: "suite",
+          title: "Suite",
+          kind: "qa-suite-summary",
+          artifact: "suite/qa-suite-summary.json",
+          required: true,
+        },
+      ],
+    };
+    for (const [runStatus, expectedPass, expectedLaneStatus, expectedDetails] of [
+      ["running", false, "unknown", "still running"],
+      ["completed", true, "pass", "counts.failed=0"],
+      ["paused", false, "unknown", "unsupported run.status=paused"],
+    ] as const) {
+      await writeJson("suite/qa-suite-summary.json", {
+        run: { status: runStatus },
+        counts: { total: 1, passed: 1, skipped: 0, failed: 0 },
+        scenarios: [{ name: "completed prefix", status: "pass" }],
+      });
+
+      const report = await buildQaConfidenceReport({
+        manifest,
+        artifactRoot: tempRoot,
+        strictGlobalPass: true,
+      });
+
+      expect(report.pass).toBe(expectedPass);
+      expect(report.lanes[0]).toMatchObject({ status: expectedLaneStatus });
+      expect(report.lanes[0]?.details).toContain(expectedDetails);
+    }
+  });
+
   it("does not let optional lanes block strict gates", async () => {
     await writeJson("required/qa-suite-summary.json", {
       counts: { total: 1, passed: 1, skipped: 0, failed: 0 },

extensions/qa-lab/src/confidence-report.ts

@@ -365,6 +365,17 @@ function evaluateQaSuiteSummary(payload: unknown): QaConfidenceLaneEvaluation {
       details: "qa-suite-summary payload was not an object",
     };
   }
+  const runStatus = isRecord(payload.run) ? payload.run.status : undefined;
+  if (runStatus !== undefined && runStatus !== "completed") {
+    return {
+      passed: false,
+      status: "unknown",
+      details:
+        runStatus === "running"
+          ? "qa-suite-summary is still running"
+          : `qa-suite-summary has unsupported run.status=${readString(runStatus) ?? typeof runStatus}`,
+    };
+  }
   const accountingError = findQaSuiteSummaryAccountingError(payload);
   if (accountingError) {
     return {

extensions/qa-lab/src/report.ts

@@ -28,6 +28,7 @@ function formatQaReportCheck(check: QaReportCheck, indent = "") {
 
 export function renderQaMarkdownReport(params: {
   title: string;
+  inProgress?: boolean;
   startedAt: Date;
   finishedAt: Date;
   checks?: QaReportCheck[];
@@ -48,10 +49,11 @@ export function renderQaMarkdownReport(params: {
     scenarios.filter((scenario) => scenario.status === "skip").length;
 
   const lines = [
-    `# ${params.title}`,
+    `# ${params.title}${params.inProgress ? " (In Progress)" : ""}`,
     "",
+    ...(params.inProgress ? ["- Status: running"] : []),
     `- Started: ${params.startedAt.toISOString()}`,
-    `- Finished: ${params.finishedAt.toISOString()}`,
+    `- ${params.inProgress ? "Updated" : "Finished"}: ${params.finishedAt.toISOString()}`,
     `- Duration ms: ${params.finishedAt.getTime() - params.startedAt.getTime()}`,
     `- Passed: ${passCount}`,
     `- Failed: ${failCount}`,

extensions/qa-lab/src/suite-artifacts.ts

@@ -49,6 +49,7 @@ export async function publishQaSuiteArtifactFiles(params: {
 }
 
 export type QaSuiteSummaryJsonParams = {
+  status?: QaSuiteSummaryJson["run"]["status"];
   scenarios: QaSuiteScenarioResult[];
   startedAt: Date;
   finishedAt: Date;
@@ -108,6 +109,7 @@ export function buildQaSuiteSummaryJson(params: QaSuiteSummaryJsonParams): QaSui
     ...(params.metrics ? { metrics: params.metrics } : {}),
     ...(params.evidence ? { evidence: params.evidence } : {}),
     run: {
+      status: params.status ?? "completed",
       startedAt: params.startedAt.toISOString(),
       finishedAt: params.finishedAt.toISOString(),
       providerMode: params.providerMode,
@@ -131,6 +133,7 @@ export function buildQaSuiteSummaryJson(params: QaSuiteSummaryJsonParams): QaSui
 }
 
 export async function writeQaSuiteArtifacts(params: {
+  status?: QaSuiteSummaryJson["run"]["status"];
   repoRoot?: string;
   outputDir: string;
   startedAt: Date;
@@ -195,6 +198,7 @@ export async function writeQaSuiteArtifacts(params: {
       : crablineChannelDriverSelection;
   const report = renderQaMarkdownReport({
     title: "OpenClaw QA Scenario Suite",
+    inProgress: params.status === "running",
     startedAt: params.startedAt,
     finishedAt: params.finishedAt,
     checks: [],

extensions/qa-lab/src/suite-process-lifecycle.test-support.ts

@@ -0,0 +1,35 @@
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import { resolveQaGatewayChildCommand } from "./gateway-child-command.js";
+import { runQaSuite } from "./suite-launch.runtime.js";
+
+const repoRoot = fileURLToPath(new URL("../../../", import.meta.url));
+const outputDir = process.argv[2];
+const scenarioIds = process.argv.slice(3);
+
+if (!outputDir || scenarioIds.length === 0) {
+  throw new Error("suite process fixture requires an output directory and scenario ids");
+}
+
+try {
+  const sutOpenClawCommand = {
+    ...resolveQaGatewayChildCommand(repoRoot),
+    usePackagedPlugins: false,
+  };
+  const result = await runQaSuite({
+    repoRoot,
+    outputDir: path.relative(repoRoot, outputDir),
+    providerMode: "mock-openai",
+    scenarioIds,
+    concurrency: 4,
+    sutOpenClawCommand,
+  });
+  const failed = result.result.scenarios.filter((scenario) => scenario.status !== "pass");
+  if (failed.length > 0) {
+    throw new Error(`suite process fixture failed ${failed.length} scenario(s)`);
+  }
+} catch (error) {
+  process.stderr.write(`${formatErrorMessage(error)}\n`);
+  process.exitCode = 1;
+}

extensions/qa-lab/src/suite-process-lifecycle.test.ts

@@ -0,0 +1,290 @@
+import { spawn, spawnSync, type ChildProcess } from "node:child_process";
+import { existsSync } from "node:fs";
+import fs from "node:fs/promises";
+import net from "node:net";
+import path from "node:path";
+import { setTimeout as sleep } from "node:timers/promises";
+import { fileURLToPath } from "node:url";
+import { useAutoCleanupTempDirTracker } from "openclaw/plugin-sdk/test-env";
+import { afterEach, describe, expect, it } from "vitest";
+import type { QaSuiteSummaryJson } from "./suite-summary.js";
+import { runQaWindowsTaskkill } from "./windows-system-tools.js";
+
+const repoRoot = fileURLToPath(new URL("../../../", import.meta.url));
+const fixturePath = fileURLToPath(
+  new URL("./suite-process-lifecycle.test-support.ts", import.meta.url),
+);
+const artifactsRoot = path.join(repoRoot, ".artifacts", "qa-e2e");
+const tempDirs = useAutoCleanupTempDirTracker(afterEach);
+const activeChildren = new Set<ChildProcess>();
+
+const PROCESS_LIFECYCLE_SCENARIO = "channel-chat-baseline";
+// Suite execution contends with the surrounding extension shard; only the bounded
+// post-summary close window is the lifecycle contract this regression enforces.
+const SUITE_COMPLETION_TIMEOUT_MS = 420_000;
+const POST_SUMMARY_EXIT_TIMEOUT_MS = 45_000;
+
+function buildSuiteProcessEnv(outputDir: string) {
+  const home = path.join(outputDir, "process-home");
+  const env: NodeJS.ProcessEnv = {
+    ...process.env,
+    HOME: home,
+    USERPROFILE: home,
+    OPENCLAW_HOME: home,
+    OPENCLAW_STATE_DIR: path.join(home, ".openclaw"),
+    OPENCLAW_CONFIG_PATH: path.join(home, ".openclaw", "openclaw.json"),
+    OPENCLAW_BUILD_PRIVATE_QA: "1",
+    OPENCLAW_QA_SUITE_PROGRESS: "1",
+    OPENCLAW_RUN_NODE_SKIP_DTS_BUILD: "1",
+  };
+  if (!existsSync(path.join(repoRoot, "dist", "index.js"))) {
+    env.OPENCLAW_FORCE_BUILD = "1";
+  }
+  delete env.VITEST;
+  delete env.VITEST_POOL_ID;
+  delete env.VITEST_WORKER_ID;
+  delete env.OPENCLAW_VITEST_FS_MODULE_CACHE_PATH;
+  delete env.OPENCLAW_VITEST_FS_MODULE_CACHE_WRITER;
+  delete env.NODE_COMPILE_CACHE;
+  delete env.NODE_DISABLE_COMPILE_CACHE;
+  delete env.OPENCLAW_NODE_COMPILE_CACHE_WRITER;
+  if (env.NODE_ENV === "test") {
+    delete env.NODE_ENV;
+  }
+  return env;
+}
+
+function forceStopProcessTree(child: ChildProcess) {
+  if (!child.pid || child.exitCode !== null || child.signalCode !== null) {
+    return;
+  }
+  if (process.platform === "win32") {
+    if (!runQaWindowsTaskkill({ pid: child.pid, signal: "SIGKILL" })) {
+      child.kill("SIGKILL");
+    }
+    return;
+  }
+  const rows = spawnSync("ps", ["-axo", "pid=,ppid="], { encoding: "utf8" })
+    .stdout.trim()
+    .split("\n")
+    .flatMap((line) => {
+      const [pidText, parentPidText] = line.trim().split(/\s+/u);
+      const pid = Number(pidText);
+      const parentPid = Number(parentPidText);
+      return Number.isSafeInteger(pid) && Number.isSafeInteger(parentPid)
+        ? [{ pid, parentPid }]
+        : [];
+    });
+  const owned = new Set([child.pid]);
+  let foundDescendant = true;
+  while (foundDescendant) {
+    foundDescendant = false;
+    for (const { pid, parentPid } of rows) {
+      if (owned.has(parentPid) && !owned.has(pid)) {
+        owned.add(pid);
+        foundDescendant = true;
+      }
+    }
+  }
+  for (const pid of [...owned].toReversed()) {
+    try {
+      process.kill(pid, "SIGKILL");
+    } catch {
+      // Already gone.
+    }
+  }
+}
+
+afterEach(async () => {
+  for (const child of activeChildren) {
+    forceStopProcessTree(child);
+  }
+  await Promise.all(
+    [...activeChildren].map(
+      (child) =>
+        new Promise<void>((resolve) => {
+          if (child.exitCode !== null || child.signalCode !== null) {
+            resolve();
+            return;
+          }
+          child.once("close", () => resolve());
+        }),
+    ),
+  );
+  activeChildren.clear();
+});
+
+function startSuiteProcess(outputDir: string, scenarioIds: readonly string[]) {
+  const child = spawn(
+    process.execPath,
+    ["--import", "tsx", fixturePath, outputDir, ...scenarioIds],
+    {
+      cwd: repoRoot,
+      env: buildSuiteProcessEnv(outputDir),
+      stdio: ["ignore", "pipe", "pipe"],
+    },
+  );
+  activeChildren.add(child);
+  let stdout = "";
+  let stderr = "";
+  const gatewayPorts = new Set<number>();
+  child.stdout?.setEncoding("utf8");
+  child.stderr?.setEncoding("utf8");
+  child.stdout?.on("data", (chunk: string) => {
+    stdout += chunk;
+  });
+  child.stderr?.on("data", (chunk: string) => {
+    stderr += chunk;
+    for (const match of chunk.matchAll(/gateway ready: http:\/\/127\.0\.0\.1:(\d+)/gu)) {
+      gatewayPorts.add(Number(match[1]));
+    }
+  });
+  const closed = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>(
+    (resolve, reject) => {
+      child.once("error", reject);
+      child.once("close", (code, signal) => {
+        activeChildren.delete(child);
+        resolve({ code, signal });
+      });
+    },
+  );
+  return {
+    child,
+    closed,
+    gatewayPorts,
+    output: () => ({ stderr, stdout }),
+  };
+}
+
+async function isTcpPortOpen(port: number) {
+  return await new Promise<boolean>((resolve) => {
+    const socket = net.createConnection({ host: "127.0.0.1", port });
+    const finish = (open: boolean) => {
+      socket.destroy();
+      resolve(open);
+    };
+    socket.setTimeout(500, () => finish(false));
+    socket.once("connect", () => finish(true));
+    socket.once("error", () => finish(false));
+  });
+}
+
+async function waitForCompletedSummary(params: {
+  outputDir: string;
+  timeoutMs: number;
+  closed: Promise<{ code: number | null; signal: NodeJS.Signals | null }>;
+  output: () => { stderr: string; stdout: string };
+}) {
+  const summaryPath = path.join(params.outputDir, "qa-suite-summary.json");
+  const deadline = Date.now() + params.timeoutMs;
+  const processState: {
+    error?: unknown;
+    outcome?: { code: number | null; signal: NodeJS.Signals | null };
+  } = {};
+  void params.closed.then(
+    (outcome) => {
+      processState.outcome = outcome;
+    },
+    (error: unknown) => {
+      processState.error = error;
+    },
+  );
+  const throwIfProcessClosed = () => {
+    if (!processState.error && !processState.outcome) {
+      return;
+    }
+    const output = params.output();
+    throw new Error(
+      `QA suite process exited before writing a completed summary: ${JSON.stringify(processState.outcome ?? { error: String(processState.error) })}\nstdout:\n${output.stdout.slice(-8_000)}\nstderr:\n${output.stderr.slice(-8_000)}`,
+    );
+  };
+  while (Date.now() < deadline) {
+    let summary: QaSuiteSummaryJson;
+    try {
+      summary = JSON.parse(await fs.readFile(summaryPath, "utf8")) as QaSuiteSummaryJson;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+        throw error;
+      }
+      throwIfProcessClosed();
+      await sleep(50);
+      continue;
+    }
+    const runStatus: unknown = summary.run.status;
+    if (runStatus === "completed") {
+      return summary;
+    }
+    if (runStatus !== "running") {
+      throw new Error(`QA suite summary is missing lifecycle status: ${String(runStatus)}`);
+    }
+    throwIfProcessClosed();
+    await sleep(50);
+  }
+  const output = params.output();
+  throw new Error(
+    `QA suite did not write a completed summary within ${params.timeoutMs}ms\nstdout:\n${output.stdout.slice(-8_000)}\nstderr:\n${output.stderr.slice(-8_000)}`,
+  );
+}
+
+async function waitForProcessClose(
+  closed: Promise<{ code: number | null; signal: NodeJS.Signals | null }>,
+  timeoutMs: number,
+) {
+  let timeout: NodeJS.Timeout | undefined;
+  try {
+    return await Promise.race([
+      closed,
+      new Promise<never>((_resolve, reject) => {
+        timeout = setTimeout(
+          () =>
+            reject(
+              new Error(
+                `QA suite process did not exit within ${timeoutMs}ms of summary completion`,
+              ),
+            ),
+          timeoutMs,
+        );
+      }),
+    ]);
+  } finally {
+    if (timeout) {
+      clearTimeout(timeout);
+    }
+  }
+}
+
+describe("qa suite command process lifecycle", () => {
+  it(
+    "exits after the terminal summary and leaves no gateway listener",
+    { timeout: SUITE_COMPLETION_TIMEOUT_MS + POST_SUMMARY_EXIT_TIMEOUT_MS + 30_000 },
+    async () => {
+      await fs.mkdir(artifactsRoot, { recursive: true });
+      const outputDir = tempDirs.make("suite-process-lifecycle-", artifactsRoot);
+      const run = startSuiteProcess(outputDir, [PROCESS_LIFECYCLE_SCENARIO]);
+      const startedWaitingAt = Date.now();
+      const heartbeat = setInterval(() => {
+        const output = run.output();
+        process.stderr.write(
+          `[qa-process-lifecycle] waiting for completed summary elapsedMs=${Date.now() - startedWaitingAt} gatewayPorts=${run.gatewayPorts.size} stderrBytes=${Buffer.byteLength(output.stderr)}\n`,
+        );
+      }, 30_000);
+      heartbeat.unref();
+      const summary = await waitForCompletedSummary({
+        outputDir,
+        timeoutMs: SUITE_COMPLETION_TIMEOUT_MS,
+        closed: run.closed,
+        output: run.output,
+      }).finally(() => clearInterval(heartbeat));
+      const outcome = await waitForProcessClose(run.closed, POST_SUMMARY_EXIT_TIMEOUT_MS);
+      const output = run.output();
+
+      expect(outcome, output.stderr).toEqual({ code: 0, signal: null });
+      expect(run.gatewayPorts.size, output.stderr).toBeGreaterThan(0);
+      expect(summary.run.status).toBe("completed");
+      expect(summary.counts).toEqual({ total: 1, passed: 1, failed: 0, skipped: 0 });
+      await expect(
+        Promise.all([...run.gatewayPorts].map((port) => isTcpPortOpen(port))),
+      ).resolves.toEqual([...run.gatewayPorts].map(() => false));
+    },
+  );
+});

extensions/qa-lab/src/suite-run-isolated.cleanup.test.ts

@@ -158,6 +158,13 @@ describe("isolated QA suite transport cleanup", () => {
     expect(lab.setLatestReport).toHaveBeenCalledWith(
       expect.objectContaining({ outputPath: "/qa-output/qa-suite-report.md" }),
     );
+    expect(mocks.writeQaSuiteArtifacts).toHaveBeenNthCalledWith(
+      1,
+      expect.objectContaining({ status: "running" }),
+    );
+    expect(mocks.writeQaSuiteArtifacts).toHaveBeenLastCalledWith(
+      expect.not.objectContaining({ status: "running" }),
+    );
     expect((thrown as Error).message.split("\n")[0]).toBe(
       "QA scenarios passed, but cleanup failed",
     );

extensions/qa-lab/src/suite-run-isolated.ts

@@ -98,6 +98,7 @@ export async function runQaFlowSuiteIsolated(
       .then(async () => {
         const partialFinishedAt = new Date();
         const { report, reportPath } = await writeQaSuiteArtifacts({
+          status: "running",
           repoRoot,
           outputDir,
           startedAt,

extensions/qa-lab/src/suite-summary.ts

@@ -49,6 +49,7 @@ export type QaSuiteSummaryJson = {
   };
   evidence?: QaEvidenceSummaryJson;
   run: {
+    status: "running" | "completed";
     startedAt: string;
     finishedAt: string;
     providerMode: QaProviderMode;

extensions/qa-lab/src/suite.summary-json.test.ts

@@ -23,6 +23,7 @@ describe("buildQaSuiteSummaryJson", () => {
 
   it("records provider/model/mode so parity gates can verify labels", () => {
     const json = buildQaSuiteSummaryJson(baseParams);
+    expect(json.run.status).toBe("completed");
     expect(json.run.startedAt).toBe("2026-04-11T00:00:00.000Z");
     expect(json.run.finishedAt).toBe("2026-04-11T00:05:00.000Z");
     expect(json.run.providerMode).toBe("mock-openai");
@@ -41,6 +42,12 @@ describe("buildQaSuiteSummaryJson", () => {
     expect(json.run.scenarioIds).toBeNull();
   });
 
+  it("distinguishes an in-progress artifact from terminal suite output", () => {
+    const json = buildQaSuiteSummaryJson({ ...baseParams, status: "running" });
+
+    expect(json.run.status).toBe("running");
+  });
+
   it("records Crabline channel-driver metadata when selected", () => {
     const json = buildQaSuiteSummaryJson({
       ...baseParams,

extensions/qa-lab/src/suite.test.ts

@@ -618,6 +618,41 @@ describe("qa suite", () => {
     }
   });
 
+  it("distinguishes partial Markdown from the terminal report shape", async () => {
+    const outputDir = await tempDirs.makeTempDir("qa-suite-report-lifecycle-");
+    const baseParams = {
+      outputDir,
+      startedAt: new Date("2026-04-11T00:00:00.000Z"),
+      finishedAt: new Date("2026-04-11T00:01:00.000Z"),
+      scenarios: [{ name: "Baseline", status: "pass" as const, steps: [] }],
+      transport: {
+        id: "qa-channel",
+        createReportNotes: () => [],
+      } as unknown as QaTransportAdapter,
+      providerMode: "mock-openai" as const,
+      primaryModel: "mock-openai/gpt-5.6-luna",
+      alternateModel: "mock-openai/gpt-5.6-luna-alt",
+      fastMode: true,
+      concurrency: 1,
+    };
+
+    try {
+      const partial = await writeQaSuiteArtifacts({ ...baseParams, status: "running" });
+      expect(partial.report).toContain("# OpenClaw QA Scenario Suite (In Progress)");
+      expect(partial.report).toContain("- Status: running");
+      expect(partial.report).toContain("- Updated: 2026-04-11T00:01:00.000Z");
+      expect(partial.report).not.toContain("- Finished:");
+
+      const terminal = await writeQaSuiteArtifacts(baseParams);
+      expect(terminal.report).toContain("# OpenClaw QA Scenario Suite\n");
+      expect(terminal.report).toContain("- Finished: 2026-04-11T00:01:00.000Z");
+      expect(terminal.report).not.toContain("In Progress");
+      expect(terminal.report).not.toContain("- Status: running");
+    } finally {
+      await fs.rm(outputDir, { recursive: true, force: true });
+    }
+  });
+
   it("writes the selected Crabline driver with an honest failed result", async () => {
     const outputDir = await tempDirs.makeTempDir("qa-suite-crabline-");
     try {