From 8a0d28d6bae2c97047f07923903a32bfcf55cb27 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 18 Aug 2026 02:03:37 -0700 Subject: [PATCH] refactor(qa): simplify scenario runner infrastructure (#125738) --- config/max-lines-baseline.txt | 1 - .../qa-lab/src/evidence-gallery.test.ts | 24 +- .../scenarios/scenario-runtime-cli-process.ts | 36 +- .../scenarios/scenario-runtime-cli.test.ts | 89 +- extensions/qa-lab/src/qa-cli-process.ts | 19 +- .../src/suite-runtime-agent-process.test.ts | 91 +- ...st-file-scenario-command-lifecycle.test.ts | 30 +- .../test-file-scenario-command-lifecycle.ts | 32 +- .../test-file-scenario-runner.docker.test.ts | 306 +++ .../test-file-scenario-runner.native.test.ts | 487 ++++ .../test-file-scenario-runner.process.test.ts | 120 + ...le-scenario-runner.script-evidence.test.ts | 594 +++++ .../test-file-scenario-runner.test-support.ts | 230 ++ .../src/test-file-scenario-runner.test.ts | 2027 ----------------- extensions/qa-lab/web/src/ui-render.test.ts | 8 +- scripts/qa/ux-matrix-evidence-producer.ts | 814 ------- .../qa-ux-matrix-evidence-producer.test.ts | 230 -- 17 files changed, 1858 insertions(+), 3280 deletions(-) create mode 100644 extensions/qa-lab/src/test-file-scenario-runner.docker.test.ts create mode 100644 extensions/qa-lab/src/test-file-scenario-runner.native.test.ts create mode 100644 extensions/qa-lab/src/test-file-scenario-runner.process.test.ts create mode 100644 extensions/qa-lab/src/test-file-scenario-runner.script-evidence.test.ts create mode 100644 extensions/qa-lab/src/test-file-scenario-runner.test-support.ts delete mode 100644 extensions/qa-lab/src/test-file-scenario-runner.test.ts delete mode 100644 scripts/qa/ux-matrix-evidence-producer.ts delete mode 100644 test/scripts/qa-ux-matrix-evidence-producer.test.ts diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index 3ccd0f7e6728..bedad4a57731 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -208,7 +208,6 @@ extensions/qa-lab/src/runtime-tool-fixture.ts extensions/qa-lab/src/scorecard-taxonomy.ts extensions/qa-lab/src/suite-launch.runtime.test.ts extensions/qa-lab/src/suite-launch.runtime.ts -extensions/qa-lab/src/test-file-scenario-runner.test.ts extensions/qa-lab/web/src/app.ts extensions/signal/src/client-container.test.ts extensions/signal/src/client-container.ts diff --git a/extensions/qa-lab/src/evidence-gallery.test.ts b/extensions/qa-lab/src/evidence-gallery.test.ts index f5d34620f42a..267bdc7b9e19 100644 --- a/extensions/qa-lab/src/evidence-gallery.test.ts +++ b/extensions/qa-lab/src/evidence-gallery.test.ts @@ -368,9 +368,9 @@ describe("evidence gallery", () => { coverageIds: [], runner: { availability: "local", - command: `node --import tsx ${repoRoot}/scripts/qa/ux-matrix-evidence-producer.ts --artifact-base ${runDir}`, + command: `node ${repoRoot}/external/qa/ux-matrix-producer.mjs --artifact-base ${runDir}`, lane: "web-ui-playwright", - workflow: `${repoRoot}/.github/workflows/ux-matrix-qa.yml#ux-matrix-local`, + workflow: `${repoRoot}/external/ci/ux-matrix.yml#matrix-local`, }, stage: "first-run", status: "pass", @@ -381,9 +381,9 @@ describe("evidence gallery", () => { runner: { availability: "local", command: - "node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix", + "node external/qa/ux-matrix-producer.mjs --artifact-base .artifacts/external-qa/ux-matrix", lane: "cli-status", - workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local", + workflow: "external/ci/ux-matrix.yml#matrix-local", }, stage: "first-run", status: "proof-gap", @@ -402,7 +402,7 @@ describe("evidence gallery", () => { await fs.writeFile(path.join(runDir, "scorecard.md"), "# UX Matrix\n\n- pass: 1\n", "utf8"); await fs.writeFile( path.join(runDir, "commands.txt"), - "node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix\n", + "node external/qa/ux-matrix-producer.mjs --artifact-base .artifacts/external-qa/ux-matrix\n", "utf8", ); await fs.mkdir(path.join(runDir, "preflight"), { recursive: true }); @@ -424,7 +424,7 @@ describe("evidence gallery", () => { kind: "ux-matrix-cell", id: "ux-matrix.web-ui.first-run", title: `UX Matrix: web-ui / first-run at ${repoRoot}`, - source: { path: "scripts/qa/ux-matrix-evidence-producer.ts" }, + source: { path: "external/qa/ux-matrix-producer.mjs" }, }, coverage: [], execution: { @@ -463,7 +463,7 @@ describe("evidence gallery", () => { kind: "ux-matrix-cell", id: "qa-lab.wrapper-cli-error", title: "UX Matrix: cli / error-state", - source: { path: "scripts/qa/ux-matrix-evidence-producer.ts" }, + source: { path: "external/qa/ux-matrix-producer.mjs" }, }, coverage: [], execution: { @@ -539,9 +539,9 @@ describe("evidence gallery", () => { runner: { availability: "local", command: - "node --import tsx /scripts/qa/ux-matrix-evidence-producer.ts --artifact-base /.artifacts/qa-e2e/suite/script/nested/ux-matrix-producer/run-1", + "node /external/qa/ux-matrix-producer.mjs --artifact-base /.artifacts/qa-e2e/suite/script/nested/ux-matrix-producer/run-1", lane: "web-ui-playwright", - workflow: "/.github/workflows/ux-matrix-qa.yml#ux-matrix-local", + workflow: "/external/ci/ux-matrix.yml#matrix-local", }, stage: "first-run", status: "pass", @@ -556,9 +556,9 @@ describe("evidence gallery", () => { runner: { availability: "local", command: - "node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix", + "node external/qa/ux-matrix-producer.mjs --artifact-base .artifacts/external-qa/ux-matrix", lane: "cli-status", - workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local", + workflow: "external/ci/ux-matrix.yml#matrix-local", }, stage: "first-run", status: "proof-gap", @@ -584,7 +584,7 @@ describe("evidence gallery", () => { repoRoot, ); expect(model.producerContext?.commands?.preview).toBe( - "node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix\n", + "node external/qa/ux-matrix-producer.mjs --artifact-base .artifacts/external-qa/ux-matrix\n", ); expect(model.producerContext?.commands?.path).toContain("commands.txt"); expect(decodeURIComponent(model.producerContext?.commands?.href ?? "")).not.toContain(repoRoot); diff --git a/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-cli-process.ts b/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-cli-process.ts index 52c9a40e77f2..daf2e7ea00a8 100644 --- a/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-cli-process.ts +++ b/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-cli-process.ts @@ -1,7 +1,9 @@ -import { spawn as startOpenClawCliProcess, spawnSync } from "node:child_process"; +import { spawn as startOpenClawCliProcess } from "node:child_process"; import { existsSync } from "node:fs"; import path from "node:path"; -import { resolveQaWindowsSystem32ExePath } from "../../../windows-system-tools.js"; +import { runQaWindowsTaskkill } from "../../../windows-system-tools.js"; + +type MatrixQaTaskkillRunner = NonNullable[0]["runCommand"]>; export function resolveMatrixQaOpenClawCliEntryPath(cwd: string): string { const mjsEntryPath = path.join(cwd, "dist", "index.mjs"); @@ -14,28 +16,18 @@ export function resolveMatrixQaOpenClawCliEntryPath(cwd: string): string { export function killMatrixQaCliChild( child: ReturnType, signal: NodeJS.Signals, - runTaskkill: typeof spawnSync = spawnSync, + runTaskkill?: MatrixQaTaskkillRunner, ): void { if (process.platform === "win32") { - if (child.pid) { - const taskkillPath = resolveQaWindowsSystem32ExePath("taskkill.exe"); - const args = ["/PID", String(child.pid), "/T"]; - if (signal === "SIGKILL") { - args.push("/F"); - } - const result = runTaskkill(taskkillPath, args, { stdio: "ignore", windowsHide: true }); - if (!result.error && result.status === 0) { - return; - } - if (signal !== "SIGKILL") { - const forceResult = runTaskkill(taskkillPath, [...args, "/F"], { - stdio: "ignore", - windowsHide: true, - }); - if (!forceResult.error && forceResult.status === 0) { - return; - } - } + if ( + child.pid && + runQaWindowsTaskkill({ + pid: child.pid, + signal, + ...(runTaskkill ? { runCommand: runTaskkill } : {}), + }) + ) { + return; } child.kill(signal); return; diff --git a/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-cli.test.ts b/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-cli.test.ts index 42c2caad2055..96a7630d98a2 100644 --- a/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-cli.test.ts +++ b/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-cli.test.ts @@ -78,13 +78,12 @@ describe("Matrix QA CLI runtime", () => { ).toBe("GET /_matrix/client/v3/sync?access_token=abcdef…ghij"); }); - it("force-kills Windows CLI process trees when graceful taskkill fails", () => { + it.each([ + { label: "succeeds after escalation", statuses: [1, 0], fallsBack: false }, + { label: "fails completely", statuses: [1, 1], fallsBack: true }, + ])("composes Windows cleanup when canonical taskkill $label", ({ fallsBack, statuses }) => { const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform"); - const originalSystemRoot = process.env.SystemRoot; - const originalWindir = process.env.WINDIR; Object.defineProperty(process, "platform", { value: "win32", configurable: true }); - process.env.SystemRoot = "C:\\Windows"; - delete process.env.WINDIR; try { const killMock = vi.fn(); const child = { @@ -93,35 +92,21 @@ describe("Matrix QA CLI runtime", () => { } as unknown as Parameters[0]; const runTaskkill = vi .fn() - .mockReturnValueOnce({ status: 1 }) - .mockReturnValueOnce({ status: 0 }); + .mockReturnValueOnce({ status: statuses[0] }) + .mockReturnValueOnce({ status: statuses[1] }); testing.killMatrixQaCliChild(child, "SIGTERM", runTaskkill); - const taskkillPath = path.win32.join("C:\\Windows", "System32", "taskkill.exe"); - expect(runTaskkill).toHaveBeenNthCalledWith(1, taskkillPath, ["/PID", "12345", "/T"], { - stdio: "ignore", - windowsHide: true, - }); - expect(runTaskkill).toHaveBeenNthCalledWith(2, taskkillPath, ["/PID", "12345", "/T", "/F"], { - stdio: "ignore", - windowsHide: true, - }); - expect(killMock).not.toHaveBeenCalled(); + expect(runTaskkill).toHaveBeenCalledTimes(2); + if (fallsBack) { + expect(killMock).toHaveBeenCalledWith("SIGTERM"); + } else { + expect(killMock).not.toHaveBeenCalled(); + } } finally { if (platformDescriptor) { Object.defineProperty(process, "platform", platformDescriptor); } - if (originalSystemRoot === undefined) { - delete process.env.SystemRoot; - } else { - process.env.SystemRoot = originalSystemRoot; - } - if (originalWindir === undefined) { - delete process.env.WINDIR; - } else { - process.env.WINDIR = originalWindir; - } } }); @@ -389,56 +374,6 @@ describe("Matrix QA CLI runtime", () => { } }); - it("kills ignored-stdio descendants after a timed-out CLI exits gracefully", async () => { - if (process.platform === "win32") { - return; - } - const root = await mkdtemp( - path.join(resolvePreferredOpenClawTmpDir(), "matrix-qa-cli-timeout-ignored-stdio-"), - ); - const childPidPath = path.join(root, "child.pid"); - const grandchildPidPath = path.join(root, "grandchild.pid"); - let childPid: number | undefined; - let grandchildPid: number | undefined; - try { - await mkdir(path.join(root, "dist")); - await writeFile( - path.join(root, "dist", "index.mjs"), - [ - "import { spawn } from 'node:child_process';", - "import { writeFileSync } from 'node:fs';", - `writeFileSync(${JSON.stringify(childPidPath)}, String(process.pid));`, - "const grandchild = spawn(process.execPath, ['-e', 'process.on(\\'SIGTERM\\', () => {}); setInterval(() => {}, 1000);'], { stdio: 'ignore' });", - "grandchild.unref();", - `writeFileSync(${JSON.stringify(grandchildPidPath)}, String(grandchild.pid));`, - "process.on('SIGTERM', () => process.exit(0));", - "setInterval(() => {}, 1000);", - ].join("\n"), - ); - - const run = runMatrixQaOpenClawCli({ - args: ["matrix", "verify", "self"], - cwd: root, - env: process.env, - timeoutMs: 500, - }); - grandchildPid = await waitForPidFile(grandchildPidPath, 2_000); - - await expect(run).rejects.toThrow(/timed out after 500ms/u); - - childPid = await waitForPidFile(childPidPath, 2_000); - expect(isProcessRunning(childPid)).toBe(false); - expect(isProcessRunning(grandchildPid)).toBe(false); - } finally { - for (const pid of [grandchildPid, childPid]) { - if (pid && isProcessRunning(pid)) { - process.kill(pid, "SIGKILL"); - } - } - await rm(root, { force: true, recursive: true }); - } - }); - it("kills ignored-stdio descendants after manual CLI session kill", async () => { if (process.platform === "win32") { return; diff --git a/extensions/qa-lab/src/qa-cli-process.ts b/extensions/qa-lab/src/qa-cli-process.ts index b1840d5225ae..c112f6157eab 100644 --- a/extensions/qa-lab/src/qa-cli-process.ts +++ b/extensions/qa-lab/src/qa-cli-process.ts @@ -1,5 +1,5 @@ // Qa Lab plugin module runs CLI processes and parses their structured output. -import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; import path from "node:path"; import { resolveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime"; import { isRecord as isJsonRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; @@ -17,7 +17,7 @@ import { QaSuiteInfraError } from "./errors.js"; import { resolveQaNodeExecPath } from "./node-exec.js"; import { createQaPosixCommandSettlement } from "./posix-command-settlement.js"; import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js"; -import { resolveQaWindowsSystem32ExePath } from "./windows-system-tools.js"; +import { runQaWindowsTaskkill } from "./windows-system-tools.js"; const ANSI_ESCAPE_PATTERN = new RegExp(String.raw`\x1B\[[0-?]*[ -/]*[@-~]`, "g"); @@ -181,19 +181,8 @@ function parseQaCliJsonOutput(text: string, args: readonly string[]) { } function killQaCliWindowsProcessTree(child: Pick) { - if (child.pid) { - const result = spawnSync( - resolveQaWindowsSystem32ExePath("taskkill.exe"), - ["/PID", String(child.pid), "/T", "/F"], - { - stdio: "ignore", - windowsHide: true, - timeout: 5_000, - }, - ); - if (!result.error && result.status === 0) { - return; - } + if (child.pid && runQaWindowsTaskkill({ pid: child.pid, signal: "SIGKILL" })) { + return; } child.kill("SIGKILL"); } diff --git a/extensions/qa-lab/src/suite-runtime-agent-process.test.ts b/extensions/qa-lab/src/suite-runtime-agent-process.test.ts index ce1f577291b6..97e6ca93adf6 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-process.test.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-process.test.ts @@ -5,7 +5,7 @@ import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime"; import { beforeEach, describe, expect, it, vi } from "vitest"; const spawnMock = vi.hoisted(() => vi.fn()); -const spawnSyncMock = vi.hoisted(() => vi.fn()); +const runQaWindowsTaskkillMock = vi.hoisted(() => vi.fn()); const resolveQaNodeExecPathMock = vi.hoisted(() => vi.fn(async () => "/usr/bin/node")); const waitForGatewayHealthyMock = vi.hoisted(() => vi.fn(async () => undefined)); const waitForTransportReadyMock = vi.hoisted(() => vi.fn(async () => undefined)); @@ -13,7 +13,11 @@ const readSessionTranscriptSummaryMock = vi.hoisted(() => vi.fn()); vi.mock("node:child_process", () => ({ spawn: spawnMock, - spawnSync: spawnSyncMock, +})); + +vi.mock("./windows-system-tools.js", async (importOriginal) => ({ + ...(await importOriginal()), + runQaWindowsTaskkill: runQaWindowsTaskkillMock, })); vi.mock("./node-exec.js", () => ({ @@ -141,7 +145,7 @@ function createAgentPromptEnv(gatewayCall: ReturnType) { describe("qa suite runtime agent process helpers", () => { beforeEach(() => { spawnMock.mockReset(); - spawnSyncMock.mockReset(); + runQaWindowsTaskkillMock.mockReset(); resolveQaNodeExecPathMock.mockClear(); waitForGatewayHealthyMock.mockClear(); waitForTransportReadyMock.mockClear(); @@ -274,53 +278,44 @@ describe("qa suite runtime agent process helpers", () => { }, ); - it("force-kills timed-out Windows qa cli process trees with taskkill", async () => { - const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform"); - const originalSystemRoot = process.env.SystemRoot; - const originalWindir = process.env.WINDIR; - Object.defineProperty(process, "platform", { value: "win32", configurable: true }); - process.env.SystemRoot = "C:\\Windows"; - delete process.env.WINDIR; - try { - const child = createSpawnedProcess({ pid: 12345 }); - spawnSyncMock.mockReturnValue({ status: 0 }); - const { pending } = startMockQaCli({ - args: ["qa", "suite"], - child, - options: { timeoutMs: 1 }, - }); - const timeoutAssertion = expect(pending).rejects.toThrow( - "qa cli timed out: openclaw qa suite", - ); + it.each([ + { label: "succeeds", taskkillSucceeded: true }, + { label: "falls back", taskkillSucceeded: false }, + ])( + "preserves the Windows timeout result when canonical cleanup $label", + async ({ taskkillSucceeded }) => { + const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform"); + Object.defineProperty(process, "platform", { value: "win32", configurable: true }); + try { + const child = createSpawnedProcess({ pid: 12345 }); + runQaWindowsTaskkillMock.mockReturnValue(taskkillSucceeded); + const { pending } = startMockQaCli({ + args: ["qa", "suite"], + child, + options: { timeoutMs: 1 }, + }); + const timeoutAssertion = expect(pending).rejects.toThrow( + "qa cli timed out: openclaw qa suite", + ); - await waitForSpawnCount(1); - await timeoutAssertion; - expect(spawnSyncMock).toHaveBeenCalledWith( - path.win32.join("C:\\Windows", "System32", "taskkill.exe"), - ["/PID", "12345", "/T", "/F"], - { - stdio: "ignore", - windowsHide: true, - timeout: 5_000, - }, - ); - expect(child.kill).not.toHaveBeenCalled(); - } finally { - if (platformDescriptor) { - Object.defineProperty(process, "platform", platformDescriptor); + await waitForSpawnCount(1); + await timeoutAssertion; + expect(runQaWindowsTaskkillMock).toHaveBeenCalledWith({ + pid: 12345, + signal: "SIGKILL", + }); + if (taskkillSucceeded) { + expect(child.kill).not.toHaveBeenCalled(); + } else { + expect(child.kill).toHaveBeenCalledWith("SIGKILL"); + } + } finally { + if (platformDescriptor) { + Object.defineProperty(process, "platform", platformDescriptor); + } } - if (originalSystemRoot === undefined) { - delete process.env.SystemRoot; - } else { - process.env.SystemRoot = originalSystemRoot; - } - if (originalWindir === undefined) { - delete process.env.WINDIR; - } else { - process.env.WINDIR = originalWindir; - } - } - }); + }, + ); it("merges isolated env overrides into qa cli runs", async () => { const { child, pending } = startMockQaCli({ diff --git a/extensions/qa-lab/src/test-file-scenario-command-lifecycle.test.ts b/extensions/qa-lab/src/test-file-scenario-command-lifecycle.test.ts index 20c7e7acf4d0..6ae0d3106ffd 100644 --- a/extensions/qa-lab/src/test-file-scenario-command-lifecycle.test.ts +++ b/extensions/qa-lab/src/test-file-scenario-command-lifecycle.test.ts @@ -4,6 +4,8 @@ import { mkdtemp, readFile, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { build as esbuild } from "esbuild"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const spawnMock = vi.hoisted(() => vi.fn()); @@ -211,7 +213,25 @@ describe.skipIf(process.platform === "win32")("qa scenario command real POSIX li it("cleans the command group before re-raising a parent SIGTERM", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "qa-command-parent-signal-")); const descendantPidPath = path.join(root, "descendant.pid"); - const moduleUrl = new URL("./test-file-scenario-command-lifecycle.ts", import.meta.url).href; + const bundlePath = path.join(root, "test-file-scenario-command-lifecycle.mjs"); + // Compile before the readiness window; a cold tsx child can exceed it under the full QA suite. + // The bundle needs only the UTF-16 helper behind this SDK import, not the full plugin runtime. + await esbuild({ + alias: { + "openclaw/plugin-sdk/text-utility-runtime": fileURLToPath( + new URL("../../../packages/normalization-core/src/utf16-slice.ts", import.meta.url), + ), + }, + bundle: true, + entryPoints: [ + fileURLToPath(new URL("./test-file-scenario-command-lifecycle.ts", import.meta.url)), + ], + format: "esm", + outfile: bundlePath, + platform: "node", + target: "node22", + }); + const moduleUrl = pathToFileURL(bundlePath).href; let descendantPid: number | undefined; if (!actualSpawn.value) { throw new Error("real spawn unavailable"); @@ -235,7 +255,7 @@ describe.skipIf(process.platform === "win32")("qa scenario command real POSIX li ].join("\n"); const controller = actualSpawn.value( process.execPath, - ["--import", "tsx", "--input-type=module", "-e", controllerScript], + ["--input-type=module", "-e", controllerScript], { cwd: process.cwd(), env: process.env, stdio: "ignore" }, ); try { @@ -355,8 +375,10 @@ describe.skipIf(process.platform === "win32")("qa scenario command lifecycle", ( signal: null, }); expect(spawnSyncMock).toHaveBeenCalledTimes(2); - expect(spawnSyncMock.mock.calls[0]?.[1]).toEqual(["/pid", "12345", "/T"]); - expect(spawnSyncMock.mock.calls[1]?.[1]).toEqual(["/pid", "12345", "/T", "/F"]); + expect(spawnSyncMock.mock.calls.map((call) => call[1])).toEqual([ + ["/PID", "12345", "/T"], + ["/PID", "12345", "/T", "/F"], + ]); } finally { if (platformDescriptor) { Object.defineProperty(process, "platform", platformDescriptor); diff --git a/extensions/qa-lab/src/test-file-scenario-command-lifecycle.ts b/extensions/qa-lab/src/test-file-scenario-command-lifecycle.ts index 3943cfe53c4a..0c54030eaabd 100644 --- a/extensions/qa-lab/src/test-file-scenario-command-lifecycle.ts +++ b/extensions/qa-lab/src/test-file-scenario-command-lifecycle.ts @@ -1,7 +1,7 @@ -import { spawn, spawnSync } from "node:child_process"; +import { spawn } from "node:child_process"; import path from "node:path"; import { createQaPosixCommandSettlement } from "./posix-command-settlement.js"; -import { resolveQaWindowsSystem32ExePath } from "./windows-system-tools.js"; +import { runQaWindowsTaskkill } from "./windows-system-tools.js"; export type QaScenarioCommandExecution = { args: string[]; @@ -24,33 +24,11 @@ type QaScenarioCommandTerminalResult = Pick< "exitCode" | "failureMessage" | "signal" >; -type QaScenarioTaskkillRunner = typeof spawnSync; - const QA_SCENARIO_COMMAND_TIMEOUT_KILL_GRACE_MS = 2_000; const QA_SCENARIO_COMMAND_TIMEOUT_FORCE_SETTLE_MS = 500; let timeoutKillGraceMs = QA_SCENARIO_COMMAND_TIMEOUT_KILL_GRACE_MS; let timeoutForceSettleMs = QA_SCENARIO_COMMAND_TIMEOUT_FORCE_SETTLE_MS; -export function killQaScenarioWindowsProcessTree( - pid: number | undefined, - signal: NodeJS.Signals, - runTaskkill: QaScenarioTaskkillRunner = spawnSync, -) { - if (pid === undefined) { - return false; - } - const taskkillPath = resolveQaWindowsSystem32ExePath("taskkill.exe"); - const args = ["/pid", String(pid), "/T"]; - const run = (force: boolean) => { - const result = runTaskkill(taskkillPath, force ? [...args, "/F"] : args, { - stdio: "ignore", - windowsHide: true, - }); - return !result.error && result.status === 0; - }; - return signal === "SIGKILL" ? run(true) : run(false) || run(true); -} - export function runQaScenarioCommandLifecycle( execution: QaScenarioCommandExecution, ): Promise { @@ -72,10 +50,12 @@ export function runQaScenarioCommandLifecycle( ...(isWindows ? { windowsCleanup: { - alive: () => child.pid !== undefined, signal: (signal: NodeJS.Signals) => { try { - if (!killQaScenarioWindowsProcessTree(child.pid, signal)) { + if ( + child.pid === undefined || + !runQaWindowsTaskkill({ pid: child.pid, signal }) + ) { child.kill(signal); } return undefined; diff --git a/extensions/qa-lab/src/test-file-scenario-runner.docker.test.ts b/extensions/qa-lab/src/test-file-scenario-runner.docker.test.ts new file mode 100644 index 000000000000..956a2a651ddd --- /dev/null +++ b/extensions/qa-lab/src/test-file-scenario-runner.docker.test.ts @@ -0,0 +1,306 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + dockerE2eLaneName, + prepareDockerE2eEnvironment, +} from "./test-file-scenario-docker-batch.js"; +import { + runQaTestFileScenarios, + type QaScenarioCommandExecution, +} from "./test-file-scenario-runner.js"; +import { + QA_TEST_RUNNER_DEFAULTS, + createScenarioRunnerTestHarness, + makeDockerE2eScenario, + writeDockerCandidateManifest, +} from "./test-file-scenario-runner.test-support.js"; + +const harness = createScenarioRunnerTestHarness(); +const makeTempRepo = (prefix: string) => harness.makeTempRepo(prefix); + +afterEach(async () => { + vi.unstubAllEnvs(); + await harness.cleanup(); +}); + +it("only batches the canonical Docker lane argument shape", () => { + const scenario = makeDockerE2eScenario("docker-lane", "gateway-network"); + if (scenario.execution.kind !== "script") { + throw new Error("expected script scenario"); + } + expect(dockerE2eLaneName(scenario)).toBe("gateway-network"); + expect( + dockerE2eLaneName({ + ...scenario, + execution: { ...scenario.execution, args: ["--lane", "gateway-network", "--extra"] }, + }), + ).toBeUndefined(); +}); + +it("prepares the exact Docker lane union in a sanitized bound environment", async () => { + const repoRoot = await makeTempRepo("qa-docker-candidate-"); + const outputDir = path.join(repoRoot, "out"); + const packagePath = path.join(repoRoot, "openclaw.tgz"); + const registryDir = path.join(repoRoot, "registry"); + const runCommand = vi.fn(async (command: QaScenarioCommandExecution) => { + expect(command.env).toMatchObject({ + KEEP_ME: "yes", + OPENCLAW_DOCKER_ALL_LANES: "gateway-network,openai-chat-tools", + OPENCLAW_DOCKER_E2E_REPO_ROOT: repoRoot, + }); + expect(command.env).not.toHaveProperty("OPENCLAW_DOCKER_ALL_BUILD"); + expect(command.env).not.toHaveProperty("OPENCLAW_CURRENT_PACKAGE_TGZ"); + expect(command.env).not.toHaveProperty("OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR"); + return await writeDockerCandidateManifest(command, { + schema: "openclaw.qa-docker-candidate/v1", + schemaVersion: 1, + sourceSha: "a".repeat(40), + candidate: { + package: { + path: packagePath, + name: "openclaw", + version: "2026.8.1", + sha256: "b".repeat(64), + }, + registry: { + dir: registryDir, + candidateVersion: "2026.8.1", + manifestSha256: "c".repeat(64), + }, + }, + }); + }); + const env = await prepareDockerE2eEnvironment({ + env: { + KEEP_ME: "yes", + OPENCLAW_DOCKER_ALL_BUILD: "1", + OPENCLAW_CURRENT_PACKAGE_TGZ: "/stale.tgz", + OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR: "/stale-registry", + }, + outputDir, + repoRoot, + runCommand, + scenarios: [ + makeDockerE2eScenario("one", "gateway-network"), + makeDockerE2eScenario("duplicate", "gateway-network"), + makeDockerE2eScenario("two", "openai-chat-tools"), + ], + }); + + expect(runCommand).toHaveBeenCalledTimes(1); + expect(Object.isFrozen(env)).toBe(true); + expect(env).toEqual({ + KEEP_ME: "yes", + OPENCLAW_DOCKER_E2E_REPO_ROOT: repoRoot, + OPENCLAW_DOCKER_E2E_SELECTED_SHA: "a".repeat(40), + OPENCLAW_CURRENT_PACKAGE_TGZ: packagePath, + OPENCLAW_CURRENT_PACKAGE_VERSION: "2026.8.1", + OPENCLAW_CURRENT_PACKAGE_SHA256: "b".repeat(64), + OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR: registryDir, + OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_CANDIDATE_VERSION: "2026.8.1", + OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_MANIFEST_SHA256: "c".repeat(64), + }); +}); + +it("returns a sanitized bound env for a package-free candidate", async () => { + const repoRoot = await makeTempRepo("qa-docker-candidate-null-"); + const env = await prepareDockerE2eEnvironment({ + env: { + KEEP_ME: "yes", + OPENCLAW_DOCKER_ALL_BUILD: "1", + OPENCLAW_CURRENT_PACKAGE_TGZ: "/stale.tgz", + }, + outputDir: path.join(repoRoot, "out"), + repoRoot, + runCommand: (command) => + writeDockerCandidateManifest(command, { + schema: "openclaw.qa-docker-candidate/v1", + schemaVersion: 1, + sourceSha: "a".repeat(40), + candidate: null, + }), + scenarios: [makeDockerE2eScenario("one", "gateway-network")], + }); + + expect(env).toEqual({ KEEP_ME: "yes", OPENCLAW_DOCKER_E2E_REPO_ROOT: repoRoot }); + expect(Object.isFrozen(env)).toBe(true); +}); + +it.each([ + { label: "extra field", patch: { extra: true } }, + { label: "malformed candidate", patch: { candidate: { package: null, registry: null } } }, +])("rejects a $label in the Docker candidate manifest", async ({ patch }) => { + const repoRoot = await makeTempRepo("qa-docker-candidate-invalid-"); + await expect( + prepareDockerE2eEnvironment({ + env: process.env, + outputDir: path.join(repoRoot, "out"), + repoRoot, + runCommand: (command) => + writeDockerCandidateManifest(command, { + schema: "openclaw.qa-docker-candidate/v1", + schemaVersion: 1, + sourceSha: "a".repeat(40), + candidate: null, + ...patch, + }), + scenarios: [makeDockerE2eScenario("one", "gateway-network")], + }), + ).rejects.toThrow(); +}); + +describe("qa test file scenario runner", () => { + it.each([ + { label: "package", candidate: "package" as const }, + { label: "package-free", candidate: "none" as const }, + ])("keeps hostile inherited Docker state out of a prepared $label run", async ({ candidate }) => { + const repoRoot = await makeTempRepo("qa-docker-replace-env-"); + const packagePath = path.join(repoRoot, "openclaw.tgz"); + vi.stubEnv("OPENCLAW_DOCKER_ALL_POISON", "hostile"); + vi.stubEnv("OPENCLAW_CURRENT_PACKAGE_TGZ", "/hostile.tgz"); + vi.stubEnv("OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR", "/hostile-registry"); + const prepared = await prepareDockerE2eEnvironment({ + env: process.env, + outputDir: path.join(repoRoot, "prep"), + repoRoot, + runCommand: (command) => + writeDockerCandidateManifest(command, { + schema: "openclaw.qa-docker-candidate/v1", + schemaVersion: 1, + sourceSha: "a".repeat(40), + candidate: + candidate === "package" + ? { + package: { + path: packagePath, + name: "openclaw", + version: "2026.8.1", + sha256: "b".repeat(64), + }, + registry: null, + } + : null, + }), + scenarios: [makeDockerE2eScenario("one", "gateway-network")], + }); + + await runQaTestFileScenarios({ + env: prepared, + envMode: "replace", + outputDir: path.join(repoRoot, "run"), + ...QA_TEST_RUNNER_DEFAULTS, + repoRoot, + scenarios: [makeDockerE2eScenario("one", "gateway-network")], + runCommand: async (command) => { + expect(command.env.OPENCLAW_DOCKER_ALL_POISON).toBeUndefined(); + expect(command.env.OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR).toBeUndefined(); + expect(command.env.OPENCLAW_CURRENT_PACKAGE_TGZ).toBe( + candidate === "package" ? packagePath : undefined, + ); + expect(command.env.OPENCLAW_DOCKER_E2E_REPO_ROOT).toBe(repoRoot); + const logDir = command.env.OPENCLAW_DOCKER_ALL_LOG_DIR!; + await fs.mkdir(logDir, { recursive: true }); + await fs.writeFile( + path.join(logDir, "summary.json"), + JSON.stringify({ + failures: [], + lanes: [{ elapsedSeconds: 1, name: "gateway-network", status: 0 }], + selectedLanes: ["gateway-network"], + }), + ); + return { exitCode: 0, stdout: "", stderr: "" }; + }, + }); + }); + + it("preserves individual Docker lane success without generic producer evidence", async () => { + const repoRoot = await makeTempRepo("qa-script-docker-individual-no-producer-evidence-"); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "docker-individual"), + ...QA_TEST_RUNNER_DEFAULTS, + failFast: true, + scenarios: [makeDockerE2eScenario("docker-gateway-network", "gateway-network")], + runCommand: async () => ({ exitCode: 0, stdout: "Docker lane passed\n", stderr: "" }), + }); + + expect(result.results[0]).toMatchObject({ + scenario: { id: "docker-gateway-network" }, + status: "pass", + }); + expect(result.evidence.entries[0]?.result.status).toBe("pass"); + }); + + it("runs Docker script scenarios through one aggregate scheduler invocation", async () => { + const repoRoot = await makeTempRepo("qa-script-docker-batch-"); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "docker-batch"); + const staleSummaryPath = path.join(outputDir, "docker-e2e-1800000ms", "summary.json"); + await fs.mkdir(path.dirname(staleSummaryPath), { recursive: true }); + await fs.writeFile(staleSummaryPath, '{"status":"passed"}\n', "utf8"); + const commands: QaScenarioCommandExecution[] = []; + const scenarios = [ + makeDockerE2eScenario("openai-tools", "openai-chat-tools"), + makeDockerE2eScenario("bundled-plugins", "bundled-plugin-install-uninstall"), + makeDockerE2eScenario("prefix-lane", "gateway"), + makeDockerE2eScenario("failing-lane", "gateway-network"), + ]; + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios, + runCommand: async (command) => { + commands.push(command); + await expect(fs.access(staleSummaryPath)).rejects.toThrow(); + const logDir = command.env.OPENCLAW_DOCKER_ALL_LOG_DIR; + if (!logDir) { + throw new Error("missing Docker scheduler log dir"); + } + await fs.mkdir(logDir, { recursive: true }); + const failedLane = { elapsedSeconds: 2, name: "gateway-network", status: 1 }; + await fs.writeFile( + path.join(logDir, "summary.json"), + `${JSON.stringify({ + failures: [failedLane], + lanes: [ + { elapsedSeconds: 4, name: "openai-chat-tools", status: 0 }, + { elapsedSeconds: 7, name: "bundled-plugin-install-uninstall-0", status: 0 }, + { elapsedSeconds: 6, name: "bundled-plugin-install-uninstall-1", status: 0 }, + { elapsedSeconds: 1, name: "gateway", status: 0 }, + failedLane, + ], + selectedLanes: [ + "openai-chat-tools", + "bundled-plugin-install-uninstall-0", + "bundled-plugin-install-uninstall-1", + "gateway", + "gateway-network", + ], + })}\n`, + "utf8", + ); + return { exitCode: 1, stdout: "", stderr: "scheduler failed\n" }; + }, + }); + + expect(commands).toHaveLength(1); + expect(commands[0]).toMatchObject({ + args: ["scripts/test-docker-all.mjs"], + command: process.execPath, + env: { + OPENCLAW_DOCKER_ALL_FAIL_FAST: "0", + OPENCLAW_DOCKER_ALL_LANES: + "openai-chat-tools,bundled-plugin-install-uninstall,gateway,gateway-network", + OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS: "1800000", + }, + }); + expect(result.results).toMatchObject([ + { scenario: { id: "openai-tools" }, status: "pass" }, + { scenario: { id: "bundled-plugins" }, status: "pass" }, + { scenario: { id: "prefix-lane" }, status: "pass" }, + { scenario: { id: "failing-lane" }, status: "fail" }, + ]); + expect(result.results[3]?.failureMessage).toBe("gateway-network exited with 1"); + }); +}); diff --git a/extensions/qa-lab/src/test-file-scenario-runner.native.test.ts b/extensions/qa-lab/src/test-file-scenario-runner.native.test.ts new file mode 100644 index 000000000000..5e5ca2528ba6 --- /dev/null +++ b/extensions/qa-lab/src/test-file-scenario-runner.native.test.ts @@ -0,0 +1,487 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { validateQaEvidenceSummaryJson } from "./evidence-summary.js"; +import { + runQaTestFileScenarios, + type QaScenarioCommandExecution, +} from "./test-file-scenario-runner.js"; +import { + QA_TEST_RUNNER_DEFAULTS, + createScenarioRunnerTestHarness, + makeTestFileScenario, + writeNativeVitestReport, +} from "./test-file-scenario-runner.test-support.js"; + +const harness = createScenarioRunnerTestHarness(); +const makeTempRepo = (prefix: string) => harness.makeTempRepo(prefix); + +afterEach(async () => { + await harness.cleanup(); +}); + +describe("qa test file scenario runner", () => { + it("runs Playwright scenarios with the repo UI e2e command and writes Playwright evidence", async () => { + const repoRoot = await makeTempRepo("qa-playwright-scenario-"); + const commands: QaScenarioCommandExecution[] = []; + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-playwright"), + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [ + makeTestFileScenario( + "playwright", + "ui/src/e2e/chat-flow.e2e.test.ts", + "sends a chat turn through the GUI", + ), + ], + runCommand: async (command) => { + commands.push(command); + await writeNativeVitestReport(command, { passed: 1 }); + return { + exitCode: 0, + stdout: "pass\n", + stderr: "", + }; + }, + env: { + OPENCLAW_QA_REF: "scenario-ref", + } as NodeJS.ProcessEnv, + }); + + expect(result.executionKind).toBe("playwright"); + expect(commands.map((command) => command.args)).toEqual([ + ["--import", "tsx", "scripts/ensure-playwright-chromium.mts"], + [ + "scripts/run-vitest.mjs", + "run", + "--config", + "test/vitest/vitest.ui-e2e.config.ts", + "--configLoader", + "runner", + "ui/src/e2e/chat-flow.e2e.test.ts", + "--reporter=verbose", + "--reporter=json", + `--outputFile.json=${path.join( + repoRoot, + ".artifacts", + "qa-e2e", + "scenario-playwright", + "scenario-playwright.vitest-report.json", + )}`, + "--testNamePattern", + "sends a chat turn through the GUI", + ], + ]); + expect(commands.map((command) => command.timeoutMs)).toEqual([1_800_000, 1_800_000]); + const evidence = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(result.evidencePath, "utf8")), + ); + expect(evidence.schemaVersion).toBe(2); + expect(evidence.entries).toHaveLength(1); + expect(evidence.entries[0]).toMatchObject({ + test: { + kind: "playwright-test", + id: "scenario-playwright", + source: { + path: "ui/src/e2e/chat-flow.e2e.test.ts", + }, + }, + coverage: [ + { + id: "ui.control", + role: "primary", + }, + { + id: "ui.streaming", + role: "secondary", + }, + ], + refs: [ + { + kind: "docs", + path: "docs/concepts/qa-e2e-automation.md", + }, + { + kind: "code", + path: "ui/src/e2e/chat-flow.e2e.test.ts", + }, + ], + execution: { + runner: "playwright", + artifacts: [ + { + kind: "log", + path: ".artifacts/qa-e2e/scenario-playwright/scenario-playwright.log", + source: "playwright", + }, + ], + }, + result: { + status: "pass", + }, + }); + }); + + it("can return aggregate evidence without writing a duplicate evidence file", async () => { + const repoRoot = await makeTempRepo("qa-playwright-memory-evidence-"); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-playwright"), + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [makeTestFileScenario("playwright", "ui/src/e2e/chat-flow.e2e.test.ts")], + writeEvidenceFile: false, + runCommand: async (command) => { + await writeNativeVitestReport(command, { passed: 1 }); + return { + exitCode: 0, + stdout: "pass\n", + stderr: "", + }; + }, + }); + + expect(result.evidence.entries).toHaveLength(1); + await expect(fs.access(result.evidencePath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("runs Vitest scenarios with the declared test path and writes Vitest evidence", async () => { + const repoRoot = await makeTempRepo("qa-vitest-scenario-"); + const commands: QaScenarioCommandExecution[] = []; + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-vitest"), + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [makeTestFileScenario("vitest", "extensions/qa-lab/src/coverage-report.test.ts")], + runCommand: async (command) => { + commands.push(command); + return { + exitCode: 1, + stdout: "", + stderr: "failed\n", + }; + }, + }); + + expect(result.executionKind).toBe("vitest"); + expect(commands.map((command) => command.args)).toEqual([ + [ + "scripts/run-vitest.mjs", + "extensions/qa-lab/src/coverage-report.test.ts", + "--reporter=verbose", + "--reporter=json", + `--outputFile.json=${path.join( + repoRoot, + ".artifacts", + "qa-e2e", + "scenario-vitest", + "scenario-vitest.vitest-report.json", + )}`, + ], + ]); + expect(commands.map((command) => command.timeoutMs)).toEqual([1_800_000]); + const evidence = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(result.evidencePath, "utf8")), + ); + expect(evidence.entries[0]).toMatchObject({ + test: { + kind: "vitest-test", + id: "scenario-vitest", + source: { + path: "extensions/qa-lab/src/coverage-report.test.ts", + }, + }, + coverage: [ + { + id: "qa.coverage", + role: "primary", + }, + { + id: "qa.reporting", + role: "secondary", + }, + ], + execution: { + runner: "vitest", + artifacts: [ + { + kind: "log", + path: ".artifacts/qa-e2e/scenario-vitest/scenario-vitest.log", + source: "vitest", + }, + ], + }, + result: { + status: "fail", + failure: { + reason: "node exited with 1", + }, + }, + }); + }); + + it.each([ + { executionKind: "vitest" as const, passed: 0, expectedStatus: "fail" as const }, + { executionKind: "playwright" as const, passed: 0, expectedStatus: "fail" as const }, + { executionKind: "vitest" as const, passed: 1, expectedStatus: "pass" as const }, + { executionKind: "playwright" as const, passed: 1, expectedStatus: "pass" as const }, + ])( + "requires an actually passed $executionKind test when the native child exits successfully ($passed passed)", + async ({ executionKind, expectedStatus, passed }) => { + const repoRoot = await makeTempRepo(`qa-${executionKind}-executed-tests-`); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`); + const scenarioPath = + executionKind === "playwright" + ? "ui/src/e2e/chat-flow.e2e.test.ts" + : "extensions/qa-lab/src/coverage-report.test.ts"; + const commands: QaScenarioCommandExecution[] = []; + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [makeTestFileScenario(executionKind, scenarioPath)], + runCommand: async (command) => { + commands.push(command); + await writeNativeVitestReport(command, { passed }); + return { exitCode: 0, stdout: "child exited successfully\n", stderr: "" }; + }, + }); + + expect(result.results[0]).toMatchObject({ status: expectedStatus }); + expect(result.evidence.entries[0]?.result.status).toBe(expectedStatus); + expect( + commands.filter((command) => command.args[0] === "scripts/run-vitest.mjs"), + ).toHaveLength(1); + if (expectedStatus === "fail") { + expect(result.results[0]?.failureMessage).toBe( + "Vitest exited successfully without reporting a successfully executed test.", + ); + } + }, + ); + + it.each([{ executionKind: "vitest" as const }, { executionKind: "playwright" as const }])( + "rejects a passing $executionKind report for an unrelated test file", + async ({ executionKind }) => { + const repoRoot = await makeTempRepo(`qa-${executionKind}-wrong-report-file-`); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [ + makeTestFileScenario( + executionKind, + executionKind === "playwright" + ? "ui/src/e2e/chat-flow.e2e.test.ts" + : "extensions/qa-lab/src/coverage-report.test.ts", + ), + ], + runCommand: async (command) => { + await writeNativeVitestReport(command, { + passed: 1, + testFilePath: "extensions/qa-lab/src/unrelated.test.ts", + }); + return { exitCode: 0, stdout: "unrelated test passed\n", stderr: "" }; + }, + }); + + expect(result.results[0]).toMatchObject({ + failureMessage: expect.stringContaining("requested test file"), + status: "fail", + }); + expect(result.evidence.entries[0]?.result.status).toBe("fail"); + }, + ); + + it.each([{ executionKind: "vitest" as const }, { executionKind: "playwright" as const }])( + "rejects a passing $executionKind report when the requested test file does not exist", + async ({ executionKind }) => { + const repoRoot = await makeTempRepo(`qa-${executionKind}-missing-requested-test-`); + const scenarioPath = + executionKind === "playwright" + ? "ui/src/e2e/chat-flow.e2e.test.ts" + : "extensions/qa-lab/src/coverage-report.test.ts"; + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`), + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [makeTestFileScenario(executionKind, scenarioPath)], + runCommand: async (command) => { + await writeNativeVitestReport(command, { + createRequestedTestFile: false, + passed: 1, + }); + return { exitCode: 0, stdout: "missing test reportedly passed\n", stderr: "" }; + }, + }); + + expect(result.results[0]).toMatchObject({ + failureMessage: expect.stringContaining("existing requested test file"), + status: "fail", + }); + expect(result.evidence.entries[0]?.result.status).toBe("fail"); + }, + ); + + it.skipIf(process.platform === "win32")( + "authenticates requested tests when the checkout root is a symlink", + async () => { + const canonicalRoot = await fs.realpath(await makeTempRepo("qa-vitest-symlinked-checkout-")); + const symlinkedRoot = path.join(canonicalRoot, "checkout-alias"); + await fs.symlink(canonicalRoot, symlinkedRoot, "dir"); + const scenarioPath = "extensions/qa-lab/src/coverage-report.test.ts"; + const result = await runQaTestFileScenarios({ + repoRoot: symlinkedRoot, + outputDir: path.join(symlinkedRoot, ".artifacts", "qa-e2e", "scenario-vitest"), + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [makeTestFileScenario("vitest", scenarioPath)], + runCommand: async (command) => { + await writeNativeVitestReport(command, { + passed: 1, + testFilePath: path.join(canonicalRoot, scenarioPath), + }); + return { exitCode: 0, stdout: "canonical test passed\n", stderr: "" }; + }, + }); + + expect(result.results[0]).toMatchObject({ status: "pass" }); + expect(result.evidence.entries[0]?.result.status).toBe("pass"); + }, + ); + + it("rejects a passing Playwright report that misses the requested test name", async () => { + const repoRoot = await makeTempRepo("qa-playwright-wrong-report-test-"); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-playwright"), + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [ + makeTestFileScenario( + "playwright", + "ui/src/e2e/chat-flow.e2e.test.ts", + "required visual assertion", + ), + ], + runCommand: async (command) => { + await writeNativeVitestReport(command, { + passed: 1, + testName: "unrelated visual assertion", + }); + return { exitCode: 0, stdout: "unrelated assertion passed\n", stderr: "" }; + }, + }); + + expect(result.results[0]).toMatchObject({ + failureMessage: expect.stringContaining("requested test name"), + status: "fail", + }); + expect(result.evidence.entries[0]?.result.status).toBe("fail"); + }); + + it("records invalid Playwright test-name patterns as failed scenario evidence", async () => { + const repoRoot = await makeTempRepo("qa-playwright-invalid-report-pattern-"); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-playwright"), + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [makeTestFileScenario("playwright", "ui/src/e2e/chat-flow.e2e.test.ts", "[")], + runCommand: async (command) => { + await writeNativeVitestReport(command, { + passed: 1, + testName: "executed visual assertion", + }); + return { exitCode: 0, stdout: "visual assertion passed\n", stderr: "" }; + }, + }); + + expect(result.results[0]).toMatchObject({ + failureMessage: expect.stringContaining("invalid requested test name pattern"), + status: "fail", + }); + expect(result.evidence.entries[0]?.result.status).toBe("fail"); + }); + + it.each([{ executionKind: "vitest" as const }, { executionKind: "playwright" as const }])( + "does not reuse a prior passing $executionKind report when the next child writes none", + async ({ executionKind }) => { + const repoRoot = await makeTempRepo(`qa-${executionKind}-stale-vitest-report-`); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`); + const scenarioPath = + executionKind === "playwright" + ? "ui/src/e2e/chat-flow.e2e.test.ts" + : "extensions/qa-lab/src/coverage-report.test.ts"; + const reportPath = path.join(outputDir, `scenario-${executionKind}.vitest-report.json`); + let writeReport = true; + const runParams = { + repoRoot, + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [makeTestFileScenario(executionKind, scenarioPath)], + runCommand: async (command: QaScenarioCommandExecution) => { + if (writeReport) { + await writeNativeVitestReport(command, { passed: 1 }); + } + return { exitCode: 0, stdout: "child exited successfully\n", stderr: "" }; + }, + }; + + const firstRun = await runQaTestFileScenarios(runParams); + expect(firstRun.results[0]).toMatchObject({ status: "pass" }); + await expect(fs.access(reportPath)).resolves.toBeUndefined(); + + writeReport = false; + const secondRun = await runQaTestFileScenarios(runParams); + expect(secondRun.results[0]).toMatchObject({ + failureMessage: `Vitest exited successfully without writing a valid JSON test report at ${reportPath}.`, + status: "fail", + }); + expect(secondRun.evidence.entries[0]?.result.status).toBe("fail"); + await expect(fs.access(reportPath)).rejects.toMatchObject({ code: "ENOENT" }); + }, + ); + + it.each([ + { failFast: true, expectedScenarioIds: ["first-native-scenario"] }, + { + failFast: false, + expectedScenarioIds: ["first-native-scenario", "later-native-scenario"], + }, + { + failFast: undefined, + expectedScenarioIds: ["first-native-scenario", "later-native-scenario"], + }, + ])( + "honors native scenario fail-fast mode ($failFast)", + async ({ failFast, expectedScenarioIds }) => { + const repoRoot = await makeTempRepo("qa-vitest-fail-fast-"); + const runCommand = vi.fn(async () => ({ + exitCode: 1, + stdout: "", + stderr: "native scenario failed\n", + })); + const firstScenario = { + ...makeTestFileScenario("vitest", "extensions/qa-lab/src/coverage-report.test.ts"), + id: "first-native-scenario", + }; + const laterScenario = { + ...makeTestFileScenario("vitest", "extensions/qa-lab/src/cli.test.ts"), + id: "later-native-scenario", + }; + + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "native-fail-fast"), + ...QA_TEST_RUNNER_DEFAULTS, + failFast, + scenarios: [firstScenario, laterScenario], + runCommand, + }); + + expect(runCommand).toHaveBeenCalledTimes(expectedScenarioIds.length); + expect(result.results.map((scenario) => scenario.scenario.id)).toEqual(expectedScenarioIds); + expect(result.results.every((scenario) => scenario.status === "fail")).toBe(true); + expect(result.evidence.entries.map((entry) => entry.test.id)).toEqual(expectedScenarioIds); + }, + ); +}); diff --git a/extensions/qa-lab/src/test-file-scenario-runner.process.test.ts b/extensions/qa-lab/src/test-file-scenario-runner.process.test.ts new file mode 100644 index 000000000000..3fbe9b1d3440 --- /dev/null +++ b/extensions/qa-lab/src/test-file-scenario-runner.process.test.ts @@ -0,0 +1,120 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { runQaScenarioCommandLifecycle } from "./test-file-scenario-command-lifecycle.js"; +import { + runQaTestFileScenarios, + type QaScenarioCommandExecution, +} from "./test-file-scenario-runner.js"; +import { + QA_TEST_RUNNER_DEFAULTS, + createScenarioRunnerTestHarness, + makeTestFileScenario, + writeNativeVitestReport, +} from "./test-file-scenario-runner.test-support.js"; + +const harness = createScenarioRunnerTestHarness(); +const makeTempDir = (prefix: string) => harness.makeTempDir(prefix); +const makeTempRepo = (prefix: string) => harness.makeTempRepo(prefix); + +afterEach(async () => { + await harness.cleanup(); +}); + +describe("qa test file scenario runner", () => { + it.each([ + { executionKind: "vitest" as const, commandCount: 1 }, + { executionKind: "playwright" as const, commandCount: 2 }, + ])( + "applies the resolved command timeout to every $executionKind subprocess", + async ({ commandCount, executionKind }) => { + const repoRoot = await makeTempRepo(`qa-${executionKind}-command-timeout-`); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`); + const commands: QaScenarioCommandExecution[] = []; + + await runQaTestFileScenarios({ + repoRoot, + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [ + makeTestFileScenario( + executionKind, + executionKind === "playwright" + ? "ui/src/e2e/chat-flow.e2e.test.ts" + : "extensions/qa-lab/src/coverage-report.test.ts", + ), + ], + commandTimeoutMs: 321, + runCommand: async (command) => { + commands.push(command); + await writeNativeVitestReport(command, { passed: 1 }); + return { exitCode: 0, stdout: "native pass\n", stderr: "" }; + }, + }); + + expect(commands).toHaveLength(commandCount); + expect(commands.map((command) => command.timeoutMs)).toEqual( + Array.from({ length: commandCount }, () => 321), + ); + }, + ); + + it.each(["vitest", "playwright"] as const)( + "terminates a hanging $executionKind subprocess with failure evidence", + async (executionKind) => { + const repoRoot = await makeTempRepo(`qa-${executionKind}-hung-command-`); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [ + makeTestFileScenario( + executionKind, + executionKind === "playwright" + ? "ui/src/e2e/chat-flow.e2e.test.ts" + : "extensions/qa-lab/src/coverage-report.test.ts", + ), + ], + commandTimeoutMs: 100, + runCommand: (execution) => + runQaScenarioCommandLifecycle({ + ...execution, + args: ["-e", "setInterval(() => {}, 1_000)"], + }), + }); + + expect(result.results[0]).toMatchObject({ + failureMessage: expect.stringContaining("timed out after 100ms"), + status: "fail", + }); + expect(result.evidence.entries[0]?.result.status).toBe("fail"); + }, + ); + + it("fails script scenarios that exit cleanly after timeout termination", async () => { + const repoRoot = process.cwd(); + const tempRoot = await makeTempDir("qa-script-timeout-clean-exit-"); + const scriptPath = path.join(tempRoot, "clean-exit-after-timeout.ts"); + await fs.writeFile( + scriptPath, + [ + "process.stdout.write('waiting for timeout\\n');", + "process.on('SIGTERM', () => process.exit(0));", + "setInterval(() => {}, 1000);", + ].join("\n"), + "utf8", + ); + + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir: path.join(tempRoot, "out"), + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [makeTestFileScenario("script", scriptPath)], + commandTimeoutMs: 100, + }); + + expect(result.results[0]?.status).toBe("fail"); + expect(result.results[0]?.failureMessage).toMatch(/timed out after 100ms/u); + }); +}); diff --git a/extensions/qa-lab/src/test-file-scenario-runner.script-evidence.test.ts b/extensions/qa-lab/src/test-file-scenario-runner.script-evidence.test.ts new file mode 100644 index 000000000000..3bb3cfefefc3 --- /dev/null +++ b/extensions/qa-lab/src/test-file-scenario-runner.script-evidence.test.ts @@ -0,0 +1,594 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { validateQaEvidenceSummaryJson } from "./evidence-summary.js"; +import type { QaSeedScenarioWithSource } from "./scenario-catalog.js"; +import { + runQaTestFileScenarios, + type QaScenarioCommandExecution, +} from "./test-file-scenario-runner.js"; +import { + buildScriptProducerEvidence, + QA_TEST_RUNNER_DEFAULTS, + createScenarioRunnerTestHarness, + makeTestFileScenario, + writeScriptProducerEvidence, +} from "./test-file-scenario-runner.test-support.js"; + +const harness = createScenarioRunnerTestHarness(); +const makeTempRepo = (prefix: string) => harness.makeTempRepo(prefix); + +afterEach(async () => { + await harness.cleanup(); +}); + +describe("qa test file scenario runner", () => { + it.each([ + { evidence: "missing", expectedFailure: /without writing fresh producer QA evidence/u }, + { evidence: "stale", expectedFailure: /without writing fresh producer QA evidence/u }, + { evidence: "empty", expectedFailure: /without reporting an executed producer check/u }, + { evidence: "malformed", expectedFailure: /invalid JSON/u }, + { evidence: "outside", expectedFailure: /inside its scenario output directory/u }, + ] as const)( + "fails a successful script with $evidence producer evidence", + async ({ evidence, expectedFailure }) => { + const repoRoot = await makeTempRepo(`qa-script-${evidence}-producer-evidence-`); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script"); + const scenarioOutputDir = path.join(outputDir, "scenario-script"); + const latestRunPath = path.join(scenarioOutputDir, "latest-run.json"); + const evidencePath = path.join(scenarioOutputDir, "qa-evidence.json"); + + if (evidence === "stale") { + await writeScriptProducerEvidence({ outputDir, status: "pass" }); + const staleEvidencePath = path.join(scenarioOutputDir, "run-1", "qa-evidence.json"); + await fs.copyFile(staleEvidencePath, evidencePath); + const staleTimestamp = new Date(Date.now() - 60_000); + await Promise.all([ + fs.utimes(staleEvidencePath, staleTimestamp, staleTimestamp), + fs.utimes(evidencePath, staleTimestamp, staleTimestamp), + ]); + } + + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], + runCommand: async () => { + await fs.mkdir(scenarioOutputDir, { recursive: true }); + if (evidence === "stale") { + await expect(fs.access(latestRunPath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fs.access(evidencePath)).rejects.toMatchObject({ code: "ENOENT" }); + await fs.writeFile( + latestRunPath, + JSON.stringify({ + qaEvidence: path.join(scenarioOutputDir, "run-1", "qa-evidence.json"), + }), + "utf8", + ); + } else if (evidence === "empty") { + await fs.writeFile( + evidencePath, + JSON.stringify({ + kind: "openclaw.qa.evidence-summary", + schemaVersion: 2, + generatedAt: new Date().toISOString(), + evidenceMode: "full", + entries: [], + }), + "utf8", + ); + } else if (evidence === "malformed") { + await fs.writeFile(evidencePath, "{not valid JSON", "utf8"); + } else if (evidence === "outside") { + await writeScriptProducerEvidence({ + outputDir, + scenarioId: "different-script-scenario", + status: "pass", + }); + await fs.writeFile( + latestRunPath, + JSON.stringify({ + qaEvidence: path.join( + outputDir, + "different-script-scenario", + "run-1", + "qa-evidence.json", + ), + }), + "utf8", + ); + } + return { exitCode: 0, stdout: "script exited successfully\n", stderr: "" }; + }, + }); + + expect(result.results[0]).toMatchObject({ status: "fail" }); + expect(result.results[0]?.failureMessage).toMatch(expectedFailure); + expect(result.evidence.entries).toHaveLength(1); + expect(result.evidence.entries[0]).toMatchObject({ + test: { id: "scenario-script" }, + result: { status: "fail" }, + }); + }, + ); + + it("runs script scenarios and imports producer QA evidence artifacts", async () => { + const repoRoot = await makeTempRepo("qa-script-scenario-"); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script"); + const commands: QaScenarioCommandExecution[] = []; + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], + runCommand: async (command) => { + commands.push(command); + const runRoot = path.join(outputDir, "scenario-script", "run-1"); + await fs.mkdir(path.join(runRoot, "surfaces", "web-ui"), { recursive: true }); + await fs.writeFile(path.join(runRoot, "surfaces", "web-ui", "screenshot.png"), "png"); + await writeScriptProducerEvidence({ + artifacts: [ + { + kind: "screenshot", + path: "surfaces/web-ui/screenshot.png", + source: "script-producer:web-ui:smoke", + }, + ], + outputDir, + status: "pass", + }); + return { exitCode: 0, stdout: "script pass\n", stderr: "" }; + }, + env: { OPENCLAW_QA_REF: "scenario-ref" } as NodeJS.ProcessEnv, + }); + + expect(result.executionKind).toBe("script"); + expect(commands.map((command) => command.args)).toEqual([ + [ + "--import", + "tsx", + "scripts/evidence-producer.ts", + "--once", + "--artifact-base", + path.join(outputDir, "scenario-script"), + ], + ]); + expect(commands.map((command) => command.timeoutMs)).toEqual([30 * 60_000]); + const evidence = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(result.evidencePath, "utf8")), + ); + expect(evidence.entries).toHaveLength(1); + expect(evidence.entries[0]).toMatchObject({ + test: { kind: "script-producer-check", id: "script-producer.web-ui.smoke" }, + coverage: [ + { id: "qa.coverage", role: "primary" }, + { id: "qa.reporting", role: "secondary" }, + ], + execution: { + runner: "evidence-producer-script", + artifacts: [ + { + kind: "screenshot", + path: ".artifacts/qa-e2e/scenario-script/scenario-script/run-1/surfaces/web-ui/screenshot.png", + source: "script-producer:web-ui:smoke", + }, + ], + }, + result: { status: "pass" }, + }); + }); + it("uses script scenario timeout overrides when running producer commands", async () => { + const repoRoot = await makeTempRepo("qa-script-scenario-timeout-"); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-timeout"); + const scenario = makeTestFileScenario("script", "scripts/evidence-producer.ts"); + if (scenario.execution.kind !== "script") { + throw new Error("expected script scenario"); + } + scenario.execution.timeoutMs = 3 * 60 * 60_000; + + const commands: QaScenarioCommandExecution[] = []; + await runQaTestFileScenarios({ + repoRoot, + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [scenario], + commandTimeoutMs: 30 * 60_000, + runCommand: async (command) => { + commands.push(command); + await writeScriptProducerEvidence({ + outputDir, + status: "pass", + }); + return { + exitCode: 0, + stdout: "script pass\n", + stderr: "", + }; + }, + env: { + OPENCLAW_QA_REF: "scenario-ref", + } as NodeJS.ProcessEnv, + }); + + expect(commands.map((command) => command.timeoutMs)).toEqual([3 * 60 * 60_000]); + }); + + it("imports producer QA evidence artifacts from failed script scenarios", async () => { + const repoRoot = await makeTempRepo("qa-script-failed-scenario-"); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-failed"); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], + runCommand: async () => { + await writeScriptProducerEvidence({ + failureReason: "Script producer check failed.", + outputDir, + status: "fail", + }); + return { exitCode: 1, stdout: "", stderr: "script failed\n" }; + }, + env: { OPENCLAW_QA_REF: "scenario-ref" } as NodeJS.ProcessEnv, + }); + + expect(result.results[0]).toMatchObject({ + status: "fail", + failureMessage: "node exited with 1", + producerEvidence: { + entries: [ + { + test: { id: "script-producer.web-ui.smoke" }, + result: { status: "fail" }, + }, + ], + }, + }); + const evidence = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(result.evidencePath, "utf8")), + ); + expect(evidence.entries).toHaveLength(2); + expect(evidence.entries[0]).toMatchObject({ + test: { kind: "script-producer-check", id: "script-producer.web-ui.smoke" }, + coverage: [ + { id: "qa.coverage", role: "primary" }, + { id: "qa.reporting", role: "secondary" }, + ], + result: { + status: "fail", + failure: { reason: "Script producer check failed." }, + }, + }); + expect(evidence.entries[1]).toMatchObject({ + test: { + kind: "script-test", + id: "scenario-script", + source: { path: "scripts/evidence-producer.ts" }, + }, + result: { + status: "fail", + failure: { reason: "node exited with 1" }, + }, + }); + }); + it("suppresses a failed-script fallback row already owned by producer scenario evidence", async () => { + const repoRoot = await makeTempRepo("qa-script-duplicate-scenario-evidence-"); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-duplicate"); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], + runCommand: async () => { + await writeScriptProducerEvidence({ + outputDir, + producerId: "scenario-script", + status: "fail", + failureReason: "producer recorded the script failure", + }); + return { exitCode: 1, stdout: "", stderr: "script failed\n" }; + }, + env: { OPENCLAW_QA_REF: "scenario-ref" } as NodeJS.ProcessEnv, + }); + + expect(result.results[0]).toMatchObject({ status: "fail" }); + expect(result.evidence.entries).toHaveLength(1); + expect(result.evidence.entries[0]).toMatchObject({ + test: { id: "scenario-script" }, + result: { failure: { reason: "producer recorded the script failure" }, status: "fail" }, + }); + }); + + it("fails script scenario results when imported producer evidence fails", async () => { + const repoRoot = await makeTempRepo("qa-script-producer-fail-"); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-producer-fail"); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], + runCommand: async () => { + await writeScriptProducerEvidence({ + failureReason: "Script producer check failed.", + outputDir, + status: "fail", + }); + return { exitCode: 0, stdout: "script pass\n", stderr: "" }; + }, + env: { OPENCLAW_QA_REF: "scenario-ref" } as NodeJS.ProcessEnv, + }); + + expect(result.results[0]).toMatchObject({ + status: "fail", + failureMessage: "Script producer check failed.", + }); + const evidence = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(result.evidencePath, "utf8")), + ); + expect(evidence.entries).toHaveLength(1); + expect(evidence.entries[0]).toMatchObject({ + test: { id: "script-producer.web-ui.smoke" }, + result: { status: "fail" }, + }); + }); + it("fails script scenario results when imported producer evidence is blocked by default", async () => { + const repoRoot = await makeTempRepo("qa-script-producer-blocked-"); + const outputDir = path.join( + repoRoot, + ".artifacts", + "qa-e2e", + "scenario-script-producer-blocked", + ); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], + runCommand: async () => { + await writeScriptProducerEvidence({ + outputDir, + status: "blocked", + failureReason: "Playwright browser is missing.", + }); + return { + exitCode: 0, + stdout: "script blocked\n", + stderr: "", + }; + }, + env: { + OPENCLAW_QA_REF: "scenario-ref", + } as NodeJS.ProcessEnv, + }); + + expect(result.results[0]).toMatchObject({ + status: "blocked", + failureMessage: "Playwright browser is missing.", + }); + }); + + it("keeps all-blocked producer evidence blocked for opt-in script scenarios", async () => { + const repoRoot = await makeTempRepo("qa-script-producer-blocked-allowed-"); + const outputDir = path.join( + repoRoot, + ".artifacts", + "qa-e2e", + "scenario-script-producer-blocked-allowed", + ); + const scenario = makeTestFileScenario("script", "scripts/evidence-producer.ts"); + if (scenario.execution.kind !== "script") { + throw new Error("expected script scenario"); + } + scenario.execution.allowBlockedEvidence = true; + + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [scenario], + runCommand: async () => { + await writeScriptProducerEvidence({ + outputDir, + status: "blocked", + failureReason: "Playwright browser is missing.", + }); + return { + exitCode: 0, + stdout: "script blocked\n", + stderr: "", + }; + }, + env: { + OPENCLAW_QA_REF: "scenario-ref", + } as NodeJS.ProcessEnv, + }); + + expect(result.results[0]).toMatchObject({ + status: "blocked", + failureMessage: "Playwright browser is missing.", + producerEvidence: { + entries: [ + { + test: { + id: "script-producer.web-ui.smoke", + }, + result: { + status: "blocked", + }, + }, + ], + }, + }); + }); + + it("allows blocked producer checks when another check genuinely passes", async () => { + const repoRoot = await makeTempRepo("qa-script-producer-blocked-mixed-"); + const outputDir = path.join( + repoRoot, + ".artifacts", + "qa-e2e", + "scenario-script-producer-blocked-mixed", + ); + const scenario = makeTestFileScenario("script", "scripts/evidence-producer.ts"); + if (scenario.execution.kind !== "script") { + throw new Error("expected script scenario"); + } + scenario.execution.allowBlockedEvidence = true; + + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [scenario], + runCommand: async () => { + await writeScriptProducerEvidence({ + additionalEntries: buildScriptProducerEvidence({ + producerId: "script-producer.web-ui.executed", + status: "pass", + }).entries, + outputDir, + status: "blocked", + failureReason: "Playwright browser is missing.", + }); + return { + exitCode: 0, + stdout: "script mixed\n", + stderr: "", + }; + }, + env: { + OPENCLAW_QA_REF: "scenario-ref", + } as NodeJS.ProcessEnv, + }); + + expect(result.results[0]).toMatchObject({ + status: "pass", + producerEvidence: { + entries: [{ result: { status: "blocked" } }, { result: { status: "pass" } }], + }, + }); + }); + + it("carries the suite profile into merged producer evidence", async () => { + const repoRoot = await makeTempRepo("qa-script-profile-"); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-profile"); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], + runCommand: async () => { + await writeScriptProducerEvidence({ + evidenceLocation: "scenario-root", + latestRun: "none", + outputDir, + profile: "smoke-ci", + status: "pass", + }); + return { exitCode: 0, stdout: "script pass\n", stderr: "" }; + }, + env: { + OPENCLAW_QA_REF: "scenario-ref", + OPENCLAW_QA_PROFILE: "smoke-ci", + } as NodeJS.ProcessEnv, + }); + + expect(result.evidence.profile).toBe("smoke-ci"); + }); + it("keeps producer artifacts outside the repo root absolute instead of emitting ../ paths", async () => { + const repoRoot = await makeTempRepo("qa-script-external-artifact-"); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-external"); + const externalArtifact = path.join(os.tmpdir(), "qa-external-artifact.png"); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], + runCommand: async () => { + await writeScriptProducerEvidence({ + artifacts: [{ kind: "screenshot", path: externalArtifact }], + outputDir, + status: "pass", + }); + return { exitCode: 0, stdout: "script pass\n", stderr: "" }; + }, + env: { OPENCLAW_QA_REF: "scenario-ref" } as NodeJS.ProcessEnv, + }); + + const artifactPath = result.evidence.entries[0]?.execution?.artifacts[0]?.path; + expect(artifactPath).toBe(path.normalize(externalArtifact)); + expect(artifactPath?.includes("..")).toBe(false); + }); + it("imports coverage-free structured evidence through the real script lifecycle", async () => { + const tempRoot = await harness.makeTempDir("qa-script-real-evidence-"); + const outputDir = path.join(tempRoot, "out"); + const scriptPath = path.join(tempRoot, "minimal-evidence-producer.mjs"); + const producerEvidence = buildScriptProducerEvidence({ + artifacts: [{ kind: "log", path: "artifact.log" }], + coverage: [], + status: "pass", + }); + await fs.writeFile( + scriptPath, + [ + "import fs from 'node:fs/promises';", + "import path from 'node:path';", + "const artifactBaseIndex = process.argv.indexOf('--artifact-base');", + "if (artifactBaseIndex < 0) throw new Error('missing --artifact-base');", + "const artifactBase = process.argv[artifactBaseIndex + 1];", + "const runRoot = path.join(artifactBase, 'run-1');", + "await fs.mkdir(runRoot, { recursive: true });", + "await fs.writeFile(path.join(runRoot, 'artifact.log'), 'structured evidence\\n', 'utf8');", + `const evidence = ${JSON.stringify(producerEvidence)};`, + "await fs.writeFile(path.join(runRoot, 'qa-evidence.json'), JSON.stringify(evidence), 'utf8');", + "await fs.writeFile(path.join(artifactBase, 'latest-run.json'), JSON.stringify({ qaEvidence: 'run-1/qa-evidence.json' }), 'utf8');", + ].join("\n"), + "utf8", + ); + const infrastructureFixture: QaSeedScenarioWithSource = { + id: "scenario-script", + title: "Temporary script evidence fixture", + surface: "qa-lab", + objective: "Exercise structured evidence import through the real script lifecycle.", + successCriteria: ["The runner imports coverage-free producer evidence and artifacts."], + codeRefs: ["external/qa/minimal-evidence-producer.mjs"], + sourcePath: "external/qa/minimal-evidence-scenario.yaml", + execution: { + kind: "script", + path: scriptPath, + args: ["--artifact-base", "${outputDir}"], + }, + }; + + const result = await runQaTestFileScenarios({ + repoRoot: process.cwd(), + outputDir, + ...QA_TEST_RUNNER_DEFAULTS, + scenarios: [infrastructureFixture], + commandTimeoutMs: 20_000, + env: { OPENCLAW_QA_REF: "temporary-script-fixture" } as NodeJS.ProcessEnv, + }); + + expect(result.executionKind).toBe("script"); + expect(result.results[0]).toMatchObject({ + status: "pass", + producerEvidence: { + entries: [{ test: { id: "script-producer.web-ui.smoke" } }], + }, + }); + expect(result.evidence.entries[0]).toMatchObject({ + coverage: [], + execution: { + artifacts: [ + { + kind: "log", + path: path.join(outputDir, "scenario-script", "run-1", "artifact.log"), + }, + ], + }, + test: { id: "script-producer.web-ui.smoke" }, + }); + }); +}); diff --git a/extensions/qa-lab/src/test-file-scenario-runner.test-support.ts b/extensions/qa-lab/src/test-file-scenario-runner.test-support.ts new file mode 100644 index 000000000000..6bd8a0e49d55 --- /dev/null +++ b/extensions/qa-lab/src/test-file-scenario-runner.test-support.ts @@ -0,0 +1,230 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { + QA_EVIDENCE_FILENAME, + QA_EVIDENCE_SUMMARY_KIND, + QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, + type QaEvidenceSummaryJson, + validateQaEvidenceSummaryJson, +} from "./evidence-summary.js"; +import type { QaSeedScenarioWithSource } from "./scenario-catalog.js"; +import { createTempDirHarness } from "./temp-dir.test-helper.js"; +import type { + QaScenarioCommandExecution, + runQaTestFileScenarios, +} from "./test-file-scenario-runner.js"; + +export const QA_TEST_RUNNER_DEFAULTS = { + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.6-luna", +} satisfies Pick[0], "primaryModel" | "providerMode">; + +export function createScenarioRunnerTestHarness() { + const tempDirs = createTempDirHarness(); + + return { + makeTempDir: tempDirs.makeTempDir, + makeTempRepo: (prefix: string) => tempDirs.makeTempDir(prefix), + cleanup: tempDirs.cleanup, + }; +} + +export function makeTestFileScenario( + executionKind: "script" | "vitest" | "playwright", + pathLocal: string, + testNamePattern?: string, +): QaSeedScenarioWithSource { + return { + id: `scenario-${executionKind}`, + title: `${executionKind} scenario`, + surface: executionKind === "playwright" ? "control-ui" : "qa-lab", + category: executionKind === "playwright" ? "control-ui.browser-ui" : "qa-lab.coverage", + coverage: { + primary: [executionKind === "playwright" ? "ui.control" : "qa.coverage"], + secondary: [executionKind === "playwright" ? "ui.streaming" : "qa.reporting"], + }, + objective: `Exercise ${executionKind} scenario evidence.`, + successCriteria: ["The scenario writes structured evidence."], + docsRefs: ["docs/concepts/qa-e2e-automation.md"], + codeRefs: [pathLocal], + sourcePath: `qa/scenarios/ui/scenario-${executionKind}.md`, + execution: { + kind: executionKind, + path: pathLocal, + ...(testNamePattern ? { testNamePattern } : {}), + ...(executionKind === "script" + ? { args: ["--once", "--artifact-base", "${outputDir}"] } + : {}), + }, + }; +} + +export function makeDockerE2eScenario(id: string, lane: string): QaSeedScenarioWithSource { + const scenario = makeTestFileScenario("script", "test/e2e/qa-lab/runtime/docker-e2e-lane.ts"); + if (scenario.execution.kind !== "script") { + throw new Error("expected script scenario"); + } + return { + ...scenario, + id, + execution: { + ...scenario.execution, + args: ["--lane", lane], + }, + }; +} + +export async function writeDockerCandidateManifest( + command: QaScenarioCommandExecution, + manifest: unknown, +) { + const manifestArg = command.args.find((arg) => arg.startsWith("--prepare-only=")); + if (!manifestArg) { + throw new Error("missing prep-only manifest argument"); + } + await fs.writeFile(manifestArg.slice("--prepare-only=".length), `${JSON.stringify(manifest)}\n`); + return { exitCode: 0, stdout: "", stderr: "" }; +} + +export async function writeNativeVitestReport( + command: QaScenarioCommandExecution, + counts: { + createRequestedTestFile?: boolean; + failed?: number; + passed: number; + testFilePath?: string; + testName?: string; + }, +) { + const reportArg = command.args.find((arg) => arg.startsWith("--outputFile.json=")); + if (!reportArg) { + return; + } + const requestedTestPath = command.args.find((arg) => arg.endsWith(".test.ts")); + if (requestedTestPath && counts.createRequestedTestFile !== false) { + const requestedTestFile = path.resolve(command.cwd, requestedTestPath); + await fs.mkdir(path.dirname(requestedTestFile), { recursive: true }); + await fs.writeFile(requestedTestFile, "// native scenario fixture\n", "utf8"); + } + const testNamePatternIndex = command.args.indexOf("--testNamePattern"); + const testName = + counts.testName ?? + (testNamePatternIndex < 0 ? undefined : command.args[testNamePatternIndex + 1]) ?? + "executes the requested scenario"; + await fs.writeFile( + reportArg.slice("--outputFile.json=".length), + JSON.stringify({ + numFailedTests: counts.failed ?? 0, + numPassedTests: counts.passed, + success: (counts.failed ?? 0) === 0, + testResults: [ + { + name: path.resolve(command.cwd, counts.testFilePath ?? requestedTestPath ?? "unknown"), + status: counts.passed > 0 ? "passed" : "skipped", + assertionResults: + counts.passed > 0 ? [{ fullName: testName, title: testName, status: "passed" }] : [], + }, + ], + }), + "utf8", + ); +} + +type ScriptEvidenceArtifact = { + kind: string; + path: string; + source?: string; +}; + +type ScriptProducerEvidenceParams = { + additionalEntries?: QaEvidenceSummaryJson["entries"]; + artifacts?: ScriptEvidenceArtifact[]; + coverage?: QaEvidenceSummaryJson["entries"][number]["coverage"]; + failureReason?: string; + producerId?: string; + profile?: string; + status: "blocked" | "fail" | "pass"; +}; + +export function buildScriptProducerEvidence( + params: ScriptProducerEvidenceParams, +): QaEvidenceSummaryJson { + return validateQaEvidenceSummaryJson({ + kind: QA_EVIDENCE_SUMMARY_KIND, + schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, + generatedAt: "2026-06-14T00:00:00.000Z", + evidenceMode: "full", + ...(params.profile ? { profile: params.profile } : {}), + entries: [ + { + test: { + kind: "script-producer-check", + id: params.producerId ?? "script-producer.web-ui.smoke", + title: "Script producer: web-ui smoke", + source: { path: "external/qa/evidence-producer.mjs" }, + }, + coverage: params.coverage ?? [{ id: "ui.control", role: "primary" }], + execution: { + runner: "evidence-producer-script", + environment: { ref: "scenario-ref", os: "darwin", nodeVersion: "v24.0.0" }, + provider: { + id: "script-producer", + live: false, + model: { name: null, ref: null }, + fixture: "synthetic-script-evidence", + }, + packageSource: { kind: "source-checkout", sha: "abc123" }, + artifacts: (params.artifacts ?? []).map((artifact) => + Object.assign({ source: "script-producer:web-ui:smoke" }, artifact), + ), + }, + result: { + status: params.status, + ...(params.failureReason ? { failure: { reason: params.failureReason } } : {}), + timing: { wallMs: 1 }, + }, + }, + ...(params.additionalEntries ?? []), + ], + }); +} + +export async function writeScriptProducerEvidence( + params: ScriptProducerEvidenceParams & { + evidenceLocation?: "run" | "scenario-root"; + latestRun?: "absolute" | "none" | "relative"; + outputDir: string; + scenarioId?: string; + }, +) { + const scenarioArtifactBase = path.join(params.outputDir, params.scenarioId ?? "scenario-script"); + const evidenceDir = + params.evidenceLocation === "scenario-root" + ? scenarioArtifactBase + : path.join(scenarioArtifactBase, "run-1"); + const evidencePath = path.join(evidenceDir, QA_EVIDENCE_FILENAME); + await fs.mkdir(evidenceDir, { recursive: true }); + await fs.writeFile( + evidencePath, + `${JSON.stringify(buildScriptProducerEvidence(params), null, 2)}\n`, + "utf8", + ); + const latestRun = params.latestRun ?? "absolute"; + if (latestRun !== "none") { + await fs.writeFile( + path.join(scenarioArtifactBase, "latest-run.json"), + `${JSON.stringify( + { + qaEvidence: + latestRun === "relative" + ? path.relative(scenarioArtifactBase, evidencePath) + : evidencePath, + }, + null, + 2, + )}\n`, + "utf8", + ); + } + return { evidenceDir, evidencePath, scenarioArtifactBase }; +} diff --git a/extensions/qa-lab/src/test-file-scenario-runner.test.ts b/extensions/qa-lab/src/test-file-scenario-runner.test.ts deleted file mode 100644 index af4211fb397c..000000000000 --- a/extensions/qa-lab/src/test-file-scenario-runner.test.ts +++ /dev/null @@ -1,2027 +0,0 @@ -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { setTimeout as sleep } from "node:timers/promises"; -import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; -import { validateQaEvidenceSummaryJson } from "./evidence-summary.js"; -import type { QaSeedScenarioWithSource } from "./scenario-catalog.js"; -import { createTempDirHarness } from "./temp-dir.test-helper.js"; -import { - resetQaScenarioCommandCleanupTimings, - runQaScenarioCommandLifecycle, - setQaScenarioCommandCleanupTimings, -} from "./test-file-scenario-command-lifecycle.js"; -import { - dockerE2eLaneName, - prepareDockerE2eEnvironment, -} from "./test-file-scenario-docker-batch.js"; -import { - runQaTestFileScenarios, - type QaScenarioCommandExecution, -} from "./test-file-scenario-runner.js"; - -const tempRoots: string[] = []; -const { cleanup: cleanupTempDirs, makeTempDir } = createTempDirHarness(); - -function isProcessRunning(pid: number) { - try { - process.kill(pid, 0); - return true; - } catch { - return false; - } -} - -async function readPid(filePath: string, timeoutMs: number) { - const deadlineAt = Date.now() + timeoutMs; - while (Date.now() < deadlineAt) { - try { - const pid = Number(await fs.readFile(filePath, "utf8")); - if (Number.isInteger(pid) && pid > 0) { - return pid; - } - } catch { - // retry until the process writes its pid - } - await sleep(5); - } - throw new Error(`timeout waiting for pid in ${filePath}`); -} - -async function waitForDead(pid: number, timeoutMs: number) { - const deadlineAt = Date.now() + timeoutMs; - while (Date.now() < deadlineAt) { - if (!isProcessRunning(pid)) { - return; - } - await sleep(5); - } - throw new Error(`process ${pid} still alive`); -} - -function makeTestFileScenario( - executionKind: "script" | "vitest" | "playwright", - pathLocal: string, - testNamePattern?: string, -): QaSeedScenarioWithSource { - return { - id: `scenario-${executionKind}`, - title: `${executionKind} scenario`, - surface: executionKind === "playwright" ? "control-ui" : "qa-lab", - category: executionKind === "playwright" ? "control-ui.browser-ui" : "qa-lab.coverage", - coverage: { - primary: [executionKind === "playwright" ? "ui.control" : "qa.coverage"], - secondary: [executionKind === "playwright" ? "ui.streaming" : "qa.reporting"], - }, - objective: `Exercise ${executionKind} scenario evidence.`, - successCriteria: ["The scenario writes structured evidence."], - docsRefs: ["docs/concepts/qa-e2e-automation.md"], - codeRefs: [pathLocal], - sourcePath: `qa/scenarios/ui/scenario-${executionKind}.md`, - execution: { - kind: executionKind, - path: pathLocal, - ...(testNamePattern ? { testNamePattern } : {}), - ...(executionKind === "script" - ? { args: ["--once", "--artifact-base", "${outputDir}"] } - : {}), - }, - }; -} - -function makeDockerE2eScenario(id: string, lane: string): QaSeedScenarioWithSource { - const scenario = makeTestFileScenario("script", "test/e2e/qa-lab/runtime/docker-e2e-lane.ts"); - if (scenario.execution.kind !== "script") { - throw new Error("expected script scenario"); - } - return { - ...scenario, - id, - execution: { - ...scenario.execution, - args: ["--lane", lane], - }, - }; -} - -async function writeDockerCandidateManifest( - command: QaScenarioCommandExecution, - manifest: unknown, -) { - const manifestArg = command.args.find((arg) => arg.startsWith("--prepare-only=")); - if (!manifestArg) { - throw new Error("missing prep-only manifest argument"); - } - await fs.writeFile(manifestArg.slice("--prepare-only=".length), `${JSON.stringify(manifest)}\n`); - return { exitCode: 0, stdout: "", stderr: "" }; -} - -it("only batches the canonical Docker lane argument shape", () => { - const scenario = makeDockerE2eScenario("docker-lane", "gateway-network"); - if (scenario.execution.kind !== "script") { - throw new Error("expected script scenario"); - } - expect(dockerE2eLaneName(scenario)).toBe("gateway-network"); - expect( - dockerE2eLaneName({ - ...scenario, - execution: { ...scenario.execution, args: ["--lane", "gateway-network", "--extra"] }, - }), - ).toBeUndefined(); -}); - -it("prepares the exact Docker lane union in a sanitized bound environment", async () => { - const repoRoot = await makeTempRepo("qa-docker-candidate-"); - const outputDir = path.join(repoRoot, "out"); - const packagePath = path.join(repoRoot, "openclaw.tgz"); - const registryDir = path.join(repoRoot, "registry"); - const runCommand = vi.fn(async (command: QaScenarioCommandExecution) => { - expect(command.env).toMatchObject({ - KEEP_ME: "yes", - OPENCLAW_DOCKER_ALL_LANES: "gateway-network,openai-chat-tools", - OPENCLAW_DOCKER_E2E_REPO_ROOT: repoRoot, - }); - expect(command.env).not.toHaveProperty("OPENCLAW_DOCKER_ALL_BUILD"); - expect(command.env).not.toHaveProperty("OPENCLAW_CURRENT_PACKAGE_TGZ"); - expect(command.env).not.toHaveProperty("OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR"); - return await writeDockerCandidateManifest(command, { - schema: "openclaw.qa-docker-candidate/v1", - schemaVersion: 1, - sourceSha: "a".repeat(40), - candidate: { - package: { - path: packagePath, - name: "openclaw", - version: "2026.8.1", - sha256: "b".repeat(64), - }, - registry: { - dir: registryDir, - candidateVersion: "2026.8.1", - manifestSha256: "c".repeat(64), - }, - }, - }); - }); - const env = await prepareDockerE2eEnvironment({ - env: { - KEEP_ME: "yes", - OPENCLAW_DOCKER_ALL_BUILD: "1", - OPENCLAW_CURRENT_PACKAGE_TGZ: "/stale.tgz", - OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR: "/stale-registry", - }, - outputDir, - repoRoot, - runCommand, - scenarios: [ - makeDockerE2eScenario("one", "gateway-network"), - makeDockerE2eScenario("duplicate", "gateway-network"), - makeDockerE2eScenario("two", "openai-chat-tools"), - ], - }); - - expect(runCommand).toHaveBeenCalledTimes(1); - expect(Object.isFrozen(env)).toBe(true); - expect(env).toEqual({ - KEEP_ME: "yes", - OPENCLAW_DOCKER_E2E_REPO_ROOT: repoRoot, - OPENCLAW_DOCKER_E2E_SELECTED_SHA: "a".repeat(40), - OPENCLAW_CURRENT_PACKAGE_TGZ: packagePath, - OPENCLAW_CURRENT_PACKAGE_VERSION: "2026.8.1", - OPENCLAW_CURRENT_PACKAGE_SHA256: "b".repeat(64), - OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR: registryDir, - OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_CANDIDATE_VERSION: "2026.8.1", - OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_MANIFEST_SHA256: "c".repeat(64), - }); -}); - -it("returns a sanitized bound env for a package-free candidate", async () => { - const repoRoot = await makeTempRepo("qa-docker-candidate-null-"); - const env = await prepareDockerE2eEnvironment({ - env: { - KEEP_ME: "yes", - OPENCLAW_DOCKER_ALL_BUILD: "1", - OPENCLAW_CURRENT_PACKAGE_TGZ: "/stale.tgz", - }, - outputDir: path.join(repoRoot, "out"), - repoRoot, - runCommand: (command) => - writeDockerCandidateManifest(command, { - schema: "openclaw.qa-docker-candidate/v1", - schemaVersion: 1, - sourceSha: "a".repeat(40), - candidate: null, - }), - scenarios: [makeDockerE2eScenario("one", "gateway-network")], - }); - - expect(env).toEqual({ KEEP_ME: "yes", OPENCLAW_DOCKER_E2E_REPO_ROOT: repoRoot }); - expect(Object.isFrozen(env)).toBe(true); -}); - -it.each([ - { label: "extra field", patch: { extra: true } }, - { label: "malformed candidate", patch: { candidate: { package: null, registry: null } } }, -])("rejects a $label in the Docker candidate manifest", async ({ patch }) => { - const repoRoot = await makeTempRepo("qa-docker-candidate-invalid-"); - await expect( - prepareDockerE2eEnvironment({ - env: process.env, - outputDir: path.join(repoRoot, "out"), - repoRoot, - runCommand: (command) => - writeDockerCandidateManifest(command, { - schema: "openclaw.qa-docker-candidate/v1", - schemaVersion: 1, - sourceSha: "a".repeat(40), - candidate: null, - ...patch, - }), - scenarios: [makeDockerE2eScenario("one", "gateway-network")], - }), - ).rejects.toThrow(); -}); - -async function makeTempRepo(prefix: string) { - const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); - tempRoots.push(repoRoot); - await fs.mkdir(path.join(repoRoot, ".artifacts", "qa-e2e"), { recursive: true }); - return repoRoot; -} - -async function writeNativeVitestReport( - command: QaScenarioCommandExecution, - counts: { - createRequestedTestFile?: boolean; - failed?: number; - passed: number; - testFilePath?: string; - testName?: string; - }, -) { - const reportArg = command.args.find((arg) => arg.startsWith("--outputFile.json=")); - if (!reportArg) { - return; - } - const requestedTestPath = command.args.find((arg) => arg.endsWith(".test.ts")); - if (requestedTestPath && counts.createRequestedTestFile !== false) { - const requestedTestFile = path.resolve(command.cwd, requestedTestPath); - await fs.mkdir(path.dirname(requestedTestFile), { recursive: true }); - await fs.writeFile(requestedTestFile, "// native scenario fixture\n", "utf8"); - } - const testNamePatternIndex = command.args.indexOf("--testNamePattern"); - const testName = - counts.testName ?? - (testNamePatternIndex < 0 ? undefined : command.args[testNamePatternIndex + 1]) ?? - "executes the requested scenario"; - await fs.writeFile( - reportArg.slice("--outputFile.json=".length), - JSON.stringify({ - numFailedTests: counts.failed ?? 0, - numPassedTests: counts.passed, - success: (counts.failed ?? 0) === 0, - testResults: [ - { - name: path.resolve(command.cwd, counts.testFilePath ?? requestedTestPath ?? "unknown"), - status: counts.passed > 0 ? "passed" : "skipped", - assertionResults: - counts.passed > 0 ? [{ fullName: testName, title: testName, status: "passed" }] : [], - }, - ], - }), - "utf8", - ); -} - -async function writeScriptProducerEvidence(params: { - outputDir: string; - producerId?: string; - scenarioId?: string; - status: "blocked" | "fail" | "pass"; - failureReason?: string; -}) { - const scenarioArtifactBase = path.join(params.outputDir, params.scenarioId ?? "scenario-script"); - const runRoot = path.join(scenarioArtifactBase, "run-1"); - await fs.mkdir(runRoot, { recursive: true }); - await fs.writeFile( - path.join(runRoot, "qa-evidence.json"), - `${JSON.stringify( - { - kind: "openclaw.qa.evidence-summary", - schemaVersion: 2, - generatedAt: "2026-06-14T00:00:00.000Z", - evidenceMode: "full", - entries: [ - { - test: { - kind: "script-producer-check", - id: params.producerId ?? "script-producer.web-ui.smoke", - title: "Script producer: web-ui smoke", - source: { path: "scripts/evidence-producer.ts" }, - }, - coverage: [{ id: "ui.control", role: "primary" }], - execution: { - runner: "evidence-producer-script", - environment: { ref: "scenario-ref", os: "darwin", nodeVersion: "v24.0.0" }, - provider: { - id: "script-producer", - live: false, - model: { name: null, ref: null }, - fixture: "mocked-script-evidence", - }, - packageSource: { kind: "source-checkout", sha: "abc123" }, - artifacts: [], - }, - result: { - status: params.status, - ...(params.failureReason ? { failure: { reason: params.failureReason } } : {}), - timing: { wallMs: 1 }, - }, - }, - ], - }, - null, - 2, - )}\n`, - "utf8", - ); - await fs.writeFile( - path.join(scenarioArtifactBase, "latest-run.json"), - `${JSON.stringify({ qaEvidence: path.join(runRoot, "qa-evidence.json") }, null, 2)}\n`, - "utf8", - ); -} - -describe("qa test file scenario runner", () => { - afterEach(async () => { - vi.unstubAllEnvs(); - resetQaScenarioCommandCleanupTimings(); - await Promise.all([ - cleanupTempDirs(), - ...tempRoots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true })), - ]); - }); - - it.each([ - { label: "package", candidate: "package" as const }, - { label: "package-free", candidate: "none" as const }, - ])("keeps hostile inherited Docker state out of a prepared $label run", async ({ candidate }) => { - const repoRoot = await makeTempRepo("qa-docker-replace-env-"); - const packagePath = path.join(repoRoot, "openclaw.tgz"); - vi.stubEnv("OPENCLAW_DOCKER_ALL_POISON", "hostile"); - vi.stubEnv("OPENCLAW_CURRENT_PACKAGE_TGZ", "/hostile.tgz"); - vi.stubEnv("OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR", "/hostile-registry"); - const prepared = await prepareDockerE2eEnvironment({ - env: process.env, - outputDir: path.join(repoRoot, "prep"), - repoRoot, - runCommand: (command) => - writeDockerCandidateManifest(command, { - schema: "openclaw.qa-docker-candidate/v1", - schemaVersion: 1, - sourceSha: "a".repeat(40), - candidate: - candidate === "package" - ? { - package: { - path: packagePath, - name: "openclaw", - version: "2026.8.1", - sha256: "b".repeat(64), - }, - registry: null, - } - : null, - }), - scenarios: [makeDockerE2eScenario("one", "gateway-network")], - }); - - await runQaTestFileScenarios({ - env: prepared, - envMode: "replace", - outputDir: path.join(repoRoot, "run"), - primaryModel: "mock-openai/gpt-5.6-luna", - providerMode: "mock-openai", - repoRoot, - scenarios: [makeDockerE2eScenario("one", "gateway-network")], - runCommand: async (command) => { - expect(command.env.OPENCLAW_DOCKER_ALL_POISON).toBeUndefined(); - expect(command.env.OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR).toBeUndefined(); - expect(command.env.OPENCLAW_CURRENT_PACKAGE_TGZ).toBe( - candidate === "package" ? packagePath : undefined, - ); - expect(command.env.OPENCLAW_DOCKER_E2E_REPO_ROOT).toBe(repoRoot); - const logDir = command.env.OPENCLAW_DOCKER_ALL_LOG_DIR!; - await fs.mkdir(logDir, { recursive: true }); - await fs.writeFile( - path.join(logDir, "summary.json"), - JSON.stringify({ - failures: [], - lanes: [{ elapsedSeconds: 1, name: "gateway-network", status: 0 }], - selectedLanes: ["gateway-network"], - }), - ); - return { exitCode: 0, stdout: "", stderr: "" }; - }, - }); - }); - - it("runs Playwright scenarios with the repo UI e2e command and writes Playwright evidence", async () => { - const repoRoot = await makeTempRepo("qa-playwright-scenario-"); - const commands: QaScenarioCommandExecution[] = []; - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-playwright"), - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [ - makeTestFileScenario( - "playwright", - "ui/src/e2e/chat-flow.e2e.test.ts", - "sends a chat turn through the GUI", - ), - ], - runCommand: async (command) => { - commands.push(command); - await writeNativeVitestReport(command, { passed: 1 }); - return { - exitCode: 0, - stdout: "pass\n", - stderr: "", - }; - }, - env: { - OPENCLAW_QA_REF: "scenario-ref", - } as NodeJS.ProcessEnv, - }); - - expect(result.executionKind).toBe("playwright"); - expect(commands.map((command) => command.args)).toEqual([ - ["--import", "tsx", "scripts/ensure-playwright-chromium.mts"], - [ - "scripts/run-vitest.mjs", - "run", - "--config", - "test/vitest/vitest.ui-e2e.config.ts", - "--configLoader", - "runner", - "ui/src/e2e/chat-flow.e2e.test.ts", - "--reporter=verbose", - "--reporter=json", - `--outputFile.json=${path.join( - repoRoot, - ".artifacts", - "qa-e2e", - "scenario-playwright", - "scenario-playwright.vitest-report.json", - )}`, - "--testNamePattern", - "sends a chat turn through the GUI", - ], - ]); - expect(commands.map((command) => command.timeoutMs)).toEqual([1_800_000, 1_800_000]); - const evidence = validateQaEvidenceSummaryJson( - JSON.parse(await fs.readFile(result.evidencePath, "utf8")), - ); - expect(evidence.schemaVersion).toBe(2); - expect(evidence.entries).toHaveLength(1); - expect(evidence.entries[0]).toMatchObject({ - test: { - kind: "playwright-test", - id: "scenario-playwright", - source: { - path: "ui/src/e2e/chat-flow.e2e.test.ts", - }, - }, - coverage: [ - { - id: "ui.control", - role: "primary", - }, - { - id: "ui.streaming", - role: "secondary", - }, - ], - refs: [ - { - kind: "docs", - path: "docs/concepts/qa-e2e-automation.md", - }, - { - kind: "code", - path: "ui/src/e2e/chat-flow.e2e.test.ts", - }, - ], - execution: { - runner: "playwright", - artifacts: [ - { - kind: "log", - path: ".artifacts/qa-e2e/scenario-playwright/scenario-playwright.log", - source: "playwright", - }, - ], - }, - result: { - status: "pass", - }, - }); - }); - - it("can return aggregate evidence without writing a duplicate evidence file", async () => { - const repoRoot = await makeTempRepo("qa-playwright-memory-evidence-"); - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-playwright"), - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [makeTestFileScenario("playwright", "ui/src/e2e/chat-flow.e2e.test.ts")], - writeEvidenceFile: false, - runCommand: async (command) => { - await writeNativeVitestReport(command, { passed: 1 }); - return { - exitCode: 0, - stdout: "pass\n", - stderr: "", - }; - }, - }); - - expect(result.evidence.entries).toHaveLength(1); - await expect(fs.access(result.evidencePath)).rejects.toMatchObject({ code: "ENOENT" }); - }); - - it("runs Vitest scenarios with the declared test path and writes Vitest evidence", async () => { - const repoRoot = await makeTempRepo("qa-vitest-scenario-"); - const commands: QaScenarioCommandExecution[] = []; - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-vitest"), - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [makeTestFileScenario("vitest", "extensions/qa-lab/src/coverage-report.test.ts")], - runCommand: async (command) => { - commands.push(command); - return { - exitCode: 1, - stdout: "", - stderr: "failed\n", - }; - }, - }); - - expect(result.executionKind).toBe("vitest"); - expect(commands.map((command) => command.args)).toEqual([ - [ - "scripts/run-vitest.mjs", - "extensions/qa-lab/src/coverage-report.test.ts", - "--reporter=verbose", - "--reporter=json", - `--outputFile.json=${path.join( - repoRoot, - ".artifacts", - "qa-e2e", - "scenario-vitest", - "scenario-vitest.vitest-report.json", - )}`, - ], - ]); - expect(commands.map((command) => command.timeoutMs)).toEqual([1_800_000]); - const evidence = validateQaEvidenceSummaryJson( - JSON.parse(await fs.readFile(result.evidencePath, "utf8")), - ); - expect(evidence.entries[0]).toMatchObject({ - test: { - kind: "vitest-test", - id: "scenario-vitest", - source: { - path: "extensions/qa-lab/src/coverage-report.test.ts", - }, - }, - coverage: [ - { - id: "qa.coverage", - role: "primary", - }, - { - id: "qa.reporting", - role: "secondary", - }, - ], - execution: { - runner: "vitest", - artifacts: [ - { - kind: "log", - path: ".artifacts/qa-e2e/scenario-vitest/scenario-vitest.log", - source: "vitest", - }, - ], - }, - result: { - status: "fail", - failure: { - reason: "node exited with 1", - }, - }, - }); - }); - - it.each([ - { executionKind: "vitest" as const, passed: 0, expectedStatus: "fail" as const }, - { executionKind: "playwright" as const, passed: 0, expectedStatus: "fail" as const }, - { executionKind: "vitest" as const, passed: 1, expectedStatus: "pass" as const }, - { executionKind: "playwright" as const, passed: 1, expectedStatus: "pass" as const }, - ])( - "requires an actually passed $executionKind test when the native child exits successfully ($passed passed)", - async ({ executionKind, expectedStatus, passed }) => { - const repoRoot = await makeTempRepo(`qa-${executionKind}-executed-tests-`); - const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`); - const scenarioPath = - executionKind === "playwright" - ? "ui/src/e2e/chat-flow.e2e.test.ts" - : "extensions/qa-lab/src/coverage-report.test.ts"; - const commands: QaScenarioCommandExecution[] = []; - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir, - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [makeTestFileScenario(executionKind, scenarioPath)], - runCommand: async (command) => { - commands.push(command); - await writeNativeVitestReport(command, { passed }); - return { exitCode: 0, stdout: "child exited successfully\n", stderr: "" }; - }, - }); - - expect(result.results[0]).toMatchObject({ status: expectedStatus }); - expect(result.evidence.entries[0]?.result.status).toBe(expectedStatus); - expect( - commands.filter((command) => command.args[0] === "scripts/run-vitest.mjs"), - ).toHaveLength(1); - if (expectedStatus === "fail") { - expect(result.results[0]?.failureMessage).toBe( - "Vitest exited successfully without reporting a successfully executed test.", - ); - } - }, - ); - - it.each([{ executionKind: "vitest" as const }, { executionKind: "playwright" as const }])( - "rejects a passing $executionKind report for an unrelated test file", - async ({ executionKind }) => { - const repoRoot = await makeTempRepo(`qa-${executionKind}-wrong-report-file-`); - const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`); - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir, - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [ - makeTestFileScenario( - executionKind, - executionKind === "playwright" - ? "ui/src/e2e/chat-flow.e2e.test.ts" - : "extensions/qa-lab/src/coverage-report.test.ts", - ), - ], - runCommand: async (command) => { - await writeNativeVitestReport(command, { - passed: 1, - testFilePath: "extensions/qa-lab/src/unrelated.test.ts", - }); - return { exitCode: 0, stdout: "unrelated test passed\n", stderr: "" }; - }, - }); - - expect(result.results[0]).toMatchObject({ - failureMessage: expect.stringContaining("requested test file"), - status: "fail", - }); - expect(result.evidence.entries[0]?.result.status).toBe("fail"); - }, - ); - - it.each([{ executionKind: "vitest" as const }, { executionKind: "playwright" as const }])( - "rejects a passing $executionKind report when the requested test file does not exist", - async ({ executionKind }) => { - const repoRoot = await makeTempRepo(`qa-${executionKind}-missing-requested-test-`); - const scenarioPath = - executionKind === "playwright" - ? "ui/src/e2e/chat-flow.e2e.test.ts" - : "extensions/qa-lab/src/coverage-report.test.ts"; - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`), - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [makeTestFileScenario(executionKind, scenarioPath)], - runCommand: async (command) => { - await writeNativeVitestReport(command, { - createRequestedTestFile: false, - passed: 1, - }); - return { exitCode: 0, stdout: "missing test reportedly passed\n", stderr: "" }; - }, - }); - - expect(result.results[0]).toMatchObject({ - failureMessage: expect.stringContaining("existing requested test file"), - status: "fail", - }); - expect(result.evidence.entries[0]?.result.status).toBe("fail"); - }, - ); - - it.skipIf(process.platform === "win32")( - "authenticates requested tests when the checkout root is a symlink", - async () => { - const canonicalRoot = await fs.realpath(await makeTempRepo("qa-vitest-symlinked-checkout-")); - const symlinkedRoot = path.join(canonicalRoot, "checkout-alias"); - await fs.symlink(canonicalRoot, symlinkedRoot, "dir"); - const scenarioPath = "extensions/qa-lab/src/coverage-report.test.ts"; - const result = await runQaTestFileScenarios({ - repoRoot: symlinkedRoot, - outputDir: path.join(symlinkedRoot, ".artifacts", "qa-e2e", "scenario-vitest"), - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [makeTestFileScenario("vitest", scenarioPath)], - runCommand: async (command) => { - await writeNativeVitestReport(command, { - passed: 1, - testFilePath: path.join(canonicalRoot, scenarioPath), - }); - return { exitCode: 0, stdout: "canonical test passed\n", stderr: "" }; - }, - }); - - expect(result.results[0]).toMatchObject({ status: "pass" }); - expect(result.evidence.entries[0]?.result.status).toBe("pass"); - }, - ); - - it("rejects a passing Playwright report that misses the requested test name", async () => { - const repoRoot = await makeTempRepo("qa-playwright-wrong-report-test-"); - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-playwright"), - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [ - makeTestFileScenario( - "playwright", - "ui/src/e2e/chat-flow.e2e.test.ts", - "required visual assertion", - ), - ], - runCommand: async (command) => { - await writeNativeVitestReport(command, { - passed: 1, - testName: "unrelated visual assertion", - }); - return { exitCode: 0, stdout: "unrelated assertion passed\n", stderr: "" }; - }, - }); - - expect(result.results[0]).toMatchObject({ - failureMessage: expect.stringContaining("requested test name"), - status: "fail", - }); - expect(result.evidence.entries[0]?.result.status).toBe("fail"); - }); - - it("records invalid Playwright test-name patterns as failed scenario evidence", async () => { - const repoRoot = await makeTempRepo("qa-playwright-invalid-report-pattern-"); - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-playwright"), - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [makeTestFileScenario("playwright", "ui/src/e2e/chat-flow.e2e.test.ts", "[")], - runCommand: async (command) => { - await writeNativeVitestReport(command, { - passed: 1, - testName: "executed visual assertion", - }); - return { exitCode: 0, stdout: "visual assertion passed\n", stderr: "" }; - }, - }); - - expect(result.results[0]).toMatchObject({ - failureMessage: expect.stringContaining("invalid requested test name pattern"), - status: "fail", - }); - expect(result.evidence.entries[0]?.result.status).toBe("fail"); - }); - - it.each([{ executionKind: "vitest" as const }, { executionKind: "playwright" as const }])( - "does not reuse a prior passing $executionKind report when the next child writes none", - async ({ executionKind }) => { - const repoRoot = await makeTempRepo(`qa-${executionKind}-stale-vitest-report-`); - const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`); - const scenarioPath = - executionKind === "playwright" - ? "ui/src/e2e/chat-flow.e2e.test.ts" - : "extensions/qa-lab/src/coverage-report.test.ts"; - const reportPath = path.join(outputDir, `scenario-${executionKind}.vitest-report.json`); - let writeReport = true; - const runParams = { - repoRoot, - outputDir, - providerMode: "mock-openai" as const, - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [makeTestFileScenario(executionKind, scenarioPath)], - runCommand: async (command: QaScenarioCommandExecution) => { - if (writeReport) { - await writeNativeVitestReport(command, { passed: 1 }); - } - return { exitCode: 0, stdout: "child exited successfully\n", stderr: "" }; - }, - }; - - const firstRun = await runQaTestFileScenarios(runParams); - expect(firstRun.results[0]).toMatchObject({ status: "pass" }); - await expect(fs.access(reportPath)).resolves.toBeUndefined(); - - writeReport = false; - const secondRun = await runQaTestFileScenarios(runParams); - expect(secondRun.results[0]).toMatchObject({ - failureMessage: `Vitest exited successfully without writing a valid JSON test report at ${reportPath}.`, - status: "fail", - }); - expect(secondRun.evidence.entries[0]?.result.status).toBe("fail"); - await expect(fs.access(reportPath)).rejects.toMatchObject({ code: "ENOENT" }); - }, - ); - - it.each([ - { failFast: true, expectedScenarioIds: ["first-native-scenario"] }, - { - failFast: false, - expectedScenarioIds: ["first-native-scenario", "later-native-scenario"], - }, - { - failFast: undefined, - expectedScenarioIds: ["first-native-scenario", "later-native-scenario"], - }, - ])( - "honors native scenario fail-fast mode ($failFast)", - async ({ failFast, expectedScenarioIds }) => { - const repoRoot = await makeTempRepo("qa-vitest-fail-fast-"); - const runCommand = vi.fn(async () => ({ - exitCode: 1, - stdout: "", - stderr: "native scenario failed\n", - })); - const firstScenario = { - ...makeTestFileScenario("vitest", "extensions/qa-lab/src/coverage-report.test.ts"), - id: "first-native-scenario", - }; - const laterScenario = { - ...makeTestFileScenario("vitest", "extensions/qa-lab/src/cli.test.ts"), - id: "later-native-scenario", - }; - - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "native-fail-fast"), - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - failFast, - scenarios: [firstScenario, laterScenario], - runCommand, - }); - - expect(runCommand).toHaveBeenCalledTimes(expectedScenarioIds.length); - expect(result.results.map((scenario) => scenario.scenario.id)).toEqual(expectedScenarioIds); - expect(result.results.every((scenario) => scenario.status === "fail")).toBe(true); - expect(result.evidence.entries.map((entry) => entry.test.id)).toEqual(expectedScenarioIds); - }, - ); - - it.each([ - { evidence: "missing", expectedFailure: /without writing fresh producer QA evidence/u }, - { evidence: "stale", expectedFailure: /without writing fresh producer QA evidence/u }, - { evidence: "empty", expectedFailure: /without reporting an executed producer check/u }, - { evidence: "malformed", expectedFailure: /invalid JSON/u }, - { evidence: "outside", expectedFailure: /inside its scenario output directory/u }, - ] as const)( - "fails a successful script with $evidence producer evidence", - async ({ evidence, expectedFailure }) => { - const repoRoot = await makeTempRepo(`qa-script-${evidence}-producer-evidence-`); - const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script"); - const scenarioOutputDir = path.join(outputDir, "scenario-script"); - const latestRunPath = path.join(scenarioOutputDir, "latest-run.json"); - const evidencePath = path.join(scenarioOutputDir, "qa-evidence.json"); - - if (evidence === "stale") { - await writeScriptProducerEvidence({ outputDir, status: "pass" }); - const staleEvidencePath = path.join(scenarioOutputDir, "run-1", "qa-evidence.json"); - await fs.copyFile(staleEvidencePath, evidencePath); - const staleTimestamp = new Date(Date.now() - 60_000); - await Promise.all([ - fs.utimes(staleEvidencePath, staleTimestamp, staleTimestamp), - fs.utimes(evidencePath, staleTimestamp, staleTimestamp), - ]); - } - - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir, - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], - runCommand: async () => { - await fs.mkdir(scenarioOutputDir, { recursive: true }); - if (evidence === "stale") { - await expect(fs.access(latestRunPath)).rejects.toMatchObject({ code: "ENOENT" }); - await expect(fs.access(evidencePath)).rejects.toMatchObject({ code: "ENOENT" }); - await fs.writeFile( - latestRunPath, - JSON.stringify({ - qaEvidence: path.join(scenarioOutputDir, "run-1", "qa-evidence.json"), - }), - "utf8", - ); - } else if (evidence === "empty") { - await fs.writeFile( - evidencePath, - JSON.stringify({ - kind: "openclaw.qa.evidence-summary", - schemaVersion: 2, - generatedAt: new Date().toISOString(), - evidenceMode: "full", - entries: [], - }), - "utf8", - ); - } else if (evidence === "malformed") { - await fs.writeFile(evidencePath, "{not valid JSON", "utf8"); - } else if (evidence === "outside") { - await writeScriptProducerEvidence({ - outputDir, - scenarioId: "different-script-scenario", - status: "pass", - }); - await fs.writeFile( - latestRunPath, - JSON.stringify({ - qaEvidence: path.join( - outputDir, - "different-script-scenario", - "run-1", - "qa-evidence.json", - ), - }), - "utf8", - ); - } - return { exitCode: 0, stdout: "script exited successfully\n", stderr: "" }; - }, - }); - - expect(result.results[0]).toMatchObject({ status: "fail" }); - expect(result.results[0]?.failureMessage).toMatch(expectedFailure); - expect(result.evidence.entries).toHaveLength(1); - expect(result.evidence.entries[0]).toMatchObject({ - test: { id: "scenario-script" }, - result: { status: "fail" }, - }); - }, - ); - - it("preserves individual Docker lane success without generic producer evidence", async () => { - const repoRoot = await makeTempRepo("qa-script-docker-individual-no-producer-evidence-"); - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "docker-individual"), - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - failFast: true, - scenarios: [makeDockerE2eScenario("docker-gateway-network", "gateway-network")], - runCommand: async () => ({ exitCode: 0, stdout: "Docker lane passed\n", stderr: "" }), - }); - - expect(result.results[0]).toMatchObject({ - scenario: { id: "docker-gateway-network" }, - status: "pass", - }); - expect(result.evidence.entries[0]?.result.status).toBe("pass"); - }); - - it("runs script scenarios and imports producer QA evidence artifacts", async () => { - const repoRoot = await makeTempRepo("qa-script-scenario-"); - const commands: QaScenarioCommandExecution[] = []; - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script"), - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], - runCommand: async (command) => { - commands.push(command); - const scenarioArtifactBase = path.join( - repoRoot, - ".artifacts", - "qa-e2e", - "scenario-script", - "scenario-script", - ); - const runRoot = path.join(scenarioArtifactBase, "run-1"); - await fs.mkdir(path.join(runRoot, "surfaces", "web-ui"), { recursive: true }); - await fs.writeFile(path.join(runRoot, "surfaces", "web-ui", "screenshot.png"), "png"); - await fs.writeFile( - path.join(runRoot, "qa-evidence.json"), - `${JSON.stringify( - { - kind: "openclaw.qa.evidence-summary", - schemaVersion: 2, - generatedAt: "2026-06-14T00:00:00.000Z", - evidenceMode: "full", - entries: [ - { - test: { - kind: "script-producer-check", - id: "script-producer.web-ui.smoke", - title: "Script producer: web-ui smoke", - source: { path: "scripts/evidence-producer.ts" }, - }, - coverage: [{ id: "ui.control", role: "primary" }], - execution: { - runner: "evidence-producer-script", - environment: { - ref: "scenario-ref", - os: "darwin", - nodeVersion: "v24.0.0", - }, - provider: { - id: "script-producer", - live: false, - model: { name: null, ref: null }, - fixture: "mocked-script-evidence", - }, - packageSource: { kind: "source-checkout", sha: "abc123" }, - artifacts: [ - { - kind: "screenshot", - path: "surfaces/web-ui/screenshot.png", - source: "script-producer:web-ui:smoke", - }, - ], - }, - result: { status: "pass", timing: { wallMs: 1 } }, - }, - ], - }, - null, - 2, - )}\n`, - "utf8", - ); - await fs.writeFile( - path.join(scenarioArtifactBase, "latest-run.json"), - `${JSON.stringify({ qaEvidence: path.join(runRoot, "qa-evidence.json") }, null, 2)}\n`, - "utf8", - ); - return { - exitCode: 0, - stdout: "script pass\n", - stderr: "", - }; - }, - env: { - OPENCLAW_QA_REF: "scenario-ref", - } as NodeJS.ProcessEnv, - }); - - expect(result.executionKind).toBe("script"); - expect(commands.map((command) => command.args)).toEqual([ - [ - "--import", - "tsx", - "scripts/evidence-producer.ts", - "--once", - "--artifact-base", - path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script", "scenario-script"), - ], - ]); - expect(commands.map((command) => command.timeoutMs)).toEqual([30 * 60_000]); - const evidence = validateQaEvidenceSummaryJson( - JSON.parse(await fs.readFile(result.evidencePath, "utf8")), - ); - expect(evidence.entries).toHaveLength(1); - expect(evidence.entries[0]).toMatchObject({ - test: { - kind: "script-producer-check", - id: "script-producer.web-ui.smoke", - }, - coverage: [ - { - id: "qa.coverage", - role: "primary", - }, - { - id: "qa.reporting", - role: "secondary", - }, - ], - execution: { - runner: "evidence-producer-script", - artifacts: [ - { - kind: "screenshot", - path: ".artifacts/qa-e2e/scenario-script/scenario-script/run-1/surfaces/web-ui/screenshot.png", - source: "script-producer:web-ui:smoke", - }, - ], - }, - result: { - status: "pass", - }, - }); - }); - - it("runs Docker script scenarios through one aggregate scheduler invocation", async () => { - const repoRoot = await makeTempRepo("qa-script-docker-batch-"); - const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "docker-batch"); - const staleSummaryPath = path.join(outputDir, "docker-e2e-1800000ms", "summary.json"); - await fs.mkdir(path.dirname(staleSummaryPath), { recursive: true }); - await fs.writeFile(staleSummaryPath, '{"status":"passed"}\n', "utf8"); - const commands: QaScenarioCommandExecution[] = []; - const scenarios = [ - makeDockerE2eScenario("openai-tools", "openai-chat-tools"), - makeDockerE2eScenario("bundled-plugins", "bundled-plugin-install-uninstall"), - makeDockerE2eScenario("prefix-lane", "gateway"), - makeDockerE2eScenario("failing-lane", "gateway-network"), - ]; - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir, - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios, - runCommand: async (command) => { - commands.push(command); - await expect(fs.access(staleSummaryPath)).rejects.toThrow(); - const logDir = command.env.OPENCLAW_DOCKER_ALL_LOG_DIR; - if (!logDir) { - throw new Error("missing Docker scheduler log dir"); - } - await fs.mkdir(logDir, { recursive: true }); - const failedLane = { elapsedSeconds: 2, name: "gateway-network", status: 1 }; - await fs.writeFile( - path.join(logDir, "summary.json"), - `${JSON.stringify({ - failures: [failedLane], - lanes: [ - { elapsedSeconds: 4, name: "openai-chat-tools", status: 0 }, - { elapsedSeconds: 7, name: "bundled-plugin-install-uninstall-0", status: 0 }, - { elapsedSeconds: 6, name: "bundled-plugin-install-uninstall-1", status: 0 }, - { elapsedSeconds: 1, name: "gateway", status: 0 }, - failedLane, - ], - selectedLanes: [ - "openai-chat-tools", - "bundled-plugin-install-uninstall-0", - "bundled-plugin-install-uninstall-1", - "gateway", - "gateway-network", - ], - })}\n`, - "utf8", - ); - return { exitCode: 1, stdout: "", stderr: "scheduler failed\n" }; - }, - }); - - expect(commands).toHaveLength(1); - expect(commands[0]).toMatchObject({ - args: ["scripts/test-docker-all.mjs"], - command: process.execPath, - env: { - OPENCLAW_DOCKER_ALL_FAIL_FAST: "0", - OPENCLAW_DOCKER_ALL_LANES: - "openai-chat-tools,bundled-plugin-install-uninstall,gateway,gateway-network", - OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS: "1800000", - }, - }); - expect(result.results).toMatchObject([ - { scenario: { id: "openai-tools" }, status: "pass" }, - { scenario: { id: "bundled-plugins" }, status: "pass" }, - { scenario: { id: "prefix-lane" }, status: "pass" }, - { scenario: { id: "failing-lane" }, status: "fail" }, - ]); - expect(result.results[3]?.failureMessage).toBe("gateway-network exited with 1"); - }); - - it("uses script scenario timeout overrides when running producer commands", async () => { - const repoRoot = await makeTempRepo("qa-script-scenario-timeout-"); - const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-timeout"); - const scenario = makeTestFileScenario("script", "scripts/evidence-producer.ts"); - if (scenario.execution.kind !== "script") { - throw new Error("expected script scenario"); - } - scenario.execution.timeoutMs = 3 * 60 * 60_000; - - const commands: QaScenarioCommandExecution[] = []; - await runQaTestFileScenarios({ - repoRoot, - outputDir, - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [scenario], - commandTimeoutMs: 30 * 60_000, - runCommand: async (command) => { - commands.push(command); - await writeScriptProducerEvidence({ - outputDir, - status: "pass", - }); - return { - exitCode: 0, - stdout: "script pass\n", - stderr: "", - }; - }, - env: { - OPENCLAW_QA_REF: "scenario-ref", - } as NodeJS.ProcessEnv, - }); - - expect(commands.map((command) => command.timeoutMs)).toEqual([3 * 60 * 60_000]); - }); - - it.each([ - { executionKind: "vitest" as const, commandCount: 1 }, - { executionKind: "playwright" as const, commandCount: 2 }, - ])( - "applies the resolved command timeout to every $executionKind subprocess", - async ({ commandCount, executionKind }) => { - const repoRoot = await makeTempRepo(`qa-${executionKind}-command-timeout-`); - const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`); - const commands: QaScenarioCommandExecution[] = []; - - await runQaTestFileScenarios({ - repoRoot, - outputDir, - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [ - makeTestFileScenario( - executionKind, - executionKind === "playwright" - ? "ui/src/e2e/chat-flow.e2e.test.ts" - : "extensions/qa-lab/src/coverage-report.test.ts", - ), - ], - commandTimeoutMs: 321, - runCommand: async (command) => { - commands.push(command); - await writeNativeVitestReport(command, { passed: 1 }); - return { exitCode: 0, stdout: "native pass\n", stderr: "" }; - }, - }); - - expect(commands).toHaveLength(commandCount); - expect(commands.map((command) => command.timeoutMs)).toEqual( - Array.from({ length: commandCount }, () => 321), - ); - }, - ); - - it.each(["vitest", "playwright"] as const)( - "terminates a hanging $executionKind subprocess with failure evidence", - async (executionKind) => { - const repoRoot = await makeTempRepo(`qa-${executionKind}-hung-command-`); - const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`); - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir, - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [ - makeTestFileScenario( - executionKind, - executionKind === "playwright" - ? "ui/src/e2e/chat-flow.e2e.test.ts" - : "extensions/qa-lab/src/coverage-report.test.ts", - ), - ], - commandTimeoutMs: 100, - runCommand: (execution) => - runQaScenarioCommandLifecycle({ - ...execution, - args: ["-e", "setInterval(() => {}, 1_000)"], - }), - }); - - expect(result.results[0]).toMatchObject({ - failureMessage: expect.stringContaining("timed out after 100ms"), - status: "fail", - }); - expect(result.evidence.entries[0]?.result.status).toBe("fail"); - }, - ); - - describe.skipIf(process.platform === "win32")("script timeout process groups", () => { - const commandTimeoutMs = 1_500; - let descendantPid: number | undefined; - let result: Awaited>; - - beforeAll(async () => { - const tempRoot = await makeTempDir("qa-script-timeout-"); - const scriptPath = path.join(tempRoot, "hanging-producer.mjs"); - const descendantPidPath = path.join(tempRoot, "descendant.pid"); - const descendantScript = [ - "process.on('SIGTERM', () => {});", - "setInterval(() => {}, 1000);", - ].join("\n"); - await fs.writeFile( - scriptPath, - [ - "import { spawn } from 'node:child_process';", - "import { writeFileSync } from 'node:fs';", - `const descendant = spawn(process.execPath, ['-e', ${JSON.stringify(descendantScript)}], { stdio: 'ignore' });`, - `writeFileSync(${JSON.stringify(descendantPidPath)}, String(descendant.pid));`, - "process.stdout.write('script still running\\n');", - "process.on('SIGTERM', () => {});", - "setInterval(() => {}, 1000);", - ].join("\n"), - "utf8", - ); - - setQaScenarioCommandCleanupTimings({ - forceSettleMs: 25, - killGraceMs: 50, - }); - const run = runQaTestFileScenarios({ - repoRoot: process.cwd(), - outputDir: path.join(tempRoot, "out"), - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [makeTestFileScenario("script", scriptPath)], - commandTimeoutMs, - // Exercise the real process-group lifecycle without spending its - // bounded startup budget on an unrelated cold tsx import. - runCommand: (execution) => - runQaScenarioCommandLifecycle({ ...execution, args: [scriptPath] }), - }); - const [pidResult, runResult] = await Promise.allSettled([ - readPid(descendantPidPath, commandTimeoutMs), - run, - ]); - if (pidResult.status === "rejected") { - throw pidResult.reason; - } - if (runResult.status === "rejected") { - throw runResult.reason; - } - descendantPid = pidResult.value; - result = runResult.value; - await waitForDead(descendantPid, 2_000); - }); - - afterAll(() => { - if (descendantPid !== undefined && isProcessRunning(descendantPid)) { - process.kill(descendantPid, "SIGKILL"); - } - }); - - it("times out script scenarios and kills descendant process groups", () => { - expect(result.results[0]?.status).toBe("fail"); - expect(result.results[0]?.failureMessage).toMatch( - new RegExp(`timed out after ${commandTimeoutMs}ms`, "u"), - ); - if (descendantPid === undefined) { - throw new Error("descendant pid was not captured"); - } - expect(isProcessRunning(descendantPid)).toBe(false); - }); - }); - - it("fails script scenarios that exit cleanly after timeout termination", async () => { - const repoRoot = process.cwd(); - const tempRoot = await makeTempDir("qa-script-timeout-clean-exit-"); - const scriptPath = path.join(tempRoot, "clean-exit-after-timeout.ts"); - await fs.writeFile( - scriptPath, - [ - "process.stdout.write('waiting for timeout\\n');", - "process.on('SIGTERM', () => process.exit(0));", - "setInterval(() => {}, 1000);", - ].join("\n"), - "utf8", - ); - - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir: path.join(tempRoot, "out"), - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [makeTestFileScenario("script", scriptPath)], - commandTimeoutMs: 100, - }); - - expect(result.results[0]?.status).toBe("fail"); - expect(result.results[0]?.failureMessage).toMatch(/timed out after 100ms/u); - }); - - it("imports producer QA evidence artifacts from failed script scenarios", async () => { - const repoRoot = await makeTempRepo("qa-script-failed-scenario-"); - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-failed"), - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], - runCommand: async () => { - const scenarioArtifactBase = path.join( - repoRoot, - ".artifacts", - "qa-e2e", - "scenario-script-failed", - "scenario-script", - ); - const runRoot = path.join(scenarioArtifactBase, "run-1"); - await fs.mkdir(runRoot, { recursive: true }); - await fs.writeFile( - path.join(runRoot, "qa-evidence.json"), - `${JSON.stringify( - { - kind: "openclaw.qa.evidence-summary", - schemaVersion: 2, - generatedAt: "2026-06-14T00:00:00.000Z", - evidenceMode: "full", - entries: [ - { - test: { - kind: "script-producer-check", - id: "script-producer.web-ui.smoke", - title: "Script producer: web-ui smoke", - source: { path: "scripts/evidence-producer.ts" }, - }, - coverage: [{ id: "ui.control", role: "primary" }], - execution: { - runner: "evidence-producer-script", - environment: { - ref: "scenario-ref", - os: "darwin", - nodeVersion: "v24.0.0", - }, - provider: { - id: "script-producer", - live: false, - model: { name: null, ref: null }, - fixture: "failed-producer-evidence", - }, - packageSource: { kind: "source-checkout", sha: "abc123" }, - artifacts: [], - }, - result: { - status: "fail", - failure: { - reason: "Script producer check failed.", - }, - timing: { wallMs: 1 }, - }, - }, - ], - }, - null, - 2, - )}\n`, - "utf8", - ); - await fs.writeFile( - path.join(scenarioArtifactBase, "latest-run.json"), - `${JSON.stringify({ qaEvidence: path.join(runRoot, "qa-evidence.json") }, null, 2)}\n`, - "utf8", - ); - return { - exitCode: 1, - stdout: "", - stderr: "script failed\n", - }; - }, - env: { - OPENCLAW_QA_REF: "scenario-ref", - } as NodeJS.ProcessEnv, - }); - - expect(result.results[0]).toMatchObject({ - status: "fail", - failureMessage: "node exited with 1", - producerEvidence: { - entries: [ - { - test: { - id: "script-producer.web-ui.smoke", - }, - result: { - status: "fail", - }, - }, - ], - }, - }); - const evidence = validateQaEvidenceSummaryJson( - JSON.parse(await fs.readFile(result.evidencePath, "utf8")), - ); - expect(evidence.entries).toHaveLength(2); - expect(evidence.entries[0]).toMatchObject({ - test: { - kind: "script-producer-check", - id: "script-producer.web-ui.smoke", - }, - coverage: [ - { id: "qa.coverage", role: "primary" }, - { id: "qa.reporting", role: "secondary" }, - ], - result: { - status: "fail", - failure: { - reason: "Script producer check failed.", - }, - }, - }); - expect(evidence.entries[1]).toMatchObject({ - test: { - kind: "script-test", - id: "scenario-script", - source: { - path: "scripts/evidence-producer.ts", - }, - }, - result: { - status: "fail", - failure: { - reason: "node exited with 1", - }, - }, - }); - }); - - it("suppresses a failed-script fallback row already owned by producer scenario evidence", async () => { - const repoRoot = await makeTempRepo("qa-script-duplicate-scenario-evidence-"); - const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-duplicate"); - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir, - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], - runCommand: async () => { - await writeScriptProducerEvidence({ - outputDir, - producerId: "scenario-script", - status: "fail", - failureReason: "producer recorded the script failure", - }); - return { exitCode: 1, stdout: "", stderr: "script failed\n" }; - }, - env: { OPENCLAW_QA_REF: "scenario-ref" } as NodeJS.ProcessEnv, - }); - - expect(result.results[0]).toMatchObject({ status: "fail" }); - expect(result.evidence.entries).toHaveLength(1); - expect(result.evidence.entries[0]).toMatchObject({ - test: { id: "scenario-script" }, - result: { failure: { reason: "producer recorded the script failure" }, status: "fail" }, - }); - }); - - it("fails script scenario results when imported producer evidence fails", async () => { - const repoRoot = await makeTempRepo("qa-script-producer-fail-"); - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-producer-fail"), - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], - runCommand: async () => { - const scenarioArtifactBase = path.join( - repoRoot, - ".artifacts", - "qa-e2e", - "scenario-script-producer-fail", - "scenario-script", - ); - const runRoot = path.join(scenarioArtifactBase, "run-1"); - await fs.mkdir(runRoot, { recursive: true }); - await fs.writeFile( - path.join(runRoot, "qa-evidence.json"), - `${JSON.stringify( - { - kind: "openclaw.qa.evidence-summary", - schemaVersion: 2, - generatedAt: "2026-06-14T00:00:00.000Z", - evidenceMode: "full", - entries: [ - { - test: { - kind: "script-producer-check", - id: "script-producer.web-ui.smoke", - title: "Script producer: web-ui smoke", - source: { path: "scripts/evidence-producer.ts" }, - }, - coverage: [{ id: "ui.control", role: "primary" }], - execution: { - runner: "evidence-producer-script", - environment: { - ref: "scenario-ref", - os: "darwin", - nodeVersion: "v24.0.0", - }, - provider: { - id: "script-producer", - live: false, - model: { name: null, ref: null }, - fixture: "mocked-script-evidence", - }, - packageSource: { kind: "source-checkout", sha: "abc123" }, - artifacts: [], - }, - result: { - status: "fail", - failure: { - reason: "Script producer check failed.", - }, - timing: { wallMs: 1 }, - }, - }, - ], - }, - null, - 2, - )}\n`, - "utf8", - ); - await fs.writeFile( - path.join(scenarioArtifactBase, "latest-run.json"), - `${JSON.stringify({ qaEvidence: path.join(runRoot, "qa-evidence.json") }, null, 2)}\n`, - "utf8", - ); - return { - exitCode: 0, - stdout: "script pass\n", - stderr: "", - }; - }, - env: { - OPENCLAW_QA_REF: "scenario-ref", - } as NodeJS.ProcessEnv, - }); - - expect(result.results[0]).toMatchObject({ - status: "fail", - failureMessage: "Script producer check failed.", - }); - const evidence = validateQaEvidenceSummaryJson( - JSON.parse(await fs.readFile(result.evidencePath, "utf8")), - ); - expect(evidence.entries).toHaveLength(1); - expect(evidence.entries[0]).toMatchObject({ - test: { - id: "script-producer.web-ui.smoke", - }, - result: { - status: "fail", - }, - }); - }); - - it("fails script scenario results when imported producer evidence is blocked by default", async () => { - const repoRoot = await makeTempRepo("qa-script-producer-blocked-"); - const outputDir = path.join( - repoRoot, - ".artifacts", - "qa-e2e", - "scenario-script-producer-blocked", - ); - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir, - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], - runCommand: async () => { - await writeScriptProducerEvidence({ - outputDir, - status: "blocked", - failureReason: "Playwright browser is missing.", - }); - return { - exitCode: 0, - stdout: "script blocked\n", - stderr: "", - }; - }, - env: { - OPENCLAW_QA_REF: "scenario-ref", - } as NodeJS.ProcessEnv, - }); - - expect(result.results[0]).toMatchObject({ - status: "blocked", - failureMessage: "Playwright browser is missing.", - }); - }); - - it("keeps all-blocked producer evidence blocked for opt-in script scenarios", async () => { - const repoRoot = await makeTempRepo("qa-script-producer-blocked-allowed-"); - const outputDir = path.join( - repoRoot, - ".artifacts", - "qa-e2e", - "scenario-script-producer-blocked-allowed", - ); - const scenario = makeTestFileScenario("script", "scripts/evidence-producer.ts"); - if (scenario.execution.kind !== "script") { - throw new Error("expected script scenario"); - } - scenario.execution.allowBlockedEvidence = true; - - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir, - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [scenario], - runCommand: async () => { - await writeScriptProducerEvidence({ - outputDir, - status: "blocked", - failureReason: "Playwright browser is missing.", - }); - return { - exitCode: 0, - stdout: "script blocked\n", - stderr: "", - }; - }, - env: { - OPENCLAW_QA_REF: "scenario-ref", - } as NodeJS.ProcessEnv, - }); - - expect(result.results[0]).toMatchObject({ - status: "blocked", - failureMessage: "Playwright browser is missing.", - producerEvidence: { - entries: [ - { - test: { - id: "script-producer.web-ui.smoke", - }, - result: { - status: "blocked", - }, - }, - ], - }, - }); - }); - - it("allows blocked producer checks when another check genuinely passes", async () => { - const repoRoot = await makeTempRepo("qa-script-producer-blocked-mixed-"); - const outputDir = path.join( - repoRoot, - ".artifacts", - "qa-e2e", - "scenario-script-producer-blocked-mixed", - ); - const scenario = makeTestFileScenario("script", "scripts/evidence-producer.ts"); - if (scenario.execution.kind !== "script") { - throw new Error("expected script scenario"); - } - scenario.execution.allowBlockedEvidence = true; - - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir, - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [scenario], - runCommand: async () => { - await writeScriptProducerEvidence({ - outputDir, - status: "blocked", - failureReason: "Playwright browser is missing.", - }); - const evidencePath = path.join(outputDir, "scenario-script", "run-1", "qa-evidence.json"); - const evidence = JSON.parse(await fs.readFile(evidencePath, "utf8")); - evidence.entries.push({ - ...evidence.entries[0], - test: { - ...evidence.entries[0].test, - id: "script-producer.web-ui.executed", - }, - result: { - status: "pass", - timing: { wallMs: 1 }, - }, - }); - await fs.writeFile(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`, "utf8"); - return { - exitCode: 0, - stdout: "script mixed\n", - stderr: "", - }; - }, - env: { - OPENCLAW_QA_REF: "scenario-ref", - } as NodeJS.ProcessEnv, - }); - - expect(result.results[0]).toMatchObject({ - status: "pass", - producerEvidence: { - entries: [{ result: { status: "blocked" } }, { result: { status: "pass" } }], - }, - }); - }); - - it("carries the suite profile into merged producer evidence", async () => { - const repoRoot = await makeTempRepo("qa-script-profile-"); - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-profile"), - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], - runCommand: async () => { - const scenarioOutputDir = path.join( - repoRoot, - ".artifacts", - "qa-e2e", - "scenario-script-profile", - "scenario-script", - ); - await fs.mkdir(scenarioOutputDir, { recursive: true }); - await fs.writeFile( - path.join(scenarioOutputDir, "qa-evidence.json"), - `${JSON.stringify({ - kind: "openclaw.qa.evidence-summary", - schemaVersion: 2, - generatedAt: "2026-06-14T00:00:00.000Z", - evidenceMode: "full", - entries: [ - { - test: { - kind: "script-producer-check", - id: "script-producer.web-ui.smoke", - title: "Script producer: web-ui smoke", - source: { path: "scripts/evidence-producer.ts" }, - }, - coverage: [{ id: "ui.control", role: "primary" }], - result: { status: "pass", timing: { wallMs: 1 } }, - }, - ], - })}\n`, - "utf8", - ); - return { exitCode: 0, stdout: "script pass\n", stderr: "" }; - }, - env: { - OPENCLAW_QA_REF: "scenario-ref", - OPENCLAW_QA_PROFILE: "smoke-ci", - } as NodeJS.ProcessEnv, - }); - - const evidence = validateQaEvidenceSummaryJson( - JSON.parse(await fs.readFile(result.evidencePath, "utf8")), - ); - expect(evidence.profile).toBe("smoke-ci"); - }); - - it("keeps producer artifacts outside the repo root absolute instead of emitting ../ paths", async () => { - const repoRoot = await makeTempRepo("qa-script-external-artifact-"); - const externalArtifact = path.join(os.tmpdir(), "qa-external-artifact.png"); - const result = await runQaTestFileScenarios({ - repoRoot, - outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-external"), - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], - runCommand: async () => { - const scenarioOutputDir = path.join( - repoRoot, - ".artifacts", - "qa-e2e", - "scenario-script-external", - "scenario-script", - ); - await fs.mkdir(scenarioOutputDir, { recursive: true }); - await fs.writeFile( - path.join(scenarioOutputDir, "qa-evidence.json"), - `${JSON.stringify({ - kind: "openclaw.qa.evidence-summary", - schemaVersion: 2, - generatedAt: "2026-06-14T00:00:00.000Z", - evidenceMode: "full", - entries: [ - { - test: { - kind: "script-producer-check", - id: "script-producer.web-ui.smoke", - title: "Script producer: web-ui smoke", - source: { path: "scripts/evidence-producer.ts" }, - }, - coverage: [{ id: "ui.control", role: "primary" }], - execution: { - runner: "evidence-producer-script", - environment: { ref: "scenario-ref", os: "darwin", nodeVersion: "v24.0.0" }, - provider: { - id: "script-producer", - live: false, - model: { name: null, ref: null }, - fixture: "mocked-script-evidence", - }, - packageSource: { kind: "source-checkout", sha: "abc123" }, - artifacts: [ - { - kind: "screenshot", - path: externalArtifact, - source: "script-producer:web-ui:smoke", - }, - ], - }, - result: { status: "pass", timing: { wallMs: 1 } }, - }, - ], - })}\n`, - "utf8", - ); - return { exitCode: 0, stdout: "script pass\n", stderr: "" }; - }, - env: { OPENCLAW_QA_REF: "scenario-ref" } as NodeJS.ProcessEnv, - }); - - const evidence = validateQaEvidenceSummaryJson( - JSON.parse(await fs.readFile(result.evidencePath, "utf8")), - ); - const artifactPath = evidence.entries[0]?.execution?.artifacts[0]?.path; - expect(artifactPath).toBe(path.normalize(externalArtifact)); - expect(artifactPath?.includes("..")).toBe(false); - }); - - it("imports the standalone UX Matrix producer as coverage-free infrastructure", async () => { - const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-ux-matrix-producer-")); - tempRoots.push(outputDir); - // The runner accepts scenario-shaped execution input, but this test fixture is not cataloged - // and deliberately carries no product taxonomy coverage. - const infrastructureFixture: QaSeedScenarioWithSource = { - id: "scenario-script", - title: "UX Matrix producer infrastructure fixture", - surface: "qa-lab", - objective: "Exercise the standalone UX Matrix evidence producer through the script runner.", - successCriteria: ["The runner imports the producer's structured evidence bundle."], - codeRefs: ["scripts/qa/ux-matrix-evidence-producer.ts"], - sourcePath: "test/scripts/qa-ux-matrix-evidence-producer.test.ts", - execution: { - kind: "script", - path: "scripts/qa/ux-matrix-evidence-producer.ts", - allowBlockedEvidence: true, - args: ["--artifact-base", "${outputDir}", "--skip-visual-proof"], - }, - }; - - const result = await runQaTestFileScenarios({ - repoRoot: process.cwd(), - outputDir, - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [infrastructureFixture], - env: { OPENCLAW_QA_REF: "infrastructure-fixture" } as NodeJS.ProcessEnv, - }); - const evidence = validateQaEvidenceSummaryJson( - JSON.parse(await fs.readFile(result.evidencePath, "utf8")), - ); - - expect(result.executionKind).toBe("script"); - const producerEntries = result.results[0]?.producerEvidence?.entries ?? []; - const producerStatuses = producerEntries.map((entry) => entry.result.status); - expect(producerEntries).toHaveLength(3); - expect(producerStatuses).toContain("blocked"); - expect(result.results[0]?.status).toBe(producerStatuses.includes("pass") ? "pass" : "blocked"); - expect(evidence.entries.map((entry) => entry.test.id)).toEqual([ - "ux-matrix.qa-lab.producer-artifact-fixture", - "ux-matrix.control-ui.screenshot-artifact", - "ux-matrix.cli.entrypoint-help", - ]); - expect(evidence.entries.every((entry) => entry.coverage.length === 0)).toBe(true); - expect( - evidence.entries.flatMap( - (entry) => entry.execution?.artifacts.map((artifact) => artifact.kind) ?? [], - ), - ).toEqual(expect.arrayContaining(["html", "log"])); - expect( - evidence.entries - .flatMap((entry) => entry.execution?.artifacts.map((artifact) => artifact.path) ?? []) - .some((artifactPath) => artifactPath.includes(path.join(outputDir, "scenario-script"))), - ).toBe(true); - }); -}); -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/extensions/qa-lab/web/src/ui-render.test.ts b/extensions/qa-lab/web/src/ui-render.test.ts index 193c8a700c26..b5d0940c0300 100644 --- a/extensions/qa-lab/web/src/ui-render.test.ts +++ b/extensions/qa-lab/web/src/ui-render.test.ts @@ -460,9 +460,9 @@ describe("QA Lab UI evidence render", () => { runner: { availability: "local", command: - "node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix", + "node external/qa/ux-matrix-producer.mjs --artifact-base .artifacts/external-qa/ux-matrix", lane: "web-ui-playwright", - workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local", + workflow: "external/ci/ux-matrix.yml#matrix-local", }, stage: "first-run", status: "pass", @@ -477,9 +477,9 @@ describe("QA Lab UI evidence render", () => { runner: { availability: "local", command: - "node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix", + "node external/qa/ux-matrix-producer.mjs --artifact-base .artifacts/external-qa/ux-matrix", lane: "cli-status", - workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local", + workflow: "external/ci/ux-matrix.yml#matrix-local", }, stage: "first-run", status: "proof-gap", diff --git a/scripts/qa/ux-matrix-evidence-producer.ts b/scripts/qa/ux-matrix-evidence-producer.ts deleted file mode 100644 index b13fbfd3ba62..000000000000 --- a/scripts/qa/ux-matrix-evidence-producer.ts +++ /dev/null @@ -1,814 +0,0 @@ -// Produces standalone QA Lab UX Matrix fixture artifacts for gallery and evidence tests. -import { execFile } from "node:child_process"; -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { pathToFileURL } from "node:url"; -import { promisify } from "node:util"; -import { toRepoRelativePath } from "../../extensions/qa-lab/src/cli-paths.js"; -import { resolveQaEvidenceEnvironment } from "../../extensions/qa-lab/src/evidence-environment.js"; -import { - QA_EVIDENCE_FILENAME, - QA_EVIDENCE_SUMMARY_KIND, - QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, - validateQaEvidenceSummaryJson, - type QaEvidenceStatus, - type QaEvidenceSummaryEntry, - type QaEvidenceSummaryJson, -} from "../../extensions/qa-lab/src/evidence-summary.js"; -import { - ensurePlaywrightChromium, - resolveSystemChromiumExecutablePath, -} from "../ensure-playwright-chromium.mts"; - -const execFileAsync = promisify(execFile); -const SOURCE_PATH = "scripts/qa/ux-matrix-evidence-producer.ts"; - -type MatrixCell = { - artifacts: Array<{ kind: string; path: string }>; - failureReason?: string; - stage: string; - status: QaEvidenceStatus; - surface: string; - title: string; - wallMs: number; -}; - -type ChromiumLauncher = Awaited["chromium"]; -type ChromiumBrowser = Awaited>; - -type ProducerOptions = { - artifactBase: string; - repoRoot: string; - skipVisualProof: boolean; -}; - -function usage() { - return `Usage: node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base [options] - -Produces a QA Lab UX Matrix evidence bundle. - -Options: - --artifact-base Evidence artifact directory - --repo-root Repository root - --skip-visual-proof Use fixture visual evidence instead of Playwright screenshots - -h, --help Show this help -`; -} - -function readOptionValue(argv: readonly string[], index: number, arg: string) { - const value = argv[index + 1] ?? ""; - if (!value || value.startsWith("-")) { - throw new Error(`${arg} requires a value`); - } - return value; -} - -function isHelpRequest(argv: readonly string[]) { - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - if (arg === "--artifact-base" || arg === "--repo-root") { - index += 1; - continue; - } - if (arg === "--help" || arg === "-h") { - return true; - } - } - return false; -} - -function parseOptions(argv: readonly string[]): ProducerOptions { - let artifactBase = ""; - let repoRoot = process.cwd(); - let skipVisualProof = false; - const seen = new Set(); - const recordOnce = (flag: string) => { - if (seen.has(flag)) { - throw new Error(`${flag} was provided more than once`); - } - seen.add(flag); - }; - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - if (arg === "--artifact-base") { - const value = readOptionValue(argv, index, arg); - recordOnce(arg); - artifactBase = value; - index += 1; - continue; - } - if (arg === "--repo-root") { - const value = readOptionValue(argv, index, arg); - recordOnce(arg); - repoRoot = value; - index += 1; - continue; - } - if (arg === "--skip-visual-proof") { - recordOnce(arg); - skipVisualProof = true; - continue; - } - throw new Error(`unsupported UX Matrix producer arg: ${arg}`); - } - if (!artifactBase.trim()) { - throw new Error("--artifact-base is required"); - } - if (!repoRoot.trim()) { - throw new Error("--repo-root must not be empty"); - } - return { - artifactBase: path.resolve(repoRoot, artifactBase), - repoRoot: path.resolve(repoRoot), - skipVisualProof, - }; -} - -type ProducerCliOutput = { - error: (message: string) => void; - log: (message: string) => void; -}; - -export async function runUxMatrixEvidenceProducerCli( - argv: readonly string[], - output: ProducerCliOutput = console, -): Promise { - try { - if (isHelpRequest(argv)) { - output.log(usage()); - return 0; - } - const result = await runUxMatrixEvidenceProducer(parseOptions(argv)); - output.log(`UX Matrix evidence: ${path.join(result.artifactBase, QA_EVIDENCE_FILENAME)}`); - output.log(`UX Matrix entries: ${result.evidence.entries.length}`); - return 0; - } catch (error) { - output.error(error instanceof Error ? error.message : String(error)); - return 1; - } -} - -async function writeJson(filePath: string, value: unknown) { - await fs.mkdir(path.dirname(filePath), { recursive: true }); - await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); -} - -async function writeText(filePath: string, value: string) { - await fs.mkdir(path.dirname(filePath), { recursive: true }); - await fs.writeFile(filePath, value, "utf8"); -} - -function cellDir(artifactBase: string, cell: Pick) { - return path.join(artifactBase, "surfaces", cell.surface, "stages", cell.stage); -} - -function relativeToArtifactBase(artifactBase: string, filePath: string) { - return path.relative(artifactBase, filePath).split(path.sep).join("/"); -} - -function sanitizeArtifactText( - value: string, - params: { - artifactBase?: string; - repoRoot: string; - }, -) { - const roots = [ - { from: path.resolve(params.repoRoot), to: "" }, - { from: pathToFileURL(path.resolve(params.repoRoot)).href, to: "file://" }, - ...(params.artifactBase - ? [ - { from: path.resolve(params.artifactBase), to: "" }, - { - from: pathToFileURL(path.resolve(params.artifactBase)).href, - to: "file://", - }, - ] - : []), - { from: os.homedir(), to: "" }, - { from: pathToFileURL(os.homedir()).href, to: "file://" }, - ].filter((entry) => entry.from && entry.from !== path.parse(entry.from).root); - return roots - .toSorted((a, b) => b.from.length - a.from.length) - .reduce((text, entry) => text.replaceAll(entry.from, entry.to), value); -} - -function buildExecution(params: { - artifacts: MatrixCell["artifacts"]; - repoRoot: string; - source: string; -}): QaEvidenceSummaryEntry["execution"] { - return { - runner: "ux-matrix-script-producer", - environment: resolveQaEvidenceEnvironment({ - env: process.env, - repoRoot: params.repoRoot, - }), - provider: { - id: "ux-matrix", - live: false, - model: { - name: null, - ref: null, - }, - fixture: "local-qa-lab-script-producer", - }, - packageSource: { - kind: "source-checkout", - }, - artifacts: params.artifacts.map((artifact) => ({ - ...artifact, - source: params.source, - })), - }; -} - -function buildEvidenceEntry(cell: MatrixCell, repoRoot: string): QaEvidenceSummaryEntry { - const source = `ux-matrix:${cell.surface}:${cell.stage}`; - return { - test: { - kind: "ux-matrix-cell", - id: `ux-matrix.${cell.surface}.${cell.stage}`, - title: cell.title, - source: { path: SOURCE_PATH }, - }, - // These entries prove the evidence/gallery infrastructure, not product taxonomy behavior. - coverage: [], - refs: [ - { kind: "code", path: SOURCE_PATH }, - { kind: "docs", path: "docs/concepts/qa-e2e-automation.md" }, - ], - execution: buildExecution({ - artifacts: cell.artifacts, - repoRoot, - source, - }), - result: { - status: cell.status, - ...(cell.status === "pass" - ? {} - : { - failure: { - class: cell.status, - reason: cell.failureReason ?? `${cell.status} UX Matrix cell`, - }, - }), - timing: { - wallMs: Math.max(1, cell.wallMs), - }, - }, - }; -} - -function buildEvidenceSummary(params: { - cells: readonly MatrixCell[]; - generatedAt: string; - repoRoot: string; -}): QaEvidenceSummaryJson { - return validateQaEvidenceSummaryJson({ - kind: QA_EVIDENCE_SUMMARY_KIND, - schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, - generatedAt: params.generatedAt, - evidenceMode: "full", - entries: params.cells.map((cell) => buildEvidenceEntry(cell, params.repoRoot)), - }); -} - -async function runCommandForCell(params: { - args: string[]; - artifactBase: string; - command: string; - cwd: string; - logPath: string; - timeoutMs: number; -}) { - const startedAt = Date.now(); - const commandLine = [params.command, ...params.args].join(" "); - try { - const { stdout, stderr } = await execFileAsync(params.command, params.args, { - cwd: params.cwd, - timeout: params.timeoutMs, - env: process.env, - maxBuffer: 1024 * 1024, - }); - await writeText( - params.logPath, - sanitizeArtifactText(`$ ${commandLine}\n${stdout}${stderr}`, { - artifactBase: params.artifactBase, - repoRoot: params.cwd, - }), - ); - return { - status: "pass" as const, - wallMs: Date.now() - startedAt, - }; - } catch (error) { - const details = sanitizeArtifactText(error instanceof Error ? error.message : String(error), { - artifactBase: params.artifactBase, - repoRoot: params.cwd, - }); - await writeText(params.logPath, `$ ${commandLine}\nblocked: ${details}\n`); - return { - failureReason: details, - status: "blocked" as const, - wallMs: Date.now() - startedAt, - }; - } -} - -async function writePreflight(artifactBase: string) { - await writeText( - path.join(artifactBase, "preflight", "runtime.txt"), - [ - `platform=${process.platform}`, - `arch=${process.arch}`, - `node=${process.version}`, - `freeMemoryBytes=${os.freemem()}`, - `totalMemoryBytes=${os.totalmem()}`, - ].join("\n") + "\n", - ); -} - -async function writeSkippedVisualProof(logPath: string) { - const startedAt = Date.now(); - await writeText(logPath, "blocked: --skip-visual-proof was set\n"); - return { - failureReason: "--skip-visual-proof was set", - status: "blocked" as const, - wallMs: Date.now() - startedAt, - }; -} - -function isMissingManagedPlaywrightBrowser(error: unknown) { - const message = error instanceof Error ? error.message : String(error); - return ( - message.includes("Executable doesn't exist") && - message.includes(".cache/ms-playwright") && - message.includes("playwright install") - ); -} - -export async function launchUxMatrixChromium(params?: { - chromium?: Pick; - systemExecutablePath?: string; -}): Promise<{ browser: ChromiumBrowser; usedSystemExecutablePath?: string }> { - const chromium = params?.chromium ?? (await import("playwright")).chromium; - try { - return { browser: await chromium.launch() }; - } catch (error) { - if (!isMissingManagedPlaywrightBrowser(error)) { - throw error; - } - const executablePath = params?.systemExecutablePath ?? resolveSystemChromiumExecutablePath(); - if (!executablePath) { - throw error; - } - return { - browser: await chromium.launch({ executablePath }), - usedSystemExecutablePath: executablePath, - }; - } -} - -export function ensureUxMatrixVideoDependencies(params: { - ensureChromium?: typeof ensurePlaywrightChromium; - usedSystemExecutablePath?: string; -}) { - if (!params.usedSystemExecutablePath) { - return; - } - const ensureChromium = params.ensureChromium ?? ensurePlaywrightChromium; - const status = ensureChromium({ - ensureFfmpeg: true, - systemExecutablePath: params.usedSystemExecutablePath, - }); - if (status !== 0) { - throw new Error(`Playwright ffmpeg install failed with status ${status}`); - } -} - -async function captureControlUiScreenshot(params: { - artifactBase: string; - htmlPath: string; - logPath: string; - repoRoot: string; - screenshotPath: string; - skipVisualProof: boolean; -}) { - if (params.skipVisualProof) { - return writeSkippedVisualProof(params.logPath); - } - const startedAt = Date.now(); - try { - const { browser } = await launchUxMatrixChromium(); - try { - const page = await browser.newPage({ viewport: { width: 1024, height: 720 } }); - await page.goto(pathToFileURL(params.htmlPath).href); - await page.screenshot({ path: params.screenshotPath, fullPage: true }); - } finally { - await browser.close(); - } - await writeText( - params.logPath, - `Captured ${relativeToArtifactBase(params.artifactBase, params.screenshotPath)}\n`, - ); - return { - status: "pass" as const, - wallMs: Date.now() - startedAt, - }; - } catch (error) { - const details = sanitizeArtifactText(error instanceof Error ? error.message : String(error), { - artifactBase: params.artifactBase, - repoRoot: params.repoRoot, - }); - await writeText(params.logPath, `blocked: ${details}\n`); - return { - failureReason: details, - status: "blocked" as const, - wallMs: Date.now() - startedAt, - }; - } -} - -function escapeHtml(value: string) { - return value - .replaceAll("&", "&") - .replaceAll("<", "<") - .replaceAll(">", ">") - .replaceAll('"', """); -} - -async function writeProducerArtifactFixtureHtml(params: { - artifactBase: string; - evidence: QaEvidenceSummaryJson; - htmlPath: string; - logPreview: string; -}) { - const previewArtifacts = params.evidence.entries - .filter((entry) => entry.test.id !== "ux-matrix.qa-lab.producer-artifact-fixture") - .flatMap((entry) => entry.execution?.artifacts ?? []); - const screenshotArtifact = previewArtifacts.find((artifact) => artifact.kind === "screenshot"); - const logArtifact = previewArtifacts.find((artifact) => artifact.kind === "log"); - const screenshotPath = screenshotArtifact - ? relativeToArtifactBase( - path.dirname(params.htmlPath), - path.join(params.artifactBase, screenshotArtifact.path), - ) - : ""; - const entryRows = params.evidence.entries - .map( - (entry) => - `
  • ${escapeHtml(entry.test.id)} - ${escapeHtml( - entry.result.status, - )} - ${entry.coverage.map((coverage) => escapeHtml(coverage.id)).join(", ")}
    • `, - ) - .join(""); - await writeText( - params.htmlPath, - ` - -UX Matrix producer artifact fixture - -
    -

    UX Matrix Producer Artifact Fixture

    -

    Standalone script-produced ${escapeHtml( - QA_EVIDENCE_FILENAME, - )}; this fixture is not the QA Lab Evidence Archive UI or product-behavior proof.

    -
    -

    UX Matrix entries

    -
      ${entryRows}
    -
    -
    -

    Artifact preview

    -

    ${escapeHtml(logArtifact?.path ?? "no log")} - ${escapeHtml( - screenshotArtifact?.path ?? "no screenshot", - )}

    -
    - - -
    -
    Choose an artifact to preview.
    -
    -
    - -`, - ); -} - -async function captureProducerArtifactFixtureProof(params: { - artifactBase: string; - htmlPath: string; - logPath: string; - repoRoot: string; - screenshotPath: string; - skipVisualProof: boolean; - videoPath: string; -}) { - if (params.skipVisualProof) { - return writeSkippedVisualProof(params.logPath); - } - const startedAt = Date.now(); - try { - const videoDir = path.join(path.dirname(params.videoPath), "recording"); - await fs.mkdir(videoDir, { recursive: true }); - const { browser, usedSystemExecutablePath } = await launchUxMatrixChromium(); - let recordedVideo: string | undefined; - try { - ensureUxMatrixVideoDependencies({ usedSystemExecutablePath }); - const context = await browser.newContext({ - viewport: { width: 1280, height: 820 }, - recordVideo: { - dir: videoDir, - size: { width: 1280, height: 820 }, - }, - }); - const page = await context.newPage(); - const video = page.video(); - await page.goto(pathToFileURL(params.htmlPath).href); - await page.locator("#preview-screenshot").click(); - await page.locator("#preview img").waitFor({ state: "visible", timeout: 5_000 }); - await page.screenshot({ path: params.screenshotPath, fullPage: true }); - await page.waitForTimeout(350); - await page.locator("#preview-log").click(); - await page.waitForTimeout(350); - await context.close(); - recordedVideo = await video?.path(); - } finally { - await browser.close(); - } - if (!recordedVideo) { - throw new Error("Playwright did not provide a recording path"); - } - await fs.mkdir(path.dirname(params.videoPath), { recursive: true }); - await fs.copyFile(recordedVideo, params.videoPath); - await writeText( - params.logPath, - [ - `Captured screenshot ${path.basename(params.screenshotPath)}`, - `Captured recording ${path.basename(params.videoPath)}`, - ].join("\n") + "\n", - ); - return { - status: "pass" as const, - wallMs: Date.now() - startedAt, - }; - } catch (error) { - const details = sanitizeArtifactText(error instanceof Error ? error.message : String(error), { - artifactBase: params.artifactBase, - repoRoot: params.repoRoot, - }); - await writeText(params.logPath, `blocked: ${details}\n`); - return { - failureReason: details, - status: "blocked" as const, - wallMs: Date.now() - startedAt, - }; - } -} - -async function writeProducerMetadata(params: { - artifactBase: string; - cells: readonly MatrixCell[]; - repoRoot: string; -}) { - const counts = params.cells.reduce>((acc, cell) => { - acc[cell.status] = (acc[cell.status] ?? 0) + 1; - return acc; - }, {}); - await writeJson(path.join(params.artifactBase, "manifest.json"), { - kind: "openclaw.qa.ux-matrix", - run: { - status: counts.fail ? "fail" : counts.blocked ? "blocked" : "pass", - }, - }); - await writeJson(path.join(params.artifactBase, "matrix.json"), { - cells: params.cells.map((cell) => ({ - coverageIds: [], - stage: cell.stage, - status: cell.status, - surface: cell.surface, - })), - counts, - }); - await writeJson(path.join(params.artifactBase, "release-ledger.json"), { - entries: params.cells.map((cell) => ({ - coverageIds: [], - stage: cell.stage, - status: cell.status, - surface: cell.surface, - })), - kind: "openclaw.qa.ux-matrix.release-ledger", - }); - await writeText( - path.join(params.artifactBase, "commands.txt"), - `node --import tsx ${SOURCE_PATH} --artifact-base ${toRepoRelativePath( - params.repoRoot, - params.artifactBase, - )}\n`, - ); - await writeText( - path.join(params.artifactBase, "scorecard.md"), - ["# UX Matrix", "", ...Object.entries(counts).map(([status, count]) => `- ${status}: ${count}`)] - .join("\n") - .trimEnd() + "\n", - ); -} - -async function runUxMatrixEvidenceProducer(options: ProducerOptions) { - await fs.mkdir(options.artifactBase, { recursive: true }); - await writePreflight(options.artifactBase); - - const cliLogPath = path.join( - cellDir(options.artifactBase, { surface: "cli", stage: "entrypoint-help" }), - "logs.txt", - ); - const cliResult = await runCommandForCell({ - args: ["openclaw.mjs", "--help"], - artifactBase: options.artifactBase, - command: process.execPath, - cwd: options.repoRoot, - logPath: cliLogPath, - timeoutMs: 30_000, - }); - - const screenshotCellDir = cellDir(options.artifactBase, { - surface: "control-ui", - stage: "screenshot-artifact", - }); - const matrixHtmlPath = path.join(screenshotCellDir, "matrix-preview.html"); - await writeText( - matrixHtmlPath, - 'UX Matrix

    UX Matrix

    Control UI artifact preview fixture generated by the scenario.

    ', - ); - const matrixScreenshotPath = path.join(screenshotCellDir, "screenshot.png"); - const matrixScreenshotResult = await captureControlUiScreenshot({ - artifactBase: options.artifactBase, - htmlPath: matrixHtmlPath, - logPath: path.join(screenshotCellDir, "logs.txt"), - repoRoot: options.repoRoot, - screenshotPath: matrixScreenshotPath, - skipVisualProof: options.skipVisualProof, - }); - - const initialCells: MatrixCell[] = [ - { - artifacts: [ - { - kind: "log", - path: relativeToArtifactBase( - options.artifactBase, - path.join(screenshotCellDir, "logs.txt"), - ), - }, - ...(matrixScreenshotResult.status === "pass" - ? [ - { - kind: "screenshot", - path: relativeToArtifactBase(options.artifactBase, matrixScreenshotPath), - }, - ] - : []), - ], - failureReason: - "failureReason" in matrixScreenshotResult - ? matrixScreenshotResult.failureReason - : undefined, - stage: "screenshot-artifact", - status: matrixScreenshotResult.status, - surface: "control-ui", - title: "UX Matrix: screenshot artifact", - wallMs: matrixScreenshotResult.wallMs, - }, - { - artifacts: [{ kind: "log", path: relativeToArtifactBase(options.artifactBase, cliLogPath) }], - failureReason: cliResult.failureReason, - stage: "entrypoint-help", - status: cliResult.status, - surface: "cli", - title: "UX Matrix: CLI entrypoint help", - wallMs: cliResult.wallMs, - }, - ]; - - const fixtureProofDir = cellDir(options.artifactBase, { - surface: "qa-lab", - stage: "producer-artifact-fixture", - }); - const fixtureHtmlPath = path.join(fixtureProofDir, "producer-artifact-fixture.html"); - const previewEvidence = buildEvidenceSummary({ - cells: initialCells, - generatedAt: new Date().toISOString(), - repoRoot: options.repoRoot, - }); - const screenshotLog = await fs.readFile(path.join(screenshotCellDir, "logs.txt"), "utf8"); - await writeProducerArtifactFixtureHtml({ - artifactBase: options.artifactBase, - evidence: previewEvidence, - htmlPath: fixtureHtmlPath, - logPreview: screenshotLog, - }); - - const fixtureProofResult = await captureProducerArtifactFixtureProof({ - artifactBase: options.artifactBase, - htmlPath: fixtureHtmlPath, - logPath: path.join(fixtureProofDir, "logs.txt"), - repoRoot: options.repoRoot, - screenshotPath: path.join(fixtureProofDir, "producer-artifact-fixture.png"), - skipVisualProof: options.skipVisualProof, - videoPath: path.join(fixtureProofDir, "producer-artifact-fixture.webm"), - }); - - const cells: MatrixCell[] = [ - { - artifacts: [ - { kind: "html", path: relativeToArtifactBase(options.artifactBase, fixtureHtmlPath) }, - { - kind: "log", - path: relativeToArtifactBase( - options.artifactBase, - path.join(fixtureProofDir, "logs.txt"), - ), - }, - ...(fixtureProofResult.status === "pass" - ? [ - { - kind: "screenshot", - path: relativeToArtifactBase( - options.artifactBase, - path.join(fixtureProofDir, "producer-artifact-fixture.png"), - ), - }, - { - kind: "video", - path: relativeToArtifactBase( - options.artifactBase, - path.join(fixtureProofDir, "producer-artifact-fixture.webm"), - ), - }, - ] - : []), - ], - failureReason: - "failureReason" in fixtureProofResult ? fixtureProofResult.failureReason : undefined, - stage: "producer-artifact-fixture", - status: fixtureProofResult.status, - surface: "qa-lab", - title: "UX Matrix: producer artifact fixture", - wallMs: fixtureProofResult.wallMs, - }, - ...initialCells, - ]; - - const evidence = buildEvidenceSummary({ - cells, - generatedAt: new Date().toISOString(), - repoRoot: options.repoRoot, - }); - await writeProducerArtifactFixtureHtml({ - artifactBase: options.artifactBase, - evidence, - htmlPath: fixtureHtmlPath, - logPreview: screenshotLog, - }); - await writeJson(path.join(options.artifactBase, QA_EVIDENCE_FILENAME), evidence); - await writeJson(path.join(options.artifactBase, "latest-run.json"), { - qaEvidence: QA_EVIDENCE_FILENAME, - }); - await writeProducerMetadata({ - artifactBase: options.artifactBase, - cells, - repoRoot: options.repoRoot, - }); - return { - artifactBase: options.artifactBase, - evidence, - }; -} - -if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { - process.exitCode = await runUxMatrixEvidenceProducerCli(process.argv.slice(2)); -} diff --git a/test/scripts/qa-ux-matrix-evidence-producer.test.ts b/test/scripts/qa-ux-matrix-evidence-producer.test.ts deleted file mode 100644 index 349c57db0d2f..000000000000 --- a/test/scripts/qa-ux-matrix-evidence-producer.test.ts +++ /dev/null @@ -1,230 +0,0 @@ -// QA UX Matrix evidence producer tests cover operator-facing CLI behavior. -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { describe, expect, it, vi } from "vitest"; -import { - ensureUxMatrixVideoDependencies, - launchUxMatrixChromium, - runUxMatrixEvidenceProducerCli, -} from "../../scripts/qa/ux-matrix-evidence-producer.js"; - -async function runCli(...args: string[]) { - const stdout: string[] = []; - const stderr: string[] = []; - const status = await runUxMatrixEvidenceProducerCli(args, { - error: (message) => stderr.push(message), - log: (message) => stdout.push(message), - }); - return { - status, - stderr: stderr.length > 0 ? `${stderr.join("\n")}\n` : "", - stdout: stdout.length > 0 ? `${stdout.join("\n")}\n` : "", - }; -} - -function expectNoNodeStack(stderr: string) { - expect(stderr).not.toContain("Node.js"); - expect(stderr).not.toContain("\n at "); -} - -describe("QA UX Matrix evidence producer CLI", () => { - it("prints help without generating evidence", async () => { - const result = await runCli("--help"); - - expect(result.status).toBe(0); - expect(result.stdout).toContain( - "Usage: node --import tsx scripts/qa/ux-matrix-evidence-producer.ts", - ); - expect(result.stderr).toBe(""); - }); - - it("prints help after boolean options without consuming valued option slots", async () => { - const result = await runCli("--skip-visual-proof", "--help"); - - expect(result.status).toBe(0); - expect(result.stdout).toContain( - "Usage: node --import tsx scripts/qa/ux-matrix-evidence-producer.ts", - ); - expect(result.stderr).toBe(""); - }); - - it("reports invalid args without a Node stack trace", async () => { - const result = await runCli("--wat"); - - expect(result.status).toBe(1); - expect(result.stdout).toBe(""); - expect(result.stderr.trim()).toBe("unsupported UX Matrix producer arg: --wat"); - expectNoNodeStack(result.stderr); - }); - - it("reports missing valued args without a Node stack trace", async () => { - const result = await runCli("--artifact-base", "--repo-root", "."); - - expect(result.status).toBe(1); - expect(result.stdout).toBe(""); - expect(result.stderr.trim()).toBe("--artifact-base requires a value"); - expectNoNodeStack(result.stderr); - }); - - it("reports duplicate evidence producer args without a Node stack trace", async () => { - const duplicateCases = [ - ["--artifact-base", ["--artifact-base", ".artifacts/a", "--artifact-base", ".artifacts/b"]], - ["--repo-root", ["--artifact-base", ".artifacts/a", "--repo-root", ".", "--repo-root", ".."]], - [ - "--skip-visual-proof", - ["--artifact-base", ".artifacts/a", "--skip-visual-proof", "--skip-visual-proof"], - ], - ] satisfies Array<[string, string[]]>; - - for (const [flag, args] of duplicateCases) { - const result = await runCli(...args); - - expect(result.status).toBe(1); - expect(result.stdout).toBe(""); - expect(result.stderr.trim()).toBe(`${flag} was provided more than once`); - expectNoNodeStack(result.stderr); - } - }); - - it("reports short flag values without treating them as help", async () => { - const artifactBaseResult = await runCli("--artifact-base", "-h"); - const repoRootResult = await runCli( - "--artifact-base", - "/tmp/openclaw-ux-test", - "--repo-root", - "-h", - ); - - expect(artifactBaseResult.status).toBe(1); - expect(artifactBaseResult.stdout).toBe(""); - expect(artifactBaseResult.stderr.trim()).toBe("--artifact-base requires a value"); - expectNoNodeStack(artifactBaseResult.stderr); - expect(repoRootResult.status).toBe(1); - expect(repoRootResult.stdout).toBe(""); - expect(repoRootResult.stderr.trim()).toBe("--repo-root requires a value"); - expectNoNodeStack(repoRootResult.stderr); - }); - - it("sanitizes local checkout paths from generated evidence artifacts", async () => { - const artifactBase = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-ux-evidence-test-")); - const fakeRepoRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-ux-repo-test-")); - try { - const result = await runCli( - "--artifact-base", - artifactBase, - "--repo-root", - fakeRepoRoot, - "--skip-visual-proof", - ); - - expect(result.status).toBe(0); - const evidence = fs.readFileSync(path.join(artifactBase, "qa-evidence.json"), "utf8"); - const cliLog = fs.readFileSync( - path.join(artifactBase, "surfaces", "cli", "stages", "entrypoint-help", "logs.txt"), - "utf8", - ); - const visualLog = fs.readFileSync( - path.join( - artifactBase, - "surfaces", - "control-ui", - "stages", - "screenshot-artifact", - "logs.txt", - ), - "utf8", - ); - expect(evidence).not.toContain(fakeRepoRoot); - expect(cliLog).not.toContain(fakeRepoRoot); - expect(`${evidence}\n${cliLog}`).toContain(""); - expect(visualLog).toBe("blocked: --skip-visual-proof was set\n"); - - const evidenceJson = JSON.parse(evidence) as { - entries: Array<{ - coverage: unknown[]; - execution?: { artifacts: Array<{ kind: string }> }; - test: { id: string }; - }>; - }; - const matrix = JSON.parse( - fs.readFileSync(path.join(artifactBase, "matrix.json"), "utf8"), - ) as { cells: Array<{ coverageIds: unknown[] }> }; - const releaseLedger = JSON.parse( - fs.readFileSync(path.join(artifactBase, "release-ledger.json"), "utf8"), - ) as { entries: Array<{ coverageIds: unknown[] }> }; - const manifest = JSON.parse( - fs.readFileSync(path.join(artifactBase, "manifest.json"), "utf8"), - ) as { run: Record }; - const commands = fs.readFileSync(path.join(artifactBase, "commands.txt"), "utf8"); - - expect(evidenceJson.entries.map((entry) => entry.test.id)).toEqual([ - "ux-matrix.qa-lab.producer-artifact-fixture", - "ux-matrix.control-ui.screenshot-artifact", - "ux-matrix.cli.entrypoint-help", - ]); - expect(evidenceJson.entries.every((entry) => entry.coverage.length === 0)).toBe(true); - expect(matrix.cells.every((cell) => cell.coverageIds.length === 0)).toBe(true); - expect(releaseLedger.entries.every((entry) => entry.coverageIds.length === 0)).toBe(true); - expect(manifest.run).not.toHaveProperty("scenarioId"); - expect(commands).toContain( - "node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base", - ); - expect(commands).not.toContain("qa suite --scenario"); - expect( - evidenceJson.entries.flatMap( - (entry) => entry.execution?.artifacts.map((artifact) => artifact.kind) ?? [], - ), - ).toEqual(expect.arrayContaining(["html", "log"])); - } finally { - fs.rmSync(artifactBase, { recursive: true, force: true }); - fs.rmSync(fakeRepoRoot, { recursive: true, force: true }); - } - }); - - it("falls back to system Chromium when the managed Playwright browser is missing", async () => { - const browser = { close: vi.fn() }; - const launch = vi - .fn() - .mockRejectedValueOnce( - new Error( - [ - "browserType.launch: Executable doesn't exist at /home/user/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell", - "Please run the following command to download new browsers:", - "pnpm exec playwright install", - ].join("\n"), - ), - ) - .mockResolvedValueOnce(browser); - - const result = await launchUxMatrixChromium({ - chromium: { launch } as unknown as NonNullable< - Parameters[0] - >["chromium"], - systemExecutablePath: "/usr/bin/chromium-browser", - }); - - expect(result).toEqual({ - browser, - usedSystemExecutablePath: "/usr/bin/chromium-browser", - }); - expect(launch).toHaveBeenNthCalledWith(1); - expect(launch).toHaveBeenNthCalledWith(2, { - executablePath: "/usr/bin/chromium-browser", - }); - }); - - it("ensures Playwright ffmpeg when video proof uses system Chromium", () => { - const ensureChromium = vi.fn(() => 0); - - ensureUxMatrixVideoDependencies({ - ensureChromium, - usedSystemExecutablePath: "/usr/bin/chromium-browser", - }); - - expect(ensureChromium).toHaveBeenCalledWith({ - ensureFfmpeg: true, - systemExecutablePath: "/usr/bin/chromium-browser", - }); - }); -});