mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-23 02:45:38 -06:00
refactor(qa): simplify scenario runner infrastructure (#125738)
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
6ed136b22d
commit
8a0d28d6ba
@@ -208,7 +208,6 @@ extensions/qa-lab/src/runtime-tool-fixture.ts
|
||||
extensions/qa-lab/src/scorecard-taxonomy.ts
|
||||
extensions/qa-lab/src/suite-launch.runtime.test.ts
|
||||
extensions/qa-lab/src/suite-launch.runtime.ts
|
||||
extensions/qa-lab/src/test-file-scenario-runner.test.ts
|
||||
extensions/qa-lab/web/src/app.ts
|
||||
extensions/signal/src/client-container.test.ts
|
||||
extensions/signal/src/client-container.ts
|
||||
|
||||
@@ -368,9 +368,9 @@ describe("evidence gallery", () => {
|
||||
coverageIds: [],
|
||||
runner: {
|
||||
availability: "local",
|
||||
command: `node --import tsx ${repoRoot}/scripts/qa/ux-matrix-evidence-producer.ts --artifact-base ${runDir}`,
|
||||
command: `node ${repoRoot}/external/qa/ux-matrix-producer.mjs --artifact-base ${runDir}`,
|
||||
lane: "web-ui-playwright",
|
||||
workflow: `${repoRoot}/.github/workflows/ux-matrix-qa.yml#ux-matrix-local`,
|
||||
workflow: `${repoRoot}/external/ci/ux-matrix.yml#matrix-local`,
|
||||
},
|
||||
stage: "first-run",
|
||||
status: "pass",
|
||||
@@ -381,9 +381,9 @@ describe("evidence gallery", () => {
|
||||
runner: {
|
||||
availability: "local",
|
||||
command:
|
||||
"node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix",
|
||||
"node external/qa/ux-matrix-producer.mjs --artifact-base .artifacts/external-qa/ux-matrix",
|
||||
lane: "cli-status",
|
||||
workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local",
|
||||
workflow: "external/ci/ux-matrix.yml#matrix-local",
|
||||
},
|
||||
stage: "first-run",
|
||||
status: "proof-gap",
|
||||
@@ -402,7 +402,7 @@ describe("evidence gallery", () => {
|
||||
await fs.writeFile(path.join(runDir, "scorecard.md"), "# UX Matrix\n\n- pass: 1\n", "utf8");
|
||||
await fs.writeFile(
|
||||
path.join(runDir, "commands.txt"),
|
||||
"node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix\n",
|
||||
"node external/qa/ux-matrix-producer.mjs --artifact-base .artifacts/external-qa/ux-matrix\n",
|
||||
"utf8",
|
||||
);
|
||||
await fs.mkdir(path.join(runDir, "preflight"), { recursive: true });
|
||||
@@ -424,7 +424,7 @@ describe("evidence gallery", () => {
|
||||
kind: "ux-matrix-cell",
|
||||
id: "ux-matrix.web-ui.first-run",
|
||||
title: `UX Matrix: web-ui / first-run at ${repoRoot}`,
|
||||
source: { path: "scripts/qa/ux-matrix-evidence-producer.ts" },
|
||||
source: { path: "external/qa/ux-matrix-producer.mjs" },
|
||||
},
|
||||
coverage: [],
|
||||
execution: {
|
||||
@@ -463,7 +463,7 @@ describe("evidence gallery", () => {
|
||||
kind: "ux-matrix-cell",
|
||||
id: "qa-lab.wrapper-cli-error",
|
||||
title: "UX Matrix: cli / error-state",
|
||||
source: { path: "scripts/qa/ux-matrix-evidence-producer.ts" },
|
||||
source: { path: "external/qa/ux-matrix-producer.mjs" },
|
||||
},
|
||||
coverage: [],
|
||||
execution: {
|
||||
@@ -539,9 +539,9 @@ describe("evidence gallery", () => {
|
||||
runner: {
|
||||
availability: "local",
|
||||
command:
|
||||
"node --import tsx <repo-root>/scripts/qa/ux-matrix-evidence-producer.ts --artifact-base <repo-root>/.artifacts/qa-e2e/suite/script/nested<repo-root>/ux-matrix-producer/run-1",
|
||||
"node <repo-root>/external/qa/ux-matrix-producer.mjs --artifact-base <repo-root>/.artifacts/qa-e2e/suite/script/nested<repo-root>/ux-matrix-producer/run-1",
|
||||
lane: "web-ui-playwright",
|
||||
workflow: "<repo-root>/.github/workflows/ux-matrix-qa.yml#ux-matrix-local",
|
||||
workflow: "<repo-root>/external/ci/ux-matrix.yml#matrix-local",
|
||||
},
|
||||
stage: "first-run",
|
||||
status: "pass",
|
||||
@@ -556,9 +556,9 @@ describe("evidence gallery", () => {
|
||||
runner: {
|
||||
availability: "local",
|
||||
command:
|
||||
"node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix",
|
||||
"node external/qa/ux-matrix-producer.mjs --artifact-base .artifacts/external-qa/ux-matrix",
|
||||
lane: "cli-status",
|
||||
workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local",
|
||||
workflow: "external/ci/ux-matrix.yml#matrix-local",
|
||||
},
|
||||
stage: "first-run",
|
||||
status: "proof-gap",
|
||||
@@ -584,7 +584,7 @@ describe("evidence gallery", () => {
|
||||
repoRoot,
|
||||
);
|
||||
expect(model.producerContext?.commands?.preview).toBe(
|
||||
"node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix\n",
|
||||
"node external/qa/ux-matrix-producer.mjs --artifact-base .artifacts/external-qa/ux-matrix\n",
|
||||
);
|
||||
expect(model.producerContext?.commands?.path).toContain("commands.txt");
|
||||
expect(decodeURIComponent(model.producerContext?.commands?.href ?? "")).not.toContain(repoRoot);
|
||||
|
||||
+14
-22
@@ -1,7 +1,9 @@
|
||||
import { spawn as startOpenClawCliProcess, spawnSync } from "node:child_process";
|
||||
import { spawn as startOpenClawCliProcess } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { resolveQaWindowsSystem32ExePath } from "../../../windows-system-tools.js";
|
||||
import { runQaWindowsTaskkill } from "../../../windows-system-tools.js";
|
||||
|
||||
type MatrixQaTaskkillRunner = NonNullable<Parameters<typeof runQaWindowsTaskkill>[0]["runCommand"]>;
|
||||
|
||||
export function resolveMatrixQaOpenClawCliEntryPath(cwd: string): string {
|
||||
const mjsEntryPath = path.join(cwd, "dist", "index.mjs");
|
||||
@@ -14,28 +16,18 @@ export function resolveMatrixQaOpenClawCliEntryPath(cwd: string): string {
|
||||
export function killMatrixQaCliChild(
|
||||
child: ReturnType<typeof startOpenClawCliProcess>,
|
||||
signal: NodeJS.Signals,
|
||||
runTaskkill: typeof spawnSync = spawnSync,
|
||||
runTaskkill?: MatrixQaTaskkillRunner,
|
||||
): void {
|
||||
if (process.platform === "win32") {
|
||||
if (child.pid) {
|
||||
const taskkillPath = resolveQaWindowsSystem32ExePath("taskkill.exe");
|
||||
const args = ["/PID", String(child.pid), "/T"];
|
||||
if (signal === "SIGKILL") {
|
||||
args.push("/F");
|
||||
}
|
||||
const result = runTaskkill(taskkillPath, args, { stdio: "ignore", windowsHide: true });
|
||||
if (!result.error && result.status === 0) {
|
||||
return;
|
||||
}
|
||||
if (signal !== "SIGKILL") {
|
||||
const forceResult = runTaskkill(taskkillPath, [...args, "/F"], {
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
});
|
||||
if (!forceResult.error && forceResult.status === 0) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (
|
||||
child.pid &&
|
||||
runQaWindowsTaskkill({
|
||||
pid: child.pid,
|
||||
signal,
|
||||
...(runTaskkill ? { runCommand: runTaskkill } : {}),
|
||||
})
|
||||
) {
|
||||
return;
|
||||
}
|
||||
child.kill(signal);
|
||||
return;
|
||||
|
||||
+12
-77
@@ -78,13 +78,12 @@ describe("Matrix QA CLI runtime", () => {
|
||||
).toBe("GET /_matrix/client/v3/sync?access_token=abcdef…ghij");
|
||||
});
|
||||
|
||||
it("force-kills Windows CLI process trees when graceful taskkill fails", () => {
|
||||
it.each([
|
||||
{ label: "succeeds after escalation", statuses: [1, 0], fallsBack: false },
|
||||
{ label: "fails completely", statuses: [1, 1], fallsBack: true },
|
||||
])("composes Windows cleanup when canonical taskkill $label", ({ fallsBack, statuses }) => {
|
||||
const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform");
|
||||
const originalSystemRoot = process.env.SystemRoot;
|
||||
const originalWindir = process.env.WINDIR;
|
||||
Object.defineProperty(process, "platform", { value: "win32", configurable: true });
|
||||
process.env.SystemRoot = "C:\\Windows";
|
||||
delete process.env.WINDIR;
|
||||
try {
|
||||
const killMock = vi.fn();
|
||||
const child = {
|
||||
@@ -93,35 +92,21 @@ describe("Matrix QA CLI runtime", () => {
|
||||
} as unknown as Parameters<typeof testing.killMatrixQaCliChild>[0];
|
||||
const runTaskkill = vi
|
||||
.fn()
|
||||
.mockReturnValueOnce({ status: 1 })
|
||||
.mockReturnValueOnce({ status: 0 });
|
||||
.mockReturnValueOnce({ status: statuses[0] })
|
||||
.mockReturnValueOnce({ status: statuses[1] });
|
||||
|
||||
testing.killMatrixQaCliChild(child, "SIGTERM", runTaskkill);
|
||||
|
||||
const taskkillPath = path.win32.join("C:\\Windows", "System32", "taskkill.exe");
|
||||
expect(runTaskkill).toHaveBeenNthCalledWith(1, taskkillPath, ["/PID", "12345", "/T"], {
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
});
|
||||
expect(runTaskkill).toHaveBeenNthCalledWith(2, taskkillPath, ["/PID", "12345", "/T", "/F"], {
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
});
|
||||
expect(killMock).not.toHaveBeenCalled();
|
||||
expect(runTaskkill).toHaveBeenCalledTimes(2);
|
||||
if (fallsBack) {
|
||||
expect(killMock).toHaveBeenCalledWith("SIGTERM");
|
||||
} else {
|
||||
expect(killMock).not.toHaveBeenCalled();
|
||||
}
|
||||
} finally {
|
||||
if (platformDescriptor) {
|
||||
Object.defineProperty(process, "platform", platformDescriptor);
|
||||
}
|
||||
if (originalSystemRoot === undefined) {
|
||||
delete process.env.SystemRoot;
|
||||
} else {
|
||||
process.env.SystemRoot = originalSystemRoot;
|
||||
}
|
||||
if (originalWindir === undefined) {
|
||||
delete process.env.WINDIR;
|
||||
} else {
|
||||
process.env.WINDIR = originalWindir;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -389,56 +374,6 @@ describe("Matrix QA CLI runtime", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("kills ignored-stdio descendants after a timed-out CLI exits gracefully", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
const root = await mkdtemp(
|
||||
path.join(resolvePreferredOpenClawTmpDir(), "matrix-qa-cli-timeout-ignored-stdio-"),
|
||||
);
|
||||
const childPidPath = path.join(root, "child.pid");
|
||||
const grandchildPidPath = path.join(root, "grandchild.pid");
|
||||
let childPid: number | undefined;
|
||||
let grandchildPid: number | undefined;
|
||||
try {
|
||||
await mkdir(path.join(root, "dist"));
|
||||
await writeFile(
|
||||
path.join(root, "dist", "index.mjs"),
|
||||
[
|
||||
"import { spawn } from 'node:child_process';",
|
||||
"import { writeFileSync } from 'node:fs';",
|
||||
`writeFileSync(${JSON.stringify(childPidPath)}, String(process.pid));`,
|
||||
"const grandchild = spawn(process.execPath, ['-e', 'process.on(\\'SIGTERM\\', () => {}); setInterval(() => {}, 1000);'], { stdio: 'ignore' });",
|
||||
"grandchild.unref();",
|
||||
`writeFileSync(${JSON.stringify(grandchildPidPath)}, String(grandchild.pid));`,
|
||||
"process.on('SIGTERM', () => process.exit(0));",
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
const run = runMatrixQaOpenClawCli({
|
||||
args: ["matrix", "verify", "self"],
|
||||
cwd: root,
|
||||
env: process.env,
|
||||
timeoutMs: 500,
|
||||
});
|
||||
grandchildPid = await waitForPidFile(grandchildPidPath, 2_000);
|
||||
|
||||
await expect(run).rejects.toThrow(/timed out after 500ms/u);
|
||||
|
||||
childPid = await waitForPidFile(childPidPath, 2_000);
|
||||
expect(isProcessRunning(childPid)).toBe(false);
|
||||
expect(isProcessRunning(grandchildPid)).toBe(false);
|
||||
} finally {
|
||||
for (const pid of [grandchildPid, childPid]) {
|
||||
if (pid && isProcessRunning(pid)) {
|
||||
process.kill(pid, "SIGKILL");
|
||||
}
|
||||
}
|
||||
await rm(root, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("kills ignored-stdio descendants after manual CLI session kill", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Qa Lab plugin module runs CLI processes and parses their structured output.
|
||||
import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import path from "node:path";
|
||||
import { resolveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime";
|
||||
import { isRecord as isJsonRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
@@ -17,7 +17,7 @@ import { QaSuiteInfraError } from "./errors.js";
|
||||
import { resolveQaNodeExecPath } from "./node-exec.js";
|
||||
import { createQaPosixCommandSettlement } from "./posix-command-settlement.js";
|
||||
import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js";
|
||||
import { resolveQaWindowsSystem32ExePath } from "./windows-system-tools.js";
|
||||
import { runQaWindowsTaskkill } from "./windows-system-tools.js";
|
||||
|
||||
const ANSI_ESCAPE_PATTERN = new RegExp(String.raw`\x1B\[[0-?]*[ -/]*[@-~]`, "g");
|
||||
|
||||
@@ -181,19 +181,8 @@ function parseQaCliJsonOutput(text: string, args: readonly string[]) {
|
||||
}
|
||||
|
||||
function killQaCliWindowsProcessTree(child: Pick<ChildProcessWithoutNullStreams, "kill" | "pid">) {
|
||||
if (child.pid) {
|
||||
const result = spawnSync(
|
||||
resolveQaWindowsSystem32ExePath("taskkill.exe"),
|
||||
["/PID", String(child.pid), "/T", "/F"],
|
||||
{
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
timeout: 5_000,
|
||||
},
|
||||
);
|
||||
if (!result.error && result.status === 0) {
|
||||
return;
|
||||
}
|
||||
if (child.pid && runQaWindowsTaskkill({ pid: child.pid, signal: "SIGKILL" })) {
|
||||
return;
|
||||
}
|
||||
child.kill("SIGKILL");
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const spawnMock = vi.hoisted(() => vi.fn());
|
||||
const spawnSyncMock = vi.hoisted(() => vi.fn());
|
||||
const runQaWindowsTaskkillMock = vi.hoisted(() => vi.fn());
|
||||
const resolveQaNodeExecPathMock = vi.hoisted(() => vi.fn(async () => "/usr/bin/node"));
|
||||
const waitForGatewayHealthyMock = vi.hoisted(() => vi.fn(async () => undefined));
|
||||
const waitForTransportReadyMock = vi.hoisted(() => vi.fn(async () => undefined));
|
||||
@@ -13,7 +13,11 @@ const readSessionTranscriptSummaryMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("node:child_process", () => ({
|
||||
spawn: spawnMock,
|
||||
spawnSync: spawnSyncMock,
|
||||
}));
|
||||
|
||||
vi.mock("./windows-system-tools.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./windows-system-tools.js")>()),
|
||||
runQaWindowsTaskkill: runQaWindowsTaskkillMock,
|
||||
}));
|
||||
|
||||
vi.mock("./node-exec.js", () => ({
|
||||
@@ -141,7 +145,7 @@ function createAgentPromptEnv(gatewayCall: ReturnType<typeof vi.fn>) {
|
||||
describe("qa suite runtime agent process helpers", () => {
|
||||
beforeEach(() => {
|
||||
spawnMock.mockReset();
|
||||
spawnSyncMock.mockReset();
|
||||
runQaWindowsTaskkillMock.mockReset();
|
||||
resolveQaNodeExecPathMock.mockClear();
|
||||
waitForGatewayHealthyMock.mockClear();
|
||||
waitForTransportReadyMock.mockClear();
|
||||
@@ -274,53 +278,44 @@ describe("qa suite runtime agent process helpers", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("force-kills timed-out Windows qa cli process trees with taskkill", async () => {
|
||||
const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform");
|
||||
const originalSystemRoot = process.env.SystemRoot;
|
||||
const originalWindir = process.env.WINDIR;
|
||||
Object.defineProperty(process, "platform", { value: "win32", configurable: true });
|
||||
process.env.SystemRoot = "C:\\Windows";
|
||||
delete process.env.WINDIR;
|
||||
try {
|
||||
const child = createSpawnedProcess({ pid: 12345 });
|
||||
spawnSyncMock.mockReturnValue({ status: 0 });
|
||||
const { pending } = startMockQaCli({
|
||||
args: ["qa", "suite"],
|
||||
child,
|
||||
options: { timeoutMs: 1 },
|
||||
});
|
||||
const timeoutAssertion = expect(pending).rejects.toThrow(
|
||||
"qa cli timed out: openclaw qa suite",
|
||||
);
|
||||
it.each([
|
||||
{ label: "succeeds", taskkillSucceeded: true },
|
||||
{ label: "falls back", taskkillSucceeded: false },
|
||||
])(
|
||||
"preserves the Windows timeout result when canonical cleanup $label",
|
||||
async ({ taskkillSucceeded }) => {
|
||||
const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform");
|
||||
Object.defineProperty(process, "platform", { value: "win32", configurable: true });
|
||||
try {
|
||||
const child = createSpawnedProcess({ pid: 12345 });
|
||||
runQaWindowsTaskkillMock.mockReturnValue(taskkillSucceeded);
|
||||
const { pending } = startMockQaCli({
|
||||
args: ["qa", "suite"],
|
||||
child,
|
||||
options: { timeoutMs: 1 },
|
||||
});
|
||||
const timeoutAssertion = expect(pending).rejects.toThrow(
|
||||
"qa cli timed out: openclaw qa suite",
|
||||
);
|
||||
|
||||
await waitForSpawnCount(1);
|
||||
await timeoutAssertion;
|
||||
expect(spawnSyncMock).toHaveBeenCalledWith(
|
||||
path.win32.join("C:\\Windows", "System32", "taskkill.exe"),
|
||||
["/PID", "12345", "/T", "/F"],
|
||||
{
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
timeout: 5_000,
|
||||
},
|
||||
);
|
||||
expect(child.kill).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
if (platformDescriptor) {
|
||||
Object.defineProperty(process, "platform", platformDescriptor);
|
||||
await waitForSpawnCount(1);
|
||||
await timeoutAssertion;
|
||||
expect(runQaWindowsTaskkillMock).toHaveBeenCalledWith({
|
||||
pid: 12345,
|
||||
signal: "SIGKILL",
|
||||
});
|
||||
if (taskkillSucceeded) {
|
||||
expect(child.kill).not.toHaveBeenCalled();
|
||||
} else {
|
||||
expect(child.kill).toHaveBeenCalledWith("SIGKILL");
|
||||
}
|
||||
} finally {
|
||||
if (platformDescriptor) {
|
||||
Object.defineProperty(process, "platform", platformDescriptor);
|
||||
}
|
||||
}
|
||||
if (originalSystemRoot === undefined) {
|
||||
delete process.env.SystemRoot;
|
||||
} else {
|
||||
process.env.SystemRoot = originalSystemRoot;
|
||||
}
|
||||
if (originalWindir === undefined) {
|
||||
delete process.env.WINDIR;
|
||||
} else {
|
||||
process.env.WINDIR = originalWindir;
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("merges isolated env overrides into qa cli runs", async () => {
|
||||
const { child, pending } = startMockQaCli({
|
||||
|
||||
@@ -4,6 +4,8 @@ import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { build as esbuild } from "esbuild";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const spawnMock = vi.hoisted(() => vi.fn());
|
||||
@@ -211,7 +213,25 @@ describe.skipIf(process.platform === "win32")("qa scenario command real POSIX li
|
||||
it("cleans the command group before re-raising a parent SIGTERM", async () => {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "qa-command-parent-signal-"));
|
||||
const descendantPidPath = path.join(root, "descendant.pid");
|
||||
const moduleUrl = new URL("./test-file-scenario-command-lifecycle.ts", import.meta.url).href;
|
||||
const bundlePath = path.join(root, "test-file-scenario-command-lifecycle.mjs");
|
||||
// Compile before the readiness window; a cold tsx child can exceed it under the full QA suite.
|
||||
// The bundle needs only the UTF-16 helper behind this SDK import, not the full plugin runtime.
|
||||
await esbuild({
|
||||
alias: {
|
||||
"openclaw/plugin-sdk/text-utility-runtime": fileURLToPath(
|
||||
new URL("../../../packages/normalization-core/src/utf16-slice.ts", import.meta.url),
|
||||
),
|
||||
},
|
||||
bundle: true,
|
||||
entryPoints: [
|
||||
fileURLToPath(new URL("./test-file-scenario-command-lifecycle.ts", import.meta.url)),
|
||||
],
|
||||
format: "esm",
|
||||
outfile: bundlePath,
|
||||
platform: "node",
|
||||
target: "node22",
|
||||
});
|
||||
const moduleUrl = pathToFileURL(bundlePath).href;
|
||||
let descendantPid: number | undefined;
|
||||
if (!actualSpawn.value) {
|
||||
throw new Error("real spawn unavailable");
|
||||
@@ -235,7 +255,7 @@ describe.skipIf(process.platform === "win32")("qa scenario command real POSIX li
|
||||
].join("\n");
|
||||
const controller = actualSpawn.value(
|
||||
process.execPath,
|
||||
["--import", "tsx", "--input-type=module", "-e", controllerScript],
|
||||
["--input-type=module", "-e", controllerScript],
|
||||
{ cwd: process.cwd(), env: process.env, stdio: "ignore" },
|
||||
);
|
||||
try {
|
||||
@@ -355,8 +375,10 @@ describe.skipIf(process.platform === "win32")("qa scenario command lifecycle", (
|
||||
signal: null,
|
||||
});
|
||||
expect(spawnSyncMock).toHaveBeenCalledTimes(2);
|
||||
expect(spawnSyncMock.mock.calls[0]?.[1]).toEqual(["/pid", "12345", "/T"]);
|
||||
expect(spawnSyncMock.mock.calls[1]?.[1]).toEqual(["/pid", "12345", "/T", "/F"]);
|
||||
expect(spawnSyncMock.mock.calls.map((call) => call[1])).toEqual([
|
||||
["/PID", "12345", "/T"],
|
||||
["/PID", "12345", "/T", "/F"],
|
||||
]);
|
||||
} finally {
|
||||
if (platformDescriptor) {
|
||||
Object.defineProperty(process, "platform", platformDescriptor);
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { spawn } from "node:child_process";
|
||||
import path from "node:path";
|
||||
import { createQaPosixCommandSettlement } from "./posix-command-settlement.js";
|
||||
import { resolveQaWindowsSystem32ExePath } from "./windows-system-tools.js";
|
||||
import { runQaWindowsTaskkill } from "./windows-system-tools.js";
|
||||
|
||||
export type QaScenarioCommandExecution = {
|
||||
args: string[];
|
||||
@@ -24,33 +24,11 @@ type QaScenarioCommandTerminalResult = Pick<
|
||||
"exitCode" | "failureMessage" | "signal"
|
||||
>;
|
||||
|
||||
type QaScenarioTaskkillRunner = typeof spawnSync;
|
||||
|
||||
const QA_SCENARIO_COMMAND_TIMEOUT_KILL_GRACE_MS = 2_000;
|
||||
const QA_SCENARIO_COMMAND_TIMEOUT_FORCE_SETTLE_MS = 500;
|
||||
let timeoutKillGraceMs = QA_SCENARIO_COMMAND_TIMEOUT_KILL_GRACE_MS;
|
||||
let timeoutForceSettleMs = QA_SCENARIO_COMMAND_TIMEOUT_FORCE_SETTLE_MS;
|
||||
|
||||
export function killQaScenarioWindowsProcessTree(
|
||||
pid: number | undefined,
|
||||
signal: NodeJS.Signals,
|
||||
runTaskkill: QaScenarioTaskkillRunner = spawnSync,
|
||||
) {
|
||||
if (pid === undefined) {
|
||||
return false;
|
||||
}
|
||||
const taskkillPath = resolveQaWindowsSystem32ExePath("taskkill.exe");
|
||||
const args = ["/pid", String(pid), "/T"];
|
||||
const run = (force: boolean) => {
|
||||
const result = runTaskkill(taskkillPath, force ? [...args, "/F"] : args, {
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
});
|
||||
return !result.error && result.status === 0;
|
||||
};
|
||||
return signal === "SIGKILL" ? run(true) : run(false) || run(true);
|
||||
}
|
||||
|
||||
export function runQaScenarioCommandLifecycle(
|
||||
execution: QaScenarioCommandExecution,
|
||||
): Promise<QaScenarioCommandResult> {
|
||||
@@ -72,10 +50,12 @@ export function runQaScenarioCommandLifecycle(
|
||||
...(isWindows
|
||||
? {
|
||||
windowsCleanup: {
|
||||
alive: () => child.pid !== undefined,
|
||||
signal: (signal: NodeJS.Signals) => {
|
||||
try {
|
||||
if (!killQaScenarioWindowsProcessTree(child.pid, signal)) {
|
||||
if (
|
||||
child.pid === undefined ||
|
||||
!runQaWindowsTaskkill({ pid: child.pid, signal })
|
||||
) {
|
||||
child.kill(signal);
|
||||
}
|
||||
return undefined;
|
||||
|
||||
@@ -0,0 +1,306 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
dockerE2eLaneName,
|
||||
prepareDockerE2eEnvironment,
|
||||
} from "./test-file-scenario-docker-batch.js";
|
||||
import {
|
||||
runQaTestFileScenarios,
|
||||
type QaScenarioCommandExecution,
|
||||
} from "./test-file-scenario-runner.js";
|
||||
import {
|
||||
QA_TEST_RUNNER_DEFAULTS,
|
||||
createScenarioRunnerTestHarness,
|
||||
makeDockerE2eScenario,
|
||||
writeDockerCandidateManifest,
|
||||
} from "./test-file-scenario-runner.test-support.js";
|
||||
|
||||
const harness = createScenarioRunnerTestHarness();
|
||||
const makeTempRepo = (prefix: string) => harness.makeTempRepo(prefix);
|
||||
|
||||
afterEach(async () => {
|
||||
vi.unstubAllEnvs();
|
||||
await harness.cleanup();
|
||||
});
|
||||
|
||||
it("only batches the canonical Docker lane argument shape", () => {
|
||||
const scenario = makeDockerE2eScenario("docker-lane", "gateway-network");
|
||||
if (scenario.execution.kind !== "script") {
|
||||
throw new Error("expected script scenario");
|
||||
}
|
||||
expect(dockerE2eLaneName(scenario)).toBe("gateway-network");
|
||||
expect(
|
||||
dockerE2eLaneName({
|
||||
...scenario,
|
||||
execution: { ...scenario.execution, args: ["--lane", "gateway-network", "--extra"] },
|
||||
}),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("prepares the exact Docker lane union in a sanitized bound environment", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-docker-candidate-");
|
||||
const outputDir = path.join(repoRoot, "out");
|
||||
const packagePath = path.join(repoRoot, "openclaw.tgz");
|
||||
const registryDir = path.join(repoRoot, "registry");
|
||||
const runCommand = vi.fn(async (command: QaScenarioCommandExecution) => {
|
||||
expect(command.env).toMatchObject({
|
||||
KEEP_ME: "yes",
|
||||
OPENCLAW_DOCKER_ALL_LANES: "gateway-network,openai-chat-tools",
|
||||
OPENCLAW_DOCKER_E2E_REPO_ROOT: repoRoot,
|
||||
});
|
||||
expect(command.env).not.toHaveProperty("OPENCLAW_DOCKER_ALL_BUILD");
|
||||
expect(command.env).not.toHaveProperty("OPENCLAW_CURRENT_PACKAGE_TGZ");
|
||||
expect(command.env).not.toHaveProperty("OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR");
|
||||
return await writeDockerCandidateManifest(command, {
|
||||
schema: "openclaw.qa-docker-candidate/v1",
|
||||
schemaVersion: 1,
|
||||
sourceSha: "a".repeat(40),
|
||||
candidate: {
|
||||
package: {
|
||||
path: packagePath,
|
||||
name: "openclaw",
|
||||
version: "2026.8.1",
|
||||
sha256: "b".repeat(64),
|
||||
},
|
||||
registry: {
|
||||
dir: registryDir,
|
||||
candidateVersion: "2026.8.1",
|
||||
manifestSha256: "c".repeat(64),
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
const env = await prepareDockerE2eEnvironment({
|
||||
env: {
|
||||
KEEP_ME: "yes",
|
||||
OPENCLAW_DOCKER_ALL_BUILD: "1",
|
||||
OPENCLAW_CURRENT_PACKAGE_TGZ: "/stale.tgz",
|
||||
OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR: "/stale-registry",
|
||||
},
|
||||
outputDir,
|
||||
repoRoot,
|
||||
runCommand,
|
||||
scenarios: [
|
||||
makeDockerE2eScenario("one", "gateway-network"),
|
||||
makeDockerE2eScenario("duplicate", "gateway-network"),
|
||||
makeDockerE2eScenario("two", "openai-chat-tools"),
|
||||
],
|
||||
});
|
||||
|
||||
expect(runCommand).toHaveBeenCalledTimes(1);
|
||||
expect(Object.isFrozen(env)).toBe(true);
|
||||
expect(env).toEqual({
|
||||
KEEP_ME: "yes",
|
||||
OPENCLAW_DOCKER_E2E_REPO_ROOT: repoRoot,
|
||||
OPENCLAW_DOCKER_E2E_SELECTED_SHA: "a".repeat(40),
|
||||
OPENCLAW_CURRENT_PACKAGE_TGZ: packagePath,
|
||||
OPENCLAW_CURRENT_PACKAGE_VERSION: "2026.8.1",
|
||||
OPENCLAW_CURRENT_PACKAGE_SHA256: "b".repeat(64),
|
||||
OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR: registryDir,
|
||||
OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_CANDIDATE_VERSION: "2026.8.1",
|
||||
OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_MANIFEST_SHA256: "c".repeat(64),
|
||||
});
|
||||
});
|
||||
|
||||
it("returns a sanitized bound env for a package-free candidate", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-docker-candidate-null-");
|
||||
const env = await prepareDockerE2eEnvironment({
|
||||
env: {
|
||||
KEEP_ME: "yes",
|
||||
OPENCLAW_DOCKER_ALL_BUILD: "1",
|
||||
OPENCLAW_CURRENT_PACKAGE_TGZ: "/stale.tgz",
|
||||
},
|
||||
outputDir: path.join(repoRoot, "out"),
|
||||
repoRoot,
|
||||
runCommand: (command) =>
|
||||
writeDockerCandidateManifest(command, {
|
||||
schema: "openclaw.qa-docker-candidate/v1",
|
||||
schemaVersion: 1,
|
||||
sourceSha: "a".repeat(40),
|
||||
candidate: null,
|
||||
}),
|
||||
scenarios: [makeDockerE2eScenario("one", "gateway-network")],
|
||||
});
|
||||
|
||||
expect(env).toEqual({ KEEP_ME: "yes", OPENCLAW_DOCKER_E2E_REPO_ROOT: repoRoot });
|
||||
expect(Object.isFrozen(env)).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ label: "extra field", patch: { extra: true } },
|
||||
{ label: "malformed candidate", patch: { candidate: { package: null, registry: null } } },
|
||||
])("rejects a $label in the Docker candidate manifest", async ({ patch }) => {
|
||||
const repoRoot = await makeTempRepo("qa-docker-candidate-invalid-");
|
||||
await expect(
|
||||
prepareDockerE2eEnvironment({
|
||||
env: process.env,
|
||||
outputDir: path.join(repoRoot, "out"),
|
||||
repoRoot,
|
||||
runCommand: (command) =>
|
||||
writeDockerCandidateManifest(command, {
|
||||
schema: "openclaw.qa-docker-candidate/v1",
|
||||
schemaVersion: 1,
|
||||
sourceSha: "a".repeat(40),
|
||||
candidate: null,
|
||||
...patch,
|
||||
}),
|
||||
scenarios: [makeDockerE2eScenario("one", "gateway-network")],
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
describe("qa test file scenario runner", () => {
|
||||
it.each([
|
||||
{ label: "package", candidate: "package" as const },
|
||||
{ label: "package-free", candidate: "none" as const },
|
||||
])("keeps hostile inherited Docker state out of a prepared $label run", async ({ candidate }) => {
|
||||
const repoRoot = await makeTempRepo("qa-docker-replace-env-");
|
||||
const packagePath = path.join(repoRoot, "openclaw.tgz");
|
||||
vi.stubEnv("OPENCLAW_DOCKER_ALL_POISON", "hostile");
|
||||
vi.stubEnv("OPENCLAW_CURRENT_PACKAGE_TGZ", "/hostile.tgz");
|
||||
vi.stubEnv("OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR", "/hostile-registry");
|
||||
const prepared = await prepareDockerE2eEnvironment({
|
||||
env: process.env,
|
||||
outputDir: path.join(repoRoot, "prep"),
|
||||
repoRoot,
|
||||
runCommand: (command) =>
|
||||
writeDockerCandidateManifest(command, {
|
||||
schema: "openclaw.qa-docker-candidate/v1",
|
||||
schemaVersion: 1,
|
||||
sourceSha: "a".repeat(40),
|
||||
candidate:
|
||||
candidate === "package"
|
||||
? {
|
||||
package: {
|
||||
path: packagePath,
|
||||
name: "openclaw",
|
||||
version: "2026.8.1",
|
||||
sha256: "b".repeat(64),
|
||||
},
|
||||
registry: null,
|
||||
}
|
||||
: null,
|
||||
}),
|
||||
scenarios: [makeDockerE2eScenario("one", "gateway-network")],
|
||||
});
|
||||
|
||||
await runQaTestFileScenarios({
|
||||
env: prepared,
|
||||
envMode: "replace",
|
||||
outputDir: path.join(repoRoot, "run"),
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
repoRoot,
|
||||
scenarios: [makeDockerE2eScenario("one", "gateway-network")],
|
||||
runCommand: async (command) => {
|
||||
expect(command.env.OPENCLAW_DOCKER_ALL_POISON).toBeUndefined();
|
||||
expect(command.env.OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR).toBeUndefined();
|
||||
expect(command.env.OPENCLAW_CURRENT_PACKAGE_TGZ).toBe(
|
||||
candidate === "package" ? packagePath : undefined,
|
||||
);
|
||||
expect(command.env.OPENCLAW_DOCKER_E2E_REPO_ROOT).toBe(repoRoot);
|
||||
const logDir = command.env.OPENCLAW_DOCKER_ALL_LOG_DIR!;
|
||||
await fs.mkdir(logDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(logDir, "summary.json"),
|
||||
JSON.stringify({
|
||||
failures: [],
|
||||
lanes: [{ elapsedSeconds: 1, name: "gateway-network", status: 0 }],
|
||||
selectedLanes: ["gateway-network"],
|
||||
}),
|
||||
);
|
||||
return { exitCode: 0, stdout: "", stderr: "" };
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves individual Docker lane success without generic producer evidence", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-script-docker-individual-no-producer-evidence-");
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "docker-individual"),
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
failFast: true,
|
||||
scenarios: [makeDockerE2eScenario("docker-gateway-network", "gateway-network")],
|
||||
runCommand: async () => ({ exitCode: 0, stdout: "Docker lane passed\n", stderr: "" }),
|
||||
});
|
||||
|
||||
expect(result.results[0]).toMatchObject({
|
||||
scenario: { id: "docker-gateway-network" },
|
||||
status: "pass",
|
||||
});
|
||||
expect(result.evidence.entries[0]?.result.status).toBe("pass");
|
||||
});
|
||||
|
||||
it("runs Docker script scenarios through one aggregate scheduler invocation", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-script-docker-batch-");
|
||||
const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "docker-batch");
|
||||
const staleSummaryPath = path.join(outputDir, "docker-e2e-1800000ms", "summary.json");
|
||||
await fs.mkdir(path.dirname(staleSummaryPath), { recursive: true });
|
||||
await fs.writeFile(staleSummaryPath, '{"status":"passed"}\n', "utf8");
|
||||
const commands: QaScenarioCommandExecution[] = [];
|
||||
const scenarios = [
|
||||
makeDockerE2eScenario("openai-tools", "openai-chat-tools"),
|
||||
makeDockerE2eScenario("bundled-plugins", "bundled-plugin-install-uninstall"),
|
||||
makeDockerE2eScenario("prefix-lane", "gateway"),
|
||||
makeDockerE2eScenario("failing-lane", "gateway-network"),
|
||||
];
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios,
|
||||
runCommand: async (command) => {
|
||||
commands.push(command);
|
||||
await expect(fs.access(staleSummaryPath)).rejects.toThrow();
|
||||
const logDir = command.env.OPENCLAW_DOCKER_ALL_LOG_DIR;
|
||||
if (!logDir) {
|
||||
throw new Error("missing Docker scheduler log dir");
|
||||
}
|
||||
await fs.mkdir(logDir, { recursive: true });
|
||||
const failedLane = { elapsedSeconds: 2, name: "gateway-network", status: 1 };
|
||||
await fs.writeFile(
|
||||
path.join(logDir, "summary.json"),
|
||||
`${JSON.stringify({
|
||||
failures: [failedLane],
|
||||
lanes: [
|
||||
{ elapsedSeconds: 4, name: "openai-chat-tools", status: 0 },
|
||||
{ elapsedSeconds: 7, name: "bundled-plugin-install-uninstall-0", status: 0 },
|
||||
{ elapsedSeconds: 6, name: "bundled-plugin-install-uninstall-1", status: 0 },
|
||||
{ elapsedSeconds: 1, name: "gateway", status: 0 },
|
||||
failedLane,
|
||||
],
|
||||
selectedLanes: [
|
||||
"openai-chat-tools",
|
||||
"bundled-plugin-install-uninstall-0",
|
||||
"bundled-plugin-install-uninstall-1",
|
||||
"gateway",
|
||||
"gateway-network",
|
||||
],
|
||||
})}\n`,
|
||||
"utf8",
|
||||
);
|
||||
return { exitCode: 1, stdout: "", stderr: "scheduler failed\n" };
|
||||
},
|
||||
});
|
||||
|
||||
expect(commands).toHaveLength(1);
|
||||
expect(commands[0]).toMatchObject({
|
||||
args: ["scripts/test-docker-all.mjs"],
|
||||
command: process.execPath,
|
||||
env: {
|
||||
OPENCLAW_DOCKER_ALL_FAIL_FAST: "0",
|
||||
OPENCLAW_DOCKER_ALL_LANES:
|
||||
"openai-chat-tools,bundled-plugin-install-uninstall,gateway,gateway-network",
|
||||
OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS: "1800000",
|
||||
},
|
||||
});
|
||||
expect(result.results).toMatchObject([
|
||||
{ scenario: { id: "openai-tools" }, status: "pass" },
|
||||
{ scenario: { id: "bundled-plugins" }, status: "pass" },
|
||||
{ scenario: { id: "prefix-lane" }, status: "pass" },
|
||||
{ scenario: { id: "failing-lane" }, status: "fail" },
|
||||
]);
|
||||
expect(result.results[3]?.failureMessage).toBe("gateway-network exited with 1");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,487 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { validateQaEvidenceSummaryJson } from "./evidence-summary.js";
|
||||
import {
|
||||
runQaTestFileScenarios,
|
||||
type QaScenarioCommandExecution,
|
||||
} from "./test-file-scenario-runner.js";
|
||||
import {
|
||||
QA_TEST_RUNNER_DEFAULTS,
|
||||
createScenarioRunnerTestHarness,
|
||||
makeTestFileScenario,
|
||||
writeNativeVitestReport,
|
||||
} from "./test-file-scenario-runner.test-support.js";
|
||||
|
||||
const harness = createScenarioRunnerTestHarness();
|
||||
const makeTempRepo = (prefix: string) => harness.makeTempRepo(prefix);
|
||||
|
||||
afterEach(async () => {
|
||||
await harness.cleanup();
|
||||
});
|
||||
|
||||
describe("qa test file scenario runner", () => {
|
||||
it("runs Playwright scenarios with the repo UI e2e command and writes Playwright evidence", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-playwright-scenario-");
|
||||
const commands: QaScenarioCommandExecution[] = [];
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-playwright"),
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [
|
||||
makeTestFileScenario(
|
||||
"playwright",
|
||||
"ui/src/e2e/chat-flow.e2e.test.ts",
|
||||
"sends a chat turn through the GUI",
|
||||
),
|
||||
],
|
||||
runCommand: async (command) => {
|
||||
commands.push(command);
|
||||
await writeNativeVitestReport(command, { passed: 1 });
|
||||
return {
|
||||
exitCode: 0,
|
||||
stdout: "pass\n",
|
||||
stderr: "",
|
||||
};
|
||||
},
|
||||
env: {
|
||||
OPENCLAW_QA_REF: "scenario-ref",
|
||||
} as NodeJS.ProcessEnv,
|
||||
});
|
||||
|
||||
expect(result.executionKind).toBe("playwright");
|
||||
expect(commands.map((command) => command.args)).toEqual([
|
||||
["--import", "tsx", "scripts/ensure-playwright-chromium.mts"],
|
||||
[
|
||||
"scripts/run-vitest.mjs",
|
||||
"run",
|
||||
"--config",
|
||||
"test/vitest/vitest.ui-e2e.config.ts",
|
||||
"--configLoader",
|
||||
"runner",
|
||||
"ui/src/e2e/chat-flow.e2e.test.ts",
|
||||
"--reporter=verbose",
|
||||
"--reporter=json",
|
||||
`--outputFile.json=${path.join(
|
||||
repoRoot,
|
||||
".artifacts",
|
||||
"qa-e2e",
|
||||
"scenario-playwright",
|
||||
"scenario-playwright.vitest-report.json",
|
||||
)}`,
|
||||
"--testNamePattern",
|
||||
"sends a chat turn through the GUI",
|
||||
],
|
||||
]);
|
||||
expect(commands.map((command) => command.timeoutMs)).toEqual([1_800_000, 1_800_000]);
|
||||
const evidence = validateQaEvidenceSummaryJson(
|
||||
JSON.parse(await fs.readFile(result.evidencePath, "utf8")),
|
||||
);
|
||||
expect(evidence.schemaVersion).toBe(2);
|
||||
expect(evidence.entries).toHaveLength(1);
|
||||
expect(evidence.entries[0]).toMatchObject({
|
||||
test: {
|
||||
kind: "playwright-test",
|
||||
id: "scenario-playwright",
|
||||
source: {
|
||||
path: "ui/src/e2e/chat-flow.e2e.test.ts",
|
||||
},
|
||||
},
|
||||
coverage: [
|
||||
{
|
||||
id: "ui.control",
|
||||
role: "primary",
|
||||
},
|
||||
{
|
||||
id: "ui.streaming",
|
||||
role: "secondary",
|
||||
},
|
||||
],
|
||||
refs: [
|
||||
{
|
||||
kind: "docs",
|
||||
path: "docs/concepts/qa-e2e-automation.md",
|
||||
},
|
||||
{
|
||||
kind: "code",
|
||||
path: "ui/src/e2e/chat-flow.e2e.test.ts",
|
||||
},
|
||||
],
|
||||
execution: {
|
||||
runner: "playwright",
|
||||
artifacts: [
|
||||
{
|
||||
kind: "log",
|
||||
path: ".artifacts/qa-e2e/scenario-playwright/scenario-playwright.log",
|
||||
source: "playwright",
|
||||
},
|
||||
],
|
||||
},
|
||||
result: {
|
||||
status: "pass",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("can return aggregate evidence without writing a duplicate evidence file", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-playwright-memory-evidence-");
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-playwright"),
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [makeTestFileScenario("playwright", "ui/src/e2e/chat-flow.e2e.test.ts")],
|
||||
writeEvidenceFile: false,
|
||||
runCommand: async (command) => {
|
||||
await writeNativeVitestReport(command, { passed: 1 });
|
||||
return {
|
||||
exitCode: 0,
|
||||
stdout: "pass\n",
|
||||
stderr: "",
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.evidence.entries).toHaveLength(1);
|
||||
await expect(fs.access(result.evidencePath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
|
||||
it("runs Vitest scenarios with the declared test path and writes Vitest evidence", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-vitest-scenario-");
|
||||
const commands: QaScenarioCommandExecution[] = [];
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-vitest"),
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [makeTestFileScenario("vitest", "extensions/qa-lab/src/coverage-report.test.ts")],
|
||||
runCommand: async (command) => {
|
||||
commands.push(command);
|
||||
return {
|
||||
exitCode: 1,
|
||||
stdout: "",
|
||||
stderr: "failed\n",
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.executionKind).toBe("vitest");
|
||||
expect(commands.map((command) => command.args)).toEqual([
|
||||
[
|
||||
"scripts/run-vitest.mjs",
|
||||
"extensions/qa-lab/src/coverage-report.test.ts",
|
||||
"--reporter=verbose",
|
||||
"--reporter=json",
|
||||
`--outputFile.json=${path.join(
|
||||
repoRoot,
|
||||
".artifacts",
|
||||
"qa-e2e",
|
||||
"scenario-vitest",
|
||||
"scenario-vitest.vitest-report.json",
|
||||
)}`,
|
||||
],
|
||||
]);
|
||||
expect(commands.map((command) => command.timeoutMs)).toEqual([1_800_000]);
|
||||
const evidence = validateQaEvidenceSummaryJson(
|
||||
JSON.parse(await fs.readFile(result.evidencePath, "utf8")),
|
||||
);
|
||||
expect(evidence.entries[0]).toMatchObject({
|
||||
test: {
|
||||
kind: "vitest-test",
|
||||
id: "scenario-vitest",
|
||||
source: {
|
||||
path: "extensions/qa-lab/src/coverage-report.test.ts",
|
||||
},
|
||||
},
|
||||
coverage: [
|
||||
{
|
||||
id: "qa.coverage",
|
||||
role: "primary",
|
||||
},
|
||||
{
|
||||
id: "qa.reporting",
|
||||
role: "secondary",
|
||||
},
|
||||
],
|
||||
execution: {
|
||||
runner: "vitest",
|
||||
artifacts: [
|
||||
{
|
||||
kind: "log",
|
||||
path: ".artifacts/qa-e2e/scenario-vitest/scenario-vitest.log",
|
||||
source: "vitest",
|
||||
},
|
||||
],
|
||||
},
|
||||
result: {
|
||||
status: "fail",
|
||||
failure: {
|
||||
reason: "node exited with 1",
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ executionKind: "vitest" as const, passed: 0, expectedStatus: "fail" as const },
|
||||
{ executionKind: "playwright" as const, passed: 0, expectedStatus: "fail" as const },
|
||||
{ executionKind: "vitest" as const, passed: 1, expectedStatus: "pass" as const },
|
||||
{ executionKind: "playwright" as const, passed: 1, expectedStatus: "pass" as const },
|
||||
])(
|
||||
"requires an actually passed $executionKind test when the native child exits successfully ($passed passed)",
|
||||
async ({ executionKind, expectedStatus, passed }) => {
|
||||
const repoRoot = await makeTempRepo(`qa-${executionKind}-executed-tests-`);
|
||||
const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`);
|
||||
const scenarioPath =
|
||||
executionKind === "playwright"
|
||||
? "ui/src/e2e/chat-flow.e2e.test.ts"
|
||||
: "extensions/qa-lab/src/coverage-report.test.ts";
|
||||
const commands: QaScenarioCommandExecution[] = [];
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [makeTestFileScenario(executionKind, scenarioPath)],
|
||||
runCommand: async (command) => {
|
||||
commands.push(command);
|
||||
await writeNativeVitestReport(command, { passed });
|
||||
return { exitCode: 0, stdout: "child exited successfully\n", stderr: "" };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.results[0]).toMatchObject({ status: expectedStatus });
|
||||
expect(result.evidence.entries[0]?.result.status).toBe(expectedStatus);
|
||||
expect(
|
||||
commands.filter((command) => command.args[0] === "scripts/run-vitest.mjs"),
|
||||
).toHaveLength(1);
|
||||
if (expectedStatus === "fail") {
|
||||
expect(result.results[0]?.failureMessage).toBe(
|
||||
"Vitest exited successfully without reporting a successfully executed test.",
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each([{ executionKind: "vitest" as const }, { executionKind: "playwright" as const }])(
|
||||
"rejects a passing $executionKind report for an unrelated test file",
|
||||
async ({ executionKind }) => {
|
||||
const repoRoot = await makeTempRepo(`qa-${executionKind}-wrong-report-file-`);
|
||||
const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`);
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [
|
||||
makeTestFileScenario(
|
||||
executionKind,
|
||||
executionKind === "playwright"
|
||||
? "ui/src/e2e/chat-flow.e2e.test.ts"
|
||||
: "extensions/qa-lab/src/coverage-report.test.ts",
|
||||
),
|
||||
],
|
||||
runCommand: async (command) => {
|
||||
await writeNativeVitestReport(command, {
|
||||
passed: 1,
|
||||
testFilePath: "extensions/qa-lab/src/unrelated.test.ts",
|
||||
});
|
||||
return { exitCode: 0, stdout: "unrelated test passed\n", stderr: "" };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.results[0]).toMatchObject({
|
||||
failureMessage: expect.stringContaining("requested test file"),
|
||||
status: "fail",
|
||||
});
|
||||
expect(result.evidence.entries[0]?.result.status).toBe("fail");
|
||||
},
|
||||
);
|
||||
|
||||
it.each([{ executionKind: "vitest" as const }, { executionKind: "playwright" as const }])(
|
||||
"rejects a passing $executionKind report when the requested test file does not exist",
|
||||
async ({ executionKind }) => {
|
||||
const repoRoot = await makeTempRepo(`qa-${executionKind}-missing-requested-test-`);
|
||||
const scenarioPath =
|
||||
executionKind === "playwright"
|
||||
? "ui/src/e2e/chat-flow.e2e.test.ts"
|
||||
: "extensions/qa-lab/src/coverage-report.test.ts";
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`),
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [makeTestFileScenario(executionKind, scenarioPath)],
|
||||
runCommand: async (command) => {
|
||||
await writeNativeVitestReport(command, {
|
||||
createRequestedTestFile: false,
|
||||
passed: 1,
|
||||
});
|
||||
return { exitCode: 0, stdout: "missing test reportedly passed\n", stderr: "" };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.results[0]).toMatchObject({
|
||||
failureMessage: expect.stringContaining("existing requested test file"),
|
||||
status: "fail",
|
||||
});
|
||||
expect(result.evidence.entries[0]?.result.status).toBe("fail");
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"authenticates requested tests when the checkout root is a symlink",
|
||||
async () => {
|
||||
const canonicalRoot = await fs.realpath(await makeTempRepo("qa-vitest-symlinked-checkout-"));
|
||||
const symlinkedRoot = path.join(canonicalRoot, "checkout-alias");
|
||||
await fs.symlink(canonicalRoot, symlinkedRoot, "dir");
|
||||
const scenarioPath = "extensions/qa-lab/src/coverage-report.test.ts";
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot: symlinkedRoot,
|
||||
outputDir: path.join(symlinkedRoot, ".artifacts", "qa-e2e", "scenario-vitest"),
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [makeTestFileScenario("vitest", scenarioPath)],
|
||||
runCommand: async (command) => {
|
||||
await writeNativeVitestReport(command, {
|
||||
passed: 1,
|
||||
testFilePath: path.join(canonicalRoot, scenarioPath),
|
||||
});
|
||||
return { exitCode: 0, stdout: "canonical test passed\n", stderr: "" };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.results[0]).toMatchObject({ status: "pass" });
|
||||
expect(result.evidence.entries[0]?.result.status).toBe("pass");
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects a passing Playwright report that misses the requested test name", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-playwright-wrong-report-test-");
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-playwright"),
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [
|
||||
makeTestFileScenario(
|
||||
"playwright",
|
||||
"ui/src/e2e/chat-flow.e2e.test.ts",
|
||||
"required visual assertion",
|
||||
),
|
||||
],
|
||||
runCommand: async (command) => {
|
||||
await writeNativeVitestReport(command, {
|
||||
passed: 1,
|
||||
testName: "unrelated visual assertion",
|
||||
});
|
||||
return { exitCode: 0, stdout: "unrelated assertion passed\n", stderr: "" };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.results[0]).toMatchObject({
|
||||
failureMessage: expect.stringContaining("requested test name"),
|
||||
status: "fail",
|
||||
});
|
||||
expect(result.evidence.entries[0]?.result.status).toBe("fail");
|
||||
});
|
||||
|
||||
it("records invalid Playwright test-name patterns as failed scenario evidence", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-playwright-invalid-report-pattern-");
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-playwright"),
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [makeTestFileScenario("playwright", "ui/src/e2e/chat-flow.e2e.test.ts", "[")],
|
||||
runCommand: async (command) => {
|
||||
await writeNativeVitestReport(command, {
|
||||
passed: 1,
|
||||
testName: "executed visual assertion",
|
||||
});
|
||||
return { exitCode: 0, stdout: "visual assertion passed\n", stderr: "" };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.results[0]).toMatchObject({
|
||||
failureMessage: expect.stringContaining("invalid requested test name pattern"),
|
||||
status: "fail",
|
||||
});
|
||||
expect(result.evidence.entries[0]?.result.status).toBe("fail");
|
||||
});
|
||||
|
||||
it.each([{ executionKind: "vitest" as const }, { executionKind: "playwright" as const }])(
|
||||
"does not reuse a prior passing $executionKind report when the next child writes none",
|
||||
async ({ executionKind }) => {
|
||||
const repoRoot = await makeTempRepo(`qa-${executionKind}-stale-vitest-report-`);
|
||||
const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`);
|
||||
const scenarioPath =
|
||||
executionKind === "playwright"
|
||||
? "ui/src/e2e/chat-flow.e2e.test.ts"
|
||||
: "extensions/qa-lab/src/coverage-report.test.ts";
|
||||
const reportPath = path.join(outputDir, `scenario-${executionKind}.vitest-report.json`);
|
||||
let writeReport = true;
|
||||
const runParams = {
|
||||
repoRoot,
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [makeTestFileScenario(executionKind, scenarioPath)],
|
||||
runCommand: async (command: QaScenarioCommandExecution) => {
|
||||
if (writeReport) {
|
||||
await writeNativeVitestReport(command, { passed: 1 });
|
||||
}
|
||||
return { exitCode: 0, stdout: "child exited successfully\n", stderr: "" };
|
||||
},
|
||||
};
|
||||
|
||||
const firstRun = await runQaTestFileScenarios(runParams);
|
||||
expect(firstRun.results[0]).toMatchObject({ status: "pass" });
|
||||
await expect(fs.access(reportPath)).resolves.toBeUndefined();
|
||||
|
||||
writeReport = false;
|
||||
const secondRun = await runQaTestFileScenarios(runParams);
|
||||
expect(secondRun.results[0]).toMatchObject({
|
||||
failureMessage: `Vitest exited successfully without writing a valid JSON test report at ${reportPath}.`,
|
||||
status: "fail",
|
||||
});
|
||||
expect(secondRun.evidence.entries[0]?.result.status).toBe("fail");
|
||||
await expect(fs.access(reportPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ failFast: true, expectedScenarioIds: ["first-native-scenario"] },
|
||||
{
|
||||
failFast: false,
|
||||
expectedScenarioIds: ["first-native-scenario", "later-native-scenario"],
|
||||
},
|
||||
{
|
||||
failFast: undefined,
|
||||
expectedScenarioIds: ["first-native-scenario", "later-native-scenario"],
|
||||
},
|
||||
])(
|
||||
"honors native scenario fail-fast mode ($failFast)",
|
||||
async ({ failFast, expectedScenarioIds }) => {
|
||||
const repoRoot = await makeTempRepo("qa-vitest-fail-fast-");
|
||||
const runCommand = vi.fn(async () => ({
|
||||
exitCode: 1,
|
||||
stdout: "",
|
||||
stderr: "native scenario failed\n",
|
||||
}));
|
||||
const firstScenario = {
|
||||
...makeTestFileScenario("vitest", "extensions/qa-lab/src/coverage-report.test.ts"),
|
||||
id: "first-native-scenario",
|
||||
};
|
||||
const laterScenario = {
|
||||
...makeTestFileScenario("vitest", "extensions/qa-lab/src/cli.test.ts"),
|
||||
id: "later-native-scenario",
|
||||
};
|
||||
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "native-fail-fast"),
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
failFast,
|
||||
scenarios: [firstScenario, laterScenario],
|
||||
runCommand,
|
||||
});
|
||||
|
||||
expect(runCommand).toHaveBeenCalledTimes(expectedScenarioIds.length);
|
||||
expect(result.results.map((scenario) => scenario.scenario.id)).toEqual(expectedScenarioIds);
|
||||
expect(result.results.every((scenario) => scenario.status === "fail")).toBe(true);
|
||||
expect(result.evidence.entries.map((entry) => entry.test.id)).toEqual(expectedScenarioIds);
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,120 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { runQaScenarioCommandLifecycle } from "./test-file-scenario-command-lifecycle.js";
|
||||
import {
|
||||
runQaTestFileScenarios,
|
||||
type QaScenarioCommandExecution,
|
||||
} from "./test-file-scenario-runner.js";
|
||||
import {
|
||||
QA_TEST_RUNNER_DEFAULTS,
|
||||
createScenarioRunnerTestHarness,
|
||||
makeTestFileScenario,
|
||||
writeNativeVitestReport,
|
||||
} from "./test-file-scenario-runner.test-support.js";
|
||||
|
||||
const harness = createScenarioRunnerTestHarness();
|
||||
const makeTempDir = (prefix: string) => harness.makeTempDir(prefix);
|
||||
const makeTempRepo = (prefix: string) => harness.makeTempRepo(prefix);
|
||||
|
||||
afterEach(async () => {
|
||||
await harness.cleanup();
|
||||
});
|
||||
|
||||
describe("qa test file scenario runner", () => {
|
||||
it.each([
|
||||
{ executionKind: "vitest" as const, commandCount: 1 },
|
||||
{ executionKind: "playwright" as const, commandCount: 2 },
|
||||
])(
|
||||
"applies the resolved command timeout to every $executionKind subprocess",
|
||||
async ({ commandCount, executionKind }) => {
|
||||
const repoRoot = await makeTempRepo(`qa-${executionKind}-command-timeout-`);
|
||||
const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`);
|
||||
const commands: QaScenarioCommandExecution[] = [];
|
||||
|
||||
await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [
|
||||
makeTestFileScenario(
|
||||
executionKind,
|
||||
executionKind === "playwright"
|
||||
? "ui/src/e2e/chat-flow.e2e.test.ts"
|
||||
: "extensions/qa-lab/src/coverage-report.test.ts",
|
||||
),
|
||||
],
|
||||
commandTimeoutMs: 321,
|
||||
runCommand: async (command) => {
|
||||
commands.push(command);
|
||||
await writeNativeVitestReport(command, { passed: 1 });
|
||||
return { exitCode: 0, stdout: "native pass\n", stderr: "" };
|
||||
},
|
||||
});
|
||||
|
||||
expect(commands).toHaveLength(commandCount);
|
||||
expect(commands.map((command) => command.timeoutMs)).toEqual(
|
||||
Array.from({ length: commandCount }, () => 321),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["vitest", "playwright"] as const)(
|
||||
"terminates a hanging $executionKind subprocess with failure evidence",
|
||||
async (executionKind) => {
|
||||
const repoRoot = await makeTempRepo(`qa-${executionKind}-hung-command-`);
|
||||
const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", `scenario-${executionKind}`);
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [
|
||||
makeTestFileScenario(
|
||||
executionKind,
|
||||
executionKind === "playwright"
|
||||
? "ui/src/e2e/chat-flow.e2e.test.ts"
|
||||
: "extensions/qa-lab/src/coverage-report.test.ts",
|
||||
),
|
||||
],
|
||||
commandTimeoutMs: 100,
|
||||
runCommand: (execution) =>
|
||||
runQaScenarioCommandLifecycle({
|
||||
...execution,
|
||||
args: ["-e", "setInterval(() => {}, 1_000)"],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(result.results[0]).toMatchObject({
|
||||
failureMessage: expect.stringContaining("timed out after 100ms"),
|
||||
status: "fail",
|
||||
});
|
||||
expect(result.evidence.entries[0]?.result.status).toBe("fail");
|
||||
},
|
||||
);
|
||||
|
||||
it("fails script scenarios that exit cleanly after timeout termination", async () => {
|
||||
const repoRoot = process.cwd();
|
||||
const tempRoot = await makeTempDir("qa-script-timeout-clean-exit-");
|
||||
const scriptPath = path.join(tempRoot, "clean-exit-after-timeout.ts");
|
||||
await fs.writeFile(
|
||||
scriptPath,
|
||||
[
|
||||
"process.stdout.write('waiting for timeout\\n');",
|
||||
"process.on('SIGTERM', () => process.exit(0));",
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir: path.join(tempRoot, "out"),
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [makeTestFileScenario("script", scriptPath)],
|
||||
commandTimeoutMs: 100,
|
||||
});
|
||||
|
||||
expect(result.results[0]?.status).toBe("fail");
|
||||
expect(result.results[0]?.failureMessage).toMatch(/timed out after 100ms/u);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,594 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { validateQaEvidenceSummaryJson } from "./evidence-summary.js";
|
||||
import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
|
||||
import {
|
||||
runQaTestFileScenarios,
|
||||
type QaScenarioCommandExecution,
|
||||
} from "./test-file-scenario-runner.js";
|
||||
import {
|
||||
buildScriptProducerEvidence,
|
||||
QA_TEST_RUNNER_DEFAULTS,
|
||||
createScenarioRunnerTestHarness,
|
||||
makeTestFileScenario,
|
||||
writeScriptProducerEvidence,
|
||||
} from "./test-file-scenario-runner.test-support.js";
|
||||
|
||||
const harness = createScenarioRunnerTestHarness();
|
||||
const makeTempRepo = (prefix: string) => harness.makeTempRepo(prefix);
|
||||
|
||||
afterEach(async () => {
|
||||
await harness.cleanup();
|
||||
});
|
||||
|
||||
describe("qa test file scenario runner", () => {
|
||||
it.each([
|
||||
{ evidence: "missing", expectedFailure: /without writing fresh producer QA evidence/u },
|
||||
{ evidence: "stale", expectedFailure: /without writing fresh producer QA evidence/u },
|
||||
{ evidence: "empty", expectedFailure: /without reporting an executed producer check/u },
|
||||
{ evidence: "malformed", expectedFailure: /invalid JSON/u },
|
||||
{ evidence: "outside", expectedFailure: /inside its scenario output directory/u },
|
||||
] as const)(
|
||||
"fails a successful script with $evidence producer evidence",
|
||||
async ({ evidence, expectedFailure }) => {
|
||||
const repoRoot = await makeTempRepo(`qa-script-${evidence}-producer-evidence-`);
|
||||
const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script");
|
||||
const scenarioOutputDir = path.join(outputDir, "scenario-script");
|
||||
const latestRunPath = path.join(scenarioOutputDir, "latest-run.json");
|
||||
const evidencePath = path.join(scenarioOutputDir, "qa-evidence.json");
|
||||
|
||||
if (evidence === "stale") {
|
||||
await writeScriptProducerEvidence({ outputDir, status: "pass" });
|
||||
const staleEvidencePath = path.join(scenarioOutputDir, "run-1", "qa-evidence.json");
|
||||
await fs.copyFile(staleEvidencePath, evidencePath);
|
||||
const staleTimestamp = new Date(Date.now() - 60_000);
|
||||
await Promise.all([
|
||||
fs.utimes(staleEvidencePath, staleTimestamp, staleTimestamp),
|
||||
fs.utimes(evidencePath, staleTimestamp, staleTimestamp),
|
||||
]);
|
||||
}
|
||||
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")],
|
||||
runCommand: async () => {
|
||||
await fs.mkdir(scenarioOutputDir, { recursive: true });
|
||||
if (evidence === "stale") {
|
||||
await expect(fs.access(latestRunPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(fs.access(evidencePath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await fs.writeFile(
|
||||
latestRunPath,
|
||||
JSON.stringify({
|
||||
qaEvidence: path.join(scenarioOutputDir, "run-1", "qa-evidence.json"),
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
} else if (evidence === "empty") {
|
||||
await fs.writeFile(
|
||||
evidencePath,
|
||||
JSON.stringify({
|
||||
kind: "openclaw.qa.evidence-summary",
|
||||
schemaVersion: 2,
|
||||
generatedAt: new Date().toISOString(),
|
||||
evidenceMode: "full",
|
||||
entries: [],
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
} else if (evidence === "malformed") {
|
||||
await fs.writeFile(evidencePath, "{not valid JSON", "utf8");
|
||||
} else if (evidence === "outside") {
|
||||
await writeScriptProducerEvidence({
|
||||
outputDir,
|
||||
scenarioId: "different-script-scenario",
|
||||
status: "pass",
|
||||
});
|
||||
await fs.writeFile(
|
||||
latestRunPath,
|
||||
JSON.stringify({
|
||||
qaEvidence: path.join(
|
||||
outputDir,
|
||||
"different-script-scenario",
|
||||
"run-1",
|
||||
"qa-evidence.json",
|
||||
),
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
return { exitCode: 0, stdout: "script exited successfully\n", stderr: "" };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.results[0]).toMatchObject({ status: "fail" });
|
||||
expect(result.results[0]?.failureMessage).toMatch(expectedFailure);
|
||||
expect(result.evidence.entries).toHaveLength(1);
|
||||
expect(result.evidence.entries[0]).toMatchObject({
|
||||
test: { id: "scenario-script" },
|
||||
result: { status: "fail" },
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("runs script scenarios and imports producer QA evidence artifacts", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-script-scenario-");
|
||||
const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script");
|
||||
const commands: QaScenarioCommandExecution[] = [];
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")],
|
||||
runCommand: async (command) => {
|
||||
commands.push(command);
|
||||
const runRoot = path.join(outputDir, "scenario-script", "run-1");
|
||||
await fs.mkdir(path.join(runRoot, "surfaces", "web-ui"), { recursive: true });
|
||||
await fs.writeFile(path.join(runRoot, "surfaces", "web-ui", "screenshot.png"), "png");
|
||||
await writeScriptProducerEvidence({
|
||||
artifacts: [
|
||||
{
|
||||
kind: "screenshot",
|
||||
path: "surfaces/web-ui/screenshot.png",
|
||||
source: "script-producer:web-ui:smoke",
|
||||
},
|
||||
],
|
||||
outputDir,
|
||||
status: "pass",
|
||||
});
|
||||
return { exitCode: 0, stdout: "script pass\n", stderr: "" };
|
||||
},
|
||||
env: { OPENCLAW_QA_REF: "scenario-ref" } as NodeJS.ProcessEnv,
|
||||
});
|
||||
|
||||
expect(result.executionKind).toBe("script");
|
||||
expect(commands.map((command) => command.args)).toEqual([
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
"scripts/evidence-producer.ts",
|
||||
"--once",
|
||||
"--artifact-base",
|
||||
path.join(outputDir, "scenario-script"),
|
||||
],
|
||||
]);
|
||||
expect(commands.map((command) => command.timeoutMs)).toEqual([30 * 60_000]);
|
||||
const evidence = validateQaEvidenceSummaryJson(
|
||||
JSON.parse(await fs.readFile(result.evidencePath, "utf8")),
|
||||
);
|
||||
expect(evidence.entries).toHaveLength(1);
|
||||
expect(evidence.entries[0]).toMatchObject({
|
||||
test: { kind: "script-producer-check", id: "script-producer.web-ui.smoke" },
|
||||
coverage: [
|
||||
{ id: "qa.coverage", role: "primary" },
|
||||
{ id: "qa.reporting", role: "secondary" },
|
||||
],
|
||||
execution: {
|
||||
runner: "evidence-producer-script",
|
||||
artifacts: [
|
||||
{
|
||||
kind: "screenshot",
|
||||
path: ".artifacts/qa-e2e/scenario-script/scenario-script/run-1/surfaces/web-ui/screenshot.png",
|
||||
source: "script-producer:web-ui:smoke",
|
||||
},
|
||||
],
|
||||
},
|
||||
result: { status: "pass" },
|
||||
});
|
||||
});
|
||||
it("uses script scenario timeout overrides when running producer commands", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-script-scenario-timeout-");
|
||||
const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-timeout");
|
||||
const scenario = makeTestFileScenario("script", "scripts/evidence-producer.ts");
|
||||
if (scenario.execution.kind !== "script") {
|
||||
throw new Error("expected script scenario");
|
||||
}
|
||||
scenario.execution.timeoutMs = 3 * 60 * 60_000;
|
||||
|
||||
const commands: QaScenarioCommandExecution[] = [];
|
||||
await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [scenario],
|
||||
commandTimeoutMs: 30 * 60_000,
|
||||
runCommand: async (command) => {
|
||||
commands.push(command);
|
||||
await writeScriptProducerEvidence({
|
||||
outputDir,
|
||||
status: "pass",
|
||||
});
|
||||
return {
|
||||
exitCode: 0,
|
||||
stdout: "script pass\n",
|
||||
stderr: "",
|
||||
};
|
||||
},
|
||||
env: {
|
||||
OPENCLAW_QA_REF: "scenario-ref",
|
||||
} as NodeJS.ProcessEnv,
|
||||
});
|
||||
|
||||
expect(commands.map((command) => command.timeoutMs)).toEqual([3 * 60 * 60_000]);
|
||||
});
|
||||
|
||||
it("imports producer QA evidence artifacts from failed script scenarios", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-script-failed-scenario-");
|
||||
const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-failed");
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")],
|
||||
runCommand: async () => {
|
||||
await writeScriptProducerEvidence({
|
||||
failureReason: "Script producer check failed.",
|
||||
outputDir,
|
||||
status: "fail",
|
||||
});
|
||||
return { exitCode: 1, stdout: "", stderr: "script failed\n" };
|
||||
},
|
||||
env: { OPENCLAW_QA_REF: "scenario-ref" } as NodeJS.ProcessEnv,
|
||||
});
|
||||
|
||||
expect(result.results[0]).toMatchObject({
|
||||
status: "fail",
|
||||
failureMessage: "node exited with 1",
|
||||
producerEvidence: {
|
||||
entries: [
|
||||
{
|
||||
test: { id: "script-producer.web-ui.smoke" },
|
||||
result: { status: "fail" },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
const evidence = validateQaEvidenceSummaryJson(
|
||||
JSON.parse(await fs.readFile(result.evidencePath, "utf8")),
|
||||
);
|
||||
expect(evidence.entries).toHaveLength(2);
|
||||
expect(evidence.entries[0]).toMatchObject({
|
||||
test: { kind: "script-producer-check", id: "script-producer.web-ui.smoke" },
|
||||
coverage: [
|
||||
{ id: "qa.coverage", role: "primary" },
|
||||
{ id: "qa.reporting", role: "secondary" },
|
||||
],
|
||||
result: {
|
||||
status: "fail",
|
||||
failure: { reason: "Script producer check failed." },
|
||||
},
|
||||
});
|
||||
expect(evidence.entries[1]).toMatchObject({
|
||||
test: {
|
||||
kind: "script-test",
|
||||
id: "scenario-script",
|
||||
source: { path: "scripts/evidence-producer.ts" },
|
||||
},
|
||||
result: {
|
||||
status: "fail",
|
||||
failure: { reason: "node exited with 1" },
|
||||
},
|
||||
});
|
||||
});
|
||||
it("suppresses a failed-script fallback row already owned by producer scenario evidence", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-script-duplicate-scenario-evidence-");
|
||||
const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-duplicate");
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")],
|
||||
runCommand: async () => {
|
||||
await writeScriptProducerEvidence({
|
||||
outputDir,
|
||||
producerId: "scenario-script",
|
||||
status: "fail",
|
||||
failureReason: "producer recorded the script failure",
|
||||
});
|
||||
return { exitCode: 1, stdout: "", stderr: "script failed\n" };
|
||||
},
|
||||
env: { OPENCLAW_QA_REF: "scenario-ref" } as NodeJS.ProcessEnv,
|
||||
});
|
||||
|
||||
expect(result.results[0]).toMatchObject({ status: "fail" });
|
||||
expect(result.evidence.entries).toHaveLength(1);
|
||||
expect(result.evidence.entries[0]).toMatchObject({
|
||||
test: { id: "scenario-script" },
|
||||
result: { failure: { reason: "producer recorded the script failure" }, status: "fail" },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails script scenario results when imported producer evidence fails", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-script-producer-fail-");
|
||||
const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-producer-fail");
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")],
|
||||
runCommand: async () => {
|
||||
await writeScriptProducerEvidence({
|
||||
failureReason: "Script producer check failed.",
|
||||
outputDir,
|
||||
status: "fail",
|
||||
});
|
||||
return { exitCode: 0, stdout: "script pass\n", stderr: "" };
|
||||
},
|
||||
env: { OPENCLAW_QA_REF: "scenario-ref" } as NodeJS.ProcessEnv,
|
||||
});
|
||||
|
||||
expect(result.results[0]).toMatchObject({
|
||||
status: "fail",
|
||||
failureMessage: "Script producer check failed.",
|
||||
});
|
||||
const evidence = validateQaEvidenceSummaryJson(
|
||||
JSON.parse(await fs.readFile(result.evidencePath, "utf8")),
|
||||
);
|
||||
expect(evidence.entries).toHaveLength(1);
|
||||
expect(evidence.entries[0]).toMatchObject({
|
||||
test: { id: "script-producer.web-ui.smoke" },
|
||||
result: { status: "fail" },
|
||||
});
|
||||
});
|
||||
it("fails script scenario results when imported producer evidence is blocked by default", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-script-producer-blocked-");
|
||||
const outputDir = path.join(
|
||||
repoRoot,
|
||||
".artifacts",
|
||||
"qa-e2e",
|
||||
"scenario-script-producer-blocked",
|
||||
);
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")],
|
||||
runCommand: async () => {
|
||||
await writeScriptProducerEvidence({
|
||||
outputDir,
|
||||
status: "blocked",
|
||||
failureReason: "Playwright browser is missing.",
|
||||
});
|
||||
return {
|
||||
exitCode: 0,
|
||||
stdout: "script blocked\n",
|
||||
stderr: "",
|
||||
};
|
||||
},
|
||||
env: {
|
||||
OPENCLAW_QA_REF: "scenario-ref",
|
||||
} as NodeJS.ProcessEnv,
|
||||
});
|
||||
|
||||
expect(result.results[0]).toMatchObject({
|
||||
status: "blocked",
|
||||
failureMessage: "Playwright browser is missing.",
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps all-blocked producer evidence blocked for opt-in script scenarios", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-script-producer-blocked-allowed-");
|
||||
const outputDir = path.join(
|
||||
repoRoot,
|
||||
".artifacts",
|
||||
"qa-e2e",
|
||||
"scenario-script-producer-blocked-allowed",
|
||||
);
|
||||
const scenario = makeTestFileScenario("script", "scripts/evidence-producer.ts");
|
||||
if (scenario.execution.kind !== "script") {
|
||||
throw new Error("expected script scenario");
|
||||
}
|
||||
scenario.execution.allowBlockedEvidence = true;
|
||||
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [scenario],
|
||||
runCommand: async () => {
|
||||
await writeScriptProducerEvidence({
|
||||
outputDir,
|
||||
status: "blocked",
|
||||
failureReason: "Playwright browser is missing.",
|
||||
});
|
||||
return {
|
||||
exitCode: 0,
|
||||
stdout: "script blocked\n",
|
||||
stderr: "",
|
||||
};
|
||||
},
|
||||
env: {
|
||||
OPENCLAW_QA_REF: "scenario-ref",
|
||||
} as NodeJS.ProcessEnv,
|
||||
});
|
||||
|
||||
expect(result.results[0]).toMatchObject({
|
||||
status: "blocked",
|
||||
failureMessage: "Playwright browser is missing.",
|
||||
producerEvidence: {
|
||||
entries: [
|
||||
{
|
||||
test: {
|
||||
id: "script-producer.web-ui.smoke",
|
||||
},
|
||||
result: {
|
||||
status: "blocked",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("allows blocked producer checks when another check genuinely passes", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-script-producer-blocked-mixed-");
|
||||
const outputDir = path.join(
|
||||
repoRoot,
|
||||
".artifacts",
|
||||
"qa-e2e",
|
||||
"scenario-script-producer-blocked-mixed",
|
||||
);
|
||||
const scenario = makeTestFileScenario("script", "scripts/evidence-producer.ts");
|
||||
if (scenario.execution.kind !== "script") {
|
||||
throw new Error("expected script scenario");
|
||||
}
|
||||
scenario.execution.allowBlockedEvidence = true;
|
||||
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [scenario],
|
||||
runCommand: async () => {
|
||||
await writeScriptProducerEvidence({
|
||||
additionalEntries: buildScriptProducerEvidence({
|
||||
producerId: "script-producer.web-ui.executed",
|
||||
status: "pass",
|
||||
}).entries,
|
||||
outputDir,
|
||||
status: "blocked",
|
||||
failureReason: "Playwright browser is missing.",
|
||||
});
|
||||
return {
|
||||
exitCode: 0,
|
||||
stdout: "script mixed\n",
|
||||
stderr: "",
|
||||
};
|
||||
},
|
||||
env: {
|
||||
OPENCLAW_QA_REF: "scenario-ref",
|
||||
} as NodeJS.ProcessEnv,
|
||||
});
|
||||
|
||||
expect(result.results[0]).toMatchObject({
|
||||
status: "pass",
|
||||
producerEvidence: {
|
||||
entries: [{ result: { status: "blocked" } }, { result: { status: "pass" } }],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("carries the suite profile into merged producer evidence", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-script-profile-");
|
||||
const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-profile");
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")],
|
||||
runCommand: async () => {
|
||||
await writeScriptProducerEvidence({
|
||||
evidenceLocation: "scenario-root",
|
||||
latestRun: "none",
|
||||
outputDir,
|
||||
profile: "smoke-ci",
|
||||
status: "pass",
|
||||
});
|
||||
return { exitCode: 0, stdout: "script pass\n", stderr: "" };
|
||||
},
|
||||
env: {
|
||||
OPENCLAW_QA_REF: "scenario-ref",
|
||||
OPENCLAW_QA_PROFILE: "smoke-ci",
|
||||
} as NodeJS.ProcessEnv,
|
||||
});
|
||||
|
||||
expect(result.evidence.profile).toBe("smoke-ci");
|
||||
});
|
||||
it("keeps producer artifacts outside the repo root absolute instead of emitting ../ paths", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-script-external-artifact-");
|
||||
const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-external");
|
||||
const externalArtifact = path.join(os.tmpdir(), "qa-external-artifact.png");
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")],
|
||||
runCommand: async () => {
|
||||
await writeScriptProducerEvidence({
|
||||
artifacts: [{ kind: "screenshot", path: externalArtifact }],
|
||||
outputDir,
|
||||
status: "pass",
|
||||
});
|
||||
return { exitCode: 0, stdout: "script pass\n", stderr: "" };
|
||||
},
|
||||
env: { OPENCLAW_QA_REF: "scenario-ref" } as NodeJS.ProcessEnv,
|
||||
});
|
||||
|
||||
const artifactPath = result.evidence.entries[0]?.execution?.artifacts[0]?.path;
|
||||
expect(artifactPath).toBe(path.normalize(externalArtifact));
|
||||
expect(artifactPath?.includes("..")).toBe(false);
|
||||
});
|
||||
it("imports coverage-free structured evidence through the real script lifecycle", async () => {
|
||||
const tempRoot = await harness.makeTempDir("qa-script-real-evidence-");
|
||||
const outputDir = path.join(tempRoot, "out");
|
||||
const scriptPath = path.join(tempRoot, "minimal-evidence-producer.mjs");
|
||||
const producerEvidence = buildScriptProducerEvidence({
|
||||
artifacts: [{ kind: "log", path: "artifact.log" }],
|
||||
coverage: [],
|
||||
status: "pass",
|
||||
});
|
||||
await fs.writeFile(
|
||||
scriptPath,
|
||||
[
|
||||
"import fs from 'node:fs/promises';",
|
||||
"import path from 'node:path';",
|
||||
"const artifactBaseIndex = process.argv.indexOf('--artifact-base');",
|
||||
"if (artifactBaseIndex < 0) throw new Error('missing --artifact-base');",
|
||||
"const artifactBase = process.argv[artifactBaseIndex + 1];",
|
||||
"const runRoot = path.join(artifactBase, 'run-1');",
|
||||
"await fs.mkdir(runRoot, { recursive: true });",
|
||||
"await fs.writeFile(path.join(runRoot, 'artifact.log'), 'structured evidence\\n', 'utf8');",
|
||||
`const evidence = ${JSON.stringify(producerEvidence)};`,
|
||||
"await fs.writeFile(path.join(runRoot, 'qa-evidence.json'), JSON.stringify(evidence), 'utf8');",
|
||||
"await fs.writeFile(path.join(artifactBase, 'latest-run.json'), JSON.stringify({ qaEvidence: 'run-1/qa-evidence.json' }), 'utf8');",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
const infrastructureFixture: QaSeedScenarioWithSource = {
|
||||
id: "scenario-script",
|
||||
title: "Temporary script evidence fixture",
|
||||
surface: "qa-lab",
|
||||
objective: "Exercise structured evidence import through the real script lifecycle.",
|
||||
successCriteria: ["The runner imports coverage-free producer evidence and artifacts."],
|
||||
codeRefs: ["external/qa/minimal-evidence-producer.mjs"],
|
||||
sourcePath: "external/qa/minimal-evidence-scenario.yaml",
|
||||
execution: {
|
||||
kind: "script",
|
||||
path: scriptPath,
|
||||
args: ["--artifact-base", "${outputDir}"],
|
||||
},
|
||||
};
|
||||
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot: process.cwd(),
|
||||
outputDir,
|
||||
...QA_TEST_RUNNER_DEFAULTS,
|
||||
scenarios: [infrastructureFixture],
|
||||
commandTimeoutMs: 20_000,
|
||||
env: { OPENCLAW_QA_REF: "temporary-script-fixture" } as NodeJS.ProcessEnv,
|
||||
});
|
||||
|
||||
expect(result.executionKind).toBe("script");
|
||||
expect(result.results[0]).toMatchObject({
|
||||
status: "pass",
|
||||
producerEvidence: {
|
||||
entries: [{ test: { id: "script-producer.web-ui.smoke" } }],
|
||||
},
|
||||
});
|
||||
expect(result.evidence.entries[0]).toMatchObject({
|
||||
coverage: [],
|
||||
execution: {
|
||||
artifacts: [
|
||||
{
|
||||
kind: "log",
|
||||
path: path.join(outputDir, "scenario-script", "run-1", "artifact.log"),
|
||||
},
|
||||
],
|
||||
},
|
||||
test: { id: "script-producer.web-ui.smoke" },
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,230 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
QA_EVIDENCE_FILENAME,
|
||||
QA_EVIDENCE_SUMMARY_KIND,
|
||||
QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
|
||||
type QaEvidenceSummaryJson,
|
||||
validateQaEvidenceSummaryJson,
|
||||
} from "./evidence-summary.js";
|
||||
import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
|
||||
import { createTempDirHarness } from "./temp-dir.test-helper.js";
|
||||
import type {
|
||||
QaScenarioCommandExecution,
|
||||
runQaTestFileScenarios,
|
||||
} from "./test-file-scenario-runner.js";
|
||||
|
||||
export const QA_TEST_RUNNER_DEFAULTS = {
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.6-luna",
|
||||
} satisfies Pick<Parameters<typeof runQaTestFileScenarios>[0], "primaryModel" | "providerMode">;
|
||||
|
||||
export function createScenarioRunnerTestHarness() {
|
||||
const tempDirs = createTempDirHarness();
|
||||
|
||||
return {
|
||||
makeTempDir: tempDirs.makeTempDir,
|
||||
makeTempRepo: (prefix: string) => tempDirs.makeTempDir(prefix),
|
||||
cleanup: tempDirs.cleanup,
|
||||
};
|
||||
}
|
||||
|
||||
export function makeTestFileScenario(
|
||||
executionKind: "script" | "vitest" | "playwright",
|
||||
pathLocal: string,
|
||||
testNamePattern?: string,
|
||||
): QaSeedScenarioWithSource {
|
||||
return {
|
||||
id: `scenario-${executionKind}`,
|
||||
title: `${executionKind} scenario`,
|
||||
surface: executionKind === "playwright" ? "control-ui" : "qa-lab",
|
||||
category: executionKind === "playwright" ? "control-ui.browser-ui" : "qa-lab.coverage",
|
||||
coverage: {
|
||||
primary: [executionKind === "playwright" ? "ui.control" : "qa.coverage"],
|
||||
secondary: [executionKind === "playwright" ? "ui.streaming" : "qa.reporting"],
|
||||
},
|
||||
objective: `Exercise ${executionKind} scenario evidence.`,
|
||||
successCriteria: ["The scenario writes structured evidence."],
|
||||
docsRefs: ["docs/concepts/qa-e2e-automation.md"],
|
||||
codeRefs: [pathLocal],
|
||||
sourcePath: `qa/scenarios/ui/scenario-${executionKind}.md`,
|
||||
execution: {
|
||||
kind: executionKind,
|
||||
path: pathLocal,
|
||||
...(testNamePattern ? { testNamePattern } : {}),
|
||||
...(executionKind === "script"
|
||||
? { args: ["--once", "--artifact-base", "${outputDir}"] }
|
||||
: {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function makeDockerE2eScenario(id: string, lane: string): QaSeedScenarioWithSource {
|
||||
const scenario = makeTestFileScenario("script", "test/e2e/qa-lab/runtime/docker-e2e-lane.ts");
|
||||
if (scenario.execution.kind !== "script") {
|
||||
throw new Error("expected script scenario");
|
||||
}
|
||||
return {
|
||||
...scenario,
|
||||
id,
|
||||
execution: {
|
||||
...scenario.execution,
|
||||
args: ["--lane", lane],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function writeDockerCandidateManifest(
|
||||
command: QaScenarioCommandExecution,
|
||||
manifest: unknown,
|
||||
) {
|
||||
const manifestArg = command.args.find((arg) => arg.startsWith("--prepare-only="));
|
||||
if (!manifestArg) {
|
||||
throw new Error("missing prep-only manifest argument");
|
||||
}
|
||||
await fs.writeFile(manifestArg.slice("--prepare-only=".length), `${JSON.stringify(manifest)}\n`);
|
||||
return { exitCode: 0, stdout: "", stderr: "" };
|
||||
}
|
||||
|
||||
export async function writeNativeVitestReport(
|
||||
command: QaScenarioCommandExecution,
|
||||
counts: {
|
||||
createRequestedTestFile?: boolean;
|
||||
failed?: number;
|
||||
passed: number;
|
||||
testFilePath?: string;
|
||||
testName?: string;
|
||||
},
|
||||
) {
|
||||
const reportArg = command.args.find((arg) => arg.startsWith("--outputFile.json="));
|
||||
if (!reportArg) {
|
||||
return;
|
||||
}
|
||||
const requestedTestPath = command.args.find((arg) => arg.endsWith(".test.ts"));
|
||||
if (requestedTestPath && counts.createRequestedTestFile !== false) {
|
||||
const requestedTestFile = path.resolve(command.cwd, requestedTestPath);
|
||||
await fs.mkdir(path.dirname(requestedTestFile), { recursive: true });
|
||||
await fs.writeFile(requestedTestFile, "// native scenario fixture\n", "utf8");
|
||||
}
|
||||
const testNamePatternIndex = command.args.indexOf("--testNamePattern");
|
||||
const testName =
|
||||
counts.testName ??
|
||||
(testNamePatternIndex < 0 ? undefined : command.args[testNamePatternIndex + 1]) ??
|
||||
"executes the requested scenario";
|
||||
await fs.writeFile(
|
||||
reportArg.slice("--outputFile.json=".length),
|
||||
JSON.stringify({
|
||||
numFailedTests: counts.failed ?? 0,
|
||||
numPassedTests: counts.passed,
|
||||
success: (counts.failed ?? 0) === 0,
|
||||
testResults: [
|
||||
{
|
||||
name: path.resolve(command.cwd, counts.testFilePath ?? requestedTestPath ?? "unknown"),
|
||||
status: counts.passed > 0 ? "passed" : "skipped",
|
||||
assertionResults:
|
||||
counts.passed > 0 ? [{ fullName: testName, title: testName, status: "passed" }] : [],
|
||||
},
|
||||
],
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
|
||||
type ScriptEvidenceArtifact = {
|
||||
kind: string;
|
||||
path: string;
|
||||
source?: string;
|
||||
};
|
||||
|
||||
type ScriptProducerEvidenceParams = {
|
||||
additionalEntries?: QaEvidenceSummaryJson["entries"];
|
||||
artifacts?: ScriptEvidenceArtifact[];
|
||||
coverage?: QaEvidenceSummaryJson["entries"][number]["coverage"];
|
||||
failureReason?: string;
|
||||
producerId?: string;
|
||||
profile?: string;
|
||||
status: "blocked" | "fail" | "pass";
|
||||
};
|
||||
|
||||
export function buildScriptProducerEvidence(
|
||||
params: ScriptProducerEvidenceParams,
|
||||
): QaEvidenceSummaryJson {
|
||||
return validateQaEvidenceSummaryJson({
|
||||
kind: QA_EVIDENCE_SUMMARY_KIND,
|
||||
schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
|
||||
generatedAt: "2026-06-14T00:00:00.000Z",
|
||||
evidenceMode: "full",
|
||||
...(params.profile ? { profile: params.profile } : {}),
|
||||
entries: [
|
||||
{
|
||||
test: {
|
||||
kind: "script-producer-check",
|
||||
id: params.producerId ?? "script-producer.web-ui.smoke",
|
||||
title: "Script producer: web-ui smoke",
|
||||
source: { path: "external/qa/evidence-producer.mjs" },
|
||||
},
|
||||
coverage: params.coverage ?? [{ id: "ui.control", role: "primary" }],
|
||||
execution: {
|
||||
runner: "evidence-producer-script",
|
||||
environment: { ref: "scenario-ref", os: "darwin", nodeVersion: "v24.0.0" },
|
||||
provider: {
|
||||
id: "script-producer",
|
||||
live: false,
|
||||
model: { name: null, ref: null },
|
||||
fixture: "synthetic-script-evidence",
|
||||
},
|
||||
packageSource: { kind: "source-checkout", sha: "abc123" },
|
||||
artifacts: (params.artifacts ?? []).map((artifact) =>
|
||||
Object.assign({ source: "script-producer:web-ui:smoke" }, artifact),
|
||||
),
|
||||
},
|
||||
result: {
|
||||
status: params.status,
|
||||
...(params.failureReason ? { failure: { reason: params.failureReason } } : {}),
|
||||
timing: { wallMs: 1 },
|
||||
},
|
||||
},
|
||||
...(params.additionalEntries ?? []),
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
export async function writeScriptProducerEvidence(
|
||||
params: ScriptProducerEvidenceParams & {
|
||||
evidenceLocation?: "run" | "scenario-root";
|
||||
latestRun?: "absolute" | "none" | "relative";
|
||||
outputDir: string;
|
||||
scenarioId?: string;
|
||||
},
|
||||
) {
|
||||
const scenarioArtifactBase = path.join(params.outputDir, params.scenarioId ?? "scenario-script");
|
||||
const evidenceDir =
|
||||
params.evidenceLocation === "scenario-root"
|
||||
? scenarioArtifactBase
|
||||
: path.join(scenarioArtifactBase, "run-1");
|
||||
const evidencePath = path.join(evidenceDir, QA_EVIDENCE_FILENAME);
|
||||
await fs.mkdir(evidenceDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
evidencePath,
|
||||
`${JSON.stringify(buildScriptProducerEvidence(params), null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
const latestRun = params.latestRun ?? "absolute";
|
||||
if (latestRun !== "none") {
|
||||
await fs.writeFile(
|
||||
path.join(scenarioArtifactBase, "latest-run.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
qaEvidence:
|
||||
latestRun === "relative"
|
||||
? path.relative(scenarioArtifactBase, evidencePath)
|
||||
: evidencePath,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
return { evidenceDir, evidencePath, scenarioArtifactBase };
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -460,9 +460,9 @@ describe("QA Lab UI evidence render", () => {
|
||||
runner: {
|
||||
availability: "local",
|
||||
command:
|
||||
"node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix",
|
||||
"node external/qa/ux-matrix-producer.mjs --artifact-base .artifacts/external-qa/ux-matrix",
|
||||
lane: "web-ui-playwright",
|
||||
workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local",
|
||||
workflow: "external/ci/ux-matrix.yml#matrix-local",
|
||||
},
|
||||
stage: "first-run",
|
||||
status: "pass",
|
||||
@@ -477,9 +477,9 @@ describe("QA Lab UI evidence render", () => {
|
||||
runner: {
|
||||
availability: "local",
|
||||
command:
|
||||
"node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix",
|
||||
"node external/qa/ux-matrix-producer.mjs --artifact-base .artifacts/external-qa/ux-matrix",
|
||||
lane: "cli-status",
|
||||
workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local",
|
||||
workflow: "external/ci/ux-matrix.yml#matrix-local",
|
||||
},
|
||||
stage: "first-run",
|
||||
status: "proof-gap",
|
||||
|
||||
@@ -1,814 +0,0 @@
|
||||
// Produces standalone QA Lab UX Matrix fixture artifacts for gallery and evidence tests.
|
||||
import { execFile } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { toRepoRelativePath } from "../../extensions/qa-lab/src/cli-paths.js";
|
||||
import { resolveQaEvidenceEnvironment } from "../../extensions/qa-lab/src/evidence-environment.js";
|
||||
import {
|
||||
QA_EVIDENCE_FILENAME,
|
||||
QA_EVIDENCE_SUMMARY_KIND,
|
||||
QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
|
||||
validateQaEvidenceSummaryJson,
|
||||
type QaEvidenceStatus,
|
||||
type QaEvidenceSummaryEntry,
|
||||
type QaEvidenceSummaryJson,
|
||||
} from "../../extensions/qa-lab/src/evidence-summary.js";
|
||||
import {
|
||||
ensurePlaywrightChromium,
|
||||
resolveSystemChromiumExecutablePath,
|
||||
} from "../ensure-playwright-chromium.mts";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const SOURCE_PATH = "scripts/qa/ux-matrix-evidence-producer.ts";
|
||||
|
||||
type MatrixCell = {
|
||||
artifacts: Array<{ kind: string; path: string }>;
|
||||
failureReason?: string;
|
||||
stage: string;
|
||||
status: QaEvidenceStatus;
|
||||
surface: string;
|
||||
title: string;
|
||||
wallMs: number;
|
||||
};
|
||||
|
||||
type ChromiumLauncher = Awaited<typeof import("playwright")>["chromium"];
|
||||
type ChromiumBrowser = Awaited<ReturnType<ChromiumLauncher["launch"]>>;
|
||||
|
||||
type ProducerOptions = {
|
||||
artifactBase: string;
|
||||
repoRoot: string;
|
||||
skipVisualProof: boolean;
|
||||
};
|
||||
|
||||
function usage() {
|
||||
return `Usage: node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base <dir> [options]
|
||||
|
||||
Produces a QA Lab UX Matrix evidence bundle.
|
||||
|
||||
Options:
|
||||
--artifact-base <dir> Evidence artifact directory
|
||||
--repo-root <dir> Repository root
|
||||
--skip-visual-proof Use fixture visual evidence instead of Playwright screenshots
|
||||
-h, --help Show this help
|
||||
`;
|
||||
}
|
||||
|
||||
function readOptionValue(argv: readonly string[], index: number, arg: string) {
|
||||
const value = argv[index + 1] ?? "";
|
||||
if (!value || value.startsWith("-")) {
|
||||
throw new Error(`${arg} requires a value`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function isHelpRequest(argv: readonly string[]) {
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--artifact-base" || arg === "--repo-root") {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function parseOptions(argv: readonly string[]): ProducerOptions {
|
||||
let artifactBase = "";
|
||||
let repoRoot = process.cwd();
|
||||
let skipVisualProof = false;
|
||||
const seen = new Set<string>();
|
||||
const recordOnce = (flag: string) => {
|
||||
if (seen.has(flag)) {
|
||||
throw new Error(`${flag} was provided more than once`);
|
||||
}
|
||||
seen.add(flag);
|
||||
};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--artifact-base") {
|
||||
const value = readOptionValue(argv, index, arg);
|
||||
recordOnce(arg);
|
||||
artifactBase = value;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--repo-root") {
|
||||
const value = readOptionValue(argv, index, arg);
|
||||
recordOnce(arg);
|
||||
repoRoot = value;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--skip-visual-proof") {
|
||||
recordOnce(arg);
|
||||
skipVisualProof = true;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`unsupported UX Matrix producer arg: ${arg}`);
|
||||
}
|
||||
if (!artifactBase.trim()) {
|
||||
throw new Error("--artifact-base is required");
|
||||
}
|
||||
if (!repoRoot.trim()) {
|
||||
throw new Error("--repo-root must not be empty");
|
||||
}
|
||||
return {
|
||||
artifactBase: path.resolve(repoRoot, artifactBase),
|
||||
repoRoot: path.resolve(repoRoot),
|
||||
skipVisualProof,
|
||||
};
|
||||
}
|
||||
|
||||
type ProducerCliOutput = {
|
||||
error: (message: string) => void;
|
||||
log: (message: string) => void;
|
||||
};
|
||||
|
||||
export async function runUxMatrixEvidenceProducerCli(
|
||||
argv: readonly string[],
|
||||
output: ProducerCliOutput = console,
|
||||
): Promise<number> {
|
||||
try {
|
||||
if (isHelpRequest(argv)) {
|
||||
output.log(usage());
|
||||
return 0;
|
||||
}
|
||||
const result = await runUxMatrixEvidenceProducer(parseOptions(argv));
|
||||
output.log(`UX Matrix evidence: ${path.join(result.artifactBase, QA_EVIDENCE_FILENAME)}`);
|
||||
output.log(`UX Matrix entries: ${result.evidence.entries.length}`);
|
||||
return 0;
|
||||
} catch (error) {
|
||||
output.error(error instanceof Error ? error.message : String(error));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
async function writeJson(filePath: string, value: unknown) {
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
||||
await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
async function writeText(filePath: string, value: string) {
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
||||
await fs.writeFile(filePath, value, "utf8");
|
||||
}
|
||||
|
||||
function cellDir(artifactBase: string, cell: Pick<MatrixCell, "stage" | "surface">) {
|
||||
return path.join(artifactBase, "surfaces", cell.surface, "stages", cell.stage);
|
||||
}
|
||||
|
||||
function relativeToArtifactBase(artifactBase: string, filePath: string) {
|
||||
return path.relative(artifactBase, filePath).split(path.sep).join("/");
|
||||
}
|
||||
|
||||
function sanitizeArtifactText(
|
||||
value: string,
|
||||
params: {
|
||||
artifactBase?: string;
|
||||
repoRoot: string;
|
||||
},
|
||||
) {
|
||||
const roots = [
|
||||
{ from: path.resolve(params.repoRoot), to: "<repo-root>" },
|
||||
{ from: pathToFileURL(path.resolve(params.repoRoot)).href, to: "file://<repo-root>" },
|
||||
...(params.artifactBase
|
||||
? [
|
||||
{ from: path.resolve(params.artifactBase), to: "<artifact-base>" },
|
||||
{
|
||||
from: pathToFileURL(path.resolve(params.artifactBase)).href,
|
||||
to: "file://<artifact-base>",
|
||||
},
|
||||
]
|
||||
: []),
|
||||
{ from: os.homedir(), to: "<home>" },
|
||||
{ from: pathToFileURL(os.homedir()).href, to: "file://<home>" },
|
||||
].filter((entry) => entry.from && entry.from !== path.parse(entry.from).root);
|
||||
return roots
|
||||
.toSorted((a, b) => b.from.length - a.from.length)
|
||||
.reduce((text, entry) => text.replaceAll(entry.from, entry.to), value);
|
||||
}
|
||||
|
||||
function buildExecution(params: {
|
||||
artifacts: MatrixCell["artifacts"];
|
||||
repoRoot: string;
|
||||
source: string;
|
||||
}): QaEvidenceSummaryEntry["execution"] {
|
||||
return {
|
||||
runner: "ux-matrix-script-producer",
|
||||
environment: resolveQaEvidenceEnvironment({
|
||||
env: process.env,
|
||||
repoRoot: params.repoRoot,
|
||||
}),
|
||||
provider: {
|
||||
id: "ux-matrix",
|
||||
live: false,
|
||||
model: {
|
||||
name: null,
|
||||
ref: null,
|
||||
},
|
||||
fixture: "local-qa-lab-script-producer",
|
||||
},
|
||||
packageSource: {
|
||||
kind: "source-checkout",
|
||||
},
|
||||
artifacts: params.artifacts.map((artifact) => ({
|
||||
...artifact,
|
||||
source: params.source,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function buildEvidenceEntry(cell: MatrixCell, repoRoot: string): QaEvidenceSummaryEntry {
|
||||
const source = `ux-matrix:${cell.surface}:${cell.stage}`;
|
||||
return {
|
||||
test: {
|
||||
kind: "ux-matrix-cell",
|
||||
id: `ux-matrix.${cell.surface}.${cell.stage}`,
|
||||
title: cell.title,
|
||||
source: { path: SOURCE_PATH },
|
||||
},
|
||||
// These entries prove the evidence/gallery infrastructure, not product taxonomy behavior.
|
||||
coverage: [],
|
||||
refs: [
|
||||
{ kind: "code", path: SOURCE_PATH },
|
||||
{ kind: "docs", path: "docs/concepts/qa-e2e-automation.md" },
|
||||
],
|
||||
execution: buildExecution({
|
||||
artifacts: cell.artifacts,
|
||||
repoRoot,
|
||||
source,
|
||||
}),
|
||||
result: {
|
||||
status: cell.status,
|
||||
...(cell.status === "pass"
|
||||
? {}
|
||||
: {
|
||||
failure: {
|
||||
class: cell.status,
|
||||
reason: cell.failureReason ?? `${cell.status} UX Matrix cell`,
|
||||
},
|
||||
}),
|
||||
timing: {
|
||||
wallMs: Math.max(1, cell.wallMs),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildEvidenceSummary(params: {
|
||||
cells: readonly MatrixCell[];
|
||||
generatedAt: string;
|
||||
repoRoot: string;
|
||||
}): QaEvidenceSummaryJson {
|
||||
return validateQaEvidenceSummaryJson({
|
||||
kind: QA_EVIDENCE_SUMMARY_KIND,
|
||||
schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
|
||||
generatedAt: params.generatedAt,
|
||||
evidenceMode: "full",
|
||||
entries: params.cells.map((cell) => buildEvidenceEntry(cell, params.repoRoot)),
|
||||
});
|
||||
}
|
||||
|
||||
async function runCommandForCell(params: {
|
||||
args: string[];
|
||||
artifactBase: string;
|
||||
command: string;
|
||||
cwd: string;
|
||||
logPath: string;
|
||||
timeoutMs: number;
|
||||
}) {
|
||||
const startedAt = Date.now();
|
||||
const commandLine = [params.command, ...params.args].join(" ");
|
||||
try {
|
||||
const { stdout, stderr } = await execFileAsync(params.command, params.args, {
|
||||
cwd: params.cwd,
|
||||
timeout: params.timeoutMs,
|
||||
env: process.env,
|
||||
maxBuffer: 1024 * 1024,
|
||||
});
|
||||
await writeText(
|
||||
params.logPath,
|
||||
sanitizeArtifactText(`$ ${commandLine}\n${stdout}${stderr}`, {
|
||||
artifactBase: params.artifactBase,
|
||||
repoRoot: params.cwd,
|
||||
}),
|
||||
);
|
||||
return {
|
||||
status: "pass" as const,
|
||||
wallMs: Date.now() - startedAt,
|
||||
};
|
||||
} catch (error) {
|
||||
const details = sanitizeArtifactText(error instanceof Error ? error.message : String(error), {
|
||||
artifactBase: params.artifactBase,
|
||||
repoRoot: params.cwd,
|
||||
});
|
||||
await writeText(params.logPath, `$ ${commandLine}\nblocked: ${details}\n`);
|
||||
return {
|
||||
failureReason: details,
|
||||
status: "blocked" as const,
|
||||
wallMs: Date.now() - startedAt,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function writePreflight(artifactBase: string) {
|
||||
await writeText(
|
||||
path.join(artifactBase, "preflight", "runtime.txt"),
|
||||
[
|
||||
`platform=${process.platform}`,
|
||||
`arch=${process.arch}`,
|
||||
`node=${process.version}`,
|
||||
`freeMemoryBytes=${os.freemem()}`,
|
||||
`totalMemoryBytes=${os.totalmem()}`,
|
||||
].join("\n") + "\n",
|
||||
);
|
||||
}
|
||||
|
||||
async function writeSkippedVisualProof(logPath: string) {
|
||||
const startedAt = Date.now();
|
||||
await writeText(logPath, "blocked: --skip-visual-proof was set\n");
|
||||
return {
|
||||
failureReason: "--skip-visual-proof was set",
|
||||
status: "blocked" as const,
|
||||
wallMs: Date.now() - startedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function isMissingManagedPlaywrightBrowser(error: unknown) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return (
|
||||
message.includes("Executable doesn't exist") &&
|
||||
message.includes(".cache/ms-playwright") &&
|
||||
message.includes("playwright install")
|
||||
);
|
||||
}
|
||||
|
||||
export async function launchUxMatrixChromium(params?: {
|
||||
chromium?: Pick<ChromiumLauncher, "launch">;
|
||||
systemExecutablePath?: string;
|
||||
}): Promise<{ browser: ChromiumBrowser; usedSystemExecutablePath?: string }> {
|
||||
const chromium = params?.chromium ?? (await import("playwright")).chromium;
|
||||
try {
|
||||
return { browser: await chromium.launch() };
|
||||
} catch (error) {
|
||||
if (!isMissingManagedPlaywrightBrowser(error)) {
|
||||
throw error;
|
||||
}
|
||||
const executablePath = params?.systemExecutablePath ?? resolveSystemChromiumExecutablePath();
|
||||
if (!executablePath) {
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
browser: await chromium.launch({ executablePath }),
|
||||
usedSystemExecutablePath: executablePath,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function ensureUxMatrixVideoDependencies(params: {
|
||||
ensureChromium?: typeof ensurePlaywrightChromium;
|
||||
usedSystemExecutablePath?: string;
|
||||
}) {
|
||||
if (!params.usedSystemExecutablePath) {
|
||||
return;
|
||||
}
|
||||
const ensureChromium = params.ensureChromium ?? ensurePlaywrightChromium;
|
||||
const status = ensureChromium({
|
||||
ensureFfmpeg: true,
|
||||
systemExecutablePath: params.usedSystemExecutablePath,
|
||||
});
|
||||
if (status !== 0) {
|
||||
throw new Error(`Playwright ffmpeg install failed with status ${status}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function captureControlUiScreenshot(params: {
|
||||
artifactBase: string;
|
||||
htmlPath: string;
|
||||
logPath: string;
|
||||
repoRoot: string;
|
||||
screenshotPath: string;
|
||||
skipVisualProof: boolean;
|
||||
}) {
|
||||
if (params.skipVisualProof) {
|
||||
return writeSkippedVisualProof(params.logPath);
|
||||
}
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
const { browser } = await launchUxMatrixChromium();
|
||||
try {
|
||||
const page = await browser.newPage({ viewport: { width: 1024, height: 720 } });
|
||||
await page.goto(pathToFileURL(params.htmlPath).href);
|
||||
await page.screenshot({ path: params.screenshotPath, fullPage: true });
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
await writeText(
|
||||
params.logPath,
|
||||
`Captured ${relativeToArtifactBase(params.artifactBase, params.screenshotPath)}\n`,
|
||||
);
|
||||
return {
|
||||
status: "pass" as const,
|
||||
wallMs: Date.now() - startedAt,
|
||||
};
|
||||
} catch (error) {
|
||||
const details = sanitizeArtifactText(error instanceof Error ? error.message : String(error), {
|
||||
artifactBase: params.artifactBase,
|
||||
repoRoot: params.repoRoot,
|
||||
});
|
||||
await writeText(params.logPath, `blocked: ${details}\n`);
|
||||
return {
|
||||
failureReason: details,
|
||||
status: "blocked" as const,
|
||||
wallMs: Date.now() - startedAt,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function escapeHtml(value: string) {
|
||||
return value
|
||||
.replaceAll("&", "&")
|
||||
.replaceAll("<", "<")
|
||||
.replaceAll(">", ">")
|
||||
.replaceAll('"', """);
|
||||
}
|
||||
|
||||
async function writeProducerArtifactFixtureHtml(params: {
|
||||
artifactBase: string;
|
||||
evidence: QaEvidenceSummaryJson;
|
||||
htmlPath: string;
|
||||
logPreview: string;
|
||||
}) {
|
||||
const previewArtifacts = params.evidence.entries
|
||||
.filter((entry) => entry.test.id !== "ux-matrix.qa-lab.producer-artifact-fixture")
|
||||
.flatMap((entry) => entry.execution?.artifacts ?? []);
|
||||
const screenshotArtifact = previewArtifacts.find((artifact) => artifact.kind === "screenshot");
|
||||
const logArtifact = previewArtifacts.find((artifact) => artifact.kind === "log");
|
||||
const screenshotPath = screenshotArtifact
|
||||
? relativeToArtifactBase(
|
||||
path.dirname(params.htmlPath),
|
||||
path.join(params.artifactBase, screenshotArtifact.path),
|
||||
)
|
||||
: "";
|
||||
const entryRows = params.evidence.entries
|
||||
.map(
|
||||
(entry) =>
|
||||
`<li><strong>${escapeHtml(entry.test.id)}</strong> - ${escapeHtml(
|
||||
entry.result.status,
|
||||
)} - ${entry.coverage.map((coverage) => escapeHtml(coverage.id)).join(", ")}</li>`,
|
||||
)
|
||||
.join("");
|
||||
await writeText(
|
||||
params.htmlPath,
|
||||
`<!doctype html>
|
||||
<meta charset="utf-8">
|
||||
<title>UX Matrix producer artifact fixture</title>
|
||||
<style>
|
||||
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; margin: 0; background: #f6f7f9; color: #121417; }
|
||||
main { max-width: 980px; margin: 0 auto; padding: 28px; }
|
||||
h1 { font-size: 30px; margin: 0 0 8px; }
|
||||
.panel { background: white; border: 1px solid #d9dee7; border-radius: 8px; padding: 18px; margin-top: 18px; }
|
||||
.toolbar { display: flex; gap: 10px; flex-wrap: wrap; margin-top: 12px; }
|
||||
button { border: 1px solid #8c98aa; background: #fff; border-radius: 6px; padding: 8px 12px; font: inherit; cursor: pointer; }
|
||||
button:focus { outline: 3px solid #91c5ff; }
|
||||
img { max-width: 100%; border: 1px solid #ccd3dd; border-radius: 6px; background: #fff; }
|
||||
pre { white-space: pre-wrap; background: #111827; color: #f9fafb; padding: 14px; border-radius: 6px; overflow: auto; }
|
||||
.meta { color: #526070; }
|
||||
</style>
|
||||
<main>
|
||||
<h1>UX Matrix Producer Artifact Fixture</h1>
|
||||
<p class="meta">Standalone script-produced ${escapeHtml(
|
||||
QA_EVIDENCE_FILENAME,
|
||||
)}; this fixture is not the QA Lab Evidence Archive UI or product-behavior proof.</p>
|
||||
<section class="panel">
|
||||
<h2>UX Matrix entries</h2>
|
||||
<ul>${entryRows}</ul>
|
||||
</section>
|
||||
<section class="panel">
|
||||
<h2>Artifact preview</h2>
|
||||
<p class="meta">${escapeHtml(logArtifact?.path ?? "no log")} - ${escapeHtml(
|
||||
screenshotArtifact?.path ?? "no screenshot",
|
||||
)}</p>
|
||||
<div class="toolbar">
|
||||
<button id="preview-screenshot" type="button">Preview screenshot artifact</button>
|
||||
<button id="preview-log" type="button">Preview log artifact</button>
|
||||
</div>
|
||||
<div id="preview" class="panel" aria-live="polite">Choose an artifact to preview.</div>
|
||||
</section>
|
||||
</main>
|
||||
<script>
|
||||
const preview = document.querySelector("#preview");
|
||||
document.querySelector("#preview-screenshot").addEventListener("click", () => {
|
||||
preview.innerHTML = ${JSON.stringify(
|
||||
screenshotPath
|
||||
? `<img alt="UX Matrix screenshot artifact" src="${escapeHtml(screenshotPath)}">`
|
||||
: "<p>No screenshot artifact was produced.</p>",
|
||||
)};
|
||||
});
|
||||
document.querySelector("#preview-log").addEventListener("click", () => {
|
||||
preview.innerHTML = ${JSON.stringify(`<pre>${escapeHtml(params.logPreview)}</pre>`)};
|
||||
});
|
||||
</script>
|
||||
`,
|
||||
);
|
||||
}
|
||||
|
||||
async function captureProducerArtifactFixtureProof(params: {
|
||||
artifactBase: string;
|
||||
htmlPath: string;
|
||||
logPath: string;
|
||||
repoRoot: string;
|
||||
screenshotPath: string;
|
||||
skipVisualProof: boolean;
|
||||
videoPath: string;
|
||||
}) {
|
||||
if (params.skipVisualProof) {
|
||||
return writeSkippedVisualProof(params.logPath);
|
||||
}
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
const videoDir = path.join(path.dirname(params.videoPath), "recording");
|
||||
await fs.mkdir(videoDir, { recursive: true });
|
||||
const { browser, usedSystemExecutablePath } = await launchUxMatrixChromium();
|
||||
let recordedVideo: string | undefined;
|
||||
try {
|
||||
ensureUxMatrixVideoDependencies({ usedSystemExecutablePath });
|
||||
const context = await browser.newContext({
|
||||
viewport: { width: 1280, height: 820 },
|
||||
recordVideo: {
|
||||
dir: videoDir,
|
||||
size: { width: 1280, height: 820 },
|
||||
},
|
||||
});
|
||||
const page = await context.newPage();
|
||||
const video = page.video();
|
||||
await page.goto(pathToFileURL(params.htmlPath).href);
|
||||
await page.locator("#preview-screenshot").click();
|
||||
await page.locator("#preview img").waitFor({ state: "visible", timeout: 5_000 });
|
||||
await page.screenshot({ path: params.screenshotPath, fullPage: true });
|
||||
await page.waitForTimeout(350);
|
||||
await page.locator("#preview-log").click();
|
||||
await page.waitForTimeout(350);
|
||||
await context.close();
|
||||
recordedVideo = await video?.path();
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
if (!recordedVideo) {
|
||||
throw new Error("Playwright did not provide a recording path");
|
||||
}
|
||||
await fs.mkdir(path.dirname(params.videoPath), { recursive: true });
|
||||
await fs.copyFile(recordedVideo, params.videoPath);
|
||||
await writeText(
|
||||
params.logPath,
|
||||
[
|
||||
`Captured screenshot ${path.basename(params.screenshotPath)}`,
|
||||
`Captured recording ${path.basename(params.videoPath)}`,
|
||||
].join("\n") + "\n",
|
||||
);
|
||||
return {
|
||||
status: "pass" as const,
|
||||
wallMs: Date.now() - startedAt,
|
||||
};
|
||||
} catch (error) {
|
||||
const details = sanitizeArtifactText(error instanceof Error ? error.message : String(error), {
|
||||
artifactBase: params.artifactBase,
|
||||
repoRoot: params.repoRoot,
|
||||
});
|
||||
await writeText(params.logPath, `blocked: ${details}\n`);
|
||||
return {
|
||||
failureReason: details,
|
||||
status: "blocked" as const,
|
||||
wallMs: Date.now() - startedAt,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function writeProducerMetadata(params: {
|
||||
artifactBase: string;
|
||||
cells: readonly MatrixCell[];
|
||||
repoRoot: string;
|
||||
}) {
|
||||
const counts = params.cells.reduce<Record<string, number>>((acc, cell) => {
|
||||
acc[cell.status] = (acc[cell.status] ?? 0) + 1;
|
||||
return acc;
|
||||
}, {});
|
||||
await writeJson(path.join(params.artifactBase, "manifest.json"), {
|
||||
kind: "openclaw.qa.ux-matrix",
|
||||
run: {
|
||||
status: counts.fail ? "fail" : counts.blocked ? "blocked" : "pass",
|
||||
},
|
||||
});
|
||||
await writeJson(path.join(params.artifactBase, "matrix.json"), {
|
||||
cells: params.cells.map((cell) => ({
|
||||
coverageIds: [],
|
||||
stage: cell.stage,
|
||||
status: cell.status,
|
||||
surface: cell.surface,
|
||||
})),
|
||||
counts,
|
||||
});
|
||||
await writeJson(path.join(params.artifactBase, "release-ledger.json"), {
|
||||
entries: params.cells.map((cell) => ({
|
||||
coverageIds: [],
|
||||
stage: cell.stage,
|
||||
status: cell.status,
|
||||
surface: cell.surface,
|
||||
})),
|
||||
kind: "openclaw.qa.ux-matrix.release-ledger",
|
||||
});
|
||||
await writeText(
|
||||
path.join(params.artifactBase, "commands.txt"),
|
||||
`node --import tsx ${SOURCE_PATH} --artifact-base ${toRepoRelativePath(
|
||||
params.repoRoot,
|
||||
params.artifactBase,
|
||||
)}\n`,
|
||||
);
|
||||
await writeText(
|
||||
path.join(params.artifactBase, "scorecard.md"),
|
||||
["# UX Matrix", "", ...Object.entries(counts).map(([status, count]) => `- ${status}: ${count}`)]
|
||||
.join("\n")
|
||||
.trimEnd() + "\n",
|
||||
);
|
||||
}
|
||||
|
||||
async function runUxMatrixEvidenceProducer(options: ProducerOptions) {
|
||||
await fs.mkdir(options.artifactBase, { recursive: true });
|
||||
await writePreflight(options.artifactBase);
|
||||
|
||||
const cliLogPath = path.join(
|
||||
cellDir(options.artifactBase, { surface: "cli", stage: "entrypoint-help" }),
|
||||
"logs.txt",
|
||||
);
|
||||
const cliResult = await runCommandForCell({
|
||||
args: ["openclaw.mjs", "--help"],
|
||||
artifactBase: options.artifactBase,
|
||||
command: process.execPath,
|
||||
cwd: options.repoRoot,
|
||||
logPath: cliLogPath,
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
|
||||
const screenshotCellDir = cellDir(options.artifactBase, {
|
||||
surface: "control-ui",
|
||||
stage: "screenshot-artifact",
|
||||
});
|
||||
const matrixHtmlPath = path.join(screenshotCellDir, "matrix-preview.html");
|
||||
await writeText(
|
||||
matrixHtmlPath,
|
||||
'<!doctype html><meta charset="utf-8"><title>UX Matrix</title><h1>UX Matrix</h1><p>Control UI artifact preview fixture generated by the scenario.</p>',
|
||||
);
|
||||
const matrixScreenshotPath = path.join(screenshotCellDir, "screenshot.png");
|
||||
const matrixScreenshotResult = await captureControlUiScreenshot({
|
||||
artifactBase: options.artifactBase,
|
||||
htmlPath: matrixHtmlPath,
|
||||
logPath: path.join(screenshotCellDir, "logs.txt"),
|
||||
repoRoot: options.repoRoot,
|
||||
screenshotPath: matrixScreenshotPath,
|
||||
skipVisualProof: options.skipVisualProof,
|
||||
});
|
||||
|
||||
const initialCells: MatrixCell[] = [
|
||||
{
|
||||
artifacts: [
|
||||
{
|
||||
kind: "log",
|
||||
path: relativeToArtifactBase(
|
||||
options.artifactBase,
|
||||
path.join(screenshotCellDir, "logs.txt"),
|
||||
),
|
||||
},
|
||||
...(matrixScreenshotResult.status === "pass"
|
||||
? [
|
||||
{
|
||||
kind: "screenshot",
|
||||
path: relativeToArtifactBase(options.artifactBase, matrixScreenshotPath),
|
||||
},
|
||||
]
|
||||
: []),
|
||||
],
|
||||
failureReason:
|
||||
"failureReason" in matrixScreenshotResult
|
||||
? matrixScreenshotResult.failureReason
|
||||
: undefined,
|
||||
stage: "screenshot-artifact",
|
||||
status: matrixScreenshotResult.status,
|
||||
surface: "control-ui",
|
||||
title: "UX Matrix: screenshot artifact",
|
||||
wallMs: matrixScreenshotResult.wallMs,
|
||||
},
|
||||
{
|
||||
artifacts: [{ kind: "log", path: relativeToArtifactBase(options.artifactBase, cliLogPath) }],
|
||||
failureReason: cliResult.failureReason,
|
||||
stage: "entrypoint-help",
|
||||
status: cliResult.status,
|
||||
surface: "cli",
|
||||
title: "UX Matrix: CLI entrypoint help",
|
||||
wallMs: cliResult.wallMs,
|
||||
},
|
||||
];
|
||||
|
||||
const fixtureProofDir = cellDir(options.artifactBase, {
|
||||
surface: "qa-lab",
|
||||
stage: "producer-artifact-fixture",
|
||||
});
|
||||
const fixtureHtmlPath = path.join(fixtureProofDir, "producer-artifact-fixture.html");
|
||||
const previewEvidence = buildEvidenceSummary({
|
||||
cells: initialCells,
|
||||
generatedAt: new Date().toISOString(),
|
||||
repoRoot: options.repoRoot,
|
||||
});
|
||||
const screenshotLog = await fs.readFile(path.join(screenshotCellDir, "logs.txt"), "utf8");
|
||||
await writeProducerArtifactFixtureHtml({
|
||||
artifactBase: options.artifactBase,
|
||||
evidence: previewEvidence,
|
||||
htmlPath: fixtureHtmlPath,
|
||||
logPreview: screenshotLog,
|
||||
});
|
||||
|
||||
const fixtureProofResult = await captureProducerArtifactFixtureProof({
|
||||
artifactBase: options.artifactBase,
|
||||
htmlPath: fixtureHtmlPath,
|
||||
logPath: path.join(fixtureProofDir, "logs.txt"),
|
||||
repoRoot: options.repoRoot,
|
||||
screenshotPath: path.join(fixtureProofDir, "producer-artifact-fixture.png"),
|
||||
skipVisualProof: options.skipVisualProof,
|
||||
videoPath: path.join(fixtureProofDir, "producer-artifact-fixture.webm"),
|
||||
});
|
||||
|
||||
const cells: MatrixCell[] = [
|
||||
{
|
||||
artifacts: [
|
||||
{ kind: "html", path: relativeToArtifactBase(options.artifactBase, fixtureHtmlPath) },
|
||||
{
|
||||
kind: "log",
|
||||
path: relativeToArtifactBase(
|
||||
options.artifactBase,
|
||||
path.join(fixtureProofDir, "logs.txt"),
|
||||
),
|
||||
},
|
||||
...(fixtureProofResult.status === "pass"
|
||||
? [
|
||||
{
|
||||
kind: "screenshot",
|
||||
path: relativeToArtifactBase(
|
||||
options.artifactBase,
|
||||
path.join(fixtureProofDir, "producer-artifact-fixture.png"),
|
||||
),
|
||||
},
|
||||
{
|
||||
kind: "video",
|
||||
path: relativeToArtifactBase(
|
||||
options.artifactBase,
|
||||
path.join(fixtureProofDir, "producer-artifact-fixture.webm"),
|
||||
),
|
||||
},
|
||||
]
|
||||
: []),
|
||||
],
|
||||
failureReason:
|
||||
"failureReason" in fixtureProofResult ? fixtureProofResult.failureReason : undefined,
|
||||
stage: "producer-artifact-fixture",
|
||||
status: fixtureProofResult.status,
|
||||
surface: "qa-lab",
|
||||
title: "UX Matrix: producer artifact fixture",
|
||||
wallMs: fixtureProofResult.wallMs,
|
||||
},
|
||||
...initialCells,
|
||||
];
|
||||
|
||||
const evidence = buildEvidenceSummary({
|
||||
cells,
|
||||
generatedAt: new Date().toISOString(),
|
||||
repoRoot: options.repoRoot,
|
||||
});
|
||||
await writeProducerArtifactFixtureHtml({
|
||||
artifactBase: options.artifactBase,
|
||||
evidence,
|
||||
htmlPath: fixtureHtmlPath,
|
||||
logPreview: screenshotLog,
|
||||
});
|
||||
await writeJson(path.join(options.artifactBase, QA_EVIDENCE_FILENAME), evidence);
|
||||
await writeJson(path.join(options.artifactBase, "latest-run.json"), {
|
||||
qaEvidence: QA_EVIDENCE_FILENAME,
|
||||
});
|
||||
await writeProducerMetadata({
|
||||
artifactBase: options.artifactBase,
|
||||
cells,
|
||||
repoRoot: options.repoRoot,
|
||||
});
|
||||
return {
|
||||
artifactBase: options.artifactBase,
|
||||
evidence,
|
||||
};
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
process.exitCode = await runUxMatrixEvidenceProducerCli(process.argv.slice(2));
|
||||
}
|
||||
@@ -1,230 +0,0 @@
|
||||
// QA UX Matrix evidence producer tests cover operator-facing CLI behavior.
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
ensureUxMatrixVideoDependencies,
|
||||
launchUxMatrixChromium,
|
||||
runUxMatrixEvidenceProducerCli,
|
||||
} from "../../scripts/qa/ux-matrix-evidence-producer.js";
|
||||
|
||||
async function runCli(...args: string[]) {
|
||||
const stdout: string[] = [];
|
||||
const stderr: string[] = [];
|
||||
const status = await runUxMatrixEvidenceProducerCli(args, {
|
||||
error: (message) => stderr.push(message),
|
||||
log: (message) => stdout.push(message),
|
||||
});
|
||||
return {
|
||||
status,
|
||||
stderr: stderr.length > 0 ? `${stderr.join("\n")}\n` : "",
|
||||
stdout: stdout.length > 0 ? `${stdout.join("\n")}\n` : "",
|
||||
};
|
||||
}
|
||||
|
||||
function expectNoNodeStack(stderr: string) {
|
||||
expect(stderr).not.toContain("Node.js");
|
||||
expect(stderr).not.toContain("\n at ");
|
||||
}
|
||||
|
||||
describe("QA UX Matrix evidence producer CLI", () => {
|
||||
it("prints help without generating evidence", async () => {
|
||||
const result = await runCli("--help");
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain(
|
||||
"Usage: node --import tsx scripts/qa/ux-matrix-evidence-producer.ts",
|
||||
);
|
||||
expect(result.stderr).toBe("");
|
||||
});
|
||||
|
||||
it("prints help after boolean options without consuming valued option slots", async () => {
|
||||
const result = await runCli("--skip-visual-proof", "--help");
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain(
|
||||
"Usage: node --import tsx scripts/qa/ux-matrix-evidence-producer.ts",
|
||||
);
|
||||
expect(result.stderr).toBe("");
|
||||
});
|
||||
|
||||
it("reports invalid args without a Node stack trace", async () => {
|
||||
const result = await runCli("--wat");
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stdout).toBe("");
|
||||
expect(result.stderr.trim()).toBe("unsupported UX Matrix producer arg: --wat");
|
||||
expectNoNodeStack(result.stderr);
|
||||
});
|
||||
|
||||
it("reports missing valued args without a Node stack trace", async () => {
|
||||
const result = await runCli("--artifact-base", "--repo-root", ".");
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stdout).toBe("");
|
||||
expect(result.stderr.trim()).toBe("--artifact-base requires a value");
|
||||
expectNoNodeStack(result.stderr);
|
||||
});
|
||||
|
||||
it("reports duplicate evidence producer args without a Node stack trace", async () => {
|
||||
const duplicateCases = [
|
||||
["--artifact-base", ["--artifact-base", ".artifacts/a", "--artifact-base", ".artifacts/b"]],
|
||||
["--repo-root", ["--artifact-base", ".artifacts/a", "--repo-root", ".", "--repo-root", ".."]],
|
||||
[
|
||||
"--skip-visual-proof",
|
||||
["--artifact-base", ".artifacts/a", "--skip-visual-proof", "--skip-visual-proof"],
|
||||
],
|
||||
] satisfies Array<[string, string[]]>;
|
||||
|
||||
for (const [flag, args] of duplicateCases) {
|
||||
const result = await runCli(...args);
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stdout).toBe("");
|
||||
expect(result.stderr.trim()).toBe(`${flag} was provided more than once`);
|
||||
expectNoNodeStack(result.stderr);
|
||||
}
|
||||
});
|
||||
|
||||
it("reports short flag values without treating them as help", async () => {
|
||||
const artifactBaseResult = await runCli("--artifact-base", "-h");
|
||||
const repoRootResult = await runCli(
|
||||
"--artifact-base",
|
||||
"/tmp/openclaw-ux-test",
|
||||
"--repo-root",
|
||||
"-h",
|
||||
);
|
||||
|
||||
expect(artifactBaseResult.status).toBe(1);
|
||||
expect(artifactBaseResult.stdout).toBe("");
|
||||
expect(artifactBaseResult.stderr.trim()).toBe("--artifact-base requires a value");
|
||||
expectNoNodeStack(artifactBaseResult.stderr);
|
||||
expect(repoRootResult.status).toBe(1);
|
||||
expect(repoRootResult.stdout).toBe("");
|
||||
expect(repoRootResult.stderr.trim()).toBe("--repo-root requires a value");
|
||||
expectNoNodeStack(repoRootResult.stderr);
|
||||
});
|
||||
|
||||
it("sanitizes local checkout paths from generated evidence artifacts", async () => {
|
||||
const artifactBase = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-ux-evidence-test-"));
|
||||
const fakeRepoRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-ux-repo-test-"));
|
||||
try {
|
||||
const result = await runCli(
|
||||
"--artifact-base",
|
||||
artifactBase,
|
||||
"--repo-root",
|
||||
fakeRepoRoot,
|
||||
"--skip-visual-proof",
|
||||
);
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
const evidence = fs.readFileSync(path.join(artifactBase, "qa-evidence.json"), "utf8");
|
||||
const cliLog = fs.readFileSync(
|
||||
path.join(artifactBase, "surfaces", "cli", "stages", "entrypoint-help", "logs.txt"),
|
||||
"utf8",
|
||||
);
|
||||
const visualLog = fs.readFileSync(
|
||||
path.join(
|
||||
artifactBase,
|
||||
"surfaces",
|
||||
"control-ui",
|
||||
"stages",
|
||||
"screenshot-artifact",
|
||||
"logs.txt",
|
||||
),
|
||||
"utf8",
|
||||
);
|
||||
expect(evidence).not.toContain(fakeRepoRoot);
|
||||
expect(cliLog).not.toContain(fakeRepoRoot);
|
||||
expect(`${evidence}\n${cliLog}`).toContain("<repo-root>");
|
||||
expect(visualLog).toBe("blocked: --skip-visual-proof was set\n");
|
||||
|
||||
const evidenceJson = JSON.parse(evidence) as {
|
||||
entries: Array<{
|
||||
coverage: unknown[];
|
||||
execution?: { artifacts: Array<{ kind: string }> };
|
||||
test: { id: string };
|
||||
}>;
|
||||
};
|
||||
const matrix = JSON.parse(
|
||||
fs.readFileSync(path.join(artifactBase, "matrix.json"), "utf8"),
|
||||
) as { cells: Array<{ coverageIds: unknown[] }> };
|
||||
const releaseLedger = JSON.parse(
|
||||
fs.readFileSync(path.join(artifactBase, "release-ledger.json"), "utf8"),
|
||||
) as { entries: Array<{ coverageIds: unknown[] }> };
|
||||
const manifest = JSON.parse(
|
||||
fs.readFileSync(path.join(artifactBase, "manifest.json"), "utf8"),
|
||||
) as { run: Record<string, unknown> };
|
||||
const commands = fs.readFileSync(path.join(artifactBase, "commands.txt"), "utf8");
|
||||
|
||||
expect(evidenceJson.entries.map((entry) => entry.test.id)).toEqual([
|
||||
"ux-matrix.qa-lab.producer-artifact-fixture",
|
||||
"ux-matrix.control-ui.screenshot-artifact",
|
||||
"ux-matrix.cli.entrypoint-help",
|
||||
]);
|
||||
expect(evidenceJson.entries.every((entry) => entry.coverage.length === 0)).toBe(true);
|
||||
expect(matrix.cells.every((cell) => cell.coverageIds.length === 0)).toBe(true);
|
||||
expect(releaseLedger.entries.every((entry) => entry.coverageIds.length === 0)).toBe(true);
|
||||
expect(manifest.run).not.toHaveProperty("scenarioId");
|
||||
expect(commands).toContain(
|
||||
"node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base",
|
||||
);
|
||||
expect(commands).not.toContain("qa suite --scenario");
|
||||
expect(
|
||||
evidenceJson.entries.flatMap(
|
||||
(entry) => entry.execution?.artifacts.map((artifact) => artifact.kind) ?? [],
|
||||
),
|
||||
).toEqual(expect.arrayContaining(["html", "log"]));
|
||||
} finally {
|
||||
fs.rmSync(artifactBase, { recursive: true, force: true });
|
||||
fs.rmSync(fakeRepoRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("falls back to system Chromium when the managed Playwright browser is missing", async () => {
|
||||
const browser = { close: vi.fn() };
|
||||
const launch = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(
|
||||
new Error(
|
||||
[
|
||||
"browserType.launch: Executable doesn't exist at /home/user/.cache/ms-playwright/chromium_headless_shell-1223/chrome-headless-shell-linux64/chrome-headless-shell",
|
||||
"Please run the following command to download new browsers:",
|
||||
"pnpm exec playwright install",
|
||||
].join("\n"),
|
||||
),
|
||||
)
|
||||
.mockResolvedValueOnce(browser);
|
||||
|
||||
const result = await launchUxMatrixChromium({
|
||||
chromium: { launch } as unknown as NonNullable<
|
||||
Parameters<typeof launchUxMatrixChromium>[0]
|
||||
>["chromium"],
|
||||
systemExecutablePath: "/usr/bin/chromium-browser",
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
browser,
|
||||
usedSystemExecutablePath: "/usr/bin/chromium-browser",
|
||||
});
|
||||
expect(launch).toHaveBeenNthCalledWith(1);
|
||||
expect(launch).toHaveBeenNthCalledWith(2, {
|
||||
executablePath: "/usr/bin/chromium-browser",
|
||||
});
|
||||
});
|
||||
|
||||
it("ensures Playwright ffmpeg when video proof uses system Chromium", () => {
|
||||
const ensureChromium = vi.fn(() => 0);
|
||||
|
||||
ensureUxMatrixVideoDependencies({
|
||||
ensureChromium,
|
||||
usedSystemExecutablePath: "/usr/bin/chromium-browser",
|
||||
});
|
||||
|
||||
expect(ensureChromium).toHaveBeenCalledWith({
|
||||
ensureFfmpeg: true,
|
||||
systemExecutablePath: "/usr/bin/chromium-browser",
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user