mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
test: add QA Lab UX Matrix evidence scenario (#94306)
* test: add qa lab ux matrix script scenario * fix(qa-lab): annotate UX Matrix producer catch callback as unknown for oxlint --------- Co-authored-by: Dallin Romney <dallinromney@gmail.com>
This commit is contained in:
Colin Johnson
@@ -60,6 +60,7 @@ describe("qa scenario catalog", () => {
|
||||
"package-openclaw-for-docker",
|
||||
"plugin-lifecycle-probe",
|
||||
"qa-otel-smoke",
|
||||
"ux-matrix-evidence-dashboard",
|
||||
].toSorted(),
|
||||
);
|
||||
expect(
|
||||
@@ -140,6 +141,7 @@ describe("qa scenario catalog", () => {
|
||||
|
||||
it("loads native test execution scenarios from YAML", () => {
|
||||
const scenario = readQaScenarioById("control-ui-chat-flow-playwright");
|
||||
const uxMatrix = readQaScenarioById("ux-matrix-evidence-dashboard");
|
||||
|
||||
expect(scenario.execution.kind).toBe("playwright");
|
||||
if (scenario.execution.kind !== "playwright") {
|
||||
@@ -148,6 +150,14 @@ describe("qa scenario catalog", () => {
|
||||
expect(scenario.execution.path).toBe("ui/src/ui/e2e/chat-flow.e2e.test.ts");
|
||||
expect(scenario.execution.flow).toBeUndefined();
|
||||
expect(scenario.coverage?.primary).toContain("ui.control");
|
||||
expect(uxMatrix.execution.kind).toBe("script");
|
||||
if (uxMatrix.execution.kind !== "script") {
|
||||
throw new Error(`expected script scenario, got ${uxMatrix.execution.kind}`);
|
||||
}
|
||||
expect(uxMatrix.execution.path).toBe("scripts/qa/ux-matrix-evidence-producer.ts");
|
||||
expect(uxMatrix.execution.args).toStrictEqual(["--artifact-base", "${outputDir}"]);
|
||||
expect(uxMatrix.execution.config).toBeUndefined();
|
||||
expect(uxMatrix.coverage?.primary).toContain("qa.artifact-safety");
|
||||
});
|
||||
|
||||
it("loads runtime parity tier metadata for first-hour and soak lanes", () => {
|
||||
|
||||
@@ -3,7 +3,7 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { validateQaEvidenceSummaryJson } from "./evidence-summary.js";
|
||||
import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
|
||||
import { readQaScenarioById, type QaSeedScenarioWithSource } from "./scenario-catalog.js";
|
||||
import {
|
||||
runQaTestFileScenarios,
|
||||
type QaScenarioCommandExecution,
|
||||
@@ -689,4 +689,65 @@ describe("qa test file scenario runner", () => {
|
||||
expect(artifactPath).toBe(path.normalize(externalArtifact));
|
||||
expect(artifactPath?.includes("..")).toBe(false);
|
||||
});
|
||||
|
||||
it("runs the UX Matrix script producer and imports its evidence bundle", async () => {
|
||||
const repoRoot = process.cwd();
|
||||
const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-ux-matrix-script-"));
|
||||
tempRoots.push(outputDir);
|
||||
const scenario = readQaScenarioById("ux-matrix-evidence-dashboard");
|
||||
|
||||
expect(scenario.execution.kind).toBe("script");
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
scenarios: [scenario],
|
||||
env: {
|
||||
OPENCLAW_QA_REF: "scenario-ref",
|
||||
} as NodeJS.ProcessEnv,
|
||||
});
|
||||
|
||||
expect(result.executionKind).toBe("script");
|
||||
expect(result.results[0]?.producerEvidence?.entries).toHaveLength(3);
|
||||
const evidence = validateQaEvidenceSummaryJson(
|
||||
JSON.parse(await fs.readFile(result.evidencePath, "utf8")),
|
||||
);
|
||||
expect(evidence.entries.map((entry) => entry.test.id)).toEqual([
|
||||
"ux-matrix.qa-lab.producer-artifact-fixture",
|
||||
"ux-matrix.control-ui.screenshot-artifact",
|
||||
"ux-matrix.cli.entrypoint-help",
|
||||
]);
|
||||
expect(
|
||||
evidence.entries.flatMap((entry) => entry.coverage.map((coverage) => coverage.id)),
|
||||
).toEqual(
|
||||
expect.arrayContaining([
|
||||
"qa.artifact-safety",
|
||||
"tools.evidence",
|
||||
"workspace.artifacts",
|
||||
"ui.control",
|
||||
"control-ui",
|
||||
"cli-entrypoint",
|
||||
"status-snapshots",
|
||||
]),
|
||||
);
|
||||
const artifactKinds = evidence.entries.flatMap(
|
||||
(entry) => entry.execution?.artifacts.map((artifact) => artifact.kind) ?? [],
|
||||
);
|
||||
expect(artifactKinds).toEqual(expect.arrayContaining(["html", "log"]));
|
||||
const fixtureEntry = evidence.entries.find(
|
||||
(entry) => entry.test.id === "ux-matrix.qa-lab.producer-artifact-fixture",
|
||||
);
|
||||
expect(fixtureEntry?.execution?.artifacts.map((artifact) => artifact.path)).toContain(
|
||||
path.join(
|
||||
outputDir,
|
||||
"ux-matrix-evidence-dashboard",
|
||||
"surfaces",
|
||||
"qa-lab",
|
||||
"stages",
|
||||
"producer-artifact-fixture",
|
||||
"producer-artifact-fixture.html",
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
title: UX Matrix evidence dashboard
|
||||
|
||||
scenario:
|
||||
id: ux-matrix-evidence-dashboard
|
||||
surface: control-ui
|
||||
category: browser-control-ui-and-webchat.browser-ui
|
||||
coverage:
|
||||
primary:
|
||||
- qa.artifact-safety
|
||||
secondary:
|
||||
- ui.control
|
||||
- control-ui
|
||||
- cli-entrypoint
|
||||
- status-snapshots
|
||||
- tools.evidence
|
||||
- workspace.artifacts
|
||||
objective: Produce UX Matrix evidence artifacts through the QA Lab script producer contract.
|
||||
successCriteria:
|
||||
- The script producer writes manifest.json, matrix.json, release-ledger.json, scorecard.md, latest-run.json, and qa-evidence.json.
|
||||
- Evidence entries declare coverage IDs and artifact paths for generated logs, screenshot proof, and video-style proof.
|
||||
- Missing local browser or CLI prerequisites are represented as blocked evidence with logs instead of fake artifacts.
|
||||
docsRefs:
|
||||
- docs/concepts/qa-e2e-automation.md
|
||||
- docs/web/control-ui.md
|
||||
codeRefs:
|
||||
- scripts/qa/ux-matrix-evidence-producer.ts
|
||||
- extensions/qa-lab/src/test-file-scenario-runner.ts
|
||||
execution:
|
||||
kind: script
|
||||
path: scripts/qa/ux-matrix-evidence-producer.ts
|
||||
summary: Script-backed UX Matrix producer that emits QA Lab evidence artifacts.
|
||||
args:
|
||||
- --artifact-base
|
||||
- ${outputDir}
|
||||
@@ -0,0 +1,637 @@
|
||||
// Produces a QA Lab UX Matrix evidence bundle through the script scenario contract.
|
||||
import { execFile } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { toRepoRelativePath } from "../../extensions/qa-lab/src/cli-paths.js";
|
||||
import {
|
||||
QA_EVIDENCE_FILENAME,
|
||||
QA_EVIDENCE_SUMMARY_KIND,
|
||||
QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
|
||||
validateQaEvidenceSummaryJson,
|
||||
type QaEvidenceStatus,
|
||||
type QaEvidenceSummaryEntry,
|
||||
type QaEvidenceSummaryJson,
|
||||
} from "../../extensions/qa-lab/src/evidence-summary.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const SCENARIO_ID = "ux-matrix-evidence-dashboard";
|
||||
const SOURCE_PATH = "scripts/qa/ux-matrix-evidence-producer.ts";
|
||||
const SUITE_COMMAND = `pnpm openclaw qa suite --scenario ${SCENARIO_ID}`;
|
||||
|
||||
type MatrixCell = {
|
||||
artifacts: Array<{ kind: string; path: string }>;
|
||||
coverageIds: string[];
|
||||
failureReason?: string;
|
||||
stage: string;
|
||||
status: QaEvidenceStatus;
|
||||
surface: string;
|
||||
title: string;
|
||||
wallMs: number;
|
||||
};
|
||||
|
||||
export type ProducerOptions = {
|
||||
artifactBase: string;
|
||||
repoRoot: string;
|
||||
skipVisualProof: boolean;
|
||||
};
|
||||
|
||||
function parseOptions(argv: readonly string[]): ProducerOptions {
|
||||
let artifactBase = "";
|
||||
let repoRoot = process.cwd();
|
||||
let skipVisualProof = false;
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--artifact-base") {
|
||||
artifactBase = argv[index + 1] ?? "";
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--repo-root") {
|
||||
repoRoot = argv[index + 1] ?? "";
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--skip-visual-proof") {
|
||||
skipVisualProof = true;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`unsupported UX Matrix producer arg: ${arg}`);
|
||||
}
|
||||
if (!artifactBase.trim()) {
|
||||
throw new Error("--artifact-base is required");
|
||||
}
|
||||
if (!repoRoot.trim()) {
|
||||
throw new Error("--repo-root must not be empty");
|
||||
}
|
||||
return {
|
||||
artifactBase: path.resolve(repoRoot, artifactBase),
|
||||
repoRoot: path.resolve(repoRoot),
|
||||
skipVisualProof,
|
||||
};
|
||||
}
|
||||
|
||||
async function writeJson(filePath: string, value: unknown) {
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
||||
await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
async function writeText(filePath: string, value: string) {
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
||||
await fs.writeFile(filePath, value, "utf8");
|
||||
}
|
||||
|
||||
function cellDir(artifactBase: string, cell: Pick<MatrixCell, "stage" | "surface">) {
|
||||
return path.join(artifactBase, "surfaces", cell.surface, "stages", cell.stage);
|
||||
}
|
||||
|
||||
function relativeToArtifactBase(artifactBase: string, filePath: string) {
|
||||
return path.relative(artifactBase, filePath).split(path.sep).join("/");
|
||||
}
|
||||
|
||||
function buildExecution(params: {
|
||||
artifacts: MatrixCell["artifacts"];
|
||||
source: string;
|
||||
}): QaEvidenceSummaryEntry["execution"] {
|
||||
return {
|
||||
runner: "ux-matrix-script-producer",
|
||||
environment: {
|
||||
ref: process.env.OPENCLAW_QA_REF?.trim() || process.env.GITHUB_SHA?.trim() || null,
|
||||
os: process.platform,
|
||||
nodeVersion: process.version,
|
||||
},
|
||||
provider: {
|
||||
id: "ux-matrix",
|
||||
live: false,
|
||||
model: {
|
||||
name: null,
|
||||
ref: null,
|
||||
},
|
||||
fixture: "local-qa-lab-script-producer",
|
||||
},
|
||||
packageSource: {
|
||||
kind: "source-checkout",
|
||||
},
|
||||
artifacts: params.artifacts.map((artifact) => ({
|
||||
...artifact,
|
||||
source: params.source,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function buildEvidenceEntry(cell: MatrixCell): QaEvidenceSummaryEntry {
|
||||
const source = `ux-matrix:${cell.surface}:${cell.stage}`;
|
||||
return {
|
||||
test: {
|
||||
kind: "ux-matrix-cell",
|
||||
id: `ux-matrix.${cell.surface}.${cell.stage}`,
|
||||
title: cell.title,
|
||||
source: { path: SOURCE_PATH },
|
||||
},
|
||||
coverage: cell.coverageIds.map((id, index) => ({
|
||||
id,
|
||||
role: index === 0 ? "primary" : "secondary",
|
||||
})),
|
||||
refs: [
|
||||
{ kind: "code", path: SOURCE_PATH },
|
||||
{ kind: "docs", path: "docs/concepts/qa-e2e-automation.md" },
|
||||
],
|
||||
execution: buildExecution({
|
||||
artifacts: cell.artifacts,
|
||||
source,
|
||||
}),
|
||||
result: {
|
||||
status: cell.status,
|
||||
...(cell.status === "pass"
|
||||
? {}
|
||||
: {
|
||||
failure: {
|
||||
class: cell.status,
|
||||
reason: cell.failureReason ?? `${cell.status} UX Matrix cell`,
|
||||
},
|
||||
}),
|
||||
timing: {
|
||||
wallMs: Math.max(1, cell.wallMs),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildEvidenceSummary(params: {
|
||||
cells: readonly MatrixCell[];
|
||||
generatedAt: string;
|
||||
}): QaEvidenceSummaryJson {
|
||||
return validateQaEvidenceSummaryJson({
|
||||
kind: QA_EVIDENCE_SUMMARY_KIND,
|
||||
schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
|
||||
generatedAt: params.generatedAt,
|
||||
evidenceMode: "full",
|
||||
entries: params.cells.map(buildEvidenceEntry),
|
||||
});
|
||||
}
|
||||
|
||||
async function runCommandForCell(params: {
|
||||
args: string[];
|
||||
command: string;
|
||||
cwd: string;
|
||||
logPath: string;
|
||||
timeoutMs: number;
|
||||
}) {
|
||||
const startedAt = Date.now();
|
||||
const commandLine = [params.command, ...params.args].join(" ");
|
||||
try {
|
||||
const { stdout, stderr } = await execFileAsync(params.command, params.args, {
|
||||
cwd: params.cwd,
|
||||
timeout: params.timeoutMs,
|
||||
env: process.env,
|
||||
maxBuffer: 1024 * 1024,
|
||||
});
|
||||
await writeText(params.logPath, `$ ${commandLine}\n${stdout}${stderr}`);
|
||||
return {
|
||||
status: "pass" as const,
|
||||
wallMs: Date.now() - startedAt,
|
||||
};
|
||||
} catch (error) {
|
||||
const details = error instanceof Error ? error.message : String(error);
|
||||
await writeText(params.logPath, `$ ${commandLine}\nblocked: ${details}\n`);
|
||||
return {
|
||||
failureReason: details,
|
||||
status: "blocked" as const,
|
||||
wallMs: Date.now() - startedAt,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function writePreflight(artifactBase: string) {
|
||||
await writeText(
|
||||
path.join(artifactBase, "preflight", "runtime.txt"),
|
||||
[
|
||||
`platform=${process.platform}`,
|
||||
`arch=${process.arch}`,
|
||||
`node=${process.version}`,
|
||||
`freeMemoryBytes=${os.freemem()}`,
|
||||
`totalMemoryBytes=${os.totalmem()}`,
|
||||
].join("\n") + "\n",
|
||||
);
|
||||
}
|
||||
|
||||
async function captureControlUiScreenshot(params: {
|
||||
artifactBase: string;
|
||||
htmlPath: string;
|
||||
logPath: string;
|
||||
screenshotPath: string;
|
||||
}) {
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
const { chromium } = await import("playwright");
|
||||
const browser = await chromium.launch();
|
||||
try {
|
||||
const page = await browser.newPage({ viewport: { width: 1024, height: 720 } });
|
||||
await page.goto(pathToFileURL(params.htmlPath).href);
|
||||
await page.screenshot({ path: params.screenshotPath, fullPage: true });
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
await writeText(
|
||||
params.logPath,
|
||||
`Captured ${relativeToArtifactBase(params.artifactBase, params.screenshotPath)}\n`,
|
||||
);
|
||||
return {
|
||||
status: "pass" as const,
|
||||
wallMs: Date.now() - startedAt,
|
||||
};
|
||||
} catch (error) {
|
||||
const details = error instanceof Error ? error.message : String(error);
|
||||
await writeText(params.logPath, `blocked: ${details}\n`);
|
||||
return {
|
||||
failureReason: details,
|
||||
status: "blocked" as const,
|
||||
wallMs: Date.now() - startedAt,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function escapeHtml(value: string) {
|
||||
return value
|
||||
.replaceAll("&", "&")
|
||||
.replaceAll("<", "<")
|
||||
.replaceAll(">", ">")
|
||||
.replaceAll('"', """);
|
||||
}
|
||||
|
||||
async function writeProducerArtifactFixtureHtml(params: {
|
||||
artifactBase: string;
|
||||
evidence: QaEvidenceSummaryJson;
|
||||
htmlPath: string;
|
||||
logPreview: string;
|
||||
}) {
|
||||
const previewArtifacts = params.evidence.entries
|
||||
.filter((entry) => entry.test.id !== "ux-matrix.qa-lab.producer-artifact-fixture")
|
||||
.flatMap((entry) => entry.execution?.artifacts ?? []);
|
||||
const screenshotArtifact = previewArtifacts.find((artifact) => artifact.kind === "screenshot");
|
||||
const logArtifact = previewArtifacts.find((artifact) => artifact.kind === "log");
|
||||
const screenshotPath = screenshotArtifact
|
||||
? relativeToArtifactBase(
|
||||
path.dirname(params.htmlPath),
|
||||
path.join(params.artifactBase, screenshotArtifact.path),
|
||||
)
|
||||
: "";
|
||||
const entryRows = params.evidence.entries
|
||||
.map(
|
||||
(entry) =>
|
||||
`<li><strong>${escapeHtml(entry.test.id)}</strong> - ${escapeHtml(
|
||||
entry.result.status,
|
||||
)} - ${entry.coverage.map((coverage) => escapeHtml(coverage.id)).join(", ")}</li>`,
|
||||
)
|
||||
.join("");
|
||||
await writeText(
|
||||
params.htmlPath,
|
||||
`<!doctype html>
|
||||
<meta charset="utf-8">
|
||||
<title>UX Matrix producer artifact fixture</title>
|
||||
<style>
|
||||
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; margin: 0; background: #f6f7f9; color: #121417; }
|
||||
main { max-width: 980px; margin: 0 auto; padding: 28px; }
|
||||
h1 { font-size: 30px; margin: 0 0 8px; }
|
||||
.panel { background: white; border: 1px solid #d9dee7; border-radius: 8px; padding: 18px; margin-top: 18px; }
|
||||
.toolbar { display: flex; gap: 10px; flex-wrap: wrap; margin-top: 12px; }
|
||||
button { border: 1px solid #8c98aa; background: #fff; border-radius: 6px; padding: 8px 12px; font: inherit; cursor: pointer; }
|
||||
button:focus { outline: 3px solid #91c5ff; }
|
||||
img { max-width: 100%; border: 1px solid #ccd3dd; border-radius: 6px; background: #fff; }
|
||||
pre { white-space: pre-wrap; background: #111827; color: #f9fafb; padding: 14px; border-radius: 6px; overflow: auto; }
|
||||
.meta { color: #526070; }
|
||||
</style>
|
||||
<main>
|
||||
<h1>UX Matrix Producer Artifact Fixture</h1>
|
||||
<p class="meta">Script-produced ${escapeHtml(QA_EVIDENCE_FILENAME)} for ${escapeHtml(
|
||||
SCENARIO_ID,
|
||||
)}; this fixture is not the QA Lab Evidence Archive UI.</p>
|
||||
<section class="panel">
|
||||
<h2>UX Matrix entries</h2>
|
||||
<ul>${entryRows}</ul>
|
||||
</section>
|
||||
<section class="panel">
|
||||
<h2>Artifact preview</h2>
|
||||
<p class="meta">${escapeHtml(logArtifact?.path ?? "no log")} - ${escapeHtml(
|
||||
screenshotArtifact?.path ?? "no screenshot",
|
||||
)}</p>
|
||||
<div class="toolbar">
|
||||
<button id="preview-screenshot" type="button">Preview screenshot artifact</button>
|
||||
<button id="preview-log" type="button">Preview log artifact</button>
|
||||
</div>
|
||||
<div id="preview" class="panel" aria-live="polite">Choose an artifact to preview.</div>
|
||||
</section>
|
||||
</main>
|
||||
<script>
|
||||
const preview = document.querySelector("#preview");
|
||||
document.querySelector("#preview-screenshot").addEventListener("click", () => {
|
||||
preview.innerHTML = ${JSON.stringify(
|
||||
screenshotPath
|
||||
? `<img alt="UX Matrix screenshot artifact" src="${escapeHtml(screenshotPath)}">`
|
||||
: "<p>No screenshot artifact was produced.</p>",
|
||||
)};
|
||||
});
|
||||
document.querySelector("#preview-log").addEventListener("click", () => {
|
||||
preview.innerHTML = ${JSON.stringify(`<pre>${escapeHtml(params.logPreview)}</pre>`)};
|
||||
});
|
||||
</script>
|
||||
`,
|
||||
);
|
||||
}
|
||||
|
||||
async function captureProducerArtifactFixtureProof(params: {
|
||||
htmlPath: string;
|
||||
logPath: string;
|
||||
screenshotPath: string;
|
||||
skipVisualProof: boolean;
|
||||
videoPath: string;
|
||||
}) {
|
||||
const startedAt = Date.now();
|
||||
if (params.skipVisualProof) {
|
||||
await writeText(params.logPath, "blocked: --skip-visual-proof was set\n");
|
||||
return {
|
||||
failureReason: "--skip-visual-proof was set",
|
||||
status: "blocked" as const,
|
||||
wallMs: Date.now() - startedAt,
|
||||
};
|
||||
}
|
||||
try {
|
||||
const { chromium } = await import("playwright");
|
||||
const videoDir = path.join(path.dirname(params.videoPath), "recording");
|
||||
await fs.mkdir(videoDir, { recursive: true });
|
||||
const browser = await chromium.launch();
|
||||
let recordedVideo: string | undefined;
|
||||
try {
|
||||
const context = await browser.newContext({
|
||||
viewport: { width: 1280, height: 820 },
|
||||
recordVideo: {
|
||||
dir: videoDir,
|
||||
size: { width: 1280, height: 820 },
|
||||
},
|
||||
});
|
||||
const page = await context.newPage();
|
||||
const video = page.video();
|
||||
await page.goto(pathToFileURL(params.htmlPath).href);
|
||||
await page.locator("#preview-screenshot").click();
|
||||
await page.locator("#preview img").waitFor({ state: "visible", timeout: 5_000 });
|
||||
await page.screenshot({ path: params.screenshotPath, fullPage: true });
|
||||
await page.waitForTimeout(350);
|
||||
await page.locator("#preview-log").click();
|
||||
await page.waitForTimeout(350);
|
||||
await context.close();
|
||||
recordedVideo = await video?.path();
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
if (!recordedVideo) {
|
||||
throw new Error("Playwright did not provide a recording path");
|
||||
}
|
||||
await fs.mkdir(path.dirname(params.videoPath), { recursive: true });
|
||||
await fs.copyFile(recordedVideo, params.videoPath);
|
||||
await writeText(
|
||||
params.logPath,
|
||||
[
|
||||
`Captured screenshot ${path.basename(params.screenshotPath)}`,
|
||||
`Captured recording ${path.basename(params.videoPath)}`,
|
||||
].join("\n") + "\n",
|
||||
);
|
||||
return {
|
||||
status: "pass" as const,
|
||||
wallMs: Date.now() - startedAt,
|
||||
};
|
||||
} catch (error) {
|
||||
const details = error instanceof Error ? error.message : String(error);
|
||||
await writeText(params.logPath, `blocked: ${details}\n`);
|
||||
return {
|
||||
failureReason: details,
|
||||
status: "blocked" as const,
|
||||
wallMs: Date.now() - startedAt,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function writeProducerMetadata(params: {
|
||||
artifactBase: string;
|
||||
cells: readonly MatrixCell[];
|
||||
repoRoot: string;
|
||||
}) {
|
||||
const counts = params.cells.reduce<Record<string, number>>((acc, cell) => {
|
||||
acc[cell.status] = (acc[cell.status] ?? 0) + 1;
|
||||
return acc;
|
||||
}, {});
|
||||
await writeJson(path.join(params.artifactBase, "manifest.json"), {
|
||||
kind: "openclaw.qa.ux-matrix",
|
||||
run: {
|
||||
scenarioId: SCENARIO_ID,
|
||||
status: counts.fail ? "fail" : counts.blocked ? "blocked" : "pass",
|
||||
},
|
||||
});
|
||||
await writeJson(path.join(params.artifactBase, "matrix.json"), {
|
||||
cells: params.cells.map((cell) => ({
|
||||
coverageIds: cell.coverageIds,
|
||||
stage: cell.stage,
|
||||
status: cell.status,
|
||||
surface: cell.surface,
|
||||
})),
|
||||
counts,
|
||||
});
|
||||
await writeJson(path.join(params.artifactBase, "release-ledger.json"), {
|
||||
entries: params.cells.map((cell) => ({
|
||||
coverageIds: cell.coverageIds,
|
||||
stage: cell.stage,
|
||||
status: cell.status,
|
||||
surface: cell.surface,
|
||||
})),
|
||||
kind: "openclaw.qa.ux-matrix.release-ledger",
|
||||
});
|
||||
await writeText(
|
||||
path.join(params.artifactBase, "commands.txt"),
|
||||
[
|
||||
`${SUITE_COMMAND} --output-dir ${toRepoRelativePath(params.repoRoot, params.artifactBase)}`,
|
||||
`node --import tsx ${SOURCE_PATH} --artifact-base ${toRepoRelativePath(
|
||||
params.repoRoot,
|
||||
params.artifactBase,
|
||||
)}`,
|
||||
].join("\n") + "\n",
|
||||
);
|
||||
await writeText(
|
||||
path.join(params.artifactBase, "scorecard.md"),
|
||||
["# UX Matrix", "", ...Object.entries(counts).map(([status, count]) => `- ${status}: ${count}`)]
|
||||
.join("\n")
|
||||
.trimEnd() + "\n",
|
||||
);
|
||||
}
|
||||
|
||||
export async function runUxMatrixEvidenceProducer(options: ProducerOptions) {
|
||||
await fs.mkdir(options.artifactBase, { recursive: true });
|
||||
await writePreflight(options.artifactBase);
|
||||
|
||||
const cliLogPath = path.join(
|
||||
cellDir(options.artifactBase, { surface: "cli", stage: "entrypoint-help" }),
|
||||
"logs.txt",
|
||||
);
|
||||
const cliResult = await runCommandForCell({
|
||||
command: process.execPath,
|
||||
args: ["openclaw.mjs", "--help"],
|
||||
cwd: options.repoRoot,
|
||||
logPath: cliLogPath,
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
|
||||
const screenshotCellDir = cellDir(options.artifactBase, {
|
||||
surface: "control-ui",
|
||||
stage: "screenshot-artifact",
|
||||
});
|
||||
const matrixHtmlPath = path.join(screenshotCellDir, "matrix-preview.html");
|
||||
await writeText(
|
||||
matrixHtmlPath,
|
||||
'<!doctype html><meta charset="utf-8"><title>UX Matrix</title><h1>UX Matrix</h1><p>Control UI artifact preview fixture generated by the scenario.</p>',
|
||||
);
|
||||
const matrixScreenshotPath = path.join(screenshotCellDir, "screenshot.png");
|
||||
const matrixScreenshotResult = await captureControlUiScreenshot({
|
||||
artifactBase: options.artifactBase,
|
||||
htmlPath: matrixHtmlPath,
|
||||
logPath: path.join(screenshotCellDir, "logs.txt"),
|
||||
screenshotPath: matrixScreenshotPath,
|
||||
});
|
||||
|
||||
const initialCells: MatrixCell[] = [
|
||||
{
|
||||
artifacts: [
|
||||
{
|
||||
kind: "log",
|
||||
path: relativeToArtifactBase(
|
||||
options.artifactBase,
|
||||
path.join(screenshotCellDir, "logs.txt"),
|
||||
),
|
||||
},
|
||||
...(matrixScreenshotResult.status === "pass"
|
||||
? [
|
||||
{
|
||||
kind: "screenshot",
|
||||
path: relativeToArtifactBase(options.artifactBase, matrixScreenshotPath),
|
||||
},
|
||||
]
|
||||
: []),
|
||||
],
|
||||
coverageIds: ["ui.control", "control-ui"],
|
||||
failureReason: matrixScreenshotResult.failureReason,
|
||||
stage: "screenshot-artifact",
|
||||
status: matrixScreenshotResult.status,
|
||||
surface: "control-ui",
|
||||
title: "UX Matrix: screenshot artifact",
|
||||
wallMs: matrixScreenshotResult.wallMs,
|
||||
},
|
||||
{
|
||||
artifacts: [{ kind: "log", path: relativeToArtifactBase(options.artifactBase, cliLogPath) }],
|
||||
coverageIds: ["cli-entrypoint", "status-snapshots"],
|
||||
failureReason: cliResult.failureReason,
|
||||
stage: "entrypoint-help",
|
||||
status: cliResult.status,
|
||||
surface: "cli",
|
||||
title: "UX Matrix: CLI entrypoint help",
|
||||
wallMs: cliResult.wallMs,
|
||||
},
|
||||
];
|
||||
|
||||
const fixtureProofDir = cellDir(options.artifactBase, {
|
||||
surface: "qa-lab",
|
||||
stage: "producer-artifact-fixture",
|
||||
});
|
||||
const fixtureHtmlPath = path.join(fixtureProofDir, "producer-artifact-fixture.html");
|
||||
const previewEvidence = buildEvidenceSummary({
|
||||
cells: initialCells,
|
||||
generatedAt: new Date().toISOString(),
|
||||
});
|
||||
const screenshotLog = await fs.readFile(path.join(screenshotCellDir, "logs.txt"), "utf8");
|
||||
await writeProducerArtifactFixtureHtml({
|
||||
artifactBase: options.artifactBase,
|
||||
evidence: previewEvidence,
|
||||
htmlPath: fixtureHtmlPath,
|
||||
logPreview: screenshotLog,
|
||||
});
|
||||
|
||||
const fixtureProofResult = await captureProducerArtifactFixtureProof({
|
||||
htmlPath: fixtureHtmlPath,
|
||||
logPath: path.join(fixtureProofDir, "logs.txt"),
|
||||
screenshotPath: path.join(fixtureProofDir, "producer-artifact-fixture.png"),
|
||||
skipVisualProof: options.skipVisualProof,
|
||||
videoPath: path.join(fixtureProofDir, "producer-artifact-fixture.webm"),
|
||||
});
|
||||
|
||||
const cells: MatrixCell[] = [
|
||||
{
|
||||
artifacts: [
|
||||
{ kind: "html", path: relativeToArtifactBase(options.artifactBase, fixtureHtmlPath) },
|
||||
{
|
||||
kind: "log",
|
||||
path: relativeToArtifactBase(
|
||||
options.artifactBase,
|
||||
path.join(fixtureProofDir, "logs.txt"),
|
||||
),
|
||||
},
|
||||
...(fixtureProofResult.status === "pass"
|
||||
? [
|
||||
{
|
||||
kind: "screenshot",
|
||||
path: relativeToArtifactBase(
|
||||
options.artifactBase,
|
||||
path.join(fixtureProofDir, "producer-artifact-fixture.png"),
|
||||
),
|
||||
},
|
||||
{
|
||||
kind: "video",
|
||||
path: relativeToArtifactBase(
|
||||
options.artifactBase,
|
||||
path.join(fixtureProofDir, "producer-artifact-fixture.webm"),
|
||||
),
|
||||
},
|
||||
]
|
||||
: []),
|
||||
],
|
||||
coverageIds: ["qa.artifact-safety", "tools.evidence", "workspace.artifacts"],
|
||||
failureReason: fixtureProofResult.failureReason,
|
||||
stage: "producer-artifact-fixture",
|
||||
status: fixtureProofResult.status,
|
||||
surface: "qa-lab",
|
||||
title: "UX Matrix: producer artifact fixture",
|
||||
wallMs: fixtureProofResult.wallMs,
|
||||
},
|
||||
...initialCells,
|
||||
];
|
||||
|
||||
const evidence = buildEvidenceSummary({ cells, generatedAt: new Date().toISOString() });
|
||||
await writeProducerArtifactFixtureHtml({
|
||||
artifactBase: options.artifactBase,
|
||||
evidence,
|
||||
htmlPath: fixtureHtmlPath,
|
||||
logPreview: screenshotLog,
|
||||
});
|
||||
await writeJson(path.join(options.artifactBase, QA_EVIDENCE_FILENAME), evidence);
|
||||
await writeJson(path.join(options.artifactBase, "latest-run.json"), {
|
||||
qaEvidence: QA_EVIDENCE_FILENAME,
|
||||
});
|
||||
await writeProducerMetadata({
|
||||
artifactBase: options.artifactBase,
|
||||
cells,
|
||||
repoRoot: options.repoRoot,
|
||||
});
|
||||
return {
|
||||
artifactBase: options.artifactBase,
|
||||
evidence,
|
||||
};
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
runUxMatrixEvidenceProducer(parseOptions(process.argv.slice(2)))
|
||||
.then((result) => {
|
||||
console.log(`UX Matrix evidence: ${path.join(result.artifactBase, QA_EVIDENCE_FILENAME)}`);
|
||||
console.log(`UX Matrix entries: ${result.evidence.entries.length}`);
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
console.error(error instanceof Error ? error.stack || error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user