From d5a27b0b96843b954568d10807e12a4e34536611 Mon Sep 17 00:00:00 2001 From: Colin Johnson Date: Thu, 18 Jun 2026 17:10:06 -0400 Subject: [PATCH] test: add QA Lab UX Matrix evidence scenario (#94306) * test: add qa lab ux matrix script scenario * fix(qa-lab): annotate UX Matrix producer catch callback as unknown for oxlint --------- Co-authored-by: Dallin Romney --- .../qa-lab/src/scenario-catalog.test.ts | 10 + .../src/test-file-scenario-runner.test.ts | 63 +- .../ui/ux-matrix-evidence-dashboard.yaml | 34 + scripts/qa/ux-matrix-evidence-producer.ts | 637 ++++++++++++++++++ 4 files changed, 743 insertions(+), 1 deletion(-) create mode 100644 qa/scenarios/ui/ux-matrix-evidence-dashboard.yaml create mode 100644 scripts/qa/ux-matrix-evidence-producer.ts diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts index 8f0ed4394bb9..774068fe32b3 100644 --- a/extensions/qa-lab/src/scenario-catalog.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.test.ts @@ -60,6 +60,7 @@ describe("qa scenario catalog", () => { "package-openclaw-for-docker", "plugin-lifecycle-probe", "qa-otel-smoke", + "ux-matrix-evidence-dashboard", ].toSorted(), ); expect( @@ -140,6 +141,7 @@ describe("qa scenario catalog", () => { it("loads native test execution scenarios from YAML", () => { const scenario = readQaScenarioById("control-ui-chat-flow-playwright"); + const uxMatrix = readQaScenarioById("ux-matrix-evidence-dashboard"); expect(scenario.execution.kind).toBe("playwright"); if (scenario.execution.kind !== "playwright") { @@ -148,6 +150,14 @@ describe("qa scenario catalog", () => { expect(scenario.execution.path).toBe("ui/src/ui/e2e/chat-flow.e2e.test.ts"); expect(scenario.execution.flow).toBeUndefined(); expect(scenario.coverage?.primary).toContain("ui.control"); + expect(uxMatrix.execution.kind).toBe("script"); + if (uxMatrix.execution.kind !== "script") { + throw new Error(`expected script scenario, got ${uxMatrix.execution.kind}`); + } + expect(uxMatrix.execution.path).toBe("scripts/qa/ux-matrix-evidence-producer.ts"); + expect(uxMatrix.execution.args).toStrictEqual(["--artifact-base", "${outputDir}"]); + expect(uxMatrix.execution.config).toBeUndefined(); + expect(uxMatrix.coverage?.primary).toContain("qa.artifact-safety"); }); it("loads runtime parity tier metadata for first-hour and soak lanes", () => { diff --git a/extensions/qa-lab/src/test-file-scenario-runner.test.ts b/extensions/qa-lab/src/test-file-scenario-runner.test.ts index 12a0b41df325..926b22f8b1da 100644 --- a/extensions/qa-lab/src/test-file-scenario-runner.test.ts +++ b/extensions/qa-lab/src/test-file-scenario-runner.test.ts @@ -3,7 +3,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { validateQaEvidenceSummaryJson } from "./evidence-summary.js"; -import type { QaSeedScenarioWithSource } from "./scenario-catalog.js"; +import { readQaScenarioById, type QaSeedScenarioWithSource } from "./scenario-catalog.js"; import { runQaTestFileScenarios, type QaScenarioCommandExecution, @@ -689,4 +689,65 @@ describe("qa test file scenario runner", () => { expect(artifactPath).toBe(path.normalize(externalArtifact)); expect(artifactPath?.includes("..")).toBe(false); }); + + it("runs the UX Matrix script producer and imports its evidence bundle", async () => { + const repoRoot = process.cwd(); + const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-ux-matrix-script-")); + tempRoots.push(outputDir); + const scenario = readQaScenarioById("ux-matrix-evidence-dashboard"); + + expect(scenario.execution.kind).toBe("script"); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir, + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.5", + scenarios: [scenario], + env: { + OPENCLAW_QA_REF: "scenario-ref", + } as NodeJS.ProcessEnv, + }); + + expect(result.executionKind).toBe("script"); + expect(result.results[0]?.producerEvidence?.entries).toHaveLength(3); + const evidence = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(result.evidencePath, "utf8")), + ); + expect(evidence.entries.map((entry) => entry.test.id)).toEqual([ + "ux-matrix.qa-lab.producer-artifact-fixture", + "ux-matrix.control-ui.screenshot-artifact", + "ux-matrix.cli.entrypoint-help", + ]); + expect( + evidence.entries.flatMap((entry) => entry.coverage.map((coverage) => coverage.id)), + ).toEqual( + expect.arrayContaining([ + "qa.artifact-safety", + "tools.evidence", + "workspace.artifacts", + "ui.control", + "control-ui", + "cli-entrypoint", + "status-snapshots", + ]), + ); + const artifactKinds = evidence.entries.flatMap( + (entry) => entry.execution?.artifacts.map((artifact) => artifact.kind) ?? [], + ); + expect(artifactKinds).toEqual(expect.arrayContaining(["html", "log"])); + const fixtureEntry = evidence.entries.find( + (entry) => entry.test.id === "ux-matrix.qa-lab.producer-artifact-fixture", + ); + expect(fixtureEntry?.execution?.artifacts.map((artifact) => artifact.path)).toContain( + path.join( + outputDir, + "ux-matrix-evidence-dashboard", + "surfaces", + "qa-lab", + "stages", + "producer-artifact-fixture", + "producer-artifact-fixture.html", + ), + ); + }); }); diff --git a/qa/scenarios/ui/ux-matrix-evidence-dashboard.yaml b/qa/scenarios/ui/ux-matrix-evidence-dashboard.yaml new file mode 100644 index 000000000000..5c54cd608983 --- /dev/null +++ b/qa/scenarios/ui/ux-matrix-evidence-dashboard.yaml @@ -0,0 +1,34 @@ +title: UX Matrix evidence dashboard + +scenario: + id: ux-matrix-evidence-dashboard + surface: control-ui + category: browser-control-ui-and-webchat.browser-ui + coverage: + primary: + - qa.artifact-safety + secondary: + - ui.control + - control-ui + - cli-entrypoint + - status-snapshots + - tools.evidence + - workspace.artifacts + objective: Produce UX Matrix evidence artifacts through the QA Lab script producer contract. + successCriteria: + - The script producer writes manifest.json, matrix.json, release-ledger.json, scorecard.md, latest-run.json, and qa-evidence.json. + - Evidence entries declare coverage IDs and artifact paths for generated logs, screenshot proof, and video-style proof. + - Missing local browser or CLI prerequisites are represented as blocked evidence with logs instead of fake artifacts. + docsRefs: + - docs/concepts/qa-e2e-automation.md + - docs/web/control-ui.md + codeRefs: + - scripts/qa/ux-matrix-evidence-producer.ts + - extensions/qa-lab/src/test-file-scenario-runner.ts + execution: + kind: script + path: scripts/qa/ux-matrix-evidence-producer.ts + summary: Script-backed UX Matrix producer that emits QA Lab evidence artifacts. + args: + - --artifact-base + - ${outputDir} diff --git a/scripts/qa/ux-matrix-evidence-producer.ts b/scripts/qa/ux-matrix-evidence-producer.ts new file mode 100644 index 000000000000..a56273063063 --- /dev/null +++ b/scripts/qa/ux-matrix-evidence-producer.ts @@ -0,0 +1,637 @@ +// Produces a QA Lab UX Matrix evidence bundle through the script scenario contract. +import { execFile } from "node:child_process"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { toRepoRelativePath } from "../../extensions/qa-lab/src/cli-paths.js"; +import { + QA_EVIDENCE_FILENAME, + QA_EVIDENCE_SUMMARY_KIND, + QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, + validateQaEvidenceSummaryJson, + type QaEvidenceStatus, + type QaEvidenceSummaryEntry, + type QaEvidenceSummaryJson, +} from "../../extensions/qa-lab/src/evidence-summary.js"; + +const execFileAsync = promisify(execFile); +const SCENARIO_ID = "ux-matrix-evidence-dashboard"; +const SOURCE_PATH = "scripts/qa/ux-matrix-evidence-producer.ts"; +const SUITE_COMMAND = `pnpm openclaw qa suite --scenario ${SCENARIO_ID}`; + +type MatrixCell = { + artifacts: Array<{ kind: string; path: string }>; + coverageIds: string[]; + failureReason?: string; + stage: string; + status: QaEvidenceStatus; + surface: string; + title: string; + wallMs: number; +}; + +export type ProducerOptions = { + artifactBase: string; + repoRoot: string; + skipVisualProof: boolean; +}; + +function parseOptions(argv: readonly string[]): ProducerOptions { + let artifactBase = ""; + let repoRoot = process.cwd(); + let skipVisualProof = false; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--artifact-base") { + artifactBase = argv[index + 1] ?? ""; + index += 1; + continue; + } + if (arg === "--repo-root") { + repoRoot = argv[index + 1] ?? ""; + index += 1; + continue; + } + if (arg === "--skip-visual-proof") { + skipVisualProof = true; + continue; + } + throw new Error(`unsupported UX Matrix producer arg: ${arg}`); + } + if (!artifactBase.trim()) { + throw new Error("--artifact-base is required"); + } + if (!repoRoot.trim()) { + throw new Error("--repo-root must not be empty"); + } + return { + artifactBase: path.resolve(repoRoot, artifactBase), + repoRoot: path.resolve(repoRoot), + skipVisualProof, + }; +} + +async function writeJson(filePath: string, value: unknown) { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); +} + +async function writeText(filePath: string, value: string) { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, value, "utf8"); +} + +function cellDir(artifactBase: string, cell: Pick) { + return path.join(artifactBase, "surfaces", cell.surface, "stages", cell.stage); +} + +function relativeToArtifactBase(artifactBase: string, filePath: string) { + return path.relative(artifactBase, filePath).split(path.sep).join("/"); +} + +function buildExecution(params: { + artifacts: MatrixCell["artifacts"]; + source: string; +}): QaEvidenceSummaryEntry["execution"] { + return { + runner: "ux-matrix-script-producer", + environment: { + ref: process.env.OPENCLAW_QA_REF?.trim() || process.env.GITHUB_SHA?.trim() || null, + os: process.platform, + nodeVersion: process.version, + }, + provider: { + id: "ux-matrix", + live: false, + model: { + name: null, + ref: null, + }, + fixture: "local-qa-lab-script-producer", + }, + packageSource: { + kind: "source-checkout", + }, + artifacts: params.artifacts.map((artifact) => ({ + ...artifact, + source: params.source, + })), + }; +} + +function buildEvidenceEntry(cell: MatrixCell): QaEvidenceSummaryEntry { + const source = `ux-matrix:${cell.surface}:${cell.stage}`; + return { + test: { + kind: "ux-matrix-cell", + id: `ux-matrix.${cell.surface}.${cell.stage}`, + title: cell.title, + source: { path: SOURCE_PATH }, + }, + coverage: cell.coverageIds.map((id, index) => ({ + id, + role: index === 0 ? "primary" : "secondary", + })), + refs: [ + { kind: "code", path: SOURCE_PATH }, + { kind: "docs", path: "docs/concepts/qa-e2e-automation.md" }, + ], + execution: buildExecution({ + artifacts: cell.artifacts, + source, + }), + result: { + status: cell.status, + ...(cell.status === "pass" + ? {} + : { + failure: { + class: cell.status, + reason: cell.failureReason ?? `${cell.status} UX Matrix cell`, + }, + }), + timing: { + wallMs: Math.max(1, cell.wallMs), + }, + }, + }; +} + +function buildEvidenceSummary(params: { + cells: readonly MatrixCell[]; + generatedAt: string; +}): QaEvidenceSummaryJson { + return validateQaEvidenceSummaryJson({ + kind: QA_EVIDENCE_SUMMARY_KIND, + schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, + generatedAt: params.generatedAt, + evidenceMode: "full", + entries: params.cells.map(buildEvidenceEntry), + }); +} + +async function runCommandForCell(params: { + args: string[]; + command: string; + cwd: string; + logPath: string; + timeoutMs: number; +}) { + const startedAt = Date.now(); + const commandLine = [params.command, ...params.args].join(" "); + try { + const { stdout, stderr } = await execFileAsync(params.command, params.args, { + cwd: params.cwd, + timeout: params.timeoutMs, + env: process.env, + maxBuffer: 1024 * 1024, + }); + await writeText(params.logPath, `$ ${commandLine}\n${stdout}${stderr}`); + return { + status: "pass" as const, + wallMs: Date.now() - startedAt, + }; + } catch (error) { + const details = error instanceof Error ? error.message : String(error); + await writeText(params.logPath, `$ ${commandLine}\nblocked: ${details}\n`); + return { + failureReason: details, + status: "blocked" as const, + wallMs: Date.now() - startedAt, + }; + } +} + +async function writePreflight(artifactBase: string) { + await writeText( + path.join(artifactBase, "preflight", "runtime.txt"), + [ + `platform=${process.platform}`, + `arch=${process.arch}`, + `node=${process.version}`, + `freeMemoryBytes=${os.freemem()}`, + `totalMemoryBytes=${os.totalmem()}`, + ].join("\n") + "\n", + ); +} + +async function captureControlUiScreenshot(params: { + artifactBase: string; + htmlPath: string; + logPath: string; + screenshotPath: string; +}) { + const startedAt = Date.now(); + try { + const { chromium } = await import("playwright"); + const browser = await chromium.launch(); + try { + const page = await browser.newPage({ viewport: { width: 1024, height: 720 } }); + await page.goto(pathToFileURL(params.htmlPath).href); + await page.screenshot({ path: params.screenshotPath, fullPage: true }); + } finally { + await browser.close(); + } + await writeText( + params.logPath, + `Captured ${relativeToArtifactBase(params.artifactBase, params.screenshotPath)}\n`, + ); + return { + status: "pass" as const, + wallMs: Date.now() - startedAt, + }; + } catch (error) { + const details = error instanceof Error ? error.message : String(error); + await writeText(params.logPath, `blocked: ${details}\n`); + return { + failureReason: details, + status: "blocked" as const, + wallMs: Date.now() - startedAt, + }; + } +} + +function escapeHtml(value: string) { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """); +} + +async function writeProducerArtifactFixtureHtml(params: { + artifactBase: string; + evidence: QaEvidenceSummaryJson; + htmlPath: string; + logPreview: string; +}) { + const previewArtifacts = params.evidence.entries + .filter((entry) => entry.test.id !== "ux-matrix.qa-lab.producer-artifact-fixture") + .flatMap((entry) => entry.execution?.artifacts ?? []); + const screenshotArtifact = previewArtifacts.find((artifact) => artifact.kind === "screenshot"); + const logArtifact = previewArtifacts.find((artifact) => artifact.kind === "log"); + const screenshotPath = screenshotArtifact + ? relativeToArtifactBase( + path.dirname(params.htmlPath), + path.join(params.artifactBase, screenshotArtifact.path), + ) + : ""; + const entryRows = params.evidence.entries + .map( + (entry) => + `
  • ${escapeHtml(entry.test.id)} - ${escapeHtml( + entry.result.status, + )} - ${entry.coverage.map((coverage) => escapeHtml(coverage.id)).join(", ")}
    • `, + ) + .join(""); + await writeText( + params.htmlPath, + ` + +UX Matrix producer artifact fixture + +
    +

    UX Matrix Producer Artifact Fixture

    +

    Script-produced ${escapeHtml(QA_EVIDENCE_FILENAME)} for ${escapeHtml( + SCENARIO_ID, + )}; this fixture is not the QA Lab Evidence Archive UI.

    +
    +

    UX Matrix entries

    +
      ${entryRows}
    +
    +
    +

    Artifact preview

    +

    ${escapeHtml(logArtifact?.path ?? "no log")} - ${escapeHtml( + screenshotArtifact?.path ?? "no screenshot", + )}

    +
    + + +
    +
    Choose an artifact to preview.
    +
    +
    + +`, + ); +} + +async function captureProducerArtifactFixtureProof(params: { + htmlPath: string; + logPath: string; + screenshotPath: string; + skipVisualProof: boolean; + videoPath: string; +}) { + const startedAt = Date.now(); + if (params.skipVisualProof) { + await writeText(params.logPath, "blocked: --skip-visual-proof was set\n"); + return { + failureReason: "--skip-visual-proof was set", + status: "blocked" as const, + wallMs: Date.now() - startedAt, + }; + } + try { + const { chromium } = await import("playwright"); + const videoDir = path.join(path.dirname(params.videoPath), "recording"); + await fs.mkdir(videoDir, { recursive: true }); + const browser = await chromium.launch(); + let recordedVideo: string | undefined; + try { + const context = await browser.newContext({ + viewport: { width: 1280, height: 820 }, + recordVideo: { + dir: videoDir, + size: { width: 1280, height: 820 }, + }, + }); + const page = await context.newPage(); + const video = page.video(); + await page.goto(pathToFileURL(params.htmlPath).href); + await page.locator("#preview-screenshot").click(); + await page.locator("#preview img").waitFor({ state: "visible", timeout: 5_000 }); + await page.screenshot({ path: params.screenshotPath, fullPage: true }); + await page.waitForTimeout(350); + await page.locator("#preview-log").click(); + await page.waitForTimeout(350); + await context.close(); + recordedVideo = await video?.path(); + } finally { + await browser.close(); + } + if (!recordedVideo) { + throw new Error("Playwright did not provide a recording path"); + } + await fs.mkdir(path.dirname(params.videoPath), { recursive: true }); + await fs.copyFile(recordedVideo, params.videoPath); + await writeText( + params.logPath, + [ + `Captured screenshot ${path.basename(params.screenshotPath)}`, + `Captured recording ${path.basename(params.videoPath)}`, + ].join("\n") + "\n", + ); + return { + status: "pass" as const, + wallMs: Date.now() - startedAt, + }; + } catch (error) { + const details = error instanceof Error ? error.message : String(error); + await writeText(params.logPath, `blocked: ${details}\n`); + return { + failureReason: details, + status: "blocked" as const, + wallMs: Date.now() - startedAt, + }; + } +} + +async function writeProducerMetadata(params: { + artifactBase: string; + cells: readonly MatrixCell[]; + repoRoot: string; +}) { + const counts = params.cells.reduce>((acc, cell) => { + acc[cell.status] = (acc[cell.status] ?? 0) + 1; + return acc; + }, {}); + await writeJson(path.join(params.artifactBase, "manifest.json"), { + kind: "openclaw.qa.ux-matrix", + run: { + scenarioId: SCENARIO_ID, + status: counts.fail ? "fail" : counts.blocked ? "blocked" : "pass", + }, + }); + await writeJson(path.join(params.artifactBase, "matrix.json"), { + cells: params.cells.map((cell) => ({ + coverageIds: cell.coverageIds, + stage: cell.stage, + status: cell.status, + surface: cell.surface, + })), + counts, + }); + await writeJson(path.join(params.artifactBase, "release-ledger.json"), { + entries: params.cells.map((cell) => ({ + coverageIds: cell.coverageIds, + stage: cell.stage, + status: cell.status, + surface: cell.surface, + })), + kind: "openclaw.qa.ux-matrix.release-ledger", + }); + await writeText( + path.join(params.artifactBase, "commands.txt"), + [ + `${SUITE_COMMAND} --output-dir ${toRepoRelativePath(params.repoRoot, params.artifactBase)}`, + `node --import tsx ${SOURCE_PATH} --artifact-base ${toRepoRelativePath( + params.repoRoot, + params.artifactBase, + )}`, + ].join("\n") + "\n", + ); + await writeText( + path.join(params.artifactBase, "scorecard.md"), + ["# UX Matrix", "", ...Object.entries(counts).map(([status, count]) => `- ${status}: ${count}`)] + .join("\n") + .trimEnd() + "\n", + ); +} + +export async function runUxMatrixEvidenceProducer(options: ProducerOptions) { + await fs.mkdir(options.artifactBase, { recursive: true }); + await writePreflight(options.artifactBase); + + const cliLogPath = path.join( + cellDir(options.artifactBase, { surface: "cli", stage: "entrypoint-help" }), + "logs.txt", + ); + const cliResult = await runCommandForCell({ + command: process.execPath, + args: ["openclaw.mjs", "--help"], + cwd: options.repoRoot, + logPath: cliLogPath, + timeoutMs: 30_000, + }); + + const screenshotCellDir = cellDir(options.artifactBase, { + surface: "control-ui", + stage: "screenshot-artifact", + }); + const matrixHtmlPath = path.join(screenshotCellDir, "matrix-preview.html"); + await writeText( + matrixHtmlPath, + 'UX Matrix

    UX Matrix

    Control UI artifact preview fixture generated by the scenario.

    ', + ); + const matrixScreenshotPath = path.join(screenshotCellDir, "screenshot.png"); + const matrixScreenshotResult = await captureControlUiScreenshot({ + artifactBase: options.artifactBase, + htmlPath: matrixHtmlPath, + logPath: path.join(screenshotCellDir, "logs.txt"), + screenshotPath: matrixScreenshotPath, + }); + + const initialCells: MatrixCell[] = [ + { + artifacts: [ + { + kind: "log", + path: relativeToArtifactBase( + options.artifactBase, + path.join(screenshotCellDir, "logs.txt"), + ), + }, + ...(matrixScreenshotResult.status === "pass" + ? [ + { + kind: "screenshot", + path: relativeToArtifactBase(options.artifactBase, matrixScreenshotPath), + }, + ] + : []), + ], + coverageIds: ["ui.control", "control-ui"], + failureReason: matrixScreenshotResult.failureReason, + stage: "screenshot-artifact", + status: matrixScreenshotResult.status, + surface: "control-ui", + title: "UX Matrix: screenshot artifact", + wallMs: matrixScreenshotResult.wallMs, + }, + { + artifacts: [{ kind: "log", path: relativeToArtifactBase(options.artifactBase, cliLogPath) }], + coverageIds: ["cli-entrypoint", "status-snapshots"], + failureReason: cliResult.failureReason, + stage: "entrypoint-help", + status: cliResult.status, + surface: "cli", + title: "UX Matrix: CLI entrypoint help", + wallMs: cliResult.wallMs, + }, + ]; + + const fixtureProofDir = cellDir(options.artifactBase, { + surface: "qa-lab", + stage: "producer-artifact-fixture", + }); + const fixtureHtmlPath = path.join(fixtureProofDir, "producer-artifact-fixture.html"); + const previewEvidence = buildEvidenceSummary({ + cells: initialCells, + generatedAt: new Date().toISOString(), + }); + const screenshotLog = await fs.readFile(path.join(screenshotCellDir, "logs.txt"), "utf8"); + await writeProducerArtifactFixtureHtml({ + artifactBase: options.artifactBase, + evidence: previewEvidence, + htmlPath: fixtureHtmlPath, + logPreview: screenshotLog, + }); + + const fixtureProofResult = await captureProducerArtifactFixtureProof({ + htmlPath: fixtureHtmlPath, + logPath: path.join(fixtureProofDir, "logs.txt"), + screenshotPath: path.join(fixtureProofDir, "producer-artifact-fixture.png"), + skipVisualProof: options.skipVisualProof, + videoPath: path.join(fixtureProofDir, "producer-artifact-fixture.webm"), + }); + + const cells: MatrixCell[] = [ + { + artifacts: [ + { kind: "html", path: relativeToArtifactBase(options.artifactBase, fixtureHtmlPath) }, + { + kind: "log", + path: relativeToArtifactBase( + options.artifactBase, + path.join(fixtureProofDir, "logs.txt"), + ), + }, + ...(fixtureProofResult.status === "pass" + ? [ + { + kind: "screenshot", + path: relativeToArtifactBase( + options.artifactBase, + path.join(fixtureProofDir, "producer-artifact-fixture.png"), + ), + }, + { + kind: "video", + path: relativeToArtifactBase( + options.artifactBase, + path.join(fixtureProofDir, "producer-artifact-fixture.webm"), + ), + }, + ] + : []), + ], + coverageIds: ["qa.artifact-safety", "tools.evidence", "workspace.artifacts"], + failureReason: fixtureProofResult.failureReason, + stage: "producer-artifact-fixture", + status: fixtureProofResult.status, + surface: "qa-lab", + title: "UX Matrix: producer artifact fixture", + wallMs: fixtureProofResult.wallMs, + }, + ...initialCells, + ]; + + const evidence = buildEvidenceSummary({ cells, generatedAt: new Date().toISOString() }); + await writeProducerArtifactFixtureHtml({ + artifactBase: options.artifactBase, + evidence, + htmlPath: fixtureHtmlPath, + logPreview: screenshotLog, + }); + await writeJson(path.join(options.artifactBase, QA_EVIDENCE_FILENAME), evidence); + await writeJson(path.join(options.artifactBase, "latest-run.json"), { + qaEvidence: QA_EVIDENCE_FILENAME, + }); + await writeProducerMetadata({ + artifactBase: options.artifactBase, + cells, + repoRoot: options.repoRoot, + }); + return { + artifactBase: options.artifactBase, + evidence, + }; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + runUxMatrixEvidenceProducer(parseOptions(process.argv.slice(2))) + .then((result) => { + console.log(`UX Matrix evidence: ${path.join(result.artifactBase, QA_EVIDENCE_FILENAME)}`); + console.log(`UX Matrix entries: ${result.evidence.entries.length}`); + }) + .catch((error: unknown) => { + console.error(error instanceof Error ? error.stack || error.message : String(error)); + process.exitCode = 1; + }); +}