From 366a686cba490c09d4e77c235a2d88d4b2c2fd98 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 31 Jul 2026 05:24:06 -0700 Subject: [PATCH] fix(qa): authenticate producer and generated-media evidence (#116828) Co-authored-by: Peter Steinberger --- .../src/suite-runtime-agent-media.test.ts | 48 +++++++++++++- .../qa-lab/src/suite-runtime-agent-media.ts | 8 ++- .../src/test-file-scenario-runner.test.ts | 66 ++++++++++++++++++- .../qa-lab/src/test-file-scenario-runner.ts | 13 +++- 4 files changed, 126 insertions(+), 9 deletions(-) diff --git a/extensions/qa-lab/src/suite-runtime-agent-media.test.ts b/extensions/qa-lab/src/suite-runtime-agent-media.test.ts index ddcbd822ca17..9de2c05ed87c 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-media.test.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-media.test.ts @@ -71,6 +71,9 @@ describe("qa suite runtime agent media helpers", () => { }); it("resolves generated image paths from mock request logs first", async () => { + const tempRoot = await makeTempDir("qa-generated-image-request-"); + const mediaPath = path.join(tempRoot, "generated.png"); + await fs.writeFile(mediaPath, "png", "utf8"); fetchJsonMock.mockResolvedValue([ { allInputText: "irrelevant", @@ -78,7 +81,7 @@ describe("qa suite runtime agent media helpers", () => { }, { allInputText: "prompt snippet", - toolOutput: JSON.stringify({ details: { media: { mediaUrls: ["/tmp/generated.png"] } } }), + toolOutput: JSON.stringify({ details: { media: { mediaUrls: [mediaPath] } } }), }, ]); @@ -86,18 +89,57 @@ describe("qa suite runtime agent media helpers", () => { resolveGeneratedImagePath({ env: { mock: { baseUrl: "http://127.0.0.1:9999" }, - gateway: { tempRoot: "/tmp/runtime" }, + gateway: { tempRoot }, } as never, promptSnippet: "prompt snippet", startedAtMs: Date.now(), timeoutMs: 2_000, }), - ).resolves.toBe("/tmp/generated.png"); + ).resolves.toBe(mediaPath); expect(fetchJsonMock).toHaveBeenCalledOnce(); expect(fetchJsonMock).toHaveBeenCalledWith(expect.any(String), expect.any(Number)); expect(fetchJsonMock.mock.calls[0]?.[1]).toBeLessThanOrEqual(2_000); }); + it.each(["missing", "stale", "empty"] as const)( + "ignores %s generated media paths returned by matching mock requests", + async (artifactState) => { + const tempRoot = await makeTempDir("qa-generated-image-invalid-request-"); + const mediaDir = path.join(tempRoot, "state", "media", "tool-image-generation"); + await fs.mkdir(mediaDir, { recursive: true }); + const freshMediaPath = path.join(mediaDir, "fresh-generated.png"); + await fs.writeFile(freshMediaPath, "fresh png", "utf8"); + const invalidMediaPath = path.join(tempRoot, `invalid-${artifactState}.png`); + if (artifactState !== "missing") { + await fs.writeFile(invalidMediaPath, artifactState === "empty" ? "" : "stale png", "utf8"); + } + if (artifactState === "stale") { + const staleTimestamp = new Date(Date.now() - 60_000); + await fs.utimes(invalidMediaPath, staleTimestamp, staleTimestamp); + } + fetchJsonMock.mockResolvedValue([ + { + allInputText: "prompt snippet", + toolOutput: JSON.stringify({ + details: { media: { mediaUrls: [invalidMediaPath] } }, + }), + }, + ]); + + await expect( + resolveGeneratedImagePath({ + env: { + mock: { baseUrl: "http://127.0.0.1:9999" }, + gateway: { tempRoot }, + } as never, + promptSnippet: "prompt snippet", + startedAtMs: Date.now(), + timeoutMs: 2_000, + }), + ).resolves.toBe(freshMediaPath); + }, + ); + it("falls back to generated image files under the gateway temp root", async () => { const tempRoot = await makeTempDir("qa-generated-image-"); const mediaDir = path.join(tempRoot, "state", "media", "tool-image-generation"); diff --git a/extensions/qa-lab/src/suite-runtime-agent-media.ts b/extensions/qa-lab/src/suite-runtime-agent-media.ts index 035217b8ade4..437992348d4f 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-media.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-media.ts @@ -100,7 +100,11 @@ async function resolveGeneratedImagePath(params: { } const mediaPath = extractMediaPathFromText(request.toolOutput); if (mediaPath) { - return mediaPath; + const stat = await fs.stat(mediaPath).catch(() => null); + // Request snapshots include previous runs; only fresh, nonempty files prove this run. + if (stat?.isFile() && stat.size > 0 && stat.mtimeMs >= params.startedAtMs - 1_000) { + return mediaPath; + } } } } catch { @@ -119,7 +123,7 @@ async function resolveGeneratedImagePath(params: { entries.map(async (entry) => { const fullPath = path.join(mediaDir, entry); const stat = await fs.stat(fullPath).catch(() => null); - if (!stat?.isFile()) { + if (!stat?.isFile() || stat.size === 0) { return null; } return { diff --git a/extensions/qa-lab/src/test-file-scenario-runner.test.ts b/extensions/qa-lab/src/test-file-scenario-runner.test.ts index 2daa843720fc..ee4a04a8283f 100644 --- a/extensions/qa-lab/src/test-file-scenario-runner.test.ts +++ b/extensions/qa-lab/src/test-file-scenario-runner.test.ts @@ -1373,7 +1373,7 @@ describe("qa test file scenario runner", () => { }); }); - it("allows blocked imported producer evidence for opt-in script scenarios", async () => { + it("keeps all-blocked producer evidence blocked for opt-in script scenarios", async () => { const repoRoot = await makeTempRepo("qa-script-producer-blocked-allowed-"); const outputDir = path.join( repoRoot, @@ -1411,7 +1411,8 @@ describe("qa test file scenario runner", () => { }); expect(result.results[0]).toMatchObject({ - status: "pass", + status: "blocked", + failureMessage: "Playwright browser is missing.", producerEvidence: { entries: [ { @@ -1427,6 +1428,65 @@ describe("qa test file scenario runner", () => { }); }); + it("allows blocked producer checks when another check genuinely passes", async () => { + const repoRoot = await makeTempRepo("qa-script-producer-blocked-mixed-"); + const outputDir = path.join( + repoRoot, + ".artifacts", + "qa-e2e", + "scenario-script-producer-blocked-mixed", + ); + const scenario = makeTestFileScenario("script", "scripts/evidence-producer.ts"); + if (scenario.execution.kind !== "script") { + throw new Error("expected script scenario"); + } + scenario.execution.allowBlockedEvidence = true; + + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir, + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.6-luna", + scenarios: [scenario], + runCommand: async () => { + await writeScriptProducerEvidence({ + outputDir, + status: "blocked", + failureReason: "Playwright browser is missing.", + }); + const evidencePath = path.join(outputDir, "scenario-script", "run-1", "qa-evidence.json"); + const evidence = JSON.parse(await fs.readFile(evidencePath, "utf8")); + evidence.entries.push({ + ...evidence.entries[0], + test: { + ...evidence.entries[0].test, + id: "script-producer.web-ui.executed", + }, + result: { + status: "pass", + timing: { wallMs: 1 }, + }, + }); + await fs.writeFile(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`, "utf8"); + return { + exitCode: 0, + stdout: "script mixed\n", + stderr: "", + }; + }, + env: { + OPENCLAW_QA_REF: "scenario-ref", + } as NodeJS.ProcessEnv, + }); + + expect(result.results[0]).toMatchObject({ + status: "pass", + producerEvidence: { + entries: [{ result: { status: "blocked" } }, { result: { status: "pass" } }], + }, + }); + }); + it("carries the suite profile into merged producer evidence", async () => { const repoRoot = await makeTempRepo("qa-script-profile-"); const result = await runQaTestFileScenarios({ @@ -1585,7 +1645,7 @@ describe("qa test file scenario runner", () => { ); expect(result.executionKind).toBe("script"); - expect(result.results[0]).toMatchObject({ status: "pass" }); + expect(result.results[0]).toMatchObject({ status: "blocked" }); expect(result.results[0]?.producerEvidence?.entries).toHaveLength(3); expect(evidence.entries.map((entry) => entry.test.id)).toEqual([ "ux-matrix.qa-lab.producer-artifact-fixture", diff --git a/extensions/qa-lab/src/test-file-scenario-runner.ts b/extensions/qa-lab/src/test-file-scenario-runner.ts index 5b6ae5264665..aa56845ef890 100644 --- a/extensions/qa-lab/src/test-file-scenario-runner.ts +++ b/extensions/qa-lab/src/test-file-scenario-runner.ts @@ -438,7 +438,18 @@ function statusFromProducerEvidence(params: { status: blockingEntry.result.status, }; } - if (producerEvidence.entries.every((entry) => entry.result.status === "skipped")) { + if (!producerEvidence.entries.some((entry) => entry.result.status === "pass")) { + // Allowing blocked checks does not make an entirely unexecuted producer a successful run. + const blockedEntry = producerEvidence.entries.find( + (entry) => entry.result.status === "blocked", + ); + if (blockedEntry) { + return { + failureMessage: + blockedEntry.result.failure?.reason ?? `${blockedEntry.test.id} reported blocked`, + status: "blocked", + }; + } return { status: "skipped" }; } return { status: "pass" };