fix(qa): authenticate producer and generated-media evidence (#116828)

Co-authored-by: Peter Steinberger <steipete@macos.shared>

extensions/qa-lab/src/suite-runtime-agent-media.test.ts

@@ -71,6 +71,9 @@ describe("qa suite runtime agent media helpers", () => {
   });
 
   it("resolves generated image paths from mock request logs first", async () => {
+    const tempRoot = await makeTempDir("qa-generated-image-request-");
+    const mediaPath = path.join(tempRoot, "generated.png");
+    await fs.writeFile(mediaPath, "png", "utf8");
     fetchJsonMock.mockResolvedValue([
       {
         allInputText: "irrelevant",
@@ -78,7 +81,7 @@ describe("qa suite runtime agent media helpers", () => {
       },
       {
         allInputText: "prompt snippet",
-        toolOutput: JSON.stringify({ details: { media: { mediaUrls: ["/tmp/generated.png"] } } }),
+        toolOutput: JSON.stringify({ details: { media: { mediaUrls: [mediaPath] } } }),
       },
     ]);
 
@@ -86,18 +89,57 @@ describe("qa suite runtime agent media helpers", () => {
       resolveGeneratedImagePath({
         env: {
           mock: { baseUrl: "http://127.0.0.1:9999" },
-          gateway: { tempRoot: "/tmp/runtime" },
+          gateway: { tempRoot },
         } as never,
         promptSnippet: "prompt snippet",
         startedAtMs: Date.now(),
         timeoutMs: 2_000,
       }),
-    ).resolves.toBe("/tmp/generated.png");
+    ).resolves.toBe(mediaPath);
     expect(fetchJsonMock).toHaveBeenCalledOnce();
     expect(fetchJsonMock).toHaveBeenCalledWith(expect.any(String), expect.any(Number));
     expect(fetchJsonMock.mock.calls[0]?.[1]).toBeLessThanOrEqual(2_000);
   });
 
+  it.each(["missing", "stale", "empty"] as const)(
+    "ignores %s generated media paths returned by matching mock requests",
+    async (artifactState) => {
+      const tempRoot = await makeTempDir("qa-generated-image-invalid-request-");
+      const mediaDir = path.join(tempRoot, "state", "media", "tool-image-generation");
+      await fs.mkdir(mediaDir, { recursive: true });
+      const freshMediaPath = path.join(mediaDir, "fresh-generated.png");
+      await fs.writeFile(freshMediaPath, "fresh png", "utf8");
+      const invalidMediaPath = path.join(tempRoot, `invalid-${artifactState}.png`);
+      if (artifactState !== "missing") {
+        await fs.writeFile(invalidMediaPath, artifactState === "empty" ? "" : "stale png", "utf8");
+      }
+      if (artifactState === "stale") {
+        const staleTimestamp = new Date(Date.now() - 60_000);
+        await fs.utimes(invalidMediaPath, staleTimestamp, staleTimestamp);
+      }
+      fetchJsonMock.mockResolvedValue([
+        {
+          allInputText: "prompt snippet",
+          toolOutput: JSON.stringify({
+            details: { media: { mediaUrls: [invalidMediaPath] } },
+          }),
+        },
+      ]);
+
+      await expect(
+        resolveGeneratedImagePath({
+          env: {
+            mock: { baseUrl: "http://127.0.0.1:9999" },
+            gateway: { tempRoot },
+          } as never,
+          promptSnippet: "prompt snippet",
+          startedAtMs: Date.now(),
+          timeoutMs: 2_000,
+        }),
+      ).resolves.toBe(freshMediaPath);
+    },
+  );
+
   it("falls back to generated image files under the gateway temp root", async () => {
     const tempRoot = await makeTempDir("qa-generated-image-");
     const mediaDir = path.join(tempRoot, "state", "media", "tool-image-generation");

extensions/qa-lab/src/suite-runtime-agent-media.ts

@@ -100,7 +100,11 @@ async function resolveGeneratedImagePath(params: {
           }
           const mediaPath = extractMediaPathFromText(request.toolOutput);
           if (mediaPath) {
-            return mediaPath;
+            const stat = await fs.stat(mediaPath).catch(() => null);
+            // Request snapshots include previous runs; only fresh, nonempty files prove this run.
+            if (stat?.isFile() && stat.size > 0 && stat.mtimeMs >= params.startedAtMs - 1_000) {
+              return mediaPath;
+            }
           }
         }
       } catch {
@@ -119,7 +123,7 @@ async function resolveGeneratedImagePath(params: {
       entries.map(async (entry) => {
         const fullPath = path.join(mediaDir, entry);
         const stat = await fs.stat(fullPath).catch(() => null);
-        if (!stat?.isFile()) {
+        if (!stat?.isFile() || stat.size === 0) {
           return null;
         }
         return {

extensions/qa-lab/src/test-file-scenario-runner.test.ts

@@ -1373,7 +1373,7 @@ describe("qa test file scenario runner", () => {
     });
   });
 
-  it("allows blocked imported producer evidence for opt-in script scenarios", async () => {
+  it("keeps all-blocked producer evidence blocked for opt-in script scenarios", async () => {
     const repoRoot = await makeTempRepo("qa-script-producer-blocked-allowed-");
     const outputDir = path.join(
       repoRoot,
@@ -1411,7 +1411,8 @@ describe("qa test file scenario runner", () => {
     });
 
     expect(result.results[0]).toMatchObject({
-      status: "pass",
+      status: "blocked",
+      failureMessage: "Playwright browser is missing.",
       producerEvidence: {
         entries: [
           {
@@ -1427,6 +1428,65 @@ describe("qa test file scenario runner", () => {
     });
   });
 
+  it("allows blocked producer checks when another check genuinely passes", async () => {
+    const repoRoot = await makeTempRepo("qa-script-producer-blocked-mixed-");
+    const outputDir = path.join(
+      repoRoot,
+      ".artifacts",
+      "qa-e2e",
+      "scenario-script-producer-blocked-mixed",
+    );
+    const scenario = makeTestFileScenario("script", "scripts/evidence-producer.ts");
+    if (scenario.execution.kind !== "script") {
+      throw new Error("expected script scenario");
+    }
+    scenario.execution.allowBlockedEvidence = true;
+
+    const result = await runQaTestFileScenarios({
+      repoRoot,
+      outputDir,
+      providerMode: "mock-openai",
+      primaryModel: "mock-openai/gpt-5.6-luna",
+      scenarios: [scenario],
+      runCommand: async () => {
+        await writeScriptProducerEvidence({
+          outputDir,
+          status: "blocked",
+          failureReason: "Playwright browser is missing.",
+        });
+        const evidencePath = path.join(outputDir, "scenario-script", "run-1", "qa-evidence.json");
+        const evidence = JSON.parse(await fs.readFile(evidencePath, "utf8"));
+        evidence.entries.push({
+          ...evidence.entries[0],
+          test: {
+            ...evidence.entries[0].test,
+            id: "script-producer.web-ui.executed",
+          },
+          result: {
+            status: "pass",
+            timing: { wallMs: 1 },
+          },
+        });
+        await fs.writeFile(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`, "utf8");
+        return {
+          exitCode: 0,
+          stdout: "script mixed\n",
+          stderr: "",
+        };
+      },
+      env: {
+        OPENCLAW_QA_REF: "scenario-ref",
+      } as NodeJS.ProcessEnv,
+    });
+
+    expect(result.results[0]).toMatchObject({
+      status: "pass",
+      producerEvidence: {
+        entries: [{ result: { status: "blocked" } }, { result: { status: "pass" } }],
+      },
+    });
+  });
+
   it("carries the suite profile into merged producer evidence", async () => {
     const repoRoot = await makeTempRepo("qa-script-profile-");
     const result = await runQaTestFileScenarios({
@@ -1585,7 +1645,7 @@ describe("qa test file scenario runner", () => {
     );
 
     expect(result.executionKind).toBe("script");
-    expect(result.results[0]).toMatchObject({ status: "pass" });
+    expect(result.results[0]).toMatchObject({ status: "blocked" });
     expect(result.results[0]?.producerEvidence?.entries).toHaveLength(3);
     expect(evidence.entries.map((entry) => entry.test.id)).toEqual([
       "ux-matrix.qa-lab.producer-artifact-fixture",

extensions/qa-lab/src/test-file-scenario-runner.ts

@@ -438,7 +438,18 @@ function statusFromProducerEvidence(params: {
       status: blockingEntry.result.status,
     };
   }
-  if (producerEvidence.entries.every((entry) => entry.result.status === "skipped")) {
+  if (!producerEvidence.entries.some((entry) => entry.result.status === "pass")) {
+    // Allowing blocked checks does not make an entirely unexecuted producer a successful run.
+    const blockedEntry = producerEvidence.entries.find(
+      (entry) => entry.result.status === "blocked",
+    );
+    if (blockedEntry) {
+      return {
+        failureMessage:
+          blockedEntry.result.failure?.reason ?? `${blockedEntry.test.id} reported blocked`,
+        status: "blocked",
+      };
+    }
     return { status: "skipped" };
   }
   return { status: "pass" };