From bdceeeb88cd058b453bad662efd255ad270eef42 Mon Sep 17 00:00:00 2001
From: Peter Steinberger <steipete@gmail.com>
Date: Wed, 19 Aug 2026 16:16:21 -0700
Subject: [PATCH] fix(qa): preserve terminal script failures (#126466)

---
 ...le-scenario-runner.script-evidence.test.ts | 78 +++++++++++++++++++
 .../qa-lab/src/test-file-scenario-runner.ts   | 15 +++-
 2 files changed, 89 insertions(+), 4 deletions(-)

diff --git a/extensions/qa-lab/src/test-file-scenario-runner.script-evidence.test.ts b/extensions/qa-lab/src/test-file-scenario-runner.script-evidence.test.ts
index 74e7e0c4a377..e6bbe7ad4b6f 100644
--- a/extensions/qa-lab/src/test-file-scenario-runner.script-evidence.test.ts
+++ b/extensions/qa-lab/src/test-file-scenario-runner.script-evidence.test.ts
@@ -301,6 +301,84 @@ describe("qa test file scenario runner", () => {
     });
   });
 
+  it.each([
+    { producerStatus: "pass" as const, terminal: "exit" as const },
+    { producerStatus: "blocked" as const, terminal: "timeout" as const },
+    { producerStatus: "skipped" as const, terminal: "exit" as const },
+  ])(
+    "makes a real $terminal failure override colliding $producerStatus producer evidence",
+    async ({ producerStatus, terminal }) => {
+      const commandName = path.basename(process.execPath);
+      const tempRoot = await makeTempRepo(`qa-script-terminal-${terminal}-${producerStatus}-`);
+      const outputDir = path.join(tempRoot, "out");
+      const scriptPath = path.join(tempRoot, "terminal-evidence-producer.mjs");
+      const producerEvidence = buildScriptProducerEvidence({
+        additionalEntries: buildScriptProducerEvidence({
+          producerId: "producer-diagnostic",
+          status: "pass",
+        }).entries,
+        artifacts: [{ kind: "log", path: "producer.log" }],
+        producerId: "scenario-script",
+        status: producerStatus,
+      });
+      await fs.writeFile(
+        scriptPath,
+        [
+          "import fs from 'node:fs/promises';",
+          "import path from 'node:path';",
+          "const artifactBase = process.argv[process.argv.indexOf('--artifact-base') + 1];",
+          "const runRoot = path.join(artifactBase, 'run-1');",
+          "await fs.mkdir(runRoot, { recursive: true });",
+          "await fs.writeFile(path.join(runRoot, 'producer.log'), 'producer evidence\\n');",
+          `await fs.writeFile(path.join(runRoot, 'qa-evidence.json'), ${JSON.stringify(JSON.stringify(producerEvidence))});`,
+          "await fs.writeFile(path.join(artifactBase, 'latest-run.json'), JSON.stringify({ qaEvidence: 'run-1/qa-evidence.json' }));",
+          terminal === "timeout" ? "setInterval(() => {}, 1_000);" : "process.exitCode = 7;",
+        ].join("\n"),
+        "utf8",
+      );
+
+      const result = await runQaTestFileScenarios({
+        repoRoot: process.cwd(),
+        outputDir,
+        ...QA_TEST_RUNNER_DEFAULTS,
+        scenarios: [makeTestFileScenario("script", scriptPath)],
+        commandTimeoutMs: 1_000,
+      });
+
+      expect(result.results[0]).toMatchObject({
+        failureMessage:
+          terminal === "timeout"
+            ? expect.stringContaining("timed out after 1000ms")
+            : `${commandName} exited with 7`,
+        status: "fail",
+      });
+      expect(result.evidence.entries).toHaveLength(2);
+      expect(
+        result.evidence.entries.find((entry) => entry.test.id === "scenario-script"),
+      ).toMatchObject({
+        coverage: [
+          { id: "qa.coverage", role: "primary" },
+          { id: "qa.reporting", role: "secondary" },
+        ],
+        execution: {
+          artifacts: [{ kind: "log", path: expect.stringContaining("producer.log") }],
+          runner: "evidence-producer-script",
+        },
+        result: {
+          failure: {
+            reason: `${commandName} ${
+              terminal === "timeout" ? "timed out after 1000ms" : "exited with 7"
+            }`,
+          },
+          status: "fail",
+        },
+      });
+      expect(
+        result.evidence.entries.find((entry) => entry.test.id === "producer-diagnostic"),
+      ).toMatchObject({ result: { status: "pass" } });
+    },
+  );
+
   it("fails script scenario results when imported producer evidence fails", async () => {
     const repoRoot = await makeTempRepo("qa-script-producer-fail-");
     const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-producer-fail");
diff --git a/extensions/qa-lab/src/test-file-scenario-runner.ts b/extensions/qa-lab/src/test-file-scenario-runner.ts
index 0acb2ec017f8..aac232a5e021 100644
--- a/extensions/qa-lab/src/test-file-scenario-runner.ts
+++ b/extensions/qa-lab/src/test-file-scenario-runner.ts
@@ -448,8 +448,8 @@ function buildTestFileEvidence(params: {
   );
   if (producerEntries.length > 0) {
     const definition = testFileRunnerDefinitions[params.kind];
-    // Failed scripts still need generic fallback evidence unless their producer
-    // already recorded that scenario identity at the authoritative boundary.
+    // Producer failures stay authoritative; parent terminal failures replace
+    // colliding non-fail results without discarding producer execution facts.
     const producerEntryIds = new Set(producerEntries.map((entry) => entry.test.id));
     const fallbackResults = params.results.filter(
       (result) => !result.producerEvidence || result.includeFallbackEvidence,
@@ -486,10 +486,17 @@ function buildTestFileEvidence(params: {
       profile: resolveQaEvidenceProfile({ env: params.env }),
       entries: [
         ...producerEntries.map((entry) => {
+          const fallbackFailure = fallbackEvidence?.entries.find(
+            (fallback) => fallback.test.id === entry.test.id && fallback.result.status === "fail",
+          );
+          const resolvedEntry =
+            entry.result.status !== "fail" && fallbackFailure
+              ? Object.assign({}, entry, { result: fallbackFailure.result })
+              : entry;
           if (evidenceMode !== "slim") {
-            return entry;
+            return resolvedEntry;
           }
-          const { execution: _execution, ...withoutExecution } = entry;
+          const { execution: _execution, ...withoutExecution } = resolvedEntry;
           return withoutExecution;
         }),
         ...(fallbackEvidence?.entries.filter((entry) => !producerEntryIds.has(entry.test.id)) ??