From 6d2566682a3fc4759017e99013d2dd2437a39ff3 Mon Sep 17 00:00:00 2001
From: Vincent Koc <vincentkoc@ieee.org>
Date: Sun, 7 Jun 2026 02:49:02 +0200
Subject: [PATCH] fix(qa): fail suite on summary scenario failures

---
 extensions/qa-lab/src/cli.runtime.test.ts   |  8 +-------
 extensions/qa-lab/src/cli.runtime.ts        |  2 +-
 extensions/qa-lab/src/suite-summary.test.ts |  9 ++++++++-
 extensions/qa-lab/src/suite-summary.ts      | 17 ++++++++++++-----
 4 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts
index decbbb470211..f1805e2b69b4 100644
--- a/extensions/qa-lab/src/cli.runtime.test.ts
+++ b/extensions/qa-lab/src/cli.runtime.test.ts
@@ -503,13 +503,7 @@ describe("qa cli runtime", () => {
       watchUrl: "http://127.0.0.1:43124",
       reportPath: suiteReportPath,
       summaryPath: suiteSummaryPath,
-      scenarios: [
-        {
-          name: "channel chat baseline",
-          status: "fail",
-          steps: [],
-        },
-      ],
+      scenarios: [],
     });
 
     try {
diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts
index 8c3d7ea05837..ac5def4750e2 100644
--- a/extensions/qa-lab/src/cli.runtime.ts
+++ b/extensions/qa-lab/src/cli.runtime.ts
@@ -706,7 +706,7 @@ export async function runQaSuiteCommand(opts: {
   process.stdout.write(`QA suite watch: ${result.watchUrl}\n`);
   process.stdout.write(`QA suite report: ${result.reportPath}\n`);
   process.stdout.write(`QA suite summary: ${result.summaryPath}\n`);
-  const failedScenarioCount = readQaSuiteFailedScenarioCountFromSummary(result);
+  const failedScenarioCount = await readQaFailedScenarioCountFromSummary(result.summaryPath);
   if (!allowFailures && failedScenarioCount !== null && failedScenarioCount > 0) {
     process.exitCode = 1;
   }
diff --git a/extensions/qa-lab/src/suite-summary.test.ts b/extensions/qa-lab/src/suite-summary.test.ts
index 20673ed6b454..d989fe9c81ab 100644
--- a/extensions/qa-lab/src/suite-summary.test.ts
+++ b/extensions/qa-lab/src/suite-summary.test.ts
@@ -12,7 +12,14 @@ describe("qa suite summary helpers", () => {
     ).toBe(2);
   });
 
-  it("prefers counts.failed when available", () => {
+  it("uses the larger failure signal when counts and scenarios disagree", () => {
+    expect(
+      readQaSuiteFailedScenarioCountFromSummary({
+        counts: { failed: 0 },
+        scenarios: [{ status: "pass" }, { status: "fail" }],
+      }),
+    ).toBe(1);
+
     expect(
       readQaSuiteFailedScenarioCountFromSummary({
         counts: { failed: 3.8 },
diff --git a/extensions/qa-lab/src/suite-summary.ts b/extensions/qa-lab/src/suite-summary.ts
index f5c0e0dbba26..9c629e123187 100644
--- a/extensions/qa-lab/src/suite-summary.ts
+++ b/extensions/qa-lab/src/suite-summary.ts
@@ -79,11 +79,18 @@ export function readQaSuiteFailedScenarioCountFromSummary(summary: unknown): num
     };
     scenarios?: Array<QaSuiteScenarioStatus>;
   };
-  if (typeof payload.counts?.failed === "number" && Number.isFinite(payload.counts.failed)) {
-    return Math.max(0, Math.floor(payload.counts.failed));
+  const countedFailures =
+    typeof payload.counts?.failed === "number" && Number.isFinite(payload.counts.failed)
+      ? Math.max(0, Math.floor(payload.counts.failed))
+      : null;
+  const scenarioFailures = Array.isArray(payload.scenarios)
+    ? countQaSuiteFailedScenarios(payload.scenarios)
+    : null;
+  if (countedFailures !== null && scenarioFailures !== null) {
+    return Math.max(countedFailures, scenarioFailures);
   }
-  if (Array.isArray(payload.scenarios)) {
-    return countQaSuiteFailedScenarios(payload.scenarios);
+  if (scenarioFailures !== null) {
+    return scenarioFailures;
   }
-  return null;
+  return countedFailures;
 }