From d9dfcd6c8a0a7be53a469091d2598a9548a8ecbd Mon Sep 17 00:00:00 2001
From: Vincent Koc <vincentkoc@ieee.org>
Date: Sun, 21 Jun 2026 14:30:36 +0200
Subject: [PATCH] fix(qa): reject impossible confidence counts

---
 .../qa-lab/src/confidence-report.test.ts      | 48 +++++++++++++++++++
 extensions/qa-lab/src/confidence-report.ts    | 46 ++++++++++++++++--
 2 files changed, 90 insertions(+), 4 deletions(-)

diff --git a/extensions/qa-lab/src/confidence-report.test.ts b/extensions/qa-lab/src/confidence-report.test.ts
index 8047b302fe5c..b2ec4c2b58fa 100644
--- a/extensions/qa-lab/src/confidence-report.test.ts
+++ b/extensions/qa-lab/src/confidence-report.test.ts
@@ -719,6 +719,54 @@ describe("qa confidence report", () => {
     expect(report.lanes[0]?.details).toContain("count/scenario mismatch");
   });
 
+  it("treats impossible suite counts as unknown", async () => {
+    for (const [artifact, expectedDetail] of [
+      [
+        { counts: { total: 1, passed: -1, skipped: 0, failed: 0 } },
+        "counts.passed must be a non-negative integer",
+      ],
+      [
+        { counts: { total: 1, passed: 2, failed: 0 } },
+        "counts.total=1 is less than provided count sum=2",
+      ],
+      [
+        { counts: { total: 1, skipped: 2, failed: 0 } },
+        "counts.total=1 is less than provided count sum=2",
+      ],
+      [
+        { counts: { total: 5, passed: 2, skipped: 2, failed: 0 } },
+        "counts.total=5 does not match counts.passed+counts.failed+counts.skipped=4",
+      ],
+    ] as const) {
+      await writeJson("live/qa-suite-summary.json", artifact);
+
+      const report = await buildQaConfidenceReport({
+        manifest: {
+          version: 1,
+          profile: "codex-100",
+          lanes: [
+            {
+              id: "first-hour-live",
+              title: "First hour live",
+              kind: "qa-suite-summary",
+              artifact: "live/qa-suite-summary.json",
+              required: true,
+              failureVerdict: "qa-harness-bug",
+            },
+          ],
+        },
+        artifactRoot: tempRoot,
+        strictZeroUnknowns: true,
+        generatedAt: "2026-05-13T00:00:00.000Z",
+      });
+
+      expect(report.pass).toBe(false);
+      expect(report.counts).toMatchObject({ failed: 0, unknown: 1 });
+      expect(report.lanes[0]).toMatchObject({ status: "unknown" });
+      expect(report.lanes[0]?.details).toContain(expectedDetail);
+    }
+  });
+
   it("requires generic summary lanes to expose an explicit pass signal", async () => {
     await writeJson("runtime/qa-runtime-parity-summary.json", {});
 
diff --git a/extensions/qa-lab/src/confidence-report.ts b/extensions/qa-lab/src/confidence-report.ts
index e3c6dbddd372..6c4fa2bb3a3e 100644
--- a/extensions/qa-lab/src/confidence-report.ts
+++ b/extensions/qa-lab/src/confidence-report.ts
@@ -152,6 +152,10 @@ function readNumber(value: unknown): number | undefined {
   return typeof value === "number" && Number.isFinite(value) ? value : undefined;
 }
 
+function readCount(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : undefined;
+}
+
 function readBoolean(value: unknown): boolean | undefined {
   return typeof value === "boolean" ? value : undefined;
 }
@@ -371,9 +375,44 @@ function evaluateQaSuiteSummary(payload: unknown): QaConfidenceLaneEvaluation {
     };
   }
   const counts = isRecord(payload.counts) ? payload.counts : undefined;
-  const totalCount = readNumber(counts?.total);
-  const passedCount = readNumber(counts?.passed);
-  const failedCount = readNumber(counts?.failed);
+  for (const key of ["total", "passed", "failed", "skipped"] as const) {
+    if (counts && Object.hasOwn(counts, key) && readCount(counts[key]) === undefined) {
+      return {
+        passed: false,
+        status: "unknown",
+        details: `qa-suite-summary counts.${key} must be a non-negative integer`,
+      };
+    }
+  }
+  const totalCount = readCount(counts?.total);
+  const passedCount = readCount(counts?.passed);
+  const failedCount = readCount(counts?.failed);
+  const explicitSkippedCount = readCount(counts?.skipped);
+  if (totalCount !== undefined) {
+    const providedCountSum =
+      (passedCount ?? 0) + (failedCount ?? 0) + (explicitSkippedCount ?? 0);
+    if (totalCount < providedCountSum) {
+      return {
+        passed: false,
+        status: "unknown",
+        details: `qa-suite-summary counts.total=${totalCount} is less than provided count sum=${providedCountSum}`,
+      };
+    }
+    if (
+      passedCount !== undefined &&
+      failedCount !== undefined &&
+      explicitSkippedCount !== undefined &&
+      totalCount !== providedCountSum
+    ) {
+      return {
+        passed: false,
+        status: "unknown",
+        details: `qa-suite-summary counts.total=${totalCount} does not match counts.passed+counts.failed+counts.skipped=${
+          providedCountSum
+        }`,
+      };
+    }
+  }
   const scenarios = Array.isArray(payload.scenarios) ? payload.scenarios : undefined;
   const failedScenarios = scenarios?.filter(
     (scenario) => isRecord(scenario) && scenario.status === "fail",
@@ -446,7 +485,6 @@ function evaluateQaSuiteSummary(payload: unknown): QaConfidenceLaneEvaluation {
         details: `qa-suite-summary has ${unknownBlockingScenarioCount} scenario row(s) with unsupported non-pass status`,
       };
     }
-    const explicitSkippedCount = readNumber(counts?.skipped);
     const inferredSkippedCount =
       totalCount === undefined || passedCount === undefined
         ? undefined