fix(qa): fail whatsapp skipped scenarios

extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts

@@ -0,0 +1,89 @@
+// Qa Lab tests cover WhatsApp live transport cli runtime behavior.
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { runQaWhatsAppCommand } from "./cli.runtime.js";
+
+const runWhatsAppQaLiveMock = vi.hoisted(() => vi.fn());
+
+vi.mock("../shared/live-artifacts.js", () => ({
+  printLiveTransportQaArtifacts: vi.fn(),
+}));
+
+vi.mock("../shared/live-transport-cli.runtime.js", () => ({
+  resolveLiveTransportQaRunOptions: (opts: Record<string, unknown>) => ({
+    outputDir: opts.repoRoot,
+    providerMode: "mock-openai",
+    repoRoot: opts.repoRoot,
+    ...opts,
+  }),
+}));
+
+vi.mock("./whatsapp-live.runtime.js", () => ({
+  runWhatsAppQaLive: runWhatsAppQaLiveMock,
+}));
+
+const tempDirs: string[] = [];
+let originalExitCode: string | number | undefined;
+
+afterEach(async () => {
+  process.exitCode = originalExitCode;
+  runWhatsAppQaLiveMock.mockReset();
+  for (const dir of tempDirs.splice(0)) {
+    await fs.rm(dir, { recursive: true, force: true });
+  }
+});
+
+async function writeSummary(summary: unknown) {
+  const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-whatsapp-cli-"));
+  tempDirs.push(outputDir);
+  const summaryPath = path.join(outputDir, "whatsapp-qa-summary.json");
+  await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8");
+  return { outputDir, summaryPath };
+}
+
+describe("WhatsApp QA CLI runtime", () => {
+  it("fails when a standard scenario is skipped by default", async () => {
+    originalExitCode = process.exitCode;
+    process.exitCode = undefined;
+    const { outputDir, summaryPath } = await writeSummary({
+      counts: { total: 1, passed: 0, failed: 0, skipped: 1 },
+      scenarios: [
+        {
+          id: "whatsapp-mention-gating",
+          status: "skip",
+        },
+      ],
+    });
+    runWhatsAppQaLiveMock.mockResolvedValueOnce({
+      observedMessagesPath: path.join(outputDir, "observed.json"),
+      reportPath: path.join(outputDir, "report.md"),
+      scenarios: [],
+      summaryPath,
+    });
+
+    await runQaWhatsAppCommand({ repoRoot: outputDir });
+
+    expect(process.exitCode).toBe(1);
+  });
+
+  it("allows skipped scenarios when failures are explicitly allowed", async () => {
+    originalExitCode = process.exitCode;
+    process.exitCode = undefined;
+    const { outputDir, summaryPath } = await writeSummary({
+      counts: { total: 1, passed: 0, failed: 0, skipped: 1 },
+      scenarios: [{ id: "whatsapp-mention-gating", status: "skip" }],
+    });
+    runWhatsAppQaLiveMock.mockResolvedValueOnce({
+      observedMessagesPath: path.join(outputDir, "observed.json"),
+      reportPath: path.join(outputDir, "report.md"),
+      scenarios: [],
+      summaryPath,
+    });
+
+    await runQaWhatsAppCommand({ allowFailures: true, repoRoot: outputDir });
+
+    expect(process.exitCode).toBeUndefined();
+  });
+});

extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.ts

@@ -1,4 +1,4 @@
-import { readQaSuiteFailedScenarioCountFromFile } from "../../suite-summary.js";
+import { readQaSuiteFailedOrSkippedScenarioCountFromFile } from "../../suite-summary.js";
 // Qa Lab plugin module implements cli behavior.
 import { printLiveTransportQaArtifacts } from "../shared/live-artifacts.js";
 import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js";
@@ -15,8 +15,10 @@ export async function runQaWhatsAppCommand(opts: LiveTransportQaCommandOptions)
     ...(result.gatewayDebugDirPath ? { "gateway debug logs": result.gatewayDebugDirPath } : {}),
   });
   if (!runOptions.allowFailures) {
-    const failedScenarioCount = await readQaSuiteFailedScenarioCountFromFile(result.summaryPath);
-    if (failedScenarioCount > 0) {
+    const blockingScenarioCount = await readQaSuiteFailedOrSkippedScenarioCountFromFile(
+      result.summaryPath,
+    );
+    if (blockingScenarioCount > 0) {
       process.exitCode = 1;
     }
   }

extensions/qa-lab/src/suite-summary.test.ts

@@ -4,7 +4,10 @@ import os from "node:os";
 import path from "node:path";
 import { describe, expect, it } from "vitest";
 import {
+  countQaSuiteFailedOrSkippedScenarios,
   countQaSuiteFailedScenarios,
+  readQaSuiteFailedOrSkippedScenarioCountFromFile,
+  readQaSuiteFailedOrSkippedScenarioCountFromSummary,
   readQaSuiteFailedScenarioCountFromFile,
   readQaSuiteFailedScenarioCountFromSummary,
 } from "./suite-summary.js";
@@ -16,6 +19,17 @@ describe("qa suite summary helpers", () => {
     ).toBe(2);
   });
 
+  it("counts failed and skipped scenarios from scenario statuses", () => {
+    expect(
+      countQaSuiteFailedOrSkippedScenarios([
+        { status: "pass" },
+        { status: "skip" },
+        { status: "skipped" },
+        { status: "fail" },
+      ]),
+    ).toBe(3);
+  });
+
   it("uses the larger failure signal when counts and scenarios disagree", () => {
     expect(
       readQaSuiteFailedScenarioCountFromSummary({
@@ -41,6 +55,22 @@ describe("qa suite summary helpers", () => {
     ).toBe(1);
   });
 
+  it("uses the larger blocking signal when skipped counts and scenarios disagree", () => {
+    expect(
+      readQaSuiteFailedOrSkippedScenarioCountFromSummary({
+        counts: { failed: 0, skipped: 1 },
+        scenarios: [{ status: "pass" }],
+      }),
+    ).toBe(1);
+
+    expect(
+      readQaSuiteFailedOrSkippedScenarioCountFromSummary({
+        counts: { failed: 0, skipped: 0 },
+        scenarios: [{ status: "skip" }, { status: "fail" }],
+      }),
+    ).toBe(2);
+  });
+
   it("returns null for unsupported summary shapes", () => {
     expect(readQaSuiteFailedScenarioCountFromSummary({ counts: { total: 2 } })).toBeNull();
     expect(readQaSuiteFailedScenarioCountFromSummary("not-json-object")).toBeNull();
@@ -65,6 +95,25 @@ describe("qa suite summary helpers", () => {
     }
   });
 
+  it("reads failed or skipped scenario counts from summary files", async () => {
+    const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-suite-summary-"));
+    const summaryPath = path.join(outputDir, "qa-suite-summary.json");
+    await fs.writeFile(
+      summaryPath,
+      JSON.stringify({
+        counts: { failed: 0, skipped: 1 },
+        scenarios: [{ status: "pass" }],
+      }),
+      "utf8",
+    );
+
+    try {
+      await expect(readQaSuiteFailedOrSkippedScenarioCountFromFile(summaryPath)).resolves.toBe(1);
+    } finally {
+      await fs.rm(outputDir, { recursive: true, force: true });
+    }
+  });
+
   it("fails summary files without a failure signal", async () => {
     const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-suite-summary-"));
     const summaryPath = path.join(outputDir, "qa-suite-summary.json");

extensions/qa-lab/src/suite-summary.ts

@@ -6,7 +6,7 @@ import type { RuntimeId, RuntimeParityResult } from "./runtime-parity.js";
 
 type QaSuiteSummaryScenario = {
   name: string;
-  status: "pass" | "fail";
+  status: "pass" | "fail" | "skip" | "skipped";
   steps: unknown[];
   details?: string;
   runtimeParity?: RuntimeParityResult;
@@ -59,6 +59,12 @@ export type QaSuiteSummaryJson = {
 
 type QaSuiteScenarioStatus = Pick<QaSuiteSummaryScenario, "status">;
 
+function readNonNegativeCount(value: unknown): number | null {
+  return typeof value === "number" && Number.isFinite(value)
+    ? Math.max(0, Math.floor(value))
+    : null;
+}
+
 export function countQaSuiteFailedScenarios(
   scenarios: ReadonlyArray<QaSuiteScenarioStatus>,
 ): number {
@@ -71,6 +77,18 @@ export function countQaSuiteFailedScenarios(
   return failed;
 }
 
+export function countQaSuiteFailedOrSkippedScenarios(
+  scenarios: ReadonlyArray<QaSuiteScenarioStatus>,
+): number {
+  let blocking = 0;
+  for (const scenario of scenarios) {
+    if (scenario.status === "fail" || scenario.status === "skip" || scenario.status === "skipped") {
+      blocking += 1;
+    }
+  }
+  return blocking;
+}
+
 export function readQaSuiteFailedScenarioCountFromSummary(summary: unknown): number | null {
   if (!summary || typeof summary !== "object") {
     return null;
@@ -81,10 +99,7 @@ export function readQaSuiteFailedScenarioCountFromSummary(summary: unknown): num
     };
     scenarios?: Array<QaSuiteScenarioStatus>;
   };
-  const countedFailures =
-    typeof payload.counts?.failed === "number" && Number.isFinite(payload.counts.failed)
-      ? Math.max(0, Math.floor(payload.counts.failed))
-      : null;
+  const countedFailures = readNonNegativeCount(payload.counts?.failed);
   const scenarioFailures = Array.isArray(payload.scenarios)
     ? countQaSuiteFailedScenarios(payload.scenarios)
     : null;
@@ -97,6 +112,37 @@ export function readQaSuiteFailedScenarioCountFromSummary(summary: unknown): num
   return countedFailures;
 }
 
+export function readQaSuiteFailedOrSkippedScenarioCountFromSummary(
+  summary: unknown,
+): number | null {
+  if (!summary || typeof summary !== "object") {
+    return null;
+  }
+  const payload = summary as {
+    counts?: {
+      failed?: unknown;
+      skipped?: unknown;
+    };
+    scenarios?: Array<QaSuiteScenarioStatus>;
+  };
+  const countedFailures = readNonNegativeCount(payload.counts?.failed);
+  const countedSkipped = readNonNegativeCount(payload.counts?.skipped);
+  const countedBlocking =
+    countedFailures !== null || countedSkipped !== null
+      ? (countedFailures ?? 0) + (countedSkipped ?? 0)
+      : null;
+  const scenarioBlocking = Array.isArray(payload.scenarios)
+    ? countQaSuiteFailedOrSkippedScenarios(payload.scenarios)
+    : null;
+  if (countedBlocking !== null && scenarioBlocking !== null) {
+    return Math.max(countedBlocking, scenarioBlocking);
+  }
+  if (scenarioBlocking !== null) {
+    return scenarioBlocking;
+  }
+  return countedBlocking;
+}
+
 export async function readQaSuiteFailedScenarioCountFromFile(summaryPath: string): Promise<number> {
   let summaryText: string;
   try {
@@ -124,3 +170,33 @@ export async function readQaSuiteFailedScenarioCountFromFile(summaryPath: string
     `QA summary at ${summaryPath} did not include counts.failed or scenarios[].status.`,
   );
 }
+
+export async function readQaSuiteFailedOrSkippedScenarioCountFromFile(
+  summaryPath: string,
+): Promise<number> {
+  let summaryText: string;
+  try {
+    summaryText = await fs.readFile(summaryPath, "utf8");
+  } catch (error) {
+    throw new Error(
+      `Could not read QA summary JSON at ${summaryPath}: ${formatErrorMessage(error)}`,
+      { cause: error },
+    );
+  }
+  let payload: unknown;
+  try {
+    payload = JSON.parse(summaryText) as unknown;
+  } catch (error) {
+    throw new Error(
+      `Could not parse QA summary JSON at ${summaryPath}: ${formatErrorMessage(error)}`,
+      { cause: error },
+    );
+  }
+  const blockingScenarioCount = readQaSuiteFailedOrSkippedScenarioCountFromSummary(payload);
+  if (blockingScenarioCount !== null) {
+    return blockingScenarioCount;
+  }
+  throw new Error(
+    `QA summary at ${summaryPath} did not include counts.failed, counts.skipped, or scenarios[].status.`,
+  );
+}