From 9b85b36d92bc1bd5e7d96ae0a43bdcf957e80d3d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 7 Jun 2026 10:30:15 +0200 Subject: [PATCH] fix(qa): fail whatsapp skipped scenarios --- .../whatsapp/cli.runtime.test.ts | 89 +++++++++++++++++++ .../live-transports/whatsapp/cli.runtime.ts | 8 +- extensions/qa-lab/src/suite-summary.test.ts | 49 ++++++++++ extensions/qa-lab/src/suite-summary.ts | 86 ++++++++++++++++-- 4 files changed, 224 insertions(+), 8 deletions(-) create mode 100644 extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts diff --git a/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts b/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts new file mode 100644 index 000000000000..499ca1cf27d6 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts @@ -0,0 +1,89 @@ +// Qa Lab tests cover WhatsApp live transport cli runtime behavior. +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { runQaWhatsAppCommand } from "./cli.runtime.js"; + +const runWhatsAppQaLiveMock = vi.hoisted(() => vi.fn()); + +vi.mock("../shared/live-artifacts.js", () => ({ + printLiveTransportQaArtifacts: vi.fn(), +})); + +vi.mock("../shared/live-transport-cli.runtime.js", () => ({ + resolveLiveTransportQaRunOptions: (opts: Record) => ({ + outputDir: opts.repoRoot, + providerMode: "mock-openai", + repoRoot: opts.repoRoot, + ...opts, + }), +})); + +vi.mock("./whatsapp-live.runtime.js", () => ({ + runWhatsAppQaLive: runWhatsAppQaLiveMock, +})); + +const tempDirs: string[] = []; +let originalExitCode: string | number | undefined; + +afterEach(async () => { + process.exitCode = originalExitCode; + runWhatsAppQaLiveMock.mockReset(); + for (const dir of tempDirs.splice(0)) { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +async function writeSummary(summary: unknown) { + const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-whatsapp-cli-")); + tempDirs.push(outputDir); + const summaryPath = path.join(outputDir, "whatsapp-qa-summary.json"); + await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8"); + return { outputDir, summaryPath }; +} + +describe("WhatsApp QA CLI runtime", () => { + it("fails when a standard scenario is skipped by default", async () => { + originalExitCode = process.exitCode; + process.exitCode = undefined; + const { outputDir, summaryPath } = await writeSummary({ + counts: { total: 1, passed: 0, failed: 0, skipped: 1 }, + scenarios: [ + { + id: "whatsapp-mention-gating", + status: "skip", + }, + ], + }); + runWhatsAppQaLiveMock.mockResolvedValueOnce({ + observedMessagesPath: path.join(outputDir, "observed.json"), + reportPath: path.join(outputDir, "report.md"), + scenarios: [], + summaryPath, + }); + + await runQaWhatsAppCommand({ repoRoot: outputDir }); + + expect(process.exitCode).toBe(1); + }); + + it("allows skipped scenarios when failures are explicitly allowed", async () => { + originalExitCode = process.exitCode; + process.exitCode = undefined; + const { outputDir, summaryPath } = await writeSummary({ + counts: { total: 1, passed: 0, failed: 0, skipped: 1 }, + scenarios: [{ id: "whatsapp-mention-gating", status: "skip" }], + }); + runWhatsAppQaLiveMock.mockResolvedValueOnce({ + observedMessagesPath: path.join(outputDir, "observed.json"), + reportPath: path.join(outputDir, "report.md"), + scenarios: [], + summaryPath, + }); + + await runQaWhatsAppCommand({ allowFailures: true, repoRoot: outputDir }); + + expect(process.exitCode).toBeUndefined(); + }); +}); diff --git a/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.ts b/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.ts index 8d84bc8744c2..457e234f7e87 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.ts @@ -1,4 +1,4 @@ -import { readQaSuiteFailedScenarioCountFromFile } from "../../suite-summary.js"; +import { readQaSuiteFailedOrSkippedScenarioCountFromFile } from "../../suite-summary.js"; // Qa Lab plugin module implements cli behavior. import { printLiveTransportQaArtifacts } from "../shared/live-artifacts.js"; import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js"; @@ -15,8 +15,10 @@ export async function runQaWhatsAppCommand(opts: LiveTransportQaCommandOptions) ...(result.gatewayDebugDirPath ? { "gateway debug logs": result.gatewayDebugDirPath } : {}), }); if (!runOptions.allowFailures) { - const failedScenarioCount = await readQaSuiteFailedScenarioCountFromFile(result.summaryPath); - if (failedScenarioCount > 0) { + const blockingScenarioCount = await readQaSuiteFailedOrSkippedScenarioCountFromFile( + result.summaryPath, + ); + if (blockingScenarioCount > 0) { process.exitCode = 1; } } diff --git a/extensions/qa-lab/src/suite-summary.test.ts b/extensions/qa-lab/src/suite-summary.test.ts index 6e813e510fa5..b98c765fb493 100644 --- a/extensions/qa-lab/src/suite-summary.test.ts +++ b/extensions/qa-lab/src/suite-summary.test.ts @@ -4,7 +4,10 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; import { + countQaSuiteFailedOrSkippedScenarios, countQaSuiteFailedScenarios, + readQaSuiteFailedOrSkippedScenarioCountFromFile, + readQaSuiteFailedOrSkippedScenarioCountFromSummary, readQaSuiteFailedScenarioCountFromFile, readQaSuiteFailedScenarioCountFromSummary, } from "./suite-summary.js"; @@ -16,6 +19,17 @@ describe("qa suite summary helpers", () => { ).toBe(2); }); + it("counts failed and skipped scenarios from scenario statuses", () => { + expect( + countQaSuiteFailedOrSkippedScenarios([ + { status: "pass" }, + { status: "skip" }, + { status: "skipped" }, + { status: "fail" }, + ]), + ).toBe(3); + }); + it("uses the larger failure signal when counts and scenarios disagree", () => { expect( readQaSuiteFailedScenarioCountFromSummary({ @@ -41,6 +55,22 @@ describe("qa suite summary helpers", () => { ).toBe(1); }); + it("uses the larger blocking signal when skipped counts and scenarios disagree", () => { + expect( + readQaSuiteFailedOrSkippedScenarioCountFromSummary({ + counts: { failed: 0, skipped: 1 }, + scenarios: [{ status: "pass" }], + }), + ).toBe(1); + + expect( + readQaSuiteFailedOrSkippedScenarioCountFromSummary({ + counts: { failed: 0, skipped: 0 }, + scenarios: [{ status: "skip" }, { status: "fail" }], + }), + ).toBe(2); + }); + it("returns null for unsupported summary shapes", () => { expect(readQaSuiteFailedScenarioCountFromSummary({ counts: { total: 2 } })).toBeNull(); expect(readQaSuiteFailedScenarioCountFromSummary("not-json-object")).toBeNull(); @@ -65,6 +95,25 @@ describe("qa suite summary helpers", () => { } }); + it("reads failed or skipped scenario counts from summary files", async () => { + const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-suite-summary-")); + const summaryPath = path.join(outputDir, "qa-suite-summary.json"); + await fs.writeFile( + summaryPath, + JSON.stringify({ + counts: { failed: 0, skipped: 1 }, + scenarios: [{ status: "pass" }], + }), + "utf8", + ); + + try { + await expect(readQaSuiteFailedOrSkippedScenarioCountFromFile(summaryPath)).resolves.toBe(1); + } finally { + await fs.rm(outputDir, { recursive: true, force: true }); + } + }); + it("fails summary files without a failure signal", async () => { const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-suite-summary-")); const summaryPath = path.join(outputDir, "qa-suite-summary.json"); diff --git a/extensions/qa-lab/src/suite-summary.ts b/extensions/qa-lab/src/suite-summary.ts index a9250acba8bc..31aee479444e 100644 --- a/extensions/qa-lab/src/suite-summary.ts +++ b/extensions/qa-lab/src/suite-summary.ts @@ -6,7 +6,7 @@ import type { RuntimeId, RuntimeParityResult } from "./runtime-parity.js"; type QaSuiteSummaryScenario = { name: string; - status: "pass" | "fail"; + status: "pass" | "fail" | "skip" | "skipped"; steps: unknown[]; details?: string; runtimeParity?: RuntimeParityResult; @@ -59,6 +59,12 @@ export type QaSuiteSummaryJson = { type QaSuiteScenarioStatus = Pick; +function readNonNegativeCount(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) + ? Math.max(0, Math.floor(value)) + : null; +} + export function countQaSuiteFailedScenarios( scenarios: ReadonlyArray, ): number { @@ -71,6 +77,18 @@ export function countQaSuiteFailedScenarios( return failed; } +export function countQaSuiteFailedOrSkippedScenarios( + scenarios: ReadonlyArray, +): number { + let blocking = 0; + for (const scenario of scenarios) { + if (scenario.status === "fail" || scenario.status === "skip" || scenario.status === "skipped") { + blocking += 1; + } + } + return blocking; +} + export function readQaSuiteFailedScenarioCountFromSummary(summary: unknown): number | null { if (!summary || typeof summary !== "object") { return null; @@ -81,10 +99,7 @@ export function readQaSuiteFailedScenarioCountFromSummary(summary: unknown): num }; scenarios?: Array; }; - const countedFailures = - typeof payload.counts?.failed === "number" && Number.isFinite(payload.counts.failed) - ? Math.max(0, Math.floor(payload.counts.failed)) - : null; + const countedFailures = readNonNegativeCount(payload.counts?.failed); const scenarioFailures = Array.isArray(payload.scenarios) ? countQaSuiteFailedScenarios(payload.scenarios) : null; @@ -97,6 +112,37 @@ export function readQaSuiteFailedScenarioCountFromSummary(summary: unknown): num return countedFailures; } +export function readQaSuiteFailedOrSkippedScenarioCountFromSummary( + summary: unknown, +): number | null { + if (!summary || typeof summary !== "object") { + return null; + } + const payload = summary as { + counts?: { + failed?: unknown; + skipped?: unknown; + }; + scenarios?: Array; + }; + const countedFailures = readNonNegativeCount(payload.counts?.failed); + const countedSkipped = readNonNegativeCount(payload.counts?.skipped); + const countedBlocking = + countedFailures !== null || countedSkipped !== null + ? (countedFailures ?? 0) + (countedSkipped ?? 0) + : null; + const scenarioBlocking = Array.isArray(payload.scenarios) + ? countQaSuiteFailedOrSkippedScenarios(payload.scenarios) + : null; + if (countedBlocking !== null && scenarioBlocking !== null) { + return Math.max(countedBlocking, scenarioBlocking); + } + if (scenarioBlocking !== null) { + return scenarioBlocking; + } + return countedBlocking; +} + export async function readQaSuiteFailedScenarioCountFromFile(summaryPath: string): Promise { let summaryText: string; try { @@ -124,3 +170,33 @@ export async function readQaSuiteFailedScenarioCountFromFile(summaryPath: string `QA summary at ${summaryPath} did not include counts.failed or scenarios[].status.`, ); } + +export async function readQaSuiteFailedOrSkippedScenarioCountFromFile( + summaryPath: string, +): Promise { + let summaryText: string; + try { + summaryText = await fs.readFile(summaryPath, "utf8"); + } catch (error) { + throw new Error( + `Could not read QA summary JSON at ${summaryPath}: ${formatErrorMessage(error)}`, + { cause: error }, + ); + } + let payload: unknown; + try { + payload = JSON.parse(summaryText) as unknown; + } catch (error) { + throw new Error( + `Could not parse QA summary JSON at ${summaryPath}: ${formatErrorMessage(error)}`, + { cause: error }, + ); + } + const blockingScenarioCount = readQaSuiteFailedOrSkippedScenarioCountFromSummary(payload); + if (blockingScenarioCount !== null) { + return blockingScenarioCount; + } + throw new Error( + `QA summary at ${summaryPath} did not include counts.failed, counts.skipped, or scenarios[].status.`, + ); +}