From eab05ddffb87b46d9bd6b1a863eb1bdb77c5c948 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Wed, 22 Jul 2026 16:54:36 +0900 Subject: [PATCH] fix(qa): preserve partial maturity evidence (#112569) --- .github/workflows/qa-profile-evidence.yml | 4 +- .../qa-lab/src/suite-launch.runtime.test.ts | 50 ++++++++++++- extensions/qa-lab/src/suite-launch.runtime.ts | 70 +++++++++++++++++++ test/scripts/ci-workflow-guards.test.ts | 12 ++-- 4 files changed, 127 insertions(+), 9 deletions(-) diff --git a/.github/workflows/qa-profile-evidence.yml b/.github/workflows/qa-profile-evidence.yml index 791ccfd27e48..68bb70491b4c 100644 --- a/.github/workflows/qa-profile-evidence.yml +++ b/.github/workflows/qa-profile-evidence.yml @@ -289,7 +289,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} - OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000" + OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "120000" OPENCLAW_QA_CREDENTIAL_ROLE: ci OPENCLAW_QA_CREDENTIAL_SOURCE: convex shell: bash @@ -303,7 +303,7 @@ jobs: pnpm openclaw qa run \ --repo-root . \ --qa-profile "${QA_PROFILE}" \ - --concurrency 6 \ + --concurrency 3 \ --fast \ --output-dir "${output_dir}" || qa_exit_code=$? diff --git a/extensions/qa-lab/src/suite-launch.runtime.test.ts b/extensions/qa-lab/src/suite-launch.runtime.test.ts index f08ebc6ab5c0..6c463be53a1f 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.test.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.test.ts @@ -1128,7 +1128,13 @@ describe("qa suite runtime launcher", () => { const testFileBlocked = new Promise((resolve) => { releaseTestFile = resolve; }); - runQaFlowSuite.mockRejectedValueOnce(new Error("flow partition failed")); + runQaFlowSuite.mockRejectedValueOnce( + new Error("flow partition failed", { + cause: Object.assign(new Error("unrelated capacity failure"), { + code: "POOL_EXHAUSTED", + }), + }), + ); runQaTestFileScenarios.mockImplementationOnce( async (params: { outputDir: string; @@ -1172,6 +1178,48 @@ describe("qa suite runtime launcher", () => { expect(rejected).toBe(true); }); + it("records unavailable channel credentials as blocked evidence", async () => { + const repoRoot = await makeTempRepo("qa-suite-credential-unavailable-"); + const poolError = Object.assign(new Error("no WhatsApp credential is available"), { + code: "POOL_EXHAUSTED", + }); + runQaFlowSuite.mockRejectedValueOnce( + new Error("failed to create QA transport live:whatsapp: credential acquire failed", { + cause: new Error("credential acquire timed out", { cause: poolError }), + }), + ); + + const result = await runQaSuite({ + repoRoot, + outputDir: ".artifacts/qa-e2e/credential-unavailable", + providerMode: "mock-openai", + channelDriver: "live", + adapterFactories: [{ id: "whatsapp", matches: () => true, create: vi.fn() }], + scenarioIds: ["whatsapp-status-command", "control-ui-chat-flow-playwright"], + }); + + expect(result.executionKind).toBe("suite"); + if (result.executionKind !== "suite") { + throw new Error("expected unified suite result"); + } + expect(result.result.scenarios[0]).toMatchObject({ + status: "fail", + details: expect.stringContaining("channel credential unavailable"), + }); + const evidence = JSON.parse(await fs.readFile(result.result.evidencePath, "utf8")) as { + entries?: Array<{ + execution?: { channel?: { id?: string } }; + result?: { status?: string }; + test?: { id?: string }; + }>; + }; + const blocked = evidence.entries?.find((entry) => entry.test?.id === "whatsapp-status-command"); + expect(blocked).toMatchObject({ + execution: { channel: { id: "whatsapp" } }, + result: { status: "blocked" }, + }); + }); + it("shares ordinary flow scenarios and isolates flow scenarios with config patches", async () => { const repoRoot = await makeTempRepo("qa-suite-partition-"); const result = await runQaSuite({ diff --git a/extensions/qa-lab/src/suite-launch.runtime.ts b/extensions/qa-lab/src/suite-launch.runtime.ts index 008e1b10dac1..35362ac733c6 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.ts @@ -1,11 +1,13 @@ // Qa Lab plugin module implements suite launch behavior. import fs from "node:fs/promises"; import path from "node:path"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { isRepoRootRelativeRef, toRepoRelativePath } from "./cli-paths.js"; import { QA_EVIDENCE_FILENAME, QA_EVIDENCE_SUMMARY_KIND, QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, + buildQaSuiteEvidenceSummary, validateQaEvidenceSummaryJson, type QaEvidenceSummaryJson, } from "./evidence-summary.js"; @@ -77,6 +79,7 @@ type QaSuiteExecutionPlan = const MAX_SHARED_FLOW_PARTITIONS = 4; const MAX_ISOLATED_FLOW_CONCURRENCY = 8; const ISOLATED_FLOW_WORKER_START_STAGGER_MS = 1_500; +const CREDENTIAL_POOL_UNAVAILABLE_CODES = new Set(["NO_CREDENTIAL_AVAILABLE", "POOL_EXHAUSTED"]); type QaUnifiedPartitionResult = { evidenceSummaries: QaEvidenceSummaryJson[]; @@ -466,6 +469,30 @@ function mergeQaEvidenceSummaries(params: { }); } +function hasCredentialPoolUnavailableCode(error: unknown): boolean { + if (!(error instanceof Error)) { + return false; + } + return ( + ("code" in error && CREDENTIAL_POOL_UNAVAILABLE_CODES.has(String(error.code))) || + hasCredentialPoolUnavailableCode(error.cause) + ); +} + +function isChannelCredentialPoolUnavailable( + error: unknown, + channelId: string | undefined, +): boolean { + if (!channelId || !(error instanceof Error)) { + return false; + } + return ( + (error.message.startsWith(`failed to create QA transport live:${channelId}:`) && + hasCredentialPoolUnavailableCode(error.cause)) || + isChannelCredentialPoolUnavailable(error.cause, channelId) + ); +} + function testFileScenarioResultToSuiteScenario( result: QaTestFileScenarioRunResult["results"][number], repoRoot: string, @@ -721,7 +748,50 @@ async function runUnifiedQaSuite(params: { )) : params.runParams?.workerStartStaggerMs, scenarioIds: partition.scenarios.map((scenario) => scenario.id), + }).catch((error: unknown) => { + if (!isChannelCredentialPoolUnavailable(error, channelGroup.channelId)) { + throw error; + } + // Preserve other channels' evidence, but keep the suite failed: maturity + // docs must not publish until every required channel can run. + const details = `channel credential unavailable: ${formatErrorMessage(error)}`; + const blockedResults = partition.scenarios.map((scenario) => ({ + name: scenario.title, + status: "blocked" as const, + details, + })); + return { + evidenceSummaries: [ + buildQaSuiteEvidenceSummary({ + artifactPaths: [], + evidenceMode: params.runParams?.evidenceMode, + channelId: channelGroup.channelId ?? transportId, + channelDriver: + params.runParams?.channelDriver ?? + channelGroup.channelDriverSelection?.channelDriver, + env: process.env, + generatedAt: new Date().toISOString(), + primaryModel, + providerMode, + repoRoot, + scenarioDefinitions: partition.scenarios, + scenarioResults: blockedResults, + }), + ], + scenarioResults: partition.scenarios.map((scenario) => ({ + scenarioId: scenario.id, + result: { + name: scenario.title, + status: "fail", + details, + steps: [{ name: "Acquire channel credential", status: "fail", details }], + }, + })), + } satisfies QaUnifiedPartitionResult; }); + if ("evidenceSummaries" in result) { + return result; + } const scenarioResults: QaUnifiedPartitionResult["scenarioResults"] = []; for (const [index, scenario] of partition.scenarios.entries()) { const scenarioResult = result.scenarios[index]; diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 504f78b13b14..d426401eb88e 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -4641,8 +4641,13 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" const runProfileStep = qaRunJob.steps.find( (step: WorkflowStep) => step.name === "Run QA profile", ); - expect(runProfileStep.run).toContain("--concurrency 6"); + expect(runProfileStep.env?.OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS).toBe("120000"); + expect(runProfileStep.run).toContain("--concurrency 3"); expect(runProfileStep.run).toContain("--fast"); + const failProfileStep = qaRunJob.steps.find( + (step: WorkflowStep) => step.name === "Fail if QA profile failed", + ); + expect(failProfileStep.if).toBe("always()"); expect(generateJob.needs).toEqual(["validate_selected_ref", "publisher_preflight"]); expect(generateJob.if.replace(/\s+/gu, " ")).toBe( "${{ always() && needs.validate_selected_ref.result == 'success' && (!inputs.publish_pull_request || needs.publisher_preflight.result == 'success') && inputs.qa_evidence_run_id == '' }}", @@ -4812,11 +4817,6 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" "if-no-files-found": "error", }); - const qaFailStep = qaRunJob.steps.find( - (step: WorkflowStep) => step.name === "Fail if QA profile failed", - ); - expect(qaFailStep.if).toBe("always()"); - const renderCheckoutStep = publishJob.steps.find( (step: WorkflowStep) => step.name === "Checkout selected ref", );