ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(qa): preserve partial maturity evidence (#112569)

Browse Source
This commit is contained in:
Dallin Romney
2026-07-22 16:54:36 +09:00
committed by GitHub
parent f0de4cfe75
commit eab05ddffb
4 changed files with 127 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/qa-profile-evidence.yml
+2 -2
View File
@@ -289,7 +289,7 @@ jobs:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }}
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000"
OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "120000"
OPENCLAW_QA_CREDENTIAL_ROLE: ci
OPENCLAW_QA_CREDENTIAL_SOURCE: convex
shell: bash
@@ -303,7 +303,7 @@ jobs:
pnpm openclaw qa run \
--repo-root . \
--qa-profile "${QA_PROFILE}" \
--concurrency 6 \
--concurrency 3 \
--fast \
--output-dir "${output_dir}" || qa_exit_code=$?
extensions/qa-lab/src/suite-launch.runtime.test.ts
+49 -1
View File
@@ -1128,7 +1128,13 @@ describe("qa suite runtime launcher", () => {
const testFileBlocked = new Promise<void>((resolve) => {
releaseTestFile = resolve;
});
runQaFlowSuite.mockRejectedValueOnce(new Error("flow partition failed"));
runQaFlowSuite.mockRejectedValueOnce(
new Error("flow partition failed", {
cause: Object.assign(new Error("unrelated capacity failure"), {
code: "POOL_EXHAUSTED",
}),
}),
);
runQaTestFileScenarios.mockImplementationOnce(
async (params: {
outputDir: string;
@@ -1172,6 +1178,48 @@ describe("qa suite runtime launcher", () => {
expect(rejected).toBe(true);
});
it("records unavailable channel credentials as blocked evidence", async () => {
const repoRoot = await makeTempRepo("qa-suite-credential-unavailable-");
const poolError = Object.assign(new Error("no WhatsApp credential is available"), {
code: "POOL_EXHAUSTED",
});
runQaFlowSuite.mockRejectedValueOnce(
new Error("failed to create QA transport live:whatsapp: credential acquire failed", {
cause: new Error("credential acquire timed out", { cause: poolError }),
}),
);
const result = await runQaSuite({
repoRoot,
outputDir: ".artifacts/qa-e2e/credential-unavailable",
providerMode: "mock-openai",
channelDriver: "live",
adapterFactories: [{ id: "whatsapp", matches: () => true, create: vi.fn() }],
scenarioIds: ["whatsapp-status-command", "control-ui-chat-flow-playwright"],
});
expect(result.executionKind).toBe("suite");
if (result.executionKind !== "suite") {
throw new Error("expected unified suite result");
}
expect(result.result.scenarios[0]).toMatchObject({
status: "fail",
details: expect.stringContaining("channel credential unavailable"),
});
const evidence = JSON.parse(await fs.readFile(result.result.evidencePath, "utf8")) as {
entries?: Array<{
execution?: { channel?: { id?: string } };
result?: { status?: string };
test?: { id?: string };
}>;
};
const blocked = evidence.entries?.find((entry) => entry.test?.id === "whatsapp-status-command");
expect(blocked).toMatchObject({
execution: { channel: { id: "whatsapp" } },
result: { status: "blocked" },
});
});
it("shares ordinary flow scenarios and isolates flow scenarios with config patches", async () => {
const repoRoot = await makeTempRepo("qa-suite-partition-");
const result = await runQaSuite({
extensions/qa-lab/src/suite-launch.runtime.ts
+70
View File
@@ -1,11 +1,13 @@
// Qa Lab plugin module implements suite launch behavior.
import fs from "node:fs/promises";
import path from "node:path";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import { isRepoRootRelativeRef, toRepoRelativePath } from "./cli-paths.js";
import {
QA_EVIDENCE_FILENAME,
QA_EVIDENCE_SUMMARY_KIND,
QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
buildQaSuiteEvidenceSummary,
validateQaEvidenceSummaryJson,
type QaEvidenceSummaryJson,
} from "./evidence-summary.js";
@@ -77,6 +79,7 @@ type QaSuiteExecutionPlan =
const MAX_SHARED_FLOW_PARTITIONS = 4;
const MAX_ISOLATED_FLOW_CONCURRENCY = 8;
const ISOLATED_FLOW_WORKER_START_STAGGER_MS = 1_500;
const CREDENTIAL_POOL_UNAVAILABLE_CODES = new Set(["NO_CREDENTIAL_AVAILABLE", "POOL_EXHAUSTED"]);
type QaUnifiedPartitionResult = {
evidenceSummaries: QaEvidenceSummaryJson[];
@@ -466,6 +469,30 @@ function mergeQaEvidenceSummaries(params: {
});
}
function hasCredentialPoolUnavailableCode(error: unknown): boolean {
if (!(error instanceof Error)) {
return false;
}
return (
("code" in error && CREDENTIAL_POOL_UNAVAILABLE_CODES.has(String(error.code))) ||
hasCredentialPoolUnavailableCode(error.cause)
);
}
function isChannelCredentialPoolUnavailable(
error: unknown,
channelId: string | undefined,
): boolean {
if (!channelId || !(error instanceof Error)) {
return false;
}
return (
(error.message.startsWith(`failed to create QA transport live:${channelId}:`) &&
hasCredentialPoolUnavailableCode(error.cause)) ||
isChannelCredentialPoolUnavailable(error.cause, channelId)
);
}
function testFileScenarioResultToSuiteScenario(
result: QaTestFileScenarioRunResult["results"][number],
repoRoot: string,
@@ -721,7 +748,50 @@ async function runUnifiedQaSuite(params: {
))
: params.runParams?.workerStartStaggerMs,
scenarioIds: partition.scenarios.map((scenario) => scenario.id),
}).catch((error: unknown) => {
if (!isChannelCredentialPoolUnavailable(error, channelGroup.channelId)) {
throw error;
}
// Preserve other channels' evidence, but keep the suite failed: maturity
// docs must not publish until every required channel can run.
const details = `channel credential unavailable: ${formatErrorMessage(error)}`;
const blockedResults = partition.scenarios.map((scenario) => ({
name: scenario.title,
status: "blocked" as const,
details,
}));
return {
evidenceSummaries: [
buildQaSuiteEvidenceSummary({
artifactPaths: [],
evidenceMode: params.runParams?.evidenceMode,
channelId: channelGroup.channelId ?? transportId,
channelDriver:
params.runParams?.channelDriver ??
channelGroup.channelDriverSelection?.channelDriver,
env: process.env,
generatedAt: new Date().toISOString(),
primaryModel,
providerMode,
repoRoot,
scenarioDefinitions: partition.scenarios,
scenarioResults: blockedResults,
}),
],
scenarioResults: partition.scenarios.map((scenario) => ({
scenarioId: scenario.id,
result: {
name: scenario.title,
status: "fail",
details,
steps: [{ name: "Acquire channel credential", status: "fail", details }],
},
})),
} satisfies QaUnifiedPartitionResult;
});
if ("evidenceSummaries" in result) {
return result;
}
const scenarioResults: QaUnifiedPartitionResult["scenarioResults"] = [];
for (const [index, scenario] of partition.scenarios.entries()) {
const scenarioResult = result.scenarios[index];
test/scripts/ci-workflow-guards.test.ts
+6 -6
View File
@@ -4641,8 +4641,13 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}"
const runProfileStep = qaRunJob.steps.find(
(step: WorkflowStep) => step.name === "Run QA profile",
);
expect(runProfileStep.run).toContain("--concurrency 6");
expect(runProfileStep.env?.OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS).toBe("120000");
expect(runProfileStep.run).toContain("--concurrency 3");
expect(runProfileStep.run).toContain("--fast");
const failProfileStep = qaRunJob.steps.find(
(step: WorkflowStep) => step.name === "Fail if QA profile failed",
);
expect(failProfileStep.if).toBe("always()");
expect(generateJob.needs).toEqual(["validate_selected_ref", "publisher_preflight"]);
expect(generateJob.if.replace(/\s+/gu, " ")).toBe(
"${{ always() && needs.validate_selected_ref.result == 'success' && (!inputs.publish_pull_request || needs.publisher_preflight.result == 'success') && inputs.qa_evidence_run_id == '' }}",
@@ -4812,11 +4817,6 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}"
"if-no-files-found": "error",
});
const qaFailStep = qaRunJob.steps.find(
(step: WorkflowStep) => step.name === "Fail if QA profile failed",
);
expect(qaFailStep.if).toBe("always()");
const renderCheckoutStep = publishJob.steps.find(
(step: WorkflowStep) => step.name === "Checkout selected ref",
);