From b86062e5b4461ea06eb8826a9cf1a20caffca7ed Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 22 Aug 2026 01:56:22 -0700 Subject: [PATCH] fix(release): accept frozen QA completion evidence (#127806) --- .github/workflows/openclaw-release-checks.yml | 3 + scripts/validate-qa-runtime-pair-summary.mts | 71 +++++- test/scripts/ci-workflow-guards.test.ts | 20 ++ .../validate-qa-runtime-pair-summary.test.ts | 203 ++++++++++++++++++ 4 files changed, 293 insertions(+), 4 deletions(-) diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index 61f4140d29fe..5dc214a42ea0 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -1745,6 +1745,7 @@ jobs: summary=".artifacts/qa-e2e/${output_dir}/qa-suite-summary.json" validator_args=( --summary "$summary" + --candidate-suite-outcome "$CANDIDATE_SUITE_OUTCOME" --target-sha "$RELEASE_CHECK_TARGET_SHA" --lane "$RUNTIME_PAIR_LANE" ) @@ -1806,6 +1807,7 @@ jobs: if: always() env: CANDIDATE_REPORT_OUTCOME: ${{ steps.candidate_runtime_parity_report.outcome }} + CANDIDATE_SUITE_OUTCOME: ${{ steps.candidate_runtime_pair.outcome }} RELEASE_CHECK_TARGET_SHA: ${{ needs.resolve_target.outputs.revision }} RUNTIME_PAIR_LANE: ${{ matrix.lane }} run: | @@ -1817,6 +1819,7 @@ jobs: --summary "$summary" --report-summary "$report_dir/qa-runtime-parity-summary.json" --report-markdown "$report_dir/qa-runtime-parity-report.md" + --candidate-suite-outcome "$CANDIDATE_SUITE_OUTCOME" --target-sha "$RELEASE_CHECK_TARGET_SHA" --lane "$RUNTIME_PAIR_LANE" ) diff --git a/scripts/validate-qa-runtime-pair-summary.mts b/scripts/validate-qa-runtime-pair-summary.mts index 9584430dddc3..30028d81ca45 100644 --- a/scripts/validate-qa-runtime-pair-summary.mts +++ b/scripts/validate-qa-runtime-pair-summary.mts @@ -62,6 +62,25 @@ const FROZEN_RUNTIME_PAIR_MANIFESTS = new Map([ ["311047822ecdde24e824d839ab105ef08f17be00:core", FROZEN_CORE_RUNTIME_PAIR_MANIFEST], ["c37af96b18776fecc9e24268f27fc89b563481bf:core", FROZEN_CORE_RUNTIME_PAIR_MANIFEST], ]); +const FROZEN_RUNTIME_PAIR_COMPATIBILITY_PROFILES = new Map([ + [ + "ee5ead24b1b46a3560f28f8d57e0afcd911acacb:core", + { + allowMissingRunStatus: true, + scenarioIds: FROZEN_CORE_RUNTIME_PAIR_MANIFEST.scenarioIds.filter( + (scenarioId) => + scenarioId !== "codex-plugin-pinned-new" && scenarioId !== "codex-plugin-pinned-old", + ), + }, + ], +]); + +type RuntimePairValidationOptions = { + candidateSuiteOutcome?: string; + lane?: string; + requireExplicitGap?: boolean; + targetSha?: string; +}; function isPassableCell(cell: unknown) { if (!isRecord(cell) || typeof cell.transportErrorClass === "string") { @@ -168,14 +187,50 @@ function requireCanonicalRuntimePair(runtimePair: unknown) { ); } +function isCanonicalIsoTimestamp(value: unknown): value is string { + if (typeof value !== "string") { + return false; + } + const timestamp = Date.parse(value); + return Number.isFinite(timestamp) && new Date(timestamp).toISOString() === value; +} + +function isTrustedMissingStatusRun( + run: Record, + options: RuntimePairValidationOptions, +) { + const profile = FROZEN_RUNTIME_PAIR_COMPATIBILITY_PROFILES.get( + `${options.targetSha}:${options.lane}`, + ); + if ( + Object.hasOwn(run, "status") || + options.candidateSuiteOutcome !== "success" || + profile?.allowMissingRunStatus !== true || + !isCanonicalIsoTimestamp(run.startedAt) || + !isCanonicalIsoTimestamp(run.finishedAt) || + Date.parse(run.finishedAt) < Date.parse(run.startedAt) + ) { + return false; + } + const scenarioIds = run.scenarioIds; + return ( + Array.isArray(scenarioIds) && + scenarioIds.length === profile.scenarioIds.length && + profile.scenarioIds.every((scenarioId, index) => scenarioIds[index] === scenarioId) + ); +} + export function validateQaRuntimePairSummary( summary: unknown, - options: { requireExplicitGap?: boolean; targetSha?: string; lane?: string } = {}, + options: RuntimePairValidationOptions = {}, ) { if (!isRecord(summary) || !isRecord(summary.run) || !Array.isArray(summary.scenarios)) { throw new Error("runtime-pair summary is missing run or scenario evidence"); } - if (summary.run.status !== "completed") { + // Frozen compatibility profiles may admit a specifically attested legacy + // producer shape. The trusted workflow outcome and exact manifest prevent a + // partial or failed producer from masquerading as terminal evidence. + if (summary.run.status !== "completed" && !isTrustedMissingStatusRun(summary.run, options)) { throw new Error("runtime-pair summary is not completed"); } if (!requireCanonicalRuntimePair(summary.run.runtimePair)) { @@ -253,7 +308,7 @@ export function validateQaRuntimePairReport( summary: unknown, reportSummary: unknown, reportMarkdown: string, - options: { requireExplicitGap?: boolean; targetSha?: string; lane?: string } = {}, + options: RuntimePairValidationOptions = {}, ) { const counts = validateQaRuntimePairSummary(summary, options); if ( @@ -355,6 +410,7 @@ export function parseArgs(argv: string[]) { let reportSummaryPath; let reportMarkdownPath; let requireExplicitGap = false; + let candidateSuiteOutcome; let targetSha; let lane; for (let index = 0; index < argv.length; index += 1) { @@ -378,6 +434,12 @@ export function parseArgs(argv: string[]) { index += 1; } else if (arg === "--require-explicit-gap") { requireExplicitGap = true; + } else if (arg === "--candidate-suite-outcome") { + candidateSuiteOutcome = argv[index + 1]; + if (!candidateSuiteOutcome || candidateSuiteOutcome.startsWith("-")) { + throw new Error("--candidate-suite-outcome requires a value"); + } + index += 1; } else if (arg === "--target-sha" || arg === "--lane") { const value = argv[index + 1]; if (!value || value.startsWith("-")) { @@ -391,7 +453,7 @@ export function parseArgs(argv: string[]) { index += 1; } else if (arg === "--help" || arg === "-h") { process.stdout.write( - "Usage: node --import tsx scripts/validate-qa-runtime-pair-summary.mts --summary [--require-explicit-gap] [--report-summary --report-markdown ]\n", + "Usage: node --import tsx scripts/validate-qa-runtime-pair-summary.mts --summary [--candidate-suite-outcome ] [--require-explicit-gap] [--report-summary --report-markdown ]\n", ); return undefined; } else { @@ -411,6 +473,7 @@ export function parseArgs(argv: string[]) { summaryPath, reportSummaryPath, reportMarkdownPath, + candidateSuiteOutcome, requireExplicitGap, targetSha, lane, diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 4779ab44759c..46f3cb40a596 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -1504,6 +1504,26 @@ ${actionRun}`; } describe("ci workflow guards", () => { + it("gates frozen runtime-pair compatibility on the trusted suite outcome", () => { + const workflow = readReleaseChecksWorkflow(); + const laneJob = workflow.jobs.qa_lab_runtime_pair_lane_release_checks; + const suiteValidation = laneJob.steps.find( + (step: WorkflowStep) => step.name === "Validate runtime-pair lane", + ); + const reportValidation = laneJob.steps.find( + (step: WorkflowStep) => step.name === "Validate runtime-pair lane report", + ); + + for (const step of [suiteValidation, reportValidation]) { + expect(step?.env?.CANDIDATE_SUITE_OUTCOME).toBe( + "${{ steps.candidate_runtime_pair.outcome }}", + ); + expect(step?.run).toContain('--candidate-suite-outcome "$CANDIDATE_SUITE_OUTCOME"'); + expect(step?.run).toContain('--target-sha "$RELEASE_CHECK_TARGET_SHA"'); + expect(step?.run).toContain('--lane "$RUNTIME_PAIR_LANE"'); + } + }); + it("retains pending same-SHA QA calls in the shared concurrency group", () => { const workflowPath = ".github/workflows/qa-live-transports-convex.yml"; const workflowSource = readFileSync(workflowPath, "utf8"); diff --git a/test/scripts/validate-qa-runtime-pair-summary.test.ts b/test/scripts/validate-qa-runtime-pair-summary.test.ts index e27036e61c71..a90c11d1b93c 100644 --- a/test/scripts/validate-qa-runtime-pair-summary.test.ts +++ b/test/scripts/validate-qa-runtime-pair-summary.test.ts @@ -106,6 +106,11 @@ const frozenCoreGapScenarioIds = new Set([ "runtime-tool-fs-write", "runtime-tool-grep", ]); +const frozenLegacyCoreScenarioIds = frozenCoreScenarioIds.filter( + (scenarioId) => + scenarioId !== "codex-plugin-pinned-new" && scenarioId !== "codex-plugin-pinned-old", +); +const frozenLegacyTargetSha = "ee5ead24b1b46a3560f28f8d57e0afcd911acacb"; function frozenCoreSummary() { return summary( @@ -125,6 +130,62 @@ function frozenCoreSummary() { ); } +function frozenLegacyStatuslessSummary() { + const fixture = summary( + frozenLegacyCoreScenarioIds.map((scenarioId) => + scenario({ + name: scenarioId, + status: "pass", + }), + ), + ); + delete (fixture.run as { status?: string }).status; + return Object.assign(fixture, { + run: { + ...fixture.run, + startedAt: "2026-08-22T06:02:37.608Z", + finishedAt: "2026-08-22T06:14:49.336Z", + }, + }); +} + +function reportFor(scenarios: ReturnType[]) { + return { + runtimePair: ["openclaw", "codex"], + totalScenarios: scenarios.length, + passedScenarios: scenarios.length, + failedScenarios: 0, + scenarios: scenarios.map((entry) => ({ + name: entry.name, + status: "pass", + drift: entry.runtimeParity.drift, + driftDetails: undefined, + openclawStatus: "pass", + codexStatus: "pass", + })), + failures: [], + pass: true, + }; +} + +function markdownFor(scenarios: ReturnType[]) { + return [ + "# OpenClaw Runtime Parity Report — openclaw vs codex", + "", + "- Verdict: pass", + ...scenarios.flatMap((entry) => [ + "", + `### ${entry.name}`, + "", + "- status: pass", + `- drift: ${entry.runtimeParity.drift}`, + "- openclaw: pass (0 tool calls)", + "- codex: pass (0 tool calls)", + ]), + "", + ].join("\n"); +} + describe("frozen QA runtime-pair summary validation", () => { it("rejects a nonterminal runtime-pair summary", () => { const fixture = summary([scenario({ name: "running", status: "pass" })]); @@ -185,6 +246,148 @@ describe("frozen QA runtime-pair summary validation", () => { }); }); + it("accepts the exact statusless frozen core profile after a successful suite", () => { + expect( + validateQaRuntimePairSummary(frozenLegacyStatuslessSummary(), { + candidateSuiteOutcome: "success", + targetSha: frozenLegacyTargetSha, + lane: "core", + }), + ).toEqual({ + total: 25, + passed: 25, + failed: 0, + skipped: 0, + }); + }); + + it("rejects a nonterminal status even for the exact frozen profile", () => { + const fixture = frozenLegacyStatuslessSummary(); + fixture.run.status = "running"; + + expect(() => + validateQaRuntimePairSummary(fixture, { + candidateSuiteOutcome: "success", + targetSha: frozenLegacyTargetSha, + lane: "core", + }), + ).toThrow("runtime-pair summary is not completed"); + }); + + it.each([ + ["missing suite outcome", { targetSha: frozenLegacyTargetSha, lane: "core" }], + [ + "failed suite", + { candidateSuiteOutcome: "failure", targetSha: frozenLegacyTargetSha, lane: "core" }, + ], + [ + "skipped suite", + { candidateSuiteOutcome: "skipped", targetSha: frozenLegacyTargetSha, lane: "core" }, + ], + [ + "cancelled suite", + { candidateSuiteOutcome: "cancelled", targetSha: frozenLegacyTargetSha, lane: "core" }, + ], + [ + "unknown suite outcome", + { candidateSuiteOutcome: "unknown", targetSha: frozenLegacyTargetSha, lane: "core" }, + ], + ["wrong target", { candidateSuiteOutcome: "success", targetSha: "0".repeat(40), lane: "core" }], + [ + "wrong lane", + { candidateSuiteOutcome: "success", targetSha: frozenLegacyTargetSha, lane: "soak" }, + ], + ])("rejects statusless frozen evidence with %s", (_label, options) => { + expect(() => validateQaRuntimePairSummary(frozenLegacyStatuslessSummary(), options)).toThrow( + "runtime-pair summary is not completed", + ); + }); + + it.each([ + ["missing startedAt", undefined, "2026-08-22T06:14:49.336Z"], + ["missing finishedAt", "2026-08-22T06:02:37.608Z", undefined], + ["invalid startedAt", "not-a-date", "2026-08-22T06:14:49.336Z"], + ["invalid finishedAt", "2026-08-22T06:02:37.608Z", "not-a-date"], + ["reversed timestamps", "2026-08-22T06:14:49.336Z", "2026-08-22T06:02:37.608Z"], + ])("rejects statusless frozen evidence with %s", (_label, startedAt, finishedAt) => { + const fixture = frozenLegacyStatuslessSummary(); + if (startedAt === undefined) { + delete (fixture.run as { startedAt?: string }).startedAt; + } else { + fixture.run.startedAt = startedAt; + } + if (finishedAt === undefined) { + delete (fixture.run as { finishedAt?: string }).finishedAt; + } else { + fixture.run.finishedAt = finishedAt; + } + + expect(() => + validateQaRuntimePairSummary(fixture, { + candidateSuiteOutcome: "success", + targetSha: frozenLegacyTargetSha, + lane: "core", + }), + ).toThrow("runtime-pair summary is not completed"); + }); + + it("rejects manifest drift before accepting a statusless frozen profile", () => { + const fixture = frozenLegacyStatuslessSummary(); + fixture.run.scenarioIds.reverse(); + fixture.scenarios.reverse(); + + expect(() => + validateQaRuntimePairSummary(fixture, { + candidateSuiteOutcome: "success", + targetSha: frozenLegacyTargetSha, + lane: "core", + }), + ).toThrow("runtime-pair summary is not completed"); + }); + + it("still rejects scenario and count failures after frozen profile admission", () => { + const options = { + candidateSuiteOutcome: "success", + targetSha: frozenLegacyTargetSha, + lane: "core", + }; + const failedScenario = frozenLegacyStatuslessSummary(); + failedScenario.scenarios[0]!.status = "fail"; + failedScenario.counts.passed -= 1; + failedScenario.counts.failed += 1; + expect(() => validateQaRuntimePairSummary(failedScenario, options)).toThrow( + "runtime-pair failure or unsupported skip", + ); + + const countDrift = frozenLegacyStatuslessSummary(); + countDrift.counts.passed -= 1; + expect(() => validateQaRuntimePairSummary(countDrift, options)).toThrow( + "counts do not match validated scenario evidence", + ); + }); + + it("applies the trusted suite outcome gate to frozen report validation", () => { + const fixture = frozenLegacyStatuslessSummary(); + const reportSummary = reportFor(fixture.scenarios); + const markdown = markdownFor(fixture.scenarios); + const options = { + candidateSuiteOutcome: "success", + targetSha: frozenLegacyTargetSha, + lane: "core", + }; + + expect(validateQaRuntimePairReport(fixture, reportSummary, markdown, options)).toMatchObject({ + total: 25, + passed: 25, + }); + expect(() => + validateQaRuntimePairReport(fixture, reportSummary, markdown, { + ...options, + candidateSuiteOutcome: "failure", + }), + ).toThrow("runtime-pair summary is not completed"); + }); + it("requires skipped count when validated evidence contains skips", () => { const fixture = summary([ scenario({