fix(release): accept frozen QA completion evidence (#127806)

.github/workflows/openclaw-release-checks.yml

@@ -1745,6 +1745,7 @@ jobs:
           summary=".artifacts/qa-e2e/${output_dir}/qa-suite-summary.json"
           validator_args=(
             --summary "$summary"
+            --candidate-suite-outcome "$CANDIDATE_SUITE_OUTCOME"
             --target-sha "$RELEASE_CHECK_TARGET_SHA"
             --lane "$RUNTIME_PAIR_LANE"
           )
@@ -1806,6 +1807,7 @@ jobs:
         if: always()
         env:
           CANDIDATE_REPORT_OUTCOME: ${{ steps.candidate_runtime_parity_report.outcome }}
+          CANDIDATE_SUITE_OUTCOME: ${{ steps.candidate_runtime_pair.outcome }}
           RELEASE_CHECK_TARGET_SHA: ${{ needs.resolve_target.outputs.revision }}
           RUNTIME_PAIR_LANE: ${{ matrix.lane }}
         run: |
@@ -1817,6 +1819,7 @@ jobs:
             --summary "$summary"
             --report-summary "$report_dir/qa-runtime-parity-summary.json"
             --report-markdown "$report_dir/qa-runtime-parity-report.md"
+            --candidate-suite-outcome "$CANDIDATE_SUITE_OUTCOME"
             --target-sha "$RELEASE_CHECK_TARGET_SHA"
             --lane "$RUNTIME_PAIR_LANE"
           )

scripts/validate-qa-runtime-pair-summary.mts

@@ -62,6 +62,25 @@ const FROZEN_RUNTIME_PAIR_MANIFESTS = new Map([
   ["311047822ecdde24e824d839ab105ef08f17be00:core", FROZEN_CORE_RUNTIME_PAIR_MANIFEST],
   ["c37af96b18776fecc9e24268f27fc89b563481bf:core", FROZEN_CORE_RUNTIME_PAIR_MANIFEST],
 ]);
+const FROZEN_RUNTIME_PAIR_COMPATIBILITY_PROFILES = new Map([
+  [
+    "ee5ead24b1b46a3560f28f8d57e0afcd911acacb:core",
+    {
+      allowMissingRunStatus: true,
+      scenarioIds: FROZEN_CORE_RUNTIME_PAIR_MANIFEST.scenarioIds.filter(
+        (scenarioId) =>
+          scenarioId !== "codex-plugin-pinned-new" && scenarioId !== "codex-plugin-pinned-old",
+      ),
+    },
+  ],
+]);
+
+type RuntimePairValidationOptions = {
+  candidateSuiteOutcome?: string;
+  lane?: string;
+  requireExplicitGap?: boolean;
+  targetSha?: string;
+};
 
 function isPassableCell(cell: unknown) {
   if (!isRecord(cell) || typeof cell.transportErrorClass === "string") {
@@ -168,14 +187,50 @@ function requireCanonicalRuntimePair(runtimePair: unknown) {
   );
 }
 
+function isCanonicalIsoTimestamp(value: unknown): value is string {
+  if (typeof value !== "string") {
+    return false;
+  }
+  const timestamp = Date.parse(value);
+  return Number.isFinite(timestamp) && new Date(timestamp).toISOString() === value;
+}
+
+function isTrustedMissingStatusRun(
+  run: Record<string, unknown>,
+  options: RuntimePairValidationOptions,
+) {
+  const profile = FROZEN_RUNTIME_PAIR_COMPATIBILITY_PROFILES.get(
+    `${options.targetSha}:${options.lane}`,
+  );
+  if (
+    Object.hasOwn(run, "status") ||
+    options.candidateSuiteOutcome !== "success" ||
+    profile?.allowMissingRunStatus !== true ||
+    !isCanonicalIsoTimestamp(run.startedAt) ||
+    !isCanonicalIsoTimestamp(run.finishedAt) ||
+    Date.parse(run.finishedAt) < Date.parse(run.startedAt)
+  ) {
+    return false;
+  }
+  const scenarioIds = run.scenarioIds;
+  return (
+    Array.isArray(scenarioIds) &&
+    scenarioIds.length === profile.scenarioIds.length &&
+    profile.scenarioIds.every((scenarioId, index) => scenarioIds[index] === scenarioId)
+  );
+}
+
 export function validateQaRuntimePairSummary(
   summary: unknown,
-  options: { requireExplicitGap?: boolean; targetSha?: string; lane?: string } = {},
+  options: RuntimePairValidationOptions = {},
 ) {
   if (!isRecord(summary) || !isRecord(summary.run) || !Array.isArray(summary.scenarios)) {
     throw new Error("runtime-pair summary is missing run or scenario evidence");
   }
-  if (summary.run.status !== "completed") {
+  // Frozen compatibility profiles may admit a specifically attested legacy
+  // producer shape. The trusted workflow outcome and exact manifest prevent a
+  // partial or failed producer from masquerading as terminal evidence.
+  if (summary.run.status !== "completed" && !isTrustedMissingStatusRun(summary.run, options)) {
     throw new Error("runtime-pair summary is not completed");
   }
   if (!requireCanonicalRuntimePair(summary.run.runtimePair)) {
@@ -253,7 +308,7 @@ export function validateQaRuntimePairReport(
   summary: unknown,
   reportSummary: unknown,
   reportMarkdown: string,
-  options: { requireExplicitGap?: boolean; targetSha?: string; lane?: string } = {},
+  options: RuntimePairValidationOptions = {},
 ) {
   const counts = validateQaRuntimePairSummary(summary, options);
   if (
@@ -355,6 +410,7 @@ export function parseArgs(argv: string[]) {
   let reportSummaryPath;
   let reportMarkdownPath;
   let requireExplicitGap = false;
+  let candidateSuiteOutcome;
   let targetSha;
   let lane;
   for (let index = 0; index < argv.length; index += 1) {
@@ -378,6 +434,12 @@ export function parseArgs(argv: string[]) {
       index += 1;
     } else if (arg === "--require-explicit-gap") {
       requireExplicitGap = true;
+    } else if (arg === "--candidate-suite-outcome") {
+      candidateSuiteOutcome = argv[index + 1];
+      if (!candidateSuiteOutcome || candidateSuiteOutcome.startsWith("-")) {
+        throw new Error("--candidate-suite-outcome requires a value");
+      }
+      index += 1;
     } else if (arg === "--target-sha" || arg === "--lane") {
       const value = argv[index + 1];
       if (!value || value.startsWith("-")) {
@@ -391,7 +453,7 @@ export function parseArgs(argv: string[]) {
       index += 1;
     } else if (arg === "--help" || arg === "-h") {
       process.stdout.write(
-        "Usage: node --import tsx scripts/validate-qa-runtime-pair-summary.mts --summary <qa-suite-summary.json> [--require-explicit-gap] [--report-summary <json> --report-markdown <md>]\n",
+        "Usage: node --import tsx scripts/validate-qa-runtime-pair-summary.mts --summary <qa-suite-summary.json> [--candidate-suite-outcome <outcome>] [--require-explicit-gap] [--report-summary <json> --report-markdown <md>]\n",
       );
       return undefined;
     } else {
@@ -411,6 +473,7 @@ export function parseArgs(argv: string[]) {
     summaryPath,
     reportSummaryPath,
     reportMarkdownPath,
+    candidateSuiteOutcome,
     requireExplicitGap,
     targetSha,
     lane,

test/scripts/ci-workflow-guards.test.ts

@@ -1504,6 +1504,26 @@ ${actionRun}`;
 }
 
 describe("ci workflow guards", () => {
+  it("gates frozen runtime-pair compatibility on the trusted suite outcome", () => {
+    const workflow = readReleaseChecksWorkflow();
+    const laneJob = workflow.jobs.qa_lab_runtime_pair_lane_release_checks;
+    const suiteValidation = laneJob.steps.find(
+      (step: WorkflowStep) => step.name === "Validate runtime-pair lane",
+    );
+    const reportValidation = laneJob.steps.find(
+      (step: WorkflowStep) => step.name === "Validate runtime-pair lane report",
+    );
+
+    for (const step of [suiteValidation, reportValidation]) {
+      expect(step?.env?.CANDIDATE_SUITE_OUTCOME).toBe(
+        "${{ steps.candidate_runtime_pair.outcome }}",
+      );
+      expect(step?.run).toContain('--candidate-suite-outcome "$CANDIDATE_SUITE_OUTCOME"');
+      expect(step?.run).toContain('--target-sha "$RELEASE_CHECK_TARGET_SHA"');
+      expect(step?.run).toContain('--lane "$RUNTIME_PAIR_LANE"');
+    }
+  });
+
   it("retains pending same-SHA QA calls in the shared concurrency group", () => {
     const workflowPath = ".github/workflows/qa-live-transports-convex.yml";
     const workflowSource = readFileSync(workflowPath, "utf8");

test/scripts/validate-qa-runtime-pair-summary.test.ts

@@ -106,6 +106,11 @@ const frozenCoreGapScenarioIds = new Set<string>([
   "runtime-tool-fs-write",
   "runtime-tool-grep",
 ]);
+const frozenLegacyCoreScenarioIds = frozenCoreScenarioIds.filter(
+  (scenarioId) =>
+    scenarioId !== "codex-plugin-pinned-new" && scenarioId !== "codex-plugin-pinned-old",
+);
+const frozenLegacyTargetSha = "ee5ead24b1b46a3560f28f8d57e0afcd911acacb";
 
 function frozenCoreSummary() {
   return summary(
@@ -125,6 +130,62 @@ function frozenCoreSummary() {
   );
 }
 
+function frozenLegacyStatuslessSummary() {
+  const fixture = summary(
+    frozenLegacyCoreScenarioIds.map((scenarioId) =>
+      scenario({
+        name: scenarioId,
+        status: "pass",
+      }),
+    ),
+  );
+  delete (fixture.run as { status?: string }).status;
+  return Object.assign(fixture, {
+    run: {
+      ...fixture.run,
+      startedAt: "2026-08-22T06:02:37.608Z",
+      finishedAt: "2026-08-22T06:14:49.336Z",
+    },
+  });
+}
+
+function reportFor(scenarios: ReturnType<typeof scenario>[]) {
+  return {
+    runtimePair: ["openclaw", "codex"],
+    totalScenarios: scenarios.length,
+    passedScenarios: scenarios.length,
+    failedScenarios: 0,
+    scenarios: scenarios.map((entry) => ({
+      name: entry.name,
+      status: "pass",
+      drift: entry.runtimeParity.drift,
+      driftDetails: undefined,
+      openclawStatus: "pass",
+      codexStatus: "pass",
+    })),
+    failures: [],
+    pass: true,
+  };
+}
+
+function markdownFor(scenarios: ReturnType<typeof scenario>[]) {
+  return [
+    "# OpenClaw Runtime Parity Report — openclaw vs codex",
+    "",
+    "- Verdict: pass",
+    ...scenarios.flatMap((entry) => [
+      "",
+      `### ${entry.name}`,
+      "",
+      "- status: pass",
+      `- drift: ${entry.runtimeParity.drift}`,
+      "- openclaw: pass (0 tool calls)",
+      "- codex: pass (0 tool calls)",
+    ]),
+    "",
+  ].join("\n");
+}
+
 describe("frozen QA runtime-pair summary validation", () => {
   it("rejects a nonterminal runtime-pair summary", () => {
     const fixture = summary([scenario({ name: "running", status: "pass" })]);
@@ -185,6 +246,148 @@ describe("frozen QA runtime-pair summary validation", () => {
     });
   });
 
+  it("accepts the exact statusless frozen core profile after a successful suite", () => {
+    expect(
+      validateQaRuntimePairSummary(frozenLegacyStatuslessSummary(), {
+        candidateSuiteOutcome: "success",
+        targetSha: frozenLegacyTargetSha,
+        lane: "core",
+      }),
+    ).toEqual({
+      total: 25,
+      passed: 25,
+      failed: 0,
+      skipped: 0,
+    });
+  });
+
+  it("rejects a nonterminal status even for the exact frozen profile", () => {
+    const fixture = frozenLegacyStatuslessSummary();
+    fixture.run.status = "running";
+
+    expect(() =>
+      validateQaRuntimePairSummary(fixture, {
+        candidateSuiteOutcome: "success",
+        targetSha: frozenLegacyTargetSha,
+        lane: "core",
+      }),
+    ).toThrow("runtime-pair summary is not completed");
+  });
+
+  it.each([
+    ["missing suite outcome", { targetSha: frozenLegacyTargetSha, lane: "core" }],
+    [
+      "failed suite",
+      { candidateSuiteOutcome: "failure", targetSha: frozenLegacyTargetSha, lane: "core" },
+    ],
+    [
+      "skipped suite",
+      { candidateSuiteOutcome: "skipped", targetSha: frozenLegacyTargetSha, lane: "core" },
+    ],
+    [
+      "cancelled suite",
+      { candidateSuiteOutcome: "cancelled", targetSha: frozenLegacyTargetSha, lane: "core" },
+    ],
+    [
+      "unknown suite outcome",
+      { candidateSuiteOutcome: "unknown", targetSha: frozenLegacyTargetSha, lane: "core" },
+    ],
+    ["wrong target", { candidateSuiteOutcome: "success", targetSha: "0".repeat(40), lane: "core" }],
+    [
+      "wrong lane",
+      { candidateSuiteOutcome: "success", targetSha: frozenLegacyTargetSha, lane: "soak" },
+    ],
+  ])("rejects statusless frozen evidence with %s", (_label, options) => {
+    expect(() => validateQaRuntimePairSummary(frozenLegacyStatuslessSummary(), options)).toThrow(
+      "runtime-pair summary is not completed",
+    );
+  });
+
+  it.each([
+    ["missing startedAt", undefined, "2026-08-22T06:14:49.336Z"],
+    ["missing finishedAt", "2026-08-22T06:02:37.608Z", undefined],
+    ["invalid startedAt", "not-a-date", "2026-08-22T06:14:49.336Z"],
+    ["invalid finishedAt", "2026-08-22T06:02:37.608Z", "not-a-date"],
+    ["reversed timestamps", "2026-08-22T06:14:49.336Z", "2026-08-22T06:02:37.608Z"],
+  ])("rejects statusless frozen evidence with %s", (_label, startedAt, finishedAt) => {
+    const fixture = frozenLegacyStatuslessSummary();
+    if (startedAt === undefined) {
+      delete (fixture.run as { startedAt?: string }).startedAt;
+    } else {
+      fixture.run.startedAt = startedAt;
+    }
+    if (finishedAt === undefined) {
+      delete (fixture.run as { finishedAt?: string }).finishedAt;
+    } else {
+      fixture.run.finishedAt = finishedAt;
+    }
+
+    expect(() =>
+      validateQaRuntimePairSummary(fixture, {
+        candidateSuiteOutcome: "success",
+        targetSha: frozenLegacyTargetSha,
+        lane: "core",
+      }),
+    ).toThrow("runtime-pair summary is not completed");
+  });
+
+  it("rejects manifest drift before accepting a statusless frozen profile", () => {
+    const fixture = frozenLegacyStatuslessSummary();
+    fixture.run.scenarioIds.reverse();
+    fixture.scenarios.reverse();
+
+    expect(() =>
+      validateQaRuntimePairSummary(fixture, {
+        candidateSuiteOutcome: "success",
+        targetSha: frozenLegacyTargetSha,
+        lane: "core",
+      }),
+    ).toThrow("runtime-pair summary is not completed");
+  });
+
+  it("still rejects scenario and count failures after frozen profile admission", () => {
+    const options = {
+      candidateSuiteOutcome: "success",
+      targetSha: frozenLegacyTargetSha,
+      lane: "core",
+    };
+    const failedScenario = frozenLegacyStatuslessSummary();
+    failedScenario.scenarios[0]!.status = "fail";
+    failedScenario.counts.passed -= 1;
+    failedScenario.counts.failed += 1;
+    expect(() => validateQaRuntimePairSummary(failedScenario, options)).toThrow(
+      "runtime-pair failure or unsupported skip",
+    );
+
+    const countDrift = frozenLegacyStatuslessSummary();
+    countDrift.counts.passed -= 1;
+    expect(() => validateQaRuntimePairSummary(countDrift, options)).toThrow(
+      "counts do not match validated scenario evidence",
+    );
+  });
+
+  it("applies the trusted suite outcome gate to frozen report validation", () => {
+    const fixture = frozenLegacyStatuslessSummary();
+    const reportSummary = reportFor(fixture.scenarios);
+    const markdown = markdownFor(fixture.scenarios);
+    const options = {
+      candidateSuiteOutcome: "success",
+      targetSha: frozenLegacyTargetSha,
+      lane: "core",
+    };
+
+    expect(validateQaRuntimePairReport(fixture, reportSummary, markdown, options)).toMatchObject({
+      total: 25,
+      passed: 25,
+    });
+    expect(() =>
+      validateQaRuntimePairReport(fixture, reportSummary, markdown, {
+        ...options,
+        candidateSuiteOutcome: "failure",
+      }),
+    ).toThrow("runtime-pair summary is not completed");
+  });
+
   it("requires skipped count when validated evidence contains skips", () => {
     const fixture = summary([
       scenario({