From a7dcb21dde47a801b5bb9b001fc367443bea628f Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Thu, 20 Aug 2026 21:56:50 -0700
Subject: [PATCH] fix(release): require terminal frozen QA evidence

---
 scripts/validate-qa-runtime-pair-summary.mts  | 37 ++++++++++++-------
 .../validate-qa-runtime-pair-summary.test.ts  | 30 +++++++++++----
 2 files changed, 47 insertions(+), 20 deletions(-)

diff --git a/scripts/validate-qa-runtime-pair-summary.mts b/scripts/validate-qa-runtime-pair-summary.mts
index 5a8f98168ba5..6cf041a24004 100644
--- a/scripts/validate-qa-runtime-pair-summary.mts
+++ b/scripts/validate-qa-runtime-pair-summary.mts
@@ -168,6 +168,21 @@ function requireCanonicalRuntimePair(runtimePair: unknown) {
   );
 }
 
+function matchesFrozenRuntimePairManifest(
+  options: { targetSha?: string; lane?: string },
+  scenarioIds: unknown[],
+  skippedScenarioIds: unknown[],
+) {
+  const manifest = FROZEN_RUNTIME_PAIR_MANIFESTS.get(`${options.targetSha}:${options.lane}`);
+  return (
+    manifest !== undefined &&
+    manifest.scenarioIds.length === scenarioIds.length &&
+    manifest.scenarioIds.every((scenarioId, index) => scenarioId === scenarioIds[index]) &&
+    manifest.gapScenarioIds.length === skippedScenarioIds.length &&
+    manifest.gapScenarioIds.every((scenarioId, index) => scenarioId === skippedScenarioIds[index])
+  );
+}
+
 export function validateQaRuntimePairSummary(
   summary: unknown,
   options: { requireExplicitGap?: boolean; targetSha?: string; lane?: string } = {},
@@ -175,11 +190,7 @@ export function validateQaRuntimePairSummary(
   if (!isRecord(summary) || !isRecord(summary.run) || !Array.isArray(summary.scenarios)) {
     throw new Error("runtime-pair summary is missing run or scenario evidence");
   }
-  const completedLegacyRun =
-    summary.run.status === undefined &&
-    typeof summary.run.finishedAt === "string" &&
-    Number.isFinite(Date.parse(summary.run.finishedAt));
-  if (summary.run.status !== "completed" && !completedLegacyRun) {
+  if (summary.run.status !== "completed" && summary.run.status !== undefined) {
     throw new Error("runtime-pair summary is not completed");
   }
   if (!requireCanonicalRuntimePair(summary.run.runtimePair)) {
@@ -218,6 +229,13 @@ export function validateQaRuntimePairSummary(
     }
   }
 
+  if (
+    run.status === undefined &&
+    !matchesFrozenRuntimePairManifest(options, scenarioIds, skippedScenarioIds)
+  ) {
+    throw new Error("runtime-pair summary is not completed");
+  }
+
   const expectedCounts = {
     total: summary.scenarios.length,
     passed,
@@ -239,14 +257,7 @@ export function validateQaRuntimePairSummary(
     throw new Error("nonzero candidate suite exit requires an explicit Codex-native harness gap");
   }
   if (options.requireExplicitGap === true) {
-    const manifest = FROZEN_RUNTIME_PAIR_MANIFESTS.get(`${options.targetSha}:${options.lane}`);
-    if (
-      !manifest ||
-      manifest.scenarioIds.length !== scenarioIds.length ||
-      manifest.scenarioIds.some((scenarioId, index) => scenarioId !== scenarioIds[index]) ||
-      manifest.gapScenarioIds.length !== skippedScenarioIds.length ||
-      manifest.gapScenarioIds.some((scenarioId, index) => scenarioId !== skippedScenarioIds[index])
-    ) {
+    if (!matchesFrozenRuntimePairManifest(options, scenarioIds, skippedScenarioIds)) {
       throw new Error("nonzero candidate exit is not covered by a trusted frozen-lane manifest");
     }
   }
diff --git a/test/scripts/validate-qa-runtime-pair-summary.test.ts b/test/scripts/validate-qa-runtime-pair-summary.test.ts
index d8b7aacdb8ae..4e30c6124d30 100644
--- a/test/scripts/validate-qa-runtime-pair-summary.test.ts
+++ b/test/scripts/validate-qa-runtime-pair-summary.test.ts
@@ -135,16 +135,20 @@ describe("frozen QA runtime-pair summary validation", () => {
     );
   });
 
-  it("accepts a completed legacy summary that used its finish timestamp", () => {
-    const fixture = summary([scenario({ name: "legacy terminal", status: "pass" })]);
+  it("accepts a statusless summary only for an exact frozen terminal manifest", () => {
+    const fixture = frozenCoreSummary();
     Reflect.deleteProperty(fixture.run, "status");
-    Object.assign(fixture.run, { finishedAt: "2026-08-21T03:34:17.434Z" });
 
-    expect(validateQaRuntimePairSummary(fixture)).toEqual({
-      total: 1,
-      passed: 1,
+    expect(
+      validateQaRuntimePairSummary(fixture, {
+        targetSha: "311047822ecdde24e824d839ab105ef08f17be00",
+        lane: "core",
+      }),
+    ).toEqual({
+      total: 27,
+      passed: 19,
       failed: 0,
-      skipped: 0,
+      skipped: 8,
     });
   });
 
@@ -157,6 +161,18 @@ describe("frozen QA runtime-pair summary validation", () => {
     );
   });
 
+  it("rejects a partial statusless artifact for a frozen candidate", () => {
+    const fixture = summary([scenario({ name: frozenCoreScenarioIds[0], status: "pass" })]);
+    Reflect.deleteProperty(fixture.run, "status");
+
+    expect(() =>
+      validateQaRuntimePairSummary(fixture, {
+        targetSha: "311047822ecdde24e824d839ab105ef08f17be00",
+        lane: "core",
+      }),
+    ).toThrow("runtime-pair summary is not completed");
+  });
+
   it("accepts only passing scenarios and explicit one-sided Codex-native gaps", () => {
     const fixture = summary([
       scenario({ name: "passing", status: "pass" }),