fix(maint): enforce 24-hour hosted gate freshness (#104651)

* fix(maint): enforce 24-hour hosted gate freshness

* style(maint): avoid hosted gate run shadowing

scripts/verify-pr-hosted-gates.d.mts

@@ -27,7 +27,8 @@ export function collectHostedGateEvidence({
   changelogOnly?: boolean | undefined;
   nowMs?: number | undefined;
 }): {
-  headSha: unknown;
+  headSha: string;
+  evidenceHeadSha?: string;
   workflows: {
     id: unknown;
     name: unknown;
@@ -40,6 +41,11 @@ export function collectHostedGateEvidence({
     updatedAt: unknown;
     url: unknown;
   }[];
+  fallbackCoveredWorkflows?: {
+    name: string;
+    coveredBy: string;
+    reason: string;
+  }[];
 };
 export function workflowRunQueryPaths(
   repo: string,
@@ -56,4 +62,4 @@ export function workflowRunQueryPaths(
 ): string[];
 export function main(argv?: string[]): void;
 export const SCHEDULED_HOSTED_WORKFLOWS: string[];
-export const HOSTED_GATE_MAX_AGE_HOURS: 12;
+export const HOSTED_GATE_MAX_AGE_HOURS: 24;

scripts/verify-pr-hosted-gates.mjs

@@ -19,7 +19,7 @@ const ARTIFACT_FALLBACK_REQUIRED_WORKFLOWS = [
 ];
 const WORKFLOW_RUNS_PAGE_SIZE = 100;
 const MAX_WORKFLOW_RUN_SEARCH_RESULTS = 1_000;
-export const HOSTED_GATE_MAX_AGE_HOURS = 12;
+export const HOSTED_GATE_MAX_AGE_HOURS = 24;
 const HOSTED_GATE_MAX_AGE_MS = HOSTED_GATE_MAX_AGE_HOURS * 60 * 60 * 1_000;
 const HOSTED_GATE_CLOCK_SKEW_MS = 5 * 60 * 1_000;
 
@@ -143,65 +143,71 @@ function isRecentRun(run, nowMs) {
   );
 }
 
-function preferredCiRun(runs) {
+function isSuccessfulRecentRun(run, nowMs) {
+  return run?.status === "completed" && run.conclusion === "success" && isRecentRun(run, nowMs);
+}
+
+function preferredCiRun(runs, nowMs) {
   const scheduledRuns = runs.filter((run) => run.event === "pull_request");
   const latestScheduledRun = latestRun(scheduledRuns);
-  const failedScheduledRun = latestRun(
-    scheduledRuns.filter(
-      (run) =>
-        run.status === "completed" && !["success", "cancelled", "skipped"].includes(run.conclusion),
-    ),
+  const latestCompletedScheduledRun = latestRun(
+    scheduledRuns.filter((run) => run.status === "completed"),
   );
-  if (failedScheduledRun && latestScheduledRun?.status !== "completed") {
-    return failedScheduledRun;
+  const latestManualRun = latestRun(runs.filter((run) => run.event === "workflow_dispatch"));
+
+  // Manual proof may replace stale scheduled success or a pending run,
+  // never an unresolved terminal non-success.
+  if (latestCompletedScheduledRun && latestCompletedScheduledRun.conclusion !== "success") {
+    return latestCompletedScheduledRun;
   }
-  if (latestScheduledRun?.status === "completed") {
+  if (latestScheduledRun?.status === "completed" && isRecentRun(latestScheduledRun, nowMs)) {
     return latestScheduledRun;
   }
-  return latestRun(runs.filter((run) => run.event === "workflow_dispatch")) ?? latestScheduledRun;
+  return latestManualRun ?? latestScheduledRun;
 }
 
 function successfulRunOrThrow(
   runs,
   workflowName,
   sha,
-  { allowManual = true, requireRecent = false, nowMs = Date.now() } = {},
+  { allowManual = true, nowMs = Date.now() } = {},
 ) {
-  const matchingRuns = matchingAuthoritativeRuns(runs, workflowName, sha, allowManual).filter(
-    (run) => !requireRecent || (run?.event === "pull_request" && isRecentRun(run, nowMs)),
-  );
-  const run = workflowName === "CI" ? preferredCiRun(matchingRuns) : latestRun(matchingRuns);
-  if (!run || run.status !== "completed" || run.conclusion !== "success") {
+  const matchingRuns = matchingAuthoritativeRuns(runs, workflowName, sha, allowManual);
+  const run = workflowName === "CI" ? preferredCiRun(matchingRuns, nowMs) : latestRun(matchingRuns);
+  if (!isSuccessfulRecentRun(run, nowMs)) {
     throw new Error(
-      `Missing successful ${requireRecent ? "recent " : ""}${workflowName} workflow for ${sha}. Observed: ${formatObservedRuns(matchingRuns)}`,
+      `Missing successful recent ${workflowName} workflow for ${sha}. Observed: ${formatObservedRuns(matchingRuns)}`,
     );
   }
   return run;
 }
 
-function successfulReleaseGateFallback(workflowRuns, sha) {
-  const fallback = latestRun(workflowRuns.filter((run) => isReleaseGateCiRun(run, sha)));
-  if (fallback?.status !== "completed" || fallback.conclusion !== "success") {
-    return null;
-  }
-  return fallback;
+function hasSuccessfulRecentReleaseGate(workflowRuns, sha, nowMs) {
+  const releaseGate = latestRun(workflowRuns.filter((run) => isReleaseGateCiRun(run, sha)));
+  return isSuccessfulRecentRun(releaseGate, nowMs);
 }
 
-function canCoverQueuedBuildArtifacts(workflowRuns, sha) {
-  if (!successfulReleaseGateFallback(workflowRuns, sha)) {
+function canCoverQueuedBuildArtifacts(workflowRuns, sha, nowMs) {
+  if (!hasSuccessfulRecentReleaseGate(workflowRuns, sha, nowMs)) {
     return false;
   }
   const supportingGatesPassed = ARTIFACT_FALLBACK_REQUIRED_WORKFLOWS.every((workflowName) => {
-    const run = latestRun(matchingAuthoritativeRuns(workflowRuns, workflowName, sha));
-    return run?.status === "completed" && run.conclusion === "success";
+    const run = latestRun(matchingAuthoritativeRuns(workflowRuns, workflowName, sha, false));
+    return isSuccessfulRecentRun(run, nowMs);
   });
   if (!supportingGatesPassed) {
     return false;
   }
-  const buildArtifactRuns = matchingAuthoritativeRuns(workflowRuns, BUILD_ARTIFACTS_WORKFLOW, sha);
+  const buildArtifactRuns = matchingAuthoritativeRuns(
+    workflowRuns,
+    BUILD_ARTIFACTS_WORKFLOW,
+    sha,
+    false,
+  );
   const latestBuildArtifactRun = latestRun(buildArtifactRuns);
   return (
     latestBuildArtifactRun?.status === "queued" &&
+    isRecentRun(latestBuildArtifactRun, nowMs) &&
     buildArtifactRuns.every(
       (run) =>
         run.status === "queued" || (run.status === "completed" && run.conclusion === "success"),
@@ -241,15 +247,13 @@ export function collectHostedGateEvidence({
     throw new Error("workflowRuns must be an array.");
   }
 
-  const collectForSha = (evidenceSha, requireRecent, requiredScheduledWorkflows = new Set()) => {
-    const allowManual = !requireRecent;
+  const collectForSha = (evidenceSha, { allowManual, requiredScheduledWorkflows = new Set() }) => {
     const workflows = [];
     const fallbackCoveredWorkflows = [];
     if (!changelogOnly) {
       workflows.push(
         successfulRunOrThrow(workflowRuns, "CI", evidenceSha, {
           allowManual,
-          requireRecent,
           nowMs,
         }),
       );
@@ -267,7 +271,7 @@ export function collectHostedGateEvidence({
       if (
         allowManual &&
         workflowName === BUILD_ARTIFACTS_WORKFLOW &&
-        canCoverQueuedBuildArtifacts(workflowRuns, evidenceSha)
+        canCoverQueuedBuildArtifacts(workflowRuns, evidenceSha, nowMs)
       ) {
         fallbackCoveredWorkflows.push({
           name: workflowName,
@@ -279,7 +283,6 @@ export function collectHostedGateEvidence({
       workflows.push(
         successfulRunOrThrow(workflowRuns, workflowName, evidenceSha, {
           allowManual,
-          requireRecent,
           nowMs,
         }),
       );
@@ -290,7 +293,7 @@ export function collectHostedGateEvidence({
   let evidenceSha = sha;
   let selected;
   try {
-    selected = collectForSha(sha, false);
+    selected = collectForSha(sha, { allowManual: true });
   } catch (exactError) {
     const currentWorkflowNames = ["CI", ...SCHEDULED_HOSTED_WORKFLOWS];
     const currentHeadHasTerminalNonSuccess = currentWorkflowNames.some((workflowName) => {
@@ -337,7 +340,10 @@ export function collectHostedGateEvidence({
     let fallbackError;
     for (const fallbackSha of new Set(fallbackShas)) {
       try {
-        selected = collectForSha(fallbackSha, true, targetScheduledWorkflows);
+        selected = collectForSha(fallbackSha, {
+          allowManual: false,
+          requiredScheduledWorkflows: targetScheduledWorkflows,
+        });
         evidenceSha = fallbackSha;
         break;
       } catch (error) {

test/scripts/verify-pr-hosted-gates.test.ts

@@ -42,6 +42,29 @@ function successfulRun(name: string, id: number, updatedAt: string) {
   };
 }
 
+function releaseGateRun(id: number, updatedAt: string) {
+  return {
+    ...successfulRun(`CI release gate ${sha}`, id, updatedAt),
+    event: "workflow_dispatch",
+    display_title: `CI release gate ${sha}`,
+  };
+}
+
+function queuedBuildArtifactFallbackRuns() {
+  return [
+    releaseGateRun(1, "2026-06-17T10:49:00Z"),
+    successfulRun("CI", 3, "2026-06-17T10:51:00Z"),
+    successfulRun("Blacksmith Testbox", 4, "2026-06-17T10:52:00Z"),
+    successfulRun("Blacksmith ARM Testbox", 5, "2026-06-17T10:53:00Z"),
+    successfulRun("Workflow Sanity", 6, "2026-06-17T10:54:00Z"),
+    {
+      ...successfulRun(BUILD_ARTIFACTS_WORKFLOW, 2, "2026-06-17T10:50:00Z"),
+      status: "queued",
+      conclusion: null,
+    },
+  ];
+}
+
 function collectHostedGateEvidence(
   options: Omit<Parameters<typeof collectHostedGateEvidenceRaw>[0], "nowMs" | "pr">,
 ) {
@@ -88,15 +111,15 @@ describe("verify-pr-hosted-gates", () => {
     };
 
     expect(() => collectHostedGateEvidence({ sha, workflowRuns })).toThrow(
-      "Missing successful Blacksmith ARM Testbox workflow",
+      "Missing successful recent Blacksmith ARM Testbox workflow",
     );
   });
 
-  it("accepts a non-docs PR when CI is the only scheduled authoritative workflow", () => {
+  it("accepts a sole scheduled CI run at the 24-hour boundary", () => {
     expect(
       collectHostedGateEvidence({
         sha,
-        workflowRuns: [successfulRun("CI", 1, "2026-06-17T10:47:00Z")],
+        workflowRuns: [successfulRun("CI", 1, "2026-06-16T10:55:00Z")],
       }),
     ).toEqual({
       headSha: sha,
@@ -104,14 +127,14 @@ describe("verify-pr-hosted-gates", () => {
     });
   });
 
-  it("accepts recent green evidence from the recorded pre-rebase head while current CI is pending", () => {
+  it("accepts 13-hour green evidence from the recorded pre-rebase head", () => {
     const previousSha = "8d86c44c6144f8f726a460914cddb8c9c201f119";
     const evidence = collectHostedGateEvidence({
       sha,
       recentSha: previousSha,
       workflowRuns: [
         {
-          ...successfulRun("CI", 1, "2026-06-17T10:50:00Z"),
+          ...successfulRun("CI", 1, "2026-06-16T21:55:00Z"),
           head_sha: previousSha,
         },
         {
@@ -213,7 +236,7 @@ describe("verify-pr-hosted-gates", () => {
             },
           ],
         }),
-      ).toThrow(`Missing successful CI workflow for ${sha}`);
+      ).toThrow(`Missing successful recent CI workflow for ${sha}`);
     },
   );
 
@@ -235,7 +258,7 @@ describe("verify-pr-hosted-gates", () => {
           },
         ],
       }),
-    ).toThrow(`Missing successful CI workflow for ${sha}`);
+    ).toThrow(`Missing successful recent CI workflow for ${sha}`);
   });
 
   it("requires the complete recent gate cohort from the recorded head", () => {
@@ -313,7 +336,7 @@ describe("verify-pr-hosted-gates", () => {
           },
         ],
       }),
-    ).toThrow(`Missing successful CI workflow for ${sha}`);
+    ).toThrow(`Missing successful recent CI workflow for ${sha}`);
   });
 
   it("rejects stale or unrecorded fallback heads", () => {
@@ -353,7 +376,7 @@ describe("verify-pr-hosted-gates", () => {
         sha,
         workflowRuns: [{ ...recentUnrelatedRun, head_sha: previousSha }, currentPending],
       }),
-    ).toThrow(`Missing successful CI workflow for ${sha}`);
+    ).toThrow(`Missing successful recent CI workflow for ${sha}`);
   });
 
   it("allows a later scheduled success to clear an earlier current-head failure", () => {
@@ -412,13 +435,13 @@ describe("verify-pr-hosted-gates", () => {
     });
   });
 
-  it("accepts the explicit exact-SHA manual CI release gate", () => {
+  it("accepts an exact-SHA manual CI release gate at the 24-hour boundary", () => {
     expect(
       collectHostedGateEvidence({
         sha,
         workflowRuns: [
           {
-            ...successfulRun(`CI release gate ${sha}`, 1, "2026-06-17T10:47:00Z"),
+            ...successfulRun(`CI release gate ${sha}`, 1, "2026-06-16T10:55:00Z"),
             event: "workflow_dispatch",
             path: ".github/workflows/ci.yml@refs/heads/release-controls",
             display_title: `CI release gate ${sha}`,
@@ -431,23 +454,30 @@ describe("verify-pr-hosted-gates", () => {
     });
   });
 
-  it("prefers the exact release-gate fallback while scheduled CI remains queued", () => {
+  it.each([
+    ["scheduled", successfulRun("CI", 1, "2026-06-16T10:54:59Z")],
+    ["manual", releaseGateRun(2, "2026-06-16T10:54:59Z")],
+  ])("rejects exact-head %s CI evidence older than 24 hours", (_kind, run) => {
+    expect(() => collectHostedGateEvidence({ sha, workflowRuns: [run] })).toThrow(
+      `Missing successful recent CI workflow for ${sha}`,
+    );
+  });
+
+  it.each([
+    [
+      "queued",
+      {
+        ...successfulRun("CI", 1, "2026-06-17T10:50:00Z"),
+        status: "queued",
+        conclusion: null,
+      },
+    ],
+    ["stale", successfulRun("CI", 1, "2026-06-16T10:54:59Z")],
+  ])("prefers a fresh exact release gate while scheduled CI is %s", (_state, scheduledRun) => {
     expect(
       collectHostedGateEvidence({
         sha,
-        workflowRuns: [
-          {
-            ...successfulRun("CI", 1, "2026-06-17T10:47:00Z"),
-            status: "queued",
-            conclusion: null,
-            updated_at: "2026-06-17T10:50:00Z",
-          },
-          {
-            ...successfulRun(`CI release gate ${sha}`, 2, "2026-06-17T10:49:00Z"),
-            event: "workflow_dispatch",
-            display_title: `CI release gate ${sha}`,
-          },
-        ],
+        workflowRuns: [scheduledRun, releaseGateRun(2, "2026-06-17T10:49:00Z")],
       }),
     ).toEqual({
       headSha: sha,
@@ -461,7 +491,7 @@ describe("verify-pr-hosted-gates", () => {
         sha,
         workflowRuns: [
           {
-            ...successfulRun("CI", 1, "2026-06-17T10:50:00Z"),
+            ...successfulRun("CI", 1, "2026-06-16T10:54:59Z"),
             conclusion: "failure",
           },
           {
@@ -471,7 +501,7 @@ describe("verify-pr-hosted-gates", () => {
           },
         ],
       }),
-    ).toThrow("Missing successful CI workflow");
+    ).toThrow("Missing successful recent CI workflow");
   });
 
   it("does not mask a failed CI run with a queued rerun and release-gate fallback", () => {
@@ -495,29 +525,14 @@ describe("verify-pr-hosted-gates", () => {
           },
         ],
       }),
-    ).toThrow("Missing successful CI workflow");
+    ).toThrow("Missing successful recent CI workflow");
   });
 
   it("covers a queued artifact Testbox only with a completed exact CI fallback", () => {
     expect(
       collectHostedGateEvidence({
         sha,
-        workflowRuns: [
-          {
-            ...successfulRun(`CI release gate ${sha}`, 1, "2026-06-17T10:49:00Z"),
-            event: "workflow_dispatch",
-            display_title: `CI release gate ${sha}`,
-          },
-          successfulRun("CI", 3, "2026-06-17T10:51:00Z"),
-          successfulRun("Blacksmith Testbox", 4, "2026-06-17T10:52:00Z"),
-          successfulRun("Blacksmith ARM Testbox", 5, "2026-06-17T10:53:00Z"),
-          successfulRun("Workflow Sanity", 6, "2026-06-17T10:54:00Z"),
-          {
-            ...successfulRun(BUILD_ARTIFACTS_WORKFLOW, 2, "2026-06-17T10:50:00Z"),
-            status: "queued",
-            conclusion: null,
-          },
-        ],
+        workflowRuns: queuedBuildArtifactFallbackRuns(),
       }),
     ).toEqual({
       headSha: sha,
@@ -537,6 +552,32 @@ describe("verify-pr-hosted-gates", () => {
     });
   });
 
+  it.each([
+    ["release gate", 0],
+    ["supporting gate", 4],
+    ["queued artifact run", 5],
+  ])("does not cover queued artifacts with a stale %s", (_kind, staleRunIndex) => {
+    const workflowRuns = queuedBuildArtifactFallbackRuns().map((run, index) =>
+      index === staleRunIndex ? { ...run, updated_at: "2026-06-16T10:54:59Z" } : run,
+    );
+    expect(() => collectHostedGateEvidence({ sha, workflowRuns })).toThrow(
+      "Missing successful recent Blacksmith Build Artifacts Testbox workflow",
+    );
+  });
+
+  it("keeps an older failed artifact run blocking a fresh queued retry", () => {
+    const workflowRuns = [
+      ...queuedBuildArtifactFallbackRuns(),
+      {
+        ...successfulRun(BUILD_ARTIFACTS_WORKFLOW, 7, "2026-06-16T10:54:59Z"),
+        conclusion: "failure",
+      },
+    ];
+    expect(() => collectHostedGateEvidence({ sha, workflowRuns })).toThrow(
+      "Missing successful recent Blacksmith Build Artifacts Testbox workflow",
+    );
+  });
+
   it("does not cover queued artifacts until all supporting workflow gates pass", () => {
     expect(() =>
       collectHostedGateEvidence({
@@ -554,7 +595,7 @@ describe("verify-pr-hosted-gates", () => {
           },
         ],
       }),
-    ).toThrow("Missing successful Blacksmith Build Artifacts Testbox workflow");
+    ).toThrow("Missing successful recent Blacksmith Build Artifacts Testbox workflow");
   });
 
   it("keeps active or terminal non-successful artifact Testboxes blocking", () => {
@@ -580,7 +621,7 @@ describe("verify-pr-hosted-gates", () => {
           sha,
           workflowRuns: [ciFallback, artifactRun],
         }),
-      ).toThrow("Missing successful Blacksmith Build Artifacts Testbox workflow");
+      ).toThrow("Missing successful recent Blacksmith Build Artifacts Testbox workflow");
     }
 
     expect(() =>
@@ -599,7 +640,7 @@ describe("verify-pr-hosted-gates", () => {
           },
         ],
       }),
-    ).toThrow("Missing successful Blacksmith Build Artifacts Testbox workflow");
+    ).toThrow("Missing successful recent Blacksmith Build Artifacts Testbox workflow");
   });
 
   it("rejects an unmarked manual CI run", () => {
@@ -614,7 +655,7 @@ describe("verify-pr-hosted-gates", () => {
           },
         ],
       }),
-    ).toThrow("Missing successful CI workflow");
+    ).toThrow("Missing successful recent CI workflow");
   });
 
   it("rejects a manual release-gate title from another workflow", () => {
@@ -630,12 +671,12 @@ describe("verify-pr-hosted-gates", () => {
           },
         ],
       }),
-    ).toThrow("Missing successful CI workflow");
+    ).toThrow("Missing successful recent CI workflow");
   });
 
   it("requires CI for docs unless the head changes only CHANGELOG.md", () => {
     expect(() => collectHostedGateEvidence({ sha, workflowRuns: [] })).toThrow(
-      "Missing successful CI workflow",
+      "Missing successful recent CI workflow",
     );
     expect(collectHostedGateEvidence({ sha, workflowRuns: [], changelogOnly: true })).toEqual({
       headSha: sha,
@@ -697,7 +738,7 @@ describe("verify-pr-hosted-gates", () => {
       `repos/openclaw/openclaw/actions/runs?head_sha=${sha}&per_page=100&page=1`,
       `repos/openclaw/openclaw/actions/runs?head_sha=${previousSha}&per_page=100&page=1`,
     ]);
-    expect(HOSTED_GATE_MAX_AGE_HOURS).toBe(12);
+    expect(HOSTED_GATE_MAX_AGE_HOURS).toBe(24);
   });
 
   it("queries recent pull-request runs for the head branch", () => {