diff --git a/scripts/verify-pr-hosted-gates.d.mts b/scripts/verify-pr-hosted-gates.d.mts index 2d3a5af27eae..88d0ddcf9c2e 100644 --- a/scripts/verify-pr-hosted-gates.d.mts +++ b/scripts/verify-pr-hosted-gates.d.mts @@ -27,7 +27,8 @@ export function collectHostedGateEvidence({ changelogOnly?: boolean | undefined; nowMs?: number | undefined; }): { - headSha: unknown; + headSha: string; + evidenceHeadSha?: string; workflows: { id: unknown; name: unknown; @@ -40,6 +41,11 @@ export function collectHostedGateEvidence({ updatedAt: unknown; url: unknown; }[]; + fallbackCoveredWorkflows?: { + name: string; + coveredBy: string; + reason: string; + }[]; }; export function workflowRunQueryPaths( repo: string, @@ -56,4 +62,4 @@ export function workflowRunQueryPaths( ): string[]; export function main(argv?: string[]): void; export const SCHEDULED_HOSTED_WORKFLOWS: string[]; -export const HOSTED_GATE_MAX_AGE_HOURS: 12; +export const HOSTED_GATE_MAX_AGE_HOURS: 24; diff --git a/scripts/verify-pr-hosted-gates.mjs b/scripts/verify-pr-hosted-gates.mjs index 27523f6c1f8e..b40a78dd7def 100644 --- a/scripts/verify-pr-hosted-gates.mjs +++ b/scripts/verify-pr-hosted-gates.mjs @@ -19,7 +19,7 @@ const ARTIFACT_FALLBACK_REQUIRED_WORKFLOWS = [ ]; const WORKFLOW_RUNS_PAGE_SIZE = 100; const MAX_WORKFLOW_RUN_SEARCH_RESULTS = 1_000; -export const HOSTED_GATE_MAX_AGE_HOURS = 12; +export const HOSTED_GATE_MAX_AGE_HOURS = 24; const HOSTED_GATE_MAX_AGE_MS = HOSTED_GATE_MAX_AGE_HOURS * 60 * 60 * 1_000; const HOSTED_GATE_CLOCK_SKEW_MS = 5 * 60 * 1_000; @@ -143,65 +143,71 @@ function isRecentRun(run, nowMs) { ); } -function preferredCiRun(runs) { +function isSuccessfulRecentRun(run, nowMs) { + return run?.status === "completed" && run.conclusion === "success" && isRecentRun(run, nowMs); +} + +function preferredCiRun(runs, nowMs) { const scheduledRuns = runs.filter((run) => run.event === "pull_request"); const latestScheduledRun = latestRun(scheduledRuns); - const failedScheduledRun = latestRun( - scheduledRuns.filter( - (run) => - run.status === "completed" && !["success", "cancelled", "skipped"].includes(run.conclusion), - ), + const latestCompletedScheduledRun = latestRun( + scheduledRuns.filter((run) => run.status === "completed"), ); - if (failedScheduledRun && latestScheduledRun?.status !== "completed") { - return failedScheduledRun; + const latestManualRun = latestRun(runs.filter((run) => run.event === "workflow_dispatch")); + + // Manual proof may replace stale scheduled success or a pending run, + // never an unresolved terminal non-success. + if (latestCompletedScheduledRun && latestCompletedScheduledRun.conclusion !== "success") { + return latestCompletedScheduledRun; } - if (latestScheduledRun?.status === "completed") { + if (latestScheduledRun?.status === "completed" && isRecentRun(latestScheduledRun, nowMs)) { return latestScheduledRun; } - return latestRun(runs.filter((run) => run.event === "workflow_dispatch")) ?? latestScheduledRun; + return latestManualRun ?? latestScheduledRun; } function successfulRunOrThrow( runs, workflowName, sha, - { allowManual = true, requireRecent = false, nowMs = Date.now() } = {}, + { allowManual = true, nowMs = Date.now() } = {}, ) { - const matchingRuns = matchingAuthoritativeRuns(runs, workflowName, sha, allowManual).filter( - (run) => !requireRecent || (run?.event === "pull_request" && isRecentRun(run, nowMs)), - ); - const run = workflowName === "CI" ? preferredCiRun(matchingRuns) : latestRun(matchingRuns); - if (!run || run.status !== "completed" || run.conclusion !== "success") { + const matchingRuns = matchingAuthoritativeRuns(runs, workflowName, sha, allowManual); + const run = workflowName === "CI" ? preferredCiRun(matchingRuns, nowMs) : latestRun(matchingRuns); + if (!isSuccessfulRecentRun(run, nowMs)) { throw new Error( - `Missing successful ${requireRecent ? "recent " : ""}${workflowName} workflow for ${sha}. Observed: ${formatObservedRuns(matchingRuns)}`, + `Missing successful recent ${workflowName} workflow for ${sha}. Observed: ${formatObservedRuns(matchingRuns)}`, ); } return run; } -function successfulReleaseGateFallback(workflowRuns, sha) { - const fallback = latestRun(workflowRuns.filter((run) => isReleaseGateCiRun(run, sha))); - if (fallback?.status !== "completed" || fallback.conclusion !== "success") { - return null; - } - return fallback; +function hasSuccessfulRecentReleaseGate(workflowRuns, sha, nowMs) { + const releaseGate = latestRun(workflowRuns.filter((run) => isReleaseGateCiRun(run, sha))); + return isSuccessfulRecentRun(releaseGate, nowMs); } -function canCoverQueuedBuildArtifacts(workflowRuns, sha) { - if (!successfulReleaseGateFallback(workflowRuns, sha)) { +function canCoverQueuedBuildArtifacts(workflowRuns, sha, nowMs) { + if (!hasSuccessfulRecentReleaseGate(workflowRuns, sha, nowMs)) { return false; } const supportingGatesPassed = ARTIFACT_FALLBACK_REQUIRED_WORKFLOWS.every((workflowName) => { - const run = latestRun(matchingAuthoritativeRuns(workflowRuns, workflowName, sha)); - return run?.status === "completed" && run.conclusion === "success"; + const run = latestRun(matchingAuthoritativeRuns(workflowRuns, workflowName, sha, false)); + return isSuccessfulRecentRun(run, nowMs); }); if (!supportingGatesPassed) { return false; } - const buildArtifactRuns = matchingAuthoritativeRuns(workflowRuns, BUILD_ARTIFACTS_WORKFLOW, sha); + const buildArtifactRuns = matchingAuthoritativeRuns( + workflowRuns, + BUILD_ARTIFACTS_WORKFLOW, + sha, + false, + ); const latestBuildArtifactRun = latestRun(buildArtifactRuns); return ( latestBuildArtifactRun?.status === "queued" && + isRecentRun(latestBuildArtifactRun, nowMs) && buildArtifactRuns.every( (run) => run.status === "queued" || (run.status === "completed" && run.conclusion === "success"), @@ -241,15 +247,13 @@ export function collectHostedGateEvidence({ throw new Error("workflowRuns must be an array."); } - const collectForSha = (evidenceSha, requireRecent, requiredScheduledWorkflows = new Set()) => { - const allowManual = !requireRecent; + const collectForSha = (evidenceSha, { allowManual, requiredScheduledWorkflows = new Set() }) => { const workflows = []; const fallbackCoveredWorkflows = []; if (!changelogOnly) { workflows.push( successfulRunOrThrow(workflowRuns, "CI", evidenceSha, { allowManual, - requireRecent, nowMs, }), ); @@ -267,7 +271,7 @@ export function collectHostedGateEvidence({ if ( allowManual && workflowName === BUILD_ARTIFACTS_WORKFLOW && - canCoverQueuedBuildArtifacts(workflowRuns, evidenceSha) + canCoverQueuedBuildArtifacts(workflowRuns, evidenceSha, nowMs) ) { fallbackCoveredWorkflows.push({ name: workflowName, @@ -279,7 +283,6 @@ export function collectHostedGateEvidence({ workflows.push( successfulRunOrThrow(workflowRuns, workflowName, evidenceSha, { allowManual, - requireRecent, nowMs, }), ); @@ -290,7 +293,7 @@ export function collectHostedGateEvidence({ let evidenceSha = sha; let selected; try { - selected = collectForSha(sha, false); + selected = collectForSha(sha, { allowManual: true }); } catch (exactError) { const currentWorkflowNames = ["CI", ...SCHEDULED_HOSTED_WORKFLOWS]; const currentHeadHasTerminalNonSuccess = currentWorkflowNames.some((workflowName) => { @@ -337,7 +340,10 @@ export function collectHostedGateEvidence({ let fallbackError; for (const fallbackSha of new Set(fallbackShas)) { try { - selected = collectForSha(fallbackSha, true, targetScheduledWorkflows); + selected = collectForSha(fallbackSha, { + allowManual: false, + requiredScheduledWorkflows: targetScheduledWorkflows, + }); evidenceSha = fallbackSha; break; } catch (error) { diff --git a/test/scripts/verify-pr-hosted-gates.test.ts b/test/scripts/verify-pr-hosted-gates.test.ts index c0f546fb89e4..81f230c3785b 100644 --- a/test/scripts/verify-pr-hosted-gates.test.ts +++ b/test/scripts/verify-pr-hosted-gates.test.ts @@ -42,6 +42,29 @@ function successfulRun(name: string, id: number, updatedAt: string) { }; } +function releaseGateRun(id: number, updatedAt: string) { + return { + ...successfulRun(`CI release gate ${sha}`, id, updatedAt), + event: "workflow_dispatch", + display_title: `CI release gate ${sha}`, + }; +} + +function queuedBuildArtifactFallbackRuns() { + return [ + releaseGateRun(1, "2026-06-17T10:49:00Z"), + successfulRun("CI", 3, "2026-06-17T10:51:00Z"), + successfulRun("Blacksmith Testbox", 4, "2026-06-17T10:52:00Z"), + successfulRun("Blacksmith ARM Testbox", 5, "2026-06-17T10:53:00Z"), + successfulRun("Workflow Sanity", 6, "2026-06-17T10:54:00Z"), + { + ...successfulRun(BUILD_ARTIFACTS_WORKFLOW, 2, "2026-06-17T10:50:00Z"), + status: "queued", + conclusion: null, + }, + ]; +} + function collectHostedGateEvidence( options: Omit[0], "nowMs" | "pr">, ) { @@ -88,15 +111,15 @@ describe("verify-pr-hosted-gates", () => { }; expect(() => collectHostedGateEvidence({ sha, workflowRuns })).toThrow( - "Missing successful Blacksmith ARM Testbox workflow", + "Missing successful recent Blacksmith ARM Testbox workflow", ); }); - it("accepts a non-docs PR when CI is the only scheduled authoritative workflow", () => { + it("accepts a sole scheduled CI run at the 24-hour boundary", () => { expect( collectHostedGateEvidence({ sha, - workflowRuns: [successfulRun("CI", 1, "2026-06-17T10:47:00Z")], + workflowRuns: [successfulRun("CI", 1, "2026-06-16T10:55:00Z")], }), ).toEqual({ headSha: sha, @@ -104,14 +127,14 @@ describe("verify-pr-hosted-gates", () => { }); }); - it("accepts recent green evidence from the recorded pre-rebase head while current CI is pending", () => { + it("accepts 13-hour green evidence from the recorded pre-rebase head", () => { const previousSha = "8d86c44c6144f8f726a460914cddb8c9c201f119"; const evidence = collectHostedGateEvidence({ sha, recentSha: previousSha, workflowRuns: [ { - ...successfulRun("CI", 1, "2026-06-17T10:50:00Z"), + ...successfulRun("CI", 1, "2026-06-16T21:55:00Z"), head_sha: previousSha, }, { @@ -213,7 +236,7 @@ describe("verify-pr-hosted-gates", () => { }, ], }), - ).toThrow(`Missing successful CI workflow for ${sha}`); + ).toThrow(`Missing successful recent CI workflow for ${sha}`); }, ); @@ -235,7 +258,7 @@ describe("verify-pr-hosted-gates", () => { }, ], }), - ).toThrow(`Missing successful CI workflow for ${sha}`); + ).toThrow(`Missing successful recent CI workflow for ${sha}`); }); it("requires the complete recent gate cohort from the recorded head", () => { @@ -313,7 +336,7 @@ describe("verify-pr-hosted-gates", () => { }, ], }), - ).toThrow(`Missing successful CI workflow for ${sha}`); + ).toThrow(`Missing successful recent CI workflow for ${sha}`); }); it("rejects stale or unrecorded fallback heads", () => { @@ -353,7 +376,7 @@ describe("verify-pr-hosted-gates", () => { sha, workflowRuns: [{ ...recentUnrelatedRun, head_sha: previousSha }, currentPending], }), - ).toThrow(`Missing successful CI workflow for ${sha}`); + ).toThrow(`Missing successful recent CI workflow for ${sha}`); }); it("allows a later scheduled success to clear an earlier current-head failure", () => { @@ -412,13 +435,13 @@ describe("verify-pr-hosted-gates", () => { }); }); - it("accepts the explicit exact-SHA manual CI release gate", () => { + it("accepts an exact-SHA manual CI release gate at the 24-hour boundary", () => { expect( collectHostedGateEvidence({ sha, workflowRuns: [ { - ...successfulRun(`CI release gate ${sha}`, 1, "2026-06-17T10:47:00Z"), + ...successfulRun(`CI release gate ${sha}`, 1, "2026-06-16T10:55:00Z"), event: "workflow_dispatch", path: ".github/workflows/ci.yml@refs/heads/release-controls", display_title: `CI release gate ${sha}`, @@ -431,23 +454,30 @@ describe("verify-pr-hosted-gates", () => { }); }); - it("prefers the exact release-gate fallback while scheduled CI remains queued", () => { + it.each([ + ["scheduled", successfulRun("CI", 1, "2026-06-16T10:54:59Z")], + ["manual", releaseGateRun(2, "2026-06-16T10:54:59Z")], + ])("rejects exact-head %s CI evidence older than 24 hours", (_kind, run) => { + expect(() => collectHostedGateEvidence({ sha, workflowRuns: [run] })).toThrow( + `Missing successful recent CI workflow for ${sha}`, + ); + }); + + it.each([ + [ + "queued", + { + ...successfulRun("CI", 1, "2026-06-17T10:50:00Z"), + status: "queued", + conclusion: null, + }, + ], + ["stale", successfulRun("CI", 1, "2026-06-16T10:54:59Z")], + ])("prefers a fresh exact release gate while scheduled CI is %s", (_state, scheduledRun) => { expect( collectHostedGateEvidence({ sha, - workflowRuns: [ - { - ...successfulRun("CI", 1, "2026-06-17T10:47:00Z"), - status: "queued", - conclusion: null, - updated_at: "2026-06-17T10:50:00Z", - }, - { - ...successfulRun(`CI release gate ${sha}`, 2, "2026-06-17T10:49:00Z"), - event: "workflow_dispatch", - display_title: `CI release gate ${sha}`, - }, - ], + workflowRuns: [scheduledRun, releaseGateRun(2, "2026-06-17T10:49:00Z")], }), ).toEqual({ headSha: sha, @@ -461,7 +491,7 @@ describe("verify-pr-hosted-gates", () => { sha, workflowRuns: [ { - ...successfulRun("CI", 1, "2026-06-17T10:50:00Z"), + ...successfulRun("CI", 1, "2026-06-16T10:54:59Z"), conclusion: "failure", }, { @@ -471,7 +501,7 @@ describe("verify-pr-hosted-gates", () => { }, ], }), - ).toThrow("Missing successful CI workflow"); + ).toThrow("Missing successful recent CI workflow"); }); it("does not mask a failed CI run with a queued rerun and release-gate fallback", () => { @@ -495,29 +525,14 @@ describe("verify-pr-hosted-gates", () => { }, ], }), - ).toThrow("Missing successful CI workflow"); + ).toThrow("Missing successful recent CI workflow"); }); it("covers a queued artifact Testbox only with a completed exact CI fallback", () => { expect( collectHostedGateEvidence({ sha, - workflowRuns: [ - { - ...successfulRun(`CI release gate ${sha}`, 1, "2026-06-17T10:49:00Z"), - event: "workflow_dispatch", - display_title: `CI release gate ${sha}`, - }, - successfulRun("CI", 3, "2026-06-17T10:51:00Z"), - successfulRun("Blacksmith Testbox", 4, "2026-06-17T10:52:00Z"), - successfulRun("Blacksmith ARM Testbox", 5, "2026-06-17T10:53:00Z"), - successfulRun("Workflow Sanity", 6, "2026-06-17T10:54:00Z"), - { - ...successfulRun(BUILD_ARTIFACTS_WORKFLOW, 2, "2026-06-17T10:50:00Z"), - status: "queued", - conclusion: null, - }, - ], + workflowRuns: queuedBuildArtifactFallbackRuns(), }), ).toEqual({ headSha: sha, @@ -537,6 +552,32 @@ describe("verify-pr-hosted-gates", () => { }); }); + it.each([ + ["release gate", 0], + ["supporting gate", 4], + ["queued artifact run", 5], + ])("does not cover queued artifacts with a stale %s", (_kind, staleRunIndex) => { + const workflowRuns = queuedBuildArtifactFallbackRuns().map((run, index) => + index === staleRunIndex ? { ...run, updated_at: "2026-06-16T10:54:59Z" } : run, + ); + expect(() => collectHostedGateEvidence({ sha, workflowRuns })).toThrow( + "Missing successful recent Blacksmith Build Artifacts Testbox workflow", + ); + }); + + it("keeps an older failed artifact run blocking a fresh queued retry", () => { + const workflowRuns = [ + ...queuedBuildArtifactFallbackRuns(), + { + ...successfulRun(BUILD_ARTIFACTS_WORKFLOW, 7, "2026-06-16T10:54:59Z"), + conclusion: "failure", + }, + ]; + expect(() => collectHostedGateEvidence({ sha, workflowRuns })).toThrow( + "Missing successful recent Blacksmith Build Artifacts Testbox workflow", + ); + }); + it("does not cover queued artifacts until all supporting workflow gates pass", () => { expect(() => collectHostedGateEvidence({ @@ -554,7 +595,7 @@ describe("verify-pr-hosted-gates", () => { }, ], }), - ).toThrow("Missing successful Blacksmith Build Artifacts Testbox workflow"); + ).toThrow("Missing successful recent Blacksmith Build Artifacts Testbox workflow"); }); it("keeps active or terminal non-successful artifact Testboxes blocking", () => { @@ -580,7 +621,7 @@ describe("verify-pr-hosted-gates", () => { sha, workflowRuns: [ciFallback, artifactRun], }), - ).toThrow("Missing successful Blacksmith Build Artifacts Testbox workflow"); + ).toThrow("Missing successful recent Blacksmith Build Artifacts Testbox workflow"); } expect(() => @@ -599,7 +640,7 @@ describe("verify-pr-hosted-gates", () => { }, ], }), - ).toThrow("Missing successful Blacksmith Build Artifacts Testbox workflow"); + ).toThrow("Missing successful recent Blacksmith Build Artifacts Testbox workflow"); }); it("rejects an unmarked manual CI run", () => { @@ -614,7 +655,7 @@ describe("verify-pr-hosted-gates", () => { }, ], }), - ).toThrow("Missing successful CI workflow"); + ).toThrow("Missing successful recent CI workflow"); }); it("rejects a manual release-gate title from another workflow", () => { @@ -630,12 +671,12 @@ describe("verify-pr-hosted-gates", () => { }, ], }), - ).toThrow("Missing successful CI workflow"); + ).toThrow("Missing successful recent CI workflow"); }); it("requires CI for docs unless the head changes only CHANGELOG.md", () => { expect(() => collectHostedGateEvidence({ sha, workflowRuns: [] })).toThrow( - "Missing successful CI workflow", + "Missing successful recent CI workflow", ); expect(collectHostedGateEvidence({ sha, workflowRuns: [], changelogOnly: true })).toEqual({ headSha: sha, @@ -697,7 +738,7 @@ describe("verify-pr-hosted-gates", () => { `repos/openclaw/openclaw/actions/runs?head_sha=${sha}&per_page=100&page=1`, `repos/openclaw/openclaw/actions/runs?head_sha=${previousSha}&per_page=100&page=1`, ]); - expect(HOSTED_GATE_MAX_AGE_HOURS).toBe(12); + expect(HOSTED_GATE_MAX_AGE_HOURS).toBe(24); }); it("queries recent pull-request runs for the head branch", () => {