ci: hourly sweeper re-fires dropped pull_request CI runs (#110889)

* ci: add hourly PR CI sweeper for dropped pull_request runs Fresh PRs race GitHub's merge-ref computation: the open-event CI run can drop entirely or be created as an un-rerunnable startup_failure (~10-16 runs daily). The sweeper lists recently updated open PRs hourly, finds heads whose only pull_request-event CI runs are startup failures (or missing), and re-fires the event by close/reopen with the Barnacle app token (GITHUB_TOKEN events would not trigger workflows). Safety: 10-minute quiet window, 24h lookback, skips drafts, merge conflicts, pending mergeability, and auto-merge PRs (close cancels auto-merge); revalidates state, head, and CI attachment immediately before mutating; per-PR budget of two sweeper closes and a per-sweep cap of ten; reopen-on-unknown ownership so a stranded close (silent) always loses to a spurious reopen (visible); manual dispatch supports dry_run. Accepted tradeoffs are documented inline: shared-SHA PR topologies can mask a dropped run (skip-only miss; run.pull_requests matching would break fork PRs), and app-auth failover at worst doubles the close budget. * test(ci): exercise pr-ci-sweeper runner with a faked client * fix(test): lint-clean pr-ci-sweeper runner fakes
2026-08-12 21:53:00 -06:00 · 2026-07-18 20:18:57 +01:00
parent 36cba351e5
commit 53ad69c6ee
4 changed files with 581 additions and 0 deletions
@@ -0,0 +1,67 @@
+name: PR CI Sweeper
+
+# Re-fires pull_request CI runs that GitHub dropped at PR open (merge-ref
+# race: no CI run attaches, or an un-rerunnable startup_failure is created).
+# Close/reopen with an app token re-fires the event; GITHUB_TOKEN events
+# would not trigger workflows.
+
+on:
+  schedule:
+    - cron: "7 * * * *"
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: Log intended re-fires without closing/reopening.
+        required: false
+        default: false
+        type: boolean
+
+env:
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+
+concurrency:
+  group: pr-ci-sweeper
+  cancel-in-progress: false
+
+permissions: {}
+
+jobs:
+  sweep:
+    permissions:
+      contents: read
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+      - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3
+        id: app-token
+        continue-on-error: true
+        with:
+          app-id: "2729701"
+          private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
+      - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3
+        id: app-token-fallback
+        if: steps.app-token.outcome == 'failure'
+        with:
+          app-id: "2971289"
+          private-key: ${{ secrets.GH_APP_PRIVATE_KEY_FALLBACK }}
+      - name: Sweep dropped PR CI runs
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
+        with:
+          github-token: ${{ steps.app-token.outputs.token || steps.app-token-fallback.outputs.token }}
+          script: |
+            const { pathToFileURL } = require("node:url");
+            const moduleUrl = pathToFileURL(
+              `${process.env.GITHUB_WORKSPACE}/scripts/github/pr-ci-sweeper.mjs`,
+            );
+            const { runPrCiSweeper } = await import(moduleUrl.href);
+
+            await runPrCiSweeper({
+              github,
+              context,
+              core,
+              dryRun: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false }},
+              appSlug: ${{ toJSON(steps.app-token.outputs.app-slug || steps.app-token-fallback.outputs.app-slug || '') }},
+            });
@@ -0,0 +1,19 @@
+export function classifyPrForSweep(params: {
+  pr: {
+    draft?: boolean;
+    created_at: string;
+    updated_at: string;
+    mergeable?: boolean | null;
+    auto_merge?: object | null;
+  };
+  ciRuns: Array<{ conclusion: string | null }>;
+  botCloseCount: number;
+  now: number;
+}): { action: "refire" | "skip"; reason: string };
+export function runPrCiSweeper(params: {
+  github: Record<string, unknown>;
+  context: Record<string, unknown>;
+  core: Pick<Console, "info"> & { setFailed: (message: string) => void };
+  dryRun?: boolean;
+  appSlug?: string;
+}): Promise<Array<{ number: number; sha: string; action: "refire" | "skip"; reason: string }>>;
@@ -0,0 +1,254 @@
+// Re-fires pull_request CI runs that GitHub dropped at PR open. Fresh PRs can
+// race merge-ref computation: the open event's CI run either never attaches to
+// the head SHA or is created as an un-rerunnable startup_failure. Closing and
+// reopening the PR re-fires the event once the merge ref exists. The workflow
+// authenticates with a GitHub App token because GITHUB_TOKEN-authored events
+// do not trigger new workflow runs.
+
+const CI_WORKFLOW_FILE = "ci.yml";
+const LOOKBACK_MS = 24 * 60 * 60 * 1000;
+// Give GitHub time to settle merge-ref computation and late run attachment
+// before judging a head SHA as dropped.
+const MIN_QUIET_MS = 10 * 60 * 1000;
+// Two bot closes per PR: a head that still has no CI after two re-fires needs
+// a human, not an hourly close/reopen loop.
+const MAX_BOT_CLOSES = 2;
+const MAX_REFIRES_PER_SWEEP = 10;
+// Known sweeper identities for the close budget. The fallback app's login is
+// only recognized while it is the active identity, so an auth failover can at
+// worst double the budget to four re-fires — still bounded, and the
+// newest-close ownership check keeps human closes authoritative regardless.
+const KNOWN_SWEEPER_LOGINS = ["openclaw-barnacle[bot]"];
+const REOPEN_DELAY_MS = 5_000;
+
+const sleep = (ms) =>
+  new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+
+export function classifyPrForSweep({ pr, ciRuns, botCloseCount, now }) {
+  if (pr.draft) {
+    return { action: "skip", reason: "draft" };
+  }
+  if (now - Date.parse(pr.created_at) > LOOKBACK_MS) {
+    return { action: "skip", reason: "outside-lookback" };
+  }
+  if (now - Date.parse(pr.updated_at) < MIN_QUIET_MS) {
+    return { action: "skip", reason: "recently-updated" };
+  }
+  // A conflicted PR legitimately has no merge ref; CI cannot attach until the
+  // author resolves, so re-firing would loop forever.
+  if (pr.mergeable === false) {
+    return { action: "skip", reason: "merge-conflict" };
+  }
+  // Pending computation resolves by the next hourly sweep; do not close/reopen
+  // on an unknown merge state.
+  if (pr.mergeable === null || pr.mergeable === undefined) {
+    return { action: "skip", reason: "mergeability-pending" };
+  }
+  // Closing a PR silently cancels enabled auto-merge and reopening does not
+  // restore it; leave those PRs (e.g. generated locale refreshes) to a human.
+  if (pr.auto_merge) {
+    return { action: "skip", reason: "auto-merge-enabled" };
+  }
+  // Queued and in-progress runs have a null conclusion and count as attached.
+  if (ciRuns.some((run) => run.conclusion !== "startup_failure")) {
+    return { action: "skip", reason: "ci-attached" };
+  }
+  if (botCloseCount >= MAX_BOT_CLOSES) {
+    return { action: "skip", reason: "refire-budget-exhausted" };
+  }
+  return {
+    action: "refire",
+    reason: ciRuns.length === 0 ? "ci-run-missing" : "ci-startup-failure",
+  };
+}
+
+async function listPullRequestCiRuns({ github, owner, repo, headSha }) {
+  // Manual dispatches or other events against the same SHA neither prove nor
+  // repair the dropped pull_request run; judge only pull_request-event runs,
+  // filtered server-side and paginated so unrelated runs cannot crowd them out.
+  // Accepted tradeoff: a SHA shared by two PRs can mask one PR's dropped run
+  // behind the other's — a skip-only miss. Matching run.pull_requests instead
+  // would misclassify fork PRs, where GitHub leaves that array empty.
+  return await github.paginate(github.rest.actions.listWorkflowRuns, {
+    owner,
+    repo,
+    workflow_id: CI_WORKFLOW_FILE,
+    head_sha: headSha,
+    event: "pull_request",
+    per_page: 100,
+  });
+}
+
+// Our close call succeeded against a verified-open PR, so the sweeper owns the
+// transition unless a newer close event by someone else is positively visible
+// (a human close in the millisecond race window makes our update an eventless
+// no-op). Stale or lagging event reads must therefore default to "ours".
+async function someoneElseClosed({
+  github,
+  owner,
+  repo,
+  pullNumber,
+  sweeperLogins,
+  knownCloseIds,
+}) {
+  const events = await github.paginate(github.rest.issues.listEvents, {
+    owner,
+    repo,
+    issue_number: pullNumber,
+    per_page: 100,
+  });
+  const newClose = events.findLast(
+    (event) => event.event === "closed" && !knownCloseIds.has(event.id),
+  );
+  if (!newClose?.actor) {
+    return false;
+  }
+  return !(newClose.actor.type === "Bot" && sweeperLogins.has(newClose.actor.login));
+}
+
+async function reopenWithRetry({ github, core, owner, repo, pullNumber }) {
+  let lastError;
+  for (let attempt = 1; attempt <= 3; attempt += 1) {
+    try {
+      await github.rest.pulls.update({ owner, repo, pull_number: pullNumber, state: "open" });
+      return true;
+    } catch (error) {
+      lastError = error;
+      await sleep(REOPEN_DELAY_MS * attempt);
+    }
+  }
+  // Never leave a swept PR closed silently: surface on the PR and fail the run.
+  await github.rest.issues
+    .createComment({
+      owner,
+      repo,
+      issue_number: pullNumber,
+      body: "PR CI Sweeper closed this PR to re-fire a dropped CI run but could not reopen it. Please reopen manually.",
+    })
+    .catch(() => undefined);
+  core.setFailed(`pr-ci-sweeper: failed to reopen #${pullNumber}: ${String(lastError)}`);
+  return false;
+}
+
+export async function runPrCiSweeper({ github, context, core, dryRun = false, appSlug = "" }) {
+  const sweeperLogins = new Set(KNOWN_SWEEPER_LOGINS);
+  if (appSlug) {
+    sweeperLogins.add(`${appSlug}[bot]`);
+  }
+  const { owner, repo } = context.repo;
+  const now = Date.now();
+  const results = [];
+  let refires = 0;
+  const openPrs = await github.paginate(github.rest.pulls.list, {
+    owner,
+    repo,
+    state: "open",
+    sort: "updated",
+    direction: "desc",
+    per_page: 100,
+  });
+  for (const listed of openPrs) {
+    if (now - Date.parse(listed.updated_at) > LOOKBACK_MS) {
+      break;
+    }
+    if (refires >= MAX_REFIRES_PER_SWEEP) {
+      core.info(`pr-ci-sweeper: per-sweep re-fire cap (${MAX_REFIRES_PER_SWEEP}) reached`);
+      break;
+    }
+    if (listed.draft) {
+      continue;
+    }
+    const ciRuns = await listPullRequestCiRuns({ github, owner, repo, headSha: listed.head.sha });
+    if (ciRuns.some((run) => run.conclusion !== "startup_failure")) {
+      continue;
+    }
+    // Candidate: fetch authoritative state (mergeable, current head) and the
+    // close history so a racing push or human action wins over the sweep.
+    const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: listed.number });
+    if (pr.state !== "open" || pr.head.sha !== listed.head.sha) {
+      continue;
+    }
+    const events = await github.paginate(github.rest.issues.listEvents, {
+      owner,
+      repo,
+      issue_number: pr.number,
+      per_page: 100,
+    });
+    // Budget counts only this sweeper's own closes so unrelated bot
+    // automation cannot exhaust a PR's re-fire allowance.
+    const botCloseCount = events.filter(
+      (event) =>
+        event.event === "closed" &&
+        event.actor?.type === "Bot" &&
+        sweeperLogins.has(event.actor.login),
+    ).length;
+    const verdict = classifyPrForSweep({ pr, ciRuns, botCloseCount, now });
+    results.push({ number: pr.number, sha: pr.head.sha.slice(0, 12), ...verdict });
+    if (verdict.action !== "refire") {
+      core.info(`pr-ci-sweeper: skip #${pr.number} (${verdict.reason})`);
+      continue;
+    }
+    refires += 1;
+    if (dryRun) {
+      core.info(`pr-ci-sweeper: dry-run, would re-fire #${pr.number} (${verdict.reason})`);
+      continue;
+    }
+    core.info(`pr-ci-sweeper: re-firing CI for #${pr.number} (${verdict.reason})`);
+    // Revalidate immediately before mutating: a human close or a fresh push in
+    // the classify gap must win over the sweep.
+    const { data: fresh } = await github.rest.pulls.get({ owner, repo, pull_number: pr.number });
+    if (
+      fresh.state !== "open" ||
+      fresh.head.sha !== pr.head.sha ||
+      fresh.auto_merge ||
+      fresh.mergeable !== true
+    ) {
+      core.info(`pr-ci-sweeper: #${pr.number} changed during sweep; leaving it alone`);
+      continue;
+    }
+    // CI can attach late during the scan's own API calls; closing then would
+    // cancel a live run. Re-check the head immediately before mutating.
+    const latestRuns = await listPullRequestCiRuns({
+      github,
+      owner,
+      repo,
+      headSha: fresh.head.sha,
+    });
+    if (latestRuns.some((run) => run.conclusion !== "startup_failure")) {
+      core.info(`pr-ci-sweeper: #${pr.number} CI attached during sweep; leaving it alone`);
+      continue;
+    }
+    const knownCloseIds = new Set(
+      events.filter((event) => event.event === "closed").map((event) => event.id),
+    );
+    await github.rest.pulls.update({ owner, repo, pull_number: pr.number, state: "closed" });
+    await sleep(REOPEN_DELAY_MS);
+    // Skip the reopen only on positive evidence that someone else performed a
+    // newer close. Verification errors and stale event reads fail toward
+    // reopening: stranding our own close is the worse outcome.
+    let humanClosed = false;
+    try {
+      humanClosed = await someoneElseClosed({
+        github,
+        owner,
+        repo,
+        pullNumber: pr.number,
+        sweeperLogins,
+        knownCloseIds,
+      });
+    } catch (error) {
+      core.info(`pr-ci-sweeper: close-ownership check failed (${String(error)}); reopening`);
+    }
+    if (humanClosed) {
+      core.info(`pr-ci-sweeper: #${pr.number} was closed by someone else; not reopening`);
+      continue;
+    }
+    await reopenWithRetry({ github, core, owner, repo, pullNumber: pr.number });
+  }
+  core.info(
+    `pr-ci-sweeper: checked ${openPrs.length} open PRs, ${results.length} candidates, ${refires} re-fire${refires === 1 ? "" : "s"}${dryRun ? " (dry-run)" : ""}`,
+  );
+  return results;
+}
@@ -0,0 +1,241 @@
+import { describe, expect, it } from "vitest";
+import { classifyPrForSweep, runPrCiSweeper } from "../../scripts/github/pr-ci-sweeper.mjs";
+
+const NOW = Date.parse("2026-07-18T12:00:00Z");
+const MINUTES = 60 * 1000;
+const HOURS = 60 * MINUTES;
+
+function pr(overrides: Partial<Parameters<typeof classifyPrForSweep>[0]["pr"]> = {}) {
+  return {
+    draft: false,
+    created_at: new Date(NOW - 2 * HOURS).toISOString(),
+    updated_at: new Date(NOW - 30 * MINUTES).toISOString(),
+    mergeable: true,
+    auto_merge: null,
+    ...overrides,
+  };
+}
+
+describe("classifyPrForSweep", () => {
+  const cases: Array<{
+    name: string;
+    input: Parameters<typeof classifyPrForSweep>[0];
+    expected: ReturnType<typeof classifyPrForSweep>;
+  }> = [
+    {
+      name: "re-fires when no CI run attached",
+      input: { pr: pr(), ciRuns: [], botCloseCount: 0, now: NOW },
+      expected: { action: "refire", reason: "ci-run-missing" },
+    },
+    {
+      name: "re-fires when only startup failures attached",
+      input: {
+        pr: pr(),
+        ciRuns: [{ conclusion: "startup_failure" }],
+        botCloseCount: 1,
+        now: NOW,
+      },
+      expected: { action: "refire", reason: "ci-startup-failure" },
+    },
+    {
+      name: "skips drafts",
+      input: { pr: pr({ draft: true }), ciRuns: [], botCloseCount: 0, now: NOW },
+      expected: { action: "skip", reason: "draft" },
+    },
+    {
+      name: "skips PRs outside the 24h lookback",
+      input: {
+        pr: pr({ created_at: new Date(NOW - 25 * HOURS).toISOString() }),
+        ciRuns: [],
+        botCloseCount: 0,
+        now: NOW,
+      },
+      expected: { action: "skip", reason: "outside-lookback" },
+    },
+    {
+      name: "skips recently updated PRs so merge-ref computation can settle",
+      input: {
+        pr: pr({ updated_at: new Date(NOW - 5 * MINUTES).toISOString() }),
+        ciRuns: [],
+        botCloseCount: 0,
+        now: NOW,
+      },
+      expected: { action: "skip", reason: "recently-updated" },
+    },
+    {
+      name: "skips merge conflicts whose merge ref legitimately cannot exist",
+      input: { pr: pr({ mergeable: false }), ciRuns: [], botCloseCount: 0, now: NOW },
+      expected: { action: "skip", reason: "merge-conflict" },
+    },
+    {
+      name: "skips PRs with auto-merge enabled (close would cancel it)",
+      input: {
+        pr: pr({ auto_merge: { merge_method: "squash" } }),
+        ciRuns: [],
+        botCloseCount: 0,
+        now: NOW,
+      },
+      expected: { action: "skip", reason: "auto-merge-enabled" },
+    },
+    {
+      name: "treats a completed run as attached",
+      input: {
+        pr: pr(),
+        ciRuns: [{ conclusion: "success" }],
+        botCloseCount: 0,
+        now: NOW,
+      },
+      expected: { action: "skip", reason: "ci-attached" },
+    },
+    {
+      name: "treats a queued run (null conclusion) as attached",
+      input: {
+        pr: pr(),
+        ciRuns: [{ conclusion: null }, { conclusion: "startup_failure" }],
+        botCloseCount: 0,
+        now: NOW,
+      },
+      expected: { action: "skip", reason: "ci-attached" },
+    },
+    {
+      name: "treats a failed run as attached (rerunnable, not sweepable)",
+      input: {
+        pr: pr(),
+        ciRuns: [{ conclusion: "failure" }],
+        botCloseCount: 0,
+        now: NOW,
+      },
+      expected: { action: "skip", reason: "ci-attached" },
+    },
+    {
+      name: "stops after two bot closes",
+      input: { pr: pr(), ciRuns: [], botCloseCount: 2, now: NOW },
+      expected: { action: "skip", reason: "refire-budget-exhausted" },
+    },
+    {
+      name: "skips while mergeability is still computing",
+      input: { pr: pr({ mergeable: null }), ciRuns: [], botCloseCount: 0, now: NOW },
+      expected: { action: "skip", reason: "mergeability-pending" },
+    },
+  ];
+
+  it.each(cases)("$name", ({ input, expected }) => {
+    expect(classifyPrForSweep(input)).toEqual(expected);
+  });
+});
+
+type FakeCall = { method: string; args: Record<string, unknown> };
+
+function fakeGithub(options: {
+  prs: Array<Record<string, unknown>>;
+  runsBySha: Record<string, Array<{ conclusion: string | null; event?: string }>>;
+  events?: Array<Record<string, unknown>>;
+}) {
+  const calls: FakeCall[] = [];
+  const record = (method: string, args: Record<string, unknown>) => {
+    calls.push({ method, args });
+  };
+  const github = {
+    paginate: (endpoint: { endpointName: string }, args: Record<string, unknown>) => {
+      record(endpoint.endpointName, args);
+      if (endpoint.endpointName === "pulls.list") {
+        return Promise.resolve(options.prs);
+      }
+      if (endpoint.endpointName === "actions.listWorkflowRuns") {
+        return Promise.resolve(
+          (options.runsBySha[args.head_sha as string] ?? []).map((run) => ({
+            event: run.event ?? "pull_request",
+            conclusion: run.conclusion,
+          })),
+        );
+      }
+      if (endpoint.endpointName === "issues.listEvents") {
+        return Promise.resolve(options.events ?? []);
+      }
+      throw new Error(`unexpected paginate ${endpoint.endpointName}`);
+    },
+    rest: {
+      pulls: {
+        list: { endpointName: "pulls.list" },
+        get: (args: Record<string, unknown>) => {
+          record("pulls.get", args);
+          const match = options.prs.find((entry) => entry.number === args.pull_number);
+          return Promise.resolve({ data: match });
+        },
+        update: (args: Record<string, unknown>) => {
+          record("pulls.update", args);
+          return Promise.resolve({});
+        },
+      },
+      actions: { listWorkflowRuns: { endpointName: "actions.listWorkflowRuns" } },
+      issues: {
+        listEvents: { endpointName: "issues.listEvents" },
+        createComment: (args: Record<string, unknown>) => {
+          record("issues.createComment", args);
+          return Promise.resolve({});
+        },
+      },
+    },
+  };
+  return { github, calls };
+}
+
+const context = { repo: { owner: "openclaw", repo: "openclaw" } };
+const core = { info: () => {}, setFailed: () => {} };
+
+describe("runPrCiSweeper", () => {
+  it("classifies a dropped-CI PR as refire in dry-run without mutating", async () => {
+    const dropped = {
+      ...pr(),
+      number: 7,
+      state: "open",
+      head: { sha: "a".repeat(40) },
+    };
+    const attached = {
+      ...pr(),
+      number: 8,
+      state: "open",
+      head: { sha: "b".repeat(40) },
+    };
+    const { github, calls } = fakeGithub({
+      prs: [dropped, attached],
+      runsBySha: {
+        [dropped.head.sha]: [{ conclusion: "startup_failure" }],
+        [attached.head.sha]: [{ conclusion: "success" }],
+      },
+    });
+    const results = await runPrCiSweeper({
+      github: github as never,
+      context: context as never,
+      core: core as never,
+      dryRun: true,
+      appSlug: "openclaw-barnacle",
+    });
+    expect(results).toEqual([
+      { number: 7, sha: "a".repeat(12), action: "refire", reason: "ci-startup-failure" },
+    ]);
+    expect(calls.filter((call) => call.method === "pulls.update")).toEqual([]);
+  });
+
+  it("closes and reopens a dropped-CI PR in live mode", async () => {
+    const dropped = {
+      ...pr(),
+      number: 9,
+      state: "open",
+      head: { sha: "c".repeat(40) },
+    };
+    const { github, calls } = fakeGithub({ prs: [dropped], runsBySha: {} });
+    const results = await runPrCiSweeper({
+      github: github as never,
+      context: context as never,
+      core: core as never,
+      appSlug: "openclaw-barnacle",
+    });
+    expect(results).toEqual([
+      { number: 9, sha: "c".repeat(12), action: "refire", reason: "ci-run-missing" },
+    ]);
+    expect(
+      calls.filter((call) => call.method === "pulls.update").map((call) => call.args.state),
+    ).toEqual(["closed", "open"]);
+  });
+});