diff --git a/.agents/skills/openclaw-ci-limits/SKILL.md b/.agents/skills/openclaw-ci-limits/SKILL.md index e46a6e33681a..28d378640b0f 100644 --- a/.agents/skills/openclaw-ci-limits/SKILL.md +++ b/.agents/skills/openclaw-ci-limits/SKILL.md @@ -114,9 +114,10 @@ such as `preflight`, `security-fast`, `build-artifacts`, and platform lanes. For repeated pull-request pushes, multiply by the number of runs expected to reach Blacksmith admission in the same 5-minute window, including runs canceled -after admission. Canonical `main` is single-flight: one run completes while -GitHub's default single pending slot is replaced by the newest push. Count one -active main matrix plus its next pending matrix, not every intermediate merge. +after admission. Canonical `main` uses two run-number-parity slots. Each slot +keeps one active non-canceling run and one coalesced pending tip. Budget for up +to two active main matrices plus their two pending tips entering the next +admission wave, not every intermediate merge. Reject a change unless the org-level worst case stays below about 60% of the live bucket. With the current 10,000-registration bucket, keep planned @@ -128,8 +129,8 @@ ClawSweeper, ClawHub, Clownfish, OpenClaw RTT, and Clawbench. Prefer these in order: 1. Preserve cancel-in-progress for superseded pull-request heads. -2. Preserve canonical `main` single-flight without canceling its running - integration cycle; GitHub's default pending slot coalesces to the newest tip. +2. Preserve canonical `main` as two non-canceling parity slots; each slot's + default pending run coalesces to the newest tip. 3. Move high-frequency, short, non-build jobs to `ubuntu-24.04`. 4. Reduce matrix rows by bundling related tests inside one runner job when the combined job stays under timeout and keeps useful failure names. @@ -155,8 +156,8 @@ Do not: These are intentionally guarded by `test/scripts/ci-workflow-guards.test.ts`: -- `CI` concurrency key version, PR cancellation, and non-canceling canonical - `main` single-flight with one coalesced pending tip. +- `CI` concurrency key version, PR cancellation, and canonical `main`'s two + non-canceling parity slots, each with one coalesced pending tip. - `preflight` and hosted `security-fast` start immediately without a debounce or standalone admission job. On Node-relevant canonical main pushes and same-repo pull requests, preflight owns the sole immutable semantic diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5637f915ac1..77b8dcdb95f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,9 +57,9 @@ permissions: run-name: ${{ github.event_name == 'workflow_dispatch' && inputs.dispatch_id != '' && format('CI {0}', inputs.dispatch_id) || (github.event_name == 'workflow_dispatch' && inputs.release_gate && format('CI release gate {0}', inputs.target_ref) || 'CI') }} concurrency: - group: ${{ github.event_name == 'workflow_dispatch' && format('{0}-manual-v1-{1}', github.workflow, github.run_id) || (github.event_name == 'pull_request' && format('{0}-v7-{1}', github.workflow, github.event.pull_request.number) || (github.repository == 'openclaw/openclaw' && format('{0}-v7-{1}', github.workflow, github.ref) || format('{0}-v7-{1}-{2}', github.workflow, github.ref, github.sha))) }} - # PRs want newest-head feedback. Canonical main instead runs one complete - # integration cycle while GitHub's single pending slot coalesces later pushes. + # Canonical main uses run-number parity for two non-canceling slots. Each + # slot keeps GitHub's single coalesced pending tip while its active run finishes. + group: ${{ github.event_name == 'workflow_dispatch' && format('{0}-manual-v1-{1}', github.workflow, github.run_id) || (github.event_name == 'pull_request' && format('{0}-v7-{1}', github.workflow, github.event.pull_request.number) || (github.repository == 'openclaw/openclaw' && github.event_name == 'push' && github.ref == 'refs/heads/main' && format('{0}-v8-{1}-{2}', github.workflow, github.ref, (endsWith(format('{0}', github.run_number), '0') || endsWith(format('{0}', github.run_number), '2') || endsWith(format('{0}', github.run_number), '4') || endsWith(format('{0}', github.run_number), '6') || endsWith(format('{0}', github.run_number), '8')) && 'a' || 'b') || (github.repository == 'openclaw/openclaw' && format('{0}-v7-{1}', github.workflow, github.ref) || format('{0}-v7-{1}-{2}', github.workflow, github.ref, github.sha)))) }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: diff --git a/docs/ci.md b/docs/ci.md index 714be4125671..2727e50e6870 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -10,14 +10,16 @@ read_when: OpenClaw CI runs on pushes to `main` (Markdown and `docs/**` paths are ignored at the trigger), on every non-draft pull request, and on manual dispatch. -Canonical `main` pushes are single-flight: the `CI` concurrency group lets one -complete integration cycle run while GitHub keeps only the newest pending push. -New merges replace that pending run instead of canceling work that already -registered a Blacksmith matrix. Pull requests still cancel superseded heads, -and manual dispatches use isolated groups. `preflight` classifies the diff and -turns expensive lanes off when only unrelated areas changed. Ordinary manual -`workflow_dispatch` runs intentionally bypass smart scoping and fan out the -full graph for release candidates and broad validation. Exact-head +Canonical `main` pushes use a two-slot pipeline keyed by run-number parity, so +at most two integration runs overlap. Each slot is non-canceling and keeps one +coalesced pending tip: a new merge replaces that slot's older pending run +instead of canceling work that already registered a Blacksmith matrix. Runs in +the two slots can complete out of order; exact-head consumers remain bound to +their requested SHA and are unaffected. Pull requests still cancel superseded +heads, and manual dispatches use isolated groups. `preflight` classifies the +diff and turns expensive lanes off when only unrelated areas changed. Ordinary +manual `workflow_dispatch` runs intentionally bypass smart scoping and fan out +the full graph for release candidates and broad validation. Exact-head `release_gate` fallbacks retain the pull request's macOS and iOS scope instead of forcing unrelated Apple lanes. Android lanes stay opt-in through `include_android` (or the `release_gate` input). Release-only @@ -59,7 +61,7 @@ Standalone Periphery workflows enforce zero dead-code findings for the iOS and m ## Fail-fast order -1. `preflight` decides which lanes exist at all. The `docs-scope` and `changed-scope` logic are steps inside this job, not standalone jobs. Canonical `main` starts immediately, but its concurrency group admits only one complete run and coalesces later pushes into one newest pending run. On Node-relevant canonical `main` pushes and same-repository pull requests, preflight is the sole exact dependency-cache writer; downstream jobs wait for it, then restore the immutable archive or fall back to the ordinary pnpm-store cache on a miss. +1. `preflight` decides which lanes exist at all. The `docs-scope` and `changed-scope` logic are steps inside this job, not standalone jobs. Canonical `main` starts immediately in one of two parity slots; each slot admits one complete run and coalesces later pushes into its newest pending tip. On Node-relevant canonical `main` pushes and same-repository pull requests, preflight is the sole exact dependency-cache writer; downstream jobs wait for it, then restore the immutable archive or fall back to the ordinary pnpm-store cache on a miss. 2. `security-fast`, `check-*`, `check-additional-*`, `check-docs`, and `skills-python` fail quickly without waiting on the heavier artifact and platform matrix jobs. 3. `build-artifacts` and the locale checks overlap with the fast Linux lanes. Control UI and native app source PRs exclude generated locale snapshots/resources; their serialized refresh workflows repair and auto-merge isolated generated PRs in the background. Source CI still blocks stale source inventories and unsafe localization calls. Generated PRs, manual CI, and release prep enforce full translated/platform-generated parity. Canonical `release/YYYY.M.PATCH` branches may include release-prep locale repairs with the other generated release output. 4. Heavier platform and runtime lanes fan out after that: `checks-fast-core`, `checks-fast-contracts-plugins-*`, `checks-fast-contracts-channels-*`, `checks-node-*`, `checks-windows`, `macos-node`, `macos-swift`, `ios-build`, and `android`. @@ -74,7 +76,7 @@ that unchanged head during the freshness window. The default-branch ruleset requires the GitHub Actions-owned `openclaw/ci-gate` check. Repository maintainers and admins have an audited break-glass bypass intended only for signed direct fast-forward landings; the organization ruleset still blocks deletion and non-fast-forward updates. Normal pull-request merges should continue to use the gate rather than bypass failed CI. The separate strict App-owned test-merge check still binds the head to current `main`. -GitHub may mark superseded pull-request jobs as `cancelled` when a newer head lands. Treat that as CI noise unless the newest run for the same PR is also failing. Canonical `main` runs are not canceled after admission; when merge traffic arrives, GitHub replaces only the older pending run with the newest tip. Matrix jobs use `fail-fast: false`, and `build-artifacts` reports embedded channel, core-support-boundary, and gateway-watch failures directly instead of queuing tiny verifier jobs. The automatic CI concurrency key is versioned (`CI-v7-*`) so a GitHub-side zombie in an old queue group cannot indefinitely block newer main runs. Manual full-suite runs use `CI-manual-v1-*` and do not cancel in-progress runs. The plugin-list startup-memory guard keeps a 350 MiB ceiling on self-hosted Blacksmith Linux and allows 425 MiB on GitHub-hosted Linux, whose RSS baseline is higher for the same built CLI. +GitHub may mark superseded pull-request jobs as `cancelled` when a newer head lands. Treat that as CI noise unless the newest run for the same PR is also failing. Canonical `main` runs are not canceled after admission; each of the two parity slots replaces only its older pending run with the newest tip. Matrix jobs use `fail-fast: false`, and `build-artifacts` reports embedded channel, core-support-boundary, and gateway-watch failures directly instead of queuing tiny verifier jobs. The canonical-main CI concurrency key is versioned (`CI-v8-*`) so GitHub-side zombies in the old group cannot block the two-slot pipeline; other automatic groups remain on `CI-v7-*`. Manual full-suite runs use `CI-manual-v1-*` and do not cancel in-progress runs. The plugin-list startup-memory guard keeps a 350 MiB ceiling on self-hosted Blacksmith Linux and allows 425 MiB on GitHub-hosted Linux, whose RSS baseline is higher for the same built CLI. Use `pnpm ci:timings`, `pnpm ci:timings:recent`, or `node scripts/ci-run-timings.mjs ` to summarize wall time, start delay, slowest jobs, failures, and the `pnpm-store-warmup` fanout barrier from GitHub Actions. Use `pnpm ci:timings:trend` for a 72-hour baseline and a latest-12-hours versus prior-12-hours comparison. Trend mode includes every main push outcome, cancellation/pass rates, and successful-run wall time, then loads a balanced latest/prior sample of at most 100 successful runs by default. Its detailed sample separates workflow admission, job dependency/gate delay (`job.created_at` minus the first job's creation), runner queue/start latency (`job.started_at` minus `job.created_at`), and execution; it also reports critical-path ownership and the actual GitHub API request count. Reruns use attempt-specific jobs and are excluded from run-level wall/admission distributions because GitHub retains the original workflow creation time. Raise or lower the detailed-run selection cap with `--detail-runs` (a run with more than 100 jobs requires multiple requests), emit JSON to stdout with `--json`, or save the same report with `--output .artifacts/ci-timings/trend.json`; missing output directories are created automatically. The baseline must cover at least two comparison windows. diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 4c7d0a796686..918460c192b1 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -2804,12 +2804,12 @@ NODE expect(nativeResourcesSetup.with).toMatchObject({ "install-bun": "false" }); }); - it("runs canonical main CI single-flight while coalescing the pending tip", () => { + it("pipelines canonical main CI across two non-canceling slots", () => { const workflow = readCiWorkflow(); - // GitHub concurrency keeps one running and one pending run by default. - // Replacing only the pending run preserves a complete integration cycle - // while coalescing merge bursts to the newest main tip. + expect(workflow.concurrency.group).toBe( + "${{ github.event_name == 'workflow_dispatch' && format('{0}-manual-v1-{1}', github.workflow, github.run_id) || (github.event_name == 'pull_request' && format('{0}-v7-{1}', github.workflow, github.event.pull_request.number) || (github.repository == 'openclaw/openclaw' && github.event_name == 'push' && github.ref == 'refs/heads/main' && format('{0}-v8-{1}-{2}', github.workflow, github.ref, (endsWith(format('{0}', github.run_number), '0') || endsWith(format('{0}', github.run_number), '2') || endsWith(format('{0}', github.run_number), '4') || endsWith(format('{0}', github.run_number), '6') || endsWith(format('{0}', github.run_number), '8')) && 'a' || 'b') || (github.repository == 'openclaw/openclaw' && format('{0}-v7-{1}', github.workflow, github.ref) || format('{0}-v7-{1}-{2}', github.workflow, github.ref, github.sha)))) }}", + ); expect(workflow.concurrency["cancel-in-progress"]).toBe( "${{ github.event_name == 'pull_request' }}", );