diff --git a/docs/ci.md b/docs/ci.md index e1bf61d6bf4d..82956ab3fa8d 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -118,7 +118,7 @@ The slowest Node test families are split or balanced so each job stays small wit - Core unit fast/support lanes run separately; core runtime infra splits into process, shared, hooks, secrets, and three cron domain shards. - Auto-reply runs as balanced workers, with the reply subtree split into agent-runner, commands, dispatch, session, and state-routing shards. - Agentic gateway/server (control-plane) configs split across chat, auth, model, HTTP/plugin, runtime, and startup lanes instead of waiting on built artifacts. -- Normal CI packs only isolated infra include-pattern shards into deterministic bundles of at most 64 test files, reducing the Node matrix without merging non-isolated command/cron, stateful agents-core, or gateway/server suites. Heavy fixed suites stay on 8 vCPU while the bundled and lower-weight lanes use 4 vCPU. +- Normal CI packs only isolated infra include-pattern shards into deterministic bundles of at most 64 test files, reducing the Node matrix without merging non-isolated command/cron, stateful agents-core, or gateway/server suites. Heavy fixed suites stay on 8 vCPU while most bundled and lower-weight lanes use 4 vCPU. Compact-small bins 2, 5, and 8 use existing 8-vCPU capacity because recent hosted runs showed they repeatedly owned the critical path while the 4-vCPU queue was materially longer; routing happens after packing, so group ownership, coverage, and the existing registration count do not change. - Pull requests on the canonical repository reuse the changed-test resolver against the synthetic merged-tree diff. Precise changes run one targeted Node job; each selected test file gets its own process so stateful suite isolation remains intact. The planner combines sibling tests with import-graph dependents and falls back to the existing 23-job compact full-suite plan for workspace package, package/lockfile, shared harness, split-config, renamed, or deleted changes, public extension-contract changes, tests with special shard setup, partially resolved or empty targets, oversized path or target plans, and planner errors. Targeted plans always retain the full built-artifact boundary gate because its repository scanners cannot be derived from imports. `main` pushes run the same full compact suite: pending intermediate push events can be coalesced, so the newest surviving run must validate the complete integration tree rather than only its final single-push diff. Manual dispatches and release gates retain the full named per-shard matrix. Compact packing uses median group walls refreshed from multiple successful hosted runs without changing the bounded job count; the high-variance source/security group remains isolated so its tail does not serialize unrelated groups. - The full Node matrix admits the consistently slow serial tooling, auto-reply command shards, and broad core-fast cache writer first. This keeps the 28-job cap while preventing critical-path work and the next run's transform seed from slipping into a later wave. - The three serial Control UI browser shards greedily pack discovered test files by source byte size. This zero-state duration proxy avoids Vitest's equal-file-count hash clustering, automatically accounts for new and changed files, and preserves the same complete test inventory without adding runners. @@ -190,15 +190,15 @@ for commands and recovery. ## Runners -| Runner | Jobs | -| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `ubuntu-24.04` | `security-fast`, manual CI dispatch and non-canonical repository fallbacks, pull-request retries of both UI E2E jobs, the QA Smoke aggregate, CodeQL security and quality scans, workflow-sanity, labeler, auto-response, the standalone Docs workflow, and the whole Install Smoke workflow | -| `blacksmith-4vcpu-ubuntu-2404` | `preflight`, `pnpm-store-warmup`, `native-i18n`, `checks-fast-core` except QA Smoke CI, plugin/channel contract shards, most bundled/lower-weight Linux Node shards, `check-*` lanes except `check-lint`, selected `check-additional-*` shards, `check-docs`, and `skills-python` | -| `blacksmith-8vcpu-ubuntu-2404` | Retained heavy Linux Node suites, first-attempt same-repo pull requests and pushes for the serial Chromium/Vite `checks-ui-e2e` lane (three Control UI shards plus one browser extension shard), boundary/extension-heavy `check-additional-*` shards, `check-sqlite-session-lifecycle`, and `android` | -| `blacksmith-16vcpu-ubuntu-2404` | Automatic QA Smoke CI shards, first-attempt same-repo pull requests and pushes for `checks-ui-e2e-real-gateway`, `build-artifacts` in CI and Testbox, and `check-lint` (CPU-sensitive enough that 8 vCPU cost more than they saved) | -| `blacksmith-8vcpu-windows-2025` | `checks-windows` | -| `blacksmith-6vcpu-macos-15` | `macos-node` on `openclaw/openclaw`; forks fall back to `macos-15` | -| `blacksmith-12vcpu-macos-26` | `macos-swift` and `ios-build` on `openclaw/openclaw`; forks fall back to `macos-26` | +| Runner | Jobs | +| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ubuntu-24.04` | `security-fast`, manual CI dispatch and non-canonical repository fallbacks, pull-request retries of both UI E2E jobs, the QA Smoke aggregate, CodeQL security and quality scans, workflow-sanity, labeler, auto-response, the standalone Docs workflow, and the whole Install Smoke workflow | +| `blacksmith-4vcpu-ubuntu-2404` | `preflight`, `pnpm-store-warmup`, `native-i18n`, `checks-fast-core` except QA Smoke CI, plugin/channel contract shards, most bundled/lower-weight Linux Node shards, `check-*` lanes except `check-lint`, selected `check-additional-*` shards, `check-docs`, and `skills-python` | +| `blacksmith-8vcpu-ubuntu-2404` | Retained heavy Linux Node suites, compact-small queue-tail bins 2, 5, and 8, first-attempt same-repo pull requests and pushes for the serial Chromium/Vite `checks-ui-e2e` lane (three Control UI shards plus one browser extension shard), boundary/extension-heavy `check-additional-*` shards, `check-sqlite-session-lifecycle`, and `android` | +| `blacksmith-16vcpu-ubuntu-2404` | Automatic QA Smoke CI shards, first-attempt same-repo pull requests and pushes for `checks-ui-e2e-real-gateway`, `build-artifacts` in CI and Testbox, and `check-lint` (CPU-sensitive enough that 8 vCPU cost more than they saved) | +| `blacksmith-8vcpu-windows-2025` | `checks-windows` | +| `blacksmith-6vcpu-macos-15` | `macos-node` on `openclaw/openclaw`; forks fall back to `macos-15` | +| `blacksmith-12vcpu-macos-26` | `macos-swift` and `ios-build` on `openclaw/openclaw`; forks fall back to `macos-26` | ## Runner registration budget diff --git a/scripts/lib/ci-node-test-plan.mts b/scripts/lib/ci-node-test-plan.mts index 0bc023da892c..ddf130e8835b 100644 --- a/scripts/lib/ci-node-test-plan.mts +++ b/scripts/lib/ci-node-test-plan.mts @@ -165,6 +165,13 @@ const COMPACT_NODE_TEST_JOB_SECONDS = 310; const COMPACT_NODE_TEST_JOB_GROUPS = 10; const COMPACT_TOOLING_NODE_TEST_GROUPS = 4; const COMPACT_WHOLE_NODE_TEST_TIMEOUT_MINUTES = 120; +// Route measured queue-tail bins to existing 8-vCPU capacity after packing so +// the planner keeps the same groups, coverage, and runner-registration count. +const COMPACT_8VCPU_CHECK_NAMES = new Set([ + "checks-node-compact-small-2", + "checks-node-compact-small-5", + "checks-node-compact-small-8", +]); const AUTO_REPLY_COMMANDS_STRIPES = 3; const AGENTS_CORE_RUNNER_CLI_STRIPES = 3; const UNIT_FAST_NODE_TEST_STRIPES = 2; @@ -1763,11 +1770,18 @@ function createCompactNodeTestShardBundles( } const runnerClass = firstGroup.runner.includes("-8vcpu-") ? "large" : "small"; const distSuffix = firstGroup.requiresDist ? "-dist" : ""; + const checkName = `checks-node-compact-${runnerClass}${distSuffix}-${index + 1}`; + const runner = COMPACT_8VCPU_CHECK_NAMES.has(checkName) + ? DEFAULT_NODE_TEST_RUNNER + : firstGroup.runner; + for (const group of bin.groups) { + group.runner = runner; + } compactJobs.push({ - checkName: `checks-node-compact-${runnerClass}${distSuffix}-${index + 1}`, + checkName, groups: bin.groups, requiresDist: firstGroup.requiresDist, - runner: firstGroup.runner, + runner, shardName: `compact-${runnerClass}${distSuffix}-${index + 1}`, // Whole-config groups run entire suites; keep their generous timeout. ...(bin.hasWholeConfigGroup diff --git a/test/scripts/ci-node-test-plan.test.ts b/test/scripts/ci-node-test-plan.test.ts index 1870d3c0fa18..4649a5492f5a 100644 --- a/test/scripts/ci-node-test-plan.test.ts +++ b/test/scripts/ci-node-test-plan.test.ts @@ -35,6 +35,7 @@ const PLUGIN_PRERELEASE_NPM_SPEC_TEST = "src/plugins/install.npm-spec.test.ts"; const PLUGIN_NPM_INSTALL_SECURITY_SCAN_TEST = "src/plugins/npm-install-security-scan.release.test.ts"; const DEFAULT_NODE_TEST_RUNNER = "blacksmith-8vcpu-ubuntu-2404"; +const BUNDLED_NODE_TEST_RUNNER = "blacksmith-4vcpu-ubuntu-2404"; function listTestFiles(rootDir: string): string[] { const gitFiles = listGitTrackedFiles({ pathspecs: rootDir }); expect(gitFiles).not.toBeNull(); @@ -323,10 +324,10 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { .find((group) => group.shard_name === "agentic-control-plane-startup-health-runtime")?.env, ).toEqual({ OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "60000" }); const largeJobs = compact.filter( - (shard) => shard.runner === DEFAULT_NODE_TEST_RUNNER && !shard.requiresDist, + (shard) => !shard.requiresDist && shard.checkName.startsWith("checks-node-compact-large-"), ); const smallJobs = compact.filter( - (shard) => shard.runner !== DEFAULT_NODE_TEST_RUNNER && !shard.requiresDist, + (shard) => !shard.requiresDist && shard.checkName.startsWith("checks-node-compact-small-"), ); const distJobs = compact.filter((shard) => shard.requiresDist); expect(largeJobs).toHaveLength(7); @@ -336,6 +337,21 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { shard.groups.every((group) => !exclusiveGroupRe.test(group.shard_name)), ); expect(regularSmallJobs).toHaveLength(10); + const routed8VcpuCheckNames = [ + "checks-node-compact-small-2", + "checks-node-compact-small-5", + "checks-node-compact-small-8", + ]; + expect( + regularSmallJobs + .filter((shard) => shard.runner === DEFAULT_NODE_TEST_RUNNER) + .map((shard) => shard.checkName), + ).toEqual(routed8VcpuCheckNames); + expect( + smallJobs + .filter((shard) => !routed8VcpuCheckNames.includes(shard.checkName)) + .every((shard) => shard.runner === BUNDLED_NODE_TEST_RUNNER), + ).toBe(true); // The refreshed hosted medians give every regular bin one known tail // anchor. Stale hints paired two of these slow groups in each runner class. const largeTailAnchors = [