diff --git a/docs/ci.md b/docs/ci.md index 82956ab3fa8d..ab65affaa561 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -119,7 +119,7 @@ The slowest Node test families are split or balanced so each job stays small wit - Auto-reply runs as balanced workers, with the reply subtree split into agent-runner, commands, dispatch, session, and state-routing shards. - Agentic gateway/server (control-plane) configs split across chat, auth, model, HTTP/plugin, runtime, and startup lanes instead of waiting on built artifacts. - Normal CI packs only isolated infra include-pattern shards into deterministic bundles of at most 64 test files, reducing the Node matrix without merging non-isolated command/cron, stateful agents-core, or gateway/server suites. Heavy fixed suites stay on 8 vCPU while most bundled and lower-weight lanes use 4 vCPU. Compact-small bins 2, 5, and 8 use existing 8-vCPU capacity because recent hosted runs showed they repeatedly owned the critical path while the 4-vCPU queue was materially longer; routing happens after packing, so group ownership, coverage, and the existing registration count do not change. -- Pull requests on the canonical repository reuse the changed-test resolver against the synthetic merged-tree diff. Precise changes run one targeted Node job; each selected test file gets its own process so stateful suite isolation remains intact. The planner combines sibling tests with import-graph dependents and falls back to the existing 23-job compact full-suite plan for workspace package, package/lockfile, shared harness, split-config, renamed, or deleted changes, public extension-contract changes, tests with special shard setup, partially resolved or empty targets, oversized path or target plans, and planner errors. Targeted plans always retain the full built-artifact boundary gate because its repository scanners cannot be derived from imports. `main` pushes run the same full compact suite: pending intermediate push events can be coalesced, so the newest surviving run must validate the complete integration tree rather than only its final single-push diff. Manual dispatches and release gates retain the full named per-shard matrix. Compact packing uses median group walls refreshed from multiple successful hosted runs without changing the bounded job count; the high-variance source/security group remains isolated so its tail does not serialize unrelated groups. +- Pull requests on the canonical repository reuse the changed-test resolver against the synthetic merged-tree diff. Precise changes run one targeted Node job; each selected test file gets its own process so stateful suite isolation remains intact. The planner combines sibling tests with import-graph dependents and falls back to the existing 23-job compact full-suite plan for workspace package, package/lockfile, shared harness, split-config, renamed, or deleted changes, public extension-contract changes, tests with special shard setup, partially resolved or empty targets, oversized path or target plans, and planner errors. Targeted plans always retain the full built-artifact boundary gate because its repository scanners cannot be derived from imports. `main` pushes run the same full compact suite: pending intermediate push events can be coalesced, so the newest surviving run must validate the complete integration tree rather than only its final single-push diff. Manual dispatches and release gates retain the full named per-shard matrix. Compact packing uses hosted means to tail-balance regular 8-vCPU bins while retaining median admission and 4-vCPU striping weights, so recurrent slow tails rebalance without changing the bounded job count or post-pack runner advisory; the high-variance source/security group remains isolated so its tail does not serialize unrelated groups. - The full Node matrix admits the consistently slow serial tooling, auto-reply command shards, and broad core-fast cache writer first. This keeps the 28-job cap while preventing critical-path work and the next run's transform seed from slipping into a later wave. - The three serial Control UI browser shards greedily pack discovered test files by source byte size. This zero-state duration proxy avoids Vitest's equal-file-count hash clustering, automatically accounts for new and changed files, and preserves the same complete test inventory without adding runners. - Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. Include-pattern shards record timing entries using the CI shard name, so `.artifacts/vitest-shard-timings.json` can distinguish a whole config from a filtered shard. diff --git a/scripts/lib/ci-node-test-plan.mts b/scripts/lib/ci-node-test-plan.mts index ddf130e8835b..ff03b54f01cd 100644 --- a/scripts/lib/ci-node-test-plan.mts +++ b/scripts/lib/ci-node-test-plan.mts @@ -178,8 +178,8 @@ const UNIT_FAST_NODE_TEST_STRIPES = 2; // Advisory runtime estimates (seconds) per split shard: median [shard:*] // begin->end wall across nine successful hosted compact runs (31568650453, // 31569157374, 31569912984, 31570693513, 31571644856, 31572044913, -// 31572489294, 31574210928, 31574367637). -// Packing only: a stale entry skews job balance but never correctness. +// 31572489294, 31574210928, 31574367637). Admission and 4-vCPU striping +// retain these weights so the bounded job count and runner advisory stay fixed. // Unknown shards fall back to a per-file estimate. const COMPACT_GROUP_SECONDS_HINTS = new Map([ ["agentic-agents-core-auth", 28], @@ -297,6 +297,43 @@ const COMPACT_GROUP_SECONDS_HINTS = new Map([ ["core-unit-src-security", 252], ["core-unit-support", 18], ]); + +// Rounded mean of the same 8-vCPU groups across successful canonical-main +// compact runs 31624370014, 31625101669, 31625905392, 31629769941, +// 31632097578, 31632768372, 31634233096, 31635221353, and 31636058167. +// Means expose recurrent slow tails hidden by medians without moving the +// post-pack 4-vCPU runner advisory. +const COMPACT_LARGE_GROUP_STRIPE_SECONDS_HINTS = new Map([ + ["agentic-agents-core-auth", 35], + ["agentic-agents-core-models", 47], + ["agentic-agents-core-runner-cli-1", 21], + ["agentic-agents-core-runner-cli-2", 9], + ["agentic-agents-core-runner-cli-3", 21], + ["agentic-agents-core-runner-commands", 33], + ["agentic-agents-core-runner-embedded", 11], + ["agentic-agents-core-runner-sessions", 12], + ["agentic-agents-core-runtime", 128], + ["agentic-agents-core-subagents", 31], + ["agentic-agents-core-tools", 61], + ["agentic-agents-embedded-base", 106], + ["agentic-agents-embedded-incomplete-turn", 24], + ["agentic-agents-embedded-overflow-compaction", 24], + ["agentic-agents-embedded-run", 46], + ["agentic-agents-support", 175], + ["agentic-control-plane-startup-core", 39], + ["agentic-gateway-core", 244], + ["agentic-gateway-methods", 154], + ["auto-reply-reply-commands-1", 40], + ["auto-reply-reply-commands-2", 20], + ["auto-reply-reply-commands-3", 32], + ["auto-reply-reply-dispatch", 82], + ["core-runtime-media-ui", 249], + ["core-unit-fast-1", 72], + ["core-unit-fast-2", 64], + ["core-unit-fast-isolated", 107], + ["core-unit-src-security", 266], +]); + // Advisory per-file wall-clock hints (seconds) for stripe balancing, measured // from single-file local runs (M4 Max) and static import-graph size. Packing // only: a stale entry skews stripe balance but never correctness. Unlisted @@ -374,6 +411,13 @@ function estimateCompactGroupSeconds(group: NodeTestShardGroup): number { return DEFAULT_WHOLE_GROUP_SECONDS; } +function estimateCompactStripeSeconds(group: NodeTestShardGroup): number { + return ( + COMPACT_LARGE_GROUP_STRIPE_SECONDS_HINTS.get(group.shard_name) ?? + estimateCompactGroupSeconds(group) + ); +} + function expandCompactGroup(group: NodeTestShardGroup): NodeTestShardGroup[] { if (group.shard_name !== "agentic-agents-embedded") { return [group]; @@ -1737,20 +1781,29 @@ function createCompactNodeTestShardBundles( } } - // First-fit above determines the bounded worker count. Stripe every regular - // group across those workers afterward; only re-striping the expanded - // embedded group left the 4-vCPU matrix with full early bins and nearly - // empty tail bins despite accurate timing hints. + // First-fit above determines the bounded worker count. Keep the + // high-variance source/security group isolated, then stripe every other + // regular group across the remaining workers. Re-striping the expanded + // embedded group avoids full early bins and nearly empty tail bins. const expandedGroups = groups.flatMap(expandCompactGroup); const regularGroups = expandedGroups .filter((group) => !isExclusiveCompactGroup(group)) .toSorted((a, b) => a.shard_name.localeCompare(b.shard_name)); const regularBinCount = bins.filter((bin) => !bin.exclusive).length; - const regularBatches = createStripedBatches( - regularGroups, - regularBinCount, - estimateCompactGroupSeconds, + const isolatedGroups = regularGroups.filter( + (group) => group.shard_name === "core-unit-src-security", ); + const stripedGroups = regularGroups.filter( + (group) => group.shard_name !== "core-unit-src-security", + ); + const regularBatches = [ + ...isolatedGroups.map((group) => [group]), + ...createStripedBatches( + stripedGroups, + regularBinCount - isolatedGroups.length, + estimateCompactStripeSeconds, + ), + ]; if (regularBatches.some((batch) => batch.length > COMPACT_NODE_TEST_JOB_GROUPS)) { throw new Error("striped compact job exceeds its group capacity"); } @@ -1758,7 +1811,7 @@ function createCompactNodeTestShardBundles( exclusive: false, groups: batch, hasWholeConfigGroup: batch.some((group) => !group.includePatterns), - weight: batch.reduce((sum, group) => sum + estimateCompactGroupSeconds(group), 0), + weight: batch.reduce((sum, group) => sum + estimateCompactStripeSeconds(group), 0), })); const exclusiveBins = bins.filter((bin) => bin.exclusive); bins.splice(0, bins.length, ...regularBins, ...exclusiveBins); diff --git a/test/scripts/ci-node-test-plan.test.ts b/test/scripts/ci-node-test-plan.test.ts index 4649a5492f5a..19701b67d782 100644 --- a/test/scripts/ci-node-test-plan.test.ts +++ b/test/scripts/ci-node-test-plan.test.ts @@ -241,6 +241,9 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { // pairing them starves model visibility and repeatedly hits its timeout. expect(jobOf("agentic-agents-core-models")).not.toBe(jobOf("core-runtime-media-ui")); expect(jobOf("core-runtime-media-ui")).not.toBe(jobOf("core-unit-src-security")); + // Means expose recurrent 8-vCPU tails hidden by the median-only plan. Keep + // the observed pairing that dominated replayed job walls separated. + expect(jobOf("agentic-agents-core-tools")).not.toBe(jobOf("agentic-agents-embedded-base")); expect( compact[jobOf("core-unit-src-security")]?.groups.map((group) => group.shard_name), ).toEqual(["core-unit-src-security"]); @@ -352,8 +355,8 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { .filter((shard) => !routed8VcpuCheckNames.includes(shard.checkName)) .every((shard) => shard.runner === BUNDLED_NODE_TEST_RUNNER), ).toBe(true); - // The refreshed hosted medians give every regular bin one known tail - // anchor. Stale hints paired two of these slow groups in each runner class. + // The refreshed hosted estimates give every regular bin one known tail + // anchor. Stale hints paired two slow groups in each runner class. const largeTailAnchors = [ "core-unit-src-security", "agentic-gateway-core",