From 786c646d88949a89c0419eae094c1956d67a1731 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 12 Aug 2026 03:41:34 -0700 Subject: [PATCH] improve(ci): rebalance compact Node shards from hosted medians (#122547) * ci: refresh compact Node shard timings Amp-Thread-ID: https://ampcode.com/threads/T-019ff3db-c467-70ad-8ed3-81f2ba94b0c0 * docs(ci): clarify compact planner counts Amp-Thread-ID: https://ampcode.com/threads/T-019ff3db-c467-70ad-8ed3-81f2ba94b0c0 --------- Co-authored-by: Amp --- docs/ci.md | 2 +- scripts/lib/ci-node-test-plan.mts | 216 ++++++++++++------------- test/scripts/ci-node-test-plan.test.ts | 39 +++++ 3 files changed, 142 insertions(+), 115 deletions(-) diff --git a/docs/ci.md b/docs/ci.md index 67cd1137dd5e..e1bf61d6bf4d 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -119,7 +119,7 @@ The slowest Node test families are split or balanced so each job stays small wit - Auto-reply runs as balanced workers, with the reply subtree split into agent-runner, commands, dispatch, session, and state-routing shards. - Agentic gateway/server (control-plane) configs split across chat, auth, model, HTTP/plugin, runtime, and startup lanes instead of waiting on built artifacts. - Normal CI packs only isolated infra include-pattern shards into deterministic bundles of at most 64 test files, reducing the Node matrix without merging non-isolated command/cron, stateful agents-core, or gateway/server suites. Heavy fixed suites stay on 8 vCPU while the bundled and lower-weight lanes use 4 vCPU. -- Pull requests on the canonical repository reuse the changed-test resolver against the synthetic merged-tree diff. Precise changes run one targeted Node job; each selected test file gets its own process so stateful suite isolation remains intact. The planner combines sibling tests with import-graph dependents and falls back to the existing 14-job compact full-suite plan for workspace package, package/lockfile, shared harness, split-config, renamed, or deleted changes, public extension-contract changes, tests with special shard setup, partially resolved or empty targets, oversized path or target plans, and planner errors. Targeted plans always retain the full built-artifact boundary gate because its repository scanners cannot be derived from imports. `main` pushes run the same full compact suite: pending intermediate push events can be coalesced, so the newest surviving run must validate the complete integration tree rather than only its final single-push diff. Manual dispatches and release gates retain the full named per-shard matrix. Compact packing uses fleet timing hints without changing the bounded job count; the high-variance source/security group remains isolated so its tail does not serialize unrelated groups. +- Pull requests on the canonical repository reuse the changed-test resolver against the synthetic merged-tree diff. Precise changes run one targeted Node job; each selected test file gets its own process so stateful suite isolation remains intact. The planner combines sibling tests with import-graph dependents and falls back to the existing 23-job compact full-suite plan for workspace package, package/lockfile, shared harness, split-config, renamed, or deleted changes, public extension-contract changes, tests with special shard setup, partially resolved or empty targets, oversized path or target plans, and planner errors. Targeted plans always retain the full built-artifact boundary gate because its repository scanners cannot be derived from imports. `main` pushes run the same full compact suite: pending intermediate push events can be coalesced, so the newest surviving run must validate the complete integration tree rather than only its final single-push diff. Manual dispatches and release gates retain the full named per-shard matrix. Compact packing uses median group walls refreshed from multiple successful hosted runs without changing the bounded job count; the high-variance source/security group remains isolated so its tail does not serialize unrelated groups. - The full Node matrix admits the consistently slow serial tooling, auto-reply command shards, and broad core-fast cache writer first. This keeps the 28-job cap while preventing critical-path work and the next run's transform seed from slipping into a later wave. - The three serial Control UI browser shards greedily pack discovered test files by source byte size. This zero-state duration proxy avoids Vitest's equal-file-count hash clustering, automatically accounts for new and changed files, and preserves the same complete test inventory without adding runners. - Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. Include-pattern shards record timing entries using the CI shard name, so `.artifacts/vitest-shard-timings.json` can distinguish a whole config from a filtered shard. diff --git a/scripts/lib/ci-node-test-plan.mts b/scripts/lib/ci-node-test-plan.mts index f8b085ce9000..7a1c71f912d8 100644 --- a/scripts/lib/ci-node-test-plan.mts +++ b/scripts/lib/ci-node-test-plan.mts @@ -168,139 +168,127 @@ const COMPACT_WHOLE_NODE_TEST_TIMEOUT_MINUTES = 120; const AUTO_REPLY_COMMANDS_STRIPES = 3; const AGENTS_CORE_RUNNER_CLI_STRIPES = 3; const UNIT_FAST_NODE_TEST_STRIPES = 2; -// Advisory runtime estimates (seconds) per split shard: [shard:*] begin->end -// wall clock across seven green Blacksmith compact PR runs after the -// cli-runner reliability whale fix (29605136624, 29605203485, 29605983019, -// 29606701461, 29611308972, 29611457693, 29611500865), averaged after -// dropping cache-warm/contention outliers outside [median/1.5, median*1.5]. +// Advisory runtime estimates (seconds) per split shard: median [shard:*] +// begin->end wall across nine successful hosted compact runs (31568650453, +// 31569157374, 31569912984, 31570693513, 31571644856, 31572044913, +// 31572489294, 31574210928, 31574367637). // Packing only: a stale entry skews job balance but never correctness. // Unknown shards fall back to a per-file estimate. -// Outlier hints were refreshed from child-process walls in runs 31453973052 -// and 31455822921. const COMPACT_GROUP_SECONDS_HINTS = new Map([ - ["agentic-agents-core-auth", 27], - ["agentic-agents-core-isolated", 9], - // Model catalog and full UI both cold-load broad graphs; preserve their - // measured separation when striping the expanded groups. - ["agentic-agents-core-models", 37], - // Reliability's runtime-free provider check dropped its wall time from - // ~245s to ~5s; the narrow anthropic cli-api artifact removes the same - // full-barrel evaluation for the remaining facade importers (spawn). - // The live-session extraction rebalanced these stripes without changing the - // fleet-scale import wall that dominates each compact group. - ["agentic-agents-core-runner-cli-1", 8], - ["agentic-agents-core-runner-cli-2", 8], - ["agentic-agents-core-runner-cli-3", 8], + ["agentic-agents-core-auth", 28], + ["agentic-agents-core-isolated", 16], + ["agentic-agents-core-models", 39], + ["agentic-agents-core-runner-cli-1", 7], + ["agentic-agents-core-runner-cli-2", 17], + ["agentic-agents-core-runner-cli-3", 13], ["agentic-agents-core-runner-commands", 27], ["agentic-agents-core-runner-embedded", 20], - ["agentic-agents-core-runner-sessions", 13], - ["agentic-agents-core-runtime", 104], - ["agentic-agents-core-subagents", 10], - ["agentic-agents-core-tools", 52], + ["agentic-agents-core-runner-sessions", 18], + ["agentic-agents-core-runtime", 113], + ["agentic-agents-core-subagents", 17], + ["agentic-agents-core-tools", 45], // The composite hint sets the job count before its independent configs are - // striped across those jobs. Split hints use the same loaded-fleet run as - // the rest of this map rather than older 2-core measurements. - ["agentic-agents-embedded", 150], - ["agentic-agents-embedded-base", 88], - ["agentic-agents-embedded-incomplete-turn", 14], - ["agentic-agents-embedded-overflow-compaction", 12], - ["agentic-agents-embedded-run", 30], - ["agentic-agents-support", 201], - ["agentic-agents-tools", 42], - ["agentic-cli", 145], - ["agentic-command-support", 65], - ["agentic-commands-agent-channel", 74], - ["agentic-commands-doctor", 19], - ["agentic-commands-doctor-auth", 11], - ["agentic-commands-doctor-config-state", 112], - ["agentic-commands-doctor-device", 2], - ["agentic-commands-doctor-gateway", 4], - ["agentic-commands-doctor-platform", 3], - ["agentic-commands-doctor-plugins-tools", 11], - ["agentic-commands-doctor-sessions-cron", 24], - ["agentic-commands-doctor-shared", 16], + // striped across those jobs; its estimate is the sum of the split medians. + ["agentic-agents-embedded", 162], + ["agentic-agents-embedded-base", 90], + ["agentic-agents-embedded-incomplete-turn", 17], + ["agentic-agents-embedded-overflow-compaction", 18], + ["agentic-agents-embedded-run", 37], + ["agentic-agents-support", 144], + ["agentic-agents-tools", 76], + ["agentic-cli", 111], + ["agentic-command-support", 61], + ["agentic-commands-agent-channel", 71], + ["agentic-commands-doctor", 23], + ["agentic-commands-doctor-auth", 19], + ["agentic-commands-doctor-config-state", 69], + ["agentic-commands-doctor-device", 3], + ["agentic-commands-doctor-gateway", 3], + ["agentic-commands-doctor-platform", 4], + ["agentic-commands-doctor-plugins-tools", 27], + ["agentic-commands-doctor-sessions-cron", 21], + ["agentic-commands-doctor-shared", 27], ["agentic-commands-doctor-whatsapp", 1], ["agentic-commands-doctor-workspace", 1], - ["agentic-commands-models", 16], - ["agentic-commands-onboard-config", 11], - ["agentic-commands-status-tools", 21], - ["agentic-control-plane-agent-chat", 123], - ["agentic-control-plane-auth-node", 128], - ["agentic-control-plane-http-models", 33], - ["agentic-control-plane-http-plugin-ws", 39], - ["agentic-control-plane-runtime-config", 14], - ["agentic-control-plane-runtime-cron", 15], + ["agentic-commands-models", 24], + ["agentic-commands-onboard-config", 26], + ["agentic-commands-status-tools", 28], + ["agentic-control-plane-agent-chat", 140], + ["agentic-control-plane-auth-node", 153], + ["agentic-control-plane-http-models", 25], + ["agentic-control-plane-http-plugin-ws", 49], + ["agentic-control-plane-runtime", 20], + ["agentic-control-plane-runtime-config", 8], + ["agentic-control-plane-runtime-cron", 31], ["agentic-control-plane-runtime-network", 1], - ["agentic-control-plane-runtime-server", 29], - ["agentic-control-plane-runtime-shared-token", 22], - ["agentic-control-plane-runtime-state", 13], - ["agentic-control-plane-runtime-ui-tools", 11], - ["agentic-control-plane-startup-core", 28], - ["agentic-control-plane-startup-health-runtime", 22], - ["agentic-control-plane-startup-restart-close", 8], - ["agentic-gateway-core", 197], - ["agentic-gateway-methods", 136], - ["agentic-plugin-sdk", 47], - ["auto-reply-core-top-level", 30], - ["auto-reply-reply-agent-runner", 40], - ["auto-reply-reply-commands-1", 44], - ["auto-reply-reply-commands-2", 18], - ["auto-reply-reply-commands-3", 36], - ["auto-reply-reply-dispatch", 64], - ["auto-reply-reply-session", 19], - ["auto-reply-reply-state-routing", 54], - ["core-runtime-cron-core", 16], - ["core-runtime-cron-isolated-agent", 94], - ["core-runtime-cron-service", 49], - ["core-runtime-hooks", 9], - ["core-runtime-infra-approval-exec", 30], - ["core-runtime-infra-channel-plugin", 17], - ["core-runtime-infra-cli-ui", 1], - ["core-runtime-infra-core-utils", 3], - ["core-runtime-infra-diagnostics-state", 19], - ["core-runtime-infra-events-runtime", 4], + ["agentic-control-plane-runtime-server", 25], + ["agentic-control-plane-runtime-shared-token", 8], + ["agentic-control-plane-runtime-state", 34], + ["agentic-control-plane-runtime-ui-tools", 9], + ["agentic-control-plane-startup-config", 5], + ["agentic-control-plane-startup-core", 27], + ["agentic-control-plane-startup-health-runtime", 11], + ["agentic-control-plane-startup-restart-close", 16], + ["agentic-gateway-core", 214], + ["agentic-gateway-methods", 119], + ["agentic-plugin-sdk", 44], + ["auto-reply-core-top-level", 27], + ["auto-reply-reply-agent-runner", 68], + ["auto-reply-reply-commands-1", 27], + ["auto-reply-reply-commands-2", 16], + ["auto-reply-reply-commands-3", 27], + ["auto-reply-reply-dispatch", 65], + ["auto-reply-reply-session", 40], + ["auto-reply-reply-state-routing", 48], + ["core-runtime-cron-core", 24], + ["core-runtime-cron-isolated-agent", 110], + ["core-runtime-cron-service", 51], + ["core-runtime-hooks", 18], + ["core-runtime-infra-approval-exec", 23], + ["core-runtime-infra-channel-plugin", 7], + ["core-runtime-infra-cli-ui", 2], + ["core-runtime-infra-core-utils", 4], + ["core-runtime-infra-device", 8], + ["core-runtime-infra-diagnostics-state", 12], + ["core-runtime-infra-env-auth", 5], + ["core-runtime-infra-events-runtime", 7], ["core-runtime-infra-file-safety", 2], - ["core-runtime-infra-files-commands", 5], + ["core-runtime-infra-files-commands", 4], ["core-runtime-infra-gateway-lock-argv", 2], ["core-runtime-infra-gateway-processes", 1], ["core-runtime-infra-gateway-watch", 1], - ["core-runtime-infra-heartbeat-core", 4], - ["core-runtime-infra-heartbeat-runner", 123], - ["core-runtime-infra-misc", 9], + ["core-runtime-infra-heartbeat-core", 6], + ["core-runtime-infra-heartbeat-runner", 54], + ["core-runtime-infra-misc", 12], ["core-runtime-infra-misc-dedupe-disk", 1], ["core-runtime-infra-misc-os", 1], ["core-runtime-infra-misc-values", 1], - ["core-runtime-infra-net-install", 13], - ["core-runtime-infra-network-node", 2], + ["core-runtime-infra-net-install", 9], + ["core-runtime-infra-network-node", 4], ["core-runtime-infra-network-platform", 4], - ["core-runtime-infra-outbound-actions", 19], - ["core-runtime-infra-outbound-core", 45], - ["core-runtime-infra-process", 118], - ["core-runtime-infra-provider-push", 17], + ["core-runtime-infra-outbound-actions", 31], + ["core-runtime-infra-outbound-core", 57], + ["core-runtime-infra-process", 134], + ["core-runtime-infra-provider-push", 15], ["core-runtime-infra-repo-tooling", 4], - ["core-runtime-infra-storage-state", 96], - ["core-runtime-infra-system-runtime", 40], - ["core-runtime-media-ui", 174], - ["core-runtime-secrets", 37], - ["core-runtime-shared", 48], - // PTY timing suites still need a lightly packed lane; the exclusive-bin cap - // leaves only trivial co-groups next to this measured runtime. + ["core-runtime-infra-storage-state", 86], + ["core-runtime-infra-system-runtime", 35], + ["core-runtime-media-ui", 196], + ["core-runtime-secrets", 58], + ["core-runtime-shared", 52], + // This dist-only group is outside the sampled nondist logs and retains its + // prior measured hint. The exclusive-bin cap keeps its lane lightly packed. ["core-runtime-tui-pty", 116], - ["core-tooling-1", 94], - ["core-tooling-2", 95], - ["core-tooling-3", 108], - ["core-tooling-4", 125], - ["core-tooling-isolated", 49], - ["core-unit-fast-1", 89], - ["core-unit-fast-2", 92], - // Fork-per-file isolation parallelizes poorly on 4 vCPU; keep it on the - // 8 vCPU class, where it still runs a measured ~90s under fleet load. - ["core-unit-fast-isolated", 90], - // In 35 green main runs on 2026-08-11/12, compact large jobs owned the - // critical tail 15 times and reached p90=457s. This group's former 205s - // hint repeatedly packed another 58s of serial work beside that tail. - ["core-unit-src-security", 295], - ["core-unit-support", 17], + ["core-tooling-1", 112], + ["core-tooling-2", 128], + ["core-tooling-3", 163], + ["core-tooling-4", 123], + ["core-tooling-isolated", 34], + ["core-unit-fast-1", 54], + ["core-unit-fast-2", 60], + ["core-unit-fast-isolated", 79], + ["core-unit-src-security", 252], + ["core-unit-support", 18], ]); // Advisory per-file wall-clock hints (seconds) for stripe balancing, measured // from single-file local runs (M4 Max) and static import-graph size. Packing diff --git a/test/scripts/ci-node-test-plan.test.ts b/test/scripts/ci-node-test-plan.test.ts index ef681c84c5aa..480e2ec4de10 100644 --- a/test/scripts/ci-node-test-plan.test.ts +++ b/test/scripts/ci-node-test-plan.test.ts @@ -332,6 +332,45 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { expect(largeJobs).toHaveLength(7); expect(smallJobs).toHaveLength(14); expect(distJobs).toHaveLength(2); + const regularSmallJobs = smallJobs.filter((shard) => + shard.groups.every((group) => !exclusiveGroupRe.test(group.shard_name)), + ); + expect(regularSmallJobs).toHaveLength(10); + // The refreshed hosted medians give every regular bin one known tail + // anchor. Stale hints paired two of these slow groups in each runner class. + const largeTailAnchors = [ + "core-unit-src-security", + "agentic-gateway-core", + "core-runtime-media-ui", + "agentic-agents-support", + "agentic-gateway-methods", + "agentic-agents-core-runtime", + "agentic-agents-embedded-base", + ]; + const smallTailAnchors = [ + "agentic-control-plane-auth-node", + "agentic-control-plane-agent-chat", + "core-runtime-infra-process", + "agentic-cli", + "core-runtime-cron-isolated-agent", + "core-runtime-infra-storage-state", + "agentic-agents-tools", + "agentic-commands-agent-channel", + "agentic-commands-doctor-config-state", + "auto-reply-reply-agent-runner", + ]; + expect( + largeJobs.map( + (shard) => + shard.groups.filter((group) => largeTailAnchors.includes(group.shard_name)).length, + ), + ).toEqual(Array.from({ length: largeTailAnchors.length }, () => 1)); + expect( + regularSmallJobs.map( + (shard) => + shard.groups.filter((group) => smallTailAnchors.includes(group.shard_name)).length, + ), + ).toEqual(Array.from({ length: smallTailAnchors.length }, () => 1)); expect(compact).toEqual( createNodeTestShardBundles({ includeReleaseOnlyPluginShards: false,