diff --git a/scripts/lib/ci-node-test-plan.mts b/scripts/lib/ci-node-test-plan.mts index 8a08e1693f52..5bf7e52b5cb0 100644 --- a/scripts/lib/ci-node-test-plan.mts +++ b/scripts/lib/ci-node-test-plan.mts @@ -98,10 +98,10 @@ const COMPACT_EMBEDDED_GROUP_NAMES = [ const MAX_BUNDLED_NODE_TEST_PATTERNS = 64; // PR-only bundles trade a little serial work for fewer ephemeral runner registrations. // Keep runner classes and subprocess isolation intact while bounding each combined job. -// The group hints below are loaded-fleet CI walls. The 220s cap keeps the -// established 24-worker compact matrix after refreshing underestimated groups; -// expanded composite groups are then striped evenly across those jobs. -const COMPACT_NODE_TEST_JOB_SECONDS = 220; +// The group hints below are loaded-fleet CI walls. The 310s admission cap +// reduces the compact matrix from 24 to 23 workers; expanded composite groups +// are then striped evenly across those jobs. +const COMPACT_NODE_TEST_JOB_SECONDS = 310; const COMPACT_NODE_TEST_JOB_GROUPS = 10; const COMPACT_TOOLING_NODE_TEST_GROUPS = 4; const COMPACT_WHOLE_NODE_TEST_TIMEOUT_MINUTES = 120; @@ -115,13 +115,14 @@ const UNIT_FAST_NODE_TEST_STRIPES = 2; // dropping cache-warm/contention outliers outside [median/1.5, median*1.5]. // Packing only: a stale entry skews job balance but never correctness. // Unknown shards fall back to a per-file estimate. -// Four outlier hints were refreshed from child-process walls in run 31450296338. +// Outlier hints were refreshed from child-process walls in runs 31453973052 +// and 31455822921. const COMPACT_GROUP_SECONDS_HINTS = new Map([ ["agentic-agents-core-auth", 27], ["agentic-agents-core-isolated", 9], - // Model catalog and full UI both cold-load broad graphs. Keep their combined - // hint above the compact-bin cap so model visibility cannot starve for 120s. - ["agentic-agents-core-models", 70], + // Model catalog and full UI both cold-load broad graphs; preserve their + // measured separation when striping the expanded groups. + ["agentic-agents-core-models", 37], // Reliability's runtime-free provider check dropped its wall time from // ~245s to ~5s; the narrow anthropic cli-api artifact removes the same // full-barrel evaluation for the remaining facade importers (spawn). @@ -133,74 +134,93 @@ const COMPACT_GROUP_SECONDS_HINTS = new Map([ ["agentic-agents-core-runner-commands", 27], ["agentic-agents-core-runner-embedded", 20], ["agentic-agents-core-runner-sessions", 13], - ["agentic-agents-core-runtime", 130], - ["agentic-agents-core-subagents", 32], + ["agentic-agents-core-runtime", 104], + ["agentic-agents-core-subagents", 10], ["agentic-agents-core-tools", 52], - // The composite hint sets the existing job count before its independent - // configs are striped across those jobs. Split hints are medians from four - // recent 2-core runs - // (30532132189, 30532967046, 30534273298, 30536274496). - ["agentic-agents-embedded", 430], - ["agentic-agents-embedded-base", 363], - ["agentic-agents-embedded-incomplete-turn", 146], - ["agentic-agents-embedded-overflow-compaction", 150], + // The composite hint sets the job count before its independent configs are + // striped across those jobs. Split hints use the same loaded-fleet run as + // the rest of this map rather than older 2-core measurements. + ["agentic-agents-embedded", 150], + ["agentic-agents-embedded-base", 88], + ["agentic-agents-embedded-incomplete-turn", 14], + ["agentic-agents-embedded-overflow-compaction", 12], ["agentic-agents-embedded-run", 30], - ["agentic-agents-support", 110], + ["agentic-agents-support", 201], ["agentic-agents-tools", 42], - ["agentic-cli", 72], - ["agentic-command-support", 41], - ["agentic-commands-agent-channel", 51], + ["agentic-cli", 145], + ["agentic-command-support", 65], + ["agentic-commands-agent-channel", 74], ["agentic-commands-doctor", 19], ["agentic-commands-doctor-auth", 11], - ["agentic-commands-doctor-config-state", 42], + ["agentic-commands-doctor-config-state", 112], + ["agentic-commands-doctor-device", 2], ["agentic-commands-doctor-gateway", 4], + ["agentic-commands-doctor-platform", 3], ["agentic-commands-doctor-plugins-tools", 11], ["agentic-commands-doctor-sessions-cron", 24], ["agentic-commands-doctor-shared", 16], + ["agentic-commands-doctor-whatsapp", 1], + ["agentic-commands-doctor-workspace", 1], ["agentic-commands-models", 16], ["agentic-commands-onboard-config", 11], ["agentic-commands-status-tools", 21], - ["agentic-control-plane-agent-chat", 74], - ["agentic-control-plane-auth-node", 89], + ["agentic-control-plane-agent-chat", 123], + ["agentic-control-plane-auth-node", 128], ["agentic-control-plane-http-models", 33], ["agentic-control-plane-http-plugin-ws", 39], ["agentic-control-plane-runtime-config", 14], ["agentic-control-plane-runtime-cron", 15], + ["agentic-control-plane-runtime-network", 1], ["agentic-control-plane-runtime-server", 29], ["agentic-control-plane-runtime-shared-token", 22], ["agentic-control-plane-runtime-state", 13], ["agentic-control-plane-runtime-ui-tools", 11], - ["agentic-control-plane-startup-core", 156], + ["agentic-control-plane-startup-core", 28], ["agentic-control-plane-startup-health-runtime", 22], ["agentic-control-plane-startup-restart-close", 8], - ["agentic-gateway-core", 124], - ["agentic-gateway-methods", 69], + ["agentic-gateway-core", 197], + ["agentic-gateway-methods", 136], ["agentic-plugin-sdk", 47], ["auto-reply-core-top-level", 30], ["auto-reply-reply-agent-runner", 40], - ["auto-reply-reply-commands-1", 24], + ["auto-reply-reply-commands-1", 44], ["auto-reply-reply-commands-2", 18], - ["auto-reply-reply-commands-3", 12], - ["auto-reply-reply-dispatch", 40], + ["auto-reply-reply-commands-3", 36], + ["auto-reply-reply-dispatch", 64], ["auto-reply-reply-session", 19], - ["auto-reply-reply-state-routing", 18], + ["auto-reply-reply-state-routing", 54], ["core-runtime-cron-core", 16], - ["core-runtime-cron-isolated-agent", 59], - ["core-runtime-cron-service", 23], + ["core-runtime-cron-isolated-agent", 94], + ["core-runtime-cron-service", 49], ["core-runtime-hooks", 9], ["core-runtime-infra-approval-exec", 30], ["core-runtime-infra-channel-plugin", 17], + ["core-runtime-infra-cli-ui", 1], + ["core-runtime-infra-core-utils", 3], ["core-runtime-infra-diagnostics-state", 19], - ["core-runtime-infra-heartbeat-runner", 53], + ["core-runtime-infra-events-runtime", 4], + ["core-runtime-infra-file-safety", 2], + ["core-runtime-infra-files-commands", 5], + ["core-runtime-infra-gateway-lock-argv", 2], + ["core-runtime-infra-gateway-processes", 1], + ["core-runtime-infra-gateway-watch", 1], + ["core-runtime-infra-heartbeat-core", 4], + ["core-runtime-infra-heartbeat-runner", 123], ["core-runtime-infra-misc", 9], + ["core-runtime-infra-misc-dedupe-disk", 1], + ["core-runtime-infra-misc-os", 1], + ["core-runtime-infra-misc-values", 1], ["core-runtime-infra-net-install", 13], + ["core-runtime-infra-network-node", 2], + ["core-runtime-infra-network-platform", 4], ["core-runtime-infra-outbound-actions", 19], ["core-runtime-infra-outbound-core", 45], - ["core-runtime-infra-process", 91], + ["core-runtime-infra-process", 118], ["core-runtime-infra-provider-push", 17], - ["core-runtime-infra-storage-state", 70], + ["core-runtime-infra-repo-tooling", 4], + ["core-runtime-infra-storage-state", 96], ["core-runtime-infra-system-runtime", 40], - ["core-runtime-media-ui", 124], + ["core-runtime-media-ui", 174], ["core-runtime-secrets", 37], ["core-runtime-shared", 48], // PTY timing suites still need a lightly packed lane; the exclusive-bin cap @@ -211,12 +231,12 @@ const COMPACT_GROUP_SECONDS_HINTS = new Map([ ["core-tooling-3", 108], ["core-tooling-4", 125], ["core-tooling-isolated", 49], - ["core-unit-fast-1", 41], + ["core-unit-fast-1", 89], ["core-unit-fast-2", 92], // Fork-per-file isolation parallelizes poorly on 4 vCPU; keep it on the // 8 vCPU class, where it still runs a measured ~90s under fleet load. ["core-unit-fast-isolated", 90], - ["core-unit-src-security", 95], + ["core-unit-src-security", 205], ["core-unit-support", 17], ]); // Advisory per-file wall-clock hints (seconds) for stripe balancing, measured @@ -1620,29 +1640,31 @@ function createCompactNodeTestShardBundles( } } + // First-fit above determines the bounded worker count. Stripe every regular + // group across those workers afterward; only re-striping the expanded + // embedded group left the 4-vCPU matrix with full early bins and nearly + // empty tail bins despite accurate timing hints. const expandedGroups = groups.flatMap(expandCompactGroup); - if (expandedGroups.length !== groups.length) { - const regularGroups = expandedGroups - .filter((group) => !isExclusiveCompactGroup(group)) - .toSorted((a, b) => a.shard_name.localeCompare(b.shard_name)); - const regularBinCount = bins.filter((bin) => !bin.exclusive).length; - const regularBatches = createStripedBatches( - regularGroups, - regularBinCount, - estimateCompactGroupSeconds, - ); - if (regularBatches.some((batch) => batch.length > COMPACT_NODE_TEST_JOB_GROUPS)) { - throw new Error("striped compact job exceeds its group capacity"); - } - const regularBins = regularBatches.map((batch) => ({ - exclusive: false, - groups: batch, - hasWholeConfigGroup: batch.some((group) => !group.includePatterns), - weight: batch.reduce((sum, group) => sum + estimateCompactGroupSeconds(group), 0), - })); - const exclusiveBins = bins.filter((bin) => bin.exclusive); - bins.splice(0, bins.length, ...regularBins, ...exclusiveBins); + const regularGroups = expandedGroups + .filter((group) => !isExclusiveCompactGroup(group)) + .toSorted((a, b) => a.shard_name.localeCompare(b.shard_name)); + const regularBinCount = bins.filter((bin) => !bin.exclusive).length; + const regularBatches = createStripedBatches( + regularGroups, + regularBinCount, + estimateCompactGroupSeconds, + ); + if (regularBatches.some((batch) => batch.length > COMPACT_NODE_TEST_JOB_GROUPS)) { + throw new Error("striped compact job exceeds its group capacity"); } + const regularBins = regularBatches.map((batch) => ({ + exclusive: false, + groups: batch, + hasWholeConfigGroup: batch.some((group) => !group.includePatterns), + weight: batch.reduce((sum, group) => sum + estimateCompactGroupSeconds(group), 0), + })); + const exclusiveBins = bins.filter((bin) => bin.exclusive); + bins.splice(0, bins.length, ...regularBins, ...exclusiveBins); for (const [index, bin] of bins.entries()) { const [firstGroup] = bin.groups; diff --git a/test/scripts/ci-node-test-plan.test.ts b/test/scripts/ci-node-test-plan.test.ts index e86dd75f09d7..5f9b1758367d 100644 --- a/test/scripts/ci-node-test-plan.test.ts +++ b/test/scripts/ci-node-test-plan.test.ts @@ -210,13 +210,15 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { }); // Rebalancing may change ownership but must not add CI workers. - expect(compact).toHaveLength(24); + expect(compact).toHaveLength(23); expect(compact.every((shard) => Array.isArray(shard.groups))).toBe(true); expect(compact.every((shard) => shard.groups.length <= 10)).toBe(true); expect(compact.some((shard) => shard.requiresDist)).toBe(true); expect( compact.every((shard) => - shard.groups.every((group) => group.requiresDist === shard.requiresDist), + shard.groups.every( + (group) => group.requiresDist === shard.requiresDist && group.runner === shard.runner, + ), ), ).toBe(true); // Runtime-balanced packing must keep the two heaviest measured groups in @@ -231,13 +233,6 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { // Cheap stripes may legally co-locate in one bin; only existence matters. expect(jobOf("core-unit-fast-1")).toBeGreaterThanOrEqual(0); expect(jobOf("core-unit-fast-2")).toBeGreaterThanOrEqual(0); - const refreshedOutliers = [ - "agentic-agents-core-runtime", - "agentic-agents-support", - "auto-reply-reply-commands-2", - "core-unit-fast-2", - ]; - expect(new Set(refreshedOutliers.map(jobOf)).size).toBe(refreshedOutliers.length); // Spawn/signal-timing suites never mix with regular groups, and every // compact bin runs serially: overlapping Vitest runs flake timing- // sensitive tests on both runner classes. @@ -256,11 +251,26 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { shard.groups.some((group) => exclusiveGroupRe.test(group.shard_name)), ).length, ).toBeGreaterThan(0); - // Both plans carry the same split stripes now; compact bundling must - // preserve base include coverage exactly. + const expectedEmbeddedAgentGroupNames = [ + "agentic-agents-embedded-base", + "agentic-agents-embedded-incomplete-turn", + "agentic-agents-embedded-overflow-compaction", + "agentic-agents-embedded-run", + ]; + const compactGroups = compact.flatMap((shard) => shard.groups); + const expectedGroupNames = base.flatMap((shard) => + shard.shardName === "agentic-agents-embedded" + ? expectedEmbeddedAgentGroupNames + : [shard.shardName], + ); + expect(compactGroups.map((group) => group.shard_name).toSorted()).toEqual( + expectedGroupNames.toSorted(), + ); + // Both plans carry the same split stripes now; compact bundling must also + // preserve include-pattern coverage exactly. expect( - compact - .flatMap((shard) => shard.groups.flatMap((group) => group.includePatterns ?? [])) + compactGroups + .flatMap((group) => group.includePatterns ?? []) .toSorted((a, b) => a.localeCompare(b)), ).toEqual( base.flatMap((shard) => shard.includePatterns ?? []).toSorted((a, b) => a.localeCompare(b)), @@ -302,16 +312,25 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { const largeJobs = compact.filter( (shard) => shard.runner === DEFAULT_NODE_TEST_RUNNER && !shard.requiresDist, ); - expect(largeJobs).toHaveLength(8); + const smallJobs = compact.filter( + (shard) => shard.runner !== DEFAULT_NODE_TEST_RUNNER && !shard.requiresDist, + ); + const distJobs = compact.filter((shard) => shard.requiresDist); + expect(largeJobs).toHaveLength(7); + expect(smallJobs).toHaveLength(14); + expect(distJobs).toHaveLength(2); + expect(compact).toEqual( + createNodeTestShardBundles({ + includeReleaseOnlyPluginShards: false, + compact: true, + }), + ); const embeddedAgentGroups = compact .flatMap((shard) => shard.groups) .filter((group) => group.shard_name.startsWith("agentic-agents-embedded-")); - expect(embeddedAgentGroups.map((group) => group.shard_name).toSorted()).toEqual([ - "agentic-agents-embedded-base", - "agentic-agents-embedded-incomplete-turn", - "agentic-agents-embedded-overflow-compaction", - "agentic-agents-embedded-run", - ]); + expect(embeddedAgentGroups.map((group) => group.shard_name).toSorted()).toEqual( + expectedEmbeddedAgentGroupNames, + ); expect( compact.some((shard) => shard.groups.some((group) => group.shard_name === "agentic-agents-embedded"), @@ -325,10 +344,6 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { (group) => group.env?.OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS === "660000", ), ).toBe(true); - const embeddedBaseJob = compact.find((shard) => - shard.groups.some((group) => group.shard_name === "agentic-agents-embedded-base"), - ); - expect(embeddedBaseJob?.groups).toHaveLength(1); expect( compact .filter((shard) => shard.groups.some((group) => !group.includePatterns))