From 2a70d849e8a8bde3d66562bc441d96fb95388baf Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 13 Aug 2026 10:46:38 -0700 Subject: [PATCH] improve(ci): stripe giant compact test groups (#123142) * ci: stripe media-ui, gateway-core, and security compact groups * fix(ci): preserve compact stripe config ownership --- docs/ci.md | 6 +- scripts/lib/ci-node-test-plan.mts | 223 +++++++++++++++++-------- test/scripts/ci-node-test-plan.test.ts | 187 +++++++++++++++++---- 3 files changed, 311 insertions(+), 105 deletions(-) diff --git a/docs/ci.md b/docs/ci.md index 766e5f3daf9e..9bbff85d0872 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -117,11 +117,11 @@ Separate iOS and macOS Periphery workflows enforce a zero-findings dead-code pol The slowest Node test families are split or balanced so each job stays small without over-reserving runners: - Plugin contracts and channel contracts each run as two weighted Blacksmith-backed shards with the standard GitHub runner fallback. -- Core unit fast/support lanes run separately; core runtime infra splits into process, shared, hooks, secrets, and three cron domain shards. +- Core unit fast/support lanes run separately; unit-src, Control UI, and gateway-core each use two deterministic file-weighted stripes, while the security and media/UI companion configs retain their scoped whole-config support groups; core runtime infra splits into process, shared, hooks, secrets, and three cron domain shards. - Auto-reply runs as balanced workers, with the reply subtree split into agent-runner, commands, dispatch, session, and state-routing shards. - Agentic gateway/server (control-plane) configs split across chat, auth, model, HTTP/plugin, runtime, and startup lanes instead of waiting on built artifacts. - Normal CI packs only isolated infra include-pattern shards into deterministic bundles of at most 64 test files, reducing the Node matrix without merging non-isolated command/cron, stateful agents-core, or gateway/server suites. Heavy fixed suites stay on 8 vCPU while most bundled and lower-weight lanes use 4 vCPU. Compact-small bins 2, 5, and 8 use existing 8-vCPU capacity because recent hosted runs showed they repeatedly owned the critical path while the 4-vCPU queue was materially longer; routing happens after packing, so group ownership, coverage, and the existing registration count do not change. -- Pull requests on the canonical repository reuse the changed-test resolver against the synthetic merged-tree diff. Precise changes run one targeted Node job; each selected test file gets its own process so stateful suite isolation remains intact. The planner combines sibling tests with import-graph dependents and falls back to the existing 25-job compact full-suite plan for workspace package, package/lockfile, shared harness, split-config, renamed, or deleted changes, public extension-contract changes, tests with special shard setup, partially resolved or empty targets, oversized path or target plans, and planner errors. Targeted plans always retain the full built-artifact boundary gate because its repository scanners cannot be derived from imports. `main` pushes run the same full compact suite: pending intermediate push events can be coalesced, so the newest surviving run must validate the complete integration tree rather than only its final single-push diff. Manual dispatches and release gates retain the full named per-shard matrix. Compact packing uses hosted means to tail-balance regular 8-vCPU bins while retaining median admission and 4-vCPU striping weights, so recurrent slow tails rebalance without changing the bounded job count or post-pack runner advisory; the high-variance source/security group remains isolated so its tail does not serialize unrelated groups. +- Pull requests on the canonical repository reuse the changed-test resolver against the synthetic merged-tree diff. Precise changes run one targeted Node job; each selected test file gets its own process so stateful suite isolation remains intact. The planner combines sibling tests with import-graph dependents and falls back to the 28-descriptor compact full-suite plan for workspace package, package/lockfile, shared harness, split-config, renamed, or deleted changes, public extension-contract changes, tests with special shard setup, partially resolved or empty targets, oversized path or target plans, and planner errors. Twenty-six nondist descriptors run as Node jobs; two dist descriptors fold into the built-artifact boundary. Targeted plans always retain that full boundary gate because its repository scanners cannot be derived from imports. `main` pushes run the same full compact suite: pending intermediate push events can be coalesced, so the newest surviving run must validate the complete integration tree rather than only its final single-push diff. Manual dispatches and release gates retain the full named per-shard matrix. Compact packing uses hosted timings to stripe the former media/UI, gateway-core, and source/security floor groups, then tail-balances regular 8-vCPU bins while retaining the existing post-pack runner advisory. - The full Node matrix admits the consistently slow serial tooling, auto-reply command shards, and broad core-fast cache writer first. This keeps the 28-job cap while preventing critical-path work and the next run's transform seed from slipping into a later wave. - The three serial Control UI browser shards greedily pack discovered test files by source byte size. This zero-state duration proxy avoids Vitest's equal-file-count hash clustering, automatically accounts for new and changed files, and preserves the same complete test inventory without adding runners. - Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. Include-pattern shards record timing entries using the CI shard name, so `.artifacts/vitest-shard-timings.json` can distinguish a whole config from a filtered shard. @@ -238,7 +238,7 @@ target below about 60% of the live bucket. With the current 10,000-registration bucket, that means a 6,000-registration operating target, leaving headroom for concurrent repositories, retries, and burst overlap. -The changed-target PR plan reduces the common Node test burst from 14 Blacksmith registrations to one. Broad-risk PRs keep the 14-registration compact fallback, so the worst case does not increase. +The changed-target PR plan reduces the common Node test burst from 26 Blacksmith registrations to one. Broad-risk PRs keep the 26-registration compact fallback. Canonical-repo CI keeps Blacksmith as the default runner path for pushes and first-attempt same-repo pull-request runs. Pull-request retries of both UI E2E jobs use GitHub-hosted Ubuntu; push retries stay on Blacksmith. All `workflow_dispatch` runs, including `release_gate`, and non-canonical repository runs use GitHub-hosted runners. The [Blacksmith outage circuit breaker](#blacksmith-outage-circuit-breaker) provides a manual repository-wide fallback; canonical runs do not probe Blacksmith queue health or flip it automatically. diff --git a/scripts/lib/ci-node-test-plan.mts b/scripts/lib/ci-node-test-plan.mts index d3701352ee2e..a0048276c823 100644 --- a/scripts/lib/ci-node-test-plan.mts +++ b/scripts/lib/ci-node-test-plan.mts @@ -6,18 +6,20 @@ import { } from "../../test/vitest/vitest.agents-paths.mjs"; import { commandsLightTestFiles } from "../../test/vitest/vitest.commands-light-paths.mjs"; import { + gatewayServerExcludedTestFiles, isGatewayServerBackedHttpTestFile, isGatewayServerTestFile, } from "../../test/vitest/vitest.gateway-server-paths.mjs"; import { fullSuiteVitestShards } from "../../test/vitest/vitest.test-shards.mjs"; import { toolingIsolatedTestFiles } from "../../test/vitest/vitest.tooling-isolated-paths.mjs"; +import { uiIsolatedTestFiles } from "../../test/vitest/vitest.ui-isolated-paths.mjs"; import { getUnitFastIsolatedTestFiles, getUnitFastTestFiles, getUnitFastTestFilesForIncludePatterns, getUnitFastTimerTestFiles, } from "../../test/vitest/vitest.unit-fast-paths.mjs"; -import { boundaryTestFiles } from "../../test/vitest/vitest.unit-paths.mjs"; +import { boundaryTestFiles, isUnitConfigTestFile } from "../../test/vitest/vitest.unit-paths.mjs"; import { listTrackedTestFiles } from "./list-test-files.mts"; type NodeTestShardGroup = { @@ -158,10 +160,10 @@ const COMPACT_EMBEDDED_GROUP_NAMES = [ const MAX_BUNDLED_NODE_TEST_PATTERNS = 64; // PR-only bundles trade a little serial work for fewer ephemeral runner registrations. // Keep runner classes and subprocess isolation intact while bounding each combined job. -// The group hints below are loaded-fleet CI walls. The 310s admission cap -// bounds the compact matrix at 25 workers; expanded composite groups are then -// striped evenly across those jobs. -const COMPACT_NODE_TEST_JOB_SECONDS = 310; +// The group hints below are loaded-fleet CI walls. After striping the three +// former floor groups, the 276s admission cap bounds the compact matrix at the +// workflow's 28-worker ceiling; composite groups then balance across those jobs. +const COMPACT_NODE_TEST_JOB_SECONDS = 276; const COMPACT_NODE_TEST_JOB_GROUPS = 10; const COMPACT_TOOLING_NODE_TEST_GROUPS = 4; const COMPACT_WHOLE_NODE_TEST_TIMEOUT_MINUTES = 120; @@ -174,6 +176,9 @@ const COMPACT_8VCPU_CHECK_NAMES = new Set([ ]); const AUTO_REPLY_COMMANDS_STRIPES = 3; const AGENTS_CORE_RUNNER_CLI_STRIPES = 3; +const AGENTIC_GATEWAY_CORE_STRIPES = 2; +const CORE_RUNTIME_MEDIA_UI_STRIPES = 2; +const CORE_UNIT_SRC_SECURITY_STRIPES = 2; const UNIT_FAST_NODE_TEST_STRIPES = 2; // Advisory runtime estimates (seconds) per split shard: median [shard:*] // begin->end wall across nine successful hosted compact runs (31684307744, @@ -238,7 +243,8 @@ const COMPACT_GROUP_SECONDS_HINTS = new Map([ ["agentic-control-plane-startup-core", 31], ["agentic-control-plane-startup-health-runtime", 11], ["agentic-control-plane-startup-restart-close", 10], - ["agentic-gateway-core", 223], + ["agentic-gateway-core-1", 149], + ["agentic-gateway-core-2", 148], ["agentic-gateway-methods", 157], ["agentic-plugin-sdk", 45], ["auto-reply-core-top-level", 27], @@ -282,7 +288,9 @@ const COMPACT_GROUP_SECONDS_HINTS = new Map([ ["core-runtime-infra-repo-tooling", 4], ["core-runtime-infra-storage-state", 104], ["core-runtime-infra-system-runtime", 36], - ["core-runtime-media-ui", 227], + ["core-runtime-media-ui-1", 160], + ["core-runtime-media-ui-2", 121], + ["core-runtime-media-ui-support", 100], ["core-runtime-secrets", 61], ["core-runtime-shared", 67], // This dist-only group is outside the sampled nondist logs and retains its @@ -296,7 +304,9 @@ const COMPACT_GROUP_SECONDS_HINTS = new Map([ ["core-unit-fast-1", 66], ["core-unit-fast-2", 64], ["core-unit-fast-isolated", 116], - ["core-unit-src-security", 290], + ["core-unit-src-security-1", 152], + ["core-unit-src-security-2", 152], + ["core-unit-src-security-support", 12], ["core-unit-support", 20], ]); @@ -323,17 +333,27 @@ const COMPACT_LARGE_GROUP_STRIPE_SECONDS_HINTS = new Map([ ["agentic-agents-embedded-run", 47], ["agentic-agents-support", 165], ["agentic-control-plane-startup-core", 33], - ["agentic-gateway-core", 230], + // Run 31691151297 measured 296.68s for gateway-core, 241.66s for UI, + // 101.58s for its companion configs, 303.93s for unit-src, and 12.07s for + // security. Run 31694057974 measured the two UI envelopes at 159.50s and + // 120.55s while exposing cross-config bleed; keep those rounded upper + // bounds after isolating the companion whole-config groups. + ["agentic-gateway-core-1", 149], + ["agentic-gateway-core-2", 148], ["agentic-gateway-methods", 153], ["auto-reply-reply-commands-1", 34], ["auto-reply-reply-commands-2", 11], ["auto-reply-reply-commands-3", 28], ["auto-reply-reply-dispatch", 86], - ["core-runtime-media-ui", 238], + ["core-runtime-media-ui-1", 160], + ["core-runtime-media-ui-2", 121], + ["core-runtime-media-ui-support", 100], ["core-unit-fast-1", 68], ["core-unit-fast-2", 67], ["core-unit-fast-isolated", 117], - ["core-unit-src-security", 287], + ["core-unit-src-security-1", 152], + ["core-unit-src-security-2", 152], + ["core-unit-src-security-support", 12], ]); // Advisory per-file wall-clock hints (seconds) for stripe balancing, measured @@ -348,13 +368,21 @@ const STRIPE_FILE_SECONDS_HINTS = new Map([ // Fresh profile: 5.1s total, 3.8s import; retain a conservative packing hint. ["src/agents/cli-runner.reliability.test.ts", 8], ["src/agents/cli-runner.spawn.test.ts", 45], + // The few CI-derived slow-file hints needed for the three new stripes are + // rounded checkmark durations from canonical-main run 31691151297. ["src/auto-reply/reply/commands-export-session.test.ts", 8], ["src/auto-reply/reply/commands-gating.test.ts", 6], ["src/auto-reply/reply/commands-learn.test.ts", 8], ["src/auto-reply/reply/commands-plugins.install.test.ts", 6], ["src/auto-reply/reply/commands-status.test.ts", 12], ["src/auto-reply/reply/commands-system-prompt.test.ts", 8], + ["src/gateway/dashboard-session-title.test.ts", 23], + ["src/gateway/managed-image-attachments.test.ts", 24], + ["src/gateway/session-message-events.test.ts", 26], + ["src/gateway/tool-resolution.test.ts", 43], ["src/scripts/test-projects.test.ts", 21], + ["ui/src/components/app-sidebar.test.ts", 28], + ["ui/src/pages/chat/chat-responsive.browser.test.ts", 30], // Focused cold proof is ~34s after right-sizing and concurrent crash phases. ["test/scripts/bench-sqlite-reliability.test.ts", 34], ["test/scripts/bundled-plugin-install-uninstall-probe.test.ts", 4], @@ -392,7 +420,7 @@ function isExclusiveCompactGroup(group: NodeTestShardGroup): boolean { // scales with the runner class. infra-process spawns child processes per test // and hit worker-startup timeouts under contention before serialization. const PINNED_WORKER_COMPACT_GROUP_RE = - /^core-tooling(?:-\d+|-isolated)$|^core-runtime-tui-pty$|^core-runtime-infra-process$|^core-runtime-media-ui$|^agentic-cli$|^agentic-gateway-(?:core|methods)$/u; + /^core-tooling(?:-\d+|-isolated)$|^core-runtime-tui-pty$|^core-runtime-infra-process$|^core-runtime-media-ui-(?:\d+|support)$|^agentic-cli$|^agentic-gateway-(?:core-\d+|methods)$/u; const PINNED_COMPACT_GROUP_ENV = { OPENCLAW_VITEST_MAX_WORKERS: "2" }; function applyCompactGroupWorkerPins(group: NodeTestShardGroup): NodeTestShardGroup { @@ -475,7 +503,8 @@ const KEEP_LARGE_NODE_TEST_RUNNER = new Set([ "agentic-agents-core-runner-sessions", "agentic-agents-core-tools", "agentic-control-plane-startup-core", - "agentic-gateway-core", + "agentic-gateway-core-1", + "agentic-gateway-core-2", "agentic-gateway-methods", "auto-reply-reply-dispatch", // The commands stripes and security suite are import-bound (30-45s of @@ -484,11 +513,15 @@ const KEEP_LARGE_NODE_TEST_RUNNER = new Set([ "auto-reply-reply-commands-1", "auto-reply-reply-commands-2", "auto-reply-reply-commands-3", - "core-runtime-media-ui", + "core-runtime-media-ui-1", + "core-runtime-media-ui-2", + "core-runtime-media-ui-support", "core-unit-fast-1", "core-unit-fast-2", "core-unit-fast-isolated", - "core-unit-src-security", + "core-unit-src-security-1", + "core-unit-src-security-2", + "core-unit-src-security-support", ]); const RELEASE_ONLY_PLUGIN_SHARDS = new Set(["agentic-plugins"]); function listTestFiles(rootDir: string): string[] { @@ -1254,23 +1287,110 @@ function createToolingSplitShards(): NodeTestSplitShard[] { ]; } +function isStripeEligibleTestFile(file: string, unitFastFiles: ReadonlySet): boolean { + return ( + !unitFastFiles.has(file) && !file.endsWith(".e2e.test.ts") && !file.endsWith(".live.test.ts") + ); +} + +function createStripedSplitShards(params: { + configs: string[]; + files: string[]; + includeExternalConfigs?: boolean; + shardName: string; + stripeCount: number; +}): NodeTestSplitShard[] { + return createStripedBatches(params.files, params.stripeCount, stripeFileWeight).map( + (includePatterns, index) => ({ + configs: params.configs, + includeExternalConfigs: params.includeExternalConfigs, + includePatterns, + requiresDist: false, + shardName: `${params.shardName}-${index + 1}`, + }), + ); +} + +function createCoreUnitSrcSecuritySplitShards(): NodeTestSplitShard[] { + const unitFastFiles = new Set(getUnitFastTestFiles()); + const files = listTestFiles("src").filter( + (file) => + isStripeEligibleTestFile(file, unitFastFiles) && + !file.startsWith("src/acp/") && + !file.startsWith("src/security/") && + isUnitConfigTestFile(file), + ); + return [ + ...createStripedSplitShards({ + configs: ["test/vitest/vitest.unit-src.config.ts"], + files, + shardName: "core-unit-src-security", + stripeCount: CORE_UNIT_SRC_SECURITY_STRIPES, + }), + { + configs: ["test/vitest/vitest.unit-security.config.ts"], + includeExternalConfigs: true, + requiresDist: false, + shardName: "core-unit-src-security-support", + }, + ]; +} + +function createCoreRuntimeMediaUiSplitShards(): NodeTestSplitShard[] { + const unitFastFiles = new Set(getUnitFastTestFiles()); + const isolatedUiFiles = new Set(uiIsolatedTestFiles); + const files = listTestFiles("ui/src").filter( + (file) => isStripeEligibleTestFile(file, unitFastFiles) && !isolatedUiFiles.has(file), + ); + return [ + ...createStripedSplitShards({ + configs: ["test/vitest/vitest.ui.config.ts"], + files, + shardName: "core-runtime-media-ui", + stripeCount: CORE_RUNTIME_MEDIA_UI_STRIPES, + }), + { + configs: [ + "test/vitest/vitest.media.config.ts", + "test/vitest/vitest.media-understanding.config.ts", + "test/vitest/vitest.tui.config.ts", + "test/vitest/vitest.ui-isolated.config.ts", + "test/vitest/vitest.wizard.config.ts", + ], + requiresDist: false, + shardName: "core-runtime-media-ui-support", + }, + ]; +} + +function createAgenticGatewayCoreSplitShards(): NodeTestSplitShard[] { + const unitFastFiles = new Set(getUnitFastTestFiles()); + const excludedGatewayFiles = new Set(gatewayServerExcludedTestFiles); + const gatewayFiles = listTestFiles("src/gateway").filter( + (file) => + isStripeEligibleTestFile(file, unitFastFiles) && + !file.startsWith("src/gateway/server-methods/") && + !isGatewayServerTestFile(file) && + !excludedGatewayFiles.has(file), + ); + const packageFiles = ["packages/gateway-client/src", "packages/gateway-protocol/src"] + .flatMap((rootDir) => listTestFiles(rootDir)) + .filter((file) => isStripeEligibleTestFile(file, unitFastFiles)); + return createStripedSplitShards({ + configs: [ + "test/vitest/vitest.gateway-core.config.ts", + "test/vitest/vitest.gateway-client.config.ts", + ], + files: [...gatewayFiles, ...packageFiles], + shardName: "agentic-gateway-core", + stripeCount: AGENTIC_GATEWAY_CORE_STRIPES, + }); +} + const SPLIT_NODE_SHARDS = new Map([ ["core-unit-fast", createUnitFastSplitShards()], ["core-tooling", createToolingSplitShards()], - [ - "core-unit-src", - [ - { - shardName: "core-unit-src-security", - configs: [ - "test/vitest/vitest.unit-src.config.ts", - "test/vitest/vitest.unit-security.config.ts", - ], - includeExternalConfigs: true, - requiresDist: false, - }, - ], - ], + ["core-unit-src", createCoreUnitSrcSecuritySplitShards()], ["core-unit-security", []], [ "core-unit-support", @@ -1318,18 +1438,7 @@ const SPLIT_NODE_SHARDS = new Map([ requiresDist: true, runner: "blacksmith-4vcpu-ubuntu-2404", }, - { - shardName: "core-runtime-media-ui", - configs: [ - "test/vitest/vitest.media.config.ts", - "test/vitest/vitest.media-understanding.config.ts", - "test/vitest/vitest.tui.config.ts", - "test/vitest/vitest.ui.config.ts", - "test/vitest/vitest.ui-isolated.config.ts", - "test/vitest/vitest.wizard.config.ts", - ], - requiresDist: false, - }, + ...createCoreRuntimeMediaUiSplitShards(), { shardName: "core-runtime-shared", configs: [ @@ -1392,14 +1501,7 @@ const SPLIT_NODE_SHARDS = new Map([ configs: [agentVitestProjectOwners.tools.config], requiresDist: false, }, - { - shardName: "agentic-gateway-core", - configs: [ - "test/vitest/vitest.gateway-core.config.ts", - "test/vitest/vitest.gateway-client.config.ts", - ], - requiresDist: false, - }, + ...createAgenticGatewayCoreSplitShards(), { shardName: "agentic-gateway-methods", configs: ["test/vitest/vitest.gateway-methods.config.ts"], @@ -1783,29 +1885,18 @@ function createCompactNodeTestShardBundles( } } - // First-fit above determines the bounded worker count. Keep the - // high-variance source/security group isolated, then stripe every other - // regular group across the remaining workers. Re-striping the expanded - // embedded group avoids full early bins and nearly empty tail bins. + // First-fit above determines the bounded worker count. Re-striping the + // expanded embedded group avoids full early bins and nearly empty tails. const expandedGroups = groups.flatMap(expandCompactGroup); const regularGroups = expandedGroups .filter((group) => !isExclusiveCompactGroup(group)) .toSorted((a, b) => a.shard_name.localeCompare(b.shard_name)); const regularBinCount = bins.filter((bin) => !bin.exclusive).length; - const isolatedGroups = regularGroups.filter( - (group) => group.shard_name === "core-unit-src-security", + const regularBatches = createStripedBatches( + regularGroups, + regularBinCount, + estimateCompactStripeSeconds, ); - const stripedGroups = regularGroups.filter( - (group) => group.shard_name !== "core-unit-src-security", - ); - const regularBatches = [ - ...isolatedGroups.map((group) => [group]), - ...createStripedBatches( - stripedGroups, - regularBinCount - isolatedGroups.length, - estimateCompactStripeSeconds, - ), - ]; if (regularBatches.some((batch) => batch.length > COMPACT_NODE_TEST_JOB_GROUPS)) { throw new Error("striped compact job exceeds its group capacity"); } diff --git a/test/scripts/ci-node-test-plan.test.ts b/test/scripts/ci-node-test-plan.test.ts index f410c2d03500..640a91ce6262 100644 --- a/test/scripts/ci-node-test-plan.test.ts +++ b/test/scripts/ci-node-test-plan.test.ts @@ -17,10 +17,19 @@ import { embeddedAgentVitestProjectOwners, } from "../vitest/vitest.agents-paths.mjs"; import { commandsLightTestFiles } from "../vitest/vitest.commands-light-paths.mjs"; +import { createGatewayClientVitestConfig } from "../vitest/vitest.gateway-client.config.ts"; +import { createGatewayCoreVitestConfig } from "../vitest/vitest.gateway-core.config.ts"; import { isGatewayServerTestFile } from "../vitest/vitest.gateway-server-paths.mjs"; import { createGatewayServerVitestConfig } from "../vitest/vitest.gateway-server.config.ts"; +import { createMediaUnderstandingVitestConfig } from "../vitest/vitest.media-understanding.config.ts"; +import { createMediaVitestConfig } from "../vitest/vitest.media.config.ts"; import { createPluginsVitestConfig } from "../vitest/vitest.plugins.config.ts"; import { createToolingVitestConfig } from "../vitest/vitest.tooling.config.ts"; +import { createTuiVitestConfig } from "../vitest/vitest.tui.config.ts"; +import { createUiIsolatedVitestConfig } from "../vitest/vitest.ui-isolated.config.ts"; +import { createUiVitestConfig } from "../vitest/vitest.ui.config.ts"; +import { createUnitVitestConfigWithOptions } from "../vitest/vitest.unit.config.ts"; +import { createWizardVitestConfig } from "../vitest/vitest.wizard.config.ts"; type VitestTestConfig = { dir?: string; @@ -268,7 +277,7 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { // Rebalancing may change ownership, but the compact plan stays within the // CI workflow's 28-worker cap. - expect(compact).toHaveLength(25); + expect(compact).toHaveLength(28); expect(compact.every((shard) => Array.isArray(shard.groups))).toBe(true); expect(compact.every((shard) => shard.groups.length <= 10)).toBe(true); expect(compact.some((shard) => shard.requiresDist)).toBe(true); @@ -279,18 +288,18 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { ), ), ).toBe(true); - // Runtime-balanced packing must keep the two heaviest measured groups in - // different jobs; regressing to per-file weights recombines them. const jobOf = (name: string) => compact.findIndex((shard) => shard.groups.some((group) => group.shard_name === name)); expect(jobOf("agentic-agents-core-runner-embedded")).toBeGreaterThanOrEqual(0); - // The complete Control UI and model catalog both cold-load broad graphs; - // pairing them starves model visibility and repeatedly hits its timeout. - expect(jobOf("agentic-agents-core-models")).not.toBe(jobOf("core-runtime-media-ui")); - expect(jobOf("core-runtime-media-ui")).not.toBe(jobOf("core-unit-src-security")); - expect( - compact[jobOf("core-unit-src-security")]?.groups.map((group) => group.shard_name), - ).toEqual(["core-unit-src-security"]); + for (const prefix of [ + "agentic-gateway-core", + "core-runtime-media-ui", + "core-unit-src-security", + ]) { + expect(jobOf(`${prefix}-1`)).toBeGreaterThanOrEqual(0); + expect(jobOf(`${prefix}-2`)).toBeGreaterThanOrEqual(0); + expect(jobOf(`${prefix}-1`)).not.toBe(jobOf(`${prefix}-2`)); + } // Cheap stripes may legally co-locate in one bin; only existence matters. expect(jobOf("core-unit-fast-1")).toBeGreaterThanOrEqual(0); expect(jobOf("core-unit-fast-2")).toBeGreaterThanOrEqual(0); @@ -356,6 +365,20 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { ).toEqual({ OPENCLAW_VITEST_MAX_WORKERS: "2", }); + for (const prefix of ["agentic-gateway-core", "core-runtime-media-ui"]) { + for (const suffix of ["1", "2"]) { + expect( + compact + .flatMap((shard) => shard.groups) + .find((group) => group.shard_name === `${prefix}-${suffix}`)?.env, + ).toEqual({ OPENCLAW_VITEST_MAX_WORKERS: "2" }); + } + } + expect( + compact + .flatMap((shard) => shard.groups) + .find((group) => group.shard_name === "core-runtime-media-ui-support")?.env, + ).toEqual({ OPENCLAW_VITEST_MAX_WORKERS: "2" }); const startupCoreJob = compact.find((shard) => shard.groups.some((group) => group.shard_name === "agentic-control-plane-startup-core"), ); @@ -377,13 +400,13 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { (shard) => !shard.requiresDist && shard.checkName.startsWith("checks-node-compact-small-"), ); const distJobs = compact.filter((shard) => shard.requiresDist); - expect(largeJobs).toHaveLength(7); - expect(smallJobs).toHaveLength(16); + expect(largeJobs).toHaveLength(9); + expect(smallJobs).toHaveLength(17); expect(distJobs).toHaveLength(2); const regularSmallJobs = smallJobs.filter((shard) => shard.groups.every((group) => !exclusiveGroupRe.test(group.shard_name)), ); - expect(regularSmallJobs).toHaveLength(11); + expect(regularSmallJobs).toHaveLength(12); const routed8VcpuCheckNames = [ "checks-node-compact-small-2", "checks-node-compact-small-5", @@ -402,12 +425,14 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { // The refreshed hosted estimates give every regular bin one known tail // anchor. Stale hints paired two slow groups in each runner class. const largeTailAnchors = [ - "core-unit-src-security", - "agentic-gateway-core", - "core-runtime-media-ui", + "core-unit-src-security-1", + "core-unit-src-security-2", + "agentic-gateway-core-1", + "agentic-gateway-core-2", + "core-runtime-media-ui-1", + "core-runtime-media-ui-2", "agentic-agents-support", "agentic-gateway-methods", - "agentic-agents-core-runtime", "core-unit-fast-isolated", ]; const smallTailAnchors = [ @@ -422,6 +447,7 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { "agentic-commands-doctor-config-state", "core-runtime-shared", "auto-reply-reply-state-routing", + "auto-reply-reply-agent-runner", ]; expect( largeJobs.map( @@ -531,12 +557,19 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { shardName: "core-unit-fast-isolated", }, { - configs: [ - "test/vitest/vitest.unit-src.config.ts", - "test/vitest/vitest.unit-security.config.ts", - ], + configs: ["test/vitest/vitest.unit-src.config.ts"], requiresDist: false, - shardName: "core-unit-src-security", + shardName: "core-unit-src-security-1", + }, + { + configs: ["test/vitest/vitest.unit-src.config.ts"], + requiresDist: false, + shardName: "core-unit-src-security-2", + }, + { + configs: ["test/vitest/vitest.unit-security.config.ts"], + requiresDist: false, + shardName: "core-unit-src-security-support", }, { configs: ["test/vitest/vitest.unit-support.config.ts"], @@ -546,6 +579,71 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { ]); }); + it("partitions each giant compact group across two deterministic stripes", () => { + const env = { ...process.env, OPENCLAW_VITEST_INCLUDE_FILE: undefined }; + const cases = [ + { + stripeConfigs: [createUiVitestConfig(env)], + supportConfigs: [ + createMediaVitestConfig(env), + createMediaUnderstandingVitestConfig(env), + createTuiVitestConfig(env), + createUiIsolatedVitestConfig(env), + createWizardVitestConfig(env), + ], + prefix: "core-runtime-media-ui", + }, + { + stripeConfigs: [createGatewayCoreVitestConfig(env), createGatewayClientVitestConfig(env)], + supportConfigs: [], + prefix: "agentic-gateway-core", + }, + { + stripeConfigs: [ + createUnitVitestConfigWithOptions(env, { + name: "unit-src", + includePatterns: ["src/**/*.test.ts"], + extraExcludePatterns: ["src/acp/**", "src/security/**"], + }), + ], + supportConfigs: [ + createUnitVitestConfigWithOptions(env, { + name: "unit-security", + includePatterns: ["src/security/**/*.test.ts"], + passWithNoTests: true, + }), + ], + prefix: "core-unit-src-security", + }, + ]; + + const shards = createNodeTestShards(); + for (const { prefix, stripeConfigs, supportConfigs } of cases) { + const stripes = shards.filter( + (shard) => /^.+-\d+$/u.test(shard.shardName) && shard.shardName.startsWith(`${prefix}-`), + ); + const actual = stripes + .flatMap((stripe) => stripe.includePatterns ?? []) + .toSorted((a, b) => a.localeCompare(b)); + const expected = stripeConfigs + .flatMap((config) => listMatchedTestFiles(config)) + .toSorted((a, b) => a.localeCompare(b)); + + expect(stripes.map((stripe) => stripe.shardName)).toEqual([`${prefix}-1`, `${prefix}-2`]); + expect(stripes.every((stripe) => (stripe.includePatterns?.length ?? 0) > 0)).toBe(true); + expect(new Set(actual).size).toBe(actual.length); + expect(actual).toEqual(expected); + + const support = shards.find((shard) => shard.shardName === `${prefix}-support`); + if (supportConfigs.length === 0) { + expect(support).toBeUndefined(); + } else { + expect(support?.includePatterns).toBeUndefined(); + expect(support?.configs).toHaveLength(supportConfigs.length); + } + } + }); + it("names the node shard checks as core test lanes", () => { const shards = createNodeTestShards(); @@ -817,18 +915,29 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { runner: "blacksmith-4vcpu-ubuntu-2404", shardName: "core-runtime-tui-pty", }, + { + configs: ["test/vitest/vitest.ui.config.ts"], + requiresDist: false, + runner: DEFAULT_NODE_TEST_RUNNER, + shardName: "core-runtime-media-ui-1", + }, + { + configs: ["test/vitest/vitest.ui.config.ts"], + requiresDist: false, + runner: DEFAULT_NODE_TEST_RUNNER, + shardName: "core-runtime-media-ui-2", + }, { configs: [ "test/vitest/vitest.media.config.ts", "test/vitest/vitest.media-understanding.config.ts", "test/vitest/vitest.tui.config.ts", - "test/vitest/vitest.ui.config.ts", "test/vitest/vitest.ui-isolated.config.ts", "test/vitest/vitest.wizard.config.ts", ], requiresDist: false, runner: DEFAULT_NODE_TEST_RUNNER, - shardName: "core-runtime-media-ui", + shardName: "core-runtime-media-ui-support", }, { configs: [ @@ -950,7 +1059,9 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { ); const commandShards = shards.filter((shard) => shard.shardName.startsWith("agentic-commands-")); const agentShards = shards.filter((shard) => shard.shardName.startsWith("agentic-agents-")); - const gatewayCoreShard = shards.find((shard) => shard.shardName === "agentic-gateway-core"); + const gatewayCoreShards = shards.filter((shard) => + shard.shardName.startsWith("agentic-gateway-core-"), + ); const gatewayMethodsShard = shards.find( (shard) => shard.shardName === "agentic-gateway-methods", ); @@ -1200,16 +1311,19 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { requiresDist: false, runner: DEFAULT_NODE_TEST_RUNNER, }); - expect(gatewayCoreShard).toEqual({ - checkName: "checks-node-agentic-gateway-core", - shardName: "agentic-gateway-core", - configs: [ - "test/vitest/vitest.gateway-core.config.ts", - "test/vitest/vitest.gateway-client.config.ts", - ], - requiresDist: false, - runner: DEFAULT_NODE_TEST_RUNNER, - }); + expect(gatewayCoreShards).toEqual( + [1, 2].map((stripe) => ({ + checkName: `checks-node-agentic-gateway-core-${stripe}`, + shardName: `agentic-gateway-core-${stripe}`, + configs: [ + "test/vitest/vitest.gateway-core.config.ts", + "test/vitest/vitest.gateway-client.config.ts", + ], + includePatterns: gatewayCoreShards[stripe - 1]?.includePatterns, + requiresDist: false, + runner: DEFAULT_NODE_TEST_RUNNER, + })), + ); expect(gatewayMethodsShard).toEqual({ checkName: "checks-node-agentic-gateway-methods", shardName: "agentic-gateway-methods", @@ -1294,7 +1408,8 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { const shardNames = shards.map((shard) => shard.shardName); expect(shardNames).not.toContain("agentic-plugins"); - expect(shardNames).toContain("agentic-gateway-core"); + expect(shardNames).toContain("agentic-gateway-core-1"); + expect(shardNames).toContain("agentic-gateway-core-2"); expect(shardNames).toContain("agentic-gateway-methods"); expect(shardNames).toContain("agentic-plugin-sdk"); });