From 2c7cf8d74599d9d43d5d0698946c4cc9a72d4c28 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 03:36:37 -0700 Subject: [PATCH] perf(ci): scale Vitest workers with runner class and share serial module cache (#108788) * perf(ci): scale Vitest workers with runner class and share serial module cache Compact node-test jobs pinned OPENCLAW_VITEST_MAX_WORKERS=2 regardless of runner size, so import-bound suites (30-45s of module-graph import per file) crawled: runner-cli spent 284s on 5s of tests, commands-1 296s on 10s. With serial plans the budget now scales with the class (6 on 8vcpu, 3 on 4vcpu); timing-sensitive groups (tooling, tui-pty, infra-process) stay pinned to 2 via plan-level env. Serial bins share one Vitest fs-module-cache path so later plans reuse the first plan's transforms. The import-bound commands stripes and security suite move to the 8vcpu class, and packing hints refresh from serial-run measurements (run 29481835688). * perf(ci): stripe cli-runner suite and pin media-ui worker budget The agents-core config runs files serially (fileParallelism false guards shared module state), so raising the worker budget cannot help its import-heavy cli-runner suite (~35s module import per file, 213s serial). Stripe it three ways so bins parallelize the imports instead. media-ui hosts browser locator tests that timed out at 6 workers on the first validation run; pin it to the proven 2-worker budget. commands-1's hint drops to its measured 6-worker runtime. --- .github/workflows/ci.yml | 13 +- scripts/ci-run-node-test-shard.d.mts | 1 + scripts/ci-run-node-test-shard.mjs | 17 +- scripts/lib/ci-node-test-plan.mjs | 205 +++++++++++++++---------- test/scripts/ci-node-test-plan.test.ts | 35 ++++- 5 files changed, 178 insertions(+), 93 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2de98cfb4f2d..3811a6305361 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1720,7 +1720,18 @@ jobs: run: test "$(go env GOVERSION)" = "go1.25.12" - name: Configure Node test resources - run: echo "OPENCLAW_VITEST_MAX_WORKERS=2" >> "$GITHUB_ENV" + # Scale the in-process Vitest worker budget with the runner class. Bins + # run one Vitest process at a time (planConcurrency 1), so workers are + # the only in-job parallelism; import-bound suites scale near-linearly. + # Timing-sensitive groups stay pinned to 2 via plan-level env. + env: + SHARD_RUNNER: ${{ matrix.runner || 'blacksmith-4vcpu-ubuntu-2404' }} + run: | + if [[ "$SHARD_RUNNER" == *"-8vcpu-"* ]]; then + echo "OPENCLAW_VITEST_MAX_WORKERS=6" >> "$GITHUB_ENV" + else + echo "OPENCLAW_VITEST_MAX_WORKERS=3" >> "$GITHUB_ENV" + fi - name: Run Node test shard # actionlint 1.7.11 lacks GitHub's current job.workflow_* fields. Serialize the diff --git a/scripts/ci-run-node-test-shard.d.mts b/scripts/ci-run-node-test-shard.d.mts index 0c5e242b4a99..6bc10288f413 100644 --- a/scripts/ci-run-node-test-shard.d.mts +++ b/scripts/ci-run-node-test-shard.d.mts @@ -24,6 +24,7 @@ export function buildChildEnv( baseEnv: Record, scratchDir: string, index: number, + options?: { serial?: boolean }, ): Record; export function resolveShardChildCommand( diff --git a/scripts/ci-run-node-test-shard.mjs b/scripts/ci-run-node-test-shard.mjs index 9ea412c4d70e..dd22447f8874 100644 --- a/scripts/ci-run-node-test-shard.mjs +++ b/scripts/ci-run-node-test-shard.mjs @@ -50,12 +50,17 @@ export function resolveShardPlans(env = process.env) { })); } -export function buildChildEnv(entry, baseEnv, scratchDir, index) { +export function buildChildEnv(entry, baseEnv, scratchDir, index, options = {}) { const childEnv = { ...baseEnv, - // Concurrent children must not share a Vitest module cache directory; - // shared caches race with ENOTEMPTY when two runs rewrite the same entries. - [FS_MODULE_CACHE_PATH_ENV_KEY]: join(scratchDir, `vitest-cache-${index}`), + // Concurrent children must not share a Vitest module cache directory + // (shared caches race with ENOTEMPTY when two runs rewrite the same + // entries), but serial bins reuse one cache so later plans skip the cold + // module transforms the first plan already paid for. + [FS_MODULE_CACHE_PATH_ENV_KEY]: join( + scratchDir, + options.serial ? "vitest-cache-shared" : `vitest-cache-${index}`, + ), OPENCLAW_TEST_PROJECTS_PARALLEL: "1", // This wrapper holds the repo heavy-check lock; children skipping it is // what lets two plans run concurrently instead of serializing on the lock. @@ -173,7 +178,9 @@ export async function runShardPlans(plans, options = {}) { exitCode = exitCode || 1; return; } - const childEnv = buildChildEnv(entry, baseEnv, scratchDir, index); + const childEnv = buildChildEnv(entry, baseEnv, scratchDir, index, { + serial: concurrency === 1, + }); const code = await runner(args, childEnv, entry.name); if (code !== 0) { // Stop scheduling new plans after a failure; the in-flight sibling diff --git a/scripts/lib/ci-node-test-plan.mjs b/scripts/lib/ci-node-test-plan.mjs index 587389979577..d1e01ee752d4 100644 --- a/scripts/lib/ci-node-test-plan.mjs +++ b/scripts/lib/ci-node-test-plan.mjs @@ -33,88 +33,91 @@ const COMPACT_NODE_TEST_JOB_GROUPS = 10; const COMPACT_TOOLING_NODE_TEST_GROUPS = 4; const COMPACT_WHOLE_NODE_TEST_TIMEOUT_MINUTES = 120; const AUTO_REPLY_COMMANDS_STRIPES = 3; +const AGENTS_CORE_RUNNER_CLI_STRIPES = 3; // Advisory runtime estimates (seconds) per split shard, measured from a -// Blacksmith compact PR run under two-plans-per-job concurrency (run -// 29395969440). Packing only: a stale entry skews job balance but never +// Blacksmith compact PR run with serial plans and 2 Vitest workers (run +// 29481835688). Packing only: a stale entry skews job balance but never // correctness. Unknown shards fall back to a per-file estimate. const COMPACT_GROUP_SECONDS_HINTS = new Map([ - ["agentic-agents-core-auth", 28], - ["agentic-agents-core-isolated", 12], - ["agentic-agents-core-models", 48], - ["agentic-agents-core-runner-cli", 120], - ["agentic-agents-core-runner-commands", 38], - ["agentic-agents-core-runner-embedded", 27], - ["agentic-agents-core-runner-sessions", 120], + ["agentic-agents-core-auth", 32], + ["agentic-agents-core-isolated", 10], + ["agentic-agents-core-models", 66], + ["agentic-agents-core-runner-cli-1", 110], + ["agentic-agents-core-runner-cli-2", 110], + ["agentic-agents-core-runner-cli-3", 110], + ["agentic-agents-core-runner-commands", 30], + ["agentic-agents-core-runner-embedded", 20], + ["agentic-agents-core-runner-sessions", 18], ["agentic-agents-core-runtime", 81], - ["agentic-agents-core-subagents", 30], - ["agentic-agents-core-tools", 41], - ["agentic-agents-embedded", 69], - ["agentic-agents-support", 104], + ["agentic-agents-core-subagents", 37], + ["agentic-agents-core-tools", 55], + ["agentic-agents-embedded", 59], + ["agentic-agents-support", 88], ["agentic-agents-tools", 43], - ["agentic-cli", 70], - ["agentic-command-support", 30], - ["agentic-commands-agent-channel", 43], - ["agentic-commands-doctor", 26], - ["agentic-commands-doctor-auth", 8], - ["agentic-commands-doctor-config-state", 58], - ["agentic-commands-doctor-gateway", 6], - ["agentic-commands-doctor-plugins-tools", 14], - ["agentic-commands-doctor-sessions-cron", 22], - ["agentic-commands-models", 17], - ["agentic-commands-onboard-config", 25], - ["agentic-commands-status-tools", 19], - ["agentic-control-plane-agent-chat", 74], - ["agentic-control-plane-auth-node", 105], - ["agentic-control-plane-http-models", 34], - ["agentic-control-plane-http-plugin-ws", 28], - ["agentic-control-plane-runtime-config", 32], - ["agentic-control-plane-runtime-cron", 22], - ["agentic-control-plane-runtime-server", 40], - ["agentic-control-plane-runtime-state", 29], - ["agentic-control-plane-runtime-ui-tools", 23], - ["agentic-control-plane-startup-core", 85], - ["agentic-control-plane-startup-health-runtime", 20], - ["agentic-control-plane-startup-restart-close", 19], - ["agentic-gateway-core", 112], - ["agentic-gateway-methods", 90], - ["agentic-plugin-sdk", 52], - ["auto-reply-core-top-level", 26], - ["auto-reply-reply-agent-runner", 33], - ["auto-reply-reply-commands-1", 179], - ["auto-reply-reply-commands-2", 39], - ["auto-reply-reply-commands-3", 28], - ["auto-reply-reply-dispatch", 41], - ["auto-reply-reply-session", 38], - ["auto-reply-reply-state-routing", 22], - ["core-runtime-cron-core", 29], - ["core-runtime-cron-isolated-agent", 55], - ["core-runtime-cron-service", 13], + ["agentic-cli", 81], + ["agentic-command-support", 28], + ["agentic-commands-agent-channel", 47], + ["agentic-commands-doctor", 23], + ["agentic-commands-doctor-auth", 10], + ["agentic-commands-doctor-config-state", 41], + ["agentic-commands-doctor-gateway", 7], + ["agentic-commands-doctor-plugins-tools", 10], + ["agentic-commands-doctor-sessions-cron", 24], + ["agentic-commands-models", 21], + ["agentic-commands-onboard-config", 15], + ["agentic-commands-status-tools", 30], + ["agentic-control-plane-agent-chat", 75], + ["agentic-control-plane-auth-node", 101], + ["agentic-control-plane-http-models", 40], + ["agentic-control-plane-http-plugin-ws", 43], + ["agentic-control-plane-runtime-config", 28], + ["agentic-control-plane-runtime-cron", 29], + ["agentic-control-plane-runtime-server", 26], + ["agentic-control-plane-runtime-state", 25], + ["agentic-control-plane-runtime-ui-tools", 24], + ["agentic-control-plane-startup-core", 113], + ["agentic-control-plane-startup-health-runtime", 24], + ["agentic-control-plane-startup-restart-close", 20], + ["agentic-gateway-core", 130], + ["agentic-gateway-methods", 79], + ["agentic-plugin-sdk", 50], + ["auto-reply-core-top-level", 33], + ["auto-reply-reply-agent-runner", 52], + ["auto-reply-reply-commands-1", 220], + ["auto-reply-reply-commands-2", 33], + ["auto-reply-reply-commands-3", 24], + ["auto-reply-reply-dispatch", 36], + ["auto-reply-reply-session", 21], + ["auto-reply-reply-state-routing", 21], + ["core-runtime-cron-core", 24], + ["core-runtime-cron-isolated-agent", 61], + ["core-runtime-cron-service", 24], ["core-runtime-hooks", 11], - ["core-runtime-infra-approval-exec", 35], - ["core-runtime-infra-channel-plugin", 20], - ["core-runtime-infra-heartbeat-runner", 44], - ["core-runtime-infra-net-install", 15], - ["core-runtime-infra-outbound-actions", 15], - ["core-runtime-infra-outbound-core", 37], - ["core-runtime-infra-process", 75], - ["core-runtime-infra-provider-push", 21], - ["core-runtime-infra-storage-state", 41], - ["core-runtime-infra-system-runtime", 29], - ["core-runtime-media-ui", 123], - ["core-runtime-secrets", 30], - ["core-runtime-shared", 49], + ["core-runtime-infra-approval-exec", 30], + ["core-runtime-infra-channel-plugin", 26], + ["core-runtime-infra-heartbeat-runner", 77], + ["core-runtime-infra-net-install", 14], + ["core-runtime-infra-outbound-actions", 21], + ["core-runtime-infra-outbound-core", 44], + ["core-runtime-infra-process", 83], + ["core-runtime-infra-provider-push", 20], + ["core-runtime-infra-storage-state", 58], + ["core-runtime-infra-system-runtime", 40], + ["core-runtime-media-ui", 131], + ["core-runtime-secrets", 43], + ["core-runtime-shared", 48], // PTY timing tests inflate badly next to co-runners; keep this group in a // lightly packed bin so its lane stays close to solo runtime. ["core-runtime-tui-pty", 200], - ["core-tooling-1", 88], - ["core-tooling-2", 84], - ["core-tooling-3", 100], - ["core-tooling-4", 79], + ["core-tooling-1", 113], + ["core-tooling-2", 111], + ["core-tooling-3", 134], + ["core-tooling-4", 94], ["core-tooling-docker", 6], - ["core-tooling-isolated", 50], - ["core-unit-fast", 157], - ["core-unit-src-security", 149], - ["core-unit-support", 28], + ["core-tooling-isolated", 55], + ["core-unit-fast", 190], + ["core-unit-src-security", 138], + ["core-unit-support", 19], ]); const DEFAULT_WHOLE_GROUP_SECONDS = 25; const DEFAULT_SECONDS_PER_TEST_FILE = 0.5; @@ -130,6 +133,21 @@ function isExclusiveCompactGroup(group) { return EXCLUSIVE_COMPACT_GROUP_RE.test(group.shard_name); } +// Spawn/signal/PTY-timing suites also flake under high in-process worker +// counts; pin them to the proven 2-worker budget while the job-level default +// scales with the runner class. infra-process spawns child processes per test +// and hit worker-startup timeouts under contention before serialization. +const PINNED_WORKER_COMPACT_GROUP_RE = + /^core-tooling(?:-\d+|-isolated|-docker)?$|^core-runtime-tui-pty$|^core-runtime-infra-process$|^core-runtime-media-ui$/u; +const PINNED_COMPACT_GROUP_ENV = { OPENCLAW_VITEST_MAX_WORKERS: "2" }; + +function applyCompactGroupWorkerPins(group) { + if (!PINNED_WORKER_COMPACT_GROUP_RE.test(group.shard_name)) { + return group; + } + return { ...group, env: { ...group.env, ...PINNED_COMPACT_GROUP_ENV } }; +} + function estimateCompactGroupSeconds(group) { const hint = COMPACT_GROUP_SECONDS_HINTS.get(group.shard_name); if (hint !== undefined) { @@ -161,7 +179,9 @@ const KEEP_LARGE_NODE_TEST_RUNNER = new Set([ "agentic-agents-core-subagents", "agentic-agents-embedded", "agentic-agents-support", - "agentic-agents-core-runner-cli", + "agentic-agents-core-runner-cli-1", + "agentic-agents-core-runner-cli-2", + "agentic-agents-core-runner-cli-3", "agentic-agents-core-runner-commands", "agentic-agents-core-runner-embedded", "agentic-agents-core-runner-sessions", @@ -170,8 +190,15 @@ const KEEP_LARGE_NODE_TEST_RUNNER = new Set([ "agentic-gateway-core", "agentic-gateway-methods", "auto-reply-reply-dispatch", + // The commands stripes and security suite are import-bound (30-45s of + // module-graph import per file); the 8 vCPU class with a higher Vitest + // worker budget cuts their wall clock roughly linearly. + "auto-reply-reply-commands-1", + "auto-reply-reply-commands-2", + "auto-reply-reply-commands-3", "core-runtime-media-ui", "core-unit-fast", + "core-unit-src-security", ]); const RELEASE_ONLY_PLUGIN_SHARDS = new Set(["agentic-plugins"]); function listTestFiles(rootDir) { @@ -439,12 +466,30 @@ function createAgentCoreSplitShards() { "agentic-agents-core-runner-sessions", "agentic-agents-core-runtime", ] - .map((shardName) => ({ - configs: ["test/vitest/vitest.agents-core.config.ts"], - includePatterns: groups.get(shardName) ?? [], - requiresDist: false, - shardName, - })) + .flatMap((shardName) => { + const includePatterns = groups.get(shardName) ?? []; + // agents-core runs files serially (fileParallelism false guards shared + // module state), so the import-heavy cli-runner suite (~35s of module + // import per file) stripes across bins to parallelize at the job level. + if (shardName === "agentic-agents-core-runner-cli") { + return createStripedBatches(includePatterns, AGENTS_CORE_RUNNER_CLI_STRIPES).map( + (batch, index) => ({ + configs: ["test/vitest/vitest.agents-core.config.ts"], + includePatterns: batch, + requiresDist: false, + shardName: `${shardName}-${index + 1}`, + }), + ); + } + return [ + { + configs: ["test/vitest/vitest.agents-core.config.ts"], + includePatterns, + requiresDist: false, + shardName, + }, + ]; + }) .filter((shard) => shard.includePatterns.length > 0); return [ @@ -1330,7 +1375,7 @@ function createCompactNodeTestShardBundles(options = {}) { runner, shard_name: shard.shardName, }; - groups.push(...expandCompactNodeTestGroup(group)); + groups.push(...expandCompactNodeTestGroup(group).map(applyCompactGroupWorkerPins)); groupsByRunner.set(key, groups); } diff --git a/test/scripts/ci-node-test-plan.test.ts b/test/scripts/ci-node-test-plan.test.ts index c073ed90de99..3aeab7e50089 100644 --- a/test/scripts/ci-node-test-plan.test.ts +++ b/test/scripts/ci-node-test-plan.test.ts @@ -241,6 +241,9 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => { .find((group) => group.shard_name === "core-runtime-tui-pty")?.env, ).toEqual({ OPENCLAW_TUI_PTY_INCLUDE_LOCAL: "1", + // Timing-sensitive groups pin the worker budget while the job-level + // default scales with the runner class. + OPENCLAW_VITEST_MAX_WORKERS: "2", }); const startupCoreJob = compact.find((shard) => shard.groups.some((group) => group.shard_name === "agentic-control-plane-startup-core"), @@ -874,18 +877,36 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => { runner: DEFAULT_NODE_TEST_RUNNER, shardName: "agentic-agents-core-subagents", }, + // cli-runner stripes: agents-core runs files serially, so the + // import-heavy suite splits across jobs to parallelize at bin level. { - checkName: "checks-node-agentic-agents-core-runner-cli", + checkName: "checks-node-agentic-agents-core-runner-cli-1", configs: ["test/vitest/vitest.agents-core.config.ts"], includePatterns: agentShards[4]?.includePatterns, requiresDist: false, runner: DEFAULT_NODE_TEST_RUNNER, - shardName: "agentic-agents-core-runner-cli", + shardName: "agentic-agents-core-runner-cli-1", + }, + { + checkName: "checks-node-agentic-agents-core-runner-cli-2", + configs: ["test/vitest/vitest.agents-core.config.ts"], + includePatterns: agentShards[5]?.includePatterns, + requiresDist: false, + runner: DEFAULT_NODE_TEST_RUNNER, + shardName: "agentic-agents-core-runner-cli-2", + }, + { + checkName: "checks-node-agentic-agents-core-runner-cli-3", + configs: ["test/vitest/vitest.agents-core.config.ts"], + includePatterns: agentShards[6]?.includePatterns, + requiresDist: false, + runner: DEFAULT_NODE_TEST_RUNNER, + shardName: "agentic-agents-core-runner-cli-3", }, { checkName: "checks-node-agentic-agents-core-runner-commands", configs: ["test/vitest/vitest.agents-core.config.ts"], - includePatterns: agentShards[5]?.includePatterns, + includePatterns: agentShards[7]?.includePatterns, requiresDist: false, runner: DEFAULT_NODE_TEST_RUNNER, shardName: "agentic-agents-core-runner-commands", @@ -893,7 +914,7 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => { { checkName: "checks-node-agentic-agents-core-runner-embedded", configs: ["test/vitest/vitest.agents-core.config.ts"], - includePatterns: agentShards[6]?.includePatterns, + includePatterns: agentShards[8]?.includePatterns, requiresDist: false, runner: DEFAULT_NODE_TEST_RUNNER, shardName: "agentic-agents-core-runner-embedded", @@ -901,7 +922,7 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => { { checkName: "checks-node-agentic-agents-core-runner-sessions", configs: ["test/vitest/vitest.agents-core.config.ts"], - includePatterns: agentShards[7]?.includePatterns, + includePatterns: agentShards[9]?.includePatterns, requiresDist: false, runner: DEFAULT_NODE_TEST_RUNNER, shardName: "agentic-agents-core-runner-sessions", @@ -909,7 +930,7 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => { { checkName: "checks-node-agentic-agents-core-runtime", configs: ["test/vitest/vitest.agents-core.config.ts"], - includePatterns: agentShards[8]?.includePatterns, + includePatterns: agentShards[10]?.includePatterns, requiresDist: false, runner: DEFAULT_NODE_TEST_RUNNER, shardName: "agentic-agents-core-runtime", @@ -917,7 +938,7 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => { { checkName: "checks-node-agentic-agents-core-isolated", configs: ["test/vitest/vitest.agents-core-isolated.config.ts"], - includePatterns: agentShards[9]?.includePatterns, + includePatterns: agentShards[11]?.includePatterns, requiresDist: false, runner: DEFAULT_NODE_TEST_RUNNER, shardName: "agentic-agents-core-isolated",