perf(ci): scale Vitest workers with runner class and share serial module cache (#108788)

* perf(ci): scale Vitest workers with runner class and share serial module cache

Compact node-test jobs pinned OPENCLAW_VITEST_MAX_WORKERS=2 regardless of
runner size, so import-bound suites (30-45s of module-graph import per
file) crawled: runner-cli spent 284s on 5s of tests, commands-1 296s on
10s. With serial plans the budget now scales with the class (6 on 8vcpu,
3 on 4vcpu); timing-sensitive groups (tooling, tui-pty, infra-process)
stay pinned to 2 via plan-level env. Serial bins share one Vitest
fs-module-cache path so later plans reuse the first plan's transforms.
The import-bound commands stripes and security suite move to the 8vcpu
class, and packing hints refresh from serial-run measurements (run
29481835688).

* perf(ci): stripe cli-runner suite and pin media-ui worker budget

The agents-core config runs files serially (fileParallelism false guards
shared module state), so raising the worker budget cannot help its
import-heavy cli-runner suite (~35s module import per file, 213s serial).
Stripe it three ways so bins parallelize the imports instead. media-ui
hosts browser locator tests that timed out at 6 workers on the first
validation run; pin it to the proven 2-worker budget. commands-1's hint
drops to its measured 6-worker runtime.
This commit is contained in:
Peter Steinberger
2026-07-16 03:36:37 -07:00
committed by GitHub
parent 7bc43f50b7
commit 2c7cf8d745
5 changed files with 178 additions and 93 deletions
+12 -1
View File
@@ -1720,7 +1720,18 @@ jobs:
run: test "$(go env GOVERSION)" = "go1.25.12"
- name: Configure Node test resources
run: echo "OPENCLAW_VITEST_MAX_WORKERS=2" >> "$GITHUB_ENV"
# Scale the in-process Vitest worker budget with the runner class. Bins
# run one Vitest process at a time (planConcurrency 1), so workers are
# the only in-job parallelism; import-bound suites scale near-linearly.
# Timing-sensitive groups stay pinned to 2 via plan-level env.
env:
SHARD_RUNNER: ${{ matrix.runner || 'blacksmith-4vcpu-ubuntu-2404' }}
run: |
if [[ "$SHARD_RUNNER" == *"-8vcpu-"* ]]; then
echo "OPENCLAW_VITEST_MAX_WORKERS=6" >> "$GITHUB_ENV"
else
echo "OPENCLAW_VITEST_MAX_WORKERS=3" >> "$GITHUB_ENV"
fi
- name: Run Node test shard
# actionlint 1.7.11 lacks GitHub's current job.workflow_* fields. Serialize the
+1
View File
@@ -24,6 +24,7 @@ export function buildChildEnv(
baseEnv: Record<string, string | undefined>,
scratchDir: string,
index: number,
options?: { serial?: boolean },
): Record<string, string | undefined>;
export function resolveShardChildCommand(
+12 -5
View File
@@ -50,12 +50,17 @@ export function resolveShardPlans(env = process.env) {
}));
}
export function buildChildEnv(entry, baseEnv, scratchDir, index) {
export function buildChildEnv(entry, baseEnv, scratchDir, index, options = {}) {
const childEnv = {
...baseEnv,
// Concurrent children must not share a Vitest module cache directory;
// shared caches race with ENOTEMPTY when two runs rewrite the same entries.
[FS_MODULE_CACHE_PATH_ENV_KEY]: join(scratchDir, `vitest-cache-${index}`),
// Concurrent children must not share a Vitest module cache directory
// (shared caches race with ENOTEMPTY when two runs rewrite the same
// entries), but serial bins reuse one cache so later plans skip the cold
// module transforms the first plan already paid for.
[FS_MODULE_CACHE_PATH_ENV_KEY]: join(
scratchDir,
options.serial ? "vitest-cache-shared" : `vitest-cache-${index}`,
),
OPENCLAW_TEST_PROJECTS_PARALLEL: "1",
// This wrapper holds the repo heavy-check lock; children skipping it is
// what lets two plans run concurrently instead of serializing on the lock.
@@ -173,7 +178,9 @@ export async function runShardPlans(plans, options = {}) {
exitCode = exitCode || 1;
return;
}
const childEnv = buildChildEnv(entry, baseEnv, scratchDir, index);
const childEnv = buildChildEnv(entry, baseEnv, scratchDir, index, {
serial: concurrency === 1,
});
const code = await runner(args, childEnv, entry.name);
if (code !== 0) {
// Stop scheduling new plans after a failure; the in-flight sibling
+125 -80
View File
@@ -33,88 +33,91 @@ const COMPACT_NODE_TEST_JOB_GROUPS = 10;
const COMPACT_TOOLING_NODE_TEST_GROUPS = 4;
const COMPACT_WHOLE_NODE_TEST_TIMEOUT_MINUTES = 120;
const AUTO_REPLY_COMMANDS_STRIPES = 3;
const AGENTS_CORE_RUNNER_CLI_STRIPES = 3;
// Advisory runtime estimates (seconds) per split shard, measured from a
// Blacksmith compact PR run under two-plans-per-job concurrency (run
// 29395969440). Packing only: a stale entry skews job balance but never
// Blacksmith compact PR run with serial plans and 2 Vitest workers (run
// 29481835688). Packing only: a stale entry skews job balance but never
// correctness. Unknown shards fall back to a per-file estimate.
const COMPACT_GROUP_SECONDS_HINTS = new Map([
["agentic-agents-core-auth", 28],
["agentic-agents-core-isolated", 12],
["agentic-agents-core-models", 48],
["agentic-agents-core-runner-cli", 120],
["agentic-agents-core-runner-commands", 38],
["agentic-agents-core-runner-embedded", 27],
["agentic-agents-core-runner-sessions", 120],
["agentic-agents-core-auth", 32],
["agentic-agents-core-isolated", 10],
["agentic-agents-core-models", 66],
["agentic-agents-core-runner-cli-1", 110],
["agentic-agents-core-runner-cli-2", 110],
["agentic-agents-core-runner-cli-3", 110],
["agentic-agents-core-runner-commands", 30],
["agentic-agents-core-runner-embedded", 20],
["agentic-agents-core-runner-sessions", 18],
["agentic-agents-core-runtime", 81],
["agentic-agents-core-subagents", 30],
["agentic-agents-core-tools", 41],
["agentic-agents-embedded", 69],
["agentic-agents-support", 104],
["agentic-agents-core-subagents", 37],
["agentic-agents-core-tools", 55],
["agentic-agents-embedded", 59],
["agentic-agents-support", 88],
["agentic-agents-tools", 43],
["agentic-cli", 70],
["agentic-command-support", 30],
["agentic-commands-agent-channel", 43],
["agentic-commands-doctor", 26],
["agentic-commands-doctor-auth", 8],
["agentic-commands-doctor-config-state", 58],
["agentic-commands-doctor-gateway", 6],
["agentic-commands-doctor-plugins-tools", 14],
["agentic-commands-doctor-sessions-cron", 22],
["agentic-commands-models", 17],
["agentic-commands-onboard-config", 25],
["agentic-commands-status-tools", 19],
["agentic-control-plane-agent-chat", 74],
["agentic-control-plane-auth-node", 105],
["agentic-control-plane-http-models", 34],
["agentic-control-plane-http-plugin-ws", 28],
["agentic-control-plane-runtime-config", 32],
["agentic-control-plane-runtime-cron", 22],
["agentic-control-plane-runtime-server", 40],
["agentic-control-plane-runtime-state", 29],
["agentic-control-plane-runtime-ui-tools", 23],
["agentic-control-plane-startup-core", 85],
["agentic-control-plane-startup-health-runtime", 20],
["agentic-control-plane-startup-restart-close", 19],
["agentic-gateway-core", 112],
["agentic-gateway-methods", 90],
["agentic-plugin-sdk", 52],
["auto-reply-core-top-level", 26],
["auto-reply-reply-agent-runner", 33],
["auto-reply-reply-commands-1", 179],
["auto-reply-reply-commands-2", 39],
["auto-reply-reply-commands-3", 28],
["auto-reply-reply-dispatch", 41],
["auto-reply-reply-session", 38],
["auto-reply-reply-state-routing", 22],
["core-runtime-cron-core", 29],
["core-runtime-cron-isolated-agent", 55],
["core-runtime-cron-service", 13],
["agentic-cli", 81],
["agentic-command-support", 28],
["agentic-commands-agent-channel", 47],
["agentic-commands-doctor", 23],
["agentic-commands-doctor-auth", 10],
["agentic-commands-doctor-config-state", 41],
["agentic-commands-doctor-gateway", 7],
["agentic-commands-doctor-plugins-tools", 10],
["agentic-commands-doctor-sessions-cron", 24],
["agentic-commands-models", 21],
["agentic-commands-onboard-config", 15],
["agentic-commands-status-tools", 30],
["agentic-control-plane-agent-chat", 75],
["agentic-control-plane-auth-node", 101],
["agentic-control-plane-http-models", 40],
["agentic-control-plane-http-plugin-ws", 43],
["agentic-control-plane-runtime-config", 28],
["agentic-control-plane-runtime-cron", 29],
["agentic-control-plane-runtime-server", 26],
["agentic-control-plane-runtime-state", 25],
["agentic-control-plane-runtime-ui-tools", 24],
["agentic-control-plane-startup-core", 113],
["agentic-control-plane-startup-health-runtime", 24],
["agentic-control-plane-startup-restart-close", 20],
["agentic-gateway-core", 130],
["agentic-gateway-methods", 79],
["agentic-plugin-sdk", 50],
["auto-reply-core-top-level", 33],
["auto-reply-reply-agent-runner", 52],
["auto-reply-reply-commands-1", 220],
["auto-reply-reply-commands-2", 33],
["auto-reply-reply-commands-3", 24],
["auto-reply-reply-dispatch", 36],
["auto-reply-reply-session", 21],
["auto-reply-reply-state-routing", 21],
["core-runtime-cron-core", 24],
["core-runtime-cron-isolated-agent", 61],
["core-runtime-cron-service", 24],
["core-runtime-hooks", 11],
["core-runtime-infra-approval-exec", 35],
["core-runtime-infra-channel-plugin", 20],
["core-runtime-infra-heartbeat-runner", 44],
["core-runtime-infra-net-install", 15],
["core-runtime-infra-outbound-actions", 15],
["core-runtime-infra-outbound-core", 37],
["core-runtime-infra-process", 75],
["core-runtime-infra-provider-push", 21],
["core-runtime-infra-storage-state", 41],
["core-runtime-infra-system-runtime", 29],
["core-runtime-media-ui", 123],
["core-runtime-secrets", 30],
["core-runtime-shared", 49],
["core-runtime-infra-approval-exec", 30],
["core-runtime-infra-channel-plugin", 26],
["core-runtime-infra-heartbeat-runner", 77],
["core-runtime-infra-net-install", 14],
["core-runtime-infra-outbound-actions", 21],
["core-runtime-infra-outbound-core", 44],
["core-runtime-infra-process", 83],
["core-runtime-infra-provider-push", 20],
["core-runtime-infra-storage-state", 58],
["core-runtime-infra-system-runtime", 40],
["core-runtime-media-ui", 131],
["core-runtime-secrets", 43],
["core-runtime-shared", 48],
// PTY timing tests inflate badly next to co-runners; keep this group in a
// lightly packed bin so its lane stays close to solo runtime.
["core-runtime-tui-pty", 200],
["core-tooling-1", 88],
["core-tooling-2", 84],
["core-tooling-3", 100],
["core-tooling-4", 79],
["core-tooling-1", 113],
["core-tooling-2", 111],
["core-tooling-3", 134],
["core-tooling-4", 94],
["core-tooling-docker", 6],
["core-tooling-isolated", 50],
["core-unit-fast", 157],
["core-unit-src-security", 149],
["core-unit-support", 28],
["core-tooling-isolated", 55],
["core-unit-fast", 190],
["core-unit-src-security", 138],
["core-unit-support", 19],
]);
const DEFAULT_WHOLE_GROUP_SECONDS = 25;
const DEFAULT_SECONDS_PER_TEST_FILE = 0.5;
@@ -130,6 +133,21 @@ function isExclusiveCompactGroup(group) {
return EXCLUSIVE_COMPACT_GROUP_RE.test(group.shard_name);
}
// Spawn/signal/PTY-timing suites also flake under high in-process worker
// counts; pin them to the proven 2-worker budget while the job-level default
// scales with the runner class. infra-process spawns child processes per test
// and hit worker-startup timeouts under contention before serialization.
const PINNED_WORKER_COMPACT_GROUP_RE =
/^core-tooling(?:-\d+|-isolated|-docker)?$|^core-runtime-tui-pty$|^core-runtime-infra-process$|^core-runtime-media-ui$/u;
const PINNED_COMPACT_GROUP_ENV = { OPENCLAW_VITEST_MAX_WORKERS: "2" };
function applyCompactGroupWorkerPins(group) {
if (!PINNED_WORKER_COMPACT_GROUP_RE.test(group.shard_name)) {
return group;
}
return { ...group, env: { ...group.env, ...PINNED_COMPACT_GROUP_ENV } };
}
function estimateCompactGroupSeconds(group) {
const hint = COMPACT_GROUP_SECONDS_HINTS.get(group.shard_name);
if (hint !== undefined) {
@@ -161,7 +179,9 @@ const KEEP_LARGE_NODE_TEST_RUNNER = new Set([
"agentic-agents-core-subagents",
"agentic-agents-embedded",
"agentic-agents-support",
"agentic-agents-core-runner-cli",
"agentic-agents-core-runner-cli-1",
"agentic-agents-core-runner-cli-2",
"agentic-agents-core-runner-cli-3",
"agentic-agents-core-runner-commands",
"agentic-agents-core-runner-embedded",
"agentic-agents-core-runner-sessions",
@@ -170,8 +190,15 @@ const KEEP_LARGE_NODE_TEST_RUNNER = new Set([
"agentic-gateway-core",
"agentic-gateway-methods",
"auto-reply-reply-dispatch",
// The commands stripes and security suite are import-bound (30-45s of
// module-graph import per file); the 8 vCPU class with a higher Vitest
// worker budget cuts their wall clock roughly linearly.
"auto-reply-reply-commands-1",
"auto-reply-reply-commands-2",
"auto-reply-reply-commands-3",
"core-runtime-media-ui",
"core-unit-fast",
"core-unit-src-security",
]);
const RELEASE_ONLY_PLUGIN_SHARDS = new Set(["agentic-plugins"]);
function listTestFiles(rootDir) {
@@ -439,12 +466,30 @@ function createAgentCoreSplitShards() {
"agentic-agents-core-runner-sessions",
"agentic-agents-core-runtime",
]
.map((shardName) => ({
configs: ["test/vitest/vitest.agents-core.config.ts"],
includePatterns: groups.get(shardName) ?? [],
requiresDist: false,
shardName,
}))
.flatMap((shardName) => {
const includePatterns = groups.get(shardName) ?? [];
// agents-core runs files serially (fileParallelism false guards shared
// module state), so the import-heavy cli-runner suite (~35s of module
// import per file) stripes across bins to parallelize at the job level.
if (shardName === "agentic-agents-core-runner-cli") {
return createStripedBatches(includePatterns, AGENTS_CORE_RUNNER_CLI_STRIPES).map(
(batch, index) => ({
configs: ["test/vitest/vitest.agents-core.config.ts"],
includePatterns: batch,
requiresDist: false,
shardName: `${shardName}-${index + 1}`,
}),
);
}
return [
{
configs: ["test/vitest/vitest.agents-core.config.ts"],
includePatterns,
requiresDist: false,
shardName,
},
];
})
.filter((shard) => shard.includePatterns.length > 0);
return [
@@ -1330,7 +1375,7 @@ function createCompactNodeTestShardBundles(options = {}) {
runner,
shard_name: shard.shardName,
};
groups.push(...expandCompactNodeTestGroup(group));
groups.push(...expandCompactNodeTestGroup(group).map(applyCompactGroupWorkerPins));
groupsByRunner.set(key, groups);
}
+28 -7
View File
@@ -241,6 +241,9 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => {
.find((group) => group.shard_name === "core-runtime-tui-pty")?.env,
).toEqual({
OPENCLAW_TUI_PTY_INCLUDE_LOCAL: "1",
// Timing-sensitive groups pin the worker budget while the job-level
// default scales with the runner class.
OPENCLAW_VITEST_MAX_WORKERS: "2",
});
const startupCoreJob = compact.find((shard) =>
shard.groups.some((group) => group.shard_name === "agentic-control-plane-startup-core"),
@@ -874,18 +877,36 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => {
runner: DEFAULT_NODE_TEST_RUNNER,
shardName: "agentic-agents-core-subagents",
},
// cli-runner stripes: agents-core runs files serially, so the
// import-heavy suite splits across jobs to parallelize at bin level.
{
checkName: "checks-node-agentic-agents-core-runner-cli",
checkName: "checks-node-agentic-agents-core-runner-cli-1",
configs: ["test/vitest/vitest.agents-core.config.ts"],
includePatterns: agentShards[4]?.includePatterns,
requiresDist: false,
runner: DEFAULT_NODE_TEST_RUNNER,
shardName: "agentic-agents-core-runner-cli",
shardName: "agentic-agents-core-runner-cli-1",
},
{
checkName: "checks-node-agentic-agents-core-runner-cli-2",
configs: ["test/vitest/vitest.agents-core.config.ts"],
includePatterns: agentShards[5]?.includePatterns,
requiresDist: false,
runner: DEFAULT_NODE_TEST_RUNNER,
shardName: "agentic-agents-core-runner-cli-2",
},
{
checkName: "checks-node-agentic-agents-core-runner-cli-3",
configs: ["test/vitest/vitest.agents-core.config.ts"],
includePatterns: agentShards[6]?.includePatterns,
requiresDist: false,
runner: DEFAULT_NODE_TEST_RUNNER,
shardName: "agentic-agents-core-runner-cli-3",
},
{
checkName: "checks-node-agentic-agents-core-runner-commands",
configs: ["test/vitest/vitest.agents-core.config.ts"],
includePatterns: agentShards[5]?.includePatterns,
includePatterns: agentShards[7]?.includePatterns,
requiresDist: false,
runner: DEFAULT_NODE_TEST_RUNNER,
shardName: "agentic-agents-core-runner-commands",
@@ -893,7 +914,7 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => {
{
checkName: "checks-node-agentic-agents-core-runner-embedded",
configs: ["test/vitest/vitest.agents-core.config.ts"],
includePatterns: agentShards[6]?.includePatterns,
includePatterns: agentShards[8]?.includePatterns,
requiresDist: false,
runner: DEFAULT_NODE_TEST_RUNNER,
shardName: "agentic-agents-core-runner-embedded",
@@ -901,7 +922,7 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => {
{
checkName: "checks-node-agentic-agents-core-runner-sessions",
configs: ["test/vitest/vitest.agents-core.config.ts"],
includePatterns: agentShards[7]?.includePatterns,
includePatterns: agentShards[9]?.includePatterns,
requiresDist: false,
runner: DEFAULT_NODE_TEST_RUNNER,
shardName: "agentic-agents-core-runner-sessions",
@@ -909,7 +930,7 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => {
{
checkName: "checks-node-agentic-agents-core-runtime",
configs: ["test/vitest/vitest.agents-core.config.ts"],
includePatterns: agentShards[8]?.includePatterns,
includePatterns: agentShards[10]?.includePatterns,
requiresDist: false,
runner: DEFAULT_NODE_TEST_RUNNER,
shardName: "agentic-agents-core-runtime",
@@ -917,7 +938,7 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => {
{
checkName: "checks-node-agentic-agents-core-isolated",
configs: ["test/vitest/vitest.agents-core-isolated.config.ts"],
includePatterns: agentShards[9]?.includePatterns,
includePatterns: agentShards[11]?.includePatterns,
requiresDist: false,
runner: DEFAULT_NODE_TEST_RUNNER,
shardName: "agentic-agents-core-isolated",