From 016c5aa772d74ebb47313a31dd236a7d4a622ead Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 12 Aug 2026 00:48:14 -0700 Subject: [PATCH] ci: balance Control UI E2E shards by source size (#122527) Co-authored-by: Amp --- docs/ci.md | 1 + test/vitest-ui-e2e-config.test.ts | 56 ++++++++++++++++++++++++++ test/vitest/vitest.ui-e2e.config.ts | 3 ++ test/vitest/vitest.ui-e2e.sequencer.ts | 37 +++++++++++++++++ 4 files changed, 97 insertions(+) create mode 100644 test/vitest-ui-e2e-config.test.ts create mode 100644 test/vitest/vitest.ui-e2e.sequencer.ts diff --git a/docs/ci.md b/docs/ci.md index cdbdbb7280a9..67cd1137dd5e 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -121,6 +121,7 @@ The slowest Node test families are split or balanced so each job stays small wit - Normal CI packs only isolated infra include-pattern shards into deterministic bundles of at most 64 test files, reducing the Node matrix without merging non-isolated command/cron, stateful agents-core, or gateway/server suites. Heavy fixed suites stay on 8 vCPU while the bundled and lower-weight lanes use 4 vCPU. - Pull requests on the canonical repository reuse the changed-test resolver against the synthetic merged-tree diff. Precise changes run one targeted Node job; each selected test file gets its own process so stateful suite isolation remains intact. The planner combines sibling tests with import-graph dependents and falls back to the existing 14-job compact full-suite plan for workspace package, package/lockfile, shared harness, split-config, renamed, or deleted changes, public extension-contract changes, tests with special shard setup, partially resolved or empty targets, oversized path or target plans, and planner errors. Targeted plans always retain the full built-artifact boundary gate because its repository scanners cannot be derived from imports. `main` pushes run the same full compact suite: pending intermediate push events can be coalesced, so the newest surviving run must validate the complete integration tree rather than only its final single-push diff. Manual dispatches and release gates retain the full named per-shard matrix. Compact packing uses fleet timing hints without changing the bounded job count; the high-variance source/security group remains isolated so its tail does not serialize unrelated groups. - The full Node matrix admits the consistently slow serial tooling, auto-reply command shards, and broad core-fast cache writer first. This keeps the 28-job cap while preventing critical-path work and the next run's transform seed from slipping into a later wave. +- The three serial Control UI browser shards greedily pack discovered test files by source byte size. This zero-state duration proxy avoids Vitest's equal-file-count hash clustering, automatically accounts for new and changed files, and preserves the same complete test inventory without adding runners. - Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. Include-pattern shards record timing entries using the CI shard name, so `.artifacts/vitest-shard-timings.json` can distinguish a whole config from a filtered shard. - Linux Node shard jobs persist Vitest's experimental filesystem module cache through the upstream Actions cache API, which Blacksmith transparently accelerates on its runners. Every CI shard is restore-only and unpacks the protected seed into its own runner-local root; the shard wrapper then gives concurrent Vitest processes separate live subdirectories. Only the non-cancelling daily or explicitly dispatched warmer saves a new immutable archive, so pull requests cannot publish transforms or mint per-PR cache families. The warmer launches each selected shard/config envelope in a fresh child process with concurrency one, preserving its include patterns and environment while reusing the same serial cache leaf. This prevents config-global state from leaking, avoids expanding filtered shards into whole configs, and retains transforms produced by the previous child. A transform-input fingerprint clears incompatible lockfile, package, tsconfig, and Vitest-config generations. The protected writer scans and prunes its restored cache to 75% after it exceeds 2 GiB. Vitest hashes module id, source content, environment, and resolved transform config, so ordinary partial source changes keep unchanged entries warm while changed modules miss safely. Coarse restore prefixes bridge workflow runs; normal Actions cache LRU and inactivity eviction bound old immutable archives. - Trusted Blacksmith Linux Node jobs also bind the pnpm store and `node_modules` from one protected dependency disk per supported Node line. GitHub-hosted jobs, including manual dispatches, fork pull requests, and same-repo retries of both UI E2E jobs, use the Actions cache path instead. Package manifests, install settings, runner platform, and the exact Node patch stay out of the disk key; an exact runtime and install-input fingerprint decides whether a job reuses the tree or reinstalls and refreshes the same disk. Manifests are canonicalized before hashing. The audited direct root hooks retain only pnpm's install lifecycle scripts, so formatting and ordinary test/build script edits keep the warm dependency tree; unaudited lifecycle-hook drift fails closed until its source inputs join the fingerprint contract. Dependency, package-manager, hook-source, and lockfile changes always invalidate the snapshot. A matching fingerprint is necessary but not sufficient: setup also checks the importer archive and manifest checksums, then verifies registry-backed lockfile dependencies retained by postinstall against the package manifests Node resolves from their importers. Missing or stale importer content falls back to a fresh install instead of serving the root hoist. A pull request whose read-only snapshot is unusable detaches the workspace bind and installs into runner-local storage, avoiding slow writes to a clone it cannot publish. Sticky cold installs disable pnpm's inner fetch retries and make up to three bounded full-install attempts from the progressively warmed store; a timeout remains a failure. After a content-validated restore or frozen-lockfile install, setup disables pnpm's redundant pre-run dependency check: the repository intentionally prunes plugin-local `node_modules`, which pnpm otherwise treats as stale and repairs through unsafe concurrent implicit installs during shard fanout. Canonical main preflight is the sole writer and measures the store on every refresh, running `pnpm store prune` only after retired package versions push it above 8 GiB. Validated warm restores no longer publish no-op snapshots: the writer uses StickyDisk's allocation-change mode, records its mount-time allocation baseline, and only a successful dependency capture creates a runner-local rebuild signal. After store pruning, preflight compares the final whole-disk allocation to that baseline and, when needed, allocates a bounded sentinel until the absolute delta has a verified 64 KiB margin over StickyDisk's 4 KiB threshold. Blacksmith snapshot publication is asynchronous even after a writer job completes, so the first run after a fresh key or fingerprint can remain cold; later content-validated exact-marker restores are the rollout proof. Required Blacksmith CI jobs and first-attempt same-repo pull requests get disposable clones, so dependency changes do not create new disks, competing snapshots, or a cache lock that can cancel builds. diff --git a/test/vitest-ui-e2e-config.test.ts b/test/vitest-ui-e2e-config.test.ts new file mode 100644 index 000000000000..b95705316a41 --- /dev/null +++ b/test/vitest-ui-e2e-config.test.ts @@ -0,0 +1,56 @@ +// Vitest UI E2E config tests protect complete, size-balanced browser sharding. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import type { TestSpecification } from "vitest/node"; +import uiE2eConfig from "./vitest/vitest.ui-e2e.config.ts"; +import { UiE2eSequencer } from "./vitest/vitest.ui-e2e.sequencer.ts"; + +const tempDirs: string[] = []; + +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + fs.rmSync(dir, { force: true, recursive: true }); + } +}); + +function requireTestConfig(config: unknown): { + sequence?: { sequencer?: unknown }; +} { + if (!config || typeof config !== "object" || !("test" in config) || !config.test) { + throw new Error("expected UI E2E Vitest test config"); + } + return config.test as { sequence?: { sequencer?: unknown } }; +} + +async function shardFiles(files: TestSpecification[], index: number, count: number) { + const sequencer = new UiE2eSequencer({ config: { shard: { count, index } } } as never); + return sequencer.shard(files); +} + +describe("Control UI E2E Vitest sharding", () => { + it("uses the source-size weighted sequencer", () => { + expect(requireTestConfig(uiE2eConfig).sequence?.sequencer).toBe(UiE2eSequencer); + }); + + it("covers every file once while balancing source bytes", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-ui-e2e-shards-")); + tempDirs.push(tempDir); + const files = [600, 500, 400, 300, 200, 100].map((bytes, index) => { + const moduleId = path.join(tempDir, `suite-${index}.e2e.test.ts`); + fs.writeFileSync(moduleId, "x".repeat(bytes)); + return { moduleId } as TestSpecification; + }); + + const shards = await Promise.all([1, 2, 3].map((index) => shardFiles(files, index, 3))); + const assignedFiles = shards.flat().map((file) => file.moduleId); + const assignedBytes = shards.map((shard) => + shard.reduce((total, file) => total + fs.statSync(file.moduleId).size, 0), + ); + + expect(assignedFiles.toSorted()).toEqual(files.map((file) => file.moduleId).toSorted()); + expect(new Set(assignedFiles).size).toBe(files.length); + expect(assignedBytes).toEqual([700, 700, 700]); + }); +}); diff --git a/test/vitest/vitest.ui-e2e.config.ts b/test/vitest/vitest.ui-e2e.config.ts index 730d8147930d..e1713e019e5e 100644 --- a/test/vitest/vitest.ui-e2e.config.ts +++ b/test/vitest/vitest.ui-e2e.config.ts @@ -2,6 +2,7 @@ import { defineConfig } from "vitest/config"; import { loadPatternListFromEnv, narrowIncludePatternsForCli } from "./vitest.pattern-file.ts"; import { sharedVitestConfig } from "./vitest.shared.config.ts"; +import { UiE2eSequencer } from "./vitest.ui-e2e.sequencer.ts"; const uiE2eIncludePatterns = ["ui/src/**/*.e2e.test.ts"]; const uiE2eRealGatewayTestFiles = [ @@ -15,6 +16,7 @@ function createUiE2eVitestConfig( ) { const base = sharedVitestConfig as Record; const baseTest = sharedVitestConfig.test ?? {}; + const baseSequence = (baseTest as { sequence?: object }).sequence; const exclude = [ ...(baseTest.exclude ?? []).filter((pattern) => pattern !== "**/*.e2e.test.ts"), ...(env.OPENCLAW_UI_E2E_SKIP_REAL_GATEWAY === "1" ? uiE2eRealGatewayTestFiles : []), @@ -44,6 +46,7 @@ function createUiE2eVitestConfig( name: "ui-e2e", pool: "forks", runner: undefined, + sequence: { ...baseSequence, sequencer: UiE2eSequencer }, setupFiles: ["test/vitest/vitest.ui-e2e.setup.ts"], }, }); diff --git a/test/vitest/vitest.ui-e2e.sequencer.ts b/test/vitest/vitest.ui-e2e.sequencer.ts new file mode 100644 index 000000000000..2cad4b71291d --- /dev/null +++ b/test/vitest/vitest.ui-e2e.sequencer.ts @@ -0,0 +1,37 @@ +// Source-size weighted sharding keeps serial Control UI E2E runners from +// clustering the largest browser suites behind Vitest's equal-file-count hash. +import { statSync } from "node:fs"; +import { BaseSequencer, type TestSpecification } from "vitest/node"; + +type ShardBucket = { + bytes: number; + files: TestSpecification[]; +}; + +export class UiE2eSequencer extends BaseSequencer { + override async shard(files: TestSpecification[]): Promise { + // Vitest invokes shard() only when config.shard is present. File size is a + // zero-state duration proxy, so new and changed tests rebalance automatically. + const { count, index } = this.ctx.config.shard!; + const buckets: ShardBucket[] = Array.from({ length: count }, () => ({ + bytes: 0, + files: [], + })); + const weightedFiles = files + .map((file) => ({ bytes: statSync(file.moduleId).size, file })) + .sort( + (left, right) => + right.bytes - left.bytes || left.file.moduleId.localeCompare(right.file.moduleId), + ); + + for (const weightedFile of weightedFiles) { + const bucket = buckets.reduce((lightest, candidate) => + candidate.bytes < lightest.bytes ? candidate : lightest, + ); + bucket.bytes += weightedFile.bytes; + bucket.files.push(weightedFile.file); + } + + return buckets[index - 1]!.files; + } +}