diff --git a/.github/workflows/vitest-cache-warm.yml b/.github/workflows/vitest-cache-warm.yml index 9a344bf617b6..444f43e234cd 100644 --- a/.github/workflows/vitest-cache-warm.yml +++ b/.github/workflows/vitest-cache-warm.yml @@ -73,11 +73,14 @@ jobs: } const shards = [...coreShards, ...additionalShards]; const configs = [...new Set(shards.flatMap((shard) => shard.configs))]; + const groups = configs.map((config) => ({ + configs: [config], + shard_name: `cache-warm:${config}`, + })); appendFileSync( process.env.GITHUB_ENV, [ - `OPENCLAW_NODE_TEST_CONFIGS_JSON=${JSON.stringify(configs)}`, - "OPENCLAW_VITEST_SHARD_NAME=core-unit-fast", + `OPENCLAW_NODE_TEST_GROUPS_JSON=${JSON.stringify(groups)}`, "OPENCLAW_NODE_TEST_PLAN_CONCURRENCY=1", ].join("\n") + "\n", ); diff --git a/docs/ci.md b/docs/ci.md index 39f6fd96540d..01191ef8270d 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -115,7 +115,7 @@ The slowest Node test families are split or balanced so each job stays small wit - Pull requests on the canonical repository reuse the changed-test resolver against the synthetic merged-tree diff. Precise changes run one targeted Node job; each selected test file gets its own process so stateful suite isolation remains intact. The planner combines sibling tests with import-graph dependents and falls back to the existing 14-job compact full-suite plan for workspace package, package/lockfile, shared harness, split-config, renamed, or deleted changes, public extension-contract changes, tests with special shard setup, partially resolved or empty targets, oversized path or target plans, and planner errors. Targeted plans always retain the full built-artifact boundary gate because its repository scanners cannot be derived from imports. `main` pushes run the same full compact suite: pending intermediate push events can be coalesced, so the newest surviving run must validate the complete integration tree rather than only its final single-push diff. Manual dispatches and release gates retain the full named per-shard matrix. - The full Node matrix admits the consistently slow serial tooling, auto-reply command shards, and broad core-fast cache writer first. This keeps the 28-job cap while preventing critical-path work and the next run's transform seed from slipping into a later wave. - Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. Include-pattern shards record timing entries using the CI shard name, so `.artifacts/vitest-shard-timings.json` can distinguish a whole config from a filtered shard. -- Linux Node shard jobs persist Vitest's experimental filesystem module cache through the upstream Actions cache API, which Blacksmith transparently accelerates on its runners. Every CI shard is restore-only and unpacks the protected seed into its own runner-local root; the shard wrapper then gives concurrent Vitest processes separate live subdirectories. Only the non-cancelling daily or explicitly dispatched warmer saves a new immutable archive, so pull requests cannot publish transforms or mint per-PR cache families. A transform-input fingerprint clears incompatible lockfile, package, tsconfig, and Vitest-config generations. The protected writer scans and prunes its restored cache to 75% after it exceeds 2 GiB. Vitest hashes module id, source content, environment, and resolved transform config, so ordinary partial source changes keep unchanged entries warm while changed modules miss safely. Coarse restore prefixes bridge workflow runs; normal Actions cache LRU and inactivity eviction bound old immutable archives. +- Linux Node shard jobs persist Vitest's experimental filesystem module cache through the upstream Actions cache API, which Blacksmith transparently accelerates on its runners. Every CI shard is restore-only and unpacks the protected seed into its own runner-local root; the shard wrapper then gives concurrent Vitest processes separate live subdirectories. Only the non-cancelling daily or explicitly dispatched warmer saves a new immutable archive, so pull requests cannot publish transforms or mint per-PR cache families. The warmer launches each selected whole config in a fresh child process with concurrency one while reusing the same serial cache leaf, preventing config-global state from leaking without discarding transforms produced by the previous config. A transform-input fingerprint clears incompatible lockfile, package, tsconfig, and Vitest-config generations. The protected writer scans and prunes its restored cache to 75% after it exceeds 2 GiB. Vitest hashes module id, source content, environment, and resolved transform config, so ordinary partial source changes keep unchanged entries warm while changed modules miss safely. Coarse restore prefixes bridge workflow runs; normal Actions cache LRU and inactivity eviction bound old immutable archives. - Trusted Linux Node jobs also bind the pnpm store and `node_modules` from one protected dependency disk per supported Node line. Package manifests, install settings, runner platform, and the exact Node patch stay out of the disk key; an exact runtime and install-input fingerprint decides whether a job reuses the tree or reinstalls and refreshes the same disk. Manifests are canonicalized before hashing. The audited direct root hooks retain only pnpm's install lifecycle scripts, so formatting and ordinary test/build script edits keep the warm dependency tree; unaudited lifecycle-hook drift fails closed until its source inputs join the fingerprint contract. Dependency, package-manager, hook-source, and lockfile changes always invalidate the snapshot. A matching fingerprint is necessary but not sufficient: setup also checks the importer archive and manifest checksums, then verifies registry-backed lockfile dependencies retained by postinstall against the package manifests Node resolves from their importers. Missing or stale importer content falls back to a fresh install instead of serving the root hoist. A pull request whose read-only snapshot is unusable detaches the workspace bind and installs into runner-local storage, avoiding slow writes to a clone it cannot publish. Sticky cold installs disable pnpm's inner fetch retries and make up to three bounded full-install attempts from the progressively warmed store; a timeout remains a failure. After a content-validated restore or frozen-lockfile install, setup disables pnpm's redundant pre-run dependency check: the repository intentionally prunes plugin-local `node_modules`, which pnpm otherwise treats as stale and repairs through unsafe concurrent implicit installs during shard fanout. Canonical main preflight is the sole writer and measures the store on every refresh, running `pnpm store prune` only after retired package versions push it above 8 GiB. Blacksmith snapshot publication is asynchronous even after a writer job completes, so the first run after a fresh key or fingerprint can remain cold; later content-validated exact-marker restores are the rollout proof. Required CI jobs and pull requests get disposable clones, so dependency changes do not create new disks, competing snapshots, or a cache lock that can cancel builds. - Node shard and build-artifact jobs also restore Node's portable on-disk compile cache through immutable Actions caches. Independent `test` and `build` namespaces prevent their writers from replacing each other's archives: the scheduled test warmer owns the protected test seed, while `build-artifacts` may publish at most one protected build archive per UTC day from trusted `main` pushes. PR and ordinary test jobs only read protected snapshots, so feature-branch bytecode never enters the shared seed and PR traffic creates no cache archives. This reuses V8 bytecode for Node-loaded orchestration, build tooling, and external dependencies across different checkout paths, including when only part of the source graph changes. Vitest child processes disable an inherited compile cache because coverage can be enabled inside dynamic configs and V8 coverage can lose source-position precision when scripts are deserialized from bytecode. - The build-artifact job also persists content-fingerprinted `build-all` step outputs. CI's self-built plugin SDK declarations hash the complete repository-owned TypeScript/JSON source graph, exclude installed and generated directories, and restore both flat declarations and package bridges after `tsdown` clears `dist`. Documentation, workflow, plugin, and other changes outside that graph can reuse the declaration snapshot; source changes rebuild it before the export gate runs. diff --git a/test/scripts/ci-run-node-test-shard.test.ts b/test/scripts/ci-run-node-test-shard.test.ts index 696a472e053c..924c839decb1 100644 --- a/test/scripts/ci-run-node-test-shard.test.ts +++ b/test/scripts/ci-run-node-test-shard.test.ts @@ -153,6 +153,58 @@ describe("scripts/ci-run-node-test-shard.mjs", () => { expect(new Set(seen.map((run) => run.cache)).size).toBe(3); }); + it("runs per-config groups serially through one persistent cache slot", async () => { + const scratchDir = makeScratchDir(); + const persistentRoot = path.join(makeScratchDir(), "persistent"); + mkdirSync(persistentRoot, { recursive: true }); + const seen: Array<{ args: string[]; cache: string | undefined; label: string }> = []; + let active = 0; + let peakActive = 0; + + const exitCode = await runShardPlans( + resolveShardPlans({ + OPENCLAW_NODE_TEST_GROUPS_JSON: JSON.stringify( + ["a", "b", "c"].map((name) => ({ + configs: [`${name}.config.ts`], + shard_name: `cache-warm:${name}`, + })), + ), + }), + { + concurrency: 1, + env: { OPENCLAW_VITEST_FS_MODULE_CACHE_PATH: persistentRoot }, + runChild: async ( + args: string[], + childEnv: Record, + label: string, + ) => { + active += 1; + peakActive = Math.max(peakActive, active); + seen.push({ + args, + cache: childEnv.OPENCLAW_VITEST_FS_MODULE_CACHE_PATH, + label, + }); + active -= 1; + return 0; + }, + scratchDir, + }, + ); + + expect(exitCode).toBe(0); + expect(peakActive).toBe(1); + expect(seen.map((run) => run.args)).toEqual([ + ["a.config.ts"], + ["b.config.ts"], + ["c.config.ts"], + ]); + expect(seen.map((run) => run.label)).toEqual(["cache-warm:a", "cache-warm:b", "cache-warm:c"]); + expect(new Set(seen.map((run) => run.cache))).toEqual( + new Set([path.join(persistentRoot, "vitest-cache-0")]), + ); + }); + it("forwards trusted Vitest arguments after the target separator", async () => { const scratchDir = makeScratchDir(); const seen: string[][] = []; diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 143075aa19de..8d9a38c7734b 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -2934,6 +2934,11 @@ describe("ci workflow guards", () => { expect(warmerSource).toContain('"agentic-agents-embedded"'); expect(warmerSource).toContain('"agentic-gateway-methods"'); expect(warmerSource).toContain('"auto-reply-reply-commands-3"'); + expect(warmerSource).toContain("const groups = configs.map((config) => ({"); + expect(warmerSource).toContain("configs: [config]"); + expect(warmerSource).toContain("`OPENCLAW_NODE_TEST_GROUPS_JSON=${JSON.stringify(groups)}`"); + expect(warmerSource).not.toContain("OPENCLAW_NODE_TEST_CONFIGS_JSON"); + expect(warmerSource).toContain('"OPENCLAW_NODE_TEST_PLAN_CONCURRENCY=1"'); expect(warmerSetup.with).toMatchObject({ "node-compile-cache-scope": "test", "save-actions-cache": "true", @@ -2950,6 +2955,45 @@ describe("ci workflow guards", () => { expect(maintainStoreStep).toBeUndefined(); expect(maintainStickyStoreStep.env.OPENCLAW_PNPM_STORE_MAX_KIB).toBe("8388608"); + const seedRoot = mkdtempSync(path.join(tmpdir(), "openclaw-cache-seed-")); + try { + const envPath = path.join(seedRoot, "github-env"); + const result = runWorkflowShellScript(seedStep.run, { + env: { + ...process.env, + GITHUB_ENV: envPath, + }, + }); + expect(result.status, result.stderr).toBe(0); + const seedEnv = Object.fromEntries( + readFileSync(envPath, "utf8") + .trim() + .split("\n") + .map((line) => { + const separator = line.indexOf("="); + return [line.slice(0, separator), line.slice(separator + 1)]; + }), + ); + const serializedGroups = seedEnv.OPENCLAW_NODE_TEST_GROUPS_JSON; + if (!serializedGroups) { + throw new Error("cache warmer did not export OPENCLAW_NODE_TEST_GROUPS_JSON"); + } + const groups = JSON.parse(serializedGroups) as Array<{ + configs: string[]; + shard_name: string; + }>; + expect(groups.length).toBeGreaterThan(1); + expect(groups.every((group) => group.configs.length === 1)).toBe(true); + expect(new Set(groups.flatMap((group) => group.configs)).size).toBe(groups.length); + expect(groups.every((group) => group.shard_name === `cache-warm:${group.configs[0]}`)).toBe( + true, + ); + expect(seedEnv.OPENCLAW_NODE_TEST_PLAN_CONCURRENCY).toBe("1"); + expect(seedEnv).not.toHaveProperty("OPENCLAW_NODE_TEST_CONFIGS_JSON"); + } finally { + rmSync(seedRoot, { force: true, recursive: true }); + } + const maintenanceRoot = mkdtempSync(path.join(tmpdir(), "openclaw-pnpm-maintenance-")); try { const storeDir = path.join(maintenanceRoot, "store");