From 3a49aa1ac68ffceff75b05d4792bef7d39b911af Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 14 Aug 2026 02:40:02 -0700 Subject: [PATCH] improve: split hosted Windows CI into three lanes (#123577) * ci: split Windows tests into three lanes * test: cover three-way Windows CI partition --- .agents/skills/openclaw-ci-limits/SKILL.md | 2 +- .github/workflows/ci.yml | 7 +++++-- docs/ci.md | 6 +++--- package.json | 7 ++++--- test/package-scripts.test.ts | 19 ++++++++++++++----- test/scripts/ci-workflow-guards.test.ts | 4 +++- 6 files changed, 30 insertions(+), 15 deletions(-) diff --git a/.agents/skills/openclaw-ci-limits/SKILL.md b/.agents/skills/openclaw-ci-limits/SKILL.md index 1f20cabdda52..bd368a0ecbce 100644 --- a/.agents/skills/openclaw-ci-limits/SKILL.md +++ b/.agents/skills/openclaw-ci-limits/SKILL.md @@ -166,7 +166,7 @@ These are intentionally guarded by `test/scripts/ci-workflow-guards.test.ts`: misses fall back to the ordinary pnpm-store cache, while hosted/fork/manual paths use only that store cache. - CI matrix caps: fast/check lanes at 12, Node test shards at 28 on Blacksmith - and 64 on the GitHub backend, Windows and Android at 2. + and 64 on the GitHub backend, Windows at 3, and Android at 2. - Canonical PR Node tests use one precise changed-target job when possible; broad, deleted, unknown, or planner-failed changes fall back to the compact full-suite plan. Targeted plans retain the full built-artifact diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffabf498b889..978cb28d0ec7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -851,7 +851,7 @@ jobs: checks_windows_matrix: createMatrix( runWindows ? usesGithubRunnerProfile - ? [1, 2].map((part) => ({ + ? [1, 2, 3].map((part) => ({ check_name: `checks-windows-node-test-${part}`, runtime: "node", task: `test-${part}`, @@ -3131,7 +3131,7 @@ jobs: shell: bash strategy: fail-fast: false - max-parallel: 2 + max-parallel: 3 matrix: ${{ fromJson(needs.preflight.outputs.checks_windows_matrix) }} steps: - &platform_checkout_step @@ -3259,6 +3259,9 @@ jobs: test-2) pnpm test:windows:ci:2 ;; + test-3) + pnpm test:windows:ci:3 + ;; *) echo "Unsupported Windows checks task: $TASK" >&2 exit 1 diff --git a/docs/ci.md b/docs/ci.md index e328f1c85dc4..3783963a0469 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -137,8 +137,8 @@ The slowest Node test families are split or balanced so each job stays small wit Once admitted, canonical Linux CI permits up to 28 concurrent Node test jobs on Blacksmith and 64 with the `github` or `hybrid` hosted planner profile. The smaller -fast/check lanes remain capped at 12 in both modes; Windows and Android stay at -two because those runner pools are narrower. Compact whole-config batches run +fast/check lanes remain capped at 12 in both modes; Windows is capped at three +and Android at two because those runner pools are narrower. Compact whole-config batches run with a 120-minute batch timeout, while include-pattern groups share the same bounded job budget. @@ -224,7 +224,7 @@ Hybrid is the normal degraded-capacity mode. If Blacksmith is down: rerun the fa gh variable set OPENCLAW_CI_RUNNER_BACKEND --repo openclaw/openclaw --body github ``` -Hosted paths use the same setup exercised by manual dispatches and fork pull requests. Blacksmith-only Docker and sticky-disk steps are skipped, dependency setup uses the ordinary Actions pnpm-store cache, and low-memory Android builds use separate Gradle processes. Hybrid attempt-1 plateau lanes deliberately keep this backend-neutral Actions-cache profile when they run on Blacksmith because preflight remains hosted. The exact workspace dependency cache intentionally stays off when preflight is hosted: GitHub can roll the runner image and Node patch between preflight and fanout in one workflow, while the safe exact key then misses; the ordinary store archive is only slightly smaller and pnpm's measured relink is already single-digit seconds. Core oxlint splits into five deterministic hosted stripes, with extension/scripts lint and optional UI/format checks in the existing `check-lint` row. The compact Node planner keeps the Blacksmith 200/276-second hint tables unchanged and selects a separate standard 4-core hosted overlay. That overlay contains measured per-group medians, targets 120/124 seconds of serial group work for the large/small source runner classes, and applies a 1.6x median scaling fallback only to unmeasured groups. Failed and timeout-and-retry samples are excluded from refreshes. Groups above 210 predicted seconds split into file-weighted hosted stripes before packing; the refreshed 305-second `core-tooling-1` median now splits that former critical-path pole in two, while the measured `core-tooling-3` outlier retains three stripes. The resulting 55 push descriptors and 64 pull-request fallback descriptors stay within the 64-row hosted cap, and every lane is at or below 209 predicted test seconds. The corresponding Blacksmith plans remain 25 and 31 total descriptors under the 28-row admission cap. Control UI E2E expands from the unchanged all-Blacksmith shape of three Vitest shards plus one browser-extension shard to seven Vitest shards plus one browser-extension shard for the `github` and `hybrid` planner profiles. QA Smoke similarly expands from four parts to six. Hybrid attempt 1 runs those expanded matrices on Blacksmith; `github` mode and hybrid retries use hosted runners. QA's planner reserves the final part's observed roughly two-minute Matrix rider before greedily assigning primary scenarios, keeping that separate run from becoming the tail. Windows expands from the unchanged single Blacksmith job to two hosted jobs with disjoint, project-aligned explicit test lists. This partitions the complete Windows-specific inventory without applying Vitest `--shard` to project-local single-file selections, which Vitest rejects. Expect slower individual builds on standard 4-core hosted runners. Blacksmith's runner-registration budget is irrelevant for hosted jobs, but GitHub-hosted concurrency limits apply. +Hosted paths use the same setup exercised by manual dispatches and fork pull requests. Blacksmith-only Docker and sticky-disk steps are skipped, dependency setup uses the ordinary Actions pnpm-store cache, and low-memory Android builds use separate Gradle processes. Hybrid attempt-1 plateau lanes deliberately keep this backend-neutral Actions-cache profile when they run on Blacksmith because preflight remains hosted. The exact workspace dependency cache intentionally stays off when preflight is hosted: GitHub can roll the runner image and Node patch between preflight and fanout in one workflow, while the safe exact key then misses; the ordinary store archive is only slightly smaller and pnpm's measured relink is already single-digit seconds. Core oxlint splits into five deterministic hosted stripes, with extension/scripts lint and optional UI/format checks in the existing `check-lint` row. The compact Node planner keeps the Blacksmith 200/276-second hint tables unchanged and selects a separate standard 4-core hosted overlay. That overlay contains measured per-group medians, targets 120/124 seconds of serial group work for the large/small source runner classes, and applies a 1.6x median scaling fallback only to unmeasured groups. Failed and timeout-and-retry samples are excluded from refreshes. Groups above 210 predicted seconds split into file-weighted hosted stripes before packing; the refreshed 305-second `core-tooling-1` median now splits that former critical-path pole in two, while the measured `core-tooling-3` outlier retains three stripes. The resulting 55 push descriptors and 64 pull-request fallback descriptors stay within the 64-row hosted cap, and every lane is at or below 209 predicted test seconds. The corresponding Blacksmith plans remain 25 and 31 total descriptors under the 28-row admission cap. Control UI E2E expands from the unchanged all-Blacksmith shape of three Vitest shards plus one browser-extension shard to seven Vitest shards plus one browser-extension shard for the `github` and `hybrid` planner profiles. QA Smoke similarly expands from four parts to six. Hybrid attempt 1 runs those expanded matrices on Blacksmith; `github` mode and hybrid retries use hosted runners. QA's planner reserves the final part's observed roughly two-minute Matrix rider before greedily assigning primary scenarios, keeping that separate run from becoming the tail. Windows expands from the unchanged single Blacksmith job to three hosted jobs with disjoint, project-aligned explicit test lists. This partitions the complete Windows-specific inventory without applying Vitest `--shard` to project-local single-file selections, which Vitest rejects. Expect slower individual builds on standard 4-core hosted runners. Blacksmith's runner-registration budget is irrelevant for hosted jobs, but GitHub-hosted concurrency limits apply. Restore all-Blacksmith routing after an outage by deleting the variable: diff --git a/package.json b/package.json index a6edd1acfc63..68e2bcf52778 100644 --- a/package.json +++ b/package.json @@ -1944,9 +1944,10 @@ "test:unit:fast:audit": "node --import tsx scripts/test-unit-fast-audit.mts", "test:voicecall:closedloop": "node --import tsx scripts/test-voicecall-closedloop.mts", "test:watch": "node --import tsx scripts/test-projects.mts --watch", - "test:windows:ci": "pnpm test:windows:ci:1 && pnpm test:windows:ci:2", - "test:windows:ci:1": "node --import tsx scripts/test-projects.mts src/state/openclaw-state-ownership.test.ts src/snapshot/local-repository.windows.test.ts src/state/openclaw-database-paths.windows.test.ts src/node-host/invoke-system-run-allowlist.test.ts src/node-host/node-worker-transfer-client.test.ts packages/terminal-core/src/display-string.test.ts src/infra/sqlite-snapshot.test.ts src/infra/ports.test.ts src/infra/advertised-lan-host.windows.test.ts src/infra/update-managed-service-handoff-command.test.ts src/infra/update-managed-service-handoff-lifecycle.test.ts src/infra/executable-path.test.ts src/infra/fs-safe-remove.test.ts src/infra/state-migrations.legacy-session-store.test.ts src/infra/windows-install-roots.test.ts src/agents/tools/media-tool-file-url.windows.test.ts src/commands/doctor-gateway-auth-token.windows.test.ts src/commands/backup-verify.test.ts src/commands/agents.commands.list.test.ts src/daemon/schtasks.startup-fallback.test.ts src/plugin-sdk/node-host.test.ts src/shared/runtime-import.test.ts test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts extensions/lobster/src/lobster-runner.test.ts", - "test:windows:ci:2": "node --import tsx scripts/test-projects.mts src/infra/ssh-client.windows.test.ts src/infra/exec-allowlist-pattern.test.ts src/infra/process-env.test.ts src/agents/sessions/windows-git-bash-path.test.ts test/scripts/npm-runner.test.ts test/scripts/ts-topology.test.ts src/test-utils/openclaw-test-state.test.ts src/utils.test.ts test/scripts/direct-run-entrypoints.test.ts test/scripts/format-generated-module.test.ts test/scripts/openclaw-cross-os-installer.windows.test.ts test/scripts/openclaw-cross-os-release-workflow.test.ts test/scripts/pnpm-runner.test.ts test/scripts/run-with-env.test.ts test/scripts/ui.test.ts test/scripts/vitest-process-group.test.ts src/config/sessions/session-accessor.sqlite-archive.worker.test.ts src/media/local-media-path.windows.test.ts src/media/web-media.file-url.windows.test.ts src/process/exec.windows.test.ts src/process/exec.windows.integration.test.ts src/process/windows-command.test.ts src/process/terminal-pty.test.ts src/tui/tui.resolve-codex-bin.test.ts src/media-understanding/attachments.file-url.windows.test.ts src/cli/daemon-cli/status.print.test.ts src/cli/mcp-cli.path-case.windows.test.ts src/auto-reply/usage-bar/template.windows.test.ts src/auto-reply/reply.triggers.trigger-handling.stages-inbound-media-into-sandbox-workspace.test.ts src/agents/provider-local-service.env-case.test.ts src/agents/bash-tools.exec.script-preflight.test.ts src/agents/agent-tools.read.windows.test.ts src/agents/agent-tools.read.host-operations.test.ts src/agents/sandbox/fs-paths.test.ts src/agents/sessions/tools/render-utils.test.ts src/agents/sessions/tools/path-utils.test.ts extensions/memory-core/src/memory-extra-file-path.windows.test.ts extensions/msteams/src/media-helpers.test.ts extensions/msteams/src/messenger.test.ts extensions/mxc/test/mxc-backend.test.ts extensions/mxc/test/sandbox-policy-loader.test.ts", + "test:windows:ci": "pnpm test:windows:ci:1 && pnpm test:windows:ci:2 && pnpm test:windows:ci:3", + "test:windows:ci:1": "node --import tsx scripts/test-projects.mts src/state/openclaw-state-ownership.test.ts src/snapshot/local-repository.windows.test.ts src/state/openclaw-database-paths.windows.test.ts src/node-host/invoke-system-run-allowlist.test.ts src/node-host/node-worker-transfer-client.test.ts packages/terminal-core/src/display-string.test.ts src/config/sessions/session-accessor.sqlite-archive.worker.test.ts src/commands/doctor-gateway-auth-token.windows.test.ts src/commands/backup-verify.test.ts src/commands/agents.commands.list.test.ts src/shared/runtime-import.test.ts test/scripts/direct-run-entrypoints.test.ts test/scripts/format-generated-module.test.ts test/scripts/openclaw-cross-os-installer.windows.test.ts test/scripts/openclaw-cross-os-release-workflow.test.ts test/scripts/pnpm-runner.test.ts test/scripts/run-with-env.test.ts test/scripts/ui.test.ts test/scripts/vitest-process-group.test.ts", + "test:windows:ci:2": "node --import tsx scripts/test-projects.mts src/infra/sqlite-snapshot.test.ts src/infra/ports.test.ts src/infra/advertised-lan-host.windows.test.ts src/infra/update-managed-service-handoff-command.test.ts src/infra/update-managed-service-handoff-lifecycle.test.ts src/infra/executable-path.test.ts src/infra/fs-safe-remove.test.ts src/infra/state-migrations.legacy-session-store.test.ts src/infra/windows-install-roots.test.ts src/agents/tools/media-tool-file-url.windows.test.ts extensions/memory-core/src/memory-extra-file-path.windows.test.ts src/daemon/schtasks.startup-fallback.test.ts src/plugin-sdk/node-host.test.ts test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts extensions/lobster/src/lobster-runner.test.ts", + "test:windows:ci:3": "node --import tsx scripts/test-projects.mts src/infra/ssh-client.windows.test.ts src/infra/exec-allowlist-pattern.test.ts src/infra/process-env.test.ts src/agents/sessions/windows-git-bash-path.test.ts test/scripts/npm-runner.test.ts test/scripts/ts-topology.test.ts src/test-utils/openclaw-test-state.test.ts src/utils.test.ts src/media/local-media-path.windows.test.ts src/media/web-media.file-url.windows.test.ts src/process/exec.windows.test.ts src/process/exec.windows.integration.test.ts src/process/windows-command.test.ts src/process/terminal-pty.test.ts src/tui/tui.resolve-codex-bin.test.ts src/media-understanding/attachments.file-url.windows.test.ts src/cli/daemon-cli/status.print.test.ts src/cli/mcp-cli.path-case.windows.test.ts src/auto-reply/usage-bar/template.windows.test.ts src/auto-reply/reply.triggers.trigger-handling.stages-inbound-media-into-sandbox-workspace.test.ts src/agents/provider-local-service.env-case.test.ts src/agents/bash-tools.exec.script-preflight.test.ts src/agents/agent-tools.read.windows.test.ts src/agents/agent-tools.read.host-operations.test.ts src/agents/sandbox/fs-paths.test.ts src/agents/sessions/tools/render-utils.test.ts src/agents/sessions/tools/path-utils.test.ts extensions/msteams/src/media-helpers.test.ts extensions/msteams/src/messenger.test.ts extensions/mxc/test/mxc-backend.test.ts extensions/mxc/test/sandbox-policy-loader.test.ts", "test:windows:schtasks:integration": "node --import tsx scripts/run-with-env.mts CI_WINDOWS_SCHTASKS_INTEGRATION=1 OPENCLAW_E2E_VERBOSE=1 OPENCLAW_VITEST_MAX_WORKERS=1 -- node scripts/run-vitest.mjs src/daemon/schtasks.integration.e2e.test.ts", "tool-display:check": "node --import tsx scripts/tool-display.ts --check", "tool-display:write": "node --import tsx scripts/tool-display.ts --write", diff --git a/test/package-scripts.test.ts b/test/package-scripts.test.ts index 85a62b8b7faf..0653adbd55f2 100644 --- a/test/package-scripts.test.ts +++ b/test/package-scripts.test.ts @@ -26,11 +26,12 @@ function readPackageJson(): RootPackageJson { return JSON.parse(fs.readFileSync("package.json", "utf8")) as RootPackageJson; } -function readWindowsCiPartScripts(): [string, string] { +function readWindowsCiPartScripts(): [string, string, string] { const scripts = readPackageJson().scripts; return [ expectDefined(scripts["test:windows:ci:1"], "Windows CI part 1 script"), expectDefined(scripts["test:windows:ci:2"], "Windows CI part 2 script"), + expectDefined(scripts["test:windows:ci:3"], "Windows CI part 3 script"), ]; } @@ -203,16 +204,24 @@ describe("package scripts", () => { expect(scripts["android:test"]).toContain(":wear:testDebugUnitTest"); }); - it("partitions Windows CI coverage into two disjoint explicit test lists", () => { + it("partitions Windows CI coverage into three disjoint explicit test lists", () => { const scripts = readPackageJson().scripts; const partScripts = readWindowsCiPartScripts(); const partTargets = partScripts.map(readWindowsCiTargets); const allTargets = partTargets.flat(); - expect(scripts["test:windows:ci"]).toBe("pnpm test:windows:ci:1 && pnpm test:windows:ci:2"); - expect(partTargets.map((targets) => targets.length)).toEqual([24, 41]); + expect(scripts["test:windows:ci"]).toBe( + "pnpm test:windows:ci:1 && pnpm test:windows:ci:2 && pnpm test:windows:ci:3", + ); + expect(partTargets.map((targets) => targets.length)).toEqual([19, 15, 31]); expect(new Set(allTargets).size).toBe(65); - expect(partTargets[0]?.filter((target) => partTargets[1]?.includes(target))).toEqual([]); + for (const [partIndex, targets] of partTargets.entries()) { + const laterTargets = new Set(partTargets.slice(partIndex + 1).flat()); + expect( + targets.filter((target) => laterTargets.has(target)), + `Windows CI part ${partIndex + 1} overlaps a later part`, + ).toEqual([]); + } }); it("runs node workspace transfer coverage in Windows CI", () => { diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index b64263e56f74..5a01bc9448d8 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -2695,7 +2695,7 @@ NODE expect(workflow.jobs["checks-fast-channel-contracts-shard"].strategy["max-parallel"]).toBe(12); expect(workflow.jobs["check-shard"].strategy["max-parallel"]).toBe(12); expect(workflow.jobs["check-additional-shard"].strategy["max-parallel"]).toBe(12); - expect(workflow.jobs["checks-windows"].strategy["max-parallel"]).toBe(2); + expect(workflow.jobs["checks-windows"].strategy["max-parallel"]).toBe(3); expect(workflow.jobs.android.strategy["max-parallel"]).toBe(2); }); @@ -2741,6 +2741,7 @@ NODE const expectedHostedWindowsMatrix = [ { check_name: "checks-windows-node-test-1", runtime: "node", task: "test-1" }, { check_name: "checks-windows-node-test-2", runtime: "node", task: "test-2" }, + { check_name: "checks-windows-node-test-3", runtime: "node", task: "test-3" }, ]; expect( JSON.parse(expectDefined(github.outputs.checks_windows_matrix, "GitHub Windows matrix")) @@ -2752,6 +2753,7 @@ NODE ).toEqual(expectedHostedWindowsMatrix); expect(runStep.run).toContain("test-1)\n pnpm test:windows:ci:1"); expect(runStep.run).toContain("test-2)\n pnpm test:windows:ci:2"); + expect(runStep.run).toContain("test-3)\n pnpm test:windows:ci:3"); }); it("installs the Android SDK platform used by Gradle", () => {