fix(ci): route UI E2E PR retries to hosted runners (#120997)

* fix(ci): route UI E2E PR retries to hosted runners

Punchcard-Session: amber-workshop-workshop-36

* fix(ci): align UI E2E retry cache routing

Punchcard-Session: amber-workshop-workshop-36

* fix(ci): clarify UI E2E retry routing

Punchcard-Session: amber-workshop-workshop-36

* fix(ci): validate workflow expression capture

Punchcard-Session: amber-workshop-workshop-36
This commit is contained in:
Vincent Koc
2026-08-09 19:50:24 +08:00
committed by GitHub
parent 2a57e1a1af
commit f83ba2b326
3 changed files with 150 additions and 26 deletions
+6 -7
View File
@@ -1469,10 +1469,9 @@ jobs:
# Compatibility targets pin a frozen Control UI whose e2e expectations track
# that release, not current main.
if: needs.preflight.outputs.run_ui_tests == 'true' && needs.preflight.outputs.compatibility_target != 'true'
# Chromium, Vite, and the Vitest coordinator share this runner even though
# files are serial. Four-vCPU runners let host-side CDP polling starve while
# the browser kept making progress, producing rotating timeout failures.
runs-on: ${{ github.event_name == 'workflow_dispatch' && 'ubuntu-24.04' || (github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-24.04') }}
# First-attempt same-repo runs use Blacksmith; manual dispatches, forks, and
# same-repo PR retries use GitHub-hosted capacity as an independent fallback.
runs-on: ${{ (github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.run_attempt > 1)) && 'ubuntu-24.04' || (github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-24.04') }}
# Keep Chromium ownership serial within each independently required shard.
timeout-minutes: 25
strategy:
@@ -1487,9 +1486,9 @@ jobs:
with:
node-version: "24.x"
install-bun: "false"
# Fork PRs must never read or write repository-global sticky snapshots.
sticky-disk: ${{ github.event_name != 'workflow_dispatch' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw') && 'true' || 'false' }}
use-actions-cache: ${{ github.event_name != 'workflow_dispatch' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw') && 'false' || 'true' }}
# Hosted paths use Actions cache and never touch repository-global sticky snapshots.
sticky-disk: ${{ (github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.run_attempt > 1)) && 'false' || (github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw') && 'true' || 'false') }}
use-actions-cache: ${{ (github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.run_attempt > 1)) && 'true' || (github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw') && 'false' || 'true') }}
- name: Install Playwright Chromium
run: node scripts/ensure-playwright-chromium.mjs
+11 -11
View File
@@ -116,7 +116,7 @@ The slowest Node test families are split or balanced so each job stays small wit
- The full Node matrix admits the consistently slow serial tooling, auto-reply command shards, and broad core-fast cache writer first. This keeps the 28-job cap while preventing critical-path work and the next run's transform seed from slipping into a later wave.
- Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. Include-pattern shards record timing entries using the CI shard name, so `.artifacts/vitest-shard-timings.json` can distinguish a whole config from a filtered shard.
- Linux Node shard jobs persist Vitest's experimental filesystem module cache through the upstream Actions cache API, which Blacksmith transparently accelerates on its runners. Every CI shard is restore-only and unpacks the protected seed into its own runner-local root; the shard wrapper then gives concurrent Vitest processes separate live subdirectories. Only the non-cancelling daily or explicitly dispatched warmer saves a new immutable archive, so pull requests cannot publish transforms or mint per-PR cache families. The warmer launches each selected shard/config envelope in a fresh child process with concurrency one, preserving its include patterns and environment while reusing the same serial cache leaf. This prevents config-global state from leaking, avoids expanding filtered shards into whole configs, and retains transforms produced by the previous child. A transform-input fingerprint clears incompatible lockfile, package, tsconfig, and Vitest-config generations. The protected writer scans and prunes its restored cache to 75% after it exceeds 2 GiB. Vitest hashes module id, source content, environment, and resolved transform config, so ordinary partial source changes keep unchanged entries warm while changed modules miss safely. Coarse restore prefixes bridge workflow runs; normal Actions cache LRU and inactivity eviction bound old immutable archives.
- Trusted Linux Node jobs also bind the pnpm store and `node_modules` from one protected dependency disk per supported Node line. Package manifests, install settings, runner platform, and the exact Node patch stay out of the disk key; an exact runtime and install-input fingerprint decides whether a job reuses the tree or reinstalls and refreshes the same disk. Manifests are canonicalized before hashing. The audited direct root hooks retain only pnpm's install lifecycle scripts, so formatting and ordinary test/build script edits keep the warm dependency tree; unaudited lifecycle-hook drift fails closed until its source inputs join the fingerprint contract. Dependency, package-manager, hook-source, and lockfile changes always invalidate the snapshot. A matching fingerprint is necessary but not sufficient: setup also checks the importer archive and manifest checksums, then verifies registry-backed lockfile dependencies retained by postinstall against the package manifests Node resolves from their importers. Missing or stale importer content falls back to a fresh install instead of serving the root hoist. A pull request whose read-only snapshot is unusable detaches the workspace bind and installs into runner-local storage, avoiding slow writes to a clone it cannot publish. Sticky cold installs disable pnpm's inner fetch retries and make up to three bounded full-install attempts from the progressively warmed store; a timeout remains a failure. After a content-validated restore or frozen-lockfile install, setup disables pnpm's redundant pre-run dependency check: the repository intentionally prunes plugin-local `node_modules`, which pnpm otherwise treats as stale and repairs through unsafe concurrent implicit installs during shard fanout. Canonical main preflight is the sole writer and measures the store on every refresh, running `pnpm store prune` only after retired package versions push it above 8 GiB. Blacksmith snapshot publication is asynchronous even after a writer job completes, so the first run after a fresh key or fingerprint can remain cold; later content-validated exact-marker restores are the rollout proof. Required CI jobs and pull requests get disposable clones, so dependency changes do not create new disks, competing snapshots, or a cache lock that can cancel builds.
- Trusted Blacksmith Linux Node jobs also bind the pnpm store and `node_modules` from one protected dependency disk per supported Node line. GitHub-hosted jobs, including manual dispatches, fork pull requests, and same-repo `checks-ui-e2e` retries, use the Actions cache path instead. Package manifests, install settings, runner platform, and the exact Node patch stay out of the disk key; an exact runtime and install-input fingerprint decides whether a job reuses the tree or reinstalls and refreshes the same disk. Manifests are canonicalized before hashing. The audited direct root hooks retain only pnpm's install lifecycle scripts, so formatting and ordinary test/build script edits keep the warm dependency tree; unaudited lifecycle-hook drift fails closed until its source inputs join the fingerprint contract. Dependency, package-manager, hook-source, and lockfile changes always invalidate the snapshot. A matching fingerprint is necessary but not sufficient: setup also checks the importer archive and manifest checksums, then verifies registry-backed lockfile dependencies retained by postinstall against the package manifests Node resolves from their importers. Missing or stale importer content falls back to a fresh install instead of serving the root hoist. A pull request whose read-only snapshot is unusable detaches the workspace bind and installs into runner-local storage, avoiding slow writes to a clone it cannot publish. Sticky cold installs disable pnpm's inner fetch retries and make up to three bounded full-install attempts from the progressively warmed store; a timeout remains a failure. After a content-validated restore or frozen-lockfile install, setup disables pnpm's redundant pre-run dependency check: the repository intentionally prunes plugin-local `node_modules`, which pnpm otherwise treats as stale and repairs through unsafe concurrent implicit installs during shard fanout. Canonical main preflight is the sole writer and measures the store on every refresh, running `pnpm store prune` only after retired package versions push it above 8 GiB. Blacksmith snapshot publication is asynchronous even after a writer job completes, so the first run after a fresh key or fingerprint can remain cold; later content-validated exact-marker restores are the rollout proof. Required Blacksmith CI jobs and first-attempt same-repo pull requests get disposable clones, so dependency changes do not create new disks, competing snapshots, or a cache lock that can cancel builds.
- Node shard and build-artifact jobs also restore Node's portable on-disk compile cache through immutable Actions caches. Independent `test` and `build` namespaces prevent their writers from replacing each other's archives: the scheduled test warmer owns the protected test seed, while `build-artifacts` may publish at most one protected build archive per UTC day from trusted `main` pushes. PR and ordinary test jobs only read protected snapshots, so feature-branch bytecode never enters the shared seed and PR traffic creates no cache archives. This reuses V8 bytecode for Node-loaded orchestration, build tooling, and external dependencies across different checkout paths, including when only part of the source graph changes. Vitest child processes disable an inherited compile cache because coverage can be enabled inside dynamic configs and V8 coverage can lose source-position precision when scripts are deserialized from bytecode.
- The build-artifact job also persists content-fingerprinted `build-all` step outputs. CI's self-built plugin SDK declarations hash the complete repository-owned TypeScript/JSON source graph, exclude installed and generated directories, and restore both flat declarations and package bridges after `tsdown` clears `dist`. Documentation, workflow, plugin, and other changes outside that graph can reuse the declaration snapshot; source changes rebuild it before the export gate runs.
- Full declaration builds split `tsdown` into AI, workspace-package, and unified groups. Each group caches declarations only, then still rebuilds runtime JavaScript before restoring those declarations. Core or plugin changes therefore invalidate only the large unified graph, while workspace-package changes conservatively invalidate every dependent declaration group. Public full builds generally use an immutable Actions cache; coarse restore keys seed partial changes, per-group content fingerprints reject stale data, and GitHub's cache quota evicts old generations. The weekly Node 22 lane instead publishes a 14-day artifact after successful `main` runs and restores only artifacts whose immutable producer identity resolves to that workflow on `main`, avoiding quota churn without allowing PR code to write a shared cache. Private-QA declarations are never persisted in Actions caches because cache namespaces are not confidentiality boundaries.
@@ -179,15 +179,15 @@ for commands and recovery.
## Runners
| Runner | Jobs |
| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `ubuntu-24.04` | `security-fast`, manual CI dispatch and non-canonical repository fallbacks, the QA Smoke aggregate, CodeQL security and quality scans, workflow-sanity, labeler, auto-response, the standalone Docs workflow, and the whole Install Smoke workflow |
| `blacksmith-4vcpu-ubuntu-2404` | `preflight`, `pnpm-store-warmup`, `native-i18n`, `checks-fast-core` except QA Smoke CI, plugin/channel contract shards, most bundled/lower-weight Linux Node shards, `check-*` lanes except `check-lint`, selected `check-additional-*` shards, `check-docs`, and `skills-python` |
| `blacksmith-8vcpu-ubuntu-2404` | Retained heavy Linux Node suites, the serial Chromium/Vite `checks-ui-e2e` lane, boundary/extension-heavy `check-additional-*` shards, and `android` |
| `blacksmith-16vcpu-ubuntu-2404` | Automatic QA Smoke CI shards, `build-artifacts` in CI and Testbox, and `check-lint` (CPU-sensitive enough that 8 vCPU cost more than they saved) |
| `blacksmith-8vcpu-windows-2025` | `checks-windows` |
| `blacksmith-6vcpu-macos-15` | `macos-node` on `openclaw/openclaw`; forks fall back to `macos-15` |
| `blacksmith-12vcpu-macos-26` | `macos-swift` and `ios-build` on `openclaw/openclaw`; forks fall back to `macos-26` |
| Runner | Jobs |
| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `ubuntu-24.04` | `security-fast`, manual CI dispatch and non-canonical repository fallbacks, pull-request retries of `checks-ui-e2e`, the QA Smoke aggregate, CodeQL security and quality scans, workflow-sanity, labeler, auto-response, the standalone Docs workflow, and the whole Install Smoke workflow |
| `blacksmith-4vcpu-ubuntu-2404` | `preflight`, `pnpm-store-warmup`, `native-i18n`, `checks-fast-core` except QA Smoke CI, plugin/channel contract shards, most bundled/lower-weight Linux Node shards, `check-*` lanes except `check-lint`, selected `check-additional-*` shards, `check-docs`, and `skills-python` |
| `blacksmith-8vcpu-ubuntu-2404` | Retained heavy Linux Node suites, first-attempt same-repo pull requests and pushes for the serial Chromium/Vite `checks-ui-e2e` lane, boundary/extension-heavy `check-additional-*` shards, and `android` |
| `blacksmith-16vcpu-ubuntu-2404` | Automatic QA Smoke CI shards, `build-artifacts` in CI and Testbox, and `check-lint` (CPU-sensitive enough that 8 vCPU cost more than they saved) |
| `blacksmith-8vcpu-windows-2025` | `checks-windows` |
| `blacksmith-6vcpu-macos-15` | `macos-node` on `openclaw/openclaw`; forks fall back to `macos-15` |
| `blacksmith-12vcpu-macos-26` | `macos-swift` and `ios-build` on `openclaw/openclaw`; forks fall back to `macos-26` |
## Runner registration budget
@@ -209,7 +209,7 @@ concurrent repositories, retries, and burst overlap.
The changed-target PR plan reduces the common Node test burst from 14 Blacksmith registrations to one. Broad-risk PRs keep the 14-registration compact fallback, so the worst case does not increase.
Canonical-repo CI keeps Blacksmith as the default runner path for normal push and pull-request runs. `workflow_dispatch` and non-canonical repository runs use GitHub-hosted runners, but normal canonical runs do not currently probe Blacksmith queue health or automatically fall back to GitHub-hosted labels when Blacksmith is unavailable.
Canonical-repo CI keeps Blacksmith as the default runner path for pushes and first-attempt same-repo pull-request runs. Pull-request retries of `checks-ui-e2e` use GitHub-hosted Ubuntu; push retries stay on Blacksmith. All `workflow_dispatch` runs, including `release_gate`, and non-canonical repository runs use GitHub-hosted runners. Normal canonical runs do not currently probe Blacksmith queue health or automatically fall back to GitHub-hosted labels when Blacksmith is unavailable.
## Surface ratchets
+133 -8
View File
@@ -14,6 +14,7 @@ import {
} from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { runInNewContext } from "node:vm";
import { expectDefined } from "@openclaw/normalization-core";
import { afterEach, describe, expect, it } from "vitest";
import { parse } from "yaml";
@@ -69,6 +70,42 @@ function readCiWorkflow() {
return parse(readFileSync(".github/workflows/ci.yml", "utf8"));
}
function evaluateWorkflowExpression(
expression: unknown,
context: {
eventName: "pull_request" | "push" | "workflow_dispatch";
headRepository?: string;
repository: string;
runAttempt: number;
},
) {
if (typeof expression !== "string") {
throw new TypeError("workflow expression must be a string");
}
const match = expression.match(/^\$\{\{\s*([\s\S]*?)\s*\}\}$/u);
if (!match) {
throw new Error(`invalid workflow expression: ${expression}`);
}
const source = match[1];
if (source === undefined) {
throw new Error(`workflow expression has no body: ${expression}`);
}
return runInNewContext(source, {
github: {
event_name: context.eventName,
repository: context.repository,
run_attempt: context.runAttempt,
event: context.headRepository
? {
pull_request: {
head: { repo: { full_name: context.headRepository } },
},
}
: {},
},
});
}
function runCiGateFixture(requiredResults: string, selectedResults: string) {
const gateStep = readCiWorkflow().jobs["ci-gate"].steps.find(
(step: WorkflowStep) => step.name === "Verify selected CI lanes",
@@ -2483,15 +2520,18 @@ NODE
for (const { jobName, stepWith } of stickyConsumers) {
const stickyCondition = stepWith["sticky-disk"];
const cacheCondition = stepWith["use-actions-cache"];
if (jobName === "checks-ui-e2e") {
continue;
}
expect(stickyCondition, jobName).toContain("github.event_name != 'workflow_dispatch'");
expect(cacheCondition, jobName).toContain("github.event_name != 'workflow_dispatch'");
expect(cacheCondition, jobName).toContain("&& 'false' || 'true'");
expect(stickyCondition, jobName).toContain(
"github.event.pull_request.head.repo.full_name == 'openclaw/openclaw'",
);
expect(cacheCondition, jobName).toContain("github.event_name != 'workflow_dispatch'");
expect(cacheCondition, jobName).toContain(
"github.event.pull_request.head.repo.full_name == 'openclaw/openclaw'",
);
expect(cacheCondition, jobName).toContain("&& 'false' || 'true'");
}
// Required CI jobs only clone the snapshot. The disposable warmer below
// owns commits so writer coalescing cannot cancel a required build job.
@@ -5019,7 +5059,9 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}"
expect(uiE2e.if).toBe(
"needs.preflight.outputs.run_ui_tests == 'true' && needs.preflight.outputs.compatibility_target != 'true'",
);
expect(uiE2e["runs-on"]).toContain("blacksmith-8vcpu-ubuntu-2404");
expect(uiE2e["runs-on"]).toBe(
"${{ (github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.run_attempt > 1)) && 'ubuntu-24.04' || (github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-24.04') }}",
);
expect(uiE2e["runs-on"]).not.toBe(ui["runs-on"]);
// Each Chromium worker keeps serial file ownership while all four shards
// together remain required by the aggregate CI gate.
@@ -5031,16 +5073,99 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}"
});
expect(workflow.jobs["ci-gate"].needs).toContain("checks-ui-e2e");
const uiSetup = expectDefined(
ui.steps.find((step: WorkflowStep) => step.name === "Setup Node environment"),
"Control UI Node setup",
);
const uiE2eSetup = expectDefined(
uiE2e.steps.find((step: WorkflowStep) => step.name === "Setup Node environment"),
"Control UI E2E Node setup",
);
expect(uiE2eSetup.uses).toBe("./.github/actions/setup-node-env");
expect(uiE2eSetup.with).toEqual(uiSetup.with);
expect(uiE2eSetup.with).toEqual({
"node-version": "24.x",
"install-bun": "false",
"sticky-disk":
"${{ (github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.run_attempt > 1)) && 'false' || (github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw') && 'true' || 'false') }}",
"use-actions-cache":
"${{ (github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.run_attempt > 1)) && 'true' || (github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw') && 'false' || 'true') }}",
});
const routingScenarios = [
{
name: "same-repo pull request first attempt",
context: {
eventName: "pull_request",
headRepository: "openclaw/openclaw",
repository: "openclaw/openclaw",
runAttempt: 1,
},
expected: {
runner: "blacksmith-8vcpu-ubuntu-2404",
stickyDisk: "true",
useActionsCache: "false",
},
},
{
name: "same-repo pull request retry",
context: {
eventName: "pull_request",
headRepository: "openclaw/openclaw",
repository: "openclaw/openclaw",
runAttempt: 2,
},
expected: {
runner: "ubuntu-24.04",
stickyDisk: "false",
useActionsCache: "true",
},
},
{
name: "fork pull request",
context: {
eventName: "pull_request",
headRepository: "contributor/openclaw",
repository: "openclaw/openclaw",
runAttempt: 1,
},
expected: {
runner: "ubuntu-24.04",
stickyDisk: "false",
useActionsCache: "true",
},
},
{
name: "workflow dispatch",
context: {
eventName: "workflow_dispatch",
repository: "openclaw/openclaw",
runAttempt: 1,
},
expected: {
runner: "ubuntu-24.04",
stickyDisk: "false",
useActionsCache: "true",
},
},
{
name: "canonical push retry",
context: {
eventName: "push",
repository: "openclaw/openclaw",
runAttempt: 2,
},
expected: {
runner: "blacksmith-8vcpu-ubuntu-2404",
stickyDisk: "true",
useActionsCache: "false",
},
},
] as const;
for (const { context, expected, name } of routingScenarios) {
expect(evaluateWorkflowExpression(uiE2e["runs-on"], context), name).toBe(expected.runner);
expect(evaluateWorkflowExpression(uiE2eSetup.with?.["sticky-disk"], context), name).toBe(
expected.stickyDisk,
);
expect(
evaluateWorkflowExpression(uiE2eSetup.with?.["use-actions-cache"], context),
name,
).toBe(expected.useActionsCache);
}
const chromiumInstall = expectDefined(
uiE2e.steps.find((step: WorkflowStep) => step.name === "Install Playwright Chromium"),