mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
perf(ci): repair and re-key the extension-boundary sticky lane (#109804)
check-additional-extension-package-boundary is the light-run critical-path pole. Measured on runs 29564442266/29564411446/29564259862 (8 vCPU, sticky mounts 429-failing): Setup Node ~40s cold install, shard step 142-161s = ~101s cold plugin-sdk dts prep (nine parallel tsgo builds) + ~58s compiling 122 plugins at concurrency 6 + ~1s canary; warm actions/cache path on a main push (29551077288) runs the shard in 67s. Levers chosen from that data: - Re-key the ext-boundary sticky disk from per-PR/per-config keys (v1) to one O(1) protected key (v2), following the node_modules snapshot fix in #109752. Blacksmith's installation-wide 1000-disk cap is saturated and every mount 429s; per-PR keys are a direct contributor. The config/scripts/ lockfile hash moves from the key into the in-job .source-trees marker next to the existing tree-OID + node/pnpm gate. Single semantic writer: the boundary lane commits explicitly (true, not if-missing) on protected pushes only; PR mounts and the check-lint consumer stay read-only, and the seed step is now writer-only (saves ~11s per PR cold run). - Bump the lane 8 -> 32 vCPU and raise OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY 6 -> 16: both the parallel dts prep and the 122 plugin compiles scale with cores at similar billed core-minutes (same precedent as check-lint/deps). Expected pole time: warm PR runs (protected snapshot tracks every main push) drop to roughly checkout + install-skip + restore + incremental shard, well under 2 minutes; cold runs that touch boundary trees shrink via the core bump. Guarded in ci-workflow-guards so the O(1) key, single-writer commit, and fingerprint composition cannot silently regress.
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
5024c100a3
commit
dfd457adc0
+57
-27
@@ -1977,21 +1977,28 @@ jobs:
|
||||
|
||||
# check-lint's shard runner rebuilds the same plugin-sdk boundary
|
||||
# artifacts the boundary lane snapshots (~72s cold); restore them from
|
||||
# the shared sticky. Strictly read-only (commit: false): if this lane
|
||||
# could commit a freshly formatted disk, a cold-key race with the
|
||||
# boundary lane would permanently seed an empty, marker-less snapshot
|
||||
# that if-missing then refuses to repair.
|
||||
# the shared sticky. Strictly read-only (commit: false): only the
|
||||
# protected boundary lane may publish this repository-global snapshot.
|
||||
- name: Mount extension boundary sticky disk
|
||||
if: matrix.task == 'lint' && github.event_name != 'workflow_dispatch' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw')
|
||||
uses: useblacksmith/stickydisk@5b350170ae4ef55b536b548ef5f5896e76a6b54f # v1.4.0
|
||||
with:
|
||||
key: ${{ github.repository }}-ext-boundary-v1-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || 'protected' }}-${{ hashFiles('tsconfig.json', 'tsconfig.plugin-sdk.dts.json', 'packages/plugin-sdk/tsconfig.json', 'scripts/check-extension-package-tsc-boundary.mjs', 'scripts/prepare-extension-package-boundary-artifacts.mjs', 'scripts/write-plugin-sdk-entry-dts.ts', 'scripts/lib/plugin-sdk-entrypoints.json', 'scripts/lib/plugin-sdk-entries.mjs', 'package.json', 'pnpm-lock.yaml') }}
|
||||
# One stable disk for the whole repository. The v1 per-PR/per-config
|
||||
# keys minted a new backing disk for every PR and toolchain change
|
||||
# and helped saturate Blacksmith's installation-wide sticky-disk
|
||||
# budget, 429-failing every mount. Snapshot validity lives in the
|
||||
# in-job marker checked below instead of the key.
|
||||
key: ${{ github.repository }}-ext-boundary-v2
|
||||
path: /var/tmp/openclaw-ext-boundary
|
||||
commit: "false"
|
||||
|
||||
- name: Restore extension boundary artifacts from sticky disk
|
||||
if: matrix.task == 'lint' && github.event_name != 'workflow_dispatch' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw')
|
||||
shell: bash
|
||||
env:
|
||||
# Config/toolchain inputs the tree-OID gate below cannot see; must
|
||||
# stay identical to the boundary lane's fingerprint composition.
|
||||
BOUNDARY_CONFIG_HASH: ${{ hashFiles('tsconfig.json', 'tsconfig.plugin-sdk.dts.json', 'packages/plugin-sdk/tsconfig.json', 'scripts/check-extension-package-tsc-boundary.mjs', 'scripts/prepare-extension-package-boundary-artifacts.mjs', 'scripts/write-plugin-sdk-entry-dts.ts', 'scripts/lib/plugin-sdk-entrypoints.json', 'scripts/lib/plugin-sdk-entries.mjs', 'package.json', 'pnpm-lock.yaml') }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sticky_root=/var/tmp/openclaw-ext-boundary
|
||||
@@ -2002,7 +2009,7 @@ jobs:
|
||||
# Same tree-OID gate as the boundary lane: restore only artifacts
|
||||
# produced from identical generative sources, so stale snapshots can
|
||||
# neither false-skip rebuilds nor leave ghost declarations.
|
||||
current_trees="$({ git rev-parse HEAD:src/plugin-sdk HEAD:packages HEAD:extensions HEAD:src/auto-reply HEAD:src/types HEAD:src/plugins/types.ts HEAD:src/video-generation HEAD:src/channels; node --version; corepack pnpm --version 2>/dev/null || pnpm --version; })"
|
||||
current_trees="$({ git rev-parse HEAD:src/plugin-sdk HEAD:packages HEAD:extensions HEAD:src/auto-reply HEAD:src/types HEAD:src/plugins/types.ts HEAD:src/video-generation HEAD:src/channels; node --version; corepack pnpm --version 2>/dev/null || pnpm --version; echo "$BOUNDARY_CONFIG_HASH"; })"
|
||||
if [ ! -f "$sticky_root/.source-trees" ] || [ "$current_trees" != "$(cat "$sticky_root/.source-trees")" ]; then
|
||||
echo "boundary source trees changed since snapshot; lint prepares cold"
|
||||
exit 0
|
||||
@@ -2182,7 +2189,11 @@ jobs:
|
||||
runner: blacksmith-8vcpu-ubuntu-2404
|
||||
- check_name: check-additional-extension-package-boundary
|
||||
group: extension-package-boundary
|
||||
runner: blacksmith-8vcpu-ubuntu-2404
|
||||
# Light-run critical-path pole: cold runs spend ~100s in nine
|
||||
# parallel plugin-sdk dts builds plus ~58s compiling 122 plugins
|
||||
# at concurrency 6 on 8 vCPUs. Both phases scale with cores at
|
||||
# similar billed core-minutes.
|
||||
runner: blacksmith-32vcpu-ubuntu-2404
|
||||
- check_name: check-additional-runtime-topology-architecture
|
||||
group: runtime-topology-architecture
|
||||
runner: blacksmith-4vcpu-ubuntu-2404
|
||||
@@ -2207,19 +2218,30 @@ jobs:
|
||||
if: matrix.group == 'extension-package-boundary' && github.event_name != 'workflow_dispatch' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw')
|
||||
uses: useblacksmith/stickydisk@5b350170ae4ef55b536b548ef5f5896e76a6b54f # v1.4.0
|
||||
with:
|
||||
# Coarse toolchain key only: source changes are meant to hit a stale
|
||||
# snapshot, whose restored artifacts rebuild incrementally. PR scope
|
||||
# keeps feature snapshots separate from protected pushes.
|
||||
key: ${{ github.repository }}-ext-boundary-v1-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || 'protected' }}-${{ hashFiles('tsconfig.json', 'tsconfig.plugin-sdk.dts.json', 'packages/plugin-sdk/tsconfig.json', 'scripts/check-extension-package-tsc-boundary.mjs', 'scripts/prepare-extension-package-boundary-artifacts.mjs', 'scripts/write-plugin-sdk-entry-dts.ts', 'scripts/lib/plugin-sdk-entrypoints.json', 'scripts/lib/plugin-sdk-entries.mjs', 'package.json', 'pnpm-lock.yaml') }}
|
||||
# One stable disk for the whole repository. The v1 per-PR/per-config
|
||||
# keys minted a new backing disk for every PR and toolchain change
|
||||
# and helped saturate Blacksmith's installation-wide sticky-disk
|
||||
# budget, 429-failing every mount. Snapshot validity lives in the
|
||||
# in-job marker checked below instead of the key, so source and
|
||||
# toolchain changes refresh this disk in place.
|
||||
key: ${{ github.repository }}-ext-boundary-v2
|
||||
path: /var/tmp/openclaw-ext-boundary
|
||||
# v1.4.0 skips commit after failed/cancelled steps, so an incomplete
|
||||
# first hydration cannot seed this key.
|
||||
commit: if-missing
|
||||
# Single semantic writer: only protected pushes commit, so
|
||||
# pull_request clones stay read-only and the snapshot tracks main.
|
||||
# Explicit true (not on-change/if-missing) because the allocated-byte
|
||||
# heuristic can miss a same-size refresh, permanently stranding
|
||||
# consumers on a stale marker. v1.4.0 skips commit after
|
||||
# failed/cancelled steps, so a broken build cannot poison this key.
|
||||
commit: ${{ github.event_name != 'pull_request' && 'true' || 'false' }}
|
||||
|
||||
- name: Restore extension boundary artifacts from sticky disk
|
||||
id: boundary-sticky-restore
|
||||
if: matrix.group == 'extension-package-boundary' && github.event_name != 'workflow_dispatch' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw')
|
||||
shell: bash
|
||||
env:
|
||||
# Config/toolchain inputs the tree-OID gate below cannot see; must
|
||||
# stay identical to the seed step and check-lint's restore gate.
|
||||
BOUNDARY_CONFIG_HASH: ${{ hashFiles('tsconfig.json', 'tsconfig.plugin-sdk.dts.json', 'packages/plugin-sdk/tsconfig.json', 'scripts/check-extension-package-tsc-boundary.mjs', 'scripts/prepare-extension-package-boundary-artifacts.mjs', 'scripts/write-plugin-sdk-entry-dts.ts', 'scripts/lib/plugin-sdk-entrypoints.json', 'scripts/lib/plugin-sdk-entries.mjs', 'package.json', 'pnpm-lock.yaml') }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sticky_root=/var/tmp/openclaw-ext-boundary
|
||||
@@ -2229,12 +2251,12 @@ jobs:
|
||||
fi
|
||||
# Restore only when the generative source trees AND the runner
|
||||
# toolchain match the snapshotting run. Tree OIDs cover source
|
||||
# content exactly; the node/pnpm versions cover toolchain-only
|
||||
# changes (the sticky key already hashes config/scripts/lockfile, so
|
||||
# those force a fresh key). Restored artifacts are valid by
|
||||
# construction: no clock-skew mtime race can false-skip a rebuild,
|
||||
# and deleted/renamed sources or a toolchain bump build cold.
|
||||
current_trees="$({ git rev-parse HEAD:src/plugin-sdk HEAD:packages HEAD:extensions HEAD:src/auto-reply HEAD:src/types HEAD:src/plugins/types.ts HEAD:src/video-generation HEAD:src/channels; node --version; corepack pnpm --version 2>/dev/null || pnpm --version; })"
|
||||
# content exactly; the node/pnpm versions and the config hash cover
|
||||
# toolchain/config/scripts/lockfile changes the OIDs cannot see.
|
||||
# Restored artifacts are valid by construction: no clock-skew mtime
|
||||
# race can false-skip a rebuild, and deleted/renamed sources or a
|
||||
# toolchain bump build cold.
|
||||
current_trees="$({ git rev-parse HEAD:src/plugin-sdk HEAD:packages HEAD:extensions HEAD:src/auto-reply HEAD:src/types HEAD:src/plugins/types.ts HEAD:src/video-generation HEAD:src/channels; node --version; corepack pnpm --version 2>/dev/null || pnpm --version; echo "$BOUNDARY_CONFIG_HASH"; })"
|
||||
if [ ! -f "$sticky_root/.source-trees" ] || [ "$current_trees" != "$(cat "$sticky_root/.source-trees")" ]; then
|
||||
echo "boundary source trees changed since snapshot; building cold"
|
||||
echo "restored=false" >> "$GITHUB_OUTPUT"
|
||||
@@ -2310,7 +2332,9 @@ jobs:
|
||||
RUN_PROMPT_SNAPSHOTS: ${{ needs.preflight.outputs.run_prompt_snapshots }}
|
||||
OPENCLAW_ADDITIONAL_BOUNDARY_SHARD: ${{ matrix.boundary_shard || '' }}
|
||||
OPENCLAW_ADDITIONAL_BOUNDARY_CONCURRENCY: 4
|
||||
OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY: 6
|
||||
# Matches the boundary lane's 32 vCPU runner (cores/2); the
|
||||
# script's default caps at 6 to protect laptops.
|
||||
OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY: 16
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -2401,16 +2425,22 @@ jobs:
|
||||
|
||||
exit "$failures"
|
||||
|
||||
# commit: if-missing snapshots only the first run per key; seed the
|
||||
# payload before the sticky post-action flushes. rsync -aR mirrors the
|
||||
# repo-relative layout the restore step replays.
|
||||
# Only the protected writer refreshes the snapshot (pull_request mounts
|
||||
# never commit, so seeding there would burn wall clock on a discarded
|
||||
# clone). Seed the payload before the sticky post-action flushes.
|
||||
# rsync -aR mirrors the repo-relative layout the restore step replays.
|
||||
- name: Seed extension boundary sticky disk
|
||||
if: success() && steps.boundary-sticky-restore.outputs.restored == 'false' && matrix.group == 'extension-package-boundary' && github.event_name != 'workflow_dispatch' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw')
|
||||
if: success() && steps.boundary-sticky-restore.outputs.restored == 'false' && matrix.group == 'extension-package-boundary' && github.event_name != 'workflow_dispatch' && github.event_name != 'pull_request' && github.repository == 'openclaw/openclaw'
|
||||
shell: bash
|
||||
env:
|
||||
# Must stay identical to the restore gates above.
|
||||
BOUNDARY_CONFIG_HASH: ${{ hashFiles('tsconfig.json', 'tsconfig.plugin-sdk.dts.json', 'packages/plugin-sdk/tsconfig.json', 'scripts/check-extension-package-tsc-boundary.mjs', 'scripts/prepare-extension-package-boundary-artifacts.mjs', 'scripts/write-plugin-sdk-entry-dts.ts', 'scripts/lib/plugin-sdk-entrypoints.json', 'scripts/lib/plugin-sdk-entries.mjs', 'package.json', 'pnpm-lock.yaml') }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sticky_root=/var/tmp/openclaw-ext-boundary
|
||||
rm -rf "$sticky_root/dist" "$sticky_root/packages" "$sticky_root/extensions" "$sticky_root/.snapshot-ready"
|
||||
# A stale marker must not survive a mid-seed failure: drop readiness
|
||||
# first so a partial payload can never be restored as valid.
|
||||
rm -rf "$sticky_root/.snapshot-ready" "$sticky_root/.source-trees" "$sticky_root/dist" "$sticky_root/packages" "$sticky_root/extensions"
|
||||
if [ -d dist/plugin-sdk ]; then
|
||||
rsync -aR dist/plugin-sdk "$sticky_root/"
|
||||
fi
|
||||
@@ -2419,7 +2449,7 @@ jobs:
|
||||
fi
|
||||
find extensions -maxdepth 3 \( -name '.boundary-tsc.tsbuildinfo' -o -name '.boundary-tsc.stamp' \) \
|
||||
-exec rsync -aR {} "$sticky_root/" \;
|
||||
{ git rev-parse HEAD:src/plugin-sdk HEAD:packages HEAD:extensions HEAD:src/auto-reply HEAD:src/types HEAD:src/plugins/types.ts HEAD:src/video-generation HEAD:src/channels; node --version; corepack pnpm --version 2>/dev/null || pnpm --version; } > "$sticky_root/.source-trees"
|
||||
{ git rev-parse HEAD:src/plugin-sdk HEAD:packages HEAD:extensions HEAD:src/auto-reply HEAD:src/types HEAD:src/plugins/types.ts HEAD:src/video-generation HEAD:src/channels; node --version; corepack pnpm --version 2>/dev/null || pnpm --version; echo "$BOUNDARY_CONFIG_HASH"; } > "$sticky_root/.source-trees"
|
||||
touch "$sticky_root/.snapshot-ready"
|
||||
|
||||
# Validate docs (format, lint, broken links) only when docs files changed.
|
||||
|
||||
@@ -2383,6 +2383,67 @@ describe("ci workflow guards", () => {
|
||||
expect(source).toContain("blacksmith-8vcpu-windows-2025");
|
||||
});
|
||||
|
||||
it("keeps the extension boundary sticky disk on one protected key", () => {
|
||||
const workflow = readCiWorkflow();
|
||||
const additionalJob = workflow.jobs["check-additional-shard"];
|
||||
const checkShardJob = workflow.jobs["check-shard"];
|
||||
|
||||
// Light-run pole: cold prep + 122 plugin compiles scale with cores at
|
||||
// similar billed core-minutes.
|
||||
expect(additionalJob.strategy.matrix.include).toContainEqual({
|
||||
check_name: "check-additional-extension-package-boundary",
|
||||
group: "extension-package-boundary",
|
||||
runner: "blacksmith-32vcpu-ubuntu-2404",
|
||||
});
|
||||
const runStep = additionalJob.steps.find(
|
||||
(step: WorkflowStep) => step.name === "Run additional check shard",
|
||||
);
|
||||
expect(runStep.env.OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY).toBe(16);
|
||||
|
||||
// O(1) disks: Blacksmith caps sticky disks per installation, and the old
|
||||
// per-PR/per-config keys minted new disks until every mount 429-failed
|
||||
// fleet-wide. Snapshot validity lives in the in-job marker, not the key.
|
||||
const boundaryMount = additionalJob.steps.find(
|
||||
(step: WorkflowStep) => step.name === "Mount extension boundary sticky disk",
|
||||
);
|
||||
const lintMount = checkShardJob.steps.find(
|
||||
(step: WorkflowStep) => step.name === "Mount extension boundary sticky disk",
|
||||
);
|
||||
expect(boundaryMount.with.key).toBe("${{ github.repository }}-ext-boundary-v2");
|
||||
expect(lintMount.with.key).toBe(boundaryMount.with.key);
|
||||
// Single semantic writer: protected pushes commit explicitly (not
|
||||
// on-change/if-missing, whose allocated-byte heuristic can strand a stale
|
||||
// marker); PR clones and the lint consumer stay read-only.
|
||||
expect(boundaryMount.with.commit).toBe(
|
||||
"${{ github.event_name != 'pull_request' && 'true' || 'false' }}",
|
||||
);
|
||||
expect(lintMount.with.commit).toBe("false");
|
||||
|
||||
// The key no longer hashes config/scripts/lockfile, so every gate must
|
||||
// compose the identical marker fingerprint or restores silently tear.
|
||||
const restoreStep = additionalJob.steps.find(
|
||||
(step: WorkflowStep) => step.name === "Restore extension boundary artifacts from sticky disk",
|
||||
);
|
||||
const lintRestoreStep = checkShardJob.steps.find(
|
||||
(step: WorkflowStep) => step.name === "Restore extension boundary artifacts from sticky disk",
|
||||
);
|
||||
const seedStep = additionalJob.steps.find(
|
||||
(step: WorkflowStep) => step.name === "Seed extension boundary sticky disk",
|
||||
);
|
||||
const configHash = seedStep.env.BOUNDARY_CONFIG_HASH;
|
||||
expect(configHash).toContain("hashFiles(");
|
||||
expect(configHash).toContain("pnpm-lock.yaml");
|
||||
expect(restoreStep.env.BOUNDARY_CONFIG_HASH).toBe(configHash);
|
||||
expect(lintRestoreStep.env.BOUNDARY_CONFIG_HASH).toBe(configHash);
|
||||
for (const gate of [restoreStep, lintRestoreStep, seedStep]) {
|
||||
expect(gate.run).toContain('echo "$BOUNDARY_CONFIG_HASH"');
|
||||
}
|
||||
// Seeding is writer-only work: PR mounts never commit, so seeding there
|
||||
// would burn wall clock on a discarded clone.
|
||||
expect(seedStep.if).toContain("github.event_name != 'pull_request'");
|
||||
expect(seedStep.if).toContain("steps.boundary-sticky-restore.outputs.restored == 'false'");
|
||||
});
|
||||
|
||||
it("runs the session accessor ratchet as a visible additional check", () => {
|
||||
const workflow = readCiWorkflow();
|
||||
const additionalJob = workflow.jobs["check-additional-shard"];
|
||||
|
||||
Reference in New Issue
Block a user