From 72e42eed48502bd7b6c7be7d9cd9cd0a1aaebaff Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 11 Aug 2026 22:26:39 -0700 Subject: [PATCH] improve(ci): reduce main gating delays and add timing trends (#122441) * ci: stop publishing warm dependency snapshots Amp-Thread-ID: https://ampcode.com/threads/T-019ff3db-c467-70ad-8ed3-81f2ba94b0c0 * ci: isolate the high-variance source test shard Amp-Thread-ID: https://ampcode.com/threads/T-019ff3db-c467-70ad-8ed3-81f2ba94b0c0 * ci: guarantee rebuilt dependency snapshot publication Amp-Thread-ID: https://ampcode.com/threads/T-019ff3db-c467-70ad-8ed3-81f2ba94b0c0 * ci: add balanced main timing trends Amp-Thread-ID: https://ampcode.com/threads/T-019ff3db-c467-70ad-8ed3-81f2ba94b0c0 * fix(ci): fall back when Crabbox CLI is unavailable Amp-Thread-ID: https://ampcode.com/threads/T-019ff3db-c467-70ad-8ed3-81f2ba94b0c0 * Revert "fix(ci): fall back when Crabbox CLI is unavailable" This reverts commit 0583ac8a9d39c56dcb12fd432a42d41f4487684f. --------- Co-authored-by: Amp --- .github/actions/setup-node-env/action.yml | 30 +- .../setup-node-env/sticky-importers.sh | 62 +- .github/workflows/ci.yml | 11 + docs/ci.md | 10 +- package.json | 1 + scripts/ci-run-timings.mjs | 729 +++++++++++++++++- scripts/lib/ci-node-test-plan.mts | 5 +- test/scripts/ci-node-test-plan.test.ts | 3 + test/scripts/ci-run-timings.test.ts | 197 ++++- test/scripts/ci-workflow-guards.test.ts | 102 ++- 10 files changed, 1113 insertions(+), 37 deletions(-) diff --git a/.github/actions/setup-node-env/action.yml b/.github/actions/setup-node-env/action.yml index 6fbc44fbec79..6ba48e203dd0 100644 --- a/.github/actions/setup-node-env/action.yml +++ b/.github/actions/setup-node-env/action.yml @@ -139,12 +139,28 @@ runs: # disk here, this gate binds cooperating code, not hostile code: the # enforced trust boundary is the fork/dispatch runner gate in ci.yml, # and same-repo PR authors already hold repository write access. - # Explicit true (not on-change) because the allocated-byte heuristic - # can miss a fingerprint refresh whose reinstall keeps disk usage - # stable, permanently stranding consumers on a stale marker. The action - # skips commit after failed/cancelled steps, so a broken install cannot - # seed this key. - commit: ${{ inputs.save-sticky-disk == 'true' && github.event_name != 'pull_request' && 'true' || 'false' }} + # Warm validated snapshots stay read-only so asynchronous publication + # does not perpetually chase no-op commits. The canonical writer records + # the action's allocation baseline below; after any real capture and + # store pruning, preflight forces a verified delta before this action's + # post phase. The action also skips commit after failed/cancelled steps. + commit: ${{ inputs.save-sticky-disk == 'true' && github.event_name != 'pull_request' && 'on-change' || 'false' }} + + - name: Record sticky disk allocation baseline + if: inputs.sticky-disk == 'true' && inputs.save-sticky-disk == 'true' && github.event_name != 'pull_request' + shell: bash + run: | + set -euo pipefail + sticky_root=/var/tmp/openclaw-node-deps + initial_usage_bytes="$(df -B1 --output=used "$sticky_root" | tail -n1 | tr -d '[:space:]')" + if [[ ! "$initial_usage_bytes" =~ ^[0-9]+$ ]] || [[ "$initial_usage_bytes" -le 0 ]]; then + echo "::error::Could not record sticky disk allocation baseline" + exit 1 + fi + rebuild_signal="${RUNNER_TEMP:?}/openclaw-sticky-deps-rebuilt" + rm -f "$rebuild_signal" + echo "OPENCLAW_STICKY_INITIAL_USAGE_BYTES=$initial_usage_bytes" >> "$GITHUB_ENV" + echo "OPENCLAW_STICKY_REBUILD_SIGNAL=$rebuild_signal" >> "$GITHUB_ENV" - name: Restore and save Vitest transform cache if: inputs.vitest-fs-cache == 'true' && inputs.save-vitest-fs-cache == 'true' && runner.os != 'Windows' @@ -462,7 +478,7 @@ runs: # publishes the fingerprint; read-only clones are discarded at job # end, so capturing there would only burn shard wall clock. if [ "$STICKY_DISK" = "true" ] && [ "$STICKY_WRITER" = "true" ]; then - bash "$GITHUB_ACTION_PATH/sticky-importers.sh" capture "$STICKY_ROOT" "$GITHUB_WORKSPACE" "$OPENCLAW_STICKY_DEPS_FINGERPRINT" + bash "$GITHUB_ACTION_PATH/sticky-importers.sh" capture "$STICKY_ROOT" "$GITHUB_WORKSPACE" "$OPENCLAW_STICKY_DEPS_FINGERPRINT" "${OPENCLAW_STICKY_REBUILD_SIGNAL:?}" fi fi diff --git a/.github/actions/setup-node-env/sticky-importers.sh b/.github/actions/setup-node-env/sticky-importers.sh index 010c07d521a7..d0dc46d5e926 100644 --- a/.github/actions/setup-node-env/sticky-importers.sh +++ b/.github/actions/setup-node-env/sticky-importers.sh @@ -3,11 +3,12 @@ set -euo pipefail mode="${1:?mode is required}" sticky_root="${2:?sticky root is required}" -workspace="${3:?workspace is required}" +workspace="${3:-}" archive="$sticky_root/importer-node-modules.tar" archive_checksum="$sticky_root/.openclaw-importer-archive.sha256" importer_manifest="$sticky_root/importer-node-modules.manifest" marker="$sticky_root/.openclaw-deps-fingerprint" +force_commit_sentinel="$sticky_root/.openclaw-force-commit" script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" archive_sha256() { @@ -32,7 +33,9 @@ verify_importers() { case "$mode" in capture) + workspace="${workspace:?workspace is required}" fingerprint="${4:?fingerprint is required}" + rebuild_signal="${5:?rebuild signal is required}" mkdir -p "$sticky_root" list_file="$(mktemp)" temp_archive="$archive.tmp.$$" @@ -67,8 +70,10 @@ case "$mode" in # registry-backed importer resolution before trusting this snapshot. printf '%s\n' "$fingerprint" >"$temp_marker" mv "$temp_marker" "$marker" + : >"$rebuild_signal" ;; restore) + workspace="${workspace:?workspace is required}" if [[ ! -f "$archive" || ! -f "$archive_checksum" || ! -f "$importer_manifest" ]]; then echo "sticky importer archive, manifest, or checksum is missing under $sticky_root" >&2 exit 1 @@ -97,6 +102,61 @@ case "$mode" in exit 1 fi ;; + ensure-change) + initial_usage_bytes="${3:?initial usage bytes are required}" + if [[ ! "$initial_usage_bytes" =~ ^[0-9]+$ ]] || [[ "$initial_usage_bytes" -le 0 ]]; then + echo "invalid initial sticky disk usage: $initial_usage_bytes" >&2 + exit 2 + fi + current_usage_bytes() { + df -B1 --output=used "$sticky_root" | tail -n1 | tr -d '[:space:]' + } + allocation_delta() { + local current="$1" + if [[ "$current" -ge "$initial_usage_bytes" ]]; then + echo $((current - initial_usage_bytes)) + else + echo $((initial_usage_bytes - current)) + fi + } + + # The pinned StickyDisk action commits only when the absolute whole-disk + # allocation delta exceeds 4096 bytes. Measure against the same baseline + # after store pruning, then leave a 64 KiB margin for its post phase. + target_delta_bytes=65536 + max_sentinel_bytes=1048576 + current="$(current_usage_bytes)" + if [[ ! "$current" =~ ^[0-9]+$ ]] || [[ "$current" -le 0 ]]; then + echo "could not read current sticky disk usage" >&2 + exit 1 + fi + delta="$(allocation_delta "$current")" + if [[ "$delta" -le "$target_delta_bytes" ]] && + [[ -f "$force_commit_sentinel" ]] && + [[ "$(stat -c %s "$force_commit_sentinel")" -ge "$max_sentinel_bytes" ]]; then + : >"$force_commit_sentinel" + sync + current="$(current_usage_bytes)" + delta="$(allocation_delta "$current")" + fi + for _ in 1 2 3; do + if [[ "$delta" -gt "$target_delta_bytes" ]]; then + echo "Sticky dependency rebuild changed allocation by ${delta} bytes" + exit 0 + fi + bytes_needed=$((initial_usage_bytes + target_delta_bytes + 4096 - current)) + blocks_needed=$(((bytes_needed + 4095) / 4096)) + if [[ "$blocks_needed" -lt 1 ]]; then + blocks_needed=1 + fi + dd if=/dev/zero bs=4096 count="$blocks_needed" status=none >>"$force_commit_sentinel" + sync + current="$(current_usage_bytes)" + delta="$(allocation_delta "$current")" + done + echo "could not force a detectable sticky disk allocation change (delta: ${delta} bytes)" >&2 + exit 1 + ;; *) echo "unsupported sticky importer mode: $mode" >&2 exit 2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31bdb6e7675a..42ce49012812 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -898,6 +898,17 @@ jobs: echo "::warning::pnpm store remains above its 8 GiB maintenance ceiling after prune" fi + # StickyDisk's pinned on-change mode compares whole-filesystem + # allocation to its mount-time baseline. Only a successful real + # dependency capture creates this runner-local signal. Force and + # verify the delta after pruning so a same-size rebuild commits while + # validated warm restores remain read-only. + if [ -f "${OPENCLAW_STICKY_REBUILD_SIGNAL:?}" ]; then + bash "$GITHUB_WORKSPACE/.github/actions/setup-node-env/sticky-importers.sh" \ + ensure-change /var/tmp/openclaw-node-deps \ + "${OPENCLAW_STICKY_INITIAL_USAGE_BYTES:?}" + fi + # Run dependency-free security checks on a hosted runner in parallel with # scope detection. No downstream job waits for Python/pre-commit setup. security-fast: diff --git a/docs/ci.md b/docs/ci.md index 8bbca4d093e4..76c09c505cad 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -76,7 +76,9 @@ The default-branch ruleset requires the GitHub Actions-owned `openclaw/ci-gate` GitHub may mark superseded pull-request jobs as `cancelled` when a newer head lands. Treat that as CI noise unless the newest run for the same PR is also failing. Canonical `main` runs are not canceled after admission; when merge traffic arrives, GitHub replaces only the older pending run with the newest tip. Matrix jobs use `fail-fast: false`, and `build-artifacts` reports embedded channel, core-support-boundary, and gateway-watch failures directly instead of queuing tiny verifier jobs. The automatic CI concurrency key is versioned (`CI-v7-*`) so a GitHub-side zombie in an old queue group cannot indefinitely block newer main runs. Manual full-suite runs use `CI-manual-v1-*` and do not cancel in-progress runs. The plugin-list startup-memory guard keeps a 350 MiB ceiling on self-hosted Blacksmith Linux and allows 425 MiB on GitHub-hosted Linux, whose RSS baseline is higher for the same built CLI. -Use `pnpm ci:timings`, `pnpm ci:timings:recent`, or `node scripts/ci-run-timings.mjs ` to summarize wall time, queue time, slowest jobs, failures, and the `pnpm-store-warmup` fanout barrier from GitHub Actions. The in-workflow `ci-timings-summary` job exists in `ci.yml` but is currently disabled (`if: false`); run the timing helper locally instead. For build timing, check the `build-artifacts` job's `Build dist` step: `pnpm build:ci-artifacts` prints `[build-all] phase timings:` and includes `ui:build`; the job also uploads the `startup-memory` artifact. +Use `pnpm ci:timings`, `pnpm ci:timings:recent`, or `node scripts/ci-run-timings.mjs ` to summarize wall time, start delay, slowest jobs, failures, and the `pnpm-store-warmup` fanout barrier from GitHub Actions. Use `pnpm ci:timings:trend` for a 72-hour baseline and a latest-12-hours versus prior-12-hours comparison. Trend mode includes every main push outcome, cancellation/pass rates, and successful-run wall time, then loads a balanced latest/prior sample of at most 100 successful runs by default. Its detailed sample separates workflow admission, job dependency/gate delay (`job.created_at` minus the first job's creation), runner queue/start latency (`job.started_at` minus `job.created_at`), and execution; it also reports critical-path ownership and the actual GitHub API request count. Reruns use attempt-specific jobs and are excluded from run-level wall/admission distributions because GitHub retains the original workflow creation time. Raise or lower the detailed-run selection cap with `--detail-runs` (a run with more than 100 jobs requires multiple requests), emit JSON to stdout with `--json`, or save the same report with `--output .artifacts/ci-timings/trend.json`; missing output directories are created automatically. The baseline must cover at least two comparison windows. + +The in-workflow `ci-timings-summary` job exists in `ci.yml` but is currently disabled (`if: false`); run the timing helper locally instead. For build timing, check the `build-artifacts` job's `Build dist` step: `pnpm build:ci-artifacts` prints `[build-all] phase timings:` and includes `ui:build`; the job also uploads the `startup-memory` artifact. ## PR context and evidence @@ -117,11 +119,11 @@ The slowest Node test families are split or balanced so each job stays small wit - Auto-reply runs as balanced workers, with the reply subtree split into agent-runner, commands, dispatch, session, and state-routing shards. - Agentic gateway/server (control-plane) configs split across chat, auth, model, HTTP/plugin, runtime, and startup lanes instead of waiting on built artifacts. - Normal CI packs only isolated infra include-pattern shards into deterministic bundles of at most 64 test files, reducing the Node matrix without merging non-isolated command/cron, stateful agents-core, or gateway/server suites. Heavy fixed suites stay on 8 vCPU while the bundled and lower-weight lanes use 4 vCPU. -- Pull requests on the canonical repository reuse the changed-test resolver against the synthetic merged-tree diff. Precise changes run one targeted Node job; each selected test file gets its own process so stateful suite isolation remains intact. The planner combines sibling tests with import-graph dependents and falls back to the existing 14-job compact full-suite plan for workspace package, package/lockfile, shared harness, split-config, renamed, or deleted changes, public extension-contract changes, tests with special shard setup, partially resolved or empty targets, oversized path or target plans, and planner errors. Targeted plans always retain the full built-artifact boundary gate because its repository scanners cannot be derived from imports. `main` pushes run the same full compact suite: pending intermediate push events can be coalesced, so the newest surviving run must validate the complete integration tree rather than only its final single-push diff. Manual dispatches and release gates retain the full named per-shard matrix. +- Pull requests on the canonical repository reuse the changed-test resolver against the synthetic merged-tree diff. Precise changes run one targeted Node job; each selected test file gets its own process so stateful suite isolation remains intact. The planner combines sibling tests with import-graph dependents and falls back to the existing 14-job compact full-suite plan for workspace package, package/lockfile, shared harness, split-config, renamed, or deleted changes, public extension-contract changes, tests with special shard setup, partially resolved or empty targets, oversized path or target plans, and planner errors. Targeted plans always retain the full built-artifact boundary gate because its repository scanners cannot be derived from imports. `main` pushes run the same full compact suite: pending intermediate push events can be coalesced, so the newest surviving run must validate the complete integration tree rather than only its final single-push diff. Manual dispatches and release gates retain the full named per-shard matrix. Compact packing uses fleet timing hints without changing the bounded job count; the high-variance source/security group remains isolated so its tail does not serialize unrelated groups. - The full Node matrix admits the consistently slow serial tooling, auto-reply command shards, and broad core-fast cache writer first. This keeps the 28-job cap while preventing critical-path work and the next run's transform seed from slipping into a later wave. - Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. Include-pattern shards record timing entries using the CI shard name, so `.artifacts/vitest-shard-timings.json` can distinguish a whole config from a filtered shard. - Linux Node shard jobs persist Vitest's experimental filesystem module cache through the upstream Actions cache API, which Blacksmith transparently accelerates on its runners. Every CI shard is restore-only and unpacks the protected seed into its own runner-local root; the shard wrapper then gives concurrent Vitest processes separate live subdirectories. Only the non-cancelling daily or explicitly dispatched warmer saves a new immutable archive, so pull requests cannot publish transforms or mint per-PR cache families. The warmer launches each selected shard/config envelope in a fresh child process with concurrency one, preserving its include patterns and environment while reusing the same serial cache leaf. This prevents config-global state from leaking, avoids expanding filtered shards into whole configs, and retains transforms produced by the previous child. A transform-input fingerprint clears incompatible lockfile, package, tsconfig, and Vitest-config generations. The protected writer scans and prunes its restored cache to 75% after it exceeds 2 GiB. Vitest hashes module id, source content, environment, and resolved transform config, so ordinary partial source changes keep unchanged entries warm while changed modules miss safely. Coarse restore prefixes bridge workflow runs; normal Actions cache LRU and inactivity eviction bound old immutable archives. -- Trusted Blacksmith Linux Node jobs also bind the pnpm store and `node_modules` from one protected dependency disk per supported Node line. GitHub-hosted jobs, including manual dispatches, fork pull requests, and same-repo retries of both UI E2E jobs, use the Actions cache path instead. Package manifests, install settings, runner platform, and the exact Node patch stay out of the disk key; an exact runtime and install-input fingerprint decides whether a job reuses the tree or reinstalls and refreshes the same disk. Manifests are canonicalized before hashing. The audited direct root hooks retain only pnpm's install lifecycle scripts, so formatting and ordinary test/build script edits keep the warm dependency tree; unaudited lifecycle-hook drift fails closed until its source inputs join the fingerprint contract. Dependency, package-manager, hook-source, and lockfile changes always invalidate the snapshot. A matching fingerprint is necessary but not sufficient: setup also checks the importer archive and manifest checksums, then verifies registry-backed lockfile dependencies retained by postinstall against the package manifests Node resolves from their importers. Missing or stale importer content falls back to a fresh install instead of serving the root hoist. A pull request whose read-only snapshot is unusable detaches the workspace bind and installs into runner-local storage, avoiding slow writes to a clone it cannot publish. Sticky cold installs disable pnpm's inner fetch retries and make up to three bounded full-install attempts from the progressively warmed store; a timeout remains a failure. After a content-validated restore or frozen-lockfile install, setup disables pnpm's redundant pre-run dependency check: the repository intentionally prunes plugin-local `node_modules`, which pnpm otherwise treats as stale and repairs through unsafe concurrent implicit installs during shard fanout. Canonical main preflight is the sole writer and measures the store on every refresh, running `pnpm store prune` only after retired package versions push it above 8 GiB. Blacksmith snapshot publication is asynchronous even after a writer job completes, so the first run after a fresh key or fingerprint can remain cold; later content-validated exact-marker restores are the rollout proof. Required Blacksmith CI jobs and first-attempt same-repo pull requests get disposable clones, so dependency changes do not create new disks, competing snapshots, or a cache lock that can cancel builds. +- Trusted Blacksmith Linux Node jobs also bind the pnpm store and `node_modules` from one protected dependency disk per supported Node line. GitHub-hosted jobs, including manual dispatches, fork pull requests, and same-repo retries of both UI E2E jobs, use the Actions cache path instead. Package manifests, install settings, runner platform, and the exact Node patch stay out of the disk key; an exact runtime and install-input fingerprint decides whether a job reuses the tree or reinstalls and refreshes the same disk. Manifests are canonicalized before hashing. The audited direct root hooks retain only pnpm's install lifecycle scripts, so formatting and ordinary test/build script edits keep the warm dependency tree; unaudited lifecycle-hook drift fails closed until its source inputs join the fingerprint contract. Dependency, package-manager, hook-source, and lockfile changes always invalidate the snapshot. A matching fingerprint is necessary but not sufficient: setup also checks the importer archive and manifest checksums, then verifies registry-backed lockfile dependencies retained by postinstall against the package manifests Node resolves from their importers. Missing or stale importer content falls back to a fresh install instead of serving the root hoist. A pull request whose read-only snapshot is unusable detaches the workspace bind and installs into runner-local storage, avoiding slow writes to a clone it cannot publish. Sticky cold installs disable pnpm's inner fetch retries and make up to three bounded full-install attempts from the progressively warmed store; a timeout remains a failure. After a content-validated restore or frozen-lockfile install, setup disables pnpm's redundant pre-run dependency check: the repository intentionally prunes plugin-local `node_modules`, which pnpm otherwise treats as stale and repairs through unsafe concurrent implicit installs during shard fanout. Canonical main preflight is the sole writer and measures the store on every refresh, running `pnpm store prune` only after retired package versions push it above 8 GiB. Validated warm restores no longer publish no-op snapshots: the writer uses StickyDisk's allocation-change mode, records its mount-time allocation baseline, and only a successful dependency capture creates a runner-local rebuild signal. After store pruning, preflight compares the final whole-disk allocation to that baseline and, when needed, allocates a bounded sentinel until the absolute delta has a verified 64 KiB margin over StickyDisk's 4 KiB threshold. Blacksmith snapshot publication is asynchronous even after a writer job completes, so the first run after a fresh key or fingerprint can remain cold; later content-validated exact-marker restores are the rollout proof. Required Blacksmith CI jobs and first-attempt same-repo pull requests get disposable clones, so dependency changes do not create new disks, competing snapshots, or a cache lock that can cancel builds. - Node shard and build-artifact jobs also restore Node's portable on-disk compile cache through immutable Actions caches. Independent `test` and `build` namespaces prevent their writers from replacing each other's archives: the scheduled test warmer owns the protected test seed, while `build-artifacts` may publish at most one protected build archive per UTC day from trusted `main` pushes. PR and ordinary test jobs only read protected snapshots, so feature-branch bytecode never enters the shared seed and PR traffic creates no cache archives. This reuses V8 bytecode for Node-loaded orchestration, build tooling, and external dependencies across different checkout paths, including when only part of the source graph changes. Vitest child processes disable an inherited compile cache because coverage can be enabled inside dynamic configs and V8 coverage can lose source-position precision when scripts are deserialized from bytecode. - The build-artifact job also persists content-fingerprinted `build-all` step outputs. CI's self-built plugin SDK declarations hash the complete repository-owned TypeScript/JSON source graph, exclude installed and generated directories, and restore both flat declarations and package bridges after `tsdown` clears `dist`. Documentation, workflow, plugin, and other changes outside that graph can reuse the declaration snapshot; source changes rebuild it before the export gate runs. - Full declaration builds split `tsdown` into AI, workspace-package, and unified groups. Each group caches declarations only, then still rebuilds runtime JavaScript before restoring those declarations. Core or plugin changes therefore invalidate only the large unified graph, while workspace-package changes conservatively invalidate every dependent declaration group. Public full builds generally use an immutable Actions cache; coarse restore keys seed partial changes, per-group content fingerprints reject stale data, and GitHub's cache quota evicts old generations. The weekly Node 22 lane instead publishes a 14-day artifact after successful `main` runs and restores only artifacts whose immutable producer identity resolves to that workflow on `main`, avoiding quota churn without allowing PR code to write a shared cache. Private-QA declarations are never persisted in Actions caches because cache namespaces are not confidentiality boundaries. @@ -261,9 +263,11 @@ pnpm build # build dist when CI artifact/smok pnpm ios:build # generate and build the iOS app project pnpm ci:timings # summarize the latest origin/main push CI run pnpm ci:timings:recent # compare recent successful main CI runs +pnpm ci:timings:trend # 72h main baseline; latest 12h versus prior 12h node scripts/ci-run-timings.mjs # summarize wall time, queue time, and slowest jobs node scripts/ci-run-timings.mjs --latest-main # ignore issue/comment noise and choose origin/main push CI node scripts/ci-run-timings.mjs --recent 10 # compare recent successful main CI runs +node scripts/ci-run-timings.mjs --trend-hours 72 --compare-hours 12 --detail-runs 100 --output .artifacts/ci-timings/trend.json pnpm test:perf:groups --full-suite --allow-failures --output .artifacts/test-perf/baseline-before.json pnpm test:perf:groups:compare .artifacts/test-perf/baseline-before.json .artifacts/test-perf/after-agent.json pnpm test:startup:memory diff --git a/package.json b/package.json index 92805a628d8e..51e1daadba21 100644 --- a/package.json +++ b/package.json @@ -1563,6 +1563,7 @@ "ci:full-release": "node scripts/full-release-validation-at-sha.mjs", "ci:timings": "node scripts/ci-run-timings.mjs --latest-main", "ci:timings:recent": "node scripts/ci-run-timings.mjs --recent 10", + "ci:timings:trend": "node scripts/ci-run-timings.mjs --trend-hours 72 --compare-hours 12", "clean:dist": "node -e \"require('fs').rmSync('dist', {recursive: true, force: true})\"", "codex-app-server:protocol:check": "node --import tsx scripts/check-codex-app-server-protocol.ts", "codex-app-server:protocol:sync": "node --import tsx scripts/sync-codex-app-server-protocol.ts", diff --git a/scripts/ci-run-timings.mjs b/scripts/ci-run-timings.mjs index 7a9cc22ab59a..ca648804648d 100644 --- a/scripts/ci-run-timings.mjs +++ b/scripts/ci-run-timings.mjs @@ -2,23 +2,29 @@ // Summarizes GitHub Actions run/job timings for CI analysis. import { execFileSync } from "node:child_process"; +import { mkdirSync, writeFileSync } from "node:fs"; +import path from "node:path"; import { isDirectRunUrl } from "./lib/direct-run.mjs"; import { parsePositiveInt } from "./lib/numeric-options.mjs"; import { execPlainGh } from "./lib/plain-gh.mjs"; const DEFAULT_GITHUB_REPOSITORY = "openclaw/openclaw"; -const RUN_JOBS_PAGE_SIZE = 20; +const RUN_JOBS_PAGE_SIZE = 100; const RUN_JOBS_MAX_PAGES = 25; +const TREND_RUNS_MAX_PAGES = 100; +const DEFAULT_TREND_COMPARE_HOURS = 12; +const DEFAULT_TREND_DETAIL_RUNS = 100; const GH_JSON_RETRY_DELAYS_MS = [1_000, 3_000, 6_000]; function sleepSync(ms) { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); } -function parseJsonCommand(command, args, options = {}) { +function parseJsonCommand(command, args, onAttempt = null, options = {}) { let lastError; for (let attempt = 0; attempt <= GH_JSON_RETRY_DELAYS_MS.length; attempt += 1) { try { + onAttempt?.(); const stdout = command === "gh" ? execPlainGh(args, { @@ -53,8 +59,12 @@ function normalizeRunJob(job) { return { completedAt: job.completedAt ?? job.completed_at ?? null, conclusion: job.conclusion ?? "", + createdAt: job.createdAt ?? job.created_at ?? null, databaseId: job.databaseId ?? job.id, + labels: Array.isArray(job.labels) ? job.labels : [], name: job.name, + runnerGroupName: job.runnerGroupName ?? job.runner_group_name ?? null, + runnerName: job.runnerName ?? job.runner_name ?? null, startedAt: job.startedAt ?? job.started_at ?? null, status: job.status ?? "", }; @@ -92,6 +102,16 @@ function percentile(values, percentileValue) { return sorted[index]; } +function summarizeDistribution(values) { + return { + count: values.length, + max: values.length === 0 ? null : Math.max(...values), + p50: percentile(values, 0.5), + p90: percentile(values, 0.9), + p95: percentile(values, 0.95), + }; +} + function parseRunList(raw) { const parsed = JSON.parse(raw); return Array.isArray(parsed) ? parsed : []; @@ -295,24 +315,29 @@ function listRecentSuccessfulCiRuns(limit) { .slice(0, limit); } -function loadRun(runId) { - const run = parseJsonCommand("gh", [ - "run", - "view", - runId, - "--json", - "status,conclusion,createdAt,updatedAt", - ]); +/** + * @param {string | number} runId + * @param {number | null} [runAttempt] + */ +function loadRunJobs(runId, runAttempt = null) { const repository = process.env.GITHUB_REPOSITORY || DEFAULT_GITHUB_REPOSITORY; + const runPath = runAttempt === null ? `runs/${runId}` : `runs/${runId}/attempts/${runAttempt}`; const pages = []; let totalCount = null; + let requestCount = 0; for (let page = 1; page <= RUN_JOBS_MAX_PAGES; page += 1) { - const payload = parseJsonCommand("gh", [ - "api", - "-X", - "GET", - `repos/${repository}/actions/runs/${runId}/jobs?per_page=${RUN_JOBS_PAGE_SIZE}&page=${page}`, - ]); + const payload = parseJsonCommand( + "gh", + [ + "api", + "-X", + "GET", + `repos/${repository}/actions/${runPath}/jobs?per_page=${RUN_JOBS_PAGE_SIZE}&page=${page}`, + ], + () => { + requestCount += 1; + }, + ); pages.push(payload); const jobs = Array.isArray(payload.jobs) ? payload.jobs : []; totalCount = typeof payload.total_count === "number" ? payload.total_count : totalCount; @@ -323,9 +348,68 @@ function loadRun(runId) { break; } } + return { jobs: collectRunJobsFromPages(pages), requestCount }; +} + +function loadRun(runId) { + const run = parseJsonCommand("gh", [ + "run", + "view", + runId, + "--json", + "status,conclusion,createdAt,updatedAt", + ]); return { ...run, - jobs: collectRunJobsFromPages(pages), + jobs: loadRunJobs(runId).jobs, + }; +} + +function normalizeTrendRun(run) { + return { + conclusion: run.conclusion ?? "", + createdAt: run.createdAt ?? run.created_at ?? null, + databaseId: run.databaseId ?? run.id, + headSha: run.headSha ?? run.head_sha ?? "", + runAttempt: run.runAttempt ?? run.run_attempt ?? 1, + status: run.status ?? "", + updatedAt: run.updatedAt ?? run.updated_at ?? null, + url: run.url ?? run.html_url ?? "", + }; +} + +function listTrendCiRuns(cutoffMs) { + const repository = process.env.GITHUB_REPOSITORY || DEFAULT_GITHUB_REPOSITORY; + const runs = []; + let requestCount = 0; + for (let page = 1; page <= TREND_RUNS_MAX_PAGES; page += 1) { + const payload = parseJsonCommand( + "gh", + [ + "api", + "-X", + "GET", + `repos/${repository}/actions/workflows/ci.yml/runs?branch=main&event=push&per_page=100&page=${page}`, + ], + () => { + requestCount += 1; + }, + ); + const pageRuns = Array.isArray(payload.workflow_runs) + ? payload.workflow_runs.map(normalizeTrendRun) + : []; + runs.push(...pageRuns); + const oldestCreatedAt = parseTime(pageRuns.at(-1)?.createdAt); + if (pageRuns.length < 100 || (oldestCreatedAt !== null && oldestCreatedAt < cutoffMs)) { + break; + } + } + return { + requestCount, + runs: runs.filter((run) => { + const createdAt = parseTime(run.createdAt); + return createdAt !== null && createdAt >= cutoffMs; + }), }; } @@ -365,6 +449,422 @@ function summarizeJobs(run) { }; } +function isSyntheticTimingJob(job) { + return job.name?.startsWith("matrix.") || job.name === "ci-timings-summary"; +} + +function isAggregateTimingJob(job) { + return isSyntheticTimingJob(job) || job.name === "openclaw/ci-gate"; +} + +function summarizeTrendRun(run) { + const createdAt = parseTime(run.createdAt); + const updatedAt = parseTime(run.updatedAt); + const jobs = (run.jobs ?? []).filter((job) => !isSyntheticTimingJob(job)); + const createdJobs = jobs + .map((job) => ({ job, createdAt: parseTime(job.createdAt) })) + .filter((entry) => entry.createdAt !== null); + const firstJobCreatedAt = + createdJobs.length === 0 ? null : Math.min(...createdJobs.map((entry) => entry.createdAt)); + const activeJobs = jobs + .map((job) => ({ + completedAt: parseTime(job.completedAt), + createdAt: parseTime(job.createdAt), + job, + startedAt: parseTime(job.startedAt), + })) + .filter( + (entry) => + entry.job.conclusion !== "skipped" && + entry.startedAt !== null && + entry.completedAt !== null, + ); + const jobTimings = activeJobs + .filter((entry) => !isAggregateTimingJob(entry.job)) + .map((entry) => ({ + dependencyGatedSeconds: secondsBetween(firstJobCreatedAt, entry.createdAt), + executionSeconds: secondsBetween(entry.startedAt, entry.completedAt), + labels: entry.job.labels, + name: entry.job.name, + runnerGroupName: entry.job.runnerGroupName, + runnerName: entry.job.runnerName, + runnerQueueSeconds: secondsBetween(entry.createdAt, entry.startedAt), + })); + const completionOrder = activeJobs.toSorted( + (left, right) => + right.completedAt - left.completedAt || + String(left.job.name).localeCompare(String(right.job.name)) || + Number(left.job.databaseId ?? 0) - Number(right.job.databaseId ?? 0), + ); + // The run list keeps the original workflow creation time after a rerun. + // Attempt-specific job data remains useful, but exclude cross-attempt run + // wall/admission metrics rather than mixing it with the latest attempt. + const firstAttempt = run.runAttempt === 1; + + return { + admittedWallSeconds: firstAttempt ? secondsBetween(firstJobCreatedAt, updatedAt) : null, + conclusion: run.conclusion, + createdAt: run.createdAt, + databaseId: run.databaseId, + detailsLoaded: Array.isArray(run.jobs), + headSha: run.headSha, + jobTimings, + lastWorkOwner: + completionOrder.find((entry) => !isAggregateTimingJob(entry.job))?.job.name ?? null, + runAttempt: run.runAttempt, + status: run.status, + terminalOwner: completionOrder[0]?.job.name ?? null, + url: run.url, + wallSeconds: firstAttempt ? secondsBetween(createdAt, updatedAt) : null, + workflowAdmissionSeconds: firstAttempt ? secondsBetween(createdAt, firstJobCreatedAt) : null, + }; +} + +function summarizeOutcomes(runs) { + const counts = { + actionRequired: 0, + cancelled: 0, + failure: 0, + inProgress: 0, + neutral: 0, + other: 0, + pending: 0, + queued: 0, + skipped: 0, + stale: 0, + startupFailure: 0, + success: 0, + timedOut: 0, + total: runs.length, + }; + const conclusionKeys = new Map([ + ["action_required", "actionRequired"], + ["cancelled", "cancelled"], + ["failure", "failure"], + ["neutral", "neutral"], + ["skipped", "skipped"], + ["stale", "stale"], + ["startup_failure", "startupFailure"], + ["success", "success"], + ["timed_out", "timedOut"], + ]); + let completedNonCancelled = 0; + for (const run of runs) { + if (run.status === "completed" && run.conclusion !== "cancelled") { + completedNonCancelled += 1; + } + const key = + run.status === "completed" + ? conclusionKeys.get(run.conclusion) + : run.status === "in_progress" + ? "inProgress" + : run.status; + if (key && Object.hasOwn(counts, key)) { + counts[key] += 1; + } else { + counts.other += 1; + } + } + return { + ...counts, + cancellationRate: counts.total === 0 ? null : counts.cancelled / counts.total, + nonCancelledPassRate: + completedNonCancelled === 0 ? null : counts.success / completedNonCancelled, + }; +} + +function summarizeCriticalOwners(runSummaries) { + const counts = new Map(); + for (const run of runSummaries) { + if (run.lastWorkOwner) { + counts.set(run.lastWorkOwner, (counts.get(run.lastWorkOwner) ?? 0) + 1); + } + } + return [...counts.entries()] + .map(([name, runs]) => ({ name, runs })) + .toSorted((left, right) => right.runs - left.runs || left.name.localeCompare(right.name)); +} + +function summarizeTrendCohort(runs, runSummaries) { + const successfulRuns = runSummaries.filter( + (run) => run.status === "completed" && run.conclusion === "success", + ); + const jobTimings = successfulRuns.flatMap((run) => run.jobTimings); + return { + criticalOwners: summarizeCriticalOwners(successfulRuns), + jobMetrics: { + dependencyGatedSeconds: summarizeDistribution( + jobTimings.map((job) => job.dependencyGatedSeconds).filter((value) => value !== null), + ), + executionSeconds: summarizeDistribution( + jobTimings.map((job) => job.executionSeconds).filter((value) => value !== null), + ), + runnerQueueSeconds: summarizeDistribution( + jobTimings.map((job) => job.runnerQueueSeconds).filter((value) => value !== null), + ), + }, + outcomes: summarizeOutcomes(runs), + samples: { + detailedSuccessfulRuns: successfulRuns.filter((run) => run.detailsLoaded).length, + successfulRuns: successfulRuns.length, + timedJobs: jobTimings.length, + }, + runMetrics: { + admittedWallSeconds: summarizeDistribution( + successfulRuns.map((run) => run.admittedWallSeconds).filter((value) => value !== null), + ), + successfulWallSeconds: summarizeDistribution( + successfulRuns.map((run) => run.wallSeconds).filter((value) => value !== null), + ), + workflowAdmissionSeconds: summarizeDistribution( + successfulRuns.map((run) => run.workflowAdmissionSeconds).filter((value) => value !== null), + ), + }, + }; +} + +function summarizeJobNames(runSummaries, fromMs, toMs) { + const byName = new Map(); + for (const run of runSummaries) { + const createdAt = parseTime(run.createdAt); + if ( + createdAt === null || + createdAt < fromMs || + createdAt >= toMs || + run.status !== "completed" || + run.conclusion !== "success" + ) { + continue; + } + for (const job of run.jobTimings) { + const timings = byName.get(job.name) ?? []; + timings.push(job); + byName.set(job.name, timings); + } + } + return byName; +} + +function summarizeNamedJobComparison(runSummaries, priorWindow, comparisonWindow) { + const prior = summarizeJobNames(runSummaries, priorWindow.fromMs, priorWindow.toMs); + const comparison = summarizeJobNames( + runSummaries, + comparisonWindow.fromMs, + comparisonWindow.toMs, + ); + return [...new Set([...prior.keys(), ...comparison.keys()])] + .map((name) => { + const summarize = (timings) => ({ + executionSeconds: summarizeDistribution( + (timings ?? []).map((job) => job.executionSeconds).filter((value) => value !== null), + ), + runnerQueueSeconds: summarizeDistribution( + (timings ?? []).map((job) => job.runnerQueueSeconds).filter((value) => value !== null), + ), + }); + return { + comparison: summarize(comparison.get(name)), + name, + prior: summarize(prior.get(name)), + }; + }) + .toSorted( + (left, right) => + (right.comparison.executionSeconds.p90 ?? -1) - + (left.comparison.executionSeconds.p90 ?? -1) || left.name.localeCompare(right.name), + ); +} + +function metricDelta(comparison, prior, key) { + const comparisonValue = comparison?.[key] ?? null; + const priorValue = prior?.[key] ?? null; + return comparisonValue === null || priorValue === null ? null : comparisonValue - priorValue; +} + +/** + * Aggregates main CI runs into a baseline, previous comparison window, and latest window. + */ +export function summarizeTrendTimings(runs, options) { + const { compareDurationMs, generatedAtMs, trendDurationMs } = options; + const baselineFromMs = generatedAtMs - trendDurationMs; + const comparisonFromMs = generatedAtMs - compareDurationMs; + const priorFromMs = comparisonFromMs - compareDurationMs; + const inWindow = (run, fromMs, toMs) => { + const createdAt = parseTime(run.createdAt); + return createdAt !== null && createdAt >= fromMs && createdAt < toMs; + }; + const baselineRuns = runs.filter((run) => inWindow(run, baselineFromMs, generatedAtMs)); + const priorRuns = runs.filter((run) => inWindow(run, priorFromMs, comparisonFromMs)); + const comparisonRuns = runs.filter((run) => inWindow(run, comparisonFromMs, generatedAtMs)); + const runSummaries = baselineRuns + .map(summarizeTrendRun) + .toSorted((left, right) => Date.parse(right.createdAt) - Date.parse(left.createdAt)); + const baselineSummaries = runSummaries.filter((run) => + inWindow(run, baselineFromMs, generatedAtMs), + ); + const priorSummaries = runSummaries.filter((run) => inWindow(run, priorFromMs, comparisonFromMs)); + const comparisonSummaries = runSummaries.filter((run) => + inWindow(run, comparisonFromMs, generatedAtMs), + ); + const cohorts = { + baseline: summarizeTrendCohort(baselineRuns, baselineSummaries), + comparison: summarizeTrendCohort(comparisonRuns, comparisonSummaries), + prior: summarizeTrendCohort(priorRuns, priorSummaries), + }; + + return { + changes: { + executionP90Seconds: metricDelta( + cohorts.comparison.jobMetrics.executionSeconds, + cohorts.prior.jobMetrics.executionSeconds, + "p90", + ), + runnerQueueP95Seconds: metricDelta( + cohorts.comparison.jobMetrics.runnerQueueSeconds, + cohorts.prior.jobMetrics.runnerQueueSeconds, + "p95", + ), + successfulWallP50Seconds: metricDelta( + cohorts.comparison.runMetrics.successfulWallSeconds, + cohorts.prior.runMetrics.successfulWallSeconds, + "p50", + ), + successfulWallP90Seconds: metricDelta( + cohorts.comparison.runMetrics.successfulWallSeconds, + cohorts.prior.runMetrics.successfulWallSeconds, + "p90", + ), + workflowAdmissionP95Seconds: metricDelta( + cohorts.comparison.runMetrics.workflowAdmissionSeconds, + cohorts.prior.runMetrics.workflowAdmissionSeconds, + "p95", + ), + }, + cohorts, + jobs: summarizeNamedJobComparison( + runSummaries, + { fromMs: priorFromMs, toMs: comparisonFromMs }, + { fromMs: comparisonFromMs, toMs: generatedAtMs }, + ), + runs: runSummaries, + windows: { + baseline: { + from: new Date(baselineFromMs).toISOString(), + to: new Date(generatedAtMs).toISOString(), + }, + comparison: { + from: new Date(comparisonFromMs).toISOString(), + to: new Date(generatedAtMs).toISOString(), + }, + prior: { + from: new Date(priorFromMs).toISOString(), + to: new Date(comparisonFromMs).toISOString(), + }, + }, + }; +} + +function formatDistribution(summary) { + return [ + `n=${summary.count}`, + `p50=${formatSeconds(summary.p50)}`, + `p90=${formatSeconds(summary.p90)}`, + `p95=${formatSeconds(summary.p95)}`, + `max=${formatSeconds(summary.max)}`, + ].join(" "); +} + +function formatDelta(value) { + if (value === null) { + return ""; + } + return `${value > 0 ? "+" : ""}${formatSeconds(value)}`; +} + +function formatPercent(value) { + return value === null ? "" : `${(value * 100).toFixed(1)}%`; +} + +function printTrendCohort(name, cohort) { + const outcomes = cohort.outcomes; + console.log(`\n${name}`); + console.log( + [ + `runs=${outcomes.total}`, + `success=${outcomes.success}`, + `failure=${outcomes.failure}`, + `timed-out=${outcomes.timedOut}`, + `startup-failure=${outcomes.startupFailure}`, + `action-required=${outcomes.actionRequired}`, + `neutral=${outcomes.neutral}`, + `skipped=${outcomes.skipped}`, + `stale=${outcomes.stale}`, + `cancelled=${outcomes.cancelled}`, + `queued=${outcomes.queued}`, + `pending=${outcomes.pending}`, + `in-progress=${outcomes.inProgress}`, + `other=${outcomes.other}`, + `pass=${formatPercent(outcomes.nonCancelledPassRate)}`, + `cancelled-rate=${formatPercent(outcomes.cancellationRate)}`, + ].join(" "), + ); + console.log( + `successful wall ${formatDistribution(cohort.runMetrics.successfulWallSeconds)}`, + ); + console.log( + `workflow admission ${formatDistribution(cohort.runMetrics.workflowAdmissionSeconds)}`, + ); + console.log( + `dependency gating ${formatDistribution(cohort.jobMetrics.dependencyGatedSeconds)}`, + ); + console.log(`runner queue ${formatDistribution(cohort.jobMetrics.runnerQueueSeconds)}`); + console.log(`job execution ${formatDistribution(cohort.jobMetrics.executionSeconds)}`); + console.log( + `detail sample ${cohort.samples.detailedSuccessfulRuns}/${cohort.samples.successfulRuns} successful runs, ${cohort.samples.timedJobs} jobs`, + ); +} + +function printTrendReport(report) { + const { baseline, comparison, prior } = report.cohorts; + console.log( + `CI trend: ${report.options.trendHours}h baseline; latest ${report.options.compareHours}h vs prior ${report.options.compareHours}h`, + ); + console.log( + `API requests=${report.apiRequests.total} (run-list=${report.apiRequests.runList}, jobs=${report.apiRequests.jobs}); detailed=${report.sampling.detailedSuccessfulRuns}/${report.sampling.eligibleSuccessfulRuns} successful runs`, + ); + printTrendCohort("Baseline", baseline); + printTrendCohort("Prior comparison window", prior); + printTrendCohort("Latest comparison window", comparison); + + console.log("\nLatest minus prior"); + console.log( + [ + `wall-p50=${formatDelta(report.changes.successfulWallP50Seconds)}`, + `wall-p90=${formatDelta(report.changes.successfulWallP90Seconds)}`, + `admission-p95=${formatDelta(report.changes.workflowAdmissionP95Seconds)}`, + `queue-p95=${formatDelta(report.changes.runnerQueueP95Seconds)}`, + `execution-p90=${formatDelta(report.changes.executionP90Seconds)}`, + ].join(" "), + ); + + if (comparison.criticalOwners.length > 0) { + console.log("\nLatest critical-path owners"); + for (const owner of comparison.criticalOwners.slice(0, 15)) { + console.log(`${String(owner.name).padEnd(56)} ${owner.runs} run(s)`); + } + } + + const timedJobs = report.jobs.filter((job) => job.comparison.executionSeconds.count > 0); + if (timedJobs.length > 0) { + console.log("\nLatest job execution p90"); + for (const job of timedJobs.slice(0, 15)) { + console.log( + `${String(job.name).padEnd(56)} latest=${formatSeconds(job.comparison.executionSeconds.p90).padStart(6)} prior=${formatSeconds(job.prior.executionSeconds.p90).padStart(6)}`, + ); + } + } +} + function printSection(title, jobs, metric) { console.log(title); for (const job of jobs) { @@ -378,9 +878,17 @@ function printSection(title, jobs, metric) { * Parses CI run timing CLI arguments. */ export function parseRunTimingArgs(args) { + let compareHours = DEFAULT_TREND_COMPARE_HOURS; + let compareHoursSpecified = false; + let detailRuns = DEFAULT_TREND_DETAIL_RUNS; + let detailRunsSpecified = false; let explicitRunId; + let json = false; let limit = 15; + let limitSpecified = false; + let outputPath = null; let recentLimit = null; + let trendHours = null; let useLatestMain = false; for (let index = 0; index < args.length; index += 1) { @@ -392,9 +900,14 @@ export function parseRunTimingArgs(args) { useLatestMain = true; continue; } + if (arg === "--json") { + json = true; + continue; + } const limitOption = consumePositiveIntFlag(args, index, "--limit"); if (limitOption) { limit = limitOption.value; + limitSpecified = true; index = limitOption.nextIndex; continue; } @@ -404,6 +917,32 @@ export function parseRunTimingArgs(args) { index = recentOption.nextIndex; continue; } + const trendOption = consumePositiveIntFlag(args, index, "--trend-hours"); + if (trendOption) { + trendHours = trendOption.value; + index = trendOption.nextIndex; + continue; + } + const compareOption = consumePositiveIntFlag(args, index, "--compare-hours"); + if (compareOption) { + compareHours = compareOption.value; + compareHoursSpecified = true; + index = compareOption.nextIndex; + continue; + } + const detailOption = consumePositiveIntFlag(args, index, "--detail-runs"); + if (detailOption) { + detailRuns = detailOption.value; + detailRunsSpecified = true; + index = detailOption.nextIndex; + continue; + } + const outputOption = consumeStringFlag(args, index, "--output"); + if (outputOption) { + outputPath = outputOption.value; + index = outputOption.nextIndex; + continue; + } if (arg.startsWith("-")) { throw new Error(`Unknown CI run timing option: ${arg}`); } @@ -413,10 +952,32 @@ export function parseRunTimingArgs(args) { explicitRunId = arg; } + if (recentLimit !== null && (explicitRunId || useLatestMain)) { + throw new Error("--recent cannot be combined with a run id or --latest-main"); + } + if (explicitRunId && useLatestMain) { + throw new Error("A run id cannot be combined with --latest-main"); + } + if (trendHours !== null) { + if (explicitRunId || useLatestMain || recentLimit !== null || limitSpecified) { + throw new Error("--trend-hours cannot be combined with single-run or --recent options"); + } + if (trendHours < compareHours * 2) { + throw new Error("--trend-hours must cover at least two --compare-hours windows"); + } + } else if (compareHoursSpecified || detailRunsSpecified || json || outputPath !== null) { + throw new Error("--compare-hours, --detail-runs, --json, and --output require --trend-hours"); + } + return { + compareHours, + detailRuns, explicitRunId, + json, limit, + outputPath, recentLimit, + trendHours, useLatestMain, }; } @@ -443,10 +1004,138 @@ function consumePositiveIntFlag(args, index, flag) { }; } -async function main() { - const { explicitRunId, limit, recentLimit, useLatestMain } = parseRunTimingArgs( - process.argv.slice(2), +function consumeStringFlag(args, index, flag) { + const arg = args[index]; + const inlinePrefix = `${flag}=`; + if (arg.startsWith(inlinePrefix)) { + const value = arg.slice(inlinePrefix.length); + if (!value) { + throw new Error(`${flag} requires a value`); + } + return { nextIndex: index, value }; + } + if (arg !== flag) { + return null; + } + const value = args[index + 1]; + if (!value || value.startsWith("-")) { + throw new Error(`${flag} requires a value`); + } + return { nextIndex: index + 1, value }; +} + +function selectTrendDetailCandidates(runs, generatedAtMs, compareDurationMs, limit) { + const comparisonFromMs = generatedAtMs - compareDurationMs; + const priorFromMs = comparisonFromMs - compareDurationMs; + const successfulRuns = runs.filter( + (run) => run.status === "completed" && run.conclusion === "success", ); + const comparison = successfulRuns.filter( + (run) => (parseTime(run.createdAt) ?? 0) >= comparisonFromMs, + ); + const prior = successfulRuns.filter((run) => { + const createdAt = parseTime(run.createdAt) ?? 0; + return createdAt >= priorFromMs && createdAt < comparisonFromMs; + }); + const older = successfulRuns.filter((run) => (parseTime(run.createdAt) ?? 0) < priorFromMs); + const selected = []; + let comparisonIndex = 0; + let priorIndex = 0; + while ( + selected.length < limit && + (comparisonIndex < comparison.length || priorIndex < prior.length) + ) { + if (comparisonIndex < comparison.length && selected.length < limit) { + selected.push(comparison[comparisonIndex]); + comparisonIndex += 1; + } + if (priorIndex < prior.length && selected.length < limit) { + selected.push(prior[priorIndex]); + priorIndex += 1; + } + } + return [ + ...selected, + ...comparison.slice(comparisonIndex), + ...prior.slice(priorIndex), + ...older, + ].slice(0, limit); +} + +function runTrendReport(options) { + const generatedAtMs = Date.now(); + const trendDurationMs = options.trendHours * 60 * 60 * 1000; + const compareDurationMs = options.compareHours * 60 * 60 * 1000; + const listed = listTrendCiRuns(generatedAtMs - trendDurationMs); + const eligibleRuns = listed.runs.filter( + (run) => run.status === "completed" && run.conclusion === "success", + ); + const detailCandidates = selectTrendDetailCandidates( + listed.runs, + generatedAtMs, + compareDurationMs, + options.detailRuns, + ); + console.error( + `[ci-timings] loading job details for ${detailCandidates.length}/${eligibleRuns.length} successful runs; expect at least ${detailCandidates.length} job API requests`, + ); + const detailsByRun = new Map(); + let jobsRequestCount = 0; + for (const run of detailCandidates) { + const loaded = loadRunJobs(run.databaseId, run.runAttempt); + jobsRequestCount += loaded.requestCount; + detailsByRun.set(run.databaseId, loaded.jobs); + } + const runs = listed.runs.map((run) => + detailsByRun.has(run.databaseId) ? { ...run, jobs: detailsByRun.get(run.databaseId) } : run, + ); + const summary = summarizeTrendTimings(runs, { + compareDurationMs, + generatedAtMs, + trendDurationMs, + }); + const repository = process.env.GITHUB_REPOSITORY || DEFAULT_GITHUB_REPOSITORY; + return { + apiRequests: { + jobs: jobsRequestCount, + runList: listed.requestCount, + total: listed.requestCount + jobsRequestCount, + }, + generatedAt: new Date(generatedAtMs).toISOString(), + options: { + compareHours: options.compareHours, + detailRuns: options.detailRuns, + trendHours: options.trendHours, + }, + repository, + sampling: { + detailedSuccessfulRuns: detailCandidates.length, + eligibleSuccessfulRuns: eligibleRuns.length, + }, + ...summary, + }; +} + +async function main() { + const options = parseRunTimingArgs(process.argv.slice(2)); + const { explicitRunId, limit, recentLimit, useLatestMain } = options; + if (options.trendHours !== null) { + const report = runTrendReport(options); + const reportJson = `${JSON.stringify(report, null, 2)}\n`; + if (options.outputPath) { + mkdirSync(path.dirname(options.outputPath), { recursive: true }); + writeFileSync(options.outputPath, reportJson); + } + if (options.json) { + process.stdout.write(reportJson); + } else { + printTrendReport(report); + if (options.outputPath) { + console.log(`\nJSON report: ${options.outputPath}`); + } + } + return; + } if (recentLimit !== null) { for (const run of listRecentSuccessfulCiRuns(recentLimit)) { const summary = summarizeJobs(loadRun(run.databaseId)); diff --git a/scripts/lib/ci-node-test-plan.mts b/scripts/lib/ci-node-test-plan.mts index b237c4e69e48..d116708863ef 100644 --- a/scripts/lib/ci-node-test-plan.mts +++ b/scripts/lib/ci-node-test-plan.mts @@ -296,7 +296,10 @@ const COMPACT_GROUP_SECONDS_HINTS = new Map([ // Fork-per-file isolation parallelizes poorly on 4 vCPU; keep it on the // 8 vCPU class, where it still runs a measured ~90s under fleet load. ["core-unit-fast-isolated", 90], - ["core-unit-src-security", 205], + // In 35 green main runs on 2026-08-11/12, compact large jobs owned the + // critical tail 15 times and reached p90=457s. This group's former 205s + // hint repeatedly packed another 58s of serial work beside that tail. + ["core-unit-src-security", 295], ["core-unit-support", 17], ]); // Advisory per-file wall-clock hints (seconds) for stripe balancing, measured diff --git a/test/scripts/ci-node-test-plan.test.ts b/test/scripts/ci-node-test-plan.test.ts index 9f7b8d70d9f1..ef681c84c5aa 100644 --- a/test/scripts/ci-node-test-plan.test.ts +++ b/test/scripts/ci-node-test-plan.test.ts @@ -240,6 +240,9 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { // pairing them starves model visibility and repeatedly hits its timeout. expect(jobOf("agentic-agents-core-models")).not.toBe(jobOf("core-runtime-media-ui")); expect(jobOf("core-runtime-media-ui")).not.toBe(jobOf("core-unit-src-security")); + expect( + compact[jobOf("core-unit-src-security")]?.groups.map((group) => group.shard_name), + ).toEqual(["core-unit-src-security"]); // Cheap stripes may legally co-locate in one bin; only existence matters. expect(jobOf("core-unit-fast-1")).toBeGreaterThanOrEqual(0); expect(jobOf("core-unit-fast-2")).toBeGreaterThanOrEqual(0); diff --git a/test/scripts/ci-run-timings.test.ts b/test/scripts/ci-run-timings.test.ts index 890965140522..21ed89916d50 100644 --- a/test/scripts/ci-run-timings.test.ts +++ b/test/scripts/ci-run-timings.test.ts @@ -1,4 +1,9 @@ // Ci Run Timings tests cover ci run timings script behavior. +import { spawnSync } from "node:child_process"; +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; import { collectRunJobsFromPages, @@ -128,16 +133,24 @@ describe("scripts/ci-run-timings.mjs", () => { { completedAt: "2026-06-01T13:26:16Z", conclusion: "success", + createdAt: null, databaseId: 101, + labels: [], name: "preflight", + runnerGroupName: null, + runnerName: null, startedAt: "2026-06-01T13:25:16Z", status: "completed", }, { completedAt: "2026-06-01T13:28:00Z", conclusion: "failure", + createdAt: null, databaseId: 102, + labels: [], name: "ci-timings-summary", + runnerGroupName: null, + runnerName: null, startedAt: "2026-06-01T13:27:00Z", status: "completed", }, @@ -250,18 +263,53 @@ describe("scripts/ci-run-timings.mjs", () => { it("ignores pnpm passthrough sentinels when parsing monitor args", () => { expect(parseRunTimingArgs(["--latest-main", "--", "--limit", "3"])).toEqual({ + compareHours: 12, + detailRuns: 100, explicitRunId: undefined, + json: false, limit: 3, + outputPath: null, recentLimit: null, + trendHours: null, useLatestMain: true, }); }); it("parses strict positive integer monitor limits", () => { - expect(parseRunTimingArgs(["123456", "--limit=7", "--recent", "4"])).toEqual({ + expect(parseRunTimingArgs(["123456", "--limit=7"])).toEqual({ + compareHours: 12, + detailRuns: 100, explicitRunId: "123456", + json: false, limit: 7, - recentLimit: 4, + outputPath: null, + recentLimit: null, + trendHours: null, + useLatestMain: false, + }); + expect(parseRunTimingArgs(["--recent", "4"]).recentLimit).toBe(4); + }); + + it("parses bounded trend comparison and JSON report options", () => { + expect( + parseRunTimingArgs([ + "--trend-hours=72", + "--compare-hours", + "12", + "--detail-runs=80", + "--json", + "--output", + "ci-trend.json", + ]), + ).toEqual({ + compareHours: 12, + detailRuns: 80, + explicitRunId: undefined, + json: true, + limit: 15, + outputPath: "ci-trend.json", + recentLimit: null, + trendHours: 72, useLatestMain: false, }); }); @@ -273,6 +321,9 @@ describe("scripts/ci-run-timings.mjs", () => { ["--limit=1e3"], ["--recent", "recent"], ["--recent", "0"], + ["--trend-hours", "0"], + ["--compare-hours", "1.5"], + ["--detail-runs", "all"], ]) { expect(() => parseRunTimingArgs(args)).toThrow("must be a positive integer"); } @@ -285,6 +336,10 @@ describe("scripts/ci-run-timings.mjs", () => { ["--limit", "-h"], ["--recent"], ["--recent", "-h"], + ["--trend-hours"], + ["--compare-hours", "--json"], + ["--detail-runs"], + ["--output="], ]) { expect(() => parseRunTimingArgs(args)).toThrow("requires a value"); } @@ -298,4 +353,142 @@ describe("scripts/ci-run-timings.mjs", () => { "Unexpected CI run id argument: 789012", ); }); + + it("rejects ambiguous monitor modes and incomplete comparison windows", () => { + expect(() => parseRunTimingArgs(["--recent", "3", "--latest-main"])).toThrow( + "--recent cannot be combined", + ); + expect(() => parseRunTimingArgs(["123456", "--latest-main"])).toThrow( + "A run id cannot be combined", + ); + expect(() => parseRunTimingArgs(["--trend-hours", "72", "--recent", "3"])).toThrow( + "--trend-hours cannot be combined", + ); + expect(() => parseRunTimingArgs(["--trend-hours", "23"])).toThrow("must cover at least two"); + expect(() => parseRunTimingArgs(["--json"])).toThrow("require --trend-hours"); + }); + + it("balances trend samples, keeps reruns attempt-specific, and counts API retries", () => { + const fixtureDir = mkdtempSync(path.join(tmpdir(), "openclaw-ci-timings-")); + const fakeGhPath = path.join(fixtureDir, "gh"); + const reportPath = path.join(fixtureDir, "reports", "trend.json"); + const retryMarkerPath = path.join(fixtureDir, "retried"); + const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); + const fixtureNowMs = Date.now(); + writeFileSync( + fakeGhPath, + `#!/usr/bin/env node +const { existsSync, writeFileSync } = require("node:fs"); +const args = process.argv.slice(2); +const endpoint = args.find((arg) => arg.startsWith("repos/")) ?? ""; +const now = Number(process.env.FIXTURE_NOW_MS); +const iso = (offsetMs) => new Date(now + offsetMs).toISOString(); +if (endpoint.includes("actions/workflows/ci.yml/runs?")) { + console.log(JSON.stringify({ workflow_runs: [ + { id: 101, status: "completed", conclusion: "success", created_at: iso(-60 * 60_000), updated_at: iso(-50 * 60_000), head_sha: "latest", run_attempt: 1, html_url: "https://example.test/101" }, + { id: 104, status: "completed", conclusion: "success", created_at: iso(-90 * 60_000), updated_at: iso(-80 * 60_000), head_sha: "latest-unsampled", run_attempt: 1, html_url: "https://example.test/104" }, + { id: 102, status: "completed", conclusion: "cancelled", created_at: iso(-2 * 60 * 60_000), updated_at: iso(-119 * 60_000), head_sha: "cancelled", run_attempt: 1, html_url: "https://example.test/102" }, + { id: 106, status: "completed", conclusion: "timed_out", created_at: iso(-3 * 60 * 60_000), updated_at: iso(-2 * 60 * 60_000 - 50 * 60_000), head_sha: "timed-out", run_attempt: 1, html_url: "https://example.test/106" }, + { id: 103, status: "completed", conclusion: "success", created_at: iso(-13 * 60 * 60_000), updated_at: iso(-12 * 60 * 60_000 - 50 * 60_000), head_sha: "prior-rerun", run_attempt: 2, html_url: "https://example.test/103" } + ] })); +} else if (endpoint.includes("actions/runs/101/attempts/1/jobs?")) { + if (!existsSync(process.env.FIXTURE_RETRY_MARKER)) { + writeFileSync(process.env.FIXTURE_RETRY_MARKER, "retried\\n"); + console.error("HTTP 502: fixture transient failure"); + process.exit(1); + } + const runStart = now - 60 * 60_000; + const at = (seconds) => new Date(runStart + seconds * 1000).toISOString(); + console.log(JSON.stringify({ total_count: 4, jobs: [ + { id: 1, name: "preflight", status: "completed", conclusion: "success", created_at: at(10), started_at: at(20), completed_at: at(60), labels: ["blacksmith-4vcpu-ubuntu-2404"], runner_name: "blacksmith-test", runner_group_name: "blacksmith" }, + { id: 2, name: "checks-node-compact-large-1", status: "completed", conclusion: "success", created_at: at(60), started_at: at(65), completed_at: at(500), labels: ["blacksmith-8vcpu-ubuntu-2404"], runner_name: "blacksmith-test", runner_group_name: "blacksmith" }, + { id: 3, name: "openclaw/ci-gate", status: "completed", conclusion: "success", created_at: at(500), started_at: at(501), completed_at: at(510), labels: ["ubuntu-24.04"], runner_name: "GitHub Actions", runner_group_name: "GitHub Actions" }, + { id: 4, name: "matrix.synthetic", status: "completed", conclusion: "success", created_at: at(510), started_at: at(511), completed_at: at(520), labels: ["ubuntu-24.04"], runner_name: "GitHub Actions", runner_group_name: "GitHub Actions" } + ] })); +} else if (endpoint.includes("actions/runs/103/attempts/2/jobs?")) { + const runStart = now - 12 * 60 * 60_000 - 55 * 60_000; + const at = (seconds) => new Date(runStart + seconds * 1000).toISOString(); + console.log(JSON.stringify({ total_count: 2, jobs: [ + { id: 5, name: "preflight", status: "completed", conclusion: "success", created_at: at(10), started_at: at(18), completed_at: at(58), labels: ["blacksmith-4vcpu-ubuntu-2404"], runner_name: "blacksmith-test", runner_group_name: "blacksmith" }, + { id: 6, name: "checks-node-compact-large-1", status: "completed", conclusion: "success", created_at: at(58), started_at: at(62), completed_at: at(470), labels: ["blacksmith-8vcpu-ubuntu-2404"], runner_name: "blacksmith-test", runner_group_name: "blacksmith" } + ] })); +} else { + console.error("unexpected gh invocation", args.join(" ")); + process.exit(2); +} +`, + ); + chmodSync(fakeGhPath, 0o755); + + try { + const result = spawnSync( + process.execPath, + [ + "scripts/ci-run-timings.mjs", + "--trend-hours", + "24", + "--compare-hours", + "12", + "--detail-runs", + "2", + "--json", + "--output", + reportPath, + ], + { + cwd: repositoryRoot, + encoding: "utf8", + env: { + ...process.env, + FIXTURE_NOW_MS: String(fixtureNowMs), + FIXTURE_RETRY_MARKER: retryMarkerPath, + OPENCLAW_GH_BIN: fakeGhPath, + }, + }, + ); + + expect(result.status, result.stderr).toBe(0); + const report = JSON.parse(result.stdout); + expect(JSON.parse(readFileSync(reportPath, "utf8"))).toEqual(report); + expect(report.apiRequests).toEqual({ jobs: 3, runList: 1, total: 4 }); + expect(report.sampling).toEqual({ + detailedSuccessfulRuns: 2, + eligibleSuccessfulRuns: 3, + }); + expect(report.cohorts.comparison.outcomes).toMatchObject({ + cancelled: 1, + cancellationRate: 0.25, + nonCancelledPassRate: 2 / 3, + success: 2, + timedOut: 1, + total: 4, + }); + expect(report.cohorts.prior.runMetrics.successfulWallSeconds.p50).toBeNull(); + expect(report.cohorts.prior.runMetrics.workflowAdmissionSeconds.p50).toBeNull(); + expect(report.cohorts.prior.samples.detailedSuccessfulRuns).toBe(1); + expect(report.cohorts.prior.jobMetrics.executionSeconds.count).toBe(2); + expect(report.cohorts.comparison.jobMetrics.runnerQueueSeconds).toMatchObject({ + count: 2, + max: 10, + p95: 10, + }); + expect(report.cohorts.comparison.jobMetrics.dependencyGatedSeconds.p95).toBe(50); + expect(report.cohorts.comparison.runMetrics.workflowAdmissionSeconds.p95).toBe(10); + expect(report.cohorts.comparison.criticalOwners).toEqual([ + { name: "checks-node-compact-large-1", runs: 1 }, + ]); + expect( + report.jobs.find((job: { name: string }) => job.name === "checks-node-compact-large-1"), + ).toMatchObject({ + comparison: { executionSeconds: { count: 1 } }, + prior: { executionSeconds: { count: 1 } }, + }); + expect(report.runs[0].jobTimings.map((job: { name: string }) => job.name)).toEqual([ + "preflight", + "checks-node-compact-large-1", + ]); + } finally { + rmSync(fixtureDir, { force: true, recursive: true }); + } + }); }); diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 8b7fc52f5857..dee497ac8a7f 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -2936,6 +2936,9 @@ NODE expect(maintainStep.run).toContain('store_dir="${PNPM_CONFIG_STORE_DIR:?}"'); expect(maintainStep.run).toContain('PNPM_CONFIG_STORE_DIR="$store_dir" pnpm store prune'); expect(maintainStep.run).toContain('>> "$GITHUB_STEP_SUMMARY"'); + expect(maintainStep.run).toContain('if [ -f "${OPENCLAW_STICKY_REBUILD_SIGNAL:?}" ]'); + expect(maintainStep.run).toContain("ensure-change /var/tmp/openclaw-node-deps"); + expect(maintainStep.run).toContain('"${OPENCLAW_STICKY_INITIAL_USAGE_BYTES:?}"'); expect(workflow.jobs["pnpm-store-warmup"].if).toContain("github.ref == 'refs/heads/main'"); expect(workflow.jobs["pnpm-store-warmup"].if).toContain( "github.repository == 'openclaw/openclaw'", @@ -2975,6 +2978,9 @@ NODE const mountStep = action.runs.steps.find( (step: WorkflowStep) => step.name === "Mount dependency sticky disk", ); + const baselineStep = action.runs.steps.find( + (step: WorkflowStep) => step.name === "Record sticky disk allocation baseline", + ); const cleanupStep = action.runs.steps.find( (step: WorkflowStep) => step.name === "Register sticky bind cleanup", ); @@ -3024,7 +3030,18 @@ NODE "${{ github.repository }}-node-deps-bind-v6-${{ inputs.node-version }}", ); expect(mountStep.with.commit).toBe( - "${{ inputs.save-sticky-disk == 'true' && github.event_name != 'pull_request' && 'true' || 'false' }}", + "${{ inputs.save-sticky-disk == 'true' && github.event_name != 'pull_request' && 'on-change' || 'false' }}", + ); + expect(baselineStep).toMatchObject({ + if: "inputs.sticky-disk == 'true' && inputs.save-sticky-disk == 'true' && github.event_name != 'pull_request'", + }); + expect(baselineStep.run).toContain('df -B1 --output=used "$sticky_root"'); + expect(baselineStep.run).toContain( + 'echo "OPENCLAW_STICKY_INITIAL_USAGE_BYTES=$initial_usage_bytes"', + ); + expect(baselineStep.run).toContain('echo "OPENCLAW_STICKY_REBUILD_SIGNAL=$rebuild_signal"'); + expect(action.runs.steps.indexOf(mountStep)).toBeLessThan( + action.runs.steps.indexOf(baselineStep), ); expect(cleanupStep).toMatchObject({ if: "inputs.sticky-disk == 'true'", @@ -3115,6 +3132,7 @@ NODE 'bash "$GITHUB_ACTION_PATH/sticky-importers.sh" capture "$STICKY_ROOT" "$GITHUB_WORKSPACE" "$OPENCLAW_STICKY_DEPS_FINGERPRINT"', ), ); + expect(installStep.run).toContain('"${OPENCLAW_STICKY_REBUILD_SIGNAL:?}"'); // The content-validated snapshot or successful install already owns // dependency validation. pnpm's redundant check sees intentionally pruned // plugin importers as stale, so it must not mutate during shard fanout. @@ -3256,6 +3274,7 @@ NODE const rootOptionalDependency = path.join(rootModules, "optional-ipaddr"); const importerDependency = path.join(importerModules, "ipaddr.js"); const helper = path.resolve(".github/actions/setup-node-env/sticky-importers.sh"); + const rebuildSignal = path.join(root, "rebuilt"); const lockfile = [ "lockfileVersion: '9.0'", "importers:", @@ -3324,7 +3343,15 @@ NODE ); writeFileSync(path.join(rootModules, "root-sentinel"), "before", "utf8"); - execFileSync("bash", [helper, "capture", stickyRoot, workspace, "fingerprint-a"]); + execFileSync("bash", [ + helper, + "capture", + stickyRoot, + workspace, + "fingerprint-a", + rebuildSignal, + ]); + expect(existsSync(rebuildSignal)).toBe(true); rmSync(importerModules, { recursive: true }); writeFileSync(path.join(rootModules, "root-sentinel"), "after", "utf8"); execFileSync("bash", [helper, "restore", stickyRoot, workspace]); @@ -3336,6 +3363,19 @@ NODE expect(readFileSync(path.join(stickyRoot, ".openclaw-deps-fingerprint"), "utf8")).toBe( "fingerprint-a\n", ); + rmSync(rebuildSignal); + execFileSync("bash", [ + helper, + "capture", + stickyRoot, + workspace, + "fingerprint-b", + rebuildSignal, + ]); + expect(existsSync(rebuildSignal)).toBe(true); + expect(readFileSync(path.join(stickyRoot, ".openclaw-deps-fingerprint"), "utf8")).toBe( + "fingerprint-b\n", + ); // Recreate the reported failure shape: a marker-matching archive can be // structurally valid yet omit the importer-local override, causing Node @@ -3360,12 +3400,14 @@ NODE ); expect(existsSync(importerModules)).toBe(false); + rmSync(rebuildSignal); const failedCapture = spawnSync( "bash", - [helper, "capture", stickyRoot, workspace, "fingerprint-b"], + [helper, "capture", stickyRoot, workspace, "fingerprint-c", rebuildSignal], { encoding: "utf8" }, ); expect(failedCapture.status).toBe(1); + expect(existsSync(rebuildSignal)).toBe(false); expect(failedCapture.stderr).toContain( "ipaddr.js expected ipaddr.js@2.4.0, resolved ipaddr.js@1.9.1", ); @@ -3374,6 +3416,59 @@ NODE } }); + it("forces StickyDisk's allocation delta after a successful rebuild", () => { + const root = mkdtempSync(path.join(tmpdir(), "openclaw-sticky-allocation-")); + try { + const fakeBin = path.join(root, "bin"); + const stickyRoot = path.join(root, "sticky"); + const usageFile = path.join(root, "usage"); + const helper = path.resolve(".github/actions/setup-node-env/sticky-importers.sh"); + mkdirSync(fakeBin, { recursive: true }); + mkdirSync(stickyRoot, { recursive: true }); + // Start one allocation block below the action's baseline. A fixed append + // can be cancelled by this shrink; the helper must measure the net delta. + writeFileSync(usageFile, "995904\n", "utf8"); + writeFileSync( + path.join(fakeBin, "df"), + '#!/usr/bin/env bash\necho Used\ncat "$OPENCLAW_TEST_USAGE_FILE"\n', + "utf8", + ); + writeFileSync( + path.join(fakeBin, "dd"), + `#!/usr/bin/env bash +set -euo pipefail +count=0 +for arg in "$@"; do + case "$arg" in count=*) count="\${arg#count=}" ;; esac +done +usage="$(<"$OPENCLAW_TEST_USAGE_FILE")" +printf '%s\n' "$((usage + count * 4096))" > "$OPENCLAW_TEST_USAGE_FILE" +`, + "utf8", + ); + writeFileSync(path.join(fakeBin, "sync"), "#!/usr/bin/env bash\nexit 0\n", "utf8"); + for (const command of ["df", "dd", "sync"]) { + chmodSync(path.join(fakeBin, command), 0o755); + } + + const result = spawnSync("bash", [helper, "ensure-change", stickyRoot, "1000000"], { + encoding: "utf8", + env: { + ...process.env, + OPENCLAW_TEST_USAGE_FILE: usageFile, + PATH: `${fakeBin}:${process.env.PATH ?? ""}`, + }, + }); + + expect(result.status, result.stderr).toBe(0); + const finalUsage = Number(readFileSync(usageFile, "utf8").trim()); + expect(Math.abs(finalUsage - 1_000_000)).toBeGreaterThan(65_536); + expect(result.stdout).toContain("Sticky dependency rebuild changed allocation"); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it("fingerprints dependency install inputs without ordinary script churn", () => { const root = mkdtempSync(path.join(tmpdir(), "openclaw-dependency-fingerprint-")); try { @@ -3717,6 +3812,7 @@ NODE ...process.env, GITHUB_STEP_SUMMARY: summaryPath, OPENCLAW_PNPM_STORE_MAX_KIB: "-1", + OPENCLAW_STICKY_REBUILD_SIGNAL: path.join(maintenanceRoot, "not-rebuilt"), PNPM_CONFIG_STORE_DIR: storeDir, }, });