From 7909ea698374c945cbddb7bb6ffdd07ad88a9072 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 21 Aug 2026 13:19:06 -0700 Subject: [PATCH] ci: remove the Test Performance Agent The workflow ran a Codex agent over the full test suite and pushed `test: optimize slow tests` straight to `main` under `contents: write`, with no pull request and no human review. Its gates were a path allowlist, a no-add/delete/rename rule, a non-decreasing total test count, and `pnpm check:changed` -- which covers changed lanes, not the full suite. Test optimization is exactly the class of change where a plausible edit can weaken coverage without moving the test count, so unattended landing is the wrong trade. Autonomous commits to `main` are not something this repo wants. It had also been inert since well before this. The daily-cadence gate excluded prior runs with `select(.status != "cancelled")`, but a finished cancelled run reports `status: "completed"` with `conclusion: "cancelled"` -- verified against run 32506655531, which that filter counts as a prior run. Its `concurrency` block sets `cancel-in-progress: false`, so main's push rate produced dozens of cancelled runs per hour and every trigger skipped, reporting green after ~2 minutes of doing nothing. No `test: optimize slow tests` commit has ever landed on `main`. `pnpm test:perf:groups` and the rest of the performance tooling it drove stay; they are useful by hand and documented in docs/reference/test.md. Repository secret OPENCLAW_TEST_PERF_AGENT_OPENAI_API_KEY now has no consumer and can be deleted. --- .github/workflows/test-performance-agent.yml | 280 ------------------- docs/ci.md | 5 - docs/reference/test.md | 2 +- 3 files changed, 1 insertion(+), 286 deletions(-) delete mode 100644 .github/workflows/test-performance-agent.yml diff --git a/.github/workflows/test-performance-agent.yml b/.github/workflows/test-performance-agent.yml deleted file mode 100644 index c80c9aeb76aa..000000000000 --- a/.github/workflows/test-performance-agent.yml +++ /dev/null @@ -1,280 +0,0 @@ -name: Test Performance Agent - -on: - workflow_run: # zizmor: ignore[dangerous-triggers] main-only test optimization after trusted CI; job gates repository, event, branch, actor, conclusion, current main SHA, and daily cadence before using write token - workflows: - - CI - types: - - completed - workflow_dispatch: - -permissions: - actions: read - contents: write - -concurrency: - group: test-performance-agent-main - cancel-in-progress: false - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" - TEST_PERF_BEFORE: .artifacts/test-perf/baseline-before.json - TEST_PERF_AFTER: .artifacts/test-perf/after-agent.json - TEST_PERF_COMPARE: .artifacts/test-perf/agent-compare.json - -jobs: - optimize-tests: - if: > - github.repository == 'openclaw/openclaw' && - (github.event_name == 'workflow_dispatch' || - (github.event.workflow_run.conclusion == 'success' && - github.event.workflow_run.event == 'push' && - github.event.workflow_run.head_branch == 'main' && - !endsWith(github.event.workflow_run.actor.login, '[bot]'))) - runs-on: ubuntu-24.04 - timeout-minutes: 240 - steps: - - name: Checkout - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: main - fetch-depth: 0 - persist-credentials: false - submodules: false - - - name: Gate trusted main activity and daily cadence - id: gate - env: - EVENT_NAME: ${{ github.event_name }} - GH_TOKEN: ${{ github.token }} - WORKFLOW_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} - run: | - set -euo pipefail - - if [ "$EVENT_NAME" != "workflow_run" ]; then - echo "run_agent=true" >> "$GITHUB_OUTPUT" - echo "base_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - exit 0 - fi - - for attempt in 1 2 3 4 5; do - if git fetch --no-tags origin main; then - break - fi - if [ "$attempt" = "5" ]; then - echo "Failed to fetch main after retries." >&2 - exit 1 - fi - echo "Fetch attempt ${attempt} failed; retrying." - sleep $((attempt * 2)) - done - - remote_main="$(git rev-parse origin/main)" - if [ "$remote_main" != "$WORKFLOW_HEAD_SHA" ]; then - echo "CI run is superseded by ${remote_main}; skipping test performance agent for ${WORKFLOW_HEAD_SHA}." - echo "run_agent=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - day_start="$(date -u +%Y-%m-%dT00:00:00Z)" - runs_json="$RUNNER_TEMP/test-performance-agent-runs.json" - gh api --method GET "repos/${GITHUB_REPOSITORY}/actions/workflows/test-performance-agent.yml/runs" \ - -f branch=main \ - -f event=workflow_run \ - -f per_page=50 > "$runs_json" - - prior_runs="$( - jq -r \ - --argjson current_run_id "$GITHUB_RUN_ID" \ - --arg day_start "$day_start" \ - '.workflow_runs[] - | select(.database_id != $current_run_id) - | select(.created_at >= $day_start) - | select(.status != "cancelled") - | select((.conclusion // "") != "skipped") - | [.database_id, .status, (.conclusion // ""), .created_at, .head_sha] - | @tsv' "$runs_json" - )" - - if [ -n "$prior_runs" ]; then - echo "Test performance agent already ran or is running today; skipping." - printf '%s\n' "$prior_runs" - echo "run_agent=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - echo "run_agent=true" >> "$GITHUB_OUTPUT" - echo "base_sha=${remote_main}" >> "$GITHUB_OUTPUT" - - - name: Setup Node environment - if: steps.gate.outputs.run_agent == 'true' - uses: ./.github/actions/setup-node-env - with: - cache-mode: restore - install-bun: "false" - - - name: Ensure test performance agent key exists - if: steps.gate.outputs.run_agent == 'true' - env: - OPENAI_API_KEY: ${{ secrets.OPENCLAW_TEST_PERF_AGENT_OPENAI_API_KEY || secrets.OPENAI_API_KEY }} - run: | - set -euo pipefail - if [ -z "${OPENAI_API_KEY:-}" ]; then - echo "Missing OPENCLAW_TEST_PERF_AGENT_OPENAI_API_KEY or OPENAI_API_KEY secret." >&2 - exit 1 - fi - - - name: Build baseline full-suite performance report - if: steps.gate.outputs.run_agent == 'true' - run: pnpm test:perf:groups --full-suite --allow-failures --output "$TEST_PERF_BEFORE" --limit 20 --top-files 40 - - - name: Run Codex test performance agent - if: steps.gate.outputs.run_agent == 'true' - uses: openai/codex-action@52fe01ec70a42f454c9d2ebd47598f9fd6893d56 - with: - openai-api-key: ${{ secrets.OPENCLAW_TEST_PERF_AGENT_OPENAI_API_KEY || secrets.OPENAI_API_KEY }} - prompt-file: .github/codex/prompts/test-performance-agent.md - model: ${{ vars.OPENCLAW_CI_OPENAI_MODEL_BARE }} - effort: high - sandbox: workspace-write - safety-strategy: drop-sudo - codex-args: '["--full-auto"]' - - - name: Enforce focused test performance patch - if: steps.gate.outputs.run_agent == 'true' - id: patch - run: | - set -euo pipefail - - untracked="$(git ls-files --others --exclude-standard)" - if [ -n "$untracked" ]; then - echo "Test performance agent created untracked files; forbidden:" - printf '%s\n' "$untracked" - exit 1 - fi - - added_deleted_or_renamed="$(git diff --name-status --diff-filter=ADR)" - if [ -n "$added_deleted_or_renamed" ]; then - echo "Test performance agent added, deleted, or renamed tracked files; forbidden:" - printf '%s\n' "$added_deleted_or_renamed" - exit 1 - fi - - bad_paths="$( - git diff --name-only | while IFS= read -r path; do - case "$path" in - apps/*|extensions/*|packages/*|scripts/*|src/*|test/*|ui/*) ;; - *) printf '%s\n' "$path" ;; - esac - done - )" - if [ -n "$bad_paths" ]; then - echo "Test performance agent touched forbidden paths:" - printf '%s\n' "$bad_paths" - exit 1 - fi - - if git diff --quiet; then - echo "has_changes=false" >> "$GITHUB_OUTPUT" - else - echo "has_changes=true" >> "$GITHUB_OUTPUT" - fi - - - name: Restore Node 24 path - if: steps.gate.outputs.run_agent == 'true' && steps.patch.outputs.has_changes == 'true' - run: - | # zizmor: ignore[github-env] NODE_BIN is set by the trusted local setup-node-env action in this same job - set -euo pipefail - export PATH="${NODE_BIN}:${PATH}" - echo "${NODE_BIN}" >> "$GITHUB_PATH" - node -v - corepack enable - pnpm -v - - - name: Run full-suite performance report after agent changes - if: steps.gate.outputs.run_agent == 'true' && steps.patch.outputs.has_changes == 'true' - run: pnpm test:perf:groups --full-suite --output "$TEST_PERF_AFTER" --limit 20 --top-files 40 - - - name: Compare test performance reports - if: steps.gate.outputs.run_agent == 'true' && steps.patch.outputs.has_changes == 'true' - run: pnpm test:perf:groups:compare "$TEST_PERF_BEFORE" "$TEST_PERF_AFTER" --output "$TEST_PERF_COMPARE" --limit 20 --top-files 40 - - - name: Enforce coverage-preserving test count - if: steps.gate.outputs.run_agent == 'true' && steps.patch.outputs.has_changes == 'true' - run: | - set -euo pipefail - node <<'NODE' - const fs = require("node:fs"); - const before = JSON.parse(fs.readFileSync(process.env.TEST_PERF_BEFORE, "utf8")); - const after = JSON.parse(fs.readFileSync(process.env.TEST_PERF_AFTER, "utf8")); - - if (before.failed) { - console.log("Baseline had failing configs; skipping total test-count comparison against partial report."); - process.exit(0); - } - - const beforeTests = before.totals?.testCount ?? 0; - const afterTests = after.totals?.testCount ?? 0; - if (afterTests < beforeTests) { - console.error(`Test count decreased from ${beforeTests} to ${afterTests}; refusing coverage-reducing patch.`); - process.exit(1); - } - console.log(`Test count preserved: ${beforeTests} -> ${afterTests}.`); - NODE - - - name: Check changed lanes - if: steps.gate.outputs.run_agent == 'true' && steps.patch.outputs.has_changes == 'true' - run: pnpm check:changed - - - name: Commit test performance updates - if: steps.gate.outputs.run_agent == 'true' && steps.patch.outputs.has_changes == 'true' - env: - GITHUB_TOKEN: ${{ github.token }} - TARGET_BRANCH: main - run: | - set -euo pipefail - - if git diff --quiet; then - echo "No test performance changes." - exit 0 - fi - - git config user.name "openclaw-test-performance-agent[bot]" - git config user.email "openclaw-test-performance-agent[bot]@users.noreply.github.com" - git add apps extensions packages scripts src test ui - git commit --no-verify -m "test: optimize slow tests" - - for attempt in 1 2 3 4 5; do - if ! git fetch --no-tags origin "${TARGET_BRANCH}"; then - echo "Fetch attempt ${attempt} failed; retrying." - sleep $((attempt * 2)) - continue - fi - if git push "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" HEAD:"${TARGET_BRANCH}"; then - exit 0 - fi - remote_main="$(git rev-parse "origin/${TARGET_BRANCH}")" - if [ "$remote_main" != "$(git rev-parse HEAD^)" ]; then - echo "main advanced; rebasing test performance update onto ${remote_main}." - if ! git rebase "origin/${TARGET_BRANCH}"; then - echo "Test performance update no longer applies cleanly; skipping stale update." - git rebase --abort || true - exit 0 - fi - pnpm check:changed - fi - echo "Test performance update attempt ${attempt} failed; retrying." - sleep $((attempt * 2)) - done - - echo "Failed to push test performance updates after retries." >&2 - exit 1 - - - name: Upload test performance artifacts - if: steps.gate.outputs.run_agent == 'true' && always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: test-performance-agent-${{ github.run_id }} - path: .artifacts/test-perf/ - if-no-files-found: ignore - retention-days: 14 diff --git a/docs/ci.md b/docs/ci.md index 3cc52aa816d2..17d39df2231f 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -55,7 +55,6 @@ dispatch. | `ios-build` | Swift lint, Debug and Release builds, focused simulator lifecycle tests, and the full release screenshot matrix when screenshot-pipeline owners changed | iOS/capture changes | | `android` | Android unit tests for both flavors plus one debug APK build | Android-relevant changes | | `openclaw/ci-gate` | Final aggregate: requires preflight and security; accepts skips only for manifest-disabled downstream lanes | Every non-draft CI run | -| `test-performance-agent` | Separate workflow: daily Codex slow-test optimization after trusted activity | Main CI success or manual dispatch | | `openclaw-performance` | Separate workflow: daily/on-demand Kova runtime performance reports with mock-provider, deep-profile, and GPT 5.6 live lanes | Scheduled and manual dispatch | The rare path-triggered `docker-seed-e2e` job selects only the executable @@ -726,10 +725,6 @@ Quality stays separate from security so quality findings can be scheduled, measu The `Docs Agent` workflow is an event-driven Codex maintenance lane for keeping existing docs aligned with recently landed changes. It has no pure schedule: a successful non-bot push CI run on `main` can trigger it, and manual dispatch can run it directly. Workflow-run invocations skip when `main` has moved on or when another non-skipped Docs Agent run was created in the last hour. When it runs, it reviews the commit range from the previous non-skipped Docs Agent source SHA to current `main`, so one hourly run can cover all main changes accumulated since the last docs pass. -### Test Performance Agent - -The `Test Performance Agent` workflow is an event-driven Codex maintenance lane for slow tests. It has no pure schedule: a successful non-bot push CI run on `main` can trigger it, but it skips if another workflow-run invocation already ran or is running that UTC day. Manual dispatch bypasses that daily activity gate. The lane builds a full-suite grouped Vitest performance report, lets Codex make only small coverage-preserving test performance fixes instead of broad refactors, then reruns the full-suite report and rejects changes that reduce the passing baseline test count. The grouped report records per-config wall time and max RSS on Linux and macOS, so the before/after comparison surfaces test memory deltas beside duration deltas. If the baseline has failing tests, Codex may fix only obvious failures and the after-agent full-suite report must pass before anything is committed. When `main` advances before the bot push lands, the lane rebases the validated patch, reruns `pnpm check:changed`, and retries the push; conflicting stale patches are skipped. It uses GitHub-hosted Ubuntu so the Codex action can keep the same drop-sudo safety posture as the docs agent. - ### Duplicate PRs After Merge The `Duplicate PRs After Merge` workflow is a manual maintainer workflow for post-land duplicate cleanup. It defaults to dry-run and only closes explicitly listed PRs when `apply=true`. Before mutating GitHub, it verifies that the landed PR is merged and that each duplicate has either a shared referenced issue or overlapping changed hunks. diff --git a/docs/reference/test.md b/docs/reference/test.md index b2103d742a10..5f24376016a7 100644 --- a/docs/reference/test.md +++ b/docs/reference/test.md @@ -198,7 +198,7 @@ If `pnpm test` flakes on a loaded host, rerun once before treating it as a regre - `pnpm test:perf:imports`: enables Vitest import-duration + import-breakdown reporting, while still using scoped lane routing for explicit file/directory targets. `pnpm test:perf:imports:changed` scopes the same profiling to files changed since `origin/main`. - `pnpm test:perf:changed:bench -- --ref ` benchmarks the routed changed-mode path against the native root-project run for the same committed git diff; `pnpm test:perf:changed:bench -- --worktree` benchmarks the current worktree change set without committing first. - `pnpm test:perf:profile:main` writes a CPU profile for the Vitest main thread (`.artifacts/vitest-main-profile`); `pnpm test:perf:profile:runner` writes CPU + heap profiles for the unit runner (`.artifacts/vitest-runner-profile`). -- `pnpm test:perf:groups --full-suite --allow-failures --output .artifacts/test-perf/baseline-before.json`: runs every full-suite Vitest leaf config serially and writes grouped duration data plus per-config JSON/log artifacts. Full-suite reports isolate files by default so retained module graphs and GC pauses from earlier files are not charged to later assertions; pass `-- --no-isolate` only when intentionally profiling shared-worker accumulation. The Test Performance Agent uses this as its baseline before attempting slow-test fixes. `pnpm test:perf:groups:compare .artifacts/test-perf/baseline-before.json .artifacts/test-perf/after-agent.json` compares grouped reports after a performance-focused change. +- `pnpm test:perf:groups --full-suite --allow-failures --output .artifacts/test-perf/baseline-before.json`: runs every full-suite Vitest leaf config serially and writes grouped duration data plus per-config JSON/log artifacts. Full-suite reports isolate files by default so retained module graphs and GC pauses from earlier files are not charged to later assertions; pass `-- --no-isolate` only when intentionally profiling shared-worker accumulation. `pnpm test:perf:groups:compare .artifacts/test-perf/baseline-before.json .artifacts/test-perf/after-agent.json` compares grouped reports after a performance-focused change. - Full, extension, and include-pattern shard runs update local timing data in `.artifacts/vitest-shard-timings.json`; later whole-config runs use those timings to balance slow and fast shards. Include-pattern CI shards append the shard name to the timing key, which keeps filtered shard timings visible without replacing whole-config timing data. Set `OPENCLAW_TEST_PROJECTS_TIMINGS=0` to ignore the local timing artifact. ## Benchmarks