From f3a2f9f8a7376ea430498437ed3c613eca7fca74 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 08:06:06 -0700 Subject: [PATCH] perf(ci): gate prompt-snapshot regeneration on the generator's blast radius (#109083) check-prompt-snapshots costs 200-310s per PR regenerating real prompt stacks, but snapshots only change when the generator's import graph or its fixtures do. The manifest now computes that reach (same pattern as QA smoke gating: always-run surface regex incl. the codex extension whose test API loads via a dynamic module id, plus import-graph walk from the snapshot helper; deletions and null diffs fail safe to running). Unaffected PR diffs skip the lane with an explicit log line; pushes and dispatches always run. --- .github/workflows/ci.yml | 20 +++++++++-- scripts/lib/ci-changed-node-test-plan.d.mts | 5 +++ scripts/lib/ci-changed-node-test-plan.mjs | 36 +++++++++++++++++++ .../scripts/ci-changed-node-test-plan.test.ts | 29 +++++++++++++++ 4 files changed, 88 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2fc6fc52110..db73e01258c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,6 +104,7 @@ jobs: run_checks_fast: ${{ steps.manifest.outputs.run_checks_fast }} historical_target: ${{ steps.manifest.outputs.historical_target }} run_qa_smoke_ci: ${{ steps.manifest.outputs.run_qa_smoke_ci }} + run_prompt_snapshots: ${{ steps.manifest.outputs.run_prompt_snapshots }} checks_fast_core_matrix: ${{ steps.manifest.outputs.checks_fast_core_matrix }} run_plugin_contracts_shards: ${{ steps.manifest.outputs.run_plugin_contracts_shards }} plugin_contracts_matrix: ${{ steps.manifest.outputs.plugin_contracts_matrix }} @@ -554,6 +555,13 @@ jobs: eventName !== "pull_request" || typeof changedNodeTestPlan.hasQaSmokeAffectingChange !== "function" || changedNodeTestPlan.hasQaSmokeAffectingChange(changedPaths); + // Prompt snapshots only change when the generator's import graph or + // its fixtures do; unaffected PR diffs skip the regeneration lane. + const changedScopeHasPromptSnapshotImpact = + changedPaths === null || + eventName !== "pull_request" || + typeof changedNodeTestPlan.hasPromptSnapshotAffectingChange !== "function" || + changedNodeTestPlan.hasPromptSnapshotAffectingChange(changedPaths); const runBuildArtifacts = runNodeFull && changedScopeHasBuildImpact; const runQaSmokeCi = runNodeFull && @@ -614,6 +622,7 @@ jobs: historical_target: historicalTarget, compatibility_target: compatibilityTarget, run_qa_smoke_ci: runQaSmokeCi, + run_prompt_snapshots: runNodeFull && changedScopeHasPromptSnapshotImpact, checks_fast_core_matrix: createMatrix(checksFastCoreTasks), run_plugin_contracts_shards: runPluginContractShards, plugin_contracts_matrix: createMatrix( @@ -2021,6 +2030,7 @@ jobs: - name: Run additional check shard env: ADDITIONAL_CHECK_GROUP: ${{ matrix.group }} + RUN_PROMPT_SNAPSHOTS: ${{ needs.preflight.outputs.run_prompt_snapshots }} OPENCLAW_ADDITIONAL_BOUNDARY_SHARD: ${{ matrix.boundary_shard || '' }} OPENCLAW_ADDITIONAL_BOUNDARY_CONCURRENCY: 4 OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY: 6 @@ -2051,8 +2061,14 @@ jobs: prompt-snapshots) # No presence fallback: the boundary runner previously invoked # this unconditionally, and silent success would drop snapshot - # drift coverage. - run_check "prompt:snapshots:check" pnpm prompt:snapshots:check + # drift coverage. The manifest gates the lane on the generator's + # import graph and fixtures; diffs outside both cannot change + # generated snapshots. + if [ "$RUN_PROMPT_SNAPSHOTS" != "true" ]; then + echo "[skip] changed scope cannot affect generated prompt snapshots" + else + run_check "prompt:snapshots:check" pnpm prompt:snapshots:check + fi ;; session-accessor-boundary) if [ ! -f scripts/check-session-accessor-boundary.mjs ]; then diff --git a/scripts/lib/ci-changed-node-test-plan.d.mts b/scripts/lib/ci-changed-node-test-plan.d.mts index 1b1d8d7e7626..af8e61997c49 100644 --- a/scripts/lib/ci-changed-node-test-plan.d.mts +++ b/scripts/lib/ci-changed-node-test-plan.d.mts @@ -14,4 +14,9 @@ export function createChangedNodeTestShards( export function hasBuildArtifactAffectingChange(changedPaths: string[]): boolean; +export function hasPromptSnapshotAffectingChange( + changedPaths: string[], + options?: { cwd?: string }, +): boolean; + export function hasQaSmokeAffectingChange(changedPaths: string[]): boolean; diff --git a/scripts/lib/ci-changed-node-test-plan.mjs b/scripts/lib/ci-changed-node-test-plan.mjs index e40481431d5c..6469f6e35279 100644 --- a/scripts/lib/ci-changed-node-test-plan.mjs +++ b/scripts/lib/ci-changed-node-test-plan.mjs @@ -91,6 +91,42 @@ export function hasQaSmokeAffectingChange(changedPaths, options = {}) { return hasImportGraphImpactOnTargets(sourcePaths, [QA_SMOKE_RUNTIME_ENTRY], cwd); } +// Surfaces the prompt-snapshot check exercises outside its generator's +// relative import graph: the snapshot fixtures and generator scripts, the +// codex extension (its test API loads through a dynamic bundled-plugin module +// id the graph walk cannot see), and the gate's own orchestration — changes +// to the gate must not be able to skip the gated lane. +const PROMPT_SNAPSHOT_SURFACE_RE = + /^(?:test\/(?:helpers\/agents|fixtures\/agents\/prompt-snapshots)|extensions\/codex)\/|^scripts\/(?:generate-prompt-snapshots\.ts|prompt-snapshot-files\.[cm]?[jt]s)$|^scripts\/lib\/ci-changed-node-test-plan\.mjs$|^\.github\/(?:workflows\/ci\.yml$|actions\/)|^(?:package\.json|pnpm-lock\.yaml|npm-shrinkwrap\.json|pnpm-workspace\.yaml)$/u; +// The generator renders real prompt-layer stacks, so its runtime blast radius +// is the snapshot helper's import graph (auto-reply prompts, channel typing, +// plugin-sdk agent harness, codex catalog fixtures). +const PROMPT_SNAPSHOT_ENTRY = "test/helpers/agents/happy-path-prompt-snapshots.ts"; + +/** + * True when a changed path can influence generated prompt snapshots: it + * touches the snapshot surface directly, or the generator's import graph + * reaches it. Diffs outside both cannot change generator output, so the + * manifest may skip the check lane. + */ +export function hasPromptSnapshotAffectingChange(changedPaths, options = {}) { + const cwd = options.cwd ?? process.cwd(); + if (changedPaths.some((changedPath) => PROMPT_SNAPSHOT_SURFACE_RE.test(changedPath))) { + return true; + } + const sourcePaths = changedPaths.filter( + (changedPath) => changedPath.startsWith("src/") && !isTestFileTarget(changedPath), + ); + if (sourcePaths.length === 0) { + return false; + } + // Deleted sources cannot be graphed; fail safe to running the check. + if (sourcePaths.some((changedPath) => !existsSync(path.join(cwd, changedPath)))) { + return true; + } + return hasImportGraphImpactOnTargets(sourcePaths, [PROMPT_SNAPSHOT_ENTRY], cwd); +} + function createBoundaryShard() { // Boundary tests scan the source tree (including test files) and build // their own fixtures; they do not consume the built dist artifact. When the diff --git a/test/scripts/ci-changed-node-test-plan.test.ts b/test/scripts/ci-changed-node-test-plan.test.ts index 9dc3871e4365..b546f8e978ea 100644 --- a/test/scripts/ci-changed-node-test-plan.test.ts +++ b/test/scripts/ci-changed-node-test-plan.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from "vitest"; import { createChangedNodeTestShards, hasBuildArtifactAffectingChange, + hasPromptSnapshotAffectingChange, hasQaSmokeAffectingChange, } from "../../scripts/lib/ci-changed-node-test-plan.mjs"; import { listGitTrackedFiles } from "../../src/test-utils/repo-files.js"; @@ -80,6 +81,34 @@ describe("CI changed Node test plan", () => { expect(hasQaSmokeAffectingChange(["src/infra/definitely-deleted-module.ts"])).toBe(true); }); + it("classifies prompt-snapshot impact by surface and generator import graph", () => { + // Inside the generator's import graph -> regenerated output can change. + expect(hasPromptSnapshotAffectingChange(["src/auto-reply/reply/prompt-prelude.ts"])).toBe(true); + // The codex extension loads through a dynamic bundled-plugin module id the + // graph walk cannot see; it stays on the always-run surface. + expect(hasPromptSnapshotAffectingChange(["extensions/codex/src/index.ts"])).toBe(true); + expect( + hasPromptSnapshotAffectingChange([ + "test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/README.md", + ]), + ).toBe(true); + expect(hasPromptSnapshotAffectingChange(["scripts/generate-prompt-snapshots.ts"])).toBe(true); + // The gate's own orchestration must not be able to skip the gated lane. + expect(hasPromptSnapshotAffectingChange([".github/workflows/ci.yml"])).toBe(true); + expect(hasPromptSnapshotAffectingChange(["scripts/lib/ci-changed-node-test-plan.mjs"])).toBe( + true, + ); + // Outside the surface and the generator graph -> the lane may skip. + expect(hasPromptSnapshotAffectingChange(["ui/src/app.ts"])).toBe(false); + expect(hasPromptSnapshotAffectingChange(["extensions/discord/src/index.ts"])).toBe(false); + expect(hasPromptSnapshotAffectingChange(["docs/index.md"])).toBe(false); + expect(hasPromptSnapshotAffectingChange(["test/scripts/ci-node-test-plan.test.ts"])).toBe( + false, + ); + // Deleted source files cannot be graphed; fail safe to running the check. + expect(hasPromptSnapshotAffectingChange(["src/infra/definitely-deleted-module.ts"])).toBe(true); + }); + it("fails safe to the full plan for broad changes", () => { expect(createChangedNodeTestShards(["package.json"])).toBeNull(); });