From 257eb21235fc5441645dffa6dd09b092abe1c0b1 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 13 Aug 2026 19:32:50 -0700 Subject: [PATCH] improve(ci): shorten GitHub-hosted Node matrix (#123406) * improve(ci): shorten GitHub-hosted matrix wall time * fix(ci): keep hosted dependency setup store-only * fix(ci): split the hosted tooling tail --- .agents/skills/openclaw-ci-limits/SKILL.md | 12 +-- .github/workflows/ci.yml | 6 +- docs/ci.md | 6 +- scripts/lib/ci-node-test-plan.mts | 79 ++++++++++++++++--- test/scripts/ci-node-test-plan.test.ts | 51 ++++++++---- test/scripts/ci-workflow-guards.test.ts | 4 +- ...n-codex-app-server-attempt-extra.config.ts | 1 + 7 files changed, 122 insertions(+), 37 deletions(-) diff --git a/.agents/skills/openclaw-ci-limits/SKILL.md b/.agents/skills/openclaw-ci-limits/SKILL.md index 06b434e26a00..1f20cabdda52 100644 --- a/.agents/skills/openclaw-ci-limits/SKILL.md +++ b/.agents/skills/openclaw-ci-limits/SKILL.md @@ -163,13 +163,13 @@ These are intentionally guarded by `test/scripts/ci-workflow-guards.test.ts`: same-repo pull requests, preflight owns the sole immutable semantic dependency-cache write of workspace `node_modules` plus the local pnpm store before fanout; all Blacksmith Node jobs are restore-only consumers and exact - misses fall back to the ordinary pnpm-store cache, while - hosted/fork/manual paths use only that store cache. -- CI matrix caps: fast/check lanes at 12, Node test shards at 28, Windows and - Android at 2. + misses fall back to the ordinary pnpm-store cache, while hosted/fork/manual + paths use only that store cache. +- CI matrix caps: fast/check lanes at 12, Node test shards at 28 on Blacksmith + and 64 on the GitHub backend, Windows and Android at 2. - Canonical PR Node tests use one precise changed-target job when possible; - broad, deleted, unknown, or planner-failed changes fall back to the 14-job - compact full-suite plan. Targeted plans retain the full built-artifact + broad, deleted, unknown, or planner-failed changes fall back to the compact + full-suite plan. Targeted plans retain the full built-artifact boundary gate. `main`, manual, and release runs stay full. - `build-artifacts` on `blacksmith-16vcpu-ubuntu-2404`. - lower-weight Node/check shards on `blacksmith-4vcpu-ubuntu-2404`. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43952627745e..e84affd572fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -765,6 +765,7 @@ jobs: runner: shard.runner, timeout_minutes: shard.timeoutMinutes, plan_concurrency: shard.planConcurrency, + predicted_seconds: shard.predictedSeconds, save_vitest_fs_cache: shard.saveVitestFsCache, targets: shard.targets, requires_go: @@ -2195,7 +2196,7 @@ jobs: fail-fast: false # Canonical main admits only one complete run at a time, so widen this # matrix within the current runner-registration budget. - max-parallel: ${{ vars.OPENCLAW_CI_RUNNER_BACKEND == 'github' && 48 || 28 }} + max-parallel: ${{ vars.OPENCLAW_CI_RUNNER_BACKEND == 'github' && 64 || 28 }} matrix: ${{ fromJson(needs.preflight.outputs.checks_node_core_nondist_matrix) }} steps: - *linux_node_checkout_step @@ -2236,6 +2237,7 @@ jobs: # that can overlap Vitest processes keeps the proven 2-worker cap. # Timing-sensitive groups stay pinned to 2 via plan-level env. env: + PREDICTED_TEST_SECONDS: ${{ matrix.predicted_seconds || '' }} SHARD_PLAN_CONCURRENCY: ${{ matrix.plan_concurrency || '' }} run: | cores="$(nproc)" @@ -2248,7 +2250,7 @@ jobs: else workers=3 fi - echo "detected cores=$cores plan_concurrency=${SHARD_PLAN_CONCURRENCY:-default} -> workers=$workers" + echo "detected cores=$cores plan_concurrency=${SHARD_PLAN_CONCURRENCY:-default} predicted_test_seconds=${PREDICTED_TEST_SECONDS:-unknown} -> workers=$workers" echo "OPENCLAW_VITEST_MAX_WORKERS=$workers" >> "$GITHUB_ENV" - name: Checkout trusted Node shard runner diff --git a/docs/ci.md b/docs/ci.md index fea5dd3a18f4..3950a656e7ba 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -135,7 +135,7 @@ The slowest Node test families are split or balanced so each job stays small wit - On the 32-vCPU self-hosted build runner, Gateway watch, channel tests, and the core support-boundary shard start together inside `build-artifacts` after `dist/` and `dist-runtime/` are already built. GitHub-hosted fallback runs keep Gateway watch serial so low-core contention cannot consume its readiness deadline. Both paths then run the two built TUI PTY artifact canaries alone; the pull request fallback plus manual and release full matrices own the dedicated full serial shard. Once admitted, canonical Linux CI permits up to 28 concurrent Node test jobs on -Blacksmith and 48 while the GitHub-hosted outage backend is active. The smaller +Blacksmith and 64 while the GitHub-hosted outage backend is active. The smaller fast/check lanes remain capped at 12 in both modes; Windows and Android stay at two because those runner pools are narrower. Compact whole-config batches run with a 120-minute batch timeout, while include-pattern groups share the same @@ -213,7 +213,7 @@ The repository variable `OPENCLAW_CI_RUNNER_BACKEND` controls the runner backend gh variable set OPENCLAW_CI_RUNNER_BACKEND --repo openclaw/openclaw --body github ``` -Degraded mode uses the same hosted paths exercised by manual dispatches and fork pull requests. Blacksmith-only Docker and sticky-disk steps are skipped, dependency setup uses the ordinary Actions pnpm-store cache, and low-memory Android builds use separate Gradle processes. Core oxlint splits into three deterministic hosted stripes, with extension/scripts lint and optional UI/format checks in the existing `check-lint` row. The compact Node planner keeps the Blacksmith 200/276-second hint tables unchanged and selects a separate standard 4-core hosted overlay. That overlay contains measured per-group medians, targets 240 seconds of serial group work for both source runner classes, and applies a 1.6x median scaling fallback only to unmeasured groups. Failed and timeout-and-retry samples are excluded from refreshes. It produces 32 push descriptors or 38 pull-request fallback descriptors under a 48-row hosted cap; the corresponding Blacksmith plans remain 25 and 31 total descriptors under the 28-row admission cap. Control UI E2E expands from the unchanged Blacksmith shape of three Vitest shards plus one browser-extension shard to seven Vitest shards plus one browser-extension shard. QA Smoke expands from the unchanged four-part Blacksmith matrix to six hosted parts. Windows expands from the unchanged single Blacksmith job to two hosted jobs with disjoint, project-aligned explicit test lists. This partitions the complete Windows-specific inventory without applying Vitest `--shard` to project-local single-file selections, which Vitest rejects. Expect slower individual builds on standard 4-core hosted runners. Blacksmith's runner-registration budget is irrelevant while the breaker is active, but GitHub-hosted concurrency limits apply. +Degraded mode uses the same hosted paths exercised by manual dispatches and fork pull requests. Blacksmith-only Docker and sticky-disk steps are skipped, dependency setup uses the ordinary Actions pnpm-store cache, and low-memory Android builds use separate Gradle processes. The exact workspace dependency cache intentionally stays off on hosted runners: GitHub can roll the runner image and Node patch between preflight and fanout in one workflow, while the safe exact key then misses; the ordinary store archive is only slightly smaller and pnpm's measured relink is already single-digit seconds. Core oxlint splits into three deterministic hosted stripes, with extension/scripts lint and optional UI/format checks in the existing `check-lint` row. The compact Node planner keeps the Blacksmith 200/276-second hint tables unchanged and selects a separate standard 4-core hosted overlay. That overlay contains measured per-group medians, targets 160 seconds of serial group work for both source runner classes, and applies a 1.6x median scaling fallback only to unmeasured groups. Failed and timeout-and-retry samples are excluded from refreshes. Groups above 210 predicted seconds split into file-weighted hosted stripes before packing; the measured tooling outlier uses three stripes after live proof showed its two-way split remained imbalanced. The resulting 46 push descriptors and 54 pull-request fallback descriptors stay under a 64-row hosted cap, and every lane is at or below 210 predicted test seconds. The corresponding Blacksmith plans remain 25 and 31 total descriptors under the 28-row admission cap. Control UI E2E expands from the unchanged Blacksmith shape of three Vitest shards plus one browser-extension shard to seven Vitest shards plus one browser-extension shard. QA Smoke expands from the unchanged four-part Blacksmith matrix to six hosted parts. Windows expands from the unchanged single Blacksmith job to two hosted jobs with disjoint, project-aligned explicit test lists. This partitions the complete Windows-specific inventory without applying Vitest `--shard` to project-local single-file selections, which Vitest rejects. Expect slower individual builds on standard 4-core hosted runners. Blacksmith's runner-registration budget is irrelevant while the breaker is active, but GitHub-hosted concurrency limits apply. Flip back after the outage by deleting the variable, which restores the default behavior: @@ -241,7 +241,7 @@ target below about 60% of the live bucket. With the current 10,000-registration bucket, that means a 6,000-registration operating target, leaving headroom for concurrent repositories, retries, and burst overlap. -The changed-target PR plan reduces the common Node test burst from 29 Blacksmith registrations to one. Broad-risk PRs keep the 29-registration metadata-complete compact fallback; canonical pushes use 24 nondist compact registrations. While the outage backend is `github`, the broader fallback uses 36 nondist hosted jobs and canonical pushes use 31; these jobs do not consume Blacksmith runner registrations. +The changed-target PR plan reduces the common Node test burst from 29 Blacksmith registrations to one. Broad-risk PRs keep the 29-registration metadata-complete compact fallback; canonical pushes use 24 nondist compact registrations. While the outage backend is `github`, the broader fallback uses 52 nondist hosted jobs and canonical pushes use 45; these jobs do not consume Blacksmith runner registrations. Canonical-repo CI keeps Blacksmith as the default runner path for pushes and first-attempt same-repo pull-request runs. Pull-request retries of both UI E2E jobs use GitHub-hosted Ubuntu; push retries stay on Blacksmith. All `workflow_dispatch` runs, including `release_gate`, and non-canonical repository runs use GitHub-hosted runners. The [Blacksmith outage circuit breaker](#blacksmith-outage-circuit-breaker) provides a manual repository-wide fallback; canonical runs do not probe Blacksmith queue health or flip it automatically. diff --git a/scripts/lib/ci-node-test-plan.mts b/scripts/lib/ci-node-test-plan.mts index c5e6a49a7208..be20c45e23d2 100644 --- a/scripts/lib/ci-node-test-plan.mts +++ b/scripts/lib/ci-node-test-plan.mts @@ -42,6 +42,7 @@ export type NodeTestShard = { groups?: NodeTestShardGroup[]; timeoutMinutes?: number; planConcurrency?: number; + predictedSeconds?: number; saveVitestFsCache?: boolean; }; @@ -166,13 +167,15 @@ const MAX_BUNDLED_NODE_TEST_PATTERNS = 64; // Keep runner classes and subprocess isolation intact while bounding each combined job. // The group hints below are loaded-fleet CI walls. Three-way striping plus a // Blacksmith keeps the proven 200s/276s admission caps and 28-worker ceiling. -// Standard 4-core GitHub runners use direct hosted wall hints below. Keep their -// serial group budget near four minutes so setup leaves jobs around five. +// Standard 4-core GitHub runners use direct hosted wall hints below. The +// 160-second wall budget is the hosted-hint equivalent of tightening the old +// 120/166 admission caps to 80/110, leaving setup inside a ~3.5-minute lane. const COMPACT_LARGE_NODE_TEST_JOB_SECONDS = 200; const COMPACT_SMALL_NODE_TEST_JOB_SECONDS = 276; -const COMPACT_GITHUB_LARGE_NODE_TEST_JOB_SECONDS = 240; -const COMPACT_GITHUB_SMALL_NODE_TEST_JOB_SECONDS = 240; +const COMPACT_GITHUB_LARGE_NODE_TEST_JOB_SECONDS = 160; +const COMPACT_GITHUB_SMALL_NODE_TEST_JOB_SECONDS = 160; const COMPACT_GITHUB_GROUP_SECONDS_SCALE = 1.6; +const COMPACT_GITHUB_MAX_PREDICTED_SECONDS = 210; const COMPACT_NODE_TEST_JOB_GROUPS = 10; const COMPACT_TOOLING_NODE_TEST_GROUPS = 4; const COMPACT_WHOLE_NODE_TEST_TIMEOUT_MINUTES = 120; @@ -555,7 +558,8 @@ const COMPACT_PUSH_EXCLUDED_SHARDS = new Set([ // Spawn/signal-timing suites (process-group waits, PTY smoke) flake when a // concurrent sibling Vitest run competes for the 4 vCPU runner. Pack them // into bins the shard runner executes at concurrency 1. -const EXCLUSIVE_COMPACT_GROUP_RE = /^core-tooling(?:-\d+|-isolated)$|^core-runtime-tui-pty$/u; +const EXCLUSIVE_COMPACT_GROUP_RE = + /^core-tooling(?:-\d+(?:-hosted-\d+)?|-isolated)$|^core-runtime-tui-pty$/u; // Exclusive bins run serially, so their packed estimate is their wall clock. const COMPACT_EXCLUSIVE_JOB_SECONDS = 150; @@ -568,7 +572,7 @@ function isExclusiveCompactGroup(group: NodeTestShardGroup): boolean { // scales with the runner class. infra-process spawns child processes per test // and hit worker-startup timeouts under contention before serialization. const PINNED_WORKER_COMPACT_GROUP_RE = - /^core-tooling(?:-\d+|-isolated)$|^core-runtime-tui-pty$|^core-runtime-infra-process$|^core-runtime-media-ui-(?:\d+|support)$|^agentic-cli$|^agentic-gateway-(?:core-\d+|methods)$/u; + /^core-tooling(?:-\d+(?:-hosted-\d+)?|-isolated)$|^core-runtime-tui-pty$|^core-runtime-infra-process$|^core-runtime-media-ui-(?:\d+|support)$|^agentic-cli$|^agentic-gateway-(?:core-\d+|methods)$/u; const PINNED_COMPACT_GROUP_ENV = { OPENCLAW_VITEST_MAX_WORKERS: "2" }; function applyCompactGroupWorkerPins(group: NodeTestShardGroup): NodeTestShardGroup { @@ -2024,6 +2028,50 @@ export function assignVitestFsCacheWriter + owner.include.some((pattern) => matchesGlob(file, pattern)) && + !owner.exclude.some((pattern) => matchesGlob(file, pattern)), + ); +} + +function splitOversizedGithubCompactGroup( + group: NodeTestShardGroup, +): Array<{ group: NodeTestShardGroup; seconds: number }> { + const seconds = estimateCompactGroupSeconds(group, "github"); + if (seconds <= COMPACT_GITHUB_MAX_PREDICTED_SECONDS) { + return [{ group, seconds }]; + } + + const includePatterns = + group.includePatterns ?? + (group.shard_name === "agentic-agents-support" ? listAgentSupportTestFiles() : undefined); + if (!includePatterns || includePatterns.length === 0) { + return [{ group, seconds }]; + } + + // Hosted proof showed the old four-way tooling stripe remained imbalanced + // after a two-way split (126s versus 246s). Give that measured outlier a + // third stripe; the generic ceiling remains sufficient for other groups. + const stripeCount = + group.shard_name === "core-tooling-3" + ? 3 + : Math.ceil(seconds / COMPACT_GITHUB_MAX_PREDICTED_SECONDS); + const splitSeconds = Math.ceil(seconds / stripeCount); + return createStripedBatches(includePatterns, stripeCount, stripeFileWeight).map( + (patterns, index) => ({ + group: { + ...group, + includePatterns: patterns, + shard_name: `${group.shard_name}-hosted-${index + 1}`, + }, + seconds: splitSeconds, + }), + ); +} + function createCompactNodeTestShardBundles( options: NodeTestPlanOptions, compactMode: CompactNodeTestPlanMode, @@ -2032,27 +2080,39 @@ function createCompactNodeTestShardBundles( (shard) => compactMode !== "push" || !COMPACT_PUSH_EXCLUDED_SHARDS.has(shard.shardName), ); const groupsByRunner = new Map(); + const hostedSplitSeconds = new Map(); for (const shard of shards) { const runner = resolveCiNodeTestRunner(shard); const key = JSON.stringify([runner, shard.requiresDist]); const groups = groupsByRunner.get(key) ?? []; - const group = { + const group = applyCompactGroupWorkerPins({ configs: shard.configs, ...(shard.env ? { env: shard.env } : {}), ...(shard.includePatterns ? { includePatterns: shard.includePatterns } : {}), requiresDist: shard.requiresDist, runner, shard_name: shard.shardName, - }; - groups.push(applyCompactGroupWorkerPins(group)); + }); + const plannedGroups = + options.runnerBackend === "github" + ? splitOversizedGithubCompactGroup(group) + : [{ group, seconds: estimateCompactGroupSeconds(group, options.runnerBackend) }]; + for (const planned of plannedGroups) { + groups.push(planned.group); + if (options.runnerBackend === "github") { + hostedSplitSeconds.set(planned.group.shard_name, planned.seconds); + } + } groupsByRunner.set(key, groups); } const compactJobs: CompactNodeTestShard[] = []; const estimateGroupSeconds = (group: NodeTestShardGroup) => + hostedSplitSeconds.get(group.shard_name) ?? estimateCompactGroupSeconds(group, options.runnerBackend); const estimateStripeSeconds = (group: NodeTestShardGroup) => + hostedSplitSeconds.get(group.shard_name) ?? estimateCompactStripeSeconds(group, options.runnerBackend); for (const groups of groupsByRunner.values()) { // First-fit decreasing sets the existing registration count from the @@ -2150,6 +2210,7 @@ function createCompactNodeTestShardBundles( // lock-timing flakes on 8 vCPU), and the packed weights are // contention-inflated so serializing is roughly wall-neutral. planConcurrency: 1, + predictedSeconds: bin.weight, }); } } diff --git a/test/scripts/ci-node-test-plan.test.ts b/test/scripts/ci-node-test-plan.test.ts index 6e2ef3c7265c..77d455d2ff25 100644 --- a/test/scripts/ci-node-test-plan.test.ts +++ b/test/scripts/ci-node-test-plan.test.ts @@ -300,9 +300,13 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { // Pushes retain three lanes of headroom under the workflow's 28-worker cap. expect(compact).toHaveLength(25); expect(pullRequestCompact).toHaveLength(31); - expect(githubCompact).toHaveLength(32); - expect(githubPullRequestCompact).toHaveLength(38); - expect(githubPullRequestCompact.length).toBeLessThanOrEqual(48); + expect(githubCompact).toHaveLength(46); + expect(githubPullRequestCompact).toHaveLength(54); + expect(githubPullRequestCompact.length).toBeLessThanOrEqual(64); + expect(Math.max(...githubCompact.map((shard) => shard.predictedSeconds ?? Infinity))).toBe(209); + expect( + Math.max(...githubPullRequestCompact.map((shard) => shard.predictedSeconds ?? Infinity)), + ).toBe(210); expect(compact.every((shard) => Array.isArray(shard.groups))).toBe(true); expect(compact.every((shard) => shard.groups.length <= 10)).toBe(true); expect(compact.some((shard) => shard.requiresDist)).toBe(true); @@ -331,8 +335,9 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { // Spawn/signal-timing suites never mix with regular groups, and every // compact bin runs serially: overlapping Vitest runs flake timing- // sensitive tests on both runner classes. - const exclusiveGroupRe = /^core-tooling(?:-\d+|-isolated)$|^core-runtime-tui-pty$/u; - for (const shard of pullRequestCompact) { + const exclusiveGroupRe = + /^core-tooling(?:-\d+(?:-hosted-\d+)?|-isolated)$|^core-runtime-tui-pty$/u; + for (const shard of [...pullRequestCompact, ...githubPullRequestCompact]) { const exclusiveCount = shard.groups.filter((group) => exclusiveGroupRe.test(group.shard_name), ).length; @@ -370,19 +375,35 @@ describe("scripts/lib/ci-node-test-plan.mts", () => { expect(pullRequestCompactGroups.map((group) => group.shard_name).toSorted()).toEqual( expectedGroupNames.toSorted(), ); - expect( - githubCompact.flatMap((shard) => shard.groups.map((group) => group.shard_name)).toSorted(), - ).toEqual(compactGroups.map((group) => group.shard_name).toSorted()); - expect( - githubPullRequestCompact - .flatMap((shard) => shard.groups.map((group) => group.shard_name)) - .toSorted(), - ).toEqual(pullRequestCompactGroups.map((group) => group.shard_name).toSorted()); + const hostedOwnerNames = (plan: typeof githubCompact) => + new Set( + plan.flatMap((shard) => + shard.groups.map((group) => group.shard_name.replace(/-hosted-\d+$/u, "")), + ), + ); + expect(hostedOwnerNames(githubCompact)).toEqual( + new Set(compactGroups.map((group) => group.shard_name)), + ); + expect(hostedOwnerNames(githubPullRequestCompact)).toEqual( + new Set(pullRequestCompactGroups.map((group) => group.shard_name)), + ); const groupsWith = (plan: typeof githubCompact, shardName: string) => plan.find((shard) => shard.groups.some((group) => group.shard_name === shardName))?.groups; + const hostedAgentSupportGroups = githubCompact + .flatMap((shard) => shard.groups) + .filter((group) => group.shard_name.startsWith("agentic-agents-support-hosted-")); + expect(hostedAgentSupportGroups).toHaveLength(2); expect( - groupsWith(githubCompact, "agentic-agents-support")?.map((group) => group.shard_name), - ).toEqual(["agentic-agents-support"]); + hostedAgentSupportGroups + .flatMap((group) => group.includePatterns ?? []) + .toSorted((left, right) => left.localeCompare(right)), + ).toEqual( + globSync(agentVitestProjectOwners.support.include, { + exclude: agentVitestProjectOwners.support.exclude, + }) + .map(toRepoPath) + .toSorted((left, right) => left.localeCompare(right)), + ); expect( groupsWith(compact, "agentic-agents-support")?.map((group) => group.shard_name), ).toContain("agentic-agents-embedded-overflow-compaction"); diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index e1ef9c14753c..b9cc8e482f91 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -2664,7 +2664,7 @@ NODE expect(workflow.jobs["checks-fast-core"].strategy["max-parallel"]).toBe(12); const nodeMaxParallel = workflow.jobs["checks-node-core-test-nondist-shard"].strategy["max-parallel"]; - expect(nodeMaxParallel).toBe("${{ vars.OPENCLAW_CI_RUNNER_BACKEND == 'github' && 48 || 28 }}"); + expect(nodeMaxParallel).toBe("${{ vars.OPENCLAW_CI_RUNNER_BACKEND == 'github' && 64 || 28 }}"); expect( evaluateWorkflowExpression(nodeMaxParallel, { eventName: "push", @@ -2680,7 +2680,7 @@ NODE runnerBackend: "github", runAttempt: 1, }), - ).toBe(48); + ).toBe(64); expect(workflow.jobs["checks-fast-plugin-contracts-shard"].strategy["max-parallel"]).toBe(12); expect(workflow.jobs["checks-fast-channel-contracts-shard"].strategy["max-parallel"]).toBe(12); expect(workflow.jobs["check-shard"].strategy["max-parallel"]).toBe(12); diff --git a/test/vitest/vitest.extension-codex-app-server-attempt-extra.config.ts b/test/vitest/vitest.extension-codex-app-server-attempt-extra.config.ts index 03a01cea0d95..88b69e435edf 100644 --- a/test/vitest/vitest.extension-codex-app-server-attempt-extra.config.ts +++ b/test/vitest/vitest.extension-codex-app-server-attempt-extra.config.ts @@ -7,6 +7,7 @@ function createExtensionCodexAppServerAttemptExtraVitestConfig( return createScopedVitestConfig( [ "extensions/codex/src/app-server/run-attempt-lifecycle-controller.test.ts", + "extensions/codex/src/app-server/run-attempt-state.test.ts", "extensions/codex/src/app-server/run-attempt-thread-cleanup.test.ts", "extensions/codex/src/app-server/run-attempt.configured-mcp.test.ts", "extensions/codex/src/app-server/run-attempt.context-engine.test.ts",