From 21b08f34b7e815a816710de7a2d4a7c6671313e1 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 10:46:44 -0700 Subject: [PATCH] perf(ci): run the deadcode Knip scans concurrently (#109151) * perf(ci): run the deadcode Knip scans concurrently check-dependencies spent ~194s running seven Knip child processes strictly serially (three package scripts, each iterating its scans). The scans are independent processes over separate configs: both deadcode scripts now Promise.all their scans, the CI task launches the three package scripts concurrently with buffered logs, and the lane moves to the 16 vCPU class for core/memory headroom. The missing-exports-script contract violation now fails fast before launching scans. Local: unused-files 218% CPU, 37s wall. * test(ci): align prerelease plan with the 16 vCPU dependencies lane --- .github/workflows/ci.yml | 34 ++++++++++++++++--- scripts/check-deadcode-exports.mjs | 16 ++++++--- scripts/check-deadcode-unused-files.mjs | 16 ++++++--- test/scripts/ci-workflow-guards.test.ts | 21 +++++++----- .../plugin-prerelease-test-plan.test.ts | 3 +- 5 files changed, 67 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c8d267d6b95..b5e815acfb2c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1820,7 +1820,9 @@ jobs: runner: blacksmith-16vcpu-ubuntu-2404 - check_name: check-dependencies task: dependencies - runner: blacksmith-4vcpu-ubuntu-2404 + # Concurrent Knip scans need cores and memory headroom; the wall + # clock roughly halves for similar billed core-minutes. + runner: blacksmith-16vcpu-ubuntu-2404 - check_name: check-test-types task: test-types # Slowest static lane (~3.5min on 4 vCPU); extra cores shorten the @@ -1923,14 +1925,38 @@ jobs: dependencies) if has_package_script "deadcode:dependencies" && has_package_script "deadcode:unused-files"; then - pnpm deadcode:dependencies - pnpm deadcode:unused-files + # The three deadcode scripts spawn independent Knip scans over + # separate configs; run them concurrently (with buffered logs + # so output stays readable) instead of paying ~3 minutes of + # serial scanning. + dc_scripts=(deadcode:dependencies deadcode:unused-files) if has_package_script "deadcode:exports"; then - pnpm deadcode:exports + dc_scripts+=(deadcode:exports) elif [[ "$HISTORICAL_TARGET" != "true" ]]; then echo "Current CI targets must provide the deadcode:exports package script." >&2 exit 1 fi + dc_pids=() + dc_logs=() + for dc in "${dc_scripts[@]}"; do + dc_log="$(mktemp -t deadcode-log.XXXXXX)" + dc_logs+=("$dc_log") + pnpm "$dc" >"$dc_log" 2>&1 & + dc_pids+=($!) + done + dc_failures=0 + for i in "${!dc_scripts[@]}"; do + if wait "${dc_pids[$i]}"; then + echo "[ok] ${dc_scripts[$i]}" + else + echo "::error title=${dc_scripts[$i]} failed::${dc_scripts[$i]} failed" + dc_failures=1 + fi + cat "${dc_logs[$i]}" + done + if [ "$dc_failures" -ne 0 ]; then + exit 1 + fi elif [[ "$HISTORICAL_TARGET" == "true" ]] && has_package_script "deadcode:ci"; then pnpm deadcode:ci else diff --git a/scripts/check-deadcode-exports.mjs b/scripts/check-deadcode-exports.mjs index 36fdc34fd2d4..1ae2cee772c6 100644 --- a/scripts/check-deadcode-exports.mjs +++ b/scripts/check-deadcode-exports.mjs @@ -96,9 +96,16 @@ export function checkUnusedExports(output) { } async function main() { - for (const scan of KNIP_SCANS) { - const ok = await runUnusedExportScan(scan); - if (!ok) { + // The scans are independent Knip child processes over separate configs; + // running them concurrently cuts the lane's serial wall clock roughly 2x. + const results = await Promise.all( + KNIP_SCANS.map(async (scan) => ({ + scan, + result: await runKnip([...scan.args, ...KNIP_COMMON_ARGS], { scanName: scan.name }), + })), + ); + for (const { scan, result } of results) { + if (!reportUnusedExportScan(scan, result)) { process.exitCode = 1; return; } @@ -108,8 +115,7 @@ async function main() { ); } -async function runUnusedExportScan(scan) { - const result = await runKnip([...scan.args, ...KNIP_COMMON_ARGS], { scanName: scan.name }); +function reportUnusedExportScan(scan, result) { if (result.errorCode || result.status === null) { console.error( `deadcode ${scan.name} failed: ${result.errorCode ?? result.signal ?? "unknown"}${ diff --git a/scripts/check-deadcode-unused-files.mjs b/scripts/check-deadcode-unused-files.mjs index 0971a7b05cb9..bb6950605048 100644 --- a/scripts/check-deadcode-unused-files.mjs +++ b/scripts/check-deadcode-unused-files.mjs @@ -91,9 +91,16 @@ export function checkKnipUnusedFileScanResult(result) { } async function main() { - for (const scan of KNIP_SCANS) { - const ok = await runUnusedFileScan(scan); - if (!ok) { + // The scans are independent Knip child processes over separate configs; + // running them concurrently halves the lane's serial wall clock. + const results = await Promise.all( + KNIP_SCANS.map(async (scan) => ({ + scan, + result: await runKnip([...scan.args, ...KNIP_COMMON_ARGS], { scanName: scan.name }), + })), + ); + for (const { scan, result } of results) { + if (!reportUnusedFileScan(scan, result)) { process.exitCode = 1; return; } @@ -101,8 +108,7 @@ async function main() { console.log("[deadcode] Knip production and full-tree unused-file checks passed with 0 entries."); } -async function runUnusedFileScan(scan) { - const result = await runKnip([...scan.args, ...KNIP_COMMON_ARGS], { scanName: scan.name }); +function reportUnusedFileScan(scan, result) { const validation = checkKnipUnusedFileScanResult(result); if (validation.failureReason) { console.error( diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 861b82510e45..652dd725242b 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -1798,7 +1798,8 @@ describe("ci workflow guards", () => { expect(workflow.jobs["check-shard"].strategy.matrix.include).toContainEqual({ check_name: "check-dependencies", task: "dependencies", - runner: "blacksmith-4vcpu-ubuntu-2404", + // Concurrent Knip scans need cores and memory headroom. + runner: "blacksmith-16vcpu-ubuntu-2404", }); expect(workflow.jobs["check-additional-shard"]["runs-on"]).toContain("matrix.runner"); expect(workflow.jobs["check-additional-shard"].strategy.matrix.include).toContainEqual({ @@ -2340,10 +2341,11 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" scripts: ["deadcode:dependencies", "deadcode:unused-files", "deadcode:exports"], }); expect(modern.status, modern.output).toBe(0); - expect(modern.calls).toEqual([ + // The scripts launch concurrently; completion order is nondeterministic. + expect(modern.calls.toSorted()).toEqual([ "deadcode:dependencies", - "deadcode:unused-files", "deadcode:exports", + "deadcode:unused-files", ]); const frozenWithExports = runDependencyCheckFixture({ @@ -2351,10 +2353,10 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" scripts: ["deadcode:dependencies", "deadcode:unused-files", "deadcode:exports"], }); expect(frozenWithExports.status, frozenWithExports.output).toBe(0); - expect(frozenWithExports.calls).toEqual([ + expect(frozenWithExports.calls.toSorted()).toEqual([ "deadcode:dependencies", - "deadcode:unused-files", "deadcode:exports", + "deadcode:unused-files", ]); const frozen = runDependencyCheckFixture({ @@ -2367,14 +2369,16 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" ], }); expect(frozen.status, frozen.output).toBe(0); - expect(frozen.calls).toEqual(["deadcode:dependencies", "deadcode:unused-files"]); + expect(frozen.calls.toSorted()).toEqual(["deadcode:dependencies", "deadcode:unused-files"]); const currentWithoutExports = runDependencyCheckFixture({ historicalTarget: false, scripts: ["deadcode:dependencies", "deadcode:unused-files"], }); expect(currentWithoutExports.status).toBe(1); - expect(currentWithoutExports.calls).toEqual(["deadcode:dependencies", "deadcode:unused-files"]); + // The missing-script contract violation now fails fast before launching + // the concurrent scans instead of wasting two Knip runs first. + expect(currentWithoutExports.calls).toEqual([]); expect(currentWithoutExports.output).toContain( "Current CI targets must provide the deadcode:exports package script.", ); @@ -2829,7 +2833,8 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" expect(checkShard.run).toContain('has_package_script "deadcode:dependencies"'); expect(checkShard.run).toContain('has_package_script "deadcode:unused-files"'); expect(checkShard.run).toContain('has_package_script "deadcode:exports"'); - expect(checkShard.run).toContain("pnpm deadcode:exports"); + // The concurrent launcher invokes scripts through the dc_scripts array. + expect(checkShard.run).toContain("dc_scripts+=(deadcode:exports)"); expect(checkShard.run).toContain( "Current CI targets must provide the deadcode:exports package script.", ); diff --git a/test/scripts/plugin-prerelease-test-plan.test.ts b/test/scripts/plugin-prerelease-test-plan.test.ts index 73e56efb65ab..1fedeae571a8 100644 --- a/test/scripts/plugin-prerelease-test-plan.test.ts +++ b/test/scripts/plugin-prerelease-test-plan.test.ts @@ -413,7 +413,8 @@ describe("scripts/lib/plugin-prerelease-test-plan.mjs", () => { ).toEqual({ check_name: "check-dependencies", task: "dependencies", - runner: "blacksmith-4vcpu-ubuntu-2404", + // Concurrent Knip scans need cores and memory headroom. + runner: "blacksmith-16vcpu-ubuntu-2404", }); expect( workflow.jobs["check-shard"].steps.find(