From e61ba9c1ea2f0928455cf3c4baf42569ffc5e00f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 9 Aug 2026 19:06:01 -0700 Subject: [PATCH] fix(scripts): fall back to local check:changed lanes on full Crabbox provider outage (#121323) Recognize Crabbox workload-routing provider exhaustion as a pre-dispatch backend outage so check:changed runs its documented local fallback. The observed full provider-chain outage previously exited 2 without running local lanes. The existing command-exit veto still runs first, so a remote run that actually executed and failed remains fail-closed. --- scripts/check-changed.mts | 7 +++++-- test/scripts/changed-lanes.test.ts | 11 +++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/scripts/check-changed.mts b/scripts/check-changed.mts index 721c1ea6c1d5..d956b5ead2b3 100644 --- a/scripts/check-changed.mts +++ b/scripts/check-changed.mts @@ -429,13 +429,16 @@ const DELEGATION_OUTPUT_TAIL_LIMIT = 64 * 1024; /** * Signatures of a failure that happened before the remote command was dispatched: - * the broker or its API was unreachable, or no lease was ever obtained. + * the broker or its API was unreachable, no lease was ever obtained, or workload + * routing exhausted its provider chain (every provider doctor failed). */ const BACKEND_UNAVAILABLE_SIGNATURES = [ /request failed: \w+ "https?:\/\/[^"]*blacksmith[^"]*"/iu, /context deadline exceeded/iu, /(?:no such host|dial tcp|connection refused|network is unreachable)/iu, /failed to (?:acquire|create|warm|start)\b[^\n]*\b(?:lease|testbox)/iu, + // crabbox-wrapper prints this and exits before dispatching anything remote. + /\[crabbox\] no ready provider for workload=/u, ]; /** @@ -1257,7 +1260,7 @@ async function main() { // Say this loudly: the proof below is local, so whoever reads the run // knows which machine produced it and that Linux-only lanes are unproven. console.error( - "[check:changed] Blacksmith never ran the checks (no run summary). Falling back to local execution; note this in the proof summary.", + "[check:changed] the remote backend never ran the checks (no run summary). Falling back to local execution; note this in the proof summary.", ); } process.exitCode = delegated.backendUnavailable diff --git a/test/scripts/changed-lanes.test.ts b/test/scripts/changed-lanes.test.ts index 07c86e7f222c..4d0083d4a960 100644 --- a/test/scripts/changed-lanes.test.ts +++ b/test/scripts/changed-lanes.test.ts @@ -2431,6 +2431,17 @@ describe("delegationFailedBeforeRunning", () => { expect(delegationFailedBeforeRunning(output)).toBe(false); }); + it("treats a full workload-routing provider outage as never having run", () => { + // Provider selection happens before any dispatch, so an exhausted routing + // chain (every doctor failing) can never carry a remote verdict. + const output = [ + "[crabbox] no ready provider for workload=ci-fast", + "[crabbox] provider readiness blacksmith-testbox:doctor exited 1,daytona:doctor exited 124,azure:doctor exited 124,aws:doctor exited 124", + ].join("\n"); + + expect(delegationFailedBeforeRunning(output)).toBe(true); + }); + it("does not mistake an infrastructure error kind for a command verdict", () => { const output = [ "failed to acquire lease for testbox",