fix(scripts): fall back to local check:changed lanes on full Crabbox provider outage (#121323)

Recognize Crabbox workload-routing provider exhaustion as a pre-dispatch backend outage so check:changed runs its documented local fallback.

The observed full provider-chain outage previously exited 2 without running local lanes.

The existing command-exit veto still runs first, so a remote run that actually executed and failed remains fail-closed.
This commit is contained in:
Peter Steinberger
2026-08-09 19:06:01 -07:00
committed by GitHub
parent 50026f1031
commit e61ba9c1ea
2 changed files with 16 additions and 2 deletions
+5 -2
View File
@@ -429,13 +429,16 @@ const DELEGATION_OUTPUT_TAIL_LIMIT = 64 * 1024;
/**
* Signatures of a failure that happened before the remote command was dispatched:
* the broker or its API was unreachable, or no lease was ever obtained.
* the broker or its API was unreachable, no lease was ever obtained, or workload
* routing exhausted its provider chain (every provider doctor failed).
*/
const BACKEND_UNAVAILABLE_SIGNATURES = [
/request failed: \w+ "https?:\/\/[^"]*blacksmith[^"]*"/iu,
/context deadline exceeded/iu,
/(?:no such host|dial tcp|connection refused|network is unreachable)/iu,
/failed to (?:acquire|create|warm|start)\b[^\n]*\b(?:lease|testbox)/iu,
// crabbox-wrapper prints this and exits before dispatching anything remote.
/\[crabbox\] no ready provider for workload=/u,
];
/**
@@ -1257,7 +1260,7 @@ async function main() {
// Say this loudly: the proof below is local, so whoever reads the run
// knows which machine produced it and that Linux-only lanes are unproven.
console.error(
"[check:changed] Blacksmith never ran the checks (no run summary). Falling back to local execution; note this in the proof summary.",
"[check:changed] the remote backend never ran the checks (no run summary). Falling back to local execution; note this in the proof summary.",
);
}
process.exitCode = delegated.backendUnavailable
+11
View File
@@ -2431,6 +2431,17 @@ describe("delegationFailedBeforeRunning", () => {
expect(delegationFailedBeforeRunning(output)).toBe(false);
});
it("treats a full workload-routing provider outage as never having run", () => {
// Provider selection happens before any dispatch, so an exhausted routing
// chain (every doctor failing) can never carry a remote verdict.
const output = [
"[crabbox] no ready provider for workload=ci-fast",
"[crabbox] provider readiness blacksmith-testbox:doctor exited 1,daytona:doctor exited 124,azure:doctor exited 124,aws:doctor exited 124",
].join("\n");
expect(delegationFailedBeforeRunning(output)).toBe(true);
});
it("does not mistake an infrastructure error kind for a command verdict", () => {
const output = [
"failed to acquire lease for testbox",