fix(crabbox): keep the setup failure reason and name its phase (#130425)

A Crabbox cloud-worker setup failure could report none of the reason it
failed. crabboxCommandDetail kept the first ~254 and last ~253 characters
and dropped the middle, but Crabbox always prints a fixed ~250-character
run-context banner first, so head preservation spent half the budget on
boilerplate. In a live machine0 failure the actionable line sat at
collapsed index 2570 and was deleted; the operator saw a banner and a
Node stack tail that read like a workspace-lock problem.

Keep the tail instead: a failing command's diagnosis is last, and the
capture layer in src/process/exec-output.ts already retains each stream's
suffix. Join [stdout, stderr] so stderr occupies the retained window
rather than being pushed out by a chatty stdout. The 512-character budget
is unchanged and stays inside core's 1024-unit bound.

Profile, desktop, and node enrollment setup also all failed as
"Crabbox setup failed", so the message never said which phase broke.
Each now carries its own label, matching the shared SSH bootstrap in
src/gateway/worker-environments/bootstrap.ts.

Net -4 production lines.
This commit is contained in:
Peter Steinberger
2026-08-26 15:06:05 -07:00
committed by GitHub
parent 730b6ef8fa
commit f2d344df0c
4 changed files with 84 additions and 28 deletions
@@ -1,26 +1,22 @@
import { redactSensitiveText } from "openclaw/plugin-sdk/logging-core";
import { WorkerProviderError } from "openclaw/plugin-sdk/plugin-entry";
import type { SpawnResult } from "openclaw/plugin-sdk/process-runtime";
import { sliceUtf16Safe, truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
import { sliceUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
const MAX_COMMAND_DETAIL_CHARS = 512;
function crabboxCommandDetail(result: SpawnResult): string {
const raw = [result.stderr, result.stdout].filter(Boolean).join("\n").trim();
const raw = [result.stdout, result.stderr].filter(Boolean).join("\n").trim();
if (!raw) {
return "";
}
const compressed = redactSensitiveText(raw).replace(/\s+/gu, " ");
const omitted = " ... ";
const remaining = MAX_COMMAND_DETAIL_CHARS - omitted.length;
const redacted =
compressed.length <= MAX_COMMAND_DETAIL_CHARS
? compressed
: `${truncateUtf16Safe(compressed, Math.ceil(remaining / 2))}${omitted}${sliceUtf16Safe(
compressed,
-Math.floor(remaining / 2),
)}`;
return redacted ? `: ${redacted}` : "";
// Failure diagnoses come last; Crabbox's fixed banner is leading boilerplate.
// Keep stderr last, matching the per-stream suffix capture in src/process/exec-output.ts.
const tailMarker = "... ";
return compressed.length <= MAX_COMMAND_DETAIL_CHARS
? `: ${compressed}`
: `: ${tailMarker}${sliceUtf16Safe(compressed, tailMarker.length - MAX_COMMAND_DETAIL_CHARS)}`;
}
export function crabboxCommandError(action: string, result: SpawnResult): Error {
@@ -956,17 +956,17 @@ describe("Crabbox worker provider", () => {
{
name: "fails",
result: commandResult({ code: 7, stderr: "apt exploded" }),
message: "Crabbox setup failed with exit code 7",
message: "Crabbox profile setup failed with exit code 7",
},
{
name: "times out",
result: commandResult({ code: null, killed: true, termination: "timeout" }),
message: "Crabbox setup did not exit normally (timeout)",
message: "Crabbox profile setup did not exit normally (timeout)",
},
{
name: "cannot start",
result: undefined,
message: "Crabbox setup could not start",
message: "Crabbox profile setup could not start",
},
])(
"stops the lease and removes its private env profile when setup $name",
@@ -1024,6 +1024,64 @@ describe("Crabbox worker provider", () => {
},
);
it.each([
{ phase: "profile setup", setupAttempt: 1 },
{ phase: "desktop setup", setupAttempt: 2 },
{ phase: "node enrollment setup", setupAttempt: 3 },
])("identifies the failed $phase phase", async ({ phase, setupAttempt }) => {
let attempts = 0;
const provider = providerWithRunner(async (argv) => {
if (argv[1] === "inspect") {
return commandResult({ stdout: inspectJson() });
}
if (argv[1] === "run" && ++attempts === setupAttempt) {
return commandResult({ code: 7, stderr: "setup command rejected" });
}
return commandResult();
});
await expect(
provider.provision({ ...PROFILE, setup: "install-node", desktop: true }, OPERATION_ID),
).rejects.toMatchObject({
code: "invalid_profile",
message: `Crabbox ${phase} failed with exit code 7: setup command rejected`,
});
});
it("preserves the node enrollment diagnosis after the Crabbox banner and setup noise", async () => {
const diagnosis =
"Error: Codex remote-exec requires the exact official @openclaw/codex@2026.8.1 plugin to be installed by cloudWorkers profile setup";
const stderr = [
`workspace owner acquired wait=218ms recovered=false run context: run=${"a".repeat(32)} lease=${LEASE_ID} slug=openclaw-${"b".repeat(32)} provider=machine0 ssh=openclaw@worker.example.test:2222 workspace=/workspace/openclaw`,
"x".repeat(2_000),
diagnosis,
" at prepareCodex ([eval]:20:11)",
" at runScriptInThisContext (node:internal/vm:209:10)",
" at node:internal/process/execution:446:12",
" at [eval]-wrapper:6:24",
" at runScriptInContext (node:internal/process/execution:444:60)",
" at evalFunction (node:internal/process/execution:279:30)",
"Node.js v24.15.0",
].join("\n");
const provider = providerWithRunner(async (argv) => {
if (argv[1] === "status") {
return commandResult({ stdout: inspectJson() });
}
return argv[1] === "run"
? commandResult({ code: 1, stderr, stdout: "setup progress ".repeat(200) })
: commandResult();
});
await expect(
provider.provision({ ...PROFILE, provider: "machine0" }, OPERATION_ID, {
executionMode: "remote-exec",
}),
).rejects.toMatchObject({
code: "invalid_profile",
message: expect.stringContaining(diagnosis),
});
});
it("preserves the allocated lease and both failures when setup cleanup times out", async () => {
let releaseCommitted = false;
const provider = providerWithRunner(async (argv) => {
@@ -2854,8 +2912,8 @@ describe("Crabbox worker provider", () => {
code,
termination,
killed: termination !== "exit",
stderr: `provider warning ${secret} ${"provider progress ".repeat(90)}${terminalStderr}`,
stdout: terminalStdout,
stderr: `provider warning ${secret}\n${terminalStderr}`,
stdout: `${"provider progress ".repeat(90)}\n${terminalStdout}`,
}),
);
const operation =
@@ -2870,10 +2928,11 @@ describe("Crabbox worker provider", () => {
action === "warmup"
? "Crabbox warmup failed with exit code 5: "
: "Crabbox inspect did not exit normally (timeout): ";
expect(message).toContain("provider warning");
expect(message.startsWith(`${failurePrefix}... `)).toBe(true);
expect(message).toContain(terminalStderr);
expect(message).toContain(terminalStdout);
expect(message).not.toContain(secret);
expect(message).not.toMatch(/\s{2,}/u);
expect(message.length).toBeLessThanOrEqual(failurePrefix.length + 512);
expect(hasLoneSurrogate(message)).toBe(false);
},
@@ -2882,10 +2941,11 @@ describe("Crabbox worker provider", () => {
it.each(["stderr", "stdout"] as const)(
"preserves UTF-16 boundaries and terminal detail from %s",
async (stream) => {
const terminalDetail = "😀 terminal failure";
const provider = providerWithRunner(async () =>
commandResult({
code: 2,
[stream]: `${"x".repeat(253)}😀${"y".repeat(300)}😀 terminal failure`,
[stream]: `${"x".repeat(600)}😀${"y".repeat(507 - terminalDetail.length)}${terminalDetail}`,
}),
);
@@ -305,6 +305,7 @@ async function waitForProvisionReady(
// otherwise the caller cannot release a box it never learned about.
async function runProvisionSetupAndWaitReady(
params: ProvisionInspectContext & {
phase: string;
setup: string;
timeoutMs?: number;
forwardedEnv?: Record<string, string>;
@@ -316,7 +317,7 @@ async function runProvisionSetupAndWaitReady(
params.forwardedEnv,
(names, profilePath, childEnv) =>
runCrabboxCommand({
action: "setup",
action: params.phase,
args: leaseRunArgs({ ...params, id: params.inspect.id }, names, profilePath),
binary: params.binary,
env: childEnv,
@@ -329,7 +330,7 @@ async function runProvisionSetupAndWaitReady(
}),
);
if (result.termination !== "exit" || result.code !== 0) {
throw permanentCrabboxCommandError("setup", result);
throw permanentCrabboxCommandError(params.phase, result);
}
} catch (error) {
return await failProvisionAfterCleanup({ ...params, id: params.inspect.id }, error);
@@ -588,10 +589,7 @@ export function createCrabboxWorkerProvider(
} catch (error) {
// Transport failure after warmup is indeterminate; preserve the lease for durable replay.
if (error instanceof WorkerProviderError) {
return await failProvisionAfterCleanup(
{ binary, id: leaseId, provider: parsed.provider, runCommand },
error,
);
return await failProvisionAfterCleanup({ ...context, id: leaseId, runCommand }, error);
}
throw error;
}
@@ -599,11 +597,10 @@ export function createCrabboxWorkerProvider(
throw new Error("Crabbox warmup lease was not found during inspection");
}
const inspectedParams = {
binary,
...context,
deadline,
inspect: inspected.inspect,
profile: parsed,
provider: parsed.provider,
runCommand,
};
if (isUnusableProvisionState(inspected.inspect.state)) {
@@ -617,6 +614,7 @@ export function createCrabboxWorkerProvider(
if (parsed.setup) {
inspectedParams.inspect = await runProvisionSetupAndWaitReady({
...inspectedParams,
phase: "profile setup",
setup: parsed.setup,
forwardedEnv,
sleep,
@@ -625,6 +623,7 @@ export function createCrabboxWorkerProvider(
if (parsed.desktop) {
inspectedParams.inspect = await runProvisionSetupAndWaitReady({
...inspectedParams,
phase: "desktop setup",
setup: createCrabboxWorkerDesktopSetup(leaseId, wallpaperBase64),
sleep,
});
@@ -652,6 +651,7 @@ export function createCrabboxWorkerProvider(
});
inspectedParams.inspect = await runProvisionSetupAndWaitReady({
...inspectedParams,
phase: "node enrollment setup",
setup: nodeEnrollmentSetup.command,
timeoutMs: CRABBOX_NODE_ENROLLMENT_TIMEOUT_MS,
...(nodeEnrollmentSetup.forwardedEnv
@@ -339,7 +339,7 @@ describe("Crabbox profile warm images", () => {
await expect(
provisionWarmProfile(provider, { ...PROFILE, setup: "install-node" }),
).rejects.toThrow("Crabbox setup failed");
).rejects.toThrow("Crabbox profile setup failed");
expect(calls.some(({ argv }) => argv[1] === "checkpoint")).toBe(false);
expect(calls.at(-1)?.argv[1]).toBe("stop");