diff --git a/extensions/crabbox/src/crabbox-worker-command-error.ts b/extensions/crabbox/src/crabbox-worker-command-error.ts index 5738e7f26e08..c1c599227bf3 100644 --- a/extensions/crabbox/src/crabbox-worker-command-error.ts +++ b/extensions/crabbox/src/crabbox-worker-command-error.ts @@ -1,26 +1,22 @@ import { redactSensitiveText } from "openclaw/plugin-sdk/logging-core"; import { WorkerProviderError } from "openclaw/plugin-sdk/plugin-entry"; import type { SpawnResult } from "openclaw/plugin-sdk/process-runtime"; -import { sliceUtf16Safe, truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; +import { sliceUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; const MAX_COMMAND_DETAIL_CHARS = 512; function crabboxCommandDetail(result: SpawnResult): string { - const raw = [result.stderr, result.stdout].filter(Boolean).join("\n").trim(); + const raw = [result.stdout, result.stderr].filter(Boolean).join("\n").trim(); if (!raw) { return ""; } const compressed = redactSensitiveText(raw).replace(/\s+/gu, " "); - const omitted = " ... "; - const remaining = MAX_COMMAND_DETAIL_CHARS - omitted.length; - const redacted = - compressed.length <= MAX_COMMAND_DETAIL_CHARS - ? compressed - : `${truncateUtf16Safe(compressed, Math.ceil(remaining / 2))}${omitted}${sliceUtf16Safe( - compressed, - -Math.floor(remaining / 2), - )}`; - return redacted ? `: ${redacted}` : ""; + // Failure diagnoses come last; Crabbox's fixed banner is leading boilerplate. + // Keep stderr last, matching the per-stream suffix capture in src/process/exec-output.ts. + const tailMarker = "... "; + return compressed.length <= MAX_COMMAND_DETAIL_CHARS + ? `: ${compressed}` + : `: ${tailMarker}${sliceUtf16Safe(compressed, tailMarker.length - MAX_COMMAND_DETAIL_CHARS)}`; } export function crabboxCommandError(action: string, result: SpawnResult): Error { diff --git a/extensions/crabbox/src/crabbox-worker-provider.test.ts b/extensions/crabbox/src/crabbox-worker-provider.test.ts index e15a1df87e17..5b3a7ac21816 100644 --- a/extensions/crabbox/src/crabbox-worker-provider.test.ts +++ b/extensions/crabbox/src/crabbox-worker-provider.test.ts @@ -956,17 +956,17 @@ describe("Crabbox worker provider", () => { { name: "fails", result: commandResult({ code: 7, stderr: "apt exploded" }), - message: "Crabbox setup failed with exit code 7", + message: "Crabbox profile setup failed with exit code 7", }, { name: "times out", result: commandResult({ code: null, killed: true, termination: "timeout" }), - message: "Crabbox setup did not exit normally (timeout)", + message: "Crabbox profile setup did not exit normally (timeout)", }, { name: "cannot start", result: undefined, - message: "Crabbox setup could not start", + message: "Crabbox profile setup could not start", }, ])( "stops the lease and removes its private env profile when setup $name", @@ -1024,6 +1024,64 @@ describe("Crabbox worker provider", () => { }, ); + it.each([ + { phase: "profile setup", setupAttempt: 1 }, + { phase: "desktop setup", setupAttempt: 2 }, + { phase: "node enrollment setup", setupAttempt: 3 }, + ])("identifies the failed $phase phase", async ({ phase, setupAttempt }) => { + let attempts = 0; + const provider = providerWithRunner(async (argv) => { + if (argv[1] === "inspect") { + return commandResult({ stdout: inspectJson() }); + } + if (argv[1] === "run" && ++attempts === setupAttempt) { + return commandResult({ code: 7, stderr: "setup command rejected" }); + } + return commandResult(); + }); + + await expect( + provider.provision({ ...PROFILE, setup: "install-node", desktop: true }, OPERATION_ID), + ).rejects.toMatchObject({ + code: "invalid_profile", + message: `Crabbox ${phase} failed with exit code 7: setup command rejected`, + }); + }); + + it("preserves the node enrollment diagnosis after the Crabbox banner and setup noise", async () => { + const diagnosis = + "Error: Codex remote-exec requires the exact official @openclaw/codex@2026.8.1 plugin to be installed by cloudWorkers profile setup"; + const stderr = [ + `workspace owner acquired wait=218ms recovered=false run context: run=${"a".repeat(32)} lease=${LEASE_ID} slug=openclaw-${"b".repeat(32)} provider=machine0 ssh=openclaw@worker.example.test:2222 workspace=/workspace/openclaw`, + "x".repeat(2_000), + diagnosis, + " at prepareCodex ([eval]:20:11)", + " at runScriptInThisContext (node:internal/vm:209:10)", + " at node:internal/process/execution:446:12", + " at [eval]-wrapper:6:24", + " at runScriptInContext (node:internal/process/execution:444:60)", + " at evalFunction (node:internal/process/execution:279:30)", + "Node.js v24.15.0", + ].join("\n"); + const provider = providerWithRunner(async (argv) => { + if (argv[1] === "status") { + return commandResult({ stdout: inspectJson() }); + } + return argv[1] === "run" + ? commandResult({ code: 1, stderr, stdout: "setup progress ".repeat(200) }) + : commandResult(); + }); + + await expect( + provider.provision({ ...PROFILE, provider: "machine0" }, OPERATION_ID, { + executionMode: "remote-exec", + }), + ).rejects.toMatchObject({ + code: "invalid_profile", + message: expect.stringContaining(diagnosis), + }); + }); + it("preserves the allocated lease and both failures when setup cleanup times out", async () => { let releaseCommitted = false; const provider = providerWithRunner(async (argv) => { @@ -2854,8 +2912,8 @@ describe("Crabbox worker provider", () => { code, termination, killed: termination !== "exit", - stderr: `provider warning ${secret} ${"provider progress ".repeat(90)}${terminalStderr}`, - stdout: terminalStdout, + stderr: `provider warning ${secret}\n${terminalStderr}`, + stdout: `${"provider progress ".repeat(90)}\n${terminalStdout}`, }), ); const operation = @@ -2870,10 +2928,11 @@ describe("Crabbox worker provider", () => { action === "warmup" ? "Crabbox warmup failed with exit code 5: " : "Crabbox inspect did not exit normally (timeout): "; - expect(message).toContain("provider warning"); + expect(message.startsWith(`${failurePrefix}... `)).toBe(true); expect(message).toContain(terminalStderr); expect(message).toContain(terminalStdout); expect(message).not.toContain(secret); + expect(message).not.toMatch(/\s{2,}/u); expect(message.length).toBeLessThanOrEqual(failurePrefix.length + 512); expect(hasLoneSurrogate(message)).toBe(false); }, @@ -2882,10 +2941,11 @@ describe("Crabbox worker provider", () => { it.each(["stderr", "stdout"] as const)( "preserves UTF-16 boundaries and terminal detail from %s", async (stream) => { + const terminalDetail = "😀 terminal failure"; const provider = providerWithRunner(async () => commandResult({ code: 2, - [stream]: `${"x".repeat(253)}😀${"y".repeat(300)}😀 terminal failure`, + [stream]: `${"x".repeat(600)}😀${"y".repeat(507 - terminalDetail.length)}${terminalDetail}`, }), ); diff --git a/extensions/crabbox/src/crabbox-worker-provider.ts b/extensions/crabbox/src/crabbox-worker-provider.ts index 8f22141f7ff3..752c60f97f4d 100644 --- a/extensions/crabbox/src/crabbox-worker-provider.ts +++ b/extensions/crabbox/src/crabbox-worker-provider.ts @@ -305,6 +305,7 @@ async function waitForProvisionReady( // otherwise the caller cannot release a box it never learned about. async function runProvisionSetupAndWaitReady( params: ProvisionInspectContext & { + phase: string; setup: string; timeoutMs?: number; forwardedEnv?: Record; @@ -316,7 +317,7 @@ async function runProvisionSetupAndWaitReady( params.forwardedEnv, (names, profilePath, childEnv) => runCrabboxCommand({ - action: "setup", + action: params.phase, args: leaseRunArgs({ ...params, id: params.inspect.id }, names, profilePath), binary: params.binary, env: childEnv, @@ -329,7 +330,7 @@ async function runProvisionSetupAndWaitReady( }), ); if (result.termination !== "exit" || result.code !== 0) { - throw permanentCrabboxCommandError("setup", result); + throw permanentCrabboxCommandError(params.phase, result); } } catch (error) { return await failProvisionAfterCleanup({ ...params, id: params.inspect.id }, error); @@ -588,10 +589,7 @@ export function createCrabboxWorkerProvider( } catch (error) { // Transport failure after warmup is indeterminate; preserve the lease for durable replay. if (error instanceof WorkerProviderError) { - return await failProvisionAfterCleanup( - { binary, id: leaseId, provider: parsed.provider, runCommand }, - error, - ); + return await failProvisionAfterCleanup({ ...context, id: leaseId, runCommand }, error); } throw error; } @@ -599,11 +597,10 @@ export function createCrabboxWorkerProvider( throw new Error("Crabbox warmup lease was not found during inspection"); } const inspectedParams = { - binary, + ...context, deadline, inspect: inspected.inspect, profile: parsed, - provider: parsed.provider, runCommand, }; if (isUnusableProvisionState(inspected.inspect.state)) { @@ -617,6 +614,7 @@ export function createCrabboxWorkerProvider( if (parsed.setup) { inspectedParams.inspect = await runProvisionSetupAndWaitReady({ ...inspectedParams, + phase: "profile setup", setup: parsed.setup, forwardedEnv, sleep, @@ -625,6 +623,7 @@ export function createCrabboxWorkerProvider( if (parsed.desktop) { inspectedParams.inspect = await runProvisionSetupAndWaitReady({ ...inspectedParams, + phase: "desktop setup", setup: createCrabboxWorkerDesktopSetup(leaseId, wallpaperBase64), sleep, }); @@ -652,6 +651,7 @@ export function createCrabboxWorkerProvider( }); inspectedParams.inspect = await runProvisionSetupAndWaitReady({ ...inspectedParams, + phase: "node enrollment setup", setup: nodeEnrollmentSetup.command, timeoutMs: CRABBOX_NODE_ENROLLMENT_TIMEOUT_MS, ...(nodeEnrollmentSetup.forwardedEnv diff --git a/extensions/crabbox/src/crabbox-worker-warm-image.test.ts b/extensions/crabbox/src/crabbox-worker-warm-image.test.ts index d96d8e2398c0..6105ccba3e6e 100644 --- a/extensions/crabbox/src/crabbox-worker-warm-image.test.ts +++ b/extensions/crabbox/src/crabbox-worker-warm-image.test.ts @@ -339,7 +339,7 @@ describe("Crabbox profile warm images", () => { await expect( provisionWarmProfile(provider, { ...PROFILE, setup: "install-node" }), - ).rejects.toThrow("Crabbox setup failed"); + ).rejects.toThrow("Crabbox profile setup failed"); expect(calls.some(({ argv }) => argv[1] === "checkpoint")).toBe(false); expect(calls.at(-1)?.argv[1]).toBe("stop");