diff --git a/extensions/crabbox/src/crabbox-worker-heartbeat.ts b/extensions/crabbox/src/crabbox-worker-heartbeat.ts index 655b7c321607..61edb7239c83 100644 --- a/extensions/crabbox/src/crabbox-worker-heartbeat.ts +++ b/extensions/crabbox/src/crabbox-worker-heartbeat.ts @@ -62,6 +62,7 @@ export function createCrabboxHeartbeatManager(dependencies: { const controller = new AbortController(); entry.controller = controller; let result: SpawnResult; + const startedAt = Date.now(); try { result = await dependencies.run(entry, controller.signal); } catch (error) { @@ -93,7 +94,8 @@ export function createCrabboxHeartbeatManager(dependencies: { } if (!entry.failureWarned) { entry.failureWarned = true; - warn(entry, crabboxCommandError("heartbeat", result).message); + const message = crabboxCommandError("heartbeat", result).message; + warn(entry, message.replace("(timeout)", `(timeout after ${Date.now() - startedAt} ms)`)); } schedule(entry); }; diff --git a/extensions/crabbox/src/crabbox-worker-provider.test.ts b/extensions/crabbox/src/crabbox-worker-provider.test.ts index 9d2764888d6c..8dbc6bed7ddb 100644 --- a/extensions/crabbox/src/crabbox-worker-provider.test.ts +++ b/extensions/crabbox/src/crabbox-worker-provider.test.ts @@ -2097,6 +2097,39 @@ describe("Crabbox worker provider", () => { } }); + it("reports the measured heartbeat duration when the command times out", async () => { + vi.useFakeTimers(); + const warnings: string[] = []; + const provider = providerWithRunner( + async (argv) => { + if (argv[1] === "inspect") { + return commandResult({ stdout: inspectJson() }); + } + if (argv[1] === "heartbeat") { + await new Promise((resolve) => { + setTimeout(resolve, 60_012); + }); + return commandResult({ code: null, killed: true, termination: "timeout" }); + } + return commandResult(); + }, + (message) => warnings.push(message), + ); + const lease = lifecycleLease(); + + try { + await expect(provider.inspect(lease)).resolves.toStrictEqual({ status: "active" }); + await vi.advanceTimersByTimeAsync(60_012); + + expect(warnings).toEqual([ + "Crabbox heartbeat did not exit normally (timeout after 60012 ms); cloud worker machines may be reaped after 60m of coordinator-idle time", + ]); + } finally { + await provider.destroy(lease); + vi.useRealTimers(); + } + }); + it("keeps heartbeat transport failures out of lifecycle operations and retries", async () => { vi.useFakeTimers(); let heartbeatAttempts = 0; diff --git a/src/gateway/server-methods/sessions-dispatch.ts b/src/gateway/server-methods/sessions-dispatch.ts index 6ea009c766bc..3cbc1a9e4db9 100644 --- a/src/gateway/server-methods/sessions-dispatch.ts +++ b/src/gateway/server-methods/sessions-dispatch.ts @@ -171,7 +171,7 @@ async function validateDispatchExecutionMode(params: { } respondInvalidWorkerSession( params.respond, - `runtime ${params.sessionRuntime} requires an SSH-backed cloud worker provider; choose a provider that supports remote-exec, or select an agent/model route with agentRuntime.id "openclaw"`, + `selected cloud worker provider does not support the remote-exec execution mode required by runtime ${params.sessionRuntime}; use an approved paired device or a provider that advertises remote-exec`, ); return false; } diff --git a/src/gateway/server-methods/sessions.dispatch.test.ts b/src/gateway/server-methods/sessions.dispatch.test.ts index 55fd1a476929..4d0730baeea5 100644 --- a/src/gateway/server-methods/sessions.dispatch.test.ts +++ b/src/gateway/server-methods/sessions.dispatch.test.ts @@ -437,7 +437,7 @@ describe("sessions.dispatch", () => { expect.objectContaining({ code: ErrorCodes.INVALID_REQUEST, message: - 'runtime codex requires an SSH-backed cloud worker provider; choose a provider that supports remote-exec, or select an agent/model route with agentRuntime.id "openclaw"', + "selected cloud worker provider does not support the remote-exec execution mode required by runtime codex; use an approved paired device or a provider that advertises remote-exec", }), ); });